20 video conversion for web view (#200)

* Added job for video conversion every 1 minute

* Handle get video as mp4 on the web

* Auto play video on web on hovered

* Added video player

* Added animation and video duration to thumbnail player

* Fixed issue with video not playing on hover

* Added animation when loading thumbnail
This commit is contained in:
Alex 2022-06-04 18:34:11 -05:00 committed by GitHub
parent 53c3c916a6
commit ab6909bfbd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 371 additions and 50 deletions

View file

@ -38,4 +38,4 @@ import { CommunicationModule } from '../communication/communication.module';
providers: [AssetService, AssetOptimizeService, BackgroundTaskService],
exports: [],
})
export class AssetModule {}
export class AssetModule { }

View file

@ -11,6 +11,7 @@ import { Response as Res } from 'express';
import { promisify } from 'util';
import { DeleteAssetDto } from './dto/delete-asset.dto';
import { SearchAssetDto } from './dto/search-asset.dto';
import ffmpeg from 'fluent-ffmpeg';
const fileInfo = promisify(stat);
@ -185,7 +186,15 @@ export class AssetService {
} else if (asset.type == AssetType.VIDEO) {
// Handle Video
const { size } = await fileInfo(asset.originalPath);
let videoPath = asset.originalPath;
let mimeType = asset.mimeType;
if (query.isWeb && asset.mimeType == 'video/quicktime') {
videoPath = asset.encodedVideoPath == '' ? asset.originalPath : asset.encodedVideoPath;
mimeType = asset.encodedVideoPath == '' ? asset.mimeType : 'video/mp4';
}
const { size } = await fileInfo(videoPath);
const range = headers.range;
if (range) {
@ -220,20 +229,22 @@ export class AssetService {
'Content-Range': `bytes ${start}-${end}/${size}`,
'Accept-Ranges': 'bytes',
'Content-Length': end - start + 1,
'Content-Type': asset.mimeType,
'Content-Type': mimeType,
});
const videoStream = createReadStream(asset.originalPath, { start: start, end: end });
const videoStream = createReadStream(videoPath, { start: start, end: end });
return new StreamableFile(videoStream);
} else {
res.set({
'Content-Type': asset.mimeType,
'Content-Type': mimeType,
});
return new StreamableFile(createReadStream(asset.originalPath));
return new StreamableFile(createReadStream(videoPath));
}
}
}

View file

@ -29,6 +29,9 @@ export class AssetEntity {
@Column({ nullable: true })
webpPath: string;
@Column({ nullable: true })
encodedVideoPath: string;
@Column()
createdAt: string;

View file

@ -65,7 +65,7 @@ import { ScheduleTasksModule } from './modules/schedule-tasks/schedule-tasks.mod
export class AppModule implements NestModule {
configure(consumer: MiddlewareConsumer): void {
if (process.env.NODE_ENV == 'development') {
consumer.apply(AppLoggerMiddleware).forRoutes('*');
// consumer.apply(AppLoggerMiddleware).forRoutes('*');
}
}
}

View file

@ -0,0 +1,17 @@
import { MigrationInterface, QueryRunner } from "typeorm";
export class UpdateAssetTableWithEncodeVideoPath1654299904583 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`
alter table assets
add column if not exists "encodedVideoPath" varchar default '';
`)
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`
alter table assets
drop column if exists "encodedVideoPath";
`);
}
}

View file

@ -17,9 +17,10 @@ import { BackgroundTaskService } from './background-task.service';
removeOnFail: false,
},
}),
TypeOrmModule.forFeature([AssetEntity, ExifEntity, SmartInfoEntity]),
],
providers: [BackgroundTaskService, BackgroundTaskProcessor],
exports: [BackgroundTaskService],
})
export class BackgroundTaskModule {}
export class BackgroundTaskModule { }

View file

@ -33,4 +33,4 @@ import { AssetOptimizeService } from './image-optimize.service';
providers: [AssetOptimizeService, ImageOptimizeProcessor, BackgroundTaskService],
exports: [AssetOptimizeService],
})
export class ImageOptimizeModule {}
export class ImageOptimizeModule { }

View file

@ -1,12 +1,30 @@
import { BullModule } from '@nestjs/bull';
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { AssetModule } from '../../api-v1/asset/asset.module';
import { AssetEntity } from '../../api-v1/asset/entities/asset.entity';
import { ImageConversionService } from './image-conversion.service';
import { VideoConversionProcessor } from './video-conversion.processor';
import { VideoConversionService } from './video-conversion.service';
@Module({
imports: [
TypeOrmModule.forFeature([AssetEntity]),
BullModule.registerQueue({
settings: {},
name: 'video-conversion',
limiter: {
max: 1,
duration: 60000
},
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
removeOnFail: false,
},
}),
],
providers: [ImageConversionService],
providers: [ImageConversionService, VideoConversionService, VideoConversionProcessor,],
})
export class ScheduleTasksModule { }

View file

@ -0,0 +1,56 @@
import { Process, Processor } from '@nestjs/bull';
import { InjectRepository } from '@nestjs/typeorm';
import { Job } from 'bull';
import { Repository } from 'typeorm';
import { AssetEntity } from '../../api-v1/asset/entities/asset.entity';
import { existsSync, mkdirSync } from 'fs';
import { APP_UPLOAD_LOCATION } from '../../constants/upload_location.constant';
import ffmpeg from 'fluent-ffmpeg';
import { Logger } from '@nestjs/common';
@Processor('video-conversion')
export class VideoConversionProcessor {
constructor(
@InjectRepository(AssetEntity)
private assetRepository: Repository<AssetEntity>,
) { }
@Process('to-mp4')
async convertToMp4(job: Job) {
const { asset }: { asset: AssetEntity } = job.data;
const basePath = APP_UPLOAD_LOCATION;
const encodedVideoPath = `${basePath}/${asset.userId}/encoded-video`;
if (!existsSync(encodedVideoPath)) {
mkdirSync(encodedVideoPath, { recursive: true });
}
const latestAssetInfo = await this.assetRepository.findOne({ id: asset.id });
const savedEncodedPath = encodedVideoPath + "/" + latestAssetInfo.id + '.mp4'
if (latestAssetInfo.encodedVideoPath == '') {
ffmpeg(latestAssetInfo.originalPath)
.outputOptions([
'-crf 23',
'-preset ultrafast',
'-vcodec libx264',
'-acodec mp3',
'-vf scale=1280:-2'
])
.output(savedEncodedPath)
.on('start', () => Logger.log("Start Converting", 'VideoConversionMOV2MP4'))
.on('error', (a, b, c) => {
Logger.error('Cannot Convert Video', 'VideoConversionMOV2MP4')
console.log(a, b, c)
})
.on('end', async () => {
Logger.log(`Converting Success ${latestAssetInfo.id}`, 'VideoConversionMOV2MP4')
await this.assetRepository.update({ id: latestAssetInfo.id }, { encodedVideoPath: savedEncodedPath });
}).run();
}
return {}
}
}

View file

@ -0,0 +1,50 @@
import { Injectable, Logger } from '@nestjs/common';
import { Cron, CronExpression } from '@nestjs/schedule';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { AssetEntity } from '../../api-v1/asset/entities/asset.entity';
import sharp from 'sharp';
import ffmpeg from 'fluent-ffmpeg';
import { APP_UPLOAD_LOCATION } from '../../constants/upload_location.constant';
import { existsSync, mkdirSync } from 'fs';
import { InjectQueue } from '@nestjs/bull/dist/decorators';
import { Queue } from 'bull';
import { randomUUID } from 'crypto';
@Injectable()
export class VideoConversionService {
constructor(
@InjectRepository(AssetEntity)
private assetRepository: Repository<AssetEntity>,
@InjectQueue('video-conversion')
private videoEncodingQueue: Queue
) { }
// time ffmpeg -i 15065f4a-47ff-4aed-8c3e-c9fcf1840531.mov -crf 35 -preset ultrafast -vcodec libx264 -acodec mp3 -vf "scale=1280:-1" 15065f4a-47ff-4aed-8c3e-c9fcf1840531.mp4
@Cron(CronExpression.EVERY_MINUTE
, {
name: 'video-encoding'
})
async mp4Conversion() {
const assets = await this.assetRepository.find({
where: {
type: 'VIDEO',
mimeType: 'video/quicktime',
encodedVideoPath: ''
},
order: {
createdAt: 'DESC'
},
take: 1
});
if (assets.length > 0) {
const asset = assets[0];
await this.videoEncodingQueue.add('to-mp4', { asset }, { jobId: asset.id },)
}
}
}

19
web/package-lock.json generated
View file

@ -23,6 +23,7 @@
"@types/axios": "^0.14.0",
"@types/bcrypt": "^5.0.0",
"@types/cookie": "^0.4.1",
"@types/fluent-ffmpeg": "^2.1.20",
"@types/leaflet": "^1.7.10",
"@types/lodash": "^4.14.182",
"@types/lodash-es": "^4.17.6",
@ -260,6 +261,15 @@
"integrity": "sha512-XW/Aa8APYr6jSVVA1y/DEIZX0/GMKLEVekNG727R8cs56ahETkRAy/3DR7+fJyh7oUgGwNQaRfXCun0+KbWY7Q==",
"dev": true
},
"node_modules/@types/fluent-ffmpeg": {
"version": "2.1.20",
"resolved": "https://registry.npmjs.org/@types/fluent-ffmpeg/-/fluent-ffmpeg-2.1.20.tgz",
"integrity": "sha512-B+OvhCdJ3LgEq2PhvWNOiB/EfwnXLElfMCgc4Z1K5zXgSfo9I6uGKwR/lqmNPFQuebNnes7re3gqkV77SyypLg==",
"dev": true,
"dependencies": {
"@types/node": "*"
}
},
"node_modules/@types/geojson": {
"version": "7946.0.8",
"resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.8.tgz",
@ -3418,6 +3428,15 @@
"integrity": "sha512-XW/Aa8APYr6jSVVA1y/DEIZX0/GMKLEVekNG727R8cs56ahETkRAy/3DR7+fJyh7oUgGwNQaRfXCun0+KbWY7Q==",
"dev": true
},
"@types/fluent-ffmpeg": {
"version": "2.1.20",
"resolved": "https://registry.npmjs.org/@types/fluent-ffmpeg/-/fluent-ffmpeg-2.1.20.tgz",
"integrity": "sha512-B+OvhCdJ3LgEq2PhvWNOiB/EfwnXLElfMCgc4Z1K5zXgSfo9I6uGKwR/lqmNPFQuebNnes7re3gqkV77SyypLg==",
"dev": true,
"requires": {
"@types/node": "*"
}
},
"@types/geojson": {
"version": "7946.0.8",
"resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.8.tgz",

View file

@ -19,6 +19,7 @@
"@types/axios": "^0.14.0",
"@types/bcrypt": "^5.0.0",
"@types/cookie": "^0.4.1",
"@types/fluent-ffmpeg": "^2.1.20",
"@types/leaflet": "^1.7.10",
"@types/lodash": "^4.14.182",
"@types/lodash-es": "^4.17.6",

View file

@ -12,10 +12,12 @@
import { serverEndpoint } from '../../constants';
import axios from 'axios';
import { downloadAssets } from '$lib/stores/download';
import VideoViewer from './video-viewer.svelte';
const dispatch = createEventDispatcher();
export let selectedAsset: ImmichAsset;
export let selectedIndex: number;
let viewDeviceId: string;
@ -157,7 +159,9 @@
</div>
<div
class="row-start-2 row-span-end col-start-1- col-span-full z-[1000] flex place-items-center hover:cursor-pointer w-3/4"
class={`row-start-2 row-span-end col-start-1 col-span-2 flex place-items-center hover:cursor-pointer w-3/4 ${
selectedAsset.type == 'VIDEO' ? '' : 'z-[999]'
}`}
on:mouseenter={() => {
halfLeftHover = true;
halfRightHover = false;
@ -168,7 +172,7 @@
on:click={navigateAssetBackward}
>
<button
class="rounded-full p-3 hover:bg-gray-500 hover:text-gray-700 text-gray-500 mx-4"
class="rounded-full p-3 hover:bg-gray-500 hover:text-gray-700 z-[1000] text-gray-500 mx-4"
class:navigation-button-hover={halfLeftHover}
on:click={navigateAssetBackward}
>
@ -182,19 +186,16 @@
{#if selectedAsset.type == AssetType.IMAGE}
<PhotoViewer assetId={viewAssetId} deviceId={viewDeviceId} on:close={closeViewer} />
{:else}
<div
class="w-full h-full bg-immich-primary/10 flex flex-col place-items-center place-content-center "
on:click={closeViewer}
>
<h1 class="animate-pulse font-bold text-4xl">Video viewer is under construction</h1>
</div>
<VideoViewer assetId={viewAssetId} on:close={closeViewer} />
{/if}
{/if}
{/key}
</div>
<div
class="row-start-2 row-span-full col-start-3 col-span-2 z-[1000] flex justify-end place-items-center hover:cursor-pointer w-3/4 justify-self-end"
class={`row-start-2 row-span-full col-start-3 col-span-2 flex justify-end place-items-center hover:cursor-pointer w-3/4 justify-self-end ${
selectedAsset.type == 'VIDEO' ? '' : 'z-[500]'
}`}
on:click={navigateAssetForward}
on:mouseenter={() => {
halfLeftHover = false;
@ -205,7 +206,7 @@
}}
>
<button
class="rounded-full p-3 hover:bg-gray-500 hover:text-gray-700 text-gray-500 mx-4"
class="rounded-full p-3 hover:bg-gray-500 hover:text-gray-700 text-gray-500 mx-4 z-[1000]"
class:navigation-button-hover={halfRightHover}
on:click={navigateAssetForward}
>

View file

@ -2,23 +2,30 @@
import { AssetType, type ImmichAsset } from '../../models/immich-asset';
import { session } from '$app/stores';
import { createEventDispatcher, onDestroy } from 'svelte';
import { fade } from 'svelte/transition';
import { fade, fly, slide } from 'svelte/transition';
import { serverEndpoint } from '../../constants';
import IntersectionObserver from '$lib/components/asset-viewer/intersection-observer.svelte';
import CheckCircle from 'svelte-material-icons/CheckCircle.svelte';
import PlayCircleOutline from 'svelte-material-icons/PlayCircleOutline.svelte';
import PauseCircleOutline from 'svelte-material-icons/PauseCircleOutline.svelte';
import LoadingSpinner from '../shared/loading-spinner.svelte';
const dispatch = createEventDispatcher();
export let asset: ImmichAsset;
export let groupIndex: number;
let imageContent: string;
let imageData: string;
let videoData: string;
let mouseOver: boolean = false;
$: dispatch('mouseEvent', { isMouseOver: mouseOver, selectedGroupIndex: groupIndex });
let mouseOverIcon: boolean = false;
let videoPlayerNode: HTMLVideoElement;
let isThumbnailVideoPlaying = false;
let calculateVideoDurationIntervalHandler: NodeJS.Timer;
let videoProgress = '00:00';
const loadImageData = async () => {
if ($session.user) {
@ -29,34 +36,54 @@
},
});
imageContent = URL.createObjectURL(await res.blob());
imageData = URL.createObjectURL(await res.blob());
return imageContent;
return imageData;
}
};
const loadVideoData = async () => {
const videoUrl = `/asset/file?aid=${asset.deviceAssetId}&did=${asset.deviceId}`;
isThumbnailVideoPlaying = false;
const videoUrl = `/asset/file?aid=${asset.deviceAssetId}&did=${asset.deviceId}&isWeb=true`;
if ($session.user) {
const res = await fetch(serverEndpoint + videoUrl, {
method: 'GET',
headers: {
Authorization: 'bearer ' + $session.user.accessToken,
},
});
try {
const res = await fetch(serverEndpoint + videoUrl, {
method: 'GET',
headers: {
Authorization: 'bearer ' + $session.user.accessToken,
},
});
const videoData = URL.createObjectURL(await res.blob());
videoData = URL.createObjectURL(await res.blob());
videoPlayerNode.src = videoData;
videoPlayerNode.src = videoData;
videoPlayerNode.load();
videoPlayerNode.oncanplay = () => {
console.log('Can play video');
};
videoPlayerNode.load();
return videoData;
videoPlayerNode.oncanplay = () => {
videoPlayerNode.muted = true;
videoPlayerNode.play();
isThumbnailVideoPlaying = true;
calculateVideoDurationIntervalHandler = setInterval(() => {
videoProgress = getVideoDurationInString(Math.round(videoPlayerNode.currentTime));
}, 1000);
};
return videoData;
} catch (e) {}
}
};
const getVideoDurationInString = (currentTime: number) => {
const minute = Math.floor(currentTime / 60);
const second = currentTime % 60;
const minuteText = minute >= 10 ? `${minute}` : `0${minute}`;
const secondText = second >= 10 ? `${second}` : `0${second}`;
return minuteText + ':' + secondText;
};
const parseVideoDuration = (duration: string) => {
const timePart = duration.split(':');
const hours = timePart[0];
@ -70,7 +97,9 @@
}
};
onDestroy(() => URL.revokeObjectURL(imageContent));
onDestroy(() => {
URL.revokeObjectURL(imageData);
});
const getSize = () => {
if (asset.exifInfo?.orientation === 'Rotate 90 CW') {
@ -81,19 +110,34 @@
return 'w-[235px] h-[235px]';
}
};
const handleMouseOverThumbnail = () => {
mouseOver = true;
};
const handleMouseLeaveThumbnail = () => {
mouseOver = false;
URL.revokeObjectURL(videoData);
if (calculateVideoDurationIntervalHandler) {
clearInterval(calculateVideoDurationIntervalHandler);
}
isThumbnailVideoPlaying = false;
videoProgress = '00:00';
};
</script>
<IntersectionObserver once={true} let:intersecting>
<div
class={`bg-gray-100 relative hover:cursor-pointer ${getSize()}`}
on:mouseenter={() => (mouseOver = true)}
on:mouseleave={() => (mouseOver = false)}
on:mouseenter={handleMouseOverThumbnail}
on:mouseleave={handleMouseLeaveThumbnail}
on:click={() => dispatch('viewAsset', { assetId: asset.id, deviceId: asset.deviceId })}
>
{#if mouseOver}
<div
in:fade={{ duration: 200 }}
class="w-full h-full bg-gradient-to-b from-gray-800/50 via-white/0 to-white/0 absolute p-2"
class="w-full bg-gradient-to-b from-gray-800/50 via-white/0 to-white/0 absolute p-2 z-10"
>
<div
on:mouseenter={() => (mouseOverIcon = true)}
@ -105,18 +149,44 @@
</div>
{/if}
<!-- Playback and info -->
{#if asset.type === AssetType.VIDEO}
<div class="absolute right-2 top-2 text-white text-xs font-medium flex gap-1 place-items-center">
{parseVideoDuration(asset.duration)}
<PlayCircleOutline size="24" />
<div class="absolute right-2 top-2 text-white text-xs font-medium flex gap-1 place-items-center z-10">
{#if isThumbnailVideoPlaying}
<span in:fly={{ x: -25, duration: 500 }}>
{videoProgress}
</span>
{:else}
<span in:fade={{ duration: 500 }}>
{parseVideoDuration(asset.duration)}
</span>
{/if}
{#if mouseOver}
{#if isThumbnailVideoPlaying}
<span in:fly={{ x: 25, duration: 500 }}>
<PauseCircleOutline size="24" />
</span>
{:else}
<span in:fade={{ duration: 250 }}>
<LoadingSpinner />
</span>
{/if}
{:else}
<span in:fade={{ duration: 500 }}>
<PlayCircleOutline size="24" />
</span>
{/if}
</div>
{/if}
<!-- Thumbnail -->
{#if intersecting}
{#await loadImageData()}
<div class={`bg-immich-primary/10 ${getSize()} flex place-items-center place-content-center`}>...</div>
{:then imageData}
<img
in:fade={{ duration: 250 }}
src={imageData}
alt={asset.id}
class={`object-cover ${getSize()} transition-all duration-100 z-0`}
@ -125,12 +195,12 @@
{/await}
{/if}
<!-- {#if mouseOver && asset.type === AssetType.VIDEO}
{#if mouseOver && asset.type === AssetType.VIDEO}
<div class="absolute w-full h-full top-0" on:mouseenter={loadVideoData}>
<video autoplay class="border-2 h-[200px]" width="250px" bind:this={videoPlayerNode}>
<video muted class="h-full object-cover" width="250px" bind:this={videoPlayerNode}>
<track kind="captions" />
</video>
</div>
{/if} -->
{/if}
</div>
</IntersectionObserver>

View file

@ -0,0 +1,75 @@
<script lang="ts">
import { session } from '$app/stores';
import { serverEndpoint } from '$lib/constants';
import { fade } from 'svelte/transition';
import type { ImmichAsset, ImmichExif } from '$lib/models/immich-asset';
import { createEventDispatcher, onMount } from 'svelte';
import LoadingSpinner from '../shared/loading-spinner.svelte';
export let assetId: string;
let asset: ImmichAsset;
const dispatch = createEventDispatcher();
let videoPlayerNode: HTMLVideoElement;
let isVideoLoading = true;
onMount(async () => {
if ($session.user) {
const res = await fetch(serverEndpoint + '/asset/assetById/' + assetId, {
headers: {
Authorization: 'bearer ' + $session.user.accessToken,
},
});
asset = await res.json();
await loadVideoData();
}
});
const loadVideoData = async () => {
isVideoLoading = true;
const videoUrl = `/asset/file?aid=${asset.deviceAssetId}&did=${asset.deviceId}&isWeb=true`;
if ($session.user) {
try {
const res = await fetch(serverEndpoint + videoUrl, {
method: 'GET',
headers: {
Authorization: 'bearer ' + $session.user.accessToken,
},
});
const videoData = URL.createObjectURL(await res.blob());
videoPlayerNode.src = videoData;
videoPlayerNode.load();
videoPlayerNode.oncanplay = () => {
videoPlayerNode.muted = true;
videoPlayerNode.play();
videoPlayerNode.muted = false;
isVideoLoading = false;
};
return videoData;
} catch (e) {}
}
};
</script>
<div transition:fade={{ duration: 150 }} class="flex place-items-center place-content-center h-full select-none">
{#if asset}
<video controls class="h-full object-contain" bind:this={videoPlayerNode}>
<track kind="captions" />
</video>
{#if isVideoLoading}
<div class="absolute w-full h-full bg-black/50 flex place-items-center place-content-center">
<LoadingSpinner />
</div>
{/if}
{/if}
</div>

View file

@ -1,7 +1,7 @@
<div>
<svg
role="status"
class="w-8 h-8 mr-2 text-gray-400 animate-spin dark:text-gray-600 fill-immich-primary"
class={`w-[24px] h-[24px] text-gray-400 animate-spin dark:text-gray-600 fill-immich-primary`}
viewBox="0 0 100 101"
fill="none"
xmlns="http://www.w3.org/2000/svg"

View file

@ -4,7 +4,6 @@
import type { Load } from '@sveltejs/kit';
export const load: Load = async ({ session }) => {
console.log('navigating to unknown paage');
if (!session.user) {
return {
status: 302,