diff --git a/docker/hwaccel-rkmpp.yml b/docker/hwaccel-rkmpp.yml new file mode 100644 index 000000000..6dfc527ff --- /dev/null +++ b/docker/hwaccel-rkmpp.yml @@ -0,0 +1,24 @@ +version: "3.8" + +# Hardware acceleration for transcoding using RKMPP for Rockchip SOCs +# This is only needed if you want to use hardware acceleration for transcoding. +# Supported host OS is Ubuntu Jammy 22.04 with custom ffmpeg from ppa:liujianfeng1994/rockchip-multimedia + +services: + hwaccel: + security_opt: # enables full access to /sys and /proc, still far better than privileged: true + - systempaths=unconfined + - apparmor=unconfined + group_add: + - video + devices: + - /dev/rga:/dev/rga + - /dev/dri:/dev/dri + - /dev/dma_heap:/dev/dma_heap + - /dev/mpp_service:/dev/mpp_service + volumes: + - /usr/bin/ffmpeg:/usr/bin/ffmpeg_mpp:ro + - /lib/aarch64-linux-gnu:/lib/ffmpeg-mpp:ro + - /lib/aarch64-linux-gnu/libblas.so.3:/lib/ffmpeg-mpp/libblas.so.3:ro # symlink is resolved by mounting + - /lib/aarch64-linux-gnu/liblapack.so.3:/lib/ffmpeg-mpp/liblapack.so.3:ro # symlink is resolved by mounting + - /lib/aarch64-linux-gnu/pulseaudio/libpulsecommon-15.99.so:/lib/ffmpeg-mpp/libpulsecommon-15.99.so:ro diff --git a/mobile/openapi/lib/model/transcode_hw_accel.dart b/mobile/openapi/lib/model/transcode_hw_accel.dart index 5db18bb70..9c4a85ef0 100644 --- a/mobile/openapi/lib/model/transcode_hw_accel.dart +++ b/mobile/openapi/lib/model/transcode_hw_accel.dart @@ -26,6 +26,7 @@ class TranscodeHWAccel { static const nvenc = TranscodeHWAccel._(r'nvenc'); static const qsv = TranscodeHWAccel._(r'qsv'); static const vaapi = TranscodeHWAccel._(r'vaapi'); + static const rkmpp = TranscodeHWAccel._(r'rkmpp'); static const disabled = TranscodeHWAccel._(r'disabled'); /// List of all possible values in this [enum][TranscodeHWAccel]. @@ -33,6 +34,7 @@ class TranscodeHWAccel { nvenc, qsv, vaapi, + rkmpp, disabled, ]; @@ -75,6 +77,7 @@ class TranscodeHWAccelTypeTransformer { case r'nvenc': return TranscodeHWAccel.nvenc; case r'qsv': return TranscodeHWAccel.qsv; case r'vaapi': return TranscodeHWAccel.vaapi; + case r'rkmpp': return TranscodeHWAccel.rkmpp; case r'disabled': return TranscodeHWAccel.disabled; default: if (!allowNull) { diff --git a/server/immich-openapi-specs.json b/server/immich-openapi-specs.json index ab9b16170..bd99bd8d7 100644 --- a/server/immich-openapi-specs.json +++ b/server/immich-openapi-specs.json @@ -8607,6 +8607,7 @@ "nvenc", "qsv", "vaapi", + "rkmpp", "disabled" ], "type": "string" diff --git a/server/src/domain/media/media.service.spec.ts b/server/src/domain/media/media.service.spec.ts index a17033233..401c8796e 100644 --- a/server/src/domain/media/media.service.spec.ts +++ b/server/src/domain/media/media.service.spec.ts @@ -1508,6 +1508,83 @@ describe(MediaService.name, () => { await expect(sut.handleVideoConversion({ id: assetStub.video.id })).resolves.toEqual(false); expect(mediaMock.transcode).not.toHaveBeenCalled(); }); + + it('should set vbr options for rkmpp when max bitrate is enabled', async () => { + storageMock.readdir.mockResolvedValue(['renderD128']); + mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); + configMock.load.mockResolvedValue([ + { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.RKMPP }, + { key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '10000k' }, + { key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.HEVC }, + ]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await sut.handleVideoConversion({ id: assetStub.video.id }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', + { + inputOptions: [], + outputOptions: [ + `-c:v hevc_rkmpp_encoder`, + '-c:a aac', + '-movflags faststart', + '-fps_mode passthrough', + '-map 0:0', + '-map 0:1', + '-g 256', + '-v verbose', + '-level 153', + '-rc_mode 3', + '-quality_min 0', + '-quality_max 100', + '-b:v 10000k', + '-width 1280', + '-height 720', + ], + twoPass: false, + ffmpegPath: 'ffmpeg_mpp', + ldLibraryPath: '/lib/aarch64-linux-gnu:/lib/ffmpeg-mpp', + }, + ); + }); + + it('should set cqp options for rkmpp when max bitrate is disabled', async () => { + storageMock.readdir.mockResolvedValue(['renderD128']); + mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); + configMock.load.mockResolvedValue([ + { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.RKMPP }, + { key: SystemConfigKey.FFMPEG_CRF, value: 30 }, + { key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '0' }, + ]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await sut.handleVideoConversion({ id: assetStub.video.id }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', + { + inputOptions: [], + outputOptions: [ + `-c:v h264_rkmpp_encoder`, + '-c:a aac', + '-movflags faststart', + '-fps_mode passthrough', + '-map 0:0', + '-map 0:1', + '-g 256', + '-v verbose', + '-level 51', + '-rc_mode 2', + '-quality_min 51', + '-quality_max 51', + '-width 1280', + '-height 720', + ], + twoPass: false, + ffmpegPath: 'ffmpeg_mpp', + ldLibraryPath: '/lib/aarch64-linux-gnu:/lib/ffmpeg-mpp', + }, + ); + }); }); it('should tonemap when policy is required and video is hdr', async () => { diff --git a/server/src/domain/media/media.service.ts b/server/src/domain/media/media.service.ts index 1c53752e8..4ddeca1d3 100644 --- a/server/src/domain/media/media.service.ts +++ b/server/src/domain/media/media.service.ts @@ -26,7 +26,16 @@ import { import { StorageCore, StorageFolder } from '../storage'; import { SystemConfigFFmpegDto } from '../system-config'; import { SystemConfigCore } from '../system-config/system-config.core'; -import { H264Config, HEVCConfig, NVENCConfig, QSVConfig, ThumbnailConfig, VAAPIConfig, VP9Config } from './media.util'; +import { + H264Config, + HEVCConfig, + NVENCConfig, + QSVConfig, + RKMPPConfig, + ThumbnailConfig, + VAAPIConfig, + VP9Config, +} from './media.util'; @Injectable() export class MediaService { @@ -352,6 +361,10 @@ export class MediaService { devices = await this.storageRepository.readdir('/dev/dri'); handler = new VAAPIConfig(config, devices); break; + case TranscodeHWAccel.RKMPP: + devices = await this.storageRepository.readdir('/dev/dri'); + handler = new RKMPPConfig(config, devices); + break; default: throw new UnsupportedMediaTypeException(`${config.accel.toUpperCase()} acceleration is unsupported`); } diff --git a/server/src/domain/media/media.util.ts b/server/src/domain/media/media.util.ts index 1ea2329df..38e68835b 100644 --- a/server/src/domain/media/media.util.ts +++ b/server/src/domain/media/media.util.ts @@ -143,6 +143,13 @@ class BaseConfig implements VideoCodecSWConfig { return this.isVideoVertical(videoStream) ? `${targetResolution}:-${mult}` : `-${mult}:${targetResolution}`; } + getSize(videoStream: VideoStreamInfo) { + const smaller = this.getTargetResolution(videoStream); + const factor = Math.max(videoStream.height, videoStream.width) / Math.min(videoStream.height, videoStream.width); + const larger = Math.round(smaller * factor); + return this.isVideoVertical(videoStream) ? { width: smaller, height: larger } : { width: larger, height: smaller }; + } + isVideoRotated(videoStream: VideoStreamInfo) { return Math.abs(videoStream.rotation) === 90; } @@ -555,3 +562,68 @@ export class VAAPIConfig extends BaseHWConfig { return this.config.cqMode !== CQMode.ICQ || this.config.targetVideoCodec === VideoCodec.VP9; } } + +export class RKMPPConfig extends BaseHWConfig { + getOptions(videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo): TranscodeOptions { + const options = super.getOptions(videoStream, audioStream); + options.ffmpegPath = 'ffmpeg_mpp'; + options.ldLibraryPath = '/lib/aarch64-linux-gnu:/lib/ffmpeg-mpp'; + options.outputOptions.push(...this.getSizeOptions(videoStream)); + return options; + } + + eligibleForTwoPass(): boolean { + return false; + } + + getBaseInputOptions() { + if (this.devices.length === 0) { + throw Error('No RKMPP device found'); + } + return []; + } + + getFilterOptions(videoStream: VideoStreamInfo) { + return this.shouldToneMap(videoStream) ? this.getToneMapping() : []; + } + + getSizeOptions(videoStream: VideoStreamInfo) { + if (this.shouldScale(videoStream)) { + const { width, height } = this.getSize(videoStream); + return [`-width ${width}`, `-height ${height}`]; + } + return []; + } + + getPresetOptions() { + switch (this.config.targetVideoCodec) { + case VideoCodec.H264: + // from ffmpeg_mpp help, commonly referred to as H264 level 5.1 + return ['-level 51']; + case VideoCodec.HEVC: + // from ffmpeg_mpp help, commonly referred to as HEVC level 5.1 + return ['-level 153']; + default: + throw Error(`Incompatible video codec for RKMPP: ${this.config.targetVideoCodec}`); + } + } + + getBitrateOptions() { + const bitrate = this.getMaxBitrateValue(); + if (bitrate > 0) { + return ['-rc_mode 3', '-quality_min 0', '-quality_max 100', `-b:v ${bitrate}${this.getBitrateUnit()}`]; + } else { + // convert CQP from 51-10 to 0-100, values below 10 are set to 10 + const quality = Math.floor(125 - Math.max(this.config.crf, 10) * (125 / 51)); + return ['-rc_mode 2', `-quality_min ${quality}`, `-quality_max ${quality}`]; + } + } + + getSupportedCodecs() { + return [VideoCodec.H264, VideoCodec.HEVC]; + } + + getVideoCodec(): string { + return `${this.config.targetVideoCodec}_rkmpp_encoder`; + } +} diff --git a/server/src/domain/repositories/media.repository.ts b/server/src/domain/repositories/media.repository.ts index cf2b8fa81..480f4f3eb 100644 --- a/server/src/domain/repositories/media.repository.ts +++ b/server/src/domain/repositories/media.repository.ts @@ -51,6 +51,8 @@ export interface TranscodeOptions { inputOptions: string[]; outputOptions: string[]; twoPass: boolean; + ffmpegPath?: string; + ldLibraryPath?: string; } export interface BitrateDistribution { diff --git a/server/src/infra/entities/system-config.entity.ts b/server/src/infra/entities/system-config.entity.ts index 2e0a7473c..de31bad32 100644 --- a/server/src/infra/entities/system-config.entity.ts +++ b/server/src/infra/entities/system-config.entity.ts @@ -119,6 +119,7 @@ export enum TranscodeHWAccel { NVENC = 'nvenc', QSV = 'qsv', VAAPI = 'vaapi', + RKMPP = 'rkmpp', DISABLED = 'disabled', } diff --git a/server/src/infra/repositories/media.repository.ts b/server/src/infra/repositories/media.repository.ts index 857e4587b..519094418 100644 --- a/server/src/infra/repositories/media.repository.ts +++ b/server/src/infra/repositories/media.repository.ts @@ -70,16 +70,18 @@ export class MediaRepository implements IMediaRepository { transcode(input: string, output: string | Writable, options: TranscodeOptions): Promise { if (!options.twoPass) { return new Promise((resolve, reject) => { - ffmpeg(input, { niceness: 10 }) - .inputOptions(options.inputOptions) - .outputOptions(options.outputOptions) - .output(output) - .on('error', (err, stdout, stderr) => { - this.logger.error(stderr); - reject(err); - }) - .on('end', resolve) - .run(); + const oldLdLibraryPath = process.env.LD_LIBRARY_PATH; + if (options.ldLibraryPath) { + // fluent ffmpeg does not allow to set environment variables, so we do it manually + process.env.LD_LIBRARY_PATH = this.chainPath(oldLdLibraryPath || '', options.ldLibraryPath); + } + try { + this.configureFfmpegCall(input, output, options).on('error', reject).on('end', resolve).run(); + } finally { + if (options.ldLibraryPath) { + process.env.LD_LIBRARY_PATH = oldLdLibraryPath; + } + } }); } @@ -90,29 +92,18 @@ export class MediaRepository implements IMediaRepository { // two-pass allows for precise control of bitrate at the cost of running twice // recommended for vp9 for better quality and compression return new Promise((resolve, reject) => { - ffmpeg(input, { niceness: 10 }) - .inputOptions(options.inputOptions) - .outputOptions(options.outputOptions) + // first pass output is not saved as only the .log file is needed + this.configureFfmpegCall(input, '/dev/null', options) .addOptions('-pass', '1') .addOptions('-passlogfile', output) .addOptions('-f null') - .output('/dev/null') // first pass output is not saved as only the .log file is needed - .on('error', (err, stdout, stderr) => { - this.logger.error(stderr); - reject(err); - }) + .on('error', reject) .on('end', () => { // second pass - ffmpeg(input, { niceness: 10 }) - .inputOptions(options.inputOptions) - .outputOptions(options.outputOptions) + this.configureFfmpegCall(input, output, options) .addOptions('-pass', '2') .addOptions('-passlogfile', output) - .output(output) - .on('error', (err, stdout, stderr) => { - this.logger.error(stderr); - reject(err); - }) + .on('error', reject) .on('end', () => fs.unlink(`${output}-0.log`)) .on('end', () => fs.rm(`${output}-0.log.mbtree`, { force: true })) .on('end', resolve) @@ -122,6 +113,20 @@ export class MediaRepository implements IMediaRepository { }); } + configureFfmpegCall(input: string, output: string | Writable, options: TranscodeOptions) { + return ffmpeg(input, { niceness: 10 }) + .setFfmpegPath(options.ffmpegPath || 'ffmpeg') + .inputOptions(options.inputOptions) + .outputOptions(options.outputOptions) + .output(output) + .on('error', (err, stdout, stderr) => this.logger.error(stderr || err)); + } + + chainPath(existing: string, path: string) { + const sep = existing.endsWith(':') ? '' : ':'; + return `${existing}${sep}${path}`; + } + async generateThumbhash(imagePath: string): Promise { const maxSize = 100; diff --git a/web/src/api/open-api/api.ts b/web/src/api/open-api/api.ts index b1714e276..76c16e8bc 100644 --- a/web/src/api/open-api/api.ts +++ b/web/src/api/open-api/api.ts @@ -3964,6 +3964,7 @@ export const TranscodeHWAccel = { Nvenc: 'nvenc', Qsv: 'qsv', Vaapi: 'vaapi', + Rkmpp: 'rkmpp', Disabled: 'disabled' } as const; diff --git a/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte b/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte index fc669b752..96fceb61e 100644 --- a/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte +++ b/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte @@ -281,6 +281,10 @@ value: TranscodeHWAccel.Vaapi, text: 'VAAPI', }, + { + value: TranscodeHWAccel.Rkmpp, + text: 'RKMPP (only on Rockchip SOCs)', + }, { value: TranscodeHWAccel.Disabled, text: 'Disabled',