beenull/immich
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

feat(server): hardware video acceleration for Rockchip SOCs via RKMPP (#4645)

Browse source 
* feat(server): hardware video acceleration for Rockchip SOCs via RKMPP

* add tests

* use LD_LIBRARY_PATH for custom ffmpeg

* incorporate review feedback

* code re-use for ffmpeg call

* review feedback
This commit is contained in:
Fynn Petersen-Frey 2023-10-30 15:39:37 +01:00 committed by GitHub
parent c54a188154
commit ce04e9e07a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 230 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
docker/hwaccel-rkmpp.yml Normal file
View file

@ -0,0 +1,24 @@
version: "3.8"
# Hardware acceleration for transcoding using RKMPP for Rockchip SOCs
# This is only needed if you want to use hardware acceleration for transcoding.
# Supported host OS is Ubuntu Jammy 22.04 with custom ffmpeg from ppa:liujianfeng1994/rockchip-multimedia
services:
hwaccel:
security_opt: # enables full access to /sys and /proc, still far better than privileged: true
- systempaths=unconfined
- apparmor=unconfined
group_add:
- video
devices:
- /dev/rga:/dev/rga
- /dev/dri:/dev/dri
- /dev/dma_heap:/dev/dma_heap
- /dev/mpp_service:/dev/mpp_service
volumes:
- /usr/bin/ffmpeg:/usr/bin/ffmpeg_mpp:ro
- /lib/aarch64-linux-gnu:/lib/ffmpeg-mpp:ro
- /lib/aarch64-linux-gnu/libblas.so.3:/lib/ffmpeg-mpp/libblas.so.3:ro # symlink is resolved by mounting
- /lib/aarch64-linux-gnu/liblapack.so.3:/lib/ffmpeg-mpp/liblapack.so.3:ro # symlink is resolved by mounting
- /lib/aarch64-linux-gnu/pulseaudio/libpulsecommon-15.99.so:/lib/ffmpeg-mpp/libpulsecommon-15.99.so:ro

3
mobile/openapi/lib/model/transcode_hw_accel.dart generated
View file

@ -26,6 +26,7 @@ class TranscodeHWAccel {
static const nvenc = TranscodeHWAccel._(r'nvenc');
static const qsv = TranscodeHWAccel._(r'qsv');
static const vaapi = TranscodeHWAccel._(r'vaapi');
static const rkmpp = TranscodeHWAccel._(r'rkmpp');
static const disabled = TranscodeHWAccel._(r'disabled');
/// List of all possible values in this [enum][TranscodeHWAccel].
@ -33,6 +34,7 @@ class TranscodeHWAccel {
nvenc,
qsv,
vaapi,
rkmpp,
disabled,
];
@ -75,6 +77,7 @@ class TranscodeHWAccelTypeTransformer {
case r'nvenc': return TranscodeHWAccel.nvenc;
case r'qsv': return TranscodeHWAccel.qsv;
case r'vaapi': return TranscodeHWAccel.vaapi;
case r'rkmpp': return TranscodeHWAccel.rkmpp;
case r'disabled': return TranscodeHWAccel.disabled;
default:
if (!allowNull) {

1
server/immich-openapi-specs.json
View file

@ -8607,6 +8607,7 @@
"nvenc",
"qsv",
"vaapi",
"rkmpp",
"disabled"
],
"type": "string"

77
server/src/domain/media/media.service.spec.ts
View file

@ -1508,6 +1508,83 @@ describe(MediaService.name, () => {
await expect(sut.handleVideoConversion({ id: assetStub.video.id })).resolves.toEqual(false);
expect(mediaMock.transcode).not.toHaveBeenCalled();
});
it('should set vbr options for rkmpp when max bitrate is enabled', async () => {
storageMock.readdir.mockResolvedValue(['renderD128']);
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.RKMPP },
{ key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '10000k' },
{ key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.HEVC },
]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
`-c:v hevc_rkmpp_encoder`,
'-c:a aac',
'-movflags faststart',
'-fps_mode passthrough',
'-map 0:0',
'-map 0:1',
'-g 256',
'-v verbose',
'-level 153',
'-rc_mode 3',
'-quality_min 0',
'-quality_max 100',
'-b:v 10000k',
'-width 1280',
'-height 720',
],
twoPass: false,
ffmpegPath: 'ffmpeg_mpp',
ldLibraryPath: '/lib/aarch64-linux-gnu:/lib/ffmpeg-mpp',
},
);
});
it('should set cqp options for rkmpp when max bitrate is disabled', async () => {
storageMock.readdir.mockResolvedValue(['renderD128']);
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.RKMPP },
{ key: SystemConfigKey.FFMPEG_CRF, value: 30 },
{ key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '0' },
]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
`-c:v h264_rkmpp_encoder`,
'-c:a aac',
'-movflags faststart',
'-fps_mode passthrough',
'-map 0:0',
'-map 0:1',
'-g 256',
'-v verbose',
'-level 51',
'-rc_mode 2',
'-quality_min 51',
'-quality_max 51',
'-width 1280',
'-height 720',
],
twoPass: false,
ffmpegPath: 'ffmpeg_mpp',
ldLibraryPath: '/lib/aarch64-linux-gnu:/lib/ffmpeg-mpp',
},
);
});
});
it('should tonemap when policy is required and video is hdr', async () => {

15
server/src/domain/media/media.service.ts
View file

@ -26,7 +26,16 @@ import {
import { StorageCore, StorageFolder } from '../storage';
import { SystemConfigFFmpegDto } from '../system-config';
import { SystemConfigCore } from '../system-config/system-config.core';
import { H264Config, HEVCConfig, NVENCConfig, QSVConfig, ThumbnailConfig, VAAPIConfig, VP9Config } from './media.util';
import {
H264Config,
HEVCConfig,
NVENCConfig,
QSVConfig,
RKMPPConfig,
ThumbnailConfig,
VAAPIConfig,
VP9Config,
} from './media.util';
@Injectable()
export class MediaService {
@ -352,6 +361,10 @@ export class MediaService {
devices = await this.storageRepository.readdir('/dev/dri');
handler = new VAAPIConfig(config, devices);
break;
case TranscodeHWAccel.RKMPP:
devices = await this.storageRepository.readdir('/dev/dri');
handler = new RKMPPConfig(config, devices);
break;
default:
throw new UnsupportedMediaTypeException(`${config.accel.toUpperCase()} acceleration is unsupported`);
}

72
server/src/domain/media/media.util.ts
View file

@ -143,6 +143,13 @@ class BaseConfig implements VideoCodecSWConfig {
return this.isVideoVertical(videoStream) ? `${targetResolution}:-${mult}` : `-${mult}:${targetResolution}`;
}
getSize(videoStream: VideoStreamInfo) {
const smaller = this.getTargetResolution(videoStream);
const factor = Math.max(videoStream.height, videoStream.width) / Math.min(videoStream.height, videoStream.width);
const larger = Math.round(smaller * factor);
return this.isVideoVertical(videoStream) ? { width: smaller, height: larger } : { width: larger, height: smaller };
}
isVideoRotated(videoStream: VideoStreamInfo) {
return Math.abs(videoStream.rotation) === 90;
}
@ -555,3 +562,68 @@ export class VAAPIConfig extends BaseHWConfig {
return this.config.cqMode !== CQMode.ICQ || this.config.targetVideoCodec === VideoCodec.VP9;
}
}
export class RKMPPConfig extends BaseHWConfig {
getOptions(videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo): TranscodeOptions {
const options = super.getOptions(videoStream, audioStream);
options.ffmpegPath = 'ffmpeg_mpp';
options.ldLibraryPath = '/lib/aarch64-linux-gnu:/lib/ffmpeg-mpp';
options.outputOptions.push(...this.getSizeOptions(videoStream));
return options;
}
eligibleForTwoPass(): boolean {
return false;
}
getBaseInputOptions() {
if (this.devices.length === 0) {
throw Error('No RKMPP device found');
}
return [];
}
getFilterOptions(videoStream: VideoStreamInfo) {
return this.shouldToneMap(videoStream) ? this.getToneMapping() : [];
}
getSizeOptions(videoStream: VideoStreamInfo) {
if (this.shouldScale(videoStream)) {
const { width, height } = this.getSize(videoStream);
return [`-width ${width}`, `-height ${height}`];
}
return [];
}
getPresetOptions() {
switch (this.config.targetVideoCodec) {
case VideoCodec.H264:
// from ffmpeg_mpp help, commonly referred to as H264 level 5.1
return ['-level 51'];
case VideoCodec.HEVC:
// from ffmpeg_mpp help, commonly referred to as HEVC level 5.1
return ['-level 153'];
default:
throw Error(`Incompatible video codec for RKMPP: ${this.config.targetVideoCodec}`);
}
}
getBitrateOptions() {
const bitrate = this.getMaxBitrateValue();
if (bitrate > 0) {
return ['-rc_mode 3', '-quality_min 0', '-quality_max 100', `-b:v ${bitrate}${this.getBitrateUnit()}`];
} else {
// convert CQP from 51-10 to 0-100, values below 10 are set to 10
const quality = Math.floor(125 - Math.max(this.config.crf, 10) * (125 / 51));
return ['-rc_mode 2', `-quality_min ${quality}`, `-quality_max ${quality}`];
}
}
getSupportedCodecs() {
return [VideoCodec.H264, VideoCodec.HEVC];
}
getVideoCodec(): string {
return `${this.config.targetVideoCodec}_rkmpp_encoder`;
}
}

2
server/src/domain/repositories/media.repository.ts
View file

@ -51,6 +51,8 @@ export interface TranscodeOptions {
inputOptions: string[];
outputOptions: string[];
twoPass: boolean;
ffmpegPath?: string;
ldLibraryPath?: string;
}
export interface BitrateDistribution {

1
server/src/infra/entities/system-config.entity.ts
View file

@ -119,6 +119,7 @@ export enum TranscodeHWAccel {
NVENC = 'nvenc',
QSV = 'qsv',
VAAPI = 'vaapi',
RKMPP = 'rkmpp',
DISABLED = 'disabled',
}

57
server/src/infra/repositories/media.repository.ts
View file

@ -70,16 +70,18 @@ export class MediaRepository implements IMediaRepository {
transcode(input: string, output: string | Writable, options: TranscodeOptions): Promise<void> {
if (!options.twoPass) {
return new Promise((resolve, reject) => {
ffmpeg(input, { niceness: 10 })
.inputOptions(options.inputOptions)
.outputOptions(options.outputOptions)
.output(output)
.on('error', (err, stdout, stderr) => {
this.logger.error(stderr);
reject(err);
})
.on('end', resolve)
.run();
const oldLdLibraryPath = process.env.LD_LIBRARY_PATH;
if (options.ldLibraryPath) {
// fluent ffmpeg does not allow to set environment variables, so we do it manually
process.env.LD_LIBRARY_PATH = this.chainPath(oldLdLibraryPath || '', options.ldLibraryPath);
}
try {
this.configureFfmpegCall(input, output, options).on('error', reject).on('end', resolve).run();
} finally {
if (options.ldLibraryPath) {
process.env.LD_LIBRARY_PATH = oldLdLibraryPath;
}
}
});
}
@ -90,29 +92,18 @@ export class MediaRepository implements IMediaRepository {
// two-pass allows for precise control of bitrate at the cost of running twice
// recommended for vp9 for better quality and compression
return new Promise((resolve, reject) => {
ffmpeg(input, { niceness: 10 })
.inputOptions(options.inputOptions)
.outputOptions(options.outputOptions)
// first pass output is not saved as only the .log file is needed
this.configureFfmpegCall(input, '/dev/null', options)
.addOptions('-pass', '1')
.addOptions('-passlogfile', output)
.addOptions('-f null')
.output('/dev/null') // first pass output is not saved as only the .log file is needed
.on('error', (err, stdout, stderr) => {
this.logger.error(stderr);
reject(err);
})
.on('error', reject)
.on('end', () => {
// second pass
ffmpeg(input, { niceness: 10 })
.inputOptions(options.inputOptions)
.outputOptions(options.outputOptions)
this.configureFfmpegCall(input, output, options)
.addOptions('-pass', '2')
.addOptions('-passlogfile', output)
.output(output)
.on('error', (err, stdout, stderr) => {
this.logger.error(stderr);
reject(err);
})
.on('error', reject)
.on('end', () => fs.unlink(`${output}-0.log`))
.on('end', () => fs.rm(`${output}-0.log.mbtree`, { force: true }))
.on('end', resolve)
@ -122,6 +113,20 @@ export class MediaRepository implements IMediaRepository {
});
}
configureFfmpegCall(input: string, output: string | Writable, options: TranscodeOptions) {
return ffmpeg(input, { niceness: 10 })
.setFfmpegPath(options.ffmpegPath || 'ffmpeg')
.inputOptions(options.inputOptions)
.outputOptions(options.outputOptions)
.output(output)
.on('error', (err, stdout, stderr) => this.logger.error(stderr || err));
}
chainPath(existing: string, path: string) {
const sep = existing.endsWith(':') ? '' : ':';
return `${existing}${sep}${path}`;
}
async generateThumbhash(imagePath: string): Promise<Buffer> {
const maxSize = 100;

1
web/src/api/open-api/api.ts generated
View file

@ -3964,6 +3964,7 @@ export const TranscodeHWAccel = {
Nvenc: 'nvenc',
Qsv: 'qsv',
Vaapi: 'vaapi',
Rkmpp: 'rkmpp',
Disabled: 'disabled'
} as const;

4
web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte
View file

@ -281,6 +281,10 @@
value: TranscodeHWAccel.Vaapi,
text: 'VAAPI',
},
{
value: TranscodeHWAccel.Rkmpp,
text: 'RKMPP (only on Rockchip SOCs)',
},
{
value: TranscodeHWAccel.Disabled,
text: 'Disabled',