feat(server): advanced settings for transcoding (#3775)

* set stream with `-map` flag

* updated tests

* fixed audio stream mapping

* added bframe setting to config

* updated api

* added b-frame option in dashboard

* updated tests and formatting

* "Advanced" section for FFmpeg with extra options

* updated api

* updated tests and formatting

* styling

* made vp9 bitstream filters conditional on b-frames

* fixed gop size condition

* add cq override

* simplified isEdited conditions

* simplified conditional flow for cq mode

* fixed dto

* clarified cq mode in description

* formatting

* added npl setting

* Adjusted b-frame title and description

* fixed rebase

* changed defaults for pascal compatibility, added temporal aq setting

* updated api

* added temporal aq to ui

* polished dashboard

* formatting

cli/src/api/open-api/api.ts

@@ -909,6 +909,21 @@ export const CLIPMode = {
 export type CLIPMode = typeof CLIPMode[keyof typeof CLIPMode];
 
 
+/**
+ * 
+ * @export
+ * @enum {string}
+ */
+
+export const CQMode = {
+    Auto: 'auto',
+    Cqp: 'cqp',
+    Icq: 'icq'
+} as const;
+
+export type CQMode = typeof CQMode[keyof typeof CQMode];
+
+
 /**
  * 
  * @export
@@ -2812,24 +2827,54 @@ export interface SystemConfigFFmpegDto {
      * @memberof SystemConfigFFmpegDto
      */
     'accel': TranscodeHWAccel;
+    /**
+     * 
+     * @type {number}
+     * @memberof SystemConfigFFmpegDto
+     */
+    'bframes': number;
+    /**
+     * 
+     * @type {CQMode}
+     * @memberof SystemConfigFFmpegDto
+     */
+    'cqMode': CQMode;
     /**
      * 
      * @type {number}
      * @memberof SystemConfigFFmpegDto
      */
     'crf': number;
+    /**
+     * 
+     * @type {number}
+     * @memberof SystemConfigFFmpegDto
+     */
+    'gopSize': number;
     /**
      * 
      * @type {string}
      * @memberof SystemConfigFFmpegDto
      */
     'maxBitrate': string;
+    /**
+     * 
+     * @type {number}
+     * @memberof SystemConfigFFmpegDto
+     */
+    'npl': number;
     /**
      * 
      * @type {string}
      * @memberof SystemConfigFFmpegDto
      */
     'preset': string;
+    /**
+     * 
+     * @type {number}
+     * @memberof SystemConfigFFmpegDto
+     */
+    'refs': number;
     /**
      * 
      * @type {AudioCodec}
@@ -2848,6 +2893,12 @@ export interface SystemConfigFFmpegDto {
      * @memberof SystemConfigFFmpegDto
      */
     'targetVideoCodec': VideoCodec;
+    /**
+     * 
+     * @type {boolean}
+     * @memberof SystemConfigFFmpegDto
+     */
+    'temporalAQ': boolean;
     /**
      * 
      * @type {number}

mobile/openapi/.openapi-generator/FILES

@@ -37,6 +37,7 @@ doc/BulkIdResponseDto.md
 doc/BulkIdsDto.md
 doc/CLIPConfig.md
 doc/CLIPMode.md
+doc/CQMode.md
 doc/ChangePasswordDto.md
 doc/CheckDuplicateAssetDto.md
 doc/CheckDuplicateAssetResponseDto.md
@@ -198,6 +199,7 @@ lib/model/check_existing_assets_response_dto.dart
 lib/model/classification_config.dart
 lib/model/clip_config.dart
 lib/model/clip_mode.dart
+lib/model/cq_mode.dart
 lib/model/create_album_dto.dart
 lib/model/create_profile_image_response_dto.dart
 lib/model/create_tag_dto.dart
@@ -324,6 +326,7 @@ test/check_existing_assets_response_dto_test.dart
 test/classification_config_test.dart
 test/clip_config_test.dart
 test/clip_mode_test.dart
+test/cq_mode_test.dart
 test/create_album_dto_test.dart
 test/create_profile_image_response_dto_test.dart
 test/create_tag_dto_test.dart

mobile/openapi/README.md

@@ -211,6 +211,7 @@ Class | Method | HTTP request | Description
  - [BulkIdsDto](doc//BulkIdsDto.md)
  - [CLIPConfig](doc//CLIPConfig.md)
  - [CLIPMode](doc//CLIPMode.md)
+ - [CQMode](doc//CQMode.md)
  - [ChangePasswordDto](doc//ChangePasswordDto.md)
  - [CheckDuplicateAssetDto](doc//CheckDuplicateAssetDto.md)
  - [CheckDuplicateAssetResponseDto](doc//CheckDuplicateAssetResponseDto.md)

mobile/openapi/doc/CQMode.md

@@ -0,0 +1,14 @@
+# openapi.model.CQMode
+
+## Load the model package
+```dart
+import 'package:openapi/api.dart';
+```
+
+## Properties
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+
+

mobile/openapi/doc/SystemConfigFFmpegDto.md

@@ -9,12 +9,18 @@ import 'package:openapi/api.dart';
 Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
 **accel** | [**TranscodeHWAccel**](TranscodeHWAccel.md) |  | 
+**bframes** | **int** |  | 
+**cqMode** | [**CQMode**](CQMode.md) |  | 
 **crf** | **int** |  | 
+**gopSize** | **int** |  | 
 **maxBitrate** | **String** |  | 
+**npl** | **int** |  | 
 **preset** | **String** |  | 
+**refs** | **int** |  | 
 **targetAudioCodec** | [**AudioCodec**](AudioCodec.md) |  | 
 **targetResolution** | **String** |  | 
 **targetVideoCodec** | [**VideoCodec**](VideoCodec.md) |  | 
+**temporalAQ** | **bool** |  | 
 **threads** | **int** |  | 
 **tonemap** | [**ToneMapping**](ToneMapping.md) |  | 
 **transcode** | [**TranscodePolicy**](TranscodePolicy.md) |  | 

mobile/openapi/lib/api.dart

@@ -73,6 +73,7 @@ part 'model/bulk_id_response_dto.dart';
 part 'model/bulk_ids_dto.dart';
 part 'model/clip_config.dart';
 part 'model/clip_mode.dart';
+part 'model/cq_mode.dart';
 part 'model/change_password_dto.dart';
 part 'model/check_duplicate_asset_dto.dart';
 part 'model/check_duplicate_asset_response_dto.dart';

mobile/openapi/lib/api_client.dart

@@ -239,6 +239,8 @@ class ApiClient {
           return CLIPConfig.fromJson(value);
         case 'CLIPMode':
           return CLIPModeTypeTransformer().decode(value);
+        case 'CQMode':
+          return CQModeTypeTransformer().decode(value);
         case 'ChangePasswordDto':
           return ChangePasswordDto.fromJson(value);
         case 'CheckDuplicateAssetDto':

mobile/openapi/lib/api_helper.dart

@@ -67,6 +67,9 @@ String parameterToString(dynamic value) {
   if (value is CLIPMode) {
     return CLIPModeTypeTransformer().encode(value).toString();
   }
+  if (value is CQMode) {
+    return CQModeTypeTransformer().encode(value).toString();
+  }
   if (value is DeleteAssetStatus) {
     return DeleteAssetStatusTypeTransformer().encode(value).toString();
   }

mobile/openapi/lib/model/cq_mode.dart

@@ -0,0 +1,88 @@
+//
+// AUTO-GENERATED FILE, DO NOT MODIFY!
+//
+// @dart=2.12
+
+// ignore_for_file: unused_element, unused_import
+// ignore_for_file: always_put_required_named_parameters_first
+// ignore_for_file: constant_identifier_names
+// ignore_for_file: lines_longer_than_80_chars
+
+part of openapi.api;
+
+
+class CQMode {
+  /// Instantiate a new enum with the provided [value].
+  const CQMode._(this.value);
+
+  /// The underlying value of this enum member.
+  final String value;
+
+  @override
+  String toString() => value;
+
+  String toJson() => value;
+
+  static const auto = CQMode._(r'auto');
+  static const cqp = CQMode._(r'cqp');
+  static const icq = CQMode._(r'icq');
+
+  /// List of all possible values in this [enum][CQMode].
+  static const values = <CQMode>[
+    auto,
+    cqp,
+    icq,
+  ];
+
+  static CQMode? fromJson(dynamic value) => CQModeTypeTransformer().decode(value);
+
+  static List<CQMode>? listFromJson(dynamic json, {bool growable = false,}) {
+    final result = <CQMode>[];
+    if (json is List && json.isNotEmpty) {
+      for (final row in json) {
+        final value = CQMode.fromJson(row);
+        if (value != null) {
+          result.add(value);
+        }
+      }
+    }
+    return result.toList(growable: growable);
+  }
+}
+
+/// Transformation class that can [encode] an instance of [CQMode] to String,
+/// and [decode] dynamic data back to [CQMode].
+class CQModeTypeTransformer {
+  factory CQModeTypeTransformer() => _instance ??= const CQModeTypeTransformer._();
+
+  const CQModeTypeTransformer._();
+
+  String encode(CQMode data) => data.value;
+
+  /// Decodes a [dynamic value][data] to a CQMode.
+  ///
+  /// If [allowNull] is true and the [dynamic value][data] cannot be decoded successfully,
+  /// then null is returned. However, if [allowNull] is false and the [dynamic value][data]
+  /// cannot be decoded successfully, then an [UnimplementedError] is thrown.
+  ///
+  /// The [allowNull] is very handy when an API changes and a new enum value is added or removed,
+  /// and users are still using an old app with the old code.
+  CQMode? decode(dynamic data, {bool allowNull = true}) {
+    if (data != null) {
+      switch (data) {
+        case r'auto': return CQMode.auto;
+        case r'cqp': return CQMode.cqp;
+        case r'icq': return CQMode.icq;
+        default:
+          if (!allowNull) {
+            throw ArgumentError('Unknown enum value to decode: $data');
+          }
+      }
+    }
+    return null;
+  }
+
+  /// Singleton [CQModeTypeTransformer] instance.
+  static CQModeTypeTransformer? _instance;
+}
+

mobile/openapi/lib/model/system_config_f_fmpeg_dto.dart

@@ -14,12 +14,18 @@ class SystemConfigFFmpegDto {
   /// Returns a new [SystemConfigFFmpegDto] instance.
   SystemConfigFFmpegDto({
     required this.accel,
+    required this.bframes,
+    required this.cqMode,
     required this.crf,
+    required this.gopSize,
     required this.maxBitrate,
+    required this.npl,
     required this.preset,
+    required this.refs,
     required this.targetAudioCodec,
     required this.targetResolution,
     required this.targetVideoCodec,
+    required this.temporalAQ,
     required this.threads,
     required this.tonemap,
     required this.transcode,
@@ -28,18 +34,30 @@ class SystemConfigFFmpegDto {
 
   TranscodeHWAccel accel;
 
+  int bframes;
+
+  CQMode cqMode;
+
   int crf;
 
+  int gopSize;
+
   String maxBitrate;
 
+  int npl;
+
   String preset;
 
+  int refs;
+
   AudioCodec targetAudioCodec;
 
   String targetResolution;
 
   VideoCodec targetVideoCodec;
 
+  bool temporalAQ;
+
   int threads;
 
   ToneMapping tonemap;
@@ -51,12 +69,18 @@ class SystemConfigFFmpegDto {
   @override
   bool operator ==(Object other) => identical(this, other) || other is SystemConfigFFmpegDto &&
      other.accel == accel &&
+     other.bframes == bframes &&
+     other.cqMode == cqMode &&
      other.crf == crf &&
+     other.gopSize == gopSize &&
      other.maxBitrate == maxBitrate &&
+     other.npl == npl &&
      other.preset == preset &&
+     other.refs == refs &&
      other.targetAudioCodec == targetAudioCodec &&
      other.targetResolution == targetResolution &&
      other.targetVideoCodec == targetVideoCodec &&
+     other.temporalAQ == temporalAQ &&
      other.threads == threads &&
      other.tonemap == tonemap &&
      other.transcode == transcode &&
@@ -66,29 +90,41 @@ class SystemConfigFFmpegDto {
   int get hashCode =>
     // ignore: unnecessary_parenthesis
     (accel.hashCode) +
+    (bframes.hashCode) +
+    (cqMode.hashCode) +
     (crf.hashCode) +
+    (gopSize.hashCode) +
     (maxBitrate.hashCode) +
+    (npl.hashCode) +
     (preset.hashCode) +
+    (refs.hashCode) +
     (targetAudioCodec.hashCode) +
     (targetResolution.hashCode) +
     (targetVideoCodec.hashCode) +
+    (temporalAQ.hashCode) +
     (threads.hashCode) +
     (tonemap.hashCode) +
     (transcode.hashCode) +
     (twoPass.hashCode);
 
   @override
-  String toString() => 'SystemConfigFFmpegDto[accel=$accel, crf=$crf, maxBitrate=$maxBitrate, preset=$preset, targetAudioCodec=$targetAudioCodec, targetResolution=$targetResolution, targetVideoCodec=$targetVideoCodec, threads=$threads, tonemap=$tonemap, transcode=$transcode, twoPass=$twoPass]';
+  String toString() => 'SystemConfigFFmpegDto[accel=$accel, bframes=$bframes, cqMode=$cqMode, crf=$crf, gopSize=$gopSize, maxBitrate=$maxBitrate, npl=$npl, preset=$preset, refs=$refs, targetAudioCodec=$targetAudioCodec, targetResolution=$targetResolution, targetVideoCodec=$targetVideoCodec, temporalAQ=$temporalAQ, threads=$threads, tonemap=$tonemap, transcode=$transcode, twoPass=$twoPass]';
 
   Map<String, dynamic> toJson() {
     final json = <String, dynamic>{};
       json[r'accel'] = this.accel;
+      json[r'bframes'] = this.bframes;
+      json[r'cqMode'] = this.cqMode;
       json[r'crf'] = this.crf;
+      json[r'gopSize'] = this.gopSize;
       json[r'maxBitrate'] = this.maxBitrate;
+      json[r'npl'] = this.npl;
       json[r'preset'] = this.preset;
+      json[r'refs'] = this.refs;
       json[r'targetAudioCodec'] = this.targetAudioCodec;
       json[r'targetResolution'] = this.targetResolution;
       json[r'targetVideoCodec'] = this.targetVideoCodec;
+      json[r'temporalAQ'] = this.temporalAQ;
       json[r'threads'] = this.threads;
       json[r'tonemap'] = this.tonemap;
       json[r'transcode'] = this.transcode;
@@ -105,12 +141,18 @@ class SystemConfigFFmpegDto {
 
       return SystemConfigFFmpegDto(
         accel: TranscodeHWAccel.fromJson(json[r'accel'])!,
+        bframes: mapValueOfType<int>(json, r'bframes')!,
+        cqMode: CQMode.fromJson(json[r'cqMode'])!,
         crf: mapValueOfType<int>(json, r'crf')!,
+        gopSize: mapValueOfType<int>(json, r'gopSize')!,
         maxBitrate: mapValueOfType<String>(json, r'maxBitrate')!,
+        npl: mapValueOfType<int>(json, r'npl')!,
         preset: mapValueOfType<String>(json, r'preset')!,
+        refs: mapValueOfType<int>(json, r'refs')!,
         targetAudioCodec: AudioCodec.fromJson(json[r'targetAudioCodec'])!,
         targetResolution: mapValueOfType<String>(json, r'targetResolution')!,
         targetVideoCodec: VideoCodec.fromJson(json[r'targetVideoCodec'])!,
+        temporalAQ: mapValueOfType<bool>(json, r'temporalAQ')!,
         threads: mapValueOfType<int>(json, r'threads')!,
         tonemap: ToneMapping.fromJson(json[r'tonemap'])!,
         transcode: TranscodePolicy.fromJson(json[r'transcode'])!,
@@ -163,12 +205,18 @@ class SystemConfigFFmpegDto {
   /// The list of required keys that must be present in a JSON.
   static const requiredKeys = <String>{
     'accel',
+    'bframes',
+    'cqMode',
     'crf',
+    'gopSize',
     'maxBitrate',
+    'npl',
     'preset',
+    'refs',
     'targetAudioCodec',
     'targetResolution',
     'targetVideoCodec',
+    'temporalAQ',
     'threads',
     'tonemap',
     'transcode',

mobile/openapi/test/cq_mode_test.dart

@@ -0,0 +1,21 @@
+//
+// AUTO-GENERATED FILE, DO NOT MODIFY!
+//
+// @dart=2.12
+
+// ignore_for_file: unused_element, unused_import
+// ignore_for_file: always_put_required_named_parameters_first
+// ignore_for_file: constant_identifier_names
+// ignore_for_file: lines_longer_than_80_chars
+
+import 'package:openapi/api.dart';
+import 'package:test/test.dart';
+
+// tests for CQMode
+void main() {
+
+  group('test CQMode', () {
+
+  });
+
+}

mobile/openapi/test/system_config_f_fmpeg_dto_test.dart

@@ -21,21 +21,46 @@ void main() {
       // TODO
     });
 
+    // int bframes
+    test('to test the property `bframes`', () async {
+      // TODO
+    });
+
+    // CQMode cqMode
+    test('to test the property `cqMode`', () async {
+      // TODO
+    });
+
     // int crf
     test('to test the property `crf`', () async {
       // TODO
     });
 
+    // int gopSize
+    test('to test the property `gopSize`', () async {
+      // TODO
+    });
+
     // String maxBitrate
     test('to test the property `maxBitrate`', () async {
       // TODO
     });
 
+    // int npl
+    test('to test the property `npl`', () async {
+      // TODO
+    });
+
     // String preset
     test('to test the property `preset`', () async {
       // TODO
     });
 
+    // int refs
+    test('to test the property `refs`', () async {
+      // TODO
+    });
+
     // AudioCodec targetAudioCodec
     test('to test the property `targetAudioCodec`', () async {
       // TODO
@@ -51,6 +76,11 @@ void main() {
       // TODO
     });
 
+    // bool temporalAQ
+    test('to test the property `temporalAQ`', () async {
+      // TODO
+    });
+
     // int threads
     test('to test the property `threads`', () async {
       // TODO

server/immich-openapi-specs.json

@@ -5415,6 +5415,14 @@
         ],
         "type": "string"
       },
+      "CQMode": {
+        "enum": [
+          "auto",
+          "cqp",
+          "icq"
+        ],
+        "type": "string"
+      },
       "ChangePasswordDto": {
         "properties": {
           "newPassword": {
@@ -7001,15 +7009,30 @@
           "accel": {
             "$ref": "#/components/schemas/TranscodeHWAccel"
           },
+          "bframes": {
+            "type": "integer"
+          },
+          "cqMode": {
+            "$ref": "#/components/schemas/CQMode"
+          },
           "crf": {
             "type": "integer"
           },
+          "gopSize": {
+            "type": "integer"
+          },
           "maxBitrate": {
             "type": "string"
           },
+          "npl": {
+            "type": "integer"
+          },
           "preset": {
             "type": "string"
           },
+          "refs": {
+            "type": "integer"
+          },
           "targetAudioCodec": {
             "$ref": "#/components/schemas/AudioCodec"
           },
@@ -7019,6 +7042,9 @@
           "targetVideoCodec": {
             "$ref": "#/components/schemas/VideoCodec"
           },
+          "temporalAQ": {
+            "type": "boolean"
+          },
           "threads": {
             "type": "integer"
           },
@@ -7037,12 +7063,18 @@
           "threads",
           "targetVideoCodec",
           "targetAudioCodec",
+          "bframes",
+          "refs",
+          "gopSize",
+          "npl",
+          "cqMode",
           "transcode",
           "accel",
           "tonemap",
           "preset",
           "targetResolution",
           "maxBitrate",
+          "temporalAQ",
           "twoPass"
         ],
         "type": "object"

server/src/domain/media/media.service.spec.ts

@@ -311,10 +311,12 @@ describe(MediaService.name, () => {
         {
           inputOptions: [],
           outputOptions: [
-            '-c:v:0 h264',
-            '-c:a:0 aac',
+            '-c:v h264',
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
             '-v verbose',
             '-vf format=yuv420p',
             '-preset ultrafast',
@@ -350,10 +352,12 @@ describe(MediaService.name, () => {
         {
           inputOptions: [],
           outputOptions: [
-            '-c:v:0 h264',
-            '-c:a:0 aac',
+            '-c:v h264',
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
             '-v verbose',
             '-vf format=yuv420p',
             '-preset ultrafast',
@@ -374,10 +378,12 @@ describe(MediaService.name, () => {
         {
           inputOptions: [],
           outputOptions: [
-            '-c:v:0 h264',
-            '-c:a:0 aac',
+            '-c:v h264',
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
             '-v verbose',
             '-vf scale=-2:720,format=yuv420p',
             '-preset ultrafast',
@@ -401,10 +407,12 @@ describe(MediaService.name, () => {
         {
           inputOptions: [],
           outputOptions: [
-            '-c:v:0 h264',
-            '-c:a:0 aac',
+            '-c:v h264',
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
             '-v verbose',
             '-vf format=yuv420p',
             '-preset ultrafast',
@@ -426,10 +434,12 @@ describe(MediaService.name, () => {
         {
           inputOptions: [],
           outputOptions: [
-            '-c:v:0 h264',
-            '-c:a:0 aac',
+            '-c:v h264',
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
             '-v verbose',
             '-vf scale=720:-2,format=yuv420p',
             '-preset ultrafast',
@@ -451,10 +461,12 @@ describe(MediaService.name, () => {
         {
           inputOptions: [],
           outputOptions: [
-            '-c:v:0 h264',
-            '-c:a:0 aac',
+            '-c:v h264',
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
             '-v verbose',
             '-vf scale=-2:720,format=yuv420p',
             '-preset ultrafast',
@@ -476,10 +488,12 @@ describe(MediaService.name, () => {
         {
           inputOptions: [],
           outputOptions: [
-            '-c:v:0 h264',
-            '-c:a:0 aac',
+            '-c:v h264',
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
             '-v verbose',
             '-vf scale=-2:720,format=yuv420p',
             '-preset ultrafast',
@@ -525,10 +539,12 @@ describe(MediaService.name, () => {
         {
           inputOptions: [],
           outputOptions: [
-            '-c:v:0 h264',
-            '-c:a:0 aac',
+            '-c:v h264',
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
             '-v verbose',
             '-vf scale=-2:720,format=yuv420p',
             '-preset ultrafast',
@@ -555,10 +571,12 @@ describe(MediaService.name, () => {
         {
           inputOptions: [],
           outputOptions: [
-            '-c:v:0 h264',
-            '-c:a:0 aac',
+            '-c:v h264',
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
             '-v verbose',
             '-vf scale=-2:720,format=yuv420p',
             '-preset ultrafast',
@@ -582,10 +600,12 @@ describe(MediaService.name, () => {
         {
           inputOptions: [],
           outputOptions: [
-            '-c:v:0 h264',
-            '-c:a:0 aac',
+            '-c:v h264',
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
             '-v verbose',
             '-vf scale=-2:720,format=yuv420p',
             '-preset ultrafast',
@@ -611,10 +631,12 @@ describe(MediaService.name, () => {
         {
           inputOptions: [],
           outputOptions: [
-            '-c:v:0 vp9',
-            '-c:a:0 aac',
+            '-c:v vp9',
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
             '-v verbose',
             '-vf scale=-2:720,format=yuv420p',
             '-cpu-used 5',
@@ -642,10 +664,12 @@ describe(MediaService.name, () => {
         {
           inputOptions: [],
           outputOptions: [
-            '-c:v:0 vp9',
-            '-c:a:0 aac',
+            '-c:v vp9',
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
             '-v verbose',
             '-vf scale=-2:720,format=yuv420p',
             '-cpu-used 2',
@@ -672,10 +696,12 @@ describe(MediaService.name, () => {
         {
           inputOptions: [],
           outputOptions: [
-            '-c:v:0 vp9',
-            '-c:a:0 aac',
+            '-c:v vp9',
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
             '-v verbose',
             '-vf scale=-2:720,format=yuv420p',
             '-row-mt 1',
@@ -701,10 +727,12 @@ describe(MediaService.name, () => {
         {
           inputOptions: [],
           outputOptions: [
-            '-c:v:0 vp9',
-            '-c:a:0 aac',
+            '-c:v vp9',
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
             '-v verbose',
             '-vf scale=-2:720,format=yuv420p',
             '-cpu-used 5',
@@ -729,10 +757,12 @@ describe(MediaService.name, () => {
         {
           inputOptions: [],
           outputOptions: [
-            '-c:v:0 h264',
-            '-c:a:0 aac',
+            '-c:v h264',
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
             '-v verbose',
             '-vf scale=-2:720,format=yuv420p',
             '-preset ultrafast',
@@ -757,10 +787,12 @@ describe(MediaService.name, () => {
         {
           inputOptions: [],
           outputOptions: [
-            '-c:v:0 h264',
-            '-c:a:0 aac',
+            '-c:v h264',
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
             '-v verbose',
             '-vf scale=-2:720,format=yuv420p',
             '-preset ultrafast',
@@ -785,10 +817,12 @@ describe(MediaService.name, () => {
         {
           inputOptions: [],
           outputOptions: [
-            '-c:v:0 hevc',
-            '-c:a:0 aac',
+            '-c:v hevc',
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
             '-v verbose',
             '-vf scale=-2:720,format=yuv420p',
             '-preset ultrafast',
@@ -816,10 +850,12 @@ describe(MediaService.name, () => {
         {
           inputOptions: [],
           outputOptions: [
-            '-c:v:0 hevc',
-            '-c:a:0 aac',
+            '-c:v hevc',
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
             '-v verbose',
             '-vf scale=-2:720,format=yuv420p',
             '-preset ultrafast',
@@ -878,17 +914,15 @@ describe(MediaService.name, () => {
           outputOptions: [
             '-tune hq',
             '-qmin 0',
-            '-g 250',
-            '-bf 3',
-            '-b_ref_mode middle',
-            '-temporal-aq 1',
             '-rc-lookahead 20',
             '-i_qfactor 0.75',
-            '-b_qfactor 1.1',
-            `-c:v:0 h264_nvenc`,
-            '-c:a:0 aac',
+            `-c:v h264_nvenc`,
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
+            '-g 256',
             '-v verbose',
             '-vf format=nv12,hwupload_cuda,scale_cuda=-2:720',
             '-preset p1',
@@ -918,17 +952,15 @@ describe(MediaService.name, () => {
           outputOptions: [
             '-tune hq',
             '-qmin 0',
-            '-g 250',
-            '-bf 3',
-            '-b_ref_mode middle',
-            '-temporal-aq 1',
             '-rc-lookahead 20',
             '-i_qfactor 0.75',
-            '-b_qfactor 1.1',
-            `-c:v:0 h264_nvenc`,
-            '-c:a:0 aac',
+            `-c:v h264_nvenc`,
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
+            '-g 256',
             '-v verbose',
             '-vf format=nv12,hwupload_cuda,scale_cuda=-2:720',
             '-preset p1',
@@ -954,17 +986,15 @@ describe(MediaService.name, () => {
           outputOptions: [
             '-tune hq',
             '-qmin 0',
-            '-g 250',
-            '-bf 3',
-            '-b_ref_mode middle',
-            '-temporal-aq 1',
             '-rc-lookahead 20',
             '-i_qfactor 0.75',
-            '-b_qfactor 1.1',
-            `-c:v:0 h264_nvenc`,
-            '-c:a:0 aac',
+            `-c:v h264_nvenc`,
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
+            '-g 256',
             '-v verbose',
             '-vf format=nv12,hwupload_cuda,scale_cuda=-2:720',
             '-preset p1',
@@ -991,17 +1021,15 @@ describe(MediaService.name, () => {
           outputOptions: [
             '-tune hq',
             '-qmin 0',
-            '-g 250',
-            '-bf 3',
-            '-b_ref_mode middle',
-            '-temporal-aq 1',
             '-rc-lookahead 20',
             '-i_qfactor 0.75',
-            '-b_qfactor 1.1',
-            `-c:v:0 h264_nvenc`,
-            '-c:a:0 aac',
+            `-c:v h264_nvenc`,
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
+            '-g 256',
             '-v verbose',
             '-vf format=nv12,hwupload_cuda,scale_cuda=-2:720',
             '-cq:v 23',
@@ -1024,17 +1052,15 @@ describe(MediaService.name, () => {
           outputOptions: [
             '-tune hq',
             '-qmin 0',
-            '-g 250',
-            '-bf 3',
-            '-b_ref_mode middle',
-            '-temporal-aq 1',
             '-rc-lookahead 20',
             '-i_qfactor 0.75',
-            '-b_qfactor 1.1',
-            `-c:v:0 h264_nvenc`,
-            '-c:a:0 aac',
+            `-c:v h264_nvenc`,
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
+            '-g 256',
             '-v verbose',
             '-vf format=nv12,hwupload_cuda,scale_cuda=-2:720',
             '-preset p1',
@@ -1060,14 +1086,15 @@ describe(MediaService.name, () => {
         {
           inputOptions: ['-init_hw_device qsv=hw', '-filter_hw_device hw'],
           outputOptions: [
-            '-g 256',
-            '-extbrc 1',
-            '-refs 5',
-            '-bf 7',
-            `-c:v:0 h264_qsv`,
-            '-c:a:0 aac',
+            `-c:v h264_qsv`,
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
+            '-bf 7',
+            '-refs 5',
+            '-g 256',
             '-v verbose',
             '-vf format=nv12,hwupload=extra_hw_frames=64,scale_qsv=-1:720',
             '-preset 7',
@@ -1095,14 +1122,15 @@ describe(MediaService.name, () => {
         {
           inputOptions: ['-init_hw_device qsv=hw', '-filter_hw_device hw'],
           outputOptions: [
-            '-g 256',
-            '-extbrc 1',
-            '-refs 5',
-            '-bf 7',
-            `-c:v:0 h264_qsv`,
-            '-c:a:0 aac',
+            `-c:v h264_qsv`,
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
+            '-bf 7',
+            '-refs 5',
+            '-g 256',
             '-v verbose',
             '-vf format=nv12,hwupload=extra_hw_frames=64,scale_qsv=-1:720',
             '-global_quality 23',
@@ -1127,14 +1155,15 @@ describe(MediaService.name, () => {
         {
           inputOptions: ['-init_hw_device qsv=hw', '-filter_hw_device hw'],
           outputOptions: [
-            '-g 256',
-            '-extbrc 1',
-            '-refs 5',
-            '-bf 7',
-            `-c:v:0 vp9_qsv`,
-            '-c:a:0 aac',
+            `-c:v vp9_qsv`,
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
+            '-bf 7',
+            '-refs 5',
+            '-g 256',
             '-low_power 1',
             '-v verbose',
             '-vf format=nv12,hwupload=extra_hw_frames=64,scale_qsv=-1:720',
@@ -1170,10 +1199,13 @@ describe(MediaService.name, () => {
         {
           inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD128', '-filter_hw_device accel'],
           outputOptions: [
-            `-c:v:0 h264_vaapi`,
-            '-c:a:0 aac',
+            `-c:v h264_vaapi`,
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
+            '-g 256',
             '-v verbose',
             '-vf format=nv12,hwupload,scale_vaapi=-2:720',
             '-compression_level 7',
@@ -1199,10 +1231,13 @@ describe(MediaService.name, () => {
         {
           inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD128', '-filter_hw_device accel'],
           outputOptions: [
-            `-c:v:0 h264_vaapi`,
-            '-c:a:0 aac',
+            `-c:v h264_vaapi`,
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
+            '-g 256',
             '-v verbose',
             '-vf format=nv12,hwupload,scale_vaapi=-2:720',
             '-compression_level 7',
@@ -1230,10 +1265,13 @@ describe(MediaService.name, () => {
         {
           inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD128', '-filter_hw_device accel'],
           outputOptions: [
-            `-c:v:0 h264_vaapi`,
-            '-c:a:0 aac',
+            `-c:v h264_vaapi`,
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
+            '-g 256',
             '-v verbose',
             '-vf format=nv12,hwupload,scale_vaapi=-2:720',
             '-qp 23',
@@ -1257,10 +1295,13 @@ describe(MediaService.name, () => {
         {
           inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/card1', '-filter_hw_device accel'],
           outputOptions: [
-            `-c:v:0 h264_vaapi`,
-            '-c:a:0 aac',
+            `-c:v h264_vaapi`,
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
+            '-g 256',
             '-v verbose',
             '-vf format=nv12,hwupload,scale_vaapi=-2:720',
             '-compression_level 7',
@@ -1280,10 +1321,13 @@ describe(MediaService.name, () => {
         {
           inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD129', '-filter_hw_device accel'],
           outputOptions: [
-            `-c:v:0 h264_vaapi`,
-            '-c:a:0 aac',
+            `-c:v h264_vaapi`,
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
+            '-g 256',
             '-v verbose',
             '-vf format=nv12,hwupload,scale_vaapi=-2:720',
             '-compression_level 7',
@@ -1310,10 +1354,12 @@ describe(MediaService.name, () => {
         {
           inputOptions: [],
           outputOptions: [
-            '-c:v:0 h264',
-            '-c:a:0 aac',
+            '-c:v h264',
+            '-c:a aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
             '-v verbose',
             '-vf scale=-2:720,format=yuv420p',
             '-preset ultrafast',
@@ -1345,10 +1391,12 @@ describe(MediaService.name, () => {
       {
         inputOptions: [],
         outputOptions: [
-          '-c:v:0 h264',
-          '-c:a:0 aac',
+          '-c:v h264',
+          '-c:a aac',
           '-movflags faststart',
           '-fps_mode passthrough',
+          '-map 0:0',
+          '-map 0:1',
           '-v verbose',
           '-vf zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:range=pc,format=yuv420p',
           '-preset ultrafast',
@@ -1370,10 +1418,12 @@ describe(MediaService.name, () => {
       {
         inputOptions: [],
         outputOptions: [
-          '-c:v:0 h264',
-          '-c:a:0 aac',
+          '-c:v h264',
+          '-c:a aac',
           '-movflags faststart',
           '-fps_mode passthrough',
+          '-map 0:0',
+          '-map 0:1',
           '-v verbose',
           '-vf zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:range=pc,format=yuv420p',
           '-preset ultrafast',
@@ -1395,10 +1445,12 @@ describe(MediaService.name, () => {
       {
         inputOptions: [],
         outputOptions: [
-          '-c:v:0 h264',
-          '-c:a:0 aac',
+          '-c:v h264',
+          '-c:a aac',
           '-movflags faststart',
           '-fps_mode passthrough',
+          '-map 0:0',
+          '-map 0:1',
           '-v verbose',
           '-vf zscale=t=linear:npl=250,tonemap=mobius:desat=0,zscale=p=bt709:t=bt709:m=bt709:range=pc,format=yuv420p',
           '-preset ultrafast',

server/src/domain/media/media.util.ts

@@ -1,4 +1,4 @@
-import { ToneMapping, TranscodeHWAccel, VideoCodec } from '@app/infra/entities';
+import { CQMode, ToneMapping, TranscodeHWAccel, VideoCodec } from '@app/infra/entities';
 import { SystemConfigFFmpegDto } from '../system-config/dto';
 import {
   AudioStreamInfo,
@@ -9,9 +9,10 @@ import {
   VideoStreamInfo,
 } from './media.repository';
 class BaseConfig implements VideoCodecSWConfig {
+  presets = ['veryslow', 'slower', 'slow', 'medium', 'fast', 'faster', 'veryfast', 'superfast', 'ultrafast'];
   constructor(protected config: SystemConfigFFmpegDto) {}
 
-  getOptions(videoStream: VideoStreamInfo, audioStream: AudioStreamInfo) {
+  getOptions(videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) {
     const options = {
       inputOptions: this.getBaseInputOptions(),
       outputOptions: this.getBaseOutputOptions(videoStream, audioStream).concat('-v verbose'),
@@ -32,15 +33,30 @@ class BaseConfig implements VideoCodecSWConfig {
     return [];
   }
 
-  getBaseOutputOptions(videoStream: VideoStreamInfo, audioStream: AudioStreamInfo) {
-    return [
-      `-c:v:${videoStream.index} ${this.getVideoCodec()}`,
-      `-c:a:${audioStream.index} ${this.getAudioCodec()}`,
+  getBaseOutputOptions(videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) {
+    const options = [
+      `-c:v ${this.getVideoCodec()}`,
+      `-c:a ${this.getAudioCodec()}`,
       // Makes a second pass moving the moov atom to the
       // beginning of the file for improved playback speed.
       '-movflags faststart',
       '-fps_mode passthrough',
+      // explicitly selects the video stream instead of leaving it up to FFmpeg
+      `-map 0:${videoStream.index}`,
     ];
+    if (audioStream) {
+      options.push(`-map 0:${audioStream.index}`);
+    }
+    if (this.getBFrames() > -1) {
+      options.push(`-bf ${this.getBFrames()}`);
+    }
+    if (this.getRefs() > 0) {
+      options.push(`-refs ${this.getRefs()}`);
+    }
+    if (this.getGopSize() > 0) {
+      options.push(`-g ${this.getGopSize()}`);
+    }
+    return options;
   }
 
   getFilterOptions(videoStream: VideoStreamInfo) {
@@ -72,12 +88,12 @@ class BaseConfig implements VideoCodecSWConfig {
     } else if (bitrates.max > 0) {
       // -bufsize is the peak possible bitrate at any moment, while -maxrate is the max rolling average bitrate
       return [
-        `-crf ${this.config.crf}`,
+        `-${this.useCQP() ? 'q:v' : 'crf'} ${this.config.crf}`,
         `-maxrate ${bitrates.max}${bitrates.unit}`,
         `-bufsize ${bitrates.max * 2}${bitrates.unit}`,
       ];
     } else {
-      return [`-crf ${this.config.crf}`];
+      return [`-${this.useCQP() ? 'q:v' : 'crf'} ${this.config.crf}`];
     }
   }
 
@@ -149,8 +165,7 @@ class BaseConfig implements VideoCodecSWConfig {
   }
 
   getPresetIndex() {
-    const presets = ['veryslow', 'slower', 'slow', 'medium', 'fast', 'faster', 'veryfast', 'superfast', 'ultrafast'];
-    return presets.indexOf(this.config.preset);
+    return this.presets.indexOf(this.config.preset);
   }
 
   getColors() {
@@ -161,14 +176,20 @@ class BaseConfig implements VideoCodecSWConfig {
     };
   }
 
+  getNPL() {
+    if (this.config.npl <= 0) {
+      // since hable already outputs a darker image, we use a lower npl value for it
+      return this.config.tonemap === ToneMapping.HABLE ? 100 : 250;
+    } else {
+      return this.config.npl;
+    }
+  }
+
   getToneMapping() {
     const colors = this.getColors();
-    // npl stands for nominal peak luminance
-    // lower npl values result in brighter output (compensating for dimmer screens)
-    // since hable already outputs a darker image, we use a lower npl value for it
-    const npl = this.config.tonemap === ToneMapping.HABLE ? 100 : 250;
+
     return [
-      `zscale=t=linear:npl=${npl}`,
+      `zscale=t=linear:npl=${this.getNPL()}`,
       `tonemap=${this.config.tonemap}:desat=0`,
       `zscale=p=${colors.primaries}:t=${colors.transfer}:m=${colors.matrix}:range=pc`,
     ];
@@ -181,6 +202,22 @@ class BaseConfig implements VideoCodecSWConfig {
   getVideoCodec(): string {
     return this.config.targetVideoCodec;
   }
+
+  getBFrames() {
+    return this.config.bframes;
+  }
+
+  getRefs() {
+    return this.config.refs;
+  }
+
+  getGopSize() {
+    return this.config.gopSize;
+  }
+
+  useCQP() {
+    return this.config.cqMode === CQMode.CQP;
+  }
 }
 
 export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig {
@@ -216,6 +253,13 @@ export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig {
   getVideoCodec(): string {
     return `${this.config.targetVideoCodec}_${this.config.accel}`;
   }
+
+  getGopSize() {
+    if (this.config.gopSize <= 0) {
+      return 256;
+    }
+    return this.config.gopSize;
+  }
 }
 
 export class ThumbnailConfig extends BaseConfig {
@@ -294,7 +338,7 @@ export class VP9Config extends BaseConfig {
       ];
     }
 
-    return [`-crf ${this.config.crf}`, `-b:v ${bitrates.max}${bitrates.unit}`];
+    return [`-${this.useCQP() ? 'q:v' : 'crf'} ${this.config.crf}`, `-b:v ${bitrates.max}${bitrates.unit}`];
   }
 
   getThreadOptions() {
@@ -311,20 +355,23 @@ export class NVENCConfig extends BaseHWConfig {
     return ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'];
   }
 
-  getBaseOutputOptions(videoStream: VideoStreamInfo, audioStream: AudioStreamInfo) {
-    return [
+  getBaseOutputOptions(videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) {
+    const options = [
       // below settings recommended from https://docs.nvidia.com/video-technologies/video-codec-sdk/12.0/ffmpeg-with-nvidia-gpu/index.html#command-line-for-latency-tolerant-high-quality-transcoding
       '-tune hq',
       '-qmin 0',
-      '-g 250',
-      '-bf 3',
-      '-b_ref_mode middle',
-      '-temporal-aq 1',
       '-rc-lookahead 20',
       '-i_qfactor 0.75',
-      '-b_qfactor 1.1',
       ...super.getBaseOutputOptions(videoStream, audioStream),
     ];
+    if (this.getBFrames() > 0) {
+      options.push('-b_ref_mode middle');
+      options.push('-b_qfactor 1.1');
+    }
+    if (this.config.temporalAQ) {
+      options.push('-temporal-aq 1');
+    }
+    return options;
   }
 
   getFilterOptions(videoStream: VideoStreamInfo) {
@@ -369,6 +416,14 @@ export class NVENCConfig extends BaseHWConfig {
   getThreadOptions() {
     return [];
   }
+
+  getRefs() {
+    const bframes = this.getBFrames();
+    if (bframes > 0 && bframes < 3 && this.config.refs < 3) {
+      return 0;
+    }
+    return this.config.refs;
+  }
 }
 
 export class QSVConfig extends BaseHWConfig {
@@ -379,15 +434,8 @@ export class QSVConfig extends BaseHWConfig {
     return ['-init_hw_device qsv=hw', '-filter_hw_device hw'];
   }
 
-  getBaseOutputOptions(videoStream: VideoStreamInfo, audioStream: AudioStreamInfo) {
-    // recommended from https://github.com/intel/media-delivery/blob/master/doc/benchmarks/intel-iris-xe-max-graphics/intel-iris-xe-max-graphics.md
-    const options = [
-      '-g 256',
-      '-extbrc 1',
-      '-refs 5',
-      '-bf 7',
-      ...super.getBaseOutputOptions(videoStream, audioStream),
-    ];
+  getBaseOutputOptions(videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) {
+    const options = super.getBaseOutputOptions(videoStream, audioStream);
     // VP9 requires enabling low power mode https://git.ffmpeg.org/gitweb/ffmpeg.git/commit/33583803e107b6d532def0f9d949364b01b6ad5a
     if (this.config.targetVideoCodec === VideoCodec.VP9) {
       options.push('-low_power 1');
@@ -415,11 +463,7 @@ export class QSVConfig extends BaseHWConfig {
 
   getBitrateOptions() {
     const options = [];
-    if (this.config.targetVideoCodec !== VideoCodec.VP9) {
-      options.push(`-global_quality ${this.config.crf}`);
-    } else {
-      options.push(`-q:v ${this.config.crf}`);
-    }
+    options.push(`-${this.useCQP() ? 'q:v' : 'global_quality'} ${this.config.crf}`);
     const bitrates = this.getBitrateDistribution();
     if (bitrates.max > 0) {
       options.push(`-maxrate ${bitrates.max}${bitrates.unit}`);
@@ -427,6 +471,25 @@ export class QSVConfig extends BaseHWConfig {
     }
     return options;
   }
+
+  // recommended from https://github.com/intel/media-delivery/blob/master/doc/benchmarks/intel-iris-xe-max-graphics/intel-iris-xe-max-graphics.md
+  getBFrames() {
+    if (this.config.bframes < 0) {
+      return 7;
+    }
+    return this.config.bframes;
+  }
+
+  getRefs() {
+    if (this.config.refs <= 0) {
+      return 5;
+    }
+    return this.config.refs;
+  }
+
+  useCQP() {
+    return this.config.cqMode === CQMode.CQP || this.config.targetVideoCodec === VideoCodec.VP9;
+  }
 }
 
 export class VAAPIConfig extends BaseHWConfig {
@@ -458,16 +521,30 @@ export class VAAPIConfig extends BaseHWConfig {
 
   getBitrateOptions() {
     const bitrates = this.getBitrateDistribution();
+    const options = [];
+
+    if (this.config.targetVideoCodec === VideoCodec.VP9) {
+      options.push('-bsf:v vp9_raw_reorder,vp9_superframe');
+    }
+
     // VAAPI doesn't allow setting both quality and max bitrate
     if (bitrates.max > 0) {
-      return [
+      options.push(
         `-b:v ${bitrates.target}${bitrates.unit}`,
         `-maxrate ${bitrates.max}${bitrates.unit}`,
         `-minrate ${bitrates.min}${bitrates.unit}`,
         '-rc_mode 3',
-      ]; // variable bitrate
+      ); // variable bitrate
+    } else if (this.useCQP()) {
+      options.push(`-qp ${this.config.crf}`, `-global_quality ${this.config.crf}`, '-rc_mode 1');
     } else {
-      return [`-qp ${this.config.crf}`, `-global_quality ${this.config.crf}`, '-rc_mode 1']; // constant quality
+      options.push(`-global_quality ${this.config.crf}`, '-rc_mode 4');
     }
+
+    return options;
+  }
+
+  useCQP() {
+    return this.config.cqMode !== CQMode.ICQ || this.config.targetVideoCodec === VideoCodec.VP9;
   }
 }

server/src/domain/system-config/dto/system-config-ffmpeg.dto.ts

@@ -1,4 +1,4 @@
-import { AudioCodec, ToneMapping, TranscodeHWAccel, TranscodePolicy, VideoCodec } from '@app/infra/entities';
+import { AudioCodec, CQMode, ToneMapping, TranscodeHWAccel, TranscodePolicy, VideoCodec } from '@app/infra/entities';
 import { ApiProperty } from '@nestjs/swagger';
 import { Type } from 'class-transformer';
 import { IsBoolean, IsEnum, IsInt, IsString, Max, Min } from 'class-validator';
@@ -34,6 +34,39 @@ export class SystemConfigFFmpegDto {
   @IsString()
   maxBitrate!: string;
 
+  @IsInt()
+  @Min(-1)
+  @Max(16)
+  @Type(() => Number)
+  @ApiProperty({ type: 'integer' })
+  bframes!: number;
+
+  @IsInt()
+  @Min(0)
+  @Max(6)
+  @Type(() => Number)
+  @ApiProperty({ type: 'integer' })
+  refs!: number;
+
+  @IsInt()
+  @Min(0)
+  @Type(() => Number)
+  @ApiProperty({ type: 'integer' })
+  gopSize!: number;
+
+  @IsInt()
+  @Min(0)
+  @Type(() => Number)
+  @ApiProperty({ type: 'integer' })
+  npl!: number;
+
+  @IsBoolean()
+  temporalAQ!: boolean;
+
+  @IsEnum(CQMode)
+  @ApiProperty({ enumName: 'CQMode', enum: CQMode })
+  cqMode!: CQMode;
+
   @IsBoolean()
   twoPass!: boolean;
 

server/src/domain/system-config/system-config.core.ts

@@ -1,5 +1,6 @@
 import {
   AudioCodec,
+  CQMode,
   SystemConfig,
   SystemConfigEntity,
   SystemConfigKey,
@@ -30,6 +31,12 @@ export const defaults = Object.freeze<SystemConfig>({
     targetAudioCodec: AudioCodec.AAC,
     targetResolution: '720',
     maxBitrate: '0',
+    bframes: -1,
+    refs: 0,
+    gopSize: 0,
+    npl: 0,
+    temporalAQ: false,
+    cqMode: CQMode.AUTO,
     twoPass: false,
     transcode: TranscodePolicy.REQUIRED,
     tonemap: ToneMapping.HABLE,

server/src/domain/system-config/system-config.service.spec.ts

@@ -1,5 +1,6 @@
 import {
   AudioCodec,
+  CQMode,
   SystemConfig,
   SystemConfigEntity,
   SystemConfigKey,
@@ -41,6 +42,12 @@ const updatedConfig = Object.freeze<SystemConfig>({
     targetResolution: '720',
     targetVideoCodec: VideoCodec.H264,
     maxBitrate: '0',
+    bframes: -1,
+    refs: 0,
+    gopSize: 0,
+    npl: 0,
+    temporalAQ: false,
+    cqMode: CQMode.AUTO,
     twoPass: false,
     transcode: TranscodePolicy.REQUIRED,
     accel: TranscodeHWAccel.DISABLED,

server/src/infra/entities/system-config.entity.ts

@@ -21,6 +21,12 @@ export enum SystemConfigKey {
   FFMPEG_TARGET_AUDIO_CODEC = 'ffmpeg.targetAudioCodec',
   FFMPEG_TARGET_RESOLUTION = 'ffmpeg.targetResolution',
   FFMPEG_MAX_BITRATE = 'ffmpeg.maxBitrate',
+  FFMPEG_BFRAMES = 'ffmpeg.bframes',
+  FFMPEG_REFS = 'ffmpeg.refs',
+  FFMPEG_GOP_SIZE = 'ffmpeg.gopSize',
+  FFMPEG_NPL = 'ffmpeg.npl',
+  FFMPEG_TEMPORAL_AQ = 'ffmpeg.temporalAQ',
+  FFMPEG_CQ_MODE = 'ffmpeg.cqMode',
   FFMPEG_TWO_PASS = 'ffmpeg.twoPass',
   FFMPEG_TRANSCODE = 'ffmpeg.transcode',
   FFMPEG_ACCEL = 'ffmpeg.accel',
@@ -105,6 +111,12 @@ export enum ToneMapping {
   DISABLED = 'disabled',
 }
 
+export enum CQMode {
+  AUTO = 'auto',
+  CQP = 'cqp',
+  ICQ = 'icq',
+}
+
 export interface SystemConfig {
   ffmpeg: {
     crf: number;
@@ -114,6 +126,12 @@ export interface SystemConfig {
     targetAudioCodec: AudioCodec;
     targetResolution: string;
     maxBitrate: string;
+    bframes: number;
+    refs: number;
+    gopSize: number;
+    npl: number;
+    temporalAQ: boolean;
+    cqMode: CQMode;
     twoPass: boolean;
     transcode: TranscodePolicy;
     accel: TranscodeHWAccel;

server/src/infra/repositories/media.repository.ts

@@ -32,7 +32,6 @@ export class MediaRepository implements IMediaRepository {
 
   async probe(input: string): Promise<VideoInfo> {
     const results = await probe(input);
-
     return {
       format: {
         formatName: results.format.format_name,

server/test/fixtures/media.stub.ts

@@ -20,7 +20,7 @@ const probeStubDefaultVideoStream: VideoStreamInfo[] = [
 ];
 
 const probeStubDefaultAudioStream: AudioStreamInfo[] = [
-  { index: 0, codecName: 'aac', codecType: 'audio', frameCount: 100 },
+  { index: 1, codecName: 'aac', codecType: 'audio', frameCount: 100 },
 ];
 
 const probeStubDefault: VideoInfo = {
@@ -119,7 +119,7 @@ export const probeStub = {
   }),
   audioStreamMp3: Object.freeze<VideoInfo>({
     ...probeStubDefault,
-    audioStreams: [{ index: 0, codecType: 'audio', codecName: 'aac', frameCount: 100 }],
+    audioStreams: [{ index: 1, codecType: 'audio', codecName: 'aac', frameCount: 100 }],
   }),
   matroskaContainer: Object.freeze<VideoInfo>({
     ...probeStubDefault,

web/src/api/open-api/api.ts

@@ -909,6 +909,21 @@ export const CLIPMode = {
 export type CLIPMode = typeof CLIPMode[keyof typeof CLIPMode];
 
 
+/**
+ * 
+ * @export
+ * @enum {string}
+ */
+
+export const CQMode = {
+    Auto: 'auto',
+    Cqp: 'cqp',
+    Icq: 'icq'
+} as const;
+
+export type CQMode = typeof CQMode[keyof typeof CQMode];
+
+
 /**
  * 
  * @export
@@ -2812,24 +2827,54 @@ export interface SystemConfigFFmpegDto {
      * @memberof SystemConfigFFmpegDto
      */
     'accel': TranscodeHWAccel;
+    /**
+     * 
+     * @type {number}
+     * @memberof SystemConfigFFmpegDto
+     */
+    'bframes': number;
+    /**
+     * 
+     * @type {CQMode}
+     * @memberof SystemConfigFFmpegDto
+     */
+    'cqMode': CQMode;
     /**
      * 
      * @type {number}
      * @memberof SystemConfigFFmpegDto
      */
     'crf': number;
+    /**
+     * 
+     * @type {number}
+     * @memberof SystemConfigFFmpegDto
+     */
+    'gopSize': number;
     /**
      * 
      * @type {string}
      * @memberof SystemConfigFFmpegDto
      */
     'maxBitrate': string;
+    /**
+     * 
+     * @type {number}
+     * @memberof SystemConfigFFmpegDto
+     */
+    'npl': number;
     /**
      * 
      * @type {string}
      * @memberof SystemConfigFFmpegDto
      */
     'preset': string;
+    /**
+     * 
+     * @type {number}
+     * @memberof SystemConfigFFmpegDto
+     */
+    'refs': number;
     /**
      * 
      * @type {AudioCodec}
@@ -2848,6 +2893,12 @@ export interface SystemConfigFFmpegDto {
      * @memberof SystemConfigFFmpegDto
      */
     'targetVideoCodec': VideoCodec;
+    /**
+     * 
+     * @type {boolean}
+     * @memberof SystemConfigFFmpegDto
+     */
+    'temporalAQ': boolean;
     /**
      * 
      * @type {number}

web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte

@@ -6,6 +6,7 @@
   import {
     api,
     AudioCodec,
+    CQMode,
     SystemConfigFFmpegDto,
     ToneMapping,
     TranscodeHWAccel,
@@ -19,6 +20,7 @@
   import HelpCircleOutline from 'svelte-material-icons/HelpCircleOutline.svelte';
   import { isEqual } from 'lodash-es';
   import { fade } from 'svelte/transition';
+  import SettingAccordion from '../setting-accordion.svelte';
 
   export let ffmpegConfig: SystemConfigFFmpegDto; // this is the config that is being edited
   export let disabled = false;
@@ -112,7 +114,7 @@
             desc="Video quality level. Typical values are 23 for H.264, 28 for HEVC, and 31 for VP9. Lower is better, but takes longer to encode and produces larger files."
             bind:value={ffmpegConfig.crf}
             required={true}
-            isEdited={!(ffmpegConfig.crf == savedConfig.crf)}
+            isEdited={ffmpegConfig.crf !== savedConfig.crf}
           />
 
           <SettingSelect
@@ -132,7 +134,7 @@
               { value: 'slower', text: 'slower' },
               { value: 'veryslow', text: 'veryslow' },
             ]}
-            isEdited={!(ffmpegConfig.preset == savedConfig.preset)}
+            isEdited={ffmpegConfig.preset !== savedConfig.preset}
           />
 
           <SettingSelect
@@ -146,7 +148,7 @@
               { value: AudioCodec.Opus, text: 'opus' },
             ]}
             name="acodec"
-            isEdited={!(ffmpegConfig.targetAudioCodec == savedConfig.targetAudioCodec)}
+            isEdited={ffmpegConfig.targetAudioCodec !== savedConfig.targetAudioCodec}
           />
 
           <SettingSelect
@@ -160,7 +162,7 @@
               { value: VideoCodec.Vp9, text: 'vp9' },
             ]}
             name="vcodec"
-            isEdited={!(ffmpegConfig.targetVideoCodec == savedConfig.targetVideoCodec)}
+            isEdited={ffmpegConfig.targetVideoCodec !== savedConfig.targetVideoCodec}
           />
 
           <SettingSelect
@@ -177,7 +179,7 @@
               { value: 'original', text: 'original' },
             ]}
             name="resolution"
-            isEdited={!(ffmpegConfig.targetResolution == savedConfig.targetResolution)}
+            isEdited={ffmpegConfig.targetResolution !== savedConfig.targetResolution}
           />
 
           <SettingInputField
@@ -186,7 +188,7 @@
             label="MAX BITRATE"
             desc="Setting a max bitrate can make file sizes more predictable at a minor cost to quality. At 720p, typical values are 2600k for VP9 or HEVC, or 4500k for H.264. Disabled if set to 0."
             bind:value={ffmpegConfig.maxBitrate}
-            isEdited={!(ffmpegConfig.maxBitrate == savedConfig.maxBitrate)}
+            isEdited={ffmpegConfig.maxBitrate !== savedConfig.maxBitrate}
           />
 
           <SettingInputField
@@ -195,7 +197,7 @@
             label="THREADS"
             desc="Higher values lead to faster encoding, but leave less room for the server to process other tasks while active. This value should not be more than the number of CPU cores. Maximizes utilization if set to 0."
             bind:value={ffmpegConfig.threads}
-            isEdited={!(ffmpegConfig.threads == savedConfig.threads)}
+            isEdited={ffmpegConfig.threads !== savedConfig.threads}
           />
 
           <SettingSelect
@@ -219,31 +221,7 @@
                 text: "Don't transcode any videos, may break playback on some clients",
               },
             ]}
-            isEdited={!(ffmpegConfig.transcode == savedConfig.transcode)}
-          />
-
-          <SettingSelect
-            label="HARDWARE ACCELERATION"
-            {disabled}
-            desc="Experimental. Much faster, but will have lower quality at the same bitrate. This setting is 'best effort': it will fallback to software transcoding on failure. VP9 may or may not work depending on your hardware."
-            bind:value={ffmpegConfig.accel}
-            name="accel"
-            options={[
-              { value: TranscodeHWAccel.Nvenc, text: 'NVENC (requires NVIDIA GPU)' },
-              {
-                value: TranscodeHWAccel.Qsv,
-                text: 'Quick Sync (requires 7th gen Intel CPU or later)',
-              },
-              {
-                value: TranscodeHWAccel.Vaapi,
-                text: 'VAAPI',
-              },
-              {
-                value: TranscodeHWAccel.Disabled,
-                text: 'Disabled',
-              },
-            ]}
-            isEdited={!(ffmpegConfig.accel == savedConfig.accel)}
+            isEdited={ffmpegConfig.transcode !== savedConfig.transcode}
           />
 
           <SettingSelect
@@ -270,7 +248,7 @@
                 text: 'Disabled',
               },
             ]}
-            isEdited={!(ffmpegConfig.tonemap == savedConfig.tonemap)}
+            isEdited={ffmpegConfig.tonemap !== savedConfig.tonemap}
           />
 
           <SettingSwitch
@@ -278,8 +256,95 @@
             {disabled}
             subtitle="Transcode in two passes to produce better encoded videos. When max bitrate is enabled (required for it to work with H.264 and HEVC), this mode uses a bitrate range based on the max bitrate and ignores CRF. For VP9, CRF can be used if max bitrate is disabled."
             bind:checked={ffmpegConfig.twoPass}
-            isEdited={!(ffmpegConfig.twoPass === savedConfig.twoPass)}
+            isEdited={ffmpegConfig.twoPass !== savedConfig.twoPass}
           />
+
+          <SettingAccordion
+            title="Hardware Acceleration"
+            subtitle="Experimental; much faster, but will have lower quality at the same bitrate"
+          >
+            <div class="ml-4 mt-4 flex flex-col gap-4">
+              <SettingSelect
+                label="ACCELERATION API"
+                {disabled}
+                desc="The API that will interact with your device to accelerate transcoding. This setting is 'best effort': it will fallback to software transcoding on failure. VP9 may or may not work depending on your hardware."
+                bind:value={ffmpegConfig.accel}
+                name="accel"
+                options={[
+                  { value: TranscodeHWAccel.Nvenc, text: 'NVENC (requires NVIDIA GPU)' },
+                  {
+                    value: TranscodeHWAccel.Qsv,
+                    text: 'Quick Sync (requires 7th gen Intel CPU or later)',
+                  },
+                  {
+                    value: TranscodeHWAccel.Vaapi,
+                    text: 'VAAPI',
+                  },
+                  {
+                    value: TranscodeHWAccel.Disabled,
+                    text: 'Disabled',
+                  },
+                ]}
+                isEdited={ffmpegConfig.accel !== savedConfig.accel}
+              />
+
+              <SettingSelect
+                label="CONSTANT QUALITY MODE"
+                desc="ICQ is better than CQP, but some hardware acceleration devices do not support this mode. Setting this option will prefer the specified mode when using quality-based encoding. Ignored by NVENC as it does not support ICQ."
+                bind:value={ffmpegConfig.cqMode}
+                options={[
+                  { value: CQMode.Auto, text: 'Auto' },
+                  { value: CQMode.Icq, text: 'ICQ' },
+                  { value: CQMode.Cqp, text: 'CQP' },
+                ]}
+                isEdited={ffmpegConfig.cqMode !== savedConfig.cqMode}
+              />
+
+              <SettingSwitch
+                title="TEMPORAL AQ"
+                {disabled}
+                subtitle="Applies only to NVENC. Increases quality of high-detail, low-motion scenes. May not be compatible with older devices."
+                bind:checked={ffmpegConfig.temporalAQ}
+                isEdited={ffmpegConfig.temporalAQ !== savedConfig.temporalAQ}
+              />
+            </div>
+          </SettingAccordion>
+
+          <SettingAccordion title="Advanced" subtitle="Options most users should not need to change">
+            <div class="ml-4 mt-4 flex flex-col gap-4">
+              <SettingInputField
+                inputType={SettingInputFieldType.NUMBER}
+                label="TONE-MAPPING NPL"
+                desc="Colors will be adjusted to look normal for a display of this brightness. Counter-intuitively, lower values increase the brightness of the video and vice versa since it compensates for the brightness of the display. 0 sets this value automatically."
+                bind:value={ffmpegConfig.npl}
+                isEdited={ffmpegConfig.npl !== savedConfig.npl}
+              />
+
+              <SettingInputField
+                inputType={SettingInputFieldType.NUMBER}
+                label="MAX B-FRAMES"
+                desc="Higher values improve compression efficiency, but slow down encoding. May not be compatible with hardware acceleration on older devices. 0 disables B-frames, while -1 sets this value automatically."
+                bind:value={ffmpegConfig.bframes}
+                isEdited={ffmpegConfig.bframes !== savedConfig.bframes}
+              />
+
+              <SettingInputField
+                inputType={SettingInputFieldType.NUMBER}
+                label="REFERENCE FRAMES"
+                desc="The number of frames to reference when compressing a given frame. Higher values improve compression efficiency, but slow down encoding. 0 sets this value automatically."
+                bind:value={ffmpegConfig.refs}
+                isEdited={ffmpegConfig.refs !== savedConfig.refs}
+              />
+
+              <SettingInputField
+                inputType={SettingInputFieldType.NUMBER}
+                label="MAX KEYFRAME INTERVAL"
+                desc="Sets the maximum frame distance between keyframes. Lower values worsen compression efficiency, but improve seek times and may improve quality in scenes with fast movement. 0 sets this value automatically."
+                bind:value={ffmpegConfig.gopSize}
+                isEdited={ffmpegConfig.gopSize !== savedConfig.gopSize}
+              />
+            </div>
+          </SettingAccordion>
         </div>
 
         <div class="ml-4">

web/src/lib/components/admin-page/settings/setting-input-field.svelte

@@ -13,8 +13,8 @@
 
   export let inputType: SettingInputFieldType;
   export let value: string | number;
-  export let min = Number.MIN_VALUE.toString();
-  export let max = Number.MAX_VALUE.toString();
+  export let min = Number.MIN_SAFE_INTEGER.toString();
+  export let max = Number.MAX_SAFE_INTEGER.toString();
   export let step = '1';
   export let label = '';
   export let desc = '';