feat(server): transcoding hardware acceleration (#3171)

* added transcode configs for nvenc,qsv and vaapi

* updated dev docker compose

* added software fallback

* working vaapi

* minor fixes and added tests

* updated api

* compile libvips

* move hwaccel settings to `hwaccel.yml`

* changed default dockerfile, moved `readdir` call

* removed unused import

* minor cleanup

* fix for arm build

* added documentation, minor fixes

* added intel driver

* updated docs

styling

* uppercase codec and api names

* formatting

* added tests

* updated docs

* removed semicolons

* added link to `hwaccel.yml`

* added newlines

* added `hwaccel` section to docker-compose.prod.yml

* ensure mesa drivers are installed

* switch to mimalloc for sharp

* moved build version and sha256 to json

* let libmfx set the render device

* possible fix for vp9 on qsv

* updated tests

* formatting

* review suggestions

* semicolon

* moved `LD_PRELOAD` to start script

* switched to jellyfin's ffmpeg package

* fixed dockerfile

* use cqp instead of icq for qsv vp9

* updated dockerfile

* added sha256sum for other platforms

* fixtures

.github/workflows/prepare-release.yml

@@ -83,4 +83,5 @@ jobs:
           files: |
             docker/docker-compose.yml
             docker/example.env
+            docker/hwaccel.yml
             *.apk

cli/src/api/open-api/api.ts

@@ -666,13 +666,13 @@ export interface AssetStatsResponseDto {
      * @type {number}
      * @memberof AssetStatsResponseDto
      */
-    'total': number;
+    'videos': number;
     /**
      * 
      * @type {number}
      * @memberof AssetStatsResponseDto
      */
-    'videos': number;
+    'total': number;
 }
 /**
  * 
@@ -2510,6 +2510,12 @@ export interface SystemConfigDto {
  * @interface SystemConfigFFmpegDto
  */
 export interface SystemConfigFFmpegDto {
+    /**
+     * 
+     * @type {TranscodeHWAccel}
+     * @memberof SystemConfigFFmpegDto
+     */
+    'accel': TranscodeHWAccel;
     /**
      * 
      * @type {number}
@@ -2858,6 +2864,22 @@ export const TimeGroupEnum = {
 export type TimeGroupEnum = typeof TimeGroupEnum[keyof typeof TimeGroupEnum];
 
 
+/**
+ * 
+ * @export
+ * @enum {string}
+ */
+
+export const TranscodeHWAccel = {
+    Nvenc: 'nvenc',
+    Qsv: 'qsv',
+    Vaapi: 'vaapi',
+    Disabled: 'disabled'
+} as const;
+
+export type TranscodeHWAccel = typeof TranscodeHWAccel[keyof typeof TranscodeHWAccel];
+
+
 /**
  * 
  * @export

docker/docker-compose.dev.yml

@@ -47,6 +47,9 @@ services:
   immich-microservices:
     container_name: immich_microservices
     image: immich-microservices:latest
+    # extends:
+    #   file: hwaccel.yml
+    #   service: hwaccel
     build:
       context: ../server
       dockerfile: Dockerfile

docker/docker-compose.prod.yml

@@ -33,6 +33,9 @@ services:
   immich-microservices:
     container_name: immich_microservices
     image: immich-microservices:latest
+    # extends:
+    #   file: hwaccel.yml
+    #   service: hwaccel
     build:
       context: ../server
       dockerfile: Dockerfile

docker/docker-compose.yml

@@ -18,6 +18,9 @@ services:
   immich-microservices:
     container_name: immich_microservices
     image: ghcr.io/immich-app/immich-server:${IMMICH_VERSION:-release}
+    # extends:
+    #   file: hwaccel.yml
+    #   service: hwaccel
     command: [ "start.sh", "microservices" ]
     volumes:
       - ${UPLOAD_LOCATION}:/usr/src/app/upload

docker/hwaccel.yml

@@ -0,0 +1,23 @@
+version: "3.8"
+
+# Hardware acceleration for transcoding - Optional
+# This is only needed if you want to use hardware acceleration for transcoding.
+# Depending on your hardware, you should uncomment the relevant lines below.
+
+services:
+  hwaccel:
+    # devices:
+    #   - /dev/dri:/dev/dri  # If using Intel QuickSync or VAAPI
+    # volumes:
+    #   - /usr/lib/wsl:/usr/lib/wsl # If using VAAPI in WSL2
+    # environment:
+    #   - NVIDIA_DRIVER_CAPABILITIES=all # If using NVIDIA GPU
+    #   - LD_LIBRARY_PATH=/usr/lib/wsl/lib # If using VAAPI in WSL2
+    #   - LIBVA_DRIVER_NAME=d3d12 # If using VAAPI in WSL2
+    # deploy: # Uncomment this section if using NVIDIA GPU
+    #   resources:
+    #     reservations:
+    #       devices:
+    #         - driver: nvidia
+    #           count: 1
+    #           capabilities: [gpu]

docs/docs/features/hardware-transcoding.md

@@ -0,0 +1,60 @@
+# Hardware Transcoding [Experimental]
+
+This feature allows you to use a GPU or Intel Quick Sync to accelerate transcoding and reduce CPU load.
+Note that hardware transcoding is much less efficient for file sizes.
+As this is a new feature, it is still experimental and may not work on all systems.
+
+## Supported APIs
+
+- NVENC
+  - NVIDIA GPUs
+- Quick Sync
+  - Intel CPUs
+- VAAPI
+  - GPUs
+
+## Limitations
+
+- The instructions and configurations here are specific to Docker Compose. Other container engines may require different configuration.
+- Only Linux and Windows (through WSL2) servers are supported.
+- WSL2 does not support Quick Sync.
+- Raspberry Pi is currently not supported.
+- Two-pass mode is only supported for NVENC. Other APIs will ignore this setting.
+- Only encoding is currently hardware accelerated, so the CPU is still used for software decoding.
+  - This is mainly because the original video may not be hardware-decodable.
+- Hardware dependent
+  - Codec support varies, but H.264 and HEVC are usually supported.
+    - Notably, NVIDIA and AMD GPUs do not support VP9 encoding.
+  - Newer devices tend to have higher transcoding quality.
+
+## Prerequisites
+
+#### NVENC
+
+- You must have the official NVIDIA driver installed on the server.
+- On Linux (except for WSL2), you also need to have [NVIDIA Container Runtime][nvcr] installed.
+
+#### QSV
+
+- For VP9 to work:
+  - You must have a 9th gen Intel CPU or newer
+  - If you have an 11th gen CPU or older, then you may need to follow [these][jellyfin-lp] instructions as Low-Power mode is required
+  - Additionally, if the server specifically has an 11th gen CPU and is running kernel 5.15 (shipped with Ubuntu 22.04 LTS), then you will need to upgrade this kernel (from [Jellyfin docs][jellyfin-kernel-bug])
+
+## Setup
+
+1. If you do not already have it, download the latest [`hwaccel.yml`][hw-file] file and ensure it's in the same folder as the `docker-compose.yml`.
+2. Uncomment the lines that apply to your system and desired usage.
+3. In the `docker-compose.yml` under `immich-microservices`, uncomment the lines relating to the `hwaccel.yml` file.
+4. Redeploy the `immich-microservices` container with these updated settings.
+5. In the Admin page under `FFmpeg settings`, change the hardware acceleration setting to the appropriate option and save.
+
+## Tips
+
+- You may want to choose a slower preset than for software transcoding to maintain quality and efficiency
+- While you can use VAAPI with Nvidia GPUs and Intel CPUs, prefer the more specific APIs since they're more optimized for their respective devices
+
+[hw-file]: https://github.com/immich-app/immich/releases/latest/download/hwaccel.yml
+[nvcr]: https://github.com/NVIDIA/nvidia-container-runtime/
+[jellyfin-lp]: https://jellyfin.org/docs/general/administration/hardware-acceleration/intel/#configure-and-verify-lp-mode-on-linux
+[jellyfin-kernel-bug]: https://jellyfin.org/docs/general/administration/hardware-acceleration/intel/#known-issues-and-limitations

docs/docs/install/docker-compose.md

@@ -25,10 +25,18 @@ wget https://github.com/immich-app/immich/releases/latest/download/docker-compos
 wget -O .env https://github.com/immich-app/immich/releases/latest/download/example.env
 ```
 
+```bash title="(Optional) Get hwaccel.yml file"
+wget https://github.com/immich-app/immich/releases/latest/download/hwaccel.yml
+```
+
 or by downloading from your browser and moving the files to the directory that you created.
 
 Note: If you downloaded the files from your browser, also ensure that you rename `example.env` to `.env`.
 
+:::info
+Optionally, you can use the [`hwaccel.yml`][hw-file] file to enable hardware acceleration for transcoding. See the [Hardware Transcoding](/docs/features/hardware-transcoding.md) guide for info on how to set this up.
+:::
+
 ### Step 2 - Populate the .env file with custom values
 
 <details>
@@ -186,4 +194,5 @@ Immich is currently under heavy development, which means you can expect breaking
 
 [compose-file]: https://github.com/immich-app/immich/releases/latest/download/docker-compose.yml
 [env-file]: https://github.com/immich-app/immich/releases/latest/download/example.env
+[hw-file]: https://github.com/immich-app/immich/releases/latest/download/hwaccel.yml
 [watchtower]: https://containrrr.dev/watchtower/

mobile/openapi/.openapi-generator/FILES

@@ -113,6 +113,7 @@ doc/TagResponseDto.md
 doc/TagTypeEnum.md
 doc/ThumbnailFormat.md
 doc/TimeGroupEnum.md
+doc/TranscodeHWAccel.md
 doc/TranscodePolicy.md
 doc/UpdateAlbumDto.md
 doc/UpdateAssetDto.md
@@ -245,6 +246,7 @@ lib/model/tag_response_dto.dart
 lib/model/tag_type_enum.dart
 lib/model/thumbnail_format.dart
 lib/model/time_group_enum.dart
+lib/model/transcode_hw_accel.dart
 lib/model/transcode_policy.dart
 lib/model/update_album_dto.dart
 lib/model/update_asset_dto.dart
@@ -366,6 +368,7 @@ test/tag_response_dto_test.dart
 test/tag_type_enum_test.dart
 test/thumbnail_format_test.dart
 test/time_group_enum_test.dart
+test/transcode_hw_accel_test.dart
 test/transcode_policy_test.dart
 test/update_album_dto_test.dart
 test/update_asset_dto_test.dart

mobile/openapi/README.md

@@ -275,6 +275,7 @@ Class | Method | HTTP request | Description
  - [TagTypeEnum](doc//TagTypeEnum.md)
  - [ThumbnailFormat](doc//ThumbnailFormat.md)
  - [TimeGroupEnum](doc//TimeGroupEnum.md)
+ - [TranscodeHWAccel](doc//TranscodeHWAccel.md)
  - [TranscodePolicy](doc//TranscodePolicy.md)
  - [UpdateAlbumDto](doc//UpdateAlbumDto.md)
  - [UpdateAssetDto](doc//UpdateAssetDto.md)

mobile/openapi/doc/AssetStatsResponseDto.md

@@ -9,8 +9,8 @@ import 'package:openapi/api.dart';
 Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
 **images** | **int** |  | 
-**total** | **int** |  | 
 **videos** | **int** |  | 
+**total** | **int** |  | 
 
 [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
 

mobile/openapi/doc/SystemConfigFFmpegDto.md

@@ -8,6 +8,7 @@ import 'package:openapi/api.dart';
 ## Properties
 Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
+**accel** | [**TranscodeHWAccel**](TranscodeHWAccel.md) |  | 
 **crf** | **int** |  | 
 **maxBitrate** | **String** |  | 
 **preset** | **String** |  | 

mobile/openapi/doc/TranscodeHWAccel.md

@@ -0,0 +1,14 @@
+# openapi.model.TranscodeHWAccel
+
+## Load the model package
+```dart
+import 'package:openapi/api.dart';
+```
+
+## Properties
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+
+

mobile/openapi/lib/api.dart

@@ -140,6 +140,7 @@ part 'model/tag_response_dto.dart';
 part 'model/tag_type_enum.dart';
 part 'model/thumbnail_format.dart';
 part 'model/time_group_enum.dart';
+part 'model/transcode_hw_accel.dart';
 part 'model/transcode_policy.dart';
 part 'model/update_album_dto.dart';
 part 'model/update_asset_dto.dart';

mobile/openapi/lib/api_client.dart

@@ -375,6 +375,8 @@ class ApiClient {
           return ThumbnailFormatTypeTransformer().decode(value);
         case 'TimeGroupEnum':
           return TimeGroupEnumTypeTransformer().decode(value);
+        case 'TranscodeHWAccel':
+          return TranscodeHWAccelTypeTransformer().decode(value);
         case 'TranscodePolicy':
           return TranscodePolicyTypeTransformer().decode(value);
         case 'UpdateAlbumDto':

mobile/openapi/lib/api_helper.dart

@@ -82,6 +82,9 @@ String parameterToString(dynamic value) {
   if (value is TimeGroupEnum) {
     return TimeGroupEnumTypeTransformer().encode(value).toString();
   }
+  if (value is TranscodeHWAccel) {
+    return TranscodeHWAccelTypeTransformer().encode(value).toString();
+  }
   if (value is TranscodePolicy) {
     return TranscodePolicyTypeTransformer().encode(value).toString();
   }

mobile/openapi/lib/model/asset_stats_response_dto.dart

@@ -14,37 +14,37 @@ class AssetStatsResponseDto {
   /// Returns a new [AssetStatsResponseDto] instance.
   AssetStatsResponseDto({
     required this.images,
-    required this.total,
     required this.videos,
+    required this.total,
   });
 
   int images;
 
-  int total;
-
   int videos;
 
+  int total;
+
   @override
   bool operator ==(Object other) => identical(this, other) || other is AssetStatsResponseDto &&
      other.images == images &&
-     other.total == total &&
-     other.videos == videos;
+     other.videos == videos &&
+     other.total == total;
 
   @override
   int get hashCode =>
     // ignore: unnecessary_parenthesis
     (images.hashCode) +
-    (total.hashCode) +
-    (videos.hashCode);
+    (videos.hashCode) +
+    (total.hashCode);
 
   @override
-  String toString() => 'AssetStatsResponseDto[images=$images, total=$total, videos=$videos]';
+  String toString() => 'AssetStatsResponseDto[images=$images, videos=$videos, total=$total]';
 
   Map<String, dynamic> toJson() {
     final json = <String, dynamic>{};
       json[r'images'] = this.images;
-      json[r'total'] = this.total;
       json[r'videos'] = this.videos;
+      json[r'total'] = this.total;
     return json;
   }
 
@@ -57,8 +57,8 @@ class AssetStatsResponseDto {
 
       return AssetStatsResponseDto(
         images: mapValueOfType<int>(json, r'images')!,
-        total: mapValueOfType<int>(json, r'total')!,
         videos: mapValueOfType<int>(json, r'videos')!,
+        total: mapValueOfType<int>(json, r'total')!,
       );
     }
     return null;
@@ -107,8 +107,8 @@ class AssetStatsResponseDto {
   /// The list of required keys that must be present in a JSON.
   static const requiredKeys = <String>{
     'images',
-    'total',
     'videos',
+    'total',
   };
 }
 

mobile/openapi/lib/model/system_config_f_fmpeg_dto.dart

@@ -13,6 +13,7 @@ part of openapi.api;
 class SystemConfigFFmpegDto {
   /// Returns a new [SystemConfigFFmpegDto] instance.
   SystemConfigFFmpegDto({
+    required this.accel,
     required this.crf,
     required this.maxBitrate,
     required this.preset,
@@ -24,6 +25,8 @@ class SystemConfigFFmpegDto {
     required this.twoPass,
   });
 
+  TranscodeHWAccel accel;
+
   int crf;
 
   String maxBitrate;
@@ -44,6 +47,7 @@ class SystemConfigFFmpegDto {
 
   @override
   bool operator ==(Object other) => identical(this, other) || other is SystemConfigFFmpegDto &&
+     other.accel == accel &&
      other.crf == crf &&
      other.maxBitrate == maxBitrate &&
      other.preset == preset &&
@@ -57,6 +61,7 @@ class SystemConfigFFmpegDto {
   @override
   int get hashCode =>
     // ignore: unnecessary_parenthesis
+    (accel.hashCode) +
     (crf.hashCode) +
     (maxBitrate.hashCode) +
     (preset.hashCode) +
@@ -68,10 +73,11 @@ class SystemConfigFFmpegDto {
     (twoPass.hashCode);
 
   @override
-  String toString() => 'SystemConfigFFmpegDto[crf=$crf, maxBitrate=$maxBitrate, preset=$preset, targetAudioCodec=$targetAudioCodec, targetResolution=$targetResolution, targetVideoCodec=$targetVideoCodec, threads=$threads, transcode=$transcode, twoPass=$twoPass]';
+  String toString() => 'SystemConfigFFmpegDto[accel=$accel, crf=$crf, maxBitrate=$maxBitrate, preset=$preset, targetAudioCodec=$targetAudioCodec, targetResolution=$targetResolution, targetVideoCodec=$targetVideoCodec, threads=$threads, transcode=$transcode, twoPass=$twoPass]';
 
   Map<String, dynamic> toJson() {
     final json = <String, dynamic>{};
+      json[r'accel'] = this.accel;
       json[r'crf'] = this.crf;
       json[r'maxBitrate'] = this.maxBitrate;
       json[r'preset'] = this.preset;
@@ -92,6 +98,7 @@ class SystemConfigFFmpegDto {
       final json = value.cast<String, dynamic>();
 
       return SystemConfigFFmpegDto(
+        accel: TranscodeHWAccel.fromJson(json[r'accel'])!,
         crf: mapValueOfType<int>(json, r'crf')!,
         maxBitrate: mapValueOfType<String>(json, r'maxBitrate')!,
         preset: mapValueOfType<String>(json, r'preset')!,
@@ -148,6 +155,7 @@ class SystemConfigFFmpegDto {
 
   /// The list of required keys that must be present in a JSON.
   static const requiredKeys = <String>{
+    'accel',
     'crf',
     'maxBitrate',
     'preset',

mobile/openapi/lib/model/transcode_hw_accel.dart

@@ -0,0 +1,91 @@
+//
+// AUTO-GENERATED FILE, DO NOT MODIFY!
+//
+// @dart=2.12
+
+// ignore_for_file: unused_element, unused_import
+// ignore_for_file: always_put_required_named_parameters_first
+// ignore_for_file: constant_identifier_names
+// ignore_for_file: lines_longer_than_80_chars
+
+part of openapi.api;
+
+
+class TranscodeHWAccel {
+  /// Instantiate a new enum with the provided [value].
+  const TranscodeHWAccel._(this.value);
+
+  /// The underlying value of this enum member.
+  final String value;
+
+  @override
+  String toString() => value;
+
+  String toJson() => value;
+
+  static const nvenc = TranscodeHWAccel._(r'nvenc');
+  static const qsv = TranscodeHWAccel._(r'qsv');
+  static const vaapi = TranscodeHWAccel._(r'vaapi');
+  static const disabled = TranscodeHWAccel._(r'disabled');
+
+  /// List of all possible values in this [enum][TranscodeHWAccel].
+  static const values = <TranscodeHWAccel>[
+    nvenc,
+    qsv,
+    vaapi,
+    disabled,
+  ];
+
+  static TranscodeHWAccel? fromJson(dynamic value) => TranscodeHWAccelTypeTransformer().decode(value);
+
+  static List<TranscodeHWAccel>? listFromJson(dynamic json, {bool growable = false,}) {
+    final result = <TranscodeHWAccel>[];
+    if (json is List && json.isNotEmpty) {
+      for (final row in json) {
+        final value = TranscodeHWAccel.fromJson(row);
+        if (value != null) {
+          result.add(value);
+        }
+      }
+    }
+    return result.toList(growable: growable);
+  }
+}
+
+/// Transformation class that can [encode] an instance of [TranscodeHWAccel] to String,
+/// and [decode] dynamic data back to [TranscodeHWAccel].
+class TranscodeHWAccelTypeTransformer {
+  factory TranscodeHWAccelTypeTransformer() => _instance ??= const TranscodeHWAccelTypeTransformer._();
+
+  const TranscodeHWAccelTypeTransformer._();
+
+  String encode(TranscodeHWAccel data) => data.value;
+
+  /// Decodes a [dynamic value][data] to a TranscodeHWAccel.
+  ///
+  /// If [allowNull] is true and the [dynamic value][data] cannot be decoded successfully,
+  /// then null is returned. However, if [allowNull] is false and the [dynamic value][data]
+  /// cannot be decoded successfully, then an [UnimplementedError] is thrown.
+  ///
+  /// The [allowNull] is very handy when an API changes and a new enum value is added or removed,
+  /// and users are still using an old app with the old code.
+  TranscodeHWAccel? decode(dynamic data, {bool allowNull = true}) {
+    if (data != null) {
+      switch (data) {
+        case r'nvenc': return TranscodeHWAccel.nvenc;
+        case r'qsv': return TranscodeHWAccel.qsv;
+        case r'vaapi': return TranscodeHWAccel.vaapi;
+        case r'disabled': return TranscodeHWAccel.disabled;
+        default:
+          if (!allowNull) {
+            throw ArgumentError('Unknown enum value to decode: $data');
+          }
+      }
+    }
+    return null;
+  }
+
+  /// Singleton [TranscodeHWAccelTypeTransformer] instance.
+  static TranscodeHWAccelTypeTransformer? _instance;
+}
+

mobile/openapi/test/asset_stats_response_dto_test.dart

@@ -21,13 +21,13 @@ void main() {
       // TODO
     });
 
-    // int total
-    test('to test the property `total`', () async {
+    // int videos
+    test('to test the property `videos`', () async {
       // TODO
     });
 
-    // int videos
-    test('to test the property `videos`', () async {
+    // int total
+    test('to test the property `total`', () async {
       // TODO
     });
 

mobile/openapi/test/system_config_f_fmpeg_dto_test.dart

@@ -16,6 +16,11 @@ void main() {
   // final instance = SystemConfigFFmpegDto();
 
   group('test SystemConfigFFmpegDto', () {
+    // TranscodeHWAccel accel
+    test('to test the property `accel`', () async {
+      // TODO
+    });
+
     // int crf
     test('to test the property `crf`', () async {
       // TODO

mobile/openapi/test/transcode_hw_accel_test.dart

@@ -0,0 +1,21 @@
+//
+// AUTO-GENERATED FILE, DO NOT MODIFY!
+//
+// @dart=2.12
+
+// ignore_for_file: unused_element, unused_import
+// ignore_for_file: always_put_required_named_parameters_first
+// ignore_for_file: constant_identifier_names
+// ignore_for_file: lines_longer_than_80_chars
+
+import 'package:openapi/api.dart';
+import 'package:test/test.dart';
+
+// tests for TranscodeHWAccel
+void main() {
+
+  group('test TranscodeHWAccel', () {
+
+  });
+
+}

server/Dockerfile

@@ -1,8 +1,19 @@
-FROM node:18.16.0-alpine3.18@sha256:f41850f74ff16a33daff988e2ea06ef8f5daeb6fb84913c7df09552a98caba09 as builder
+FROM node:18-bookworm@sha256:c85dc4392f44f5de1d0d72dd20a088a542734445f99bed7aa8ac895c706d370d as builder
 
 WORKDIR /usr/src/app
 
-RUN apk add --update-cache build-base imagemagick-dev python3 ffmpeg libraw-dev perl vips-dev vips-heif vips-jxl vips-magick
+COPY bin/install-ffmpeg.sh build-lock.json ./
+RUN sed -i -e's/ main/ main contrib non-free non-free-firmware/g' /etc/apt/sources.list.d/debian.sources
+RUN apt-get update && apt-get install -yqq build-essential ninja-build meson pkg-config jq \
+libglib2.0-dev libexpat1-dev librsvg2-dev libexif-dev libwebp-dev liborc-0.4-dev libtiff5-dev \
+libjpeg62-turbo-dev libgsf-1-dev libspng-dev libraw-dev libjxl-dev libheif-dev \
+mesa-va-drivers libmimalloc2.0 $(if [ $(arch) = "x86_64" ]; then echo "intel-media-va-driver-non-free"; fi) \
+&& ./install-ffmpeg.sh && apt-get autoremove && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# debian build for imagemagick has broken RAW support, so build manually
+COPY bin/build-imagemagick.sh bin/build-libvips.sh ./
+RUN ./build-imagemagick.sh
+RUN ./build-libvips.sh
 
 COPY package.json package-lock.json ./
 
@@ -15,14 +26,31 @@ FROM builder as prod
 RUN npm run build
 RUN npm prune --omit=dev --omit=optional
 
-
-FROM node:18.16.0-alpine3.18@sha256:f41850f74ff16a33daff988e2ea06ef8f5daeb6fb84913c7df09552a98caba09
+FROM node:18-bookworm-slim@sha256:a0cca98f2896135d4c0386922211c1f90f98f27a58b8f2c07850d0fbe1c2104e
 
 ENV NODE_ENV=production
 
 WORKDIR /usr/src/app
 
-RUN apk add --no-cache ffmpeg imagemagick-dev libraw-dev perl tini vips-dev vips-heif vips-jxl vips-magick
+COPY bin/install-ffmpeg.sh build-lock.json ./
+RUN sed -i -e's/ main/ main contrib non-free non-free-firmware/g' /etc/apt/sources.list.d/debian.sources
+RUN apt-get update && apt-get install -yqq tini libheif1 libwebp7 libwebpdemux2 libwebpmux3 mesa-va-drivers \
+libjpeg62-turbo libexpat1 librsvg2-2 libjxl0.7 libraw20 libtiff6 libspng0 libexif12 libgcc-s1 libglib2.0-0 \
+libgsf-1-114 libopenjp2-7 liblcms2-2 liborc-0.4-0 libopenexr-3-1-30 liblqr-1-0 libltdl7 zlib1g \
+mesa-va-drivers libmimalloc2.0 $(if [ $(arch) = "x86_64" ]; then echo "intel-media-va-driver-non-free"; fi) jq wget \
+&& ./install-ffmpeg.sh && apt-get remove -yqq jq wget && apt-get autoremove -yqq && apt-get clean && rm -rf /var/lib/apt/lists/* \
+&& rm install-ffmpeg.sh && rm build-lock.json
+ENV PATH=/usr/lib/jellyfin-ffmpeg:$PATH
+
+COPY --from=prod /usr/local/bin/magick /usr/local/bin/magick
+COPY --from=prod /usr/local/include/ImageMagick-7 /usr/local/include/ImageMagick-7
+
+COPY --from=prod /usr/local/bin/vips /usr/local/bin/vips
+COPY --from=prod /usr/local/include/vips/ /usr/local/include/vips/
+
+COPY --from=prod /usr/local/lib/ /usr/local/lib/
+
+RUN ldconfig /usr/local/lib
 
 COPY --from=prod /usr/src/app/node_modules ./node_modules
 COPY --from=prod /usr/src/app/dist ./dist
@@ -34,7 +62,6 @@ COPY package.json package-lock.json ./
 COPY start*.sh ./
 
 RUN npm link && npm cache clean --force
-
 VOLUME /usr/src/app/upload
 
 EXPOSE 3001

server/bin/build-imagemagick.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e
+
+LOCK=$(jq -c '.packages[] | select(.name == "imagemagick")' build-lock.json)
+IMAGEMAGICK_VERSION=${IMAGEMAGICK_VERSION:=$(echo $LOCK | jq -r '.version')}
+IMAGEMAGICK_SHA256=${IMAGEMAGICK_SHA256:=$(echo $LOCK | jq -r '.sha256')}
+
+echo "$IMAGEMAGICK_SHA256  $IMAGEMAGICK_VERSION.tar.gz" > imagemagick.sha256
+mkdir -p ImageMagick
+wget -nv https://github.com/ImageMagick/ImageMagick/archive/${IMAGEMAGICK_VERSION}.tar.gz
+sha256sum -c imagemagick.sha256
+tar -xvf ${IMAGEMAGICK_VERSION}.tar.gz -C ImageMagick --strip-components=1
+rm ${IMAGEMAGICK_VERSION}.tar.gz
+rm imagemagick.sha256
+cd ImageMagick
+./configure --with-modules
+make -j$(nproc)
+make install
+cd .. && rm -rf ImageMagick
+ldconfig /usr/local/lib

server/bin/build-libvips.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -e
+
+LOCK=$(jq -c '.packages[] | select(.name == "libvips")' build-lock.json)
+LIBVIPS_VERSION=${LIBVIPS_VERSION:=$(echo $LOCK | jq -r '.version')}
+LIBVIPS_SHA256=${LIBVIPS_SHA256:=$(echo $LOCK | jq -r '.sha256')}
+
+echo "$LIBVIPS_SHA256  vips-$LIBVIPS_VERSION.tar.xz" > libvips.sha256
+mkdir -p libvips
+wget -nv https://github.com/libvips/libvips/releases/download/v${LIBVIPS_VERSION}/vips-${LIBVIPS_VERSION}.tar.xz
+sha256sum -c libvips.sha256
+tar -xvf vips-${LIBVIPS_VERSION}.tar.xz -C libvips --strip-components=1
+rm vips-${LIBVIPS_VERSION}.tar.xz
+rm libvips.sha256
+cd libvips
+meson setup build --buildtype=release --libdir=lib -Dintrospection=false
+cd build
+# ninja test  # tests set concurrency too high for arm/v7
+ninja install
+cd .. && rm -rf libvips
+ldconfig /usr/local/lib

server/bin/install-ffmpeg.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e
+
+LOCK=$(jq -c '.packages[] | select(.name == "ffmpeg")' build-lock.json)
+export TARGETARCH=${TARGETARCH:=$(dpkg --print-architecture)}
+FFMPEG_VERSION=${FFMPEG_VERSION:=$(echo $LOCK | jq -r '.version')}
+FFMPEG_SHA256=${FFMPEG_SHA256:=$(echo $LOCK | jq -r '.sha256[$ENV.TARGETARCH]')}
+
+echo "$FFMPEG_SHA256  jellyfin-ffmpeg6_${FFMPEG_VERSION}-bookworm_${TARGETARCH}.deb" > ffmpeg.sha256
+
+wget -nv https://github.com/jellyfin/jellyfin-ffmpeg/releases/download/v${FFMPEG_VERSION}/jellyfin-ffmpeg6_${FFMPEG_VERSION}-bookworm_${TARGETARCH}.deb
+sha256sum -c ffmpeg.sha256
+apt-get -yqq -f install ./jellyfin-ffmpeg6_${FFMPEG_VERSION}-bookworm_${TARGETARCH}.deb
+rm jellyfin-ffmpeg6_${FFMPEG_VERSION}-bookworm_${TARGETARCH}.deb
+rm ffmpeg.sha256
+ldconfig /usr/lib/jellyfin-ffmpeg/lib

server/build-lock.json

@@ -0,0 +1,24 @@
+{
+  "packages": [
+    {
+      "name": "imagemagick",
+      "version": "7.1.1-13",
+
+      "sha256": "8e3ce1aaad19da9f2ca444072bcc631d193a219e3ee11c13ad6d3c895044142c"
+    },
+    {
+      "name": "libvips",
+      "version": "8.14.2",
+      "sha256": "27dad021f0835a5ab14e541d02abd41e4c3bd012d2196438df5a9e754984f7ce"
+    },
+    {
+      "name": "ffmpeg",
+      "version": "6.0-4",
+      "sha256": {
+        "amd64": "18d98b292b891cde86c2a08e5e989c3430e51a136cdc232bc4162fef3b4f0f44",
+        "arm64": "67eb1e5a38ac695dd253d9ac290ad0e9fb709e8260449a7445e8460b7db3c516",
+        "armhf": "a29605ab0eced3511c8a6623504fab5b8bb174a486d87f94bf5522ed9a5970e6"
+      }
+    }
+  ]
+}

server/immich-openapi-specs.json

@@ -4973,14 +4973,15 @@
         "type": "object"
       },
       "AssetStatsResponseDto": {
+        "type": "object",
         "properties": {
           "images": {
             "type": "integer"
           },
-          "total": {
+          "videos": {
             "type": "integer"
           },
-          "videos": {
+          "total": {
             "type": "integer"
           }
         },
@@ -4988,8 +4989,7 @@
           "images",
           "videos",
           "total"
-        ],
-        "type": "object"
+        ]
       },
       "AssetTypeEnum": {
         "enum": [
@@ -6547,6 +6547,9 @@
       },
       "SystemConfigFFmpegDto": {
         "properties": {
+          "accel": {
+            "$ref": "#/components/schemas/TranscodeHWAccel"
+          },
           "crf": {
             "type": "integer"
           },
@@ -6581,6 +6584,7 @@
           "targetVideoCodec",
           "targetAudioCodec",
           "transcode",
+          "accel",
           "preset",
           "targetResolution",
           "maxBitrate",
@@ -6809,6 +6813,15 @@
         ],
         "type": "string"
       },
+      "TranscodeHWAccel": {
+        "enum": [
+          "nvenc",
+          "qsv",
+          "vaapi",
+          "disabled"
+        ],
+        "type": "string"
+      },
       "TranscodePolicy": {
         "enum": [
           "all",

server/src/domain/media/media.repository.ts

@@ -1,3 +1,5 @@
+import { VideoCodec } from '@app/infra/entities';
+
 export const IMediaRepository = 'IMediaRepository';
 
 export interface ResizeOptions {
@@ -55,6 +57,10 @@ export interface VideoCodecSWConfig {
   getOptions(stream: VideoStreamInfo): TranscodeOptions;
 }
 
+export interface VideoCodecHWConfig extends VideoCodecSWConfig {
+  getSupportedCodecs(): Array<VideoCodec>;
+}
+
 export interface IMediaRepository {
   // image
   resize(input: string | Buffer, output: string, options: ResizeOptions): Promise<void>;

server/src/domain/media/media.service.spec.ts

@@ -1,4 +1,4 @@
-import { AssetType, SystemConfigKey, TranscodePolicy, VideoCodec } from '@app/infra/entities';
+import { AssetType, SystemConfigKey, TranscodeHWAccel, TranscodePolicy, VideoCodec } from '@app/infra/entities';
 import {
   assetStub,
   newAssetRepositoryMock,
@@ -272,6 +272,7 @@ describe(MediaService.name, () => {
             '-acodec aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-v verbose',
             '-preset ultrafast',
             '-crf 23',
           ],
@@ -309,6 +310,7 @@ describe(MediaService.name, () => {
             '-acodec aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-v verbose',
             '-preset ultrafast',
             '-crf 23',
           ],
@@ -331,6 +333,7 @@ describe(MediaService.name, () => {
             '-acodec aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-v verbose',
             '-vf scale=-2:720',
             '-preset ultrafast',
             '-crf 23',
@@ -357,6 +360,7 @@ describe(MediaService.name, () => {
             '-acodec aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-v verbose',
             '-preset ultrafast',
             '-crf 23',
           ],
@@ -380,6 +384,7 @@ describe(MediaService.name, () => {
             '-acodec aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-v verbose',
             '-vf scale=720:-2',
             '-preset ultrafast',
             '-crf 23',
@@ -404,6 +409,7 @@ describe(MediaService.name, () => {
             '-acodec aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-v verbose',
             '-vf scale=-2:720',
             '-preset ultrafast',
             '-crf 23',
@@ -428,6 +434,7 @@ describe(MediaService.name, () => {
             '-acodec aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-v verbose',
             '-vf scale=-2:720',
             '-preset ultrafast',
             '-crf 23',
@@ -476,6 +483,7 @@ describe(MediaService.name, () => {
             '-acodec aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-v verbose',
             '-vf scale=-2:720',
             '-preset ultrafast',
             '-crf 23',
@@ -505,6 +513,7 @@ describe(MediaService.name, () => {
             '-acodec aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-v verbose',
             '-vf scale=-2:720',
             '-preset ultrafast',
             '-b:v 3104k',
@@ -531,6 +540,7 @@ describe(MediaService.name, () => {
             '-acodec aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-v verbose',
             '-vf scale=-2:720',
             '-preset ultrafast',
             '-crf 23',
@@ -559,6 +569,7 @@ describe(MediaService.name, () => {
             '-acodec aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-v verbose',
             '-vf scale=-2:720',
             '-cpu-used 5',
             '-row-mt 1',
@@ -589,6 +600,7 @@ describe(MediaService.name, () => {
             '-acodec aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-v verbose',
             '-vf scale=-2:720',
             '-cpu-used 2',
             '-row-mt 1',
@@ -618,6 +630,7 @@ describe(MediaService.name, () => {
             '-acodec aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-v verbose',
             '-vf scale=-2:720',
             '-row-mt 1',
             '-crf 23',
@@ -646,6 +659,7 @@ describe(MediaService.name, () => {
             '-acodec aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-v verbose',
             '-vf scale=-2:720',
             '-cpu-used 5',
             '-row-mt 1',
@@ -673,6 +687,7 @@ describe(MediaService.name, () => {
             '-acodec aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-v verbose',
             '-vf scale=-2:720',
             '-preset ultrafast',
             '-threads 2',
@@ -700,6 +715,7 @@ describe(MediaService.name, () => {
             '-acodec aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-v verbose',
             '-vf scale=-2:720',
             '-preset ultrafast',
             '-crf 23',
@@ -727,6 +743,7 @@ describe(MediaService.name, () => {
             '-acodec aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-v verbose',
             '-vf scale=-2:720',
             '-preset ultrafast',
             '-threads 2',
@@ -757,6 +774,7 @@ describe(MediaService.name, () => {
             '-acodec aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-v verbose',
             '-vf scale=-2:720',
             '-preset ultrafast',
             '-crf 23',
@@ -765,5 +783,508 @@ describe(MediaService.name, () => {
         },
       );
     });
+
+    it('should skip transcoding for audioless videos with optimal policy if video codec is correct', async () => {
+      mediaMock.probe.mockResolvedValue(probeStub.noAudioStreams);
+      configMock.load.mockResolvedValue([
+        { key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.HEVC },
+        { key: SystemConfigKey.FFMPEG_TRANSCODE, value: TranscodePolicy.OPTIMAL },
+        { key: SystemConfigKey.FFMPEG_TARGET_RESOLUTION, value: '1080p' },
+      ]);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).not.toHaveBeenCalled();
+    });
+
+    it('should return false if hwaccel is enabled for an unsupported codec', async () => {
+      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
+      configMock.load.mockResolvedValue([
+        { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.NVENC },
+        { key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.VP9 },
+      ]);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await expect(sut.handleVideoConversion({ id: assetStub.video.id })).resolves.toEqual(false);
+      expect(mediaMock.transcode).not.toHaveBeenCalled();
+    });
+
+    it('should return false if hwaccel option is invalid', async () => {
+      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
+      configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_ACCEL, value: 'invalid' }]);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await expect(sut.handleVideoConversion({ id: assetStub.video.id })).resolves.toEqual(false);
+      expect(mediaMock.transcode).not.toHaveBeenCalled();
+    });
+
+    it('should set two pass options for nvenc when enabled', async () => {
+      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
+      configMock.load.mockResolvedValue([
+        { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.NVENC },
+        { key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '10000k' },
+        { key: SystemConfigKey.FFMPEG_TWO_PASS, value: true },
+      ]);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/asset-id.mp4',
+        {
+          inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'],
+          outputOptions: [
+            `-vcodec h264_nvenc`,
+            '-tune hq',
+            '-qmin 0',
+            '-g 250',
+            '-bf 3',
+            '-b_ref_mode middle',
+            '-temporal-aq 1',
+            '-rc-lookahead 20',
+            '-i_qfactor 0.75',
+            '-b_qfactor 1.1',
+            '-acodec aac',
+            '-movflags faststart',
+            '-fps_mode passthrough',
+            '-v verbose',
+            '-vf hwupload_cuda,scale_cuda=-2:720',
+            '-preset p1',
+            '-b:v 6897k',
+            '-maxrate 10000k',
+            '-bufsize 6897k',
+            '-multipass 2',
+          ],
+          twoPass: false,
+        },
+      );
+    });
+
+    it('should set vbr options for nvenc when max bitrate is enabled', async () => {
+      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
+      configMock.load.mockResolvedValue([
+        { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.NVENC },
+        { key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '10000k' },
+      ]);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/asset-id.mp4',
+        {
+          inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'],
+          outputOptions: [
+            `-vcodec h264_nvenc`,
+            '-tune hq',
+            '-qmin 0',
+            '-g 250',
+            '-bf 3',
+            '-b_ref_mode middle',
+            '-temporal-aq 1',
+            '-rc-lookahead 20',
+            '-i_qfactor 0.75',
+            '-b_qfactor 1.1',
+            '-acodec aac',
+            '-movflags faststart',
+            '-fps_mode passthrough',
+            '-v verbose',
+            '-vf hwupload_cuda,scale_cuda=-2:720',
+            '-preset p1',
+            '-cq:v 23',
+            '-maxrate 10000k',
+            '-bufsize 6897k',
+          ],
+          twoPass: false,
+        },
+      );
+    });
+
+    it('should set cq options for nvenc when max bitrate is disabled', async () => {
+      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
+      configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.NVENC }]);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/asset-id.mp4',
+        {
+          inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'],
+          outputOptions: [
+            `-vcodec h264_nvenc`,
+            '-tune hq',
+            '-qmin 0',
+            '-g 250',
+            '-bf 3',
+            '-b_ref_mode middle',
+            '-temporal-aq 1',
+            '-rc-lookahead 20',
+            '-i_qfactor 0.75',
+            '-b_qfactor 1.1',
+            '-acodec aac',
+            '-movflags faststart',
+            '-fps_mode passthrough',
+            '-v verbose',
+            '-vf hwupload_cuda,scale_cuda=-2:720',
+            '-preset p1',
+            '-cq:v 23',
+          ],
+          twoPass: false,
+        },
+      );
+    });
+
+    it('should omit preset for nvenc if invalid', async () => {
+      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
+      configMock.load.mockResolvedValue([
+        { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.NVENC },
+        { key: SystemConfigKey.FFMPEG_PRESET, value: 'invalid' },
+      ]);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/asset-id.mp4',
+        {
+          inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'],
+          outputOptions: [
+            `-vcodec h264_nvenc`,
+            '-tune hq',
+            '-qmin 0',
+            '-g 250',
+            '-bf 3',
+            '-b_ref_mode middle',
+            '-temporal-aq 1',
+            '-rc-lookahead 20',
+            '-i_qfactor 0.75',
+            '-b_qfactor 1.1',
+            '-acodec aac',
+            '-movflags faststart',
+            '-fps_mode passthrough',
+            '-v verbose',
+            '-vf hwupload_cuda,scale_cuda=-2:720',
+            '-cq:v 23',
+          ],
+          twoPass: false,
+        },
+      );
+    });
+
+    it('should ignore two pass for nvenc if max bitrate is disabled', async () => {
+      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
+      configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.NVENC }]);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/asset-id.mp4',
+        {
+          inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'],
+          outputOptions: [
+            `-vcodec h264_nvenc`,
+            '-tune hq',
+            '-qmin 0',
+            '-g 250',
+            '-bf 3',
+            '-b_ref_mode middle',
+            '-temporal-aq 1',
+            '-rc-lookahead 20',
+            '-i_qfactor 0.75',
+            '-b_qfactor 1.1',
+            '-acodec aac',
+            '-movflags faststart',
+            '-fps_mode passthrough',
+            '-v verbose',
+            '-vf hwupload_cuda,scale_cuda=-2:720',
+            '-preset p1',
+            '-cq:v 23',
+          ],
+          twoPass: false,
+        },
+      );
+    });
+
+    it('should set options for qsv', async () => {
+      storageMock.readdir.mockResolvedValue(['renderD128']);
+      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
+      configMock.load.mockResolvedValue([
+        { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.QSV },
+        { key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '10000k' },
+      ]);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/asset-id.mp4',
+        {
+          inputOptions: ['-init_hw_device qsv=hw', '-filter_hw_device hw'],
+          outputOptions: [
+            `-vcodec h264_qsv`,
+            '-g 256',
+            '-extbrc 1',
+            '-refs 5',
+            '-bf 7',
+            '-acodec aac',
+            '-movflags faststart',
+            '-fps_mode passthrough',
+            '-v verbose',
+            '-vf format=nv12,hwupload=extra_hw_frames=64,scale_qsv=-1:720',
+            '-preset 7',
+            '-global_quality 23',
+            '-maxrate 10000k',
+            '-bufsize 20000k',
+          ],
+          twoPass: false,
+        },
+      );
+    });
+
+    it('should omit preset for qsv if invalid', async () => {
+      storageMock.readdir.mockResolvedValue(['renderD128']);
+      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
+      configMock.load.mockResolvedValue([
+        { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.QSV },
+        { key: SystemConfigKey.FFMPEG_PRESET, value: 'invalid' },
+      ]);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/asset-id.mp4',
+        {
+          inputOptions: ['-init_hw_device qsv=hw', '-filter_hw_device hw'],
+          outputOptions: [
+            `-vcodec h264_qsv`,
+            '-g 256',
+            '-extbrc 1',
+            '-refs 5',
+            '-bf 7',
+            '-acodec aac',
+            '-movflags faststart',
+            '-fps_mode passthrough',
+            '-v verbose',
+            '-vf format=nv12,hwupload=extra_hw_frames=64,scale_qsv=-1:720',
+            '-global_quality 23',
+          ],
+          twoPass: false,
+        },
+      );
+    });
+
+    it('should set low power mode for qsv if target video codec is vp9', async () => {
+      storageMock.readdir.mockResolvedValue(['renderD128']);
+      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
+      configMock.load.mockResolvedValue([
+        { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.QSV },
+        { key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.VP9 },
+      ]);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/asset-id.mp4',
+        {
+          inputOptions: ['-init_hw_device qsv=hw', '-filter_hw_device hw'],
+          outputOptions: [
+            `-vcodec vp9_qsv`,
+            '-g 256',
+            '-extbrc 1',
+            '-refs 5',
+            '-bf 7',
+            '-low_power 1',
+            '-acodec aac',
+            '-movflags faststart',
+            '-fps_mode passthrough',
+            '-v verbose',
+            '-vf format=nv12,hwupload=extra_hw_frames=64,scale_qsv=-1:720',
+            '-preset 7',
+            '-q:v 23',
+          ],
+          twoPass: false,
+        },
+      );
+    });
+
+    it('should return false for qsv if no hw devices', async () => {
+      storageMock.readdir.mockResolvedValue([]);
+      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
+      configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.QSV }]);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await expect(sut.handleVideoConversion({ id: assetStub.video.id })).resolves.toEqual(false);
+      expect(mediaMock.transcode).not.toHaveBeenCalled();
+    });
+
+    it('should set vbr options for vaapi when max bitrate is enabled', async () => {
+      storageMock.readdir.mockResolvedValue(['renderD128']);
+      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
+      configMock.load.mockResolvedValue([
+        { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.VAAPI },
+        { key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '10000k' },
+      ]);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/asset-id.mp4',
+        {
+          inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD128', '-filter_hw_device accel'],
+          outputOptions: [
+            `-vcodec h264_vaapi`,
+            '-acodec aac',
+            '-movflags faststart',
+            '-fps_mode passthrough',
+            '-v verbose',
+            '-vf format=nv12,hwupload,scale_vaapi=-2:720',
+            '-compression_level 7',
+            '-b:v 6897k',
+            '-maxrate 10000k',
+            '-minrate 3448.5k',
+            '-rc_mode 3',
+          ],
+          twoPass: false,
+        },
+      );
+    });
+
+    it('should set cq options for vaapi when max bitrate is disabled', async () => {
+      storageMock.readdir.mockResolvedValue(['renderD128']);
+      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
+      configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.VAAPI }]);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/asset-id.mp4',
+        {
+          inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD128', '-filter_hw_device accel'],
+          outputOptions: [
+            `-vcodec h264_vaapi`,
+            '-acodec aac',
+            '-movflags faststart',
+            '-fps_mode passthrough',
+            '-v verbose',
+            '-vf format=nv12,hwupload,scale_vaapi=-2:720',
+            '-compression_level 7',
+            '-qp 23',
+            '-global_quality 23',
+            '-rc_mode 1',
+          ],
+          twoPass: false,
+        },
+      );
+    });
+
+    it('should omit preset for vaapi if invalid', async () => {
+      storageMock.readdir.mockResolvedValue(['renderD128']);
+      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
+      configMock.load.mockResolvedValue([
+        { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.VAAPI },
+        { key: SystemConfigKey.FFMPEG_PRESET, value: 'invalid' },
+      ]);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/asset-id.mp4',
+        {
+          inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD128', '-filter_hw_device accel'],
+          outputOptions: [
+            `-vcodec h264_vaapi`,
+            '-acodec aac',
+            '-movflags faststart',
+            '-fps_mode passthrough',
+            '-v verbose',
+            '-vf format=nv12,hwupload,scale_vaapi=-2:720',
+            '-qp 23',
+            '-global_quality 23',
+            '-rc_mode 1',
+          ],
+          twoPass: false,
+        },
+      );
+    });
+
+    it('should prefer gpu for vaapi if available', async () => {
+      storageMock.readdir.mockResolvedValue(['renderD129', 'card1', 'card0', 'renderD128']);
+      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
+      configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.VAAPI }]);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/asset-id.mp4',
+        {
+          inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/card1', '-filter_hw_device accel'],
+          outputOptions: [
+            `-vcodec h264_vaapi`,
+            '-acodec aac',
+            '-movflags faststart',
+            '-fps_mode passthrough',
+            '-v verbose',
+            '-vf format=nv12,hwupload,scale_vaapi=-2:720',
+            '-compression_level 7',
+            '-qp 23',
+            '-global_quality 23',
+            '-rc_mode 1',
+          ],
+          twoPass: false,
+        },
+      );
+
+      storageMock.readdir.mockResolvedValue(['renderD129', 'renderD128']);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/asset-id.mp4',
+        {
+          inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD129', '-filter_hw_device accel'],
+          outputOptions: [
+            `-vcodec h264_vaapi`,
+            '-acodec aac',
+            '-movflags faststart',
+            '-fps_mode passthrough',
+            '-v verbose',
+            '-vf format=nv12,hwupload,scale_vaapi=-2:720',
+            '-compression_level 7',
+            '-qp 23',
+            '-global_quality 23',
+            '-rc_mode 1',
+          ],
+          twoPass: false,
+        },
+      );
+    });
+
+    it('should fallback to sw transcoding if hw transcoding fails', async () => {
+      storageMock.readdir.mockResolvedValue(['renderD128']);
+      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
+      configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.VAAPI }]);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      mediaMock.transcode.mockRejectedValueOnce(new Error('error'));
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledTimes(2);
+      expect(mediaMock.transcode).toHaveBeenLastCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/asset-id.mp4',
+        {
+          inputOptions: [],
+          outputOptions: [
+            '-vcodec h264',
+            '-acodec aac',
+            '-movflags faststart',
+            '-fps_mode passthrough',
+            '-v verbose',
+            '-vf scale=-2:720',
+            '-preset ultrafast',
+            '-crf 23',
+          ],
+          twoPass: false,
+        },
+      );
+    });
+
+    it('should return false for vaapi if no hw devices', async () => {
+      storageMock.readdir.mockResolvedValue([]);
+      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
+      configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.VAAPI }]);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await expect(sut.handleVideoConversion({ id: assetStub.video.id })).resolves.toEqual(false);
+      expect(mediaMock.transcode).not.toHaveBeenCalled();
+    });
   });
 });

server/src/domain/media/media.service.ts

@@ -1,4 +1,4 @@
-import { AssetEntity, AssetType, TranscodePolicy, VideoCodec } from '@app/infra/entities';
+import { AssetEntity, AssetType, TranscodeHWAccel, TranscodePolicy, VideoCodec } from '@app/infra/entities';
 import { Inject, Injectable, Logger, UnsupportedMediaTypeException } from '@nestjs/common';
 import { join } from 'path';
 import { IAssetRepository, WithoutProperty } from '../asset';
@@ -8,8 +8,8 @@ import { IStorageRepository, StorageCore, StorageFolder } from '../storage';
 import { ISystemConfigRepository, SystemConfigFFmpegDto } from '../system-config';
 import { SystemConfigCore } from '../system-config/system-config.core';
 import { JPEG_THUMBNAIL_SIZE, WEBP_THUMBNAIL_SIZE } from './media.constant';
-import { AudioStreamInfo, IMediaRepository, VideoStreamInfo } from './media.repository';
-import { H264Config, HEVCConfig, VP9Config } from './media.util';
+import { AudioStreamInfo, IMediaRepository, VideoCodecHWConfig, VideoStreamInfo } from './media.repository';
+import { H264Config, HEVCConfig, NVENCConfig, QSVConfig, VAAPIConfig, VP9Config } from './media.util';
 
 @Injectable()
 export class MediaService {
@@ -155,14 +155,26 @@ export class MediaService {
 
     let transcodeOptions;
     try {
-      transcodeOptions = this.getCodecConfig(config).getOptions(mainVideoStream);
+      transcodeOptions = await this.getCodecConfig(config).then((c) => c.getOptions(mainVideoStream));
     } catch (err) {
       this.logger.error(`An error occurred while configuring transcoding options: ${err}`);
       return false;
     }
 
     this.logger.log(`Start encoding video ${asset.id} ${JSON.stringify(transcodeOptions)}`);
-    await this.mediaRepository.transcode(input, output, transcodeOptions);
+    try {
+      await this.mediaRepository.transcode(input, output, transcodeOptions);
+    } catch (err) {
+      this.logger.error(err);
+      if (config.accel && config.accel !== TranscodeHWAccel.DISABLED) {
+        this.logger.error(
+          `Error occurred during transcoding. Retrying with ${config.accel.toUpperCase()} acceleration disabled.`,
+        );
+      }
+      config.accel = TranscodeHWAccel.DISABLED;
+      transcodeOptions = await this.getCodecConfig(config).then((c) => c.getOptions(mainVideoStream));
+      await this.mediaRepository.transcode(input, output, transcodeOptions);
+    }
 
     this.logger.log(`Encoding success ${asset.id}`);
 
@@ -195,15 +207,11 @@ export class MediaService {
     const isTargetContainer = ['mov,mp4,m4a,3gp,3g2,mj2', 'mp4', 'mov'].includes(containerExtension);
     const isTargetAudioCodec = audioStream == null || audioStream.codecName === ffmpegConfig.targetAudioCodec;
 
-    if (audioStream != null) {
-      this.logger.verbose(
-        `${asset.id}: AudioCodecName ${audioStream.codecName}, AudioStreamCodecType ${audioStream.codecType}, containerExtension ${containerExtension}`,
-      );
-    } else {
-      this.logger.verbose(
-        `${asset.id}: AudioCodecName None, AudioStreamCodecType None, containerExtension ${containerExtension}`,
-      );
-    }
+    this.logger.verbose(
+      `${asset.id}: AudioCodecName ${audioStream?.codecName ?? 'None'}, AudioStreamCodecType ${
+        audioStream?.codecType ?? 'None'
+      }, containerExtension ${containerExtension}`,
+    );
 
     const allTargetsMatching = isTargetVideoCodec && isTargetAudioCodec && isTargetContainer;
     const scalingEnabled = ffmpegConfig.targetResolution !== 'original';
@@ -228,7 +236,14 @@ export class MediaService {
     }
   }
 
-  private getCodecConfig(config: SystemConfigFFmpegDto) {
+  async getCodecConfig(config: SystemConfigFFmpegDto) {
+    if (config.accel === TranscodeHWAccel.DISABLED) {
+      return this.getSWCodecConfig(config);
+    }
+    return this.getHWCodecConfig(config);
+  }
+
+  private getSWCodecConfig(config: SystemConfigFFmpegDto) {
     switch (config.targetVideoCodec) {
       case VideoCodec.H264:
         return new H264Config(config);
@@ -240,4 +255,31 @@ export class MediaService {
         throw new UnsupportedMediaTypeException(`Codec '${config.targetVideoCodec}' is unsupported`);
     }
   }
+
+  private async getHWCodecConfig(config: SystemConfigFFmpegDto) {
+    let handler: VideoCodecHWConfig;
+    let devices: string[];
+    switch (config.accel) {
+      case TranscodeHWAccel.NVENC:
+        handler = new NVENCConfig(config);
+        break;
+      case TranscodeHWAccel.QSV:
+        devices = await this.storageRepository.readdir('/dev/dri');
+        handler = new QSVConfig(config, devices);
+        break;
+      case TranscodeHWAccel.VAAPI:
+        devices = await this.storageRepository.readdir('/dev/dri');
+        handler = new VAAPIConfig(config, devices);
+        break;
+      default:
+        throw new UnsupportedMediaTypeException(`${config.accel.toUpperCase()} acceleration is unsupported`);
+    }
+    if (!handler.getSupportedCodecs().includes(config.targetVideoCodec)) {
+      throw new UnsupportedMediaTypeException(
+        `${config.accel.toUpperCase()} acceleration does not support codec '${config.targetVideoCodec.toUpperCase()}'. Supported codecs: ${handler.getSupportedCodecs()}`,
+      );
+    }
+
+    return handler;
+  }
 }

server/src/domain/media/media.util.ts

@@ -1,13 +1,26 @@
+import { TranscodeHWAccel, VideoCodec } from '@app/infra/entities';
 import { SystemConfigFFmpegDto } from '../system-config/dto';
-import { BitrateDistribution, TranscodeOptions, VideoCodecSWConfig, VideoStreamInfo } from './media.repository';
-
+import {
+  BitrateDistribution,
+  TranscodeOptions,
+  VideoCodecHWConfig,
+  VideoCodecSWConfig,
+  VideoStreamInfo,
+} from './media.repository';
 class BaseConfig implements VideoCodecSWConfig {
   constructor(protected config: SystemConfigFFmpegDto) {}
 
   getOptions(stream: VideoStreamInfo) {
     const options = {
       inputOptions: this.getBaseInputOptions(),
-      outputOptions: this.getBaseOutputOptions(),
+      outputOptions: this.getBaseOutputOptions().concat([
+        `-acodec ${this.config.targetAudioCodec}`,
+        // Makes a second pass moving the moov atom to the
+        // beginning of the file for improved playback speed.
+        '-movflags faststart',
+        '-fps_mode passthrough',
+        '-v verbose',
+      ]),
       twoPass: this.eligibleForTwoPass(),
     } as TranscodeOptions;
     const filters = this.getFilterOptions(stream);
@@ -26,14 +39,7 @@ class BaseConfig implements VideoCodecSWConfig {
   }
 
   getBaseOutputOptions() {
-    return [
-      `-vcodec ${this.config.targetVideoCodec}`,
-      `-acodec ${this.config.targetAudioCodec}`,
-      // Makes a second pass moving the moov atom to the beginning of
-      // the file for improved playback speed.
-      '-movflags faststart',
-      '-fps_mode passthrough',
-    ];
+    return [`-vcodec ${this.config.targetVideoCodec}`];
   }
 
   getFilterOptions(stream: VideoStreamInfo) {
@@ -77,11 +83,11 @@ class BaseConfig implements VideoCodecSWConfig {
   }
 
   eligibleForTwoPass() {
-    if (!this.config.twoPass) {
+    if (!this.config.twoPass || this.config.accel !== TranscodeHWAccel.DISABLED) {
       return false;
     }
 
-    return this.isBitrateConstrained() || this.config.targetVideoCodec === 'vp9';
+    return this.isBitrateConstrained() || this.config.targetVideoCodec === VideoCodec.VP9;
   }
 
   getBitrateDistribution() {
@@ -107,7 +113,8 @@ class BaseConfig implements VideoCodecSWConfig {
 
   getScaling(stream: VideoStreamInfo) {
     const targetResolution = this.getTargetResolution(stream);
-    return this.isVideoVertical(stream) ? `${targetResolution}:-2` : `-2:${targetResolution}`;
+    const mult = this.config.accel === TranscodeHWAccel.QSV ? 1 : 2; // QSV doesn't support scaling numbers below -1
+    return this.isVideoVertical(stream) ? `${targetResolution}:-${mult}` : `-${mult}:${targetResolution}`;
   }
 
   isVideoRotated(stream: VideoStreamInfo) {
@@ -137,6 +144,34 @@ class BaseConfig implements VideoCodecSWConfig {
   }
 }
 
+export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig {
+  protected devices: string[];
+
+  constructor(protected config: SystemConfigFFmpegDto, devices: string[] = []) {
+    super(config);
+    this.devices = this.validateDevices(devices);
+  }
+
+  getSupportedCodecs() {
+    return [VideoCodec.H264, VideoCodec.HEVC, VideoCodec.VP9];
+  }
+
+  validateDevices(devices: string[]) {
+    return devices
+      .filter((device) => device.startsWith('renderD') || device.startsWith('card'))
+      .sort((a, b) => {
+        // order GPU devices first
+        if (a.startsWith('card') && b.startsWith('renderD')) {
+          return -1;
+        }
+        if (a.startsWith('renderD') && b.startsWith('card')) {
+          return 1;
+        }
+        return -a.localeCompare(b);
+      });
+  }
+}
+
 export class H264Config extends BaseConfig {
   getThreadOptions() {
     if (this.config.threads <= 0) {
@@ -189,3 +224,168 @@ export class VP9Config extends BaseConfig {
     return ['-row-mt 1', ...super.getThreadOptions()];
   }
 }
+
+export class NVENCConfig extends BaseHWConfig {
+  getSupportedCodecs() {
+    return [VideoCodec.H264, VideoCodec.HEVC];
+  }
+
+  getBaseInputOptions() {
+    return ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'];
+  }
+
+  getBaseOutputOptions() {
+    return [
+      `-vcodec ${this.config.targetVideoCodec}_nvenc`,
+      // below settings recommended from https://docs.nvidia.com/video-technologies/video-codec-sdk/12.0/ffmpeg-with-nvidia-gpu/index.html#command-line-for-latency-tolerant-high-quality-transcoding
+      '-tune hq',
+      '-qmin 0',
+      '-g 250',
+      '-bf 3',
+      '-b_ref_mode middle',
+      '-temporal-aq 1',
+      '-rc-lookahead 20',
+      '-i_qfactor 0.75',
+      '-b_qfactor 1.1',
+    ];
+  }
+
+  getFilterOptions(stream: VideoStreamInfo) {
+    const options = ['hwupload_cuda'];
+    if (this.shouldScale(stream)) {
+      options.push(`scale_cuda=${this.getScaling(stream)}`);
+    }
+
+    return options;
+  }
+
+  getPresetOptions() {
+    let presetIndex = this.getPresetIndex();
+    if (presetIndex < 0) {
+      return [];
+    }
+    presetIndex = 7 - Math.min(6, presetIndex); // map to p1-p7; p7 is the highest quality, so reverse index
+    return [`-preset p${presetIndex}`];
+  }
+
+  getBitrateOptions() {
+    const bitrates = this.getBitrateDistribution();
+    if (bitrates.max > 0 && this.config.twoPass) {
+      return [
+        `-b:v ${bitrates.target}${bitrates.unit}`,
+        `-maxrate ${bitrates.max}${bitrates.unit}`,
+        `-bufsize ${bitrates.target}${bitrates.unit}`,
+        '-multipass 2',
+      ];
+    } else if (bitrates.max > 0) {
+      return [
+        `-cq:v ${this.config.crf}`,
+        `-maxrate ${bitrates.max}${bitrates.unit}`,
+        `-bufsize ${bitrates.target}${bitrates.unit}`,
+      ];
+    } else {
+      return [`-cq:v ${this.config.crf}`];
+    }
+  }
+
+  getThreadOptions() {
+    return [];
+  }
+}
+
+export class QSVConfig extends BaseHWConfig {
+  getBaseInputOptions() {
+    if (!this.devices.length) {
+      throw Error('No QSV device found');
+    }
+    return ['-init_hw_device qsv=hw', '-filter_hw_device hw'];
+  }
+
+  getBaseOutputOptions() {
+    // recommended from https://github.com/intel/media-delivery/blob/master/doc/benchmarks/intel-iris-xe-max-graphics/intel-iris-xe-max-graphics.md
+    const options = [`-vcodec ${this.config.targetVideoCodec}_qsv`, '-g 256', '-extbrc 1', '-refs 5', '-bf 7'];
+    // VP9 requires enabling low power mode https://git.ffmpeg.org/gitweb/ffmpeg.git/commit/33583803e107b6d532def0f9d949364b01b6ad5a
+    if (this.config.targetVideoCodec === VideoCodec.VP9) {
+      options.push('-low_power 1');
+    }
+    return options;
+  }
+
+  getFilterOptions(stream: VideoStreamInfo) {
+    const options = ['format=nv12', 'hwupload=extra_hw_frames=64'];
+    if (this.shouldScale(stream)) {
+      options.push(`scale_qsv=${this.getScaling(stream)}`);
+    }
+    return options;
+  }
+
+  getPresetOptions() {
+    let presetIndex = this.getPresetIndex();
+    if (presetIndex < 0) {
+      return [];
+    }
+    presetIndex = Math.min(6, presetIndex) + 1; // 1 to 7
+    return [`-preset ${presetIndex}`];
+  }
+
+  getBitrateOptions() {
+    const options = [];
+    if (this.config.targetVideoCodec !== VideoCodec.VP9) {
+      options.push(`-global_quality ${this.config.crf}`);
+    } else {
+      options.push(`-q:v ${this.config.crf}`);
+    }
+    const bitrates = this.getBitrateDistribution();
+    if (bitrates.max > 0) {
+      options.push(`-maxrate ${bitrates.max}${bitrates.unit}`);
+      options.push(`-bufsize ${bitrates.max * 2}${bitrates.unit}`);
+    }
+    return options;
+  }
+}
+
+export class VAAPIConfig extends BaseHWConfig {
+  getBaseInputOptions() {
+    if (this.devices.length === 0) {
+      throw Error('No VAAPI device found');
+    }
+    return [`-init_hw_device vaapi=accel:/dev/dri/${this.devices[0]}`, '-filter_hw_device accel'];
+  }
+
+  getBaseOutputOptions() {
+    return [`-vcodec ${this.config.targetVideoCodec}_vaapi`];
+  }
+
+  getFilterOptions(stream: VideoStreamInfo) {
+    const options = ['format=nv12', 'hwupload'];
+    if (this.shouldScale(stream)) {
+      options.push(`scale_vaapi=${this.getScaling(stream)}`);
+    }
+
+    return options;
+  }
+
+  getPresetOptions() {
+    let presetIndex = this.getPresetIndex();
+    if (presetIndex < 0) {
+      return [];
+    }
+    presetIndex = Math.min(6, presetIndex) + 1; // 1 to 7
+    return [`-compression_level ${presetIndex}`];
+  }
+
+  getBitrateOptions() {
+    const bitrates = this.getBitrateDistribution();
+    // VAAPI doesn't allow setting both quality and max bitrate
+    if (bitrates.max > 0) {
+      return [
+        `-b:v ${bitrates.target}${bitrates.unit}`,
+        `-maxrate ${bitrates.max}${bitrates.unit}`,
+        `-minrate ${bitrates.min}${bitrates.unit}`,
+        '-rc_mode 3',
+      ]; // variable bitrate
+    } else {
+      return [`-qp ${this.config.crf}`, `-global_quality ${this.config.crf}`, '-rc_mode 1']; // constant quality
+    }
+  }
+}

server/src/domain/storage/storage.repository.ts

@@ -29,4 +29,5 @@ export interface IStorageRepository {
   checkFileExists(filepath: string, mode?: number): Promise<boolean>;
   mkdirSync(filepath: string): void;
   checkDiskUsage(folder: string): Promise<DiskUsage>;
+  readdir(folder: string): Promise<string[]>;
 }

server/src/domain/system-config/dto/system-config-ffmpeg.dto.ts

@@ -1,4 +1,4 @@
-import { AudioCodec, TranscodePolicy, VideoCodec } from '@app/infra/entities';
+import { AudioCodec, TranscodeHWAccel, TranscodePolicy, VideoCodec } from '@app/infra/entities';
 import { ApiProperty } from '@nestjs/swagger';
 import { Type } from 'class-transformer';
 import { IsBoolean, IsEnum, IsInt, IsString, Max, Min } from 'class-validator';
@@ -40,4 +40,8 @@ export class SystemConfigFFmpegDto {
   @IsEnum(TranscodePolicy)
   @ApiProperty({ enumName: 'TranscodePolicy', enum: TranscodePolicy })
   transcode!: TranscodePolicy;
+
+  @IsEnum(TranscodeHWAccel)
+  @ApiProperty({ enumName: 'TranscodeHWAccel', enum: TranscodeHWAccel })
+  accel!: TranscodeHWAccel;
 }

server/src/domain/system-config/system-config.core.ts

@@ -4,6 +4,7 @@ import {
   SystemConfigEntity,
   SystemConfigKey,
   SystemConfigValue,
+  TranscodeHWAccel,
   TranscodePolicy,
   VideoCodec,
 } from '@app/infra/entities';
@@ -27,6 +28,7 @@ export const defaults = Object.freeze<SystemConfig>({
     maxBitrate: '0',
     twoPass: false,
     transcode: TranscodePolicy.REQUIRED,
+    accel: TranscodeHWAccel.DISABLED,
   },
   job: {
     [QueueName.BACKGROUND_TASK]: { concurrency: 5 },

server/src/domain/system-config/system-config.service.spec.ts

@@ -3,6 +3,7 @@ import {
   SystemConfig,
   SystemConfigEntity,
   SystemConfigKey,
+  TranscodeHWAccel,
   TranscodePolicy,
   VideoCodec,
 } from '@app/infra/entities';
@@ -41,6 +42,7 @@ const updatedConfig = Object.freeze<SystemConfig>({
     maxBitrate: '0',
     twoPass: false,
     transcode: TranscodePolicy.REQUIRED,
+    accel: TranscodeHWAccel.DISABLED,
   },
   oauth: {
     autoLaunch: true,

server/src/infra/entities/system-config.entity.ts

@@ -23,6 +23,7 @@ export enum SystemConfigKey {
   FFMPEG_MAX_BITRATE = 'ffmpeg.maxBitrate',
   FFMPEG_TWO_PASS = 'ffmpeg.twoPass',
   FFMPEG_TRANSCODE = 'ffmpeg.transcode',
+  FFMPEG_ACCEL = 'ffmpeg.accel',
 
   JOB_THUMBNAIL_GENERATION_CONCURRENCY = 'job.thumbnailGeneration.concurrency',
   JOB_METADATA_EXTRACTION_CONCURRENCY = 'job.metadataExtraction.concurrency',
@@ -71,6 +72,13 @@ export enum AudioCodec {
   OPUS = 'opus',
 }
 
+export enum TranscodeHWAccel {
+  NVENC = 'nvenc',
+  QSV = 'qsv',
+  VAAPI = 'vaapi',
+  DISABLED = 'disabled',
+}
+
 export interface SystemConfig {
   ffmpeg: {
     crf: number;
@@ -82,6 +90,7 @@ export interface SystemConfig {
     maxBitrate: string;
     twoPass: boolean;
     transcode: TranscodePolicy;
+    accel: TranscodeHWAccel;
   };
   job: Record<QueueName, { concurrency: number }>;
   oauth: {

server/src/infra/repositories/filesystem.provider.ts

@@ -1,7 +1,7 @@
 import { DiskUsage, ImmichReadStream, ImmichZipStream, IStorageRepository } from '@app/domain';
 import archiver from 'archiver';
 import { constants, createReadStream, existsSync, mkdirSync } from 'fs';
-import fs from 'fs/promises';
+import fs, { readdir } from 'fs/promises';
 import mv from 'mv';
 import { promisify } from 'node:util';
 import path from 'path';
@@ -92,4 +92,6 @@ export class FilesystemProvider implements IStorageRepository {
       total: stats.blocks * stats.bsize,
     };
   }
+
+  readdir = readdir;
 }

server/src/infra/repositories/media.repository.ts

@@ -6,6 +6,7 @@ import sharp from 'sharp';
 import { promisify } from 'util';
 
 const probe = promisify<string, FfprobeData>(ffmpeg.ffprobe);
+sharp.concurrency(0);
 
 export class MediaRepository implements IMediaRepository {
   private logger = new Logger(MediaRepository.name);
@@ -73,7 +74,7 @@ export class MediaRepository implements IMediaRepository {
         .map((stream) => ({
           height: stream.height || 0,
           width: stream.width || 0,
-          codecName: stream.codec_name,
+          codecName: stream.codec_name === 'h265' ? 'hevc' : stream.codec_name,
           codecType: stream.codec_type,
           frameCount: Number.parseInt(stream.nb_frames ?? '0'),
           rotation: Number.parseInt(`${stream.rotation ?? 0}`),
@@ -91,6 +92,7 @@ export class MediaRepository implements IMediaRepository {
     if (!options.twoPass) {
       return new Promise((resolve, reject) => {
         ffmpeg(input, { niceness: 10 })
+          .inputOptions(options.inputOptions)
           .outputOptions(options.outputOptions)
           .output(output)
           .on('error', (err, stdout, stderr) => {
@@ -106,6 +108,7 @@ export class MediaRepository implements IMediaRepository {
     // recommended for vp9 for better quality and compression
     return new Promise((resolve, reject) => {
       ffmpeg(input, { niceness: 10 })
+        .inputOptions(options.inputOptions)
         .outputOptions(options.outputOptions)
         .addOptions('-pass', '1')
         .addOptions('-passlogfile', output)
@@ -118,6 +121,7 @@ export class MediaRepository implements IMediaRepository {
         .on('end', () => {
           // second pass
           ffmpeg(input, { niceness: 10 })
+            .inputOptions(options.inputOptions)
             .outputOptions(options.outputOptions)
             .addOptions('-pass', '2')
             .addOptions('-passlogfile', output)

server/start.sh

@@ -1,5 +1,7 @@
 #!/bin/sh
 
+export LD_PRELOAD=/usr/lib/$(arch)-linux-gnu/libmimalloc.so.2
+
 if [ "$DB_URL_FILE" ]; then
 	export DB_URL=$(cat $DB_URL_FILE)
 	unset DB_URL_FILE

server/test/fixtures/media.stub.ts

@@ -7,7 +7,7 @@ const probeStubDefaultFormat: VideoFormat = {
 };
 
 const probeStubDefaultVideoStream: VideoStreamInfo[] = [
-  { height: 1080, width: 1920, codecName: 'h265', codecType: 'video', frameCount: 100, rotation: 0 },
+  { height: 1080, width: 1920, codecName: 'hevc', codecType: 'video', frameCount: 100, rotation: 0 },
 ];
 
 const probeStubDefaultAudioStream: AudioStreamInfo[] = [{ codecName: 'aac', codecType: 'audio' }];
@@ -20,13 +20,14 @@ const probeStubDefault: VideoInfo = {
 
 export const probeStub = {
   noVideoStreams: Object.freeze<VideoInfo>({ ...probeStubDefault, videoStreams: [] }),
+  noAudioStreams: Object.freeze<VideoInfo>({ ...probeStubDefault, audioStreams: [] }),
   multipleVideoStreams: Object.freeze<VideoInfo>({
     ...probeStubDefault,
     videoStreams: [
       {
         height: 1080,
         width: 400,
-        codecName: 'h265',
+        codecName: 'hevc',
         codecType: 'video',
         frameCount: 100,
         rotation: 0,
@@ -47,7 +48,7 @@ export const probeStub = {
       {
         height: 0,
         width: 400,
-        codecName: 'h265',
+        codecName: 'hevc',
         codecType: 'video',
         frameCount: 100,
         rotation: 0,

server/test/repositories/storage.repository.mock.ts

@@ -11,5 +11,6 @@ export const newStorageRepositoryMock = (): jest.Mocked<IStorageRepository> => {
     checkFileExists: jest.fn(),
     mkdirSync: jest.fn(),
     checkDiskUsage: jest.fn(),
+    readdir: jest.fn(),
   };
 };

web/src/api/open-api/api.ts

@@ -666,13 +666,13 @@ export interface AssetStatsResponseDto {
      * @type {number}
      * @memberof AssetStatsResponseDto
      */
-    'total': number;
+    'videos': number;
     /**
      * 
      * @type {number}
      * @memberof AssetStatsResponseDto
      */
-    'videos': number;
+    'total': number;
 }
 /**
  * 
@@ -2510,6 +2510,12 @@ export interface SystemConfigDto {
  * @interface SystemConfigFFmpegDto
  */
 export interface SystemConfigFFmpegDto {
+    /**
+     * 
+     * @type {TranscodeHWAccel}
+     * @memberof SystemConfigFFmpegDto
+     */
+    'accel': TranscodeHWAccel;
     /**
      * 
      * @type {number}
@@ -2858,6 +2864,22 @@ export const TimeGroupEnum = {
 export type TimeGroupEnum = typeof TimeGroupEnum[keyof typeof TimeGroupEnum];
 
 
+/**
+ * 
+ * @export
+ * @enum {string}
+ */
+
+export const TranscodeHWAccel = {
+    Nvenc: 'nvenc',
+    Qsv: 'qsv',
+    Vaapi: 'vaapi',
+    Disabled: 'disabled'
+} as const;
+
+export type TranscodeHWAccel = typeof TranscodeHWAccel[keyof typeof TranscodeHWAccel];
+
+
 /**
  * 
  * @export

web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte

@@ -3,7 +3,7 @@
     notificationController,
     NotificationType,
   } from '$lib/components/shared-components/notification/notification';
-  import { api, AudioCodec, SystemConfigFFmpegDto, TranscodePolicy, VideoCodec } from '@api';
+  import { api, AudioCodec, SystemConfigFFmpegDto, TranscodeHWAccel, TranscodePolicy, VideoCodec } from '@api';
   import SettingButtonsRow from '../setting-buttons-row.svelte';
   import SettingInputField, { SettingInputFieldType } from '../setting-input-field.svelte';
   import SettingSelect from '../setting-select.svelte';
@@ -189,6 +189,29 @@
             isEdited={!(ffmpegConfig.transcode == savedConfig.transcode)}
           />
 
+          <SettingSelect
+            label="HARDWARE ACCELERATION"
+            desc="Experimental. Much faster, but will have lower quality at the same bitrate. This setting is 'best effort': it will fallback to software transcoding on failure. VP9 may or may not work depending on your hardware."
+            bind:value={ffmpegConfig.accel}
+            name="accel"
+            options={[
+              { value: TranscodeHWAccel.Nvenc, text: 'NVENC (requires NVIDIA GPU)' },
+              {
+                value: TranscodeHWAccel.Qsv,
+                text: 'Quick Sync (requires 7th gen Intel CPU or later)',
+              },
+              {
+                value: TranscodeHWAccel.Vaapi,
+                text: 'VAAPI',
+              },
+              {
+                value: TranscodeHWAccel.Disabled,
+                text: 'Disabled',
+              },
+            ]}
+            isEdited={!(ffmpegConfig.accel == savedConfig.accel)}
+          />
+
           <SettingSwitch
             title="TWO-PASS ENCODING"
             subtitle="Transcode in two passes to produce better encoded videos. When max bitrate is enabled (required for it to work with H.264 and HEVC), this mode uses a bitrate range based on the max bitrate and ignores CRF. For VP9, CRF can be used if max bitrate is disabled."