From dfd6846deba5d64d2e9cec2e02bed5abae664523 Mon Sep 17 00:00:00 2001
From: martin <74269598+martabal@users.noreply.github.com>
Date: Sun, 3 Dec 2023 23:34:23 +0100
Subject: [PATCH] fix(server): video orientation (#5455)

* fix: video orientation

* pr feedback
---
 .../domain/metadata/metadata.service.spec.ts  | 22 +++++++++++-
 .../src/domain/metadata/metadata.service.ts   | 34 +++++++++++++++++++
 2 files changed, 55 insertions(+), 1 deletion(-)

diff --git a/server/src/domain/metadata/metadata.service.spec.ts b/server/src/domain/metadata/metadata.service.spec.ts
index 0ef5dfd73..1eefc5eba 100644
--- a/server/src/domain/metadata/metadata.service.spec.ts
+++ b/server/src/domain/metadata/metadata.service.spec.ts
@@ -5,11 +5,13 @@ import {
   newAssetRepositoryMock,
   newCryptoRepositoryMock,
   newJobRepositoryMock,
+  newMediaRepositoryMock,
   newMetadataRepositoryMock,
   newMoveRepositoryMock,
   newPersonRepositoryMock,
   newStorageRepositoryMock,
   newSystemConfigRepositoryMock,
+  probeStub,
 } from '@test';
 import { randomBytes } from 'crypto';
 import { Stats } from 'fs';
@@ -21,6 +23,7 @@ import {
   IAssetRepository,
   ICryptoRepository,
   IJobRepository,
+  IMediaRepository,
   IMetadataRepository,
   IMoveRepository,
   IPersonRepository,
@@ -30,7 +33,7 @@ import {
   WithProperty,
   WithoutProperty,
 } from '../repositories';
-import { MetadataService } from './metadata.service';
+import { MetadataService, Orientation } from './metadata.service';
 
 describe(MetadataService.name, () => {
   let albumMock: jest.Mocked<IAlbumRepository>;
@@ -40,6 +43,7 @@ describe(MetadataService.name, () => {
   let jobMock: jest.Mocked<IJobRepository>;
   let metadataMock: jest.Mocked<IMetadataRepository>;
   let moveMock: jest.Mocked<IMoveRepository>;
+  let mediaMock: jest.Mocked<IMediaRepository>;
   let personMock: jest.Mocked<IPersonRepository>;
   let storageMock: jest.Mocked<IStorageRepository>;
   let sut: MetadataService;
@@ -54,6 +58,7 @@ describe(MetadataService.name, () => {
     moveMock = newMoveRepositoryMock();
     personMock = newPersonRepositoryMock();
     storageMock = newStorageRepositoryMock();
+    mediaMock = newMediaRepositoryMock();
 
     sut = new MetadataService(
       albumMock,
@@ -63,6 +68,7 @@ describe(MetadataService.name, () => {
       metadataMock,
       storageMock,
       configMock,
+      mediaMock,
       moveMock,
       personMock,
     );
@@ -277,6 +283,7 @@ describe(MetadataService.name, () => {
 
     it('should not apply motion photos if asset is video', async () => {
       assetMock.getByIds.mockResolvedValue([{ ...assetStub.livePhotoMotionAsset, isVisible: true }]);
+      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
 
       await sut.handleMetadataExtraction({ id: assetStub.livePhotoMotionAsset.id });
       expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoMotionAsset.id]);
@@ -287,6 +294,19 @@ describe(MetadataService.name, () => {
       );
     });
 
+    it('should extract the correct video orientation', async () => {
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      mediaMock.probe.mockResolvedValue(probeStub.videoStreamVertical2160p);
+      metadataMock.readTags.mockResolvedValue(null);
+
+      await sut.handleMetadataExtraction({ id: assetStub.video.id });
+
+      expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.video.id]);
+      expect(assetMock.upsertExif).toHaveBeenCalledWith(
+        expect.objectContaining({ orientation: Orientation.Rotate270CW }),
+      );
+    });
+
     it('should apply motion photos', async () => {
       assetMock.getByIds.mockResolvedValue([{ ...assetStub.livePhotoStillAsset, livePhotoVideoId: null }]);
       metadataMock.readTags.mockResolvedValue({
diff --git a/server/src/domain/metadata/metadata.service.ts b/server/src/domain/metadata/metadata.service.ts
index e9c7ff931..9c8f887dc 100644
--- a/server/src/domain/metadata/metadata.service.ts
+++ b/server/src/domain/metadata/metadata.service.ts
@@ -14,6 +14,7 @@ import {
   IAssetRepository,
   ICryptoRepository,
   IJobRepository,
+  IMediaRepository,
   IMetadataRepository,
   IMoveRepository,
   IPersonRepository,
@@ -49,6 +50,17 @@ interface DirectoryEntry {
   Item: DirectoryItem;
 }
 
+export enum Orientation {
+  Horizontal = '1',
+  MirrorHorizontal = '2',
+  Rotate180 = '3',
+  MirrorVertical = '4',
+  MirrorHorizontalRotate270CW = '5',
+  Rotate90CW = '6',
+  MirrorHorizontalRotate90CW = '7',
+  Rotate270CW = '8',
+}
+
 type ExifEntityWithoutGeocodeAndTypeOrm = Omit<
   ExifEntity,
   'city' | 'state' | 'country' | 'description' | 'exifTextSearchableColumn'
@@ -90,6 +102,7 @@ export class MetadataService {
     @Inject(IMetadataRepository) private repository: IMetadataRepository,
     @Inject(IStorageRepository) private storageRepository: IStorageRepository,
     @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
+    @Inject(IMediaRepository) private mediaRepository: IMediaRepository,
     @Inject(IMoveRepository) moveRepository: IMoveRepository,
     @Inject(IPersonRepository) personRepository: IPersonRepository,
   ) {
@@ -182,6 +195,27 @@ export class MetadataService {
 
     const { exifData, tags } = await this.exifData(asset);
 
+    if (asset.type === AssetType.VIDEO) {
+      const { videoStreams } = await this.mediaRepository.probe(asset.originalPath);
+
+      if (videoStreams[0]) {
+        switch (videoStreams[0].rotation) {
+          case -90:
+            exifData.orientation = Orientation.Rotate90CW;
+            break;
+          case 0:
+            exifData.orientation = Orientation.Horizontal;
+            break;
+          case 90:
+            exifData.orientation = Orientation.Rotate270CW;
+            break;
+          case 180:
+            exifData.orientation = Orientation.Rotate180;
+            break;
+        }
+      }
+    }
+
     await this.applyMotionPhotos(asset, tags);
     await this.applyReverseGeocoding(asset, exifData);
     await this.assetRepository.upsertExif(exifData);