PlaybackManager.cpp 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704
  1. /*
  2. * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
  3. *
  4. * SPDX-License-Identifier: BSD-2-Clause
  5. */
  6. #include <AK/Format.h>
  7. #include <LibCore/Timer.h>
  8. #include <LibVideo/Containers/Matroska/MatroskaDemuxer.h>
  9. #include <LibVideo/VP9/Decoder.h>
  10. #include "PlaybackManager.h"
  11. namespace Video {
  12. #define TRY_OR_FATAL_ERROR(expression) \
  13. ({ \
  14. auto&& _fatal_expression = (expression); \
  15. if (_fatal_expression.is_error()) { \
  16. dispatch_fatal_error(_fatal_expression.release_error()); \
  17. return; \
  18. } \
  19. static_assert(!::AK::Detail::IsLvalueReference<decltype(_fatal_expression.release_value())>, \
  20. "Do not return a reference from a fallible expression"); \
  21. _fatal_expression.release_value(); \
  22. })
  23. DecoderErrorOr<NonnullOwnPtr<PlaybackManager>> PlaybackManager::from_file(StringView filename)
  24. {
  25. auto demuxer = TRY(Matroska::MatroskaDemuxer::from_file(filename));
  26. return create(move(demuxer));
  27. }
  28. DecoderErrorOr<NonnullOwnPtr<PlaybackManager>> PlaybackManager::from_mapped_file(NonnullRefPtr<Core::MappedFile> mapped_file)
  29. {
  30. auto demuxer = TRY(Matroska::MatroskaDemuxer::from_mapped_file(move(mapped_file)));
  31. return create(move(demuxer));
  32. }
  33. DecoderErrorOr<NonnullOwnPtr<PlaybackManager>> PlaybackManager::from_data(ReadonlyBytes data)
  34. {
  35. auto demuxer = TRY(Matroska::MatroskaDemuxer::from_data(data));
  36. return create(move(demuxer));
  37. }
  38. DecoderErrorOr<NonnullOwnPtr<PlaybackManager>> PlaybackManager::create(NonnullOwnPtr<Demuxer> demuxer)
  39. {
  40. auto video_tracks = TRY(demuxer->get_tracks_for_type(TrackType::Video));
  41. if (video_tracks.is_empty())
  42. return DecoderError::with_description(DecoderErrorCategory::Invalid, "No video track is present"sv);
  43. auto track = video_tracks[0];
  44. dbgln_if(PLAYBACK_MANAGER_DEBUG, "Selecting video track number {}", track.identifier());
  45. auto decoder = DECODER_TRY_ALLOC(try_make<VP9::Decoder>());
  46. auto frame_queue = DECODER_TRY_ALLOC(VideoFrameQueue::create());
  47. auto playback_manager = DECODER_TRY_ALLOC(try_make<PlaybackManager>(demuxer, track, move(decoder), move(frame_queue)));
  48. playback_manager->m_present_timer = DECODER_TRY_ALLOC(Core::Timer::create_single_shot(0, [&self = *playback_manager] { self.timer_callback(); }));
  49. playback_manager->m_decode_thread = DECODER_TRY_ALLOC(Threading::Thread::try_create([&self = *playback_manager] {
  50. while (!self.m_stop_decoding.load())
  51. self.decode_and_queue_one_sample();
  52. dbgln_if(PLAYBACK_MANAGER_DEBUG, "Media Decoder thread ended.");
  53. return 0;
  54. },
  55. "Media Decoder"sv));
  56. playback_manager->m_playback_handler = make<SeekingStateHandler>(*playback_manager, false, Time::zero(), SeekMode::Fast);
  57. DECODER_TRY_ALLOC(playback_manager->m_playback_handler->on_enter());
  58. playback_manager->m_decode_thread->start();
  59. return playback_manager;
  60. }
  61. PlaybackManager::PlaybackManager(NonnullOwnPtr<Demuxer>& demuxer, Track video_track, NonnullOwnPtr<VideoDecoder>&& decoder, VideoFrameQueue&& frame_queue)
  62. : m_demuxer(move(demuxer))
  63. , m_selected_video_track(video_track)
  64. , m_frame_queue(move(frame_queue))
  65. , m_decoder(move(decoder))
  66. , m_decode_wait_condition(m_decode_wait_mutex)
  67. {
  68. }
  69. PlaybackManager::~PlaybackManager()
  70. {
  71. m_stop_decoding.exchange(true);
  72. m_decode_wait_condition.broadcast();
  73. dbgln_if(PLAYBACK_MANAGER_DEBUG, "Waiting for decode thread to end...");
  74. (void)m_decode_thread->join();
  75. dbgln_if(PLAYBACK_MANAGER_DEBUG, "Successfully destroyed PlaybackManager.");
  76. }
  77. void PlaybackManager::resume_playback()
  78. {
  79. dbgln_if(PLAYBACK_MANAGER_DEBUG, "Resuming playback.");
  80. TRY_OR_FATAL_ERROR(m_playback_handler->play());
  81. }
  82. void PlaybackManager::pause_playback()
  83. {
  84. dbgln_if(PLAYBACK_MANAGER_DEBUG, "Pausing playback.");
  85. if (!m_playback_handler->is_playing())
  86. warnln("Cannot pause.");
  87. TRY_OR_FATAL_ERROR(m_playback_handler->pause());
  88. }
  89. Time PlaybackManager::current_playback_time()
  90. {
  91. return m_playback_handler->current_time();
  92. }
  93. Time PlaybackManager::duration()
  94. {
  95. auto duration_result = ({
  96. auto demuxer_locker = Threading::MutexLocker(m_demuxer_mutex);
  97. m_demuxer->duration();
  98. });
  99. if (duration_result.is_error())
  100. dispatch_decoder_error(duration_result.release_error());
  101. return duration_result.release_value();
  102. }
  103. void PlaybackManager::dispatch_fatal_error(Error error)
  104. {
  105. dbgln_if(PLAYBACK_MANAGER_DEBUG, "Encountered fatal error: {}", error.string_literal());
  106. // FIXME: For threading, this will have to use a pre-allocated event to send to the main loop
  107. // to be able to gracefully handle OOM.
  108. if (on_fatal_playback_error)
  109. on_fatal_playback_error(move(error));
  110. }
  111. void PlaybackManager::dispatch_decoder_error(DecoderError error)
  112. {
  113. switch (error.category()) {
  114. case DecoderErrorCategory::EndOfStream:
  115. dbgln_if(PLAYBACK_MANAGER_DEBUG, "{}", error.string_literal());
  116. TRY_OR_FATAL_ERROR(m_playback_handler->stop());
  117. break;
  118. default:
  119. dbgln("Playback error encountered: {}", error.string_literal());
  120. TRY_OR_FATAL_ERROR(m_playback_handler->stop());
  121. if (on_decoder_error)
  122. on_decoder_error(move(error));
  123. break;
  124. }
  125. }
  126. void PlaybackManager::dispatch_new_frame(RefPtr<Gfx::Bitmap> frame)
  127. {
  128. if (on_video_frame)
  129. on_video_frame(move(frame));
  130. }
  131. bool PlaybackManager::dispatch_frame_queue_item(FrameQueueItem&& item)
  132. {
  133. if (item.is_error()) {
  134. dispatch_decoder_error(item.release_error());
  135. return true;
  136. }
  137. dbgln_if(PLAYBACK_MANAGER_DEBUG, "Sent frame for presentation with timestamp {}ms, late by {}ms", item.timestamp().to_milliseconds(), (current_playback_time() - item.timestamp()).to_milliseconds());
  138. dispatch_new_frame(item.bitmap());
  139. return false;
  140. }
  141. void PlaybackManager::dispatch_state_change()
  142. {
  143. if (on_playback_state_change)
  144. on_playback_state_change();
  145. }
  146. void PlaybackManager::timer_callback()
  147. {
  148. TRY_OR_FATAL_ERROR(m_playback_handler->on_timer_callback());
  149. }
  150. void PlaybackManager::seek_to_timestamp(Time target_timestamp, SeekMode seek_mode)
  151. {
  152. TRY_OR_FATAL_ERROR(m_playback_handler->seek(target_timestamp, seek_mode));
  153. }
  154. Optional<Time> PlaybackManager::seek_demuxer_to_most_recent_keyframe(Time timestamp, Optional<Time> earliest_available_sample)
  155. {
  156. auto result = m_demuxer->seek_to_most_recent_keyframe(m_selected_video_track, timestamp, move(earliest_available_sample));
  157. if (result.is_error())
  158. dispatch_decoder_error(result.release_error());
  159. return result.release_value();
  160. }
  161. Optional<FrameQueueItem> PlaybackManager::dequeue_one_frame()
  162. {
  163. auto result = m_frame_queue.dequeue();
  164. m_decode_wait_condition.broadcast();
  165. if (result.is_error()) {
  166. if (result.error() != VideoFrameQueue::QueueStatus::Empty)
  167. dispatch_fatal_error(Error::from_string_literal("Dequeue failed with an unexpected error"));
  168. return {};
  169. }
  170. return result.release_value();
  171. }
  172. void PlaybackManager::restart_playback()
  173. {
  174. seek_to_timestamp(Time::zero());
  175. }
  176. void PlaybackManager::decode_and_queue_one_sample()
  177. {
  178. #if PLAYBACK_MANAGER_DEBUG
  179. auto start_time = Time::now_monotonic();
  180. #endif
  181. FrameQueueItem item_to_enqueue;
  182. while (item_to_enqueue.is_empty()) {
  183. // Get a sample to decode.
  184. auto sample_result = [&]() {
  185. // FIXME: Implement and use a class to enforce that this field is accessed through a mutex (like Kernel::MutexProtected).
  186. Threading::MutexLocker demuxer_locker(m_demuxer_mutex);
  187. return m_demuxer->get_next_video_sample_for_track(m_selected_video_track);
  188. }();
  189. if (sample_result.is_error()) {
  190. item_to_enqueue = FrameQueueItem::error_marker(sample_result.release_error(), FrameQueueItem::no_timestamp);
  191. break;
  192. }
  193. auto sample = sample_result.release_value();
  194. // Submit the sample to the decoder.
  195. auto decode_result = m_decoder->receive_sample(sample->data());
  196. if (decode_result.is_error()) {
  197. item_to_enqueue = FrameQueueItem::error_marker(decode_result.release_error(), sample->timestamp());
  198. break;
  199. }
  200. // Retrieve the last available frame to present.
  201. OwnPtr<VideoFrame> decoded_frame = nullptr;
  202. while (true) {
  203. auto frame_result = m_decoder->get_decoded_frame();
  204. if (frame_result.is_error()) {
  205. if (frame_result.error().category() == DecoderErrorCategory::NeedsMoreInput) {
  206. break;
  207. }
  208. item_to_enqueue = FrameQueueItem::error_marker(frame_result.release_error(), sample->timestamp());
  209. break;
  210. }
  211. decoded_frame = frame_result.release_value();
  212. }
  213. // Convert the frame for display.
  214. if (decoded_frame != nullptr) {
  215. auto& cicp = decoded_frame->cicp();
  216. cicp.adopt_specified_values(sample->container_cicp());
  217. cicp.default_code_points_if_unspecified({ ColorPrimaries::BT709, TransferCharacteristics::BT709, MatrixCoefficients::BT709, VideoFullRangeFlag::Studio });
  218. // BT.601, BT.709 and BT.2020 have a similar transfer function to sRGB, so other applications
  219. // (Chromium, VLC) forgo transfer characteristics conversion. We will emulate that behavior by
  220. // handling those as sRGB instead, which causes no transfer function change in the output,
  221. // unless display color management is later implemented.
  222. switch (cicp.transfer_characteristics()) {
  223. case TransferCharacteristics::BT601:
  224. case TransferCharacteristics::BT709:
  225. case TransferCharacteristics::BT2020BitDepth10:
  226. case TransferCharacteristics::BT2020BitDepth12:
  227. cicp.set_transfer_characteristics(TransferCharacteristics::SRGB);
  228. break;
  229. default:
  230. break;
  231. }
  232. auto bitmap_result = decoded_frame->to_bitmap();
  233. if (bitmap_result.is_error())
  234. item_to_enqueue = FrameQueueItem::error_marker(bitmap_result.release_error(), sample->timestamp());
  235. else
  236. item_to_enqueue = FrameQueueItem::frame(bitmap_result.release_value(), sample->timestamp());
  237. break;
  238. }
  239. }
  240. VERIFY(!item_to_enqueue.is_empty());
  241. #if PLAYBACK_MANAGER_DEBUG
  242. dbgln("Media Decoder: Sample at {}ms took {}ms to decode, queue contains ~{} items", item_to_enqueue.timestamp().to_milliseconds(), (Time::now_monotonic() - start_time).to_milliseconds(), m_frame_queue.weak_used());
  243. #endif
  244. auto wait = [&] {
  245. auto wait_locker = Threading::MutexLocker(m_decode_wait_mutex);
  246. m_decode_wait_condition.wait();
  247. };
  248. bool had_error = item_to_enqueue.is_error();
  249. while (true) {
  250. if (m_frame_queue.can_enqueue()) {
  251. MUST(m_frame_queue.enqueue(move(item_to_enqueue)));
  252. break;
  253. }
  254. if (m_stop_decoding.load()) {
  255. dbgln_if(PLAYBACK_MANAGER_DEBUG, "Media Decoder: Received signal to stop, exiting decode function...");
  256. return;
  257. }
  258. m_buffer_is_full.exchange(true);
  259. dbgln_if(PLAYBACK_MANAGER_DEBUG, "Media Decoder: Waiting for a frame to be dequeued...");
  260. wait();
  261. }
  262. if (had_error) {
  263. dbgln_if(PLAYBACK_MANAGER_DEBUG, "Media Decoder: Encountered {}, waiting...", item_to_enqueue.error().category() == DecoderErrorCategory::EndOfStream ? "end of stream"sv : "error"sv);
  264. m_buffer_is_full.exchange(true);
  265. wait();
  266. }
  267. m_buffer_is_full.exchange(false);
  268. }
  269. Time PlaybackManager::PlaybackStateHandler::current_time() const
  270. {
  271. return m_manager.m_last_present_in_media_time;
  272. }
  273. ErrorOr<void> PlaybackManager::PlaybackStateHandler::seek(Time target_timestamp, SeekMode seek_mode)
  274. {
  275. return replace_handler_and_delete_this<SeekingStateHandler>(is_playing(), target_timestamp, seek_mode);
  276. }
  277. ErrorOr<void> PlaybackManager::PlaybackStateHandler::stop()
  278. {
  279. return replace_handler_and_delete_this<StoppedStateHandler>();
  280. }
  281. template<class T, class... Args>
  282. ErrorOr<void> PlaybackManager::PlaybackStateHandler::replace_handler_and_delete_this(Args... args)
  283. {
  284. OwnPtr<PlaybackStateHandler> temp_handler = TRY(try_make<T>(m_manager, args...));
  285. m_manager.m_playback_handler.swap(temp_handler);
  286. #if PLAYBACK_MANAGER_DEBUG
  287. m_has_exited = true;
  288. dbgln("Changing state from {} to {}", temp_handler->name(), m_manager.m_playback_handler->name());
  289. #endif
  290. TRY(m_manager.m_playback_handler->on_enter());
  291. m_manager.dispatch_state_change();
  292. return {};
  293. }
  294. PlaybackManager& PlaybackManager::PlaybackStateHandler::manager() const
  295. {
  296. #if PLAYBACK_MANAGER_DEBUG
  297. VERIFY(!m_has_exited);
  298. #endif
  299. return m_manager;
  300. }
  301. class PlaybackManager::ResumingStateHandler : public PlaybackManager::PlaybackStateHandler {
  302. public:
  303. ResumingStateHandler(PlaybackManager& manager, bool playing)
  304. : PlaybackStateHandler(manager)
  305. , m_playing(playing)
  306. {
  307. }
  308. ~ResumingStateHandler() override = default;
  309. protected:
  310. ErrorOr<void> assume_next_state()
  311. {
  312. if (!m_playing)
  313. return replace_handler_and_delete_this<PausedStateHandler>();
  314. return replace_handler_and_delete_this<PlayingStateHandler>();
  315. }
  316. ErrorOr<void> play() override
  317. {
  318. m_playing = true;
  319. manager().dispatch_state_change();
  320. return {};
  321. }
  322. bool is_playing() const override { return m_playing; }
  323. ErrorOr<void> pause() override
  324. {
  325. m_playing = false;
  326. manager().dispatch_state_change();
  327. return {};
  328. }
  329. bool m_playing { false };
  330. };
  331. class PlaybackManager::PlayingStateHandler : public PlaybackManager::PlaybackStateHandler {
  332. public:
  333. PlayingStateHandler(PlaybackManager& manager)
  334. : PlaybackStateHandler(manager)
  335. {
  336. }
  337. ~PlayingStateHandler() override = default;
  338. private:
  339. ErrorOr<void> on_enter() override
  340. {
  341. m_last_present_in_real_time = Time::now_monotonic();
  342. return on_timer_callback();
  343. }
  344. StringView name() override { return "Playing"sv; }
  345. bool is_playing() const override { return true; };
  346. PlaybackState get_state() const override { return PlaybackState::Playing; }
  347. ErrorOr<void> pause() override
  348. {
  349. manager().m_last_present_in_media_time = current_time();
  350. return replace_handler_and_delete_this<PausedStateHandler>();
  351. }
  352. ErrorOr<void> buffer() override
  353. {
  354. manager().m_last_present_in_media_time = current_time();
  355. return replace_handler_and_delete_this<BufferingStateHandler>(true);
  356. }
  357. Time current_time() const override
  358. {
  359. return manager().m_last_present_in_media_time + (Time::now_monotonic() - m_last_present_in_real_time);
  360. }
  361. ErrorOr<void> on_timer_callback() override
  362. {
  363. auto set_presentation_timer = [&]() {
  364. auto frame_time_ms = (manager().m_next_frame->timestamp() - current_time()).to_milliseconds();
  365. VERIFY(frame_time_ms <= NumericLimits<int>::max());
  366. dbgln_if(PLAYBACK_MANAGER_DEBUG, "Time until next frame is {}ms", frame_time_ms);
  367. manager().m_present_timer->start(max(static_cast<int>(frame_time_ms), 0));
  368. };
  369. if (manager().m_next_frame.has_value() && current_time() < manager().m_next_frame->timestamp()) {
  370. dbgln_if(PLAYBACK_MANAGER_DEBUG, "Current time {}ms is too early to present the next frame at {}ms, delaying", current_time().to_milliseconds(), manager().m_next_frame->timestamp().to_milliseconds());
  371. set_presentation_timer();
  372. return {};
  373. }
  374. Optional<FrameQueueItem> future_frame_item;
  375. bool should_present_frame = false;
  376. // Skip frames until we find a frame past the current playback time, and keep the one that precedes it to display.
  377. while (true) {
  378. future_frame_item = manager().dequeue_one_frame();
  379. if (!future_frame_item.has_value())
  380. break;
  381. if (future_frame_item->timestamp() >= current_time() || future_frame_item->timestamp() == FrameQueueItem::no_timestamp) {
  382. dbgln_if(PLAYBACK_MANAGER_DEBUG, "Should present frame, future {} is error or after {}ms", future_frame_item->debug_string(), current_time().to_milliseconds());
  383. should_present_frame = true;
  384. break;
  385. }
  386. if (manager().m_next_frame.has_value()) {
  387. dbgln_if(PLAYBACK_MANAGER_DEBUG, "At {}ms: Dropped {} in favor of {}", current_time().to_milliseconds(), manager().m_next_frame->debug_string(), future_frame_item->debug_string());
  388. manager().m_skipped_frames++;
  389. }
  390. manager().m_next_frame.emplace(future_frame_item.release_value());
  391. }
  392. // If we don't have both of these items, we can't present, since we need to set a timer for
  393. // the next frame. Check if we need to buffer based on the current state.
  394. if (!manager().m_next_frame.has_value() || !future_frame_item.has_value()) {
  395. #if PLAYBACK_MANAGER_DEBUG
  396. StringBuilder debug_string_builder;
  397. debug_string_builder.append("We don't have "sv);
  398. if (!manager().m_next_frame.has_value()) {
  399. debug_string_builder.append("a frame to present"sv);
  400. if (!future_frame_item.has_value())
  401. debug_string_builder.append(" or a future frame"sv);
  402. } else {
  403. debug_string_builder.append("a future frame"sv);
  404. }
  405. debug_string_builder.append(", checking for error and buffering"sv);
  406. dbgln_if(PLAYBACK_MANAGER_DEBUG, debug_string_builder.to_deprecated_string());
  407. #endif
  408. if (future_frame_item.has_value()) {
  409. if (future_frame_item->is_error()) {
  410. manager().dispatch_decoder_error(future_frame_item.release_value().release_error());
  411. return {};
  412. }
  413. manager().m_next_frame.emplace(future_frame_item.release_value());
  414. }
  415. TRY(buffer());
  416. return {};
  417. }
  418. // If we have a frame, send it for presentation.
  419. if (should_present_frame) {
  420. auto now = Time::now_monotonic();
  421. manager().m_last_present_in_media_time += now - m_last_present_in_real_time;
  422. m_last_present_in_real_time = now;
  423. if (manager().dispatch_frame_queue_item(manager().m_next_frame.release_value()))
  424. return {};
  425. }
  426. // Now that we've presented the current frame, we can throw whatever error is next in queue.
  427. // This way, we always display a frame before the stream ends, and should also show any frames
  428. // we already had when a real error occurs.
  429. if (future_frame_item->is_error()) {
  430. manager().dispatch_decoder_error(future_frame_item.release_value().release_error());
  431. return {};
  432. }
  433. // The future frame item becomes the next one to present.
  434. manager().m_next_frame.emplace(future_frame_item.release_value());
  435. set_presentation_timer();
  436. return {};
  437. }
  438. Time m_last_present_in_real_time = Time::zero();
  439. };
  440. class PlaybackManager::PausedStateHandler : public PlaybackManager::PlaybackStateHandler {
  441. public:
  442. PausedStateHandler(PlaybackManager& manager)
  443. : PlaybackStateHandler(manager)
  444. {
  445. }
  446. ~PausedStateHandler() override = default;
  447. private:
  448. StringView name() override { return "Paused"sv; }
  449. ErrorOr<void> play() override
  450. {
  451. return replace_handler_and_delete_this<PlayingStateHandler>();
  452. }
  453. bool is_playing() const override { return false; };
  454. PlaybackState get_state() const override { return PlaybackState::Paused; }
  455. };
  456. // FIXME: This is a placeholder variable that could be scaled based on how long each frame decode takes to
  457. // avoid triggering the timer to check the queue constantly. However, doing so may reduce the speed
  458. // of seeking due to the decode thread having to wait for a signal to continue decoding.
  459. constexpr int buffering_or_seeking_decode_wait_time = 1;
  460. class PlaybackManager::BufferingStateHandler : public PlaybackManager::ResumingStateHandler {
  461. using PlaybackManager::ResumingStateHandler::ResumingStateHandler;
  462. ErrorOr<void> on_enter() override
  463. {
  464. manager().m_present_timer->start(buffering_or_seeking_decode_wait_time);
  465. return {};
  466. }
  467. StringView name() override { return "Buffering"sv; }
  468. ErrorOr<void> on_timer_callback() override
  469. {
  470. auto buffer_is_full = manager().m_buffer_is_full.load();
  471. dbgln_if(PLAYBACK_MANAGER_DEBUG, "Buffering timer callback has been called. Buffer is {}.", buffer_is_full ? "full, exiting"sv : "not full, waiting"sv);
  472. if (buffer_is_full)
  473. return assume_next_state();
  474. manager().m_present_timer->start(buffering_or_seeking_decode_wait_time);
  475. return {};
  476. }
  477. PlaybackState get_state() const override { return PlaybackState::Buffering; }
  478. };
  479. class PlaybackManager::SeekingStateHandler : public PlaybackManager::ResumingStateHandler {
  480. public:
  481. SeekingStateHandler(PlaybackManager& manager, bool playing, Time target_timestamp, SeekMode seek_mode)
  482. : ResumingStateHandler(manager, playing)
  483. , m_target_timestamp(target_timestamp)
  484. , m_seek_mode(seek_mode)
  485. {
  486. }
  487. ~SeekingStateHandler() override = default;
  488. private:
  489. ErrorOr<void> on_enter() override
  490. {
  491. auto earliest_available_sample = manager().m_last_present_in_media_time;
  492. if (manager().m_next_frame.has_value() && manager().m_next_frame->timestamp() != FrameQueueItem::no_timestamp) {
  493. earliest_available_sample = min(earliest_available_sample, manager().m_next_frame->timestamp());
  494. }
  495. {
  496. Threading::MutexLocker demuxer_locker(manager().m_demuxer_mutex);
  497. auto keyframe_timestamp = manager().seek_demuxer_to_most_recent_keyframe(m_target_timestamp, earliest_available_sample);
  498. #if PLAYBACK_MANAGER_DEBUG
  499. auto seek_mode_name = m_seek_mode == SeekMode::Accurate ? "Accurate"sv : "Fast"sv;
  500. if (keyframe_timestamp.has_value())
  501. dbgln("{} seeking to timestamp target {}ms, selected keyframe at {}ms", seek_mode_name, m_target_timestamp.to_milliseconds(), keyframe_timestamp->to_milliseconds());
  502. else
  503. dbgln("{} seeking to timestamp target {}ms, demuxer kept its iterator position after {}ms", seek_mode_name, m_target_timestamp.to_milliseconds(), earliest_available_sample.to_milliseconds());
  504. #endif
  505. if (m_seek_mode == SeekMode::Fast)
  506. m_target_timestamp = keyframe_timestamp.value_or(manager().m_last_present_in_media_time);
  507. if (keyframe_timestamp.has_value()) {
  508. dbgln_if(PLAYBACK_MANAGER_DEBUG, "Keyframe is nearer to the target than the current frames, emptying queue");
  509. while (manager().dequeue_one_frame().has_value()) { }
  510. manager().m_next_frame.clear();
  511. manager().m_last_present_in_media_time = keyframe_timestamp.value();
  512. } else if (m_target_timestamp >= manager().m_last_present_in_media_time && manager().m_next_frame.has_value() && manager().m_next_frame.value().timestamp() > m_target_timestamp) {
  513. dbgln_if(PLAYBACK_MANAGER_DEBUG, "Target timestamp is between the last presented frame and the next frame, exiting seek at {}ms", m_target_timestamp.to_milliseconds());
  514. manager().m_last_present_in_media_time = m_target_timestamp;
  515. return assume_next_state();
  516. }
  517. }
  518. return skip_samples_until_timestamp();
  519. }
  520. ErrorOr<void> skip_samples_until_timestamp()
  521. {
  522. while (true) {
  523. auto optional_item = manager().dequeue_one_frame();
  524. if (!optional_item.has_value())
  525. break;
  526. auto item = optional_item.release_value();
  527. dbgln_if(PLAYBACK_MANAGER_DEBUG, "Dequeuing frame at {}ms and comparing to seek target {}ms", item.timestamp().to_milliseconds(), m_target_timestamp.to_milliseconds());
  528. if (manager().m_next_frame.has_value() && (item.timestamp() > m_target_timestamp || item.timestamp() == FrameQueueItem::no_timestamp)) {
  529. // If the frame we're presenting is later than the target timestamp, skip the timestamp forward to it.
  530. if (manager().m_next_frame->timestamp() > m_target_timestamp) {
  531. manager().m_last_present_in_media_time = manager().m_next_frame->timestamp();
  532. } else {
  533. manager().m_last_present_in_media_time = m_target_timestamp;
  534. }
  535. if (manager().dispatch_frame_queue_item(manager().m_next_frame.release_value()))
  536. return {};
  537. manager().m_next_frame.emplace(item);
  538. dbgln_if(PLAYBACK_MANAGER_DEBUG, "Exiting seek to {} state at {}ms", m_playing ? "Playing" : "Paused", manager().m_last_present_in_media_time.to_milliseconds());
  539. return assume_next_state();
  540. }
  541. manager().m_next_frame.emplace(item);
  542. }
  543. dbgln_if(PLAYBACK_MANAGER_DEBUG, "Frame queue is empty while seeking, waiting for buffer to fill.");
  544. manager().m_present_timer->start(buffering_or_seeking_decode_wait_time);
  545. return {};
  546. }
  547. StringView name() override { return "Seeking"sv; }
  548. ErrorOr<void> seek(Time target_timestamp, SeekMode seek_mode) override
  549. {
  550. m_target_timestamp = target_timestamp;
  551. m_seek_mode = seek_mode;
  552. return on_enter();
  553. }
  554. Time current_time() const override
  555. {
  556. return m_target_timestamp;
  557. }
  558. // We won't need this override when threaded, the queue can pause us in on_enter().
  559. ErrorOr<void> on_timer_callback() override
  560. {
  561. dbgln_if(PLAYBACK_MANAGER_DEBUG, "Seeking wait finished, attempting to dequeue until timestamp.");
  562. return skip_samples_until_timestamp();
  563. }
  564. PlaybackState get_state() const override { return PlaybackState::Seeking; }
  565. Time m_target_timestamp { Time::zero() };
  566. SeekMode m_seek_mode { SeekMode::Accurate };
  567. };
  568. class PlaybackManager::StoppedStateHandler : public PlaybackManager::PlaybackStateHandler {
  569. public:
  570. StoppedStateHandler(PlaybackManager& manager)
  571. : PlaybackStateHandler(manager)
  572. {
  573. }
  574. ~StoppedStateHandler() override = default;
  575. private:
  576. ErrorOr<void> on_enter() override
  577. {
  578. return {};
  579. }
  580. StringView name() override { return "Stopped"sv; }
  581. ErrorOr<void> play() override
  582. {
  583. // When Stopped, the decoder thread will be waiting for a signal to start its loop going again.
  584. manager().m_decode_wait_condition.broadcast();
  585. return replace_handler_and_delete_this<SeekingStateHandler>(true, Time::zero(), SeekMode::Fast);
  586. }
  587. bool is_playing() const override { return false; };
  588. PlaybackState get_state() const override { return PlaybackState::Stopped; }
  589. };
  590. }