0ct0pu5/ladybird
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2

LibVideo: Add PlaybackManager to load and decode videos

Browse source 
This file will be the basis for abstracting away the out-of-thread or
later out-of-process decoding from applications displaying videos. For
now, the demuxer is hardcoded to be MatroskaParser, since that is all
we support so far. The demuxer should later be selected based on the
file header.

The playback and decoding are currently all done on one thread using
timers. The design of the code is such that adding threading should
be trivial, at least based on an earlier version of the code. For now,
though, it's better that this runs in one thread, as the multithreaded
approach causes the Video Player to lock up permanently after a few
frames are decoded.
This commit is contained in:
Zaggy1024 2022-10-29 18:16:14 -05:00 • committed by Andreas Kling
parent 0a4def1208
commit 353e1c2b4d
Notes: sideshowbarker 2024-07-17 05:06:13 +09:00
5 changed files with 430 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
AK/Debug.h.in
View file

@ -342,6 +342,10 @@
#cmakedefine01 PDF_DEBUG
#endif
#ifndef PLAYBACK_MANAGER_DEBUG
#cmakedefine01 PLAYBACK_MANAGER_DEBUG
#endif
#ifndef PNG_DEBUG
#cmakedefine01 PNG_DEBUG
#endif

1
Meta/CMake/all_the_debug_macros.cmake
View file

@ -141,6 +141,7 @@ set(PATA_DEBUG ON)
set(PATH_DEBUG ON)
set(PCI_DEBUG ON)
set(PDF_DEBUG ON)
set(PLAYBACK_MANAGER_DEBUG ON)
set(PNG_DEBUG ON)
set(POLL_SELECT_DEBUG ON)
set(PORTABLE_IMAGE_LOADER_DEBUG ON)

1
Userland/Libraries/LibVideo/CMakeLists.txt
View file

@ -4,6 +4,7 @@ set(SOURCES
Color/TransferCharacteristics.cpp
MatroskaDemuxer.cpp
MatroskaReader.cpp
PlaybackManager.cpp
VideoFrame.cpp
VP9/BitStream.cpp
VP9/Decoder.cpp

244
Userland/Libraries/LibVideo/PlaybackManager.cpp Normal file
View file

@ -0,0 +1,244 @@
/*
* Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <AK/Format.h>
#include <LibCore/Timer.h>
#include <LibVideo/MatroskaReader.h>
#include <LibVideo/VP9/Decoder.h>
#include "MatroskaDemuxer.h"
#include "PlaybackManager.h"
namespace Video {
// We post DecoderErrors to the event queue to be handled, since some will occur off the main thread.
#define TRY_OR_POST_ERROR_AND_RETURN(expression, return_value) \
({ \
auto _temporary_result = ((expression)); \
if (_temporary_result.is_error()) { \
dbgln("Playback error encountered: {}", _temporary_result.error().string_literal()); \
m_main_loop.post_event(*this, make<DecoderErrorEvent>(_temporary_result.release_error())); \
return return_value; \
} \
_temporary_result.release_value(); \
})
#define TRY_OR_POST_ERROR(expression) TRY_OR_POST_ERROR_AND_RETURN(expression, )
DecoderErrorOr<NonnullRefPtr<PlaybackManager>> PlaybackManager::from_file(Object* event_handler, StringView filename)
{
NonnullOwnPtr<Demuxer> demuxer = TRY(MatroskaDemuxer::from_file(filename));
auto video_tracks = demuxer->get_tracks_for_type(TrackType::Video);
if (video_tracks.is_empty())
return DecoderError::with_description(DecoderErrorCategory::Invalid, "No video track is present"sv);
auto track = video_tracks[0];
dbgln_if(PLAYBACK_MANAGER_DEBUG, "Selecting video track number {}", track.identifier());
NonnullOwnPtr<VideoDecoder> decoder = make<VP9::Decoder>();
return PlaybackManager::construct(event_handler, demuxer, track, decoder);
}
PlaybackManager::PlaybackManager(Object* event_handler, NonnullOwnPtr<Demuxer>& demuxer, Track video_track, NonnullOwnPtr<VideoDecoder>& decoder)
: Object(event_handler)
, m_main_loop(Core::EventLoop::current())
, m_demuxer(move(demuxer))
, m_selected_video_track(video_track)
, m_decoder(move(decoder))
, m_frame_queue(make<VideoFrameQueue>())
, m_present_timer(Core::Timer::construct())
, m_decode_timer(Core::Timer::construct())
{
m_present_timer->set_single_shot(true);
m_present_timer->set_interval(0);
m_present_timer->on_timeout = [&] { update_presented_frame(); };
m_decode_timer->set_single_shot(true);
m_decode_timer->set_interval(0);
m_decode_timer->on_timeout = [&] { on_decode_timer(); };
}
void PlaybackManager::set_playback_status(PlaybackStatus status)
{
if (status != m_status) {
auto old_status = m_status;
m_status = status;
dbgln_if(PLAYBACK_MANAGER_DEBUG, "Set playback status from {} to {}", playback_status_to_string(old_status), playback_status_to_string(m_status));
if (status == PlaybackStatus::Playing) {
if (old_status == PlaybackStatus::Stopped) {
restart_playback();
m_frame_queue->clear();
m_skipped_frames = 0;
}
m_last_present_in_real_time = Time::now_monotonic();
m_present_timer->start();
} else {
m_last_present_in_media_time = current_playback_time();
m_last_present_in_real_time = Time::zero();
m_present_timer->stop();
}
m_main_loop.post_event(*this, make<PlaybackStatusChangeEvent>(status, old_status));
}
}
void PlaybackManager::event(Core::Event& event)
{
if (event.type() == DecoderErrorOccurred) {
auto& error_event = static_cast<DecoderErrorEvent&>(event);
VERIFY(error_event.error().category() != DecoderErrorCategory::EndOfStream);
}
// Allow events to bubble up in all cases.
event.ignore();
}
void PlaybackManager::resume_playback()
{
set_playback_status(PlaybackStatus::Playing);
}
void PlaybackManager::pause_playback()
{
set_playback_status(PlaybackStatus::Paused);
}
bool PlaybackManager::prepare_next_frame()
{
if (m_next_frame.has_value())
return true;
if (m_frame_queue->is_empty())
return false;
auto frame_item = m_frame_queue->dequeue();
m_next_frame.emplace(frame_item);
m_decode_timer->start();
return true;
}
Time PlaybackManager::current_playback_time()
{
if (is_playing())
return m_last_present_in_media_time + (Time::now_monotonic() - m_last_present_in_real_time);
return m_last_present_in_media_time;
}
Time PlaybackManager::duration()
{
return m_demuxer->duration();
}
void PlaybackManager::update_presented_frame()
{
bool out_of_queued_frames = false;
Optional<FrameQueueItem> frame_item_to_display;
while (true) {
out_of_queued_frames = out_of_queued_frames || !prepare_next_frame();
if (out_of_queued_frames)
break;
VERIFY(m_next_frame.has_value());
if (m_next_frame->timestamp > current_playback_time() || m_next_frame->is_eos_marker())
break;
if (frame_item_to_display.has_value()) {
dbgln_if(PLAYBACK_MANAGER_DEBUG, "At {}ms: Dropped frame with timestamp {}ms for the next at {}ms", current_playback_time().to_milliseconds(), frame_item_to_display->timestamp.to_milliseconds(), m_next_frame->timestamp.to_milliseconds());
m_skipped_frames++;
}
frame_item_to_display = m_next_frame.release_value();
}
if (!out_of_queued_frames && frame_item_to_display.has_value()) {
m_main_loop.post_event(*this, make<VideoFramePresentEvent>(frame_item_to_display->bitmap));
m_last_present_in_media_time = current_playback_time();
m_last_present_in_real_time = Time::now_monotonic();
frame_item_to_display.clear();
}
if (frame_item_to_display.has_value()) {
VERIFY(!m_next_frame.has_value());
m_next_frame = frame_item_to_display;
dbgln_if(PLAYBACK_MANAGER_DEBUG, "Set next frame back to dequeued item at timestamp {}ms", m_next_frame->timestamp.to_milliseconds());
}
if (!is_playing())
return;
if (!out_of_queued_frames) {
if (m_next_frame->is_eos_marker()) {
set_playback_status(PlaybackStatus::Stopped);
m_next_frame.clear();
return;
}
auto frame_time_ms = (m_next_frame.value().timestamp - current_playback_time()).to_milliseconds();
VERIFY(frame_time_ms <= NumericLimits<int>::max());
dbgln_if(PLAYBACK_MANAGER_DEBUG, "Time until next frame is {}ms", frame_time_ms);
m_present_timer->start(max(static_cast<int>(frame_time_ms), 0));
return;
}
set_playback_status(PlaybackStatus::Buffering);
m_decode_timer->start();
}
void PlaybackManager::restart_playback()
{
m_last_present_in_media_time = Time::zero();
m_last_present_in_real_time = Time::zero();
TRY_OR_POST_ERROR(m_demuxer->seek_to_most_recent_keyframe(m_selected_video_track, 0));
}
bool PlaybackManager::decode_and_queue_one_sample()
{
if (m_frame_queue->size() >= FRAME_BUFFER_COUNT)
return false;
#if PLAYBACK_MANAGER_DEBUG
auto start_time = Time::now_monotonic();
#endif
auto frame_sample_result = m_demuxer->get_next_video_sample_for_track(m_selected_video_track);
if (frame_sample_result.is_error()) {
if (frame_sample_result.error().category() == DecoderErrorCategory::EndOfStream) {
m_frame_queue->enqueue(FrameQueueItem::eos_marker());
return false;
}
m_main_loop.post_event(*this, make<DecoderErrorEvent>(frame_sample_result.release_error()));
return false;
}
auto frame_sample = frame_sample_result.release_value();
TRY_OR_POST_ERROR_AND_RETURN(m_decoder->receive_sample(frame_sample->data()), false);
auto decoded_frame = TRY_OR_POST_ERROR_AND_RETURN(m_decoder->get_decoded_frame(), false);
auto& cicp = decoded_frame->cicp();
cicp.adopt_specified_values(frame_sample->container_cicp());
cicp.default_code_points_if_unspecified({ Video::ColorPrimaries::BT709, Video::TransferCharacteristics::BT709, Video::MatrixCoefficients::BT709, Video::ColorRange::Studio });
auto bitmap = TRY_OR_POST_ERROR_AND_RETURN(decoded_frame->to_bitmap(), false);
m_frame_queue->enqueue(FrameQueueItem { bitmap, frame_sample->timestamp() });
#if PLAYBACK_MANAGER_DEBUG
auto end_time = Time::now_monotonic();
dbgln("Decoding took {}ms", (end_time - start_time).to_milliseconds());
#endif
return true;
}
void PlaybackManager::on_decode_timer()
{
if (!decode_and_queue_one_sample() && is_buffering()) {
set_playback_status(PlaybackStatus::Playing);
return;
}
// Continually decode until buffering is complete
if (is_buffering())
m_decode_timer->start();
}
}

180
Userland/Libraries/LibVideo/PlaybackManager.h Normal file
View file

@ -0,0 +1,180 @@
/*
* Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <AK/Atomic.h>
#include <AK/Function.h>
#include <AK/NonnullOwnPtr.h>
#include <AK/Queue.h>
#include <AK/Time.h>
#include <LibCore/EventLoop.h>
#include <LibCore/SharedCircularQueue.h>
#include <LibGfx/Bitmap.h>
#include <LibThreading/ConditionVariable.h>
#include <LibThreading/Mutex.h>
#include <LibThreading/Thread.h>
#include "Demuxer.h"
#include "MatroskaDocument.h"
#include "VideoDecoder.h"
namespace Video {
enum class PlaybackStatus {
Playing,
Paused,
Buffering,
Seeking,
Stopped,
};
struct FrameQueueItem {
static FrameQueueItem eos_marker()
{
return { nullptr, Time::max() };
}
RefPtr<Gfx::Bitmap> bitmap;
Time timestamp;
bool is_eos_marker() const { return !bitmap; }
};
static constexpr size_t FRAME_BUFFER_COUNT = 4;
using VideoFrameQueue = Queue<FrameQueueItem, FRAME_BUFFER_COUNT>;
class PlaybackManager : public Core::Object {
C_OBJECT(PlaybackManager)
public:
static DecoderErrorOr<NonnullRefPtr<PlaybackManager>> from_file(Object* event_handler, StringView file);
static DecoderErrorOr<NonnullRefPtr<PlaybackManager>> from_data(Object* event_handler, Span<u8> data);
PlaybackManager(Object* event_handler, NonnullOwnPtr<Demuxer>& demuxer, Track video_track, NonnullOwnPtr<VideoDecoder>& decoder);
~PlaybackManager() override = default;
void resume_playback();
void pause_playback();
void restart_playback();
bool is_playing() const { return m_status == PlaybackStatus::Playing; }
bool is_buffering() const { return m_status == PlaybackStatus::Buffering; }
bool is_stopped() const { return m_status == PlaybackStatus::Stopped; }
u64 number_of_skipped_frames() const { return m_skipped_frames; }
void event(Core::Event& event) override;
Time current_playback_time();
Time duration();
Function<void(NonnullRefPtr<Gfx::Bitmap>, Time)> on_frame_present;
private:
void set_playback_status(PlaybackStatus status);
bool prepare_next_frame();
void update_presented_frame();
// Runs off the main thread
bool decode_and_queue_one_sample();
void on_decode_timer();
Core::EventLoop& m_main_loop;
PlaybackStatus m_status { PlaybackStatus::Stopped };
Time m_last_present_in_media_time = Time::zero();
Time m_last_present_in_real_time = Time::zero();
NonnullOwnPtr<Demuxer> m_demuxer;
Track m_selected_video_track;
NonnullOwnPtr<VideoDecoder> m_decoder;
NonnullOwnPtr<VideoFrameQueue> m_frame_queue;
Optional<FrameQueueItem> m_next_frame;
NonnullRefPtr<Core::Timer> m_present_timer;
unsigned m_decoding_buffer_time_ms = 16;
NonnullRefPtr<Core::Timer> m_decode_timer;
u64 m_skipped_frames;
};
enum EventType : unsigned {
DecoderErrorOccurred = (('v' << 2) | ('i' << 1) | 'd') << 4,
VideoFramePresent,
PlaybackStatusChange,
};
class DecoderErrorEvent : public Core::Event {
public:
explicit DecoderErrorEvent(DecoderError error)
: Core::Event(DecoderErrorOccurred)
, m_error(move(error))
{
}
virtual ~DecoderErrorEvent() = default;
DecoderError error() { return m_error; }
private:
DecoderError m_error;
};
class VideoFramePresentEvent : public Core::Event {
public:
VideoFramePresentEvent() = default;
explicit VideoFramePresentEvent(RefPtr<Gfx::Bitmap> frame)
: Core::Event(VideoFramePresent)
, m_frame(move(frame))
{
}
virtual ~VideoFramePresentEvent() = default;
RefPtr<Gfx::Bitmap> frame() { return m_frame; }
private:
RefPtr<Gfx::Bitmap> m_frame;
};
class PlaybackStatusChangeEvent : public Core::Event {
public:
PlaybackStatusChangeEvent() = default;
explicit PlaybackStatusChangeEvent(PlaybackStatus status, PlaybackStatus previous_status)
: Core::Event(PlaybackStatusChange)
, m_status(status)
, m_previous_status(previous_status)
{
}
virtual ~PlaybackStatusChangeEvent() = default;
PlaybackStatus status();
PlaybackStatus previous_status();
private:
PlaybackStatus m_status;
PlaybackStatus m_previous_status;
};
inline StringView playback_status_to_string(PlaybackStatus status)
{
switch (status) {
case PlaybackStatus::Playing:
return "Playing"sv;
case PlaybackStatus::Paused:
return "Paused"sv;
case PlaybackStatus::Buffering:
return "Buffering"sv;
case PlaybackStatus::Seeking:
return "Seeking"sv;
case PlaybackStatus::Stopped:
return "Stopped"sv;
}
return "Unknown"sv;
};
}