diff --git a/CMakeLists.txt b/CMakeLists.txt index 1a66068..f23e2ad 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -49,14 +49,31 @@ project(gdvapi LANGUAGES CXX ) +find_package(Vulkan REQUIRED) + +find_package(PkgConfig REQUIRED) + +pkg_check_modules(AVFORMAT REQUIRED IMPORTED_TARGET libavformat) +pkg_check_modules(AVCODEC REQUIRED IMPORTED_TARGET libavcodec) +pkg_check_modules(SWSCALE REQUIRED IMPORTED_TARGET libswscale) +pkg_check_modules(AVUTIL REQUIRED IMPORTED_TARGET libavutil) + add_library(${LIBNAME} SHARED) +target_link_libraries(${LIBNAME} PRIVATE + Vulkan::Headers + PkgConfig::AVFORMAT + PkgConfig::AVCODEC + PkgConfig::SWSCALE + PkgConfig::AVUTIL +) + target_sources(${LIBNAME} PRIVATE src/register_types.cpp src/register_types.h - src/udpvideostream_class.cpp - src/udpvideostream_class.h + src/filevideostream_class.cpp + src/filevideostream_class.h ) # Fetch a list of the xml files to use for documentation and add to our target diff --git a/flake.nix b/flake.nix index c613f29..5444da1 100644 --- a/flake.nix +++ b/flake.nix @@ -13,6 +13,13 @@ clang python3 godot + pkg-config + ]; + + buildInputs = with pkgs; [ + vulkan-loader + vulkan-headers + ffmpeg ]; shellHook = '' diff --git a/project/badapple.mov b/project/badapple.mov new file mode 100644 index 0000000..73bab02 Binary files /dev/null and b/project/badapple.mov differ diff --git a/project/example.gd b/project/example.gd index 5157869..d3c0927 100644 --- a/project/example.gd +++ b/project/example.gd @@ -2,4 +2,4 @@ extends Node func _ready() -> void: - $UDPVideoStream.begin("localhost:1000") + $FileVideoStream.begin("res://badapple.mov") diff --git a/project/example.tscn b/project/example.tscn index 3fd2d94..52249f1 100644 --- a/project/example.tscn +++ b/project/example.tscn @@ -5,4 +5,4 @@ [node name="Node" type="Node" unique_id=2080300232] script = ExtResource("1_jdh55") -[node name="UDPVideoStream" type="UDPVideoStream" parent="." unique_id=270982386] +[node name="FileVideoStream" type="FileVideoStream" parent="." unique_id=330752336] diff --git a/project/test.mp4 b/project/test.mp4 new file mode 100644 index 0000000..b11552f Binary files /dev/null and b/project/test.mp4 differ diff --git a/src/filevideostream_class.cpp b/src/filevideostream_class.cpp new file mode 100644 index 0000000..a09a6cd --- /dev/null +++ b/src/filevideostream_class.cpp @@ -0,0 +1,275 @@ +#include "filevideostream_class.h" +#include "godot_cpp/classes/project_settings.hpp" +#include "godot_cpp/classes/rd_texture_format.hpp" +#include "godot_cpp/classes/rd_texture_view.hpp" +#include "godot_cpp/classes/rendering_device.hpp" +#include "godot_cpp/classes/rendering_server.hpp" +#include "godot_cpp/variant/utility_functions.hpp" + +extern "C" { +#include +#include +#include +#include +#include +} + +using namespace godot; + +enum HWMode { HW_NONE, + HW_VULKAN, + HW_VAAPI }; +static HWMode active_hw_mode = HW_NONE; + +void FileVideoStream::_bind_methods() { + godot::ClassDB::bind_method(D_METHOD("begin", "source"), &FileVideoStream::begin); +} + +static enum AVPixelFormat get_hw_format(AVCodecContext *ctx, const enum AVPixelFormat *formats) { + for (const enum AVPixelFormat *p = formats; *p != -1; p++) { + if (active_hw_mode == HW_VULKAN && *p == AV_PIX_FMT_VULKAN) { + return AV_PIX_FMT_VULKAN; + } + if (active_hw_mode == HW_VAAPI && *p == AV_PIX_FMT_VAAPI) { + return AV_PIX_FMT_VAAPI; + } + } + UtilityFunctions::printerr("[gdvapi] hardware decoding unavailable"); + active_hw_mode = HW_NONE; + return AV_PIX_FMT_NONE; +} + +void FileVideoStream::begin(const Variant &source) { + String res_path = source; + String global_path = ProjectSettings::get_singleton()->globalize_path(res_path); + CharString path_utf8 = global_path.utf8(); + + fmt_ctx = nullptr; + if (avformat_open_input(&fmt_ctx, path_utf8.get_data(), nullptr, nullptr) < 0) { + UtilityFunctions::printerr("[gdvapi] failed to open input stream"); + return; + } + + if (avformat_find_stream_info(fmt_ctx, nullptr) < 0) { + return; + } + + video_stream_idx = -1; + for (unsigned int i = 0; i < fmt_ctx->nb_streams; i++) { + if (fmt_ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { + video_stream_idx = i; + break; + } + } + if (video_stream_idx == -1) return; + + AVStream *stream = fmt_ctx->streams[video_stream_idx]; + if (stream->avg_frame_rate.den > 0 && stream->avg_frame_rate.num > 0) { + double fps = av_q2d(stream->avg_frame_rate); + frame_delay = 1.0 / fps; + UtilityFunctions::print("[gdvapi] fps: ", fps, " frame delay: ", frame_delay, "s"); + } else { + frame_delay = 1.0 / 60.0; // fallback + } + + AVCodecParameters *codec_par = fmt_ctx->streams[video_stream_idx]->codecpar; + codec = avcodec_find_decoder(codec_par->codec_id); + if (!codec) + return; + + codec_ctx = avcodec_alloc_context3(codec); + if (!codec_ctx) + return; + + if (avcodec_parameters_to_context(codec_ctx, codec_par) < 0) + return; + + active_hw_mode = HW_VULKAN; + if (av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VULKAN, nullptr, nullptr, 0) < 0) { + UtilityFunctions::print("[gdvapi] vulkan video extensions missing on driver, trying va-api"); + + active_hw_mode = HW_VAAPI; + if (av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VAAPI, nullptr, nullptr, 0) < 0) { + UtilityFunctions::print("[gdvapi] va-api acceleration unavailable. using software"); + active_hw_mode = HW_NONE; + } + } + + if (active_hw_mode != HW_NONE) { + codec_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx); + codec_ctx->get_format = get_hw_format; + } + + if (avcodec_open2(codec_ctx, codec, nullptr) < 0) + return; + + frame = av_frame_alloc(); + sw_frame = av_frame_alloc(); + packet = av_packet_alloc(); + + width = codec_ctx->width; + height = codec_ctx->height; + texture_init(); + + sws_ctx = sws_getContext( + width, height, codec_ctx->pix_fmt, + width, height, AV_PIX_FMT_RGBA, + SWS_BILINEAR, nullptr, nullptr, nullptr); + + time_accumulator = 0.0; + set_process(true); +} + +void FileVideoStream::_process(double delta) { + if (!fmt_ctx || !codec_ctx) return; + + time_accumulator += delta; + + if (time_accumulator < frame_delay) { + return; + } + + time_accumulator -= frame_delay; + + bool frame_finished = false; + while (!frame_finished && av_read_frame(fmt_ctx, packet) >= 0) { + if (packet->stream_index == video_stream_idx) { + if (avcodec_send_packet(codec_ctx, packet) >= 0) { + while (avcodec_receive_frame(codec_ctx, frame) >= 0) { + frame_finished = true; + + RenderingDevice *rd = RenderingServer::get_singleton()->get_rendering_device(); + if (!rd || !texture_rd_rid.is_valid()) + continue; + + if (frame->format == AV_PIX_FMT_VULKAN || frame->format == AV_PIX_FMT_VAAPI) { + if (av_hwframe_transfer_data(sw_frame, frame, 0) >= 0) { + PackedByteArray gpu_bytes; + gpu_bytes.resize(width * height * 4); + + uint8_t *dest_pointers[4] = { gpu_bytes.ptrw(), nullptr, nullptr, nullptr }; + int dest_linesizes[4] = { width * 4, 0, 0, 0 }; + + enum AVPixelFormat src_fmt = (enum AVPixelFormat)sw_frame->format; + struct SwsContext *hw_sws = sws_getContext( + width, height, src_fmt, + width, height, AV_PIX_FMT_RGBA, + SWS_BILINEAR, nullptr, nullptr, nullptr); + + if (hw_sws) { + sws_scale(hw_sws, sw_frame->data, sw_frame->linesize, 0, height, dest_pointers, dest_linesizes); + sws_freeContext(hw_sws); + rd->texture_update(texture_rd_rid, 0, gpu_bytes); + } + } + } else if (frame->data[0] != nullptr) { + PackedByteArray rgba_data; + rgba_data.resize(width * height * 4); + uint8_t *dest_pointers[4] = { rgba_data.ptrw(), nullptr, nullptr, nullptr }; + int dest_linesizes[4] = { width * 4, 0, 0, 0 }; + + if (sws_ctx) { + sws_scale(sws_ctx, frame->data, frame->linesize, 0, height, dest_pointers, dest_linesizes); + rd->texture_update(texture_rd_rid, 0, rgba_data); + } + } + + queue_redraw(); + } + } + } + av_packet_unref(packet); + } +} + +FileVideoStream::~FileVideoStream() { + cleanup(); +} + +void FileVideoStream::_notification(int p_what) { + switch (p_what) { + case NOTIFICATION_EXIT_TREE: { + cleanup(); + } break; + } +} + +void FileVideoStream::texture_init() { + RenderingDevice *rd = RenderingServer::get_singleton()->get_rendering_device(); + if (!rd) + return; + + if (texture_rd_rid.is_valid()) { + rd->free_rid(texture_rd_rid); + } + + Ref format; + format.instantiate(); + format->set_texture_type(RenderingDevice::TEXTURE_TYPE_2D); + format->set_format(RenderingDevice::DATA_FORMAT_R8G8B8A8_UNORM); + format->set_width(width); + format->set_height(height); + format->set_depth(1); + format->set_array_layers(1); + format->set_mipmaps(1); + format->set_usage_bits(RenderingDevice::TEXTURE_USAGE_SAMPLING_BIT | + RenderingDevice::TEXTURE_USAGE_CAN_UPDATE_BIT); + + PackedByteArray byte_array; + byte_array.resize(width * height * 4); + + TypedArray data; + data.push_back(byte_array); + + Ref view; + view.instantiate(); + + texture_rd_rid = rd->texture_create(format, view, data); + + if (texture_rd_rid.is_valid()) { + godot_texture.instantiate(); + godot_texture->set_texture_rd_rid(texture_rd_rid); + } +} + +void FileVideoStream::cleanup() { + set_process(false); + RenderingDevice *rd = RenderingServer::get_singleton()->get_rendering_device(); + + godot_texture.unref(); + + if (rd && texture_rd_rid.is_valid()) { + rd->free_rid(texture_rd_rid); + texture_rd_rid = RID(); + } + + if (sws_ctx) { + sws_freeContext(sws_ctx); + sws_ctx = nullptr; + } + if (frame) { + av_frame_free(&frame); + } + if (sw_frame) { + av_frame_free(&sw_frame); + } + if (packet) { + av_packet_free(&packet); + } + if (codec_ctx) { + avcodec_free_context(&codec_ctx); + } + if (hw_device_ctx) { + av_buffer_unref(&hw_device_ctx); + hw_device_ctx = nullptr; + } + if (fmt_ctx) { + avformat_close_input(&fmt_ctx); + } +} + +void FileVideoStream::_draw() { + if (godot_texture.is_valid()) { + draw_texture(godot_texture, Point2(0, 0)); + } +} diff --git a/src/filevideostream_class.h b/src/filevideostream_class.h new file mode 100644 index 0000000..d17a78a --- /dev/null +++ b/src/filevideostream_class.h @@ -0,0 +1,56 @@ +#pragma once + +#include "godot_cpp/classes/control.hpp" +#include "godot_cpp/classes/texture2drd.hpp" +#include "godot_cpp/variant/rid.hpp" + +extern "C" { +#include +#include +#include +#include +} + +namespace godot { + +class FileVideoStream : public Control { + GDCLASS(FileVideoStream, Control) + +private: + AVFormatContext *fmt_ctx = nullptr; + AVCodecContext *codec_ctx = nullptr; + const AVCodec *codec = nullptr; + AVFrame *frame = nullptr; + AVPacket *packet = nullptr; + struct SwsContext *sws_ctx = nullptr; + int video_stream_idx = -1; + int width = 256; + int height = 256; + AVBufferRef *hw_device_ctx = nullptr; + AVFrame *sw_frame = nullptr; + + double frame_delay = 0.0; + double time_accumulator = 0.0; + + RID texture_rd_rid; + Ref godot_texture; + + void cleanup(); + void texture_init(); + +protected: + static void _bind_methods(); + +public: + FileVideoStream() = default; + ~FileVideoStream() override; + + void begin(const Variant &source); + + void _process(double delta) override; + + void _notification(int p_what); + void _draw() override; +}; + +} diff --git a/src/register_types.cpp b/src/register_types.cpp index 74f574a..bc51a55 100644 --- a/src/register_types.cpp +++ b/src/register_types.cpp @@ -5,16 +5,15 @@ #include #include -#include "udpvideostream_class.h" +#include "filevideostream_class.h" using namespace godot; -void initialize_gdextension_types(ModuleInitializationLevel p_level) -{ +void initialize_gdextension_types(ModuleInitializationLevel p_level) { if (p_level != MODULE_INITIALIZATION_LEVEL_SCENE) { return; } - GDREGISTER_CLASS(UDPVideoStream); + GDREGISTER_CLASS(FileVideoStream); } void uninitialize_gdextension_types(ModuleInitializationLevel p_level) { @@ -23,16 +22,14 @@ void uninitialize_gdextension_types(ModuleInitializationLevel p_level) { } } -extern "C" -{ - // Initialization - GDExtensionBool GDE_EXPORT gdvapi_library_init(GDExtensionInterfaceGetProcAddress p_get_proc_address, GDExtensionClassLibraryPtr p_library, GDExtensionInitialization *r_initialization) - { - GDExtensionBinding::InitObject init_obj(p_get_proc_address, p_library, r_initialization); - init_obj.register_initializer(initialize_gdextension_types); - init_obj.register_terminator(uninitialize_gdextension_types); - init_obj.set_minimum_library_initialization_level(MODULE_INITIALIZATION_LEVEL_SCENE); +extern "C" { +// Initialization +GDExtensionBool GDE_EXPORT gdvapi_library_init(GDExtensionInterfaceGetProcAddress p_get_proc_address, GDExtensionClassLibraryPtr p_library, GDExtensionInitialization *r_initialization) { + GDExtensionBinding::InitObject init_obj(p_get_proc_address, p_library, r_initialization); + init_obj.register_initializer(initialize_gdextension_types); + init_obj.register_terminator(uninitialize_gdextension_types); + init_obj.set_minimum_library_initialization_level(MODULE_INITIALIZATION_LEVEL_SCENE); - return init_obj.init(); - } + return init_obj.init(); +} } diff --git a/src/udpvideostream_class.cpp b/src/udpvideostream_class.cpp deleted file mode 100644 index ec3e43e..0000000 --- a/src/udpvideostream_class.cpp +++ /dev/null @@ -1,97 +0,0 @@ -#include "udpvideostream_class.h" -#include "godot_cpp/classes/rendering_server.hpp" -#include "godot_cpp/classes/rendering_device.hpp" -#include "godot_cpp/classes/rd_texture_format.hpp" -#include "godot_cpp/classes/rd_texture_view.hpp" - - -using namespace godot; - -void UDPVideoStream::_bind_methods() { - godot::ClassDB::bind_method(D_METHOD("begin", "variant"), &UDPVideoStream::begin); -} - -void UDPVideoStream::begin(const Variant &source) const { - print_line(vformat("[gdvapi] begin stream at %s", source.stringify())); -} - -UDPVideoStream::~UDPVideoStream() { - cleanup(); -} - -void UDPVideoStream::_notification(int p_what) { - switch (p_what) { - case NOTIFICATION_POST_ENTER_TREE: { - texture_init(); - } break; - - case NOTIFICATION_EXIT_TREE: { - cleanup(); - } break; - } -} - - -void UDPVideoStream::texture_init() { - RenderingDevice *rd = RenderingServer::get_singleton()->get_rendering_device(); - if (!rd) { - UtilityFunctions::printerr("Vulkan RenderingDevice is not available!"); - return; - } - - int width = 256; - int height = 256; - - Ref format; - format.instantiate(); - format->set_texture_type(RenderingDevice::TEXTURE_TYPE_2D); - format->set_format(RenderingDevice::DATA_FORMAT_R8G8B8A8_UNORM); - format->set_width(width); - format->set_height(height); - format->set_depth(1); - format->set_array_layers(1); - format->set_mipmaps(1); - format->set_usage_bits(RenderingDevice::TEXTURE_USAGE_SAMPLING_BIT | - RenderingDevice::TEXTURE_USAGE_CAN_UPDATE_BIT); - - PackedByteArray byte_array; - byte_array.resize(width * height * 4); - for (int i = 0; i < byte_array.size(); i += 4) { - byte_array[i] = 0; - byte_array[i + 1] = 0; - byte_array[i + 2] = 255; - byte_array[i + 3] = 255; - } - - TypedArray data; - data.push_back(byte_array); - - Ref view; - view.instantiate(); - - texture_rd_rid = rd->texture_create(format, view, data); - - if (texture_rd_rid.is_valid()) { - godot_texture.instantiate(); - godot_texture->set_texture_rd_rid(texture_rd_rid); - - queue_redraw(); - } -} - -void UDPVideoStream::cleanup() { - RenderingDevice *rd = RenderingServer::get_singleton()->get_rendering_device(); - - godot_texture.unref(); - - if (rd && texture_rd_rid.is_valid()) { - rd->free_rid(texture_rd_rid); - texture_rd_rid = RID(); - } -} - -void UDPVideoStream::_draw() { - if (godot_texture.is_valid() && godot_texture->get_texture_rd_rid().is_valid()) { - draw_texture(godot_texture, Point2(0, 0)); - } -} diff --git a/src/udpvideostream_class.h b/src/udpvideostream_class.h deleted file mode 100644 index 0e605be..0000000 --- a/src/udpvideostream_class.h +++ /dev/null @@ -1,32 +0,0 @@ -#pragma once - -#include "godot_cpp/classes/control.hpp" -#include "godot_cpp/classes/wrapped.hpp" -#include "godot_cpp/variant/variant.hpp" -#include "godot_cpp/classes/rendering_device.hpp" -#include "godot_cpp/classes/texture2drd.hpp" - -using namespace godot; - -class UDPVideoStream : public Control { - GDCLASS(UDPVideoStream, Control) - -private: - RID texture_rd_rid; - Ref godot_texture; - - void cleanup(); - void texture_init(); - -protected: - static void _bind_methods(); - -public: - UDPVideoStream() = default; - ~UDPVideoStream() override; - - void begin(const Variant &source) const; - - void _notification(int p_what); - virtual void _draw() override; -};