file decoding works

CMakeLists.txt

@@ -49,14 +49,31 @@ project(gdvapi
     LANGUAGES CXX
 )
 
+find_package(Vulkan REQUIRED)
+
+find_package(PkgConfig REQUIRED)
+
+pkg_check_modules(AVFORMAT REQUIRED IMPORTED_TARGET libavformat)
+pkg_check_modules(AVCODEC REQUIRED IMPORTED_TARGET libavcodec)
+pkg_check_modules(SWSCALE REQUIRED IMPORTED_TARGET libswscale)
+pkg_check_modules(AVUTIL REQUIRED IMPORTED_TARGET libavutil)
+
 add_library(${LIBNAME} SHARED)
 
+target_link_libraries(${LIBNAME} PRIVATE 
+    Vulkan::Headers 
+    PkgConfig::AVFORMAT
+    PkgConfig::AVCODEC 
+    PkgConfig::SWSCALE 
+    PkgConfig::AVUTIL
+)
+
 target_sources(${LIBNAME}
     PRIVATE
     src/register_types.cpp
     src/register_types.h
-    src/udpvideostream_class.cpp
-    src/udpvideostream_class.h
+    src/filevideostream_class.cpp
+    src/filevideostream_class.h
 )
 
 # Fetch a list of the xml files to use for documentation and add to our target

flake.nix

@@ -13,6 +13,13 @@
 		  clang
 		  python3
 		  godot
+		  pkg-config
+      	];
+
+      	buildInputs = with pkgs; [
+      	  vulkan-loader
+      	  vulkan-headers
+      	  ffmpeg
       	];
 
       	shellHook = ''

project/example.gd

@@ -2,4 +2,4 @@ extends Node
 
 
 func _ready() -> void:
-	$UDPVideoStream.begin("localhost:1000")
+	$FileVideoStream.begin("res://badapple.mov")

project/example.tscn

@@ -5,4 +5,4 @@
 [node name="Node" type="Node" unique_id=2080300232]
 script = ExtResource("1_jdh55")
 
-[node name="UDPVideoStream" type="UDPVideoStream" parent="." unique_id=270982386]
+[node name="FileVideoStream" type="FileVideoStream" parent="." unique_id=330752336]

src/filevideostream_class.cpp

@@ -0,0 +1,275 @@
+#include "filevideostream_class.h"
+#include "godot_cpp/classes/project_settings.hpp"
+#include "godot_cpp/classes/rd_texture_format.hpp"
+#include "godot_cpp/classes/rd_texture_view.hpp"
+#include "godot_cpp/classes/rendering_device.hpp"
+#include "godot_cpp/classes/rendering_server.hpp"
+#include "godot_cpp/variant/utility_functions.hpp"
+
+extern "C" {
+#include <libavcodec/avcodec.h>
+#include <libavformat/avformat.h>
+#include <libavutil/hwcontext.h>
+#include <libavutil/imgutils.h>
+#include <libswscale/swscale.h>
+}
+
+using namespace godot;
+
+enum HWMode { HW_NONE,
+	HW_VULKAN,
+	HW_VAAPI };
+static HWMode active_hw_mode = HW_NONE;
+
+void FileVideoStream::_bind_methods() {
+	godot::ClassDB::bind_method(D_METHOD("begin", "source"), &FileVideoStream::begin);
+}
+
+static enum AVPixelFormat get_hw_format(AVCodecContext *ctx, const enum AVPixelFormat *formats) {
+	for (const enum AVPixelFormat *p = formats; *p != -1; p++) {
+		if (active_hw_mode == HW_VULKAN && *p == AV_PIX_FMT_VULKAN) {
+			return AV_PIX_FMT_VULKAN;
+		}
+		if (active_hw_mode == HW_VAAPI && *p == AV_PIX_FMT_VAAPI) {
+			return AV_PIX_FMT_VAAPI;
+		}
+	}
+	UtilityFunctions::printerr("[gdvapi] hardware decoding unavailable");
+	active_hw_mode = HW_NONE;
+	return AV_PIX_FMT_NONE;
+}
+
+void FileVideoStream::begin(const Variant &source) {
+	String res_path = source;
+	String global_path = ProjectSettings::get_singleton()->globalize_path(res_path);
+	CharString path_utf8 = global_path.utf8();
+
+	fmt_ctx = nullptr;
+	if (avformat_open_input(&fmt_ctx, path_utf8.get_data(), nullptr, nullptr) < 0) {
+		UtilityFunctions::printerr("[gdvapi] failed to open input stream");
+		return;
+	}
+
+	if (avformat_find_stream_info(fmt_ctx, nullptr) < 0) {
+		return;
+	}
+
+	video_stream_idx = -1;
+	for (unsigned int i = 0; i < fmt_ctx->nb_streams; i++) {
+		if (fmt_ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
+			video_stream_idx = i;
+			break;
+		}
+	}
+	if (video_stream_idx == -1) return;
+	
+    AVStream *stream = fmt_ctx->streams[video_stream_idx];
+    if (stream->avg_frame_rate.den > 0 && stream->avg_frame_rate.num > 0) {
+        double fps = av_q2d(stream->avg_frame_rate);
+        frame_delay = 1.0 / fps;
+        UtilityFunctions::print("[gdvapi] fps: ", fps, " frame delay: ", frame_delay, "s");
+    } else {
+        frame_delay = 1.0 / 60.0; // fallback
+    }
+
+	AVCodecParameters *codec_par = fmt_ctx->streams[video_stream_idx]->codecpar;
+	codec = avcodec_find_decoder(codec_par->codec_id);
+	if (!codec)
+		return;
+
+	codec_ctx = avcodec_alloc_context3(codec);
+	if (!codec_ctx)
+		return;
+
+	if (avcodec_parameters_to_context(codec_ctx, codec_par) < 0)
+		return;
+
+	active_hw_mode = HW_VULKAN;
+	if (av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VULKAN, nullptr, nullptr, 0) < 0) {
+		UtilityFunctions::print("[gdvapi] vulkan video extensions missing on driver, trying va-api");
+
+		active_hw_mode = HW_VAAPI;
+		if (av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VAAPI, nullptr, nullptr, 0) < 0) {
+			UtilityFunctions::print("[gdvapi] va-api acceleration unavailable. using software");
+			active_hw_mode = HW_NONE;
+		}
+	}
+
+	if (active_hw_mode != HW_NONE) {
+		codec_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx);
+		codec_ctx->get_format = get_hw_format;
+	}
+
+	if (avcodec_open2(codec_ctx, codec, nullptr) < 0)
+		return;
+
+	frame = av_frame_alloc();
+	sw_frame = av_frame_alloc();
+	packet = av_packet_alloc();
+
+	width = codec_ctx->width;
+	height = codec_ctx->height;
+	texture_init();
+
+	sws_ctx = sws_getContext(
+			width, height, codec_ctx->pix_fmt,
+			width, height, AV_PIX_FMT_RGBA,
+			SWS_BILINEAR, nullptr, nullptr, nullptr);
+
+    time_accumulator = 0.0;
+    set_process(true);
+}
+
+void FileVideoStream::_process(double delta) {
+    if (!fmt_ctx || !codec_ctx) return;
+
+    time_accumulator += delta;
+
+    if (time_accumulator < frame_delay) {
+        return;
+    }
+
+    time_accumulator -= frame_delay;
+
+	bool frame_finished = false;
+	while (!frame_finished && av_read_frame(fmt_ctx, packet) >= 0) {
+		if (packet->stream_index == video_stream_idx) {
+			if (avcodec_send_packet(codec_ctx, packet) >= 0) {
+				while (avcodec_receive_frame(codec_ctx, frame) >= 0) {
+					frame_finished = true;
+
+					RenderingDevice *rd = RenderingServer::get_singleton()->get_rendering_device();
+					if (!rd || !texture_rd_rid.is_valid())
+						continue;
+
+					if (frame->format == AV_PIX_FMT_VULKAN || frame->format == AV_PIX_FMT_VAAPI) {
+						if (av_hwframe_transfer_data(sw_frame, frame, 0) >= 0) {
+							PackedByteArray gpu_bytes;
+							gpu_bytes.resize(width * height * 4);
+
+							uint8_t *dest_pointers[4] = { gpu_bytes.ptrw(), nullptr, nullptr, nullptr };
+							int dest_linesizes[4] = { width * 4, 0, 0, 0 };
+
+							enum AVPixelFormat src_fmt = (enum AVPixelFormat)sw_frame->format;
+							struct SwsContext *hw_sws = sws_getContext(
+									width, height, src_fmt,
+									width, height, AV_PIX_FMT_RGBA,
+									SWS_BILINEAR, nullptr, nullptr, nullptr);
+
+							if (hw_sws) {
+								sws_scale(hw_sws, sw_frame->data, sw_frame->linesize, 0, height, dest_pointers, dest_linesizes);
+								sws_freeContext(hw_sws);
+								rd->texture_update(texture_rd_rid, 0, gpu_bytes);
+							}
+						}
+					} else if (frame->data[0] != nullptr) {
+						PackedByteArray rgba_data;
+						rgba_data.resize(width * height * 4);
+						uint8_t *dest_pointers[4] = { rgba_data.ptrw(), nullptr, nullptr, nullptr };
+						int dest_linesizes[4] = { width * 4, 0, 0, 0 };
+
+						if (sws_ctx) {
+							sws_scale(sws_ctx, frame->data, frame->linesize, 0, height, dest_pointers, dest_linesizes);
+							rd->texture_update(texture_rd_rid, 0, rgba_data);
+						}
+					}
+
+					queue_redraw();
+				}
+			}
+		}
+		av_packet_unref(packet);
+	}
+}
+
+FileVideoStream::~FileVideoStream() {
+	cleanup();
+}
+
+void FileVideoStream::_notification(int p_what) {
+	switch (p_what) {
+		case NOTIFICATION_EXIT_TREE: {
+			cleanup();
+		} break;
+	}
+}
+
+void FileVideoStream::texture_init() {
+	RenderingDevice *rd = RenderingServer::get_singleton()->get_rendering_device();
+	if (!rd)
+		return;
+
+	if (texture_rd_rid.is_valid()) {
+		rd->free_rid(texture_rd_rid);
+	}
+
+	Ref<RDTextureFormat> format;
+	format.instantiate();
+	format->set_texture_type(RenderingDevice::TEXTURE_TYPE_2D);
+	format->set_format(RenderingDevice::DATA_FORMAT_R8G8B8A8_UNORM);
+	format->set_width(width);
+	format->set_height(height);
+	format->set_depth(1);
+	format->set_array_layers(1);
+	format->set_mipmaps(1);
+	format->set_usage_bits(RenderingDevice::TEXTURE_USAGE_SAMPLING_BIT |
+			RenderingDevice::TEXTURE_USAGE_CAN_UPDATE_BIT);
+
+	PackedByteArray byte_array;
+	byte_array.resize(width * height * 4);
+
+	TypedArray<PackedByteArray> data;
+	data.push_back(byte_array);
+
+	Ref<RDTextureView> view;
+	view.instantiate();
+
+	texture_rd_rid = rd->texture_create(format, view, data);
+
+	if (texture_rd_rid.is_valid()) {
+		godot_texture.instantiate();
+		godot_texture->set_texture_rd_rid(texture_rd_rid);
+	}
+}
+
+void FileVideoStream::cleanup() {
+	set_process(false);
+	RenderingDevice *rd = RenderingServer::get_singleton()->get_rendering_device();
+
+	godot_texture.unref();
+
+	if (rd && texture_rd_rid.is_valid()) {
+		rd->free_rid(texture_rd_rid);
+		texture_rd_rid = RID();
+	}
+
+	if (sws_ctx) {
+		sws_freeContext(sws_ctx);
+		sws_ctx = nullptr;
+	}
+	if (frame) {
+		av_frame_free(&frame);
+	}
+	if (sw_frame) {
+		av_frame_free(&sw_frame);
+	}
+	if (packet) {
+		av_packet_free(&packet);
+	}
+	if (codec_ctx) {
+		avcodec_free_context(&codec_ctx);
+	}
+	if (hw_device_ctx) {
+		av_buffer_unref(&hw_device_ctx);
+		hw_device_ctx = nullptr;
+	}
+	if (fmt_ctx) {
+		avformat_close_input(&fmt_ctx);
+	}
+}
+
+void FileVideoStream::_draw() {
+	if (godot_texture.is_valid()) {
+		draw_texture(godot_texture, Point2(0, 0));
+	}
+}

src/filevideostream_class.h

@@ -0,0 +1,56 @@
+#pragma once
+
+#include "godot_cpp/classes/control.hpp"
+#include "godot_cpp/classes/texture2drd.hpp"
+#include "godot_cpp/variant/rid.hpp"
+
+extern "C" {
+#include <libavcodec/avcodec.h>
+#include <libavformat/avformat.h>
+#include <libavutil/imgutils.h>
+#include <libswscale/swscale.h>
+}
+
+namespace godot {
+
+class FileVideoStream : public Control {
+	GDCLASS(FileVideoStream, Control)
+
+private:
+	AVFormatContext *fmt_ctx = nullptr;
+	AVCodecContext *codec_ctx = nullptr;
+	const AVCodec *codec = nullptr;
+	AVFrame *frame = nullptr;
+	AVPacket *packet = nullptr;
+	struct SwsContext *sws_ctx = nullptr;
+	int video_stream_idx = -1;
+	int width = 256;
+	int height = 256;
+	AVBufferRef *hw_device_ctx = nullptr;
+	AVFrame *sw_frame = nullptr;
+
+	double frame_delay = 0.0;
+    double time_accumulator = 0.0; 
+
+	RID texture_rd_rid;
+	Ref<Texture2DRD> godot_texture;
+
+	void cleanup();
+	void texture_init();
+
+protected:
+	static void _bind_methods();
+
+public:
+	FileVideoStream() = default;
+	~FileVideoStream() override;
+
+	void begin(const Variant &source);
+
+	void _process(double delta) override;
+
+	void _notification(int p_what);
+	void _draw() override;
+};
+
+}

src/register_types.cpp

@@ -5,16 +5,15 @@
 #include <godot_cpp/core/defs.hpp>
 #include <godot_cpp/godot.hpp>
 
-#include "udpvideostream_class.h"
+#include "filevideostream_class.h"
 
 using namespace godot;
 
-void initialize_gdextension_types(ModuleInitializationLevel p_level)
-{
+void initialize_gdextension_types(ModuleInitializationLevel p_level) {
 	if (p_level != MODULE_INITIALIZATION_LEVEL_SCENE) {
 		return;
 	}
-	GDREGISTER_CLASS(UDPVideoStream);
+	GDREGISTER_CLASS(FileVideoStream);
 }
 
 void uninitialize_gdextension_types(ModuleInitializationLevel p_level) {
@@ -23,16 +22,14 @@ void uninitialize_gdextension_types(ModuleInitializationLevel p_level) {
 	}
 }
 
-extern "C"
-{
-	// Initialization
-	GDExtensionBool GDE_EXPORT gdvapi_library_init(GDExtensionInterfaceGetProcAddress p_get_proc_address, GDExtensionClassLibraryPtr p_library, GDExtensionInitialization *r_initialization)
-	{
+extern "C" {
+// Initialization
+GDExtensionBool GDE_EXPORT gdvapi_library_init(GDExtensionInterfaceGetProcAddress p_get_proc_address, GDExtensionClassLibraryPtr p_library, GDExtensionInitialization *r_initialization) {
 	GDExtensionBinding::InitObject init_obj(p_get_proc_address, p_library, r_initialization);
 	init_obj.register_initializer(initialize_gdextension_types);
 	init_obj.register_terminator(uninitialize_gdextension_types);
 	init_obj.set_minimum_library_initialization_level(MODULE_INITIALIZATION_LEVEL_SCENE);
 
 	return init_obj.init();
-	}
+}
 }

src/udpvideostream_class.cpp

@@ -1,97 +0,0 @@
-#include "udpvideostream_class.h"
-#include "godot_cpp/classes/rendering_server.hpp"
-#include "godot_cpp/classes/rendering_device.hpp"
-#include "godot_cpp/classes/rd_texture_format.hpp"
-#include "godot_cpp/classes/rd_texture_view.hpp"
-
-
-using namespace godot;
-
-void UDPVideoStream::_bind_methods() {
-	godot::ClassDB::bind_method(D_METHOD("begin", "variant"), &UDPVideoStream::begin);
-}
-
-void UDPVideoStream::begin(const Variant &source) const {
-	print_line(vformat("[gdvapi] begin stream at %s", source.stringify()));
-}
-
-UDPVideoStream::~UDPVideoStream() {
-    cleanup();
-}
-
-void UDPVideoStream::_notification(int p_what) {
-    switch (p_what) {
-        case NOTIFICATION_POST_ENTER_TREE: {
-        	texture_init();
-        } break;
-        
-        case NOTIFICATION_EXIT_TREE: {
-            cleanup();
-        } break;
-    }
-}
-
-
-void UDPVideoStream::texture_init() {
-	RenderingDevice *rd = RenderingServer::get_singleton()->get_rendering_device();
-    if (!rd) {
-        UtilityFunctions::printerr("Vulkan RenderingDevice is not available!");
-        return;
-    }
-
-    int width = 256;
-    int height = 256;
-
-    Ref<RDTextureFormat> format;
-    format.instantiate();
-    format->set_texture_type(RenderingDevice::TEXTURE_TYPE_2D);
-    format->set_format(RenderingDevice::DATA_FORMAT_R8G8B8A8_UNORM);
-    format->set_width(width);
-    format->set_height(height);
-    format->set_depth(1);
-    format->set_array_layers(1);
-    format->set_mipmaps(1);
-    format->set_usage_bits(RenderingDevice::TEXTURE_USAGE_SAMPLING_BIT | 
-                           RenderingDevice::TEXTURE_USAGE_CAN_UPDATE_BIT);
-
-    PackedByteArray byte_array;
-    byte_array.resize(width * height * 4);
-    for (int i = 0; i < byte_array.size(); i += 4) {
-        byte_array[i]     = 0;
-        byte_array[i + 1] = 0;
-        byte_array[i + 2] = 255;
-        byte_array[i + 3] = 255;
-    }
-
-    TypedArray<PackedByteArray> data;
-    data.push_back(byte_array);
-
-    Ref<RDTextureView> view;
-	view.instantiate(); 
-
-	texture_rd_rid = rd->texture_create(format, view, data);
-
-    if (texture_rd_rid.is_valid()) {
-        godot_texture.instantiate();
-        godot_texture->set_texture_rd_rid(texture_rd_rid);
-        
-        queue_redraw();
-    }
-}
-
-void UDPVideoStream::cleanup() {
-    RenderingDevice *rd = RenderingServer::get_singleton()->get_rendering_device();
-    
-    godot_texture.unref();
-
-    if (rd && texture_rd_rid.is_valid()) {
-        rd->free_rid(texture_rd_rid);
-        texture_rd_rid = RID();
-    }
-}
-
-void UDPVideoStream::_draw() {
-    if (godot_texture.is_valid() && godot_texture->get_texture_rd_rid().is_valid()) {
-        draw_texture(godot_texture, Point2(0, 0));
-    }
-}

src/udpvideostream_class.h

@@ -1,32 +0,0 @@
-#pragma once
-
-#include "godot_cpp/classes/control.hpp"
-#include "godot_cpp/classes/wrapped.hpp"
-#include "godot_cpp/variant/variant.hpp"
-#include "godot_cpp/classes/rendering_device.hpp"
-#include "godot_cpp/classes/texture2drd.hpp"
-
-using namespace godot;
-
-class UDPVideoStream : public Control {
-	GDCLASS(UDPVideoStream, Control)
-
-private:
-	RID texture_rd_rid;
-	Ref<Texture2DRD> godot_texture;
-
-	void cleanup();
-	void texture_init();
-
-protected:
-	static void _bind_methods();
-
-public:
-	UDPVideoStream() = default;
-	~UDPVideoStream() override;
-
-	void begin(const Variant &source) const;
-
-	void _notification(int p_what);
-	virtual void _draw() override;
-};