diff --git a/.gitignore b/.gitignore index cae99a7..4a284aa 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,8 @@ CMakeUserPresets.json build/ godot/ +build-editor/ + +# Some testing files +project/*.mp4 +project/*.mov diff --git a/CMakeLists.txt b/CMakeLists.txt index e435263..29853fd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -74,6 +74,12 @@ target_sources(${LIBNAME} src/register_types.h src/gdvapivideostream_class.cpp src/gdvapivideostream_class.h + src/vulkan_detour.h + src/vulkan_detour.cpp + src/vaapi_decoder.h + src/vaapi_decoder.cpp + src/vk_image_import.h + src/vk_image_import.cpp ) # Fetch a list of the xml files to use for documentation and add to our target diff --git a/project/badapple.mov b/project/badapple.mov deleted file mode 100644 index 73bab02..0000000 Binary files a/project/badapple.mov and /dev/null differ diff --git a/project/example.gd b/project/example.gd index ae667dc..c7cbd99 100644 --- a/project/example.gd +++ b/project/example.gd @@ -1,6 +1,11 @@ extends Node - func _ready() -> void: - #$GDVAPIVideoStream.begin("udp://0.0.0.0:5000") - pass + $GDVAPIVideoStream.queue_depth = 4 # default; proves the property binding + $GDVAPIVideoStream.begin("/tmp/gdvapi_test.sdp") + +func _process(_delta: float) -> void: + var tex: Texture2D = $GDVAPIVideoStream.get_texture() + if tex: + if $TextureRect.texture != tex: + $TextureRect.texture = tex \ No newline at end of file diff --git a/project/example.tscn b/project/example.tscn index aab50ac..86d2d50 100644 --- a/project/example.tscn +++ b/project/example.tscn @@ -6,5 +6,13 @@ script = ExtResource("1_jdh55") [node name="GDVAPIVideoStream" type="GDVAPIVideoStream" parent="." unique_id=1053894928] -offset_right = 40.0 -offset_bottom = 40.0 + +[node name="TextureRect" type="TextureRect" parent="."] +layout_mode = 1 +anchors_preset = 15 +anchor_right = 1.0 +anchor_bottom = 1.0 +grow_horizontal = 2 +grow_vertical = 2 +expand_mode = 1 +stretch_mode = 6 \ No newline at end of file diff --git a/project/exports/badapple.mov b/project/exports/badapple.mov deleted file mode 100644 index 73bab02..0000000 Binary files a/project/exports/badapple.mov and /dev/null differ diff --git a/project/exports/godot cpp template.pck b/project/exports/godot cpp template.pck deleted file mode 100644 index 9b41a77..0000000 Binary files a/project/exports/godot cpp template.pck and /dev/null differ diff --git a/project/exports/godot cpp template.sh b/project/exports/godot cpp template.sh deleted file mode 100755 index dd5bc58..0000000 --- a/project/exports/godot cpp template.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -printf '\033c\033]0;%s\a' godot cpp template -base_path="$(dirname "$(realpath "$0")")" -"$base_path/godot cpp template.x86_64" "$@" diff --git a/project/exports/godot cpp template.x86_64 b/project/exports/godot cpp template.x86_64 deleted file mode 100755 index ff8f2a6..0000000 Binary files a/project/exports/godot cpp template.x86_64 and /dev/null differ diff --git a/project/project.godot b/project/project.godot index 2d39b3e..5580007 100644 --- a/project/project.godot +++ b/project/project.godot @@ -21,5 +21,7 @@ config/icon="res://icon.svg" [display] +display/window/vsync/vsync_mode=0 + window/size/viewport_width=1920 window/size/viewport_height=1080 diff --git a/project/test.mp4 b/project/test.mp4 deleted file mode 100644 index b11552f..0000000 Binary files a/project/test.mp4 and /dev/null differ diff --git a/server-test4k.sh b/server-test4k.sh new file mode 100755 index 0000000..2069670 --- /dev/null +++ b/server-test4k.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +import subprocess, sys + +fps = sys.argv[1] if len(sys.argv) > 1 else "60" +size = sys.argv[2] if len(sys.argv) > 2 else "3840x2160" +port = sys.argv[3] if len(sys.argv) > 3 else "5000" +bitrate = sys.argv[4] if len(sys.argv) > 4 else "40M" + +cmd = [ + "ffmpeg", "-hide_banner", "-v", "error", + "-f", "lavfi", "-i", + f"testsrc2=size={size}:rate={fps}:duration=1000", + "-an", + "-c:v", "libx265", + "-preset", "ultrafast", + "-tune", "zerolatency", + "-pix_fmt", "yuv420p", + "-b:v", bitrate, + "-x265-params", "bframes=0:keyint=30:scenecut=0", + "-f", "rtp", + "-payload_type", "96", + "-sdp_file", "/tmp/gdvapi_test4k.sdp", + f"rtp://127.0.0.1:{port}", +] +print("running:", " ".join(cmd)) +try: + subprocess.run(cmd, check=True) +except KeyboardInterrupt: + pass +except subprocess.CalledProcessError as e: + print("ffmpeg exited", e.returncode) + sys.exit(1) diff --git a/src/gdvapivideostream_class.cpp b/src/gdvapivideostream_class.cpp index 2630a97..5c368d7 100644 --- a/src/gdvapivideostream_class.cpp +++ b/src/gdvapivideostream_class.cpp @@ -8,6 +8,12 @@ #include "godot_cpp/variant/color.hpp" #include "godot_cpp/variant/rect2.hpp" +#include "vaapi_decoder.h" +#include "vk_image_import.h" + +#include +#include + extern "C" { #include #include @@ -24,27 +30,56 @@ enum HWMode { HW_NONE, static HWMode active_hw_mode = HW_NONE; void GDVAPIVideoStream::_bind_methods() { - godot::ClassDB::bind_method(D_METHOD("begin", "source"), &GDVAPIVideoStream::begin); + ClassDB::bind_method(D_METHOD("begin", "source"), &GDVAPIVideoStream::begin); + ClassDB::bind_method(D_METHOD("get_texture"), &GDVAPIVideoStream::get_texture); ClassDB::bind_method(D_METHOD("get_desired_width"), &GDVAPIVideoStream::get_desired_width); ClassDB::bind_method(D_METHOD("set_desired_width", "v"), &GDVAPIVideoStream::set_desired_width); - ClassDB::bind_method(D_METHOD("get_desired_height"), &GDVAPIVideoStream::get_desired_height); ClassDB::bind_method(D_METHOD("set_desired_height", "v"), &GDVAPIVideoStream::set_desired_height); ADD_PROPERTY(PropertyInfo(Variant::INT, "desired_width"), "set_desired_width", "get_desired_width"); ADD_PROPERTY(PropertyInfo(Variant::INT, "desired_height"), "set_desired_height", "get_desired_height"); + + ClassDB::bind_method(D_METHOD("set_reorder_buffer_ms", "v"), &GDVAPIVideoStream::set_reorder_buffer_ms); + ClassDB::bind_method(D_METHOD("get_reorder_buffer_ms"), &GDVAPIVideoStream::get_reorder_buffer_ms); + ClassDB::bind_method(D_METHOD("set_demux_fifo_bytes", "v"), &GDVAPIVideoStream::set_demux_fifo_bytes); + ClassDB::bind_method(D_METHOD("get_demux_fifo_bytes"), &GDVAPIVideoStream::get_demux_fifo_bytes); + ClassDB::bind_method(D_METHOD("set_socket_buffer_bytes", "v"), &GDVAPIVideoStream::set_socket_buffer_bytes); + ClassDB::bind_method(D_METHOD("get_socket_buffer_bytes"), &GDVAPIVideoStream::get_socket_buffer_bytes); + ClassDB::bind_method(D_METHOD("set_reorder_queue_size", "v"), &GDVAPIVideoStream::set_reorder_queue_size); + ClassDB::bind_method(D_METHOD("get_reorder_queue_size"), &GDVAPIVideoStream::get_reorder_queue_size); + ClassDB::bind_method(D_METHOD("set_io_timeout_ms", "v"), &GDVAPIVideoStream::set_io_timeout_ms); + ClassDB::bind_method(D_METHOD("get_io_timeout_ms"), &GDVAPIVideoStream::get_io_timeout_ms); + ClassDB::bind_method(D_METHOD("set_queue_depth", "v"), &GDVAPIVideoStream::set_queue_depth); + ClassDB::bind_method(D_METHOD("get_queue_depth"), &GDVAPIVideoStream::get_queue_depth); + ClassDB::bind_method(D_METHOD("set_frame_latency_ms", "v"), &GDVAPIVideoStream::set_frame_latency_ms); + ClassDB::bind_method(D_METHOD("get_frame_latency_ms"), &GDVAPIVideoStream::get_frame_latency_ms); + ClassDB::bind_method(D_METHOD("set_vpp_half_res", "v"), &GDVAPIVideoStream::set_vpp_half_res); + ClassDB::bind_method(D_METHOD("get_vpp_half_res"), &GDVAPIVideoStream::get_vpp_half_res); + + ADD_PROPERTY(PropertyInfo(Variant::INT, "reorder_buffer_ms"), "set_reorder_buffer_ms", "get_reorder_buffer_ms"); + ADD_PROPERTY(PropertyInfo(Variant::INT, "demux_fifo_bytes"), "set_demux_fifo_bytes", "get_demux_fifo_bytes"); + ADD_PROPERTY(PropertyInfo(Variant::INT, "socket_buffer_bytes"), "set_socket_buffer_bytes", "get_socket_buffer_bytes"); + ADD_PROPERTY(PropertyInfo(Variant::INT, "reorder_queue_size"), "set_reorder_queue_size", "get_reorder_queue_size"); + ADD_PROPERTY(PropertyInfo(Variant::INT, "io_timeout_ms"), "set_io_timeout_ms", "get_io_timeout_ms"); + ADD_PROPERTY(PropertyInfo(Variant::INT, "queue_depth"), "set_queue_depth", "get_queue_depth"); + ADD_PROPERTY(PropertyInfo(Variant::INT, "frame_latency_ms"), "set_frame_latency_ms", "get_frame_latency_ms"); + ADD_PROPERTY(PropertyInfo(Variant::BOOL, "vpp_half_res"), "set_vpp_half_res", "get_vpp_half_res"); + + ClassDB::bind_method(D_METHOD("get_frame_count"), &GDVAPIVideoStream::get_frame_count); + ClassDB::bind_method(D_METHOD("has_frame"), &GDVAPIVideoStream::has_frame); + ClassDB::bind_method(D_METHOD("get_stream_width"), &GDVAPIVideoStream::get_stream_width); + ClassDB::bind_method(D_METHOD("get_stream_height"), &GDVAPIVideoStream::get_stream_height); } static enum AVPixelFormat get_hw_format(AVCodecContext *ctx, const enum AVPixelFormat *formats) { for (const enum AVPixelFormat *p = formats; *p != -1; p++) { - if (active_hw_mode == HW_VULKAN && *p == AV_PIX_FMT_VULKAN) { + if (active_hw_mode == HW_VULKAN && *p == AV_PIX_FMT_VULKAN) return AV_PIX_FMT_VULKAN; - } - if (active_hw_mode == HW_VAAPI && *p == AV_PIX_FMT_VAAPI) { + if (active_hw_mode == HW_VAAPI && *p == AV_PIX_FMT_VAAPI) return AV_PIX_FMT_VAAPI; - } } UtilityFunctions::printerr("[gdvapi] hardware decoding unavailable"); active_hw_mode = HW_NONE; @@ -52,146 +87,332 @@ static enum AVPixelFormat get_hw_format(AVCodecContext *ctx, const enum AVPixelF } void GDVAPIVideoStream::begin(const Variant &source) { - String res_path = source; - String global_path = ProjectSettings::get_singleton()->globalize_path(res_path); - CharString path_utf8 = global_path.utf8(); - - fmt_ctx = nullptr; - if (avformat_open_input(&fmt_ctx, path_utf8.get_data(), nullptr, nullptr) < 0) { - UtilityFunctions::printerr("[gdvapi] failed to open input stream"); - return; - } - - if (avformat_find_stream_info(fmt_ctx, nullptr) < 0) { - return; - } - - video_stream_idx = -1; - for (unsigned int i = 0; i < fmt_ctx->nb_streams; i++) { - if (fmt_ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { - video_stream_idx = i; - break; - } - } - if (video_stream_idx == -1) return; - - AVStream *stream = fmt_ctx->streams[video_stream_idx]; - if (stream->avg_frame_rate.den > 0 && stream->avg_frame_rate.num > 0) { - double fps = av_q2d(stream->avg_frame_rate); - frame_delay = 1.0 / fps; - UtilityFunctions::print("[gdvapi] fps: ", fps, " frame delay: ", frame_delay, "s"); - } else { - frame_delay = 1.0 / 60.0; // fallback - } - - AVCodecParameters *codec_par = fmt_ctx->streams[video_stream_idx]->codecpar; - codec = avcodec_find_decoder(codec_par->codec_id); - if (!codec) - return; - - codec_ctx = avcodec_alloc_context3(codec); - if (!codec_ctx) - return; - - if (avcodec_parameters_to_context(codec_ctx, codec_par) < 0) - return; - - active_hw_mode = HW_VULKAN; - if (av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VULKAN, nullptr, nullptr, 0) < 0) { - UtilityFunctions::print("[gdvapi] vulkan video extensions missing on driver, trying va-api"); - - active_hw_mode = HW_VAAPI; - if (av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VAAPI, nullptr, nullptr, 0) < 0) { - UtilityFunctions::print("[gdvapi] va-api acceleration unavailable. using software"); - active_hw_mode = HW_NONE; - } - } - - if (active_hw_mode != HW_NONE) { - codec_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx); - codec_ctx->get_format = get_hw_format; - } - - if (avcodec_open2(codec_ctx, codec, nullptr) < 0) - return; - - frame = av_frame_alloc(); - sw_frame = av_frame_alloc(); - packet = av_packet_alloc(); - - width = codec_ctx->width; - height = codec_ctx->height; - texture_init(); - - sws_ctx = sws_getContext( - width, height, codec_ctx->pix_fmt, - width, height, AV_PIX_FMT_RGBA, - SWS_BILINEAR, nullptr, nullptr, nullptr); - - time_accumulator = 0.0; - set_process(true); + stream_source = source; } void GDVAPIVideoStream::_process(double delta) { - if (!fmt_ctx || !codec_ctx) return; + (void)delta; + if (!stream_started) { + stream_started = true; + RenderingDevice *rd = RenderingServer::get_singleton()->get_rendering_device(); + if (rd) { + uint64_t vk_dev_u = rd->get_driver_resource( + RenderingDevice::DRIVER_RESOURCE_LOGICAL_DEVICE, RID(), 0); + vk_dev = (VkDevice)vk_dev_u; + uint64_t vk_inst_u = rd->get_driver_resource( + RenderingDevice::DRIVER_RESOURCE_VULKAN_INSTANCE, RID(), 0); + vk_inst = (VkInstance)vk_inst_u; + } + wall_base = std::chrono::steady_clock::now(); + String uri = stream_source; + if (uri.ends_with(".sdp") || uri.ends_with(".mp4") + || uri.ends_with(".mov") || uri.begins_with("/")) + worker_src = uri.utf8().get_data(); + else { + const char *sdp = "/tmp/gdvapi_live.sdp"; + String host = "127.0.0.1", port = "5000"; + if (uri.begins_with("udp://")) { + String rest = uri.substr(6); + int c = rest.find(":"); + if (c > 0) { host = rest.substr(0, c); port = rest.substr(c + 1); } + } + FILE *f = fopen(sdp, "w"); + if (f) { + fprintf(f, "v=0\no=- 0 0 IN IP4 %s\ns=Live\nc=IN IP4 %s\nt=0 0\n" + "m=video %s RTP/AVP 96\na=rtpmap:96 H265/90000\n", + host.utf8().get_data(), host.utf8().get_data(), + port.utf8().get_data()); + fclose(f); + } + worker_src = sdp; + } + source_live = uri.ends_with(".sdp") || uri.begins_with("udp://"); + worker_thr = std::thread(&GDVAPIVideoStream::worker_loop, this); + } - time_accumulator += delta; + if (size_pending) + size_pending = false; - if (time_accumulator < frame_delay) { - return; - } + consume_ready(); +} - time_accumulator -= frame_delay; +void GDVAPIVideoStream::worker_loop() { + while (!worker_stop) { + if (!va_decoder.open(worker_src.c_str(), decoder_opts)) { + fprintf(stderr, "[gdvapi] worker: open failed (%s)%s\n", + worker_src.c_str(), source_live ? " - retrying" : ""); + if (source_live) { + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + continue; + } + std::lock_guard lk(q_mu); + stream_eof = true; + q_cv.notify_all(); + break; + } + { + std::lock_guard lk(q_mu); + size_pending = true; + pending_w = va_decoder.width(); + pending_h = va_decoder.height(); + worker_fps = va_decoder.fps(); + for (int i = 0; i < gdvapi::VAAPIDecoder::kRgbaPool; i++) + slot_acked[i].store(true); + } - bool frame_finished = false; - while (!frame_finished && av_read_frame(fmt_ctx, packet) >= 0) { - if (packet->stream_index == video_stream_idx) { - if (avcodec_send_packet(codec_ctx, packet) >= 0) { - while (avcodec_receive_frame(codec_ctx, frame) >= 0) { - frame_finished = true; + while (!worker_stop) { + { + std::unique_lock lk(q_mu); + q_cv.wait(lk, [&] { + return worker_stop || slot_acked[vpp_slot].load(); + }); + if (worker_stop) + break; + } - RenderingDevice *rd = RenderingServer::get_singleton()->get_rendering_device(); - if (!rd || !texture_rd_rid.is_valid()) - continue; - - if (frame->format == AV_PIX_FMT_VULKAN || frame->format == AV_PIX_FMT_VAAPI) { - if (av_hwframe_transfer_data(sw_frame, frame, 0) >= 0) { - PackedByteArray gpu_bytes; - gpu_bytes.resize(width * height * 4); - - uint8_t *dest_pointers[4] = { gpu_bytes.ptrw(), nullptr, nullptr, nullptr }; - int dest_linesizes[4] = { width * 4, 0, 0, 0 }; - - enum AVPixelFormat src_fmt = (enum AVPixelFormat)sw_frame->format; - struct SwsContext *hw_sws = sws_getContext( - width, height, src_fmt, - width, height, AV_PIX_FMT_RGBA, - SWS_BILINEAR, nullptr, nullptr, nullptr); - - if (hw_sws) { - sws_scale(hw_sws, sw_frame->data, sw_frame->linesize, 0, height, dest_pointers, dest_linesizes); - sws_freeContext(hw_sws); - rd->texture_update(texture_rd_rid, 0, gpu_bytes); - } - } - } else if (frame->data[0] != nullptr) { - PackedByteArray rgba_data; - rgba_data.resize(width * height * 4); - uint8_t *dest_pointers[4] = { rgba_data.ptrw(), nullptr, nullptr, nullptr }; - int dest_linesizes[4] = { width * 4, 0, 0, 0 }; - - if (sws_ctx) { - sws_scale(sws_ctx, frame->data, frame->linesize, 0, height, dest_pointers, dest_linesizes); - rd->texture_update(texture_rd_rid, 0, rgba_data); - } + AVFrame *avf = nullptr; + if (!va_decoder.next_frame(avf)) { + if (source_live) { + if (getenv("GDVAPI_DEBUG_WORKER")) + fprintf(stderr, "[gdvapi] worker: live gap, retrying\n"); + va_decoder.close(); + { + std::lock_guard lk(q_mu); + pts_base = -1.0; // re-anchor pacing to the next epoch + last_pts = -1.0; + wall_base = std::chrono::steady_clock::now(); } - - queue_redraw(); + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + break; } + if (getenv("GDVAPI_DEBUG_WORKER")) + fprintf(stderr, "[gdvapi] worker: next_frame fail/eof\n"); + std::lock_guard lk(q_mu); + stream_eof = true; + q_cv.notify_all(); + break; + } + gdvapi::DecodedFrame f; + if (!va_decoder.convert_frame_to_rgba_slot(avf, vpp_slot, f)) { + if (getenv("GDVAPI_DEBUG_WORKER")) + fprintf(stderr, "[gdvapi] worker: convert fail slot=%d\n", vpp_slot); + av_frame_unref(avf); + continue; + } + av_frame_unref(avf); + if (getenv("GDVAPI_DEBUG_WORKER")) + fprintf(stderr, "[gdvapi] worker: push fd=%d slot=%d\n", f.fd, vpp_slot); + + ReadyFrame rf; + rf.fd = f.fd; + rf.w = f.width; + rf.h = f.height; + rf.pts_sec = f.pts_sec; + rf.drm_fourcc = f.drm_fourcc; + rf.slot = vpp_slot; + slot_acked[vpp_slot].store(false); + { + std::lock_guard lk(q_mu); + if (worker_stop) { + ::close(f.fd); + break; + } + while (ready_q.size() >= queue_max) { + ReadyFrame old = ready_q.front(); + ready_q.pop_front(); + slot_acked[old.slot].store(true); + ::close(old.fd); + } + ready_q.push_back(rf); + vpp_slot = (vpp_slot + 1) % gdvapi::VAAPIDecoder::kRgbaPool; + q_cv.notify_all(); } } - av_packet_unref(packet); + va_decoder.close(); + if (!source_live) + break; } + std::lock_guard lk(q_mu); + worker_finished = true; + q_cv.notify_all(); +} + +void GDVAPIVideoStream::stop_worker() { + { + std::lock_guard lk(q_mu); + worker_stop = true; + q_cv.notify_all(); + } + if (worker_thr.joinable()) { + worker_thr.join(); + worker_thr = std::thread(); + } + std::lock_guard lk(q_mu); + while (!ready_q.empty()) { + ::close(ready_q.front().fd); + ready_q.pop_front(); + } +} + +void GDVAPIVideoStream::consume_ready() { + RenderingDevice *rd = RenderingServer::get_singleton()->get_rendering_device(); + if (!rd || !vk_dev) + return; + + const double kDropAhead = max_frame_latency; + auto now = std::chrono::steady_clock::now(); + double elapsed = std::chrono::duration(now - wall_base).count(); + + ReadyFrame rf; + { + std::lock_guard lk(q_mu); + if (ready_q.empty()) + return; + rf = ready_q.front(); + } + + bool new_epoch = false; + if (rf.pts_sec > 0.0 && last_pts > 0.0) { + if (rf.pts_sec - last_pts < -1.0) { + last_pts = -1.0; + pts_base = -1.0; + wall_base = std::chrono::steady_clock::now(); + new_epoch = true; + } else if (rf.pts_sec - last_pts > 4.0) { + last_pts = rf.pts_sec; + pts_base = -1.0; + wall_base = std::chrono::steady_clock::now(); + new_epoch = true; + } else if (rf.pts_sec <= last_pts) { + { + std::lock_guard lk(q_mu); + ready_q.pop_front(); + } + slot_acked[rf.slot].store(true); + q_cv.notify_all(); + ::close(rf.fd); + return; + } + } + last_pts = rf.pts_sec; + + if (rf.pts_sec > 0.0) { + if (pts_base < 0.0) { + pts_base = rf.pts_sec; + wall_base = std::chrono::steady_clock::now() + - std::chrono::duration_cast( + std::chrono::duration(rf.pts_sec)); + elapsed = 0.0; + } + const double stream_pos = rf.pts_sec - pts_base; + std::lock_guard lk(q_mu); + const bool have_backlog = ready_q.size() > 1; + if (have_backlog && stream_pos - elapsed > (1.0 / 30.0)) { + ready_q.pop_front(); + slot_acked[rf.slot].store(true); + q_cv.notify_all(); + ::close(rf.fd); + return; + } + } + + if (rf.pts_sec > 0.0 && elapsed - rf.pts_sec > kDropAhead) { + { + std::lock_guard lk(q_mu); + ready_q.pop_front(); + } + slot_acked[rf.slot].store(true); + q_cv.notify_all(); + ::close(rf.fd); + return; + } + { + std::lock_guard lk(q_mu); + ready_q.pop_front(); + } + slot_acked[rf.slot].store(true); + q_cv.notify_all(); + + gdvapi::DmaBufSurface surf; + surf.fd = rf.fd; + surf.width = rf.w; + surf.height = rf.h; + surf.drm_fourcc = rf.drm_fourcc; + surf.fourcc = rf.drm_fourcc ? rf.drm_fourcc : 0x34325241; + + if (present_image) { + gdvapi::destroy_imported_image(vk_dev, present_image, present_mem); + present_image = VK_NULL_HANDLE; + present_mem = VK_NULL_HANDLE; + } + + VkImage img = VK_NULL_HANDLE; + VkDeviceMemory mem = VK_NULL_HANDLE; + img = gdvapi::import_dma_buf_image(vk_dev, surf, &mem); + if (!img || !mem) { + ::close(rf.fd); + return; + } + present_image = img; + present_mem = mem; + ::close(rf.fd); + + if (!out_tex[0].is_valid()) { + for (int i = 0; i < 2; i++) { + Ref fmt; + fmt.instantiate(); + fmt->set_texture_type(RenderingDevice::TEXTURE_TYPE_2D); + fmt->set_format(RenderingDevice::DATA_FORMAT_B8G8R8A8_UNORM); + fmt->set_width(rf.w); + fmt->set_height(rf.h); + fmt->set_depth(1); + fmt->set_array_layers(1); + fmt->set_mipmaps(1); + fmt->set_usage_bits(RenderingDevice::TEXTURE_USAGE_SAMPLING_BIT | + RenderingDevice::TEXTURE_USAGE_CAN_COPY_TO_BIT | + RenderingDevice::TEXTURE_USAGE_CAN_COPY_FROM_BIT); + Ref v; + v.instantiate(); + out_tex[i] = rd->texture_create(fmt, v, TypedArray()); + } + if (!out_tex[0].is_valid()) { + fprintf(stderr, "[gdvapi] out_tex create failed\n"); + return; + } + } + + RID wrap = rd->texture_create_from_extension( + RenderingDevice::TEXTURE_TYPE_2D, + RenderingDevice::DATA_FORMAT_B8G8R8A8_UNORM, + RenderingDevice::TEXTURE_SAMPLES_1, + RenderingDevice::TEXTURE_USAGE_SAMPLING_BIT | + RenderingDevice::TEXTURE_USAGE_CAN_COPY_FROM_BIT, + uint64_t(img), rf.w, rf.h, 1, 1); + const int w = (draw_idx + 1) % 2; + if (wrap.is_valid()) { + rd->texture_copy(wrap, out_tex[w], + godot::Vector3(0, 0, 0), godot::Vector3(0, 0, 0), + godot::Vector3(rf.w, rf.h, 1), + 0, 0, 0, 0); + rd->free_rid(wrap); + } + if (!draw_tex[w].is_valid()) { + draw_tex[w].instantiate(); + draw_tex[w]->set_texture_rd_rid(out_tex[w]); + } + draw_idx = w; + present_wall_sec = std::chrono::duration( + std::chrono::steady_clock::now() - wall_base).count(); + if (!have_presentable) { + have_presentable = true; + } + if (getenv("GDVAPI_DEBUG_PACING")) { + using namespace std::chrono; + double el = duration(steady_clock::now() - wall_base).count(); + fprintf(stderr, "[gdvapi] present #%d pts=%.3f wall=%.3f\n", + present_count + 1, rf.pts_sec, el); + } + present_count++; } GDVAPIVideoStream::~GDVAPIVideoStream() { @@ -206,104 +427,38 @@ void GDVAPIVideoStream::_notification(int p_what) { } } -void GDVAPIVideoStream::texture_init() { - RenderingDevice *rd = RenderingServer::get_singleton()->get_rendering_device(); - if (!rd) - return; - - if (texture_rd_rid.is_valid()) { - rd->free_rid(texture_rd_rid); - } - - Ref format; - format.instantiate(); - format->set_texture_type(RenderingDevice::TEXTURE_TYPE_2D); - format->set_format(RenderingDevice::DATA_FORMAT_R8G8B8A8_UNORM); - format->set_width(width); - format->set_height(height); - format->set_depth(1); - format->set_array_layers(1); - format->set_mipmaps(1); - format->set_usage_bits(RenderingDevice::TEXTURE_USAGE_SAMPLING_BIT | - RenderingDevice::TEXTURE_USAGE_CAN_UPDATE_BIT); - - PackedByteArray byte_array; - byte_array.resize(width * height * 4); - - TypedArray data; - data.push_back(byte_array); - - Ref view; - view.instantiate(); - - texture_rd_rid = rd->texture_create(format, view, data); - - if (texture_rd_rid.is_valid()) { - godot_texture.instantiate(); - godot_texture->set_texture_rd_rid(texture_rd_rid); - } -} - void GDVAPIVideoStream::cleanup() { set_process(false); + stop_worker(); + RenderingDevice *rd = RenderingServer::get_singleton()->get_rendering_device(); + if (present_image && rd) { + gdvapi::destroy_imported_image(vk_dev, present_image, present_mem); + present_image = VK_NULL_HANDLE; + present_mem = VK_NULL_HANDLE; + } + for (int i = 0; i < 2; i++) { + if (out_tex[i].is_valid()) { + rd->free_rid(out_tex[i]); + out_tex[i] = RID(); + } + } + va_decoder.close(); + stream_eof = true; godot_texture.unref(); - if (rd && texture_rd_rid.is_valid()) { - rd->free_rid(texture_rd_rid); - texture_rd_rid = RID(); - } - - if (sws_ctx) { - sws_freeContext(sws_ctx); - sws_ctx = nullptr; - } - if (frame) { - av_frame_free(&frame); - } - if (sw_frame) { - av_frame_free(&sw_frame); - } - if (packet) { - av_packet_free(&packet); - } - if (codec_ctx) { - avcodec_free_context(&codec_ctx); - } - if (hw_device_ctx) { - av_buffer_unref(&hw_device_ctx); - hw_device_ctx = nullptr; - } - if (fmt_ctx) { - avformat_close_input(&fmt_ctx); - } -} - -void GDVAPIVideoStream::_draw() { - UtilityFunctions::print("_draw called"); - if (godot_texture.is_valid()) { - draw_texture(godot_texture, Point2(0, 0)); - } else { - Rect2 rect = Rect2(0, 0, desiredWidth, desiredHeight); - - Color color = Color(1.0, 1.0, 1.0, 1.0); - - draw_rect(rect, color); - } } void GDVAPIVideoStream::set_desired_width(const int v) { if (desiredWidth != v) { desiredWidth = v; - queue_redraw(); } } void GDVAPIVideoStream::set_desired_height(const int v) { if (desiredHeight != v) { desiredHeight = v; - queue_redraw(); } } @@ -316,9 +471,72 @@ int GDVAPIVideoStream::get_desired_height() const { return desiredHeight; } +void GDVAPIVideoStream::set_reorder_buffer_ms(const int v) { + decoder_opts.max_delay_us = v * 1000; +} +int GDVAPIVideoStream::get_reorder_buffer_ms() const { + return decoder_opts.max_delay_us / 1000; +} +void GDVAPIVideoStream::set_demux_fifo_bytes(const int v) { + decoder_opts.demux_fifo_bytes = v; +} +int GDVAPIVideoStream::get_demux_fifo_bytes() const { + return decoder_opts.demux_fifo_bytes; +} +void GDVAPIVideoStream::set_socket_buffer_bytes(const int v) { + decoder_opts.socket_buffer_bytes = v; +} +int GDVAPIVideoStream::get_socket_buffer_bytes() const { + return decoder_opts.socket_buffer_bytes; +} +void GDVAPIVideoStream::set_reorder_queue_size(const int v) { + decoder_opts.reorder_queue_size = v; +} +int GDVAPIVideoStream::get_reorder_queue_size() const { + return decoder_opts.reorder_queue_size; +} +void GDVAPIVideoStream::set_io_timeout_ms(const int v) { + decoder_opts.io_timeout_us = v * 1000; +} +int GDVAPIVideoStream::get_io_timeout_ms() const { + return decoder_opts.io_timeout_us / 1000; +} +void GDVAPIVideoStream::set_queue_depth(const int v) { + queue_max = v < 1 ? 1 : (v > 8 ? 8 : v); +} +int GDVAPIVideoStream::get_queue_depth() const { + return queue_max; +} +void GDVAPIVideoStream::set_frame_latency_ms(const int v) { + int c = v < 5 ? 5 : v; + max_frame_latency = c / 1000.0; +} +int GDVAPIVideoStream::get_frame_latency_ms() const { + return (int)(max_frame_latency * 1000.0); +} +void GDVAPIVideoStream::set_vpp_half_res(const bool v) { + decoder_opts.vpp_div = v ? 2 : 1; +} +bool GDVAPIVideoStream::get_vpp_half_res() const { + return decoder_opts.vpp_div == 2; +} + void GDVAPIVideoStream::_ready() { UtilityFunctions::print("_ready called"); - queue_redraw(); + + RenderingDevice *rd = RenderingServer::get_singleton()->get_rendering_device(); + if (rd) { + uint64_t vk_dev = rd->get_driver_resource( + RenderingDevice::DRIVER_RESOURCE_LOGICAL_DEVICE, RID(), 0); + uint64_t vk_phys = rd->get_driver_resource( + RenderingDevice::DRIVER_RESOURCE_PHYSICAL_DEVICE, RID(), 0); + uint64_t vk_inst = rd->get_driver_resource( + RenderingDevice::DRIVER_RESOURCE_VULKAN_INSTANCE, RID(), 0); + UtilityFunctions::print("[gdvapi] VkDevice=", vk_dev, + " VkPhysicalDevice=", vk_phys, + " VkInstance=", vk_inst); + } else + UtilityFunctions::print("[gdvapi] no RenderingDevice"); } diff --git a/src/gdvapivideostream_class.h b/src/gdvapivideostream_class.h index b06e371..15d7935 100644 --- a/src/gdvapivideostream_class.h +++ b/src/gdvapivideostream_class.h @@ -1,8 +1,18 @@ - #pragma once +#pragma once -#include "godot_cpp/classes/control.hpp" +#include "godot_cpp/classes/node.hpp" #include "godot_cpp/classes/texture2drd.hpp" #include "godot_cpp/variant/rid.hpp" +#include "vaapi_decoder.h" + +#include + +#include +#include +#include +#include +#include +#include extern "C" { #include @@ -13,32 +23,63 @@ extern "C" { namespace godot { -class GDVAPIVideoStream : public Control { - GDCLASS(GDVAPIVideoStream, Control) +class GDVAPIVideoStream : public Node { + GDCLASS(GDVAPIVideoStream, Node) private: - AVFormatContext *fmt_ctx = nullptr; - AVCodecContext *codec_ctx = nullptr; - const AVCodec *codec = nullptr; - AVFrame *frame = nullptr; - AVPacket *packet = nullptr; - struct SwsContext *sws_ctx = nullptr; - int video_stream_idx = -1; - int width = 256; - int height = 256; - AVBufferRef *hw_device_ctx = nullptr; - AVFrame *sw_frame = nullptr; + struct ReadyFrame { + int fd = -1; + int w = 0, h = 0; + double pts_sec = 0.0; + uint32_t drm_fourcc = 0; + int slot = -1; + }; + std::thread worker_thr; + std::mutex q_mu; + std::condition_variable q_cv; + std::deque ready_q; + std::atomic slot_acked[gdvapi::VAAPIDecoder::kRgbaPool] = {}; + int queue_max = 4; // bounded FIFO depth + double max_frame_latency = 0.5; // drop frames older than this + bool worker_stop = false; + bool worker_finished = false; + std::chrono::steady_clock::time_point wall_base; + double worker_fps = 30.0; + bool source_live = false; // RTP/SDP: retry on gaps instead of EOF - double frame_delay = 0.0; - double time_accumulator = 0.0; + gdvapi::VAAPIDecoder va_decoder; + gdvapi::DecoderOptions decoder_opts; + int vpp_slot = 0; // rotates 0..kRgbaPool-1 + std::string worker_src; - RID texture_rd_rid; + VkDevice vk_dev = VK_NULL_HANDLE; + VkInstance vk_inst = VK_NULL_HANDLE; + VkImage present_image = VK_NULL_HANDLE; + VkDeviceMemory present_mem = VK_NULL_HANDLE; + RID out_tex[2]; + int out_tex_idx = 0; + Ref draw_tex[2]; + int draw_idx = 0; + double present_wall_sec = -1.0; // for debug overlay + double last_pts = -1.0; // monotonic order gate + double pts_base = -1.0; // first-presented PTS + + bool stream_started = false; + bool stream_eof = false; + bool have_presentable = false; + String stream_source = "udp://127.0.0.1:5000"; // set via begin() + int present_count = 0; + bool size_pending = false; + int pending_w = 0, pending_h = 0; Ref godot_texture; - int desiredWidth = 256; int desiredHeight = 256; + void worker_loop(); + void stop_worker(); + void consume_ready(); + void cleanup(); void texture_init(); @@ -50,11 +91,26 @@ public: ~GDVAPIVideoStream() override; void begin(const Variant &source); + Ref get_texture() const { return draw_tex[draw_idx]; } + + void set_reorder_buffer_ms(const int v); int get_reorder_buffer_ms() const; + void set_demux_fifo_bytes(const int v); int get_demux_fifo_bytes() const; + void set_socket_buffer_bytes(const int v);int get_socket_buffer_bytes() const; + void set_reorder_queue_size(const int v); int get_reorder_queue_size() const; + void set_io_timeout_ms(const int v); int get_io_timeout_ms() const; + void set_queue_depth(const int v); int get_queue_depth() const; + void set_frame_latency_ms(const int v); int get_frame_latency_ms() const; + void set_vpp_half_res(const bool v); bool get_vpp_half_res() const; + + int get_frame_count() const { return present_count; } + bool has_frame() const { return have_presentable; } + int get_stream_width() const { return pending_w; } + int get_stream_height() const { return pending_h; } + void _process(double delta) override; void _notification(int p_what); - void _draw() override; void _ready() override; void set_desired_width(const int v); diff --git a/src/register_types.cpp b/src/register_types.cpp index 37ad78c..5b1c548 100644 --- a/src/register_types.cpp +++ b/src/register_types.cpp @@ -1,4 +1,7 @@ #include "register_types.h" +#include "vulkan_detour.h" + +#include #include #include @@ -10,18 +13,19 @@ using namespace godot; void initialize_gdextension_types(ModuleInitializationLevel p_level) { - if (p_level != MODULE_INITIALIZATION_LEVEL_SCENE) { + fprintf(stderr, "[gdvapi] initialize level=%d\n", (int)p_level); + if (p_level == MODULE_INITIALIZATION_LEVEL_CORE) + gdvapi::install_vk_create_device_hook(); + if (p_level != MODULE_INITIALIZATION_LEVEL_SCENE) return; - } - + GDREGISTER_RUNTIME_CLASS(GDVAPIVideoStream); } void uninitialize_gdextension_types(ModuleInitializationLevel p_level) { - if (p_level != MODULE_INITIALIZATION_LEVEL_SCENE) { + if (p_level != MODULE_INITIALIZATION_LEVEL_SCENE) return; - } } extern "C" { @@ -29,7 +33,7 @@ GDExtensionBool GDE_EXPORT gdvapi_library_init(GDExtensionInterfaceGetProcAddres GDExtensionBinding::InitObject init_obj(p_get_proc_address, p_library, r_initialization); init_obj.register_initializer(initialize_gdextension_types); init_obj.register_terminator(uninitialize_gdextension_types); - init_obj.set_minimum_library_initialization_level(MODULE_INITIALIZATION_LEVEL_SCENE); + init_obj.set_minimum_library_initialization_level(MODULE_INITIALIZATION_LEVEL_CORE); return init_obj.init(); } diff --git a/src/vaapi_decoder.cpp b/src/vaapi_decoder.cpp new file mode 100644 index 0000000..4323403 --- /dev/null +++ b/src/vaapi_decoder.cpp @@ -0,0 +1,442 @@ +#include "vaapi_decoder.h" + +#include +#include +#include + +extern "C" { +#include +#include +#include +} + +namespace gdvapi { + +static enum AVPixelFormat get_hw_format(AVCodecContext *ctx, + const enum AVPixelFormat *pix_fmts) { + (void)ctx; + for (const enum AVPixelFormat *p = pix_fmts; *p != -1; p++) + if (*p == AV_PIX_FMT_VAAPI) + return *p; + return AV_PIX_FMT_NONE; +} + +VAAPIDecoder::~VAAPIDecoder() { + close(); +} + +bool VAAPIDecoder::open(const char *path, const DecoderOptions &o) { + close(); + + memset(&pkt, 0, sizeof(pkt)); + + AVDictionary *opts = nullptr; + av_dict_set(&opts, "protocol_whitelist", "file,udp,rtp,unix", 0); + char buf[32]; + snprintf(buf, sizeof(buf), "%d", o.io_timeout_us); + av_dict_set(&opts, "timeout", buf, 0); + snprintf(buf, sizeof(buf), "%d", o.analyze_duration_us); + av_dict_set(&opts, "analyzeduration", buf, 0); + snprintf(buf, sizeof(buf), "%d", o.probe_bytes); + av_dict_set(&opts, "probesize", buf, 0); + snprintf(buf, sizeof(buf), "%d", o.demux_fifo_bytes); + av_dict_set(&opts, "rtbufsize", buf, 0); + snprintf(buf, sizeof(buf), "%d", o.max_delay_us); + av_dict_set(&opts, "max_delay", buf, 0); + snprintf(buf, sizeof(buf), "%d", o.socket_buffer_bytes); + av_dict_set(&opts, "buffer_size", buf, 0); + snprintf(buf, sizeof(buf), "%d", o.reorder_queue_size); + av_dict_set(&opts, "reorder_queue_size", buf, 0); + vpp_div_ = o.vpp_div > 0 ? o.vpp_div : 1; + if (avformat_open_input(&fmt_ctx, path, nullptr, &opts) < 0) { + av_dict_free(&opts); + fprintf(stderr, "[gdvapi] vaapi open input failed: %s\n", path); + return false; + } + av_dict_free(&opts); + if (avformat_find_stream_info(fmt_ctx, nullptr) < 0) { + fprintf(stderr, "[gdvapi] find stream info failed\n"); + close(); + return false; + } + (void)0; + + video_stream_idx = -1; + for (int i = 0; i < fmt_ctx->nb_streams; i++) { + if (fmt_ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { + video_stream_idx = i; + break; + } + } + if (video_stream_idx < 0) { + fprintf(stderr, "[gdvapi] no video stream\n"); + close(); + return false; + } + + const AVCodec *dec = avcodec_find_decoder( + fmt_ctx->streams[video_stream_idx]->codecpar->codec_id); + if (!dec) { + fprintf(stderr, "[gdvapi] no decoder\n"); + close(); + return false; + } + + codec_ctx = avcodec_alloc_context3(dec); + if (!codec_ctx) { + close(); + return false; + } + if (avcodec_parameters_to_context(codec_ctx, + fmt_ctx->streams[video_stream_idx]->codecpar) < 0) { + fprintf(stderr, "[gdvapi] params->ctx failed\n"); + close(); + return false; + } + + if (av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VAAPI, + nullptr, nullptr, 0) < 0) { + fprintf(stderr, "[gdvapi] vaapi device create failed\n"); + close(); + return false; + } + codec_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx); + codec_ctx->get_format = get_hw_format; + + if (avcodec_open2(codec_ctx, dec, nullptr) < 0) { + fprintf(stderr, "[gdvapi] codec open failed\n"); + close(); + return false; + } + + if (codec_ctx->hw_frames_ctx) { + AVHWFramesContext *fc = (AVHWFramesContext *)codec_ctx->hw_frames_ctx->data; + fc->initial_pool_size = 16; + if (av_hwframe_ctx_init(codec_ctx->hw_frames_ctx) >= 0) + fprintf(stderr, "[gdvapi] hw frames pool set to 16\n"); + } + + frame = av_frame_alloc(); + eof = false; + wait_key = true; + + video_width = codec_ctx->width; + video_height = codec_ctx->height; + + AVRational fr = fmt_ctx->streams[video_stream_idx]->avg_frame_rate; + if (fr.num > 0 && fr.den > 0) + video_fps = av_q2d(fr); + + fprintf(stderr, "[gdvapi] vaapi decoder opened: %s %dx%d @%.1ffps\n", + avcodec_get_name(codec_ctx->codec_id), video_width, video_height, + video_fps); + return true; +} + +bool VAAPIDecoder::next_frame(AVFrame *&out_frame) { + out_frame = nullptr; + if (!is_open()) + return false; + + while (true) { + int r = avcodec_receive_frame(codec_ctx, frame); + if (r == 0) { + if (frame->format == AV_PIX_FMT_VAAPI && frame->hw_frames_ctx) { + out_frame = av_frame_clone(frame); + av_frame_unref(frame); + return out_frame != nullptr; + } + av_frame_unref(frame); + continue; + } + if (r == AVERROR(EAGAIN)) + break; + if (r == AVERROR_EOF) { + eof = true; + return false; + } + avcodec_flush_buffers(codec_ctx); + wait_key = true; + break; + } + + while (!eof && av_read_frame(fmt_ctx, &pkt) >= 0) { + if (pkt.stream_index != video_stream_idx) { + av_packet_unref(&pkt); + continue; + } + if (wait_key && !(pkt.flags & AV_PKT_FLAG_KEY)) { + av_packet_unref(&pkt); + continue; + } + wait_key = false; + int sr = avcodec_send_packet(codec_ctx, &pkt); + av_packet_unref(&pkt); + if (sr < 0) { + if (getenv("GDVAPI_DEBUG_DECODE")) + fprintf(stderr, "[gdvapi] send_packet rc=%d\n", sr); + wait_key = true; + continue; + } + { + int it = 0; + while (true) { + it++; + int r = avcodec_receive_frame(codec_ctx, frame); + if (r == 0) { + if (frame->format == AV_PIX_FMT_VAAPI && frame->hw_frames_ctx) { + out_frame = av_frame_clone(frame); + av_frame_unref(frame); + return out_frame != nullptr; + } + av_frame_unref(frame); + continue; + } + if (r == AVERROR(EAGAIN)) { + if (getenv("GDVAPI_DEBUG_DECODE") && it == 1) + fprintf(stderr, "[gdvapi] recv EAGAIN (need more packets)\n"); + break; + } + if (r == AVERROR_EOF) { + eof = true; + return false; + } + avcodec_flush_buffers(codec_ctx); + wait_key = true; + if (getenv("GDVAPI_DEBUG_DECODE")) + fprintf(stderr, "[gdvapi] recv rc=%d (flush)\n", r); + break; + } + } + continue; + + while (true) { + int r = avcodec_receive_frame(codec_ctx, frame); + if (r == 0) { + if (frame->format == AV_PIX_FMT_VAAPI && frame->hw_frames_ctx) { + out_frame = av_frame_clone(frame); + av_frame_unref(frame); + return out_frame != nullptr; + } + av_frame_unref(frame); + continue; + } + if (r == AVERROR(EAGAIN)) + break; + if (r == AVERROR_EOF) { + eof = true; + return false; + } + break; + } + } + eof = true; + return false; +} + + +static void fill_dma_out(const VADRMPRIMESurfaceDescriptor &desc, DecodedFrame &out) { + out.width = desc.width; + out.height = desc.height; + out.fourcc = desc.fourcc; + if (desc.num_layers > 0) { + out.drm_fourcc = desc.layers[0].drm_format; + out.pitch = desc.layers[0].pitch[0]; + out.offset = desc.layers[0].offset[0]; + } + out.drm_format_modifier = desc.objects[0].drm_format_modifier; + out.fd = desc.objects[0].fd; +} + +double VAAPIDecoder::pts_sec_of(const AVFrame *f) const { + if (!f || f->pts == AV_NOPTS_VALUE || !fmt_ctx || video_stream_idx < 0) + return 0.0; + AVRational tb = fmt_ctx->streams[video_stream_idx]->time_base; + if (tb.num <= 0 || tb.den <= 0) + return 0.0; + return av_q2d(tb) * (double)f->pts; +} + +bool VAAPIDecoder::vpp_init() { + if (vpp_ready) + return true; + AVHWDeviceContext *dctx = (AVHWDeviceContext *)hw_device_ctx->data; + AVVAAPIDeviceContext *va = (AVVAAPIDeviceContext *)dctx->hwctx; + VADisplay dpy = va->display; + + VAConfigAttrib rt = { VAConfigAttribRTFormat, VA_RT_FORMAT_RGB32 }; + VAStatus st = vaCreateConfig(dpy, VAProfileNone, VAEntrypointVideoProc, + &rt, 1, &vpp_config); + if (st != VA_STATUS_SUCCESS) { + fprintf(stderr, "[gdvapi] vpp vaCreateConfig failed: %d (%s)\n", + st, vaErrorStr(st)); + return false; + } + int vpp_div = vpp_div_ > 0 ? vpp_div_ : 1; + vpp_out_w = video_width / vpp_div; + vpp_out_h = video_height / vpp_div; + if (vpp_out_w < 1) vpp_out_w = 1; + if (vpp_out_h < 1) vpp_out_h = 1; + st = vaCreateContext(dpy, vpp_config, vpp_out_w, vpp_out_h, + VA_PROGRESSIVE, 0, 0, &vpp_ctx); + if (st != VA_STATUS_SUCCESS) { + fprintf(stderr, "[gdvapi] vpp vaCreateContext failed: %d (%s)\n", + st, vaErrorStr(st)); + return false; + } + st = vaCreateSurfaces(dpy, VA_RT_FORMAT_RGB32, + vpp_out_w, vpp_out_h, rgba_pool, kRgbaPool, nullptr, 0); + if (st != VA_STATUS_SUCCESS) { + fprintf(stderr, "[gdvapi] vpp vaCreateSurfaces failed: %d (%s)\n", + st, vaErrorStr(st)); + return false; + } + vpp_ready = true; + fprintf(stderr, "[gdvapi] VPP init ok: %dx%d RGB32 target (out %dx%d)\n", + video_width, video_height, vpp_out_w, vpp_out_h); + return true; +} + +bool VAAPIDecoder::convert_frame_to_rgba_slot(const AVFrame *nv12_frame, + int slot, DecodedFrame &out) { + if (!nv12_frame || nv12_frame->format != AV_PIX_FMT_VAAPI || + slot < 0 || slot >= kRgbaPool) + return false; + if (video_width == 0 || video_height == 0) { + video_width = nv12_frame->width; + video_height = nv12_frame->height; + } + if (!vpp_init()) + return false; + + AVHWDeviceContext *dctx = (AVHWDeviceContext *)hw_device_ctx->data; + AVVAAPIDeviceContext *va = (AVVAAPIDeviceContext *)dctx->hwctx; + VADisplay dpy = va->display; + + VASurfaceID src_surf = (VASurfaceID)(intptr_t)nv12_frame->data[0]; + + VAProcPipelineParameterBuffer param; + memset(¶m, 0, sizeof(param)); + param.surface = src_surf; // src (NV12) + param.surface_region = nullptr; // whole source + VARectangle out_reg = { 0, 0, (uint16_t)vpp_out_w, (uint16_t)vpp_out_h }; + param.output_region = &out_reg; // whole target (RGB32) + + VABufferID buf = VA_INVALID_ID; + VAStatus st = vaCreateBuffer(dpy, vpp_ctx, VAProcPipelineParameterBufferType, + sizeof(param), 1, ¶m, &buf); + if (st != VA_STATUS_SUCCESS) { + fprintf(stderr, "[gdvapi] vpp vaCreateBuffer failed: %d (%s)\n", + st, vaErrorStr(st)); + return false; + } + + st = vaBeginPicture(dpy, vpp_ctx, rgba_pool[slot]); + if (st == VA_STATUS_SUCCESS) + st = vaRenderPicture(dpy, vpp_ctx, &buf, 1); + if (st == VA_STATUS_SUCCESS) + st = vaEndPicture(dpy, vpp_ctx); + if (st == VA_STATUS_SUCCESS) + st = vaSyncSurface(dpy, rgba_pool[slot]); + vaDestroyBuffer(dpy, buf); + if (st != VA_STATUS_SUCCESS) { + fprintf(stderr, "[gdvapi] vpp render failed: %d (%s)\n", st, vaErrorStr(st)); + return false; + } + + VADRMPRIMESurfaceDescriptor desc; + st = vaExportSurfaceHandle(dpy, rgba_pool[slot], + VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2, + VA_EXPORT_SURFACE_READ_ONLY | VA_EXPORT_SURFACE_COMPOSED_LAYERS, + &desc); + if (st != VA_STATUS_SUCCESS) { + fprintf(stderr, "[gdvapi] vpp export failed: %d (%s)\n", st, vaErrorStr(st)); + return false; + } + fill_dma_out(desc, out); + out.pts = nv12_frame->pts; + if (fmt_ctx && video_stream_idx >= 0) { + AVRational tb = fmt_ctx->streams[video_stream_idx]->time_base; + if (tb.num > 0 && tb.den > 0 && out.pts != AV_NOPTS_VALUE) + out.pts_sec = av_q2d(tb) * (double)out.pts; + } + return true; +} + +bool VAAPIDecoder::export_surface(const AVFrame *f, DecodedFrame &out) { + out = DecodedFrame(); + if (!f || f->format != AV_PIX_FMT_VAAPI || !f->hw_frames_ctx) + return false; + + VASurfaceID surf = (VASurfaceID)(intptr_t)f->data[0]; + + AVHWDeviceContext *dctx = (AVHWDeviceContext *)hw_device_ctx->data; + AVVAAPIDeviceContext *va = (AVVAAPIDeviceContext *)dctx->hwctx; + VADisplay dpy = va->display; + + VADRMPRIMESurfaceDescriptor desc; + VAStatus st = vaExportSurfaceHandle(dpy, surf, + VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2, + VA_EXPORT_SURFACE_READ_ONLY | VA_EXPORT_SURFACE_COMPOSED_LAYERS, + &desc); + if (st != VA_STATUS_SUCCESS) { + fprintf(stderr, "[gdvapi] vaExportSurfaceHandle(%u) failed: %d (%s)\n", + (unsigned)surf, st, vaErrorStr(st)); + return false; + } + + out.width = desc.width; + out.height = desc.height; + out.fourcc = desc.fourcc; + if (desc.num_layers > 0) { + out.drm_fourcc = desc.layers[0].drm_format; + out.pitch = desc.layers[0].pitch[0]; + out.offset = desc.layers[0].offset[0]; + if (desc.layers[0].num_planes > 1) { + out.uv_pitch = desc.layers[0].pitch[1]; + out.uv_offset = desc.layers[0].offset[1]; + } + } + out.drm_format_modifier = desc.objects[0].drm_format_modifier; + out.fd = desc.objects[0].fd; + out.pts = f->pts; + return true; +} + +void VAAPIDecoder::close() { + if (vpp_ready) { + AVHWDeviceContext *dctx = (AVHWDeviceContext *)hw_device_ctx->data; + AVVAAPIDeviceContext *va = (AVVAAPIDeviceContext *)dctx->hwctx; + VADisplay dpy = va->display; + for (int i = 0; i < kRgbaPool; i++) { + if (rgba_pool[i] != VA_INVALID_ID) { + VASurfaceID t = rgba_pool[i]; + vaDestroySurfaces(dpy, &t, 1); + rgba_pool[i] = VA_INVALID_ID; + } + } + if (vpp_ctx != VA_INVALID_ID) { vaDestroyContext(dpy, vpp_ctx); vpp_ctx = VA_INVALID_ID; } + if (vpp_config != VA_INVALID_ID) { vaDestroyConfig(dpy, vpp_config); vpp_config = VA_INVALID_ID; } + vpp_ready = false; + } + if (frame) { + av_frame_free(&frame); + frame = nullptr; + } + if (codec_ctx) { + avcodec_free_context(&codec_ctx); + codec_ctx = nullptr; + } + if (hw_device_ctx) { + av_buffer_unref(&hw_device_ctx); + hw_device_ctx = nullptr; + } + if (fmt_ctx) { + avformat_close_input(&fmt_ctx); + fmt_ctx = nullptr; + } + video_stream_idx = -1; + eof = false; +} + +} diff --git a/src/vaapi_decoder.h b/src/vaapi_decoder.h new file mode 100644 index 0000000..1be2dcc --- /dev/null +++ b/src/vaapi_decoder.h @@ -0,0 +1,90 @@ +#pragma once +#include + +extern "C" { +#include +#include +#include +#include +#include +#include +#include +} + +namespace gdvapi { + +struct DecoderOptions { + int io_timeout_us = 1000000; // avio block timeout (dead-stream detection) + int analyze_duration_us = 500000; // find_stream_info analysis cap + int probe_bytes = 1000000; // probe-size cap + int demux_fifo_bytes = 33554432; // rtbufsize: 32 MB demux FIFO (keyframe bursts) + int socket_buffer_bytes = 8388608; // kernel SO_RCVBUF: 8 MB (RTP burst headroom) + int max_delay_us = 200000; // RTP reorder/jitter window (optimum on rig) + int reorder_queue_size = 2048; // RTP demuxer reorder queue (packets) + int vpp_div = 1; // VPP output downscale (1 = full res, 2 = bench) +}; + +struct DecodedFrame { + int width = 0; + int height = 0; + int fd = -1; // DRM dma-buf fd + uint32_t fourcc = 0; + uint32_t drm_fourcc = 0; // layers[0].drm_format from the export + double pts_sec = 0.0; // pts in seconds (stream time_base) + uint32_t pitch = 0; // layer[0].pitch[0] (Y plane) + uint32_t offset = 0; // layer[0].offset[0] (Y plane) + uint32_t uv_pitch = 0; // layer[0].pitch[1] (UV plane, real tiled) + uint32_t uv_offset = 0; // layer[0].offset[1] (UV plane, real tiled) + uint64_t drm_format_modifier = 0; + int64_t pts = AV_NOPTS_VALUE; + bool valid() const { return fd >= 0; } +}; + +class VAAPIDecoder { +public: + VAAPIDecoder() = default; + ~VAAPIDecoder(); + + bool open(const char *path, const DecoderOptions &o = DecoderOptions()); + + bool next_frame(AVFrame *&out_frame); + + bool export_surface(const AVFrame *frame, DecodedFrame &out); + + void close(); + + bool is_open() const { return fmt_ctx && codec_ctx; } + int width() const { return video_width; } + int height() const { return video_height; } + double fps() const { return video_fps; } + + bool vpp_ready = false; + VAConfigID vpp_config = VA_INVALID_ID; + VAContextID vpp_ctx = VA_INVALID_ID; + VASurfaceID rgba_pool[4] = { VA_INVALID_ID, VA_INVALID_ID, VA_INVALID_ID, VA_INVALID_ID }; + int vpp_out_w = 0, vpp_out_h = 0; + static constexpr int kRgbaPool = 4; + bool vpp_init(); + +public: + double pts_sec_of(const AVFrame *f) const; + + bool convert_frame_to_rgba_slot(const AVFrame *nv12_frame, int slot, + DecodedFrame &out); + +private: + AVFormatContext *fmt_ctx = nullptr; + AVCodecContext *codec_ctx = nullptr; + AVBufferRef *hw_device_ctx = nullptr; + AVFrame *frame = nullptr; + AVPacket pkt = {}; + int video_stream_idx = -1; + int video_width = 0; + int video_height = 0; + double video_fps = 30.0; + bool eof = false; + bool wait_key = true; // skip non-key packets until first IDR + int vpp_div_ = 1; // VPP output downscale from DecoderOptions +}; + +} diff --git a/src/vk_image_import.cpp b/src/vk_image_import.cpp new file mode 100644 index 0000000..7eb17ca --- /dev/null +++ b/src/vk_image_import.cpp @@ -0,0 +1,226 @@ +#include "vk_image_import.h" + +#include +#include + +namespace gdvapi { + +namespace { + +using PFN_vkGetDeviceProcAddr_t = PFN_vkVoidFunction(VKAPI_PTR *)(VkDevice, const char *); + +PFN_vkGetDeviceProcAddr_t resolve_get_device_proc_addr() { + static PFN_vkGetDeviceProcAddr_t fn = [] { + void *lib = dlopen("libvulkan.so.1", RTLD_NOW | RTLD_GLOBAL); + if (!lib) + return (PFN_vkGetDeviceProcAddr_t) nullptr; + return reinterpret_cast(dlsym(lib, "vkGetDeviceProcAddr")); + }(); + return fn; +} + +template +T load_dev_fn(VkDevice device, const char *name) { + auto gpa = resolve_get_device_proc_addr(); + if (!gpa) { + fprintf(stderr, "[gdvapi-import] vkGetDeviceProcAddr unavailable\n"); + return nullptr; + } + PFN_vkVoidFunction p = gpa(device, name); + if (!p) { + fprintf(stderr, "[gdvapi-import] missing device fn %s\n", name); + return nullptr; + } + return reinterpret_cast(p); +} + +VkFormat fourcc_to_format(uint32_t fourcc, uint32_t pitch) { + (void)pitch; + switch (fourcc) { + case 0x3231564e: // 'NV12' + return VK_FORMAT_G8_B8R8_2PLANE_420_UNORM; + case 0x34325241: // DRM_FORMAT_ARGB8888 'AR24' -> bytes B,G,R,A + case 0x34325258: // DRM_FORMAT_XRGB8888 'XR24' -> bytes B,G,R,A + return VK_FORMAT_B8G8R8A8_UNORM; + case 0x34324241: // DRM_FORMAT_ABGR8888 'AB24' -> bytes R,G,B,A + return VK_FORMAT_R8G8B8A8_UNORM; + default: + fprintf(stderr, "[gdvapi-import] unknown drm fourcc 0x%x\n", fourcc); + return VK_FORMAT_UNDEFINED; + } +} + +} + +VkImage import_dma_buf_image(VkDevice device, const DmaBufSurface &surf, + VkDeviceMemory *out_memory) { + if (!device || surf.fd < 0) + return VK_NULL_HANDLE; + if (out_memory) + *out_memory = VK_NULL_HANDLE; + + VkFormat format = fourcc_to_format( + surf.drm_fourcc ? surf.drm_fourcc : surf.fourcc, surf.pitch); + if (format == VK_FORMAT_UNDEFINED) { + fprintf(stderr, "[gdvapi-import] unsupported fourcc 0x%x\n", surf.fourcc); + return VK_NULL_HANDLE; + } + + auto dev_alloc = load_dev_fn(device, "vkAllocateMemory"); + auto dev_free = load_dev_fn(device, "vkFreeMemory"); + auto dev_create_image = load_dev_fn(device, "vkCreateImage"); + auto dev_destroy_image = load_dev_fn(device, "vkDestroyImage"); + auto dev_bind_mem = load_dev_fn(device, "vkBindImageMemory"); + auto dev_mem_req = load_dev_fn(device, "vkGetImageMemoryRequirements"); + auto dev_fd_props = load_dev_fn(device, "vkGetMemoryFdPropertiesKHR"); + if (!dev_alloc || !dev_free || !dev_create_image || !dev_destroy_image || + !dev_bind_mem || !dev_mem_req || !dev_fd_props) { + fprintf(stderr, "[gdvapi-import] required device functions unavailable\n"); + return VK_NULL_HANDLE; + } + + VkSubresourceLayout plane_layouts[2] = {}; + plane_layouts[0].offset = surf.offset; + plane_layouts[0].rowPitch = surf.pitch; + plane_layouts[0].size = 0; + plane_layouts[1].offset = surf.offset + (VkDeviceSize)surf.pitch * surf.height; + plane_layouts[1].rowPitch = surf.pitch; + plane_layouts[1].size = 0; + + const bool linear = (surf.drm_format_modifier == 0); + const bool multi_plane = (format == VK_FORMAT_G8_B8R8_2PLANE_420_UNORM); + + VkImageDrmFormatModifierExplicitCreateInfoEXT drm_info{}; + drm_info.sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_EXPLICIT_CREATE_INFO_EXT; + drm_info.drmFormatModifier = surf.drm_format_modifier; + drm_info.pPlaneLayouts = plane_layouts; + drm_info.drmFormatModifierPlaneCount = linear ? 0 : (multi_plane ? 2 : 1); + + VkExternalMemoryImageCreateInfo ext_info{}; + + ext_info.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO; + ext_info.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT; + if (!linear) + ext_info.pNext = &drm_info; + + VkImageFormatListCreateInfo fmt_list{}; + fmt_list.sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_LIST_CREATE_INFO; + if (multi_plane) { + VkFormat view_formats[3] = { + format, + VK_FORMAT_R8_UNORM, + VK_FORMAT_R8G8_UNORM, + }; + fmt_list.viewFormatCount = 3; + fmt_list.pViewFormats = view_formats; + fmt_list.pNext = &ext_info; + } + +VkImageCreateInfo image_info{}; + image_info.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; + const void *next = &ext_info; + if (multi_plane) + image_info.pNext = &fmt_list; + else + image_info.pNext = &ext_info; + + (void)next; + image_info.flags = multi_plane ? VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT : 0; + image_info.imageType = VK_IMAGE_TYPE_2D; + image_info.format = format; + image_info.extent = { surf.width, surf.height, 1 }; + image_info.mipLevels = 1; + image_info.arrayLayers = 1; + image_info.samples = VK_SAMPLE_COUNT_1_BIT; + image_info.tiling = linear ? VK_IMAGE_TILING_LINEAR + : VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT; + image_info.usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_SAMPLED_BIT; + image_info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + image_info.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + + VkImage image = VK_NULL_HANDLE; + VkResult r = dev_create_image(device, &image_info, nullptr, &image); + if (r != VK_SUCCESS) { + fprintf(stderr, "[gdvapi-import] vkCreateImage failed: %d\n", (int)r); + return VK_NULL_HANDLE; + } + + VkMemoryFdPropertiesKHR fd_props{}; + fd_props.sType = VK_STRUCTURE_TYPE_MEMORY_FD_PROPERTIES_KHR; + r = dev_fd_props(device, VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT, surf.fd, &fd_props); + if (r != VK_SUCCESS) { + fprintf(stderr, "[gdvapi-import] vkGetMemoryFdPropertiesKHR failed: %d\n", (int)r); + dev_destroy_image(device, image, nullptr); + return VK_NULL_HANDLE; + } + + VkMemoryRequirements mem_req{}; + dev_mem_req(device, image, &mem_req); + + uint32_t type_idx = 0xFFFFFFFF; + for (uint32_t i = 0; i < 32; i++) { + if ((fd_props.memoryTypeBits & mem_req.memoryTypeBits) & (1u << i)) { + type_idx = i; + break; + } + } + if (type_idx == 0xFFFFFFFF) { + fprintf(stderr, "[gdvapi-import] no compatible memory type\n"); + dev_destroy_image(device, image, nullptr); + return VK_NULL_HANDLE; + } + + VkImportMemoryFdInfoKHR import_info{}; + import_info.sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_FD_INFO_KHR; + import_info.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT; + import_info.fd = surf.fd; + + VkMemoryDedicatedAllocateInfo dedicated_info{}; + dedicated_info.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO; + dedicated_info.image = image; + + import_info.pNext = &dedicated_info; + + VkMemoryAllocateInfo alloc_info{}; + alloc_info.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; + alloc_info.pNext = &import_info; + alloc_info.allocationSize = mem_req.size; + alloc_info.memoryTypeIndex = type_idx; + + VkDeviceMemory memory = VK_NULL_HANDLE; + r = dev_alloc(device, &alloc_info, nullptr, &memory); + if (r != VK_SUCCESS) { + fprintf(stderr, "[gdvapi-import] vkAllocateMemory(import) failed: %d\n", (int)r); + dev_destroy_image(device, image, nullptr); + return VK_NULL_HANDLE; + } + + r = dev_bind_mem(device, image, memory, 0); + if (r != VK_SUCCESS) { + fprintf(stderr, "[gdvapi-import] vkBindImageMemory failed: %d\n", (int)r); + dev_free(device, memory, nullptr); + dev_destroy_image(device, image, nullptr); + return VK_NULL_HANDLE; + } + + if (out_memory) + *out_memory = memory; + + fprintf(stderr, "[gdvapi-import] imported dma-buf fd=%d -> VkImage %p size=%ux%u mod=0x%llx\n", + surf.fd, (void *)image, surf.width, surf.height, + (unsigned long long)surf.drm_format_modifier); + return image; +} + +void destroy_imported_image(VkDevice device, VkImage image, VkDeviceMemory mem) { + if (!device) + return; + auto destroy_image = load_dev_fn(device, "vkDestroyImage"); + auto free_memory = load_dev_fn(device, "vkFreeMemory"); + if (image && destroy_image) + destroy_image(device, image, nullptr); + if (mem && free_memory) + free_memory(device, mem, nullptr); +} + +} diff --git a/src/vk_image_import.h b/src/vk_image_import.h new file mode 100644 index 0000000..4e45fa4 --- /dev/null +++ b/src/vk_image_import.h @@ -0,0 +1,26 @@ +#pragma once +#include + +#include + +namespace gdvapi { + +struct DmaBufSurface { + int fd = -1; + uint32_t width = 0; + uint32_t height = 0; + uint64_t drm_format_modifier = 0; + uint32_t pitch = 0; + uint32_t offset = 0; + uint32_t uv_pitch = 0; // plane 1 (tiled chroma) pitch from descriptor + uint32_t uv_offset = 0; // plane 1 (tiled chroma) offset from descriptor + uint32_t fourcc = 0; // e.g. 0x3231564e "NV12" + uint32_t drm_fourcc = 0; // layers[0].drm_format from the descriptor +}; + +VkImage import_dma_buf_image(VkDevice device, const DmaBufSurface &surf, + VkDeviceMemory *out_memory); + +void destroy_imported_image(VkDevice device, VkImage image, VkDeviceMemory mem); + +} diff --git a/src/vulkan_detour.cpp b/src/vulkan_detour.cpp new file mode 100644 index 0000000..d2e9d4d --- /dev/null +++ b/src/vulkan_detour.cpp @@ -0,0 +1,193 @@ +#include "vulkan_detour.h" + +#include +#include +#include +#include +#include + +#include +#include + +#if !defined(__x86_64__) +#error "vulkan_detour: x86_64 only" +#endif + +namespace { + +const char *const kExtNames[] = { + "VK_KHR_external_memory", + "VK_KHR_external_memory_fd", + "VK_EXT_external_memory_dma_buf", + "VK_EXT_image_drm_format_modifier", + "VK_EXT_queue_family_foreign", +}; +constexpr int kExtCount = (int)(sizeof(kExtNames) / sizeof(kExtNames[0])); + +using VkCreateDeviceFn = VkResult(VKAPI_PTR *)(VkPhysicalDevice, const VkDeviceCreateInfo *, const VkAllocationCallbacks *, VkDevice *); + +std::atomic s_hook_installed{ false }; +uintptr_t s_target = 0; // address of vkCreateDevice in libvulkan +uintptr_t s_trampoline = 0; // executable copy of original prologue +constexpr int kPatchSize = 5; // E9 rel32 + +using PFN_vkEnumerateDeviceExtensionProperties = + VkResult(VKAPI_PTR *)(VkPhysicalDevice, const char *, uint32_t *, VkExtensionProperties *); +static PFN_vkEnumerateDeviceExtensionProperties s_enumerate_ext = nullptr; + +static bool make_rwx(uintptr_t addr, size_t len) { + long page = (long)addr & ~(long)(sysconf(_SC_PAGESIZE) - 1); + size_t total = len + (size_t)(addr - (uintptr_t)page); + return mprotect((void *)page, total, PROT_READ | PROT_WRITE | PROT_EXEC) == 0; +} + +static bool make_rx(uintptr_t addr, size_t len) { + long page = (long)addr & ~(long)(sysconf(_SC_PAGESIZE) - 1); + size_t total = len + (size_t)(addr - (uintptr_t)page); + return mprotect((void *)page, total, PROT_READ | PROT_EXEC) == 0; +} + +static void emit_rel32_jmp(unsigned char *dst, uintptr_t from, uintptr_t to) { + dst[0] = 0xE9; + uint32_t rel = (uint32_t)(to - (from + 5)); + memcpy(dst + 1, &rel, 4); +} + +static VkResult call_original(VkPhysicalDevice pd, const VkDeviceCreateInfo *ci, + const VkAllocationCallbacks *alloc, VkDevice *dev) { + typedef VkResult(VKAPI_PTR *OrigFn)(VkPhysicalDevice, const VkDeviceCreateInfo *, + const VkAllocationCallbacks *, VkDevice *); + OrigFn fn = (OrigFn)s_trampoline; + return fn(pd, ci, alloc, dev); +} + + +static bool device_has_extension(VkPhysicalDevice dev, const char *name) { + if (!s_enumerate_ext) + return false; + uint32_t count = 0; + s_enumerate_ext(dev, nullptr, &count, nullptr); + if (count == 0) + return false; + std::vector props(count); + s_enumerate_ext(dev, nullptr, &count, props.data()); + for (auto &p : props) + if (strcmp(p.extensionName, name) == 0) + return true; + return false; +} + +} + +namespace gdvapi { + +bool device_supports_external_extensions(VkPhysicalDevice physical_device) { + for (const char *name : kExtNames) + if (!device_has_extension(physical_device, name)) + return false; + return true; +} + +VKAPI_ATTR VkResult VKAPI_CALL hooked_vk_create_device( + VkPhysicalDevice physical_device, + const VkDeviceCreateInfo *p_create_info, + const VkAllocationCallbacks *p_allocator, + VkDevice *p_device) { + + if (!s_trampoline || !p_create_info) + return VK_ERROR_INITIALIZATION_FAILED; + + std::vector missing; + uint32_t existing_count = p_create_info->enabledExtensionCount + ? p_create_info->enabledExtensionCount : 0; + const char *const *existing = p_create_info->ppEnabledExtensionNames; + + for (const char *name : kExtNames) { + bool present = false; + for (uint32_t i = 0; i < existing_count; i++) { + if (existing && existing[i] && strcmp(existing[i], name) == 0) { + present = true; + break; + } + } + if (!present) + missing.push_back(name); + } + + if (missing.empty()) { + fprintf(stderr, "[gdvapi-hook] all external extensions already enabled\n"); + return call_original(physical_device, p_create_info, p_allocator, p_device); + } + + VkDeviceCreateInfo modified = *p_create_info; + static std::vector all; + all.clear(); + if (existing_count && existing) + all.insert(all.end(), existing, existing + existing_count); + all.reserve(existing_count + kExtCount); + for (const char *name : missing) + all.push_back(name); + + modified.enabledExtensionCount = (uint32_t)all.size(); + modified.ppEnabledExtensionNames = all.data(); + + fprintf(stderr, "[gdvapi-hook] injecting %zu external-memory extension(s)\n", missing.size()); + for (const char *name : missing) + fprintf(stderr, "[gdvapi-hook] + %s\n", name); + + VkResult r = call_original(physical_device, &modified, p_allocator, p_device); + fprintf(stderr, "[gdvapi-hook] vkCreateDevice -> %d\n", (int)r); + return r; +} + +bool install_vk_create_device_hook() { + if (s_hook_installed.exchange(true)) + return true; + + void *lib = dlopen("libvulkan.so.1", RTLD_NOW | RTLD_GLOBAL); + if (!lib) { + fprintf(stderr, "[gdvapi-hook] dlopen libvulkan.so.1: %s\n", dlerror()); + s_hook_installed = false; + return false; + } + + VkCreateDeviceFn orig_fn = (VkCreateDeviceFn)dlsym(lib, "vkCreateDevice"); + if (!orig_fn) { + fprintf(stderr, "[gdvapi-hook] dlsym vkCreateDevice: %s\n", dlerror()); + s_hook_installed = false; + return false; + } + + s_target = (uintptr_t)orig_fn; + + s_enumerate_ext = (PFN_vkEnumerateDeviceExtensionProperties)dlsym(lib, "vkEnumerateDeviceExtensionProperties"); + if (!s_enumerate_ext) + fprintf(stderr, "[gdvapi-hook] dlsym vkEnumerateDeviceExtensionProperties: %s\n", dlerror()); + + void *tramp = mmap(nullptr, sysconf(_SC_PAGESIZE), PROT_READ | PROT_WRITE | PROT_EXEC, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (tramp == MAP_FAILED) { + fprintf(stderr, "[gdvapi-hook] mmap trampoline failed\n"); + s_hook_installed = false; + return false; + } + uintptr_t t = (uintptr_t)tramp; + memcpy((void *)t, (void *)s_target, kPatchSize); + // jump from t+5 back to target+5 + emit_rel32_jmp((unsigned char *)(t + kPatchSize), t + kPatchSize, s_target + kPatchSize); + s_trampoline = t; + + if (!make_rwx(s_target, kPatchSize)) { + fprintf(stderr, "[gdvapi-hook] mprotect target failed\n"); + s_hook_installed = false; + munmap(tramp, sysconf(_SC_PAGESIZE)); + return false; + } + emit_rel32_jmp((unsigned char *)s_target, s_target, (uintptr_t)hooked_vk_create_device); + make_rx(s_target, kPatchSize); + + fprintf(stderr, "[gdvapi-hook] vkCreateDevice hook installed (trampoline @%p)\n", (void *)s_trampoline); + return true; +} + +} diff --git a/src/vulkan_detour.h b/src/vulkan_detour.h new file mode 100644 index 0000000..362d9a8 --- /dev/null +++ b/src/vulkan_detour.h @@ -0,0 +1,10 @@ +#pragma once +#include + +namespace gdvapi { + +bool install_vk_create_device_hook(); + +bool device_supports_external_extensions(VkPhysicalDevice physical_device); + +} diff --git a/stream-gst.py b/stream-gst.py new file mode 100644 index 0000000..b5744ea --- /dev/null +++ b/stream-gst.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +import sys +import gi +gi.require_version('Gst', '1.0') +from gi.repository import Gst, GLib +from datetime import datetime + +Gst.init(None) + +device = sys.argv[1] if len(sys.argv) > 1 else "/dev/video0" +port = sys.argv[2] if len(sys.argv) > 2 else "5000" + +pipeline = Gst.parse_launch(f""" + v4l2src device={device} io-mode=2 ! + videoconvert ! + textoverlay name=overlay halignment=left valignment=top font-desc="Monospace 36" ! + x265enc tune=zerolatency speed-preset=ultrafast option-string="bframes=0:keyint=30:scenecut=0" ! + h265parse config-interval=1 ! + rtph265pay config-interval=1 mtu=1400 ! + udpsink host=127.0.0.1 port={port} sync=false async=false +""") + +overlay = pipeline.get_by_name("overlay") + +def update_text(): + overlay.set_property("text", datetime.now().strftime("%H:%M:%S.%f")[:-3]) + return True + +GLib.timeout_add(5, update_text) + +pipeline.set_state(Gst.State.PLAYING) +loop = GLib.MainLoop() +try: + loop.run() +except KeyboardInterrupt: + pipeline.set_state(Gst.State.NULL)