From d14ccf251750a63cc9f84afa9c8df3c6e0420022 Mon Sep 17 00:00:00 2001 From: neatnoise Date: Tue, 21 Apr 2026 18:20:49 +0200 Subject: [PATCH] fix(linux): multi-GPU segfault + wlr GPU auto selection, DMA-BUF metadata planes and revert wlr vulkan support (#5030) --- src/platform/linux/vulkan_encode.cpp | 48 +++++++++++++++++++++++++--- src/platform/linux/wayland.cpp | 25 +++------------ src/platform/linux/wayland.h | 2 +- src/platform/linux/wlgrab.cpp | 12 ++----- 4 files changed, 50 insertions(+), 37 deletions(-) diff --git a/src/platform/linux/vulkan_encode.cpp b/src/platform/linux/vulkan_encode.cpp index ac194f98..7104609d 100644 --- a/src/platform/linux/vulkan_encode.cpp +++ b/src/platform/linux/vulkan_encode.cpp @@ -50,11 +50,20 @@ namespace vk { VkApplicationInfo app = {VK_STRUCTURE_TYPE_APPLICATION_INFO}; app.apiVersion = VK_API_VERSION_1_1; + + static const std::array instance_exts = {VK_EXT_PHYSICAL_DEVICE_DRM_EXTENSION_NAME}; VkInstanceCreateInfo ci = {VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO}; ci.pApplicationInfo = &app; + ci.enabledExtensionCount = instance_exts.size(); + ci.ppEnabledExtensionNames = instance_exts.data(); VkInstance inst = VK_NULL_HANDLE; if (vkCreateInstance(&ci, nullptr, &inst) != VK_SUCCESS) { - return {}; + // Retry without the extension for loaders that don't support it + ci.enabledExtensionCount = 0; + ci.ppEnabledExtensionNames = nullptr; + if (vkCreateInstance(&ci, nullptr, &inst) != VK_SUCCESS) { + return {}; + } } uint32_t count = 0; @@ -435,6 +444,26 @@ namespace vk { } } + /** + * @brief Query the driver-expected plane count for a format+modifier pair. + * @return Expected plane count, or 0 if unknown. + */ + int query_modifier_plane_count(VkFormat format, uint64_t modifier) { + VkDrmFormatModifierPropertiesListEXT mod_list = {VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_EXT}; + VkFormatProperties2 fmt_props2 = {VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2}; + fmt_props2.pNext = &mod_list; + vkGetPhysicalDeviceFormatProperties2(vk_dev.phys_dev, format, &fmt_props2); + std::vector mod_props(mod_list.drmFormatModifierCount); + mod_list.pDrmFormatModifierProperties = mod_props.data(); + vkGetPhysicalDeviceFormatProperties2(vk_dev.phys_dev, format, &fmt_props2); + for (const auto &mp : mod_props) { + if (mp.drmFormatModifier == modifier) { + return mp.drmFormatModifierPlaneCount; + } + } + return 0; + } + bool import_dmabuf(const egl::surface_descriptor_t &sd) { destroy_src_image(); @@ -459,12 +488,22 @@ namespace vk { }; VkImageTiling tiling; + auto [vk_format, vk_swizzle] = drm_fourcc_to_vk_format(sd.fourcc); + if (sd.modifier != DRM_FORMAT_MOD_INVALID) { - int plane_count = 0; + int dmabuf_planes = 0; for (int i = 0; i < 4 && sd.fds[i] >= 0; ++i) { + dmabuf_planes++; + } + + // Query driver for the expected plane count for this format+modifier. + // DMA-BUF exports may include extra metadata planes (e.g. AMD DCC). + int expected = query_modifier_plane_count(vk_format, sd.modifier); + int plane_count = (expected > 0 && expected <= dmabuf_planes) ? expected : dmabuf_planes; + + for (int i = 0; i < plane_count; ++i) { drm_layouts[i].offset = sd.offsets[i]; drm_layouts[i].rowPitch = sd.pitches[i]; - plane_count++; } drm_ci.drmFormatModifier = sd.modifier; drm_ci.drmFormatModifierPlaneCount = plane_count; @@ -475,8 +514,6 @@ namespace vk { tiling = VK_IMAGE_TILING_LINEAR; } - auto [vk_format, vk_swizzle] = drm_fourcc_to_vk_format(sd.fourcc); - VkImageCreateInfo img_ci = {VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO}; img_ci.pNext = &ext_ci; img_ci.imageType = VK_IMAGE_TYPE_2D; @@ -781,6 +818,7 @@ namespace vk { for (int i = 0; i < AV_NUM_DATA_POINTERS && vk_frame->img[i]; i++) { vk_frame->layout[i] = VK_IMAGE_LAYOUT_GENERAL; vk_frame->access[i] = VK_ACCESS_SHADER_WRITE_BIT; + vk_frame->queue_family[i] = vk_dev.compute_qf; } target.initialized = true; diff --git a/src/platform/linux/wayland.cpp b/src/platform/linux/wayland.cpp index 6d6195ff..f9336061 100644 --- a/src/platform/linux/wayland.cpp +++ b/src/platform/linux/wayland.cpp @@ -226,27 +226,10 @@ namespace wl { return true; } - // Find render node - drmDevice *devices[16]; - int n = drmGetDevices2(0, devices, 16); - if (n <= 0) { - BOOST_LOG(error) << "[wayland] No DRM devices found"sv; - return false; - } - - int drm_fd = -1; - for (int i = 0; i < n; i++) { - if (devices[i]->available_nodes & (1 << DRM_NODE_RENDER)) { - drm_fd = open(devices[i]->nodes[DRM_NODE_RENDER], O_RDWR); - if (drm_fd >= 0) { - break; - } - } - } - drmFreeDevices(devices, n); - + auto render_path = platf::resolve_render_device(); + int drm_fd = open(render_path.c_str(), O_RDWR); if (drm_fd < 0) { - BOOST_LOG(error) << "[wayland] Failed to open DRM render node"sv; + BOOST_LOG(error) << "[wayland] Failed to open DRM render node: "sv << render_path; return false; } @@ -374,7 +357,7 @@ namespace wl { } // Create GBM buffer - current_bo = gbm_bo_create(gbm_device, dmabuf_info.width, dmabuf_info.height, dmabuf_info.format, GBM_BO_USE_RENDERING | GBM_BO_USE_LINEAR); + current_bo = gbm_bo_create(gbm_device, dmabuf_info.width, dmabuf_info.height, dmabuf_info.format, GBM_BO_USE_RENDERING); if (!current_bo) { BOOST_LOG(error) << "Failed to create GBM buffer"sv; zwlr_screencopy_frame_v1_destroy(frame); diff --git a/src/platform/linux/wayland.h b/src/platform/linux/wayland.h index d765950f..286c247b 100644 --- a/src/platform/linux/wayland.h +++ b/src/platform/linux/wayland.h @@ -69,7 +69,6 @@ namespace wl { std::array frames; frame_t *current_frame; zwlr_screencopy_frame_v1_listener listener; - bool y_invert {false}; private: bool init_gbm(); @@ -96,6 +95,7 @@ namespace wl { struct gbm_device *gbm_device {nullptr}; struct gbm_bo *current_bo {nullptr}; struct wl_buffer *current_wl_buffer {nullptr}; + bool y_invert {false}; }; class monitor_t { diff --git a/src/platform/linux/wlgrab.cpp b/src/platform/linux/wlgrab.cpp index 917ea861..848e5121 100644 --- a/src/platform/linux/wlgrab.cpp +++ b/src/platform/linux/wlgrab.cpp @@ -11,7 +11,6 @@ #include "src/platform/common.h" #include "src/video.h" #include "vaapi.h" -#include "vulkan_encode.h" #include "wayland.h" using namespace std::literals; @@ -349,7 +348,6 @@ namespace wl { img->sd = current_frame->sd; img->frame_timestamp = current_frame->frame_timestamp; - img->y_invert = dmabuf.y_invert; // Prevent dmabuf from closing the file descriptors. std::fill_n(current_frame->sd.fds, 4, -1); @@ -379,12 +377,6 @@ namespace wl { } #endif -#ifdef SUNSHINE_BUILD_VULKAN - if (mem_type == platf::mem_type_e::vulkan) { - return vk::make_avcodec_encode_device_vram(width, height, 0, 0); - } -#endif - #ifdef SUNSHINE_BUILD_CUDA if (mem_type == platf::mem_type_e::cuda) { return cuda::make_avcodec_gl_encode_device(width, height, 0, 0); @@ -406,12 +398,12 @@ namespace wl { namespace platf { std::shared_ptr wl_display(mem_type_e hwdevice_type, const std::string &display_name, const video::config_t &config) { - if (hwdevice_type != platf::mem_type_e::system && hwdevice_type != platf::mem_type_e::vaapi && hwdevice_type != platf::mem_type_e::cuda && hwdevice_type != platf::mem_type_e::vulkan) { + if (hwdevice_type != platf::mem_type_e::system && hwdevice_type != platf::mem_type_e::vaapi && hwdevice_type != platf::mem_type_e::cuda) { BOOST_LOG(error) << "[wlgrab] Could not initialize display with the given hw device type."sv; return nullptr; } - if (hwdevice_type == platf::mem_type_e::vaapi || hwdevice_type == platf::mem_type_e::cuda || hwdevice_type == platf::mem_type_e::vulkan) { + if (hwdevice_type == platf::mem_type_e::vaapi || hwdevice_type == platf::mem_type_e::cuda) { auto wlr = std::make_shared(); if (wlr->init(hwdevice_type, display_name, config)) { return nullptr;