fix(linux): multi-GPU segfault + wlr GPU auto selection, DMA-BUF metadata planes and revert wlr vulkan support (#5030)

src/platform/linux/vulkan_encode.cpp

@@ -50,11 +50,20 @@ namespace vk {
 
     VkApplicationInfo app = {VK_STRUCTURE_TYPE_APPLICATION_INFO};
     app.apiVersion = VK_API_VERSION_1_1;
+
+    static const std::array<const char *, 1> instance_exts = {VK_EXT_PHYSICAL_DEVICE_DRM_EXTENSION_NAME};
     VkInstanceCreateInfo ci = {VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO};
     ci.pApplicationInfo = &app;
+    ci.enabledExtensionCount = instance_exts.size();
+    ci.ppEnabledExtensionNames = instance_exts.data();
     VkInstance inst = VK_NULL_HANDLE;
     if (vkCreateInstance(&ci, nullptr, &inst) != VK_SUCCESS) {
-      return {};
+      // Retry without the extension for loaders that don't support it
+      ci.enabledExtensionCount = 0;
+      ci.ppEnabledExtensionNames = nullptr;
+      if (vkCreateInstance(&ci, nullptr, &inst) != VK_SUCCESS) {
+        return {};
+      }
     }
 
     uint32_t count = 0;
@@ -435,6 +444,26 @@ namespace vk {
       }
     }
 
+    /**
+     * @brief Query the driver-expected plane count for a format+modifier pair.
+     * @return Expected plane count, or 0 if unknown.
+     */
+    int query_modifier_plane_count(VkFormat format, uint64_t modifier) {
+      VkDrmFormatModifierPropertiesListEXT mod_list = {VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_EXT};
+      VkFormatProperties2 fmt_props2 = {VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2};
+      fmt_props2.pNext = &mod_list;
+      vkGetPhysicalDeviceFormatProperties2(vk_dev.phys_dev, format, &fmt_props2);
+      std::vector<VkDrmFormatModifierPropertiesEXT> mod_props(mod_list.drmFormatModifierCount);
+      mod_list.pDrmFormatModifierProperties = mod_props.data();
+      vkGetPhysicalDeviceFormatProperties2(vk_dev.phys_dev, format, &fmt_props2);
+      for (const auto &mp : mod_props) {
+        if (mp.drmFormatModifier == modifier) {
+          return mp.drmFormatModifierPlaneCount;
+        }
+      }
+      return 0;
+    }
+
     bool import_dmabuf(const egl::surface_descriptor_t &sd) {
       destroy_src_image();
 
@@ -459,12 +488,22 @@ namespace vk {
       };
       VkImageTiling tiling;
 
+      auto [vk_format, vk_swizzle] = drm_fourcc_to_vk_format(sd.fourcc);
+
       if (sd.modifier != DRM_FORMAT_MOD_INVALID) {
-        int plane_count = 0;
+        int dmabuf_planes = 0;
         for (int i = 0; i < 4 && sd.fds[i] >= 0; ++i) {
+          dmabuf_planes++;
+        }
+
+        // Query driver for the expected plane count for this format+modifier.
+        // DMA-BUF exports may include extra metadata planes (e.g. AMD DCC).
+        int expected = query_modifier_plane_count(vk_format, sd.modifier);
+        int plane_count = (expected > 0 && expected <= dmabuf_planes) ? expected : dmabuf_planes;
+
+        for (int i = 0; i < plane_count; ++i) {
           drm_layouts[i].offset = sd.offsets[i];
           drm_layouts[i].rowPitch = sd.pitches[i];
-          plane_count++;
         }
         drm_ci.drmFormatModifier = sd.modifier;
         drm_ci.drmFormatModifierPlaneCount = plane_count;
@@ -475,8 +514,6 @@ namespace vk {
         tiling = VK_IMAGE_TILING_LINEAR;
       }
 
-      auto [vk_format, vk_swizzle] = drm_fourcc_to_vk_format(sd.fourcc);
-
       VkImageCreateInfo img_ci = {VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO};
       img_ci.pNext = &ext_ci;
       img_ci.imageType = VK_IMAGE_TYPE_2D;
@@ -781,6 +818,7 @@ namespace vk {
       for (int i = 0; i < AV_NUM_DATA_POINTERS && vk_frame->img[i]; i++) {
         vk_frame->layout[i] = VK_IMAGE_LAYOUT_GENERAL;
         vk_frame->access[i] = VK_ACCESS_SHADER_WRITE_BIT;
+        vk_frame->queue_family[i] = vk_dev.compute_qf;
       }
 
       target.initialized = true;

src/platform/linux/wayland.cpp

@@ -226,27 +226,10 @@ namespace wl {
       return true;
     }
 
-    // Find render node
+    auto render_path = platf::resolve_render_device();
-    drmDevice *devices[16];
+    int drm_fd = open(render_path.c_str(), O_RDWR);
-    int n = drmGetDevices2(0, devices, 16);
-    if (n <= 0) {
-      BOOST_LOG(error) << "[wayland] No DRM devices found"sv;
-      return false;
-    }
-
-    int drm_fd = -1;
-    for (int i = 0; i < n; i++) {
-      if (devices[i]->available_nodes & (1 << DRM_NODE_RENDER)) {
-        drm_fd = open(devices[i]->nodes[DRM_NODE_RENDER], O_RDWR);
-        if (drm_fd >= 0) {
-          break;
-        }
-      }
-    }
-    drmFreeDevices(devices, n);
-
     if (drm_fd < 0) {
-      BOOST_LOG(error) << "[wayland] Failed to open DRM render node"sv;
+      BOOST_LOG(error) << "[wayland] Failed to open DRM render node: "sv << render_path;
       return false;
     }
 
@@ -374,7 +357,7 @@ namespace wl {
     }
 
     // Create GBM buffer
-    current_bo = gbm_bo_create(gbm_device, dmabuf_info.width, dmabuf_info.height, dmabuf_info.format, GBM_BO_USE_RENDERING | GBM_BO_USE_LINEAR);
+    current_bo = gbm_bo_create(gbm_device, dmabuf_info.width, dmabuf_info.height, dmabuf_info.format, GBM_BO_USE_RENDERING);
     if (!current_bo) {
       BOOST_LOG(error) << "Failed to create GBM buffer"sv;
       zwlr_screencopy_frame_v1_destroy(frame);

src/platform/linux/wayland.h

@@ -69,7 +69,6 @@ namespace wl {
     std::array<frame_t, 2> frames;
     frame_t *current_frame;
     zwlr_screencopy_frame_v1_listener listener;
-    bool y_invert {false};
 
   private:
     bool init_gbm();
@@ -96,6 +95,7 @@ namespace wl {
     struct gbm_device *gbm_device {nullptr};
     struct gbm_bo *current_bo {nullptr};
     struct wl_buffer *current_wl_buffer {nullptr};
+    bool y_invert {false};
   };
 
   class monitor_t {

src/platform/linux/wlgrab.cpp

@@ -11,7 +11,6 @@
 #include "src/platform/common.h"
 #include "src/video.h"
 #include "vaapi.h"
-#include "vulkan_encode.h"
 #include "wayland.h"
 
 using namespace std::literals;
@@ -349,7 +348,6 @@ namespace wl {
 
       img->sd = current_frame->sd;
       img->frame_timestamp = current_frame->frame_timestamp;
-      img->y_invert = dmabuf.y_invert;
 
       // Prevent dmabuf from closing the file descriptors.
       std::fill_n(current_frame->sd.fds, 4, -1);
@@ -379,12 +377,6 @@ namespace wl {
       }
 #endif
 
-#ifdef SUNSHINE_BUILD_VULKAN
-      if (mem_type == platf::mem_type_e::vulkan) {
-        return vk::make_avcodec_encode_device_vram(width, height, 0, 0);
-      }
-#endif
-
 #ifdef SUNSHINE_BUILD_CUDA
       if (mem_type == platf::mem_type_e::cuda) {
         return cuda::make_avcodec_gl_encode_device(width, height, 0, 0);
@@ -406,12 +398,12 @@ namespace wl {
 
 namespace platf {
   std::shared_ptr<display_t> wl_display(mem_type_e hwdevice_type, const std::string &display_name, const video::config_t &config) {
-    if (hwdevice_type != platf::mem_type_e::system && hwdevice_type != platf::mem_type_e::vaapi && hwdevice_type != platf::mem_type_e::cuda && hwdevice_type != platf::mem_type_e::vulkan) {
+    if (hwdevice_type != platf::mem_type_e::system && hwdevice_type != platf::mem_type_e::vaapi && hwdevice_type != platf::mem_type_e::cuda) {
       BOOST_LOG(error) << "[wlgrab] Could not initialize display with the given hw device type."sv;
       return nullptr;
     }
 
-    if (hwdevice_type == platf::mem_type_e::vaapi || hwdevice_type == platf::mem_type_e::cuda || hwdevice_type == platf::mem_type_e::vulkan) {
+    if (hwdevice_type == platf::mem_type_e::vaapi || hwdevice_type == platf::mem_type_e::cuda) {
       auto wlr = std::make_shared<wl::wlr_vram_t>();
       if (wlr->init(hwdevice_type, display_name, config)) {
         return nullptr;