fix(linux/vulkan): preserve host aspect ratio in encoder output (#5130)

src/platform/linux/vulkan_encode.cpp

@@ -3,6 +3,7 @@
  * @brief Vulkan-native encoder: DMA-BUF -> Vulkan compute (RGB->YUV) -> Vulkan Video encode.
  *        No EGL/GL dependency — all GPU work stays in a single Vulkan queue.
  */
+#include <algorithm>
 #include <array>
 #include <cstdint>
 #include <drm_fourcc.h>
@@ -113,7 +114,9 @@ namespace vk {
     std::array<float, 2> range_uv;
     std::array<int32_t, 2> src_offset;
     std::array<int32_t, 2> src_size;
+    std::array<int32_t, 2> dst_offset;
     std::array<int32_t, 2> dst_size;
+    std::array<int32_t, 2> dst_full_size;
     std::array<int32_t, 2> cursor_pos;
     std::array<int32_t, 2> cursor_size;
     int32_t y_invert;
@@ -284,20 +287,34 @@ namespace vk {
         descriptors_dirty = false;
       }
 
+      // Preserve aspect ratio: fit src into dst, center with black bars.
+      // UV plane is subsampled 2x, so keep effective size and offset even.
+      float scalar = std::min((float) frame->width / width, (float) frame->height / height);
+      int32_t eff_w = std::min<int32_t>(((int32_t) (width * scalar)) & ~1, frame->width & ~1);
+      int32_t eff_h = std::min<int32_t>(((int32_t) (height * scalar)) & ~1, frame->height & ~1);
+      int32_t dst_off_x = ((frame->width - eff_w) / 2) & ~1;
+      int32_t dst_off_y = ((frame->height - eff_h) / 2) & ~1;
+      eff_w = std::min(eff_w, (frame->width - dst_off_x) & ~1);
+      eff_h = std::min(eff_h, (frame->height - dst_off_y) & ~1);
+
       // Fill push constants
       push.src_offset[0] = offset_x;
       push.src_offset[1] = offset_y;
       push.src_size[0] = width;
       push.src_size[1] = height;
-      push.dst_size[0] = frame->width;
+      push.dst_offset[0] = dst_off_x;
-      push.dst_size[1] = frame->height;
+      push.dst_offset[1] = dst_off_y;
+      push.dst_size[0] = eff_w;
+      push.dst_size[1] = eff_h;
+      push.dst_full_size[0] = frame->width;
+      push.dst_full_size[1] = frame->height;
       push.y_invert = descriptor.y_invert ? 1 : 0;
 
       if (descriptor.data) {
-        float scale_x = (float) frame->width / width;
+        float scale_x = (float) eff_w / width;
-        float scale_y = (float) frame->height / height;
+        float scale_y = (float) eff_h / height;
-        push.cursor_pos[0] = (int32_t) ((descriptor.x - offset_x) * scale_x);
+        push.cursor_pos[0] = (int32_t) ((descriptor.x - offset_x) * scale_x) + dst_off_x;
-        push.cursor_pos[1] = (int32_t) ((descriptor.y - offset_y) * scale_y);
+        push.cursor_pos[1] = (int32_t) ((descriptor.y - offset_y) * scale_y) + dst_off_y;
         push.cursor_size[0] = (int32_t) (descriptor.width * scale_x);
         push.cursor_size[1] = (int32_t) (descriptor.height * scale_y);
       } else {

src_assets/linux/assets/shaders/vulkan/rgb2yuv.comp

@@ -15,7 +15,9 @@ layout(push_constant) uniform PushConstants {
   vec2 range_uv;
   ivec2 src_offset;
   ivec2 src_size;
-  ivec2 dst_size;
+  ivec2 dst_offset;  // top-left of the aspect-preserved viewport
+  ivec2 dst_size;  // effective viewport (src after aspect-fit)
+  ivec2 dst_full_size;  // full encoder frame size
   ivec2 cursor_pos;
   ivec2 cursor_size;  // w=0 means no cursor
   int y_invert;
@@ -32,13 +34,27 @@ vec3 blend_cursor(vec3 rgb, ivec2 pos) {
 
 void main() {
   ivec2 pos = ivec2(gl_GlobalInvocationID.xy);
-  if (pos.x >= pc.dst_size.x || pos.y >= pc.dst_size.y)
+  if (pos.x >= pc.dst_full_size.x || pos.y >= pc.dst_full_size.y)
     return;
 
+  ivec2 d = pos - pc.dst_offset;
+  if (d.x < 0 || d.y < 0 || d.x >= pc.dst_size.x || d.y >= pc.dst_size.y) {
+    // Outside the aspect-fit viewport: write encoded black.
+    // Use the same conversion formula as RGB=(0,0,0), where the dot product
+    // is zero and the encoded value is color_vec_*.w adjusted by range_*.
+    float y_black = pc.color_vec_y.w * pc.range_y.x + pc.range_y.y;
+    float u_black = pc.color_vec_u.w * pc.range_uv.x + pc.range_uv.y;
+    float v_black = pc.color_vec_v.w * pc.range_uv.x + pc.range_uv.y;
+    imageStore(y_out, pos, vec4(y_black, 0, 0, 0));
+    if ((pos.x & 1) == 0 && (pos.y & 1) == 0)
+      imageStore(uv_out, pos >> 1, vec4(u_black, v_black, 0, 0));
+    return;
+  }
+
   vec2 inv_tex = 1.0 / vec2(textureSize(rgb_in, 0));
   vec2 scale = vec2(pc.src_size) / vec2(pc.dst_size);
 
-  vec2 uv = (vec2(pc.src_offset) + (vec2(pos) + 0.5) * scale) * inv_tex;
+  vec2 uv = (vec2(pc.src_offset) + (vec2(d) + 0.5) * scale) * inv_tex;
   if (pc.y_invert != 0)
     uv.y = 1.0 - uv.y;
   vec3 rgb = texture(rgb_in, uv).rgb;