feat(linux): Add Vulkan video encoder (#4603)

cmake/compile_definitions/linux.cmake

@@ -120,6 +120,73 @@ if(LIBVA_FOUND)
             "${CMAKE_SOURCE_DIR}/src/platform/linux/vaapi.cpp")
 endif()
 
+# vulkan video encoding (via FFmpeg)
+if(${SUNSHINE_ENABLE_VULKAN})
+    # use Vulkan headers from build-deps submodule (system headers may be too old, e.g. Ubuntu 22.04)
+    set(VULKAN_HEADERS_DIR "${CMAKE_SOURCE_DIR}/third-party/build-deps/third-party/FFmpeg/Vulkan-Headers/include")
+    if(NOT EXISTS "${VULKAN_HEADERS_DIR}/vulkan/vulkan.h")
+        message(FATAL_ERROR "Vulkan headers not found in build-deps submodule")
+    endif()
+
+    find_library(VULKAN_LIBRARY NAMES vulkan vulkan-1)
+    if(NOT VULKAN_LIBRARY)
+        message(FATAL_ERROR "libvulkan not found")
+    endif()
+
+    # prefer glslc, fall back to glslangValidator
+    find_program(GLSLC_EXECUTABLE glslc)
+    if(NOT GLSLC_EXECUTABLE)
+        find_program(GLSLANG_EXECUTABLE glslangValidator)
+    endif()
+    if(NOT GLSLC_EXECUTABLE AND NOT GLSLANG_EXECUTABLE)
+        message(FATAL_ERROR "Vulkan shader compiler not found (need glslc or glslangValidator)")
+    endif()
+
+    list(APPEND SUNSHINE_DEFINITIONS SUNSHINE_BUILD_VULKAN=1)
+    include_directories(SYSTEM ${VULKAN_HEADERS_DIR})
+    list(APPEND PLATFORM_LIBRARIES ${VULKAN_LIBRARY})
+    list(APPEND PLATFORM_TARGET_FILES
+            "${CMAKE_SOURCE_DIR}/src/platform/linux/vulkan_encode.h"
+            "${CMAKE_SOURCE_DIR}/src/platform/linux/vulkan_encode.cpp")
+
+    # compile GLSL -> SPIR-V -> C include at build time
+    set(VULKAN_SHADER_DIR "${CMAKE_BINARY_DIR}/generated-src/shaders")
+    set(VULKAN_SHADER_SOURCE "${SUNSHINE_SOURCE_ASSETS_DIR}/linux/assets/shaders/vulkan/rgb2yuv.comp")
+    set(VULKAN_SHADER_SPV "${VULKAN_SHADER_DIR}/rgb2yuv.spv")
+    set(VULKAN_SHADER_DATA "${VULKAN_SHADER_DIR}/rgb2yuv.spv.inc")
+
+    file(MAKE_DIRECTORY "${VULKAN_SHADER_DIR}")
+
+    if(GLSLC_EXECUTABLE)
+        add_custom_command(
+                OUTPUT "${VULKAN_SHADER_SPV}"
+                COMMAND ${GLSLC_EXECUTABLE} -O "${VULKAN_SHADER_SOURCE}" -o "${VULKAN_SHADER_SPV}"
+                DEPENDS "${VULKAN_SHADER_SOURCE}"
+                COMMENT "Compiling Vulkan shader rgb2yuv.comp (glslc)"
+                VERBATIM)
+    else()
+        add_custom_command(
+                OUTPUT "${VULKAN_SHADER_SPV}"
+                COMMAND ${GLSLANG_EXECUTABLE} -V -o "${VULKAN_SHADER_SPV}" "${VULKAN_SHADER_SOURCE}"
+                DEPENDS "${VULKAN_SHADER_SOURCE}"
+                COMMENT "Compiling Vulkan shader rgb2yuv.comp (glslangValidator)"
+                VERBATIM)
+    endif()
+
+    add_custom_command(
+            OUTPUT "${VULKAN_SHADER_DATA}"
+            COMMAND ${CMAKE_COMMAND} -DSPV_FILE=${VULKAN_SHADER_SPV} -DOUT_FILE=${VULKAN_SHADER_DATA}
+                -P "${CMAKE_SOURCE_DIR}/cmake/scripts/binary_to_c.cmake"
+            DEPENDS "${VULKAN_SHADER_SPV}"
+            COMMENT "Generating C include from rgb2yuv.spv"
+            VERBATIM)
+
+    add_custom_target(vulkan_shaders
+            DEPENDS "${VULKAN_SHADER_DATA}"
+            COMMENT "Vulkan shader compilation")
+    set(SUNSHINE_TARGET_DEPENDENCIES ${SUNSHINE_TARGET_DEPENDENCIES} vulkan_shaders)
+endif()
+
 # wayland
 if(${SUNSHINE_ENABLE_WAYLAND})
     find_package(Wayland REQUIRED)

cmake/prep/options.cmake

@@ -58,6 +58,8 @@ elseif(UNIX)  # Linux
             "Enable KMS grab if available." ON)
     option(SUNSHINE_ENABLE_VAAPI
             "Enable building vaapi specific code." ON)
+    option(SUNSHINE_ENABLE_VULKAN
+            "Enable Vulkan video encoding." ON)
     option(SUNSHINE_ENABLE_WAYLAND
             "Enable building wayland specific code." ON)
     option(SUNSHINE_ENABLE_X11

cmake/scripts/binary_to_c.cmake

@@ -0,0 +1,35 @@
+# binary_to_c.cmake - Convert a binary file to a C uint32_t initializer list.
+# Input:  SPV_FILE  - path to SPIR-V binary
+# Output: OUT_FILE  - path to write C initializer (e.g. {0x07230203, ...})
+
+file(READ "${SPV_FILE}" data HEX)
+string(LENGTH "${data}" hex_len)
+math(EXPR num_bytes "${hex_len} / 2")
+math(EXPR num_words "${num_bytes} / 4")
+math(EXPR last "${num_words} - 1")
+
+set(_out "{")
+set(_idx 0)
+while(_idx LESS_EQUAL last)
+    math(EXPR off "${_idx} * 8")
+    math(EXPR off1 "${off} + 2")
+    math(EXPR off2 "${off} + 4")
+    math(EXPR off3 "${off} + 6")
+    string(SUBSTRING "${data}" ${off} 2 b0)
+    string(SUBSTRING "${data}" ${off1} 2 b1)
+    string(SUBSTRING "${data}" ${off2} 2 b2)
+    string(SUBSTRING "${data}" ${off3} 2 b3)
+    # little-endian to uint32_t
+    string(APPEND _out "0x${b3}${b2}${b1}${b0}")
+    if(NOT _idx EQUAL last)
+        string(APPEND _out ",")
+    endif()
+    math(EXPR _col "(${_idx} + 1) % 8")
+    if(_col EQUAL 0)
+        string(APPEND _out "\n")
+    endif()
+    math(EXPR _idx "${_idx} + 1")
+endwhile()
+string(APPEND _out "}\n")
+
+file(WRITE "${OUT_FILE}" "${_out}")