Vulkan/D3D12: Fix HDR rendering, ribbon shaders, and pipeline barriers (#18734)

* WIP: Vulkan/D3D12 ribbon shader and mipmap fixes

Save point for in-progress gfx driver changes including
Vulkan ribbon shader updates and D3D12 fixes.

* Vulkan: Fix HDR render pass mismatch and pipeline barrier

Fix render pass / framebuffer format mismatch when rendering through
the SDR offscreen buffer in HDR mode. The filter chain's final pass
and the frame render pass were using vk->render_pass (A2B10G10R10)
while the offscreen buffer framebuffer was created with
vk->sdr_render_pass (B8G8R8A8), causing a Vulkan validation error.

Select the correct render pass at frame time based on
use_offscreen_buffer, and rebuild the filter chain's final pass
pipeline to use the SDR render pass both at shader preset load and
at swapchain recreation.

Also fix an incorrect srcStageMask in the image layout transition
after the HDR pipeline: SHADER_READ_BIT requires FRAGMENT_SHADER_BIT,
not COLOR_ATTACHMENT_OUTPUT_BIT.

* Vulkan/D3D12: Use BLEND_OP_MAX for alpha channel blending

Change the alpha blend operation from ADD to MAX in the blend-enabled
pipeline states for both Vulkan and D3D12. This ensures the render
target alpha channel is always 1.0 (since the background establishes
alpha=1.0 and max(src, 1.0) preserves it) while leaving RGB blending
with SRC_ALPHA / ONE_MINUS_SRC_ALPHA completely unchanged.

Author: MajorPainTheCactus

gfx/common/vulkan_common.c

@@ -911,6 +911,9 @@ static const char *vulkan_optional_instance_extensions[] = {
 #ifdef __APPLE__
    VK_KHR_PORTABILITY_ENUMERATION_EXTENSION_NAME,
 #endif
+#ifdef _WIN32
+   "VK_KHR_get_surface_capabilities2",
+#endif
 #ifdef VULKAN_HDR_SWAPCHAIN
    VULKAN_COLORSPACE_EXTENSION_NAME
 #endif

gfx/drivers/d3d12.c

@@ -247,6 +247,7 @@ typedef struct ALIGN(16)
       float height;
    } OutputSize;
    float time;
+   float alpha;
 } d3d12_uniform_t;
 
 typedef struct
@@ -455,7 +456,7 @@ static D3D12_RENDER_TARGET_BLEND_DESC d3d12_blend_enable_desc = {
    D3D12_BLEND_OP_ADD,
    D3D12_BLEND_SRC_ALPHA,
    D3D12_BLEND_INV_SRC_ALPHA,
-   D3D12_BLEND_OP_ADD,
+   D3D12_BLEND_OP_MAX,
    D3D12_LOGIC_OP_NOOP,
    D3D12_COLOR_WRITE_ENABLE_ALL,
 };
@@ -1199,7 +1200,8 @@ static void gfx_display_d3d12_draw_pipeline(gfx_display_ctx_draw_t *draw,
    cmd->lpVtbl->IASetPrimitiveTopology(cmd,
          D3D_PRIMITIVE_TOPOLOGY_TRIANGLESTRIP);
 
-   d3d12->ubo_values.time += 0.01f;
+   d3d12->ubo_values.time  += 0.01f;
+   d3d12->ubo_values.alpha  = draw->color ? draw->color[3] : 1.0f;
 
    {
       D3D12_RANGE read_range;
@@ -3528,6 +3530,7 @@ static void *d3d12_gfx_init(const video_info_t* video,
    d3d12->ubo_values.mvp               = d3d12->mvp_no_rot;
    d3d12->ubo_values.OutputSize.width  = d3d12->chain.viewport.Width;
    d3d12->ubo_values.OutputSize.height = d3d12->chain.viewport.Height;
+   d3d12->ubo_values.alpha             = 1.0f;
 
    {
       math_matrix_4x4* mvp;

gfx/drivers/d3d_shaders/ribbon_simple_sm4.hlsl.h

@@ -7,6 +7,7 @@ SRC(
          float4x4 modelViewProj;
          float2 Outputsize;
          float time;
+         float alpha;
       };
       uniform UBO global;
 
@@ -46,6 +47,6 @@ SRC(
 
       float4 PSMain() : SV_TARGET
       {
-         return float4(0.05, 0.05, 0.05, 1.0);
+         return float4(0.05, 0.05, 0.05, global.alpha);
       };
 )

gfx/drivers/d3d_shaders/ribbon_sm4.hlsl.h

@@ -13,6 +13,7 @@ SRC(
          float4x4 modelViewProj;
          float2 Outputsize;
          float time;
+         float alpha;
       };
       uniform UBO global;
 
@@ -70,7 +71,7 @@ SRC(
          float3 normal = normalize(cross(x, y));
          float c = 1.0 - dot(normal, up);
          c = (1.0 - cos(c * c)) / 13.0;
-         return  float4(c, c, c, 1.0);
+         return  float4(c, c, c, global.alpha);
    //      return  float4(c, c, c, c);
    //      return float4(1.0, 1.0, 1.0, c);
    //      return float4(1.0, 0.0, 1.0, 1.0);

gfx/drivers/vulkan.c

@@ -1352,14 +1352,18 @@ static void gfx_display_vk_draw_pipeline(
       default:
       case VIDEO_SHADER_MENU:
       case VIDEO_SHADER_MENU_2:
-         ca                               = &p_disp->dispca;
-         draw->coords                     = (struct video_coords*)&ca->coords;
-         draw->backend_data               = ubo_scratch_data;
-         draw->backend_data_size          = 2 * sizeof(float);
-
-         /* Match UBO layout in shader. */
-         memcpy(ubo_scratch_data, &t, sizeof(t));
-         memcpy(ubo_scratch_data + sizeof(float), &yflip, sizeof(yflip));
+         {
+            float alpha                   = draw->color ? draw->color[3] : 1.0f;
+            ca                            = &p_disp->dispca;
+            draw->coords                  = (struct video_coords*)&ca->coords;
+            draw->backend_data            = ubo_scratch_data;
+            draw->backend_data_size       = 3 * sizeof(float);
+
+            /* Match UBO layout in shader. */
+            memcpy(ubo_scratch_data, &t, sizeof(t));
+            memcpy(ubo_scratch_data + sizeof(float), &yflip, sizeof(yflip));
+            memcpy(ubo_scratch_data + 2 * sizeof(float), &alpha, sizeof(alpha));
+         }
          break;
 
       /* Snow simple */
@@ -2624,7 +2628,7 @@ static void vulkan_init_pipelines(vk_t *vk)
    blend_attachment.colorBlendOp        = VK_BLEND_OP_ADD;
    blend_attachment.srcAlphaBlendFactor = VK_BLEND_FACTOR_SRC_ALPHA;
    blend_attachment.dstAlphaBlendFactor = VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA;
-   blend_attachment.alphaBlendOp        = VK_BLEND_OP_ADD;
+   blend_attachment.alphaBlendOp        = VK_BLEND_OP_MAX;
 
    /* Glyph pipeline */
    module_info.codeSize                 = sizeof(font_frag);
@@ -3303,6 +3307,20 @@ static bool vulkan_init_filter_chain_preset(vk_t *vk, const char *shader_path)
          vulkan_set_hdr_inverse_tonemap(vk, vk->filter_chain, !emits_hdr10);
          vulkan_set_hdr10(vk, vk->filter_chain, !emits_hdr10);
          vk->flags |= VK_FLAG_SHOULD_RESIZE;
+
+         if (!emits_hdr10)
+         {
+            /* Rendering goes through the SDR offscreen buffer;
+             * rebuild the filter chain's final pass pipeline
+             * to use the SDR render pass. */
+            struct vulkan_filter_chain_swapchain_info sdr_swapchain;
+            sdr_swapchain.vp          = vk->vk_vp;
+            sdr_swapchain.format      = vk->context->swapchain_format;
+            sdr_swapchain.render_pass = vk->sdr_render_pass;
+            sdr_swapchain.num_indices = vk->context->num_swapchain_images;
+            vulkan_filter_chain_update_swapchain_info(
+                  vk->filter_chain, &sdr_swapchain);
+         }
       }
       else if (rt_format == VK_FORMAT_R16G16B16A16_SFLOAT)
       {
@@ -4028,6 +4046,26 @@ static void vulkan_check_swapchain(vk_t *vk)
    filter_info.format               = vk->context->swapchain_format;
    filter_info.render_pass          = vk->render_pass;
    filter_info.num_indices          = vk->context->num_swapchain_images;
+
+#ifdef VULKAN_HDR_SWAPCHAIN
+   /* When rendering through the SDR offscreen buffer, the filter chain's
+    * final pass must use the SDR render pass to match the framebuffer format. */
+   if (vk->context->flags & VK_CTX_FLAG_HDR_ENABLE)
+   {
+      vulkan_filter_chain_t *chain   = vk->filter_chain
+         ? vk->filter_chain : vk->filter_chain_default;
+      struct video_shader *preset    = vulkan_filter_chain_get_preset(chain);
+      if (preset && preset->passes)
+      {
+         VkFormat rt_format = vulkan_filter_chain_get_pass_rt_format(
+               chain, preset->passes - 1);
+         if (  vulkan_is_hdr10_format(rt_format)
+            && !vulkan_filter_chain_emits_hdr10(chain))
+            filter_info.render_pass = vk->sdr_render_pass;
+      }
+   }
+#endif
+
    if (
        !vulkan_filter_chain_update_swapchain_info(
           (vk->filter_chain) ? vk->filter_chain : vk->filter_chain_default,
@@ -5050,6 +5088,12 @@ static bool vulkan_frame(void *data, const void *frame,
          else if (tex->image)
             vulkan_transition_texture(vk, vk->cmd, tex);
 
+         /* If the texture hasn't been uploaded yet (e.g. after
+          * reinit from HDR toggle with no cached frame),
+          * fall back to the default black texture. */
+         if (tex->layout == VK_IMAGE_LAYOUT_UNDEFINED)
+            tex = &vk->default_texture;
+
          input.image  = tex->image;
          input.view   = tex->view;
          input.layout = tex->layout;
@@ -5105,7 +5149,12 @@ static bool vulkan_frame(void *data, const void *frame,
    {
       rp_info.sType                    = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO;
       rp_info.pNext                    = NULL;
+#ifdef VULKAN_HDR_SWAPCHAIN
+      rp_info.renderPass               = use_offscreen_buffer
+         ? vk->sdr_render_pass : vk->render_pass;
+#else
       rp_info.renderPass               = vk->render_pass;
+#endif
       rp_info.framebuffer              = backbuffer->framebuffer;
       rp_info.renderArea.offset.x      = 0;
       rp_info.renderArea.offset.y      = 0;
@@ -5153,7 +5202,7 @@ static bool vulkan_frame(void *data, const void *frame,
          VULKAN_IMAGE_LAYOUT_TRANSITION(vk->cmd, vk->offscreen_buffer.image,
                VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
                VK_ACCESS_SHADER_READ_BIT, VK_ACCESS_TRANSFER_WRITE_BIT,
-               VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT,
+               VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT,
                VK_PIPELINE_STAGE_TRANSFER_BIT);
 
          end_main_pass = false;

gfx/drivers/vulkan_shaders/pipeline_ribbon.frag

@@ -5,6 +5,7 @@ layout(std140, set = 0, binding = 0) uniform UBO
 {
    float time;
    float yflip;
+   float alpha;
 } constants;
 
 layout(location = 0) in vec3 vEC;
@@ -19,5 +20,5 @@ void main()
    vec3 normal = normalize(cross(x, y));
    float c = 1.0 - dot(normal, up);
    c = (1.0 - cos(c * c)) / 3.0;
-   FragColor = vec4(c, c, c, 1.0);
+   FragColor = vec4(c, c, c, constants.alpha);
 }

gfx/drivers/vulkan_shaders/pipeline_ribbon.frag.inc

@@ -1,4 +1,4 @@
-{0x07230203,0x00010000,0x000d0007,0x00000036,
+{0x07230203,0x00010000,0x000d000b,0x00000037,
 0x00000000,0x00020011,0x00000001,0x0006000b,
 0x00000001,0x4c534c47,0x6474732e,0x3035342e,
 0x00000000,0x0003000e,0x00000000,0x00000001,
@@ -16,76 +16,80 @@
 0x00030005,0x00000011,0x004f4255,0x00050006,
 0x00000011,0x00000000,0x656d6974,0x00000000,
 0x00050006,0x00000011,0x00000001,0x696c6679,
-0x00000070,0x00050005,0x00000013,0x736e6f63,
-0x746e6174,0x00000073,0x00040005,0x0000001b,
-0x6d726f6e,0x00006c61,0x00030005,0x00000021,
-0x00000063,0x00050005,0x00000031,0x67617246,
-0x6f6c6f43,0x00000072,0x00040047,0x0000000b,
-0x0000001e,0x00000000,0x00050048,0x00000011,
+0x00000070,0x00050006,0x00000011,0x00000002,
+0x68706c61,0x00000061,0x00050005,0x00000013,
+0x736e6f63,0x746e6174,0x00000073,0x00040005,
+0x0000001b,0x6d726f6e,0x00006c61,0x00030005,
+0x00000021,0x00000063,0x00050005,0x00000031,
+0x67617246,0x6f6c6f43,0x00000072,0x00040047,
+0x0000000b,0x0000001e,0x00000000,0x00030047,
+0x00000011,0x00000002,0x00050048,0x00000011,
 0x00000000,0x00000023,0x00000000,0x00050048,
 0x00000011,0x00000001,0x00000023,0x00000004,
-0x00030047,0x00000011,0x00000002,0x00040047,
-0x00000013,0x00000022,0x00000000,0x00040047,
-0x00000013,0x00000021,0x00000000,0x00040047,
-0x00000031,0x0000001e,0x00000000,0x00020013,
-0x00000002,0x00030021,0x00000003,0x00000002,
-0x00030016,0x00000006,0x00000020,0x00040017,
-0x00000007,0x00000006,0x00000003,0x00040020,
-0x00000008,0x00000007,0x00000007,0x00040020,
-0x0000000a,0x00000001,0x00000007,0x0004003b,
-0x0000000a,0x0000000b,0x00000001,0x0004001e,
-0x00000011,0x00000006,0x00000006,0x00040020,
-0x00000012,0x00000002,0x00000011,0x0004003b,
-0x00000012,0x00000013,0x00000002,0x00040015,
-0x00000014,0x00000020,0x00000001,0x0004002b,
-0x00000014,0x00000015,0x00000001,0x00040020,
-0x00000016,0x00000002,0x00000006,0x00040020,
-0x00000020,0x00000007,0x00000006,0x0004002b,
-0x00000006,0x00000022,0x3f800000,0x0004002b,
-0x00000006,0x00000024,0x00000000,0x0006002c,
-0x00000007,0x00000025,0x00000024,0x00000024,
-0x00000022,0x0004002b,0x00000006,0x0000002d,
-0x40400000,0x00040017,0x0000002f,0x00000006,
-0x00000004,0x00040020,0x00000030,0x00000003,
-0x0000002f,0x0004003b,0x00000030,0x00000031,
-0x00000003,0x00050036,0x00000002,0x00000004,
-0x00000000,0x00000003,0x000200f8,0x00000005,
-0x0004003b,0x00000008,0x00000009,0x00000007,
-0x0004003b,0x00000008,0x0000000e,0x00000007,
-0x0004003b,0x00000008,0x0000001b,0x00000007,
-0x0004003b,0x00000020,0x00000021,0x00000007,
-0x0004003d,0x00000007,0x0000000c,0x0000000b,
-0x000400cf,0x00000007,0x0000000d,0x0000000c,
-0x0003003e,0x00000009,0x0000000d,0x0004003d,
-0x00000007,0x0000000f,0x0000000b,0x000400d0,
-0x00000007,0x00000010,0x0000000f,0x0003003e,
-0x0000000e,0x00000010,0x00050041,0x00000016,
-0x00000017,0x00000013,0x00000015,0x0004003d,
-0x00000006,0x00000018,0x00000017,0x0004003d,
-0x00000007,0x00000019,0x0000000e,0x0005008e,
-0x00000007,0x0000001a,0x00000019,0x00000018,
-0x0003003e,0x0000000e,0x0000001a,0x0004003d,
-0x00000007,0x0000001c,0x00000009,0x0004003d,
-0x00000007,0x0000001d,0x0000000e,0x0007000c,
-0x00000007,0x0000001e,0x00000001,0x00000044,
-0x0000001c,0x0000001d,0x0006000c,0x00000007,
-0x0000001f,0x00000001,0x00000045,0x0000001e,
-0x0003003e,0x0000001b,0x0000001f,0x0004003d,
-0x00000007,0x00000023,0x0000001b,0x00050094,
-0x00000006,0x00000026,0x00000023,0x00000025,
-0x00050083,0x00000006,0x00000027,0x00000022,
-0x00000026,0x0003003e,0x00000021,0x00000027,
-0x0004003d,0x00000006,0x00000028,0x00000021,
-0x0004003d,0x00000006,0x00000029,0x00000021,
-0x00050085,0x00000006,0x0000002a,0x00000028,
-0x00000029,0x0006000c,0x00000006,0x0000002b,
-0x00000001,0x0000000e,0x0000002a,0x00050083,
-0x00000006,0x0000002c,0x00000022,0x0000002b,
-0x00050088,0x00000006,0x0000002e,0x0000002c,
-0x0000002d,0x0003003e,0x00000021,0x0000002e,
-0x0004003d,0x00000006,0x00000032,0x00000021,
-0x0004003d,0x00000006,0x00000033,0x00000021,
-0x0004003d,0x00000006,0x00000034,0x00000021,
-0x00070050,0x0000002f,0x00000035,0x00000032,
-0x00000033,0x00000034,0x00000022,0x0003003e,
-0x00000031,0x00000035,0x000100fd,0x00010038}
+0x00050048,0x00000011,0x00000002,0x00000023,
+0x00000008,0x00040047,0x00000013,0x00000021,
+0x00000000,0x00040047,0x00000013,0x00000022,
+0x00000000,0x00040047,0x00000031,0x0000001e,
+0x00000000,0x00020013,0x00000002,0x00030021,
+0x00000003,0x00000002,0x00030016,0x00000006,
+0x00000020,0x00040017,0x00000007,0x00000006,
+0x00000003,0x00040020,0x00000008,0x00000007,
+0x00000007,0x00040020,0x0000000a,0x00000001,
+0x00000007,0x0004003b,0x0000000a,0x0000000b,
+0x00000001,0x0005001e,0x00000011,0x00000006,
+0x00000006,0x00000006,0x00040020,0x00000012,
+0x00000002,0x00000011,0x0004003b,0x00000012,
+0x00000013,0x00000002,0x00040015,0x00000014,
+0x00000020,0x00000001,0x0004002b,0x00000014,
+0x00000015,0x00000001,0x00040020,0x00000016,
+0x00000002,0x00000006,0x00040020,0x00000020,
+0x00000007,0x00000006,0x0004002b,0x00000006,
+0x00000022,0x3f800000,0x0004002b,0x00000006,
+0x00000024,0x00000000,0x0006002c,0x00000007,
+0x00000025,0x00000024,0x00000024,0x00000022,
+0x0004002b,0x00000006,0x0000002d,0x40400000,
+0x00040017,0x0000002f,0x00000006,0x00000004,
+0x00040020,0x00000030,0x00000003,0x0000002f,
+0x0004003b,0x00000030,0x00000031,0x00000003,
+0x0004002b,0x00000014,0x00000033,0x00000002,
+0x00050036,0x00000002,0x00000004,0x00000000,
+0x00000003,0x000200f8,0x00000005,0x0004003b,
+0x00000008,0x00000009,0x00000007,0x0004003b,
+0x00000008,0x0000000e,0x00000007,0x0004003b,
+0x00000008,0x0000001b,0x00000007,0x0004003b,
+0x00000020,0x00000021,0x00000007,0x0004003d,
+0x00000007,0x0000000c,0x0000000b,0x000400cf,
+0x00000007,0x0000000d,0x0000000c,0x0003003e,
+0x00000009,0x0000000d,0x0004003d,0x00000007,
+0x0000000f,0x0000000b,0x000400d0,0x00000007,
+0x00000010,0x0000000f,0x0003003e,0x0000000e,
+0x00000010,0x00050041,0x00000016,0x00000017,
+0x00000013,0x00000015,0x0004003d,0x00000006,
+0x00000018,0x00000017,0x0004003d,0x00000007,
+0x00000019,0x0000000e,0x0005008e,0x00000007,
+0x0000001a,0x00000019,0x00000018,0x0003003e,
+0x0000000e,0x0000001a,0x0004003d,0x00000007,
+0x0000001c,0x00000009,0x0004003d,0x00000007,
+0x0000001d,0x0000000e,0x0007000c,0x00000007,
+0x0000001e,0x00000001,0x00000044,0x0000001c,
+0x0000001d,0x0006000c,0x00000007,0x0000001f,
+0x00000001,0x00000045,0x0000001e,0x0003003e,
+0x0000001b,0x0000001f,0x0004003d,0x00000007,
+0x00000023,0x0000001b,0x00050094,0x00000006,
+0x00000026,0x00000023,0x00000025,0x00050083,
+0x00000006,0x00000027,0x00000022,0x00000026,
+0x0003003e,0x00000021,0x00000027,0x0004003d,
+0x00000006,0x00000028,0x00000021,0x0004003d,
+0x00000006,0x00000029,0x00000021,0x00050085,
+0x00000006,0x0000002a,0x00000028,0x00000029,
+0x0006000c,0x00000006,0x0000002b,0x00000001,
+0x0000000e,0x0000002a,0x00050083,0x00000006,
+0x0000002c,0x00000022,0x0000002b,0x00050088,
+0x00000006,0x0000002e,0x0000002c,0x0000002d,
+0x0003003e,0x00000021,0x0000002e,0x0004003d,
+0x00000006,0x00000032,0x00000021,0x00050041,
+0x00000016,0x00000034,0x00000013,0x00000033,
+0x0004003d,0x00000006,0x00000035,0x00000034,
+0x00070050,0x0000002f,0x00000036,0x00000032,
+0x00000032,0x00000032,0x00000035,0x0003003e,
+0x00000031,0x00000036,0x000100fd,0x00010038}

gfx/drivers/vulkan_shaders/pipeline_ribbon.vert

@@ -7,6 +7,7 @@ layout(std140, set = 0, binding = 0) uniform UBO
 {
    float time;
    float yflip;
+   float alpha;
 } constants;
 
 float iqhash(float n)