Support capturing rotated displays on Windows (LizardByte#1602)

src/nvenc/nvenc_base.cpp

@@ -500,6 +500,8 @@ namespace nvenc {
       return true;
     }
 
+    encoder_state.rfi_needs_confirmation = true;
+
     if (last_frame < first_frame) {
       BOOST_LOG(error) << "NvEnc: invaid rfi request " << first_frame << "-" << last_frame << ", generating IDR";
       return false;
@@ -508,14 +510,13 @@ namespace nvenc {
     BOOST_LOG(debug) << "NvEnc: rfi request " << first_frame << "-" << last_frame << " expanding to last encoded frame " << encoder_state.last_encoded_frame_index;
     last_frame = encoder_state.last_encoded_frame_index;
 
+    encoder_state.last_rfi_range = { first_frame, last_frame };
+
     if (last_frame - first_frame + 1 >= encoder_params.ref_frames_in_dpb) {
       BOOST_LOG(debug) << "NvEnc: rfi request too large, generating IDR";
       return false;
     }
 
-    encoder_state.rfi_needs_confirmation = true;
-    encoder_state.last_rfi_range = { first_frame, last_frame };
-
     for (auto i = first_frame; i <= last_frame; i++) {
       if (nvenc_failed(nvenc->nvEncInvalidateRefFrames(encoder, i))) {
         BOOST_LOG(error) << "NvEncInvalidateRefFrames " << i << " failed: " << last_error_string;

src/platform/windows/display.h

@@ -81,23 +81,71 @@ namespace platf::dxgi {
   public:
     gpu_cursor_t():
         cursor_view { 0, 0, 0, 0, 0.0f, 1.0f } {};
-    void
-    set_pos(LONG rel_x, LONG rel_y, bool visible) {
-      cursor_view.TopLeftX = rel_x;
-      cursor_view.TopLeftY = rel_y;
 
+    void
+    set_pos(LONG topleft_x, LONG topleft_y, LONG display_width, LONG display_height, DXGI_MODE_ROTATION display_rotation, bool visible) {
+      this->topleft_x = topleft_x;
+      this->topleft_y = topleft_y;
+      this->display_width = display_width;
+      this->display_height = display_height;
+      this->display_rotation = display_rotation;
       this->visible = visible;
+      update_viewport();
     }
 
     void
-    set_texture(LONG width, LONG height, texture2d_t &&texture) {
-      cursor_view.Width = width;
-      cursor_view.Height = height;
-
+    set_texture(LONG texture_width, LONG texture_height, texture2d_t &&texture) {
       this->texture = std::move(texture);
+      this->texture_width = texture_width;
+      this->texture_height = texture_height;
+      update_viewport();
+    }
+
+    void
+    update_viewport() {
+      switch (display_rotation) {
+        case DXGI_MODE_ROTATION_UNSPECIFIED:
+        case DXGI_MODE_ROTATION_IDENTITY:
+          cursor_view.TopLeftX = topleft_x;
+          cursor_view.TopLeftY = topleft_y;
+          cursor_view.Width = texture_width;
+          cursor_view.Height = texture_height;
+          break;
+
+        case DXGI_MODE_ROTATION_ROTATE90:
+          cursor_view.TopLeftX = topleft_y;
+          cursor_view.TopLeftY = display_width - texture_width - topleft_x;
+          cursor_view.Width = texture_height;
+          cursor_view.Height = texture_width;
+          break;
+
+        case DXGI_MODE_ROTATION_ROTATE180:
+          cursor_view.TopLeftX = display_width - texture_width - topleft_x;
+          cursor_view.TopLeftY = display_height - texture_height - topleft_y;
+          cursor_view.Width = texture_width;
+          cursor_view.Height = texture_height;
+          break;
+
+        case DXGI_MODE_ROTATION_ROTATE270:
+          cursor_view.TopLeftX = display_height - texture_height - topleft_y;
+          cursor_view.TopLeftY = topleft_x;
+          cursor_view.Width = texture_height;
+          cursor_view.Height = texture_width;
+          break;
+      }
     }
 
     texture2d_t texture;
+    LONG texture_width;
+    LONG texture_height;
+
+    LONG topleft_x;
+    LONG topleft_y;
+
+    LONG display_width;
+    LONG display_height;
+    DXGI_MODE_ROTATION display_rotation;
+
     shader_res_t input_res;
 
     D3D11_VIEWPORT cursor_view;
@@ -141,6 +189,10 @@ namespace platf::dxgi {
     DXGI_RATIONAL display_refresh_rate;
     int display_refresh_rate_rounded;
 
+    DXGI_MODE_ROTATION display_rotation = DXGI_MODE_ROTATION_UNSPECIFIED;
+    int width_before_rotation;
+    int height_before_rotation;
+
     int client_frame_rate;
 
     DXGI_FORMAT capture_format;
@@ -277,8 +329,8 @@ namespace platf::dxgi {
     blend_t blend_invert;
     blend_t blend_disable;
 
-    ps_t scene_ps;
-    vs_t scene_vs;
+    ps_t cursor_ps;
+    vs_t cursor_vs;
 
     gpu_cursor_t cursor_alpha;
     gpu_cursor_t cursor_xor;

src/platform/windows/display_base.cpp

@@ -461,6 +461,17 @@ namespace platf::dxgi {
             width = desc.DesktopCoordinates.right - offset_x;
             height = desc.DesktopCoordinates.bottom - offset_y;
 
+            display_rotation = desc.Rotation;
+            if (display_rotation == DXGI_MODE_ROTATION_ROTATE90 ||
+                display_rotation == DXGI_MODE_ROTATION_ROTATE270) {
+              width_before_rotation = height;
+              height_before_rotation = width;
+            }
+            else {
+              width_before_rotation = width;
+              height_before_rotation = height;
+            }
+
             // left and bottom may be negative, yet absolute mouse coordinates start at 0x0
             // Ensure offset starts at 0x0
             offset_x -= GetSystemMetrics(SM_XVIRTUALSCREEN);

src/platform/windows/display_vram.cpp

@@ -107,6 +107,7 @@ namespace platf::dxgi {
   blob_t convert_UV_linear_ps_hlsl;
   blob_t convert_UV_PQ_ps_hlsl;
   blob_t scene_vs_hlsl;
+  blob_t cursor_vs_hlsl;
   blob_t convert_Y_ps_hlsl;
   blob_t convert_Y_linear_ps_hlsl;
   blob_t convert_Y_PQ_ps_hlsl;
@@ -472,6 +473,17 @@ namespace platf::dxgi {
       }
       device_ctx->VSSetConstantBuffers(0, 1, &info_scene);
 
+      {
+        int32_t rotation_modifier = display->display_rotation == DXGI_MODE_ROTATION_UNSPECIFIED ? 0 : display->display_rotation - 1;
+        int32_t rotation_data[16 / sizeof(int32_t)] { -rotation_modifier };  // aligned to 16-byte
+        auto rotation = make_buffer(device.get(), rotation_data);
+        if (!rotation) {
+          BOOST_LOG(error) << "Failed to create display rotation vertex constant buffer";
+          return -1;
+        }
+        device_ctx->VSSetConstantBuffers(1, 1, &rotation);
+      }
+
       D3D11_RENDER_TARGET_VIEW_DESC nv12_rt_desc {
         format == DXGI_FORMAT_P010 ? DXGI_FORMAT_R16_UNORM : DXGI_FORMAT_R8_UNORM,
         D3D11_RTV_DIMENSION_TEXTURE2D
@@ -618,7 +630,11 @@ namespace platf::dxgi {
         convert_UV_vs_hlsl->GetBufferPointer(), convert_UV_vs_hlsl->GetBufferSize(),
         &input_layout);
 
-      this->display = std::move(display);
+      this->display = std::dynamic_pointer_cast<display_base_t>(display);
+      if (!this->display) {
+        return -1;
+      }
+      display = nullptr;
 
       blend_disable = make_blend(device.get(), false, false);
       if (!blend_disable) {
@@ -735,7 +751,7 @@ namespace platf::dxgi {
     // amongst multiple hwdevice_t objects (and therefore multiple ID3D11Devices).
     std::map<uint32_t, encoder_img_ctx_t> img_ctx_map;
 
-    std::shared_ptr<platf::display_t> display;
+    std::shared_ptr<display_base_t> display;
 
     vs_t convert_UV_vs;
     ps_t convert_UV_ps;
@@ -988,8 +1004,11 @@ namespace platf::dxgi {
     }
 
     if (frame_info.LastMouseUpdateTime.QuadPart) {
-      cursor_alpha.set_pos(frame_info.PointerPosition.Position.x, frame_info.PointerPosition.Position.y, frame_info.PointerPosition.Visible);
-      cursor_xor.set_pos(frame_info.PointerPosition.Position.x, frame_info.PointerPosition.Position.y, frame_info.PointerPosition.Visible);
+      cursor_alpha.set_pos(frame_info.PointerPosition.Position.x, frame_info.PointerPosition.Position.y,
+        width, height, display_rotation, frame_info.PointerPosition.Visible);
+
+      cursor_xor.set_pos(frame_info.PointerPosition.Position.x, frame_info.PointerPosition.Position.y,
+        width, height, display_rotation, frame_info.PointerPosition.Visible);
     }
 
     const bool blend_mouse_cursor_flag = (cursor_alpha.visible || cursor_xor.visible) && cursor_visible;
@@ -1008,7 +1027,7 @@ namespace platf::dxgi {
 
       // It's possible for our display enumeration to race with mode changes and result in
       // mismatched image pool and desktop texture sizes. If this happens, just reinit again.
-      if (desc.Width != width || desc.Height != height) {
+      if (desc.Width != width_before_rotation || desc.Height != height_before_rotation) {
         BOOST_LOG(info) << "Capture size changed ["sv << width << 'x' << height << " -> "sv << desc.Width << 'x' << desc.Height << ']';
         return capture_e::reinit;
       }
@@ -1109,8 +1128,8 @@ namespace platf::dxgi {
 
       // Otherwise create a new surface.
       D3D11_TEXTURE2D_DESC t {};
-      t.Width = width;
-      t.Height = height;
+      t.Width = width_before_rotation;
+      t.Height = height_before_rotation;
       t.MipLevels = 1;
       t.ArraySize = 1;
       t.SampleDesc.Count = 1;
@@ -1217,8 +1236,8 @@ namespace platf::dxgi {
     }
 
     auto blend_cursor = [&](img_d3d_t &d3d_img) {
-      device_ctx->VSSetShader(scene_vs.get(), nullptr, 0);
-      device_ctx->PSSetShader(scene_ps.get(), nullptr, 0);
+      device_ctx->VSSetShader(cursor_vs.get(), nullptr, 0);
+      device_ctx->PSSetShader(cursor_ps.get(), nullptr, 0);
       device_ctx->OMSetRenderTargets(1, &d3d_img.capture_rt, nullptr);
 
       if (cursor_alpha.texture.get()) {
@@ -1338,15 +1357,26 @@ namespace platf::dxgi {
       return -1;
     }
 
-    status = device->CreateVertexShader(scene_vs_hlsl->GetBufferPointer(), scene_vs_hlsl->GetBufferSize(), nullptr, &scene_vs);
+    status = device->CreateVertexShader(cursor_vs_hlsl->GetBufferPointer(), cursor_vs_hlsl->GetBufferSize(), nullptr, &cursor_vs);
     if (status) {
       BOOST_LOG(error) << "Failed to create scene vertex shader [0x"sv << util::hex(status).to_string_view() << ']';
       return -1;
     }
 
+    {
+      int32_t rotation_modifier = display_rotation == DXGI_MODE_ROTATION_UNSPECIFIED ? 0 : display_rotation - 1;
+      int32_t rotation_data[16 / sizeof(int32_t)] { rotation_modifier };  // aligned to 16-byte
+      auto rotation = make_buffer(device.get(), rotation_data);
+      if (!rotation) {
+        BOOST_LOG(error) << "Failed to create display rotation vertex constant buffer";
+        return -1;
+      }
+      device_ctx->VSSetConstantBuffers(2, 1, &rotation);
+    }
+
     if (config.dynamicRange && is_hdr()) {
       // This shader will normalize scRGB white levels to a user-defined white level
-      status = device->CreatePixelShader(scene_NW_ps_hlsl->GetBufferPointer(), scene_NW_ps_hlsl->GetBufferSize(), nullptr, &scene_ps);
+      status = device->CreatePixelShader(scene_NW_ps_hlsl->GetBufferPointer(), scene_NW_ps_hlsl->GetBufferSize(), nullptr, &cursor_ps);
       if (status) {
         BOOST_LOG(error) << "Failed to create scene pixel shader [0x"sv << util::hex(status).to_string_view() << ']';
         return -1;
@@ -1365,7 +1395,7 @@ namespace platf::dxgi {
       device_ctx->PSSetConstantBuffers(1, 1, &sdr_multiplier);
     }
     else {
-      status = device->CreatePixelShader(scene_ps_hlsl->GetBufferPointer(), scene_ps_hlsl->GetBufferSize(), nullptr, &scene_ps);
+      status = device->CreatePixelShader(scene_ps_hlsl->GetBufferPointer(), scene_ps_hlsl->GetBufferSize(), nullptr, &cursor_ps);
       if (status) {
         BOOST_LOG(error) << "Failed to create scene pixel shader [0x"sv << util::hex(status).to_string_view() << ']';
         return -1;
@@ -1392,8 +1422,8 @@ namespace platf::dxgi {
     auto img = std::make_shared<img_d3d_t>();
 
     // Initialize format-independent fields
-    img->width = width;
-    img->height = height;
+    img->width = width_before_rotation;
+    img->height = height_before_rotation;
     img->id = next_image_id++;
 
     return img;
@@ -1645,6 +1675,11 @@ namespace platf::dxgi {
       return -1;
     }
 
+    cursor_vs_hlsl = compile_vertex_shader(SUNSHINE_SHADERS_DIR "/CursorVS.hlsl");
+    if (!cursor_vs_hlsl) {
+      return -1;
+    }
+
     convert_Y_ps_hlsl = compile_pixel_shader(SUNSHINE_SHADERS_DIR "/ConvertYPS.hlsl");
     if (!convert_Y_ps_hlsl) {
       return -1;

src_assets/windows/assets/shaders/directx/ConvertUVVS.hlsl

@@ -1,29 +1,46 @@
 struct VertTexPosWide {
-	float3 uuv : TEXCOORD;
-	float4 pos : SV_POSITION;
+    float3 uuv : TEXCOORD;
+    float4 pos : SV_POSITION;
 };
 
 cbuffer info : register(b0) {
-	float width_i;
+    float width_i;
+};
+
+cbuffer rotation_info : register(b1) {
+    int rotation;
 };
 
 //--------------------------------------------------------------------------------------
 // Vertex Shader
 //--------------------------------------------------------------------------------------
 VertTexPosWide main_vs(uint vI : SV_VERTEXID)
 {
-	float idHigh = float(vI >> 1);
-	float idLow = float(vI & uint(1));
+    VertTexPosWide output;
+    float2 tex;
+
+    if (vI == 0) {
+        output.pos = float4(-1, -1, 0, 1);
+        tex = float2(0, 1);
+    }
+    else if (vI == 1) {
+        output.pos = float4(-1, 3, 0, 1);
+        tex = float2(0, -1);
+    }
+    else if (vI == 2) {
+        output.pos = float4(3, -1, 0, 1);
+        tex = float2(2, 1);
+    }
 
-	float x = idHigh * 4.0 - 1.0;
-	float y = idLow * 4.0 - 1.0;
+    if (rotation != 0) {
+        float rotation_radians = radians(90 * rotation);
+        float2x2 rotation_matrix = { cos(rotation_radians), -sin(rotation_radians),
+                                     sin(rotation_radians), cos(rotation_radians) };
+        float2 rotation_center = { 0.5, 0.5 };
+        tex = round(rotation_center + mul(rotation_matrix, tex - rotation_center));
+    }
 
-	float u_right = idHigh * 2.0;
-	float u_left = u_right - width_i;
-	float v = 1.0 - idLow * 2.0;
+    output.uuv = float3(tex.x, tex.x - width_i, tex.y);
 
-	VertTexPosWide vert_out;
-	vert_out.uuv = float3(u_left, u_right, v);
-	vert_out.pos = float4(x, y, 0.0, 1.0);
-	return vert_out;
-}
+    return output;
+}