From e98d7577bb42763e17b88481feed1eb5a29331e2 Mon Sep 17 00:00:00 2001 From: ns6089 <61738816+ns6089@users.noreply.github.com> Date: Wed, 13 Sep 2023 17:34:26 +0300 Subject: [PATCH] Support capturing rotated displays on Windows (#1602) --- src/nvenc/nvenc_base.cpp | 7 +- src/platform/windows/display.h | 72 ++++++++++++++++--- src/platform/windows/display_base.cpp | 11 +++ src/platform/windows/display_vram.cpp | 63 ++++++++++++---- .../assets/shaders/directx/ConvertUVVS.hlsl | 47 ++++++++---- .../assets/shaders/directx/CursorVS.hlsl | 37 ++++++++++ .../assets/shaders/directx/SceneVS.hlsl | 39 ++++++---- 7 files changed, 222 insertions(+), 54 deletions(-) create mode 100644 src_assets/windows/assets/shaders/directx/CursorVS.hlsl diff --git a/src/nvenc/nvenc_base.cpp b/src/nvenc/nvenc_base.cpp index b3bc923b522..f305f7682ad 100644 --- a/src/nvenc/nvenc_base.cpp +++ b/src/nvenc/nvenc_base.cpp @@ -500,6 +500,8 @@ namespace nvenc { return true; } + encoder_state.rfi_needs_confirmation = true; + if (last_frame < first_frame) { BOOST_LOG(error) << "NvEnc: invaid rfi request " << first_frame << "-" << last_frame << ", generating IDR"; return false; @@ -508,14 +510,13 @@ namespace nvenc { BOOST_LOG(debug) << "NvEnc: rfi request " << first_frame << "-" << last_frame << " expanding to last encoded frame " << encoder_state.last_encoded_frame_index; last_frame = encoder_state.last_encoded_frame_index; + encoder_state.last_rfi_range = { first_frame, last_frame }; + if (last_frame - first_frame + 1 >= encoder_params.ref_frames_in_dpb) { BOOST_LOG(debug) << "NvEnc: rfi request too large, generating IDR"; return false; } - encoder_state.rfi_needs_confirmation = true; - encoder_state.last_rfi_range = { first_frame, last_frame }; - for (auto i = first_frame; i <= last_frame; i++) { if (nvenc_failed(nvenc->nvEncInvalidateRefFrames(encoder, i))) { BOOST_LOG(error) << "NvEncInvalidateRefFrames " << i << " failed: " << last_error_string; diff --git a/src/platform/windows/display.h b/src/platform/windows/display.h index cd21081221a..2d480c5954c 100644 --- a/src/platform/windows/display.h +++ b/src/platform/windows/display.h @@ -81,23 +81,71 @@ namespace platf::dxgi { public: gpu_cursor_t(): cursor_view { 0, 0, 0, 0, 0.0f, 1.0f } {}; - void - set_pos(LONG rel_x, LONG rel_y, bool visible) { - cursor_view.TopLeftX = rel_x; - cursor_view.TopLeftY = rel_y; + void + set_pos(LONG topleft_x, LONG topleft_y, LONG display_width, LONG display_height, DXGI_MODE_ROTATION display_rotation, bool visible) { + this->topleft_x = topleft_x; + this->topleft_y = topleft_y; + this->display_width = display_width; + this->display_height = display_height; + this->display_rotation = display_rotation; this->visible = visible; + update_viewport(); } void - set_texture(LONG width, LONG height, texture2d_t &&texture) { - cursor_view.Width = width; - cursor_view.Height = height; - + set_texture(LONG texture_width, LONG texture_height, texture2d_t &&texture) { this->texture = std::move(texture); + this->texture_width = texture_width; + this->texture_height = texture_height; + update_viewport(); + } + + void + update_viewport() { + switch (display_rotation) { + case DXGI_MODE_ROTATION_UNSPECIFIED: + case DXGI_MODE_ROTATION_IDENTITY: + cursor_view.TopLeftX = topleft_x; + cursor_view.TopLeftY = topleft_y; + cursor_view.Width = texture_width; + cursor_view.Height = texture_height; + break; + + case DXGI_MODE_ROTATION_ROTATE90: + cursor_view.TopLeftX = topleft_y; + cursor_view.TopLeftY = display_width - texture_width - topleft_x; + cursor_view.Width = texture_height; + cursor_view.Height = texture_width; + break; + + case DXGI_MODE_ROTATION_ROTATE180: + cursor_view.TopLeftX = display_width - texture_width - topleft_x; + cursor_view.TopLeftY = display_height - texture_height - topleft_y; + cursor_view.Width = texture_width; + cursor_view.Height = texture_height; + break; + + case DXGI_MODE_ROTATION_ROTATE270: + cursor_view.TopLeftX = display_height - texture_height - topleft_y; + cursor_view.TopLeftY = topleft_x; + cursor_view.Width = texture_height; + cursor_view.Height = texture_width; + break; + } } texture2d_t texture; + LONG texture_width; + LONG texture_height; + + LONG topleft_x; + LONG topleft_y; + + LONG display_width; + LONG display_height; + DXGI_MODE_ROTATION display_rotation; + shader_res_t input_res; D3D11_VIEWPORT cursor_view; @@ -141,6 +189,10 @@ namespace platf::dxgi { DXGI_RATIONAL display_refresh_rate; int display_refresh_rate_rounded; + DXGI_MODE_ROTATION display_rotation = DXGI_MODE_ROTATION_UNSPECIFIED; + int width_before_rotation; + int height_before_rotation; + int client_frame_rate; DXGI_FORMAT capture_format; @@ -277,8 +329,8 @@ namespace platf::dxgi { blend_t blend_invert; blend_t blend_disable; - ps_t scene_ps; - vs_t scene_vs; + ps_t cursor_ps; + vs_t cursor_vs; gpu_cursor_t cursor_alpha; gpu_cursor_t cursor_xor; diff --git a/src/platform/windows/display_base.cpp b/src/platform/windows/display_base.cpp index 30f85224d78..50f72698859 100644 --- a/src/platform/windows/display_base.cpp +++ b/src/platform/windows/display_base.cpp @@ -461,6 +461,17 @@ namespace platf::dxgi { width = desc.DesktopCoordinates.right - offset_x; height = desc.DesktopCoordinates.bottom - offset_y; + display_rotation = desc.Rotation; + if (display_rotation == DXGI_MODE_ROTATION_ROTATE90 || + display_rotation == DXGI_MODE_ROTATION_ROTATE270) { + width_before_rotation = height; + height_before_rotation = width; + } + else { + width_before_rotation = width; + height_before_rotation = height; + } + // left and bottom may be negative, yet absolute mouse coordinates start at 0x0 // Ensure offset starts at 0x0 offset_x -= GetSystemMetrics(SM_XVIRTUALSCREEN); diff --git a/src/platform/windows/display_vram.cpp b/src/platform/windows/display_vram.cpp index 5c0ead06ee1..bf8fb569160 100644 --- a/src/platform/windows/display_vram.cpp +++ b/src/platform/windows/display_vram.cpp @@ -107,6 +107,7 @@ namespace platf::dxgi { blob_t convert_UV_linear_ps_hlsl; blob_t convert_UV_PQ_ps_hlsl; blob_t scene_vs_hlsl; + blob_t cursor_vs_hlsl; blob_t convert_Y_ps_hlsl; blob_t convert_Y_linear_ps_hlsl; blob_t convert_Y_PQ_ps_hlsl; @@ -472,6 +473,17 @@ namespace platf::dxgi { } device_ctx->VSSetConstantBuffers(0, 1, &info_scene); + { + int32_t rotation_modifier = display->display_rotation == DXGI_MODE_ROTATION_UNSPECIFIED ? 0 : display->display_rotation - 1; + int32_t rotation_data[16 / sizeof(int32_t)] { -rotation_modifier }; // aligned to 16-byte + auto rotation = make_buffer(device.get(), rotation_data); + if (!rotation) { + BOOST_LOG(error) << "Failed to create display rotation vertex constant buffer"; + return -1; + } + device_ctx->VSSetConstantBuffers(1, 1, &rotation); + } + D3D11_RENDER_TARGET_VIEW_DESC nv12_rt_desc { format == DXGI_FORMAT_P010 ? DXGI_FORMAT_R16_UNORM : DXGI_FORMAT_R8_UNORM, D3D11_RTV_DIMENSION_TEXTURE2D @@ -618,7 +630,11 @@ namespace platf::dxgi { convert_UV_vs_hlsl->GetBufferPointer(), convert_UV_vs_hlsl->GetBufferSize(), &input_layout); - this->display = std::move(display); + this->display = std::dynamic_pointer_cast(display); + if (!this->display) { + return -1; + } + display = nullptr; blend_disable = make_blend(device.get(), false, false); if (!blend_disable) { @@ -735,7 +751,7 @@ namespace platf::dxgi { // amongst multiple hwdevice_t objects (and therefore multiple ID3D11Devices). std::map img_ctx_map; - std::shared_ptr display; + std::shared_ptr display; vs_t convert_UV_vs; ps_t convert_UV_ps; @@ -988,8 +1004,11 @@ namespace platf::dxgi { } if (frame_info.LastMouseUpdateTime.QuadPart) { - cursor_alpha.set_pos(frame_info.PointerPosition.Position.x, frame_info.PointerPosition.Position.y, frame_info.PointerPosition.Visible); - cursor_xor.set_pos(frame_info.PointerPosition.Position.x, frame_info.PointerPosition.Position.y, frame_info.PointerPosition.Visible); + cursor_alpha.set_pos(frame_info.PointerPosition.Position.x, frame_info.PointerPosition.Position.y, + width, height, display_rotation, frame_info.PointerPosition.Visible); + + cursor_xor.set_pos(frame_info.PointerPosition.Position.x, frame_info.PointerPosition.Position.y, + width, height, display_rotation, frame_info.PointerPosition.Visible); } const bool blend_mouse_cursor_flag = (cursor_alpha.visible || cursor_xor.visible) && cursor_visible; @@ -1008,7 +1027,7 @@ namespace platf::dxgi { // It's possible for our display enumeration to race with mode changes and result in // mismatched image pool and desktop texture sizes. If this happens, just reinit again. - if (desc.Width != width || desc.Height != height) { + if (desc.Width != width_before_rotation || desc.Height != height_before_rotation) { BOOST_LOG(info) << "Capture size changed ["sv << width << 'x' << height << " -> "sv << desc.Width << 'x' << desc.Height << ']'; return capture_e::reinit; } @@ -1109,8 +1128,8 @@ namespace platf::dxgi { // Otherwise create a new surface. D3D11_TEXTURE2D_DESC t {}; - t.Width = width; - t.Height = height; + t.Width = width_before_rotation; + t.Height = height_before_rotation; t.MipLevels = 1; t.ArraySize = 1; t.SampleDesc.Count = 1; @@ -1217,8 +1236,8 @@ namespace platf::dxgi { } auto blend_cursor = [&](img_d3d_t &d3d_img) { - device_ctx->VSSetShader(scene_vs.get(), nullptr, 0); - device_ctx->PSSetShader(scene_ps.get(), nullptr, 0); + device_ctx->VSSetShader(cursor_vs.get(), nullptr, 0); + device_ctx->PSSetShader(cursor_ps.get(), nullptr, 0); device_ctx->OMSetRenderTargets(1, &d3d_img.capture_rt, nullptr); if (cursor_alpha.texture.get()) { @@ -1338,15 +1357,26 @@ namespace platf::dxgi { return -1; } - status = device->CreateVertexShader(scene_vs_hlsl->GetBufferPointer(), scene_vs_hlsl->GetBufferSize(), nullptr, &scene_vs); + status = device->CreateVertexShader(cursor_vs_hlsl->GetBufferPointer(), cursor_vs_hlsl->GetBufferSize(), nullptr, &cursor_vs); if (status) { BOOST_LOG(error) << "Failed to create scene vertex shader [0x"sv << util::hex(status).to_string_view() << ']'; return -1; } + { + int32_t rotation_modifier = display_rotation == DXGI_MODE_ROTATION_UNSPECIFIED ? 0 : display_rotation - 1; + int32_t rotation_data[16 / sizeof(int32_t)] { rotation_modifier }; // aligned to 16-byte + auto rotation = make_buffer(device.get(), rotation_data); + if (!rotation) { + BOOST_LOG(error) << "Failed to create display rotation vertex constant buffer"; + return -1; + } + device_ctx->VSSetConstantBuffers(2, 1, &rotation); + } + if (config.dynamicRange && is_hdr()) { // This shader will normalize scRGB white levels to a user-defined white level - status = device->CreatePixelShader(scene_NW_ps_hlsl->GetBufferPointer(), scene_NW_ps_hlsl->GetBufferSize(), nullptr, &scene_ps); + status = device->CreatePixelShader(scene_NW_ps_hlsl->GetBufferPointer(), scene_NW_ps_hlsl->GetBufferSize(), nullptr, &cursor_ps); if (status) { BOOST_LOG(error) << "Failed to create scene pixel shader [0x"sv << util::hex(status).to_string_view() << ']'; return -1; @@ -1365,7 +1395,7 @@ namespace platf::dxgi { device_ctx->PSSetConstantBuffers(1, 1, &sdr_multiplier); } else { - status = device->CreatePixelShader(scene_ps_hlsl->GetBufferPointer(), scene_ps_hlsl->GetBufferSize(), nullptr, &scene_ps); + status = device->CreatePixelShader(scene_ps_hlsl->GetBufferPointer(), scene_ps_hlsl->GetBufferSize(), nullptr, &cursor_ps); if (status) { BOOST_LOG(error) << "Failed to create scene pixel shader [0x"sv << util::hex(status).to_string_view() << ']'; return -1; @@ -1392,8 +1422,8 @@ namespace platf::dxgi { auto img = std::make_shared(); // Initialize format-independent fields - img->width = width; - img->height = height; + img->width = width_before_rotation; + img->height = height_before_rotation; img->id = next_image_id++; return img; @@ -1645,6 +1675,11 @@ namespace platf::dxgi { return -1; } + cursor_vs_hlsl = compile_vertex_shader(SUNSHINE_SHADERS_DIR "/CursorVS.hlsl"); + if (!cursor_vs_hlsl) { + return -1; + } + convert_Y_ps_hlsl = compile_pixel_shader(SUNSHINE_SHADERS_DIR "/ConvertYPS.hlsl"); if (!convert_Y_ps_hlsl) { return -1; diff --git a/src_assets/windows/assets/shaders/directx/ConvertUVVS.hlsl b/src_assets/windows/assets/shaders/directx/ConvertUVVS.hlsl index 77ff38d7deb..f36128fefcd 100644 --- a/src_assets/windows/assets/shaders/directx/ConvertUVVS.hlsl +++ b/src_assets/windows/assets/shaders/directx/ConvertUVVS.hlsl @@ -1,10 +1,14 @@ struct VertTexPosWide { - float3 uuv : TEXCOORD; - float4 pos : SV_POSITION; + float3 uuv : TEXCOORD; + float4 pos : SV_POSITION; }; cbuffer info : register(b0) { - float width_i; + float width_i; +}; + +cbuffer rotation_info : register(b1) { + int rotation; }; //-------------------------------------------------------------------------------------- @@ -12,18 +16,31 @@ cbuffer info : register(b0) { //-------------------------------------------------------------------------------------- VertTexPosWide main_vs(uint vI : SV_VERTEXID) { - float idHigh = float(vI >> 1); - float idLow = float(vI & uint(1)); + VertTexPosWide output; + float2 tex; + + if (vI == 0) { + output.pos = float4(-1, -1, 0, 1); + tex = float2(0, 1); + } + else if (vI == 1) { + output.pos = float4(-1, 3, 0, 1); + tex = float2(0, -1); + } + else if (vI == 2) { + output.pos = float4(3, -1, 0, 1); + tex = float2(2, 1); + } - float x = idHigh * 4.0 - 1.0; - float y = idLow * 4.0 - 1.0; + if (rotation != 0) { + float rotation_radians = radians(90 * rotation); + float2x2 rotation_matrix = { cos(rotation_radians), -sin(rotation_radians), + sin(rotation_radians), cos(rotation_radians) }; + float2 rotation_center = { 0.5, 0.5 }; + tex = round(rotation_center + mul(rotation_matrix, tex - rotation_center)); + } - float u_right = idHigh * 2.0; - float u_left = u_right - width_i; - float v = 1.0 - idLow * 2.0; + output.uuv = float3(tex.x, tex.x - width_i, tex.y); - VertTexPosWide vert_out; - vert_out.uuv = float3(u_left, u_right, v); - vert_out.pos = float4(x, y, 0.0, 1.0); - return vert_out; -} \ No newline at end of file + return output; +} diff --git a/src_assets/windows/assets/shaders/directx/CursorVS.hlsl b/src_assets/windows/assets/shaders/directx/CursorVS.hlsl new file mode 100644 index 00000000000..a93935d7496 --- /dev/null +++ b/src_assets/windows/assets/shaders/directx/CursorVS.hlsl @@ -0,0 +1,37 @@ +struct PS_INPUT +{ + float4 pos : SV_POSITION; + float2 tex : TEXCOORD; +}; + +cbuffer rotation_info : register(b2) { + int rotation; +}; + +PS_INPUT main_vs(uint vI : SV_VERTEXID) +{ + PS_INPUT output; + + if (vI == 0) { + output.pos = float4(-1, -1, 0, 1); + output.tex = float2(0, 1); + } + else if (vI == 1) { + output.pos = float4(-1, 3, 0, 1); + output.tex = float2(0, -1); + } + else if (vI == 2) { + output.pos = float4(3, -1, 0, 1); + output.tex = float2(2, 1); + } + + if (rotation != 0) { + float rotation_radians = radians(90 * rotation); + float2x2 rotation_matrix = { cos(rotation_radians), -sin(rotation_radians), + sin(rotation_radians), cos(rotation_radians) }; + float2 rotation_center = { 0.5, 0.5 }; + output.tex = round(rotation_center + mul(rotation_matrix, output.tex - rotation_center)); + } + + return output; +} \ No newline at end of file diff --git a/src_assets/windows/assets/shaders/directx/SceneVS.hlsl b/src_assets/windows/assets/shaders/directx/SceneVS.hlsl index 3afaffc60f9..3e72aa0cd0d 100644 --- a/src_assets/windows/assets/shaders/directx/SceneVS.hlsl +++ b/src_assets/windows/assets/shaders/directx/SceneVS.hlsl @@ -1,22 +1,37 @@ struct PS_INPUT { - float4 pos : SV_POSITION; - float2 tex : TEXCOORD; + float4 pos : SV_POSITION; + float2 tex : TEXCOORD; +}; + +cbuffer rotation_info : register(b1) { + int rotation; }; PS_INPUT main_vs(uint vI : SV_VERTEXID) { - float idHigh = float(vI >> 1); - float idLow = float(vI & uint(1)); + PS_INPUT output; - float x = idHigh * 4.0 - 1.0; - float y = idLow * 4.0 - 1.0; + if (vI == 0) { + output.pos = float4(-1, -1, 0, 1); + output.tex = float2(0, 1); + } + else if (vI == 1) { + output.pos = float4(-1, 3, 0, 1); + output.tex = float2(0, -1); + } + else if (vI == 2) { + output.pos = float4(3, -1, 0, 1); + output.tex = float2(2, 1); + } - float u = idHigh * 2.0; - float v = 1.0 - idLow * 2.0; + if (rotation != 0) { + float rotation_radians = radians(90 * rotation); + float2x2 rotation_matrix = { cos(rotation_radians), -sin(rotation_radians), + sin(rotation_radians), cos(rotation_radians) }; + float2 rotation_center = { 0.5, 0.5 }; + output.tex = round(rotation_center + mul(rotation_matrix, output.tex - rotation_center)); + } - PS_INPUT vert_out; - vert_out.pos = float4(x, y, 0.0, 1.0); - vert_out.tex = float2(u, v); - return vert_out; + return output; } \ No newline at end of file