From 68f9a89857d0bcca9a87b82637ce79f88ec97ec3 Mon Sep 17 00:00:00 2001
From: ns6089 <61738816+ns6089@users.noreply.github.com>
Date: Sun, 10 Sep 2023 11:55:02 +0300
Subject: [PATCH] nvenc: generate less IDR frames if rfi is delayed

If IDR frame was generated due too rfi request being too large, we
didn't save it in last rfi range.

Also move rfi_needs_confirmation variable modification to a more
logical place, even if doesn't affect how current code works.
Any unfulfilled rfi request needs confirmation, whether it's direct rfi
flag in transported frame or indirect IDR frame.
---
 src/nvenc/nvenc_base.cpp | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/nvenc/nvenc_base.cpp b/src/nvenc/nvenc_base.cpp
index b3bc923b522..f305f7682ad 100644
--- a/src/nvenc/nvenc_base.cpp
+++ b/src/nvenc/nvenc_base.cpp
@@ -500,6 +500,8 @@ namespace nvenc {
       return true;
     }
 
+    encoder_state.rfi_needs_confirmation = true;
+
     if (last_frame < first_frame) {
       BOOST_LOG(error) << "NvEnc: invaid rfi request " << first_frame << "-" << last_frame << ", generating IDR";
       return false;
@@ -508,14 +510,13 @@ namespace nvenc {
     BOOST_LOG(debug) << "NvEnc: rfi request " << first_frame << "-" << last_frame << " expanding to last encoded frame " << encoder_state.last_encoded_frame_index;
     last_frame = encoder_state.last_encoded_frame_index;
 
+    encoder_state.last_rfi_range = { first_frame, last_frame };
+
     if (last_frame - first_frame + 1 >= encoder_params.ref_frames_in_dpb) {
       BOOST_LOG(debug) << "NvEnc: rfi request too large, generating IDR";
       return false;
     }
 
-    encoder_state.rfi_needs_confirmation = true;
-    encoder_state.last_rfi_range = { first_frame, last_frame };
-
     for (auto i = first_frame; i <= last_frame; i++) {
       if (nvenc_failed(nvenc->nvEncInvalidateRefFrames(encoder, i))) {
         BOOST_LOG(error) << "NvEncInvalidateRefFrames " << i << " failed: " << last_error_string;