WIP Fixes for transformed outputs with mirroring

src/backend/kms/surface/mod.rs

@@ -77,7 +77,7 @@ use tracing::{error, trace, warn};
 
 use std::{
     borrow::BorrowMut,
-    collections::{HashMap, HashSet},
+    collections::{hash_map, HashMap, HashSet},
     mem,
     sync::{
         atomic::{AtomicBool, Ordering},
@@ -147,41 +147,72 @@ pub struct SurfaceThreadState {
     egui: EguiState,
 }
 
+#[derive(Debug, PartialEq)]
+struct MirroringOutputConfig {
+    size: Size<i32, Physical>,
+    fractional_scale: f64,
+}
+
+impl MirroringOutputConfig {
+    fn for_output_untransformed(output: &Output) -> Self {
+        Self {
+            // Apply inverse of output transform to mode size to get correct size
+            // for an untransformed render.
+            size: output.current_transform().invert().transform_size(
+                output
+                    .current_mode()
+                    .map(|mode| mode.size)
+                    .unwrap_or_default(),
+            ),
+            fractional_scale: output.current_scale().fractional_scale(),
+        }
+    }
+
+    fn for_output(output: &Output) -> Self {
+        Self {
+            size: output
+                .current_mode()
+                .map(|mode| mode.size)
+                .unwrap_or_default(),
+            fractional_scale: output.current_scale().fractional_scale(),
+        }
+    }
+}
+
 #[derive(Debug)]
 struct MirroringState {
     texture: TextureRenderBuffer<GlesTexture>,
     damage_tracker: OutputDamageTracker,
+    output_config: MirroringOutputConfig,
 }
 
 impl MirroringState {
     fn new_with_renderer(
         renderer: &mut GlMultiRenderer,
         format: Fourcc,
-        output: &Output,
+        output_config: MirroringOutputConfig,
     ) -> Result<Self> {
-        let size = output
-            .current_mode()
-            .map(|mode| mode.size)
-            .unwrap_or_default()
-            .to_logical(1)
-            .to_buffer(1, Transform::Normal);
-        let opaque_regions = vec![Rectangle::from_loc_and_size((0, 0), size)];
-
-        let texture = Offscreen::<GlesTexture>::create_buffer(renderer, format, size)?;
-        let transform = output.current_transform();
+        let size = output_config.size;
+        let buffer_size = size.to_logical(1).to_buffer(1, Transform::Normal);
+        let opaque_regions = vec![Rectangle::from_loc_and_size((0, 0), buffer_size)];
+
+        let texture = Offscreen::<GlesTexture>::create_buffer(renderer, format, buffer_size)?;
         let texture_buffer = TextureRenderBuffer::from_texture(
             renderer,
             texture,
             1,
-            transform,
+            Transform::Normal,
             Some(opaque_regions),
         );
 
-        let damage_tracker = OutputDamageTracker::from_output(output);
+        // Don't use `from_output` to avoid applying output transform
+        let damage_tracker =
+            OutputDamageTracker::new(size, output_config.fractional_scale, Transform::Normal);
 
         Ok(MirroringState {
             texture: texture_buffer,
             damage_tracker,
+            output_config,
         })
     }
 }
@@ -1083,25 +1114,30 @@ impl SurfaceThreadState {
 
         // actual rendering
         let res = if let Some(mirrored_output) = self.mirroring.as_ref().filter(|mirrored_output| {
-            mirrored_output.current_mode().is_some_and(|mirror_mode| {
-                self.output
-                    .current_mode()
-                    .is_some_and(|mode| mode != mirror_mode)
-            }) || mirrored_output.current_scale().fractional_scale()
-                != self.output.current_scale().fractional_scale()
+            MirroringOutputConfig::for_output_untransformed(mirrored_output)
+                != MirroringOutputConfig::for_output(&self.output)
         }) {
-            let mirroring_state = {
-                let entry = self.mirroring_textures.entry(self.target_node);
-                let mut new_state = None;
-                if matches!(entry, std::collections::hash_map::Entry::Vacant(_)) {
-                    new_state = Some(MirroringState::new_with_renderer(
+            let mirrored_output_config =
+                MirroringOutputConfig::for_output_untransformed(mirrored_output);
+            let mirroring_state = match self.mirroring_textures.entry(self.target_node) {
+                hash_map::Entry::Occupied(occupied) => {
+                    let mirroring_state = occupied.into_mut();
+                    if mirroring_state.output_config != mirrored_output_config {
+                        *mirroring_state = MirroringState::new_with_renderer(
+                            &mut renderer,
+                            compositor.format(),
+                            mirrored_output_config,
+                        )?
+                    }
+                    mirroring_state
+                }
+                hash_map::Entry::Vacant(vacant) => {
+                    vacant.insert(MirroringState::new_with_renderer(
                         &mut renderer,
                         compositor.format(),
-                        mirrored_output,
-                    )?);
+                        mirrored_output_config,
+                    )?)
                 }
-                // I really want a failable initializer...
-                entry.or_insert_with(|| new_state.unwrap())
             };
 
             mirroring_state