Merge pull request #55 from nus-vv-streams/feat/upsample-vvplay-integ…

…ration

Feat/upsample vvplay integration

README.md

@@ -575,7 +575,7 @@ Options:
       --decoder <DECODER_TYPE>       [default: noop] [possible values: noop, draco]
       --decoder-path <DECODER_PATH>  
       --bg-color <BG_COLOR>          [default: rgb(255,255,255)]
-  --lodify                           [default: False]
+  --adaptive-upsampling              [default: False]
   -h, --help                         Print help
 ```
 

src/bin/vvplay.rs

@@ -4,7 +4,7 @@ use std::path::Path;
 
 use vivotk::render::wgpu::{
     builder::RenderBuilder, camera::Camera, controls::Controller, metrics_reader::MetricsReader,
-    render_manager::AdaptiveManager, renderer::Renderer,
+    render_manager::AdaptiveUpsamplingManager, renderer::Renderer,
 };
 
 /// Plays a folder of pcd files in lexicographical order
@@ -60,7 +60,7 @@ struct Args {
     #[clap(long, default_value = "rgb(255,255,255)")]
     bg_color: OsString,
     #[clap(long, default_value = "false")]
-    lod: bool,
+    adaptive_upsampling: bool,
 }
 
 #[derive(clap::ValueEnum, Clone, Copy)]
@@ -71,7 +71,7 @@ enum DecoderType {
 
 fn main() {
     let args: Args = Args::parse();
-    let adaptive_manager = AdaptiveManager::new(&args.src, args.lod);
+    let adaptive_manager = AdaptiveUpsamplingManager::new(&args.src, args.adaptive_upsampling);
 
     let camera = Camera::new(
         (args.camera_x, args.camera_y, args.camera_z),

src/formats/pointxyzrgba.rs

@@ -2,6 +2,7 @@ use serde::{
     ser::{Serialize, SerializeStruct, Serializer},
     Deserialize,
 };
+use std::cmp::Ordering;
 #[repr(C)]
 #[derive(Debug, Copy, Clone, PartialEq, bytemuck::Pod, bytemuck::Zeroable, Deserialize)]
 pub struct PointXyzRgba {
@@ -13,6 +14,24 @@ pub struct PointXyzRgba {
     pub b: u8,
     pub a: u8,
 }
+
+impl PartialOrd for PointXyzRgba {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl Ord for PointXyzRgba {
+    fn cmp(&self, other: &Self) -> Ordering {
+        self.x
+            .total_cmp(&other.x)
+            .then_with(|| self.y.total_cmp(&other.y))
+            .then_with(|| self.z.total_cmp(&other.z))
+    }
+}
+
+impl Eq for PointXyzRgba {}
+
 impl Serialize for PointXyzRgba {
     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
     where

src/pipeline/subcommands/read.rs

@@ -44,7 +44,6 @@ impl Read {
 
 impl Subcommand for Read {
     fn handle(&mut self, messages: Vec<PipelineMessage>, channel: &Channel) {
-        println!("Reading files");
         if messages.is_empty() {
             let mut files = find_all_files(&self.args.files);
             files.sort();

src/pipeline/subcommands/upsample.rs

@@ -5,7 +5,9 @@ use std::time::Instant;
 use crate::{
     pipeline::{channel::Channel, PipelineMessage},
     reconstruct::poisson_reconstruct::reconstruct,
-    upsample::{interpolate::upsample, upsample_methods::UpsampleMethod},
+    upsample::{
+        interpolate::upsample, interpolate::upsample_grid, upsample_methods::UpsampleMethod,
+    },
 };
 
 use super::Subcommand;
@@ -68,8 +70,9 @@ impl Subcommand for Upsampler {
         for message in messages {
             match message {
                 PipelineMessage::IndexedPointCloud(pc, i) => {
-                    println!("Doing upsample");
-                    let upsampled_pc = upsample(pc, self.factor);
+                    let upsampled_pc = upsample_grid(pc, self.factor);
+                    // println!("Doing upsample");
+                    // let upsampled_pc = upsample(pc, self.factor);
                     channel.send(PipelineMessage::IndexedPointCloud(upsampled_pc, i));
                 }
                 PipelineMessage::SubcommandMessage(subcommand_object, i) => {

src/render/wgpu/camera.rs

@@ -6,10 +6,10 @@ use wgpu::util::DeviceExt;
 use winit::dpi::PhysicalPosition;
 use winit::event::*;
 
-const CAMERA_SPEED: f32 = 2.0;
-const CAMERA_SENSITIVITY: f32 = 0.5;
+const CAMERA_SPEED: f32 = 1.0;
+const CAMERA_SENSITIVITY: f32 = 0.2;
 const PROJECTION_FOXY: f32 = 45.0;
-const PROJECTION_ZNEAR: f32 = 0.1;
+const PROJECTION_ZNEAR: f32 = 0.001;
 const PROJECTION_ZFAR: f32 = 100.0;
 
 #[derive(Clone)]
@@ -169,8 +169,8 @@ impl CameraState {
 /// Create a uniform buffer: a blob of data that is available to every invocation of a set of shaders.
 /// This buffer is used to store our view projection matrix
 pub struct CameraUniform {
-    view_position: [f32; 4],
-    view_proj: [[f32; 4]; 4],
+    pub view_position: [f32; 4],
+    pub view_proj: [[f32; 4]; 4],
 }
 
 impl Default for CameraUniform {

src/render/wgpu/mod.rs

@@ -10,3 +10,4 @@ pub mod render_manager;
 pub mod renderable;
 pub mod renderer;
 pub mod resolution_controller;
+pub mod upsampler;

src/render/wgpu/render_manager.rs

@@ -6,9 +6,10 @@ use std::path::Path;
 use std::process::exit;
 
 use super::camera::CameraState;
-use super::reader::{LODFileReader, RenderReader};
+use super::reader::{LODFileReader, PointCloudFileReader, RenderReader};
 use super::renderable::Renderable;
 use super::resolution_controller::ResolutionController;
+use super::upsampler::Upsampler;
 
 pub trait RenderManager<T: Renderable> {
     fn start(&mut self) -> Option<T>;
@@ -20,7 +21,7 @@ pub trait RenderManager<T: Renderable> {
     fn should_redraw(&mut self, camera_state: &CameraState) -> bool;
 }
 
-pub struct AdaptiveManager {
+pub struct AdaptiveLODManager {
     reader: LODFileReader,
 
     // For adaptive loading
@@ -35,48 +36,7 @@ pub struct AdaptiveManager {
     additional_points_loaded: Vec<usize>,
 }
 
-fn infer_format(src: &String) -> String {
-    let choices = ["pcd", "ply", "bin", "http"];
-    const PCD: usize = 0;
-    const PLY: usize = 1;
-    const BIN: usize = 2;
-
-    if choices.contains(&src.as_str()) {
-        return src.clone();
-    }
-
-    let path = Path::new(src);
-    // infer by counting extension numbers (pcd ply and bin)
-
-    let mut choice_count = [0, 0, 0];
-    for file_entry in path.read_dir().unwrap() {
-        match file_entry {
-            Ok(entry) => {
-                if let Some(ext) = entry.path().extension() {
-                    if ext.eq("pcd") {
-                        choice_count[PCD] += 1;
-                    } else if ext.eq("ply") {
-                        choice_count[PLY] += 1;
-                    } else if ext.eq("bin") {
-                        choice_count[BIN] += 1;
-                    }
-                }
-            }
-            Err(e) => {
-                eprintln!("{e}")
-            }
-        }
-    }
-
-    let max_index = choice_count
-        .iter()
-        .enumerate()
-        .max_by_key(|(_, &item)| item)
-        .map(|(index, _)| index);
-    choices[max_index.unwrap()].to_string()
-}
-
-impl AdaptiveManager {
+impl AdaptiveLODManager {
     pub fn new(src: &String, lod: bool) -> Self {
         let base_path = if lod {
             src.clone() + "/base"
@@ -149,10 +109,7 @@ impl AdaptiveManager {
     }
 
     pub fn get_desired_point_cloud(&mut self, index: usize) -> Option<PointCloud<PointXyzRgba>> {
-        // let now = std::time::Instant::now();
-
         if self.metadata.is_none() {
-            // println!("get base pc: {:?}", now.elapsed());
             let pc = self.reader.get_at(index).unwrap();
             return Some(pc);
         }
@@ -231,7 +188,7 @@ impl AdaptiveManager {
     }
 }
 
-impl RenderManager<PointCloud<PointXyzRgba>> for AdaptiveManager {
+impl RenderManager<PointCloud<PointXyzRgba>> for AdaptiveLODManager {
     fn start(&mut self) -> Option<PointCloud<PointXyzRgba>> {
         self.get_desired_point_cloud(0)
     }
@@ -261,6 +218,102 @@ impl RenderManager<PointCloud<PointXyzRgba>> for AdaptiveManager {
     }
 }
 
+pub struct AdaptiveUpsamplingManager {
+    reader: PointCloudFileReader,
+
+    // For adaptive loading
+    camera_state: Option<CameraState>,
+
+    // As the temporary cache
+    current_index: usize,
+    pc: Option<PointCloud<PointXyzRgba>>,
+
+    should_adaptive_upsample: bool,
+}
+
+impl AdaptiveUpsamplingManager {
+    pub fn new(src: &String, should_adaptive_upsample: bool) -> Self {
+        let play_format = infer_format(src);
+        let base_path = Path::new(src);
+
+        let reader = PointCloudFileReader::from_directory(base_path, &play_format);
+
+        if reader.is_empty() {
+            eprintln!("Must provide at least one file!");
+            exit(1);
+        }
+
+        Self {
+            pc: None,
+            reader,
+            camera_state: None,
+            current_index: usize::MAX,
+            should_adaptive_upsample,
+        }
+    }
+
+    pub fn len(&self) -> usize {
+        self.reader.len()
+    }
+}
+
+impl RenderManager<PointCloud<PointXyzRgba>> for AdaptiveUpsamplingManager {
+    fn start(&mut self) -> Option<PointCloud<PointXyzRgba>> {
+        self.reader.get_at(0)
+    }
+
+    fn get_at(&mut self, index: usize) -> Option<PointCloud<PointXyzRgba>> {
+        if !self.should_adaptive_upsample {
+            return self.reader.get_at(index);
+        }
+
+        const PARTITION_SIZE: usize = 6;
+        if index != self.current_index || self.pc.is_none() {
+            let pc = self.reader.get_at(index)?;
+            self.pc = Some(pc);
+            self.current_index = index;
+        }
+
+        let camera_state = self.camera_state.as_ref().unwrap();
+
+        let mut visible_pc =
+            Upsampler::get_visible_points(self.pc.as_ref().unwrap().clone(), camera_state);
+        let mut needs_upsampling = true;
+        while needs_upsampling {
+            let upsampled_points = Upsampler::upsample(&visible_pc, camera_state, PARTITION_SIZE);
+
+            if let Some(upsampled_points) = upsampled_points {
+                self.pc.as_mut().unwrap().combine(&upsampled_points);
+                visible_pc.combine(&upsampled_points);
+            } else {
+                needs_upsampling = false;
+            }
+        }
+
+        Some(visible_pc)
+    }
+
+    fn len(&self) -> usize {
+        self.reader.len()
+    }
+
+    fn is_empty(&self) -> bool {
+        self.reader.is_empty()
+    }
+
+    fn set_len(&mut self, len: usize) {
+        self.reader.set_len(len);
+    }
+
+    fn set_camera_state(&mut self, camera_state: Option<CameraState>) {
+        self.camera_state = camera_state;
+    }
+
+    fn should_redraw(&mut self, _camera_state: &CameraState) -> bool {
+        true
+    }
+}
+
 /// Dummy wrapper for RenderReader
 pub struct RenderReaderWrapper<T, U>
 where
@@ -315,3 +368,44 @@ where
         false
     }
 }
+
+fn infer_format(src: &String) -> String {
+    let choices = ["pcd", "ply", "bin", "http"];
+    const PCD: usize = 0;
+    const PLY: usize = 1;
+    const BIN: usize = 2;
+
+    if choices.contains(&src.as_str()) {
+        return src.clone();
+    }
+
+    let path = Path::new(src);
+    // infer by counting extension numbers (pcd ply and bin)
+
+    let mut choice_count = [0, 0, 0];
+    for file_entry in path.read_dir().unwrap() {
+        match file_entry {
+            Ok(entry) => {
+                if let Some(ext) = entry.path().extension() {
+                    if ext.eq("pcd") {
+                        choice_count[PCD] += 1;
+                    } else if ext.eq("ply") {
+                        choice_count[PLY] += 1;
+                    } else if ext.eq("bin") {
+                        choice_count[BIN] += 1;
+                    }
+                }
+            }
+            Err(e) => {
+                eprintln!("{e}")
+            }
+        }
+    }
+
+    let max_index = choice_count
+        .iter()
+        .enumerate()
+        .max_by_key(|(_, &item)| item)
+        .map(|(index, _)| index);
+    choices[max_index.unwrap()].to_string()
+}

src/render/wgpu/renderer.rs

@@ -529,9 +529,12 @@ where
     pub fn update_vertices(&mut self, device: &Device, queue: &Queue, data: &T) {
         let vertices = data.num_vertices();
         if vertices > self.num_vertices {
+            // print!("creating new device");
+
             self.vertex_buffer.destroy();
             self.vertex_buffer = data.create_buffer(device);
         } else {
+            // print!("writing to buffer length: {}", data.bytes().len());
             queue.write_buffer(&self.vertex_buffer, 0, data.bytes());
         }
         self.num_vertices = vertices;