Merge pull request #52 from nus-vv-streams/feat/lod-integration

Support adaptive playback in `vvplay`

Cargo.toml

@@ -75,6 +75,7 @@ kdtree = "0.7.0"
 num-traits = "0.2"
 float-ord = "0.3"
 color_space = "0.5"
+serde_json = "1.0.114"
 #ffmpeg-next = "6"
 
 [dev-dependencies]

README.md

@@ -41,6 +41,7 @@ Commands:
   info        Get the info of a pointcloud file or directory.
                   Supported formats are .pcd and .ply.
                   If no option is specified, all info will be printed.
+  lodify       Preprocesses point cloud data for adaptive playback in vvplay
   dash        Dash will simulate a varying network conditions. 
                   Dash reads in one of our supported file formats. 
                   Files can be of the type .pcd .ply. 
@@ -308,6 +309,44 @@ vv convert --input ./pcd_b --output ./ply_a --storage-type ascii --output-format
 vv convert --input ./pcd_b --output ./pcd_a --storage-type ascii --output-format pcd
 ```
 
+#### `lodify`
+
+A preprocessing step to optimize point cloud data for adaptive playback in `vvplay`
+
+Hyperparameters:
+* Threshold: Manages the structural property (e.g. distribution of the points)
+* Partitions: Manages how the point cloud is segmented for detail distribution.
+
+
+```shell
+Usage: lodify [OPTIONS] <PATH>
+
+Arguments:
+  <PATH>  
+
+Options:
+  -x, --x-partition <X_PARTITION>                [default: 2]
+  -y, --y-partition <Y_PARTITION>                [default: 2]
+  -z, --z-partition <Z_PARTITION>                [default: 2]
+  -b, --base-proportion <BASE_PROPORTION>        [default: 30]
+  -t, --threshold <POINTS_PER_VOXEL_THRESHOLD>   [default: 10]
+  -h, --help           Print help
+```
+
+***Lodifying Point Clouds***
+
+This command constructs a base layer using about 30% of the original points to capture the core structure, while the remaining 70% are reserved to incrementally add detail as needed.
+
+
+```shell
+vv read ./Pcd_b  +output=pcdb \
+    lodify +input=pcdb +output=pcdb_lod \
+    write ./Pcd_lod \
+             +input=pcdb_lod \
+             --storage-type binary \
+             --output-format pcd
+```
+
 #### `info`
 
 Get the info of a pointcloud file or directory. Supported formats are .pcd and .ply. If no option is specified, all info will be printed.
@@ -465,6 +504,7 @@ Options:
       --decoder <DECODER_TYPE>       [default: noop] [possible values: noop, draco]
       --decoder-path <DECODER_PATH>  
       --bg-color <BG_COLOR>          [default: rgb(255,255,255)]
+  --lodify                           [default: False]
   -h, --help                         Print help
 ```
 

src/bin/vvplay.rs

@@ -3,8 +3,8 @@ use std::ffi::OsString;
 use std::path::Path;
 
 use vivotk::render::wgpu::{
-    adaptive_reader::AdaptiveReader, builder::RenderBuilder, camera::Camera, controls::Controller,
-    metrics_reader::MetricsReader, renderer::Renderer,
+    builder::RenderBuilder, camera::Camera, controls::Controller, metrics_reader::MetricsReader,
+    render_manager::AdaptiveManager, renderer::Renderer,
 };
 
 /// Plays a folder of pcd files in lexicographical order
@@ -13,7 +13,7 @@ struct Args {
     /// src can be:
     /// 1. Directory with all the pcd files in lexicographical order
     /// 2. location of the mpd file
-    src: Vec<String>,
+    src: String,
     #[clap(short = 'q', long, default_value_t = 0)]
     quality: u8,
     #[clap(short, long, default_value_t = 30.0)]
@@ -59,6 +59,8 @@ struct Args {
     decoder_path: Option<OsString>,
     #[clap(long, default_value = "rgb(255,255,255)")]
     bg_color: OsString,
+    #[clap(long, default_value = "false")]
+    lod: bool,
 }
 
 #[derive(clap::ValueEnum, Clone, Copy)]
@@ -69,7 +71,7 @@ enum DecoderType {
 
 fn main() {
     let args: Args = Args::parse();
-    let adaptive_reader = AdaptiveReader::new(&args.src);
+    let adaptive_manager = AdaptiveManager::new(&args.src, args.lod);
 
     let camera = Camera::new(
         (args.camera_x, args.camera_y, args.camera_z),
@@ -80,9 +82,9 @@ fn main() {
         .metrics
         .map(|os_str| MetricsReader::from_directory(Path::new(&os_str)));
     let mut builder = RenderBuilder::default();
-    let slider_end = adaptive_reader.len() - 1;
+    let slider_end = adaptive_manager.len() - 1;
     let render = builder.add_window(Renderer::new(
-        adaptive_reader,
+        adaptive_manager,
         args.fps,
         camera,
         (args.width, args.height),

src/bin/vvplay_async.rs

@@ -9,7 +9,7 @@ use vivotk::codec::decoder::{DracoDecoder, NoopDecoder, Tmc2rsDecoder};
 use vivotk::codec::Decoder;
 use vivotk::dash::fetcher::{FetchResult, Fetcher};
 use vivotk::dash::{ThroughputPrediction, ViewportPrediction};
-use vivotk::render::wgpu::reader::RenderReaderCameraPos;
+use vivotk::render::wgpu::render_manager::{RenderManager, RenderReaderWrapper};
 use vivotk::render::wgpu::{
     builder::{EventType, RenderBuilder, RenderEvent},
     camera::{Camera, CameraPosition},
@@ -384,9 +384,10 @@ fn main() {
             .await
     });
     // let mut pcd_reader = PcdAsyncReader::new(buf_out_rx, out_buf_sx, args.buffer_size);
-    let mut pcd_reader = PcdAsyncReader::new(buf_out_rx, to_buf_sx);
+    let pcd_reader = PcdAsyncReader::new(buf_out_rx, to_buf_sx);
+    let mut pcd_manager = RenderReaderWrapper::new(pcd_reader);
     // set the reader max length
-    pcd_reader.set_len(total_frames);
+    pcd_manager.set_len(total_frames);
 
     let camera = Camera::new(
         (args.camera_x, args.camera_y, args.camera_z),
@@ -398,7 +399,7 @@ fn main() {
         .map(|os_str| MetricsReader::from_directory(Path::new(&os_str)));
 
     let mut builder = RenderBuilder::default();
-    let slider_end = pcd_reader.len() - 1;
+    let slider_end = pcd_manager.len() - 1;
 
     // This is the main window that renders the point cloud
     let render_window_id =
@@ -413,7 +414,7 @@ fn main() {
     // } else {
         //t: pcd reader still using normal render reader, and it is not implemented now
         builder.add_window(Renderer::new(
-            pcd_reader,
+            pcd_manager,
             args.fps,
             camera,
             (args.width, args.height),

src/downsample/octree.rs

@@ -1,135 +1,20 @@
-use crate::formats::{pointxyzrgba::PointXyzRgba, PointCloud};
+use crate::{
+    formats::{bounds::Bounds, pointxyzrgba::PointXyzRgba, PointCloud},
+    utils::get_pc_bound,
+};
 
 use rayon::prelude::*;
 
-const DELTA: f32 = 1e-4;
-
 pub fn downsample(
     points: PointCloud<PointXyzRgba>,
     points_per_voxel: usize,
 ) -> PointCloud<PointXyzRgba> {
     if points.points.is_empty() {
         points
     } else {
-        let first_point = points.points[0];
-        let mut min_x = first_point.x;
-        let mut max_x = first_point.x;
-        let mut min_y = first_point.y;
-        let mut max_y = first_point.y;
-        let mut min_z = first_point.z;
-        let mut max_z = first_point.z;
-
-        for &point in &points.points {
-            min_x = min_x.min(point.x);
-            max_x = max_x.max(point.x);
-            min_y = min_y.min(point.y);
-            max_y = max_y.max(point.y);
-            min_z = min_z.min(point.z);
-            max_z = max_z.max(point.z);
-        }
-
-        let points = octree_downsample(
-            points.points,
-            Bounds {
-                min_x,
-                max_x,
-                min_y,
-                max_y,
-                min_z,
-                max_z,
-            },
-            points_per_voxel,
-        );
-
-        PointCloud {
-            number_of_points: points.len(),
-            points,
-        }
-    }
-}
-
-struct Bounds {
-    min_x: f32,
-    max_x: f32,
-    min_y: f32,
-    max_y: f32,
-    min_z: f32,
-    max_z: f32,
-}
-
-impl Bounds {
-    fn new(min_x: f32, max_x: f32, min_y: f32, max_y: f32, min_z: f32, max_z: f32) -> Self {
-        Self {
-            min_x,
-            max_x,
-            min_y,
-            max_y,
-            min_z,
-            max_z,
-        }
-    }
-
-    fn split(&self) -> Vec<Bounds> {
-        let &Bounds {
-            min_x,
-            max_x,
-            min_y,
-            max_y,
-            min_z,
-            max_z,
-        } = self;
-
-        let bisect_x = (max_x + min_x) / 2f32;
-        let bisect_y = (max_y + min_y) / 2f32;
-        let bisect_z = (max_z + min_z) / 2f32;
-
-        vec![
-            Bounds::new(min_x, bisect_x, min_y, bisect_y, min_z, bisect_z),
-            Bounds::new(min_x, bisect_x, min_y, bisect_y, bisect_z + DELTA, max_z),
-            Bounds::new(min_x, bisect_x, bisect_y + DELTA, max_y, min_z, bisect_z),
-            Bounds::new(
-                min_x,
-                bisect_x,
-                bisect_y + DELTA,
-                max_y,
-                bisect_z + DELTA,
-                max_z,
-            ),
-            Bounds::new(bisect_x + DELTA, max_x, min_y, bisect_y, min_z, bisect_z),
-            Bounds::new(
-                bisect_x + DELTA,
-                max_x,
-                min_y,
-                bisect_y,
-                bisect_z + DELTA,
-                max_z,
-            ),
-            Bounds::new(
-                bisect_x + DELTA,
-                max_x,
-                bisect_y + DELTA,
-                max_y,
-                min_z,
-                bisect_z,
-            ),
-            Bounds::new(
-                bisect_x + DELTA,
-                max_x,
-                bisect_y + DELTA,
-                max_y,
-                bisect_z + DELTA,
-                max_z,
-            ),
-        ]
-    }
-
-    fn contains(&self, point: &PointXyzRgba) -> bool {
-        point.x >= self.min_x
-            && point.x <= self.max_x
-            && point.y >= self.min_y
-            && point.y <= self.max_y
-            && point.z >= self.min_z
-            && point.z <= self.max_z
+        let bound = get_pc_bound(&points);
+        let points = octree_downsample(points.points, bound, points_per_voxel);
+        PointCloud::new(points.len(), points)
     }
 }