From 09d9c7775469b8b2c38b36a78f893f24ce9cf9e3 Mon Sep 17 00:00:00 2001 From: JonathonLuiten Date: Sat, 27 Jul 2024 22:56:40 +1200 Subject: [PATCH 1/3] Fix projection for images with non-centered camera (e.g. crops) The current code assumes [cx,cy] are at [w/2, h/2], if this is not the case (very common if you want to render a non-centered crop) the code gives VERY bad and incorrect results. This quick fix uses the proper [cx, cy]. --- gsplat/cuda/csrc/utils.cuh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/gsplat/cuda/csrc/utils.cuh b/gsplat/cuda/csrc/utils.cuh index 7e23a0f8e..a7abba309 100644 --- a/gsplat/cuda/csrc/utils.cuh +++ b/gsplat/cuda/csrc/utils.cuh @@ -153,13 +153,15 @@ inline __device__ void persp_proj( T tan_fovx = 0.5f * width / fx; T tan_fovy = 0.5f * height / fy; - T lim_x = 1.3f * tan_fovx; - T lim_y = 1.3f * tan_fovy; + T lim_x_pos = (width - cx) / fx + 0.3f * tan_fovx; + T lim_x_neg = cx / fx + 0.3f * tan_fovx; + T lim_y_pos = (height - cy) / fy + 0.3f * tan_fovy; + T lim_y_neg = cy / fy + 0.3f * tan_fovy; T rz = 1.f / z; T rz2 = rz * rz; - T tx = z * min(lim_x, max(-lim_x, x * rz)); - T ty = z * min(lim_y, max(-lim_y, y * rz)); + T tx = z * min(lim_x_pos, max(-lim_x_neg, x * rz)); + T ty = z * min(lim_y_pos, max(-lim_y_neg, y * rz)); // mat3x2 is 3 columns x 2 rows. mat3x2 J = mat3x2(fx * rz, 0.f, // 1st column @@ -341,4 +343,4 @@ inline __device__ void add_blur_vjp(const T eps2d, const mat2 conic_blur, v_sqr_comp * (one_minus_sqr_comp * conic_blur[1][1] - eps2d * det_conic_blur); } -#endif // GSPLAT_CUDA_UTILS_H \ No newline at end of file +#endif // GSPLAT_CUDA_UTILS_H From 8dca74759fff1b85b6d1462bd893d94a073e47ff Mon Sep 17 00:00:00 2001 From: Ruilong Li <397653553@qq.com> Date: Wed, 31 Jul 2024 21:57:03 +0000 Subject: [PATCH 2/3] fix crop for backward --- gsplat/cuda/csrc/utils.cuh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/gsplat/cuda/csrc/utils.cuh b/gsplat/cuda/csrc/utils.cuh index a7abba309..64df5abf8 100644 --- a/gsplat/cuda/csrc/utils.cuh +++ b/gsplat/cuda/csrc/utils.cuh @@ -185,13 +185,15 @@ inline __device__ void persp_proj_vjp( T tan_fovx = 0.5f * width / fx; T tan_fovy = 0.5f * height / fy; - T lim_x = 1.3f * tan_fovx; - T lim_y = 1.3f * tan_fovy; + T lim_x_pos = (width - cx) / fx + 0.3f * tan_fovx; + T lim_x_neg = cx / fx + 0.3f * tan_fovx; + T lim_y_pos = (height - cy) / fy + 0.3f * tan_fovy; + T lim_y_neg = cy / fy + 0.3f * tan_fovy; T rz = 1.f / z; T rz2 = rz * rz; - T tx = z * min(lim_x, max(-lim_x, x * rz)); - T ty = z * min(lim_y, max(-lim_y, y * rz)); + T tx = z * min(lim_x_pos, max(-lim_x_neg, x * rz)); + T ty = z * min(lim_y_pos, max(-lim_y_neg, y * rz)); // mat3x2 is 3 columns x 2 rows. mat3x2 J = mat3x2(fx * rz, 0.f, // 1st column From 1a25203db314ba4feedd18491a2a67efa3a2c3c2 Mon Sep 17 00:00:00 2001 From: Ruilong Li <397653553@qq.com> Date: Wed, 31 Jul 2024 23:11:28 +0000 Subject: [PATCH 3/3] continous fix. also torch impl. --- gsplat/cuda/_torch_impl.py | 12 ++++++++---- gsplat/cuda/csrc/utils.cuh | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/gsplat/cuda/_torch_impl.py b/gsplat/cuda/_torch_impl.py index e73cdd86a..50f9e81c3 100644 --- a/gsplat/cuda/_torch_impl.py +++ b/gsplat/cuda/_torch_impl.py @@ -83,13 +83,17 @@ def _persp_proj( fx = Ks[..., 0, 0, None] # [C, 1] fy = Ks[..., 1, 1, None] # [C, 1] + cx = Ks[..., 0, 2, None] # [C, 1] + cy = Ks[..., 1, 2, None] # [C, 1] tan_fovx = 0.5 * width / fx # [C, 1] tan_fovy = 0.5 * height / fy # [C, 1] - lim_x = 1.3 * tan_fovx - lim_y = 1.3 * tan_fovy - tx = tz * torch.clamp(tx / tz, min=-lim_x, max=lim_x) - ty = tz * torch.clamp(ty / tz, min=-lim_y, max=lim_y) + lim_x_pos = (width - cx) / fx + 0.3 * tan_fovx + lim_x_neg = cx / fx + 0.3 * tan_fovx + lim_y_pos = (height - cy) / fy + 0.3 * tan_fovy + lim_y_neg = cy / fy + 0.3 * tan_fovy + tx = tz * torch.clamp(tx / tz, min=-lim_x_neg, max=lim_x_pos) + ty = tz * torch.clamp(ty / tz, min=-lim_y_neg, max=lim_y_pos) O = torch.zeros((C, N), device=means.device, dtype=means.dtype) J = torch.stack( diff --git a/gsplat/cuda/csrc/utils.cuh b/gsplat/cuda/csrc/utils.cuh index 64df5abf8..7519cc5c3 100644 --- a/gsplat/cuda/csrc/utils.cuh +++ b/gsplat/cuda/csrc/utils.cuh @@ -221,12 +221,12 @@ inline __device__ void persp_proj_vjp( v_cov2d * J * glm::transpose(cov3d) + glm::transpose(v_cov2d) * J * cov3d; // fov clipping - if (x * rz <= lim_x && x * rz >= -lim_x) { + if (x * rz <= lim_x_pos && x * rz >= -lim_x_neg) { v_mean3d.x += -fx * rz2 * v_J[2][0]; } else { v_mean3d.z += -fx * rz3 * v_J[2][0] * tx; } - if (y * rz <= lim_y && y * rz >= -lim_y) { + if (y * rz <= lim_y_pos && y * rz >= -lim_y_neg) { v_mean3d.y += -fy * rz2 * v_J[2][1]; } else { v_mean3d.z += -fy * rz3 * v_J[2][1] * ty;