Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix projection for images with non-centered camera (e.g. crops) #305

Merged
merged 3 commits into from
Aug 1, 2024
Merged

Fix projection for images with non-centered camera (e.g. crops) #305

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
09d9c77
Fix projection for images with non-centered camera (e.g. crops)
JonathonLuiten Jul 27, 2024
8dca747
fix crop for backward
Jul 31, 2024
1a25203
continous fix. also torch impl.
Jul 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions gsplat/cuda/_torch_impl.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,13 +83,17 @@ def _persp_proj(

fx = Ks[..., 0, 0, None] # [C, 1]
fy = Ks[..., 1, 1, None] # [C, 1]
cx = Ks[..., 0, 2, None] # [C, 1]
cy = Ks[..., 1, 2, None] # [C, 1]
tan_fovx = 0.5 * width / fx # [C, 1]
tan_fovy = 0.5 * height / fy # [C, 1]

lim_x = 1.3 * tan_fovx
lim_y = 1.3 * tan_fovy
tx = tz * torch.clamp(tx / tz, min=-lim_x, max=lim_x)
ty = tz * torch.clamp(ty / tz, min=-lim_y, max=lim_y)
lim_x_pos = (width - cx) / fx + 0.3 * tan_fovx
lim_x_neg = cx / fx + 0.3 * tan_fovx
lim_y_pos = (height - cy) / fy + 0.3 * tan_fovy
lim_y_neg = cy / fy + 0.3 * tan_fovy
tx = tz * torch.clamp(tx / tz, min=-lim_x_neg, max=lim_x_pos)
ty = tz * torch.clamp(ty / tz, min=-lim_y_neg, max=lim_y_pos)

O = torch.zeros((C, N), device=means.device, dtype=means.dtype)
J = torch.stack(
Expand Down
26 changes: 15 additions & 11 deletions gsplat/cuda/csrc/utils.cuh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,13 +153,15 @@ inline __device__ void persp_proj(

T tan_fovx = 0.5f * width / fx;
T tan_fovy = 0.5f * height / fy;
T lim_x = 1.3f * tan_fovx;
T lim_y = 1.3f * tan_fovy;
T lim_x_pos = (width - cx) / fx + 0.3f * tan_fovx;
T lim_x_neg = cx / fx + 0.3f * tan_fovx;
T lim_y_pos = (height - cy) / fy + 0.3f * tan_fovy;
T lim_y_neg = cy / fy + 0.3f * tan_fovy;

T rz = 1.f / z;
T rz2 = rz * rz;
T tx = z * min(lim_x, max(-lim_x, x * rz));
T ty = z * min(lim_y, max(-lim_y, y * rz));
T tx = z * min(lim_x_pos, max(-lim_x_neg, x * rz));
T ty = z * min(lim_y_pos, max(-lim_y_neg, y * rz));

// mat3x2 is 3 columns x 2 rows.
mat3x2<T> J = mat3x2<T>(fx * rz, 0.f, // 1st column
Expand All @@ -183,13 +185,15 @@ inline __device__ void persp_proj_vjp(

T tan_fovx = 0.5f * width / fx;
T tan_fovy = 0.5f * height / fy;
T lim_x = 1.3f * tan_fovx;
T lim_y = 1.3f * tan_fovy;
T lim_x_pos = (width - cx) / fx + 0.3f * tan_fovx;
T lim_x_neg = cx / fx + 0.3f * tan_fovx;
T lim_y_pos = (height - cy) / fy + 0.3f * tan_fovy;
T lim_y_neg = cy / fy + 0.3f * tan_fovy;

T rz = 1.f / z;
T rz2 = rz * rz;
T tx = z * min(lim_x, max(-lim_x, x * rz));
T ty = z * min(lim_y, max(-lim_y, y * rz));
T tx = z * min(lim_x_pos, max(-lim_x_neg, x * rz));
T ty = z * min(lim_y_pos, max(-lim_y_neg, y * rz));

// mat3x2 is 3 columns x 2 rows.
mat3x2<T> J = mat3x2<T>(fx * rz, 0.f, // 1st column
Expand Down Expand Up @@ -217,12 +221,12 @@ inline __device__ void persp_proj_vjp(
v_cov2d * J * glm::transpose(cov3d) + glm::transpose(v_cov2d) * J * cov3d;

// fov clipping
if (x * rz <= lim_x && x * rz >= -lim_x) {
if (x * rz <= lim_x_pos && x * rz >= -lim_x_neg) {
v_mean3d.x += -fx * rz2 * v_J[2][0];
} else {
v_mean3d.z += -fx * rz3 * v_J[2][0] * tx;
}
if (y * rz <= lim_y && y * rz >= -lim_y) {
if (y * rz <= lim_y_pos && y * rz >= -lim_y_neg) {
v_mean3d.y += -fy * rz2 * v_J[2][1];
} else {
v_mean3d.z += -fy * rz3 * v_J[2][1] * ty;
Expand Down Expand Up @@ -341,4 +345,4 @@ inline __device__ void add_blur_vjp(const T eps2d, const mat2<T> conic_blur,
v_sqr_comp * (one_minus_sqr_comp * conic_blur[1][1] - eps2d * det_conic_blur);
}

#endif // GSPLAT_CUDA_UTILS_H
#endif // GSPLAT_CUDA_UTILS_H
Loading