diff --git a/examples/benchmarks/compression/eval_all_datasets.py b/examples/benchmarks/compression/eval_all_datasets.py new file mode 100644 index 000000000..ab236d94c --- /dev/null +++ b/examples/benchmarks/compression/eval_all_datasets.py @@ -0,0 +1,108 @@ +# skript for running evaluations on all datasets and writing per scene csv files for 3dgs.zip survey + +import os +import json +import csv +import subprocess +import shutil + +RESULT_DIR_STRING="results/benchmark_{dataset}_mcmc_{num_g}_png_compression" + +datasets = ["tt", "db", "mip"] + +num_gaussians= [ + (360000, "0_36M"), + (490000, "0_49M"), + (1000000, "1M"), + (4000000, "4M") + ] + +named_methods = { + 1000000: "-1.00M" +} + +def run_all_evals(): + for dataset in datasets: + for num_g in num_gaussians: + + RESULT_DIR = RESULT_DIR_STRING.format(dataset=dataset, num_g=num_g[1]) + CAP_MAX = num_g[0] + + if dataset == "mip": + filename = "mcmc.sh" + else: + filename = f"mcmc_{dataset}.sh" + + print(f"Running {dataset} with {CAP_MAX} Gaussians, writing results to {RESULT_DIR}") + os.system(f"bash benchmarks/compression/{filename} {RESULT_DIR} {CAP_MAX}") + + +def write_results(): + # fetch per-scene results and write into csvs + for dataset in datasets: + + if dataset == "tt": + dataset_name = "TanksAndTemples" + elif dataset == "db": + dataset_name = "DeepBlending" + elif dataset == "mip": + dataset_name = "MipNeRF360" + + csv_dir = os.path.join("benchmarks", "compression", "results", dataset_name) + + if os.path.exists(csv_dir): + shutil.rmtree(csv_dir) + os.makedirs(csv_dir) + + for num_g in num_gaussians: + RESULT_DIR = RESULT_DIR_STRING.format(dataset=dataset, num_g=num_g[1]) + try: + scenes = os.listdir(RESULT_DIR) + except: + print("no results for", RESULT_DIR) + + for scene in scenes: + scene_dir = os.path.join(RESULT_DIR, scene) + csv_file_path = os.path.join(csv_dir, scene+".csv") + metrics_path = os.path.join(RESULT_DIR, scene, "stats", "compress_step29999.json") + if not os.path.exists(scene_dir): + continue + + #extract size + zip_path = f"{scene_dir}/compression.zip" + if os.path.exists(zip_path): + subprocess.run(f"rm {zip_path}", shell=True) + subprocess.run(f"zip -r {zip_path} {scene_dir}/compression/", shell=True) + out = subprocess.run( + f"stat -c%s {zip_path}", shell=True, capture_output=True + ) + size = int(out.stdout) + + #extract metrics + with open(metrics_path, 'r') as file: + data = json.load(file) + + if num_g[0] in named_methods: + name = named_methods[num_g[0]] + else: + name = "" + + new_row = [name, data["psnr"], data["ssim"], data["lpips"], size, num_g[0]] + + with open(csv_file_path, 'a', newline='') as csv_file: + writer = csv.writer(csv_file) + + # Check if the file is empty before writing the header + if os.path.getsize(csv_file_path) == 0: + header = ["Submethod", "PSNR", "SSIM", "LPIPS", "Size [Bytes]", "#Gaussians"] + writer.writerow(header) + + writer.writerow(new_row) + +if __name__ == "__main__": + run_all_evals() + write_results() + + + + diff --git a/examples/benchmarks/compression/mcmc.sh b/examples/benchmarks/compression/mcmc.sh index 4c7165f3d..ca66f29b7 100644 --- a/examples/benchmarks/compression/mcmc.sh +++ b/examples/benchmarks/compression/mcmc.sh @@ -18,6 +18,10 @@ CAP_MAX=1000000 # RESULT_DIR="results/benchmark_mcmc_4M_png_compression" # CAP_MAX=4000000 +# Override default values if provided as arguments +[ ! -z "$1" ] && RESULT_DIR="$1" +[ ! -z "$2" ] && CAP_MAX="$2" + for SCENE in $SCENE_LIST; do diff --git a/examples/benchmarks/compression/mcmc_db.sh b/examples/benchmarks/compression/mcmc_db.sh new file mode 100644 index 000000000..edfe5c93d --- /dev/null +++ b/examples/benchmarks/compression/mcmc_db.sh @@ -0,0 +1,52 @@ +SCENE_DIR="data/DeepBlending" + +SCENE_LIST="drjohnson playroom" + +# # 0.36M GSs +# RESULT_DIR="results/benchmark_db_mcmc_0_36M_png_compression" +# CAP_MAX=360000 + +# # 0.49M GSs +# RESULT_DIR="results/benchmark_db_mcmc_0_49M_png_compression" +# CAP_MAX=490000 + +# 1M GSs +RESULT_DIR="results/benchmark_db_mcmc_1M_png_compression" +CAP_MAX=1000000 + +# # 4M GSs +# RESULT_DIR="results/benchmark_db_mcmc_4M_png_compression" +# CAP_MAX=4000000 + +# Override default values if provided as arguments +[ ! -z "$1" ] && RESULT_DIR="$1" +[ ! -z "$2" ] && CAP_MAX="$2" + +for SCENE in $SCENE_LIST; +do + echo "Running $SCENE" + + # train without eval + CUDA_VISIBLE_DEVICES=0 python simple_trainer.py mcmc --eval_steps -1 --disable_viewer --data_factor 1 \ + --strategy.cap-max $CAP_MAX \ + --data_dir $SCENE_DIR/$SCENE/ \ + --result_dir $RESULT_DIR/$SCENE/ + + # eval: use vgg for lpips to align with other benchmarks + CUDA_VISIBLE_DEVICES=0 python simple_trainer.py mcmc --disable_viewer --data_factor 1 \ + --strategy.cap-max $CAP_MAX \ + --data_dir $SCENE_DIR/$SCENE/ \ + --result_dir $RESULT_DIR/$SCENE/ \ + --lpips_net vgg \ + --compression png \ + --ckpt $RESULT_DIR/$SCENE/ckpts/ckpt_29999_rank0.pt +done + +# Zip the compressed files and summarize the stats +if command -v zip &> /dev/null +then + echo "Zipping results" + python benchmarks/compression/summarize_stats.py --results_dir $RESULT_DIR --scenes $SCENE_LIST +else + echo "zip command not found, skipping zipping" +fi \ No newline at end of file diff --git a/examples/benchmarks/compression/mcmc_tt.sh b/examples/benchmarks/compression/mcmc_tt.sh index 054920929..1f0e5e887 100644 --- a/examples/benchmarks/compression/mcmc_tt.sh +++ b/examples/benchmarks/compression/mcmc_tt.sh @@ -1,5 +1,5 @@ SCENE_DIR="data/tandt" -# eval all 9 scenes for benchmarking + SCENE_LIST="train truck" # # 0.36M GSs @@ -18,6 +18,10 @@ CAP_MAX=1000000 # RESULT_DIR="results/benchmark_tt_mcmc_4M_png_compression" # CAP_MAX=4000000 +# Override default values if provided as arguments +[ ! -z "$1" ] && RESULT_DIR="$1" +[ ! -z "$2" ] && CAP_MAX="$2" + for SCENE in $SCENE_LIST; do echo "Running $SCENE" diff --git a/examples/benchmarks/compression/results/DeepBlending/drjohnson.csv b/examples/benchmarks/compression/results/DeepBlending/drjohnson.csv new file mode 100644 index 000000000..51f4a7da5 --- /dev/null +++ b/examples/benchmarks/compression/results/DeepBlending/drjohnson.csv @@ -0,0 +1,5 @@ +Submethod,PSNR,SSIM,LPIPS,Size [Bytes],#Gaussians +,29.201324462890625,0.9001765251159668,0.2728179395198822,6696497,360000 +,29.313091278076172,0.9025779962539673,0.263129860162735,8469791,490000 +-1.00M,29.479978561401367,0.9068419933319092,0.2507980763912201,15664460,1000000 +,29.039730072021484,0.9078600406646729,0.24312639236450195,56163073,4000000 diff --git a/examples/benchmarks/compression/results/DeepBlending/playroom.csv b/examples/benchmarks/compression/results/DeepBlending/playroom.csv new file mode 100644 index 000000000..26f6359eb --- /dev/null +++ b/examples/benchmarks/compression/results/DeepBlending/playroom.csv @@ -0,0 +1,5 @@ +Submethod,PSNR,SSIM,LPIPS,Size [Bytes],#Gaussians +,30.262693405151367,0.9101507067680359,0.2688414454460144,6867113,360000 +,30.337688446044922,0.9118587970733643,0.26298797130584717,8706740,490000 +-1.00M,29.99859619140625,0.9106613397598267,0.252819299697876,15990548,1000000 +,29.75582504272461,0.9131292104721069,0.23756566643714905,57211413,4000000 diff --git a/examples/benchmarks/compression/results/MipNeRF360.csv b/examples/benchmarks/compression/results/MipNeRF360.csv deleted file mode 100644 index faf69be9f..000000000 --- a/examples/benchmarks/compression/results/MipNeRF360.csv +++ /dev/null @@ -1,5 +0,0 @@ -Submethod,PSNR,SSIM,LPIPS,Size [Bytes],#Gaussians -,26.64,0.788,0.270,6916294,360000 -,26.88,0.796,0.256,8796870,490000 --1.00M,27.29,0.811,0.229,16038022,1000000 -,27.70,0.825,0.197,57812682,4000000 \ No newline at end of file diff --git a/examples/benchmarks/compression/results/MipNeRF360/bicycle.csv b/examples/benchmarks/compression/results/MipNeRF360/bicycle.csv new file mode 100644 index 000000000..039efa47f --- /dev/null +++ b/examples/benchmarks/compression/results/MipNeRF360/bicycle.csv @@ -0,0 +1,5 @@ +Submethod,PSNR,SSIM,LPIPS,Size [Bytes],#Gaussians +,24.314237594604492,0.7095159888267517,0.3013736307621002,7055770,360000 +,24.534364700317383,0.7251359224319458,0.2785482406616211,8813338,490000 +-1.00M,24.911056518554688,0.7516974210739136,0.23711256682872772,15926996,1000000 +,25.325092315673828,0.7788066864013672,0.1876075118780136,58617041,4000000 diff --git a/examples/benchmarks/compression/results/MipNeRF360/bonsai.csv b/examples/benchmarks/compression/results/MipNeRF360/bonsai.csv new file mode 100644 index 000000000..d44ba4489 --- /dev/null +++ b/examples/benchmarks/compression/results/MipNeRF360/bonsai.csv @@ -0,0 +1,5 @@ +Submethod,PSNR,SSIM,LPIPS,Size [Bytes],#Gaussians +,31.02326011657715,0.9329836368560791,0.22257398068904877,6830786,360000 +,31.281402587890625,0.9321386218070984,0.215091735124588,8648204,490000 +-1.00M,31.807077407836914,0.9414350390434265,0.20146769285202026,15826537,1000000 +,32.30672836303711,0.9459878206253052,0.1877104789018631,56735351,4000000 diff --git a/examples/benchmarks/compression/results/MipNeRF360/counter.csv b/examples/benchmarks/compression/results/MipNeRF360/counter.csv new file mode 100644 index 000000000..d24b086be --- /dev/null +++ b/examples/benchmarks/compression/results/MipNeRF360/counter.csv @@ -0,0 +1,5 @@ +Submethod,PSNR,SSIM,LPIPS,Size [Bytes],#Gaussians +,28.405691146850586,0.8973114490509033,0.2164955884218216,6878603,360000 +,28.529481887817383,0.9007075428962708,0.2095101773738861,8730556,490000 +-1.00M,28.866003036499023,0.9071822762489319,0.19578693807125092,15947230,1000000 +,29.177431106567383,0.9131742119789124,0.17931070923805237,57486357,4000000 diff --git a/examples/benchmarks/compression/results/MipNeRF360/flowers.csv b/examples/benchmarks/compression/results/MipNeRF360/flowers.csv new file mode 100644 index 000000000..a8e990f21 --- /dev/null +++ b/examples/benchmarks/compression/results/MipNeRF360/flowers.csv @@ -0,0 +1,5 @@ +Submethod,PSNR,SSIM,LPIPS,Size [Bytes],#Gaussians +,21.042749404907227,0.5605924725532532,0.40565335750579834,7161082,360000 +,21.23011589050293,0.574547290802002,0.38778236508369446,9020882,490000 +-1.00M,21.494901657104492,0.6000087261199951,0.3538331985473633,16371725,1000000 +,21.674463272094727,0.6261774301528931,0.30014270544052124,59397940,4000000 diff --git a/examples/benchmarks/compression/results/MipNeRF360/garden.csv b/examples/benchmarks/compression/results/MipNeRF360/garden.csv new file mode 100644 index 000000000..d751ed1a5 --- /dev/null +++ b/examples/benchmarks/compression/results/MipNeRF360/garden.csv @@ -0,0 +1,5 @@ +Submethod,PSNR,SSIM,LPIPS,Size [Bytes],#Gaussians +,25.743114471435547,0.7966399192810059,0.2254912108182907,7095776,360000 +,26.028152465820312,0.8122509717941284,0.20099054276943207,9005181,490000 +-1.00M,26.60516357421875,0.8374983668327332,0.15973007678985596,16546320,1000000 +,27.302112579345703,0.8619086742401123,0.1209760457277298,59521047,4000000 diff --git a/examples/benchmarks/compression/results/MipNeRF360/kitchen.csv b/examples/benchmarks/compression/results/MipNeRF360/kitchen.csv new file mode 100644 index 000000000..59cfa4120 --- /dev/null +++ b/examples/benchmarks/compression/results/MipNeRF360/kitchen.csv @@ -0,0 +1,5 @@ +Submethod,PSNR,SSIM,LPIPS,Size [Bytes],#Gaussians +,29.573772430419922,0.9109970331192017,0.1568634957075119,6886401,360000 +,29.951499938964844,0.9146948456764221,0.14950363337993622,8583446,490000 +-1.00M,30.56005859375,0.9218958020210266,0.13678385317325592,15597179,1000000 +,31.493282318115234,0.9290380477905273,0.1236552819609642,56478553,4000000 diff --git a/examples/benchmarks/compression/results/MipNeRF360/room.csv b/examples/benchmarks/compression/results/MipNeRF360/room.csv new file mode 100644 index 000000000..c6e3df552 --- /dev/null +++ b/examples/benchmarks/compression/results/MipNeRF360/room.csv @@ -0,0 +1,5 @@ +Submethod,PSNR,SSIM,LPIPS,Size [Bytes],#Gaussians +,31.20783233642578,0.9134507179260254,0.2248891294002533,6680829,360000 +,31.438037872314453,0.919510006904602,0.21646593511104584,8558287,490000 +-1.00M,31.8409366607666,0.9252573847770691,0.20395036041736603,15785810,1000000 +,32.10432434082031,0.927442729473114,0.19184806942939758,57016712,4000000 diff --git a/examples/benchmarks/compression/results/MipNeRF360/stump.csv b/examples/benchmarks/compression/results/MipNeRF360/stump.csv new file mode 100644 index 000000000..9de3956f4 --- /dev/null +++ b/examples/benchmarks/compression/results/MipNeRF360/stump.csv @@ -0,0 +1,5 @@ +Submethod,PSNR,SSIM,LPIPS,Size [Bytes],#Gaussians +,25.656526565551758,0.732469379901886,0.2942078113555908,6919767,360000 +,25.865047454833984,0.7448430061340332,0.2767064571380615,8828910,490000 +-1.00M,26.30782699584961,0.7667227983474731,0.24127236008644104,15854509,1000000 +,26.782581329345703,0.7874260544776917,0.20132017135620117,57520011,4000000 diff --git a/examples/benchmarks/compression/results/MipNeRF360/treehill.csv b/examples/benchmarks/compression/results/MipNeRF360/treehill.csv new file mode 100644 index 000000000..2d113463c --- /dev/null +++ b/examples/benchmarks/compression/results/MipNeRF360/treehill.csv @@ -0,0 +1,5 @@ +Submethod,PSNR,SSIM,LPIPS,Size [Bytes],#Gaussians +,22.8242130279541,0.6230133175849915,0.3928559720516205,6875408,360000 +,22.85292625427246,0.6296589970588684,0.3755051791667938,8739408,490000 +-1.00M,22.878782272338867,0.6389139890670776,0.3419164717197418,16072601,1000000 +,22.965383529663086,0.6463788747787476,0.28769150376319885,58275545,4000000 diff --git a/examples/benchmarks/compression/results/TanksAndTemples.csv b/examples/benchmarks/compression/results/TanksAndTemples.csv deleted file mode 100644 index 5845808d9..000000000 --- a/examples/benchmarks/compression/results/TanksAndTemples.csv +++ /dev/null @@ -1,5 +0,0 @@ -Submethod,PSNR,SSIM,LPIPS,Size [Bytes],#Gaussians -,23.54,0.838,0.200,6875669,360000 -,23.62,0.845,0.188,8728572,490000 --1.00M,24.03,0.857,0.163,16100628,1000000 -,24.47,0.872,0.132,58239022,4000000 diff --git a/examples/benchmarks/compression/results/TanksAndTemples/train.csv b/examples/benchmarks/compression/results/TanksAndTemples/train.csv new file mode 100644 index 000000000..625dc6cd0 --- /dev/null +++ b/examples/benchmarks/compression/results/TanksAndTemples/train.csv @@ -0,0 +1,5 @@ +Submethod,PSNR,SSIM,LPIPS,Size [Bytes],#Gaussians +,21.811771392822266,0.8045595288276672,0.23488155007362366,6802125,360000 +,21.922565460205078,0.8111316561698914,0.2212243378162384,8693415,490000 +-1.00M,22.31643295288086,0.8283930420875549,0.19330424070358276,16086616,1000000 +,22.71592903137207,0.8432173728942871,0.16197781264781952,57268346,4000000 diff --git a/examples/benchmarks/compression/results/TanksAndTemples/truck.csv b/examples/benchmarks/compression/results/TanksAndTemples/truck.csv new file mode 100644 index 000000000..0eb2b2321 --- /dev/null +++ b/examples/benchmarks/compression/results/TanksAndTemples/truck.csv @@ -0,0 +1,5 @@ +Submethod,PSNR,SSIM,LPIPS,Size [Bytes],#Gaussians +,25.094594955444336,0.871437668800354,0.1670973002910614,6831513,360000 +,25.42046356201172,0.876853883266449,0.1554252803325653,8691262,490000 +-1.00M,25.811182022094727,0.8866869211196899,0.1334751546382904,16115589,1000000 +,26.319414138793945,0.8979613184928894,0.10413430631160736,59028839,4000000