From 5f85e0549a55f31d324ab9e3bd00dd96822dec1f Mon Sep 17 00:00:00 2001 From: e-koch Date: Sat, 19 Sep 2015 17:27:28 -0600 Subject: [PATCH 01/10] Updates to analysis pipeline --- Examples/analysis_pipeline.py | 27 +++++++++++++++++---------- Examples/noise_validation.r | 2 +- Examples/signal_validation.r | 2 +- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/Examples/analysis_pipeline.py b/Examples/analysis_pipeline.py index c563b5e4..24312390 100644 --- a/Examples/analysis_pipeline.py +++ b/Examples/analysis_pipeline.py @@ -1,6 +1,7 @@ import numpy as np import turbustat.analysis as ta +from turbustat.statistics import statistics_list import os import subprocess import sys @@ -106,20 +107,21 @@ print "Making distance plots." ta.comparison_plot(path, comparisons=good_comparison, - out_path=path+"Distance Plots/") + out_path=path+"Distance Plots/", + design_matrix=design_matrix) # Run the R-script to fit the data to the model # Must have 0_0 and 2_2 comparisons to run -if not "0_0" in good_comparison and not "2_2" in comparison_plot: +if "0_0" not in good_comparison and "2_2" not in good_comparison: raise StandardError("Model fitting requires 0_0 and 2_2 to be available.") os.chdir(path) print "Fitting model of given design." -subprocess.call(['Rscript', path+"FactorialAnalysis.R"]) +subprocess.call(['Rscript', "FactorialAnalysis.R"]) # This should create two output tables of the whole dataset. @@ -128,23 +130,28 @@ print "Running metric validation." subprocess.call(['Rscript', - "/Users/eric/Dropbox/code_development/TurbuStat/Examples/noise_validation.r", + os.path.join(turbustat_path, "Examples/noise_validation.r"), path, "10000"]) subprocess.call(['Rscript', - "/Users/eric/Dropbox/code_development/TurbuStat/Examples/signal_validation.r", + os.path.join(turbustat_path, "Examples/signal_validation.r"), path, "10000"]) # Finally, create the model plots print "Creating model plots." -execfile(turbustat_path+"Examples/effect_plots.py") +execfile(os.path.join(turbustat_path, "Examples/effect_plots.py")) -effect_plots(path+"DataforFits.csv", path+"ResultsFactorial.csv", save=True, - out_path=path+'Model Plots/') +# Remove PDF_AD from the list -map_all_results(path+"ResultsFactorial.csv", save=True, normed=True, - out_path=path+'Model Plots/') +statistics_list.remove("PDF_AD") + +effect_plots("DataforFits.csv", "ResultsFactorial.csv", save=True, + out_path='Model Plots/') + +map_all_results("ResultsFactorial.csv", normed=True, max_order=2, + save_name="map_all_results.pdf", + out_path='Model Plots/') print "Finished!" diff --git a/Examples/noise_validation.r b/Examples/noise_validation.r index 8ac608cb..46265c4f 100644 --- a/Examples/noise_validation.r +++ b/Examples/noise_validation.r @@ -4,7 +4,7 @@ args = commandArgs(TRUE) startTime = Sys.time() -setwd(args[1]) +# setwd(args[1]) FidDes00 = read.csv('distances_0_0.csv', header = T) FidFid00 = read.csv('fiducials_0_0.csv', header = T) diff --git a/Examples/signal_validation.r b/Examples/signal_validation.r index 6558c100..6ec832c1 100644 --- a/Examples/signal_validation.r +++ b/Examples/signal_validation.r @@ -3,7 +3,7 @@ args = commandArgs(TRUE) startTime = Sys.time() -setwd(args[1]) +# setwd(args[1]) FidDes00 = read.csv('distances_0_0.csv', header = T) FidDes22 = read.csv('distances_2_2.csv', header = T) From df685ea4bf3eaa8ffa10410e1f8f0317d747b164 Mon Sep 17 00:00:00 2001 From: e-koch Date: Tue, 22 Sep 2015 17:20:30 -0600 Subject: [PATCH 02/10] Updates to analysis scripts --- Examples/analysis_pipeline.py | 13 +++++++++-- Examples/effect_plots.py | 2 +- Examples/make_distance_plots.py | 5 +++-- Examples/make_obs_tables.py | 38 +++++++++++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 5 deletions(-) create mode 100644 Examples/make_obs_tables.py diff --git a/Examples/analysis_pipeline.py b/Examples/analysis_pipeline.py index 24312390..63420e80 100644 --- a/Examples/analysis_pipeline.py +++ b/Examples/analysis_pipeline.py @@ -150,8 +150,17 @@ effect_plots("DataforFits.csv", "ResultsFactorial.csv", save=True, out_path='Model Plots/') -map_all_results("ResultsFactorial.csv", normed=True, max_order=2, +# Only show results of the good statistics +good_stats = ["Cramer", "DeltaVariance", "Dendrogram_Hist", + "Dendrogram_Num", "PCA", "PDF_Hellinger", "SCF", "VCA", "VCS", + "VCS_Density", "VCS_Velocity", "Skewness", "Kurtosis"] + +# THE ASPECT RATIO IS FUNKY +# NEED TO ADJUST BY HAND +# Use: p.ion(), and set save_name=None to alter by-hand + +map_all_results("ResultsFactorial.csv", normed=False, max_order=2, save_name="map_all_results.pdf", - out_path='Model Plots/') + out_path='Model Plots/', statistics=good_stats) print "Finished!" diff --git a/Examples/effect_plots.py b/Examples/effect_plots.py index 998bed2e..e3ade8bc 100644 --- a/Examples/effect_plots.py +++ b/Examples/effect_plots.py @@ -253,7 +253,6 @@ def map_all_results(effects_file, min_zscore=2.0, save_name=None, 10, 2) - p.figure(figsize=(16, 7)) p.imshow(values, vmin=0, vmax=10, cmap=milagro, interpolation="nearest") p.xticks(np.arange(len(model_effects)), model_effects, rotation=90, @@ -265,6 +264,7 @@ def map_all_results(effects_file, min_zscore=2.0, save_name=None, # Avoid white lines in the pdf rendering cbar.solids.set_edgecolor("face") + p.axes().set_aspect('auto') p.tight_layout() # Save if save_name has been given diff --git a/Examples/make_distance_plots.py b/Examples/make_distance_plots.py index 8ea7e0b2..4811a8d8 100644 --- a/Examples/make_distance_plots.py +++ b/Examples/make_distance_plots.py @@ -46,9 +46,10 @@ obs_to_fid_path, comparisons=obs_to_fid_comparisons, out_path=os.path.join(obs_to_fid_path, "Distance Plots"), num_fids=5, design_matrix=design_matrix, obs_to_fid=True, legend=False, + obs_legend=True, statistics=["Cramer", "DeltaVariance", "Dendrogram_Hist", "Dendrogram_Num", "PCA", "PDF_Hellinger", "SCF", "VCA", "VCS", - "VCS_Density", "VCS_Velocity"]) + "VCS_Density", "VCS_Velocity", "Skewness", "Kurtosis"]) # Des to Obs @@ -62,4 +63,4 @@ legend_labels=["Oph A", "IC 348", "NGC 1333"], statistics=["Cramer", "DeltaVariance", "Dendrogram_Hist", "Dendrogram_Num", "PCA", "PDF_Hellinger", "SCF", "VCA", "VCS", - "VCS_Density", "VCS_Velocity"]) + "VCS_Density", "VCS_Velocity", "Skewness", "Kurtosis"]) diff --git a/Examples/make_obs_tables.py b/Examples/make_obs_tables.py new file mode 100644 index 00000000..be0c7c88 --- /dev/null +++ b/Examples/make_obs_tables.py @@ -0,0 +1,38 @@ + +# Create data tables of the observational results +# Run from Dropbox/AstroStatistics/Full Factorial/Observational Results/ + +import os +import shutil + +from turbustat.analysis.convert_results import concat_convert_HDF5 +from turbustat.analysis import convert_format + + +# Obs to Fids + +path = "Obs_to_Fid/" +hdf5_path = "Obs_to_Fid/HDF5/" + +convert_format(hdf5_path, 0) +shutil.move(hdf5_path+"complete_distances_face_0.csv", path) + +convert_format(hdf5_path, 1) +shutil.move(hdf5_path+"complete_distances_face_1.csv", path) + +convert_format(hdf5_path, 2) +shutil.move(hdf5_path+"complete_distances_face_2.csv", path) + +# Des to Obs + +hdf5_path = "Des_to_Obs/HDF5/" + +concat_convert_HDF5(hdf5_path, face=0, interweave=True, average_axis=0) +shutil.move(os.path.join(hdf5_path, "des_0.csv"), "Des_to_Obs") +shutil.move("Des_to_Obs/des_0.csv", "Des_to_Obs/des_0_obs.csv") + +concat_convert_HDF5(hdf5_path, face=2, interweave=True, average_axis=0) +shutil.move(os.path.join(hdf5_path, "des_2.csv"), "Des_to_Obs") +shutil.move("Des_to_Obs/des_2.csv", "Des_to_Obs/des_2_obs.csv") + +# Make plots using make_distance_plots.py From 79f33ff51eaeda0bc3d4620fc2a8abf6117212ae Mon Sep 17 00:00:00 2001 From: e-koch Date: Thu, 1 Oct 2015 17:28:45 -0600 Subject: [PATCH 03/10] Increase font size --- Examples/effect_plots.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Examples/effect_plots.py b/Examples/effect_plots.py index e3ade8bc..a3c19e16 100644 --- a/Examples/effect_plots.py +++ b/Examples/effect_plots.py @@ -256,11 +256,11 @@ def map_all_results(effects_file, min_zscore=2.0, save_name=None, p.imshow(values, vmin=0, vmax=10, cmap=milagro, interpolation="nearest") p.xticks(np.arange(len(model_effects)), model_effects, rotation=90, - fontsize=18) - p.yticks(np.arange(len(statistics)), stat_labels, fontsize=18) + fontsize=24) + p.yticks(np.arange(len(statistics)), stat_labels, fontsize=24) cbar = p.colorbar(fraction=0.05, shrink=0.9) - cbar.ax.set_ylabel(r'$t$-value', size=18) - cbar.ax.tick_params(labelsize=18) + cbar.ax.set_ylabel(r'$t$-value', size=24) + cbar.ax.tick_params(labelsize=24) # Avoid white lines in the pdf rendering cbar.solids.set_edgecolor("face") From 23a555e21f09af7f162312516e5037ef8b766159 Mon Sep 17 00:00:00 2001 From: e-koch Date: Sun, 11 Oct 2015 16:01:50 -0600 Subject: [PATCH 04/10] Just use titles --- turbustat/analysis/comparison_plot.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/turbustat/analysis/comparison_plot.py b/turbustat/analysis/comparison_plot.py index 3a1999cb..e5908348 100644 --- a/turbustat/analysis/comparison_plot.py +++ b/turbustat/analysis/comparison_plot.py @@ -271,9 +271,10 @@ def _plotter(ax, data, fid_data, num_fids, title, stat, bottom, left, ax.plot(x_vals, y_vals, "-o", label="Fiducial " + str(i), alpha=0.6) # Set title in upper left hand corner - ax.annotate(title, xy=(0, 1), xytext=(12, -6), va='top', - xycoords='axes fraction', textcoords='offset points', - fontsize=12, alpha=0.75) + ax.set_title(title, fontsize=12) + # ax.annotate(title, xy=(1, 0), xytext=(0.9, 0.05), va='top', + # xycoords='axes fraction', textcoords='axes fraction', + # fontsize=12, alpha=0.75) if left: # Set the ylabel using the stat name. Replace underscores ax.set_ylabel(stat.replace("_", " ")+"\nDistance", fontsize=10, From cd789de4dca282edc33ceb781fc3e05d4eb64843 Mon Sep 17 00:00:00 2001 From: e-koch Date: Sun, 11 Oct 2015 16:07:15 -0600 Subject: [PATCH 05/10] Add extra space for legend --- Examples/make_distance_plots.py | 6 +++--- turbustat/analysis/comparison_plot.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Examples/make_distance_plots.py b/Examples/make_distance_plots.py index 4811a8d8..3eaf98d8 100644 --- a/Examples/make_distance_plots.py +++ b/Examples/make_distance_plots.py @@ -48,7 +48,7 @@ num_fids=5, design_matrix=design_matrix, obs_to_fid=True, legend=False, obs_legend=True, statistics=["Cramer", "DeltaVariance", "Dendrogram_Hist", - "Dendrogram_Num", "PCA", "PDF_Hellinger", "SCF", "VCA", "VCS", + "Dendrogram_Num", "PCA", "SCF", "VCA", "VCS", "VCS_Density", "VCS_Velocity", "Skewness", "Kurtosis"]) # Des to Obs @@ -60,7 +60,7 @@ des_to_obs_path, comparisons=des_to_obs_comparisons, out_path=os.path.join(des_to_obs_path, "Distance Plots"), num_fids=3, design_matrix=design_matrix, - legend_labels=["Oph A", "IC 348", "NGC 1333"], + legend_labels=["IC 348", "NGC 1333", "Oph A"], statistics=["Cramer", "DeltaVariance", "Dendrogram_Hist", - "Dendrogram_Num", "PCA", "PDF_Hellinger", "SCF", "VCA", "VCS", + "Dendrogram_Num", "PCA", "SCF", "VCA", "VCS", "VCS_Density", "VCS_Velocity", "Skewness", "Kurtosis"]) diff --git a/turbustat/analysis/comparison_plot.py b/turbustat/analysis/comparison_plot.py index e5908348..0df1a8c9 100644 --- a/turbustat/analysis/comparison_plot.py +++ b/turbustat/analysis/comparison_plot.py @@ -314,11 +314,11 @@ def _plotter(ax, data, fid_data, num_fids, title, stat, bottom, left, if legend: ax.legend(loc="upper right", prop={'size': 10}) if labels is None: - ax.set_xlim([-1, num_design + num_fids + 5]) + ax.set_xlim([-1, num_design + num_fids + 8]) ax.set_xticks(np.append(x_vals, x_fid_vals)) ax.set_xticklabels(xtick_labels+fid_labels, rotation=90, size=12) else: - ax.set_xlim([-2, num_design + num_fids + 5]) + ax.set_xlim([-2, num_design + num_fids + 8]) xticks = np.append([-1], np.append(x_vals, x_fid_vals)) ax.set_xticks(xticks) ax.set_xticklabels(labels+fid_labels, rotation=90, size=12) From 38e9e16a871cc9a11d041afd5f43e3682fb38a0c Mon Sep 17 00:00:00 2001 From: e-koch Date: Sun, 11 Oct 2015 16:18:43 -0600 Subject: [PATCH 06/10] Revert order --- Examples/make_distance_plots.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Examples/make_distance_plots.py b/Examples/make_distance_plots.py index 3eaf98d8..9e949a94 100644 --- a/Examples/make_distance_plots.py +++ b/Examples/make_distance_plots.py @@ -60,7 +60,7 @@ des_to_obs_path, comparisons=des_to_obs_comparisons, out_path=os.path.join(des_to_obs_path, "Distance Plots"), num_fids=3, design_matrix=design_matrix, - legend_labels=["IC 348", "NGC 1333", "Oph A"], + legend_labels=["Oph A", "IC 348", "NGC 1333"], statistics=["Cramer", "DeltaVariance", "Dendrogram_Hist", "Dendrogram_Num", "PCA", "SCF", "VCA", "VCS", "VCS_Density", "VCS_Velocity", "Skewness", "Kurtosis"]) From 178d414704fbe8c9d5878011221d5eba8e111364 Mon Sep 17 00:00:00 2001 From: e-koch Date: Sun, 11 Oct 2015 16:25:32 -0600 Subject: [PATCH 07/10] Match colour order --- turbustat/analysis/comparison_plot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/turbustat/analysis/comparison_plot.py b/turbustat/analysis/comparison_plot.py index 0df1a8c9..2c5d1927 100644 --- a/turbustat/analysis/comparison_plot.py +++ b/turbustat/analysis/comparison_plot.py @@ -339,7 +339,7 @@ def _horiz_obs_plot(ax, data, num_fids, shading=False, legend=False): "ic348.13co.fits": "IC 348"} # Also needs to be generalized - colors = ["r", "g", "b"] + colors = ["b", "g", "r"] x_vals = ax.axis()[:2] From 8f8e8e764b3c1f9e2d43814b1aad32ee1e7f4eb7 Mon Sep 17 00:00:00 2001 From: e-koch Date: Sun, 11 Oct 2015 16:26:44 -0600 Subject: [PATCH 08/10] Build error due to new numpy? --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 26e71e5c..9fb9768d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,13 +19,13 @@ install: - source activate test # Now install dependencies - - conda install --yes numpy + - conda install --yes numpy==1.10.0 - conda install --yes scipy - conda install --yes matplotlib - conda install --yes pandas - conda install --yes statsmodels - conda install --yes scikit-learn - - conda install --yes astropy==0.4rc1 + - conda install --yes astropy # Use pip to install development versions - pip install git+https://github.com/dendrograms/astrodendro.git From 586416ff09686174c1e4307eeace6db37612c928 Mon Sep 17 00:00:00 2001 From: e-koch Date: Sun, 11 Oct 2015 16:36:48 -0600 Subject: [PATCH 09/10] Change order again --- turbustat/analysis/comparison_plot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/turbustat/analysis/comparison_plot.py b/turbustat/analysis/comparison_plot.py index 2c5d1927..16ec37d3 100644 --- a/turbustat/analysis/comparison_plot.py +++ b/turbustat/analysis/comparison_plot.py @@ -339,7 +339,7 @@ def _horiz_obs_plot(ax, data, num_fids, shading=False, legend=False): "ic348.13co.fits": "IC 348"} # Also needs to be generalized - colors = ["b", "g", "r"] + colors = ["g", "r", "b"] x_vals = ax.axis()[:2] From 5738a134d915b4b8fddd1b67272021c0e82255d1 Mon Sep 17 00:00:00 2001 From: e-koch Date: Sun, 11 Oct 2015 16:37:42 -0600 Subject: [PATCH 10/10] Stick to 1.9 for python 2.6 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9fb9768d..9d4ecea5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ install: - source activate test # Now install dependencies - - conda install --yes numpy==1.10.0 + - conda install --yes numpy==1.9.0 - conda install --yes scipy - conda install --yes matplotlib - conda install --yes pandas