Field

.gitignore

@@ -8,5 +8,6 @@
 /cresis-toolbox/cluster/requiredMCRProducts.txt
 /cresis-toolbox/cluster/run_cluster_job.sh
 /cresis-toolbox/cluster/cluster_job
-
-
+*~
+cresis-toolbox/cluster/unresolvedSymbols.txt
+cresis-toolbox/cluster/includedSupportPackages.txt

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 The Center for Remote Sensing of Ice and Snow
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,23 @@
+# cresis-toolbox
+CReSIS Toolbox: Polar radar software toolbox (mostly Matlab based)
+
+Contains code for calibrating, processing, analyzing, and viewing radar data. The tools are focused on cryospheric radar sounders.
+
+Main guide for setting up and using the cresis toolbox:
+https://ops.cresis.ku.edu/wiki/
+
+cresis-toolbox: Matlab code. This is the main directory with all of the source code. This is primarily Matlab code with a few other languages used for certain specific operations that are called from the Matlab code.
+
+idl: IDL code. Only includes support for reading data product files
+
+python: Python code. Only includes support for reading ECMWF weather model files.
+
+vim: Vim code. Only includes setup files for using vimdiff.
+
+## To cite usage of the CReSIS Toolbox please use the following:
+
+CReSIS. (2021). cresis-toolbox (Version 3.0.1) [Computer software]. https://doi.org/10.5281/zenodo.5683959
+
+## To acknowledge the use of the CReSIS Toolbox, please note Kansas, NSF and NASA contributions. For example:
+
+We acknowledge the use of the CReSIS toolbox from CReSIS generated with support from the University of Kansas, NASA Operation IceBridge grant NNX16AH54G, and NSF grants ACI-1443054, OPP-1739003, and IIS-1838230.

bash/SDA_file_check.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+# $DATE should be passed from qsub when it's queued up
+htar -c -v -f /hpss/c/r/cresis/2017_Antarctica_Basler/$1.tar -P -Y auto -H copies=1 -H verify=1 /N/dcwan/projects/cresis/2017_Antarctica_Basler/$1

bash/count_files_in_dir.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+# How to use:
+# Enable only one of the "find" statement options
+# Navigate to the correct directory as explained in the comments of the
+# option and then run this command.
+
+# OPTION 1: USE THIS FOR CT_TMP
+# -bash-4.1$ pwd
+# /cresis/snfs1/dataproducts/ct_data/ct_tmp (gRadar.ct_tmp_path)
+#find . -maxdepth 3 -mindepth 3 -type d -print0 | sort | while read -d '' -r dir; do
+
+# OPTION 2: USE THIS FOR RAW DATA STORED IN fmcw AND mcords DIRECTORIES
+# bash-4.1$ pwd
+# /cresis/snfs2/data/2019_Antarctica_GV
+# bash-4.1$ ls
+# 20191017  20191027  20191030  20191104	20191108  20191114  20191118
+# bash-4.1$ ls 20191030
+# fmcw  mcords
+#find . -maxdepth 2 -mindepth 2 -iname "mcords" -type d -print0 | sort | while read -d '' -r dir; do
+
+# OPTION 3: USE THIS FOR OUTPUT DIRECTORY (gRadar.out_path)
+# Run this command in the instrument directory
+# -bash-4.1$ pwd
+# /cresis/snfs1/dataproducts/ct_data/accum
+# /cresis/snfs1/dataproducts/ct_data/kuband
+# /cresis/snfs1/dataproducts/ct_data/rds
+# /cresis/snfs1/dataproducts/ct_data/snow
+find . -maxdepth 2 -mindepth 2 -iname "*" -type d -print0 | sort | while read -d '' -r dir; do
+
+# ALL OPTIONS USE THE FOLLOWING CODE (NO NEED TO CHANGE BELOW THIS POINT)
+  #files=("$dir"/*)
+  num=$(find "$dir" -ls | wc -l);
+  #printf "%5d files in directory %s\n" "$num" "$dir"
+  printf "%5d\tDir:\t%s\n" "$num" "$dir"
+done

bash/tape_or_disk.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+# This script helps determine if files are on disk or on tape
+# If the apparent size is larger, than this generally means that
+# at least some of the files are on tape.
+#
+# SYNTAX:
+# ./tape_or_disk.sh DIRECTORY_OR_FILE_LIST
+#
+# EXAMPLES:
+# ./tape_or_disk.sh 2013*
+# ./tape_or_disk.sh *
+
+if (($# == 0))
+then
+ printf "Syntax is:\n  ./$0 DIRECTORY_OR_FILE_LIST\n" $0
+ exit
+else
+ fns=$@
+fi
+
+# Find the maximum length of each filename
+max_length=0
+for fn in $fns
+do
+  length=${#fn}
+  if ((length > max_length))
+  then
+    max_length=$length
+  fi
+done
+format_string=`printf "%%%ds %%20s %%20s %%20s%s" $max_length "\n"` # Comment out for spreadsheet
+#format_string=`printf "%%s\t%%s\t%%s\t%%s%s" "\n"` # Uncomment for spreadsheet
+
+# Print out the headers
+printf "$format_string" Filename "Disk Usage" "Apparent Disk Usage" "Tape/Disk Estimate"
+
+# Print out the disk usage information for each file
+units[0]="K"
+units[1]="M"
+units[2]="G"
+units[3]="T"
+units[4]="P"
+for fn in $fns
+do
+  if [ ! -e $fn ]
+  then
+    printf "$format_string" $fn "-" "-" "File Not Found"
+    continue;
+  fi
+
+  # Get the disk size of this file/directory in kilobytes
+  du_val_kb=`du -sk $fn | awk '{print $1}' `
+  du_actual_kb=`du -sk --apparent-size $fn | awk '{print $1}' `
+
+  # Unit Conversion to human readable du_val (Comment out for spreadsheet)
+  for (( tmp=$du_val_kb,unit_num=0; tmp > 1024; tmp/=1024,unit_num++ ))
+  do
+  printf "" 
+  done
+  du_val=`echo $tmp${units[$unit_num]}`
+  #Alternate KB format (Uncomment for spreadsheet)
+  #du_val=`echo $du_val_kb`
+
+  # Unit conversion to human readable for du_actual (Comment out for spreadsheet)
+  for (( tmp=$du_actual_kb,unit_num=0; tmp > 1024; tmp/=1024,unit_num++ ))
+  do
+  printf "" 
+  done
+  du_actual=`echo $tmp${units[$unit_num]}`
+  #Alternate KB format (Uncomment for spreadsheet)
+  #du_actual=`echo $du_actual_kb`
+
+  # Print size information to screen
+  if ((du_actual_kb > 100*du_val_kb))
+  then
+    printf "$format_string" $fn $du_val $du_actual "TAPE"
+  elif ((du_actual_kb > du_val_kb))
+  then
+    printf "$format_string" $fn $du_val $du_actual "PART-ON-TAPE"
+  else
+    printf "$format_string" $fn $du_val $du_actual "DISK"
+  fi
+done
+

bash/tape_small_file_archive_script.sh

@@ -0,0 +1,18 @@
+# small_file_archive_script.sh
+# Puts all small files (<1048576 bytes) contained in the current directory
+# into a tar file in the current directory. Also creates a text file with a
+# listing of all the small files.
+
+# CREATE SMALL FILE ARCHIVE SCRIPT
+echo "First and only input should be the filename prefix for the tar file and is usually the radar directory concatenated with the season name folder (e.g. MCoRDS_2018_Antarctica_DC8)"
+SEASON=$1
+# CREATE 1 MB FILLER FILE CALLED delete_this_zero_file
+dd if=/dev/zero bs=1 count=1048576 | tr '\0' '0' >delete_this_zero_file
+# CREATE A TEXT FILE CONTAINING PATH TO ALL THE SMALL FILES
+find ./ -type f -size -1048576c >$SEASON"_small_file_list.txt"
+# CREATE AN ARCHIVE OF ALL THE SMALL FILES PLUS THE ZERO FILE WHICH ENSURES THE TAR IS >=1 MB
+tar cf $SEASON"_small_file_archive.tar" delete_this_zero_file -T $SEASON"_small_file_list.txt"
+# DELETE THE 1 MB FILLER FILE
+rm delete_this_zero_file
+
+

bash/watch_cpu_mem.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+TOTAL_MEM=`free | awk 'NR==2 {print $2}'`
+
+user=`whoami`
+LOG_FILE=/tmp/watch_cpu_mem.$USER.log
+
+while (( 1 ))
+do
+  clear
+  printf "Writing to file %s\n" $LOG_FILE
+  ps -C MATLAB,hf3d,hfss,worker_task,java,httpd,postmaster,cluster_job -o "comm,user,rss,cputime,pid,pcpu" | awk -v total_mem=$TOTAL_MEM 'NR==1 {printf "%10s %10s %10s %10s %10s %10s %10s\n", $1, $2, $3, "%MEM", $4, $5, $6;} NR>1 {printf "%10s %10s %10.1f %10.0f%% %10s %10d %10.1f\n", $1, $2, $3/1024/1024, $3/total_mem*100, $4, $5, $6}'
+  date >>$LOG_FILE
+  ps -C MATLAB,hf3d,hfss,worker_task,java,httpd,postmaster,cluster_job -o "comm,user,rss,cputime,pid,pcpu" | awk -v total_mem=$TOTAL_MEM 'NR==1 {printf "%10s %10s %10s %10s %10s %10s %10s\n", $1, $2, $3, "%MEM", $4, $5, $6;} NR>1 {printf "%10s %10s %10.1f %10.0f%% %10s %10d %10.1f\n", $1, $2, $3/1024/1024, $3/total_mem*100, $4, $5, $6}' >>$LOG_FILE
+  sleep 1
+done
+

citation.cff

@@ -0,0 +1,11 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+authors:
+- family-names: CReSIS
+  given-names:
+  orcid: https://ror.org/04t2m2598
+title: "cresis-toolbox"
+version: 3.0.1
+doi: 10.5281/zenodo.5683959
+date-released: 2021-11-12
+url: "https://github.com/CReSIS/cresis-toolbox"

cresis-toolbox/+deconv/auto_tag_artifacts.m

@@ -194,8 +194,7 @@
   % Set the current artifact to on for just the bad frames
   frames.(update_field)(bad_frms) = frames.(update_field)(bad_frms) + 2^(artifact_type-1);
 
-  records_fn = ct_filename_support(param,'','records');
-  records = load(records_fn);
+  records = records_load(records_fn);
   frm_time_axis = gps_time_to_frame(records.gps_time,frames.frame_idxs);
   frm_time = interp1(records.gps_time,frm_time_axis,gps_time);
 

cresis-toolbox/+deconv/test_deconv_wfs.m

@@ -129,7 +129,7 @@
 
 end
 % blue green red
-load(ct_filename_support(param,'','frames'));
+frames = frames_load(param);
 frm = find(mean(param.load.recs) >= frames.frame_idxs,1,'last');
 
 figure(7); clf;

cresis-toolbox/+imb/@ascopewin/ascopeCM_callback.m

@@ -114,7 +114,7 @@ function reorder_ascopes(obj,val,new_val)
 
 if new_val ~= val
   % Reorder layers
-  Nascopes = length(obj.eg.layers.lyr_name);
+  Nascopes = length(obj.ascope.echowin);
   new_order = [1:val-1, val+1:Nascopes];
   new_order = [new_order(1:new_val-1) val new_order(new_val:Nascopes-1)];
 
@@ -128,7 +128,9 @@ function reorder_ascopes(obj,val,new_val)
   obj.ascope.surf_twtt = obj.ascope.surf_twtt(new_order);
   obj.ascope.lat = obj.ascope.lat(new_order);
   obj.ascope.lon = obj.ascope.lon(new_order);
-  obj.ascope.elev = obj.ascope.elev(new_order);
+  obj.ascope.target_twtt = obj.ascope.target_twtt(new_order);
+  obj.ascope.xlims = obj.ascope.xlims(new_order);
+  obj.ascope.ylims = obj.ascope.ylims(new_order);
   obj.ascope.visible = obj.ascope.visible(new_order);
   obj.ascope.selected = obj.ascope.selected(new_order);
   obj.plot_update();

cresis-toolbox/+imb/@ascopewin/create_ui.m

@@ -105,7 +105,7 @@ function create_ui(obj)
 % =========================================================================
 obj.left_panel.xaxisPM = uicontrol('Parent',obj.left_panel.handle);
 set(obj.left_panel.xaxisPM,'Style','PopupMenu');
-set(obj.left_panel.xaxisPM,'String',{'Twtt','Depth'});
+set(obj.left_panel.xaxisPM,'String',{'Twtt','Depth Air','Depth Ice'});
 set(obj.left_panel.xaxisPM,'Value',1);
 set(obj.left_panel.xaxisPM,'Callback',@obj.xaxisPM_callback);
 set(obj.left_panel.xaxisPM,'TooltipString','Set the x-axis units');

cresis-toolbox/+imb/@ascopewin/plot_update.m

@@ -1,5 +1,8 @@
 function plot_update(obj)
 % plot_update(obj)
+%
+% This function is called any time anything changes that requires an update
+% in the ascope plot (change of order, new ascope, hide/visible, etc.)
 
 % Update plot based on selection
 set(obj.h_ascope(obj.ascope.selected),'Color','red');
@@ -12,7 +15,7 @@ function plot_update(obj)
 set(obj.h_cursor(~obj.ascope.visible),'Visible','off');
 
 first_plot = ~isfinite(obj.xlims(1));
-    
+
 % Update list box entries
 LB_strings = cell(1,length(obj.ascope.echowin));
 obj.xlims = [inf -inf];
@@ -54,7 +57,13 @@ function plot_update(obj)
 val = get(obj.left_panel.xaxisPM,'Value');
 if val == 2
   physical_constants;
+  % Depth in air for above the surface
+  obj.xlims = obj.xlims/1e6 * c/2;
+elseif val == 3
+  physical_constants;
+  % Depth in air for above the surface
   obj.xlims(obj.xlims<0) = obj.xlims(obj.xlims<0)/1e6 * c/2;
+  % Depth in ice for below the surface
   obj.xlims(obj.xlims>0) = obj.xlims(obj.xlims>0)/1e6 * c/2/sqrt(er_ice);
 end
 if first_plot

cresis-toolbox/+imb/@ascopewin/update_ascope.m

@@ -47,7 +47,8 @@ function update_ascope(obj,ascope)
 val = get(obj.left_panel.xaxisPM,'Value');
 idx = (obj.ascope.target_twtt(new_idx)-obj.ascope.twtt{new_idx}(1))/(obj.ascope.twtt{new_idx}(2)-obj.ascope.twtt{new_idx}(1));
 idx = min(length(obj.ascope.data{new_idx}),max(1,round(idx + 1)));
-if val == 2
+if val == 3
+  % Depth in Air above surface and Ice below surface
   physical_constants;
   depth = zeros(size(obj.ascope.twtt{new_idx}));
   above_surf = obj.ascope.twtt{new_idx} < obj.ascope.surf_twtt(new_idx);
@@ -64,7 +65,18 @@ function update_ascope(obj,ascope)
   set(obj.h_cursor(new_idx),{'XData','YData','Visible'}, ...
     {depth, 10*log10(obj.ascope.data{new_idx}(idx)), ...
     'on'});
+elseif val == 2
+  % Depth in Air
+  set(obj.h_ascope(new_idx),{'XData','YData','Visible','Color'}, ...
+    {(obj.ascope.twtt{new_idx} - obj.ascope.surf_twtt(new_idx))*c/2, ...
+    10*log10(obj.ascope.data{new_idx}), ...
+    'on','red'});
+  set(obj.h_cursor(new_idx),{'XData','YData','Visible'}, ...
+    {(obj.ascope.target_twtt(new_idx) - obj.ascope.surf_twtt(new_idx))*c/2, ...
+    10*log10(obj.ascope.data{new_idx}(idx)), ...
+    'on'});
 else
+  % TWTT
   set(obj.h_ascope(new_idx),{'XData','YData','Visible','Color'}, ...
     {(obj.ascope.twtt{new_idx} - obj.ascope.surf_twtt(new_idx))*1e6, ...
     10*log10(obj.ascope.data{new_idx}), ...