diff --git a/.gitignore b/.gitignore index da4c492b..5fb26d7a 100644 --- a/.gitignore +++ b/.gitignore @@ -8,5 +8,6 @@ /cresis-toolbox/cluster/requiredMCRProducts.txt /cresis-toolbox/cluster/run_cluster_job.sh /cresis-toolbox/cluster/cluster_job - - +*~ +cresis-toolbox/cluster/unresolvedSymbols.txt +cresis-toolbox/cluster/includedSupportPackages.txt diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..5b3fd5da --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 The Center for Remote Sensing of Ice and Snow + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 00000000..6b6d072f --- /dev/null +++ b/README.md @@ -0,0 +1,23 @@ +# cresis-toolbox +CReSIS Toolbox: Polar radar software toolbox (mostly Matlab based) + +Contains code for calibrating, processing, analyzing, and viewing radar data. The tools are focused on cryospheric radar sounders. + +Main guide for setting up and using the cresis toolbox: +https://ops.cresis.ku.edu/wiki/ + +cresis-toolbox: Matlab code. This is the main directory with all of the source code. This is primarily Matlab code with a few other languages used for certain specific operations that are called from the Matlab code. + +idl: IDL code. Only includes support for reading data product files + +python: Python code. Only includes support for reading ECMWF weather model files. + +vim: Vim code. Only includes setup files for using vimdiff. + +## To cite usage of the CReSIS Toolbox please use the following: + +CReSIS. (2021). cresis-toolbox (Version 3.0.1) [Computer software]. https://doi.org/10.5281/zenodo.5683959 + +## To acknowledge the use of the CReSIS Toolbox, please note Kansas, NSF and NASA contributions. For example: + +We acknowledge the use of the CReSIS toolbox from CReSIS generated with support from the University of Kansas, NASA Operation IceBridge grant NNX16AH54G, and NSF grants ACI-1443054, OPP-1739003, and IIS-1838230. diff --git a/bash/SDA_file_check.sh b/bash/SDA_file_check.sh new file mode 100755 index 00000000..6624c1dc --- /dev/null +++ b/bash/SDA_file_check.sh @@ -0,0 +1,3 @@ +#!/bin/bash +# $DATE should be passed from qsub when it's queued up +htar -c -v -f /hpss/c/r/cresis/2017_Antarctica_Basler/$1.tar -P -Y auto -H copies=1 -H verify=1 /N/dcwan/projects/cresis/2017_Antarctica_Basler/$1 diff --git a/bash/count_files_in_dir.sh b/bash/count_files_in_dir.sh new file mode 100644 index 00000000..82bf6d70 --- /dev/null +++ b/bash/count_files_in_dir.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# How to use: +# Enable only one of the "find" statement options +# Navigate to the correct directory as explained in the comments of the +# option and then run this command. + +# OPTION 1: USE THIS FOR CT_TMP +# -bash-4.1$ pwd +# /cresis/snfs1/dataproducts/ct_data/ct_tmp (gRadar.ct_tmp_path) +#find . -maxdepth 3 -mindepth 3 -type d -print0 | sort | while read -d '' -r dir; do + +# OPTION 2: USE THIS FOR RAW DATA STORED IN fmcw AND mcords DIRECTORIES +# bash-4.1$ pwd +# /cresis/snfs2/data/2019_Antarctica_GV +# bash-4.1$ ls +# 20191017 20191027 20191030 20191104 20191108 20191114 20191118 +# bash-4.1$ ls 20191030 +# fmcw mcords +#find . -maxdepth 2 -mindepth 2 -iname "mcords" -type d -print0 | sort | while read -d '' -r dir; do + +# OPTION 3: USE THIS FOR OUTPUT DIRECTORY (gRadar.out_path) +# Run this command in the instrument directory +# -bash-4.1$ pwd +# /cresis/snfs1/dataproducts/ct_data/accum +# /cresis/snfs1/dataproducts/ct_data/kuband +# /cresis/snfs1/dataproducts/ct_data/rds +# /cresis/snfs1/dataproducts/ct_data/snow +find . -maxdepth 2 -mindepth 2 -iname "*" -type d -print0 | sort | while read -d '' -r dir; do + +# ALL OPTIONS USE THE FOLLOWING CODE (NO NEED TO CHANGE BELOW THIS POINT) + #files=("$dir"/*) + num=$(find "$dir" -ls | wc -l); + #printf "%5d files in directory %s\n" "$num" "$dir" + printf "%5d\tDir:\t%s\n" "$num" "$dir" +done diff --git a/bash/tape_or_disk.sh b/bash/tape_or_disk.sh new file mode 100644 index 00000000..d60d8387 --- /dev/null +++ b/bash/tape_or_disk.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# This script helps determine if files are on disk or on tape +# If the apparent size is larger, than this generally means that +# at least some of the files are on tape. +# +# SYNTAX: +# ./tape_or_disk.sh DIRECTORY_OR_FILE_LIST +# +# EXAMPLES: +# ./tape_or_disk.sh 2013* +# ./tape_or_disk.sh * + +if (($# == 0)) +then + printf "Syntax is:\n ./$0 DIRECTORY_OR_FILE_LIST\n" $0 + exit +else + fns=$@ +fi + +# Find the maximum length of each filename +max_length=0 +for fn in $fns +do + length=${#fn} + if ((length > max_length)) + then + max_length=$length + fi +done +format_string=`printf "%%%ds %%20s %%20s %%20s%s" $max_length "\n"` # Comment out for spreadsheet +#format_string=`printf "%%s\t%%s\t%%s\t%%s%s" "\n"` # Uncomment for spreadsheet + +# Print out the headers +printf "$format_string" Filename "Disk Usage" "Apparent Disk Usage" "Tape/Disk Estimate" + +# Print out the disk usage information for each file +units[0]="K" +units[1]="M" +units[2]="G" +units[3]="T" +units[4]="P" +for fn in $fns +do + if [ ! -e $fn ] + then + printf "$format_string" $fn "-" "-" "File Not Found" + continue; + fi + + # Get the disk size of this file/directory in kilobytes + du_val_kb=`du -sk $fn | awk '{print $1}' ` + du_actual_kb=`du -sk --apparent-size $fn | awk '{print $1}' ` + + # Unit Conversion to human readable du_val (Comment out for spreadsheet) + for (( tmp=$du_val_kb,unit_num=0; tmp > 1024; tmp/=1024,unit_num++ )) + do + printf "" + done + du_val=`echo $tmp${units[$unit_num]}` + #Alternate KB format (Uncomment for spreadsheet) + #du_val=`echo $du_val_kb` + + # Unit conversion to human readable for du_actual (Comment out for spreadsheet) + for (( tmp=$du_actual_kb,unit_num=0; tmp > 1024; tmp/=1024,unit_num++ )) + do + printf "" + done + du_actual=`echo $tmp${units[$unit_num]}` + #Alternate KB format (Uncomment for spreadsheet) + #du_actual=`echo $du_actual_kb` + + # Print size information to screen + if ((du_actual_kb > 100*du_val_kb)) + then + printf "$format_string" $fn $du_val $du_actual "TAPE" + elif ((du_actual_kb > du_val_kb)) + then + printf "$format_string" $fn $du_val $du_actual "PART-ON-TAPE" + else + printf "$format_string" $fn $du_val $du_actual "DISK" + fi +done + diff --git a/bash/tape_small_file_archive_script.sh b/bash/tape_small_file_archive_script.sh new file mode 100644 index 00000000..d24a017d --- /dev/null +++ b/bash/tape_small_file_archive_script.sh @@ -0,0 +1,18 @@ +# small_file_archive_script.sh +# Puts all small files (<1048576 bytes) contained in the current directory +# into a tar file in the current directory. Also creates a text file with a +# listing of all the small files. + +# CREATE SMALL FILE ARCHIVE SCRIPT +echo "First and only input should be the filename prefix for the tar file and is usually the radar directory concatenated with the season name folder (e.g. MCoRDS_2018_Antarctica_DC8)" +SEASON=$1 +# CREATE 1 MB FILLER FILE CALLED delete_this_zero_file +dd if=/dev/zero bs=1 count=1048576 | tr '\0' '0' >delete_this_zero_file +# CREATE A TEXT FILE CONTAINING PATH TO ALL THE SMALL FILES +find ./ -type f -size -1048576c >$SEASON"_small_file_list.txt" +# CREATE AN ARCHIVE OF ALL THE SMALL FILES PLUS THE ZERO FILE WHICH ENSURES THE TAR IS >=1 MB +tar cf $SEASON"_small_file_archive.tar" delete_this_zero_file -T $SEASON"_small_file_list.txt" +# DELETE THE 1 MB FILLER FILE +rm delete_this_zero_file + + diff --git a/bash/watch_cpu_mem.sh b/bash/watch_cpu_mem.sh new file mode 100644 index 00000000..5e3e15e2 --- /dev/null +++ b/bash/watch_cpu_mem.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +TOTAL_MEM=`free | awk 'NR==2 {print $2}'` + +user=`whoami` +LOG_FILE=/tmp/watch_cpu_mem.$USER.log + +while (( 1 )) +do + clear + printf "Writing to file %s\n" $LOG_FILE + ps -C MATLAB,hf3d,hfss,worker_task,java,httpd,postmaster,cluster_job -o "comm,user,rss,cputime,pid,pcpu" | awk -v total_mem=$TOTAL_MEM 'NR==1 {printf "%10s %10s %10s %10s %10s %10s %10s\n", $1, $2, $3, "%MEM", $4, $5, $6;} NR>1 {printf "%10s %10s %10.1f %10.0f%% %10s %10d %10.1f\n", $1, $2, $3/1024/1024, $3/total_mem*100, $4, $5, $6}' + date >>$LOG_FILE + ps -C MATLAB,hf3d,hfss,worker_task,java,httpd,postmaster,cluster_job -o "comm,user,rss,cputime,pid,pcpu" | awk -v total_mem=$TOTAL_MEM 'NR==1 {printf "%10s %10s %10s %10s %10s %10s %10s\n", $1, $2, $3, "%MEM", $4, $5, $6;} NR>1 {printf "%10s %10s %10.1f %10.0f%% %10s %10d %10.1f\n", $1, $2, $3/1024/1024, $3/total_mem*100, $4, $5, $6}' >>$LOG_FILE + sleep 1 +done + diff --git a/citation.cff b/citation.cff new file mode 100644 index 00000000..0332e768 --- /dev/null +++ b/citation.cff @@ -0,0 +1,11 @@ +cff-version: 1.2.0 +message: "If you use this software, please cite it as below." +authors: +- family-names: CReSIS + given-names: + orcid: https://ror.org/04t2m2598 +title: "cresis-toolbox" +version: 3.0.1 +doi: 10.5281/zenodo.5683959 +date-released: 2021-11-12 +url: "https://github.com/CReSIS/cresis-toolbox" diff --git a/cresis-toolbox/+deconv/auto_tag_artifacts.m b/cresis-toolbox/+deconv/auto_tag_artifacts.m index 58b0b759..8db04adb 100644 --- a/cresis-toolbox/+deconv/auto_tag_artifacts.m +++ b/cresis-toolbox/+deconv/auto_tag_artifacts.m @@ -194,8 +194,7 @@ % Set the current artifact to on for just the bad frames frames.(update_field)(bad_frms) = frames.(update_field)(bad_frms) + 2^(artifact_type-1); - records_fn = ct_filename_support(param,'','records'); - records = load(records_fn); + records = records_load(records_fn); frm_time_axis = gps_time_to_frame(records.gps_time,frames.frame_idxs); frm_time = interp1(records.gps_time,frm_time_axis,gps_time); diff --git a/cresis-toolbox/+deconv/test_deconv_wfs.m b/cresis-toolbox/+deconv/test_deconv_wfs.m index 12ca064b..541e8b16 100644 --- a/cresis-toolbox/+deconv/test_deconv_wfs.m +++ b/cresis-toolbox/+deconv/test_deconv_wfs.m @@ -129,7 +129,7 @@ end % blue green red -load(ct_filename_support(param,'','frames')); +frames = frames_load(param); frm = find(mean(param.load.recs) >= frames.frame_idxs,1,'last'); figure(7); clf; diff --git a/cresis-toolbox/+imb/@ascopewin/ascopeCM_callback.m b/cresis-toolbox/+imb/@ascopewin/ascopeCM_callback.m index bae72654..2ebbf75d 100644 --- a/cresis-toolbox/+imb/@ascopewin/ascopeCM_callback.m +++ b/cresis-toolbox/+imb/@ascopewin/ascopeCM_callback.m @@ -114,7 +114,7 @@ function reorder_ascopes(obj,val,new_val) if new_val ~= val % Reorder layers - Nascopes = length(obj.eg.layers.lyr_name); + Nascopes = length(obj.ascope.echowin); new_order = [1:val-1, val+1:Nascopes]; new_order = [new_order(1:new_val-1) val new_order(new_val:Nascopes-1)]; @@ -128,7 +128,9 @@ function reorder_ascopes(obj,val,new_val) obj.ascope.surf_twtt = obj.ascope.surf_twtt(new_order); obj.ascope.lat = obj.ascope.lat(new_order); obj.ascope.lon = obj.ascope.lon(new_order); - obj.ascope.elev = obj.ascope.elev(new_order); + obj.ascope.target_twtt = obj.ascope.target_twtt(new_order); + obj.ascope.xlims = obj.ascope.xlims(new_order); + obj.ascope.ylims = obj.ascope.ylims(new_order); obj.ascope.visible = obj.ascope.visible(new_order); obj.ascope.selected = obj.ascope.selected(new_order); obj.plot_update(); diff --git a/cresis-toolbox/+imb/@ascopewin/create_ui.m b/cresis-toolbox/+imb/@ascopewin/create_ui.m index 2290e133..6ab4f6f8 100644 --- a/cresis-toolbox/+imb/@ascopewin/create_ui.m +++ b/cresis-toolbox/+imb/@ascopewin/create_ui.m @@ -105,7 +105,7 @@ function create_ui(obj) % ========================================================================= obj.left_panel.xaxisPM = uicontrol('Parent',obj.left_panel.handle); set(obj.left_panel.xaxisPM,'Style','PopupMenu'); -set(obj.left_panel.xaxisPM,'String',{'Twtt','Depth'}); +set(obj.left_panel.xaxisPM,'String',{'Twtt','Depth Air','Depth Ice'}); set(obj.left_panel.xaxisPM,'Value',1); set(obj.left_panel.xaxisPM,'Callback',@obj.xaxisPM_callback); set(obj.left_panel.xaxisPM,'TooltipString','Set the x-axis units'); diff --git a/cresis-toolbox/+imb/@ascopewin/plot_update.m b/cresis-toolbox/+imb/@ascopewin/plot_update.m index 96aa19dc..7285f775 100644 --- a/cresis-toolbox/+imb/@ascopewin/plot_update.m +++ b/cresis-toolbox/+imb/@ascopewin/plot_update.m @@ -1,5 +1,8 @@ function plot_update(obj) % plot_update(obj) +% +% This function is called any time anything changes that requires an update +% in the ascope plot (change of order, new ascope, hide/visible, etc.) % Update plot based on selection set(obj.h_ascope(obj.ascope.selected),'Color','red'); @@ -12,7 +15,7 @@ function plot_update(obj) set(obj.h_cursor(~obj.ascope.visible),'Visible','off'); first_plot = ~isfinite(obj.xlims(1)); - + % Update list box entries LB_strings = cell(1,length(obj.ascope.echowin)); obj.xlims = [inf -inf]; @@ -54,7 +57,13 @@ function plot_update(obj) val = get(obj.left_panel.xaxisPM,'Value'); if val == 2 physical_constants; + % Depth in air for above the surface + obj.xlims = obj.xlims/1e6 * c/2; +elseif val == 3 + physical_constants; + % Depth in air for above the surface obj.xlims(obj.xlims<0) = obj.xlims(obj.xlims<0)/1e6 * c/2; + % Depth in ice for below the surface obj.xlims(obj.xlims>0) = obj.xlims(obj.xlims>0)/1e6 * c/2/sqrt(er_ice); end if first_plot diff --git a/cresis-toolbox/+imb/@ascopewin/update_ascope.m b/cresis-toolbox/+imb/@ascopewin/update_ascope.m index ad587ebb..11f82771 100644 --- a/cresis-toolbox/+imb/@ascopewin/update_ascope.m +++ b/cresis-toolbox/+imb/@ascopewin/update_ascope.m @@ -47,7 +47,8 @@ function update_ascope(obj,ascope) val = get(obj.left_panel.xaxisPM,'Value'); idx = (obj.ascope.target_twtt(new_idx)-obj.ascope.twtt{new_idx}(1))/(obj.ascope.twtt{new_idx}(2)-obj.ascope.twtt{new_idx}(1)); idx = min(length(obj.ascope.data{new_idx}),max(1,round(idx + 1))); -if val == 2 +if val == 3 + % Depth in Air above surface and Ice below surface physical_constants; depth = zeros(size(obj.ascope.twtt{new_idx})); above_surf = obj.ascope.twtt{new_idx} < obj.ascope.surf_twtt(new_idx); @@ -64,7 +65,18 @@ function update_ascope(obj,ascope) set(obj.h_cursor(new_idx),{'XData','YData','Visible'}, ... {depth, 10*log10(obj.ascope.data{new_idx}(idx)), ... 'on'}); +elseif val == 2 + % Depth in Air + set(obj.h_ascope(new_idx),{'XData','YData','Visible','Color'}, ... + {(obj.ascope.twtt{new_idx} - obj.ascope.surf_twtt(new_idx))*c/2, ... + 10*log10(obj.ascope.data{new_idx}), ... + 'on','red'}); + set(obj.h_cursor(new_idx),{'XData','YData','Visible'}, ... + {(obj.ascope.target_twtt(new_idx) - obj.ascope.surf_twtt(new_idx))*c/2, ... + 10*log10(obj.ascope.data{new_idx}(idx)), ... + 'on'}); else + % TWTT set(obj.h_ascope(new_idx),{'XData','YData','Visible','Color'}, ... {(obj.ascope.twtt{new_idx} - obj.ascope.surf_twtt(new_idx))*1e6, ... 10*log10(obj.ascope.data{new_idx}), ... diff --git a/cresis-toolbox/+imb/@ascopewin/xaxisPM_callback.m b/cresis-toolbox/+imb/@ascopewin/xaxisPM_callback.m index 8ee9dc0a..6eeb9357 100644 --- a/cresis-toolbox/+imb/@ascopewin/xaxisPM_callback.m +++ b/cresis-toolbox/+imb/@ascopewin/xaxisPM_callback.m @@ -9,11 +9,14 @@ function xaxisPM_callback(obj,hObj,event) % uicontrol. uicontrol(obj.right_panel.status_panel.statusText); -xlims = xlim(obj.h_axes); % Convert all x-limits to twtt +xlims = xlim(obj.h_axes); if obj.cur_xaxis == 2 - xlims(xlims<0) = xlims(xlims<0)*2/c*1e6; - xlims(xlims>0) = xlims(xlims>0)*2*sqrt(er_ice)/c*1e6; + xlims = xlims*2/c*1e6; + obj.xlims = obj.xlims*2/c*1e6; +elseif obj.cur_xaxis == 3 + xlims(obj.xlims<0) = xlims(obj.xlims<0)*2/c*1e6; + xlims(obj.xlims>0) = xlims(obj.xlims>0)*2*sqrt(er_ice)/c*1e6; obj.xlims(obj.xlims<0) = obj.xlims(obj.xlims<0)*2/c*1e6; obj.xlims(obj.xlims>0) = obj.xlims(obj.xlims>0)*2*sqrt(er_ice)/c*1e6; end @@ -24,21 +27,27 @@ function xaxisPM_callback(obj,hObj,event) if val == 1 xlabel(obj.h_axes,'TWTT (us)'); elseif val == 2 - xlabel(obj.h_axes,'Depth (m)'); + xlabel(obj.h_axes,'Depth (m, e_r=1.0)'); +elseif val == 3 + xlabel(obj.h_axes,'Depth (m, e_r=3.15 below surface)'); end for idx = 1:length(obj.ascope.echowin) % Update label if val == 1 - % Update ascope - xlabel(obj.h_axes,'TWTT (us)'); + % Update ascope for TWTT set(obj.h_ascope(idx),'XData',(obj.ascope.twtt{idx} - obj.ascope.surf_twtt(idx))*1e6); % Update cursor set(obj.h_cursor(idx),{'XData'}, ... {(obj.ascope.target_twtt(idx) - obj.ascope.surf_twtt(idx))*1e6}); elseif val == 2 - % Update ascope - xlabel(obj.h_axes,'Depth (m)'); + % Update ascope for Depth in Air + set(obj.h_ascope(idx),'XData',(obj.ascope.twtt{idx} - obj.ascope.surf_twtt(idx))*c/2); + % Update cursor + set(obj.h_cursor(idx),{'XData'}, ... + {(obj.ascope.target_twtt(idx) - obj.ascope.surf_twtt(idx))*c/2}); + elseif val == 3 + % Update ascope for Depth in Ice depth = zeros(size(obj.ascope.twtt{idx})); above_surf = obj.ascope.twtt{idx} < obj.ascope.surf_twtt(idx); depth(above_surf) = (obj.ascope.twtt{idx}(above_surf) - obj.ascope.surf_twtt(idx)) * c/2; @@ -56,10 +65,13 @@ function xaxisPM_callback(obj,hObj,event) % Convert all x-limits to depth if needed if val == 2 - xlims(xlims<0) = xlims(xlims<0)/1e6 * c/2; - xlims(xlims>0) = xlims(xlims>0)/1e6 * c/2/sqrt(er_ice); + xlims = xlims/1e6 * c/2; + obj.xlims = obj.xlims/1e6 * c/2; +elseif val == 3 + xlims(obj.xlims<0) = xlims(obj.xlims<0)/1e6 * c/2; + xlims(obj.xlims>0) = xlims(obj.xlims>0)/1e6 * c/2/sqrt(er_ice); obj.xlims(obj.xlims<0) = obj.xlims(obj.xlims<0)/1e6 * c/2; obj.xlims(obj.xlims>0) = obj.xlims(obj.xlims>0)/1e6 * c/2/sqrt(er_ice); end -xlim(obj.h_axes,obj.xlims); +xlim(obj.h_axes,xlims); obj.cur_xaxis = val; diff --git a/cresis-toolbox/+imb/@echowin/button_motion.m b/cresis-toolbox/+imb/@echowin/button_motion.m index edbf8526..9920f19a 100644 --- a/cresis-toolbox/+imb/@echowin/button_motion.m +++ b/cresis-toolbox/+imb/@echowin/button_motion.m @@ -54,8 +54,14 @@ function button_motion(obj,src,event) % Get the CData cdata = 10*log10(obj.eg.image_data(min(size(obj.eg.image_data,1),max(1,1+round((y-obj.eg.image_yaxis(1))/(obj.eg.image_yaxis(2)-obj.eg.image_yaxis(1))))), rline)); + % Get the frame number + cur_frm = find(obj.eg.image_gps_time(rline) >= obj.eg.start_gps_time,1,'last'); + if isempty(cur_frm) + cur_frm = 1; + end + % Print mouse position to status bar - set(obj.right_panel.status_panel.mouseCoordText,'String',sprintf('%.3fN %.3fW %.0f %g %.0f',obj.eg.image_lat(rline),obj.eg.image_lon(rline),x,y,cdata)); + set(obj.right_panel.status_panel.mouseCoordText,'String',sprintf('%03d %.3fN %.3fW |%.0f|%g|%.0f',cur_frm,obj.eg.image_lat(rline),obj.eg.image_lon(rline),x,y,cdata)); % OLD: Print mouse position to status bar % yaxis_unit = get(obj.left_panel.yaxisPM,'Value'); diff --git a/cresis-toolbox/+imb/@echowin/button_up.m b/cresis-toolbox/+imb/@echowin/button_up.m index 4111053e..c343265d 100644 --- a/cresis-toolbox/+imb/@echowin/button_up.m +++ b/cresis-toolbox/+imb/@echowin/button_up.m @@ -167,7 +167,7 @@ function button_up(obj,src,event) param.cur_quality = get(obj.left_panel.qualityPM,'Value'); % Current layers param.cur_layers = find(obj.eg.layers.selected_layers).'; - param.layer.x = obj.eg.layers.x_curUnit{1}; + param.layer.x = obj.eg.layers.x_curUnit; param.layer.y = obj.eg.layers.y_curUnit; param.layer.type = obj.eg.layers.type; param.layer.qual = obj.eg.layers.qual; @@ -176,6 +176,8 @@ function button_up(obj,src,event) param.image_y = get(obj.left_panel.imagewin.img,'YData'); param.image_c = get(obj.left_panel.imagewin.img,'CData'); + param.echo_time = obj.eg.time; + %% Find proper param window position (for browse tool) if ~obj.tool.accessed param.keep_tool_pos = false; diff --git a/cresis-toolbox/+imb/@echowin/cmds_execute.m b/cresis-toolbox/+imb/@echowin/cmds_execute.m index 635db465..992871cb 100644 --- a/cresis-toolbox/+imb/@echowin/cmds_execute.m +++ b/cresis-toolbox/+imb/@echowin/cmds_execute.m @@ -98,10 +98,10 @@ function cmds_execute(obj,cmds_list,cmds_direction) elseif yaxis_choice == 2 % WGS_84 Elevation elevation = interp1(obj.eg.gps_time,... obj.eg.elev,... - obj.eg.layers.x{layer_idx},'linear','extrap'); + obj.eg.layers.x,'linear','extrap'); surface = interp1(obj.eg.gps_time,... obj.eg.surf_twtt,... - obj.eg.layers.x{layer_idx},'linear','extrap'); + obj.eg.layers.x,'linear','extrap'); for point_idx = point_idxs if isnan(obj.eg.layers.y{layer_idx}(point_idx)) obj.eg.layers.y_curUnit{layer_idx}(point_idx) = NaN; @@ -115,7 +115,7 @@ function cmds_execute(obj,cmds_list,cmds_direction) elseif yaxis_choice == 3 % Depth/Range surface = interp1(obj.eg.gps_time,... obj.eg.surf_twtt,... - obj.eg.layers.x{layer_idx},'linear','extrap'); + obj.eg.layers.x,'linear','extrap'); for point_idx = point_idxs if isnan(obj.eg.layers.y{layer_idx}(point_idx)) obj.eg.layers.y_curUnit{layer_idx}(point_idx) = NaN; @@ -133,7 +133,7 @@ function cmds_execute(obj,cmds_list,cmds_direction) elseif yaxis_choice == 5 % Surface flat surface = interp1(obj.eg.gps_time,... obj.eg.surf_twtt,... - obj.eg.layers.x{layer_idx},'linear','extrap'); + obj.eg.layers.x,'linear','extrap'); for point_idx = point_idxs if isnan(obj.eg.layers.y{layer_idx}(point_idx)) obj.eg.layers.y_curUnit{layer_idx}(point_idx) = NaN; @@ -156,11 +156,11 @@ function cmds_execute(obj,cmds_list,cmds_direction) % args{2} = [x_min x_max y_min y_max] with x in gps-time and y in twtt % units -% obj.eg.layers.x{layer_idx} % gps-time +% obj.eg.layers.x % gps-time % obj.eg.layers.y{layer_idx} % twtt % obj.eg.layers.qual{layer_idx} % integer 1-3 % obj.eg.layers.type{layer_idx} % this is either 1 (manual) or 2 (auto) -% obj.eg.layers.x_curUnit{layer_idx} % current x-axis units (e.g. along-track km) +% obj.eg.layers.x_curUnit % current x-axis units (e.g. along-track km) % obj.eg.layers.y_curUnit{layer_idx} % current y-axis units (e.g. WGS-84 m) @@ -173,7 +173,7 @@ function cmds_execute(obj,cmds_list,cmds_direction) end %% cmds_execute_delete: Determine which point indices need to be updated -point_idxs = find(obj.eg.layers.x{layer_idx} > args{2}(1) & obj.eg.layers.x{layer_idx} < args{2}(2) ... +point_idxs = find(obj.eg.layers.x > args{2}(1) & obj.eg.layers.x < args{2}(2) ... & obj.eg.layers.y{layer_idx} > args{2}(3) & obj.eg.layers.y{layer_idx} < args{2}(4)); obj.eg.layers.y{layer_idx}(point_idxs) = NaN; @@ -207,12 +207,11 @@ function cmds_execute(obj,cmds_list,cmds_direction) obj.eg.layers.selected_layers = false(1,length(obj.eg.layers.lyr_name)); obj.eg.layers.selected_layers(val) = true; obj.eg.layers.visible_layers = [obj.eg.layers.visible_layers(1:val-1) true obj.eg.layers.visible_layers(val:end)]; -obj.eg.layers.x = [obj.eg.layers.x(1:val-1) obj.eg.layers.x(1) obj.eg.layers.x(val:end)]; -obj.eg.layers.y = [obj.eg.layers.y(1:val-1) {nan(size(obj.eg.layers.x{1}))} obj.eg.layers.y(val:end)]; -obj.eg.layers.qual = [obj.eg.layers.qual(1:val-1) {ones(size(obj.eg.layers.x{1}))} obj.eg.layers.qual(val:end)]; -obj.eg.layers.type = [obj.eg.layers.type(1:val-1) {2*ones(size(obj.eg.layers.x{1}))} obj.eg.layers.type(val:end)]; -obj.eg.layers.x_curUnit = [obj.eg.layers.x_curUnit(1:val-1) obj.eg.layers.x_curUnit(1) obj.eg.layers.x_curUnit(val:end)]; -obj.eg.layers.y_curUnit = [obj.eg.layers.y_curUnit(1:val-1) {nan(size(obj.eg.layers.x_curUnit{1}))} obj.eg.layers.y_curUnit(val:end)]; + +obj.eg.layers.y = [obj.eg.layers.y(1:val-1) {nan(size(obj.eg.layers.x))} obj.eg.layers.y(val:end)]; +obj.eg.layers.qual = [obj.eg.layers.qual(1:val-1) {ones(size(obj.eg.layers.x))} obj.eg.layers.qual(val:end)]; +obj.eg.layers.type = [obj.eg.layers.type(1:val-1) {2*ones(size(obj.eg.layers.x))} obj.eg.layers.type(val:end)]; +obj.eg.layers.y_curUnit = [obj.eg.layers.y_curUnit(1:val-1) {nan(size(obj.eg.layers.x_curUnit))} obj.eg.layers.y_curUnit(val:end)]; obj.h_layer = [obj.h_layer(1:2*(val-1)) nan(1,2) obj.h_layer(2*val-1:end)]; obj.h_quality = [obj.h_quality(1:6*(val-1)) nan(1,6) obj.h_quality(6*val-5:end)]; @@ -259,11 +258,9 @@ function cmds_execute(obj,cmds_list,cmds_direction) obj.eg.layers.lyr_order = obj.eg.layers.lyr_order(new_order); obj.eg.layers.selected_layers = obj.eg.layers.selected_layers(new_order); obj.eg.layers.visible_layers = obj.eg.layers.visible_layers(new_order); -obj.eg.layers.x = obj.eg.layers.x(new_order); obj.eg.layers.y = obj.eg.layers.y(new_order); obj.eg.layers.qual = obj.eg.layers.qual(new_order); obj.eg.layers.type = obj.eg.layers.type(new_order); -obj.eg.layers.x_curUnit = obj.eg.layers.x_curUnit(new_order); obj.eg.layers.y_curUnit = obj.eg.layers.y_curUnit(new_order); obj.h_layer = obj.h_layer(reshape(bsxfun(@plus,repmat(new_order,[2 1])*2,[-1:0].'),[1 Nlayers*2])); @@ -272,7 +269,7 @@ function cmds_execute(obj,cmds_list,cmds_direction) obj.plot_layers(); obj.set_visibility(); -obj.layerLB_str(); +obj.layerLB_str(true); set(obj.left_panel.layerLB,'Value',val); end @@ -293,11 +290,9 @@ function cmds_execute(obj,cmds_list,cmds_direction) obj.eg.layers.lyr_order = [obj.eg.layers.lyr_order(1:val-1) obj.eg.layers.lyr_order(val+1:end)]; obj.eg.layers.selected_layers = false(1,length(obj.eg.layers.lyr_name)); obj.eg.layers.visible_layers = [obj.eg.layers.visible_layers(1:val-1) obj.eg.layers.visible_layers(val+1:end)]; -obj.eg.layers.x = [obj.eg.layers.x(1:val-1) obj.eg.layers.x(val+1:end)]; obj.eg.layers.y = [obj.eg.layers.y(1:val-1) obj.eg.layers.y(val+1:end)]; obj.eg.layers.qual = [obj.eg.layers.qual(1:val-1) obj.eg.layers.qual(val+1:end)]; obj.eg.layers.type = [obj.eg.layers.type(1:val-1) obj.eg.layers.type(val+1:end)]; -obj.eg.layers.x_curUnit = [obj.eg.layers.x_curUnit(1:val-1) obj.eg.layers.x_curUnit(val+1:end)]; obj.eg.layers.y_curUnit = [obj.eg.layers.y_curUnit(1:val-1) obj.eg.layers.y_curUnit(val+1:end)]; delete(obj.h_layer((val-1)*2+(1:2))); @@ -305,7 +300,7 @@ function cmds_execute(obj,cmds_list,cmds_direction) obj.h_layer = [obj.h_layer(1:2*(val-1)) obj.h_layer(2*(val+1)-1:end)]; obj.h_quality = [obj.h_quality(1:6*(val-1)) obj.h_quality(6*(val+1)-5:end)]; -obj.layerLB_str(); +obj.layerLB_str(true); set(obj.left_panel.layerLB,'Value',[]); end @@ -345,17 +340,15 @@ function cmds_execute(obj,cmds_list,cmds_direction) obj.eg.layers.lyr_order = obj.eg.layers.lyr_order(new_order); obj.eg.layers.selected_layers = obj.eg.layers.selected_layers(new_order); obj.eg.layers.visible_layers = obj.eg.layers.visible_layers(new_order); -obj.eg.layers.x = obj.eg.layers.x(new_order); obj.eg.layers.y = obj.eg.layers.y(new_order); obj.eg.layers.qual = obj.eg.layers.qual(new_order); obj.eg.layers.type = obj.eg.layers.type(new_order); -obj.eg.layers.x_curUnit = obj.eg.layers.x_curUnit(new_order); obj.eg.layers.y_curUnit = obj.eg.layers.y_curUnit(new_order); obj.h_layer = obj.h_layer(reshape(bsxfun(@plus,repmat(new_order,[2 1])*2,[-1:0].'),[1 Nlayers*2])); obj.h_quality = obj.h_quality(reshape(bsxfun(@plus,repmat(new_order,[6 1])*6,[-5:0].'),[1 Nlayers*6])); -obj.layerLB_str(); +obj.layerLB_str(true); set(obj.left_panel.layerLB,'Value',val); end diff --git a/cresis-toolbox/+imb/@echowin/cmds_set_undo_stack.m b/cresis-toolbox/+imb/@echowin/cmds_set_undo_stack.m index 5b9c638a..7c1b41dd 100644 --- a/cresis-toolbox/+imb/@echowin/cmds_set_undo_stack.m +++ b/cresis-toolbox/+imb/@echowin/cmds_set_undo_stack.m @@ -1,5 +1,5 @@ -function cmds_set_undo_stack(obj,undo_stack) -% cmds_set_undo_stack(obj,undo_stack) +function cmds_list = cmds_set_undo_stack(obj,undo_stack) +% cmds_list = cmds_set_undo_stack(obj,undo_stack) % % Attach or detach undo stack. Also adds or deletes listeners. % @@ -7,6 +7,14 @@ function cmds_set_undo_stack(obj,undo_stack) % undo_stack = undo stack handle class % Detach: % undo_stack = empty ([]) +% +% cmds_list: list of commands to be run with cmds_execute +% +% Returns list of commands that need to be run (i.e. commands that have not +% been saved yet). After echo.draw() is called, then run the second part of +% this function: +% obj.cmds_set_undo_stack_after_draw(cmds_list); + if isempty(undo_stack) % Detach the current undo stack @@ -22,6 +30,7 @@ function cmds_set_undo_stack(obj,undo_stack) end obj.undo_stack = []; end + cmds_list = {}; else obj.undo_stack = undo_stack; @@ -30,11 +39,8 @@ function cmds_set_undo_stack(obj,undo_stack) obj.undo_stack_synchronize_listener ... = addlistener(obj.undo_stack,'synchronize_event',@obj.cmds_synchronize); - % An undo stack already exists for this system-segment pair, so attach this echowin - % to it. Since there may be commands in the undo stack already, we will - % run these commands so that the new echowin is synced up with the stack. + % Attach echowin to undo stack and get any commands that have not been run + % yet. cmds_list = obj.undo_stack.attach_document(obj); - obj.cmds_execute(cmds_list,'redo'); -end - + end diff --git a/cresis-toolbox/+imb/@echowin/cmds_set_undo_stack_after_draw.m b/cresis-toolbox/+imb/@echowin/cmds_set_undo_stack_after_draw.m new file mode 100644 index 00000000..2fc676c9 --- /dev/null +++ b/cresis-toolbox/+imb/@echowin/cmds_set_undo_stack_after_draw.m @@ -0,0 +1,18 @@ +function cmds_set_undo_stack_after_draw(obj,cmds_list) +% cmds_set_undo_stack_after_draw(obj,cmds_list) +% +% Run commands from cmds_set_undo_stack and updates save button. +% +% cmds_list: list of commands to be run with cmds_execute + +% Since there may be commands in the undo stack already, we will run these +% commands so that the new echowin is synced up with the stack. +obj.cmds_execute(cmds_list,'redo'); + +% Check to see if any modifications have been done relative to the last +% save. +if obj.undo_stack.ismodified() + set(obj.left_panel.savePB,'String','(S)ave Layer*'); +else + set(obj.left_panel.savePB,'String','(S)ave Layer'); +end diff --git a/cresis-toolbox/+imb/@echowin/create_ui.m b/cresis-toolbox/+imb/@echowin/create_ui.m index a1252a11..cbab495a 100644 --- a/cresis-toolbox/+imb/@echowin/create_ui.m +++ b/cresis-toolbox/+imb/@echowin/create_ui.m @@ -91,13 +91,13 @@ function create_ui(obj) % Any double click: Nothing % Ctrl + double click: Zoom reset -obj.tool.list{end+1} = imb.picktool_quality; +obj.tool.list{end+1} = imb.picktool_viterbi; addlistener(obj.tool.list{end},'hide_param',@obj.toolparam_close_callback); -% Left click: Nothing -% Left click and drag: Sets all points contains in draw to the currently -% selected quality level +% Left click: Enters a manual point based on parameters +% Find max in range (specify range line/bin extent to search) +% Left click and drag: Runs tomo.viterbi algorithm on selected data. % Right click: Set cursor point -% Right click and drag: Nothing +% Right click and drag: Delete all points in range % Scroll: Zooms in/out % Ctrl + any click: Select layer % Ctrl + any click and drag: Zoom @@ -120,10 +120,10 @@ function create_ui(obj) % Any double click: Nothing % Ctrl + double click: Zoom reset -obj.tool.list{end+1} = imb.picktool_browse; +obj.tool.list{end+1} = imb.picktool_copy([],obj); addlistener(obj.tool.list{end},'hide_param',@obj.toolparam_close_callback); -% Left click: Open A-scope window -% Left click and drag: Nothing +% Left click and drag: Copy source layers to the selected layers +% Deletes all previous points in range % Right click: Set cursor point % Right click and drag: Nothing % Scroll: Zooms in/out @@ -132,10 +132,11 @@ function create_ui(obj) % Any double click: Nothing % Ctrl + double click: Zoom reset -obj.tool.list{end+1} = imb.picktool_convert([],obj); +obj.tool.list{end+1} = imb.picktool_quality; addlistener(obj.tool.list{end},'hide_param',@obj.toolparam_close_callback); -% Left click and drag: Converts selected layers to the specified layer -% Deletes all previous points in range +% Left click: Nothing +% Left click and drag: Sets all points contains in draw to the currently +% selected quality level % Right click: Set cursor point % Right click and drag: Nothing % Scroll: Zooms in/out @@ -144,19 +145,33 @@ function create_ui(obj) % Any double click: Nothing % Ctrl + double click: Zoom reset -obj.tool.list{end+1} = imb.picktool_viterbi; +obj.tool.list{end+1} = imb.picktool_stat; addlistener(obj.tool.list{end},'hide_param',@obj.toolparam_close_callback); -% Left click: Enters a manual point based on parameters -% Find max in range (specify range line/bin extent to search) -% Left click and drag: Runs tomo.viterbi algorithm on selected data. +% Left click: Nothing +% Left click and drag: Sets all points contains in draw to the currently +% selected quality level % Right click: Set cursor point -% Right click and drag: Delete all points in range +% Right click and drag: Nothing % Scroll: Zooms in/out % Ctrl + any click: Select layer % Ctrl + any click and drag: Zoom % Any double click: Nothing % Ctrl + double click: Zoom reset + +obj.tool.list{end+1} = imb.picktool_browse; +addlistener(obj.tool.list{end},'hide_param',@obj.toolparam_close_callback); +% Left click: Open A-scope window +% Left click and drag: Nothing +% Right click: Set cursor point +% Right click and drag: Nothing +% Scroll: Zooms in/out +% Ctrl + any click: Select layer +% Ctrl + any click and drag: Zoom +% Any double click: Nothing +% Ctrl + double click: Zoom reset + + % obj.tool.list{end+1} = imb.picktool_landmark; % Left click: Create landmark point % Left click and drag: Create landmark region @@ -353,6 +368,7 @@ function create_ui(obj) % Define the context menu items and install their callbacks obj.left_panel.layerCM_visible = uimenu(obj.left_panel.layerCM, 'Label', '&Visible', 'Callback', @obj.layerCM_callback); obj.left_panel.layerCM_hide = uimenu(obj.left_panel.layerCM, 'Label', '&Hide', 'Callback', @obj.layerCM_callback); +obj.left_panel.layerCM_set_surf = uimenu(obj.left_panel.layerCM, 'Label', 'Set surface layer', 'Callback', @obj.layerCM_callback); uimenu(obj.left_panel.layerCM, 'Label', '---', 'Callback', @obj.layerCM_callback); obj.left_panel.layerCM_new = uimenu(obj.left_panel.layerCM, 'Label', '&New layer', 'Callback', @obj.layerCM_callback); obj.left_panel.layerCM_copy = uimenu(obj.left_panel.layerCM, 'Label', '&Copy layer', 'Callback', @obj.layerCM_callback); @@ -364,6 +380,9 @@ function create_ui(obj) obj.left_panel.layerCM_down = uimenu(obj.left_panel.layerCM, 'Label', '&Down', 'Callback', @obj.layerCM_callback); obj.left_panel.layerCM_top = uimenu(obj.left_panel.layerCM, 'Label', '&Top', 'Callback', @obj.layerCM_callback); obj.left_panel.layerCM_bottom = uimenu(obj.left_panel.layerCM, 'Label', '&Bottom', 'Callback', @obj.layerCM_callback); +obj.left_panel.layerCM_detrend = uimenu(obj.left_panel.layerCM, 'Label', 'Detrend', 'Callback', @obj.layerCM_callback,'Checked','off'); +obj.left_panel.layerCM_multiple = uimenu(obj.left_panel.layerCM, 'Label', 'Surface Multiple Suppression', 'Callback', @obj.layerCM_callback,'Checked','off'); +obj.left_panel.layerCM_properties = uimenu(obj.left_panel.layerCM, 'Label', 'Set properties', 'Callback', @obj.layerCM_callback); uimenu(obj.left_panel.layerCM, 'Label', '---', 'Callback', @obj.layerCM_callback); obj.left_panel.layerCM_merge = uimenu(obj.left_panel.layerCM, 'Label', '&Merge layers', 'Callback', @obj.layerCM_callback); obj.left_panel.layerCM_delete = uimenu(obj.left_panel.layerCM, 'Label', 'Delete layer', 'Callback', @obj.layerCM_callback); @@ -478,7 +497,7 @@ function create_ui(obj) end set(obj.right_panel.status_panel.statusText,'HorizontalAlignment','left'); set(obj.right_panel.status_panel.statusText,'String',''); -set(obj.right_panel.status_panel.statusText,'TooltipString','Right click to copy status bar text'); +set(obj.right_panel.status_panel.statusText,'TooltipString','Right click or ctrl-C to copy status bar text. Status bar text shows time and location at cursor; shows layer depth if a layer is selected.'); % mouse coordinate info display obj.right_panel.status_panel.mouseCoordText = uicontrol('parent',obj.right_panel.status_panel.handle); @@ -491,7 +510,7 @@ function create_ui(obj) end set(obj.right_panel.status_panel.mouseCoordText,'HorizontalAlignment','left'); set(obj.right_panel.status_panel.mouseCoordText,'String',''); -set(obj.right_panel.status_panel.mouseCoordText,'TooltipString','Latitude deg, Longitude deg (X, Y, ColorData)'); +set(obj.right_panel.status_panel.mouseCoordText,'TooltipString','Frame Latitude (deg, N) Longitude (deg, W) |X-coordinate|Y-coordinate|Z-coordinate/intensity.'); %----echogram context menu obj.right_panel.echoCM= uicontextmenu('parent',obj.h_fig); @@ -511,7 +530,7 @@ function create_ui(obj) obj.right_panel.status_panel.table.height_margin(row,col) = 0; row = 1; col = 2; obj.right_panel.status_panel.table.handles{row,col} = obj.right_panel.status_panel.mouseCoordText; -obj.right_panel.status_panel.table.width(row,col) = 200; +obj.right_panel.status_panel.table.width(row,col) = 250; obj.right_panel.status_panel.table.height(row,col) = inf; obj.right_panel.status_panel.table.width_margin(row,col) = 0; obj.right_panel.status_panel.table.height_margin(row,col) = 0; diff --git a/cresis-toolbox/+imb/@echowin/crossovers_closest.m b/cresis-toolbox/+imb/@echowin/crossovers_closest.m deleted file mode 100644 index e69de29b..00000000 diff --git a/cresis-toolbox/+imb/@echowin/draw.m b/cresis-toolbox/+imb/@echowin/draw.m index 167bc7f0..6fe19017 100644 --- a/cresis-toolbox/+imb/@echowin/draw.m +++ b/cresis-toolbox/+imb/@echowin/draw.m @@ -72,34 +72,16 @@ function draw(obj,param) obj.eg.lat = []; obj.eg.lon = []; obj.eg.elev = []; +obj.eg.roll = []; obj.eg.surf_twtt = []; obj.eg.frms = []; %% Get all the frames for this segment -if strcmp(obj.eg.layers.source,'layerdata') - obj.eg.start_gps_time = param.start_gps_time; - obj.eg.stop_gps_time = param.stop_gps_time; - obj.eg.frm_strs = {}; - for frm = 1:length(obj.eg.start_gps_time) - obj.eg.frm_strs{frm} = sprintf('%s_%03d', obj.eg.cur_sel.day_seg, frm); - end -else - ops_param = struct('properties',[]); - ops_param.properties.location = obj.eg.cur_sel.location; - ops_param.properties.segment_id = obj.eg.cur_sel.seg_id; - [status,data] = opsGetSegmentInfo(obj.eg.system,ops_param); - obj.eg.frm_strs = {}; - obj.eg.start_gps_time = []; - obj.eg.stop_gps_time = []; - for idx = 1:length(data.properties.frame) - obj.eg.frm_strs{idx} = data.properties.frame{idx}; - obj.eg.start_gps_time(idx) = double(data.properties.start_gps_time(idx)); - obj.eg.stop_gps_time(idx) = double(data.properties.stop_gps_time(idx)); - end - % Sort the list of frames in case database isn't sorted - [obj.eg.frm_strs sorted_idxs] = sort(obj.eg.frm_strs); - obj.eg.start_gps_time = obj.eg.start_gps_time(sorted_idxs); - obj.eg.stop_gps_time = obj.eg.stop_gps_time(sorted_idxs); +obj.eg.start_gps_time = param.start_gps_time; +obj.eg.stop_gps_time = param.stop_gps_time; +obj.eg.frm_strs = {}; +for frm = 1:length(obj.eg.start_gps_time) + obj.eg.frm_strs{frm} = sprintf('%s_%03d', obj.eg.cur_sel.day_seg, frm); end %% Update file listing of all source files @@ -137,7 +119,7 @@ function draw(obj,param) obj.eg.layers.selected_layers = false(size(obj.eg.layers.lyr_id)); obj.eg.layers.visible_layers = true(size(obj.eg.layers.lyr_id)); -obj.layerLB_str(); +obj.layerLB_str(false); %% Load flightlines, layers, crossovers and place the cursor diff --git a/cresis-toolbox/+imb/@echowin/echowin.m b/cresis-toolbox/+imb/@echowin/echowin.m index 43abc48d..0061c573 100644 --- a/cresis-toolbox/+imb/@echowin/echowin.m +++ b/cresis-toolbox/+imb/@echowin/echowin.m @@ -198,6 +198,8 @@ % x_label: string containing x-axis label % y_label: string containing x-axis label % y_order: string containing "normal" or "reverse" for obj.h_axes + % detrend: detrend properties structure + % multiple_suppression: surface multiple suppression properties structure % % layers % show_manual_pts % Logical scalar, Show manual points in layer plots @@ -217,11 +219,11 @@ % selected (tools and operations will act on the layer) % visible_layers: Nlayer length logical vector, true means layer visible % x: 1 by Nx vector of x-values in GPS time - % y: 1 by Nx vector of y-values in twtt - % qual: 1 by Nx vector of quality values (1=good,2=medium,3=bad,NaN=unassigned) - % type: 1 by Nx vector of type values (1= + % y: Cell array of 1 by Nx vectors of y-values in twtt + % qual: Cell array of 1 by Nx vectors of quality values (1=good,2=medium,3=bad,NaN=unassigned) + % type: Cell array of 1 by Nx vectors of type values (1= % x_curUnit: 1 by Nx vector of x-values in current x-axis units - % y_curUnit: 1 by Nx vector of y-values in current y-axis units + % y_curUnit: Cell array of 1 by Nx vectors of y-values in current y-axis units % saved.lyr_name = {}; % Last saved version % saved.lyr_group_name = {}; % Last saved version % saved.lyr_id = {}; % Last saved version @@ -344,8 +346,9 @@ obj.eg.data = []; % original Nt by Nx single data matrix (a copy must be kept in case operations are done on the matrix) obj.eg.gps_time = []; % GPS time of data matrix, represents seconds since Jan 1, 1970 (ANSI-C standard), 1 by Nx double vector obj.eg.elev = []; % Elevation of data matrix in meters, 1 by Nx double vector - obj.eg.lat = []; % Latitude of data matrix, 1 by Nx double vector - obj.eg.lon = []; % Longitude of data matrix, 1 by Nx double vector + obj.eg.lat = []; % Latitude of data matrix, 1 by Nx double vector, degrees N + obj.eg.lon = []; % Longitude of data matrix, 1 by Nx double vector, degrees E + obj.eg.roll = []; % Roll of data matrix, 1 by Nx double vector, radians right wing tip down from level obj.eg.surf_twtt = []; % Surface of data matrix, 1 by Nx double vector obj.eg.time = []; % Fast time of data matrix, Nt by 1 double vector @@ -356,6 +359,9 @@ obj.eg.x_label = ''; % string containing x-axis label obj.eg.y_label = ''; % string containing x-axis label obj.eg.y_order = ''; % string containing "normal" or "reverse" for obj.h_axes + obj.eg.detrend.top = 1; % index or name to top layer + obj.eg.detrend.bottom = 2; % index or name to bottom layer + obj.eg.detrend.order = 7; % order of detrend polynomial obj.eg.layers = []; obj.eg.layers.show_manual_pts = true; % Logical scalar, Show manual points in layer plots @@ -373,12 +379,12 @@ obj.eg.layers.surf_id = []; % surface ID (set in draw) obj.eg.layers.selected_layers = []; % Nlayer length logical vector, true means layer is selected (tools and operations will act on the layer) (set in draw) obj.eg.layers.visible_layers = []; % Nlayer length logical vector, true means layer visible (set in draw) - obj.eg.layers.x = {}; % 1 by Nx vector of x-values in GPS time (set in load_layers) - obj.eg.layers.y = {}; % 1 by Nx vector of y-values in twtt (set in load_layers) - obj.eg.layers.qual = {}; % 1 by Nx vector of quality values (1=good,2=medium,3=bad,NaN=unassigned) (set in load_layers) - obj.eg.layers.type = {}; % 1 by Nx vector of type values 1 (manual) or 2 (auto) (set in load_layers) - obj.eg.layers.x_curUnit = {}; % 1 by Nx vector of x-values in current x-axis units (set in plot_layers) - obj.eg.layers.y_curUnit = {}; % 1 by Nx vector of y-values in current y-axis units (set in plot_layers) + obj.eg.layers.x = []; % 1 by Nx vector of x-values in GPS time (set in load_layers) + obj.eg.layers.y = {}; % Cell array of 1 by Nx vectors of y-values in twtt (set in load_layers) + obj.eg.layers.qual = {}; % Cell array of 1 by Nx vectors of quality values (1=good,2=medium,3=bad,NaN=unassigned) (set in load_layers) + obj.eg.layers.type = {}; % Cell array of 1 by Nx vectors of type values 1 (manual) or 2 (auto) (set in load_layers) + obj.eg.layers.x_curUnit = []; % 1 by Nx vector of x-values in current x-axis units (set in plot_layers) + obj.eg.layers.y_curUnit = {}; % Cell array of 1 by Nx vectors of y-values in current y-axis units (set in plot_layers) obj.eg.layers.saved.lyr_age = {}; % Last saved version obj.eg.layers.saved.lyr_age_source = {}; % Last saved version obj.eg.layers.saved.lyr_desc = {}; % Last saved version @@ -487,7 +493,6 @@ function delete(obj) xaxisPM_callback(obj,hObj,event); yaxisPM_callback(obj,hObj,event); - idx = crossovers_closest(obj,gps_time,twtt); open_crossover(obj,source,event); cursor_crossover(obj,source,event); cur_frame = get_crossover(pbj); @@ -517,15 +522,16 @@ function delete(obj) set_visibility(obj,varargin); % Layer colors change_dynamic_range(obj); change_display_c(obj); - layerLB_str(obj); + layerLB_str(obj,keep_value); new_layerPB_OKbutton_callback(obj,hObj,event); new_layerPB_close_callback(obj,hObj,event); - cancel_operation = undo_stack_modified_check(obj); + cancel_operation = undo_stack_modified_check(obj,force_save_or_cancel_flag); toggle_imagewin_visibility(obj,h_obj,event); update_layer_plots(obj); % Update layer plots, called from cmds_execute %% Commands/Undo stack Methods - cmds_set_undo_stack(obj,undo_stack); % Attaches and detaches undo stack and listener + cmds_list = cmds_set_undo_stack(obj,undo_stack); % Attaches and detaches undo stack and listener, call before draw + cmds_set_undo_stack_after_draw(obj,cmds_list); % Run this with cmds_list from cmds_set_undo_stack after draw called cmds = cmds_convert_units(obj,cmds); % Converts tool commands from current units to gps-time and twtt cmds_execute(obj,cmds_list,cmds_direction); % Executes a set of commands on this echowin cmds_synchronize(obj,varargin); % for tools (callback for the undo_stack synchronize event) diff --git a/cresis-toolbox/+imb/@echowin/frameLB_callback.m b/cresis-toolbox/+imb/@echowin/frameLB_callback.m index 2b48c323..6dc1b18e 100644 --- a/cresis-toolbox/+imb/@echowin/frameLB_callback.m +++ b/cresis-toolbox/+imb/@echowin/frameLB_callback.m @@ -33,7 +33,7 @@ function frameLB_callback(obj,hObj,event) % Regular click frame_idx = get(obj.left_panel.frameLB,'Value'); obj.update_frame_and_sourceLB(frame_idx); - if obj.eg.old_frame_idx ~= frame_idx + if ~isequal(obj.eg.old_frame_idx,frame_idx) % Let the map window know that a different frame has been selected % so that the map window can change the frame selection obj.eg.old_frame_idx = frame_idx; diff --git a/cresis-toolbox/+imb/@echowin/key_press.m b/cresis-toolbox/+imb/@echowin/key_press.m index c1430c7e..3da82134 100644 --- a/cresis-toolbox/+imb/@echowin/key_press.m +++ b/cresis-toolbox/+imb/@echowin/key_press.m @@ -105,9 +105,9 @@ function key_press(obj,src,event) case 4 % browse fprintf('Left click: Open ascope window.\n'); fprintf('Alt + Left click and drag: No function\n\n'); - case 5 % convert layers + case 5 % copy layers fprintf('Left click: No function\n'); - fprintf('Left click and drag: Convert layers within selected region to the layers specified in the param window (p)\n\n'); + fprintf('Left click and drag: Copy layers from source layer specified in the param window (p) to the selected region of the selected layers\n\n'); case 6 % HMM detection fprintf('Left click: Enter point.\n'); fprintf('Left click and drag: Perform HMM detection on selected region.\n\n'); @@ -174,11 +174,11 @@ function key_press(obj,src,event) obj.set_visibility(); case 'b' - set(obj.left_panel.toolPM,'Value',4); - tmp = obj.tool.list{4}; obj.tool.left_click_fh = @tmp.left_click; - tmp = obj.tool.list{4}; obj.tool.left_click_and_drag_fh = @tmp.left_click_and_drag; - tmp = obj.tool.list{4}; obj.tool.right_click_fh = @tmp.right_click; - tmp = obj.tool.list{4}; obj.tool.right_click_and_drag_fh = @tmp.right_click_and_drag; + set(obj.left_panel.toolPM,'Value',7); + tmp = obj.tool.list{7}; obj.tool.left_click_fh = @tmp.left_click; + tmp = obj.tool.list{7}; obj.tool.left_click_and_drag_fh = @tmp.left_click_and_drag; + tmp = obj.tool.list{7}; obj.tool.right_click_fh = @tmp.right_click; + tmp = obj.tool.list{7}; obj.tool.right_click_and_drag_fh = @tmp.right_click_and_drag; obj.toolPM_callback(); case 'c' @@ -193,12 +193,12 @@ function key_press(obj,src,event) % enable or disable crossovers obj.crossovers.gui.visibility_toggle(); else - % convert layer tool hotkey - set(obj.left_panel.toolPM,'Value',5); - tmp = obj.tool.list{5}; obj.tool.left_click_fh = @tmp.left_click; - tmp = obj.tool.list{5}; obj.tool.left_click_and_drag_fh = @tmp.left_click_and_drag; - tmp = obj.tool.list{5}; obj.tool.right_click_fh = @tmp.right_click; - tmp = obj.tool.list{5}; obj.tool.right_click_and_drag_fh = @tmp.right_click_and_drag; + % copy layer tool hotkey + set(obj.left_panel.toolPM,'Value',4); + tmp = obj.tool.list{4}; obj.tool.left_click_fh = @tmp.left_click; + tmp = obj.tool.list{4}; obj.tool.left_click_and_drag_fh = @tmp.left_click_and_drag; + tmp = obj.tool.list{4}; obj.tool.right_click_fh = @tmp.right_click; + tmp = obj.tool.list{4}; obj.tool.right_click_and_drag_fh = @tmp.right_click_and_drag; obj.toolPM_callback(); end @@ -232,11 +232,11 @@ function key_press(obj,src,event) case 'q' if isempty(event.Modifier) - set(obj.left_panel.toolPM,'Value',2); - tmp = obj.tool.list{2}; obj.tool.left_click_fh = @tmp.left_click; - tmp = obj.tool.list{2}; obj.tool.left_click_and_drag_fh = @tmp.left_click_and_drag; - tmp = obj.tool.list{2}; obj.tool.right_click_fh = @tmp.right_click; - tmp = obj.tool.list{2}; obj.tool.right_click_and_drag_fh = @tmp.right_click_and_drag; + set(obj.left_panel.toolPM,'Value',5); + tmp = obj.tool.list{5}; obj.tool.left_click_fh = @tmp.left_click; + tmp = obj.tool.list{5}; obj.tool.left_click_and_drag_fh = @tmp.left_click_and_drag; + tmp = obj.tool.list{5}; obj.tool.right_click_fh = @tmp.right_click; + tmp = obj.tool.list{5}; obj.tool.right_click_and_drag_fh = @tmp.right_click_and_drag; obj.toolPM_callback(); elseif obj.shift_pressed && ~obj.alt_pressed && ~obj.control_pressed new_quality = 1+mod(get(obj.left_panel.qualityPM,'Value'), ... @@ -255,7 +255,8 @@ function key_press(obj,src,event) tmp = obj.tool.list{3}; obj.tool.left_click_and_drag_fh = @tmp.left_click_and_drag; tmp = obj.tool.list{3}; obj.tool.right_click_fh = @tmp.right_click; tmp = obj.tool.list{3}; obj.tool.right_click_and_drag_fh = @tmp.right_click_and_drag; - obj.toolPM_callback(); + obj.toolPM_callback(); + elseif ~obj.shift_pressed && ~obj.alt_pressed && obj.control_pressed %% Save screenshot of current echowin if ~exist('gRadar','var') @@ -278,14 +279,21 @@ function key_press(obj,src,event) obj.savePB_callback(); end + case 't' + set(obj.left_panel.toolPM,'Value',6); + tmp = obj.tool.list{6}; obj.tool.left_click_fh = @tmp.left_click; + tmp = obj.tool.list{6}; obj.tool.left_click_and_drag_fh = @tmp.left_click_and_drag; + tmp = obj.tool.list{6}; obj.tool.right_click_fh = @tmp.right_click; + tmp = obj.tool.list{6}; obj.tool.right_click_and_drag_fh = @tmp.right_click_and_drag; + obj.toolPM_callback(); + case 'v' - if isempty(event.Modifier) - set(obj.left_panel.toolPM,'Value',6); - tmp = obj.tool.list{3}; obj.tool.left_click_fh = @tmp.left_click; - tmp = obj.tool.list{3}; obj.tool.left_click_and_drag_fh = @tmp.left_click_and_drag; - tmp = obj.tool.list{3}; obj.tool.right_click_and_drag_fh = @tmp.right_click_and_drag; - obj.toolPM_callback(); - end + set(obj.left_panel.toolPM,'Value',2); + tmp = obj.tool.list{2}; obj.tool.left_click_fh = @tmp.left_click; + tmp = obj.tool.list{2}; obj.tool.left_click_and_drag_fh = @tmp.left_click_and_drag; + tmp = obj.tool.list{2}; obj.tool.right_click_fh = @tmp.right_click; + tmp = obj.tool.list{2}; obj.tool.right_click_and_drag_fh = @tmp.right_click_and_drag; + obj.toolPM_callback(); case 'u' %% Undo last tool operation @@ -353,7 +361,7 @@ function key_press(obj,src,event) end end end - obj.layerLB_str(); + obj.layerLB_str(true); obj.set_visibility(); case 'downarrow' % Down-arrow: Move Echogram Down @@ -377,6 +385,8 @@ function key_press(obj,src,event) % clip new axis to limits of loaded data obj.redraw(xlims(1),xlims(2),cur_axis(3),cur_axis(4),struct('clipped',true,'ylim_force',obj.shift_pressed)); + button_motion(obj); + case 'uparrow' % Up-arrow: Move Echogram Up if ~isempty(current_object) && (current_object == obj.left_panel.layerLB || current_object == obj.left_panel.sourceLB) return @@ -398,6 +408,8 @@ function key_press(obj,src,event) % clip new axis to limits of loaded data obj.redraw(xlims(1),xlims(2),cur_axis(3),cur_axis(4),struct('clipped',true,'ylim_force',obj.shift_pressed)); + button_motion(obj); + case 'rightarrow' % Right arrow cur_axis = [get(obj.h_axes,'Xlim') ... get(obj.h_axes,'YLim')]; @@ -415,6 +427,8 @@ function key_press(obj,src,event) % Draw data with new axis obj.redraw(xlims(1),xlims(2),cur_axis(3),cur_axis(4),struct('clipped',false,'ylim_force',obj.shift_pressed)); + button_motion(obj); + case 'leftarrow' % Left arrow cur_axis = [get(obj.h_axes,'Xlim') ... get(obj.h_axes,'YLim')]; @@ -432,5 +446,7 @@ function key_press(obj,src,event) % Draw data with new axis obj.redraw(xlims(1),xlims(2),cur_axis(3),cur_axis(4),struct('clipped',false,'ylim_force',obj.shift_pressed)); + button_motion(obj); + end -end \ No newline at end of file +end diff --git a/cresis-toolbox/+imb/@echowin/layerCM_callback.m b/cresis-toolbox/+imb/@echowin/layerCM_callback.m index 26997702..62b8e70c 100644 --- a/cresis-toolbox/+imb/@echowin/layerCM_callback.m +++ b/cresis-toolbox/+imb/@echowin/layerCM_callback.m @@ -6,21 +6,42 @@ function layerCM_callback(obj,source,event) % uicontrol(obj.right_panel.status_panel.statusText); if source == obj.left_panel.layerCM_visible || source == obj.left_panel.layerCM_hide + %% visible + %% hide val = get(obj.left_panel.layerLB,'Value'); obj.eg.layers.visible_layers(val) = source == obj.left_panel.layerCM_visible; - obj.layerLB_str(); + obj.layerLB_str(true); % Update plot based on selection obj.set_visibility(); + +elseif source == obj.left_panel.layerCM_set_surf + %% set_surf + val = get(obj.left_panel.layerLB,'Value'); + if ~isempty(val) + obj.eg.layers.surf_id = obj.eg.layers.lyr_id(val(1)); + % good_mask: logical vector with 1 where the twtt of the surface is a number and 0 + % when NaN. + surf_idx = find(obj.eg.layers.lyr_id==obj.eg.layers.surf_id); + good_mask = ~isnan(obj.eg.layers.y{surf_idx}); + if sum(good_mask) > 2 + obj.eg.surf_twtt = interp1(obj.eg.layers.x(good_mask),obj.eg.layers.y{surf_idx}(good_mask),obj.eg.gps_time); + obj.eg.surf_twtt = interp_finite(obj.eg.surf_twtt,0); + end + obj.yaxisPM_callback(); + end + elseif source == obj.left_panel.layerCM_new || source == obj.left_panel.layerCM_copy || source == obj.left_panel.layerCM_insert - if strcmpi(obj.eg.layers.source,'layerData') + %% new + %% copy + %% insert + if strcmpi(obj.eg.layers.source,'layerdata') % Get the currently selected layers. The new layer will be inserted % before the first of the currently selected layers or at the bottom % of the listbox. val = get(obj.left_panel.layerLB,'Value'); - val = val(val>2); % Ensure that there is at least one layer selected if copying if source == obj.left_panel.layerCM_new || ~isempty(val) if isempty(val) @@ -140,11 +161,16 @@ function layerCM_callback(obj,source,event) else % Get id for new layer - new_lyr_id = max(max(obj.undo_stack.user_data.layer_organizer.lyr_id),max(obj.eg.layers.lyr_id)) + 1; + new_lyr_id = max([obj.undo_stack.user_data.layer_organizer.lyr_id obj.eg.layers.lyr_id]) + 1; + if isempty(new_lyr_id) + new_lyr_id = 1; + end id = new_lyr_id; fprintf('Add layer %s:%s age %g desc "%s"\n', group_name, name, age, desc); - if val > length(obj.eg.layers.lyr_id) + if isempty(obj.eg.layers.lyr_order) + order = 1; + elseif val > length(obj.eg.layers.lyr_id) order = obj.eg.layers.lyr_order(end) + 1; else order = obj.eg.layers.lyr_order(val); @@ -226,13 +252,15 @@ function layerCM_callback(obj,source,event) end end end + else + error('Create new layer in OPS via the prefs window.'); end elseif source == obj.left_panel.layerCM_edit - if strcmpi(obj.eg.layers.source,'layerData') + %% edit + if strcmpi(obj.eg.layers.source,'layerdata') % Get the currently selected layers. vals = get(obj.left_panel.layerLB,'Value'); - vals = vals(vals>2); if length(vals) == 1 val = vals(1); prompt = {'Layer Age:','Layer Age Source:','Layer Age Source Age:','Layer Age Source Type:','Description:','Layer Group Name:','Layer Name:'}; @@ -319,9 +347,10 @@ function layerCM_callback(obj,source,event) end elseif source == obj.left_panel.layerCM_sequence - if strcmpi(obj.eg.layers.source,'layerData') + %% sequence + if strcmpi(obj.eg.layers.source,'layerdata') vals = get(obj.left_panel.layerLB,'Value'); - vals = vals(vals>2); + vals = vals(vals>1); prompt = {'Basename (BASENAME_001):','Zero padding length ("003" is 3):','Start count at:'}; old_base_name = obj.eg.layers.lyr_name{vals(1)}; @@ -374,9 +403,10 @@ function layerCM_callback(obj,source,event) end elseif source == obj.left_panel.layerCM_order - if strcmpi(obj.eg.layers.source,'layerData') + %% order + if strcmpi(obj.eg.layers.source,'layerdata') vals = get(obj.left_panel.layerLB,'Value'); - vals = vals(vals>2); + vals = vals(vals>1); done = false; order_unsorted = obj.eg.layers.lyr_order; name_unsorted = obj.eg.layers.lyr_name; @@ -451,10 +481,11 @@ function layerCM_callback(obj,source,event) end elseif source == obj.left_panel.layerCM_up - if strcmpi(obj.eg.layers.source,'layerData') + %% up + if strcmpi(obj.eg.layers.source,'layerdata') % Get the currently selected layers. val = get(obj.left_panel.layerLB,'Value'); - val = val(val>3); + val = val(val>1); if ~isempty(val) val = val(1); age = obj.eg.layers.lyr_age(val); @@ -490,10 +521,11 @@ function layerCM_callback(obj,source,event) end elseif source == obj.left_panel.layerCM_down - if strcmpi(obj.eg.layers.source,'layerData') + %% down + if strcmpi(obj.eg.layers.source,'layerdata') % Get the currently selected layers. val = get(obj.left_panel.layerLB,'Value'); - val = val(val>2 & val3); + val = val(val>2); if ~isempty(val) val = val(1); age = obj.eg.layers.lyr_age(val); @@ -542,7 +575,7 @@ function layerCM_callback(obj,source,event) name = obj.eg.layers.lyr_name{val}; order = obj.eg.layers.lyr_order(val); - new_val = 3; + new_val = 2; new_order = obj.eg.layers.lyr_order(new_val); fprintf('Move layer top %s:%s\n', group_name, name); @@ -573,10 +606,10 @@ function layerCM_callback(obj,source,event) end elseif source == obj.left_panel.layerCM_bottom - if strcmpi(obj.eg.layers.source,'layerData') + %% bottom + if strcmpi(obj.eg.layers.source,'layerdata') % Get the currently selected layers. val = get(obj.left_panel.layerLB,'Value'); - val = val(val>2); if ~isempty(val) val = val(1); age = obj.eg.layers.lyr_age(val); @@ -617,21 +650,25 @@ function layerCM_callback(obj,source,event) end elseif source == obj.left_panel.layerCM_merge - if strcmpi(obj.eg.layers.source,'layerData') + %% merge + if strcmpi(obj.eg.layers.source,'layerdata') % Get the currently selected layers. vals = get(obj.left_panel.layerLB,'Value'); - vals = vals(vals>2); + vals = vals(vals>1); if length(vals) < 2 - return; + fprintf('Merge layers requires that two layers are selected.'); + return; end - - cancel_operation = obj.undo_stack_modified_check(); + + cancel_operation = obj.undo_stack_modified_check(true); if cancel_operation return end % Copy points from all layers (except the first) to the first layer + % This operates on the source because all frames have to be merged and + % not just the frames that are loaded. y = nan(length(vals),length(obj.undo_stack.user_data.frame)); quality = ones(length(vals),length(obj.undo_stack.user_data.frame)); type = 2*ones(length(vals),length(obj.undo_stack.user_data.frame)); @@ -648,13 +685,13 @@ function layerCM_callback(obj,source,event) type(val_idx,point_ids) = obj.undo_stack.user_data.layer_info(frm).type(lay_idx,:); else % Layer does not exist in this file, set to defaults - quality(val_idx,point_ids) = ones(size(obj.undo_stack.user_data.layer_info(frm).GPS_time)); - y(val_idx,point_ids) = nan(size(obj.undo_stack.user_data.layer_info(frm).GPS_time)); - type(val_idx,point_ids) = 2*ones(size(obj.undo_stack.user_data.layer_info(frm).GPS_time)); + quality(val_idx,point_ids) = ones(size(obj.undo_stack.user_data.layer_info(frm).gps_time)); + y(val_idx,point_ids) = nan(size(obj.undo_stack.user_data.layer_info(frm).gps_time)); + type(val_idx,point_ids) = 2*ones(size(obj.undo_stack.user_data.layer_info(frm).gps_time)); end end end - + % Merge the layers and determine a mask of where changes will be made merged_y = nan(size(obj.undo_stack.user_data.frame)); merged_quality = ones(size(obj.undo_stack.user_data.frame)); @@ -669,13 +706,16 @@ function layerCM_callback(obj,source,event) h_fig = figure; clf(h_fig); h_axes = axes('parent',h_fig); - h_plot = plot(y(1,:).','b.','parent',h_axes,'LineWidth',2); hold(h_axes,'on'); - h_plot(end+(1:length(vals)-1)) = plot(y(2:end,:).','r^','parent',h_axes,'LineWidth',2); - h_plot(end+1) = plot(find(mask),merged_y(mask),'go','parent',h_axes,'LineWidth',2); - legend(h_plot([1 2 end]),'Original','Merging layers','Merged'); - - prompt = questdlg(sprintf('Are you sure you want to merge the %d selected layers? See plot showing the merge operation. All layers will be deleted except for layer %d.', ... + h_plot = plot(y(1,:).'*1e6,'b.','parent',h_axes,'LineWidth',2); + xlabel('GPS time index'); + ylabel('Two-way propagation (\mus)'); + set(h_axes,'YDir',obj.eg.y_order); + h_plot(end+(1:length(vals)-1)) = plot(y(2:end,:).'*1e6,'r^','parent',h_axes,'LineWidth',2); + h_plot(end+1) = plot(find(mask),merged_y(mask)*1e6,'go','parent',h_axes,'LineWidth',2); + legend(h_plot([1 2 end]),sprintf('Master Layer %d',vals(1)),'Merged Layers','Merged Result'); + + prompt = questdlg(sprintf('Are you sure you want to merge the %d selected layers? See plot showing the merge operation. All selected layers will be deleted except for layer %d.', ... length(vals), vals(1)), ... 'Merge Layers','Yes','Cancel','Cancel'); if ~strcmpi(prompt,'Yes') @@ -706,7 +746,7 @@ function layerCM_callback(obj,source,event) name = obj.eg.layers.lyr_name{val}; order = obj.eg.layers.lyr_order(val); fprintf('Delete layer %s:%s\n', group_name, name); - + % Delete all the points in the layer nan_mask = ~isnan(y(val_idx,:)); cmds(end+1).undo_cmd = 'insert'; @@ -733,10 +773,10 @@ function layerCM_callback(obj,source,event) end elseif source == obj.left_panel.layerCM_delete - if strcmpi(obj.eg.layers.source,'layerData') + %% delete + if strcmpi(obj.eg.layers.source,'layerdata') % Get the currently selected layers. vals = get(obj.left_panel.layerLB,'Value'); - vals = vals(vals>2); if length(vals) > 1 prompt = questdlg(sprintf('Are you sure you want to delete the %d selected layers?', ... @@ -798,4 +838,96 @@ function layerCM_callback(obj,source,event) end +elseif source == obj.left_panel.layerCM_detrend + %% detrend + + state = get(obj.left_panel.layerCM_detrend,'Checked'); + if strcmp(state,'off') + set(obj.left_panel.layerCM_detrend,'Checked','on') + else + set(obj.left_panel.layerCM_detrend,'Checked','off') + end + + % Replot image + cur_axis = axis(obj.h_axes); + + % Convert x_min, x_max to GPS time + xlims = interp1(obj.eg.image_xaxis,obj.eg.image_gps_time,cur_axis(1:2),'linear','extrap'); + + % Update echogram with new settings + obj.plot_echogram(obj.eg.image_gps_time(1),obj.eg.image_gps_time(end),-inf,inf); + + % Draw data with new axis + obj.redraw(xlims(1),xlims(2),cur_axis(3),cur_axis(4),struct('clipped',3)); + +elseif source == obj.left_panel.layerCM_multiple + %% multiple suppression + state = get(obj.left_panel.layerCM_multiple,'Checked'); + if strcmp(state,'off') + set(obj.left_panel.layerCM_multiple,'Checked','on') + else + set(obj.left_panel.layerCM_multiple,'Checked','off') + end + + % Replot image + cur_axis = axis(obj.h_axes); + + % Convert x_min, x_max to GPS time + xlims = interp1(obj.eg.image_xaxis,obj.eg.image_gps_time,cur_axis(1:2),'linear','extrap'); + + % Update echogram with new settings + obj.plot_echogram(obj.eg.image_gps_time(1),obj.eg.image_gps_time(end),-inf,inf); + + % Draw data with new axis + obj.redraw(xlims(1),xlims(2),cur_axis(3),cur_axis(4),struct('clipped',3)); + +elseif source == obj.left_panel.layerCM_properties + %% properties + obj.eg.detrend.top = 1; + obj.eg.detrend.bottom = 2; + obj.eg.detrend.order = 7; + def = {sprintf('%d',obj.eg.detrend.top),sprintf('%d',obj.eg.detrend.bottom),sprintf('%d',obj.eg.detrend.order)}; + + prompt = {'Detrend top layer','Detrend bottom layer','Detrend polynomial order'}; + answer = inputdlg(prompt,'Detrend properties',1,def); + + if length(answer) == 3 && ~isempty(answer{3}) + % detrend top + try + obj.eg.detrend.top = eval(answer{1}); + catch + obj.eg.detrend.top = 1; + end + if isempty(obj.eg.detrend.top) + obj.eg.detrend.top = 1; + end + + % detrend bottom + try + obj.eg.detrend.bottom = eval(answer{2}); + catch + obj.eg.detrend.bottom = 2; + end + if isempty(obj.eg.detrend.bottom) + obj.eg.detrend.bottom = 2; + end + + % detrend order + try + obj.eg.detrend.order = eval(answer{3}); + catch + obj.eg.detrend.order = 7; + end + if isempty(obj.eg.detrend.order) + obj.eg.detrend.order = 7; + end + + detrend_state = get(obj.left_panel.layerCM_detrend,'Checked'); + mult_state = get(obj.left_panel.layerCM_multiple,'Checked'); + if strcmp(detrend_state,'on') || strcmp(mult_state,'on') + % Replot image + obj.plot_echogram(obj.eg.image_gps_time(1),obj.eg.image_gps_time(end),-inf,inf); + end + + end end diff --git a/cresis-toolbox/+imb/@echowin/layerLB_callback.m b/cresis-toolbox/+imb/@echowin/layerLB_callback.m index 7be3ac6b..aede6abb 100644 --- a/cresis-toolbox/+imb/@echowin/layerLB_callback.m +++ b/cresis-toolbox/+imb/@echowin/layerLB_callback.m @@ -26,11 +26,18 @@ function layerLB_callback(obj,source,event) end else + + str = 'Select layers to operate on, right click to open context menu to manipulate list of layers. Red font indicates layer visibility is off.'; val = get(source,'Value'); - + for i = 1:length(val) + str = sprintf('%s\n(%d):%s %s',str,val(i),obj.eg.layers.lyr_name{val(i)},obj.eg.layers.lyr_desc{val(i)}); + end + sprintf(str) + set(obj.left_panel.layerLB,'TooltipString',str); + str = ''; obj.eg.layers.selected_layers(:)=false; obj.eg.layers.selected_layers(val)=true; - % Update plot based on selection + % Update plot based on selectio obj.set_visibility(); end \ No newline at end of file diff --git a/cresis-toolbox/+imb/@echowin/layerLB_str.m b/cresis-toolbox/+imb/@echowin/layerLB_str.m index de1fcf74..ed15764f 100644 --- a/cresis-toolbox/+imb/@echowin/layerLB_str.m +++ b/cresis-toolbox/+imb/@echowin/layerLB_str.m @@ -1,5 +1,12 @@ -function layerLB_str(obj) +function layerLB_str(obj,keep_value) +if exist('keep_value','var') || isempty(keep_value) + keep_value = false; +end + +if ~keep_value + set(obj.left_panel.layerLB,'Value',[]); +end LB_strings = cell(1,length(obj.eg.layers.lyr_name)); for idx = 1:length(obj.eg.layers.lyr_name) name = sprintf(obj.eg.layers.lyr_name{idx},idx); diff --git a/cresis-toolbox/+imb/@echowin/load_echogram.m b/cresis-toolbox/+imb/@echowin/load_echogram.m index 2e9d55a9..8c241763 100644 --- a/cresis-toolbox/+imb/@echowin/load_echogram.m +++ b/cresis-toolbox/+imb/@echowin/load_echogram.m @@ -25,8 +25,24 @@ % Determine the desired image sourceMenus = get(obj.left_panel.sourceCM,'Children'); -img = length(sourceMenus)-strmatch('on',get(sourceMenus,'Checked'))-3; - +img = []; +for idx = 1:length(sourceMenus) + if strncmp(sourceMenus(idx).Label,'Image',5) + if isequal(sourceMenus(idx).Checked,'on') + img = str2double(sourceMenus(idx).Label(7:end)); + end + elseif strcmp(sourceMenus(idx).Label,'Combined') + if isequal(sourceMenus(idx).Checked,'on') + img = 0; + end + % Combined is the last one so break + break; + end +end +if isempty(img) + sourceMenus(idx).Checked = 'on'; +end + % Determine if the sources exist for the desired frames, source, and image desire_exists = obj.eg.source_fns_existence(desire_frame_idxs,source_idx,img+1); @@ -42,6 +58,12 @@ if isempty(img) first_match = find(obj.eg.source_fns_existence(most_desire_frame_idx,:,:),1); if isempty(first_match) + fprintf('Searching for echogram source files in:\n'); + for source_idx = 1:length(obj.eg.sources) + fn_dir = ct_filename_out(obj.eg.cur_sel,obj.eg.sources{source_idx}); + fprintf(' %s\n', fn_dir); + end + errordlg(sprintf('Searched echogram sources specified in preferences and no matching echogram files exist for frame %03d.', most_desire_frame_idx), 'No matching echograms found.') error('No source files exist for frame %03d.', most_desire_frame_idx); else warning('Source %s does not exist for frame %03d.', obj.eg.sources{source_idx}, most_desire_frame_idx); @@ -108,8 +130,60 @@ obj.eg.lat = obj.eg.lat(valid_mask); obj.eg.lon = obj.eg.lon(valid_mask); obj.eg.elev = obj.eg.elev(valid_mask); +obj.eg.roll = obj.eg.roll(valid_mask); obj.eg.surf_twtt = obj.eg.surf_twtt(valid_mask); +%% Determine new time axis +min_time = inf; +max_time = -inf; +dt = 0; +for frame_idx = 1:length(desire_frame_idxs) + cur_frame = desire_frame_idxs(frame_idx); + + % load EG + fn = fullfile(ct_filename_out(obj.eg.cur_sel,obj.eg.sources{source_idx},'',0),sprintf('Data_%s%s.mat',fn_img_str,obj.eg.frm_strs{cur_frame})); + if ~exist(fn,'file') + warning('File %s not found', fn); + keyboard + end + % Load EG data and metadata + tmp_vars = whos('-file',fn); + if any(strcmp('Truncate_Bins',{tmp_vars.name})) + tmp = load(fn,'Time','Truncate_Bins','Elevation_Correction'); + else + tmp = load(fn); + end + tmp = uncompress_echogram(tmp); + if isempty(tmp.Time) + % Handle special case of file with all bad data + warning('File does not have any good data so skipping for obj.eg.time axis creation: %s', fn); + else + if tmp.Time(1) < min_time + min_time = tmp.Time(1); + end + if tmp.Time(end) > max_time + max_time = tmp.Time(end); + end + if length(tmp.Time) >= 2 + if dt == 0 || dt < tmp.Time(2) - tmp.Time(1) + dt = tmp.Time(2) - tmp.Time(1); + end + end + end +end +if dt == 0 + dt = 1; +end +if ~isfinite(min_time) + min_time = 0; + max_time = dt; +end +new_time = (dt*round(min_time/dt) : dt : max_time).'; +if ~isempty(obj.eg.data) + obj.eg.data = interp1(obj.eg.time, obj.eg.data, new_time); +end +obj.eg.time = new_time; + %% Loading new data fprintf(' Loading echogram (%s)\n',datestr(now,'HH:MM:SS')); physical_constants; @@ -126,8 +200,15 @@ end % Load EG data and metadata tmp = load(fn); + if ~isfield(tmp,'Roll') + tmp.Roll = zeros(size(tmp.GPS_time)); + end tmp.Time = reshape(tmp.Time,[length(tmp.Time) 1]); % Fixes a bug in some echograms tmp = uncompress_echogram(tmp); + if length(tmp.Time) < 2 + tmp.Time = min_time+[0 dt]; + tmp.Data(1:2,:) = nan; + end % Remove any data in echogram that is not part of the frame being loaded % (e.g. some echograms were created with some data from neighboring @@ -138,6 +219,7 @@ tmp.Latitude = tmp.Latitude(valid_mask); tmp.Longitude = tmp.Longitude(valid_mask); tmp.Elevation = tmp.Elevation(valid_mask); + tmp.Roll = tmp.Roll(valid_mask); tmp.GPS_time = tmp.GPS_time(valid_mask); if isfield(tmp,'Surface') && ~isempty(tmp.Surface) tmp.Surface = tmp.Surface(valid_mask); @@ -178,6 +260,7 @@ obj.eg.lat = cat(2,obj.eg.lat,tmp.Latitude); obj.eg.lon = cat(2,obj.eg.lon,tmp.Longitude); obj.eg.elev = cat(2,obj.eg.elev,tmp.Elevation); + obj.eg.roll = cat(2,obj.eg.roll,tmp.Roll); obj.eg.gps_time = cat(2,obj.eg.gps_time,tmp.GPS_time); obj.eg.surf_twtt = cat(2,obj.eg.surf_twtt,tmp.Surface); obj.eg.data = cat(2,obj.eg.data,tmp.Data); @@ -189,6 +272,8 @@ obj.eg.lon(splice_idx:end)); obj.eg.elev = cat(2,obj.eg.elev(1:splice_idx-1),tmp.Elevation, ... obj.eg.elev(splice_idx:end)); + obj.eg.roll = cat(2,obj.eg.roll(1:splice_idx-1),tmp.Roll, ... + obj.eg.roll(splice_idx:end)); obj.eg.gps_time = cat(2,obj.eg.gps_time(1:splice_idx-1),tmp.GPS_time, ... obj.eg.gps_time(splice_idx:end)); obj.eg.surf_twtt = cat(2,obj.eg.surf_twtt(1:splice_idx-1),tmp.Surface, ... diff --git a/cresis-toolbox/+imb/@echowin/load_flightline.m b/cresis-toolbox/+imb/@echowin/load_flightline.m index f0350ac6..7ea86185 100644 --- a/cresis-toolbox/+imb/@echowin/load_flightline.m +++ b/cresis-toolbox/+imb/@echowin/load_flightline.m @@ -19,13 +19,16 @@ function load_flightline(obj) [lat,lon] = projinv(obj.eg.proj,data.properties.X,data.properties.Y); [data.properties.X,data.properties.Y] = google_map.latlon_to_world(lat,lon); data.properties.Y = 256-data.properties.Y; + elseif obj.eg.map.source == 3 + [data.properties.Y,data.properties.X] = projinv(obj.eg.proj,data.properties.X,data.properties.Y); end obj.eg.map_x = double(data.properties.X)/obj.eg.map.scale; obj.eg.map_y = double(data.properties.Y)/obj.eg.map.scale; -%% LayerData: Loading flight path from layerData +%% LayerData: Loading flight path from tracks files else - fprintf(' Loading flight path from layerData(%s)\n', datestr(now,'HH:MM:SS')); + fprintf(' Loading flight path from csarp_support/tracks files (%s)\n', datestr(now,'HH:MM:SS')); + obj.eg.map_id = []; obj.eg.map_gps_time = []; obj.eg.map_elev = []; obj.eg.map_x = []; @@ -39,6 +42,9 @@ function load_flightline(obj) if obj.eg.map.source == 1 [X,Y] = google_map.latlon_to_world(obj.undo_stack.user_data.layer_info(obj.eg.frms(idx)).lat,obj.undo_stack.user_data.layer_info(obj.eg.frms(idx)).lon); Y = 256-Y; + elseif obj.eg.map.source == 3 + X = obj.undo_stack.user_data.layer_info(obj.eg.frms(idx)).lon; + Y = obj.undo_stack.user_data.layer_info(obj.eg.frms(idx)).lat; else [X,Y] = projfwd(obj.eg.proj,obj.undo_stack.user_data.layer_info(obj.eg.frms(idx)).lat,obj.undo_stack.user_data.layer_info(obj.eg.frms(idx)).lon); end diff --git a/cresis-toolbox/+imb/@echowin/load_layers.m b/cresis-toolbox/+imb/@echowin/load_layers.m index 64e5eaa6..15c32383 100644 --- a/cresis-toolbox/+imb/@echowin/load_layers.m +++ b/cresis-toolbox/+imb/@echowin/load_layers.m @@ -32,12 +32,11 @@ function load_layers(obj) [status,data] = opsGetLayerPoints(obj.eg.system,ops_param); %% OPS: Preallocate layer arrays - obj.eg.layers.x = {}; obj.eg.layers.y = {}; obj.eg.layers.qual = {}; obj.eg.layers.type = {}; + obj.eg.layers.x = double(obj.eg.map_gps_time); % gps-time for idx = 1:length(obj.eg.layers.lyr_name) - obj.eg.layers.x{idx} = double(obj.eg.map_gps_time); % gps-time obj.eg.layers.y{idx} = nan(size(obj.eg.map_id)); % twtt obj.eg.layers.qual{idx} = nan(size(obj.eg.map_id)); % integer 1-3 obj.eg.layers.type{idx} = nan(size(obj.eg.map_id)); % this is either 1 (manual) or 2 (auto) @@ -69,25 +68,22 @@ function load_layers(obj) %% LayerData: Load layer points from layerdata %% LayerData: Preallocate layer arrays - fprintf(' Loading layer points from layerData (%s)\n',datestr(now)); - obj.eg.layers.x = {}; + fprintf(' Loading layer points from layer data files (%s)\n',datestr(now)); + obj.eg.layers.x = []; obj.eg.layers.y = {}; obj.eg.layers.qual = {}; obj.eg.layers.type = {}; for idx = 1:length(obj.eg.layers.lyr_id) - obj.eg.layers.x{idx} = []; % gps time obj.eg.layers.y{idx} = []; % twtt obj.eg.layers.qual{idx} = []; % integer 1-3 obj.eg.layers.type{idx} = []; % this is either 1 (manual) or 2 (auto) end %% LayerData: Load gps_time, quality, twtt and type of layers from undo_stack - lGPS = []; for frm = obj.eg.frms(1):obj.eg.frms(end); gps_time = obj.undo_stack.user_data.layer_info(frm).gps_time; - lGPS = cat(2,lGPS,gps_time); % concatenates the layer GPS time + obj.eg.layers.x = cat(2,obj.eg.layers.x,gps_time); % concatenates the layer GPS time for idx=1:length(obj.eg.layers.lyr_name) - obj.eg.layers.x{idx} = cat(2,obj.eg.layers.x{idx},gps_time); % gps time lay_idx = find(obj.eg.layers.lyr_id(idx) == obj.undo_stack.user_data.layer_info(frm).id); if ~isempty(lay_idx) qual = obj.undo_stack.user_data.layer_info(frm).quality(lay_idx,:); @@ -105,17 +101,18 @@ function load_layers(obj) end end - %% LayerData: Update echogram surface if there are enough good points - % Find good surface points - if ~any(obj.eg.layers.lyr_id==1) - error('standard:surface must be added in the "Layers" preference window before loading echograms.'); - end - - % logical vector with 1 where the twtt of the surface is a number and 0 - % when NaN. - good_mask = ~isnan(obj.eg.layers.y{obj.eg.layers.lyr_id==1}); - if sum(good_mask) > 2 - obj.eg.surf_twtt = interp1(lGPS(good_mask),obj.eg.layers.y{obj.eg.layers.lyr_id==1}(good_mask),obj.eg.gps_time); - obj.eg.surf_twtt = interp_finite(obj.eg.surf_twtt,0); + %% LayerData: Update echogram surface if there are enough good points + if ~isempty(obj.eg.layers.lyr_id) + if isempty(obj.eg.layers.surf_id) || all(obj.eg.layers.surf_id ~= obj.eg.layers.lyr_id) + % Surface ID not set yet, assume it is the minimum + obj.eg.layers.surf_id = min(obj.eg.layers.lyr_id); + end + % good_mask: logical vector with 1 where the twtt of the surface is a number and 0 + % when NaN. + good_mask = ~isnan(obj.eg.layers.y{obj.eg.layers.lyr_id==obj.eg.layers.surf_id}); + if sum(good_mask) > 2 + obj.eg.surf_twtt = interp1(obj.eg.layers.x(good_mask),obj.eg.layers.y{obj.eg.layers.lyr_id==obj.eg.layers.surf_id}(good_mask),obj.eg.gps_time); + obj.eg.surf_twtt = interp_finite(obj.eg.surf_twtt,0); + end end end diff --git a/cresis-toolbox/+imb/@echowin/load_layers_init.m b/cresis-toolbox/+imb/@echowin/load_layers_init.m index 9d9e4fa4..26a247e4 100644 --- a/cresis-toolbox/+imb/@echowin/load_layers_init.m +++ b/cresis-toolbox/+imb/@echowin/load_layers_init.m @@ -5,8 +5,8 @@ function load_layers_init(obj) if strcmpi(obj.eg.layers.source,'OPS') %% OPS: Preallocating layer arrays + obj.eg.layers.x = double(obj.eg.map_gps_time); % gps-time for idx = 1:length(obj.eg.layers.lyr_id) - obj.eg.layers.x{idx} = double(obj.eg.map_gps_time); % gps-time obj.eg.layers.y{idx} = NaN*zeros(size(obj.eg.map_id)); % twtt obj.eg.layers.qual{idx} = NaN*zeros(size(obj.eg.map_id)); % integer 1-3 obj.eg.layers.type{idx} = NaN*zeros(size(obj.eg.map_id)); % this is either 1 (manual) or 2 (auto) @@ -14,8 +14,8 @@ function load_layers_init(obj) %% LayerData: Preallocating layer arrays else + obj.eg.layers.x = []; %gps time for idx = 1:length(obj.eg.layers.lyr_id) - obj.eg.layers.x{idx} = []; %gps time obj.eg.layers.y{idx} = []; % twtt obj.eg.layers.qual{idx} = []; % integer 1-3 obj.eg.layers.type{idx} = []; % this is either 1 (manual) or 2 (auto) @@ -25,12 +25,13 @@ function load_layers_init(obj) %% Plot layers delete(obj.h_layer); delete(obj.h_quality); +obj.h_layer = []; +obj.h_quality = []; % ------------------------------------------------------------------------- % WARNING: DO NOT IMPLEMENT WITH SCATTER... TOO SLOW RENDERING % ------------------------------------------------------------------------- -layer_data_x = obj.eg.layers.x; -for idx = 1:length(layer_data_x) +for idx = 1:length(obj.eg.layers.y) % Manual points (plot this way to handle empty XData or YData obj.h_layer(2*(idx-1)+1) = plot(obj.h_axes,NaN,NaN,'bx'); % Auto and manual points diff --git a/cresis-toolbox/+imb/@echowin/plot_echogram.m b/cresis-toolbox/+imb/@echowin/plot_echogram.m index 15091c53..fa657cbe 100644 --- a/cresis-toolbox/+imb/@echowin/plot_echogram.m +++ b/cresis-toolbox/+imb/@echowin/plot_echogram.m @@ -3,16 +3,68 @@ function plot_echogram(obj,x_min,x_max,y_min,y_max) % % Plot echogram data from echogram files +if ~obj.busy_mode + set_busy_mode = true; + obj.busy_mode = true; + set(obj.h_fig,'Pointer','watch'); + obj.status_text_set(sprintf('(%s) Plotting echogram...', datestr(now,'HH:MM:SS')),'replace'); + drawnow; +else + set_busy_mode = false; +end + physical_constants; + +%% Apply optional multiple suppression % ====================================================================== + +state = get(obj.left_panel.layerCM_multiple,'Checked'); +if strcmp(state,'on') + try + mult_param = struct('window_units','b'); + pdata.Data = obj.eg.data; + pdata.Time = obj.eg.time; + pdata.Roll = obj.eg.roll; + pdata = echo_mult_suppress(pdata, obj.eg.surf_twtt, mult_param); + + catch ME + warning('Multiple suppression failed, multiple supppression properties may be incorrect. Error:\n%s', ME.getReport); + pdata = obj.eg.data; + end +else + pdata = obj.eg.data; +end + +%% Apply optional detrending +% ====================================================================== + +state = get(obj.left_panel.layerCM_detrend,'Checked'); +if strcmp(state,'on') + try + detrend_param = struct('method','polynomial','units','b'); + detrend_param.layer_top = round(interp1(obj.eg.time,... + 1:length(obj.eg.time),... + obj.eg.layers.y{obj.eg.detrend.top},'linear','extrap')); + detrend_param.layer_bottom = round(interp1(obj.eg.time,... + 1:length(obj.eg.time),... + obj.eg.layers.y{obj.eg.detrend.bottom},'linear','extrap')); + detrend_param.order = obj.eg.detrend.order; + pdata = 10.^(echo_detrend(10*log10(pdata),detrend_param)/10); + catch ME + warning('Detrend failed, detrend properties may be incorrect. Error:\n%s', ME.getReport); + pdata = obj.eg.data; + end +end + %% Convert the data along the x-axis according to the units +% ====================================================================== xaxis_choice = get(obj.left_panel.xaxisPM,'Value'); if xaxis_choice == 1 % rangeline % update image_xaxis and image_gps_time obj.eg.image_xaxis = 1:length(obj.eg.gps_time); obj.eg.image_gps_time = obj.eg.gps_time; % update image_data according to xaxis_gpstime - obj.eg.image_data = obj.eg.data; + obj.eg.image_data = pdata; % update x label obj.eg.x_label = 'Range Line'; @@ -27,7 +79,7 @@ function plot_echogram(obj,x_min,x_max,y_min,y_max) along_track_uniform,'linear'); % update image_data according to image_gps_time obj.eg.image_data = interp1(obj.eg.gps_time,... - obj.eg.data.',obj.eg.image_gps_time,'linear').'; + pdata.',obj.eg.image_gps_time,'linear').'; % update x label obj.eg.x_label = 'Along track (km)'; @@ -41,7 +93,7 @@ function plot_echogram(obj,x_min,x_max,y_min,y_max) obj.eg.image_xaxis = gps_time_uniform; % update display_data according to xaxis_gpstime obj.eg.image_data = interp1(obj.eg.gps_time,... - obj.eg.data.',obj.eg.image_gps_time,'linear').'; + pdata.',obj.eg.image_gps_time,'linear').'; % update x label obj.eg.x_label = 'GPS time'; end @@ -54,8 +106,11 @@ function plot_echogram(obj,x_min,x_max,y_min,y_max) obj.eg.image_ecef = zeros(3,length(obj.eg.image_gps_time)); [obj.eg.image_ecef(1,:),obj.eg.image_ecef(2,:),obj.eg.image_ecef(3,:)] ... - = geodetic2ecef(obj.eg.image_lat/180*pi,obj.eg.image_lon/180*pi,obj.eg.image_elev,WGS84.ellipsoid); -[z_vec_x,z_vec_y,z_vec_z] = geodetic2ecef(obj.eg.image_lat/180*pi,obj.eg.image_lon/180*pi,obj.eg.image_elev-1,WGS84.ellipsoid); + = ct_lla2ecef(obj.eg.image_lat/180*pi,obj.eg.image_lon/180*pi,zeros(size(obj.eg.image_elev))); +[z_vec_x,z_vec_y,z_vec_z] = ct_lla2ecef(obj.eg.image_lat/180*pi,obj.eg.image_lon/180*pi,obj.eg.image_elev-1); +% [obj.eg.image_ecef(1,:),obj.eg.image_ecef(2,:),obj.eg.image_ecef(3,:)] ... +% = geodetic2ecef(obj.eg.image_lat/180*pi,obj.eg.image_lon/180*pi,zeros(size(obj.eg.image_elev)),WGS84.ellipsoid); +% [z_vec_x,z_vec_y,z_vec_z] = geodetic2ecef(obj.eg.image_lat/180*pi,obj.eg.image_lon/180*pi,obj.eg.image_elev-1,WGS84.ellipsoid); obj.eg.image_y_vec = zeros(3,length(obj.eg.image_gps_time)); obj.eg.image_z_vec = zeros(3,length(obj.eg.image_gps_time)); @@ -86,7 +141,7 @@ function plot_echogram(obj,x_min,x_max,y_min,y_max) % update yaxis and yaxis_time obj.eg.image_yaxis = obj.eg.time*1e6; % update y label - obj.eg.y_label = 'Two-way Propagation (\mus)'; + obj.eg.y_label = 'Two-way propagation (\mus)'; obj.eg.y_order = 'reverse'; elseif yaxis_choice == 2 % WGS_84 Elevation @@ -96,7 +151,7 @@ function plot_echogram(obj,x_min,x_max,y_min,y_max) surface = interp1(obj.eg.gps_time,... obj.eg.surf_twtt,obj.eg.image_gps_time,'linear'); physical_constants; - elev_max = max(elevation - time(1)*vel_air); + elev_max = max(elevation - surface*vel_air - (time(1)-surface)*vel_ice); elev_min = min(elevation - surface*vel_air - (time(end)-surface)*vel_ice); dt = time(2) - time(1); drange = dt * vel_ice; @@ -170,14 +225,16 @@ function plot_echogram(obj,x_min,x_max,y_min,y_max) depth_uniform = (depth_min:d_depth:depth_max).'; % depth axis we will interpolate onto % update image_data Nt = size(obj.eg.image_data,1); - obj.eg.image_data = [obj.eg.image_data;... - zeros(length(depth_uniform)-Nt,size(obj.eg.image_data,2))]; + % HACK: Not sure why, but using new_img instead of obj.eg.image_data speeds + % this code up in some situations + new_img = zeros(length(depth_uniform), size(obj.eg.image_data,2)); for idx = 1:length(surface) depth = min(0,time-surface(idx)) * vel_air ... + max(0,time-surface(idx)) * vel_ice; - obj.eg.image_data(:,idx) = interp1(depth,... - obj.eg.image_data(1:Nt,idx),depth_uniform,'linear'); + new_img(:,idx) = interp1(depth,... + obj.eg.image_data(:,idx),depth_uniform,'linear'); end + obj.eg.image_data = new_img; % update image_yaxis obj.eg.image_yaxis = depth_uniform; % update y label @@ -228,6 +285,18 @@ function plot_echogram(obj,x_min,x_max,y_min,y_max) ylim(obj.h_axes,sort([y_min y_max])) %% Apply the display mode -obj.left_panel.imagewin.set_cdata(obj.eg.image_data); +% obj.left_panel.imagewin.set_cdata(obj.eg.image_data); % <-- This might be +% needed: If so, document the need better when it is uncommented because +% this is the second call to set_cdata. + +if set_busy_mode + obj.busy_mode = false; + if obj.zoom_mode + set(obj.h_fig,'Pointer','custom'); + else + set(obj.h_fig,'Pointer','Arrow'); + end + obj.status_text_set(sprintf(' done. (%s)', datestr(now,'HH:MM:SS')),'append'); +end end diff --git a/cresis-toolbox/+imb/@echowin/plot_layers.m b/cresis-toolbox/+imb/@echowin/plot_layers.m index 9f7fde5a..c1eb8e5e 100644 --- a/cresis-toolbox/+imb/@echowin/plot_layers.m +++ b/cresis-toolbox/+imb/@echowin/plot_layers.m @@ -23,10 +23,10 @@ function plot_layers(obj) layer_y_curUnit = obj.eg.layers.y; elevation = interp1(obj.eg.gps_time,... obj.eg.elev,... - obj.eg.layers.x{1},'linear','extrap'); + obj.eg.layers.x,'linear','extrap'); surface = interp1(obj.eg.gps_time,... obj.eg.surf_twtt,... - obj.eg.layers.x{1},'linear','extrap'); + obj.eg.layers.x,'linear','extrap'); for idx = 1: length(layer_y_curUnit) range = min(layer_y_curUnit{idx},surface)*vel_air ... +max(0,layer_y_curUnit{idx}-surface) * vel_ice; @@ -39,7 +39,7 @@ function plot_layers(obj) layer_y_curUnit = obj.eg.layers.y; surface = interp1(obj.eg.gps_time,... obj.eg.surf_twtt,... - obj.eg.layers.x{1},'linear','extrap'); + obj.eg.layers.x,'linear','extrap'); for idx = 1:length(layer_y_curUnit) layer_y_curUnit{idx} = min(layer_y_curUnit{idx},surface)*vel_air ... +max(0,layer_y_curUnit{idx}-surface)*vel_ice; @@ -60,7 +60,7 @@ function plot_layers(obj) layer_y_curUnit = obj.eg.layers.y; surface = interp1(obj.eg.gps_time,... obj.eg.surf_twtt,... - obj.eg.layers.x{1},'linear','extrap'); + obj.eg.layers.x,'linear','extrap'); for idx = 1:length(layer_y_curUnit) layer_y_curUnit{idx} = min(0,(layer_y_curUnit{idx}-surface))*vel_air ... +max(0,(layer_y_curUnit{idx}-surface))*vel_ice; @@ -71,14 +71,21 @@ function plot_layers(obj) %% Plot layers %% WARNING: DO NOT IMPLEMENT WITH SCATTER... TOO SLOW RENDERING -layer_data_x = obj.eg.layers.x; -obj.eg.layers.x_curUnit = {}; +obj.eg.layers.x_curUnit = []; obj.eg.layers.y_curUnit = {}; -for idx = 1:length(layer_data_x) +try + delete(obj.h_layer(2*length(obj.eg.layers.y) + 1:end)); +end +obj.h_layer = obj.h_layer(1:2*length(obj.eg.layers.y)); +try + delete(obj.h_quality(6*length(obj.eg.layers.y)+1:end)); +end +obj.h_quality = obj.h_quality(1:6*length(obj.eg.layers.y)); +for idx = 1:length(obj.eg.layers.y) % Convert x-axis units layer_x_curUnit = interp1(obj.eg.image_gps_time,... obj.eg.image_xaxis,... - layer_data_x{idx},'linear','extrap'); + obj.eg.layers.x,'linear','extrap'); % get manual/auto pts (use them for layer handles) layer_manual = obj.eg.layers.type{idx} == 1; @@ -125,7 +132,7 @@ function plot_layers(obj) {layer_x_curUnit,layer_y_curUnit_derived}); %% Update obj.eg layers with current units - obj.eg.layers.x_curUnit{idx} = layer_x_curUnit; + obj.eg.layers.x_curUnit = layer_x_curUnit; obj.eg.layers.y_curUnit{idx} = layer_y_curUnit{idx}; end diff --git a/cresis-toolbox/+imb/@echowin/redraw.m b/cresis-toolbox/+imb/@echowin/redraw.m index 51e3d02b..57ce6db3 100644 --- a/cresis-toolbox/+imb/@echowin/redraw.m +++ b/cresis-toolbox/+imb/@echowin/redraw.m @@ -178,12 +178,20 @@ function redraw(obj,x_min,x_max,y_min,y_max,param) % No frames changed % Update echogram surface if there are enough good points from OPS % Find good surface points - good_mask = ~isnan(obj.eg.layers.y{obj.eg.layers.lyr_id==1}); - if sum(good_mask) > 2 - % There are surface layer points in the database, overwrite the surface - % with these - obj.eg.surf_twtt = interp1(obj.eg.map_gps_time(good_mask),obj.eg.layers.y{obj.eg.layers.lyr_id==1}(good_mask),obj.eg.gps_time); - obj.eg.surf_twtt = interp_finite(obj.eg.surf_twtt,0); + if ~isempty(obj.eg.layers.lyr_id) + if isempty(obj.eg.layers.surf_id) || all(obj.eg.layers.surf_id ~= obj.eg.layers.lyr_id) + % Surface ID not set yet, assume it is the minimum + obj.eg.layers.surf_id = min(obj.eg.layers.lyr_id); + end + % good_mask: logical vector with 1 where the twtt of the surface is a number and 0 + % when NaN. + good_mask = ~isnan(obj.eg.layers.y{obj.eg.layers.lyr_id==obj.eg.layers.surf_id}); + if sum(good_mask) > 2 + % There are surface layer points in the database, overwrite the surface + % with these + obj.eg.surf_twtt = interp1(obj.eg.map_gps_time(good_mask),obj.eg.layers.y{obj.eg.layers.lyr_id==1}(good_mask),obj.eg.gps_time); + obj.eg.surf_twtt = interp_finite(obj.eg.surf_twtt,0); + end end obj.plot_echogram(x_min,x_max,y_min,y_max); obj.plot_layers(); diff --git a/cresis-toolbox/+imb/@echowin/set_cursor_by_map.m b/cresis-toolbox/+imb/@echowin/set_cursor_by_map.m index f91c8ea0..7814c573 100644 --- a/cresis-toolbox/+imb/@echowin/set_cursor_by_map.m +++ b/cresis-toolbox/+imb/@echowin/set_cursor_by_map.m @@ -9,7 +9,8 @@ function set_cursor_by_map(obj,lat,lon,type,elev) if ~obj.busy_mode physical_constants; - [x,y,z] = geodetic2ecef(lat/180*pi,lon/180*pi,0,WGS84.ellipsoid); + [x,y,z] = ct_lla2ecef(lat/180*pi,lon/180*pi,0); + %[x,y,z] = geodetic2ecef(lat/180*pi,lon/180*pi,0,WGS84.ellipsoid); [~,rline] = min((obj.eg.image_ecef(1,:)-x).^2 + (obj.eg.image_ecef(2,:)-y).^2 + (obj.eg.image_ecef(3,:)-z).^2); % CONVERT range/elev to y @@ -17,7 +18,8 @@ function set_cursor_by_map(obj,lat,lon,type,elev) surf_range = obj.eg.image_surf_twtt(rline)*c/2; slowness_air = 2/c; if map_caused_call - [x,y,z] = geodetic2ecef(lat/180*pi,lon/180*pi,obj.eg.image_elev(rline),WGS84.ellipsoid); + [x,y,z] = ct_lla2ecef(lat/180*pi,lon/180*pi,obj.eg.image_elev(rline)); + %[x,y,z] = geodetic2ecef(lat/180*pi,lon/180*pi,obj.eg.image_elev(rline),WGS84.ellipsoid); range = sqrt((obj.eg.image_ecef(1,rline)-x).^2 + (obj.eg.image_ecef(2,rline)-y).^2 + (obj.eg.image_ecef(3,rline)-z).^2); if yaxis_choice == 1 % TWTT y = range*slowness_air*1e6; diff --git a/cresis-toolbox/+imb/@echowin/status_text_cursor.m b/cresis-toolbox/+imb/@echowin/status_text_cursor.m index 8c7c9342..1036bc43 100644 --- a/cresis-toolbox/+imb/@echowin/status_text_cursor.m +++ b/cresis-toolbox/+imb/@echowin/status_text_cursor.m @@ -23,13 +23,13 @@ else physical_constants; elev = obj.eg.image_elev(rline); - [~,unique_idxs] = unique(obj.eg.layers.x{1}); + [~,unique_idxs] = unique(obj.eg.layers.x); warning off; - surf_y = interp1(obj.eg.layers.x{1}(unique_idxs), ... + surf_y = interp1(obj.eg.layers.x(unique_idxs), ... obj.eg.layers.y{1}(unique_idxs),obj.cursor.gps_time); - [~,unique_idxs] = unique(obj.eg.layers.x{cur_layers(1)}); + [~,unique_idxs] = unique(obj.eg.layers.x); try - layer_y = interp1(obj.eg.layers.x{cur_layers(1)}(unique_idxs), ... + layer_y = interp1(obj.eg.layers.x(unique_idxs), ... obj.eg.layers.y{cur_layers(1)}(unique_idxs),obj.cursor.gps_time); if ~all(isfinite(layer_y)) error(''); diff --git a/cresis-toolbox/+imb/@echowin/undo_stack_modified_check.m b/cresis-toolbox/+imb/@echowin/undo_stack_modified_check.m index 27baad2d..cfab0fd0 100644 --- a/cresis-toolbox/+imb/@echowin/undo_stack_modified_check.m +++ b/cresis-toolbox/+imb/@echowin/undo_stack_modified_check.m @@ -1,13 +1,26 @@ -function cancel_operation = undo_stack_modified_check(obj) - +function cancel_operation = undo_stack_modified_check(obj,force_save_or_cancel_flag) +% cancel_operation = undo_stack_modified_check(obj,force_save_or_cancel_flag) +% % Check to see if undo stack got created and, if it did, if there are % any items in the undostack (i.e. operations that have not been saved) +% +% force_save_or_cancel_flag: optional logic scalar. default is false. If +% true, then the only options are Yes or Cancel. + +if nargin < 2 || isempty(force_save_or_cancel_flag) + force_save_or_cancel_flag = false; +end if isempty(obj.undo_stack) || ~obj.undo_stack.ismodified() cancel_operation = false; else - prompt = questdlg('There are unsaved layers. You do not need to save them now, but would you like to?',... - 'Unsaved Layers','Yes','No','Cancel','Yes'); + if force_save_or_cancel_flag + prompt = questdlg('There are unsaved layers. You must save them now to continnue with this operation. Would you like to save and continue?',... + 'Unsaved Layers','Yes','Cancel','Yes'); + else + prompt = questdlg('There are unsaved layers. You do not need to save them now, but would you like to?',... + 'Unsaved Layers','Yes','No','Cancel','Yes'); + end switch prompt case 'Yes' % save diff --git a/cresis-toolbox/+imb/@echowin/update_cursor.m b/cresis-toolbox/+imb/@echowin/update_cursor.m index 81f40125..418651a3 100644 --- a/cresis-toolbox/+imb/@echowin/update_cursor.m +++ b/cresis-toolbox/+imb/@echowin/update_cursor.m @@ -82,7 +82,8 @@ ecef = ecef - z_vec*z_offset; ecef = [ecef + y_vec*cross_track, ecef - y_vec*cross_track]; - [obj.cursor.clutter_lat obj.cursor.clutter_lon] = ecef2geodetic(ecef(1,:),ecef(2,:),ecef(3,:),WGS84.ellipsoid); + [obj.cursor.clutter_lat obj.cursor.clutter_lon] = ct_ecef2lla(ecef(1,:),ecef(2,:),ecef(3,:)); + %[obj.cursor.clutter_lat obj.cursor.clutter_lon] = ecef2geodetic(ecef(1,:),ecef(2,:),ecef(3,:),WGS84.ellipsoid); obj.cursor.clutter_lat = obj.cursor.clutter_lat * 180/pi; obj.cursor.clutter_lon = obj.cursor.clutter_lon * 180/pi; diff --git a/cresis-toolbox/+imb/@echowin/update_layer_plots.m b/cresis-toolbox/+imb/@echowin/update_layer_plots.m index c2ffa9d6..68d554ac 100644 --- a/cresis-toolbox/+imb/@echowin/update_layer_plots.m +++ b/cresis-toolbox/+imb/@echowin/update_layer_plots.m @@ -7,7 +7,7 @@ function update_layer_plots(obj) layer_y_curUnit = obj.eg.layers.y_curUnit; for idx = 1:length(layer_y_curUnit) - layer_x_curUnit = obj.eg.layers.x_curUnit{idx}; + layer_x_curUnit = obj.eg.layers.x_curUnit; % get manual/auto pts (use them for layer handles) layer_manual = obj.eg.layers.type{idx} == 1; diff --git a/cresis-toolbox/+imb/@echowin/update_source_fns_existence.m b/cresis-toolbox/+imb/@echowin/update_source_fns_existence.m index 3421d30f..64ba865f 100644 --- a/cresis-toolbox/+imb/@echowin/update_source_fns_existence.m +++ b/cresis-toolbox/+imb/@echowin/update_source_fns_existence.m @@ -8,6 +8,10 @@ function update_source_fns_existence(obj) % Frames, Sources, Images obj.eg.source_fns_existence = logical(zeros(length(obj.eg.frm_strs),length(obj.eg.sources),1)); +if isempty(obj.eg.sources) + error('There must be at least one echogram source entered for "Echogram Sources" in the preferences.'); +end + found_at_least_one_good_file = false; for source_idx = 1:length(obj.eg.sources) fn_dir = ct_filename_out(obj.eg.cur_sel,obj.eg.sources{source_idx}); @@ -44,6 +48,7 @@ function update_source_fns_existence(obj) end end if ~found_at_least_one_good_file + errordlg(sprintf('No good data files found for this frame. E.g. %s. Check to make sure echogram sources in preferences matches the echogram files you have available.', fn_dir),'No matching echograms found.') error('No good data files found for this frame. E.g. %s', fn_dir); end diff --git a/cresis-toolbox/+imb/@mapwin/button_motion.m b/cresis-toolbox/+imb/@mapwin/button_motion.m index 5501d1d5..8cced5c0 100644 --- a/cresis-toolbox/+imb/@mapwin/button_motion.m +++ b/cresis-toolbox/+imb/@mapwin/button_motion.m @@ -38,6 +38,10 @@ function button_motion(obj,src,event) if (obj.map.source == 1) [lat, lon] = google_map.world_to_latlon(x*obj.map.scale, 256-y*obj.map.scale); set(obj.status_panel.mouseCoordText,'String',sprintf('%8.3fN %8.3fW; X=%8.3f Y=%8.3f ',lat,lon,x,y)); + elseif (obj.map.source == 3) + lat = y; + lon = x; + set(obj.status_panel.mouseCoordText,'String',sprintf('%8.3fN %8.3fW; X=%8.3f Y=%8.3f ',lat,lon,x,y)); else [lat,lon] = projinv(obj.map.proj,x*obj.map.scale,y*obj.map.scale); set(obj.status_panel.mouseCoordText,'String',sprintf('%8.3fN %8.3fW; X=%8.3fkm Y=%8.3fkm ',lat,lon,x,y)); diff --git a/cresis-toolbox/+imb/@mapwin/button_up.m b/cresis-toolbox/+imb/@mapwin/button_up.m index e3899d47..402141c7 100644 --- a/cresis-toolbox/+imb/@mapwin/button_up.m +++ b/cresis-toolbox/+imb/@mapwin/button_up.m @@ -66,6 +66,9 @@ function button_up(obj,src,event) if (obj.map.source == 1) [lat, lon] = google_map.world_to_latlon(x*obj.map.scale, 256-y*obj.map.scale); + elseif (obj.map.source == 3) + lon = x*obj.map.scale; + lat = y*obj.map.scale; else [lat,lon] = projinv(obj.map.proj,x*obj.map.scale,y*obj.map.scale); end @@ -78,6 +81,9 @@ function button_up(obj,src,event) % Update the echowin's cursor on the map if obj.map.source == 1 [x,y] = google_map.latlon_to_world(obj.echowin_list(idx).cursor.lat, obj.echowin_list(idx).cursor.lon); y = 256-y; + elseif obj.map.source == 3 + x = obj.echowin_list(idx).cursor.lon; + y = obj.echowin_list(idx).cursor.lat; else [x,y] = projfwd(obj.map.proj, obj.echowin_list(idx).cursor.lat, obj.echowin_list(idx).cursor.lon); end diff --git a/cresis-toolbox/+imb/@mapwin/get_closest_frame.m b/cresis-toolbox/+imb/@mapwin/get_closest_frame.m index 88280cd0..bad30cbd 100644 --- a/cresis-toolbox/+imb/@mapwin/get_closest_frame.m +++ b/cresis-toolbox/+imb/@mapwin/get_closest_frame.m @@ -11,20 +11,20 @@ function get_closest_frame(obj, param) ops_param.properties.season = obj.cur_map_pref_settings.seasons; if obj.map.fline_source == 1 - % layerdata flineslines selected + % csarp_support/track flineslines selected % ----------------------------------------------------------------------- % Find the frame of the closest point - [~,idx] = min((obj.layerdata.y-param.y).^2+(obj.layerdata.x-param.x).^2); + [~,idx] = min((obj.trackdata.y-param.y).^2+(obj.trackdata.x-param.x).^2); % Extract out frame, system, and season name - frm_id = obj.layerdata.frm_id(idx); - season_idx = obj.layerdata.season_idx(idx); + frm_id = obj.trackdata.frm_id(idx); + season_idx = obj.trackdata.season_idx(idx); season_name = obj.cur_map_pref_settings.seasons{season_idx}; [sys,season_name] = strtok(season_name,'_'); season_name = season_name(2:end); % Get a logical mask indicating all indices that match the frame - frm_mask = obj.layerdata.frm_id == frm_id; + frm_mask = obj.trackdata.frm_id == frm_id; % Generate frame string YYYYMMDD_SS_FFF frm_id = num2str(frm_id); @@ -41,8 +41,8 @@ function get_closest_frame(obj, param) data.properties.frame = frm_str; data.properties.season = season_name; data.properties.segment_id = str2num(frm_id(1:10)); - data.properties.X = obj.layerdata.x(frm_mask); - data.properties.Y = obj.layerdata.y(frm_mask); + data.properties.X = obj.trackdata.x(frm_mask); + data.properties.Y = obj.trackdata.y(frm_mask); else % Get segment id from opsGetFrameSearch @@ -63,8 +63,8 @@ function get_closest_frame(obj, param) data.properties.frame = frm_str; data.properties.season = frm_data.properties.season; data.properties.segment_id = frm_data.properties.segment_id; - data.properties.X = obj.layerdata.x(frm_mask); - data.properties.Y = obj.layerdata.y(frm_mask); + data.properties.X = obj.trackdata.x(frm_mask); + data.properties.Y = obj.trackdata.y(frm_mask); status = frm_status; end end diff --git a/cresis-toolbox/+imb/@mapwin/get_map.m b/cresis-toolbox/+imb/@mapwin/get_map.m index 98cf7d8e..3a8fb161 100644 --- a/cresis-toolbox/+imb/@mapwin/get_map.m +++ b/cresis-toolbox/+imb/@mapwin/get_map.m @@ -4,14 +4,18 @@ function get_map(obj,hObj,event) % This is the callback function which is called when the preference % window "OK" button is pressed and the prefwin "StateChange" event occurs. -%% Determine map source -if strcmp('google_map', obj.map_pref.settings.map_name) +%% Determine map source and projection +if strcmp('Google Map', obj.map_pref.settings.map_name) obj.map.source = 1; obj.map.scale = 1; -elseif strcmp('blank_map', obj.map_pref.settings.map_name) +elseif strcmp('Blank Stereographic Map', obj.map_pref.settings.map_name) % blank map selected obj.map.source = 2; obj.map.scale = 1e3; +elseif strcmp('Blank Geodetic Map', obj.map_pref.settings.map_name) + % blank map selected + obj.map.source = 3; + obj.map.scale = 1; else % OPS map selected obj.map.source = 0; @@ -28,7 +32,7 @@ function get_map(obj,hObj,event) obj.map.fline_source = 1; end -%% Check which settings have changed +%% Which settings changed? if ~strcmpi(obj.cur_map_pref_settings.system, obj.map_pref.settings.system) system_changed = true; else @@ -76,21 +80,23 @@ function get_map(obj,hObj,event) return; end -%% Copy current preference window settings over +%% Copy current preference window settings obj.cur_map_pref_settings = obj.map_pref.settings; -%% Update map selection (also called at startup) +%% Update map selection % ================================================================= flightlines = obj.cur_map_pref_settings.flightlines; map_name = obj.cur_map_pref_settings.map_name; map_zone = obj.cur_map_pref_settings.map_zone; fprintf('Loading and plotting map %s:%s (%s)\n', map_zone, map_name, datestr(now,'HH:MM:SS')); +%% Connect to OPS if needed if obj.map.source == 0 || obj.map.fline_source == 0 opsCmd; if obj.map.fline_source == 0 - %% Create season and group ID strings for OPS flightline requests + % Create season and group ID strings for OPS flightline requests + % --------------------------------------------------------------------- % 1. create seasons viewparam if ~isempty(obj.cur_map_pref_settings.seasons) @@ -141,7 +147,7 @@ function get_map(obj,hObj,event) end if obj.map.source == 0 - %% Setup OPS map and flightlines for OPS maps + %% Setup "OPS" map and flightlines % Update axes labels xlabel(obj.map_panel.h_axes,'X (km)'); @@ -187,11 +193,12 @@ function get_map(obj,hObj,event) end elseif obj.map.source == 1 - %% Get Map: Google + %% Setup "Google" map and flightlines % Setup the Google map if isempty(obj.google.map) - obj.google.map = google_map(); + global gRadar; + obj.google.map = google_map(gRadar.ops.google_map_api_key); end % Update axes labels @@ -241,7 +248,7 @@ function get_map(obj,hObj,event) obj.map.yaxis_default = sort(256-obj.map.yaxis_default); elseif obj.map.source == 2 - %% Setup blank map and flightlines for OPS maps + %% Setup "Blank Stereographic" map and flightlines % Update axes labels xlabel(obj.map_panel.h_axes,'X (km)'); @@ -281,59 +288,125 @@ function get_map(obj,hObj,event) obj.map.xaxis_default = [-3400000 3400000]/1e3; obj.map.yaxis_default = [-3400000 3400000]/1e3; end + +elseif obj.map.source == 3 + %% Setup "Blank Geodetic" map and flightlines + + % Update axes labels + xlabel(obj.map_panel.h_axes,'Lon (deg)'); + ylabel(obj.map_panel.h_axes,'Lat (deg)'); + + % Setup OPS flightlines if enabled + wms_flightline_layer = []; + if obj.map.fline_source == 0 + % Rename to layer for readability + layer = obj.map_pref.ops.wms_capabilities.Layer; + % Setup OPS flightlines + if strcmpi(flightlines,'OPS Flightlines') && ~isempty(layer.refine('line_paths')) + wms_flightline_layer = layer.refine(sprintf('%s_%s_line_paths',map_zone,obj.cur_map_pref_settings.system),'MatchType','exact'); + elseif strcmpi(flightlines,'OPS Quality Flightlines') && ~isempty(layer.refine('data_quality')) + wms_flightline_layer = layer.refine(sprintf('%s_%s_data_quality',map_zone,obj.cur_map_pref_settings.system),'MatchType','exact'); + elseif strcmpi(flightlines,'OPS Coverage Flightlines') && ~isempty(layer.refine('data_coverage')) + wms_flightline_layer = layer.refine(sprintf('%s_%s_data_coverage',map_zone,obj.cur_map_pref_settings.system),'MatchType','exact'); + elseif strcmpi(flightlines,'OPS Crossover Errors') && ~isempty(layer.refine('crossover_errors')) + wms_flightline_layer = layer.refine(sprintf('%s_%s_crossover_errors',map_zone,obj.cur_map_pref_settings.system),'MatchType','exact'); + elseif strcmpi(flightlines,'OPS Bed Elevation') && ~isempty(layer.refine('data_elevation')) + wms_flightline_layer = layer.refine(sprintf('%s_%s_data_elevation',map_zone,obj.cur_map_pref_settings.system),'MatchType','exact'); + end + + % Get request + obj.ops.request = WMSMapRequest(wms_flightline_layer); + else + obj.ops.request = []; + end + + % Set projection code and default map bounds + if strcmp(map_zone,'arctic') + obj.ops.request.CoordRefSysCode = 'EPSG:4326'; + obj.map.xaxis_default = [-90 0]; + obj.map.yaxis_default = [45 90]; + else + obj.ops.request.CoordRefSysCode = 'EPSG:4326'; + obj.map.xaxis_default = [-180 180]; + obj.map.yaxis_default = [-90 -45]; + end end +%% Plot tracks files flightlines +% ========================================================================= if obj.map.fline_source == 1 + % Init tracks files data + % ----------------------------------------------------------------------- + obj.trackdata.x = []; + obj.trackdata.y = []; + obj.trackdata.frm_id = []; + obj.trackdata.season_idx = []; + obj.trackdata.frm_info = struct('frm_id',{},'start_gps_time',{},'stop_gps_time',{}); - %% Plot flightlines - obj.layerdata.x = []; - obj.layerdata.y = []; - obj.layerdata.frm_id = []; - obj.layerdata.season_idx = []; - obj.layerdata.frm_info = struct('frm_id',{},'start_gps_time',{},'stop_gps_time',{}); - - % Looping through the seasons - layer_fn_dir = ct_filename_support(struct('radar_name','rds'),'layer',''); + % Load each tracks file (tracks file list created in + % imb.prefwin.create_ui) + % ----------------------------------------------------------------------- + tracks_fn_dir = ct_filename_support(struct('radar_name','rds'),'tracks',''); for season_idx = 1:length(obj.cur_map_pref_settings.seasons) - %Loading the season layerdata files - layer_fn_name = sprintf('layer_%s_%s.mat', obj.cur_map_pref_settings.map_zone, obj.cur_map_pref_settings.seasons{season_idx}); - layer_fn = fullfile(layer_fn_dir,layer_fn_name); - S = load(layer_fn); + % Load the csarp_support/tracks file + % --------------------------------------------------------------------- + tracks_fn_name = sprintf('tracks_%s_%s.mat', obj.cur_map_pref_settings.map_zone, obj.cur_map_pref_settings.seasons{season_idx}); + tracks_fn = fullfile(tracks_fn_dir,tracks_fn_name); + S = load(tracks_fn,'lat','lon','frm_id','frm_info'); + % Concatenate data to tracks files data + % --------------------------------------------------------------------- if obj.map.source == 1 + % Google map [x,y] = google_map.latlon_to_world(S.lat, S.lon); y = 256-y; + elseif obj.map.source == 3 + % Blank Geodetic map + x = S.lon; + y = S.lat; else + % OPS or Blank Stereographic map [x,y] = projfwd(obj.map.proj, S.lat, S.lon); end x = x/obj.map.scale; y = y/obj.map.scale; - obj.layerdata.x = [obj.layerdata.x x]; - obj.layerdata.y = [obj.layerdata.y y]; - obj.layerdata.frm_id = [obj.layerdata.frm_id S.frm_id]; - obj.layerdata.season_idx = [obj.layerdata.season_idx season_idx*ones(size(x))]; - obj.layerdata.frm_info(season_idx) = S.frm_info; + obj.trackdata.x = [obj.trackdata.x x]; + obj.trackdata.y = [obj.trackdata.y y]; + obj.trackdata.frm_id = [obj.trackdata.frm_id S.frm_id]; + obj.trackdata.season_idx = [obj.trackdata.season_idx season_idx*ones(size(x))]; + obj.trackdata.frm_info(season_idx) = S.frm_info; % Plot flight lines - set(obj.map_panel.h_flightline,'XData',obj.layerdata.x,'YData',obj.layerdata.y); + % --------------------------------------------------------------------- + set(obj.map_panel.h_flightline,'XData',obj.trackdata.x,'YData',obj.trackdata.y); end else + % OPS flightlines are integrated into the web map service (WMS) request + % so we don't need to worry about flightlines here set(obj.map_panel.h_flightline,'XData',NaN,'YData',NaN); end +%% Update GUI +% ========================================================================= + % Turn map axes on if this is the first time a map is being loaded +% ------------------------------------------------------------------------- set(obj.map_panel.h_axes,'Visible', 'on'); set(obj.map_panel.h_image,'Visible', 'on'); % Set map bounds to default if this is the first time a map is being loaded % or if the projection changed +% ------------------------------------------------------------------------- if isempty(obj.map.xaxis) || ~strcmpi(obj.ops.request.CoordRefSysCode,obj.map.CoordRefSysCode) obj.map.xaxis = obj.map.xaxis_default; obj.map.yaxis = obj.map.yaxis_default; obj.map.CoordRefSysCode = obj.ops.request.CoordRefSysCode; end +% Update map axes +% ------------------------------------------------------------------------- obj.query_redraw_map(obj.map.xaxis(1),obj.map.xaxis(end),obj.map.yaxis(1),obj.map.yaxis(end)); % Reset selection +% ------------------------------------------------------------------------- obj.map.sel.frm_str = ''; % Current frame name obj.map.sel.seg_id = []; % Current segment ID (Database ID for OPS layer source, index into obj.cur_map_pref_settings.seasons for layerdata source) obj.map.sel.season_name = ''; % Current season name @@ -343,11 +416,16 @@ function get_map(obj,hObj,event) % Change map title to the currently selected frame set(obj.top_panel.flightLabel,'String',obj.map.sel.frm_str); +% Cleanup +% ------------------------------------------------------------------------- + % Redraw table to ensure everything is the right size table_draw(obj.table); +% Force map view into foreground figure(obj.h_fig); +% Save current settings obj.save_default_params(); fprintf(' Done (%s)\n', datestr(now)); diff --git a/cresis-toolbox/+imb/@mapwin/key_press.m b/cresis-toolbox/+imb/@mapwin/key_press.m index a6b354fa..e482cd62 100644 --- a/cresis-toolbox/+imb/@mapwin/key_press.m +++ b/cresis-toolbox/+imb/@mapwin/key_press.m @@ -85,24 +85,23 @@ function key_press(obj,src,event) % move plot down % break if already at the limit - if check_limits(obj,xaxis,yaxis,'d') - + %if check_limits(obj,xaxis,yaxis,'d') %break; - else + %else new_yaxis = [yaxis(1) - y_extent*0.4, yaxis(end) - y_extent*0.4]; - if new_yaxis(1) < obj.map.yaxis_default(1) - new_yaxis(1) = obj.map.yaxis_default(1); - new_yaxis(end) = new_yaxis(1) + y_extent; - end + %if new_yaxis(1) < obj.map.yaxis_default(1) + % new_yaxis(1) = obj.map.yaxis_default(1); + % new_yaxis(end) = new_yaxis(1) + y_extent; + %end % get a new map for these limits new_yaxis = sort(new_yaxis); % don't change the x limits in this case - new_xaxis = obj.ops.request.XLim/obj.map.scale; + new_xaxis = obj.map.xaxis; obj.query_redraw_map(new_xaxis(1),new_xaxis(end),... new_yaxis(1),new_yaxis(end)); - end + %end case 'uparrow' % Up arrow % move plot up @@ -111,21 +110,21 @@ function key_press(obj,src,event) %y_extent = diff(yaxis); % break if already at the limit - if check_limits(obj,xaxis,yaxis,'u') - else + %if check_limits(obj,xaxis,yaxis,'u') + %else new_yaxis = [yaxis(1) + y_extent*0.4, yaxis(end) + y_extent*0.4]; - if new_yaxis(end) > obj.map.yaxis_default(end) - new_yaxis(end) = obj.map.yaxis_default(end); - new_yaxis(1) = new_yaxis(end) - y_extent; - end + %if new_yaxis(end) > obj.map.yaxis_default(end) + % new_yaxis(end) = obj.map.yaxis_default(end); + % new_yaxis(1) = new_yaxis(end) - y_extent; + %end % get a new map for these limits new_yaxis = sort(new_yaxis); % don't change the x limits in this case - new_xaxis = obj.ops.request.XLim/obj.map.scale; + new_xaxis = obj.map.xaxis; obj.query_redraw_map(new_xaxis(1),new_xaxis(end),... new_yaxis(1),new_yaxis(end)); - end + %end case 'rightarrow' % Right arrow % move plot right @@ -134,21 +133,21 @@ function key_press(obj,src,event) end % break if already at the limit - if check_limits(obj,xaxis,yaxis,'r') %break; - else + %if check_limits(obj,xaxis,yaxis,'r') %break; + %else new_xaxis = [xaxis(1) + x_extent*0.4, xaxis(end) + x_extent*0.4]; - if new_xaxis(end) > obj.map.xaxis_default(end) - new_xaxis(end) = obj.map.xaxis_default(end); - new_xaxis(1) = new_xaxis(end) - x_extent; - end + %if new_xaxis(end) > obj.map.xaxis_default(end) + % new_xaxis(end) = obj.map.xaxis_default(end); + % new_xaxis(1) = new_xaxis(end) - x_extent; + %end % get a new map for these limits new_xaxis = sort(new_xaxis); % don't change the y limits in this case - new_yaxis = obj.ops.request.YLim/obj.map.scale; + new_yaxis = obj.map.yaxis; obj.query_redraw_map(new_xaxis(1),new_xaxis(end),... new_yaxis(1),new_yaxis(end)); - end + %end case 'leftarrow' % Left arrow % move plot left @@ -157,22 +156,22 @@ function key_press(obj,src,event) end % break if already at the limit - if check_limits(obj,xaxis,yaxis,'l') + %if check_limits(obj,xaxis,yaxis,'l') %break; - else + %else new_xaxis = [xaxis(1) - x_extent*0.4, xaxis(end) - x_extent*0.4]; - if new_xaxis(1) < obj.map.xaxis_default(1) - new_xaxis(1) = obj.map.xaxis_default(1); - new_xaxis(end) = new_xaxis(1) + x_extent; - end + %if new_xaxis(1) < obj.map.xaxis_default(1) + % new_xaxis(1) = obj.map.xaxis_default(1); + % new_xaxis(end) = new_xaxis(1) + x_extent; + %end % get a new map for these limits new_xaxis = sort(new_xaxis); % don't change the y limits in this case - new_yaxis = obj.ops.request.YLim/obj.map.scale; + new_yaxis = obj.map.yaxis; obj.query_redraw_map(new_xaxis(1),new_xaxis(end),... new_yaxis(1),new_yaxis(end)); - end + %end case 'z' if obj.control_pressed diff --git a/cresis-toolbox/+imb/@mapwin/loadPB_callback.m b/cresis-toolbox/+imb/@mapwin/loadPB_callback.m index e047b469..547142d5 100644 --- a/cresis-toolbox/+imb/@mapwin/loadPB_callback.m +++ b/cresis-toolbox/+imb/@mapwin/loadPB_callback.m @@ -71,7 +71,7 @@ function loadPB_callback(obj,hObj,event) addlistener(obj.echowin_list(echo_idx),'update_cursors',@obj.update_echowin_cursors); addlistener(obj.echowin_list(echo_idx),'update_map_selection',@obj.update_map_selection_echowin); addlistener(obj.echowin_list(echo_idx),'open_crossover_event',@obj.open_crossover_echowin); - addlistener(obj.echowin_list(echo_idx).tool.list{4},'ascope_memory',@obj.ascope_memory); + addlistener(obj.echowin_list(echo_idx).tool.list{7},'ascope_memory',@obj.ascope_memory); % Connect picktool_browse tool to ascope % Create a selection plot that identifies the echowin on the map obj.echowin_maps(echo_idx).h_cursor = plot(obj.map_panel.h_axes, [NaN],[NaN],'kx','LineWidth',2,'MarkerSize',10); @@ -80,13 +80,14 @@ function loadPB_callback(obj,hObj,event) end % Draw the echo class in the selected echowin +param = []; param.sources = obj.cur_map_pref_settings.sources; param.layers = obj.cur_map_pref_settings.layers; param.cur_sel = obj.map.sel; param.cur_sel.frm = str2num(param.cur_sel.frm_str(13:end)); param.cur_sel.location = obj.cur_map_pref_settings.map_zone; param.cur_sel.day_seg = param.cur_sel.frm_str(1:11); -if strcmp(obj.cur_map_pref_settings.system,'layerdata') +if strcmp(obj.cur_map_pref_settings.system,'tracks') param.system = param.cur_sel.radar_name; param.cur_sel.radar_name = param.cur_sel.radar_name; param.cur_sel.season_name = param.cur_sel.season_name; @@ -130,10 +131,23 @@ function loadPB_callback(obj,hObj,event) param.map = obj.map; if strcmpi(param.layer_source,'layerdata') % Find this season in the list of seasons - season_idx = find(strcmp(system_name_full,obj.cur_map_pref_settings.seasons)); - % Create a mask that identifies the frames for the selected segment in this season - frm_idxs = find(param.cur_sel.seg_id == floor(obj.layerdata.frm_info(season_idx).frm_id/1000)); - num_frm = length(frm_idxs); + if strcmp(obj.cur_map_pref_settings.system,'tracks') + % Season layer files: Load segment information + season_idx = find(strcmp(system_name_full,obj.cur_map_pref_settings.seasons)); + % Create a mask that identifies the frames for the selected segment in this season + frm_idxs = find(param.cur_sel.seg_id == floor(obj.trackdata.frm_info(season_idx).frm_id/1000)); + num_frm = length(frm_idxs); + else + % OPS: Load segment information + ops_param = struct('properties',[]); + ops_param.properties.location = param.cur_sel.location; + ops_param.properties.segment_id = param.cur_sel.seg_id; + [status,data] = opsGetSegmentInfo(param.system,ops_param); + num_frm = length(data.properties.frame); + % Database may return them out of order + param.start_gps_time = sort(double(data.properties.start_gps_time)); + param.stop_gps_time = sort(double(data.properties.stop_gps_time)); + end % Load layer organizer file into param.layers % param.layers.lyr_age % layer_organizer.lyr_age (age of layer or NaN) @@ -153,16 +167,16 @@ function loadPB_callback(obj,hObj,event) param_layerdata.records.gps.time_offset = NaN; param_layerdata.radar.lever_arm_fh = []; layers = layerdata(param_layerdata,param.layer_data_source); - layers.check_all(); + layers.check_layer_organizer(); for frm = 1:num_frm - % stores the filename for all frames in the segment + % Stores the filename for all frames in the segment param.filename{frm} = layers.layer_fn(frm); - gps_time = layers.gps_time(frm); + gps_time = layers.gps_time(frm); % Get the gps_time to take its length Nx = length(gps_time); param.frame(end+(1:Nx)) = frm; % stores the frame number for each point path id in each frame param.frame_idxs(end+(1:Nx)) = 1:length(gps_time); % contains the point number for each individual point in each frame - % stores the layer information for all frames in the segment + % Stores the layer information for all frames in the segment if frm == 1 layer_info = layers.layer{frm}; else @@ -187,14 +201,27 @@ function loadPB_callback(obj,hObj,event) param.layers.lyr_name = param.layers.lyr_name(new_order); param.layers.lyr_order = param.layers.lyr_order(new_order); - param.start_gps_time = obj.layerdata.frm_info(season_idx).start_gps_time(frm_idxs); - param.stop_gps_time = obj.layerdata.frm_info(season_idx).stop_gps_time(frm_idxs); + if strcmp(obj.cur_map_pref_settings.system,'tracks') + param.start_gps_time = obj.trackdata.frm_info(season_idx).start_gps_time(frm_idxs); + param.stop_gps_time = obj.trackdata.frm_info(season_idx).stop_gps_time(frm_idxs); + end else + % OPS: Load segment information + ops_param = struct('properties',[]); + ops_param.properties.location = param.cur_sel.location; + ops_param.properties.segment_id = param.cur_sel.seg_id; + [status,data] = opsGetSegmentInfo(param.system,ops_param); + num_frm = length(data.properties.frame); + % Database may return them out of order + param.start_gps_time = sort(double(data.properties.start_gps_time)); + param.stop_gps_time = sort(double(data.properties.stop_gps_time)); + param.layers.lyr_age = nan(size(param.layers.lyr_id)); % layer.age (age of layer or NaN) param.layers.lyr_age_source = cell(size(param.layers.lyr_id)); % layer.age_source (struct vector of age sources) param.layers.lyr_desc = cellfun(@char,cell(size(param.layers.lyr_id)),'UniformOutput',false); % layer.desc (layer description string) param.layers.lyr_order = [1:length(param.layers.lyr_id)]; % layer.order (positive integer, 1 to N where N is the number of layers) + layers.layer_organizer = param.layers; end @@ -207,7 +234,7 @@ function loadPB_callback(obj,hObj,event) end % Attach echowin to the undo stack -obj.echowin_list(echo_idx).cmds_set_undo_stack(obj.undo_stack_list(undo_stack_match_idx)); +cmds_list = obj.echowin_list(echo_idx).cmds_set_undo_stack(obj.undo_stack_list(undo_stack_match_idx)); % user_data: This is only used for param.layer_source == 'layerdata' except % for the field param.layer_source. obj.undo_stack_list(undo_stack_match_idx).user_data.layer_source = param.layer_source; % string containing layer source ('OPS' or 'layerdata') @@ -228,5 +255,9 @@ function loadPB_callback(obj,hObj,event) rethrow(ME); end +% Since there may be commands in the undo stack already, we will run these +% commands so that the new echowin is synced up with the stack. +obj.echowin_list(echo_idx).cmds_set_undo_stack_after_draw(cmds_list); + %% Cleanup set(obj.h_fig,'Pointer','Arrow'); diff --git a/cresis-toolbox/+imb/@mapwin/mapwin.m b/cresis-toolbox/+imb/@mapwin/mapwin.m index 5a6cc052..1008e033 100644 --- a/cresis-toolbox/+imb/@mapwin/mapwin.m +++ b/cresis-toolbox/+imb/@mapwin/mapwin.m @@ -44,8 +44,8 @@ % cur_map_pref_settings.layers.lyr_name: string, OPS layer name % cur_map_pref_settings.layers.lyr_group_name: string, OPS group name % cur_map_pref_settings.layers.lyr_id: OPS layer ID - % cur_map_pref_settings.seasons: cell array of strings containing seasons loaded (layerdata flightlines include system in the season name) - % cur_map_pref_settings.system: string, system name if OPS flight lines, otherwise 'layerdata' + % cur_map_pref_settings.seasons: cell array of strings containing seasons loaded (tracks files flightlines include system in the season name) + % cur_map_pref_settings.system: string, system name if OPS flight lines, otherwise 'tracks' % cur_map_pref_settings.sources: echogram sources to load % cur_map_pref_settings.map_zone: string, 'antarctic' or 'arctic' % cur_map_pref_settings.map_name: string, name of map @@ -54,7 +54,7 @@ map % map.source % 0 for OPS map, 1 for blank, 2 for Google % map.scale % 1e3 (km to meters for OPS/Blank), 1 for Google - % map.fline_source % 0 for OPS flight lines, 1 for season layerdata + % map.fline_source % 0 for OPS flight lines, 1 for csarp_support/tracks files % map.proj % Matlab Projection Structure % map.xaxis_default % Current xaxis default bounds % map.yaxis_default % Current yaxis default bounds @@ -77,16 +77,16 @@ google % google.map % Stores imp info about the google map obj - % Season layerdata: - % (.../csarp_support/layers/layer_MAPZONE_SYSTEM_SEASON.mat) - layerdata - % layerdata.x - % layerdata.y - % layerdata.frm_id - % layerdata.season_idx - % layerdata.frm_info().frm_id - % layerdata.frm_info().start_gps_time - % layerdata.frm_info().stop_gps_time + % Tracks files: + % (.../csarp_support/tracks/tracks_MAPZONE_SYSTEM_SEASON.mat) + trackdata + % trackdata.x + % trackdata.y + % trackdata.frm_id + % trackdata.season_idx + % trackdata.frm_info().frm_id + % trackdata.frm_info().start_gps_time + % trackdata.frm_info().stop_gps_time end @@ -140,7 +140,7 @@ obj.map.scale = []; % 1e3 (km to meters for OPS), 1 for Google % Currently selected flight line information obj.map.sel.frm_str = ''; % Current frame name - obj.map.sel.seg_id = []; % Current segment ID (Database ID for OPS layer source, index into obj.cur_map_pref_settings.seasons for layerdata source) + obj.map.sel.seg_id = []; % Current segment ID (Database ID for OPS layer source, index into obj.cur_map_pref_settings.seasons for csarp_support/tracks files source) obj.map.sel.season_name = ''; % Current season name obj.map.sel.radar_name = ''; % Current radar name obj.map.xaxis = []; @@ -162,17 +162,17 @@ obj.google = []; obj.google.map = []; % Stores imp info about the google map obj - % layerdata properties + % csarp_support/tracks files properties % ------------------------------------------------------------------- - obj.layerdata = []; - obj.layerdata.x = []; % Nx length vector of flightlines in local map coordinates - obj.layerdata.y = []; % Nx length vector of flightlines in local map coordinates - obj.layerdata.frm_id = []; % Nx length vector of frame ids (as numeric) - obj.layerdata.season_idx = []; % Nx length vector of season indices into obj.cur_map_pref_settings.seasons - obj.layerdata.frm_info = []; % Frame information struct array (one struct per entry in obj.cur_map_pref_settings.seasons) - obj.layerdata.frm_info.frm_id = []; % Nf length vector of frames, frame id (numeric) - obj.layerdata.frm_info.start_gps_time = []; % Nf length vector, start GPS time of frame (ANSI-C seconds since Jan 1, 1970) - obj.layerdata.frm_info.stop_gps_time = []; % Nf length vector, stop GPS time of frame (ANSI-C seconds since Jan 1, 1970) + obj.trackdata = []; + obj.trackdata.x = []; % Nx length vector of flightlines in local map coordinates + obj.trackdata.y = []; % Nx length vector of flightlines in local map coordinates + obj.trackdata.frm_id = []; % Nx length vector of frame ids (as numeric) + obj.trackdata.season_idx = []; % Nx length vector of season indices into obj.cur_map_pref_settings.seasons + obj.trackdata.frm_info = []; % Frame information struct array (one struct per entry in obj.cur_map_pref_settings.seasons) + obj.trackdata.frm_info.frm_id = []; % Nf length vector of frames, frame id (numeric) + obj.trackdata.frm_info.start_gps_time = []; % Nf length vector, start GPS time of frame (ANSI-C seconds since Jan 1, 1970) + obj.trackdata.frm_info.stop_gps_time = []; % Nf length vector, stop GPS time of frame (ANSI-C seconds since Jan 1, 1970) % Set default parameters % ------------------------------------------------------------------- diff --git a/cresis-toolbox/+imb/@mapwin/query_redraw_map.m b/cresis-toolbox/+imb/@mapwin/query_redraw_map.m index 23125269..b646831c 100644 --- a/cresis-toolbox/+imb/@mapwin/query_redraw_map.m +++ b/cresis-toolbox/+imb/@mapwin/query_redraw_map.m @@ -12,8 +12,8 @@ function query_redraw_map(obj,x_min,x_max,y_min,y_max) % (in km) to get %% Scale the requested limits to meters -obj.ops.request.XLim = sort([x_min x_max]*obj.map.scale); -obj.ops.request.YLim = sort([y_min y_max]*obj.map.scale); +map_xlim = sort([x_min x_max]*obj.map.scale); +map_ylim = sort([y_min y_max]*obj.map.scale); %% Force requested limits to maintain the aspect ratio of the figure old_u = get(obj.map_panel.h_axes,'units'); @@ -25,21 +25,25 @@ function query_redraw_map(obj,x_min,x_max,y_min,y_max) aspect_ratio = height/width; set(obj.map_panel.h_axes,'units',old_u); -% Grow the limits so that the requested region is as big or bigger -% than the requested limits -if aspect_ratio*diff(obj.ops.request.XLim) > diff(obj.ops.request.YLim) - growth = aspect_ratio*diff(obj.ops.request.XLim) - diff(obj.ops.request.YLim); - obj.ops.request.YLim(1) = obj.ops.request.YLim(1) - growth/2; - obj.ops.request.YLim(2) = obj.ops.request.YLim(2) + growth/2; -else - growth = diff(obj.ops.request.YLim)/aspect_ratio - diff(obj.ops.request.XLim); - obj.ops.request.XLim(1) = obj.ops.request.XLim(1) - growth/2; - obj.ops.request.XLim(2) = obj.ops.request.XLim(2) + growth/2; +if obj.map.source ~= 3 + % Grow the limits so that the requested region is as big or bigger + % than the requested limits + if aspect_ratio*diff(map_xlim) > diff(map_ylim) + growth = aspect_ratio*diff(map_xlim) - diff(map_ylim); + map_ylim(1) = map_ylim(1) - growth/2; + map_ylim(2) = map_ylim(2) + growth/2; + else + growth = diff(map_ylim)/aspect_ratio - diff(map_xlim); + map_xlim(1) = map_xlim(1) - growth/2; + map_xlim(2) = map_xlim(2) + growth/2; + end end if obj.map.source == 0 %% OPS map obj.ops.request.ImageFormat = 'image/jpeg'; + obj.ops.request.XLim = map_xlim; + obj.ops.request.YLim = map_ylim; % Build the new WMS query, submit it and then retrieve the result obj.ops.request.ImageHeight = height; @@ -59,6 +63,8 @@ function query_redraw_map(obj,x_min,x_max,y_min,y_max) elseif obj.map.source == 1 %% Google map + obj.ops.request.XLim = map_xlim; + obj.ops.request.YLim = map_ylim; [A,x_data,y_data] = obj.google.map.request_google_map(obj.ops.request.XLim(1), obj.ops.request.XLim(2), 256-obj.ops.request.YLim(1), 256-obj.ops.request.YLim(2)); A = flipud(A); @@ -94,10 +100,12 @@ function query_redraw_map(obj,x_min,x_max,y_min,y_max) end elseif obj.map.source == 2 - %% Blank map + %% Blank Stereographic map if obj.map.fline_source == 0 obj.ops.request.ImageFormat = 'image/png'; + obj.ops.request.XLim = map_xlim; + obj.ops.request.YLim = map_ylim; % Build the new WMS query, submit it and then retrieve the result obj.ops.request.ImageHeight = height; @@ -112,8 +120,39 @@ function query_redraw_map(obj,x_min,x_max,y_min,y_max) y_data = R(3,2) + [0 size(A,1)*R(1,2)]; else A = ones(1,1,3); - x_data = obj.ops.request.XLim/obj.map.scale; - y_data = obj.ops.request.YLim/obj.map.scale; + x_data = map_xlim/obj.map.scale; + y_data = map_ylim/obj.map.scale; + end + +elseif obj.map.source == 3 + %% Blank Geodetic map + + if obj.map.fline_source == 0 + % THIS IS NOT SUPPORTED... MAYBE OUR GEOSERVER IS TOO OLD AND MATLAB + % DOES NOT KNOW HOW TO INTERFACE + obj.ops.request.ImageFormat = 'image/png'; + %obj.ops.request.Latlim = map_ylim; + %obj.ops.request.Lonlim = map_xlim; + obj.ops.request.Latlim = map_xlim; % HACK FOR OLD GEOSERVER? + obj.ops.request.Lonlim = map_ylim; % HACK FOR OLD GEOSERVER? + + % Build the new WMS query, submit it and then retrieve the result + obj.ops.request.ImageHeight = height; + obj.ops.request.ImageWidth = width; + modrequest = strcat(obj.ops.request.RequestURL,'&viewparams=',obj.ops.seasons_modrequest,obj.ops.season_group_ids_modrequest); + A = obj.map_pref.ops.wms.getMap(modrequest); + R = obj.ops.request.RasterRef; + R = R/obj.map.scale; + + % Create axes + %x_data = R(3,1) + [0 size(A,2)*R(2,1)]; + %y_data = R(3,2) + [0 size(A,1)*R(1,2)]; + y_data = fliplr(R(3,1) + [0 size(A,2)*R(2,1)]); % HACK FOR OLD GEOSERVER? + x_data = fliplr(R(3,2) + [0 size(A,1)*R(1,2)]); % HACK FOR OLD GEOSERVER? + else + A = ones(1,1,3); + x_data = map_xlim/obj.map.scale; + y_data = map_ylim/obj.map.scale; end end diff --git a/cresis-toolbox/+imb/@mapwin/save_default_params.m b/cresis-toolbox/+imb/@mapwin/save_default_params.m index 24da5752..807038e5 100644 --- a/cresis-toolbox/+imb/@mapwin/save_default_params.m +++ b/cresis-toolbox/+imb/@mapwin/save_default_params.m @@ -25,4 +25,9 @@ function save_default_params(obj) default_params = obj.default_params; -save(obj.default_params.picker_param_fn,'-append','-struct','default_params','mapwin','prefwin','echowin'); +try + fprintf('Saving user parameters file: %s\n', obj.default_params.picker_param_fn); + ct_save(obj.default_params.picker_param_fn,'-append','-struct','default_params','mapwin','prefwin','echowin'); +catch ME + warning('Failed to save default parameters file %s', obj.default_params.picker_param_fn); +end diff --git a/cresis-toolbox/+imb/@mapwin/search_callback.m b/cresis-toolbox/+imb/@mapwin/search_callback.m index 2a18724c..9cdfbd75 100644 --- a/cresis-toolbox/+imb/@mapwin/search_callback.m +++ b/cresis-toolbox/+imb/@mapwin/search_callback.m @@ -26,15 +26,15 @@ function search_callback(obj,src,event) if ~isempty(season_match) season_match = find(cellfun(@(x) ~isempty(regexpi(x,season_match)), obj.cur_map_pref_settings.seasons)); % Get a logical mask indicating all indices that match the frame - frm_mask = floor(obj.layerdata.frm_id/10^(13-length(frm_id_str))) == frm_id; - season_mask = false(size(obj.layerdata.season_idx)); + frm_mask = floor(obj.trackdata.frm_id/10^(13-length(frm_id_str))) == frm_id; + season_mask = false(size(obj.trackdata.season_idx)); for idx = 1:length(season_match) - season_mask = season_mask | obj.layerdata.season_idx == season_match(idx); + season_mask = season_mask | obj.trackdata.season_idx == season_match(idx); end frm_mask = frm_mask & season_mask; else % Get a logical mask indicating all indices that match the frame - frm_mask = floor(obj.layerdata.frm_id/10^(13-length(frm_id_str))) == frm_id; + frm_mask = floor(obj.trackdata.frm_id/10^(13-length(frm_id_str))) == frm_id; end % Find the first matching frame in the list @@ -44,8 +44,8 @@ function search_callback(obj,src,event) return; end % Extract out frame, system, and season name - frm_id = obj.layerdata.frm_id(idx); - season_idx = obj.layerdata.season_idx(idx); + frm_id = obj.trackdata.frm_id(idx); + season_idx = obj.trackdata.season_idx(idx); season_name = obj.cur_map_pref_settings.seasons{season_idx}; [sys,season_name] = strtok(season_name,'_'); season_name = season_name(2:end); @@ -63,8 +63,8 @@ function search_callback(obj,src,event) data.properties.frame = frm_str; data.properties.season = season_name; data.properties.segment_id = str2num(frm_id(1:10)); - data.properties.X = obj.layerdata.x(frm_mask); - data.properties.Y = obj.layerdata.y(frm_mask); + data.properties.X = obj.trackdata.x(frm_mask); + data.properties.Y = obj.trackdata.y(frm_mask); new_xdata = data.properties.X; new_ydata = data.properties.Y; else @@ -84,8 +84,8 @@ function search_callback(obj,src,event) data.properties.frame = frm_str; data.properties.season = frm_data.properties.season; data.properties.segment_id = frm_data.properties.segment_id; - data.properties.X = obj.layerdata.x(frm_mask); - data.properties.Y = obj.layerdata.y(frm_mask); + data.properties.X = obj.trackdata.x(frm_mask); + data.properties.Y = obj.trackdata.y(frm_mask); new_xdata = data.properties.X; new_ydata = data.properties.Y; end diff --git a/cresis-toolbox/+imb/@mapwin/set_default_params.m b/cresis-toolbox/+imb/@mapwin/set_default_params.m index 0704352e..c3b5e4de 100644 --- a/cresis-toolbox/+imb/@mapwin/set_default_params.m +++ b/cresis-toolbox/+imb/@mapwin/set_default_params.m @@ -248,12 +248,12 @@ function set_default_params(obj,picker_param_fn) default_params.prefwin.sources = sort({'standard','qlook','mvdr','CSARP_post/standard','CSARP_post/mvdr','CSARP_post/qlook'}); default_params.prefwin.season_names = {}; default_params.prefwin.layer_names = {'surface'}; -default_params.prefwin.system = 'layerdata'; +default_params.prefwin.system = 'tracks'; default_params.prefwin.map_name = ''; -default_params.prefwin.flightlines = 'layerdata Flightlines'; +default_params.prefwin.flightlines = 'tracks files Flightlines'; % default_params.prefwin.layer_source = 'layerdata'; -default_params.prefwin.layer_data_source = 'layerData'; +default_params.prefwin.layer_data_source = 'layer'; % default_params.prefwin.x = tmp.pref_x; default_params.prefwin.y = tmp.pref_y; @@ -273,8 +273,11 @@ function set_default_params(obj,picker_param_fn) %% Load user passed in file and override the default parameters with the contents of this file if exist(picker_param_fn,'file') + fprintf('Loading user parameters file: %s\n', picker_param_fn); default_params_override = load(picker_param_fn); default_params = merge_structs(default_params,default_params_override); +else + fprintf('Loading user parameters file, but not found: %s\n', picker_param_fn); end default_params.picker_param_fn = picker_param_fn; @@ -289,7 +292,8 @@ function set_default_params(obj,picker_param_fn) end end try - save(picker_param_fn,'-struct','default_params'); + fprintf('Saving user parameters file: %s\n', picker_param_fn); + ct_save(picker_param_fn,'-append','-struct','default_params'); catch ME warning('Failed to save default parameters file %s', picker_param_fn); end diff --git a/cresis-toolbox/+imb/@mapwin/update_ascopewin_cursors.m b/cresis-toolbox/+imb/@mapwin/update_ascopewin_cursors.m index bd3fa0d0..4fc0efd9 100644 --- a/cresis-toolbox/+imb/@mapwin/update_ascopewin_cursors.m +++ b/cresis-toolbox/+imb/@mapwin/update_ascopewin_cursors.m @@ -17,6 +17,9 @@ function update_ascopewin_cursors(obj,src,event) if obj.map.source == 1 [x_red(end+1),y_red(end+1)] ... = google_map.latlon_to_world(obj.map_ascope.ascope.lat(idx), obj.map_ascope.ascope.lon(idx)); y = 256-y; + elseif obj.map.source == 3 + x_red(end+1) = obj.map_ascope.ascope.lon(idx); + y_red(end+1) = obj.map_ascope.ascope.lat(idx); else [x_red(end+1),y_red(end+1)] ... = projfwd(obj.map.proj, obj.map_ascope.ascope.lat(idx), obj.map_ascope.ascope.lon(idx)); diff --git a/cresis-toolbox/+imb/@mapwin/update_echowin_cursors.m b/cresis-toolbox/+imb/@mapwin/update_echowin_cursors.m index 7f6a83ea..b74ee2f8 100644 --- a/cresis-toolbox/+imb/@mapwin/update_echowin_cursors.m +++ b/cresis-toolbox/+imb/@mapwin/update_echowin_cursors.m @@ -11,6 +11,9 @@ function update_echowin_cursors(obj,src,event) % Update the source echowin's cursor and clutter on the map if obj.map.source == 1 [src_x,src_y] = google_map.latlon_to_world([src.cursor.lat src.cursor.clutter_lat], [src.cursor.lon src.cursor.clutter_lon]); src_y = 256-src_y; +elseif obj.map.source == 3 + src_x = src.cursor.clutter_lon; + src_y = src.cursor.clutter_lat; else [src_x,src_y] = projfwd(obj.map.proj, [src.cursor.lat src.cursor.clutter_lat], [src.cursor.lon src.cursor.clutter_lon]); end @@ -26,6 +29,9 @@ function update_echowin_cursors(obj,src,event) % Update the echowin's cursor on the map if obj.map.source == 1 [x,y] = google_map.latlon_to_world(obj.echowin_list(idx).cursor.lat, obj.echowin_list(idx).cursor.lon); y = 256-y; + elseif obj.map.source == 3 + x = obj.echowin_list(idx).cursor.lon; + y = obj.echowin_list(idx).cursor.lat; else [x,y] = projfwd(obj.map.proj, obj.echowin_list(idx).cursor.lat, obj.echowin_list(idx).cursor.lon); end diff --git a/cresis-toolbox/+imb/@mapwin/update_echowin_flightlines.m b/cresis-toolbox/+imb/@mapwin/update_echowin_flightlines.m index 12a562cf..ac96ac28 100644 --- a/cresis-toolbox/+imb/@mapwin/update_echowin_flightlines.m +++ b/cresis-toolbox/+imb/@mapwin/update_echowin_flightlines.m @@ -13,6 +13,13 @@ function update_echowin_flightlines(obj,src,event) xlimits = xlim(src.h_axes); start_gps = src.eg.image_gps_time(find(src.eg.image_xaxis>=xlimits(1),1)); stop_gps = src.eg.image_gps_time(find(src.eg.image_xaxis<=xlimits(2),1,'last')); +% Handle special edge cases for the first and last column when the user +% selects xlimits before the first pixel or after the last pixel. +if isempty(start_gps) + start_gps = stop_gps; +elseif isempty(stop_gps) + stop_gps = start_gps; +end valid_idxs = find(src.eg.map_gps_time >= start_gps & src.eg.map_gps_time <= stop_gps); if isempty(valid_idxs) diff --git a/cresis-toolbox/+imb/@mapwin/update_map_selection.m b/cresis-toolbox/+imb/@mapwin/update_map_selection.m deleted file mode 100644 index e69de29b..00000000 diff --git a/cresis-toolbox/+imb/@mapwin/update_map_selection_echowin.m b/cresis-toolbox/+imb/@mapwin/update_map_selection_echowin.m index aa55a51e..3c95e74d 100644 --- a/cresis-toolbox/+imb/@mapwin/update_map_selection_echowin.m +++ b/cresis-toolbox/+imb/@mapwin/update_map_selection_echowin.m @@ -20,7 +20,7 @@ function update_map_selection_echowin(obj,src,event) frm_id = str2num(frm_id); % Get a logical mask indicating all indices that match the frame - frm_mask = obj.layerdata.frm_id == frm_id; + frm_mask = obj.trackdata.frm_id == frm_id; % Find the first matching frame in the list idx = find(frm_mask,1); if isempty(idx) @@ -28,8 +28,8 @@ function update_map_selection_echowin(obj,src,event) return; end % Extract out frame, system, and season name - frm_id = obj.layerdata.frm_id(idx); - season_idx = obj.layerdata.season_idx(idx); + frm_id = obj.trackdata.frm_id(idx); + season_idx = obj.trackdata.season_idx(idx); season_name = obj.cur_map_pref_settings.seasons{season_idx}; [sys,season_name] = strtok(season_name,'_'); season_name = season_name(2:end); @@ -47,8 +47,8 @@ function update_map_selection_echowin(obj,src,event) data.properties.frame = frm_str; data.properties.season = season_name; data.properties.segment_id = str2num(frm_id(1:10)); - data.properties.X = obj.layerdata.x(frm_mask); - data.properties.Y = obj.layerdata.y(frm_mask); + data.properties.X = obj.trackdata.x(frm_mask); + data.properties.Y = obj.trackdata.y(frm_mask); new_xdata = data.properties.X; new_ydata = data.properties.Y; else @@ -68,8 +68,8 @@ function update_map_selection_echowin(obj,src,event) data.properties.frame = frm_str; data.properties.season = frm_data.properties.season; data.properties.segment_id = frm_data.properties.segment_id; - data.properties.X = obj.layerdata.x(frm_mask); - data.properties.Y = obj.layerdata.y(frm_mask); + data.properties.X = obj.trackdata.x(frm_mask); + data.properties.Y = obj.trackdata.y(frm_mask); new_xdata = data.properties.X; new_ydata = data.properties.Y; end diff --git a/cresis-toolbox/+imb/@picktool/insert_pnt.m b/cresis-toolbox/+imb/@picktool/insert_pnt.m index 9abef15a..bbc2fca5 100644 --- a/cresis-toolbox/+imb/@picktool/insert_pnt.m +++ b/cresis-toolbox/+imb/@picktool/insert_pnt.m @@ -27,12 +27,17 @@ % Find the closest point in the image [~,image_x_idx] = min(abs(image_x-x)); -[~,image_y_idx] = min(abs(image_y-y)); -% Prevent search range from searching outside bounds of param.image_c -search_range = search_range(image_y_idx+search_range >= 1 ... - & image_y_idx+search_range < size(param.image_c,1)); -[~,max_idx] = max(param.image_c(image_y_idx+search_range,image_x_idx)); -max_idx = search_range(max_idx) + image_y_idx; +dy = image_y(2)-image_y(1); +image_y_idx = 1 + round((y-image_y(1))/dy); +if image_y_idx < 1 || image_y_idx > size(param.image_c,1) + max_idx = image_y_idx; +else + % Prevent search range from searching outside bounds of param.image_c + search_range = search_range(image_y_idx+search_range >= 1 ... + & image_y_idx+search_range < size(param.image_c,1)); + [~,max_idx] = max(param.image_c(image_y_idx+search_range,image_x_idx)); + max_idx = search_range(max_idx) + image_y_idx; +end [~,point_idxs] = min(abs(param.layer.x-x)); new_y = interp1(1:length(image_y),image_y,max_idx,'linear','extrap'); diff --git a/cresis-toolbox/+imb/@picktool_convert/create_ui.m b/cresis-toolbox/+imb/@picktool_copy/create_ui.m similarity index 59% rename from cresis-toolbox/+imb/@picktool_convert/create_ui.m rename to cresis-toolbox/+imb/@picktool_copy/create_ui.m index cb233bf2..a41ca7d2 100644 --- a/cresis-toolbox/+imb/@picktool_convert/create_ui.m +++ b/cresis-toolbox/+imb/@picktool_copy/create_ui.m @@ -1,4 +1,5 @@ function create_ui(obj) +% picktool_copy.create_ui(obj) set(obj.h_fig,'DockControls','off') set(obj.h_fig,'NumberTitle','off'); @@ -40,61 +41,81 @@ function create_ui(obj) obj.panel.source_label = uicontrol('Parent',obj.panel.handle); set(obj.panel.source_label,'Style','text'); set(obj.panel.source_label,'String','Source layer:'); -set(obj.panel.source_label,'FontSize',8) -set(obj.panel.source_label,'TooltipString','This is the source layer to convert all active layers to. Select parts of this layer in the echogram to convert layers to the selected section.'); +set(obj.panel.source_label,'FontSize',10); +tooptip_str = 'This is the source layer to copy into all active layers. Select parts of this layer in the echogram to copy layers for the selected section.'; +set(obj.panel.source_label,'TooltipString',tooptip_str); %----source layer edit obj.panel.sourceTB = uicontrol('Parent',obj.panel.handle); set(obj.panel.sourceTB,'Style','edit'); set(obj.panel.sourceTB,'String','1'); -set(obj.panel.sourceTB,'FontSize',8); -set(obj.panel.sourceTB,'TooltipString','This is the source layer to convert all active layers to. Select parts of this layer in the echogram to convert layers to the selected section.'); +set(obj.panel.sourceTB,'FontSize',10); +set(obj.panel.sourceTB,'TooltipString',tooptip_str); + +%----source eval label +obj.panel.source_eval_label = uicontrol('Parent',obj.panel.handle); +set(obj.panel.source_eval_label,'Style','text'); +set(obj.panel.source_eval_label,'String','Source eval:'); +set(obj.panel.source_eval_label,'FontSize',10) +tooltip_str = 'Only used for "Copy" mode. The source layer will have this operation run on it. The source layer is stored in a variable called "s". Default is "s=s" which does no operation. "s=0.5*s" would halve the value of the source layer.'; +set(obj.panel.source_eval_label,'TooltipString',tooltip_str); + +%----source eval edit +obj.panel.source_evalTB = uicontrol('Parent',obj.panel.handle); +set(obj.panel.source_evalTB,'Style','edit'); +set(obj.panel.source_evalTB,'String','s=s'); +set(obj.panel.source_evalTB,'FontSize',10); +set(obj.panel.source_evalTB,'TooltipString',tooltip_str); %----correct layer label obj.panel.correct_label = uicontrol('Parent',obj.panel.handle); set(obj.panel.correct_label,'Style','text'); set(obj.panel.correct_label,'String','Correct layer:'); -set(obj.panel.correct_label,'FontSize',8) -set(obj.panel.correct_label,'TooltipString','If Enable Diff selected, then selecting the source layer will cause active layers to change by correct - source.'); +set(obj.panel.correct_label,'FontSize',10) +tooltip_str = 'If Enable Diff selected, then selecting the source layer will cause active layers to change by correct - source.'; +set(obj.panel.correct_label,'TooltipString',tooltip_str); %----correct layer edit obj.panel.correctTB = uicontrol('Parent',obj.panel.handle); set(obj.panel.correctTB,'Style','edit'); set(obj.panel.correctTB,'String','1'); -set(obj.panel.correctTB,'FontSize',8); -set(obj.panel.correctTB,'TooltipString','If Enable Diff selected, then selecting the source layer will cause active layers to change by correct - source.'); - -%----diff mode label -obj.panel.diff_mode_label = uicontrol('Parent',obj.panel.handle); -set(obj.panel.diff_mode_label,'Style','text'); -set(obj.panel.diff_mode_label,'String','Enable Diff:'); -set(obj.panel.diff_mode_label,'FontSize',8); -set(obj.panel.diff_mode_label,'TooltipString','If Enable Diff selected, then selecting the source layer will cause active layers to change by correct - source.'); - -%----diff mode checkbox -obj.panel.diff_modeCB = uicontrol('Parent',obj.panel.handle); -set(obj.panel.diff_modeCB,'Style','checkbox'); -set(obj.panel.diff_modeCB,'Value', 0); -set(obj.panel.diff_modeCB,'TooltipString','If Enable Diff selected, then selecting the source layer will cause active layers to change by correct - source.'); +set(obj.panel.correctTB,'FontSize',10); +set(obj.panel.correctTB,'TooltipString',tooltip_str); + +%----mode label +obj.panel.mode_label = uicontrol('Parent',obj.panel.handle); +set(obj.panel.mode_label,'Style','text'); +set(obj.panel.mode_label,'String','Copy Mode:'); +set(obj.panel.mode_label,'FontSize',10); +tooltip_str = sprintf('Copy: Copies the "Source layer" to the selected layers after running "Source eval"\nMerge: Merges the "Source layer" into the undefined sections of the selected layers.\nDiff: Applies the difference between the "Correct layer" and the "Source layer" to the selected layers.'); +set(obj.panel.mode_label,'TooltipString',tooltip_str); + +%----mode checkbox +obj.panel.modePM = uicontrol('Parent',obj.panel.handle); +set(obj.panel.modePM,'Style','popupmenu'); +set(obj.panel.modePM,'String', {'Copy','Merge','Diff'}); +set(obj.panel.modePM,'Value', 1); +set(obj.panel.modePM,'TooltipString',tooltip_str); %----shift layer label obj.panel.shift_label = uicontrol('Parent',obj.panel.handle); set(obj.panel.shift_label,'Style','text'); set(obj.panel.shift_label,'String','Shift Size:'); -set(obj.panel.shift_label,'FontSize',8) +set(obj.panel.shift_label,'FontSize',10) set(obj.panel.shift_label,'TooltipString','This controls the number of range bins that the up/down functions shift a layer.'); %----shift layer edit obj.panel.shiftTB = uicontrol('Parent',obj.panel.handle); set(obj.panel.shiftTB,'Style','edit'); set(obj.panel.shiftTB,'String','1'); -set(obj.panel.shiftTB,'FontSize',8); +set(obj.panel.shiftTB,'FontSize',10); set(obj.panel.shiftTB,'TooltipString','This controls the number of range bins that the up/down functions shift a layer.'); %----up pushbutton obj.panel.upPB = uicontrol('Parent',obj.panel.handle); set(obj.panel.upPB,'Style','PushButton'); set(obj.panel.upPB,'String','Up'); +set(obj.panel.upPB,'FontSize',10); set(obj.panel.upPB,'Callback',@obj.upPB_callback); set(obj.panel.upPB,'TooltipString','Shift selected layers up by number of range bins listed in Shift Size'); @@ -102,16 +123,17 @@ function create_ui(obj) obj.panel.downPB = uicontrol('Parent',obj.panel.handle); set(obj.panel.downPB,'Style','PushButton'); set(obj.panel.downPB,'String','Down'); +set(obj.panel.downPB,'FontSize',10); set(obj.panel.downPB,'Callback',@obj.downPB_callback); set(obj.panel.downPB,'TooltipString','Shift selected layers down by number of range bins listed in Shift Size'); %--------------------------------------------------------------------------------------------- % set up top panel table obj.panel.table.ui=obj.panel.handle; -obj.panel.table.width_margin = NaN*zeros(30,30); % Just make these bigger than they have to be -obj.panel.table.height_margin = NaN*zeros(30,30); -obj.panel.table.false_width = NaN*zeros(30,30); -obj.panel.table.false_height = NaN*zeros(30,30); +obj.panel.table.width_margin = nan(30,30); % Just make these bigger than they have to be +obj.panel.table.height_margin = nan(30,30); +obj.panel.table.false_width = nan(30,30); +obj.panel.table.false_height = nan(30,30); obj.panel.table.offset = [0 0]; row = 0; @@ -119,40 +141,54 @@ function create_ui(obj) row = row+1; col = 1; obj.panel.table.handles{row,col} = obj.panel.source_label; obj.panel.table.width(row,col) = inf; -obj.panel.table.height(row,col) = 25; +obj.panel.table.height(row,col) = 20; obj.panel.table.width_margin(row,col) = 0; -obj.panel.table.height_margin(row,col) = 0; +obj.panel.table.height_margin(row,col) = 3; col = 2; obj.panel.table.handles{row,col} = obj.panel.sourceTB; obj.panel.table.width(row,col) = inf; -obj.panel.table.height(row,col) = 25; +obj.panel.table.height(row,col) = 20; obj.panel.table.width_margin(row,col) = 0; obj.panel.table.height_margin(row,col) = 0; row = row+1; col = 1; -obj.panel.table.handles{row,col} = obj.panel.correct_label; +obj.panel.table.handles{row,col} = obj.panel.source_eval_label; +obj.panel.table.width(row,col) = inf; +obj.panel.table.height(row,col) = 20; +obj.panel.table.width_margin(row,col) = 0; +obj.panel.table.height_margin(row,col) = 3; + +col = 2; +obj.panel.table.handles{row,col} = obj.panel.source_evalTB; obj.panel.table.width(row,col) = inf; -obj.panel.table.height(row,col) = 25; +obj.panel.table.height(row,col) = 20; obj.panel.table.width_margin(row,col) = 0; obj.panel.table.height_margin(row,col) = 0; +row = row+1; col = 1; +obj.panel.table.handles{row,col} = obj.panel.correct_label; +obj.panel.table.width(row,col) = inf; +obj.panel.table.height(row,col) = 20; +obj.panel.table.width_margin(row,col) = 0; +obj.panel.table.height_margin(row,col) = 3; + col = 2; obj.panel.table.handles{row,col} = obj.panel.correctTB; obj.panel.table.width(row,col) = inf; -obj.panel.table.height(row,col) = 25; +obj.panel.table.height(row,col) = 20; obj.panel.table.width_margin(row,col) = 0; obj.panel.table.height_margin(row,col) = 0; row = row+1; col = 1; -obj.panel.table.handles{row,col} = obj.panel.diff_mode_label; +obj.panel.table.handles{row,col} = obj.panel.mode_label; obj.panel.table.width(row,col) = inf; obj.panel.table.height(row,col) = 20; obj.panel.table.width_margin(row,col) = 0; -obj.panel.table.height_margin(row,col) = 0; +obj.panel.table.height_margin(row,col) = 3; col = 2; -obj.panel.table.handles{row,col} = obj.panel.diff_modeCB; +obj.panel.table.handles{row,col} = obj.panel.modePM; obj.panel.table.width(row,col) = inf; obj.panel.table.height(row,col) = 20; obj.panel.table.width_margin(row,col) = 0; @@ -161,28 +197,28 @@ function create_ui(obj) row = row+1; col = 1; obj.panel.table.handles{row,col} = obj.panel.shift_label; obj.panel.table.width(row,col) = inf; -obj.panel.table.height(row,col) = 25; +obj.panel.table.height(row,col) = 20; obj.panel.table.width_margin(row,col) = 0; -obj.panel.table.height_margin(row,col) = 0; +obj.panel.table.height_margin(row,col) = 3; col = 2; obj.panel.table.handles{row,col} = obj.panel.shiftTB; obj.panel.table.width(row,col) = inf; -obj.panel.table.height(row,col) = 25; +obj.panel.table.height(row,col) = 20; obj.panel.table.width_margin(row,col) = 0; obj.panel.table.height_margin(row,col) = 0; row = row+1; col = 1; obj.panel.table.handles{row,col} = obj.panel.upPB; obj.panel.table.width(row,col) = inf; -obj.panel.table.height(row,col) = 25; +obj.panel.table.height(row,col) = 20; obj.panel.table.width_margin(row,col) = 0; obj.panel.table.height_margin(row,col) = 0; col = 2; obj.panel.table.handles{row,col} = obj.panel.downPB; obj.panel.table.width(row,col) = inf; -obj.panel.table.height(row,col) = 25; +obj.panel.table.height(row,col) = 20; obj.panel.table.width_margin(row,col) = 0; obj.panel.table.height_margin(row,col) = 0; diff --git a/cresis-toolbox/+imb/@picktool_convert/downPB_callback.m b/cresis-toolbox/+imb/@picktool_copy/downPB_callback.m similarity index 91% rename from cresis-toolbox/+imb/@picktool_convert/downPB_callback.m rename to cresis-toolbox/+imb/@picktool_copy/downPB_callback.m index 51b47173..fc293915 100644 --- a/cresis-toolbox/+imb/@picktool_convert/downPB_callback.m +++ b/cresis-toolbox/+imb/@picktool_copy/downPB_callback.m @@ -1,12 +1,12 @@ function downPB_callback(obj,hObj,event) -% picktool_convert.downPB_callback(obj,hObj,event) +% picktool_copy.downPB_callback(obj,hObj,event) % % down push button callback. Causes active layers to be shifted down according % to the "Shift Size" field cur_layers = find(obj.parent.eg.layers.selected_layers).'; -point_idxs = 1:length(obj.parent.eg.layers.x_curUnit{1}); +point_idxs = 1:length(obj.parent.eg.layers.x_curUnit); dy = obj.parent.eg.image_yaxis(2)-obj.parent.eg.image_yaxis(1); diff --git a/cresis-toolbox/+imb/@picktool_convert/left_click.m b/cresis-toolbox/+imb/@picktool_copy/left_click.m similarity index 56% rename from cresis-toolbox/+imb/@picktool_convert/left_click.m rename to cresis-toolbox/+imb/@picktool_copy/left_click.m index ad531c03..c0da2772 100644 --- a/cresis-toolbox/+imb/@picktool_convert/left_click.m +++ b/cresis-toolbox/+imb/@picktool_copy/left_click.m @@ -1,4 +1,5 @@ function cmds = left_click(obj,param) +% cmds = picktool_copy.left_click(obj,param) cmds = []; diff --git a/cresis-toolbox/+imb/@picktool_convert/left_click_and_drag.m b/cresis-toolbox/+imb/@picktool_copy/left_click_and_drag.m similarity index 51% rename from cresis-toolbox/+imb/@picktool_convert/left_click_and_drag.m rename to cresis-toolbox/+imb/@picktool_copy/left_click_and_drag.m index 1651904e..734ce3d1 100644 --- a/cresis-toolbox/+imb/@picktool_convert/left_click_and_drag.m +++ b/cresis-toolbox/+imb/@picktool_copy/left_click_and_drag.m @@ -1,7 +1,7 @@ function cmds = left_click_and_drag(obj,param) -% cmds = left_click_and_drag(obj,param) +% cmds = picktool_copy.left_click_and_drag(obj,param) % -% Convert layer tool +% Copy layer tool image_x = param.image_x; image_y = param.image_y; @@ -10,7 +10,7 @@ y = param.y; cmds = []; -fprintf('Convert points %f to %f, %f to %f\n', x, y); +fprintf('Copy points %f to %f, %f to %f\n', x, y); try source_layer = round(str2double(get(obj.panel.sourceTB,'String'))); @@ -21,9 +21,9 @@ error('Source layer entered is invalid. Enter a single positive integer.\n'); end -diff_mode_en = get(obj.panel.diff_modeCB,'Value'); +copy_mode = get(obj.panel.modePM,'Value'); -if diff_mode_en +if copy_mode == 3 try correct_layer = round(str2double(get(obj.panel.correctTB,'String'))); if length(correct_layer) ~= 1 || correct_layer < 1 || correct_layer > length(param.layer.y) @@ -36,17 +36,18 @@ param.x_bounds = 2; param.y_bounds = 2; -if diff_mode_en - [manual_idxs,auto_idxs,point_idxs] = find_matching_pnts(obj,param,correct_layer); +if copy_mode == 3 + [manual_idxs,auto_idxs,point_idxs] = obj.find_matching_pnts(param,correct_layer); else - [manual_idxs,auto_idxs,point_idxs] = find_matching_pnts(obj,param,source_layer); + [manual_idxs,auto_idxs,point_idxs] = obj.find_matching_pnts(param,source_layer); end for layer_idx = 1:length(cur_layers) cur_layer = cur_layers(layer_idx); if ~isempty(point_idxs) - if diff_mode_en + if copy_mode == 3 + %% Diff Mode cmds(end+1).undo_cmd = 'insert'; cmds(end).undo_args = {cur_layer, point_idxs, ... param.layer.y{cur_layer}(point_idxs), ... @@ -57,15 +58,49 @@ param.layer.y{cur_layer}(point_idxs) + param.layer.y{correct_layer}(point_idxs) - param.layer.y{source_layer}(point_idxs), ... param.layer.type{cur_layer}(point_idxs), ... param.layer.qual{cur_layer}(point_idxs)}; + elseif copy_mode == 2 + %% Merge Mode + cmds(end+1).undo_cmd = 'insert'; + % Only update the NaN + mask = isnan(param.layer.y{cur_layer}(point_idxs)); + cmds(end).undo_args = {cur_layer, point_idxs(mask), ... + param.layer.y{cur_layer}(point_idxs(mask)), ... + param.layer.type{cur_layer}(point_idxs(mask)), ... + param.layer.qual{cur_layer}(point_idxs(mask))}; + cmds(end).redo_cmd = 'insert'; + % Merge vectors + merged_layer = merge_vectors(param.layer.y{cur_layer}(point_idxs),param.layer.y{source_layer}(point_idxs)); + % Insert the merged points + cmds(end).redo_args = {cur_layer, point_idxs(mask), ... + merged_layer(mask), ... + param.layer.type{source_layer}(point_idxs(mask)), ... + param.layer.qual{source_layer}(point_idxs(mask))}; else + %% Copy Mode cmds(end+1).undo_cmd = 'insert'; cmds(end).undo_args = {cur_layer, point_idxs, ... param.layer.y{cur_layer}(point_idxs), ... param.layer.type{cur_layer}(point_idxs), ... param.layer.qual{cur_layer}(point_idxs)}; cmds(end).redo_cmd = 'insert'; + s = param.layer.y{source_layer}(point_idxs); + try + size_s = size(s); + eval_cmd = get(obj.panel.source_evalTB,'String'); + evalc(eval_cmd); + % Ensure size is correct + if ~isequal(size_s, size(s)) + error('s changed in size which is not allowed. Source eval parameter should be modified to prevent this from happening.'); + end + % Ensure type is correct. + if ~isa(s,'double') + error('s changed in type which is not allowed. Source eval parameter should be modified to prevent this from happening.'); + end + catch ME; + warning(ME.getReport); + end; cmds(end).redo_args = {cur_layer, point_idxs, ... - param.layer.y{source_layer}(point_idxs), ... + s, ... param.layer.type{source_layer}(point_idxs), ... param.layer.qual{source_layer}(point_idxs)}; end diff --git a/cresis-toolbox/+imb/@picktool_convert/picktool_convert.m b/cresis-toolbox/+imb/@picktool_copy/picktool_copy.m similarity index 77% rename from cresis-toolbox/+imb/@picktool_convert/picktool_convert.m rename to cresis-toolbox/+imb/@picktool_copy/picktool_copy.m index c2be2298..b8738080 100644 --- a/cresis-toolbox/+imb/@picktool_convert/picktool_convert.m +++ b/cresis-toolbox/+imb/@picktool_copy/picktool_copy.m @@ -1,4 +1,4 @@ -classdef picktool_convert < imb.picktool +classdef picktool_copy < imb.picktool properties table @@ -11,7 +11,7 @@ end methods - function obj = picktool_convert(h_fig,parent) + function obj = picktool_copy(h_fig,parent) %%% Pre Initialization %%% % Any code not using output argument (obj) if nargin == 0 || isempty(h_fig) @@ -24,10 +24,10 @@ % Any code, including access to object obj.parent = parent; obj.h_fig = h_fig; - obj.tool_name = '(c)onvert'; - obj.tool_name_title = 'convert'; + obj.tool_name = '(c)opy'; + obj.tool_name_title = 'copy'; obj.tool_shortcut = 'c'; - obj.help_string = sprintf('Left click: No function\nLeft click and drag: Convert selected layers to match the layer specified in the param window (p)\n\n'); + obj.help_string = sprintf('Left click: No function\nLeft click and drag: Copy the selected portion of the source layer specified in the tool param window (p) into the selected layers.\n\n'); obj.w = 190; obj.h = 170; diff --git a/cresis-toolbox/+imb/@picktool_convert/upPB_callback.m b/cresis-toolbox/+imb/@picktool_copy/upPB_callback.m similarity index 91% rename from cresis-toolbox/+imb/@picktool_convert/upPB_callback.m rename to cresis-toolbox/+imb/@picktool_copy/upPB_callback.m index bb9c501d..3239a37a 100644 --- a/cresis-toolbox/+imb/@picktool_convert/upPB_callback.m +++ b/cresis-toolbox/+imb/@picktool_copy/upPB_callback.m @@ -1,12 +1,12 @@ function upPB_callback(obj,hObj,event) -% picktool_convert.upPB_callback(obj,hObj,event) +% picktool_copy.upPB_callback(obj,hObj,event) % % up push button callback. Causes active layers to be shifted up according % to the "Shift Size" field cur_layers = find(obj.parent.eg.layers.selected_layers).'; -point_idxs = 1:length(obj.parent.eg.layers.x_curUnit{1}); +point_idxs = 1:length(obj.parent.eg.layers.x_curUnit); dy = obj.parent.eg.image_yaxis(2)-obj.parent.eg.image_yaxis(1); diff --git a/cresis-toolbox/+imb/@picktool_interpolate/create_ui.m b/cresis-toolbox/+imb/@picktool_interpolate/create_ui.m index e2c1ff1c..c60deac3 100644 --- a/cresis-toolbox/+imb/@picktool_interpolate/create_ui.m +++ b/cresis-toolbox/+imb/@picktool_interpolate/create_ui.m @@ -37,51 +37,51 @@ function create_ui(obj) table_draw(obj.table); %----max rbins label -obj.panel.max_rbin_rng_label = uicontrol('Parent',obj.h_fig); -set(obj.panel.max_rbin_rng_label,'Style','text'); -set(obj.panel.max_rbin_rng_label,'String',sprintf('Max point range:')); -set(obj.panel.max_rbin_rng_label,'FontSize',8) -set(obj.panel.max_rbin_rng_label,'TooltipString','During manual (left click) entry, this sets the range of bins that will be searched to find the max. Set to "0" to not search.'); +obj.panel.manual_range_label = uicontrol('Parent',obj.h_fig); +set(obj.panel.manual_range_label,'Style','text'); +set(obj.panel.manual_range_label,'String',sprintf('Manual range:')); +set(obj.panel.manual_range_label,'FontSize',10) +set(obj.panel.manual_range_label,'TooltipString','During manual (left click) entry, this sets the range of bins that will be searched to find the max. Set to "0" to not search.'); %----max rbins entry -obj.panel.max_rbin_rng_tbox = uicontrol('Parent',obj.h_fig); -set(obj.panel.max_rbin_rng_tbox,'Style','edit'); -set(obj.panel.max_rbin_rng_tbox,'String','1'); -set(obj.panel.max_rbin_rng_tbox,'FontSize',8); -set(obj.panel.max_rbin_rng_tbox,'TooltipString','During manual (left click) entry, this sets the range of bins that will be searched to find the max. Set to "0" to not search.'); +obj.panel.manual_rangeTB = uicontrol('Parent',obj.h_fig); +set(obj.panel.manual_rangeTB,'Style','edit'); +set(obj.panel.manual_rangeTB,'String','1'); +set(obj.panel.manual_rangeTB,'FontSize',10); +set(obj.panel.manual_rangeTB,'TooltipString','During manual (left click) entry, this sets the range of bins that will be searched to find the max. Set to "0" to not search.'); % %----threshold label % obj.panel.leading_edge_thresh_label = uicontrol('Parent',obj.h_fig); % set(obj.panel.leading_edge_thresh_label,'Style','text'); % set(obj.panel.leading_edge_thresh_label,'String','LE threshold:'); -% set(obj.panel.leading_edge_thresh_label,'FontSize',8) +% set(obj.panel.leading_edge_thresh_label,'FontSize',10) % % %----threshold entry % obj.panel.leading_edge_thresh_tbox = uicontrol('Parent',obj.h_fig); % set(obj.panel.leading_edge_thresh_tbox,'Style','edit'); % set(obj.panel.leading_edge_thresh_tbox,'String','0'); -% set(obj.panel.leading_edge_thresh_tbox,'FontSize',8); +% set(obj.panel.leading_edge_thresh_tbox,'FontSize',10); % %----Interpolation label obj.panel.interp_mode_label = uicontrol('Parent',obj.h_fig); set(obj.panel.interp_mode_label,'Style','text'); set(obj.panel.interp_mode_label,'String',sprintf('Interpolation:')); -set(obj.panel.interp_mode_label,'FontSize',8) +set(obj.panel.interp_mode_label,'FontSize',10) set(obj.panel.interp_mode_label,'TooltipString','During auto-interp (ALT left click and drag), manual points will be interpolated with this method.'); %----Interpolation pulldown menu -obj.panel.interp_mode_pdmenu = uicontrol('Parent',obj.h_fig); -set(obj.panel.interp_mode_pdmenu,'Style','popupmenu'); -set(obj.panel.interp_mode_pdmenu,'String',... +obj.panel.interp_modePM = uicontrol('Parent',obj.h_fig); +set(obj.panel.interp_modePM,'Style','popupmenu'); +set(obj.panel.interp_modePM,'String',... {'linear','spline'}); -set(obj.panel.interp_mode_pdmenu,'FontSize',8); -set(obj.panel.interp_mode_pdmenu,'TooltipString','During auto-interp (ALT left click and drag), manual points will be interpolated with this method.'); +set(obj.panel.interp_modePM,'FontSize',10); +set(obj.panel.interp_modePM,'TooltipString','During auto-interp (ALT left click and drag), manual points will be interpolated with this method.'); % % %----reinterp mode label % obj.panel.reinterp_mode_label = uicontrol('Parent',obj.h_fig); % set(obj.panel.reinterp_mode_label,'Style','text'); % set(obj.panel.reinterp_mode_label,'String',sprintf('Reinterpolation: (beta)')); -% set(obj.panel.reinterp_mode_label,'FontSize',8) +% set(obj.panel.reinterp_mode_label,'FontSize',10) % % %----reinterp mode checkbox % obj.panel.reinterp_mode_cbox = uicontrol('Parent',obj.h_fig); @@ -90,67 +90,67 @@ function create_ui(obj) %--------------------------------------------------------------------------------------------- % set up top panel table obj.panel.table.ui=obj.panel.handle; -obj.panel.table.width_margin = NaN*zeros(30,30); % Just make these bigger than they have to be -obj.panel.table.height_margin = NaN*zeros(30,30); -obj.panel.table.false_width = NaN*zeros(30,30); -obj.panel.table.false_height = NaN*zeros(30,30); +obj.panel.table.width_margin = nan(30,30); % Just make these bigger than they have to be +obj.panel.table.height_margin = nan(30,30); +obj.panel.table.false_width = nan(30,30); +obj.panel.table.false_height = nan(30,30); obj.panel.table.offset = [0 0]; row = 0; row = row+1; col = 1; -obj.panel.table.handles{row,col} = obj.panel.max_rbin_rng_label; +obj.panel.table.handles{row,col} = obj.panel.manual_range_label; obj.panel.table.width(row,col) = inf; -obj.panel.table.height(row,col) = 25; +obj.panel.table.height(row,col) = 20; obj.panel.table.width_margin(row,col) = 0; -obj.panel.table.height_margin(row,col) = 0; +obj.panel.table.height_margin(row,col) = 3; col = 2; -obj.panel.table.handles{row,col} = obj.panel.max_rbin_rng_tbox; +obj.panel.table.handles{row,col} = obj.panel.manual_rangeTB; obj.panel.table.width(row,col) = inf; -obj.panel.table.height(row,col) = 25; +obj.panel.table.height(row,col) = 20; obj.panel.table.width_margin(row,col) = 0; obj.panel.table.height_margin(row,col) = 0; % row = row+1; col = 1; % obj.panel.table.handles{row,col} = obj.panel.leading_edge_thresh_label; % obj.panel.table.width(row,col) = inf; -% obj.panel.table.height(row,col) = 25; +% obj.panel.table.height(row,col) = 20; % obj.panel.table.width_margin(row,col) = 0; -% obj.panel.table.height_margin(row,col) = 0; +% obj.panel.table.height_margin(row,col) = 3; % % col = 2; % obj.panel.table.handles{row,col} = obj.panel.leading_edge_thresh_tbox; % obj.panel.table.width(row,col) = inf; -% obj.panel.table.height(row,col) = 25; +% obj.panel.table.height(row,col) = 20; % obj.panel.table.width_margin(row,col) = 0; % obj.panel.table.height_margin(row,col) = 0; % row = row+1; col = 1; obj.panel.table.handles{row,col} = obj.panel.interp_mode_label; obj.panel.table.width(row,col) = inf; -obj.panel.table.height(row,col) = 25; +obj.panel.table.height(row,col) = 20; obj.panel.table.width_margin(row,col) = 0; -obj.panel.table.height_margin(row,col) = 0; +obj.panel.table.height_margin(row,col) = 3; col = 2; -obj.panel.table.handles{row,col} = obj.panel.interp_mode_pdmenu; +obj.panel.table.handles{row,col} = obj.panel.interp_modePM; obj.panel.table.width(row,col) = inf; -obj.panel.table.height(row,col) = 25; +obj.panel.table.height(row,col) = 20; obj.panel.table.width_margin(row,col) = 0; obj.panel.table.height_margin(row,col) = 0; % % row = row+1; col = 1; % obj.panel.table.handles{row,col} = obj.panel.reinterp_mode_label; % obj.panel.table.width(row,col) = inf; -% obj.panel.table.height(row,col) = 25; +% obj.panel.table.height(row,col) = 20; % obj.panel.table.width_margin(row,col) = 0; -% obj.panel.table.height_margin(row,col) = 0; +% obj.panel.table.height_margin(row,col) = 3; % % col = 2; % obj.panel.table.handles{row,col} = obj.panel.reinterp_mode_cbox; % obj.panel.table.width(row,col) = inf; -% obj.panel.table.height(row,col) = 25; +% obj.panel.table.height(row,col) = 20; % obj.panel.table.width_margin(row,col) = 0; % obj.panel.table.height_margin(row,col) = 0; @@ -164,7 +164,3 @@ function create_ui(obj) clear row col table_draw(obj.panel.table); - - -return - diff --git a/cresis-toolbox/+imb/@picktool_interpolate/left_click.m b/cresis-toolbox/+imb/@picktool_interpolate/left_click.m index 1556a5f9..0a0011b7 100644 --- a/cresis-toolbox/+imb/@picktool_interpolate/left_click.m +++ b/cresis-toolbox/+imb/@picktool_interpolate/left_click.m @@ -1,7 +1,7 @@ function cmds = left_click(obj,param) %% Get search range from tool param window -rbin_range_str = get(obj.panel.max_rbin_rng_tbox,'String'); +rbin_range_str = get(obj.panel.manual_rangeTB,'String'); try % Assumes the value entered is a matlab expression that can be evaluated search_range = eval(sprintf('[%s]', rbin_range_str)); diff --git a/cresis-toolbox/+imb/@picktool_interpolate/left_click_and_drag.m b/cresis-toolbox/+imb/@picktool_interpolate/left_click_and_drag.m index fa8b226a..dd1bbb0b 100644 --- a/cresis-toolbox/+imb/@picktool_interpolate/left_click_and_drag.m +++ b/cresis-toolbox/+imb/@picktool_interpolate/left_click_and_drag.m @@ -13,8 +13,8 @@ fprintf('Interpolate points %f to %f, %f to %f\n', x, y); %% Get tool interpolate method -interp_idx = get(obj.panel.interp_mode_pdmenu,'Value'); -interp_type = get(obj.panel.interp_mode_pdmenu,'String'); +interp_idx = get(obj.panel.interp_modePM,'Value'); +interp_type = get(obj.panel.interp_modePM,'String'); interp_type = interp_type{interp_idx}; param.x_bounds = 3; @@ -23,7 +23,7 @@ for layer_idx = 1:length(cur_layers) cur_layer = cur_layers(layer_idx); - [manual_idxs,auto_idxs,point_idxs] = find_matching_pnts(obj,param,cur_layer); + [manual_idxs,auto_idxs,~] = find_matching_pnts(obj,param,cur_layer); if length(manual_idxs) < 2 warning('Insufficient points to interpolate'); diff --git a/cresis-toolbox/+imb/@picktool_interpolate/picktool_interpolate.m b/cresis-toolbox/+imb/@picktool_interpolate/picktool_interpolate.m index 26969959..b2c286d5 100644 --- a/cresis-toolbox/+imb/@picktool_interpolate/picktool_interpolate.m +++ b/cresis-toolbox/+imb/@picktool_interpolate/picktool_interpolate.m @@ -53,7 +53,7 @@ % Any code, including access to object % This is the first tool initialized by a new echowin obj.h_fig = h_fig; - obj.w = 185; + obj.w = 200; obj.h = 120; obj.tool_name = '(i)nterp'; obj.tool_name_title = 'enter'; diff --git a/cresis-toolbox/+imb/@picktool_snake/create_ui.m b/cresis-toolbox/+imb/@picktool_snake/create_ui.m new file mode 100644 index 00000000..6f99d468 --- /dev/null +++ b/cresis-toolbox/+imb/@picktool_snake/create_ui.m @@ -0,0 +1,163 @@ +function create_ui_components(obj) + +% create_ui_components(obj) +% +% Creates components for the snake param window's UI when the echogram +% window is first opened. +% + +set(obj.h_fig,'DockControls','off') +set(obj.h_fig,'NumberTitle','off'); +if strcmpi(class(obj.h_fig),'double') + set(obj.h_fig,'Name',sprintf('%d: %s tool parameters', obj.h_fig, obj.tool_name_title)); +else + set(obj.h_fig,'Name',sprintf('%d: %s tool parameters', obj.h_fig.Number, obj.tool_name_title)); +end +set(obj.h_fig,'ToolBar','none'); +set(obj.h_fig,'MenuBar','none'); +set(obj.h_fig,'KeyPressFcn',@obj.key_press); +set(obj.h_fig,'CloseRequestFcn',@obj.close_win); +%set(obj.h_fig,'Resize','off'); + +set(obj.h_fig,'visible','off'); + +% set default position (changed when window accessed) +set(obj.h_fig,'Units','Pixels'); +set(obj.h_fig,'Position',[0 0 obj.w obj.h]); +% show top panel (snake params) but not bottom panel (crandall params) +%set(obj.top_panel.handle,'visible','on'); +%set(obj.bottom_panel.handle,'visible','off'); + +%========================================================================== +% top panel +obj.top_panel.handle = uipanel('Parent',obj.h_fig); +set(obj.top_panel.handle,'HighlightColor',[0.8 0.8 0.8]); +set(obj.top_panel.handle,'ShadowColor',[0.6 0.6 0.6]); +%set(obj.top_panel.handle,'visible','off'); + +%-------------------------------------- +% table +obj.table.ui=obj.h_fig; + +row = 1; col = 1; +obj.table.handles{row,col} = obj.top_panel.handle; +obj.table.width(row,col) = inf; +obj.table.height(row,col) = inf; +obj.table.width_margin(row,col) = 0; +obj.table.height_margin(row,col) = 0; + +clear row col +table_draw(obj.table); + +%========================================================================== +% top panel table contents + +% +% %-----mode label +% obj.top_panel.mode_label = uicontrol('Parent',obj.top_panel.handle); +% set(obj.top_panel.mode_label,'Style','text'); +% set(obj.top_panel.mode_label,'String','Mode'); +% +% %----snake tool list box +% obj.top_panel.tool_PM = uicontrol('Parent',obj.top_panel.handle); +% set(obj.top_panel.tool_PM,'Style','popupmenu'); +% set(obj.top_panel.tool_PM,'String',{'basic'}); +% set(obj.top_panel.tool_PM,'Value',1) +% set(obj.top_panel.tool_PM,'Callback',@obj.toolPM_callback); + +%----insert range +obj.top_panel.manual_range_text = uicontrol('Parent',obj.top_panel.handle); +set(obj.top_panel.manual_range_text,'Style','text'); +set(obj.top_panel.manual_range_text,'String','Manual range:'); +set(obj.top_panel.manual_range_text,'TooltipString','During manual (left click) entry, this sets the range of bins that will be searched to find the max. Set to "0" to not search.'); +%----insert pt search range box +obj.top_panel.manual_rangeTB = uicontrol('Parent',obj.top_panel.handle); +set(obj.top_panel.manual_rangeTB,'Style','edit'); +set(obj.top_panel.manual_rangeTB,'String',obj.in_rng_sv); +set(obj.top_panel.manual_rangeTB,'TooltipString','During manual (left click) entry, this sets the range of bins that will be searched to find the max. Set to "0" to not search.'); + +%----snake search range name +obj.top_panel.snake_range_label = uicontrol('Parent',obj.top_panel.handle); +set(obj.top_panel.snake_range_label,'Style','text'); +set(obj.top_panel.snake_range_label,'String','Snake range:'); +set(obj.top_panel.snake_range_label,'TooltipString','During auto-snake (ALT left click and drag), snake will search +/- this many bins for the peak intensity.'); +%----snake search range box +obj.top_panel.snake_rangeTB = uicontrol('Parent',obj.top_panel.handle); +set(obj.top_panel.snake_rangeTB,'Style','edit'); +set(obj.top_panel.snake_rangeTB,'String',obj.sn_rng_sv); +set(obj.top_panel.snake_rangeTB,'TooltipString','During auto-snake (ALT left click and drag), snake will search +/- this many bins for the peak intensity.'); + +%-------------------------------------------------------------------------- +% set up top panel table +obj.top_panel.table.ui=obj.top_panel.handle; +obj.top_panel.table.width_margin = nan(30,30); % Just make these bigger than they have to be +obj.top_panel.table.height_margin = nan(30,30); +obj.top_panel.table.false_width = nan(30,30); +obj.top_panel.table.false_height = nan(30,30); +obj.top_panel.table.offset = [0 0]; + +row = 0; + +% row = row+1; col = 1; +% obj.top_panel.table.handles{row,col} = obj.top_panel.mode_label; +% obj.top_panel.table.width(row,col) = inf; +% obj.top_panel.table.height(row,col) = 20; +% obj.top_panel.table.width_margin(row,col) = 0; +% obj.top_panel.table.height_margin(row,col) = 3; +% +% col = 2; +% obj.top_panel.table.handles{row,col} = obj.top_panel.tool_PM; +% obj.top_panel.table.width(row,col) = inf; +% obj.top_panel.table.height(row,col) = 20; +% obj.top_panel.table.width_margin(row,col) = 0; +% obj.top_panel.table.height_margin(row,col) = 0; + +row = row+1; col = 1; +obj.top_panel.table.handles{row,col} = obj.top_panel.manual_range_text; +obj.top_panel.table.width(row,col) = inf; +obj.top_panel.table.height(row,col) = 20; +obj.top_panel.table.width_margin(row,col) = 0; +obj.top_panel.table.height_margin(row,col) = 3; + +col = 2; +obj.top_panel.table.handles{row,col} = obj.top_panel.manual_rangeTB; +obj.top_panel.table.width(row,col) = inf; +obj.top_panel.table.height(row,col) = 20; +obj.top_panel.table.width_margin(row,col) = 0; +obj.top_panel.table.height_margin(row,col) = 0; + +row = row+1; col = 1; +obj.top_panel.table.handles{row,col} = obj.top_panel.snake_range_label; +obj.top_panel.table.width(row,col) = inf; +obj.top_panel.table.height(row,col) = 20; +obj.top_panel.table.width_margin(row,col) = 0; +obj.top_panel.table.height_margin(row,col) = 3; + +col = 2; +obj.top_panel.table.handles{row,col} = obj.top_panel.snake_rangeTB; +obj.top_panel.table.width(row,col) = inf; +obj.top_panel.table.height(row,col) = 20; +obj.top_panel.table.width_margin(row,col) = 0; +obj.top_panel.table.height_margin(row,col) = 0; + +% Add spacer to fill window +row = row+1; col = 1; +obj.top_panel.table.handles{row,col} = []; +obj.top_panel.table.width(row,col) = inf; +obj.top_panel.table.height(row,col) = inf; +obj.top_panel.table.width_margin(row,col) = 0; +obj.top_panel.table.height_margin(row,col) = 0; + +clear row col + +% Draw table +table_draw(obj.top_panel.table); + +if obj.first_time + obj.first_time = false; +else + set(obj.h_fig,'visible','on'); +end + +return + diff --git a/cresis-toolbox/+imb/@picktool_snake/create_ui_basic.m b/cresis-toolbox/+imb/@picktool_snake/create_ui_basic.m deleted file mode 100644 index 060a3d5d..00000000 --- a/cresis-toolbox/+imb/@picktool_snake/create_ui_basic.m +++ /dev/null @@ -1,155 +0,0 @@ -function create_ui_basic(obj,xpos,ypos) - -% create_ui_basic(obj,xpos,ypos) -% -% Creates components for the snake param window's UI when the basic snake -% tool is selected. Plots the window at xpos,ypos. -% - -set(obj.h_fig,'visible','off'); - -% set default position (changed when window accessed) -set(obj.h_fig,'Units','Pixels'); -set(obj.h_fig,'Position',[xpos ypos obj.w obj.h]); -% show top panel (snake params) but not bottom panel (crandall params) -%set(obj.top_panel.handle,'visible','on'); -%set(obj.bottom_panel.handle,'visible','off'); - -if ~obj.first_time - figure(obj.h_fig); - clf; - obj.table = []; -end - -%========================================================================== -% top panel -obj.top_panel.handle = uipanel('Parent',obj.h_fig); -set(obj.top_panel.handle,'HighlightColor',[0.8 0.8 0.8]); -set(obj.top_panel.handle,'ShadowColor',[0.6 0.6 0.6]); -%set(obj.top_panel.handle,'visible','off'); - -%-------------------------------------- -% table -obj.table.ui=obj.h_fig; - -row = 1; col = 1; -obj.table.handles{row,col} = obj.top_panel.handle; -obj.table.width(row,col) = inf; -obj.table.height(row,col) = inf; -obj.table.width_margin(row,col) = 0; -obj.table.height_margin(row,col) = 0; - -clear row col -table_draw(obj.table); - -%============================================================================================ -% top panel table contents - -% -% %-----mode label -% obj.top_panel.mode_label = uicontrol('Parent',obj.top_panel.handle); -% set(obj.top_panel.mode_label,'Style','text'); -% set(obj.top_panel.mode_label,'String','Mode'); -% -% %----snake tool list box -% obj.top_panel.tool_PM = uicontrol('Parent',obj.top_panel.handle); -% set(obj.top_panel.tool_PM,'Style','popupmenu'); -% set(obj.top_panel.tool_PM,'String',{'basic'}); -% set(obj.top_panel.tool_PM,'Value',1) -% set(obj.top_panel.tool_PM,'Callback',@obj.toolPM_callback); - -%----insert range -obj.top_panel.insert_range_label = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.insert_range_label,'Style','text'); -set(obj.top_panel.insert_range_label,'String','Manual range:'); -set(obj.top_panel.insert_range_label,'TooltipString','During manual (left click) entry, this sets the range of bins that will be searched to find the max. Set to "0" to not search.'); -%----insert pt search range box -obj.top_panel.insert_range_TE = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.insert_range_TE,'Style','edit'); -set(obj.top_panel.insert_range_TE,'String',obj.in_rng_sv); -set(obj.top_panel.insert_range_TE,'TooltipString','During manual (left click) entry, this sets the range of bins that will be searched to find the max. Set to "0" to not search.'); - -%----snake search range name -obj.top_panel.snake_range_label = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.snake_range_label,'Style','text'); -set(obj.top_panel.snake_range_label,'String','Snake range:'); -set(obj.top_panel.snake_range_label,'TooltipString','During auto-snake (ALT left click and drag), snake will search +/- this many bins for the peak intensity.'); -%----snake search range box -obj.top_panel.snake_range_TE = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.snake_range_TE,'Style','edit'); -set(obj.top_panel.snake_range_TE,'String',obj.sn_rng_sv); -set(obj.top_panel.snake_range_TE,'TooltipString','During auto-snake (ALT left click and drag), snake will search +/- this many bins for the peak intensity.'); - -%--------------------------------------------------------------------------------------------- -% set up top panel table -obj.top_panel.table.ui=obj.top_panel.handle; -obj.top_panel.table.width_margin = NaN*zeros(30,30); % Just make these bigger than they have to be -obj.top_panel.table.height_margin = NaN*zeros(30,30); -obj.top_panel.table.false_width = NaN*zeros(30,30); -obj.top_panel.table.false_height = NaN*zeros(30,30); -obj.top_panel.table.offset = [0 0]; - -row = 0; - -% row = row+1; col = 1; -% obj.top_panel.table.handles{row,col} = obj.top_panel.mode_label; -% obj.top_panel.table.width(row,col) = inf; -% obj.top_panel.table.height(row,col) = 25; -% obj.top_panel.table.width_margin(row,col)= 1.5; -% obj.top_panel.table.height_margin(row,col)=1.5; -% -% col = 2; -% obj.top_panel.table.handles{row,col} = obj.top_panel.tool_PM; -% obj.top_panel.table.width(row,col) = inf; -% obj.top_panel.table.height(row,col) = 25; -% obj.top_panel.table.width_margin(row,col)= 1.5; -% obj.top_panel.table.height_margin(row,col)=1.5; - -row = row+1; col = 1; -obj.top_panel.table.handles{row,col} = obj.top_panel.insert_range_label; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = 25; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -col = 2; -obj.top_panel.table.handles{row,col} = obj.top_panel.insert_range_TE; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = 25; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -row = row+1; col = 1; -obj.top_panel.table.handles{row,col} = obj.top_panel.snake_range_label; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = 25; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -col = 2; -obj.top_panel.table.handles{row,col} = obj.top_panel.snake_range_TE; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = 25; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -% Add spacer to fill window -row = row+1; col = 1; -obj.top_panel.table.handles{row,col} = []; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -clear row col - -% Draw table -table_draw(obj.top_panel.table); - -if obj.first_time - obj.first_time = false; -else - set(obj.h_fig,'visible','on'); -end - -return; \ No newline at end of file diff --git a/cresis-toolbox/+imb/@picktool_snake/create_ui_components.m b/cresis-toolbox/+imb/@picktool_snake/create_ui_components.m deleted file mode 100644 index 410c8288..00000000 --- a/cresis-toolbox/+imb/@picktool_snake/create_ui_components.m +++ /dev/null @@ -1,23 +0,0 @@ -function create_ui_components(obj) - -% create_ui_components(obj) -% -% Creates components for the snake param window's UI when the echogram -% window is first opened. -% - -set(obj.h_fig,'DockControls','off') -set(obj.h_fig,'NumberTitle','off'); -if strcmpi(class(obj.h_fig),'double') - set(obj.h_fig,'Name',sprintf('%d: %s tool parameters', obj.h_fig, obj.tool_name_title)); -else - set(obj.h_fig,'Name',sprintf('%d: %s tool parameters', obj.h_fig.Number, obj.tool_name_title)); -end -set(obj.h_fig,'ToolBar','none'); -set(obj.h_fig,'MenuBar','none'); -set(obj.h_fig,'KeyPressFcn',@obj.key_press); -set(obj.h_fig,'CloseRequestFcn',@obj.close_win); -%set(obj.h_fig,'Resize','off'); - -return - diff --git a/cresis-toolbox/+imb/@picktool_snake/create_ui_crandall.m b/cresis-toolbox/+imb/@picktool_snake/create_ui_crandall.m deleted file mode 100644 index f075f208..00000000 --- a/cresis-toolbox/+imb/@picktool_snake/create_ui_crandall.m +++ /dev/null @@ -1,301 +0,0 @@ -function create_ui_crandall(obj,xpos,ypos) - -% create_ui_crandall(obj,xpos,ypos) -% -% Creates components for the snake param window's UI when the crandall pick -% tool is selected. Plots the window at (xpos,ypos). -% - -set(obj.h_fig,'visible','off'); - -figure(obj.h_fig); -clf; -obj.table = []; - -%========================================================================== -% top panel -obj.top_panel.handle = uipanel('Parent',obj.h_fig); -set(obj.top_panel.handle,'HighlightColor',[0.8 0.8 0.8]); -set(obj.top_panel.handle,'ShadowColor',[0.6 0.6 0.6]); -%set(obj.top_panel.handle,'visible','off'); -% bottom panel -obj.bottom_panel.handle= uipanel('Parent',obj.h_fig); -set(obj.bottom_panel.handle,'HighlightColor',[0.8 0.8 0.8]); -set(obj.bottom_panel.handle,'ShadowColor',[0.6 0.6 0.6]); -%set(obj.bottom_panel.handle,'visible','off'); - -%-------------------------------------- -% table -% set default position (changed when window accessed) -set(obj.h_fig,'Units','Pixels'); -set(obj.h_fig,'Position',[xpos ypos obj.w obj.crandall_h]); - -obj.table.ui=obj.h_fig; - -row = 1; col = 1; -obj.table.handles{row,col} = obj.top_panel.handle; -obj.table.width(row,col) = inf; -obj.table.height(row,col) = inf; -obj.table.width_margin(row,col) = 0; -obj.table.height_margin(row,col) = 0; - -row = 2; col = 1; -obj.table.handles{row,col} = obj.bottom_panel.handle; -obj.table.width(row,col) = inf; -obj.table.height(row,col) = inf; -obj.table.width_margin(row,col) = 0; -obj.table.height_margin(row,col) = 0; - -clear row col -table_draw(obj.table); - -%============================================================================================ -% top panel table contents - -%----snake tool list box -obj.top_panel.tool_PM = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.tool_PM,'Style','popupmenu'); -set(obj.top_panel.tool_PM,'String',{'basic','crandall','panton'}); -set(obj.top_panel.tool_PM,'Value',2) -set(obj.top_panel.tool_PM,'Callback',@obj.toolPM_callback); - -%-----mode label -obj.top_panel.mode_label = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.mode_label,'Style','text'); -set(obj.top_panel.mode_label,'String','Mode'); - -%----insert range -obj.top_panel.insert_range_label = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.insert_range_label,'Style','text'); -set(obj.top_panel.insert_range_label,'String','Max point range:'); -%----param1 box -obj.top_panel.insert_range_TE = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.insert_range_TE,'Style','edit'); -set(obj.top_panel.insert_range_TE,'String',obj.in_rng_sv); -%----param2 name -obj.top_panel.snake_range_label = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.snake_range_label,'Style','text'); -set(obj.top_panel.snake_range_label,'String','Snake range:'); -%----param2 box -obj.top_panel.snake_range_TE = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.snake_range_TE,'Style','edit'); -set(obj.top_panel.snake_range_TE,'String',obj.sn_rng_sv); -%----reinterp mode enable label -obj.top_panel.reinterp_mode_label = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.reinterp_mode_label,'Style','text'); -set(obj.top_panel.reinterp_mode_label,'String','Reinterpolation (beta):'); -%----reinterp mode enable cbox -obj.top_panel.reinterp_mode_cbox = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.reinterp_mode_cbox,'Style','checkbox'); - - -%--------------------------------------------------------------------------------------------- -% set up top panel table -obj.top_panel.table.ui=obj.top_panel.handle; -obj.top_panel.table.width_margin = NaN*zeros(30,30); % Just make these bigger than they have to be -obj.top_panel.table.height_margin = NaN*zeros(30,30); -obj.top_panel.table.false_width = NaN*zeros(30,30); -obj.top_panel.table.false_height = NaN*zeros(30,30); -obj.top_panel.table.offset = [0 0]; - -row = 1; col = 1; -obj.top_panel.table.handles{row,col} = obj.top_panel.mode_label; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -row = 1; col = 2; -obj.top_panel.table.handles{row,col} = obj.top_panel.tool_PM; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -row = 2; col = 1; -obj.top_panel.table.handles{row,col} = obj.top_panel.insert_range_label; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -row = 2; col = 2; -obj.top_panel.table.handles{row,col} = obj.top_panel.insert_range_TE; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -row = 3; col = 1; -obj.top_panel.table.handles{row,col} = obj.top_panel.snake_range_label; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -row = 3; col = 2; -obj.top_panel.table.handles{row,col} = obj.top_panel.snake_range_TE; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -row = 4; col = 1; -obj.top_panel.table.handles{row,col} = obj.top_panel.reinterp_mode_label; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -row = 4; col = 2; -obj.top_panel.table.handles{row,col} = obj.top_panel.reinterp_mode_cbox; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -clear row col - -% Draw table -table_draw(obj.top_panel.table); - -%============================================================================================= -% bottom_panel table contents -%--------------------------------------------------------------------------------------------- -%---- top smooth -obj.bottom_panel.topSmooth = uicontrol('Parent',obj.bottom_panel.handle); -set(obj.bottom_panel.topSmooth,'Style','text'); -set(obj.bottom_panel.topSmooth,'String','Top smooth:'); -%---- bottom smooth -obj.bottom_panel.bottomSmooth = uicontrol('Parent',obj.bottom_panel.handle); -set(obj.bottom_panel.bottomSmooth,'Style','text'); -set(obj.bottom_panel.bottomSmooth,'String','Bottom smooth:'); - -%---- top smooth slider -obj.bottom_panel.topSmoothSlider = uicontrol('Parent',obj.bottom_panel.handle); -set(obj.bottom_panel.topSmoothSlider,'Style','slider','Min',0,'Max',1,... - 'Value',obj.top_sm_sv,'SliderStep',[0.05 0.2]); -%---- bottom smooth slider -obj.bottom_panel.bottomSmoothSlider = uicontrol('Parent',obj.bottom_panel.handle); -set(obj.bottom_panel.bottomSmoothSlider,'Style','slider','Min',0,'Max',1,... - 'Value',obj.bot_sm_sv,'SliderStep',[0.05 0.2]); - -%---- top peakRatio -obj.bottom_panel.topPeakRatio = uicontrol('Parent',obj.bottom_panel.handle); -set(obj.bottom_panel.topPeakRatio,'Style','text'); -set(obj.bottom_panel.topPeakRatio,'String','Top peak ratio:'); -%---- bottom PeakRatio -obj.bottom_panel.bottomPeakRatio = uicontrol('Parent',obj.bottom_panel.handle); -set(obj.bottom_panel.bottomPeakRatio,'Style','text'); -set(obj.bottom_panel.bottomPeakRatio,'String','Bottom peak ratio:'); - -%---- top PeakRatio slider -obj.bottom_panel.topPeakRatioSlider = uicontrol('Parent',obj.bottom_panel.handle); -set(obj.bottom_panel.topPeakRatioSlider,'Style','slider','Min',0,'Max',1,... - 'Value',obj.top_pk_sv,'SliderStep',[0.05 0.2]); -%---- bottom PeakRatio slider -obj.bottom_panel.bottomPeakRatioSlider = uicontrol('Parent',obj.bottom_panel.handle); -set(obj.bottom_panel.bottomPeakRatioSlider,'Style','slider','Min',0,'Max',1,... - 'Value',obj.bot_pk_sv,'SliderStep',[0.05 0.2]); - -%============================== -%---- Repulse name label -obj.bottom_panel.repulseLabel = uicontrol('Parent',obj.bottom_panel.handle); -set(obj.bottom_panel.repulseLabel,'Style','text'); -set(obj.bottom_panel.repulseLabel,'String','Repulse:'); -%---- Repulse slider -obj.bottom_panel.repulseSlider = uicontrol('Parent',obj.bottom_panel.handle); -set(obj.bottom_panel.repulseSlider,'Style','slider','Min',0,'Max',1,... - 'Value',obj.rep_sv,'SliderStep',[0.05 0.2]); - -%---------------------------------------------------------------------------------------------- -% set up bottom panel table - -obj.bottom_panel.table.ui=obj.bottom_panel.handle; -obj.bottom_panel.table.width_margin = NaN*zeros(30,30); % Just make these bigger than they have to be -obj.bottom_panel.table.height_margin = NaN*zeros(30,30); -obj.bottom_panel.table.false_width = NaN*zeros(30,30); -obj.bottom_panel.table.false_height = NaN*zeros(30,30); -obj.bottom_panel.table.offset = [0 0]; - - -row = 1; col = 1; -obj.bottom_panel.table.handles{row,col} = obj.bottom_panel.topSmooth; -obj.bottom_panel.table.width(row,col) = inf; -obj.bottom_panel.table.height(row,col) = inf; -obj.bottom_panel.table.width_margin(row,col)= 1.5; -obj.bottom_panel.table.height_margin(row,col)=1.5; - -row = 1; col = 2; -obj.bottom_panel.table.handles{row,col} = obj.bottom_panel.topSmoothSlider; -obj.bottom_panel.table.width(row,col) = inf; -obj.bottom_panel.table.height(row,col) = inf; -obj.bottom_panel.table.width_margin(row,col)= 1.5; -obj.bottom_panel.table.height_margin(row,col)=1.5; - -row = 2; col = 1; -obj.bottom_panel.table.handles{row,col} = obj.bottom_panel.bottomSmooth; -obj.bottom_panel.table.width(row,col) = inf; -obj.bottom_panel.table.height(row,col) = inf; -obj.bottom_panel.table.width_margin(row,col)= 1.5; -obj.bottom_panel.table.height_margin(row,col)=1.5; - -row = 2; col=2; -obj.bottom_panel.table.handles{row,col} = obj.bottom_panel.bottomSmoothSlider; -obj.bottom_panel.table.width(row,col) = inf; -obj.bottom_panel.table.height(row,col) = inf; -obj.bottom_panel.table.width_margin(row,col)= 1.5; -obj.bottom_panel.table.height_margin(row,col)=1.5; - -row = 3; col = 1; -obj.bottom_panel.table.handles{row,col} = obj.bottom_panel.topPeakRatio; -obj.bottom_panel.table.width(row,col) = inf; -obj.bottom_panel.table.height(row,col) = inf; -obj.bottom_panel.table.width_margin(row,col)= 1.5; -obj.bottom_panel.table.height_margin(row,col)=1.5; - -row = 3; col = 2; -obj.bottom_panel.table.handles{row,col} = obj.bottom_panel.topPeakRatioSlider; -obj.bottom_panel.table.width(row,col) = inf; -obj.bottom_panel.table.height(row,col) = inf; -obj.bottom_panel.table.width_margin(row,col)= 1.5; -obj.bottom_panel.table.height_margin(row,col)=1.5; - -row = 4; col = 1; -obj.bottom_panel.table.handles{row,col} = obj.bottom_panel.bottomPeakRatio; -obj.bottom_panel.table.width(row,col) = inf; -obj.bottom_panel.table.height(row,col) = inf; -obj.bottom_panel.table.width_margin(row,col)= 1.5; -obj.bottom_panel.table.height_margin(row,col)=1.5; - -row = 4; col = 2; -obj.bottom_panel.table.handles{row,col} = obj.bottom_panel.bottomPeakRatioSlider; -obj.bottom_panel.table.width(row,col) = inf; -obj.bottom_panel.table.height(row,col) = inf; -obj.bottom_panel.table.width_margin(row,col)= 1.5; -obj.bottom_panel.table.height_margin(row,col)=1.5; - -row = 5; col = 1; -obj.bottom_panel.table.handles{row,col} = obj.bottom_panel.repulseLabel; -obj.bottom_panel.table.width(row,col) = inf; -obj.bottom_panel.table.height(row,col) = inf; -obj.bottom_panel.table.width_margin(row,col)= 1.5; -obj.bottom_panel.table.height_margin(row,col)=1.5; - -row = 5; col=2; -obj.bottom_panel.table.handles{row,col} = obj.bottom_panel.repulseSlider; -obj.bottom_panel.table.width(row,col) = inf; -obj.bottom_panel.table.height(row,col) = inf; -obj.bottom_panel.table.width_margin(row,col)= 1.5; -obj.bottom_panel.table.height_margin(row,col)=1.5; - - - -clear row col - -%---------------------------------------------------------------------------------------------- -% Draw table -table_draw(obj.bottom_panel.table); -set(obj.h_fig,'visible','on'); - -return \ No newline at end of file diff --git a/cresis-toolbox/+imb/@picktool_snake/left_click.m b/cresis-toolbox/+imb/@picktool_snake/left_click.m index b6879c0d..45e24fc1 100644 --- a/cresis-toolbox/+imb/@picktool_snake/left_click.m +++ b/cresis-toolbox/+imb/@picktool_snake/left_click.m @@ -1,7 +1,7 @@ function cmds = left_click(obj,param) %% Get search range from tool param window -rbin_range_str = get(obj.top_panel.insert_range_TE,'String'); +rbin_range_str = get(obj.top_panel.manual_rangeTB,'String'); try % Assumes the value entered is a matlab expression that can be evaluated search_range = eval(sprintf('[%s]', rbin_range_str)); diff --git a/cresis-toolbox/+imb/@picktool_snake/left_click_and_drag.m b/cresis-toolbox/+imb/@picktool_snake/left_click_and_drag.m index eda6e55b..89cefabc 100644 --- a/cresis-toolbox/+imb/@picktool_snake/left_click_and_drag.m +++ b/cresis-toolbox/+imb/@picktool_snake/left_click_and_drag.m @@ -20,13 +20,12 @@ % tool_idx = get(obj.top_panel.tool_PM,'Value'); tool_idx = 1; if tool_idx == 1 - %========================================================================= %% Basic Snake for layer_idx = 1:length(cur_layers) cur_layer = cur_layers(layer_idx); - [manual_idxs,auto_idxs,point_idxs] = find_matching_pnts(obj,param,cur_layer); + [manual_idxs,auto_idxs,~] = find_matching_pnts(obj,param,cur_layer); if length(manual_idxs) < 1 warning('Insufficient points to snake'); @@ -47,133 +46,5 @@ 2*ones(size(auto_idxs)), param.cur_quality*ones(size(auto_idxs))}; end end - -% elseif tool_idx == 2 -% %% ========================================================================== -% % Crandall -% -% % setup params -% top_smooth = 10^(get(obj.bottom_panel.topSmoothSlider, 'Value')*6); -% top_peak = get(obj.bottom_panel.topPeakRatioSlider, 'Value'); -% bottom_smooth = 10^(get(obj.bottom_panel.bottomSmoothSlider, 'Value')*6); -% bottom_peak = get(obj.bottom_panel.bottomPeakRatioSlider, 'Value'); -% repulse = 10^(get(obj.bottom_panel.repulseSlider, 'Value')*2); -% opts = [top_smooth bottom_smooth top_peak bottom_peak repulse]; -% -% for layer_idx = 1:length(cur_layers) -% cur_layer = cur_layers(layer_idx); -% x_auto = param.layer_x_auto{layer_idx}; -% y_auto = param.layer_y_auto{layer_idx}; -% x_manual = param.layer_x_manu{layer_idx}; -% y_manual = param.layer_y_manu{layer_idx}; -% manual_auto = layer_type{cur_layer}; -% -% % combine manual and auto points -% [layer_data_x,I] = sort([x_manual,x_auto]); -% y_cat = [y_manual,y_auto]; -% layer_data_y = y_cat(I); -% % Determine which points are in the rubberband/box -% dx = x_data(2) - x_data(1); -% x_idx_1 = find(layer_data_x >= x(1)-dx/2,1,'first'); -% x_idx_2 = find(layer_data_x <= x(2)+dx/2,1,'last'); -% old_idx = x_idx_1:x_idx_2; -% manual_auto = manual_auto(old_idx); -% old_idx_manual = old_idx(logical(manual_auto-2)); -% old_idx_auto = old_idx(logical(manual_auto-1)); -% y_old_manual = layer_data_y(old_idx_manual); % in current unit -% x_old_manual = layer_data_x(old_idx_manual); % in current unit -% A_indices = x_data >= (x(1)-dx/2) & x_data <= (x(2)+dx/2); -% A = A(:,A_indices); -% if ~isfield(param,'point') || isempty(param.point.x) -% pnts_y = interp1(y_data,1:length(y_data),... -% y_old_manual,'linear','extrap'); -% pnts_x = interp1(x_data,1:length(x_data),... -% x_old_manual,'linear','extrap'); -% pnts = [round(pnts_x);round(pnts_y)]; -% if length(pnts_x) < 1 % no manual points -% [AA path]=stereo(1, double(A), opts); -% elseif cur_layer == 1 % have manual points && surface -% [AA path]=stereo(1, double(A), opts,double(pnts),[]); -% elseif cur_layer == 2 % have manual points && bottom -% [AA path]=stereo(1, double(A), opts,[],double(pnts)); -% end -% y_new = interp1(1:length(y_data),y_data,path(cur_layer,:),'linear','extrap'); -% x_new = x_data(A_indices); -% % Run snake on values -% cmds(end+1).cmd = 'snake crandall'; -% cmds(end).begin_cmd = true; -% cmds(end).cur_layer = cur_layer; -% cmds(end).type = manual_auto; % auto -% cmds(end).qual = param.cur_qual; -% cmds(end).old_idx = old_idx; -% cmds(end).new_pnts.x = x_new; -% cmds(end).new_pnts.y = y_new; -% else -% x_old_manual(end+1) = param.point.x; -% y_old_manual(end+1) = param.point.y; -% [x_old_manual sort_idx] = sort(x_old_manual); -% y_old_manual = y_old_manual(sort_idx); -% pnts_y = interp1(y_data,1:length(y_data),... -% y_old_manual,'linear','extrap'); -% pnts_x = interp1(x_data,1:length(x_data),... -% x_old_manual,'linear','extrap'); -% pnts = [round(pnts_x);round(pnts_y)]; -% if length(pnts_x) < 1 % no manual points -% [AA path]=stereo(1, double(A), opts); -% elseif cur_layer == 1 % have manual points && surface -% [AA path]=stereo(1, double(A), opts,double(pnts),[]); -% elseif cur_layer == 2 % have manual points && bottom -% [AA path]=stereo(1, double(A), opts,[],double(pnts)); -% end -% y_new = interp1(1:length(y_data),y_data,path(cur_layer,:),'linear','extrap'); -% x_new = x_data(A_indices); -% %-------first delete auto points -% cmds(end+1).cmd = 'delete auto pnts'; -% cmds(end).begin_cmd = true; -% cmds(end).cur_layer = cur_layer; -% cmds(end).type = 2; % auto -% cmds(end).qual = param.cur_qual; -% cmds(end).old_idx = old_idx; -% cmds(end).new_pnts.x = []; -% cmds(end).new_pnts.y = []; -% %-------then add the single manual point -% cmds(end+1).cmd = 'insert'; -% cmds(end).begin_cmd = false; -% cmds(end).cur_layer = cur_layer; -% cmds(end).type = 1; % manual -% cmds(end).qual = param.cur_qual; -% cmds(end).old_idx = []; -% cmds(end).new_pnts.x = param.point.x; -% cmds(end).new_pnts.y = param.point.y; -% %-------last add derived points -% cmds(end+1).cmd = 'snake Crandall'; -% cmds(end).begin_cmd = false; -% cmds(end).cur_layer = cur_layer; -% cmds(end).type = 2; % auto -% cmds(end).qual = param.cur_qual; -% cmds(end).old_idx = []; -% cmds(end).new_pnts.x = x_new; -% cmds(end).new_pnts.y = y_new; -% end -% if get(obj.top_panel.reinterp_mode_cbox,'Value') == 1 -% -% obj.last_tool = tool_idx; -% obj.last_layers = cur_layers; -% obj.last_range_gps = interp1(x_data,param.image_gps_time,... -% x,'linear','extrap'); -% end -% -% end -% -% -% -% -% -% -% -% elseif tool_idx == 3; -% %%========================================================================== -% % Panton -% -% + end diff --git a/cresis-toolbox/+imb/@picktool_snake/picktool_snake.m b/cresis-toolbox/+imb/@picktool_snake/picktool_snake.m index 39aa1731..ab00afa4 100644 --- a/cresis-toolbox/+imb/@picktool_snake/picktool_snake.m +++ b/cresis-toolbox/+imb/@picktool_snake/picktool_snake.m @@ -61,13 +61,10 @@ obj.rep_sv = .5; obj.cur_mode = 1; - obj.create_ui_components(); - obj.create_ui_basic(0,0); + obj.create_ui(); end - create_ui_components(obj); - create_ui_basic(obj,xpos,ypos); - create_ui_crandall(obj,xpos,ypos); + create_ui(obj); cmds = left_click(obj,param); cmds = left_click_and_drag(obj,param); vals = snake(obj,image_c,image_x,image_y,x_old,y_old,x_new); diff --git a/cresis-toolbox/+imb/@picktool_snake/snake.m b/cresis-toolbox/+imb/@picktool_snake/snake.m index 20cfd35d..0524cad5 100644 --- a/cresis-toolbox/+imb/@picktool_snake/snake.m +++ b/cresis-toolbox/+imb/@picktool_snake/snake.m @@ -5,7 +5,7 @@ % Author: John Paden %% Get search range from tool param window -rbin_range_str = get(obj.top_panel.snake_range_TE,'String'); +rbin_range_str = get(obj.top_panel.snake_rangeTB,'String'); try % Assumes the value entered is a matlab expression that can be evaluated search_range = eval(sprintf('[%s]', rbin_range_str)); @@ -52,5 +52,3 @@ image_y,layer,'linear','extrap'); vals = interp1(image_x,vals,x_new,'linear'); - -return; diff --git a/cresis-toolbox/+imb/@picktool_stat/create_ui.m b/cresis-toolbox/+imb/@picktool_stat/create_ui.m new file mode 100644 index 00000000..06e85df6 --- /dev/null +++ b/cresis-toolbox/+imb/@picktool_stat/create_ui.m @@ -0,0 +1,122 @@ +function create_ui(obj) +% create_ui(obj) +% +% Create tool parameters user interface for imb.picker statistics tool +% +% Author: Dhagash Kapadia, John Paden + +set(obj.h_fig,'DockControls','off') +set(obj.h_fig,'NumberTitle','off'); +if strcmpi(class(obj.h_fig),'double') + set(obj.h_fig,'Name',sprintf('%d: %s tool parameters', obj.h_fig, obj.tool_name_title)); +else + set(obj.h_fig,'Name',sprintf('%d: %s tool parameters', obj.h_fig.Number, obj.tool_name_title)); +end +set(obj.h_fig,'ToolBar','none'); +set(obj.h_fig,'MenuBar','none'); +set(obj.h_fig,'CloseRequestFcn',@obj.close_win); +set(obj.h_fig,'KeyPressFcn',@obj.key_press); +%set(obj.h_fig,'Resize','off'); +% set default position (changed when window accessed) +set(obj.h_fig,'Position',[0 0 obj.w obj.h]); + +%========================================================================== +% top panel +obj.panel.handle = uipanel('Parent',obj.h_fig); +set(obj.panel.handle,'HighlightColor',[0.8 0.8 0.8]); +set(obj.panel.handle,'ShadowColor',[0.6 0.6 0.6]); +%set(obj.panel.handle,'visible','off'); + +%-------------------------------------- +% table +obj.table.ui=obj.h_fig; + +row = 1; col = 1; +obj.table.handles{row,col} = obj.panel.handle; +obj.table.width(row,col) = inf; +obj.table.height(row,col) = inf; +obj.table.width_margin(row,col) = 0; +obj.table.height_margin(row,col) = 0; + +clear row col +table_draw(obj.table); + +%----Stat Options label +obj.panel.stat_mode_label = uicontrol('Parent',obj.h_fig); +set(obj.panel.stat_mode_label,'Style','text'); +set(obj.panel.stat_mode_label,'String',sprintf('Stat Options:')); +set(obj.panel.stat_mode_label,'FontSize',10) +set(obj.panel.stat_mode_label,'TooltipString','Choose the action to be taken on the selected image pixels.'); + +%----Stat Options pulldown menu +obj.panel.stat_modePM = uicontrol('Parent',obj.h_fig); +set(obj.panel.stat_modePM,'Style','popupmenu'); +set(obj.panel.stat_modePM,'String',... + {'Overall Statistics', 'Histogram', 'Line by Line Statistics'}); +set(obj.panel.stat_modePM,'FontSize',10); +set(obj.panel.stat_modePM,'TooltipString','Choose the action to be taken on the selected image pixels.'); + +%----Close Stat Window label +obj.panel.close_stat_window_label = uicontrol('Parent',obj.h_fig); +set(obj.panel.close_stat_window_label,'Style','text'); +set(obj.panel.close_stat_window_label,'String',sprintf('Stat Options:')); +set(obj.panel.close_stat_window_label,'FontSize',10); +set(obj.panel.close_stat_window_label,'TooltipString','Choose the action to be taken on the selected image pixels.'); + +%----Close Stat Window pushbutton +obj.panel.close_stat_windowPB = uicontrol('Parent',obj.h_fig); +set(obj.panel.close_stat_windowPB,'Style','PushButton'); +set(obj.panel.close_stat_windowPB,'String','Close Stat Tool Windows'); +set(obj.panel.close_stat_windowPB,'FontSize',10); +set(obj.panel.close_stat_windowPB,'TooltipString','Closes all the windows that this stat window has opened.'); +set(obj.panel.close_stat_windowPB,'Callback',@obj.close_stat_windows); + +%--------------------------------------------------------------------------------------------- +% set up top panel table +obj.panel.table.ui=obj.panel.handle; +obj.panel.table.width_margin = nan(30,30); % Just make these bigger than they have to be +obj.panel.table.height_margin = nan(30,30); +obj.panel.table.false_width = nan(30,30); +obj.panel.table.false_height = nan(30,30); +obj.panel.table.offset = [0 0]; + +row = 0; + +row = row+1; col = 1; +obj.panel.table.handles{row,col} = obj.panel.stat_mode_label; +obj.panel.table.width(row,col) = 100; +obj.panel.table.height(row,col) = 20; +obj.panel.table.width_margin(row,col) = 0; +obj.panel.table.height_margin(row,col) = 3; + +col = 2; +obj.panel.table.handles{row,col} = obj.panel.stat_modePM; +obj.panel.table.width(row,col) = inf; +obj.panel.table.height(row,col) = 20; +obj.panel.table.width_margin(row,col) = 0; +obj.panel.table.height_margin(row,col) = 0; + +row = row+1; col = 1; +obj.panel.table.handles{row,col} = obj.panel.close_stat_window_label; +obj.panel.table.width(row,col) = 100; +obj.panel.table.height(row,col) = 20; +obj.panel.table.width_margin(row,col) = 0; +obj.panel.table.height_margin(row,col) = 3; + +col = 2; +obj.panel.table.handles{row,col} = obj.panel.close_stat_windowPB; +obj.panel.table.width(row,col) = inf; +obj.panel.table.height(row,col) = 25; +obj.panel.table.width_margin(row,col) = 0; +obj.panel.table.height_margin(row,col) = 0; + +% Add spacer to fill window +row = row+1; col = 1; +obj.panel.table.handles{row,col} = []; +obj.panel.table.width(row,col) = inf; +obj.panel.table.height(row,col) = inf; +obj.panel.table.width_margin(row,col)= 1.5; +obj.panel.table.height_margin(row,col)=1.5; + +clear row col +table_draw(obj.panel.table); diff --git a/cresis-toolbox/+imb/@picktool_stat/left_click.m b/cresis-toolbox/+imb/@picktool_stat/left_click.m new file mode 100644 index 00000000..76bb7fb3 --- /dev/null +++ b/cresis-toolbox/+imb/@picktool_stat/left_click.m @@ -0,0 +1,55 @@ +function cmds = left_click(obj,param) +% cmds = left_click(obj,param) +% +% imb.picker statistics tool +% +% Does not return any commands. +% +% Author: Dhagash Kapadia, John Paden + +% General setup +image_x = param.image_x; +image_y = param.image_y; + +x = param.x; +y = param.y; +cmds = []; + +% Find x and y limits +xlims = sort(image_x([1 end])); +ylims = sort(image_y([1 end])); + +% Check to make sure selection box is valid +if all(x < xlims(1)) || all(x > xlims(2)) + % button down and button up outside x-axis limits so ignore + return +end + +% Clip selection point to image limits +x(xxlims(2)) = xlims(2); +y(yylims(2)) = ylims(2); + +% Find image indices +x_idxs = sort(interp1(image_x,1:length(image_x),x,'nearest','extrap')); +y_idxs = sort(interp1(image_y,1:length(image_y),y,'nearest','extrap')); + +% Create frame ID string +cur_frm = num2str(find(param.echowin.eg.image_gps_time(x_idxs(1)) >= param.echowin.eg.start_gps_time,1,'last')); +if isempty(cur_frm) + cur_frm = num2str(1); +end +frm_str = strcat(param.echowin.eg.cur_sel.day_seg,'_',cur_frm); + +% Compute statistics for range line and value of selected point +data = param.image_c(:,x_idxs); +frame_stat_str = sprintf('Frame_ID \t%s\n', frm_str); +x_lim_str = sprintf('x_value \t%g\n', x); +y_lim_str = sprintf('y_value \t%g\n', y); +cur_val_str = sprintf('Cur_value \t%g\n',data(y_idxs)); +max_str = sprintf('Min_value \t%g\n',nanmax(data)); +min_str = sprintf('Max_value \t%g\n',nanmin(data)); +mean_str = sprintf('Mean_value \t%g\n',10*log10(nanmean(10.^(data./10)))); +median_str = sprintf('Median_value\t%g\n',nanmedian(data)); +fprintf([frame_stat_str x_lim_str y_lim_str cur_val_str max_str min_str mean_str median_str]); diff --git a/cresis-toolbox/+imb/@picktool_stat/left_click_and_drag.m b/cresis-toolbox/+imb/@picktool_stat/left_click_and_drag.m new file mode 100644 index 00000000..c2a0e29f --- /dev/null +++ b/cresis-toolbox/+imb/@picktool_stat/left_click_and_drag.m @@ -0,0 +1,89 @@ +function cmds = left_click_and_drag(obj,param) +% cmds = left_click_and_drag(obj,param) +% +% imb.picker statistics tool +% +% Does not return any commands. +% +% Author: Dhagash Kapadia, John Paden + +% General setup +image_x = param.image_x; +image_y = param.image_y; + +x = param.x; +y = param.y; +cmds = []; + +% Get Tool Params statistics method +stat_idx = get(obj.panel.stat_modePM,'Value'); + +% Find x and y limits +xlims = sort(image_x([1 end])); +ylims = sort(image_y([1 end])); + +% Check to make sure selection box is valid +if all(x < xlims(1)) || all(x > xlims(2)) + % button down and button up outside x-axis limits so ignore + return +end + +% Clip selection box to image limits +x(xxlims(2)) = xlims(2); +y(yylims(2)) = ylims(2); + +% Find image indices +x_idxs = sort(interp1(image_x,1:length(image_x),x,'nearest','extrap')); +y_idxs = sort(interp1(image_y,1:length(image_y),y,'nearest','extrap')); + +% Create frame ID string +cur_frm = num2str(find(param.echowin.eg.image_gps_time(x_idxs(1)) >= param.echowin.eg.start_gps_time,1,'last')); +if isempty(cur_frm) + cur_frm = num2str(1); +end +frm_str = strcat(param.echowin.eg.cur_sel.day_seg,'_',cur_frm); + +% Extract the region of interest +data = param.image_c(y_idxs(1):y_idxs(end),x_idxs(1):x_idxs(2)); + +% Run the selected Tool Params statistics method selected by user +if stat_idx == 1 + % Overall Statistics + data = data(:); + frame_stat_str = sprintf('Frame_ID \t%s\n', frm_str); + x_lim_str = sprintf('x_limits \t%g\t%g\n', x); + y_lim_str = sprintf('y_limits \t%g\t%g\n', y); + max_str = sprintf('Min_value \t%g\n',nanmax(data)); + min_str = sprintf('Max_value \t%g\n',nanmin(data)); + mean_str = sprintf('Mean_value \t%g\n',10*log10(nanmean(10.^(data./10)))); + median_str = sprintf('Median_value\t%g\n',nanmedian(data)); + fprintf([frame_stat_str x_lim_str y_lim_str max_str min_str mean_str median_str]); + +elseif stat_idx == 2 + % Histogram + obj.h_fig_stat(end+1) = figure; + clf(obj.h_fig_stat(end)); + h_axes(1) = axes('parent',obj.h_fig_stat(end)); + histogram(h_axes(1),data,max(round(0.01*numel(data)),10)); + title(h_axes(1),sprintf('%s x %g-%g y %g-%g',frm_str,x(1),x(2),y(1),y(2)),'interpreter','none'); + xlabel('Relative power (dB)'); + ylabel(sprintf('Frequency (counts out of %g)',numel(data))); + +elseif stat_idx == 3 + % Line by Line Statistics + obj.h_fig_stat(end+1) = figure; + clf(obj.h_fig_stat(end)); + h_axes(1) = axes('parent',obj.h_fig_stat(end)); + rlines = image_x(x_idxs(1):x_idxs(2)); + plot(h_axes(1),rlines,nanmax(data)); + hold(h_axes(1),'on') + plot(h_axes(1),rlines,nanmin(data)); + plot(h_axes(1),rlines,10*log10(nanmean(10.^(data./10)))); + plot(h_axes(1),rlines,nanmedian(data)); + xlabel(param.echowin.eg.x_label) + ylabel('Relative power (dB)') + title(h_axes(1),sprintf('%s x %g-%g y %g-%g',frm_str,x(1),x(2),y(1),y(2)),'interpreter','none'); + legend('Max', 'Min', 'Mean', 'Median') +end diff --git a/cresis-toolbox/+imb/@picktool_stat/picktool_stat.m b/cresis-toolbox/+imb/@picktool_stat/picktool_stat.m new file mode 100644 index 00000000..e8425b78 --- /dev/null +++ b/cresis-toolbox/+imb/@picktool_stat/picktool_stat.m @@ -0,0 +1,77 @@ +classdef picktool_stat < imb.picktool + % Print or plot statistics pick tool + % + % obj = picktool_stat(h_fig) + % Constructer. + % + % Left click: Prints the value at the point clicked + % Left click and drag: Action depends on the statistic selected: + % Line by Line Statistics + % Overall Statistics + % Histogram + % Right click: No action + % Right click and drag: Delete all points in range + % Scroll: Zooms in/out + % Ctrl + any click: Select layer + % Ctrl + any click and drag: Zoom + % Any double click: Nothing + % Ctrl + double click: Zoom reset + % + % Author: Dhagash Kapadia, John Paden + + properties + table + panel + + h_fig_stat + end + + properties (SetAccess = immutable, GetAccess = public) %constants + w % window's width + h % window's height + end + + methods + function obj = picktool_stat(h_fig) + %%% Pre Initialization %%% + % Any code not using output argument (obj) + if nargin == 0 || isempty(h_fig) + h_fig = figure('Visible','off'); + else + %figure(h_fig,'Visible','off'); + end + + %%% Post Initialization %%% + % Any code, including access to object + % This is the first tool initialized by a new echowin + obj.h_fig = h_fig; + obj.w = 280; + obj.h = 60; + obj.tool_name = 's(t)at'; + obj.tool_name_title = 'enter'; + obj.tool_shortcut = 't'; + obj.help_string = sprintf('Left click: Print pixel value.\nAlt + Left click and drag: Run selected statistics command. Change stat mode in Tool Parameters window (p).\n\n'); + + obj.h_fig_stat = []; + + obj.create_ui; + end + + create_ui(obj); + cmds = left_click(obj,h_image,layers,cur_layers,x,y,param); + cmds = left_click_and_drag(obj,h_layer,h_image,layers,cur_layers,x,y,param); + + function close_stat_windows(obj,hObj,event) + delete(obj.h_fig_stat); + obj.h_fig_stat = []; + end + + function delete(obj) + obj.close_stat_windows(); + end + + end + +end + + diff --git a/cresis-toolbox/+imb/@picktool_viterbi/create_ui.m b/cresis-toolbox/+imb/@picktool_viterbi/create_ui.m new file mode 100644 index 00000000..e86cb35b --- /dev/null +++ b/cresis-toolbox/+imb/@picktool_viterbi/create_ui.m @@ -0,0 +1,212 @@ +function create_ui(obj) +% create_ui(obj) +% +% Creates components for the window's UI when the echogram +% window is first opened. +% + +set(obj.h_fig,'DockControls','off') +set(obj.h_fig,'NumberTitle','off'); +if strcmpi(class(obj.h_fig),'double') + set(obj.h_fig,'Name',sprintf('%d: %s tool parameters', obj.h_fig, obj.tool_name_title)); +else + set(obj.h_fig,'Name',sprintf('%d: %s tool parameters', obj.h_fig.Number, obj.tool_name_title)); +end +set(obj.h_fig,'ToolBar','none'); +set(obj.h_fig,'MenuBar','none'); +set(obj.h_fig,'KeyPressFcn',@obj.key_press); +set(obj.h_fig,'CloseRequestFcn',@obj.close_win); + +set(obj.h_fig,'visible','off'); + +% set default position (changed when window accessed) +set(obj.h_fig,'Units','Pixels'); +set(obj.h_fig,'Position',[0 0 obj.w obj.h]); + +%========================================================================== +% top panel +obj.top_panel.handle = uipanel('Parent',obj.h_fig); +set(obj.top_panel.handle,'HighlightColor',[0.8 0.8 0.8]); +set(obj.top_panel.handle,'ShadowColor',[0.6 0.6 0.6]); + +%-------------------------------------- +% table +obj.table.ui=obj.h_fig; + +obj.table.handles{1,1} = obj.top_panel.handle; +obj.table.width(1,1) = inf; +obj.table.height(1,1) = inf; +obj.table.width_margin(1,1) = 0; +obj.table.height_margin(1,1) = 0; + +table_draw(obj.table); + +%========================================================================== +% top panel table contents + +%----insert range +tooltip = 'Viterbi will search +/- this many bins for the peak intensity on insert'; +obj.top_panel.insert_range_label = uicontrol('Parent',obj.top_panel.handle); +set(obj.top_panel.insert_range_label,'Style','text'); +set(obj.top_panel.insert_range_label,'String','Max point range:'); +set(obj.top_panel.insert_range_label,'TooltipString', tooltip); +%----insert pt search range box +obj.top_panel.insert_range_TE = uicontrol('Parent',obj.top_panel.handle); +set(obj.top_panel.insert_range_TE,'Style','edit'); +set(obj.top_panel.insert_range_TE,'String','5'); +set(obj.top_panel.insert_range_TE,'TooltipString', tooltip); + +%-----Horizontal Bounding +tooltip = sprintf(['How to bound the input and output of Viterbi horizontally

', ... + 'Entire Echogram: no bounding -- search entire echogram
', ... + 'Selection Box: Search within bounds of selection box
', ... + 'Extreme Groundtruth: Search between extreme groundtruth points within selection box']); +obj.top_panel.hori_bound_label = uicontrol('Parent',obj.top_panel.handle); +set(obj.top_panel.hori_bound_label,'Style','text'); +set(obj.top_panel.hori_bound_label,'String','Horizontal Bounding'); +set(obj.top_panel.hori_bound_label,'TooltipString', tooltip); +%----Horizontal Bounding popupmenu +obj.top_panel.hori_bound_PM = uicontrol('Parent',obj.top_panel.handle); +set(obj.top_panel.hori_bound_PM,'Style','popupmenu'); +set(obj.top_panel.hori_bound_PM,'String',{'Entire Echogram', 'Selection Box', 'Extreme Groundtruth'}); +set(obj.top_panel.hori_bound_PM,'Value', 2) +set(obj.top_panel.hori_bound_PM,'TooltipString', tooltip); + +%-----Vertical Bounding +tooltip = sprintf(['How to bound the input and output of Viterbi vertically

', ... + 'Entire Echogram: no bounding -- search entire echogram
', ... + 'Selection Box: Search within bounds of selection box
', ... + 'Layers: Search between top and bottom layers specified below.']); +obj.top_panel.vert_bound_label = uicontrol('Parent',obj.top_panel.handle); +set(obj.top_panel.vert_bound_label,'Style','text'); +set(obj.top_panel.vert_bound_label,'String','Vertical Bounding'); +set(obj.top_panel.vert_bound_label,'TooltipString', tooltip); +%----Vertical Bounding popupmenu +obj.top_panel.vert_bound_PM = uicontrol('Parent',obj.top_panel.handle); +set(obj.top_panel.vert_bound_PM,'Style','popupmenu'); +set(obj.top_panel.vert_bound_PM,'String',{'Entire Echogram', 'Selection Box', 'Layers'}); +set(obj.top_panel.vert_bound_PM,'Value', 2) +set(obj.top_panel.vert_bound_PM,'TooltipString', tooltip); + +%----top layer label +tooltip = 'Upper search bound specified as a layer number. "n" refers to selected layer number. Expressions accepted. Ignored if vertical bounds not set to ''Layers''.'; +obj.top_panel.top_layer_label = uicontrol('Parent',obj.top_panel.handle); +set(obj.top_panel.top_layer_label,'Style','text'); +set(obj.top_panel.top_layer_label,'String','Top Layer:'); +set(obj.top_panel.top_layer_label,'TooltipString', tooltip); +%----top layer box +obj.top_panel.top_layer_TE = uicontrol('Parent',obj.top_panel.handle); +set(obj.top_panel.top_layer_TE,'Style','edit'); +set(obj.top_panel.top_layer_TE,'String', 'n-1'); +set(obj.top_panel.top_layer_TE,'TooltipString', tooltip); + +%----bottom layer label +tooltip = 'Lower search bound specified as a layer number. "n" refers to selected layer number. Expressions accepted. Ignored if vertical bounds not set to ''Layers''.'; +obj.top_panel.bottom_layer_label = uicontrol('Parent',obj.top_panel.handle); +set(obj.top_panel.bottom_layer_label,'Style','text'); +set(obj.top_panel.bottom_layer_label,'String','Bottom Layer:'); +set(obj.top_panel.bottom_layer_label,'TooltipString', tooltip); +%----bottom layer box +obj.top_panel.bottom_layer_TE = uicontrol('Parent',obj.top_panel.handle); +set(obj.top_panel.bottom_layer_TE,'Style','edit'); +set(obj.top_panel.bottom_layer_TE,'String', 'n+1'); +set(obj.top_panel.bottom_layer_TE,'TooltipString', tooltip); + +%----layer guard label +tooltip = 'Restrict the vertical layer bounds by this many more bins. Only used for ''layer'' vertical bounding.'; +obj.top_panel.layer_guard_label = uicontrol('Parent',obj.top_panel.handle); +set(obj.top_panel.layer_guard_label,'Style','text'); +set(obj.top_panel.layer_guard_label,'String','Layer Guard:'); +set(obj.top_panel.layer_guard_label,'TooltipString', tooltip); +%----layer guard box +obj.top_panel.layer_guard_TE = uicontrol('Parent',obj.top_panel.handle); +set(obj.top_panel.layer_guard_TE,'Style','edit'); +set(obj.top_panel.layer_guard_TE,'String', '2'); +set(obj.top_panel.layer_guard_TE,'TooltipString', tooltip); + +%----along track weight label +tooltip = 'The weight by which to multiply the binary cost. Greater weight = smoother. This value should be larger (e.g. 1) for low resolution radars or smooth layers. This value should be smaller (e.g. 0.001) for fine resolution radars or where the layer is changing rapidly.'; +obj.top_panel.along_track_weight_label = uicontrol('Parent',obj.top_panel.handle); +set(obj.top_panel.along_track_weight_label,'Style','text'); +set(obj.top_panel.along_track_weight_label,'String','Smoothness weight:'); +set(obj.top_panel.along_track_weight_label,'TooltipString', tooltip); +%----along track weight box +obj.top_panel.along_track_weight_TE = uicontrol('Parent',obj.top_panel.handle); +set(obj.top_panel.along_track_weight_TE,'Style','edit'); +set(obj.top_panel.along_track_weight_TE,'String', '0.1'); +set(obj.top_panel.along_track_weight_TE,'TooltipString', tooltip); + +%----gt cutoff label +tooltip = 'Points must be chosen within this many rangebins of a ground truth point when present. -1 for any distance allowed.'; +obj.top_panel.ground_truth_cutoff_label = uicontrol('Parent',obj.top_panel.handle); +set(obj.top_panel.ground_truth_cutoff_label,'Style','text'); +set(obj.top_panel.ground_truth_cutoff_label,'String','Ground Truth Cutoff:'); +set(obj.top_panel.ground_truth_cutoff_label,'TooltipString', tooltip); +%----gt cutoff box +obj.top_panel.ground_truth_cutoff_TE = uicontrol('Parent',obj.top_panel.handle); +set(obj.top_panel.ground_truth_cutoff_TE,'Style','edit'); +set(obj.top_panel.ground_truth_cutoff_TE,'String', '0'); +set(obj.top_panel.ground_truth_cutoff_TE,'TooltipString', tooltip); +%% +%--------------------------------------------------------------------------------------------- +% set up top panel table +cols = 2; +rows = 8; % Just keep this larger or equal to actual number of rows. + +% set up top panel table +default_dimensions = NaN*zeros(rows,cols); +obj.top_panel.table.ui=obj.top_panel.handle; +obj.top_panel.table.width_margin = default_dimensions; +obj.top_panel.table.height_margin = default_dimensions; +obj.top_panel.table.false_width = default_dimensions; +obj.top_panel.table.false_height = default_dimensions; +obj.top_panel.table.offset = [0 0]; + +obj.top_panel.table.width = ones(rows, cols) * inf; +obj.top_panel.table.height = ones(rows, cols) * inf; +obj.top_panel.table.width_margin = ones(rows, cols) * 1.5; +obj.top_panel.table.height_margin = ones(rows, cols) * 1.5; + +row = 0; + +%% Insert Range +row = row + 1; +obj.top_panel.table.handles{row,1} = obj.top_panel.insert_range_label; +obj.top_panel.table.handles{row,2} = obj.top_panel.insert_range_TE; +%% Horizontal Bound +row = row + 1; +obj.top_panel.table.handles{row,1} = obj.top_panel.hori_bound_label; +obj.top_panel.table.handles{row,2} = obj.top_panel.hori_bound_PM; +%% Vertical Bound +row = row + 1; +obj.top_panel.table.handles{row,1} = obj.top_panel.vert_bound_label; +obj.top_panel.table.handles{row,2} = obj.top_panel.vert_bound_PM; +%% Top Layer +row = row + 1; +obj.top_panel.table.handles{row,1} = obj.top_panel.top_layer_label; +obj.top_panel.table.handles{row,2} = obj.top_panel.top_layer_TE; +%% Bottom Layer +row = row + 1; +obj.top_panel.table.handles{row,1} = obj.top_panel.bottom_layer_label; +obj.top_panel.table.handles{row,2} = obj.top_panel.bottom_layer_TE; +%% Layer Guard +row = row + 1; +obj.top_panel.table.handles{row,1} = obj.top_panel.layer_guard_label; +obj.top_panel.table.handles{row,2} = obj.top_panel.layer_guard_TE; +%% Along-track Weight +row = row + 1; +obj.top_panel.table.handles{row,1} = obj.top_panel.along_track_weight_label; +obj.top_panel.table.handles{row,2} = obj.top_panel.along_track_weight_TE; +%% gt cutoff +row = row + 1; +obj.top_panel.table.handles{row,1} = obj.top_panel.ground_truth_cutoff_label; +obj.top_panel.table.handles{row,2} = obj.top_panel.ground_truth_cutoff_TE; + +if row > rows + warning('Viterbi create_ui does not have default values for new rows. Update rows variable to match number of rows present.'); +end + +clear row cols + +% Draw table +table_draw(obj.top_panel.table); diff --git a/cresis-toolbox/+imb/@picktool_viterbi/create_ui_basic.m b/cresis-toolbox/+imb/@picktool_viterbi/create_ui_basic.m deleted file mode 100644 index a324f969..00000000 --- a/cresis-toolbox/+imb/@picktool_viterbi/create_ui_basic.m +++ /dev/null @@ -1,366 +0,0 @@ -function create_ui_basic(obj,xpos,ypos) - -% create_ui_basic(obj,xpos,ypos) -% -% Creates components for the HMM param window's UI when the Viterbi -% tool is selected. Plots the window at xpos,ypos. -% - -set(obj.h_fig,'visible','off'); - -% set default position (changed when window accessed) -set(obj.h_fig,'Units','Pixels'); -set(obj.h_fig,'Position',[xpos ypos obj.w obj.h]); -% show top panel -% set(obj.top_panel.handle,'visible','on'); -% set(obj.bottom_panel.handle,'visible','off'); - -if ~obj.first_time - figure(obj.h_fig); - clf; - obj.table = []; -end - -%========================================================================== -% top panel -obj.top_panel.handle = uipanel('Parent',obj.h_fig); -set(obj.top_panel.handle,'HighlightColor',[0.8 0.8 0.8]); -set(obj.top_panel.handle,'ShadowColor',[0.6 0.6 0.6]); -%set(obj.top_panel.handle,'visible','off'); - -%-------------------------------------- -% table -obj.table.ui=obj.h_fig; - -row = 1; col = 1; -obj.table.handles{row,col} = obj.top_panel.handle; -obj.table.width(row,col) = inf; -obj.table.height(row,col) = inf; -obj.table.width_margin(row,col) = 0; -obj.table.height_margin(row,col) = 0; - -clear row col -table_draw(obj.table); - -%============================================================================================ -% top panel table contents - -%----HMM tool list box -obj.top_panel.tool_PM = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.tool_PM,'Style','popupmenu'); -set(obj.top_panel.tool_PM,'String',{'basic'}); -set(obj.top_panel.tool_PM,'Value',1) -set(obj.top_panel.tool_PM,'Callback',@obj.toolPM_callback); - -%-----mode label -obj.top_panel.mode_label = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.mode_label,'Style','text'); -set(obj.top_panel.mode_label,'String','Mode'); -%----insert range -obj.top_panel.insert_range_label = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.insert_range_label,'Style','text'); -set(obj.top_panel.insert_range_label,'String','Max point range:'); -%----insert pt search range box -obj.top_panel.insert_range_TE = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.insert_range_TE,'Style','edit'); -set(obj.top_panel.insert_range_TE,'String',obj.in_rng_sv); -%----Viterbi search range name -obj.top_panel.Viterbi_range_label = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.Viterbi_range_label,'Style','text'); -set(obj.top_panel.Viterbi_range_label,'String','HMM detection range:'); -%----Viterbi search range box -obj.top_panel.Viterbi_range_TE = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.Viterbi_range_TE,'Style','edit'); -set(obj.top_panel.Viterbi_range_TE,'String',obj.sn_rng_sv); -%----reinterp mode enable label -obj.top_panel.reinterp_mode_label = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.reinterp_mode_label,'Style','text'); -set(obj.top_panel.reinterp_mode_label,'String','Reinterpolation (beta):'); -%----reinterp mode enable cbox -obj.top_panel.reinterp_mode_cbox = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.reinterp_mode_cbox,'Style','checkbox'); -%----column restriction label -obj.top_panel.column_restriction_label = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.column_restriction_label,'Style','text'); -set(obj.top_panel.column_restriction_label,'String','Column tracking restriction:'); -%----column restriction cbox -obj.top_panel.column_restriction_cbox = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.column_restriction_cbox,'Style','checkbox'); -set(obj.top_panel.column_restriction_cbox,'Value', 1); -%----top suppression label -obj.top_panel.top_sup_label = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.top_sup_label,'Style','text'); -set(obj.top_panel.top_sup_label,'String',sprintf('Top\nsuppression:')); -%----top suppression cbox -obj.top_panel.top_sup_cbox = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.top_sup_cbox,'Style','checkbox'); -set(obj.top_panel.top_sup_cbox,'Value', 1); -%----multiple suppression label -obj.top_panel.mult_sup_label = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.mult_sup_label,'Style','text'); -set(obj.top_panel.mult_sup_label,'String','Multiple suppression:'); -%----multiple suppression cbox -obj.top_panel.mult_sup_cbox = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.mult_sup_cbox,'Style','checkbox'); -set(obj.top_panel.mult_sup_cbox,'Value', 1); -%----quality output label -obj.top_panel.quality_output_label = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.quality_output_label,'Style','text'); -set(obj.top_panel.quality_output_label,'String',sprintf('Quality\noutput:')); -%----quality output cbox -obj.top_panel.quality_output_cbox = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.quality_output_cbox,'Style','checkbox'); -set(obj.top_panel.quality_output_cbox,'Value', 0); -%----quality threshold -obj.top_panel.quality_threshold_label = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.quality_threshold_label,'Style','text'); -set(obj.top_panel.quality_threshold_label,'String',sprintf('Quality\nthreshold:')); -%----quality threshold box -obj.top_panel.quality_threshold_TE = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.quality_threshold_TE,'Style','edit'); -set(obj.top_panel.quality_threshold_TE,'String','-20'); -%----smoothness weight label -obj.top_panel.smoothness_weight_label = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.smoothness_weight_label,'Style','text'); -set(obj.top_panel.smoothness_weight_label,'String',sprintf('Smoothness\nweight:')); -%----smoothness weight box -obj.top_panel.smoothness_weight_TE = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.smoothness_weight_TE,'Style','edit'); -set(obj.top_panel.smoothness_weight_TE,'String','3'); -%----smoothness variance label -obj.top_panel.smoothness_variance_label = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.smoothness_variance_label,'Style','text'); -set(obj.top_panel.smoothness_variance_label,'String',sprintf('Smoothness\nvariance:')); -%----smoothness variance box -obj.top_panel.smoothness_variance_TE = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.smoothness_variance_TE,'Style','edit'); -set(obj.top_panel.smoothness_variance_TE,'String','Inf'); -%----repulsion label -obj.top_panel.repulsion_label = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.repulsion_label,'Style','text'); -set(obj.top_panel.repulsion_label,'String',sprintf('Repulsion:')); -%----repulsion box -obj.top_panel.repulsion_TE = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.repulsion_TE,'Style','edit'); -set(obj.top_panel.repulsion_TE,'String','150000'); -%----ice bin threshold label -obj.top_panel.icebinthr_label = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.icebinthr_label,'Style','text'); -set(obj.top_panel.icebinthr_label,'String',sprintf('Ice bin\nthreshold:')); -%----ice bin threshold box -obj.top_panel.icebinthr_TE = uicontrol('Parent',obj.top_panel.handle); -set(obj.top_panel.icebinthr_TE,'Style','edit'); -set(obj.top_panel.icebinthr_TE,'String','10'); -%% -%--------------------------------------------------------------------------------------------- -% set up top panel table -obj.top_panel.table.ui=obj.top_panel.handle; -obj.top_panel.table.width_margin = NaN*zeros(30,30); % Just make these bigger than they have to be -obj.top_panel.table.height_margin = NaN*zeros(30,30); -obj.top_panel.table.false_width = NaN*zeros(30,30); -obj.top_panel.table.false_height = NaN*zeros(30,30); -obj.top_panel.table.offset = [0 0]; - -row = 1; col = 1; -obj.top_panel.table.handles{row,col} = obj.top_panel.mode_label; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -row = 1; col = 2; -obj.top_panel.table.handles{row,col} = obj.top_panel.tool_PM; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -row = 2; col = 1; -obj.top_panel.table.handles{row,col} = obj.top_panel.insert_range_label; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -row = 2; col = 2; -obj.top_panel.table.handles{row,col} = obj.top_panel.insert_range_TE; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -row = 3; col = 1; -obj.top_panel.table.handles{row,col} = obj.top_panel.Viterbi_range_label; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -row = 3; col = 2; -obj.top_panel.table.handles{row,col} = obj.top_panel.Viterbi_range_TE; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -row = 4; col = 1; -obj.top_panel.table.handles{row,col} = obj.top_panel.reinterp_mode_label; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -row = 4; col = 2; -obj.top_panel.table.handles{row,col} = obj.top_panel.reinterp_mode_cbox; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -%% Column restriction -row = 5; col = 1; -obj.top_panel.table.handles{row,col} = obj.top_panel.column_restriction_label; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -row = 5; col = 2; -obj.top_panel.table.handles{row,col} = obj.top_panel.column_restriction_cbox; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -%% Top suppression -row = 6; col = 1; -obj.top_panel.table.handles{row,col} = obj.top_panel.top_sup_label; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -row = 6; col = 2; -obj.top_panel.table.handles{row,col} = obj.top_panel.top_sup_cbox; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -%% Multiple suppression -row = 7; col = 1; -obj.top_panel.table.handles{row,col} = obj.top_panel.mult_sup_label; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -row = 7; col = 2; -obj.top_panel.table.handles{row,col} = obj.top_panel.mult_sup_cbox; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -%% Quality output -row = 8; col = 1; -obj.top_panel.table.handles{row,col} = obj.top_panel.quality_output_label; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -row = 8; col = 2; -obj.top_panel.table.handles{row,col} = obj.top_panel.quality_output_cbox; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -%% Quality threshold -row = 9; col = 1; -obj.top_panel.table.handles{row,col} = obj.top_panel.quality_threshold_label; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -row = 9; col = 2; -obj.top_panel.table.handles{row,col} = obj.top_panel.quality_threshold_TE; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -%% Smoothness weight -row = 10; col = 1; -obj.top_panel.table.handles{row,col} = obj.top_panel.smoothness_weight_label; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -row = 10; col = 2; -obj.top_panel.table.handles{row,col} = obj.top_panel.smoothness_weight_TE; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -%% Smoothness variance -row = 11; col = 1; -obj.top_panel.table.handles{row,col} = obj.top_panel.smoothness_variance_label; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -row = 11; col = 2; -obj.top_panel.table.handles{row,col} = obj.top_panel.smoothness_variance_TE; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -%% Repulsion -row = 12; col = 1; -obj.top_panel.table.handles{row,col} = obj.top_panel.repulsion_label; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -row = 12; col = 2; -obj.top_panel.table.handles{row,col} = obj.top_panel.repulsion_TE; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -%% Ice bin threshold -row = 13; col = 1; -obj.top_panel.table.handles{row,col} = obj.top_panel.icebinthr_label; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -row = 13; col = 2; -obj.top_panel.table.handles{row,col} = obj.top_panel.icebinthr_TE; -obj.top_panel.table.width(row,col) = inf; -obj.top_panel.table.height(row,col) = inf; -obj.top_panel.table.width_margin(row,col)= 1.5; -obj.top_panel.table.height_margin(row,col)=1.5; - -clear row col - -% Draw table -table_draw(obj.top_panel.table); - -if obj.first_time - obj.first_time = false; -else - set(obj.h_fig,'visible','on'); -end - -return; \ No newline at end of file diff --git a/cresis-toolbox/+imb/@picktool_viterbi/create_ui_components.m b/cresis-toolbox/+imb/@picktool_viterbi/create_ui_components.m deleted file mode 100644 index e473af20..00000000 --- a/cresis-toolbox/+imb/@picktool_viterbi/create_ui_components.m +++ /dev/null @@ -1,23 +0,0 @@ -function create_ui_components(obj) - -% create_ui_components(obj) -% -% Creates components for the window's UI when the echogram -% window is first opened. -% - -set(obj.h_fig,'DockControls','off') -set(obj.h_fig,'NumberTitle','off'); -if strcmpi(class(obj.h_fig),'double') - set(obj.h_fig,'Name',sprintf('%d: %s tool parameters', obj.h_fig, obj.tool_name_title)); -else - set(obj.h_fig,'Name',sprintf('%d: %s tool parameters', obj.h_fig.Number, obj.tool_name_title)); -end -set(obj.h_fig,'ToolBar','none'); -set(obj.h_fig,'MenuBar','none'); -set(obj.h_fig,'KeyPressFcn',@obj.key_press); -set(obj.h_fig,'CloseRequestFcn',@obj.close_win); -%set(obj.h_fig,'Resize','off'); - -return - diff --git a/cresis-toolbox/+imb/@picktool_viterbi/left_click_and_drag.m b/cresis-toolbox/+imb/@picktool_viterbi/left_click_and_drag.m index 5bc1d23f..0da9d9d4 100644 --- a/cresis-toolbox/+imb/@picktool_viterbi/left_click_and_drag.m +++ b/cresis-toolbox/+imb/@picktool_viterbi/left_click_and_drag.m @@ -6,6 +6,8 @@ % Compile with % mex -largeArrayDims viterbi.cpp +physical_constants; + image_x = param.image_x; image_y = param.image_y; image_c = param.image_c; @@ -19,191 +21,213 @@ param.x_bounds = 3; param.y_bounds = 1; -tool_idx = get(obj.top_panel.tool_PM,'Value'); -if tool_idx == 1 - - %========================================================================= - - for layer_idx = 1:length(cur_layers) - cur_layer = cur_layers(layer_idx); - - [manual_idxs,auto_idxs,point_idxs] = find_matching_pnts(obj,param,cur_layer); - - if length(manual_idxs) < 1 - warning('Insufficient points to track'); - continue; - elseif ~isempty(auto_idxs) - - % Nx: number of along track records/range lines - Nx = length(image_x); - custom_data.mu = [11.2575 11.3748 11.4393 11.4555 11.4323 11.3666 11.2668 11.1332 10.9900 10.8484 10.6916]; - custom_data.sigma = [5.4171 5.2945 5.2187 5.1939 5.2174 5.3247 5.4643 5.6571 5.8428 6.0477 6.2935]; - - % Interpolate surface layer to match image x-axis coordinates - surf_bins = interp1(param.layer.x,param.layer.y{1},image_x); - % Interpolate surface layer y-axis units to image pixels - surf_bins = interp1(image_y, 1:length(image_y),surf_bins); - % Interpolate all non-finite values using surrounding data - surf_bins = interp_finite(surf_bins); - - % Match GT points with axis coordinates - gt = [param.layer.x(manual_idxs); interp1(image_y, 1:length(image_y),param.layer.y{cur_layer}(manual_idxs))]; - - % Echogram Parameters - viterbi_data = image_c; - bottom_bin = -1; - mask = inf * ones([1 Nx]); - egt_weight = -1; - slope = round(diff(surf_bins)); - bounds = []; - viterbi_weight = ones([1 Nx]); - mu_size = 31; - mu = log10(exp(-(-(mu_size-1)/2 : (mu_size-1)/2).^4/1)); - mu(mu<-30) = -30; - mu = mu - mean(mu); - sigma = sum(abs(mu))/10*ones(1,mu_size); - mask_dist = round(bwdist(mask == 0)); - - try - smooth_weight = str2double(obj.top_panel.smoothness_weight_TE.String); - catch ME - smooth_weight = 3; - end - - try - smooth_var = str2double(obj.top_panel.smoothness_variance_TE.String); - catch ME - smooth_var = Inf; - end - - try - repulsion = str2double(obj.top_panel.repulsion_TE.String); - catch ME - repulsion = 150000; - end - - try - ice_bin_thr = str2double(obj.top_panel.icebinthr_TE.String); - catch ME - ice_bin_thr = 10; - end - - %% Detrending - if 1 - % Along track filtering - viterbi_data = fir_dec(viterbi_data,ones(1,5)/5,1); - % Estimate noise level - noise_value = mean(mean(viterbi_data(end-80:end-60,:))); - % Estimate trend - trend = mean(viterbi_data,2); - trend(trend length(param.layer.y) || layer_num < 1) + if strcmp(vert_bound_selection, 'Layers') + warning('Layer %.0f not found. Defaulting to echogram bound.', layer_num); end + continue; end + layers_bins(layers_idx, :) = interp_layer(param.layer.y{layer_num}, param.layer.x, image_x, image_y); end -else - %%% Do nothing for now + + % Get Vertical Bounds + % upper_bounds: nearest row to radar + % lower_bounds: furthest row from radar + % upper_bounds <= lower_bounds or an error will occur in c++ call + if strcmp(vert_bound_selection, 'Entire Echogram') + upper_bounds = ones(1, Nx); + lower_bounds = ones(1, Nx)*length(image_y); + elseif strcmp(vert_bound_selection, 'Selection Box') + upper_bounds = ones(1, Nx)*interp1(image_y, 1:length(image_y),param.y(1), 'nearest', 'extrap'); + lower_bounds = ones(1, Nx)*interp1(image_y, 1:length(image_y),param.y(2), 'nearest', 'extrap'); + % Sort upper and lower bounds + if upper_bounds(1) > lower_bounds(1) + temp_bounds = lower_bounds; + lower_bounds = upper_bounds; + upper_bounds = temp_bounds; + clear temp_bounds; + end + elseif strcmp(vert_bound_selection, 'Layers') + upper_bounds = layers_bins(1, :) + layer_guard; + lower_bounds = layers_bins(2, :) - layer_guard; + + % Handle overlapped bounds + half_bounds = round((upper_bounds + lower_bounds) / 2); + invalid_mask = upper_bounds > lower_bounds; + upper_bounds(invalid_mask) = half_bounds(invalid_mask); + lower_bounds(invalid_mask) = half_bounds(invalid_mask); + end + + % Get horizontal bounds + hori_bound_selection = obj.top_panel.hori_bound_PM.Value; + hori_bound_selection = obj.top_panel.hori_bound_PM.String{hori_bound_selection}; + if strcmp(hori_bound_selection, 'Entire Echogram') + % Image points + hori_bounds = [1 Nx]; + % Layer points + hori_layer_idxs = 1:length(param.layer.x); + elseif strcmp(hori_bound_selection, 'Selection Box') + % Image points + hori_bounds = interp1(image_x, 1:length(image_x),param.x, 'nearest', 'extrap'); + % Layer points + hori_layer_idxs = find(param.layer.x >= param.x(1) & param.layer.x <= param.x(2)); + elseif strcmp(hori_bound_selection, 'Extreme Groundtruth') + % Image points + selected_gt_idxs = gt_idxs(gt_idxs >= param.x(1) & gt_idxs <= param.x(2)); + if length(selected_gt_idxs) < 2 + warning('At least two ground truth points must be selected when horizontal bounding option is set to track between the "extreme gt points". Cancelling.'); + return; + end + hori_bounds = selected_gt_idxs([1 end]); + % Layer points + hori_layer_idxs = find(param.layer.x >= image_x(selected_gt_idxs(1)) & param.layer.x <= image_x(selected_gt_idxs(end))); + end + hori_bounds(1) = max(hori_bounds(1), 1); + hori_bounds(end) = min(hori_bounds(end), Nx); + + bound_gt_idxs = gt_idxs(gt_idxs >= hori_bounds(1) & gt_idxs <= hori_bounds(end)); + if any(gt(bound_gt_idxs) < upper_bounds(bound_gt_idxs) | gt(bound_gt_idxs) > lower_bounds(bound_gt_idxs)) + warning('Groundtruth points outside vertical bounds. Cancelling.'); + return; + end + + upper_bounds(gt_idxs) = max([gt(gt_idxs) - gt_cutoff; upper_bounds(gt_idxs)]); + lower_bounds(gt_idxs) = min([gt(gt_idxs) + gt_cutoff; lower_bounds(gt_idxs)]); + + elevation = param.echowin.eg.elev; + vel_air = c/2; + vel_ice = c/(sqrt(er_ice)*2); + dt = param.echo_time(2) - param.echo_time(1); + along_track_slope = diff(elevation); + + yaxis_choice = get(param.echowin.left_panel.yaxisPM,'Value'); + if yaxis_choice == 1 % TWTT + drange = dt * vel_air; + along_track_slope = round(along_track_slope / drange); + elseif yaxis_choice == 2 % WGS_84 Elevation + drange = dt * vel_ice; + along_track_slope = round(along_track_slope / drange); + elseif yaxis_choice == 3 % Range + drange = dt * vel_ice; + along_track_slope = round(along_track_slope / drange); + elseif yaxis_choice == 4 % Range bin + drange = dt * vel_air; + along_track_slope = round(along_track_slope / drange); + elseif yaxis_choice == 5 % Surface flat + drange = dt * vel_ice; + along_track_slope(:) = 0; + end + + % Crop input data to horizontal bounds + viterbi_data = viterbi_data(:,hori_bounds(1):hori_bounds(end)); + along_track_slope = along_track_slope(:,hori_bounds(1):hori_bounds(end)-1); + upper_bounds = upper_bounds(:,hori_bounds(1):hori_bounds(end)); + lower_bounds = lower_bounds(:,hori_bounds(1):hori_bounds(end)); + + % Handle NaNs and negative or too large bounds + upper_bounds(~isfinite(upper_bounds) | upper_bounds < 1) = 1; + upper_bounds(upper_bounds > size(viterbi_data, 1)) = size(viterbi_data, 1); + lower_bounds(lower_bounds < 1) = 1; + lower_bounds(~isfinite(lower_bounds) | lower_bounds > size(viterbi_data, 1)) = size(viterbi_data, 1); + + % Echogram image normalization + viterbi_data(~isfinite(viterbi_data)) = NaN; + viterbi_data = echo_norm(viterbi_data,struct('scale',[-40 90])); + viterbi_data(~isfinite(viterbi_data)) = -1e4; + + viterbi_timer = tic; + y_new = tomo.viterbi2(single(viterbi_data), along_track_slope, along_track_weight, upper_bounds, lower_bounds); + fprintf('Viterbi call took %.2f sec.\n', toc(viterbi_timer)); + + bounding_idxs = hori_bounds(1):hori_bounds(end); + + % Resample and interpolate y_new to match layer axes + y_new = interp1(1:length(image_y), image_y,y_new,'linear','extrap'); + y_new = interp1(image_x(bounding_idxs),y_new,param.layer.x(hori_layer_idxs),'linear', 'extrap'); + + cmds(end+1).undo_cmd = 'insert'; + cmds(end).undo_args = {cur_layer, hori_layer_idxs, ... + param.layer.y{cur_layer}(hori_layer_idxs), ... + param.layer.type{cur_layer}(hori_layer_idxs), ... + param.layer.qual{cur_layer}(hori_layer_idxs)}; + + cmds(end).redo_cmd = 'insert'; + type = param.layer.type{cur_layer}(hori_layer_idxs); + type(type ~= 1 | ~isfinite(param.layer.y{cur_layer}(hori_layer_idxs))) = 2; % Some day we will set different types based on the automated method used... + cmds(end).redo_args = {cur_layer, hori_layer_idxs, y_new, ... + type, param.cur_quality*ones(size(hori_layer_idxs))}; end -return +end + +function new_layer_bins = interp_layer(layer, layer_x, image_x, image_y) +% Interpolate layer to match image x-axis coordinates +new_layer_bins = interp1(layer_x, layer, image_x, 'nearest'); +% Interpolate layer y-axis units to image pixels +new_layer_bins = interp1(image_y, 1:length(image_y),new_layer_bins, 'nearest'); +end diff --git a/cresis-toolbox/+imb/@picktool_viterbi/picktool_viterbi.m b/cresis-toolbox/+imb/@picktool_viterbi/picktool_viterbi.m index 4a22fbe8..9c4fbbb4 100644 --- a/cresis-toolbox/+imb/@picktool_viterbi/picktool_viterbi.m +++ b/cresis-toolbox/+imb/@picktool_viterbi/picktool_viterbi.m @@ -4,20 +4,9 @@ top_panel bottom_panel table - last_tool; - last_layers; - last_range_gps; - first_time; - cur_mode; - - % properties to preserve tool values across deletions - in_rng_sv; - sn_rng_sv; - top_sm_sv; - bot_sm_sv; - top_pk_sv; - bot_pk_sv; - rep_sv; + last_tool + last_layers + last_range_gps end properties (SetAccess = immutable, GetAccess = public) %constants @@ -27,16 +16,12 @@ methods function obj = picktool_viterbi(h_fig) - %%% Pre Initialization %%% - % Any code not using output argument (obj) if nargin == 0 || isempty(h_fig) h_fig = figure('Visible','off'); else figure(h_fig,'Visible','off'); end - %%% Post Initialization %%% - % Any code, including access to object obj.h_fig = h_fig; obj.tool_name = '(v)iterbi'; obj.tool_name_title = 'viterbi'; @@ -45,22 +30,10 @@ obj.bottom_panel = []; obj.top_panel = []; obj.table = []; - obj.w = 200; - obj.h = 475; - obj.first_time = true; - - obj.in_rng_sv = 5; - obj.sn_rng_sv = 5; - obj.top_sm_sv = .5; - obj.bot_sm_sv = .5; - obj.top_pk_sv = .5; - obj.bot_pk_sv = .5; - obj.rep_sv = .5; - obj.cur_mode = 1; + obj.w = 350; + obj.h = 300; - obj.create_ui_components(); - obj.create_ui_basic(0,0); - + obj.create_ui(); end function cmd = apply_PB_callback(obj,sb,slices) @@ -191,147 +164,147 @@ function set_custom_data(obj,custom_data) obj.custom_data.sigma = mean(custom_data.sigma); end - function create_option_ui(obj) - obj.h_fig = figure('Visible','off','DockControls','off', ... - 'NumberTitle','off','ToolBar','none','MenuBar','none','Resize','off'); - if strcmpi(class(obj.h_fig),'double') - set(obj.h_fig,'Name',sprintf('%d: viterbi tool prefs',obj.h_fig)); - else - set(obj.h_fig,'Name',sprintf('%d: viterbi tool prefs',obj.h_fig.Number)); - end - set(obj.h_fig,'CloseRequestFcn',@obj.close_win); - pos = get(obj.h_fig,'Position'); - pos(3) = 200; - pos(4) = 130; - set(obj.h_fig,'Position',pos); - - % Slice range - obj.gui.slice_rangeTXT = uicontrol('Style','text','string','Slice range'); - set(obj.gui.slice_rangeTXT,'TooltipString','Enter a vector specifying relative range in slices. E.g. "-5:5".'); - - obj.gui.slice_rangeLE = uicontrol('parent',obj.h_fig); - set(obj.gui.slice_rangeLE,'style','edit') - set(obj.gui.slice_rangeLE,'string','0') - set(obj.gui.slice_rangeLE,'TooltipString','Enter a vector specifying relative range in slices. E.g. "-5:5".'); - - % Threshold - obj.gui.thresholdTXT = uicontrol('Style','text','string','Threshold'); - set(obj.gui.thresholdTXT,'TooltipString','Specify an image threshold.'); - - obj.gui.thresholdLE = uicontrol('parent',obj.h_fig); - set(obj.gui.thresholdLE,'style','edit') - set(obj.gui.thresholdLE,'string','13.5') - set(obj.gui.thresholdLE,'TooltipString','Specify an image threshold.'); - - % Ground Truth Weight - obj.gui.gt_weightTXT = uicontrol('Style','text','string','GT weight'); - set(obj.gui.gt_weightTXT,'TooltipString','Specify weighting of ground truth.'); - - obj.gui.gt_weightLE = uicontrol('parent',obj.h_fig); - set(obj.gui.gt_weightLE,'style','edit') - set(obj.gui.gt_weightLE,'string','10') - set(obj.gui.gt_weightLE,'TooltipString','Specify weighting of ground truth.'); - - obj.gui.slopeTXT = uicontrol('Style','text','string','Slope'); - obj.gui.slopeLE = uicontrol('parent',obj.h_fig); - set(obj.gui.slopeLE,'style','edit') - set(obj.gui.slopeLE,'string','zeros(1,63)') - - % Select mask - obj.gui.select_maskCB = uicontrol('parent',obj.h_fig); - set(obj.gui.select_maskCB,'style','checkbox') - set(obj.gui.select_maskCB,'string','Select') - set(obj.gui.select_maskCB,'value',1) - set(obj.gui.select_maskCB,'TooltipString','Check to operate only on the selected region.'); - - % Previous ground truth - obj.gui.previousCB = uicontrol('parent',obj.h_fig); - set(obj.gui.previousCB,'style','checkbox') - set(obj.gui.previousCB,'string','Previous') - set(obj.gui.previousCB,'value',1) - set(obj.gui.previousCB,'TooltipString','Use previous slice as ground truth.'); - - % GUI container table - obj.gui.table.ui = obj.h_fig; - obj.gui.table.width_margin = NaN*zeros(30,30); % Just make these bigger than they have to be - obj.gui.table.height_margin = NaN*zeros(30,30); - obj.gui.table.false_width = NaN*zeros(30,30); - obj.gui.table.false_height = NaN*zeros(30,30); - obj.gui.table.offset = [0 0]; - row = 1; - col = 1; - obj.gui.table.handles{row,col} = obj.gui.slice_rangeTXT; - obj.gui.table.width(row,col) = 70; - obj.gui.table.height(row,col) = 20; - obj.gui.table.width_margin(row,col) = 1; - obj.gui.table.height_margin(row,col) = 1; - col = 2; - obj.gui.table.handles{row,col} = obj.gui.slice_rangeLE; - obj.gui.table.width(row,col) = inf; - obj.gui.table.height(row,col) = 20; - obj.gui.table.width_margin(row,col) = 1; - obj.gui.table.height_margin(row,col) = 1; - - row = row + 1; - col = 1; - obj.gui.table.handles{row,col} = obj.gui.thresholdTXT; - obj.gui.table.width(row,col) = 70; - obj.gui.table.height(row,col) = 20; - obj.gui.table.width_margin(row,col) = 1; - obj.gui.table.height_margin(row,col) = 1; - col = 2; - obj.gui.table.handles{row,col} = obj.gui.thresholdLE; - obj.gui.table.width(row,col) = inf; - obj.gui.table.height(row,col) = 20; - obj.gui.table.width_margin(row,col) = 1; - obj.gui.table.height_margin(row,col) = 1; - - - row = row + 1; - col = 1; - obj.gui.table.handles{row,col} = obj.gui.gt_weightTXT; - obj.gui.table.width(row,col) = 70; - obj.gui.table.height(row,col) = 20; - obj.gui.table.width_margin(row,col) = 1; - obj.gui.table.height_margin(row,col) = 1; - col = 2; - obj.gui.table.handles{row,col} = obj.gui.gt_weightLE; - obj.gui.table.width(row,col) = inf; - obj.gui.table.height(row,col) = 20; - obj.gui.table.width_margin(row,col) = 1; - obj.gui.table.height_margin(row,col) = 1; - - row = row + 1; - col = 1; - obj.gui.table.handles{row,col} = obj.gui.slopeTXT; - obj.gui.table.width(row,col) = 70; - obj.gui.table.height(row,col) = 20; - obj.gui.table.width_margin(row,col) = 1; - obj.gui.table.height_margin(row,col) = 1; - col = 2; - obj.gui.table.handles{row,col} = obj.gui.slopeLE; - obj.gui.table.width(row,col) = inf; - obj.gui.table.height(row,col) = 20; - obj.gui.table.width_margin(row,col) = 1; - obj.gui.table.height_margin(row,col) = 1; - - row = row + 1; - col = 1; - obj.gui.table.handles{row,col} = obj.gui.select_maskCB; - obj.gui.table.width(row,col) = inf; - obj.gui.table.height(row,col) = 20; - obj.gui.table.width_margin(row,col) = 1; - obj.gui.table.height_margin(row,col) = 1; - col = 2; - obj.gui.table.handles{row,col} = obj.gui.previousCB; - obj.gui.table.width(row,col) = inf; - obj.gui.table.height(row,col) = 20; - obj.gui.table.width_margin(row,col) = 1; - obj.gui.table.height_margin(row,col) = 1; - - clear row col - table_draw(obj.gui.table); - end +% function create_option_ui(obj) +% obj.h_fig = figure('Visible','off','DockControls','off', ... +% 'NumberTitle','off','ToolBar','none','MenuBar','none','Resize','off'); +% if strcmpi(class(obj.h_fig),'double') +% set(obj.h_fig,'Name',sprintf('%d: viterbi tool prefs',obj.h_fig)); +% else +% set(obj.h_fig,'Name',sprintf('%d: viterbi tool prefs',obj.h_fig.Number)); +% end +% set(obj.h_fig,'CloseRequestFcn',@obj.close_win); +% pos = get(obj.h_fig,'Position'); +% pos(3) = 200; +% pos(4) = 130; +% set(obj.h_fig,'Position',pos); +% +% % Slice range +% obj.gui.slice_rangeTXT = uicontrol('Style','text','string','Slice range'); +% set(obj.gui.slice_rangeTXT,'TooltipString','Enter a vector specifying relative range in slices. E.g. "-5:5".'); +% +% obj.gui.slice_rangeLE = uicontrol('parent',obj.h_fig); +% set(obj.gui.slice_rangeLE,'style','edit') +% set(obj.gui.slice_rangeLE,'string','0') +% set(obj.gui.slice_rangeLE,'TooltipString','Enter a vector specifying relative range in slices. E.g. "-5:5".'); +% +% % Threshold +% obj.gui.thresholdTXT = uicontrol('Style','text','string','Threshold'); +% set(obj.gui.thresholdTXT,'TooltipString','Specify an image threshold.'); +% +% obj.gui.thresholdLE = uicontrol('parent',obj.h_fig); +% set(obj.gui.thresholdLE,'style','edit') +% set(obj.gui.thresholdLE,'string','13.5') +% set(obj.gui.thresholdLE,'TooltipString','Specify an image threshold.'); +% +% % Ground Truth Weight +% obj.gui.gt_weightTXT = uicontrol('Style','text','string','GT weight'); +% set(obj.gui.gt_weightTXT,'TooltipString','Specify weighting of ground truth.'); +% +% obj.gui.gt_weightLE = uicontrol('parent',obj.h_fig); +% set(obj.gui.gt_weightLE,'style','edit') +% set(obj.gui.gt_weightLE,'string','10') +% set(obj.gui.gt_weightLE,'TooltipString','Specify weighting of ground truth.'); +% +% obj.gui.slopeTXT = uicontrol('Style','text','string','Slope'); +% obj.gui.slopeLE = uicontrol('parent',obj.h_fig); +% set(obj.gui.slopeLE,'style','edit') +% set(obj.gui.slopeLE,'string','zeros(1,63)') +% +% % Select mask +% obj.gui.select_maskCB = uicontrol('parent',obj.h_fig); +% set(obj.gui.select_maskCB,'style','checkbox') +% set(obj.gui.select_maskCB,'string','Select') +% set(obj.gui.select_maskCB,'value',1) +% set(obj.gui.select_maskCB,'TooltipString','Check to operate only on the selected region.'); +% +% % Previous ground truth +% obj.gui.previousCB = uicontrol('parent',obj.h_fig); +% set(obj.gui.previousCB,'style','checkbox') +% set(obj.gui.previousCB,'string','Previous') +% set(obj.gui.previousCB,'value',1) +% set(obj.gui.previousCB,'TooltipString','Use previous slice as ground truth.'); +% +% % GUI container table +% obj.gui.table.ui = obj.h_fig; +% obj.gui.table.width_margin = NaN*zeros(30,30); % Just make these bigger than they have to be +% obj.gui.table.height_margin = NaN*zeros(30,30); +% obj.gui.table.false_width = NaN*zeros(30,30); +% obj.gui.table.false_height = NaN*zeros(30,30); +% obj.gui.table.offset = [0 0]; +% row = 1; +% col = 1; +% obj.gui.table.handles{row,col} = obj.gui.slice_rangeTXT; +% obj.gui.table.width(row,col) = 70; +% obj.gui.table.height(row,col) = 20; +% obj.gui.table.width_margin(row,col) = 1; +% obj.gui.table.height_margin(row,col) = 1; +% col = 2; +% obj.gui.table.handles{row,col} = obj.gui.slice_rangeLE; +% obj.gui.table.width(row,col) = inf; +% obj.gui.table.height(row,col) = 20; +% obj.gui.table.width_margin(row,col) = 1; +% obj.gui.table.height_margin(row,col) = 1; +% +% row = row + 1; +% col = 1; +% obj.gui.table.handles{row,col} = obj.gui.thresholdTXT; +% obj.gui.table.width(row,col) = 70; +% obj.gui.table.height(row,col) = 20; +% obj.gui.table.width_margin(row,col) = 1; +% obj.gui.table.height_margin(row,col) = 1; +% col = 2; +% obj.gui.table.handles{row,col} = obj.gui.thresholdLE; +% obj.gui.table.width(row,col) = inf; +% obj.gui.table.height(row,col) = 20; +% obj.gui.table.width_margin(row,col) = 1; +% obj.gui.table.height_margin(row,col) = 1; +% +% +% row = row + 1; +% col = 1; +% obj.gui.table.handles{row,col} = obj.gui.gt_weightTXT; +% obj.gui.table.width(row,col) = 70; +% obj.gui.table.height(row,col) = 20; +% obj.gui.table.width_margin(row,col) = 1; +% obj.gui.table.height_margin(row,col) = 1; +% col = 2; +% obj.gui.table.handles{row,col} = obj.gui.gt_weightLE; +% obj.gui.table.width(row,col) = inf; +% obj.gui.table.height(row,col) = 20; +% obj.gui.table.width_margin(row,col) = 1; +% obj.gui.table.height_margin(row,col) = 1; +% +% row = row + 1; +% col = 1; +% obj.gui.table.handles{row,col} = obj.gui.slopeTXT; +% obj.gui.table.width(row,col) = 70; +% obj.gui.table.height(row,col) = 20; +% obj.gui.table.width_margin(row,col) = 1; +% obj.gui.table.height_margin(row,col) = 1; +% col = 2; +% obj.gui.table.handles{row,col} = obj.gui.slopeLE; +% obj.gui.table.width(row,col) = inf; +% obj.gui.table.height(row,col) = 20; +% obj.gui.table.width_margin(row,col) = 1; +% obj.gui.table.height_margin(row,col) = 1; +% +% row = row + 1; +% col = 1; +% obj.gui.table.handles{row,col} = obj.gui.select_maskCB; +% obj.gui.table.width(row,col) = inf; +% obj.gui.table.height(row,col) = 20; +% obj.gui.table.width_margin(row,col) = 1; +% obj.gui.table.height_margin(row,col) = 1; +% col = 2; +% obj.gui.table.handles{row,col} = obj.gui.previousCB; +% obj.gui.table.width(row,col) = inf; +% obj.gui.table.height(row,col) = 20; +% obj.gui.table.width_margin(row,col) = 1; +% obj.gui.table.height_margin(row,col) = 1; +% +% clear row col +% table_draw(obj.gui.table); +% end end diff --git a/cresis-toolbox/+imb/@prefwin/create_ui.m b/cresis-toolbox/+imb/@prefwin/create_ui.m index 5e99b26c..b8322000 100644 --- a/cresis-toolbox/+imb/@prefwin/create_ui.m +++ b/cresis-toolbox/+imb/@prefwin/create_ui.m @@ -11,47 +11,58 @@ function create_ui(obj) obj.ops.layers = []; obj.ops.layers.lyr_name = {}; -layer_sources = {'layerdata','Connect to OPS'}; - -flightlines = {'layerdata Flightlines','Connect to OPS'}; +if obj.map_toolbox + layer_sources = {'layerdata','Connect to OPS'}; + flightlines = {'tracks files Flightlines','Connect to OPS'}; +else + layer_sources = {'layerdata'}; + flightlines = {'tracks files Flightlines'}; +end -wms_maps = {'arctic:blank_map';'antarctic:blank_map';'arctic:google_map';'antarctic:google_map';'Connect to OPS'}; +if obj.map_toolbox + wms_maps = {'arctic:Blank Geodetic Map';'antarctic:Blank Geodetic Map';'arctic:Blank Stereographic Map';'antarctic:Blank Stereographic Map';'arctic:Google Map';'antarctic:Google Map';'Connect to OPS'}; +else + %error('Mapping toolbox is not available. Mapping toolbox must be available for projections and/or WMS service. Blank Geodetic Maps not supported yet.') + wms_maps = {'arctic:Blank Geodetic Map';'antarctic:Blank Geodetic Map'}; +end -%% Get System Info from LayerData +%% Load tracks files % ========================================================================= -layer_fn_dir = ct_filename_support(struct('radar_name','rds'),'layer',''); % Setting radar_name to rds is arbitrary -fprintf('Finding the season layerdata in %s\n', layer_fn_dir); -layer_fns = get_filenames(layer_fn_dir,'layer','','.mat'); + +% Create tracks file paths +% ------------------------------------------------------------------------- +track_fn_dir = ct_filename_support(struct('radar_name','rds'),'tracks',''); % Setting radar_name to rds is arbitrary +fprintf('Finding the csarp_support/tracks files in %s\n', track_fn_dir); +track_fns = get_filenames(track_fn_dir,'tracks','','.mat'); + +% Parse tracks filenames one at a time to populate +% obj.systems +% obj.seasons +% obj.locations +% ------------------------------------------------------------------------- valid_file_count = 0; -for layer_idx = 1:length(layer_fns) - layer_fn = layer_fns{layer_idx}; - % Parse layer_fn: layer_SYSTEM_SEASONNAME.mat - [~,layer_fn_name] = fileparts(layer_fn); - [token,remain] = strtok(layer_fn_name,'_'); - if strcmpi(token,'layer') - [token,remain] = strtok(remain,'_'); - [sys_token,remain] = strtok(remain,'_'); - if strcmpi(token,'arctic') - % sys_token: accum, kuband, rds, or snow string - % remain: _YYYY_LOCATION_PLATFORM string - obj.systems{end+1} = 'layerdata'; - obj.seasons{end+1} = sprintf('%s_%s',sys_token,remain(2:end)); - obj.locations{end+1} = 'arctic'; - valid_file_count = valid_file_count+1; - elseif strcmpi(token,'antarctic') +for track_idx = 1:length(track_fns) + track_fn = track_fns{track_idx}; + % Parse track_fn: layer_SYSTEM_SEASONNAME.mat + [~,track_fn_name] = fileparts(track_fn); + [file_type_str,remain] = strtok(track_fn_name,'_'); + if strcmpi(file_type_str,'tracks') + [location_str,remain] = strtok(remain,'_'); + [sys_str,remain] = strtok(remain,'_'); + if any(strcmpi(location_str,{'arctic','antarctic'})) % sys_token: accum, kuband, rds, or snow string % remain: _YYYY_LOCATION_PLATFORM string - obj.systems{end+1} = 'layerdata'; - obj.seasons{end+1} = sprintf('%s_%s',sys_token,remain(2:end)); - obj.locations{end+1} = 'antarctic'; + obj.systems{end+1} = 'tracks'; + obj.seasons{end+1} = sprintf('%s_%s',sys_str,remain(2:end)); + obj.locations{end+1} = location_str; valid_file_count = valid_file_count+1; end end end if valid_file_count == 0 - warning('No season layerdata files found. Use create_season_layerdata_files.m to create season layerdata files. Continuing without layerdata.'); + warning('No tracks files found. Use imb.create_track_files.m to create tracks files. Continuing without tracks files.'); else - fprintf(' Found %d season layerdata files.\n', valid_file_count); + fprintf(' Found %d tracks files.\n', valid_file_count); end %% Set Prefwin Figure @@ -78,17 +89,18 @@ function create_ui(obj) set(obj.h_gui.layerSourcePM,'Style','popupmenu'); set(obj.h_gui.layerSourcePM,'HorizontalAlignment','Center'); set(obj.h_gui.layerSourcePM,'FontName','fixed'); -set(obj.h_gui.layerSourcePM,'TooltipString','Available layer sources (select one)'); +set(obj.h_gui.layerSourcePM,'TooltipString','Layer two way travel time data source (select one)'); set(obj.h_gui.layerSourcePM,'Callback',@obj.layerSourcePM_callback); % layerdata sources pop up menu (populate later from preference window)%% obj.h_gui.layerDataSourcePM = popupmenu_edit(obj.h_fig,{'layer','CSARP_post/layer'}); -set(obj.h_gui.layerDataSourcePM.h_valuePM,'TooltipString','Available layerdata filepath sources (select one). Right click to add, edit, or delete entries.'); +set(obj.h_gui.layerDataSourcePM.h_valuePM,'TooltipString',sprintf('Available layerdata filepath sources (select one). Right click to add, edit, or delete entries.\nThe entry "layer" will look in the CSARP_layer directory.\nThe entry "CSARP_post / layer" will look in the CSARP_post / CSARP_layer directory.')); % Layer selection class (populate later from preference file) obj.h_gui.h_layers = selectionbox(obj.h_fig,'Layers',[],1); -set(obj.h_gui.h_layers.h_list_available,'TooltipString','Available layers (double or right click to select).'); -set(obj.h_gui.h_layers.h_list_selected,'TooltipString','Selected layers (double or right click to remove).'); +set(obj.h_gui.h_layers.h_text,'TooltipString','Select the layer two way travel time data source. If layerdata, then select the layerdata source. If OPS, then select layers to load.'); +set(obj.h_gui.h_layers.h_list_available,'TooltipString','Available OPS layers (double or right click to select).'); +set(obj.h_gui.h_layers.h_list_selected,'TooltipString','Selected OPS layers (double or right click to remove).'); obj.h_gui.h_layers.set_enable(false); uimenu(obj.h_gui.h_layers.h_list_availableCM, 'Label', 'New', 'Callback', @obj.layers_callback); @@ -98,13 +110,13 @@ function create_ui(obj) % Season selection class (populate later from preference file) obj.h_gui.h_seasons = selectionbox(obj.h_fig,'Seasons',[],1); -set(obj.h_gui.h_seasons.h_list_available,'TooltipString','Available seasons (double click or right click to select).'); -set(obj.h_gui.h_seasons.h_list_selected,'TooltipString','Selected seasons(double click or right click to remove).'); +set(obj.h_gui.h_seasons.h_list_available,'TooltipString','Available seasons for the chosen map (double click or right click to select).'); +set(obj.h_gui.h_seasons.h_list_selected,'TooltipString','Selected seasons for the chosen map (double click or right click to remove).'); % System list box label obj.h_gui.systemsText = uicontrol('Parent',obj.h_fig); set(obj.h_gui.systemsText,'Style','Text'); -set(obj.h_gui.systemsText,'String','Systems'); +set(obj.h_gui.systemsText,'String','Radar Systems'); % Source list box label obj.h_gui.sourceText = uicontrol('Parent',obj.h_fig); @@ -113,13 +125,13 @@ function create_ui(obj) % System list box obj.h_gui.systemsLB = uicontrol('Parent',obj.h_fig); -set(obj.h_gui.systemsLB,'String',{'layerdata'}); +set(obj.h_gui.systemsLB,'String',{'tracks'}); set(obj.h_gui.systemsLB,'Style','listbox'); set(obj.h_gui.systemsLB,'HorizontalAlignment','Center'); set(obj.h_gui.systemsLB,'FontName','fixed'); set(obj.h_gui.systemsLB,'Callback',@obj.systemsLB_callback); set(obj.h_gui.systemsLB,'Min',1); % One must always be selected -set(obj.h_gui.systemsLB,'TooltipString','Systems (choose one)'); +set(obj.h_gui.systemsLB,'TooltipString','Radar systems (choose one)'); % Source list box (populate later from preference file) obj.h_gui.sourceLB = uicontrol('Parent',obj.h_fig); @@ -311,17 +323,19 @@ function create_ui(obj) % Check to see if default parameters require OPS load_ops = false; -if ~strcmp(obj.default_params.system,'layerdata') - load_ops = true; -end -if strcmp(obj.default_params.layer_source,'OPS') - load_ops = true; -end -if strcmp(obj.default_params.flightlines(1:3),'OPS') - load_ops = true; -end -if all(~strcmp(obj.default_params.map_name,{'arctic:blank_map';'antarctic:blank_map';'arctic:google_map';'antarctic:google_map'})) - load_ops = true; +if obj.map_toolbox + if ~strcmp(obj.default_params.system,'tracks') + load_ops = true; + end + if strcmp(obj.default_params.layer_source,'OPS') + load_ops = true; + end + if strcmp(obj.default_params.flightlines(1:3),'OPS') + load_ops = true; + end + if all(~strcmp(obj.default_params.map_name,{'arctic:Blank Geodetic Map';'antarctic:Blank Geodetic Map';'arctic:Blank Stereographic Map';'antarctic:Blank Stereographic Map';'arctic:Google Map';'antarctic:Google Map'})) + load_ops = true; + end end if load_ops obj.ops_connect(); diff --git a/cresis-toolbox/+imb/@prefwin/layers_callback.m b/cresis-toolbox/+imb/@prefwin/layers_callback.m index c274c9c8..a3288a05 100644 --- a/cresis-toolbox/+imb/@prefwin/layers_callback.m +++ b/cresis-toolbox/+imb/@prefwin/layers_callback.m @@ -1,6 +1,10 @@ function layers_callback(obj,status,event) h_status = get(status); +if isfield(h_status,'Text') + % Handle missing label field in newer versions of Matlab (R2017b+) + h_status.Label = h_status.Text; +end if isfield(h_status,'Label') && strcmp(get(status,'Label'),'New') h_fig = figure('NumberTitle','off','Name','New Layer','DockControls','off','NumberTitle','off','ToolBar','none','MenuBar','none'); pos = get(h_fig,'Position'); diff --git a/cresis-toolbox/+imb/@prefwin/layers_callback_refresh.m b/cresis-toolbox/+imb/@prefwin/layers_callback_refresh.m index a3786a6a..07cd80ab 100644 --- a/cresis-toolbox/+imb/@prefwin/layers_callback_refresh.m +++ b/cresis-toolbox/+imb/@prefwin/layers_callback_refresh.m @@ -10,7 +10,7 @@ function layers_callback_refresh(obj) if ~isempty(system_name) %% layerdata systems - if strcmpi(system_name,'layerdata') + if strcmpi(system_name,'tracks') layer_sources = get(obj.h_gui.layerSourcePM,'String'); layer_source = layer_sources{get(obj.h_gui.layerSourcePM,'Value')}; if strcmp(layer_source,'OPS') @@ -42,15 +42,9 @@ function layers_callback_refresh(obj) end end - % Get the current list of selected layers - selectedString = obj.h_gui.h_layers.get_selected_strings(); - % Create the new list of layers for the system that is selected - obj.h_gui.h_layers.set_available(menuString); - - % Select all the old layers that were selected (this function - % automatically ignores layers that do not exist in the new list) - obj.h_gui.h_layers.set_selected(selectedString,true); + obj.h_gui.h_layers.set_list(sort(menuString)); + obj.h_gui.h_layers.set_selected({'standard:surface','standard:bottom'},true); set(obj.h_gui.h_layers.h_list_availableCM.Children([2 3 4]),'Enable','off'); end @@ -65,16 +59,9 @@ function layers_callback_refresh(obj) menuString{idx} = sprintf('%s:%s', data.properties.lyr_group_name{idx}, data.properties.lyr_name{idx}); end - % Get the current list of selected layers - selectedString = obj.h_gui.h_layers.get_selected_strings(); - selectedString = union({'standard:surface'},selectedString); - % Create the new list of layers for the system that is selected - obj.h_gui.h_layers.set_available(menuString); - - % Select all the old layers that were selected (this function - % automatically ignores layers that do not exist in the new list) - obj.h_gui.h_layers.set_selected(selectedString,true); + obj.h_gui.h_layers.set_list(sort(menuString)); + obj.h_gui.h_layers.set_selected({'standard:surface','standard:bottom'},true); set(obj.h_gui.h_layers.h_list_availableCM.Children([2 3 4]),'Enable','on'); end diff --git a/cresis-toolbox/+imb/@prefwin/okPB_callback.m b/cresis-toolbox/+imb/@prefwin/okPB_callback.m index 1352f4ea..fed756b9 100644 --- a/cresis-toolbox/+imb/@prefwin/okPB_callback.m +++ b/cresis-toolbox/+imb/@prefwin/okPB_callback.m @@ -91,6 +91,15 @@ function okPB_callback(obj,hObj,event) end end +%% Check flightlines +flightlines = get(obj.h_gui.flightlinesPM,'String'); +flightlines = flightlines{get(obj.h_gui.flightlinesPM,'Value')}; +if strcmp(map_name,'Blank Geodetic Map') && ~strcmp(flightlines,'tracks files Flightlines') + % if so, don't plot a map + uiwait(msgbox('Only "tracks files Flightlines" supports "Blank Geodetic Map". Change either the map setting or the flightline setting.','Error updating preferences','modal')); + return; +end + %% Pass information to map window % Give information on the season, location (arctic/antarctic) and system_name % (rds accum etc.) to the map window @@ -114,8 +123,7 @@ function okPB_callback(obj,hObj,event) obj.settings.map_zone = map_zone; obj.settings.map_name = map_name; -flightlines = get(obj.h_gui.flightlinesPM,'String'); -obj.settings.flightlines = flightlines{get(obj.h_gui.flightlinesPM,'Value')}; +obj.settings.flightlines = flightlines; % Broadcast notice that StateChange event has occurred (calls imb.mapwin.get_map) notify(obj,'StateChange'); diff --git a/cresis-toolbox/+imb/@prefwin/ops_connect.m b/cresis-toolbox/+imb/@prefwin/ops_connect.m index bbac2f90..14ccc66a 100644 --- a/cresis-toolbox/+imb/@prefwin/ops_connect.m +++ b/cresis-toolbox/+imb/@prefwin/ops_connect.m @@ -39,9 +39,9 @@ return; end % Add Blank and Google maps -wms_maps = [{'arctic:blank_map';'antarctic:blank_map';'arctic:google_map';'antarctic:google_map'}; wms_maps(:)]; +wms_maps = [{'arctic:Blank Geodetic Map';'antarctic:Blank Geodetic Map';'arctic:Blank Stereographic Map';'antarctic:Blank Stereographic Map';'arctic:Google Map';'antarctic:Google Map'}; wms_maps(:)]; -flightlines = {'layerdata Flightlines','OPS Flightlines','OPS Quality Flightlines','OPS Coverage Flightlines', 'OPS Crossover Errors','OPS Bottom Elevation'}; +flightlines = {'tracks files Flightlines','OPS Flightlines','OPS Quality Flightlines','OPS Coverage Flightlines', 'OPS Crossover Errors','OPS Bottom Elevation'}; set(obj.h_gui.flightlinesPM,'String',flightlines); set(obj.h_gui.mapsPM,'String',wms_maps); diff --git a/cresis-toolbox/+imb/@prefwin/prefwin.m b/cresis-toolbox/+imb/@prefwin/prefwin.m index d2121f67..b46fb429 100644 --- a/cresis-toolbox/+imb/@prefwin/prefwin.m +++ b/cresis-toolbox/+imb/@prefwin/prefwin.m @@ -14,10 +14,10 @@ % default_params.season_names: cell array of selected season names % default_params.layer_names: cell array of selected layers % default_params.system: String containing system name - % ('accum','kuband','rds', 'snow', 'layerdata') + % ('accum','kuband','rds', 'snow', 'tracks') % default_params.map_name: string containing map selection % default_params.flightlines: string containing flightline selection - % ('layerdata Flightlines','OPS Flightlines','OPS Quality + % ('tracks files Flightlines','OPS Flightlines','OPS Quality % Flightlines','OPS Coverage Flightlines', 'OPS Crossover % Errors','OPS Bottom Elevation') % default_params.layer_source: string containing layer source ('OPS' or @@ -46,7 +46,7 @@ % settings have changed. settings % settings.layer_source: string containing the current layer source - % settings.layer_data_source: string containing the current layer layerData source + % settings.layer_data_source: string containing the current layer layerdata source % settings.layers: cell array of currently selected layers (OPS layer source only) % settings.seasons: cell array of currently selected seasons % settings.system: string containing current system @@ -55,6 +55,8 @@ % settings.map_name: string containing the map name % settings.flightlines: string containing flightline setting + % Mapping toolbox version entry + map_toolbox end @@ -91,6 +93,8 @@ obj.settings.map_name = []; obj.settings.flightlines = []; + obj.map_toolbox = license('checkout','map_toolbox'); + try create_ui(obj); catch ME diff --git a/cresis-toolbox/+imb/@prefwin/season_update.m b/cresis-toolbox/+imb/@prefwin/season_update.m index 1fb390b9..643038c2 100644 --- a/cresis-toolbox/+imb/@prefwin/season_update.m +++ b/cresis-toolbox/+imb/@prefwin/season_update.m @@ -1,9 +1,21 @@ function season_update(obj,status,event) +% imb.prefwin.season_update(obj,status,event) +% +% Update obj.h_gui.systemsLB +% Update obj.h_gui.h_seasons +% Update obj.h_gui.h_layers +% +% Author: John Paden +%% Setup + +% Determine what kind of flightline is selected (OPS or tracks files) flightlines_value = get(obj.h_gui.flightlinesPM,'Value'); flightlines_list = get(obj.h_gui.flightlinesPM,'String'); flightlines = flightlines_list(flightlines_value); +% Determine which system is selected (relevant only for OPS since tracks +% files group all systems together) systems_value = get(obj.h_gui.systemsLB,'Value'); systems = get(obj.h_gui.systemsLB,'String'); if ~isempty(systems) @@ -12,8 +24,10 @@ function season_update(obj,status,event) system_name = []; end +%% Update obj.h_gui.systemsLB if strcmp(flightlines{1}(1:3),'OPS') - unique_systems = setdiff(unique(obj.systems),{'layerdata'}); + % OPS based flight tracks + unique_systems = setdiff(unique(obj.systems),{'tracks'}); set(obj.h_gui.systemsLB,'String',unique_systems); system_value = find(strcmp(system_name,unique_systems),1); if isempty(system_value) @@ -26,25 +40,32 @@ function season_update(obj,status,event) set(obj.h_gui.systemsLB,'Value',system_value); system_name = unique_systems{system_value}; end + set(obj.h_gui.systemsText,'String','Radar Systems'); + set(obj.h_gui.systemsLB,'Enable','on'); else - system_name = 'layerdata'; - set(obj.h_gui.systemsLB,'String',{'layerdata'}); + % tracks files based flight tracks + system_name = 'tracks'; + set(obj.h_gui.systemsText,'String','-'); + set(obj.h_gui.systemsLB,'String',{'tracks'}); set(obj.h_gui.systemsLB,'Value',1); + set(obj.h_gui.systemsLB,'Enable','off'); end +%% Update obj.h_gui.h_seasons +% only list seasons that correspond to the map location selected (arctic or +% antarctic) map_value = get(obj.h_gui.mapsPM,'Value'); map_list = get(obj.h_gui.mapsPM,'String'); map = map_list(map_value); -[map_zone,map_name] = strtok(map,':'); -map_name = map_name(2:end); +[map_zone,~] = strtok(map,':'); zone_mask = strcmp(map_zone,obj.locations); systems_mask = strcmp(system_name,obj.systems); -obj.h_gui.h_seasons.set_list(obj.seasons(zone_mask & systems_mask)); +obj.h_gui.h_seasons.set_list(sort(obj.seasons(zone_mask & systems_mask))); -%% Layers +%% Update obj.h_gui.h_layers layer_sources = get(obj.h_gui.layerSourcePM,'String'); layer_source = layer_sources{get(obj.h_gui.layerSourcePM,'Value')}; if strcmp(layer_source,'layerdata') diff --git a/cresis-toolbox/+imb/create_track_files.m b/cresis-toolbox/+imb/create_track_files.m new file mode 100644 index 00000000..b4b1e327 --- /dev/null +++ b/cresis-toolbox/+imb/create_track_files.m @@ -0,0 +1,111 @@ +function [lat,lon,gps_time,frm_id,elev,surf,bottom,quality,frm_info,gps_source] = create_track_files(param,param_override) +% [lat,lon,gps_time,frm_id,elev,surf,bottom,quality,frm_info,gps_source] = create_track_files(param,param_override) +% +% Return tracks file information including lat, lon. See +% imb.run_all_create_track_files. The imb.picker loads this file when +% plotting flightlines without OPS. Loading all the individual +% CSARP_layerData files would be slow so this helps speed up the loading in +% imb.picker (and potentially other programs). +% +% param = struct with processing parameters +% param_override = parameters in this struct will override parameters +% in param. This struct must also contain the gRadar fields. +% Typically global gRadar; param_override = gRadar; +% +% param.day_seg +% param.create_track_files.layer_params: opsLoadLayers struct +% array which should define the loading of two layers. The first layer +% should be the surface and the second layer should be the bottom. +% +% Outputs: +% +% These variables are all 1 by Nx vectors of the same length, segments are +% terminated with NaN and all segments are included in these Nx length +% vectors: +% bottom: ice bottom two way travel time in seconds +% elev: elevation in meters +% frm_id: full frame ID 2019020401123 +% gps_time: GPS time in ANSI-C standard (seconds since Jan 1, 1970) +% lat: latitude in degrees +% lon: longitude in degrees +% surf: ice surface two way travel time in seconds +% quality: integer enumeration of bottom quality, 1=good, 2=moderate, +% 3=poor or derived from another source +% frm_info: structure containing frame information +% .frm_id: Nfrm element vector of frame IDs +% .start_gps_time: Nfrm element vector of start GPS times for each frame +% .stop_gps_time: Nfrm element vector of stop GPS times for each frame +% +% Author: John Paden, Rohan Choudhari +% +% See also: imb.run_all_create_track_files.m, +% imb.create_track_files.m + +%% General Setup +% ===================================================================== +param = merge_structs(param, param_override); + +% fprintf('=====================================================================\n'); +% fprintf('%s: %s (%s)\n', mfilename, param.day_seg, datestr(now)); +% fprintf('=====================================================================\n'); + +%% Reading layer data +% ===================================================================== +try + layer = opsLoadLayers(param,param.create_track_files.layer_params); +catch ME + rethrow(ME); +end + +if isempty(layer) + error('%s\tno_layers_returned!!!', param.day_seg); +end + +if isempty(layer(1).gps_time) + error('%s\tempty_layer!!!', param.day_seg); +end + +if any(isnan(layer(1).twtt)) + fprintf('%s\tsurface_NaN!!!\t%d\t%d\n', param.day_seg, sum(isnan(layer(1).twtt)), length(layer(1).twtt)); +end + +% Checking for inconsistent field lengths +if length(layer(1).lat) ~= length(layer(1).lon) ... + || length(layer(1).lat) ~= length(layer(1).gps_time) ... + || length(layer(1).lat) ~= length(layer(1).frm) ... + || length(layer(1).lat) ~= length(layer(1).elev) ... + || length(layer(1).lat) ~= length(layer(1).twtt) ... + || length(layer(1).lat) ~= length(layer(2).twtt) ... + || length(layer(1).lat) ~= length(layer(2).quality) + warning('%s\tLength of fields in layer files are not matched. This should never happen and indicates corrupt layer data files.\n'); + keyboard; + return; +end + +%% Loading frames and records files +% ===================================================================== +frames = frames_load(param); +records = records_load(param,'gps_time','gps_source'); % loads "gps_time", "gps_source" variables + +%% Create outputs +% ===================================================================== + +lat = layer(1).lat; +lon = layer(1).lon; +gps_time = layer(1).gps_time; +% Store full frame ID number 20190204_01_003 --> 2019020401003 +frm_id = str2num(param.day_seg([1:8,10:11]))*1000+layer(1).frm; +elev = layer(1).elev; +surf = layer(1).twtt; +bottom = layer(2).twtt; +quality = layer(2).quality; +if isfield(records,'gps_source') + gps_source = records.gps_source; +else + gps_source = ''; +end + +% Store frame GPS time boundaries +frm_info.frm_id = str2num(param.day_seg([1:8,10:11]))*1000+(1:length(frames.frame_idxs)); +frm_info.start_gps_time = records.gps_time(frames.frame_idxs); +frm_info.stop_gps_time = [records.gps_time(frames.frame_idxs(2:end)) records.gps_time(end)]; diff --git a/cresis-toolbox/+imb/delete_layer_pnts_outside_view.m b/cresis-toolbox/+imb/delete_layer_pnts_outside_view.m index d6b8deed..93072be2 100644 --- a/cresis-toolbox/+imb/delete_layer_pnts_outside_view.m +++ b/cresis-toolbox/+imb/delete_layer_pnts_outside_view.m @@ -23,7 +23,7 @@ end fprintf('delete_pnts_outside_view %s\n', param.day_seg); - load(ct_filename_support(param,'','frames')); + frames = frames_load(param); layers = {'surface','bottom'}; for layer = layers diff --git a/cresis-toolbox/+imb/create_season_layerdata_files.m b/cresis-toolbox/+imb/download_track_files.m similarity index 69% rename from cresis-toolbox/+imb/create_season_layerdata_files.m rename to cresis-toolbox/+imb/download_track_files.m index f3fa0915..83e24c10 100644 --- a/cresis-toolbox/+imb/create_season_layerdata_files.m +++ b/cresis-toolbox/+imb/download_track_files.m @@ -1,10 +1,11 @@ -function [lat,lon,frm_id,elev,surf,bottom,quality,frm_info,gps_source] = create_season_layerdata_files(param,param_override) -% [lat,lon,frm_id,elev,surf,bottom,quality,frm_info,gps_source] = create_season_layerdata_files(param,param_override) +function download_track_files(param,param_override) +% [lat,lon,gps_time,frm_id,elev,surf,bottom,quality,frm_info,gps_source] = create_track_files(param,param_override) % -% Return season layer file information including lat, lon. See imb. The imb.picker -% loads this file when plotting flightlines without OPS. Loading all the -% individual CSARP_layerData files would be slow so this helps speed up the -% loading in imb.picker (and potentially other programs). +% Return tracks file information including lat, lon. See +% imb.run_all_create_track_files. The imb.picker loads this file when +% plotting flightlines without OPS. Loading all the individual +% CSARP_layerData files would be slow so this helps speed up the loading in +% imb.picker (and potentially other programs). % % param = struct with processing parameters % param_override = parameters in this struct will override parameters @@ -12,15 +13,19 @@ % Typically global gRadar; param_override = gRadar; % % param.day_seg -% param.create_season_layerdata_files.layer_params: opsLoadLayers struct +% param.create_track_files.layer_params: opsLoadLayers struct % array which should define the loading of two layers. The first layer % should be the surface and the second layer should be the bottom. % % Outputs: -% These variables are all 1 by Nx vectors of the same length: +% +% These variables are all 1 by Nx vectors of the same length, segments are +% terminated with NaN and all segments are included in these Nx length +% vectors: % bottom: ice bottom two way travel time in seconds % elev: elevation in meters % frm_id: full frame ID 2019020401123 +% gps_time: GPS time in ANSI-C standard (seconds since Jan 1, 1970) % lat: latitude in degrees % lon: longitude in degrees % surf: ice surface two way travel time in seconds @@ -33,8 +38,8 @@ % % Author: John Paden, Rohan Choudhari % -% See also: imb.run_all_create_season_layerdata_files.m, -% imb.create_season_layerdata_files.m +% See also: imb.run_all_create_track_files.m, +% imb.create_track_files.m %% General Setup % ===================================================================== @@ -47,25 +52,26 @@ %% Reading layer data % ===================================================================== try - layer = opsLoadLayers(param,param.create_season_layerdata_files.layer_params); + layer = opsLoadLayers(param,param.create_track_files.layer_params); catch ME rethrow(ME); end if isempty(layer) error('%s\tno_layers_returned!!!', param.day_seg); -end; +end if isempty(layer(1).gps_time) error('%s\tempty_layer!!!', param.day_seg); -end; +end if any(isnan(layer(1).twtt)) - fprintf('%s\tsurface_NaN!!!\n', param.day_seg); -end; + fprintf('%s\tsurface_NaN!!!\t%d\t%d\n', param.day_seg, sum(isnan(layer(1).twtt)), length(layer(1).twtt)); +end % Checking for inconsistent field lengths if length(layer(1).lat) ~= length(layer(1).lon) ... + || length(layer(1).lat) ~= length(layer(1).gps_time) ... || length(layer(1).lat) ~= length(layer(1).frm) ... || length(layer(1).lat) ~= length(layer(1).elev) ... || length(layer(1).lat) ~= length(layer(1).twtt) ... @@ -78,16 +84,15 @@ %% Loading frames and records files % ===================================================================== -frames_fn = ct_filename_support(param,'','frames'); -frames = load(frames_fn); % loads "frames" variable -records_fn = ct_filename_support(param,'','records'); -records = load(records_fn,'gps_time','gps_source'); % loads "gps_time" variable +frames = frames_load(param); +records = records_load(param,'gps_time','gps_source'); % loads "gps_time", "gps_source" variables %% Create outputs % ===================================================================== lat = layer(1).lat; lon = layer(1).lon; +gps_time = layer(1).gps_time; % Store full frame ID number 20190204_01_003 --> 2019020401003 frm_id = str2num(param.day_seg([1:8,10:11]))*1000+layer(1).frm; elev = layer(1).elev; diff --git a/cresis-toolbox/+imb/get_proj_info.m b/cresis-toolbox/+imb/get_proj_info.m index 37987931..32614691 100644 --- a/cresis-toolbox/+imb/get_proj_info.m +++ b/cresis-toolbox/+imb/get_proj_info.m @@ -1,7 +1,7 @@ function [proj] = get_proj_info(loc) % [proj] = get_proj_info(loc) -standard_projections; +proj_load_standard; switch loc case 'arctic' diff --git a/cresis-toolbox/+imb/ice_mask_edit.m b/cresis-toolbox/+imb/ice_mask_edit.m index bc967eb7..6b694af5 100644 --- a/cresis-toolbox/+imb/ice_mask_edit.m +++ b/cresis-toolbox/+imb/ice_mask_edit.m @@ -112,7 +112,7 @@ % obj.mdata.twtt = param.mdata.twtt; % obj.mdata.theta = param.mdata.theta; % obj.mdata.ice_mask = param.mdata.ice_mask; - % obj.mdata.param_combine = param.mdata.param_combine; + % obj.mdata.param_array = param.mdata.param_array; % rmfield(param,'mdata'); obj.mdata = param.mdata; end @@ -728,7 +728,7 @@ function change_slice(obj,slice) if isempty(obj.mdata_loaded) || ~obj.mdata_loaded %% load data % convert from FCS to proj - origin_ecef = obj.mdata.param_combine.array_param.fcs{1}{1}.origin(:,:); + origin_ecef = obj.mdata.param_array.array_param.fcs{1}{1}.origin(:,:); physical_constants; [origin_lat,origin_lon] = ecef2geodetic(origin_ecef(1,:),origin_ecef(2,:),origin_ecef(3,:),WGS84.ellipsoid); origin_lat = origin_lat*180/pi; @@ -750,11 +750,11 @@ function change_slice(obj,slice) intersect_ecef_all = zeros([3,Nt,Nr]); for rline = 1:Nr - Tfcs_ecef = [obj.mdata.param_combine.array_param.fcs{1}{1}.x(:,rline), ... - obj.mdata.param_combine.array_param.fcs{1}{1}.y(:,rline), ... - obj.mdata.param_combine.array_param.fcs{1}{1}.z(:,rline)]; + Tfcs_ecef = [obj.mdata.param_array.array_param.fcs{1}{1}.x(:,rline), ... + obj.mdata.param_array.array_param.fcs{1}{1}.y(:,rline), ... + obj.mdata.param_array.array_param.fcs{1}{1}.z(:,rline)]; - dir = [zeros(1,Nt) ; sin(obj.mdata.theta) ; -cos(obj.mdata.theta)]; + dir = [zeros(Nt,1), sin(obj.mdata.theta), -cos(obj.mdata.theta)].'; for row = 1:Nt dir(:,row) = dir(:,row)/norm(dir(:,row)); end diff --git a/cresis-toolbox/+imb/run_all_create_season_layerdata_files.m b/cresis-toolbox/+imb/run_all_create_season_layerdata_files.m deleted file mode 100644 index e68ceb97..00000000 --- a/cresis-toolbox/+imb/run_all_create_season_layerdata_files.m +++ /dev/null @@ -1,228 +0,0 @@ -% script run_all_create_season_layerdata_files -% run_all_create_season_layerdata_files -% -% Creates a season layer file containing the lat, lon. The imb.picker loads -% this file when plotting flightlines without OPS. Loading all the -% individual CSARP_layerData files would be slow so this helps speed up the -% loading. -% -% Output filenames are of the form: -% .../csarp_support/layer/layer_SYSTEM_SEASONNAME.mat -% For example: -% .../csarp_support/layer/layer_accum_2018_Antarctica_TObas.mat -% -% Output file: -% These variables are all 1 by Nx vectors of the same length, segments are -% terminated with NaN: -% bottom: ice bottom two way travel time in seconds -% elev: elevation in meters -% frm_id: full frame ID 2019020401123 -% lat: latitude in degrees -% lon: longitude in degrees -% surf: ice surface two way travel time in seconds -% quality: integer enumeration of bottom quality, 1=good, 2=moderate, -% 3=poor or derived from another source -% frm_info: structure containing frame information -% .frm_id: Nfrm element vector of frame IDs -% .start_gps_time: Nfrm element vector of start GPS times for each frame -% .stop_gps_time: Nfrm element vector of stop GPS times for each frame -% -% Author: John Paden, Rohan Choudhari -% -% See also: imb.run_all_create_season_layerdata_files.m, -% imb.create_season_layerdata_files.m - -%% User Settings -% ========================================================================= - -%% User Settings: Select seasons -% ------------------------------------------------------------------------- -param_fns = {}; -% param_fns{end+1} = 'accum_param_2015_Antarctica_Ground.xls'; -% param_fns{end+1} = 'accum_param_2018_Antarctica_TObas.xls'; -% param_fns{end+1} = 'accum_param_2019_Antarctica_TObas.xls'; -% param_fns{end+1} = 'rds_param_1993_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_1995_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_1996_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_1997_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_1998_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_1999_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2001_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2002_Antarctica_P3chile.xls'; % May not work -% param_fns{end+1} = 'rds_param_2002_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2003_Greenland_P3.xls'; % May not work -% param_fns{end+1} = 'rds_param_2004_Antarctica_P3chile.xls'; % May not work -% param_fns{end+1} = 'rds_param_2005_Greenland_TO.xls'; % May not work -% param_fns{end+1} = 'rds_param_2006_Greenland_TO.xls'; -% param_fns{end+1} = 'rds_param_2007_Greenland_P3.xls'; % May not work -% param_fns{end+1} = 'rds_param_2008_Greenland_Ground.xls'; -% param_fns{end+1} = 'rds_param_2008_Greenland_TO.xls'; -% param_fns{end+1} = 'rds_param_2009_Antarctica_DC8.xls'; -% param_fns{end+1} = 'rds_param_2009_Antarctica_TO.xls'; -% param_fns{end+1} = 'rds_param_2009_Greenland_TO.xls'; -% param_fns{end+1} = 'rds_param_2009_Greenland_TO_wise.xls'; -% param_fns{end+1} = 'rds_param_2010_Antarctica_DC8.xls'; -% param_fns{end+1} = 'rds_param_2010_Greenland_DC8.xls'; -% param_fns{end+1} = 'rds_param_2010_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2010_Greenland_TO_wise.xls'; -% param_fns{end+1} = 'rds_param_2011_Antarctica_DC8.xls'; -% param_fns{end+1} = 'rds_param_2011_Antarctica_TO.xls'; -% param_fns{end+1} = 'rds_param_2011_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2011_Greenland_TO.xls'; -% param_fns{end+1} = 'rds_param_2012_Antarctica_DC8.xls'; -% param_fns{end+1} = 'rds_param_2012_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2013_Antarctica_Basler.xls'; -% param_fns{end+1} = 'rds_param_2013_Antarctica_P3.xls'; -% param_fns{end+1} = 'rds_param_2013_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2014_Antarctica_DC8.xls'; -% param_fns{end+1} = 'rds_param_2014_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2015_Greenland_C130.xls'; -% param_fns{end+1} = 'rds_param_2016_Antarctica_DC8.xls'; -% param_fns{end+1} = 'rds_param_2016_Greenland_G1XB.xls'; -% param_fns{end+1} = 'rds_param_2016_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2016_Greenland_Polar6.xls'; -% param_fns{end+1} = 'rds_param_2016_Greenland_TOdtu.xls'; -% param_fns{end+1} = 'rds_param_2017_Antarctica_Basler.xls'; -% param_fns{end+1} = 'rds_param_2017_Antarctica_P3.xls'; -% param_fns{end+1} = 'rds_param_2017_Antarctica_Polar6.xls'; -% param_fns{end+1} = 'rds_param_2017_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2018_Antarctica_DC8.xls'; -% param_fns{end+1} = 'rds_param_2018_Antarctica_Ground.xls'; -% param_fns{end+1} = 'rds_param_2018_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2018_Greenland_Polar6.xls'; -% param_fns{end+1} = 'rds_param_2019_Greenland_P3.xls'; -param_fns{end+1} = 'rds_param_2019_Antarctica_Ground.xls'; -% param_fns{end+1} = 'rds_param_2019_Antarctica_GV.xls'; -% param_fns{end+1} = 'snow_param_2012_Greenland_P3.xls'; -% param_fns{end+1} = 'snow_param_2019_SouthDakota_CESSNA.xls'; - -%% User Settings: Select layers -% First layer is the "surface" -% Second layer is the "bottom" -if 1 - layer_params = struct('name','surface'); - layer_params.source = 'layerdata'; - % layer_params(2).layerdata_source = 'CSARP_post/layerData'; - layer_params.existence_check = true; - layer_params(2).name = 'bottom'; - layer_params(2).source = 'layerdata'; - % layer_params(2).layerdata_source = 'CSARP_post/layerData'; - layer_params(2).existence_check = false; -else - % HACK!!! - keyboard - layer_params = struct('name','surface'); - layer_params.source = 'layerdata'; - layer_params(1).layerdata_source = 'layerData_koenig'; - layer_params(2).existence_check = true; - layer_params(1).fix_broken_layerdata = true; % Found some bad files - layer_params(2).name = 'bottom'; - layer_params(2).source = 'layerdata'; - layer_params(2).layerdata_source = 'layerData_koenig'; - layer_params(2).existence_check = false; - layer_params(2).fix_broken_layerdata = true; % Found some bad files - for lay_idx=2:30 - layer_params(lay_idx+1).name = sprintf('Koenig_%d',lay_idx); - layer_params(lay_idx+1).source = 'layerdata'; - layer_params(lay_idx+1).layerdata_source = 'layerData_koenig'; - layer_params(lay_idx+1).existence_check = false; - layer_params(lay_idx+1).fix_broken_layerdata = true; % Found some bad files - end -end - -param_override = []; -param_override.create_season_layerdata_files.layer_params = layer_params; - -%% Automated Section -% ========================================================================= -% Input checking -global gRadar; -if exist('param_override','var') - param_override = merge_structs(gRadar,param_override); -else - param_override = gRadar; -end - -%% Loop to process each season -for param_idx = 1:length(param_fns) - - % Initialize variables to be extracted from layers - lat = []; - lon = []; - frm_id = []; - surf = []; - bottom = []; - elev = []; - quality = []; - frm_info = []; - frm_info.frm_id = []; - frm_info.start_gps_time = []; - frm_info.stop_gps_time = []; - - % Read in parameter spreadsheet - param_fn = ct_filename_param(param_fns{param_idx}); - fprintf('Reading %s\n', param_fn); - params = read_param_xls(param_fn,''); - - if isempty(params) - continue; - end - - % Run all segments (except "do not process") - if 1 - params = ct_set_params(params,'cmd.generic',1); - params = ct_set_params(params,'cmd.generic',0,'cmd.notes','do not process'); - else - % HACK!!! - keyboard - params = ct_set_params(params,'cmd.generic',0); - params = ct_set_params(params,'cmd.generic',1,'day_seg','20120330_04'); - end - - %% Load each segment - for param_idx = 1:length(params) - param = params(param_idx); - - if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic - fprintf('%s\tdo not process\n', param.day_seg); - continue; - end - - try - [tmp.lat,tmp.lon,tmp.frm_id,tmp.elev,tmp.surf,tmp.bottom,tmp.quality,tmp.frm_info,gps_source] = imb.create_season_layerdata_files(param,param_override); - catch ME - fprintf('%s\terror!!!\t%s\n', param.day_seg, ME.getReport); - continue; - end - - % Concatenate data - fprintf('%s\tloaded\t%d\t%d\n', param.day_seg, length(tmp.lat), sum(~isnan(tmp.bottom))); - lat = [lat tmp.lat NaN]; - lon = [lon tmp.lon NaN]; - % Store full frame ID number 20190204_01_003 --> 2019020401003 - frm_id = [frm_id tmp.frm_id NaN]; - elev = [elev tmp.elev NaN]; - surf = [surf tmp.surf NaN]; - bottom = [bottom tmp.bottom NaN]; - quality = [quality tmp.quality NaN]; - - % Store frame GPS time boundaries - frm_info.frm_id = [frm_info.frm_id tmp.frm_info.frm_id]; - frm_info.start_gps_time = [frm_info.start_gps_time tmp.frm_info.start_gps_time]; - frm_info.stop_gps_time = [frm_info.stop_gps_time tmp.frm_info.stop_gps_time]; - end - - %% Save output for this season - out_fn_dir = ct_filename_support(param,'layer',''); - out_fn_name = sprintf('layer_%s_%s_%s.mat', param.post.ops.location, ct_output_dir(param.radar_name), param.season_name); - out_fn = fullfile(out_fn_dir,out_fn_name); - fprintf(' Saving %s\n\n', out_fn); - if ~exist(out_fn_dir,'dir') - mkdir(out_fn_dir); - end - file_type = 'layer_season'; - file_version = '1'; - param = struct('day_seg',param.day_seg,'radar_name',param.radar_name, ... - 'season_name',param.season_name,'sw_version',param.sw_version); - ct_save(out_fn,'bottom','elev','file_type','file_version','frm_id','frm_info','gps_source','lat','lon','param','quality','surf'); -end diff --git a/cresis-toolbox/+imb/run_all_create_track_files.m b/cresis-toolbox/+imb/run_all_create_track_files.m new file mode 100644 index 00000000..3a8cbb9f --- /dev/null +++ b/cresis-toolbox/+imb/run_all_create_track_files.m @@ -0,0 +1,263 @@ +% script run_all_create_track_files +% run_all_create_track_files +% +% Creates season track file(s) containing the lat, lon, and other +% information. The imb.picker loads these file when plotting flightlines +% without OPS. Loading all the individual CSARP_layer files would be +% slow so using these track files helps speed imb.picker loading. +% +% Output filenames are of the form: +% .../csarp_support/tracks/tracks_SYSTEM_SEASONNAME.mat +% For example: +% .../csarp_support/tracks/tracks_accum_2018_Antarctica_TObas.mat +% +% Output file: +% +% These variables are all 1 by Nx vectors of the same length, segments are +% terminated with NaN and all segments are included in these Nx length +% vectors: +% bottom: ice bottom two way travel time in seconds +% elev: elevation in meters +% frm_id: full frame ID 2019020401123 +% lat: latitude in degrees +% lon: longitude in degrees +% surf: ice surface two way travel time in seconds +% quality: integer enumeration of bottom quality, 1=good, 2=moderate, +% 3=poor or derived from another source +% frm_info: structure containing frame information +% .frm_id: Nfrm element vector of frame IDs +% .start_gps_time: Nfrm element vector of start GPS times for each frame +% .stop_gps_time: Nfrm element vector of stop GPS times for each frame +% +% Author: John Paden, Rohan Choudhari +% +% See also: imb.run_all_create_track_files.m, +% imb.create_track_files.m + +%% User Settings +% ========================================================================= + +% Select seasons in run_all: +run_all; + +%% User Settings: Select layers +% First layer is the "surface" +% Second layer is the "bottom" +if 1 + layer_params = struct('name','surface'); + layer_params.source = 'layerdata'; + % layer_params.layerdata_source = 'CSARP_post/layerData'; + layer_params.existence_check = true; + layer_params(2).name = 'bottom'; + layer_params(2).source = 'layerdata'; + % layer_params(2).layerdata_source = 'CSARP_post/layerData'; + layer_params(2).existence_check = false; +else + % Debug: Enable to use non-standard layer data + keyboard + layer_params = struct('name','surface'); + layer_params.source = 'layerdata'; + layer_params(1).layerdata_source = 'layer_koenig'; + layer_params(2).existence_check = true; + layer_params(1).fix_broken_layerdata = true; % Found some bad files + layer_params(2).name = 'bottom'; + layer_params(2).source = 'layerdata'; + layer_params(2).layerdata_source = 'layer_koenig'; + layer_params(2).existence_check = false; + layer_params(2).fix_broken_layerdata = true; % Found some bad files +end + +param_override = []; +param_override.create_track_files.layer_params = layer_params; +run_create_track_files.mode = 'create'; +% run_create_track_files.mode = 'append'; + +%% Automated Section +% ========================================================================= +% Input checking +global gRadar; +if exist('param_override','var') + param_override = merge_structs(gRadar,param_override); +else + param_override = gRadar; +end + +%% Loop to process each season +for param_idx = 1:length(param_fns) + + % Initialize variables to be extracted from layers + lat = []; + lon = []; + gps_time = []; + frm_id = []; + surf = []; + bottom = []; + elev = []; + quality = []; + frm_info = []; + frm_info.frm_id = []; + frm_info.start_gps_time = []; + frm_info.stop_gps_time = []; + + % Read in parameter spreadsheet + param_fn = ct_filename_param(param_fns{param_idx}); + fprintf('Reading %s\n', param_fn); + params = read_param_xls(param_fn,''); + + if isempty(params) + continue; + end + + % Run all segments (except "do not process") + if 1 + params = ct_set_params(params,'cmd.generic',1); + params = ct_set_params(params,'cmd.generic',0,'cmd.notes','do not process'); + else + % Debug: Enable to run only particular segments + keyboard + params = ct_set_params(params,'cmd.generic',0); + params = ct_set_params(params,'cmd.generic',1,'day_seg','20120330_04'); + end + + %% Load each segment + for param_idx = 1:length(params) + param = params(param_idx); + + if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic + fprintf('%s\tdo not process\n', param.day_seg); + continue; + end + + % Input checks: run_create_track_files + if ~exist('run_create_track_files','var') + run_create_track_files = []; + end + % run_create_track_files.mode: string containing the mode. Must be either + % 'create' (default) or 'append'. The create mode overwrites the existing file. The + % append mode adds segments to the existing file. Append is slower for doing whole seasons, + % but is faster if just adding a few segments. + if ~isfield(run_create_track_files,'mode') || isempty(run_create_track_files.mode) + run_create_track_files.mode = 'create'; + end + + tmp = []; + try + param.cmd.frms = []; + [tmp.lat,tmp.lon,tmp.gps_time,tmp.frm_id,tmp.elev,tmp.surf,tmp.bottom,tmp.quality,tmp.frm_info,gps_source] = imb.create_track_files(param,param_override); + catch ME + fprintf('%s\terror!!!\t%s\n', param.day_seg, ME.getReport); + continue; + end + + if strcmpi(run_create_track_files.mode,'append') + %% Append to existing season file + + % Load existing file for this season + % ------------------------------------------------------------------- + out_fn_dir = ct_filename_support(param,'tracks',''); + out_fn_name = sprintf('tracks_%s_%s_%s.mat', param.post.ops.location, ct_output_dir(param.radar_name), param.season_name); + out_fn = fullfile(out_fn_dir,out_fn_name); + if ~exist(out_fn,'file') + % If file does not exist, switch to create mode + run_create_track_files.mode = 'create'; + else + fprintf('%s\tloaded\t%d\t%d\t', param.day_seg, length(tmp.lat), sum(~isnan(tmp.bottom))); + fprintf('Appending\t%s\n', out_fn); + old = load(out_fn); + + % Determine the numeric frm_id for the segment that is being inserted + tmp_frm_id = str2num(param.day_seg([1:8,10:11]))*1000; + + % Store along-track data + % ----------------------------------------------------------------- + % stop_idx: index to stop at for old data + stop_idx = find(old.frm_id < tmp_frm_id,1,'last'); + if isempty(stop_idx) + stop_idx = 0; + else + stop_idx = stop_idx + 1; % Include the NaN at the end of the segment + end + + % start_idx: the index to start again after the top index + start_idx = find(old.frm_id > tmp_frm_id+999,1); + if isempty(start_idx) + start_idx = length(old.frm_id) + 1; + end + + % Add new segment to file; also removes old data for the segment if + % it was there. + lat = [old.lat(1:stop_idx) tmp.lat NaN old.lat(start_idx:end)]; + lon = [old.lon(1:stop_idx) tmp.lon NaN old.lon(start_idx:end)]; + gps_time = [old.gps_time(1:stop_idx) tmp.lon NaN old.gps_time(start_idx:end)]; + frm_id = [old.frm_id(1:stop_idx) tmp.frm_id NaN old.frm_id(start_idx:end)]; + elev = [old.elev(1:stop_idx) tmp.elev NaN old.elev(start_idx:end)]; + surf = [old.surf(1:stop_idx) tmp.surf NaN old.surf(start_idx:end)]; + bottom = [old.bottom(1:stop_idx) tmp.bottom NaN old.bottom(start_idx:end)]; + quality = [old.quality(1:stop_idx) tmp.quality NaN old.quality(start_idx:end)]; + + % Store frame GPS time boundaries + % ----------------------------------------------------------------- + % stop_idx: index to stop at for old data + stop_idx = find(old.frm_info.frm_id < tmp_frm_id,1,'last'); + if isempty(stop_idx) + stop_idx = 0; + end + + % start_idx: the index to start again after the top index + start_idx = find(old.frm_info.frm_id > tmp_frm_id+999,1); + if isempty(start_idx) + start_idx = length(old.frm_info.frm_id) + 1; + end + + frm_info.frm_id = [old.frm_info.frm_id(1:stop_idx) tmp.frm_info.frm_id old.frm_info.frm_id(start_idx:end)]; + frm_info.start_gps_time = [old.frm_info.start_gps_time(1:stop_idx) tmp.frm_info.start_gps_time old.frm_info.start_gps_time(start_idx:end)]; + frm_info.stop_gps_time = [old.frm_info.stop_gps_time(1:stop_idx) tmp.frm_info.stop_gps_time old.frm_info.stop_gps_time(start_idx:end)]; + + ct_save(out_fn,'-append','bottom','elev','frm_id','frm_info','lat','lon','quality','surf'); + end + end + + if strcmpi(run_create_track_files.mode,'create') + % Concatenate on new season file + % ------------------------------------------------------------------- + fprintf('%s\tloaded\t%d\t%d\n', param.day_seg, length(tmp.lat), sum(~isnan(tmp.bottom))); + lat = [lat tmp.lat NaN]; + lon = [lon tmp.lon NaN]; + gps_time = [gps_time tmp.gps_time NaN]; + % Store full frame ID number 20190204_01_003 --> 2019020401003 + frm_id = [frm_id tmp.frm_id NaN]; + elev = [elev tmp.elev NaN]; + surf = [surf tmp.surf NaN]; + bottom = [bottom tmp.bottom NaN]; + quality = [quality tmp.quality NaN]; + % Store frame GPS time boundaries + frm_info.frm_id = [frm_info.frm_id tmp.frm_info.frm_id]; + frm_info.start_gps_time = [frm_info.start_gps_time tmp.frm_info.start_gps_time]; + frm_info.stop_gps_time = [frm_info.stop_gps_time tmp.frm_info.stop_gps_time]; + elseif ~strcmpi(run_create_track_files.mode,'append') + error('Valid modes are append or create. Invalid mode specified for run_create_track_files.mode: %s', run_create_track_files.mode); + end + end + + if strcmpi(run_create_track_files.mode,'create') + if isempty(lat) + %% No data error + error('There is no data available and so no season layer file can be saved.'); + + else + %% Save output for this season + out_fn_dir = ct_filename_support(param,'tracks',''); + out_fn_name = sprintf('tracks_%s_%s_%s.mat', param.post.ops.location, ct_output_dir(param.radar_name), param.season_name); + out_fn = fullfile(out_fn_dir,out_fn_name); + fprintf(' Saving %s\n\n', out_fn); + if ~exist(out_fn_dir,'dir') + mkdir(out_fn_dir); + end + file_type = 'tracks'; + file_version = '1'; + param = struct('day_seg',param.day_seg,'radar_name',param.radar_name, ... + 'season_name',param.season_name,'sw_version',param.sw_version); + ct_save(out_fn,'bottom','elev','file_type','file_version','frm_id','frm_info','gps_source','gps_time','lat','lon','param','quality','surf'); + end + end +end diff --git a/cresis-toolbox/+imb/run_all_download_track_files.m b/cresis-toolbox/+imb/run_all_download_track_files.m new file mode 100644 index 00000000..72ee7ac5 --- /dev/null +++ b/cresis-toolbox/+imb/run_all_download_track_files.m @@ -0,0 +1,96 @@ +% script run_all_download_track_files +% run_all_download_track_files +% +% Downloads a season track file. The imb.picker loads these files when +% plotting flightlines without OPS. +% +% Filenames are of the form: +% .../csarp_support/tracks/tracks_SYSTEM_SEASONNAME.mat +% For example: +% .../csarp_support/tracks/tracks_accum_2018_Antarctica_TObas.mat +% +% Author: John Paden, Rohan Choudhari +% +% See also: imb.run_all_download_track_files.m + +%% User Settings +% ========================================================================= + +% Select seasons in run_all: +run_all; + +%% User Settings +% ========================================================================= +param_override = []; +param_override.download_track_files.force_update = false; % Only download if the files does not already exist + +%% Automated Section +% ========================================================================= +% Input checking +global gRadar; +if exist('param_override','var') + param_override = merge_structs(gRadar,param_override); +else + param_override = gRadar; +end + +%% Loop to process each season +for param_idx = 1:length(param_fns) + + % Read in parameter spreadsheet + param_fn = ct_filename_param(param_fns{param_idx}); + fprintf('Reading %s\n', param_fn); + params = read_param_xls(param_fn,''); + + if isempty(params) + continue; + end + + % Run all segments (except "do not process") + params = ct_set_params(params,'cmd.generic',1); + params = ct_set_params(params,'cmd.generic',0,'cmd.notes','do not process'); + + %% Look for a good segment + found_good_segment = false; + for param_idx = 1:length(params) + param = params(param_idx); + + if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic + fprintf('%s\tdo not process\n', param.day_seg); + continue; + else + found_good_segment = true; + param = merge_structs(param,param_override); + break; + end + end + if ~found_good_segment + warning('No good segment found in this segment so skipping this segment''s track file.') + else + %% Save output for this season + out_fn_dir = ct_filename_support(param,'tracks',''); + out_fn_name = sprintf('tracks_%s_%s_%s.mat', param.post.ops.location, ct_output_dir(param.radar_name), param.season_name); + out_fn = fullfile(out_fn_dir,out_fn_name); + url = sprintf('%s/data/csarp_support/tracks/%s', param.data.url, out_fn_name); + if exist(out_fn,'file') && ~param.download_track_files.force_update + fprintf(' File already exists %s\n Set param.download_track_files.force_update to true to redownload.\n', out_fn) + else + if exist(out_fn,'file') + fprintf(' Downloading %s\n', url); + fprintf(' to overwrite %s\n', out_fn); + else + fprintf(' Downloading %s\n', url); + fprintf(' to %s\n', out_fn); + end + if ~exist(out_fn_dir,'dir') + mkdir(out_fn_dir); + end + wget_cmd = sprintf('wget -P %s %s', out_fn_dir, url); + fprintf(' %s\n', wget_cmd); + system(wget_cmd); + if ~exist(out_fn,'file') + warning('Failed to download file.'); + end + end + end +end diff --git a/cresis-toolbox/+imb/run_create_track_files.m b/cresis-toolbox/+imb/run_create_track_files.m new file mode 100644 index 00000000..8b140481 --- /dev/null +++ b/cresis-toolbox/+imb/run_create_track_files.m @@ -0,0 +1,248 @@ +% script run_all_create_track_files +% run_all_create_track_files +% +% Creates season track file(s) containing the lat, lon, and other +% information. The imb.picker loads these file when plotting flightlines +% without OPS. Loading all the individual CSARP_layer files would be +% slow so using these track files helps speed imb.picker loading. +% +% Output filenames are of the form: +% .../csarp_support/tracks/tracks_SYSTEM_SEASONNAME.mat +% For example: +% .../csarp_support/tracks/tracks_accum_2018_Antarctica_TObas.mat +% +% Output file: +% +% These variables are all 1 by Nx vectors of the same length, segments are +% terminated with NaN and all segments are included in these Nx length +% vectors: +% bottom: ice bottom two way travel time in seconds +% elev: elevation in meters +% frm_id: full frame ID 2019020401123 +% lat: latitude in degrees +% lon: longitude in degrees +% surf: ice surface two way travel time in seconds +% quality: integer enumeration of bottom quality, 1=good, 2=moderate, +% 3=poor or derived from another source +% frm_info: structure containing frame information +% .frm_id: Nfrm element vector of frame IDs +% .start_gps_time: Nfrm element vector of start GPS times for each frame +% .stop_gps_time: Nfrm element vector of stop GPS times for each frame +% +% Author: John Paden, Rohan Choudhari +% +% See also: imb.run_all_create_track_files.m, +% imb.create_track_files.m + +%% User Settings +% ========================================================================= + +param_override = []; + +% Read in parameter spreadsheet +params = read_param_xls(param_fn,''); + +% Parameters spreadsheet to use for updating +% 1. Segment and frame list are taken from the parameter sheet +% 2. For GPS update, GPS time offsets are pulled from the parameter sheet +params = read_param_xls(ct_filename_param('accum_param_2022_Antarctica_BaslerMKB.xls')); + +% params = ct_set_params(params,'cmd.generic',0); +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20230110_02'); +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20230110_03'); +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20230113_02'); +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20230116_02'); +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20230124_02'); +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20230124_03'); +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20230125_01'); +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20230126_01'); +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20230126_03'); +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20230126_08'); +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20230127_01'); + +% Do all segments +params = ct_set_params(params,'cmd.generic',1); +params = ct_set_params(params,'cmd.generic',0,'cmd.notes','do not process'); + +%% User Settings: Select layers +% First layer is the "surface" +% Second layer is the "bottom" +layer_params = struct('name','surface'); +layer_params.source = 'layerdata'; +% layer_params.layerdata_source = 'CSARP_post/layerData'; +layer_params.existence_check = true; +layer_params(2).name = 'bottom'; +layer_params(2).source = 'layerdata'; +% layer_params(2).layerdata_source = 'CSARP_post/layerData'; +layer_params(2).existence_check = false; + +param_override = []; +param_override.create_track_files.layer_params = layer_params; +run_create_track_files.mode = 'create'; +% run_create_track_files.mode = 'append'; + +%% Automated Section +% ========================================================================= +% Input checking +global gRadar; +if exist('param_override','var') + param_override = merge_structs(gRadar,param_override); +else + param_override = gRadar; +end + +% Initialize variables to be extracted from layers +lat = []; +lon = []; +gps_time = []; +frm_id = []; +surf = []; +bottom = []; +elev = []; +quality = []; +frm_info = []; +frm_info.frm_id = []; +frm_info.start_gps_time = []; +frm_info.stop_gps_time = []; + +%% Load each segment +for param_idx = 1:length(params) + param = params(param_idx); + + if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic + fprintf('%s\tdo not process\n', param.day_seg); + continue; + end + + % Input checks: run_create_track_files + if ~exist('run_create_track_files','var') + run_create_track_files = []; + end + % run_create_track_files.mode: string containing the mode. Must be either + % 'create' (default) or 'append'. The create mode overwrites the existing file. The + % append mode adds segments to the existing file. Append is slower for doing whole seasons, + % but is faster if just adding a few segments. + if ~isfield(run_create_track_files,'mode') || isempty(run_create_track_files.mode) + run_create_track_files.mode = 'create'; + end + + tmp = []; + try + param.cmd.frms = []; + [tmp.lat,tmp.lon,tmp.gps_time,tmp.frm_id,tmp.elev,tmp.surf,tmp.bottom,tmp.quality,tmp.frm_info,gps_source] = imb.create_track_files(param,param_override); + catch ME + fprintf('%s\terror!!!\t%s\n', param.day_seg, ME.getReport); + continue; + end + + if strcmpi(run_create_track_files.mode,'append') + %% Append to existing season file + + % Load existing file for this season + % ------------------------------------------------------------------- + out_fn_dir = ct_filename_support(param,'tracks',''); + out_fn_name = sprintf('tracks_%s_%s_%s.mat', param.post.ops.location, ct_output_dir(param.radar_name), param.season_name); + out_fn = fullfile(out_fn_dir,out_fn_name); + if ~exist(out_fn,'file') + % If file does not exist, switch to create mode + run_create_track_files.mode = 'create'; + else + fprintf('%s\tloaded\t%d\t%d\t', param.day_seg, length(tmp.lat), sum(~isnan(tmp.bottom))); + fprintf('Appending\t%s\n', out_fn); + old = load(out_fn); + + % Determine the numeric frm_id for the segment that is being inserted + tmp_frm_id = str2num(param.day_seg([1:8,10:11]))*1000; + + % Store along-track data + % ----------------------------------------------------------------- + % stop_idx: index to stop at for old data + stop_idx = find(old.frm_id < tmp_frm_id,1,'last'); + if isempty(stop_idx) + stop_idx = 0; + else + stop_idx = stop_idx + 1; % Include the NaN at the end of the segment + end + + % start_idx: the index to start again after the top index + start_idx = find(old.frm_id > tmp_frm_id+999,1); + if isempty(start_idx) + start_idx = length(old.frm_id) + 1; + end + + % Add new segment to file; also removes old data for the segment if + % it was there. + lat = [old.lat(1:stop_idx) tmp.lat NaN old.lat(start_idx:end)]; + lon = [old.lon(1:stop_idx) tmp.lon NaN old.lon(start_idx:end)]; + gps_time = [old.gps_time(1:stop_idx) tmp.lon NaN old.gps_time(start_idx:end)]; + frm_id = [old.frm_id(1:stop_idx) tmp.frm_id NaN old.frm_id(start_idx:end)]; + elev = [old.elev(1:stop_idx) tmp.elev NaN old.elev(start_idx:end)]; + surf = [old.surf(1:stop_idx) tmp.surf NaN old.surf(start_idx:end)]; + bottom = [old.bottom(1:stop_idx) tmp.bottom NaN old.bottom(start_idx:end)]; + quality = [old.quality(1:stop_idx) tmp.quality NaN old.quality(start_idx:end)]; + + % Store frame GPS time boundaries + % ----------------------------------------------------------------- + % stop_idx: index to stop at for old data + stop_idx = find(old.frm_info.frm_id < tmp_frm_id,1,'last'); + if isempty(stop_idx) + stop_idx = 0; + end + + % start_idx: the index to start again after the top index + start_idx = find(old.frm_info.frm_id > tmp_frm_id+999,1); + if isempty(start_idx) + start_idx = length(old.frm_info.frm_id) + 1; + end + + frm_info.frm_id = [old.frm_info.frm_id(1:stop_idx) tmp.frm_info.frm_id old.frm_info.frm_id(start_idx:end)]; + frm_info.start_gps_time = [old.frm_info.start_gps_time(1:stop_idx) tmp.frm_info.start_gps_time old.frm_info.start_gps_time(start_idx:end)]; + frm_info.stop_gps_time = [old.frm_info.stop_gps_time(1:stop_idx) tmp.frm_info.stop_gps_time old.frm_info.stop_gps_time(start_idx:end)]; + + ct_save(out_fn,'-append','bottom','elev','frm_id','frm_info','lat','lon','quality','surf'); + end + end + + if strcmpi(run_create_track_files.mode,'create') + % Concatenate on new season file + % ------------------------------------------------------------------- + fprintf('%s\tloaded\t%d\t%d\n', param.day_seg, length(tmp.lat), sum(~isnan(tmp.bottom))); + lat = [lat tmp.lat NaN]; + lon = [lon tmp.lon NaN]; + gps_time = [gps_time tmp.gps_time NaN]; + % Store full frame ID number 20190204_01_003 --> 2019020401003 + frm_id = [frm_id tmp.frm_id NaN]; + elev = [elev tmp.elev NaN]; + surf = [surf tmp.surf NaN]; + bottom = [bottom tmp.bottom NaN]; + quality = [quality tmp.quality NaN]; + % Store frame GPS time boundaries + frm_info.frm_id = [frm_info.frm_id tmp.frm_info.frm_id]; + frm_info.start_gps_time = [frm_info.start_gps_time tmp.frm_info.start_gps_time]; + frm_info.stop_gps_time = [frm_info.stop_gps_time tmp.frm_info.stop_gps_time]; + elseif ~strcmpi(run_create_track_files.mode,'append') + error('Valid modes are append or create. Invalid mode specified for run_create_track_files.mode: %s', run_create_track_files.mode); + end +end + +if strcmpi(run_create_track_files.mode,'create') + if isempty(lat) + %% No data error + error('There is no data available and so no season layer file can be saved.'); + + else + %% Save output for this season + out_fn_dir = ct_filename_support(param,'tracks',''); + out_fn_name = sprintf('tracks_%s_%s_%s.mat', param.post.ops.location, ct_output_dir(param.radar_name), param.season_name); + out_fn = fullfile(out_fn_dir,out_fn_name); + fprintf(' Saving %s\n\n', out_fn); + if ~exist(out_fn_dir,'dir') + mkdir(out_fn_dir); + end + file_type = 'tracks'; + file_version = '1'; + param = struct('day_seg',param.day_seg,'radar_name',param.radar_name, ... + 'season_name',param.season_name,'sw_version',param.sw_version); + ct_save(out_fn,'bottom','elev','file_type','file_version','frm_id','frm_info','gps_source','gps_time','lat','lon','param','quality','surf'); + end +end diff --git a/cresis-toolbox/+imb/run_slice_browser.m b/cresis-toolbox/+imb/run_slice_browser.m index 81f24565..e4ec19dd 100644 --- a/cresis-toolbox/+imb/run_slice_browser.m +++ b/cresis-toolbox/+imb/run_slice_browser.m @@ -9,7 +9,7 @@ %% User Settings % ========================================================================= -if 1 +if 0 % DOA methods param.doa_method_flag = true; day_seg = '20110317_03'; @@ -24,20 +24,13 @@ ice_mask_fn = ''; doa_limits = [-60 60]; % DOA limits for slice browsing (outside this limits will not be displayed) nadir_doa_lim = [-2 2]; % DOA range overwhich an estimated DOA is considered as nadir. This is usually the initial DOA limits of S-MAP. -elseif 0 +elseif 1 param.radar_name = 'rds'; param.season_name = '2018_Greenland_P3'; - out_type = 'music_imgs4_Nsig2'; - surfdata_source = 'surfData'; -% surfdata_source = 'surfData_englacial'; - param.day_seg = '20180404_02'; - frm = 1; + out_type = 'music3D'; + surfdata_source = 'surf'; param.day_seg = '20180406_01'; - frm = 1; -% param.day_seg = '20180418_06'; -% frm = 13; -% param.day_seg = '20180405_01'; -% frm = 56; + frm = 2; geotiff_fn = ct_filename_gis(param,fullfile('greenland','Landsat-7','Greenland_natural_90m.tif')); ice_mask_fn = ct_filename_gis(param,fullfile('greenland','IceMask','GimpIceMask_90m_v1.1.bin')); ice_mask_fn = ''; @@ -46,19 +39,20 @@ elseif 0 param.radar_name = 'rds'; param.season_name = '2014_Greenland_P3'; - out_type = 'music3D'; - surfdata_source = 'surfData'; + out_type = 'music3D_old'; + surfdata_source = ''; param.day_seg = '20140325_05'; - frm = 2; + frm = 1; geotiff_fn = ct_filename_gis(param,fullfile('canada','Landsat-7','Canada_90m.tif')); - ice_mask_fn = ct_filename_gis(param,fullfile('canada','ice_mask','03_rgi50_ArcticCanadaNorth','03_rgi50_ArcticCanadaNorth.bin')); + %ice_mask_fn = ct_filename_gis(param,fullfile('canada','ice_mask','03_rgi50_ArcticCanadaNorth','03_rgi50_ArcticCanadaNorth.bin')); + ice_mask_fn = ''; bounds_relative = [3 2 0 0]; elseif 0 param.radar_name = 'rds'; param.season_name = '2009_Antarctica_TO'; out_type = 'music3D'; - surfdata_source = 'surfData'; + surfdata_source = ''; param.day_seg = '20091224_01'; frm = 26; geotiff_fn = ct_filename_gis(param,fullfile('antarctica','Landsat-7','Antarctica_LIMA_480m.tif')); @@ -67,14 +61,14 @@ elseif 0 param.radar_name = 'rds'; - param.season_name = '2016_Antarctica_DC8'; - out_type = 'music3D'; - surfdata_source = 'surfData'; - param.day_seg = '20161117_06'; + param.season_name = '2019_Antarctica_Ground'; + out_type = 'music3D_paden'; + surfdata_source = 'surfData_paden'; + param.day_seg = '20200107_01'; frm = 1; geotiff_fn = ct_filename_gis(param,fullfile('antarctica','Landsat-7','Antarctica_LIMA_480m.tif')); ice_mask_fn = ''; - bounds_relative = [8 8 0 0]; + bounds_relative = [0 0 0 0]; else param.radar_name = 'rds'; @@ -92,25 +86,31 @@ %% Automated Section % ========================================================================= -fn = fullfile(ct_filename_out(param,out_type,''),sprintf('Data_%s_%03d.mat',param.day_seg,frm)); -if ~exist('run_slice_browser_fn','var') || ~strcmp(run_slice_browser_fn,fn) +echogram_fn = fullfile(ct_filename_out(param,out_type,''),sprintf('Data_%s_%03d.mat',param.day_seg,frm)); +if ~exist('run_slice_browser_fn','var') || ~strcmp(run_slice_browser_fn,echogram_fn) fprintf('Loading data (%s)\n', datestr(now)); - mdata = load(fn); + mdata = load(echogram_fn); proj = geotiffinfo(geotiff_fn); [DEM, R, tmp] = geotiffread(geotiff_fn); - run_slice_browser_fn = fn; + run_slice_browser_fn = echogram_fn; fprintf(' Done loading data (%s)\n', datestr(now)); +else + fprintf('Using already loaded data. Run "clear run_slice_browser_fn" to load new data (%s)\n', datestr(now)); + end if ~exist('surfdata_source','var') || isempty(surfdata_source) - surfdata_source = 'surfData'; + surfdata_source = 'surf'; end sb_param = []; sb_param.surfdata_fn = fullfile(ct_filename_out(param,surfdata_source,''),sprintf('Data_%s_%03d.mat',param.day_seg,frm)); if ~exist(sb_param.surfdata_fn,'file') - sb_param.surfdata_fn = ''; + surf = tomo.surfdata(mdata,surfdata_source); + surf.save_surfdata(sb_param.surfdata_fn); +else + surf = tomo.surfdata.update_file(sb_param.surfdata_fn,sb_param.surfdata_fn,echogram_fn); end if ~isfield(param,'doa_method_flag') || isempty(param.doa_method_flag) @@ -126,21 +126,21 @@ try; delete(obj); end; if sb_param.doa_method_flag % DOA method (DOA is passed in degrees) - obj = imb.slice_browser(mdata.Tomo.theta * 180/pi,[],sb_param); + obj = imb.slice_browser(mdata,[],sb_param); else % Beamforming method - obj = imb.slice_browser(10*log10(mdata.Topography.img),[],sb_param); - try; delete(viterbi_tool); end; - viterbi_tool = imb.slicetool_viterbi(); - obj.insert_tool(viterbi_tool); + obj = imb.slice_browser(mdata,[],sb_param); +% try; delete(viterbi_tool); end; +% viterbi_tool = imb.slicetool_viterbi(); +% obj.insert_tool(viterbi_tool); try; delete(trws_tool); end; trws_tool = imb.slicetool_trws(); obj.insert_tool(trws_tool); - try; delete(max_tool); end; - max_tool = imb.slicetool_max(); - obj.insert_tool(max_tool); +% try; delete(max_tool); end; +% max_tool = imb.slicetool_max(); +% obj.insert_tool(max_tool); try; delete(quality_tool); end; quality_tool = imb.slicetool_quality(); @@ -150,9 +150,9 @@ delete_tool = imb.slicetool_delete(); obj.insert_tool(delete_tool); - try; delete(threshold_tool); end; - threshold_tool = imb.slicetool_threshold(); - obj.insert_tool(threshold_tool); +% try; delete(threshold_tool); end; +% threshold_tool = imb.slicetool_threshold(); +% obj.insert_tool(threshold_tool); end if ~isempty(ice_mask_fn) [ice_mask_fn_dir ice_mask_fn_name] = fileparts(ice_mask_fn); diff --git a/cresis-toolbox/+imb/save_undo_stack.m b/cresis-toolbox/+imb/save_undo_stack.m index 9ce22b7d..514357fc 100644 --- a/cresis-toolbox/+imb/save_undo_stack.m +++ b/cresis-toolbox/+imb/save_undo_stack.m @@ -168,10 +168,11 @@ function save_undo_stack(undo_stack) if isempty(lay_idx) % Layer does not exist in this layerData file yet lay_idx = length(undo_stack.user_data.layer_info(frm).id) + 1; + Nx = length(undo_stack.user_data.layer_info(frm).gps_time); undo_stack.user_data.layer_info(frm).id(lay_idx) = cur_layer; - undo_stack.user_data.layer_info(frm).quality(lay_idx,:) = ones(size(undo_stack.user_data.layer_info(frm).gps_time)); - undo_stack.user_data.layer_info(frm).twtt(lay_idx,:) = nan(size(undo_stack.user_data.layer_info(frm).gps_time)); - undo_stack.user_data.layer_info(frm).type(lay_idx,:) = 2*ones(size(undo_stack.user_data.layer_info(frm).gps_time)); + undo_stack.user_data.layer_info(frm).quality(lay_idx,:) = ones(1,Nx); + undo_stack.user_data.layer_info(frm).twtt(lay_idx,:) = nan(1,Nx); + undo_stack.user_data.layer_info(frm).type(lay_idx,:) = 2*ones(1,Nx); end % Update quality @@ -259,9 +260,9 @@ function save_undo_stack(undo_stack) % Create a mask for the modified points that is this frame only Nx = sum(undo_stack.user_data.frame == frm); lay_idx = length(undo_stack.user_data.layer_info(frm).id) + 1; - undo_stack.user_data.layer_info(frm).twtt(lay_idx,:) = nan(1,Nx); - undo_stack.user_data.layer_info(frm).quality(lay_idx,:) = ones(1,Nx); - undo_stack.user_data.layer_info(frm).type(lay_idx,:) = 2*ones(1,Nx); + undo_stack.user_data.layer_info(frm).twtt(lay_idx,1:Nx) = nan(1,Nx); + undo_stack.user_data.layer_info(frm).quality(lay_idx,1:Nx) = ones(1,Nx); + undo_stack.user_data.layer_info(frm).type(lay_idx,1:Nx) = 2*ones(1,Nx); undo_stack.user_data.layer_info(frm).id(lay_idx) = id; layerdata_frms(end+1) = frm; end @@ -342,7 +343,7 @@ function save_undo_stack(undo_stack) else lay = undo_stack.user_data.layer_info(frm); layer_fn = undo_stack.user_data.filename{frm}; - ct_save(layer_fn,'-struct','lay') % saving to layerData file + ct_save(layer_fn,'-struct','lay') % saving to layer data file end end end diff --git a/cresis-toolbox/+imb/slice_browser.m b/cresis-toolbox/+imb/slice_browser.m index 8e7ca909..e5c9aafd 100644 --- a/cresis-toolbox/+imb/slice_browser.m +++ b/cresis-toolbox/+imb/slice_browser.m @@ -79,7 +79,7 @@ methods %% constructor/slice_browser: - function obj = slice_browser(data,h_control_image,param) + function obj = slice_browser(mdata,h_control_image,param) if ~exist('param','var') param = []; end @@ -110,7 +110,7 @@ end undo_param.id = []; obj.undo_stack = imb.undo_stack(undo_param); - obj.data = data; + obj.data = 10*log10(mdata.Tomo.img); obj.slice = 1; obj.plot_visibility = true; obj.bounds_relative = param.bounds_relative; @@ -125,13 +125,11 @@ %start(obj.slice_tool.timer) % Load surface data - if isfield(param,'surfdata_fn') && ~isempty(param.surfdata_fn) - obj.sd = tomo.surfdata(param.surfdata_fn,param); - obj.surfdata_fn = param.surfdata_fn; - else - obj.sd = tomo.surfdata(param); - obj.surfdata_fn = ''; - end + obj.sd = tomo.surfdata(param.surfdata_fn); + obj.sd.theta = mdata.Tomo.theta(:,1); + obj.sd.time = mdata.Time; + obj.sd.units('bins'); + obj.surfdata_fn = param.surfdata_fn; if ~isempty(h_control_image) obj.h_control_is_child = false; @@ -144,7 +142,7 @@ obj.h_control_axes = axes('Parent',obj.h_control_fig,'YDir','reverse'); hold(obj.h_control_axes,'on'); if ~param.doa_method_flag - obj.h_control_image = imagesc(squeeze(obj.data(:,floor(size(data,2)/2)+1,:)),'Parent',obj.h_control_axes); + obj.h_control_image = imagesc(squeeze(obj.data(:,floor(size(obj.data,2)/2)+1,:)),'Parent',obj.h_control_axes); colormap(obj.h_control_axes,parula(256)); xlabel(obj.h_control_axes,'Along-track range line'); ylabel(obj.h_control_axes,'Range bin'); @@ -333,19 +331,19 @@ set(obj.gui.prev10PB,'style','pushbutton') set(obj.gui.prev10PB,'string','<<') set(obj.gui.prev10PB,'Callback',@obj.prev10_button_callback) - set(obj.gui.prev10PB,'TooltipString','Move backward ten slices (<)'); + set(obj.gui.prev10PB,'TooltipString','Move backward five slices (k)'); obj.gui.next10PB = uicontrol('parent',obj.gui.left_panel); set(obj.gui.next10PB,'style','pushbutton') set(obj.gui.next10PB,'string','>>') set(obj.gui.next10PB,'Callback',@obj.next10_button_callback) - set(obj.gui.next10PB,'TooltipString','Move forward ten slices (>)'); + set(obj.gui.next10PB,'TooltipString','Move forward five slices (l)'); if ~obj.doa_method_flag obj.gui.savePB = uicontrol('parent',obj.gui.left_panel); set(obj.gui.savePB,'style','pushbutton') set(obj.gui.savePB,'string','(S)ave') set(obj.gui.savePB,'Callback',@obj.save_button_callback) - set(obj.gui.savePB,'TooltipString','(S)ave surfaces to file'); + set(obj.gui.savePB,'TooltipString','Save surfaces to file (shift-S)'); obj.gui.helpPB = uicontrol('parent',obj.gui.left_panel); set(obj.gui.helpPB,'style','pushbutton') @@ -544,12 +542,12 @@ function save_button_callback(obj,source,callbackdata) %% next10_button_callback function next10_button_callback(obj,source,callbackdata) - obj.change_slice(obj.slice + 10,false); + obj.change_slice(obj.slice + 5,false); end %% prev10_button_callback function prev10_button_callback(obj,source,callbackdata) - obj.change_slice(obj.slice - 10,false); + obj.change_slice(obj.slice - 5,false); end %% undo_sync @@ -566,7 +564,12 @@ function undo_sync(obj,source,callbackdata) = cmds_list{cmd_idx}{subcmd_idx}.redo.y; new_slice = cmds_list{cmd_idx}{subcmd_idx}.redo.slice; elseif strcmp(cmds_list{cmd_idx}{subcmd_idx}.type,'slice_dummy') - new_slice = cmds_list{cmd_idx}{subcmd_idx}.redo.slice; + if ~any(obj.slice==cmds_list{cmd_idx}{subcmd_idx}.redo.slice) + new_slice = cmds_list{cmd_idx}{subcmd_idx}.redo.slice(1); + fprintf('Redo slices %d-%d\n', min(cmds_list{cmd_idx}{subcmd_idx}.redo.slice), max(cmds_list{cmd_idx}{subcmd_idx}.redo.slice)); + else + new_slice = obj.slice; + end end end end @@ -580,7 +583,12 @@ function undo_sync(obj,source,callbackdata) = cmds_list{cmd_idx}{subcmd_idx}.undo.y; new_slice = cmds_list{cmd_idx}{subcmd_idx}.undo.slice; elseif strcmp(cmds_list{cmd_idx}{subcmd_idx}.type,'slice_dummy') - new_slice = cmds_list{cmd_idx}{subcmd_idx}.redo.slice; + if ~any(obj.slice==cmds_list{cmd_idx}{subcmd_idx}.redo.slice) + new_slice = cmds_list{cmd_idx}{subcmd_idx}.redo.slice(1); + fprintf('Undo slices %d-%d\n', min(cmds_list{cmd_idx}{subcmd_idx}.redo.slice), max(cmds_list{cmd_idx}{subcmd_idx}.redo.slice)); + else + new_slice = obj.slice; + end end end end @@ -888,17 +896,13 @@ function key_press(obj,src,event) end case 'period' - if ~obj.shift_pressed - obj.change_slice(obj.slice + 5,false); - else - obj.change_slice(obj.slice + 1,false); - end + obj.change_slice(obj.slice + 1,false); case 'comma' - if ~obj.shift_pressed - obj.change_slice(obj.slice - 5,false); - else - obj.change_slice(obj.slice - 1,false); - end + obj.change_slice(obj.slice - 1,false); + case 'k' + obj.change_slice(obj.slice - 5,false); + case 'l' + obj.change_slice(obj.slice + 5,false); case 'delete' surf_idx = get(obj.gui.surfaceLB,'Value'); @@ -1062,7 +1066,14 @@ function update_slice(obj,param) 'YData', tmp_y); end end - if ~isempty(get(obj.gui.surfaceLB,'String')) + surf_idx = get(obj.gui.surfaceLB,'value'); + surfaceLB_str = get(obj.gui.surfaceLB,'string'); + if ~isempty(get(obj.gui.surfaceLB,'String')) ... + && (isempty(surf_idx) || surf_idx == 0 || all(surf_idx ~= 1:length(surfaceLB_str))) + set(obj.gui.surfaceLB,'value',1); + surf_idx = 1; + end + if ~isempty(get(obj.gui.surfaceLB,'String')) && ~isempty(surf_idx) % Update surface selection related plots surf_idx = get(obj.gui.surfaceLB,'value'); x_select = obj.sd.surf(surf_idx).x(:,obj.slice); @@ -1231,7 +1242,8 @@ function insert_tool(obj, slice_browser_tool) %% Help function help_menu(obj) - fprintf('Key Short Cuts\n'); + fprintf('\nMouse Operations\n'); + fprintf('======================================================\n'); fprintf('\nZoom Mode\n'); fprintf('left-click and drag: zoom to selection\n'); @@ -1239,11 +1251,16 @@ function help_menu(obj) fprintf('right-click: zoom out at point\n'); fprintf('\nPointer Mode In "slice" window\n'); - fprintf('left-click: set layer point (or toggle logical value)\n'); - fprintf('right-click and drag: select points (shift-key holds selection)\n'); + fprintf('left-click: set ground truth point or toggle logical value if mask or quality selected\n'); + fprintf('right-click and drag: select points to operate on (shift-key holds selection)\n'); - fprintf('\nPointer Mode In "layer" and "echogram" window\n'); + fprintf('\nPointer Mode In "surface" and "echogram" window\n'); fprintf('left-click: sets current slice\n'); + fprintf('right-click and drag: select region to operate on (shift-key holds selection)\n'); + + fprintf('\nPointer Mode In "surface" window\n'); + fprintf('right-click and drag: select region and apply tool\n'); + fprintf(' For the quality tool, holding shift toggles setting true/false\n'); fprintf('\nAll Modes\n'); fprintf('scroll: zoom in/out at point\n'); @@ -1257,6 +1274,27 @@ function help_menu(obj) end end end + + fprintf('\nKeyboard Shortcuts\n'); + fprintf('======================================================\n'); + + fprintf('\nGeneral\n'); + fprintf('F1: print this help menu\n'); + fprintf('space: toggle surface visibility\n'); + fprintf('u: undo the last operation\n'); + fprintf('r: redo an operation that was undone with undo\n'); + fprintf('shift-S: save surfaces to surfdata file\n'); + fprintf('delete: deletes selected points (or sets to false if logical layer)\n'); + + fprintf('\nMovement\n'); + fprintf('period . : go forward 1 frame\n'); + fprintf('comma , : go forward 1 frame\n'); + fprintf('g: go to a specific frame\n'); + fprintf('k: go back 5 frames\n'); + fprintf('l: go forward 5 frames\n'); + fprintf('arrow-keys: pan left/right/up/down in the image\n'); + fprintf('z: toggle zoom mode on/off\n'); + fprintf('ctrl-z: zoom reset (zooms all the way out to show the full image)\n'); end %% getEventData diff --git a/cresis-toolbox/+imb/slicetool_icemask.m b/cresis-toolbox/+imb/slicetool_icemask.m index 6411cf84..2fe189ed 100644 --- a/cresis-toolbox/+imb/slicetool_icemask.m +++ b/cresis-toolbox/+imb/slicetool_icemask.m @@ -34,7 +34,11 @@ function delete(obj) function set_custom_data(obj,custom_data) param.mdata.twtt = custom_data.mdata.twtt; - param.mdata.theta = custom_data.mdata.theta_cal; + if isfield(custom_data.mdata.Tomo,'theta_cal') && ~isempty(custom_data.mdata.Tomo.theta_cal) + param.mdata.theta = custom_data.mdata.Tomo.theta_cal; + else + param.mdata.theta = custom_data.mdata.Tomo.theta; + end param.mdata.ice_mask = custom_data.mdata.ice_mask; param.mdata.param_array = custom_data.mdata.param_array; % rmfield(param,'mdata'); diff --git a/cresis-toolbox/+imb/slicetool_trws.m b/cresis-toolbox/+imb/slicetool_trws.m index 1ec23c19..1a7e4505 100644 --- a/cresis-toolbox/+imb/slicetool_trws.m +++ b/cresis-toolbox/+imb/slicetool_trws.m @@ -30,11 +30,22 @@ % .slice: current slice in 3D image (third index of .data) % .surf_idx: active surface % slices: array of slices to operate on (overrides sb.slice) + if sb.surf_idx < 1 || sb.surf_idx > length(sb.sd.surf) + return + end control_idx = sb.sd.surf(sb.surf_idx).gt; active_idx = sb.sd.surf(sb.surf_idx).active; surf_idx = sb.sd.surf(sb.surf_idx).top; mask_idx = sb.sd.surf(sb.surf_idx).mask; + physical_constants; + + try + max_loops = eval(get(obj.gui.max_loopsLE,'String')); + catch ME + error('Error in number of loops: %s', ME.getReport); + end + max_loops = uint32(max_loops); try eval_cmd = get(obj.gui.slice_rangeLE,'String'); slice_range = eval(eval_cmd); @@ -43,37 +54,45 @@ eval_cmd, ME.message); end try - num_loops = eval(get(obj.gui.numloopsLE,'String')); - catch ME - error('Error in number of loops: %s', ME.getReport); - end - try - threshold = eval(get(obj.gui.thresholdLE,'String')); + normalization = eval(get(obj.gui.normalizationLE,'String')); catch ME - error('Error in threshold: %s', ME.getReport); + error('Error in normalization: %s', ME.getReport); end try - range = eval(get(obj.gui.rangeLE,'String')); + gt_range = eval(get(obj.gui.gt_rangeLE,'String')); + gt_range = gt_range(1); catch ME - error('Error in range: %s', ME.getReport); + error('Error in ground truth range: %s', ME.getReport); end try - polynomial = eval(get(obj.gui.polynomialLE,'String')); + ct_weight_max = eval(get(obj.gui.ct_weightLE,'String')); + ct_weight_max = ct_weight_max(1); catch ME - error('Error in polynomial: %s', ME.getReport); + error('Error in cross-track smoothness weight: %s', ME.getReport); end try - mu_size = eval(get(obj.gui.correlationLE,'String')); + at_weight = eval(get(obj.gui.at_weightLE,'String')); + at_weight = single(at_weight(1)); catch ME - error('Error in correlation: %s', ME.getReport); + error('Error in along-track smoothness weight: %s', ME.getReport); end try - smooth = eval(get(obj.gui.smoothLE,'String')); + eval_cmd = get(obj.gui.tomo_layersLE,'String'); + if isempty(eval_cmd) + tomo_params = []; + else + tomo_params = eval(eval_cmd); + end catch ME - error('Error in smooth: %s', ME.getReport); + error('Error in tomography layers: %s', ME.getReport); end if get(obj.gui.select_maskCB,'Value') cols = find(sb.select_mask); + if isempty(cols) + % Nothing to be done + cmd = []; + return; + end else cols = 1:size(sb.data,2); end @@ -99,139 +118,163 @@ end fprintf('Apply %s to surface %d slices %d - %d\n', obj.tool_name, active_idx, slices(1), slices(end)); - gt = []; - if ~isempty(control_idx) - % Create ground truth input - % 1. Each column is one ground truth input - % 2. Row 1: relative slice/range-line, Row 2: x, Row 3: y - for idx = 1:length(slices) - slice = slices(idx); - mask = isfinite(sb.sd.surf(control_idx).x(:,slice)) ... - & isfinite(sb.sd.surf(control_idx).y(:,slice)); - mask(1:sb.bounds_relative(1)) = 0; - mask(end-sb.bounds_relative(2)+1:end) = 0; - gt = cat(2,gt,[(idx-1)*ones(1,sum(mask)); ... - sb.sd.surf(control_idx).x(mask,slice).'-1; ... - sb.sd.surf(control_idx).y(mask,slice).'+0.5]); - bottom_bin = sb.sd.surf(control_idx).y(33,slices); - end - else - bottom_bin = NaN*zeros(1,length(slices)); + % Get 3D image data + % ------------------------------------------------------------------- + trws_data = single(sb.data(:,:,slices)); + Nt = size(trws_data,1); + Nsv = size(trws_data,2); + Nx = size(trws_data,3); + + theta = sb.sd.theta; + + % Find the bin closest to nadir + [min_nadir,nadir_col] = min(abs(theta)); + if theta(nadir_col)~=0 + warning('Nadir steering vector column (theta == 0) not present, closest steering vector is theta = %.3f.', theta(nadir_col)); end + % Get surfaces needed to apply tool + % ------------------------------------------------------------------- + if isempty(control_idx) + gt_bins = NaN*sb.sd.surf(control_idx).y(:,slices); + else + gt_bins = round(sb.sd.surf(control_idx).y(:,slices)); + end if isempty(surf_idx) surf_bins = NaN*sb.sd.surf(active_idx).y(:,slices); + Surface = ones(1,Nt); else - surf_bins = sb.sd.surf(surf_idx).y(:,slices); + surf_bins = round(sb.sd.surf(surf_idx).y(:,slices)); + Surface = interp_finite(sb.sd.surf(surf_idx).y(nadir_col,:),1); + Surface = round(Surface(slices)); end - surf_bins(isnan(surf_bins)) = -1; - - bottom_bin(isnan(bottom_bin)) = -1; - - if isempty(mask_idx) - mask = ones(size(sb.data,2),length(slices)); + if isempty(active_idx) + active_bins = NaN*sb.sd.surf(active_idx).y(:,slices); + Bottom = Surface; else - mask = sb.sd.surf(mask_idx).y(:,slices); - end - - begin_slice = max(1, min(slices)-1); - end_slice = min(size(sb.data,3), max(slices)+1); - edge = [sb.sd.surf(active_idx).y(:,begin_slice), sb.sd.surf(active_idx).y(:,end_slice)]; - edge(mask(:,1) == 0,1) = surf_bins(mask(:,1) == 0,1); - edge(mask(:,end) == 0,2) = surf_bins(mask(:,end) == 0,end); - - trws_data = sb.data(:,:,slices); - trws_data(trws_data>threshold(2)) = threshold(2); - for idx = 1:length(slices) - for col = 1:size(sb.data,2) - tmp = trws_data(1:min(end,round(surf_bins(col,idx))+70),col,idx); - tmp(tmp>threshold(1)) = threshold(1); - trws_data(1:min(end,round(surf_bins(col,idx))+70),col,idx) = tmp; - end - end - if ~left_edge_en - edge(:,1) = -1; + active_bins = (sb.sd.surf(active_idx).y(:,slices)); + Bottom = interp_finite(sb.sd.surf(active_idx).y(nadir_col,:),NaN); + Bottom = round(Bottom(slices)); + Bottom(isnan(Bottom)) = Surface(isnan(Bottom)); end - if ~right_edge_en - edge(:,end) = -1; - end - if ~isempty(polynomial) - smooth_slope = polyval(polynomial, linspace(-1,1,size(sb.data,2)-1)); + if isempty(mask_idx) + ice_mask = ones(size(sb.data,2),length(slices)); else - smooth_slope = []; + ice_mask = sb.sd.surf(mask_idx).y(:,slices); end - - smooth_weight = smooth(1:2); - smooth_var = smooth(3); - mu = sinc(linspace(-1.5,1.5,mu_size)); - sigma = sum(mu)/20*ones(1,mu_size); - mask_dist = round(bwdist(mask == 0)); - bounds = [sb.bounds_relative(1) size(trws_data,2)-sb.bounds_relative(2)-1 -1 -1]; - mask_dist = round(mask_dist .* 9); - - clear DIM DIM_costmatrix; - global gRadar; - DIM = load(fullfile(gRadar.path, '+tomo', 'Layer_tracking_3D_parameters_Matrix.mat')); - DIM_costmatrix = DIM.Layer_tracking_3D_parameters; - DIM_costmatrix = DIM_costmatrix .* (200 ./ max(DIM_costmatrix(:))); + theta_ice = asin(sin(theta(:))/sqrt(er_ice)); - %% DoA-to-DoA transition model - % Obtained from geostatistical analysis of 2014 Greenland P3 - transition_mu = [0.000000, 0.000000, 2.590611, 3.544282, 4.569263, 5.536577, 6.476430, 7.416807, 8.404554, 9.457255, 10.442658, 11.413710, 12.354409, 13.332689, 14.364614, 15.381671, 16.428969, 17.398906, 18.418794, 19.402757, 20.383026, 21.391834, 22.399259, 23.359765, 24.369957, 25.344982, 26.301805, 27.307530, 28.274756, 28.947572, 29.691010, 32.977387, 34.203212, 34.897994, 35.667128, 36.579019, 37.558978, 38.548659, 39.540715, 40.550138, 41.534781, 42.547407, 43.552700, 44.537758, 45.553618, 46.561057, 47.547331, 48.530976, 49.516588, 50.536075, 51.562886, 52.574938, 53.552979, 54.554206, 55.559657, 56.574029, 57.591999, 58.552986, 59.562937, 60.551616, 61.549909, 62.551092, 63.045791, 63.540490]; - transition_sigma = [0.457749, 0.805132, 1.152514, 1.213803, 1.290648, 1.370986, 1.586141, 1.626730, 1.785789, 1.791043, 1.782936, 1.727153, 1.770210, 1.714973, 1.687484, 1.663294, 1.633185, 1.647318, 1.619522, 1.626555, 1.649593, 1.628138, 1.699512, 1.749184, 1.809822, 1.946782, 2.126822, 2.237959, 2.313358, 2.280555, 1.419753, 1.112363, 1.426246, 2.159619, 2.140899, 2.083267, 1.687420, 1.574745, 1.480296, 1.443887, 1.415708, 1.356100, 1.401891, 1.398477, 1.365730, 1.418647, 1.407810, 1.430151, 1.391357, 1.403471, 1.454194, 1.470535, 1.417235, 1.455086, 1.436509, 1.378037, 1.415834, 1.333177, 1.298108, 1.277559, 1.358260, 1.483521, 1.674642, 1.865764]; - - if length(transition_mu) ~= size(sb.data, 2) - transition_mu = imresize(transition_mu, [1 size(sb.data, 2)]); - end - - if length(transition_sigma) ~= size(sb.data, 2) - transition_sigma = imresize(transition_sigma, [1 size(sb.data, 2)]); - end - - if top_edge_en && ~isempty(cols) && cols(1) > bounds(1)+1 - % Add ground truth from top edge as long as top edge is bounded - gt = cat(2,gt,[slices-slices(1); ... - sb.sd.surf(active_idx).x(cols(1)-1,slices)-1; ... - sb.sd.surf(active_idx).y(cols(1)-1,slices)+0.5]); - end - - if bottom_edge_en && ~isempty(cols) && cols(end) < bounds(2) - % Add ground truth from top edge as long as top edge is bounded - gt = cat(2,gt,[slices-slices(1); ... - sb.sd.surf(active_idx).x(cols(end)+1,slices)-1; ... - sb.sd.surf(active_idx).y(cols(end)+1,slices)+0.5]); + master.GPS_time = sb.sd.gps_time(slices); + [master.Latitude,master.Longitude,master.Elevation] = ecef2geodetic(sb.sd.fcs.origin(1,slices),sb.sd.fcs.origin(2,slices),sb.sd.fcs.origin(3,slices),WGS84.ellipsoid); + master.Latitude = master.Latitude/180*pi; + master.Longitude = master.Longitude/180*pi; + + dt = sb.sd.time(2)-sb.sd.time(1); + dr = dt*c/2; + at_slope = single([diff(master.Elevation / dr) 0]); + + H = interp_finite(sb.sd.time(Surface))*c/2; + T = interp_finite(sb.sd.time(Bottom))*c/2/sqrt(er_ice); + theta_threshold = theta(:); + %theta_threshold(theta_threshold > pi/2*0.75) = pi/2*0.75; + %theta_threshold(theta_threshold < -pi/2*0.75) = -pi/2*0.75; + theta_threshold(theta_threshold > pi/2*0.75) = NaN; + theta_threshold(theta_threshold < -pi/2*0.75) = NaN; + R = 1./cos(theta_threshold) * H(:).' + 1./cos(theta_ice) * T(:).'; + %ct_slope = single([zeros(1,Nx); diff(R)/dr]+[diff(R)/dr; zeros(1,Nx)]); + %ct_slope(2:end-1,:) = ct_slope(2:end-1,:)/2; + ct_slope = [diff(R)/dr; nan(1,Nx)]; + %ct_slope = interp_finite(ct_slope,0,@(x,y,z) interp1(x,y,z,'linear','extrap'),[],'interp'); + ct_slope = single(interp_finite(ct_slope,0)); + + ct_weight = 1./(3+mean(abs(ct_slope),2)); + ct_weight = ct_weight_max*ct_weight./max(ct_weight); + ct_weight = single(ct_weight); + + if isempty(tomo_params) + bounds = uint32([ones(1,Nx); Nt*ones(1,Nx)]); + + else + for lay_idx = 1:length(tomo_params) + tomo_params(lay_idx).existence_check = false; + end + tomo_layers = opsLoadLayers(sb.sd.param,tomo_params); + + %% Interpolate tomo_layers information to mdata + for lay_idx = 1:length(tomo_layers) + ops_layer = []; + ops_layer{1}.gps_time = tomo_layers(lay_idx).gps_time; + + ops_layer{1}.type = tomo_layers(lay_idx).type; + ops_layer{1}.quality = tomo_layers(lay_idx).quality; + ops_layer{1}.twtt = tomo_layers(lay_idx).twtt; + ops_layer{1}.type(isnan(ops_layer{1}.type)) = 2; + ops_layer{1}.quality(isnan(ops_layer{1}.quality)) = 1; + lay = opsInterpLayersToMasterGPSTime(master,ops_layer,[300 60]); + tomo_layers(lay_idx).twtt_ref = lay.layerData{1}.value{2}.data; + end + tomo_top_bin = round(interp_finite(interp1(sb.sd.time,1:length(sb.sd.time),tomo_layers(1).twtt_ref),NaN)); + tomo_bottom_bin = round(interp_finite(interp1(sb.sd.time,1:length(sb.sd.time),tomo_layers(2).twtt_ref),NaN)); + + tomo_top_bin(isnan(tomo_top_bin)) = 1; + tomo_bottom_bin(isnan(tomo_bottom_bin)) = Nt; + bounds = uint32([tomo_top_bin; tomo_bottom_bin]); + bounds(bounds<1) = 1; + bounds(bounds>Nt) = Nt; + bounds = bounds - 1; end - rows = []; - if isfinite(range) && size(gt,2) > 0 - % Restrict search to range of rows around ground truth - rows = max(1,min(round(gt(3,:)-range))) : min(size(trws_data,1),max(round(gt(3,:)+range))); - if length(rows) < length(mu)+1 - error('Error: Range restriction leaves too few rows of data. Increase "Row range" option.'); + % Apply ground truth, surface, surface suppression and ice mask + % ------------------------------------------------------------------- + begin_slice = max(1, min(slices)-1); + end_slice = min(size(sb.data,3), max(slices)+1); + surf_bins = round(sb.sd.surf(surf_idx).y(:,slices)); + surf_weight = 30; + surf_range = 110; + for idx = 1:Nx + slice = slices(idx); + for col = 1:Nsv + if idx == 1 && left_edge_en && isfinite(active_bins(col,idx)) ... + || idx == Nx && right_edge_en && isfinite(active_bins(col,idx)) ... + || col ~= 1 && col == cols(1) && top_edge_en && isfinite(active_bins(col,idx)) ... + || col ~= Nsv && col == cols(end) && bottom_edge_en && isfinite(active_bins(col,idx)) + % No change for edge + trws_data([1:active_bins(col,idx)-1 active_bins(col,idx)+1:end], col, idx) = -10e9; + elseif ice_mask(col,idx) + if isfinite(gt_bins(col,idx)) + % Ground truth + trws_data([1:gt_bins(col,idx)-gt_range, gt_bins(col,idx)+gt_range:end], col, idx) = -10e9; + end + if isfinite(surf_bins(col,idx)) + if surf_bins(col,idx) < Nt + % Bottom below top + trws_data(1:surf_bins(col,idx), col, idx) = -10e9; + % Surface suppression + cur_surf_range = max(0,min(surf_range,Nt-surf_bins(col,idx))); + trws_data(surf_bins(col,idx) + (1:cur_surf_range), col, idx) ... + = trws_data(surf_bins(col,idx) + (1:cur_surf_range), col, idx) ... + -surf_weight/surf_range * (cur_surf_range:-1:1).'; + elseif surf_bins(col,idx) >= Nt + % Handle special case when surface is more than the last row + trws_data(1:Nt-1, col, idx) = -10e9; + end + end + elseif isfinite(surf_bins(col,idx)) + % Bottom equals top because there is no ice here + trws_data([1:surf_bins(col,idx)-1 surf_bins(col,idx)+1:end], col, idx) = -10e9; + end end - trws_data = trws_data(rows,:,:); - surf_bins = surf_bins - rows(1) + 1; - bottom_bin = bottom_bin - rows(1) + 1; - gt(3,:) = gt(3,:) - rows(1) + 1; end - - tic; - correct_surface = tomo.trws(double(trws_data), ... - double(surf_bins), double(bottom_bin), double(gt), double(mask), ... - double(mu), double(sigma), double(smooth_weight), double(smooth_var), ... - double(smooth_slope), double(edge), double(num_loops), int64(bounds), ... - double(mask_dist), double(DIM_costmatrix), ... - double(transition_mu), double(transition_sigma)); - toc; + % TRW-S + % ------------------------------------------------------------------- + tic; + correct_surface = tomo.trws2(trws_data,at_slope,at_weight,ct_slope,ct_weight,max_loops,bounds); fprintf(' %.2f sec per slice\n', toc/size(trws_data,3)); - if ~isempty(rows) - correct_surface = correct_surface + rows(1) - 1; - end - % Create cmd for surface change + % ------------------------------------------------------------------- cmd = []; for idx = start_slice_idx:end_slice_idx slice = slices(idx); @@ -245,8 +288,19 @@ cmd{end}.redo.y = correct_surface(cols,idx); cmd{end}.type = 'standard'; end - cmd{end+1}.redo.slice = sb.slice; - cmd{end}.undo.slice = sb.slice; + if any(sb.slice == slices(start_slice_idx:end_slice_idx)) + % Current slice in slice browser is one of the affected slices, my + % placing it first in the list this is the one the viewer will jump + % to when the undo or redo operation are run + cmd{end+1}.redo.slice = [sb.slice slices(start_slice_idx:end_slice_idx)]; + cmd{end}.undo.slice = [sb.slice slices(start_slice_idx:end_slice_idx)]; + else + % Current slice in slice browser is NOT one of the affected slices. + % During the undo or redo operation, the browser will now jump to + % the first affected slice. + cmd{end+1}.redo.slice = slices(start_slice_idx:end_slice_idx); + cmd{end}.undo.slice = slices(start_slice_idx:end_slice_idx); + end cmd{end}.type = 'slice_dummy'; end @@ -267,16 +321,17 @@ function create_option_ui(obj) set(obj.h_fig,'CloseRequestFcn',@obj.close_win); pos = get(obj.h_fig,'Position'); pos(3) = 220; - pos(4) = 240; + pos(4) = 215; set(obj.h_fig,'Position',pos); % Number of loops - obj.gui.numloopsTXT = uicontrol('Style','text','string','Loops'); + obj.gui.max_loopsTXT = uicontrol('Style','text','string','Max Loops'); + set(obj.gui.max_loopsTXT,'TooltipString','Maximum number of iterations that TRW-S will run. Default is 10.'); - obj.gui.numloopsLE = uicontrol('parent',obj.h_fig); - set(obj.gui.numloopsLE,'style','edit') - set(obj.gui.numloopsLE,'string','10') - set(obj.gui.numloopsLE,'TooltipString','Number of iterations.'); + obj.gui.max_loopsLE = uicontrol('parent',obj.h_fig); + set(obj.gui.max_loopsLE,'style','edit') + set(obj.gui.max_loopsLE,'string','10') + set(obj.gui.max_loopsLE,'TooltipString',get(obj.gui.max_loopsTXT,'TooltipString')); % Slice range obj.gui.slice_rangeTXT = uicontrol('Style','text','string','Slice range'); @@ -285,59 +340,60 @@ function create_option_ui(obj) obj.gui.slice_rangeLE = uicontrol('parent',obj.h_fig); set(obj.gui.slice_rangeLE,'style','edit') set(obj.gui.slice_rangeLE,'string','-5:6') - set(obj.gui.slice_rangeLE,'TooltipString','Enter a vector specifying relative range in slices. E.g. "-5:7".'); - - % Threshold - obj.gui.thresholdTXT = uicontrol('Style','text','string','Threshold'); - set(obj.gui.thresholdTXT,'TooltipString','Specify image threshold. First number is surface. Second number is whole column.'); - - obj.gui.thresholdLE = uicontrol('parent',obj.h_fig); - set(obj.gui.thresholdLE,'style','edit') - set(obj.gui.thresholdLE,'string','[13.5 inf]') - set(obj.gui.thresholdLE,'TooltipString','Specify image threshold. First number is surface. Second number is whole column.'); - - % Row range - obj.gui.rangeTXT = uicontrol('Style','text','string','Row range'); - set(obj.gui.rangeTXT,'TooltipString','Specify a number of rows beyond the ground truth to search.'); - - obj.gui.rangeLE = uicontrol('parent',obj.h_fig); - set(obj.gui.rangeLE,'style','edit') - set(obj.gui.rangeLE,'string','inf') - set(obj.gui.rangeLE,'TooltipString','Specify a number of rows beyond the ground truth to search.'); - - % Polynomial Shape - obj.gui.polynomialTXT = uicontrol('Style','text','string','Shape poly'); - set(obj.gui.polynomialTXT,'TooltipString','Specify a polynomial coefficients (e.g. -5x^2 is [-5 0 0]).'); - - obj.gui.polynomialLE = uicontrol('parent',obj.h_fig); - set(obj.gui.polynomialLE,'style','edit') - set(obj.gui.polynomialLE,'string','[]') - set(obj.gui.polynomialLE,'TooltipString','Specify polynomial coefficients (e.g. -5x^2 is [-5 0 0]).'); - - % Correlation length - obj.gui.correlationTXT = uicontrol('Style','text','string','Correlation'); - set(obj.gui.correlationTXT,'TooltipString','Specify a correlation length for surface impulse template.'); - - obj.gui.correlationLE = uicontrol('parent',obj.h_fig); - set(obj.gui.correlationLE,'style','edit') - set(obj.gui.correlationLE,'string','11') - set(obj.gui.correlationLE,'TooltipString','Specify a correlation length for surface impulse template.'); - - % Smooth - obj.gui.smoothTXT = uicontrol('Style','text','string','Smoothness'); - set(obj.gui.smoothTXT,'TooltipString','Specify surface smoothness ["slice weight" "column weight" "edges weight"].'); - - obj.gui.smoothLE = uicontrol('parent',obj.h_fig); - set(obj.gui.smoothLE,'style','edit') - set(obj.gui.smoothLE,'string','[22 22 32]') - set(obj.gui.smoothLE,'TooltipString','Specify surface smoothness ["slice weight" "column weight" "edges weight"].'); + set(obj.gui.slice_rangeLE,'TooltipString','Enter a vector specifying relative range in slices. Default is -5:6.'); + set(obj.gui.slice_rangeLE,'TooltipString',get(obj.gui.slice_rangeTXT,'TooltipString')); + + % Normalization + obj.gui.normalizationTXT = uicontrol('Style','text','string','Normalization'); + set(obj.gui.normalizationTXT,'TooltipString','Specify image thresholds for echo_norm function. First number is the minimum noise value. Second number is the maximum signal value. Default is [-inf inf] which allows the noise floor and maximum signal level to be estimated from the data.'); + + obj.gui.normalizationLE = uicontrol('parent',obj.h_fig); + set(obj.gui.normalizationLE,'style','edit') + set(obj.gui.normalizationLE,'string','[-inf inf]') + set(obj.gui.normalizationLE,'TooltipString',get(obj.gui.normalizationTXT,'TooltipString')); + + % Ground truth row range + obj.gui.gt_rangeTXT = uicontrol('Style','text','string','GT Range'); + set(obj.gui.gt_rangeTXT,'TooltipString','Specify a number of rows around the ground truth to search. Default is 5 bins.'); + + obj.gui.gt_rangeLE = uicontrol('parent',obj.h_fig); + set(obj.gui.gt_rangeLE,'style','edit') + set(obj.gui.gt_rangeLE,'string','5') + set(obj.gui.gt_rangeLE,'TooltipString',get(obj.gui.gt_rangeTXT,'TooltipString')); + + % Crosstrack Smoothness + obj.gui.ct_weightTXT = uicontrol('Style','text','string','CT Smooth'); + set(obj.gui.ct_weightTXT,'TooltipString','Specify weight for surface smoothness in cross-track dimension. Default is 0.01.'); + + obj.gui.ct_weightLE = uicontrol('parent',obj.h_fig); + set(obj.gui.ct_weightLE,'style','edit') + set(obj.gui.ct_weightLE,'string','0.01') + set(obj.gui.ct_weightLE,'TooltipString',get(obj.gui.ct_weightTXT,'TooltipString')); + + % Alongtrack Smoothness + obj.gui.at_weightTXT = uicontrol('Style','text','string','AT Smooth'); + set(obj.gui.at_weightTXT,'TooltipString','Specify weight for surface smoothness in along-track dimension. Default is 0.01.'); + + obj.gui.at_weightLE = uicontrol('parent',obj.h_fig); + set(obj.gui.at_weightLE,'style','edit') + set(obj.gui.at_weightLE,'string','0.01') + set(obj.gui.at_weightLE,'TooltipString',get(obj.gui.at_weightTXT,'TooltipString')); + + % Tomography layers + obj.gui.tomo_layersTXT = uicontrol('Style','text','string','Tomo Layers'); + set(obj.gui.tomo_layersTXT,'TooltipString','Specify layers to constrain the top row and bottom row of the surface. Default is struct(''name'',{''tomo_top'',''tomo_bottom''}).'); + + obj.gui.tomo_layersLE = uicontrol('parent',obj.h_fig); + set(obj.gui.tomo_layersLE,'style','edit') + set(obj.gui.tomo_layersLE,'string','struct(''name'',{''tomo_top'',''tomo_bottom''})') + set(obj.gui.tomo_layersLE,'TooltipString',get(obj.gui.tomo_layersTXT,'TooltipString')); % Select mask obj.gui.select_maskCB = uicontrol('parent',obj.h_fig); set(obj.gui.select_maskCB,'style','checkbox') set(obj.gui.select_maskCB,'string','Select') set(obj.gui.select_maskCB,'value',1) - set(obj.gui.select_maskCB,'TooltipString','Check to operate only on the selected region.'); + set(obj.gui.select_maskCB,'TooltipString','Check to operate only on the selected cross-track bins.'); % Left edge obj.gui.leftEdgeCB = uicontrol('parent',obj.h_fig); @@ -376,13 +432,13 @@ function create_option_ui(obj) obj.gui.table.offset = [0 0]; row = 1; col = 1; - obj.gui.table.handles{row,col} = obj.gui.numloopsTXT; + obj.gui.table.handles{row,col} = obj.gui.max_loopsTXT; obj.gui.table.width(row,col) = 70; obj.gui.table.height(row,col) = 20; obj.gui.table.width_margin(row,col) = 1; obj.gui.table.height_margin(row,col) = 1; col = 2; - obj.gui.table.handles{row,col} = obj.gui.numloopsLE; + obj.gui.table.handles{row,col} = obj.gui.max_loopsLE; obj.gui.table.width(row,col) = inf; obj.gui.table.height(row,col) = 20; obj.gui.table.width_margin(row,col) = 1; @@ -404,13 +460,13 @@ function create_option_ui(obj) row = row + 1; col = 1; - obj.gui.table.handles{row,col} = obj.gui.thresholdTXT; + obj.gui.table.handles{row,col} = obj.gui.normalizationTXT; obj.gui.table.width(row,col) = 70; obj.gui.table.height(row,col) = 20; obj.gui.table.width_margin(row,col) = 1; obj.gui.table.height_margin(row,col) = 1; col = 2; - obj.gui.table.handles{row,col} = obj.gui.thresholdLE; + obj.gui.table.handles{row,col} = obj.gui.normalizationLE; obj.gui.table.width(row,col) = inf; obj.gui.table.height(row,col) = 20; obj.gui.table.width_margin(row,col) = 1; @@ -418,13 +474,13 @@ function create_option_ui(obj) row = row + 1; col = 1; - obj.gui.table.handles{row,col} = obj.gui.rangeTXT; + obj.gui.table.handles{row,col} = obj.gui.gt_rangeTXT; obj.gui.table.width(row,col) = 70; obj.gui.table.height(row,col) = 20; obj.gui.table.width_margin(row,col) = 1; obj.gui.table.height_margin(row,col) = 1; col = 2; - obj.gui.table.handles{row,col} = obj.gui.rangeLE; + obj.gui.table.handles{row,col} = obj.gui.gt_rangeLE; obj.gui.table.width(row,col) = inf; obj.gui.table.height(row,col) = 20; obj.gui.table.width_margin(row,col) = 1; @@ -432,13 +488,13 @@ function create_option_ui(obj) row = row + 1; col = 1; - obj.gui.table.handles{row,col} = obj.gui.polynomialTXT; + obj.gui.table.handles{row,col} = obj.gui.ct_weightTXT; obj.gui.table.width(row,col) = 70; obj.gui.table.height(row,col) = 20; obj.gui.table.width_margin(row,col) = 1; obj.gui.table.height_margin(row,col) = 1; col = 2; - obj.gui.table.handles{row,col} = obj.gui.polynomialLE; + obj.gui.table.handles{row,col} = obj.gui.ct_weightLE; obj.gui.table.width(row,col) = inf; obj.gui.table.height(row,col) = 20; obj.gui.table.width_margin(row,col) = 1; @@ -446,13 +502,13 @@ function create_option_ui(obj) row = row + 1; col = 1; - obj.gui.table.handles{row,col} = obj.gui.correlationTXT; + obj.gui.table.handles{row,col} = obj.gui.at_weightTXT; obj.gui.table.width(row,col) = 70; obj.gui.table.height(row,col) = 20; obj.gui.table.width_margin(row,col) = 1; obj.gui.table.height_margin(row,col) = 1; col = 2; - obj.gui.table.handles{row,col} = obj.gui.correlationLE; + obj.gui.table.handles{row,col} = obj.gui.at_weightLE; obj.gui.table.width(row,col) = inf; obj.gui.table.height(row,col) = 20; obj.gui.table.width_margin(row,col) = 1; @@ -460,13 +516,13 @@ function create_option_ui(obj) row = row + 1; col = 1; - obj.gui.table.handles{row,col} = obj.gui.smoothTXT; + obj.gui.table.handles{row,col} = obj.gui.tomo_layersTXT; obj.gui.table.width(row,col) = 70; obj.gui.table.height(row,col) = 20; obj.gui.table.width_margin(row,col) = 1; obj.gui.table.height_margin(row,col) = 1; col = 2; - obj.gui.table.handles{row,col} = obj.gui.smoothLE; + obj.gui.table.handles{row,col} = obj.gui.tomo_layersLE; obj.gui.table.width(row,col) = inf; obj.gui.table.height(row,col) = 20; obj.gui.table.width_margin(row,col) = 1; diff --git a/cresis-toolbox/+insar/combine_passes.m b/cresis-toolbox/+insar/combine_passes.m deleted file mode 100644 index a26e4d64..00000000 --- a/cresis-toolbox/+insar/combine_passes.m +++ /dev/null @@ -1,619 +0,0 @@ -%% User Settings -passes = struct('frm',{},'wf_adc',{},'param_fn',{}); -% master_pass = struct('frm','20140429_01_067','wf_adc',[2 9],'param_fn','rds_param_2014_Greenland_P3.xls'); -master_pass = struct('frm','20140502_01_041','wf_adc',[2 9],'param_fn','rds_param_2014_Greenland_P3.xls'); - -if 1 - %% Summit Camp: 2012-2014 - - pass_name = sprintf('summit_2012_2014_wf2'); - param_fn = 'rds_param_2014_Greenland_P3.xls'; - wf = 2; - for adc = 2:16 - passes(end+1) = struct('frm','20140502_01_041','wf_adc',[wf adc],'param_fn',param_fn); - end - param_fn = 'rds_param_2012_Greenland_P3.xls'; - wf = 1; - for adc = 2:16 - passes(end+1) = struct('frm','20120330_03_008','wf_adc',[wf adc],'param_fn',param_fn); - end - passes(end+1) = master_pass; - -% % Start: -% 20120330_03_008: 72.646053 N, -37.898030 E, X:234.152 km, Y:-1879.297 km, 2012-03-30 15:10:28.01, 3264.13 Elevation, 0.00 Depth -% 20140502_01_041: 72.646347 N, -37.897234 E, X:234.171 km, Y:-1879.267 km, 2014-05-02 15:18:08.68, 3270.24 Elevation, 0.00 Depth -% % Stop: -% 20120330_03_008: 72.791389 N, -38.461623 E, X:213.822 km, Y:-1865.521 km, 2012-03-30 15:13:28.49, 3239.21 Elevation, 0.00 Depth -% 20140502_01_041: 72.791530 N, -38.461307 E, X:213.828 km, Y:-1865.510 km, 2014-05-02 15:21:13.16, 3243.05 Elevation, 0.00 Depth - - %Found using check_region - start = struct('lat', 72.646,'lon', -37.898); - stop = struct('lat', 72.791, 'lon', -38.461); - %Set dist min - dist_min = 300; - -elseif 0 - %% OIB P3 Greenland: 2011-2014 - -% pass_name = sprintf('rds_thule_2011_2014_wf1'); -% param_fn = 'rds_param_2014_Greenland_P3.xls'; -% wf = 1; -% for adc = 2:16 -% passes(end+1) = struct('frm','20140429_01_067','wf_adc',[wf adc],'param_fn',param_fn); -% end -% param_fn = 'rds_param_2011_Greenland_P3.xls'; -% wf = 1; -% for adc = 2:16 -% passes(end+1) = struct('frm','20110502_02_032','wf_adc',[wf adc],'param_fn',param_fn); -% end - -% pass_name = sprintf('rds_thule_2011_2014_wf2'); -% param_fn = 'rds_param_2014_Greenland_P3.xls'; -% wf = 2; -% for adc = 2:16 -% passes(end+1) = struct('frm','20140429_01_067','wf_adc',[wf adc],'param_fn',param_fn); -% end -% param_fn = 'rds_param_2011_Greenland_P3.xls'; -% wf = 1; -% for adc = 2:16 -% passes(end+1) = struct('frm','20110502_02_032','wf_adc',[wf adc],'param_fn',param_fn); -% end - -% pass_name = sprintf('rds_thule_2011_2014_wf3'); -% param_fn = 'rds_param_2014_Greenland_P3.xls'; -% wf = 3; -% for adc = 2:16 -% passes(end+1) = struct('frm','20140429_01_067','wf_adc',[wf adc],'param_fn',param_fn); -% end -% param_fn = 'rds_param_2011_Greenland_P3.xls'; -% wf = 2; -% for adc = 2:16 -% passes(end+1) = struct('frm','20110502_02_032','wf_adc',[wf adc],'param_fn',param_fn); -% end - - %% OIB P3 Greenland: 2012-2014 -% pass_name = sprintf('rds_thule_2012_2014_wf1'); -% param_fn = 'rds_param_2014_Greenland_P3.xls'; -% wf = 1; -% for adc = 2:16 -% passes(end+1) = struct('frm','20140429_01_067','wf_adc',[wf adc],'param_fn',param_fn); -% end -% param_fn = 'rds_param_2012_Greenland_P3.xls'; -% wf = 1; -% for adc = 2:16 -% passes(end+1) = struct('frm','20120516_01_089','wf_adc',[wf adc],'param_fn',param_fn); -% end - -% pass_name = sprintf('rds_thule_2012_2014_wf2'); -% param_fn = 'rds_param_2014_Greenland_P3.xls'; -% wf = 2; -% for adc = 2:16 -% passes(end+1) = struct('frm','20140429_01_067','wf_adc',[wf adc],'param_fn',param_fn); -% end -% param_fn = 'rds_param_2012_Greenland_P3.xls'; -% wf = 1; -% for adc = 2:16 -% passes(end+1) = struct('frm','20120516_01_089','wf_adc',[wf adc],'param_fn',param_fn); -% end - -% pass_name = sprintf('rds_thule_2012_2014_wf3'); -% param_fn = 'rds_param_2014_Greenland_P3.xls'; -% wf = 3; -% for adc = 2:16 -% passes(end+1) = struct('frm','20140429_01_067','wf_adc',[wf adc],'param_fn',param_fn); -% end -% param_fn = 'rds_param_2012_Greenland_P3.xls'; -% wf = 2; -% for adc = 2:16 -% passes(end+1) = struct('frm','20120516_01_089','wf_adc',[wf adc],'param_fn',param_fn); -% end -% - %% OIB P3 Greenland: 2013-2014 -% pass_name = sprintf('rds_thule_2013_2014_wf1'); -% param_fn = 'rds_param_2014_Greenland_P3.xls'; -% wf = 1; -% for adc = 2:16 -% passes(end+1) = struct('frm','20140429_01_067','wf_adc',[wf adc],'param_fn',param_fn); -% end -% param_fn = 'rds_param_2013_Greenland_P3.xls'; -% wf = 1; -% for adc = 1:7 -% passes(end+1) = struct('frm','20130419_01_004','wf_adc',[wf adc],'param_fn',param_fn); -% end -% -% pass_name = sprintf('rds_thule_2013_2014_wf2'); -% param_fn = 'rds_param_2014_Greenland_P3.xls'; -% wf = 2; -% for adc = 2:16 -% passes(end+1) = struct('frm','20140429_01_067','wf_adc',[wf adc],'param_fn',param_fn); -% end -% param_fn = 'rds_param_2013_Greenland_P3.xls'; -% wf = 1; -% for adc = 1:7 -% passes(end+1) = struct('frm','20130419_01_004','wf_adc',[wf adc],'param_fn',param_fn); -% end -% -% pass_name = sprintf('rds_thule_2013_2014_wf3'); -% param_fn = 'rds_param_2014_Greenland_P3.xls'; -% wf = 3; -% for adc = 2:16 -% passes(end+1) = struct('frm','20140429_01_067','wf_adc',[wf adc],'param_fn',param_fn); -% end -% param_fn = 'rds_param_2013_Greenland_P3.xls'; -% wf = 2; -% for adc = 1:7 -% passes(end+1) = struct('frm','20130419_01_004','wf_adc',[wf adc],'param_fn',param_fn); -% end - - %% OIB P3 Greenland: 2011 -% param_fn = 'rds_param_2011_Greenland_P3.xls'; -% wf = 1; -% pass_name = sprintf('rds_thule_20110502_02_032_wf%d',wf); -% for adc = 2:16 -% passes(end+1) = struct('frm','20110502_02_032','wf_adc',[wf adc],'param_fn',param_fn); -% end -% passes(end+1) = master_pass; - -% param_fn = 'rds_param_2011_Greenland_P3.xls'; -% wf = 1; -% pass_name = sprintf('rds_thule_20110506_01_004_wf%d',wf); -% for adc = 2:16 -% passes(end+1) = struct('frm','20110506_01_004','wf_adc',[wf adc],'param_fn',param_fn); -% end - -% param_fn = 'rds_param_2011_Greenland_P3.xls'; -% wf = 1; -% pass_name = sprintf('rds_thule_20110509_01_004_wf%d',wf); -% for adc = 2:16 -% passes(end+1) = struct('frm','20110509_01_004','wf_adc',[wf adc],'param_fn',param_fn); -% end - -% param_fn = 'rds_param_2011_Greenland_P3.xls'; -% wf = 1; -% pass_name = sprintf('rds_thule_20110509_02_034_wf%d',wf); -% for adc = 2:16 -% passes(end+1) = struct('frm','20110509_02_034','wf_adc',[wf adc],'param_fn',param_fn); -% end - - %% OIB P3 Greenland: 2012 -% param_fn = 'rds_param_2012_Greenland_P3.xls'; -% wf = 1; -% pass_name = sprintf('rds_thule_20120503_03_067_wf%d',wf); -% for adc = 2:16 -% passes(end+1) = struct('frm','20120503_03_067','wf_adc',[wf adc],'param_fn',param_fn); -% end - -% param_fn = 'rds_param_2012_Greenland_P3.xls'; -% wf = 1; -% pass_name = sprintf('rds_thule_20120516_01_089_wf%d',wf); -% for adc = 2:16 -% passes(end+1) = struct('frm','20120516_01_089','wf_adc',[wf adc],'param_fn',param_fn); -% end - - %% OIB P3 Greenland: 2013 -% param_fn = 'rds_param_2013_Greenland_P3.xls'; -% wf = 1; -% pass_name = sprintf('rds_thule_20130419_01_004_wf%d',wf); -% for adc = 1:7 -% passes(end+1) = struct('frm','20130419_01_004','wf_adc',[wf adc],'param_fn',param_fn); -% end - -% param_fn = 'rds_param_2013_Greenland_P3.xls'; -% wf = 1; -% pass_name = sprintf('rds_thule_20130426_01_004_wf%d',wf); -% for adc = 1:7 -% passes(end+1) = struct('frm','20130426_01_004','wf_adc',[wf adc],'param_fn',param_fn); -% end -% passes(end+1) = master_pass; - - %% OIB P3 Greenland: 2014 - - param_fn = 'rds_param_2014_Greenland_P3.xls'; - wf = 2; - pass_name = sprintf('rds_thule_combine_wf%d',wf); - for adc = 2:16 - passes(end+1) = struct('frm','20140429_01_067','wf_adc',[wf adc],'param_fn',param_fn); - end - for adc = 2:16 - passes(end+1) = struct('frm','20140429_01_005','wf_adc',[wf adc],'param_fn',param_fn); - end - for adc = 2:16 - passes(end+1) = struct('frm','20140514_01_004','wf_adc',[wf adc],'param_fn',param_fn); - end - for adc = 2:16 - passes(end+1) = struct('frm','20140514_01_066','wf_adc',[wf adc],'param_fn',param_fn); - end - for adc = 2:16 - passes(end+1) = struct('frm','20140515_02_004','wf_adc',[wf adc],'param_fn',param_fn); - end - for adc = 2:16 - passes(end+1) = struct('frm','20140515_02_069','wf_adc',[wf adc],'param_fn',param_fn); - end - for adc = 2:16 - passes(end+1) = struct('frm','20140521_02_033','wf_adc',[wf adc],'param_fn',param_fn); - end - -% % frms{end+1} = '20140429_01_004'; %sar processed stop is far -% % frms{end+1} = '20140429_01_066'; %outside of window -% % frms{end+1} = '20140501_01_044'; %sar processed %outside of window -% % frms{end+1} = '20140501_01_045'; %sar processed %Large baseline -% % frms{end+1} = '20140502_01_060'; %High altitude flight -% % frms{end+1} = '20140507_01_068'; %outside of window -% % frms{end+1} = '20140521_01_003'; %Lots of random noise - -% param_fn = 'rds_param_2014_Greenland_P3.xls'; -% wf = 2; -% pass_name = sprintf('rds_thule_20140429_01_067_wf%d',wf); -% for adc = 2:16 -% passes(end+1) = struct('frm','20140429_01_067','wf_adc',[wf adc],'param_fn',param_fn); -% end - -% param_fn = 'rds_param_2014_Greenland_P3.xls'; -% wf = 2; -% pass_name = sprintf('rds_thule_20140429_01_005_wf%d',wf); -% for adc = 2:16 -% passes(end+1) = struct('frm','20140429_01_005','wf_adc',[wf adc],'param_fn',param_fn); -% end - - %Found using check_region - start = struct('lat', 77.10,'lon', -62.3); - stop = struct('lat', 77.13, 'lon', -61.9); - %Set dist min - dist_min = 300; -elseif 0 - %% TO DTU Greenland: North Line - param_fn = 'rds_param_2016_Greenland_TOdtu.xls'; - pass_name = sprintf('2016_Greenland_TOdtu_north'); - passes(end+1) = struct('frm','20161107_02_003','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161107_02_006','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161107_03_003','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161107_03_006','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161107_07_003','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161108_01_003','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161108_01_006','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161108_01_009','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161108_02_002','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161110_01_003','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161110_01_006','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161110_01_009','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161110_02_003','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161110_03_001','wf_adc',[1 1],'param_fn',param_fn); - start = struct('lat',69.174061,'lon',-49.668096); - stop = struct('lat',69.196834,'lon',-48.906555); - dist_min = 2000; -elseif 0 - %% TO DTU Greenland: Middle Line - param_fn = 'rds_param_2016_Greenland_TOdtu.xls'; - pass_name = sprintf('2016_Greenland_TOdtu_middle'); - passes(end+1) = struct('frm','20161107_02_001','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161107_02_004','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161107_03_001','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161107_03_004','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161107_07_001','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161108_01_001','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161108_01_004','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161108_01_007','wf_adc',[1 1],'param_fn',param_fn); % Surface does not show up in interferogram - passes(end+1) = struct('frm','20161108_01_010','wf_adc',[1 1],'param_fn',param_fn); % Surface does not show up in interferogram - passes(end+1) = struct('frm','20161110_01_001','wf_adc',[1 1],'param_fn',param_fn); % Surface is faint - passes(end+1) = struct('frm','20161110_01_004','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161110_01_007','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161110_02_001','wf_adc',[1 1],'param_fn',param_fn); % 2 km short of stop point - passes(end+1) = struct('frm','20161110_02_004','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161110_03_007','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161110_03_008','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161111_01_001','wf_adc',[1 1],'param_fn',param_fn); % 460 m short of start point - passes(end+1) = struct('frm','20161111_01_002','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161111_01_003','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161111_01_004','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161111_01_005','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161111_01_006','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161111_01_007','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161111_01_008','wf_adc',[1 1],'param_fn',param_fn); - start = struct('lat',69.163,'lon',-49.675); - stop = struct('lat',69.176,'lon',-48.888); - dist_min = 2500; -elseif 0 - %% TO DTU Greenland: South Line - param_fn = 'rds_param_2016_Greenland_TOdtu.xls'; - pass_name = sprintf('2016_Greenland_TOdtu_south'); - passes(end+1) = struct('frm','20161107_02_002','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161107_02_005','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161107_03_002','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161107_03_005','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161107_07_002','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161108_01_002','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161108_01_005','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161108_01_008','wf_adc',[1 1],'param_fn',param_fn); % Surface does not show up in interferogram - passes(end+1) = struct('frm','20161108_02_001','wf_adc',[1 1],'param_fn',param_fn); % Surface does not show up in interferogram - passes(end+1) = struct('frm','20161110_01_002','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161110_01_005','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161110_01_008','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161110_02_002','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161110_02_005','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','','wf_adc',[1 1],'param_fn',param_fn); - start = struct('lat',69.149,'lon',-49.666); - stop = struct('lat',69.162,'lon',-48.846); - dist_min = 2000; -elseif 0 - %% TO DTU Iceland: South Line - param_fn = 'rds_param_2016_Greenland_TOdtu.xls'; - pass_name = sprintf('2016_Greenland_TOdtu_iceland_south'); - passes(end+1) = struct('frm','20161101_03_004','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161101_01_002','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161101_02_002','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161101_03_002','wf_adc',[1 1],'param_fn',param_fn); - start = struct('lat',64.802,'lon',-18.850); - stop = struct('lat',64.807,'lon',-19.105); - dist_min = 2000; -elseif 1 - %% TO DTU Iceland: North Line - param_fn = 'rds_param_2016_Greenland_TOdtu.xls'; - pass_name = sprintf('2016_Greenland_TOdtu_iceland_north'); - passes(end+1) = struct('frm','20161101_01_003','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161101_02_001','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161101_02_003','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161101_03_003','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20161101_04_001','wf_adc',[1 1],'param_fn',param_fn); - start = struct('lat',64.843,'lon',-19.203); - stop = struct('lat',64.804,'lon',-18.909); - dist_min = 2000; -elseif 0 - %% G1XB Russell Glacier: Good Quality Line - param_fn = 'rds_param_2016_Greenland_G1XB.xls'; - pass_name = sprintf('2016_Greenland_G1XB_good'); - passes(end+1) = struct('frm','20160413_01_001','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20160413_01_002','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20160413_02_001','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20160413_02_002','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20160413_02_003','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20160413_02_004','wf_adc',[1 1],'param_fn',param_fn); - start = struct('lat',67.092809,'lon',-50.204091); - stop = struct('lat',67.096958,'lon',-50.054023); - dist_min = 100; -elseif 0 - %% G1XB Russell Glacier: Medium Quality Line - param_fn = 'rds_param_2016_Greenland_G1XB.xls'; - pass_name = sprintf('2016_Greenland_G1XB_medium'); - passes(end+1) = struct('frm','20160416_01_002','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20160416_01_003','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20160416_01_004','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20160416_01_005','wf_adc',[1 1],'param_fn',param_fn); - % passes(end+1) = struct('frm','20160417_04_002','wf_adc',[1 1],'param_fn',param_fn); % LARGE BASELINE - % passes(end+1) = struct('frm','20160417_04_003','wf_adc',[1 1],'param_fn',param_fn); % LARGE BASELINE - passes(end+1) = struct('frm','20160417_04_004','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20160417_04_005','wf_adc',[1 1],'param_fn',param_fn); - start = struct('lat',67.097188,'lon',-50.219048); - stop = struct('lat',67.101868,'lon',-50.047311); - dist_min = 100; -else - %% G1XB Russell Glacier: Bad Quality Line - param_fn = 'rds_param_2016_Greenland_G1XB.xls'; - pass_name = sprintf('2016_Greenland_G1XB_bad'); - passes(end+1) = struct('frm','20160417_01_001','wf_adc',[1 1],'param_fn',param_fn); - % passes(end+1) = struct('frm','20160417_01_002','wf_adc',[1 1],'param_fn',param_fn); % GPS BAD - passes(end+1) = struct('frm','20160417_02_002','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20160417_02_003','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20160417_02_004','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20160417_02_005','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20160417_03_002','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20160417_03_003','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20160417_03_004','wf_adc',[1 1],'param_fn',param_fn); - passes(end+1) = struct('frm','20160417_03_005','wf_adc',[1 1],'param_fn',param_fn); - frms{end+1} = '20160417_01_001'; - % frms{end+1} = '20160417_01_002'; GPS BAD - frms{end+1} = '20160417_02_002'; - frms{end+1} = '20160417_02_003'; - frms{end+1} = '20160417_02_004'; - frms{end+1} = '20160417_02_005'; - frms{end+1} = '20160417_03_002'; - frms{end+1} = '20160417_03_003'; - frms{end+1} = '20160417_03_004'; - frms{end+1} = '20160417_03_005'; - start = struct('lat',67.102454,'lon',-50.192878); - stop = struct('lat',67.108618,'lon',-49.968087); - dist_min = 100; -end - -%% Automated -for passes_idx = 1:length(passes) - if ~isfield(passes(passes_idx),'wf_adc') || isempty(passes(passes_idx).wf_adc) - passes(passes_idx).wf_adc = [1 1]; - end -end - -enable_load_sar = true; % Set to false if debugging to avoid reloading -if enable_load_sar - % Load SAR data - metadata = []; - data = []; - for passes_idx = 1:length(passes) - param = []; - param.day_seg = passes(passes_idx).frm(1:11); - param = read_param_xls(ct_filename_param(passes(passes_idx).param_fn),param.day_seg); - - param.load_sar_data.fn = ''; % Leave empty for default - - % Start and stop chunk to load (inf for second element loads to the end) - param.load_sar_data.chunk = [1 inf]; - - param.load_sar_data.sar_type = 'fk'; - - frm = str2double(passes(passes_idx).frm(13:end)); - param.load_sar_data.frame = frm; - - param.load_sar_data.subap = 1; - - % (wf,adc) pairs to load - param.load_sar_data.imgs = {passes(passes_idx).wf_adc}; - - % Combine waveforms parameters - param.load_sar_data.wf_comb = 10e-6; - - % Debug level (1 = default) - param.load_sar_data.debug_level = 2; - - % Combine receive channels - param.load_sar_data.combine_channels = 0; - - % Take abs()^2 of the data (only runs if combine_channels runs) - param.load_sar_data.incoherent = 0; - - % Combine waveforms (only runs if incoherent runs) - param.load_sar_data.combine_waveforms = 0; - - % Parameters for local_detrend (cmd == 5 disables, only runs if incoherent runs) - param.load_sar_data.detrend.cmd = 3; - param.load_sar_data.detrend.B_noise = [100 200]; - param.load_sar_data.detrend.B_sig = [1 10]; - param.load_sar_data.detrend.minVal = -inf; - - [data{passes_idx},metadata{passes_idx}] = load_sar_data(param); - - metadata{passes_idx}.frm = frm; - end -end - -%% Find start/stop points and extract radar passes -physical_constants; -[start.x,start.y,start.z] = geodetic2ecef(start.lat/180*pi,start.lon/180*pi,0,WGS84.ellipsoid); -[stop.x,stop.y,stop.z] = geodetic2ecef(stop.lat/180*pi,stop.lon/180*pi,0,WGS84.ellipsoid); - -pass = []; - -%% Go through each frame and extract the pass(es) from that frame -% NOTE: This code looks for every pass in the frame (i.e. a frame may -% contain multiple passes and this code should find each). -for passes_idx = 1:length(passes) - % Find the distance to the start - start_ecef = [start.x;start.y;start.z]; - stop_ecef = [stop.x;stop.y;stop.z]; - radar_ecef = []; - [radar_ecef.x,radar_ecef.y,radar_ecef.z] = geodetic2ecef(metadata{passes_idx}.lat/180*pi, ... - metadata{passes_idx}.lon/180*pi,0*metadata{passes_idx}.elev, ... - WGS84.ellipsoid); - radar_ecef = [radar_ecef.x; radar_ecef.y; radar_ecef.z]; - - %% Collect the closest point every time the trajectory passes near (pass(pass_idx).surface(rline)) = pass(pass_idx).surface(rline)*c/2 ... - + (time(time>pass(pass_idx).surface(rline)) - pass(pass_idx).surface(rline))*c/2/sqrt(er_ice); - - pixels = pass(pass_idx).ecef(:,rline) + pass(pass_idx).z(:,rline)*range; - - % Compute the location of all pixels from this range line in the master - % line coordinate system FCS. - Tfcs = [ref.x, ref.y, ref.z]; - pass(pass_idx).pixels(:,:,rline) = (pixels - ref.origin) / Tfcs; - end - end - - pass(pass_idx).along_track_slave = geodetic_to_along_track(pass(pass_idx).lat,pass(pass_idx).lon,pass(pass_idx).elev); - - if 0 - % Debug plot showing indexes for alignment of passes - figure(h_fig_ref_idx); - plot(pass(pass_idx).ref_idx) - drawnow; - end - - % Resample images and position vectors onto a common along-track axes - % 1. Oversample slave data by 10x in along track - Mx = 10; - Nx = size(pass(pass_idx).data,2); - data_oversample = interpft(pass(pass_idx).data.',Mx*Nx); - % 2. Interpolate to find the oversampled slave axes - along_track_oversample = interp1(0:Nx-1, ... - pass(pass_idx).along_track, (0:Nx*Mx-1)/Mx,'linear','extrap'); - % 3. Interpolate oversampled slave data onto master along track axes - pass(pass_idx).ref_data = interp1(along_track_oversample, ... - data_oversample, along_track,'linear','extrap').'; - - pass(pass_idx).ref_y = interp1(pass(pass_idx).along_track, ... - pass(pass_idx).ref_y, along_track,'linear','extrap').'; - pass(pass_idx).ref_z = interp1(pass(pass_idx).along_track, ... - pass(pass_idx).ref_z, along_track,'linear','extrap').'; - - % Apply fixed coregistration time shift - freq = pass(pass_idx).wfs(pass(pass_idx).wf).freq; - freq = freq - freq(1); % Remove center frequency offset - dt = coregistration_time_shift(pass_idx) * (pass(pass_idx).wfs(pass(pass_idx).wf).time(2)-pass(pass_idx).wfs(pass(pass_idx).wf).time(1)); - pass(pass_idx).ref_data = ifft(bsxfun(@times,fft(pass(pass_idx).ref_data),exp(-1i*2*pi*freq*dt))); - - if insar_mode == 1 - % Motion compensation of FCS z-motion - for rline = 1:size(pass(pass_idx).ref_data,2) - % Convert z-offset into time-offset assuming nadir DOA - dt = pass(pass_idx).ref_z(rline)/(c/2); - pass(pass_idx).ref_data(:,rline) = ifft(fft(pass(pass_idx).ref_data(:,rline)) ... - .*exp(1i*2*pi*pass(pass_idx).wfs(pass(pass_idx).wf).freq*dt) ); - end - elseif insar_mode == 2 || insar_mode == 4 - % Co-register images using GPS and nadir squint angle assumption - % - % Motion compensation of FCS z-motion without center frequency so there - % is no phase shift. - freq = pass(pass_idx).wfs(pass(pass_idx).wf).freq; - freq = freq - freq(1); % Remove center frequency offset - for rline = 1:size(pass(pass_idx).ref_data,2) - % Convert z-offset into time-offset assuming nadir DOA - dt = pass(pass_idx).ref_z(rline)/(c/2); - pass(pass_idx).ref_data(:,rline) = ifft(fft(pass(pass_idx).ref_data(:,rline)) ... - .*exp(1i*2*pi*freq*dt) ); - end - elseif insar_mode == 3 - % Motion compensation of FCS z-motion and slope compensation - - % True time delay shift for z-offset - for rline = 1:size(pass(pass_idx).ref_data,2) - % Convert z-offset into time-offset assuming nadir DOA - dt = pass(pass_idx).ref_z(rline)/(c/2); - pass(pass_idx).ref_data(:,rline) = ifft(fft(pass(pass_idx).ref_data(:,rline)) ... - .*exp(1i*2*pi*pass(pass_idx).wfs(pass(pass_idx).wf).freq*dt) ); - end - - % Phase only correction for slope - if 1 - % Using file generated from this dataset - fn_slope = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_insar/summit_2012_2014_wf2_slope.mat'; - % fn_slope = fullfile(fn_dir,[fn_name '_slope.mat']); - load(fn_slope,'slope','GPS_time','Latitude','Longitude','Elevation','Time','Surface'); - slope = interp1(GPS_time,slope.',pass(baseline_master_idx).gps_time).'; - slope = interp_finite(slope.').'; - slope = interp1(Time,slope,pass(pass_idx).wfs(pass(pass_idx).wf).time); - slope = interp_finite(slope); - - pass(pass_idx).ref_data = pass(pass_idx).ref_data .* exp(-1i*4*pi*pass(pass_idx).wfs(pass(pass_idx).wf).freq(1)/c *bsxfun(@times,sin(slope),pass(pass_idx).ref_y(:).')); - - elseif 1 - % Using file generated from another dataset - fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_insar/rds_thule_20140429_01_067_wf2_slope.mat'; - - % TBD - - end - - elseif 0 - % Co-register images using cross-correlation - keyboard - end - - % Match time axis to baseline_master_idx - if 0 - pass(pass_idx).ref_data = interp1(pass(pass_idx).wfs(pass(pass_idx).wf).time, pass(pass_idx).ref_data, pass(baseline_master_idx).wfs(pass(baseline_master_idx).wf).time, 'linear', 0); - else - Mt = 4; - Nt = length(pass(pass_idx).wfs(pass(pass_idx).wf).time); - pass(pass_idx).ref_data = interpft(pass(pass_idx).ref_data,Mt*Nt); - dt = pass(pass_idx).wfs(pass(pass_idx).wf).time(2)-pass(pass_idx).wfs(pass(pass_idx).wf).time(1); - time_Mt = pass(pass_idx).wfs(pass(pass_idx).wf).time(1) + dt/Mt*(0:Mt*Nt-1); - pass(pass_idx).ref_data = interp1(time_Mt, pass(pass_idx).ref_data, pass(baseline_master_idx).wfs(pass(baseline_master_idx).wf).time, 'linear', 0); - end - - if 0 - % Normalize surface phase - Nt = size(pass(pass_idx).ref_data,1); - Nx = size(pass(pass_idx).ref_data,2); - H = pass(baseline_master_idx).ref_data(round(ref.surface_bin)+(0:Nx-1)*Nt) .* conj(pass(pass_idx).ref_data(round(ref.surface_bin)+(0:Nx-1)*Nt)); - H = exp(1i*angle(H)); - pass(pass_idx).ref_data = bsxfun(@times,pass(pass_idx).ref_data,H); - end - - % Concatenate data into a single matrix - data = cat(3,data,pass(pass_idx).ref_data); -end - -%% Apply equalization -% ----------------------- -if insar_mode == 2 || insar_mode == 3 || insar_mode == 4 - equalization = reshape(equalization,[1 1 numel(equalization)]); - data = bsxfun(@times,data,1./equalization(:,:,pass_en_idxs)); -end - -if 0 - %% Coregister: Data Dependent method to estimate System Time Delay - % Apply fixed coregistration time shift - coherence_sum = []; - coregistration_time_shifts = -2:0.05:2; - % coregistration_time_shifts = -1.6:0.01:-1.3; % For 2013 - for pass_out_idx = 1:length(pass_en_idxs) - pass_idx = pass_en_idxs(pass_out_idx); - fprintf('%d of %d (pass %d)\n', pass_out_idx, length(pass_en_idxs), pass_idx); - - freq = pass(baseline_master_idx).wfs(pass(baseline_master_idx).wf).freq; - freq = freq - freq(1); % Remove center frequency offset - for coregistration_time_shift_idx = 1:length(coregistration_time_shifts) - coregistration_time_shift = coregistration_time_shifts(coregistration_time_shift_idx); - dt = coregistration_time_shift * (pass(pass_idx).wfs(pass(pass_idx).wf).time(2)-pass(pass_idx).wfs(pass(pass_idx).wf).time(1)); - adjusted = ifft(bsxfun(@times,fft(data(:,:,pass_idx)),exp(-1i*2*pi*freq*dt))); - coherence = fir_dec(adjusted(rbins,:) .* conj(data(rbins,:,master_idx)) ./ abs(adjusted(rbins,:) .* data(rbins,:,master_idx)),ones(1,7)/7,1); - coherence = fir_dec(coherence.',ones(1,3)/3,1).'; - coherence = abs(coherence); - % coherence_sum(coregistration_time_shift_idx) = sum(coherence(coherence>0.5)); - coherence_sum(coregistration_time_shift_idx,pass_idx) = sum(coherence(coherence>0)); - % TriangleRayIntersection fprintf('%g %.2f\n', coregistration_time_shift, coherence_sum(coregistration_time_shift_idx)); - % imagesc(coherence); colormap(1-gray(256)); - % pause - end - [~,coregistration_time_shift_idx] = max(coherence_sum); - coregistration_time_shift = coregistration_time_shifts(coregistration_time_shift_idx) - end - figure(2000); clf; - plot(coregistration_time_shifts,coherence_sum) - [~,coregistration_time_shift_idx] = max(coherence_sum); - coregistration_time_shift = coregistration_time_shifts(coregistration_time_shift_idx); - fprintf('%g ', coregistration_time_shift); fprintf('\n'); - return -end - -%% Plot interferograms -h_data_axes = []; -new_equalization = []; -master_out_idx = find(pass_en_idxs == master_idx); -for pass_out_idx = 1:length(pass_en_idxs) - pass_idx = pass_en_idxs(pass_out_idx); - - figure(pass_idx); clf; - set(pass_idx,'WindowStyle','docked') - if 0 - imagesc(lp(data(rbins,:,pass_idx))) - colormap(1-gray(256)); - ylabel('Range bin'); - xlabel('Range line'); - title(sprintf('%s_%03d %d',pass(pass_idx).param_sar.day_seg,pass(pass_idx).param_sar.load.frm,pass(pass_idx).direction),'interpreter','none') - %caxis([-90 8]); - else - % Form interferogram (couple options) - complex_data = fir_dec(data(rbins,:,pass_out_idx) .* conj(data(rbins,:,master_out_idx)),ones(1,11)/11,1); - if ~exist('equalization_rlines','var') || isempty(equalization_rlines) - new_equalization(pass_idx) = mean(complex_data(:)); % equalization only valid when motion compensation with phase is used - else - new_equalization(pass_idx) = mean(mean(complex_data(:,equalization_rlines))); % equalization only valid when motion compensation with phase is used - end - if insar_mode == 1 - complex_data = complex_data ./ new_equalization(pass_idx); - end - % Plot interferogram - if insar_mode == 4 - imagesc(lp(data(rbins,:,pass_out_idx))); - colormap(1-gray(256)); - h_colorbar = colorbar; - set(get(h_colorbar,'ylabel'),'string','Relative power (dB)'); - else - coherence = abs(fir_dec(data(rbins,:,pass_out_idx) .* conj(data(rbins,:,master_out_idx)) ./ abs(data(rbins,:,pass_out_idx) .* data(rbins,:,master_out_idx)),ones(1,11)/11,1)) ... - .* exp(1i*angle(complex_data)); - imagesc(hsv_plot_coherence(coherence,[0 1])); - colormap(hsv(256)) - h_colorbar = colorbar; - caxis([-pi pi]) - set(get(h_colorbar,'ylabel'),'string','Angle (radians)'); - end - ylabel('Range bin'); - xlabel('Range line'); - title(sprintf('%s_%03d %d',pass(pass_idx).param_sar.day_seg,pass(pass_idx).param_sar.load.frm,pass(pass_idx).direction),'interpreter','none') - - if 0 - imagesc(abs(coherence)); - colormap(1-gray(256)) - h_colorbar = colorbar; - caxis([0 1]) - set(get(h_colorbar,'ylabel'),'string','Coherence'); - ylabel('Range bin'); - xlabel('Range line'); - title(sprintf('%s_%03d %d',pass(pass_idx).param_sar.day_seg,pass(pass_idx).param_sar.load.frm,pass(pass_idx).direction),'interpreter','none') - end - - end - h_data_axes(end+1) = gca; - -end -linkaxes(h_data_axes,'xy'); - -fprintf('=============================================\n'); -fprintf('New equalization\n'); -fprintf('%.1f ', lp(new_equalization)-mean(lp(new_equalization(pass_en_idxs)))); -fprintf('\n'); -fprintf('%.1f ', angle(new_equalization)*180/pi) -fprintf('\n'); -fprintf('=============================================\n'); - -%% Plot baseline -h_fig_baseline = figure(200); clf; -h_plot_baseline = []; -h_legend_baseline = {}; -for pass_out_idx = 1:length(pass_en_idxs) - pass_idx = pass_en_idxs(pass_out_idx); - - base_line ... - = sqrt( (pass(pass_idx).ref_z - pass(master_idx).ref_z).^2 ... - + (pass(pass_idx).ref_y - pass(master_idx).ref_y).^2 ); - - h_plot_baseline(end+1) = plot(base_line); - h_legend_baseline{end+1} = sprintf('%d',pass_idx); - hold on; -end -xlabel('Range line'); -ylabel('Baseline (m)'); -grid on; -legend(h_plot_baseline,h_legend_baseline); - -fn_map = fullfile(fn_dir,[fn_name output_fn_midfix '_map.fig']); -saveas(h_fig_map,fn_map); -fn_elev = fullfile(fn_dir,[fn_name output_fn_midfix '_elev.fig']); -saveas(h_fig_elev,fn_elev); -fn_baseline = fullfile(fn_dir,[fn_name output_fn_midfix '_baseline.fig']); -saveas(h_fig_baseline,fn_baseline); - -%% Return conditions for modes 1,3,4 -if insar_mode == 4 - return -end -if insar_mode == 1 - [fn_dir,fn_name] = fileparts(fn); - fn_insar = fullfile(fn_dir,[fn_name output_fn_midfix '_insar_no_slope.mat']); - param_sar = pass(master_idx).param_sar; - param_records = pass(master_idx).param_records; - save(fn_insar,'-v7.3','data','ref','param_sar','param_records'); - return -end -if insar_mode == 3 - [fn_dir,fn_name] = fileparts(fn); - fn_insar = fullfile(fn_dir,[fn_name output_fn_midfix '_insar.mat']); - param_sar = pass(master_idx).param_sar; - param_records = pass(master_idx).param_records; - save(fn_insar,'-v7.3','data','ref','param_sar','param_records'); - return -end - -%% Array Processing - -% Package data to call array_proc.m -% 1. Data -% 2. Trajectory and attitude -% 3. Array processing parameters -data = {permute(data,[1 2 4 5 3])}; - -param.array = []; -param.array.method = 1; -% param.array.Nsv = 256; param.array = rmfield(param.array,'theta'); % Forces default theta -% param.array.theta = linspace(-20,20,256); -param.array.theta = linspace(-6,6,256); -param.array.Nsrc = 2; -param.array.bin_rng = [-4:4]; -param.array.line_rng = [-20:20]; -param.array.dbin = 1; -param.array.dline = 11; -param.array.freq_rng = 1; -for pass_out_idx = 1:length(pass_en_idxs) - pass_idx = pass_en_idxs(pass_out_idx); - - param.array.fcs{1}{pass_out_idx}.pos = along_track; - param.array.fcs{1}{pass_out_idx}.pos(2,:) = pass(pass_idx).ref_y; - param.array.fcs{1}{pass_out_idx}.pos(3,:) = pass(pass_idx).ref_z; - param.array.fcs{1}{pass_out_idx}.base_line ... - = sqrt( (pass(pass_idx).ref_z - pass(master_idx).ref_z).^2 ... - + (pass(pass_idx).ref_y - pass(master_idx).ref_y).^2 ); - - param.array.fcs{1}{pass_out_idx}.surface = ref.surface; -end - -param.array.wfs.time = ref.wfs(ref.wf).time; -dt = param.array.wfs.time(2)-param.array.wfs.time(1); -param.array_proc.bin0 = param.array.wfs.time(1)/dt; -param.array.sv_fh = @array_proc_sv; -param.array.wfs.fc = ref.wfs(ref.wf).fc; -param.array.imgs = {[ones(length(pass_en_idxs),1), (1:length(pass_en_idxs)).']}; -param.array.tomo_en = true; - -array_proc_methods; -param = array_proc(param); -param.array.method = STANDARD_METHOD; -[param_array0,result0] = array_proc(param,{data{1}(:,:,:,:,:)}); -% param.array.method = MVDR_METHOD; -% [param_array1,result1] = array_proc(param,{data{1}(:,:,:,:,:)}); -param.array.method = MUSIC_METHOD; -[param_array2,result2] = array_proc(param,{data{1}(:,:,:,:,:)}); - -figure(2001); clf; -imagesc(lp(result0.img)) -title('Periodogram') -colormap(1-gray(256)) -h_axes = gca; - -% figure(2002); clf; -% imagesc(lp(result1.img)) -% title('MVDR') -% colormap(1-gray(256)) -% h_axes(end+1) = gca; - -figure(2003); clf; -imagesc(lp(result2.img)) -title('MUSIC') -colormap(1-gray(256)) -h_axes(end+1) = gca; - -figure(2004); clf; -imagesc(lp(fir_dec(abs(data{1}(:,:,master_out_idx)).^2, ones(size(param.array.line_rng)), param.array.dline, ... - 1-param.array.line_rng(1), size(data{1}(:,:,master_out_idx),2)-length(param.array.line_rng)+1))) -title('Single Channel'); -colormap(1-gray(256)) -h_axes(end+1) = gca; - -linkaxes(h_axes,'xy'); - -%% Save Results -[fn_dir,fn_name] = fileparts(fn); - -Tomo = result0.tomo; -Data = result0.img; -GPS_time = ref.gps_time(param_array0.array_proc.lines); -Latitude = ref.lat(param_array0.array_proc.lines); -Longitude = ref.lon(param_array0.array_proc.lines); -Elevation = ref.elev(param_array0.array_proc.lines); -Roll = ref.roll(param_array0.array_proc.lines); -Pitch = ref.pitch(param_array0.array_proc.lines); -Heading = ref.heading(param_array0.array_proc.lines); -Surface = ref.surface(param_array0.array_proc.lines); -Bottom = nan(size(Surface)); -param_sar = pass(master_idx).param_sar; -param_records = pass(master_idx).param_records; -param_array = param_array0; -Time = param.array.wfs.time(param_array0.array_proc.bins); -file_version = '1'; -fn_mat = fullfile(fn_dir,[fn_name output_fn_midfix '_standard.mat']); -save('-v7.3',fn_mat,'Tomo','Data','Latitude','Longitude','Elevation','GPS_time', ... - 'Surface','Bottom','Time','param_array','param_records', ... - 'param_sar', 'Roll', 'Pitch', 'Heading', 'file_version'); - -Tomo = result2.tomo; -Data = result2.img; -GPS_time = ref.gps_time(param_array2.array_proc.lines); -Latitude = ref.lat(param_array2.array_proc.lines); -Longitude = ref.lon(param_array2.array_proc.lines); -Elevation = ref.elev(param_array2.array_proc.lines); -Roll = ref.roll(param_array2.array_proc.lines); -Pitch = ref.pitch(param_array2.array_proc.lines); -Heading = ref.heading(param_array2.array_proc.lines); -Surface = ref.surface(param_array2.array_proc.lines); -Bottom = nan(size(Surface)); -param_sar = pass(master_idx).param_sar; -param_records = pass(master_idx).param_records; -param_array = param_array2; -Time = param.array.wfs.time(param_array2.array_proc.bins); -file_version = '1'; -fn_mat = fullfile(fn_dir,[fn_name output_fn_midfix '_music.mat']); -save('-v7.3',fn_mat,'Tomo','Data','Latitude','Longitude','Elevation','GPS_time', ... - 'Surface','Bottom','Time','param_array','param_records', ... - 'param_sar', 'Roll', 'Pitch', 'Heading', 'file_version'); - -fn_standard = fullfile(fn_dir,[fn_name output_fn_midfix '_standard.fig']); -saveas(2001,fn_standard); -% fn_mvdr = fullfile(fn_dir,[fn_name output_fn_midfix '_mvdr.fig']); -% saveas(2002,fn_mvdr); -fn_music = fullfile(fn_dir,[fn_name output_fn_midfix '_music.fig']); -saveas(2003,fn_music); -fn_single = fullfile(fn_dir,[fn_name output_fn_midfix '_single.fig']); -saveas(2004,fn_single); diff --git a/cresis-toolbox/+insar/plot_trajectories.m b/cresis-toolbox/+insar/plot_trajectories.m deleted file mode 100644 index a14571be..00000000 --- a/cresis-toolbox/+insar/plot_trajectories.m +++ /dev/null @@ -1,48 +0,0 @@ -h_fig = figure(1); clf; -h_axes = axes; -hold(h_axes,'on'); -xlim(h_axes,[-200 200]); -ylim(h_axes,[-100 100]); - -y = []; -z = []; -for rline=1:size(array_param.fcs{1}{1}.pos,2) - for idx=1:length(array_param.fcs{1}) - y(idx,rline) = array_param.fcs{1}{idx}.pos(2,rline); - z(idx,rline) = array_param.fcs{1}{idx}.pos(3,rline); - end -end -% imagesc(abs(y)) -% colormap(jet); -% caxis([0 25]); - -xlim(h_axes,[min(y(:)) max(y(:))]); -ylim(h_axes,[min(z(:)) max(z(:))]); - - -for idx=1:length(array_param.fcs{1}) - h_plot(idx) = plot(h_axes, NaN,NaN,'x'); -end -h_title = title(h_axes, ''); - -for rline=1:10:size(array_param.fcs{1}{1}.pos,2) - for idx=1:length(array_param.fcs{1}) - set(h_plot(idx),'XData',array_param.fcs{1}{idx}.pos(2,rline),'YData',array_param.fcs{1}{idx}.pos(3,rline)); - end - set(h_title,'String',sprintf('%d',rline)); -% M(rline) = getframe; - pause -end -% movie(M); -return - - -% For each trajectory find the sum of all the distances to other trajectories -total_dists = zeros(length(array_param.fcs{1}),1); -for idx = 1:length(array_param.fcs{1}) - dist = bsxfun(@minus, y([1:idx-1 idx+1:end],:), y(idx,:)).^2 ... - + bsxfun(@minus, z([1:idx-1 idx+1:end],:), z(idx,:)).^2; - total_dists(idx) = sum(sum(dist)); -end - - diff --git a/cresis-toolbox/+multipass/coh_interf_base_plots.m b/cresis-toolbox/+multipass/coh_interf_base_plots.m deleted file mode 100644 index a8eab741..00000000 --- a/cresis-toolbox/+multipass/coh_interf_base_plots.m +++ /dev/null @@ -1,118 +0,0 @@ -clearvars -except gRadar tmp -close all -%File names for the different years -fn2011_2014 = '/cresis/snfs1/dataproducts/ct_data/rds/2011_Greenland_P3/CSARP_insar/rds_thule_2011_2014_wf2_insar.mat'; -fn2011_2012 = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_insar/rds_thule_2011_2012_wf2_insar3.mat'; -fn2012_2014 = '/cresis/snfs1/dataproducts/ct_data/rds/2012_Greenland_P3/CSARP_insar/rds_thule_2012_2014_wf2_insar.mat'; -fn2012_2013 = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_insar/rds_thule_2012_2013_wf2_insar3.mat'; -fn2013_2014 = '/cresis/snfs1/dataproducts/ct_data/rds/2013_Greenland_P3/CSARP_insar/rds_thule_2013_2014_wf2_insar.mat'; -fn2014_2014 = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_insar/rds_thule_2014_2Week_wf3_insar3.mat'; -fn20142 = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_insar/rds_thule_2014_1Day_wf3_insar3.mat'; -fn2014allwf = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_multipass/rds_thule_2014_SameDay_allwf_multipass03.mat'; - -fns = {fn2014allwf}; -figtitle = {'2014 Same Day'}; -saven = {'2014_SameDay_allwf'}; -masterid = 8; -%Settings -set1 = {1:15,1:15, 1:15, 1:15, 1:7, 1:15,1:15,1:15}; -set2 = {16:30,16:30, 16:30, 16:30, 8:22, 16:22, 16:30, 16:30}; -if 0 %7 Element Setting - set1 = {5:11, 5:11, 5:11, 5:11, 5:11}; - set2 = {20:26, 20:26, 16:22, 20:26, 20:26}; - saven = cellfun(@(c)[c '_7Elements'],saven,'UniformOutput',false); -end -rbins = [240:420]; -filt_x = 9; filt_y = 45; -%Convert phase to physical length -eps_ice = 3.15; wfuse = 2; -cls = physconst('LightSpeed'); -savedir = '/cresis/snfs1/scratch/bmiller/More_Multipass/Check_Figures/'; - -for fn_id = 1:length(fns) - if ~exist('tmp','var') || length(tmp) frames.gps_time(passes(passes_idx).frms(frm_idx)+1) + warning('Old echogram format with extra "overlap" data at start and/or end. Removing overlap.'); + good_mask = tmp_data.GPS_time >= frames.gps_time(passes(passes_idx).frms(frm_idx)) ... + & tmp_data.GPS_time <= frames.gps_time(passes(passes_idx).frms(frm_idx)+1); + tmp_data.GPS_time = tmp_data.GPS_time(good_mask); + tmp_data.Data = tmp_data.Data(:,good_mask); + tmp_data.Latitude = tmp_data.Latitude(good_mask); + tmp_data.Longitude = tmp_data.Longitude(good_mask); + tmp_data.Elevation = tmp_data.Elevation(good_mask); + tmp_data.Roll = tmp_data.Roll(good_mask); + tmp_data.Pitch = tmp_data.Pitch(good_mask); + tmp_data.Heading = tmp_data.Heading(good_mask); + tmp_data.Surface = tmp_data.Surface(good_mask); + tmp_data.Bottom = tmp_data.Bottom(good_mask); + end metadata{passes_idx}.gps_time = [metadata{passes_idx}.gps_time ,tmp_data.GPS_time]; metadata{passes_idx}.lat = [metadata{passes_idx}.lat ,tmp_data.Latitude]; metadata{passes_idx}.lon = [metadata{passes_idx}.lon ,tmp_data.Longitude]; @@ -204,74 +231,172 @@ metadata{passes_idx}.heading = [metadata{passes_idx}.heading ,tmp_data.Heading]; metadata{passes_idx}.surface = [metadata{passes_idx}.surface ,tmp_data.Surface]; metadata{passes_idx}.bottom = [metadata{passes_idx}.bottom ,tmp_data.Bottom]; - metadata{passes_idx}.fcs.origin = [metadata{passes_idx}.fcs.origin ,tmp_data.param_array.array_proc.fcs.origin]; - metadata{passes_idx}.fcs.x = [metadata{passes_idx}.fcs.x ,tmp_data.param_array.array_proc.fcs.x]; - metadata{passes_idx}.fcs.y = [metadata{passes_idx}.fcs.y ,tmp_data.param_array.array_proc.fcs.y]; - metadata{passes_idx}.fcs.z = [metadata{passes_idx}.fcs.z ,tmp_data.param_array.array_proc.fcs.z]; - metadata{passes_idx}.fcs.pos = [metadata{passes_idx}.fcs.pos ,tmp_data.param_array.array_proc.fcs.pos]; - data{passes_idx} = [data{passes_idx} ,tmp_data.Data]; + % Handle old format, but print error message + if isfield(tmp_data.param_array,'array_param') + warning('OLD SAR DATA FORMAT. IF MULTIPLE WF-ADC PAIRS WERE USED IN THE DATA PRODUCT, THE FLIGHT COORDINATE SYSTEM WILL BE INCORRECT AND THE DATA SHOULD BE REPROCESSED.'); + metadata{passes_idx}.fcs.origin = [metadata{passes_idx}.fcs.origin, tmp_data.param_array.array_param.fcs{1}{1}.origin]; + metadata{passes_idx}.fcs.x = [metadata{passes_idx}.fcs.x, tmp_data.param_array.array_param.fcs{1}{1}.x]; + metadata{passes_idx}.fcs.y = [metadata{passes_idx}.fcs.y, tmp_data.param_array.array_param.fcs{1}{1}.y]; + metadata{passes_idx}.fcs.z = [metadata{passes_idx}.fcs.z, tmp_data.param_array.array_param.fcs{1}{1}.z]; + metadata{passes_idx}.fcs.pos = [metadata{passes_idx}.fcs.pos, tmp_data.param_array.array_param.fcs{1}{1}.pos]; + elseif ~isfield(tmp_data.param_array,'array_proc') + % Construct FCS fields from trajectory data + records = records_load(param_pass); + records = records_reference_trajectory_load(param_pass,records); + records.lat = interp1(records.gps_time,records.lat,tmp_data.GPS_time); + records.lon = gps_interp1(records.gps_time,records.lon*(pi/180),tmp_data.GPS_time)*(180/pi); + records.elev = interp1(records.gps_time,records.elev,tmp_data.GPS_time); + records.gps_time = tmp_data.GPS_time; + [~,~,~,origin,x,y,z,pos] = trajectory_coord_system(records); + metadata{passes_idx}.fcs.origin = [metadata{passes_idx}.fcs.origin, origin]; + metadata{passes_idx}.fcs.x = [metadata{passes_idx}.fcs.x, x]; + metadata{passes_idx}.fcs.y = [metadata{passes_idx}.fcs.y, y]; + metadata{passes_idx}.fcs.z = [metadata{passes_idx}.fcs.z, z]; + metadata{passes_idx}.fcs.pos = [metadata{passes_idx}.fcs.pos, pos]; + + elseif iscell(tmp_data.param_array.array_proc.fcs) + warning('OLD SAR DATA FORMAT. IF MULTIPLE WF-ADC PAIRS WERE USED IN THE DATA PRODUCT, THE FLIGHT COORDINATE SYSTEM WILL BE INCORRECT AND THE DATA SHOULD BE REPROCESSED.'); + metadata{passes_idx}.fcs.origin = [metadata{passes_idx}.fcs.origin, tmp_data.param_array.array_proc.fcs{1}{1}.origin]; + metadata{passes_idx}.fcs.x = [metadata{passes_idx}.fcs.x, tmp_data.param_array.array_proc.fcs{1}{1}.x]; + metadata{passes_idx}.fcs.y = [metadata{passes_idx}.fcs.y, tmp_data.param_array.array_proc.fcs{1}{1}.y]; + metadata{passes_idx}.fcs.z = [metadata{passes_idx}.fcs.z, tmp_data.param_array.array_proc.fcs{1}{1}.z]; + metadata{passes_idx}.fcs.pos = [metadata{passes_idx}.fcs.pos, tmp_data.param_array.array_proc.fcs{1}{1}.pos]; + else + metadata{passes_idx}.fcs.origin = [metadata{passes_idx}.fcs.origin, tmp_data.param_array.array_proc.fcs.origin]; + metadata{passes_idx}.fcs.x = [metadata{passes_idx}.fcs.x, tmp_data.param_array.array_proc.fcs.x]; + metadata{passes_idx}.fcs.y = [metadata{passes_idx}.fcs.y, tmp_data.param_array.array_proc.fcs.y]; + metadata{passes_idx}.fcs.z = [metadata{passes_idx}.fcs.z, tmp_data.param_array.array_proc.fcs.z]; + metadata{passes_idx}.fcs.pos = [metadata{passes_idx}.fcs.pos, tmp_data.param_array.array_proc.fcs.pos]; + end + data{passes_idx} = [data{passes_idx}, tmp_data.Data]; end else - %% Load: Load "sar" Data + %% Load Data: SAR - % Setup param_sar structure used to load SAR data - param_sar = param_pass; + % Setup param_load structure used in sar_load.m + param_load = param_pass; + param_load.sar_load = param.combine_passes.sar_load; % (wf,adc) pairs to load - param_sar.load_sar_data.imgs = passes(passes_idx).imgs; + param_load.sar_load.imgs = passes(passes_idx).imgs; % Debug level (1 = default) - param_sar.load_sar_data.debug_level = 2; + param_load.sar_load.debug_level = 2; - for frm_idx = 1:length(passes(passes_idx).frms) - % Load SAR data for this frame - param_sar.load_sar_data.frm = passes(passes_idx).frms(frm_idx); - % load_sar_data loads all images at once - [data{end+1},metadata{end+1}] = load_sar_data(param_sar); - - metadata{end}.frms = passes(passes_idx).frms(frm_idx); - metadata{end}.param_pass = param_pass; - - %% Do waveform combination - param_mode = 'array'; - param_img_combine = param_pass; - param_img_combine.array.img_comb = param_pass.array.img_comb(1:... - min([length(param_pass.array.img_comb), 3*(length(passes(passes_idx).imgs)-1)])); - param_img_combine.array.imgs = passes(passes_idx).imgs; - if length(param_img_combine.array.img_comb)<3 - param_img_combine.array.img_comb = [param_img_combine.array.img_comb(1) -inf param_img_combine.array.img_comb(2)]; - end - param_img_combine.load.frm = metadata{end}.frms; - param_img_combine.day_seg = passes(passes_idx).day_seg; - - data_in = []; - data_in.Data =data{end}; - data_in.Time = {}; - for img = 1:length(passes(passes_idx).imgs) - data_in.Time{img}= metadata{end}.wfs(img).time; + % Load SAR data for this frame + param_load.sar_load.frms = passes(passes_idx).frms; + % sar_load loads all images at once + [data{end+1},metadata{end+1}] = sar_load(param_load); + + metadata{end}.frms = passes(passes_idx).frms; + + % Update param_pass radar.wfs structure + if passes_idx == 1 || ~strcmp(passes(passes_idx).day_seg,passes(passes_idx-1).day_seg) + % Load records for data_load_wfs.m and save original param_pass + % structure + param_pass_original = param_pass; + records = records_load(param_pass); + else + % Reset param_pass so it starts at the same point for each wf-adc + % pair + param_pass = param_pass_original; + end + [tmp_wfs,~] = data_load_wfs(setfield(param_pass,'load',struct('imgs',{passes(passes_idx).imgs})),records); + metadata{end}.param_pass = param_pass; + + %% Load Data SAR: equalization + param_pass.radar.wfs = merge_structs(param_pass.radar.wfs,tmp_wfs); + for img = 1:length(passes(passes_idx).imgs) + % Get the wf and adc values for this image + wf = passes(passes_idx).imgs{img}(1,1); + adc = passes(passes_idx).imgs{img}(1,2); + % Check to see if the parameter-spreadsheet/param_override values + % have been updated from when the SAR data were created. + chan_equal ... + = 10.^((param_pass.radar.wfs(wf).chan_equal_dB(param_pass.radar.wfs(wf).rx_paths(adc)) ... + - metadata{end}.wfs(wf).chan_equal_dB(param_pass.radar.wfs(wf).rx_paths(adc)) )/20) ... + .* exp(1i*( ... + param_pass.radar.wfs(wf).chan_equal_deg(param_pass.radar.wfs(wf).rx_paths(adc)) ... + - metadata{end}.wfs(wf).chan_equal_deg(param_pass.radar.wfs(wf).rx_paths(adc)) )/180*pi); + Tsys_old = metadata{end}.wfs(wf).Tsys(param_pass.radar.wfs(wf).rx_paths(adc)); + Tsys = param_pass.radar.wfs(wf).Tsys(param_pass.radar.wfs(wf).rx_paths(adc)); + dTsys = Tsys-Tsys_old; + if chan_equal ~= 1 + fprintf(' Updating chan_equal for wf %d adc %d (dchan_equal is %g dB %g deg)\n', wf, adc, db(chan_equal), angle(chan_equal)*180/pi); + data{end}{img} = data{end}{img} ./ chan_equal; end - data_in.GPS_time = metadata{end}.fcs{1}{1}.gps_time; - if length(data{end})>1 %Combine if using multiple waveforms - [data{end}, metadata{end}.time] = img_combine(param_img_combine,param_mode,surf_layer,data_in); - else - data{end} = data{end}{1}; + if dTsys ~= 0 + fprintf(' Updating Tsys for wf %d adc %d from %g to %g (dTsys is %g)\n', wf, adc, Tsys_old, Tsys, dTsys); + data{end}{img} = ifft(bsxfun(@times,fft(data{end}{img},[],1), ... + exp(-1i*2*pi*metadata{end}.wfs(wf).freq*dTsys))); end - if 0 - %Check image combine output - figure(1); clf; h_axes = []; - for idx = 1:length(data_in.Data) - h_axes(idx) = subplot(length(data_in.Data),1,idx); - imagesc(data_in.GPS_time,data_in.Time{idx},lp(data_in.Data{idx})) - xlabel('GPS Time') - ylabel('TWTT') + end + if 0 + % Debug plots + rline = 1; % <== SPECIFY WHICH RANGE LINE TO PLOT + colors_cell = {'k','r','y','g','b','c','m'}; + h_plot = []; + legend_str = {}; + figure; clf; + for img = 1:length(data{end}) + for wf_adc = 1:size(data{end}{img},3) + wf = passes(passes_idx).imgs{img}(1,1); + adc = passes(passes_idx).imgs{img}(1,2); + rline = max(1,min(rline,size(data{end}{img},2))); + h_tmp = plot(metadata{end}.wfs(wf).time, squeeze(imag(data{end}{img}(:,rline,wf_adc))), colors_cell{1+mod(img-1,length(colors_cell))}); + if wf_adc == 1 + legend_str{img} = sprintf('Img %d wf %d adc %d', img, wf, adc); + h_plot(img) = h_tmp; + end + hold on; end - - figure(2); clf; - h_axes(end+1) = axes('parent',2); - imagesc(data_in.GPS_time,metadata{end}.time,lp(data{end})) + end + legend(h_plot, legend_str); + end + + %% Load Data SAR: img_comb + % Combines low-gain and high-gain images into a single image + param_mode = 'array'; + param_img_combine = param_pass; + % array.img_comb should have 3*(length(imgs)-1) fields, if it has more + % than required, then truncate to 3*(length(imgs)-1) + param_img_combine.array.img_comb = param_pass.array.img_comb(1:... + min([length(param_pass.array.img_comb), 3*(length(passes(passes_idx).imgs)-1)])); + param_img_combine.array.imgs = passes(passes_idx).imgs; + if length(param_img_combine.array.imgs) > 1 && length(param_img_combine.array.img_comb) == 2*(length(param_img_combine.array.imgs)-1) + error('Spreadsheet has the wrong number of entries for param.array.img_comb. It has the right number for the old combine method. Usually this is fixed by inserting "-inf" for the second coefficient which controls how the receiver blanking is used. For example [3e-6 1e-6; 10e-6 3e-6] becomes [3e-6 -inf 1e-6; 10e-6 -inf 3e-6].'); + end + param_img_combine.load.frm = metadata{end}.frms; + param_img_combine.day_seg = passes(passes_idx).day_seg; + + data_in = []; + data_in.Data =data{end}; + data_in.Time = {}; + for img = 1:length(passes(passes_idx).imgs) + wf = passes(passes_idx).imgs{img}(1); + data_in.Time{img}= metadata{end}.wfs(wf).time; + end + data_in.GPS_time = metadata{end}.fcs{1}{1}.gps_time; + if length(data{end})>1 %Combine if using multiple waveforms + [data{end}, metadata{end}.time] = img_combine(param_img_combine,param_mode,surf_layer,data_in); + else + data{end} = data{end}{1}; + end + if 0 + %Check image combine output + figure(1); clf; h_axes = []; + for idx = 1:length(data_in.Data) + h_axes(idx) = subplot(length(data_in.Data),1,idx); + imagesc(data_in.GPS_time,data_in.Time{idx},lp(data_in.Data{idx})) xlabel('GPS Time') ylabel('TWTT') - linkaxes(h_axes) - keyboard end + + figure(2); clf; + h_axes(end+1) = axes('parent',2); + imagesc(data_in.GPS_time,metadata{end}.time,lp(data{end})) + xlabel('GPS Time') + ylabel('TWTT') + linkaxes(h_axes) + keyboard end end end @@ -397,6 +522,30 @@ if strcmpi(param.combine_passes.input_type,'echo') pass(end).data = data{data_idx}(:,rlines); pass(end).wfs.time = metadata{data_idx}.time; + if isfield(metadata{data_idx}.param_array,'combine') + % Old format + if iscell(metadata{data_idx}.param_array.combine.imgs{1}) + wf = metadata{data_idx}.param_array.combine.imgs{1}{1}(1); + else + wf = metadata{data_idx}.param_array.combine.imgs{1}(1); + end + else + if iscell(metadata{data_idx}.param_array.array.imgs{1}) + wf = metadata{data_idx}.param_array.array.imgs{1}{1}(1); + else + % Old format + wf = metadata{data_idx}.param_array.array.imgs{1}(1); + end + end + if ~isfield(metadata{data_idx}.param_array.radar,'wfs') + % Old format + pass(end).wfs.fc = 0.5*(metadata{data_idx}.param_pass.radar.wfs(wf).f0+metadata{data_idx}.param_pass.radar.wfs(wf).f1); + elseif ~isfield(metadata{data_idx}.param_array.radar.wfs,'fc') + % Old format + pass(end).wfs.fc = 0.5*(metadata{data_idx}.param_array.radar.wfs(wf).f0+metadata{data_idx}.param_array.radar.wfs(wf).f1); + else + pass(end).wfs.fc = metadata{data_idx}.param_array.radar.wfs(wf).fc; + end pass(end).time = metadata{data_idx}.time; pass(end).gps_time = metadata{data_idx}.gps_time(rlines); @@ -408,7 +557,12 @@ pass(end).y = metadata{data_idx}.fcs.y(:,rlines); pass(end).z = metadata{data_idx}.fcs.z(:,rlines); pass(end).origin = metadata{data_idx}.fcs.origin(:,rlines); - pass(end).pos = metadata{data_idx}.fcs.pos(:,rlines); + if size(metadata{data_idx}.fcs.pos,2) ~= size(metadata{data_idx}.fcs.origin,2) + warning('Old file format with pos field error. No valid data in pos field. Setting to all zeros (i.e. the reference position of the array and not the actual position of the wf-adc pair).'); + pass(end).pos = zeros(3,length(rlines)); + else + pass(end).pos = metadata{data_idx}.fcs.pos(:,rlines); + end pass(end).surface = metadata{data_idx}.surface(:,rlines); else pass(end).data = data{data_idx}(:,rlines); diff --git a/cresis-toolbox/+multipass/multipass.m b/cresis-toolbox/+multipass/multipass.m index a4bcbfeb..18d25d93 100644 --- a/cresis-toolbox/+multipass/multipass.m +++ b/cresis-toolbox/+multipass/multipass.m @@ -1,24 +1,36 @@ % function multipass(param,param_override) % multipass(param,param_override) % -% Combines -% param.multipass.comp_mode -% 1 to find equalization coefficients +% Synchronize, register, correct passes from combine_passes and then apply +% array processing. +% +% param.multipass.comp_mode: integer from 1 to 4 controlling the mode +% 1: 1 to find equalization coefficients % Motion compensation of FCS z-motion % (Motion compensation with phase correction) % Quits after computing equalization coefficients -% 2 to do array processing on data, 4 +% 2: to do coregistration and (if input_type=='sar') array process +% This is the mode that must be used when using input_type='echo' % Co-register images using GPS and nadir squint angle assumption % (Motion compensation without phase correction) -% Runs array processing -% 3 to differential INSAR -% Co-register images using GPS and nadir squint angle assumption -% (Motion compensation with phase correction AND slope correction) -% Saves output for interferometry -% 4 to plot results +% Runs array processing if input_type=='sar' and is used for cross-track +% slope estimation and basal swath imaging +% 3: to differential INSAR +% Co-register images using GPS and nadir squint angle assumption (Motion +% compensation with phase correction AND slope correction) Saves output +% for (differential) interferometry (coherence and interferometric phase +% images created) and vertical velocity estimation. +% 4: to plot results % Co-register images using GPS and nadir squint angle assumption % Quits after plotting results -% +% +% param.multipass.fn: string containing the full file to the file created +% by multipass.combine_passes. +% +% param.multipass.layer: opsLoadLayers layer parameter structure that +% specifies all the layers to load for each pass and to synchronize. The +% default is struct('name',{'surface','bottom'}). +% %% Setup % ========================================================================= @@ -27,8 +39,10 @@ param.multipass.pass_name, datestr(now)); fprintf('=====================================================================\n'); +param = merge_structs(param,param_override); + physical_constants; -standard_projections; +proj_load_standard; %% Load multipass.combine_passes file fn = param.multipass.fn; @@ -39,12 +53,16 @@ % ========================================================================= % Confirm either SAR or echogram data -if ~isfield(param.multipass,'input_type') - if ~isfield(pass(1),'input_type') - param.multipass.input_type = 'echo'; - else - param.multipass.input_type = pass(1).input_type; - end +if ~isfield(pass(1),'input_type') + input_type = 'echo'; +else + input_type = pass(1).input_type; +end + +% Confirm echo input type is running comp_mode==2 +if param.multipass.comp_mode ~= 2 && strcmpi(input_type,'echo') + warning('Only param.multipass.comp_mode == 2 may be used with input_type=="echo".'); + param.multipass.comp_mode = 2; end % baseline_master_idx: All images are registered to the pass indicated by @@ -55,6 +73,8 @@ baseline_master_idx = param.multipass.baseline_master_idx; % coregistration_time_shift: Fast time time shift. Default is zero. +% Positive values cause the image to move towards the radar. Units are in +% range bins. if ~isfield(param.multipass,'coregistration_time_shift') || isempty(param.multipass.coregistration_time_shift) param.multipass.coregistration_time_shift = zeros(1,length(pass)); end @@ -65,7 +85,7 @@ % debug_plots: cell array of strings that enable certain debug outputs if ~isfield(param.multipass,'debug_plots') || isempty(param.multipass.debug_plots) - if strcmpi(param.multipass.input_type,'echo') + if strcmpi(input_type,'echo') param.multipass.debug_plots = {'debug'}; else param.multipass.debug_plots = {'debug','coherent'}; @@ -76,29 +96,16 @@ % equalization: Equalization (complex weight) for each pass. Default weight is all ones. if ~isfield(param.multipass, 'equalization') - param.multipass.equalization = exp(1i*(zeros(1,length(pass))/20)/180*pi); + % Written with /20 and /180*pi to emphasize the dB and deg format. + % param.multipass.equalization = exp(1i*(zeros(1,length(pass))/20)/180*pi); + param.multipass.equalization = ones(1,length(pass)); end equalization = param.multipass.equalization; % layer: layer struct to opsLoadLayers which indicates which layers will be % loaded and coregistered along with each image if ~isfield(param.multipass,'layer') || isempty(param.multipass.layer) - param.multipass.layer = struct(); -end -if length(param.multipass.layer) < 2 - param.multipass.layer(2) = struct(); -end -if ~isfield(param.multipass.layer(1),'name') || isempty(param.multipass.layer(1).name) - param.multipass.layer(1).name = 'surface'; -end -if ~isfield(param.multipass.layer(1),'source') || isempty(param.multipass.layer(1).source) - param.multipass.layer(1).source = 'layerData'; -end -if ~isfield(param.multipass.layer(2),'name') || isempty(param.multipass.layer(2).name) - param.multipass.layer(2).name = 'bottom'; -end -if ~isfield(param.multipass.layer(1),'source') || isempty(param.multipass.layer(2).source) - param.multipass.layer(2).source = 'layerData'; + param.multipass.layer = struct('name',{'surface','bottom'},'source','layerdata'); end % master_idx: Index to pass that will be used as the master pass for the @@ -120,6 +127,7 @@ end % Assume any undefined passes are enabled param.multipass.pass_en_mask(end+1:length(pass)) = true; +param.multipass.pass_en_mask = logical(param.multipass.pass_en_mask); pass_en_mask = param.multipass.pass_en_mask; if ~pass_en_mask(master_idx) error('The master index pass %d must be enabled in param.multipass.pass_en_mask.', master_idx); @@ -203,16 +211,23 @@ if ~pass_en_mask(pass_idx) continue end + + % Update the parameters with current gRadar settings so that the paths + % are correct. A common issue is that multipass.combine_passes files are + % created in one environemnt and then loaded here under a different + % environment and the paths are different. + param_override = merge_structs(gRadar,param_override); + param_paths_updated = merge_structs(pass(pass_idx).param_pass,param_override); % Load layers - pass(pass_idx).layers = opsLoadLayers(pass(pass_idx).param_pass,param.multipass.layer); - + pass(pass_idx).layers = opsLoadLayers(param_paths_updated,param.multipass.layer); + % Interpolate all layers onto a common reference (ref) for lay_idx = 1:length(pass(pass_idx).layers) pass(pass_idx).layers(lay_idx).twtt ... = interp_finite(interp1(pass(pass_idx).layers(lay_idx).gps_time, ... pass(pass_idx).layers(lay_idx).twtt, ... - pass(pass_idx).gps_time,'linear')); + pass(pass_idx).gps_time,'linear'),0); end end @@ -294,6 +309,9 @@ legend(h_plot_map,h_legend_map); legend(h_plot_elev,h_legend_elev); h_axes_echo = h_axes_echo(pass_en_mask); + % x-axes from each pass do not line up since the images are not + % registered yet so this does not really work in the x-axis unless the + % along-track sampling rate is similar between each pass linkaxes(h_axes_echo); xlim(h_axes_echo(1),[1 max_rlines]); ylim(h_axes_echo(1),param.multipass.time_gate*1e6); @@ -332,7 +350,7 @@ end for pass_out_idx = 1:length(pass_en_idxs) pass_idx = pass_en_idxs(pass_out_idx); - fprintf('%d of %d (pass %d)\n', pass_out_idx, length(pass_en_idxs), pass_idx); + fprintf('Coregister: %d of %d (pass %d) (%s)\n', pass_out_idx, length(pass_en_idxs), pass_idx, datestr(now,'yyyymmdd_HHMMSS')); %% Pass: 1. Position in ref coordinate system pass(pass_idx).ref_idx = zeros(1,size(pass(pass_idx).origin,2)); @@ -360,7 +378,7 @@ pass(pass_idx).ref_y(rline) = offset(:,min_idx).'*ref.y(:,min_idx); pass(pass_idx).ref_z(rline) = offset(:,min_idx).'*ref.z(:,min_idx); - if 0 %strcmp('sar',param.multipass.input_type) + if 0 %strcmp('sar',input_type) % Compute the location of all pixels from this range line in ECEF pass(pass_idx).time; time = pass(pass_idx).time(2)-pass(pass_idx).time(1); @@ -388,7 +406,7 @@ %% Pass: 2. Resample in along-track % Resample images and position vectors onto a common along-track axes - if strcmp('sar',param.multipass.input_type) + if strcmp('sar',input_type) % 1. Oversample slave data by 10x in along track Mx = 10; Nx = size(pass(pass_idx).data,2); @@ -399,7 +417,7 @@ % 3. Interpolate oversampled slave data onto master along track axes pass(pass_idx).ref_data = interp1(along_track_oversample, ... data_oversample, along_track,'linear','extrap').'; - elseif strcmp('echo',param.multipass.input_type) + elseif strcmp('echo',input_type) pass(pass_idx).ref_data = interp1(pass(pass_idx).along_track, ... pass(pass_idx).data.', along_track,'linear').'; end @@ -421,13 +439,16 @@ pass(pass_idx).freq_baseband = df * ifftshift( -floor(Nt/2) : floor((Nt-1)/2) ).'; fc = pass(pass_idx).wfs(pass(pass_idx).wf).fc; pass(pass_idx).freq = fc + pass(pass_idx).freq_baseband; + if pass_idx == baseline_master_idx + ref.freq = pass(pass_idx).freq; + end time_shift = coregistration_time_shift(pass_idx) * dt; - if strcmp('echo',param.multipass.input_type) + if strcmp('echo',input_type) % Apply time shift with interpolation pass(pass_idx).ref_data = interp1(pass(pass_idx).time, pass(pass_idx).ref_data, pass(pass_idx).time+time_shift, 'linear'); - pass(pass_idx).ref_data = interp_finite(pass(pass_idx).ref_data); + pass(pass_idx).ref_data = interp_finite(pass(pass_idx).ref_data,NaN); else % Apply frequency domain time shift (envelope only shift so baseband frequency) @@ -439,7 +460,7 @@ end %% Pass: 4. Motion/slope compensation - if strcmp('sar',param.multipass.input_type) + if strcmp('sar',input_type) Htime_window = tukeywin_trim(Nt,0.5); if param.multipass.comp_mode == 1 || param.multipass.comp_mode == 3 % Motion compensation of FCS z-motion (envelope and phase) @@ -469,15 +490,15 @@ pass(pass_idx).ref_data = pass(pass_idx).ref_data .* exp(-1i*4*pi*pass(pass_idx).freq(1)/c *bsxfun(@times,sin(slope.slope),pass(pass_idx).ref_y(:).')); end - elseif strcmp('echo',param.multipass.input_type) + elseif strcmp('echo',input_type) % Motion compensation of FCS z-motion using linear interpolation for rline = 1:size(pass(pass_idx).ref_data,2) time_shift = pass(pass_idx).ref_z(rline)/(c/2); pass(pass_idx).ref_data(:,rline) = interp1(pass(pass_idx).time, pass(pass_idx).ref_data(:,rline), pass(pass_idx).time+time_shift, 'linear'); - pass(pass_idx).ref_data(:,rline) = interp_finite(pass(pass_idx).ref_data(:,rline)); + pass(pass_idx).ref_data(:,rline) = interp_finite(pass(pass_idx).ref_data(:,rline),NaN); end end - + % Motion compensation for layers time_shift = pass(pass_idx).ref_z/(c/2); for lay_idx = 1:length(pass(pass_idx).layers) @@ -492,11 +513,11 @@ Mt = 4; Nt = length(pass(pass_idx).time); dt = pass(pass_idx).time(2)-pass(pass_idx).time(1); - if strcmp('sar',param.multipass.input_type) + if strcmp('sar',input_type) pass(pass_idx).ref_data = interpft(pass(pass_idx).ref_data,Mt*Nt); time_Mt = pass(pass_idx).time(1) + dt/Mt*(0:Mt*Nt-1); pass(pass_idx).ref_data = interp1(time_Mt, pass(pass_idx).ref_data, pass(baseline_master_idx).time, 'linear', 0); - elseif strcmp('echo',param.multipass.input_type) + elseif strcmp('echo',input_type) pass(pass_idx).ref_data = interp1(pass(pass_idx).time, pass(pass_idx).ref_data, pass(baseline_master_idx).time, 'linear', 0); end end @@ -516,26 +537,26 @@ %% Apply equalization % ----------------------- -if param.multipass.comp_mode ~= 1 && strcmp('sar',param.multipass.input_type) +if param.multipass.comp_mode ~= 1 && strcmp('sar',input_type) equalization = reshape(equalization,[1 1 numel(equalization)]); data = bsxfun(@times,data,1./equalization(:,:,pass_en_idxs)); end if 0 - %% Coregister: Data Dependent method to estimate System Time Delay + %% Estimate Coregistration: Data Dependent method to estimate System Time Delay % Apply fixed coregistration time shift coherence_sum = []; coregistration_time_shifts = -2:0.05:2; + rbins = param.multipass.rbins; for pass_out_idx = 1:length(pass_en_idxs) pass_idx = pass_en_idxs(pass_out_idx); - fprintf('%d of %d (pass %d)\n', pass_out_idx, length(pass_en_idxs), pass_idx); + fprintf('Estimate Coregistration: %d of %d, pass %d (%s)\n', pass_out_idx, length(pass_en_idxs), pass_idx, datestr(now,'yyyymmdd_HHMMSS')); - freq = pass(pass_idx).freq; + freq = ref.freq; freq = freq - freq(1); % Remove center frequency offset - coherence_sum = []; for coregistration_time_shift_idx = 1:length(coregistration_time_shifts) coregistration_time_shift = coregistration_time_shifts(coregistration_time_shift_idx); - dt = coregistration_time_shift * (pass(pass_idx).time(2)-pass(pass_idx).time(1)); + dt = coregistration_time_shift * (ref.time(2)-ref.time(1)); adjusted = ifft(bsxfun(@times,fft(data(:,:,pass_idx)),exp(-1i*2*pi*freq*dt))); coherence = fir_dec(adjusted(rbins,:) .* conj(data(rbins,:,master_idx)) ./ abs(adjusted(rbins,:) .* data(rbins,:,master_idx)),ones(1,7)/7,1); coherence = fir_dec(coherence.',ones(1,3)/3,1).'; @@ -546,12 +567,12 @@ % imagesc(coherence); colormap(1-gray(256)); % pause end - [~,coregistration_time_shift_idx] = max(coherence_sum); + [~,coregistration_time_shift_idx] = max(coherence_sum,[],1); coregistration_time_shift = coregistration_time_shifts(coregistration_time_shift_idx) end figure(2000); clf; plot(coregistration_time_shifts,coherence_sum) - [~,coregistration_time_shift_idx] = max(coherence_sum); + [~,coregistration_time_shift_idx] = max(coherence_sum,[],1); coregistration_time_shift = coregistration_time_shifts(coregistration_time_shift_idx); fprintf('%g ', coregistration_time_shift); fprintf('\n'); return @@ -564,10 +585,11 @@ master_out_idx = find(pass_en_idxs == master_idx); for pass_out_idx = 1:length(pass_en_idxs) pass_idx = pass_en_idxs(pass_out_idx); + fprintf('Plot: %d of %d, pass %d (%s)\n', pass_out_idx, length(pass_en_idxs), pass_idx, datestr(now,'yyyymmdd_HHMMSS')); figure(pass_idx); clf; set(pass_idx,'WindowStyle','docked') - if strcmp('echo',param.multipass.input_type) + if strcmp('echo',input_type) % Echogram: Power detected image % ===================================================================== if strcmp(param.multipass.units,'meters') @@ -603,7 +625,6 @@ set(gca,'YDir','normal'); % Convert layerPnts_y from TWTT to WGS_84 Elevation - elevation = pass(master_idx).elev; surface = pass(pass_idx).layers(1).twtt_ref; for lay_idx = 1:length(pass(pass_idx).layers) layer_y_curUnit = pass(pass_idx).layers(lay_idx).twtt_ref; @@ -617,14 +638,14 @@ hold on; plot(pass(master_idx).along_track/1e3, pass(pass_idx).layers(lay_idx).layer_elev); end - + else imagesc(lp(data(rbins,:,pass_out_idx))) ylabel('Range bin'); xlabel('Range line'); end colormap(1-gray(256)); - title(sprintf('%s_%03d %d',pass(pass_idx).param_sar.day_seg,pass(pass_idx).param_pass.cmd.frms(1),pass(pass_idx).direction),'interpreter','none') + title(sprintf('%s_%03d %d',pass(pass_idx).param_pass.day_seg,pass(pass_idx).param_pass.cmd.frms(1),pass(pass_idx).direction),'interpreter','none') %caxis([-90 8]); else @@ -676,10 +697,10 @@ end linkaxes(h_data_axes,'xy'); -if ~strcmp('echo',param.multipass.input_type) +if ~strcmp('echo',input_type) fprintf('=============================================\n'); fprintf('New equalization\n'); - fprintf('%.1f ', lp(new_equalization)-mean(lp(new_equalization(pass_en_idxs)))); + fprintf('%.1f ', db(new_equalization,'voltage')-mean(db(new_equalization(pass_en_idxs),'voltage'))); fprintf('\n'); fprintf('%.1f ', angle(new_equalization)*180/pi) fprintf('\n'); @@ -732,9 +753,9 @@ if ~exist(out_fn_dir,'dir') mkdir(out_fn_dir); end -save(out_fn,'-v7.3','pass','data','ref','param_combine_passes','param_multipass'); +ct_save(out_fn,'-v7.3','pass','data','ref','param_combine_passes','param_multipass'); -if param.multipass.comp_mode ~= 2 || strcmp('echo',param.multipass.input_type) +if param.multipass.comp_mode ~= 2 || strcmp('echo',input_type) return end @@ -755,7 +776,6 @@ param.array.line_rng = [-20:20]; param.array.dbin = 1; param.array.dline = 11; -param.array.freq_rng = 1; h_fig_baseline = figure(200); clf; h_plot_baseline = []; h_legend_baseline = {}; @@ -766,10 +786,10 @@ param.array.fcs{1}{pass_out_idx}.pos = along_track; param.array.fcs{1}{pass_out_idx}.pos(2,:) = pass(pass_idx).ref_y; param.array.fcs{1}{pass_out_idx}.pos(3,:) = pass(pass_idx).ref_z; - param.array.fcs{1}{pass_out_idx}.base_line ... + param.array.fcs{1}{pass_out_idx}.base_line ... = sqrt( (pass(pass_idx).ref_z - pass(master_idx).ref_z).^2 ... - + (pass(pass_idx).ref_y - pass(master_idx).ref_y).^2 ); - + + (pass(pass_idx).ref_y - pass(master_idx).ref_y).^2 ); + param.array.fcs{1}{pass_out_idx}.surface = ref.surface; end @@ -814,7 +834,7 @@ file_version = '1'; fn_mat = fullfile(fn_dir,[fn_name output_fn_midfix '_standard.mat']); fprintf('Saving %s (%s)\n', fn_mat, datestr(now)); -save('-v7.3',fn_mat,'Tomo','Data','Latitude','Longitude','Elevation','GPS_time', ... +ct_save('-v7.3',fn_mat,'Tomo','Data','Latitude','Longitude','Elevation','GPS_time', ... 'Surface','Bottom','Time','param_array','param_records', ... 'param_sar', 'Roll', 'Pitch', 'Heading', 'file_version'); @@ -836,6 +856,6 @@ file_version = '1'; fn_mat = fullfile(fn_dir,[fn_name output_fn_midfix '_music.mat']); fprintf('Saving %s (%s)\n', fn_mat, datestr(now)); -save('-v7.3',fn_mat,'Tomo','Data','Latitude','Longitude','Elevation','GPS_time', ... +ct_save('-v7.3',fn_mat,'Tomo','Data','Latitude','Longitude','Elevation','GPS_time', ... 'Surface','Bottom','Time','param_array','param_records', ... 'param_sar', 'Roll', 'Pitch', 'Heading', 'file_version'); diff --git a/cresis-toolbox/+multipass/phaseslope_bydepth.m b/cresis-toolbox/+multipass/phaseslope_bydepth.m deleted file mode 100644 index c34c887f..00000000 --- a/cresis-toolbox/+multipass/phaseslope_bydepth.m +++ /dev/null @@ -1,139 +0,0 @@ -clearvars -except gRadar -close all -%File names for the different years -fn2011_2014 = '/cresis/snfs1/dataproducts/ct_data/rds/2011_Greenland_P3/CSARP_insar/rds_thule_2011_2014_wf2_insar.mat'; -fn2011_2012 = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_insar/rds_thule_2011_2012_wf2_insar3.mat'; -fn2012_2014 = '/cresis/snfs1/dataproducts/ct_data/rds/2012_Greenland_P3/CSARP_insar/rds_thule_2012_2014_wf2_insar.mat'; -fn2012_2013 = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_insar/rds_thule_2012_2013_wf2_insar3.mat'; -fn2013_2014 = '/cresis/snfs1/dataproducts/ct_data/rds/2013_Greenland_P3/CSARP_insar/rds_thule_2013_2014_wf2_insar.mat'; -fn2014_2014 = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_insar/rds_thule_2014_1Day_wf2_insar3.mat'; - -fns = {fn2011_2014, fn2011_2012, fn2012_2014, fn2012_2013, fn2013_2014,fn2014_2014}; -leg = {'2011 to 2014', '2011 to 2012', '2012 to 2014', '2012 to 2013', '2013 to 2014','2014 to 2014'}; -saven = {'PhaseSlope_Data_1Chunk'}; -%Settings -set1 = {1:15, 1:15, 1:15, 1:7, 1:15,1:15}; -set2 = {16:30, 16:30, 16:30, 8:22, 16:22, 16:30}; -if 0 %7 Element Setting - set1 = {5:11, 5:11, 5:11, 5:11, 5:11}; - set2 = {20:26, 20:26, 16:22, 20:26, 20:26}; - saven = cellfun(@(c)[c '_7Elements'],saven,'UniformOutput',false); -end -rbins = [240:420]; -filt_x = 3; filt_y = 31; -filt_med = 5; -filt_psd = 301; %Also called snapshots -PSDthresh = 500; -for fn_id = 1:length(fns) - if ~exist('tmp','var') || length(tmp)pi) = phaseraw(phaseraw>pi)-2*pi; - phaseout{fn_id}(id_ch,:) = phaseraw; - - %Check phaseout - if 0 - figure(300+id_ch) - hold on - plot(1:length(PSDmaxind),phaseout{fn_id}(id_ch,:)) - if fn_id == length(fns) - hold off - legend(leg) - grid on - title(sprintf('Phase Out Chunk %d',id_ch)) - end - end - end -end - -%Convert phase to physical length -eps_ice = 3.15; wfuse = 2; -kappa = 1/sqrt(eps_ice); -cls = physconst('LightSpeed'); -centfreq = (tmp{1}.ref.wfs(wfuse).f0+tmp{1}.ref.wfs(wfuse).f1)/2; -BW = tmp{1}.ref.wfs(wfuse).f1-tmp{1}.ref.wfs(wfuse).f0; -efflambda = centfreq/(kappa*cls); -alongx={}; -for pid = 1:length(fns) - disp_off{pid} = (phaseout{pid}/(4*pi))*efflambda; %m - %Convert geodetic to along track - alongx{pid} = geodetic_to_along_track(tmp{pid}.ref.lat,tmp{pid}.ref.lon,tmp{pid}.ref.elev)/1000; -end - -%Convert rbins to depth - %Grab the rbin corresponding to the middle of the overlapping segment - for id_ch = 1:numchunks - startind = (sampchunk*(id_ch-1)+1)-sampchunk/2*(id_ch-1); - endind = (sampchunk*(id_ch))-sampchunk/2*(id_ch-1); - middleind(id_ch) = mean([startind endind])+rbins(1); - %Grab the associated time value - timemid(id_ch) = tmp{1}.ref.wfs(wfuse).time(floor(middleind(id_ch))); - end - %Convert time to depth - binsize = (cls/(2*BW*sqrt(eps_ice))); - crossy = middleind*binsize; - -%% Save everything -save([saven{1} '.mat'],'disp_off','phaseout','middleind','alongx','leg',... - 'crossy', 'numchunks', 'BW', 'eps_ice', 'fns','daydiff') -% PlotPhaseSlope_1Chunk \ No newline at end of file diff --git a/cresis-toolbox/+multipass/plot_phaseslope.m b/cresis-toolbox/+multipass/plot_phaseslope.m deleted file mode 100644 index ef5162c5..00000000 --- a/cresis-toolbox/+multipass/plot_phaseslope.m +++ /dev/null @@ -1,117 +0,0 @@ -%% Plot everything -clearvars -except gRadar tmp -load('PhaseSlope_Data_3Chunk.mat') -cls = physconst('lightspeed'); -savedir='/cresis/snfs1/scratch/bmiller/MultiPass_Results_12_1/Official_Paper_Results'; - -% % Plot each line together -% leg2 = {'Top', 'Mid', 'Bot'}; legger={}; -% linsty = {'-','--','.-'}; lincol = {'r','b','g'}; -% figure(50) -% for pid = 1:length(fns) -% colnow = lincol{pid}; -% dispnow = phaseout{pid}; -% for id_ch = 1:numchunks -% stynow = linsty{id_ch}; -% ynow = dispnow(id_ch,:)./middleind(id_ch); -% plot(alongx, ynow, [colnow stynow]) -% hold on -% legger{end+1} = sprintf('%s %s',leg{pid},leg2{id_ch}); -% end -% end -% ylabel('radians by range bine (rad/~)') -% xlabel('Along Track') -% title('Depth Change Across Years') -% legend(legger) -% grid on -% hold off - - -filt_disp = 301; -%Plot each line together -leg2 = {'Top', 'Mid', 'Bot'}; legger={}; -linsty = {'-','--',':'}; lincol = {'r','b','g','c','m','k'}; -figure(60) -for pid = fliplr(1:length(fns)) - colnow = lincol{pid}; - dispnow = disp_off{pid}; - %Extra averaging of dispout - dispnow = fir_dec(dispnow,ones(1,filt_disp)/filt_disp); - for id_ch = 1:numchunks - stynow = linsty{id_ch}; -% ynow = dispnow(id_ch,:)./crossy(id_ch); - ynow = dispnow(id_ch,:)./(cls/(2*BW*sqrt(eps_ice))); - plot(alongx{pid}, ynow, [colnow stynow]) - hold on - legger{end+1} = sprintf('%s %s',leg{pid},leg2{id_ch}); - end -end -ylabel('Depth change by Depth (m\cdotm^{-1})') -xlabel('Along Track (km)') -title('Depth Change Across Years') -legend(legger,'Position',[0.8078 0.1196 0.1825 0.7972]) -set(findall(gcf,'type','line'),'LineWidth',2) -set(get(gcf,'Children'),'fontsize',14) -set(gcf,'Position',[147 236 1264 563]) -grid on -hold off -savefn = '3Chunk_PhaseEstimate_DepthbyDepth'; -saveas(gcf,fullfile(savedir,[savefn '.fig'])) -saveas(gcf,fullfile(savedir,[savefn '.png'])) - -%Plot the delta of depth change -%Plot each line together -leg2 = {'Top', 'Mid', 'Bot'}; legger={}; -figure(61) -for pid = fliplr(1:length(fns)) - colnow = lincol{pid}; - dispnow = disp_off{pid}; - %Extra averaging of dispout - dispnow = fir_dec(dispnow,ones(1,filt_disp)/filt_disp); - for id_ch = 1:numchunks - stynow = linsty{id_ch}; -% ynow = dispnow(id_ch,:)./crossy(id_ch); - ynow = dispnow(id_ch,:); - plot(alongx{pid}, ynow, [colnow stynow]) - hold on - legger{end+1} = sprintf('%s %s',leg{pid},leg2{id_ch}); - end -end -ylabel('Depth Change (m)') -xlabel('Along Track (km)') -title('Depth Change Across Years') -legend(legger,'Position',[0.8078 0.1196 0.1825 0.7972]) -set(findall(gcf,'type','line'),'LineWidth',2) -set(get(gcf,'Children'),'fontsize',14) -set(gcf,'Position',[147 236 1264 563]) -grid on -hold off -savefn = '3Chunk_PhaseEstimate_DeltaDepth'; -saveas(gcf,fullfile(savedir,[savefn '.fig'])) -saveas(gcf,fullfile(savedir,[savefn '.png'])) - -% %Plot the phase output -% for pid = 1:length(fns) -% figure(30+pid) -% imagesc(alongx, crossy, phaseout{pid}) -% % [Con, h] = contourf(phaseout{pid},100); -% % set(h,'LineColor','none') -% c = colorbar; -% c.Label.String = 'Phase (rad)'; -% xlabel('Along Track (m)') -% ylabel('Depth (m)') -% title(sprintf('%s to 2014',leg{pid})) -% caxis([-0.07 0]) -% end -% %Plot the displacement output -% for pid = 1:length(fns) -% figure(40+pid) -% imagesc(alongx, crossy, disp_off{pid}) -% c = colorbar; -% c.Label.String = 'Displacement (m)'; -% xlabel('Along Track (km)') -% ylabel('Depth (m)') -% title(sprintf('%s to 2014',leg{pid})) -% caxis([-10 0]*1e-3) -% end - diff --git a/cresis-toolbox/+multipass/run_combine_passes.m b/cresis-toolbox/+multipass/run_combine_passes.m index 3107ae99..fc95a716 100644 --- a/cresis-toolbox/+multipass/run_combine_passes.m +++ b/cresis-toolbox/+multipass/run_combine_passes.m @@ -12,235 +12,580 @@ param_override = []; param = []; -passes = struct('day_seg',{},'frms',{},'param_fn',{},'in_path',{},'imgs',[]); +passes = []; -%% Petermann Line 1 2002 -% pass_name = sprintf('Petermann_line1_2002'); -% dist_min = 500; -% master_pass_idx = 1 -% -% param_fn = 'rds_param_2002_Greenland_P3.xls'; -% day_seg = '20020528_06'; -% frms = 5:6; -% passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); -% start = struct('lat',80.492292,'lon',-59.975190); -% stop = struct('lat',81.116866,'lon',-62.041787); -% input_type = 'echo'; +example_str = 'egig_2011_2012_2014_2018_allwf'; -%% Petermann Line 1 2011 -% pass_name = sprintf('Petermann_line1_2011'); -% dist_min = 500; -% master_pass_idx = 1; +if strcmpi(example_str,'Thwaites_201902_201912_202001') + %% Thwaites Line 1 20190201_01, 20191225_01, 20200127_01 + pass_name = sprintf('Thwaites_201902_201912_202001'); + dist_min = 300; + master_pass_idx = 1; + start = struct('lat',-75.137955,'lon',-105.538244); + stop = struct('lat',-75.259933,'lon',-105.418265); + input_type = 'echo'; + passes = struct('day_seg',{},'frms',{},'param_fn',{},'in_path',{}); + + param_fn = 'accum_param_2018_Antarctica_TObas.xls'; + day_seg = '20190201_01'; + frms = 33:34; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post/standard'); + + param_fn = 'accum_param_2019_Antarctica_TObas.xls'; + day_seg = '20191225_01'; + frms = 20:22; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); + + param_fn = 'accum_param_2019_Antarctica_TObas.xls'; + day_seg = '20200127_01'; + frms = 33:34; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); +end -% param_fn = 'rds_param_2011_Greenland_P3.xls'; -% day_seg = '20110507_02'; -% frms = 9:11; -% passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post/csarp-combined'); -% start = struct('lat',80.499370,'lon',-60.013440); -% stop = struct('lat',80.952663,'lon',-61.586491); -% input_type = 'echo'; +if strcmpi(example_str,'Petermann_line1_2011_2014_2015_2017_2018_2019') + %% Petermann Line 1 2011, 2014, 2015, 2017, 2018, 2019 + pass_name = sprintf('Petermann_line1_2011_2014_2015_2017_2018_2019'); + dist_min = 5000; + master_pass_idx = 1; + start = struct('lat',80.499370,'lon',-60.013440); + stop = struct('lat',81.100,'lon',-61.800); + input_type = 'echo'; + passes = struct('day_seg',{},'frms',{},'param_fn',{},'in_path',{}); + + param_fn = 'rds_param_2011_Greenland_P3.xls'; + day_seg = '20110507_02'; + frms = 9:11; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); + + param_fn = 'rds_param_2014_Greenland_P3.xls'; + day_seg = '20140505_01'; + frms = 17:19; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); + + param_fn = 'rds_param_2015_Greenland_C130.xls'; + day_seg = '20150505_02'; + frms = 16:18; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post/standard'); + + param_fn = 'rds_param_2017_Greenland_P3.xls'; + day_seg = '20170331_01'; + frms = 16:17; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post/standard'); + + param_fn = 'rds_param_2018_Greenland_P3.xls'; + day_seg = '20180405_01'; + frms = 15:17; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post/standard'); + + param_fn = 'rds_param_2019_Greenland_P3.xls'; + day_seg = '20190417_01'; + frms = 15:16; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); +end -%% Petermann Line 1 2014 -% pass_name = sprintf('Petermann_line1_2014'); -% dist_min = 500; -% master_pass_idx = 1; -% start = struct('lat',80.503546,'lon',-60.034880); -% stop = struct('lat',80.929117,'lon',-61.524422); -% input_type = 'echo'; -% -% param_fn = 'rds_param_2014_Greenland_P3.xls'; -% day_seg = '20140505_01'; -% frms = 17:18; -% passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); +if strcmpi(example_str,'Petermann_line2_2013_2014_2017') + %% Petermann Line 2 2013, 2014, 2017 + pass_name = sprintf('Petermann_line2_2013_2014_2017'); + dist_min = 500; + master_pass_idx = 1; + start = struct('lat',80.517191,'lon',-59.844969); + stop = struct('lat',81.031,'lon',-61.672); + + input_type = 'echo'; + passes = struct('day_seg',{},'frms',{},'param_fn',{},'in_path',{}); + param_fn = 'rds_param_2013_Greenland_P3.xls'; + day_seg = '20130420_02'; + frms = 4:5; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); + + param_fn = 'rds_param_2014_Greenland_P3.xls'; + day_seg = '20140512_01'; + frms = 16:17; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); + + param_fn = 'rds_param_2017_Greenland_P3.xls'; + day_seg = '20170331_01'; + frms = 22:23; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post\standard'); +end + +if strcmpi(example_str,'Petermann_line3_2010_2017') + %% Petermann Line 2 2013, 2014, 2017 + pass_name = sprintf('Petermann_line3_2010_2017'); + dist_min = 500; + master_pass_idx = 2; + start = struct('lat',80.517191,'lon',-59.844969); + stop = struct('lat',80.903411,'lon',-61.332159); + input_type = 'echo'; + passes = struct('day_seg',{},'frms',{},'param_fn',{},'in_path',{}); + + % Currently not processed for 2010 DC8 and 2010 P3 segments + param_fn = 'rds_param_2010_Greenland_DC8.xls'; + day_seg = '20100420_03'; + frms = 8:9; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post\standard'); + + param_fn = 'rds_param_2017_Greenland_P3.xls'; + day_seg = '201700414_01'; + frms = 16:17; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post\standard'); +end + +if strcmpi(example_str,'Petermann_line4_2010_2011_2013_2014_2017') + %% Petermann Line 4 2010, 2011, 2013, 2014, 2017 + pass_name = sprintf('Petermann_line4_2010_2011_2013_2014_2017'); + dist_min = 500; + master_pass_idx = 1; + start = struct('lat',80.5295278,'lon',-59.567146); + stop = struct('lat',81.100,'lon',-61.686); + input_type = 'echo'; + passes = struct('day_seg',{},'frms',{},'param_fn',{},'in_path',{}); + + param_fn = 'rds_param_2010_Greenland_DC8.xls'; + day_seg = '20100324_01'; + frms = 25:27; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); + + param_fn = 'rds_param_2011_Greenland_P3.xls'; + day_seg = '20110507_02'; + frms = 13:15; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); + + param_fn = 'rds_param_2013_Greenland_P3.xls'; + day_seg = '20130420_02'; + frms = 7:8; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); + + param_fn = 'rds_param_2014_Greenland_P3.xls'; + day_seg = '20140512_01'; + frms = 13:15; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); + + param_fn = 'rds_param_2017_Greenland_P3.xls'; + day_seg = '20170331_01'; + frms = 13:14; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post\standard'); +end + +if strcmpi(example_str,'Petermann_oblique_2014_2015_2017_2018') + %% Petermann Oblique 2014, 2017, 2018 + pass_name = sprintf('Petermann_oblique_2014_2015_2017_2018'); + dist_min = 500; + master_pass_idx = 1; + start = struct('lat',80.775,'lon',-60.166); + stop = struct('lat',80.468,'lon',-60.575); + input_type = 'echo'; + passes = struct('day_seg',{},'frms',{},'param_fn',{},'in_path',{}); + + param_fn = 'rds_param_2014_Greenland_P3.xls'; + day_seg = '20140505_01'; + frms = 20:21; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post/standard'); + + param_fn = 'rds_param_2015_Greenland_C130.xls'; + day_seg = '20150505_02'; + frms = 20:21; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post/standard'); + + param_fn = 'rds_param_2017_Greenland_P3.xls'; + day_seg = '20170414_01'; + frms = 19:20; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post/standard'); + + param_fn = 'rds_param_2018_Greenland_P3.xls'; + day_seg = '20180405_01'; + frms = 18:19; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); +end -%% 2017 -% param_fn = 'rds_param_2017_Greenland_P3.xls'; -% day_seg = '20170331_01'; -% frms = 16:17; -% passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post/standard','imgs'); -% start = struct('lat',80.504179,'lon',-60.037068); -% stop = struct('lat',80.962162,'lon',-61.612482); -% pass_name = sprintf('2017_Greenland_P3'); -% input_type = 'echo'; +if strcmpi(example_str,'79N_line1_2010_2014_2016_2018') + %% 79N Line 1 2010, 2014, 2016, 2018 + pass_name = sprintf('79N_line1_2010_2014_2016_2018'); + dist_min = 16000; + master_pass_idx = 2; + start = struct('lat',79.346109,'lon',-22.575643); + stop = struct('lat',79.559002,'lon',-19.329911); + input_type = 'echo'; + passes = struct('day_seg',{},'frms',{},'param_fn',{},'in_path',{}); + + param_fn = 'rds_param_2010_Greenland_P3.xls'; + day_seg = '20100525_04'; + frms = 11:13; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post/standard'); + + param_fn = 'rds_param_2014_Greenland_P3.xls'; + day_seg = '20140429_01'; + frms = 43:44; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post/standard'); + + param_fn = 'rds_param_2016_Greenland_P3.xls'; + day_seg = '20160509_10'; + frms = 1; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post/standard'); + + param_fn = 'rds_param_2018_Greenland_P3.xls'; + day_seg = '20180418_05'; + frms = 1:2; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post/standard'); +end -%% Petermann Line 1 2018 -% pass_name = sprintf('Petermann_line1_2018'); -% dist_min = 500; -% master_pass_idx = 1; -% start = struct('lat',80.521230,'lon',-60.127320); -% stop = struct('lat',80.964739,'lon',-61.618307); -% input_type = 'echo'; -% -% param_fn = 'rds_param_2018_Greenland_P3.xls'; -% day_seg = '20180405_01'; -% frms = 15:16; -% passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); +if strcmpi(example_str,'79N_Cross1_2012_2017_2018_2019') + %% 79N Line 1 2012, 2017, 2018, 2019 + pass_name = sprintf('79N_Cross1_2012_2017_2018_2019'); + dist_min = 16000; + master_pass_idx = 2; + start = struct('lat',79.346109,'lon',-22.575643); + stop = struct('lat',79.559002,'lon',-19.329911); + input_type = 'echo'; + passes = struct('day_seg',{},'frms',{},'param_fn',{},'in_path',{}); + + param_fn = 'rds_param_2012_Greenland_P3.xls'; + day_seg = '20120514_02'; + frms = 19; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post/standard'); + + param_fn = 'rds_param_2017_Greenland_P3.xls'; + day_seg = '20140429_01'; + frms = 43:44; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post/standard'); + + param_fn = 'rds_param_2018_Greenland_P3.xls'; + day_seg = '20160509_10'; + frms = 1; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post/standard'); + + % param_fn = 'rds_param_2019_Greenland_P3.xls'; + % day_seg = '20190405_02'; + % frms = 15; + % passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post/standard'); +end -%% Petermann Line 1 2011, 2014, 2018 -% pass_name = sprintf('Petermann_line1_2011_2014_2018'); -% dist_min = 500; -% master_pass_idx = 2; -% start = struct('lat',80.499370,'lon',-60.013440); -% stop = struct('lat',80.964739,'lon',-61.618307); -% input_type = 'echo'; -% -% param_fn = 'rds_param_2011_Greenland_P3.xls'; -% day_seg = '20110507_02'; -% frms = 9:11; -% passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); -% -% param_fn = 'rds_param_2014_Greenland_P3.xls'; -% day_seg = '20140505_01'; -% frms = 17:18; -% passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); -% -% param_fn = 'rds_param_2018_Greenland_P3.xls'; -% day_seg = '20180405_01'; -% frms = 15:16; -% passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); -%% Petermann Line 2 2013, 2014 -% pass_name = sprintf('Petermann_line2_2013_2014'); -% dist_min = 500; -% master_pass_idx = 2; -% start = struct('lat',80.517191,'lon',-59.844969); -% stop = struct('lat',80.903411,'lon',-61.332159); -% input_type = 'echo'; -% -% param_fn = 'rds_param_2013_Greenland_P3.xls'; -% day_seg = '20130420_02'; -% frms = 4:5; -% passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); -% -% param_fn = 'rds_param_2014_Greenland_P3.xls'; -% day_seg = '20140512_01'; -% frms = 16:17; -% passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); +if strcmpi(example_str,'Ryder_line1_2011_2013_2015_2019') + %% Ryder line1 2011,2013,2015,2019 + pass_name = sprintf('Ryder_line1_2011_2013_2015_2019'); + dist_min = 16000; + master_pass_idx = 3; % Use 2015 since it runs parallel to glacier + start = struct('lat',81.420,'lon',-50.278); + stop = struct('lat',81.835,'lon',-51.104); + input_type = 'echo'; + passes = struct('day_seg',{},'frms',{},'param_fn',{},'in_path',{}); + + param_fn = 'rds_param_2011_Greenland_P3.xls'; + day_seg = '20110509_01'; + frms = 19:20; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); + + param_fn = 'rds_param_2013_Greenland_P3.xls'; + day_seg = '20130426_01'; + frms = 17:18; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post/standard'); + + param_fn = 'rds_param_2015_Greenland_C130.xls'; + day_seg = '20150506_02'; + frms = 19:20; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post/standard'); + + param_fn = 'rds_param_2019_Greenland_P3.xls'; + day_seg = '20190423_01'; + frms = 15:16; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); +end -%% Petermann Line 4 2011, 2014, 2018 -% pass_name = sprintf('Petermann_line4_2010_2011_2013_2014'); -% dist_min = 500; -% master_pass_idx = 4; -% start = struct('lat',80.5295278,'lon',-59.567146); -% stop = struct('lat',80.995874,'lon',-61.357055); -% input_type = 'echo'; -% -% param_fn = 'rds_param_2010_Greenland_DC8.xls'; -% day_seg = '20100324_01'; -% frms = 25:26; -% passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); -% -% param_fn = 'rds_param_2011_Greenland_P3.xls'; -% day_seg = '20110507_02'; -% frms = 13:15; -% passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); -% -% param_fn = 'rds_param_2013_Greenland_P3.xls'; -% day_seg = '20130420_02'; -% frms = 7:8; -% passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); -% -% param_fn = 'rds_param_2014_Greenland_P3.xls'; -% day_seg = '20140512_01'; -% frms = 13:14; -% passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); +if strcmpi(example_str,'Steensby_line1_2011_2013_2015_2019') + %% Steensby line1 2011,2013,2015,2019 + pass_name = sprintf('Steensby_line1_2011_2013_2015_2019'); + dist_min = 16000; + master_pass_idx = 2; + start = struct('lat',81.457622,'lon',-54.244113); + stop = struct('lat',81.687467,'lon',-54.537472); + input_type = 'echo'; + passes = struct('day_seg',{},'frms',{},'param_fn',{},'in_path',{}); + + param_fn = 'rds_param_2011_Greenland_P3.xls'; + day_seg = '20110509_01'; + frms = 16; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post/standard'); + + param_fn = 'rds_param_2013_Greenland_P3.xls'; + day_seg = '20130426_01'; + frms = 15:16; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post/standard'); + + param_fn = 'rds_param_2015_Greenland_C130.xls'; + day_seg = '20150506_02'; + frms = 16:17; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post/standard'); + + param_fn = 'rds_param_2019_Greenland_P3.xls'; + day_seg = '20190423_01'; + frms = 13:14; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); +end -%% 79N Line 1 2010, 2014, 2016, 2018 -% pass_name = sprintf('79N_line1_2010_2014_2016_2018'); -% dist_min = 16000; -% master_pass_idx = 2; -% start = struct('lat',79.346109,'lon',-22.575643); -% stop = struct('lat',79.559002,'lon',-19.329911); -% input_type = 'echo'; -% -% param_fn = 'rds_param_2010_Greenland_P3.xls'; -% day_seg = '20100525_04'; -% frms = 11:13; -% passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); -% -% param_fn = 'rds_param_2014_Greenland_P3.xls'; -% day_seg = '20140429_01'; -% frms = 43:44; -% passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); -% -% param_fn = 'rds_param_2016_Greenland_P3.xls'; -% day_seg = '20160509_10'; -% frms = 1; -% passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); -% -% param_fn = 'rds_param_2018_Greenland_P3.xls'; -% day_seg = '20180418_05'; -% frms = 1:2; -% passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); +if strcmpi(example_str,'Humboldt_line1_2012_2013_2014_2017') + %% Humboldt line1 2012,2013,2014,2017 + pass_name = sprintf('Humboldt_line1_2012_2013_2014_2017'); + dist_min = 16000; + master_pass_idx = 1; + start = struct('lat',79.801630,'lon',-64.155016); + stop = struct('lat',79.798388,'lon',-64.752165); + input_type = 'echo'; + passes = struct('day_seg',{},'frms',{},'param_fn',{},'in_path',{}); + + param_fn = 'rds_param_2012_Greenland_P3.xls'; + day_seg = '20120511_01'; + frms = 39; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post/standard'); + + param_fn = 'rds_param_2013_Greenland_P3.xls'; + day_seg = '20130420_01'; + frms = 35; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post/standard'); + + param_fn = 'rds_param_2014_Greenland_P3.xls'; + day_seg = '20140520_03'; + frms = 31; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post/standard'); + + param_fn = 'rds_param_2017_Greenland_P3.xls'; + day_seg = '20170417_01'; + frms = 40; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post/standard'); +end -%% 2014 Greenland P3 2 Week Difference -% wf = 3; -% pass_name = sprintf('rds_thule_2014_2Week_wf%d',wf); -% start = struct('lat', 77.10,'lon', -62.3); -% stop = struct('lat', 77.13, 'lon', -61.9); -% dist_min = 300; -% master_pass_idx = 8; -% input_type = 'sar'; -% -% param_fn = 'rds_param_2014_Greenland_P3.xls'; -% -% for adc = 2:16 -% passes(end+1) = struct('day_seg','20140429_01','frms',67,'param_fn',param_fn,'in_path','','imgs',[wf adc]); -% end -% for adc = 2:16 -% passes(end+1) = struct('day_seg','20140515_02','frms',4,'param_fn',param_fn,'in_path','','imgs',[wf adc]); -% end +if strcmpi(example_str,'ZI_line1_2010_2014_2016_2017_2018_2019') + %% Z.I. line1 2010,2014,2016,2017,2018, 2019 + pass_name = sprintf('ZI_line1_2010_2014_2016_2017_2018_2019'); + dist_min = 16000; + master_pass_idx = 2; + start = struct('lat',78.939138,'lon',-21.354236); + stop = struct('lat',78.927951,'lon',-19.105975); + input_type = 'echo'; + passes = struct('day_seg',{},'frms',{},'param_fn',{},'in_path',{}); + + param_fn = 'rds_param_2010_Greenland_P3.xls'; + day_seg = '20100525_04'; + frms = 09; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post/standard'); + + param_fn = 'rds_param_2014_Greenland_P3.xls'; + day_seg = '20140508_01'; + frms = 46:47; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post/standard'); + + param_fn = 'rds_param_2016_Greenland_P3.xls'; + day_seg = '20160509_07'; + frms = 10; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post/standard'); + + param_fn = 'rds_param_2017_Greenland_P3.xls'; + day_seg = '20170403_02'; + frms = 4:5; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','CSARP_post/standard'); + + param_fn = 'rds_param_2018_Greenland_P3.xls'; + day_seg = '20180418_04'; + frms = 18:19; + passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); + + % param_fn = 'rds_param_2019_Greenland_P3.xls'; + % day_seg = '20190405_01'; + % frms = 22:23; + % passes(end+1) = struct('day_seg',day_seg,'frms',frms,'param_fn',param_fn,'in_path','standard'); +end -%% 2014 Greenland P3 Same Day -% pass_name = 'rds_thule_2014_SameDay_allwf'; -% start = struct('lat', 77.10,'lon', -62.3); -% stop = struct('lat', 77.13, 'lon', -61.9); -% dist_min = 300; -% master_pass_idx = 8; -% input_type = 'sar'; -% -% param_fn = 'rds_param_2014_Greenland_P3.xls'; -% -% for adc = 2:16 -% passes(end+1) = struct('day_seg','20140429_01','frms',[67 5],'param_fn',param_fn,'in_path','','imgs',{{[1 adc], [2 adc], [3 adc]}}); -% end +if strcmpi(example_str,'camp_century_2014_2_weeks') + %% Camp Century: 2014 Greenland P3 2 weeks + pass_name = 'camp_century_2014_2_weeks'; + dist_min = 300; + master_pass_idx = 8; + start = struct('lat', 77.10,'lon', -62.3); + stop = struct('lat', 77.13, 'lon', -61.9); + input_type = 'sar'; + passes = struct('day_seg',{},'frms',{},'param_fn',{},'in_path',{},'imgs',[]); + + param_fn = 'rds_param_2014_Greenland_P3.xls'; + for adc = 2:16 + passes(end+1) = struct('day_seg','20140429_01','frms',67,'param_fn',param_fn,'in_path','','imgs',{{[1 adc], [2 adc], [3 adc]}}); + end + for adc = 2:16 + passes(end+1) = struct('day_seg','20140515_02','frms',4,'param_fn',param_fn,'in_path','','imgs',{{[1 adc], [2 adc], [3 adc]}}); + end +end -%% 2011 to 2012 Greenland P3 -% pass_name = sprintf('rds_thule_2011_2012_wf2'); -% start = struct('lat', 77.10,'lon', -62.3); -% stop = struct('lat', 77.13, 'lon', -61.9); -% dist_min = 300; -% master_pass_idx = 8; -% input_type = 'sar'; -% -% param_fn = 'rds_param_2012_Greenland_P3.xls'; -% wf = 2; -% for adc = 2:16 -% passes(end+1) = struct('day_seg','20120516_01','frms',89,'param_fn',param_fn,'in_path','','imgs',[wf adc]); -% end -% param_fn = 'rds_param_2011_Greenland_P3.xls'; -% wf = 2; -% for adc = 2:16 -% passes(end+1) = struct('day_seg','20110502_02','frms',32,'param_fn',param_fn,'in_path','','imgs',[wf adc]); -% end +if strcmpi(example_str,'camp_century_2014_same_day') + %% Camp Century: 2014 Greenland P3 same day + pass_name = 'camp_century_2014_same_day'; + dist_min = 300; + master_pass_idx = 8; + start = struct('lat', 77.10,'lon', -62.3); + stop = struct('lat', 77.13, 'lon', -61.9); + input_type = 'sar'; + passes = struct('day_seg',{},'frms',{},'param_fn',{},'in_path',{},'imgs',[]); + + param_fn = 'rds_param_2014_Greenland_P3.xls'; + for adc = 2:16 + passes(end+1) = struct('day_seg','20140429_01','frms',[67 5],'param_fn',param_fn,'in_path','','imgs',{{[1 adc], [2 adc], [3 adc]}}); + end +end -%% Summit Camp: 2012-2014 -pass_name = sprintf('summit_2012_2014_allwf'); -start = struct('lat', 72.646,'lon', -37.898); -stop = struct('lat', 72.791, 'lon', -38.461); -dist_min = 300; -master_pass_idx = 8; -input_type = 'sar'; +if strcmpi(example_str,'camp_century_2011_2012_2013_2014') + %% Camp Century: 2011, 2012, 2013, 2014 Greenland P3 + pass_name = sprintf('camp_century_2011_2012_2013_2014'); + dist_min = 300; + master_pass_idx = 15+15+7+8; + start = struct('lat', 77.10,'lon', -62.3); + stop = struct('lat', 77.13, 'lon', -61.9); + input_type = 'sar'; + passes = struct('day_seg',{},'frms',{},'param_fn',{},'in_path',{},'imgs',[]); + + param_fn = 'rds_param_2011_Greenland_P3.xls'; + for adc = 2:16 + passes(end+1) = struct('day_seg','20110502_02','frms',32,'param_fn',param_fn,'in_path','','imgs',{{[1 adc], [2 adc]}}); + end + param_fn = 'rds_param_2012_Greenland_P3.xls'; + for adc = 2:16 + passes(end+1) = struct('day_seg','20120516_01','frms',89,'param_fn',param_fn,'in_path','','imgs',{{[1 adc], [2 adc]}}); + end + param_fn = 'rds_param_2013_Greenland_P3.xls'; + for adc = 1:7 + passes(end+1) = struct('day_seg','20130419_01','frms',4,'param_fn',param_fn,'in_path','','imgs',{{[1 adc], [2 adc]}}); + end + param_fn = 'rds_param_2014_Greenland_P3.xls'; + for adc = 2:16 + passes(end+1) = struct('day_seg','20140429_01','frms',67,'param_fn',param_fn,'in_path','','imgs',{{[1 adc], [2 adc], [3 adc]}}); + end + for adc = 2:16 + passes(end+1) = struct('day_seg','20140429_01','frms',5,'param_fn',param_fn,'in_path','','imgs',{{[1 adc], [2 adc], [3 adc]}}); + end + for adc = 2:16 + passes(end+1) = struct('day_seg','20140515_02','frms',4,'param_fn',param_fn,'in_path','','imgs',{{[1 adc], [2 adc], [3 adc]}}); + end +end -param_fn = 'rds_param_2014_Greenland_P3.xls'; -for adc = 2:16 - passes(end+1) = struct('day_seg','20140502_01','frms',41,'param_fn',param_fn, ... - 'in_path','','imgs',{{[1 adc], [2 adc], [3 adc]}}); +if strcmpi(example_str,'summit_2012_2014_allwf') + %% Summit Camp: 2012-2014 + pass_name = sprintf('summit_2012_2014_allwf'); + dist_min = 300; + master_pass_idx = 8; + start = struct('lat', 72.646,'lon', -37.898); + stop = struct('lat', 72.791, 'lon', -38.461); + input_type = 'sar'; + passes = struct('day_seg',{},'frms',{},'param_fn',{},'in_path',{},'imgs',[],'param_override',[]); + + param_fn = 'rds_param_2014_Greenland_P3.xls'; + pass_override = []; + tmp_param = read_param_xls(ct_filename_param(param_fn),'20140502_01'); + for wf = 1:length(tmp_param.radar.wfs) + pass_override.radar.wfs(wf).chan_equal_dB = tmp_param.radar.wfs(wf).chan_equal_dB; + pass_override.radar.wfs(wf).chan_equal_deg = tmp_param.radar.wfs(wf).chan_equal_deg; + pass_override.radar.wfs(wf).Tsys = tmp_param.radar.wfs(wf).Tsys; + end + pass_override.radar.wfs(1).chan_equal_deg = pass_override.radar.wfs(1).chan_equal_deg + -94.9225; % wf 1 to 2 correction + pass_override.radar.wfs(1).Tsys = pass_override.radar.wfs(1).Tsys + 5.86577e-09; % wf 1 to 2 correction + pass_override.radar.wfs(3).chan_equal_deg = pass_override.radar.wfs(3).chan_equal_deg + 52.7387 + -98.5571; % wf 2 to 3 correction + pass_override.radar.wfs(3).Tsys = pass_override.radar.wfs(3).Tsys + 1.16604e-08; % wf 2 to 3 correction + for adc = 2:16 + passes(end+1) = struct('day_seg','20140502_01','frms',41,'param_fn',param_fn,'in_path','','imgs',{{[1 adc], [2 adc], [3 adc]}},'param_override',pass_override); + end + + param_fn = 'rds_param_2012_Greenland_P3.xls'; + pass_override = []; + tmp_param = read_param_xls(ct_filename_param(param_fn),'20120330_03'); + for wf = 1:length(tmp_param.radar.wfs) + pass_override.radar.wfs(wf).chan_equal_dB = tmp_param.radar.wfs(wf).chan_equal_dB; + pass_override.radar.wfs(wf).chan_equal_deg = tmp_param.radar.wfs(wf).chan_equal_deg; + pass_override.radar.wfs(wf).Tsys = tmp_param.radar.wfs(wf).Tsys; + end + pass_override.radar.wfs(1).chan_equal_deg = pass_override.radar.wfs(1).chan_equal_deg + 5.09989; % wf 1 to 2 correction + pass_override.radar.wfs(1).Tsys = pass_override.radar.wfs(1).Tsys -5.65956e-09; % wf 1 to 2 correction + for adc = 2:16 + passes(end+1) = struct('day_seg','20120330_03','frms',8,'param_fn',param_fn,'in_path','','imgs',{{[1 adc], [2 adc]}},'param_override',pass_override); + end end -param_fn = 'rds_param_2012_Greenland_P3.xls'; -for adc = 2:16 - passes(end+1) = struct('day_seg','20120330_03','frms',8,'param_fn',param_fn,'in_path','','imgs',{{[1 adc], [2 adc]}}); +if strcmpi(example_str,'egig_2011_2012_2014_2018_allwf') + %% EGIG line: 2011-2018 + pass_name = sprintf('egig_2011_2012_2014_2018_allwf'); + start = struct('lat', 71.147248,'lon', -36.921416); + stop = struct('lat', 71.179219, 'lon', -36.442813); + dist_min = 300; + + master_pass_idx = 8; + input_type = 'sar'; + passes = struct('day_seg',{},'frms',{},'param_fn',{},'in_path',{},'imgs',[],'param_override',[]); + + % Additional data to capture physical signals of interest + % 2011: 20110426_11_006 and 007 + % 2012: 20120411_02_007 and 008 + % 2014: 20140410_01_058 and 059 + % 2017: 20170506_01_058 and 059 + % 2018: 20180501_01_051 and 053, 054, and 055 + + param_fn = 'rds_param_2014_Greenland_P3.xls'; + pass_override = []; + tmp_param = read_param_xls(ct_filename_param(param_fn),'20140410_01'); + for wf = 1:length(tmp_param.radar.wfs) + pass_override.radar.wfs(wf).chan_equal_dB = tmp_param.radar.wfs(wf).chan_equal_dB; + pass_override.radar.wfs(wf).chan_equal_deg = tmp_param.radar.wfs(wf).chan_equal_deg; + pass_override.radar.wfs(wf).Tsys = tmp_param.radar.wfs(wf).Tsys; + end + pass_override.radar.wfs(1).chan_equal_deg = pass_override.radar.wfs(1).chan_equal_deg + -94.9225; % wf 1 to 2 correction + pass_override.radar.wfs(1).Tsys = pass_override.radar.wfs(1).Tsys + 5.86577e-09; % wf 1 to 2 correction + pass_override.radar.wfs(3).chan_equal_deg = pass_override.radar.wfs(3).chan_equal_deg + 52.7387 + -98.5571; % wf 2 to 3 correction + pass_override.radar.wfs(3).Tsys = pass_override.radar.wfs(3).Tsys + 1.16604e-08; % wf 2 to 3 correction + for adc = 2:16 + passes(end+1) = struct('day_seg','20140410_01','frms',57,'param_fn',param_fn,'in_path','','imgs',{{[1 adc], [2 adc], [3 adc]}},'param_override',pass_override); + end + + param_fn = 'rds_param_2011_Greenland_P3.xls'; + pass_override = []; + tmp_param = read_param_xls(ct_filename_param(param_fn),'20110426_11'); + for wf = 1:length(tmp_param.radar.wfs) + pass_override.radar.wfs(wf).chan_equal_dB = tmp_param.radar.wfs(wf).chan_equal_dB; + pass_override.radar.wfs(wf).chan_equal_deg = tmp_param.radar.wfs(wf).chan_equal_deg; + pass_override.radar.wfs(wf).Tsys = tmp_param.radar.wfs(wf).Tsys; + end + pass_override.radar.wfs(1).chan_equal_deg = pass_override.radar.wfs(1).chan_equal_deg + -26.8951 + 47.4778; % wf 1 to 2 correction + pass_override.radar.wfs(1).Tsys = pass_override.radar.wfs(1).Tsys - 5.8045e-09; % wf 1 to 2 correction + for adc = 2:16 + passes(end+1) = struct('day_seg','20110426_11','frms',5,'param_fn',param_fn,'in_path','','imgs',{{[1 adc], [2 adc]}},'param_override',pass_override); + end + + param_fn = 'rds_param_2012_Greenland_P3.xls'; + pass_override = []; + tmp_param = read_param_xls(ct_filename_param(param_fn),'20120411_02'); + for wf = 1:length(tmp_param.radar.wfs) + pass_override.radar.wfs(wf).chan_equal_dB = tmp_param.radar.wfs(wf).chan_equal_dB; + pass_override.radar.wfs(wf).chan_equal_deg = tmp_param.radar.wfs(wf).chan_equal_deg; + pass_override.radar.wfs(wf).Tsys = tmp_param.radar.wfs(wf).Tsys; + end + pass_override.radar.wfs(1).chan_equal_deg = pass_override.radar.wfs(1).chan_equal_deg + 5.09989; % wf 1 to 2 correction + pass_override.radar.wfs(1).Tsys = pass_override.radar.wfs(1).Tsys -5.65956e-09; % wf 1 to 2 correction + for adc = 2:16 + passes(end+1) = struct('day_seg','20120411_02','frms',[9 10],'param_fn',param_fn,'in_path','','imgs',{{[1 adc], [2 adc]}},'param_override',pass_override); + end + + % 2017 data on tape, no SAR data products + % param_fn = 'rds_param_2017_Greenland_P3.xls'; + % for adc = 2:16 + % passes(end+1) = struct('day_seg','20170506_01','frms',57,'param_fn',param_fn,'in_path','','imgs',{{[1 adc], [2 adc], [3 adc]}}); + % end + +% param_fn = 'rds_param_2018_Greenland_P3.xls'; +% pass_override = []; +% for adc = [1:4,6:16] +% passes(end+1) = struct('day_seg','20180501_01','frms',[52],'param_fn',param_fn, ... +% 'in_path','','imgs',{{[1 adc],[3 adc], [5 adc]}},'param_override',pass_override); +% end + % Combining left and right beams could present a problem due to them + % having common phase centers so disabling the right transmit beam data + %for adc = [1:4,6:16] + % passes(end+1) = struct('day_seg','20180501_01','frms',[52],'param_fn',param_fn, ... + % 'in_path','','imgs',{{[2 adc], [4 adc], [6 adc]}},'param_override',pass_override); + %end + end %% Automated section diff --git a/cresis-toolbox/+multipass/run_multipass.m b/cresis-toolbox/+multipass/run_multipass.m index 89af6559..4d7176d3 100644 --- a/cresis-toolbox/+multipass/run_multipass.m +++ b/cresis-toolbox/+multipass/run_multipass.m @@ -1,105 +1,186 @@ -clearvars -except gRadar +% script run_multipass +% +% Script for running multipass.m +% +% Authors: Cody Barnett, Bailey Miller, John Paden +% +% See also: multipass.combine_passes.m, multipass.run_combine_passes.m, +% multipass.multipass.m, multipass.run_multipass.m + +%% User Settings +% ========================================================================= + +global gRadar; +param_override = []; param = []; -%% Petermann Line 1 2014 -% if ispc -% fn = fullfile('X:\ct_data\rds\2014_Greenland_P3\CSARP_multipass\',sprintf('Petermann_line1_2014.mat')); -% else -% fn = fullfile('/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_multipass/',sprintf('Petermann_line1_2014')); -% end - -%% Petermann Line 1 2011, 2014, 2018 -% if ispc -% param.multipass.fn = fullfile('X:\ct_data\rds\2014_Greenland_P3\CSARP_multipass\',sprintf('Petermann_line1_2011_2014_2018.mat')); -% else -% param.multipass.fn = fullfile('/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_multipass/',sprintf('Petermann_line1_2011_2014_2018')); -% end -% -% param.multipass.rbins = []; -% -% param.multipass.baseline_master_idx = 2; -% param.multipass.master_idx = 2; -% -% param.multipass.pass_en_mask = []; -% param.multipass.output_fn_midfix = []; -% param.multipass.coregistration_time_shift = [0 0 -2]; -% param.multipass.comp_mode = 2; - - -%% Petermann Line 2 2013, 2014 -% if ispc -% param.multipass.fn = fullfile('X:\ct_data\rds\2014_Greenland_P3\CSARP_multipass\',sprintf('Petermann_line2_2013_2014.mat')); -% else -% param.multipass.fn = fullfile('/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_multipass/',sprintf('Petermann_line2_2013_2014')); -% end -% -% param.multipass.rbins = []; -% -% param.multipass.baseline_master_idx = 2; -% param.multipass.master_idx = 2; -% -% param.multipass.pass_en_mask = []; -% param.multipass.output_fn_midfix = []; -% param.multipass.coregistration_time_shift = [-0.5 0]; -% param.multipass.comp_mode = 2; - -%% Petermann Line 4 2010, 2011, 2013, 2014 -% if ispc -% param.multipass.fn = fullfile('X:\ct_data\rds\2014_Greenland_P3\CSARP_multipass\',sprintf('Petermann_line4_2010_2011_2013_2014.mat')); -% else -% param.multipass.fn = fullfile('/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_multipass/',sprintf('Petermann_line4_2010_2011_2013_2014')); -% end -% -% param.multipass.rbins = []; -% -% param.multipass.baseline_master_idx = 2; -% param.multipass.master_idx = 2; -% -% param.multipass.pass_en_mask = []; -% param.multipass.output_fn_midfix = []; -% param.multipass.coregistration_time_shift = [0 -0.5 0 0]; -% param.multipass.comp_mode = 2; - - -%% 79N Line 1 2010, 2014, 2016, 2018 -% if ispc -% param.multipass.fn = fullfile('X:\ct_data\rds\2014_Greenland_P3\CSARP_multipass\',sprintf('79N_line1_2010_2014_2016_2018.mat')); -% else -% param.multipass.fn = fullfile('/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_multipass/',sprintf('79N_line1_2010_2014_2016_2018')); -% end -% -% param.multipass.rbins = []; -% -% param.multipass.baseline_master_idx = 2; -% param.multipass.master_idx = 2; -% -% param.multipass.pass_en_mask = []; -% param.multipass.output_fn_midfix = []; -% param.multipass.coregistration_time_shift = [1 0 0 -2]; -% param.multipass.time_gate = [2e-6 13e-6]; -% -% comp_mode = 2; -%% 2011 to 2012 Greenland P3 -% radartype = 'rds'; -% passname = 'rds_thule_2011_2012'; -% param.multipass.fn = fullfile(gRadar.out_path,radartype,'2014_Greenland_P3','CSARP_multipass',passname); -% -% param.multipass.rbins = 220:420; -% -% param.multipass.baseline_master_idx = 8; -% param.multipass.master_idx = 8; -% -% param.multipass.pass_en_mask = []; -% param.multipass.output_fn_midfix = []; -% param.multipass.coregistration_time_shift = []; -% param.multipass.comp_mode = 1; -% param.multipass.time_gate = []; - -%% 2014 Same Day Greenland P3 -if 0 - radartype = 'rds'; - passname = 'rds_thule_2014_SameDay_allwf'; - param.multipass.fn = fullfile(gRadar.out_path,radartype,'2014_Greenland_P3','CSARP_multipass',passname); +example_str = 'egig_2011_2012_2014_2018_allwf'; + +% param.multipass.layer: Choose which layers to include in the coregistered output. +% Default is surface and bottom from CSARP_layer. +%param_override.multipass.layer = struct('name',{'surface' 'bottom_qc'},'source',{'ops'}); +%param_override.multipass.layer = struct('name',{'surface' 'bottom_qc' 'surface'},'source',{'ops','ops','lidar'},'lidar_source','atm'); + +if strcmpi(example_str,'Thwaites_201902_201912_202001') + %% Thwaites Line 1 20190201_01, 20191225_01, 20200127_01 + param.multipass.fn = fullfile(gRadar.out_path,'accum','2018_Antarctica_TObas','CSARP_multipass','Thwaites_201902_201912_202001.mat'); + + param.multipass.rbins = []; + + param.multipass.baseline_master_idx = 1; + param.multipass.master_idx = 1; + + param.multipass.pass_en_mask = []; + param.multipass.output_fn_midfix = []; + param.multipass.coregistration_time_shift = [0 0 0]; + param.multipass.comp_mode = 2; +end + +if strcmpi(example_str,'Petermann_line1_2011_2014_2015_2017_2018_2019') + %% Petermann Line 1 2011, 2014, 2015, 2017, 2018, 2019 + param.multipass.fn = fullfile(gRadar.out_path,'rds','2011_Greenland_P3','CSARP_multipass','Petermann_line1_2011_2014_2015_2017_2018_2019'); + + param.multipass.rbins = []; + + param.multipass.baseline_master_idx = 1; + param.multipass.master_idx = 1; + + param.multipass.pass_en_mask = []; + param.multipass.output_fn_midfix = []; + param.multipass.coregistration_time_shift = [0 0 0 0 0 0]; + param.multipass.comp_mode = 2; +end + +if strcmpi(example_str,'Petermann_line2_2013_2014_2017') + %% Petermann Line 2 2013, 2014, 2017 + param.multipass.fn = fullfile(gRadar.out_path,'rds','2013_Greenland_P3','CSARP_multipass','Petermann_line2_2013_2014_2017'); + + param.multipass.rbins = []; + + param.multipass.baseline_master_idx = 1; + param.multipass.master_idx = 1; + + param.multipass.pass_en_mask = []; + param.multipass.output_fn_midfix = []; + param.multipass.coregistration_time_shift = [0 0 0]; + param.multipass.comp_mode = 2; +end + +if strcmpi(example_str,'Petermann_line4_2010_2011_2013_2014_2017') + %% Petermann Line 4 2010, 2011, 2013, 2014, 2017 + param.multipass.fn = fullfile(gRadar.out_path,'rds','2010_Greenland_DC8','CSARP_multipass','Petermann_line4_2010_2011_2013_2014_2017'); + + param.multipass.rbins = []; + + param.multipass.baseline_master_idx = 1; + param.multipass.master_idx = 1; + + param.multipass.pass_en_mask = []; + param.multipass.output_fn_midfix = []; + param.multipass.coregistration_time_shift = [0 0 0 0 0]; + param.multipass.comp_mode = 2; + +end + +if strcmpi(example_str,'Petermann_oblique_2014_2015_2017_2018') + %% Petermann Oblique 2014, 2017, 2018 + param.multipass.fn = fullfile(gRadar.out_path,'rds','2014_Greenland_P3','CSARP_multipass','Petermann_oblique_2014_2015_2017_2018'); + + param.multipass.rbins = []; + + param.multipass.baseline_master_idx = 1; + param.multipass.master_idx = 1; + + param.multipass.pass_en_mask = []; + param.multipass.output_fn_midfix = []; + param.multipass.coregistration_time_shift = [0 0 0 0]; + param.multipass.comp_mode = 2; +end + +if strcmpi(example_str,'79N_line1_2010_2014_2016_2018') + %% 79N Line 1 2010, 2014, 2016, 2018, 2019 + param.multipass.fn = fullfile(gRadar.out_path,'rds','2014_Greenland_P3','CSARP_multipass','79N_line1_2010_2014_2016_2018'); + + param.multipass.rbins = []; + + param.multipass.baseline_master_idx = 2; + param.multipass.master_idx = 2; + + param.multipass.pass_en_mask = []; + param.multipass.output_fn_midfix = []; + param.multipass.coregistration_time_shift = [0 0 0 0 0]; + param.multipass.comp_mode = 2; + param.multipass.time_gate = [2e-6 13e-6]; +end + +if strcmpi(example_str,'Humboldt_line1_2012_2013_2014_2017') + %% Humboldt Line 1 2012, 2013, 2014, 2017 + param.multipass.fn = fullfile(gRadar.out_path,'rds','2013_Greenland_P3','CSARP_multipass','Humboldt_line1_2012_2013_2014_2017'); + + param.multipass.rbins = []; + + param.multipass.baseline_master_idx = 1; + param.multipass.master_idx = 1; + + param.multipass.pass_en_mask = []; + param.multipass.output_fn_midfix = []; + param.multipass.coregistration_time_shift = [0 0 0 0]; + param.multipass.comp_mode = 2; + param.multipass.time_gate = [2e-6 13e-6]; +end + +if strcmpi(example_str,'Ryder_line1_2011_2013_2015_2019') + %% Ryder Line 1 2011, 2013, 2015, 2019 + param.multipass.fn = fullfile(gRadar.out_path,'rds','2015_Greenland_C130','CSARP_multipass','Ryder_line1_2011_2013_2015_2019'); + + param.multipass.rbins = []; + + % Use 2015 for master_idx since it runs parallel to glacier + param.multipass.baseline_master_idx = 3; + param.multipass.master_idx = 3; + + param.multipass.pass_en_mask = []; + param.multipass.output_fn_midfix = []; + param.multipass.coregistration_time_shift = [0 0 0 0]; + param.multipass.comp_mode = 2; + param.multipass.time_gate = [2e-6 15e-6]; +end + +if strcmpi(example_str,'Steensby_line1_2011_2013_2015_2019') + %% Steensby Line 1 2011, 2013, 2015, 2019 + param.multipass.fn = fullfile(gRadar.out_path,'rds','2013_Greenland_P3','CSARP_multipass','Steensby_line1_2011_2013_2015_2019'); + + param.multipass.rbins = []; + + param.multipass.baseline_master_idx = 2; + param.multipass.master_idx = 2; + + param.multipass.pass_en_mask = []; + param.multipass.output_fn_midfix = []; + param.multipass.coregistration_time_shift = [0 0 0 0]; + param.multipass.comp_mode = 2; + param.multipass.time_gate = [2e-6 10e-6]; +end + +if strcmpi(example_str,'ZI_line1_2010_2014_2016_2017_2018_2019') + %% Zachariae Isstrom Line 1 2010, 2014, 2016, 2017, 2018, 2019 + param.multipass.fn = fullfile(gRadar.out_path,'rds','2014_Greenland_P3','CSARP_multipass','ZI_line1_2010_2014_2016_2017_2018_2019'); + + param.multipass.rbins = []; + + param.multipass.baseline_master_idx = 2; + param.multipass.master_idx = 2; + + param.multipass.pass_en_mask = []; + param.multipass.output_fn_midfix = []; + param.multipass.coregistration_time_shift = [0 0 0 0 0 0]; + param.multipass.comp_mode = 2; + param.multipass.time_gate = [2e-6 13e-6]; +end + +if strcmpi(example_str,'camp_century_2014_same_day') + %% Camp Century: 2014 Greenland P3 same day + param.multipass.fn = fullfile(gRadar.out_path,'rds','2014_Greenland_P3','CSARP_multipass','camp_century_2014_same_day'); param.multipass.rbins = []; @@ -109,6 +190,7 @@ param.multipass.pass_en_mask = []; param.multipass.output_fn_midfix = []; param.multipass.coregistration_time_shift = []; + param.multipass.comp_mode = 1:4; param.multipass.time_gate = []; %Load equalization vector (sar specific) @@ -116,66 +198,162 @@ eqvec2 = eqvec1; neweq = [42.8 45.5 48.2 50.9 53.6 56.3 58.9 0.0 2.7 5.4 8.1 93.6 96.3 99.0 101.7]/2; neweq = [neweq neweq]; - + if 1 && exist('neweq','var') && ~isempty(neweq) eqvec1 = eqvec1-neweq(1:length(eqvec1)); eqvec2 = eqvec2-neweq(length(eqvec1)+1:end); end - + equalization1 = 10.^(zeros(1,15)/20) .* exp(1i*(eqvec1)/180*pi); equalization2 = 10.^(zeros(1,15)/20) .* exp(1i*(eqvec2)/180*pi); param.multipass.equalization = [equalization1 equalization2]; param.multipass.debug_plots = {'NA'}; +end + +if strcmpi(example_str,'camp_century_2011_2012_2013_2014') + %% Camp Century: 2011, 2012, 2013, 2014 Greenland P3 + param.multipass.fn = fullfile(gRadar.out_path,'rds','2014_Greenland_P3','CSARP_multipass','camp_century_2011_2012_2013_2014'); + + param.multipass.rbins = 220:420; - comp_mode = 1:4; + param.multipass.baseline_master_idx = 8; + param.multipass.master_idx = 8; + + param.multipass.pass_en_mask = []; + param.multipass.output_fn_midfix = []; + param.multipass.coregistration_time_shift = []; + param.multipass.comp_mode = 1; + param.multipass.time_gate = []; end -%% Summit Camp: 2012-2014 -if 1 - radartype = 'rds'; - passname = 'summit_2012_2014_allwf'; - param.multipass.fn = fullfile(gRadar.out_path,radartype,'2014_Greenland_P3','CSARP_multipass',passname); +if strcmpi(example_str,'summit_2012_2014_allwf') + %% Summit Camp: 2012-2014 + param.multipass.fn = fullfile(gRadar.out_path,'rds','2014_Greenland_P3','CSARP_multipass','summit_2012_2014_allwf'); param.multipass.rbins = []; if 0 - comp_mode = 2; + % All passes: estimate coregistration + param.multipass.comp_mode = 2; + param.multipass.slope_correction_en = true; + param.multipass.baseline_master_idx = 8; + param.multipass.master_idx = 8; + param.multipass.output_fn_midfix = ''; + param.multipass.pass_en_mask = false(1,30); + param.multipass.pass_en_mask(1:30) = true; + elseif 0 + % All passes: estimate equalization all passes + param.multipass.comp_mode = 1; + param.multipass.slope_correction_en = true; + param.multipass.baseline_master_idx = 8; + param.multipass.master_idx = 8; + param.multipass.output_fn_midfix = ''; + param.multipass.pass_en_mask = false(1,30); + param.multipass.pass_en_mask(1:30) = true; + elseif 0 + % Single season only: estimate cross-track slope + % Run tomo.cross_track_slope_est after this + param.multipass.comp_mode = 2; param.multipass.baseline_master_idx = 8; param.multipass.master_idx = 8; param.multipass.output_fn_midfix = '_2014'; param.multipass.pass_en_mask = false(1,30); param.multipass.pass_en_mask(1:15) = true; elseif 0 - comp_mode = 3; + % All passes: prepare SLC images to form interferograms all passes + % Run tomo.tomo_insar_image after this + param.multipass.comp_mode = 3; param.multipass.slope_correction_en = true; param.multipass.baseline_master_idx = 8; param.multipass.master_idx = 8; param.multipass.output_fn_midfix = ''; param.multipass.pass_en_mask = false(1,30); param.multipass.pass_en_mask(1:30) = true; - elseif 1 - comp_mode = 2; + elseif 0 + % TEST: 2012 only using 2014 as master; useful for 2012 equalization + param.multipass.comp_mode = 2; param.multipass.baseline_master_idx = 8; param.multipass.master_idx = 15+8; param.multipass.output_fn_midfix = '_2012'; param.multipass.pass_en_mask = false(1,30); param.multipass.pass_en_mask(16:30) = true; else 0 - comp_mode = 2; + % TEST: 2012 only using 2012 as master; useful for 2012 equalization + param.multipass.comp_mode = 2; param.multipass.baseline_master_idx = 15+8; param.multipass.master_idx = 15+8; param.multipass.output_fn_midfix = '_2012master'; param.multipass.pass_en_mask = false(1,30); param.multipass.pass_en_mask(16:30) = true; end - + param.multipass.coregistration_time_shift = []; param.multipass.time_gate = []; - param.multipass.equalization = 10.^(zeros(1,30)) ... - .* exp(1i*([7.9 22.5 19.7 22.7 29.9 14.9 22.3 0.0 1.5 5.4 13.4 19.2 17.1 17.8 22.7 167.4 166.3 177.1 164.3 -177.6 165.9 171.6 155.3 154.6 157.5 164.6 179.0 176.7 174.8 -129.8]/180*pi)); +% param.multipass.equalization = 10.^(zeros(1,30)) ... +% .* exp(1i*([7.9 22.5 19.7 22.7 29.9 14.9 22.3 0.0 1.5 5.4 13.4 19.2 17.1 17.8 22.7 167.4 166.3 177.1 164.3 -177.6 165.9 171.6 155.3 154.6 157.5 164.6 179.0 176.7 174.8 -129.8]/180*pi)); + + param.multipass.debug_plots = {'debug','coherent'}; +end + + +if strcmpi(example_str,'egig_2011_2012_2014_2018_allwf') + %% EGIG line: 2011-2018 + param.multipass.fn = fullfile(gRadar.out_path,'rds','2014_Greenland_P3','CSARP_multipass','egig_2011_2012_2014_2018_allwf'); + + param.multipass.rbins = []; + + if 0 + % All passes: estimate coregistration + param.multipass.comp_mode = 2; + param.multipass.slope_correction_en = false; + param.multipass.baseline_master_idx = 8; + param.multipass.master_idx = 8; + param.multipass.output_fn_midfix = ''; + param.multipass.pass_en_mask = false(1,75); + param.multipass.pass_en_mask(1:75) = true; + param.multipass.rbins = 300:750; + elseif 0 + % All passes: estimate equalization all passes + param.multipass.comp_mode = 1; + param.multipass.slope_correction_en = false; + param.multipass.baseline_master_idx = 8; + param.multipass.master_idx = 8; + param.multipass.output_fn_midfix = ''; + param.multipass.pass_en_mask = false(1,75); + param.multipass.pass_en_mask(1:75) = true; + elseif 0 + % Single season only: estimate cross-track slope + % Run tomo.cross_track_slope_est after this + param.multipass.comp_mode = 2; + param.multipass.slope_correction_en = false; + param.multipass.baseline_master_idx = 8; + param.multipass.master_idx = 8; + param.multipass.output_fn_midfix = '_2014'; + param.multipass.pass_en_mask = false(1,75); + param.multipass.pass_en_mask(1:15) = true; + elseif 1 + % All passes: prepare SLC images to form interferograms all passes + % Run tomo.tomo_insar_image after this + param.multipass.comp_mode = 3; + param.multipass.slope_correction_en = true; + param.multipass.baseline_master_idx = 8; + param.multipass.master_idx = 8; + param.multipass.output_fn_midfix = ''; + param.multipass.pass_en_mask = false(1,75); + param.multipass.pass_en_mask(1:75) = true; + end + +% param.multipass.coregistration_time_shift = []; + param.multipass.coregistration_time_shift = [0.05 0 0 -0.05 0 0 0 0 0.05 -0.05 0 0 0 0.05 0 0.35 0.3 0.35 0.3 0.35 0.35 0.3 0.25 0.35 0.3 0.3 0.3 0.25 0.3 0.25 0.25 0.3 0.3 0.25 0.3 0.25 0.25 0.3 0.25 0.3 0.25 0.3 0.35 0.3 0.35 0.15 0.1 0.15 0.15 0.1 0.1 0.15 0.1 0.1 0.1 0.1 0.1 0.15 0.05 0.1 0.1 0.05 0.1 0.1 0.05 0.05 0.1 0.05 0.1 0.1 0.1 0.05 0.1 0 0.1]; + param.multipass.time_gate = []; + param.multipass.equalization = 10.^([13.7 14.1 13.7 12.9 13.2 14.3 14.0 15.0 12.5 16.0 15.1 12.6 11.9 11.2 12.6 -13.1 -10.8 -14.5 -13.2 -14.4 -15.1 -15.6 1.8 3.8 3.0 2.6 1.2 3.2 5.9 2.2 -17.7 -13.7 -18.0 -14.9 -1.9 -17.6 -19.0 -3.5 2.4 -1.2 -1.7 -1.1 2.2 10.5 -0.5 -0.0 -0.2 0.0 -0.2 -0.1 1.6 -0.8 -0.2 -0.0 -0.3 -0.3 -1.9 -0.8 -0.2 -0.7 -2.1 -2.3 -2.1 -2.2 -1.9 -0.3 -2.5 -1.9 -1.6 -1.7 -1.9 -3.3 -2.1 -1.8 -2.0]/20) ... + .* exp(1i*([4.9 18.4 15.2 19.2 23.2 15.2 22.6 0.0 0.5 6.8 13.3 9.2 5.8 6.0 10.1 44.2 40.7 47.0 49.0 41.8 40.8 47.0 29.7 32.8 30.9 38.5 37.5 36.0 37.2 30.7 -74.7 -110.5 -58.3 -104.2 -128.8 -69.7 -56.9 -128.8 -127.6 -126.6 -120.4 -124.6 -125.4 -130.3 -130.2 140.1 142.9 143.9 148.2 157.8 154.1 162.0 145.6 159.4 150.4 155.2 160.2 146.3 146.9 141.9 56.5 61.9 62.2 64.3 74.6 70.8 80.2 62.3 77.0 69.0 71.5 77.4 63.3 63.2 60.0]/180*pi)); + % param.multipass.equalization = 10.^(zeros(1,75)) ... + % .* exp(1i*(zeros(1,75)/180*pi)); + param.multipass.debug_plots = {'debug','coherent'}; end @@ -191,8 +369,5 @@ end % Run multipass -for mode_id = 1:length(comp_mode) - param.multipass.comp_mode = comp_mode(mode_id); - %multipass.multipass(param, param_override); - multipass.multipass -end \ No newline at end of file +%multipass.multipass(param, param_override); +multipass.multipass diff --git a/cresis-toolbox/+multipass/velocity_coregister.m b/cresis-toolbox/+multipass/velocity_coregister.m index 0b83198d..9d27fdc9 100644 --- a/cresis-toolbox/+multipass/velocity_coregister.m +++ b/cresis-toolbox/+multipass/velocity_coregister.m @@ -1,20 +1,29 @@ +%% User Settings +% ========================================================================= + param = []; vel_fn = {}; vel_mult = []; +if ispc + vel_fn_dir = 'Y:\cbarnett\year_greenland_vv\'; +else + vel_fn_dir = '/cresis/snfs1/scratch/cbarnett/year_greenland_vv/'; +end + %% Petermann Line 1 2011, 2014, 2018 -% if ispc -% fn = fullfile('X:\ct_data\rds\2014_Greenland_P3\CSARP_multipass\',sprintf('Petermann_line1_2011_2014_2018_multipass.mat')); -% else -% fn = fullfile('/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_multipass/',sprintf('Petermann_line1_2011_2014_2018_multipass')); -% end -% vel_fn{1}{1} = '/cresis/snfs1/scratch/cbarnett/year_greenland_vv/greenland_vel_mosaic500_2012_2013_vv_v02.1.tif'; -% vel_mult{1} = [3]; -% vel_fn{2}{1} = '/cresis/snfs1/scratch/cbarnett/year_greenland_vv/greenland_vel_mosaic200_2014_2015_vv_v02.1.tif'; -% vel_fn{2}{2} = '/cresis/snfs1/scratch/cbarnett/year_greenland_vv/greenland_vel_mosaic200_2015_2016_vv_v02.1.tif'; -% vel_fn{2}{3} = '/cresis/snfs1/scratch/cbarnett/year_greenland_vv/greenland_vel_mosaic200_2016_2017_vv_v02.1.tif'; -% vel_fn{2}{4} = '/cresis/snfs1/scratch/cbarnett/year_greenland_vv/greenland_vel_mosaic200_2017_2018_vv_v02.1.tif'; -% vel_mult{2} = [1 1 1 1]; +if ispc + fn = fullfile('X:\ct_data\rds\2014_Greenland_P3\CSARP_multipass\',sprintf('Petermann_line1_2011_2014_2018_multipass.mat')); +else + fn = fullfile('/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_multipass/',sprintf('Petermann_line1_2011_2014_2018_multipass')); +end +vel_fn{1}{1} = fullfile(vel_fn_dir,'greenland_vel_mosaic500_2012_2013_vv_v02.1.tif'); +vel_mult{1} = [3]; +vel_fn{2}{1} = fullfile(vel_fn_dir,'greenland_vel_mosaic200_2014_2015_vv_v02.1.tif'); +vel_fn{2}{2} = fullfile(vel_fn_dir,'greenland_vel_mosaic200_2015_2016_vv_v02.1.tif'); +vel_fn{2}{3} = fullfile(vel_fn_dir,'greenland_vel_mosaic200_2016_2017_vv_v02.1.tif'); +vel_fn{2}{4} = fullfile(vel_fn_dir,'greenland_vel_mosaic200_2017_2018_vv_v02.1.tif'); +vel_mult{2} = [1 1 1 1]; %% Petermann Line 2 2013, 2014 % if ispc @@ -22,7 +31,7 @@ % else % fn = fullfile('/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_multipass/',sprintf('Petermann_line2_2013_2014_multipass')); % end -% vel_fn{1}{1} = '/cresis/snfs1/scratch/cbarnett/year_greenland_vv/greenland_vel_mosaic500_2012_2013_vv_v02.1.tif'; +% vel_fn{1}{1} = fullfile(vel_fn_dir,'greenland_vel_mosaic500_2012_2013_vv_v02.1.tif'); % vel_mult{1} = [1]; %% Petermann Line 4 2010, 2011, 2013, 2014 @@ -31,32 +40,54 @@ % else % fn = fullfile('/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_multipass/',sprintf('Petermann_line4_2010_2011_2013_2014_multipass')); % end -% vel_fn{1}{1} = '/cresis/snfs1/scratch/cbarnett/year_greenland_vv/greenland_vel_mosaic500_2012_2013_vv_v02.1.tif'; +% vel_fn{1}{1} = fullfile(vel_fn_dir,'petermann','greenland_vel_mosaic500_2012_2013_vv_v02.1.tif'); % vel_mult{1} = [1]; -% vel_fn{2}{1} = '/cresis/snfs1/scratch/cbarnett/year_greenland_vv/greenland_vel_mosaic500_2012_2013_vv_v02.1.tif'; +% vel_fn{2}{1} = fullfile(vel_fn_dir,'greenland_vel_mosaic500_2012_2013_vv_v02.1.tif'); % vel_mult{2} = [2]; -% vel_fn{3}{1} = '/cresis/snfs1/scratch/cbarnett/year_greenland_vv/greenland_vel_mosaic500_2012_2013_vv_v02.1.tif'; +% vel_fn{3}{1} = fullfile(vel_fn_dir,'greenland_vel_mosaic500_2012_2013_vv_v02.1.tif'); % vel_mult{3} = [1]; %% 79N Line 1 2010, 2014, 2016, 2018 -if ispc - fn = fullfile('X:\ct_data\rds\2014_Greenland_P3\CSARP_multipass\',sprintf('79N_line1_2010_2014_2016_2018_multipass.mat')); -else - fn = fullfile('/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_multipass/',sprintf('79N_line1_2010_2014_2016_2018_multipass')); -end +% if ispc +% fn = fullfile('X:\ct_data\rds\2014_Greenland_P3\CSARP_multipass\',sprintf('79N_line1_2010_2014_2016_2018_multipass.mat')); +% else +% fn = fullfile('/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_multipass/',sprintf('79N_line1_2010_2014_2016_2018_multipass')); +% end +% +% vel_fn{1}{1} = fullfile(vel_fn_dir,'greenland_vel_mosaic500_2012_2013_vv_v02.1.tif'); +% vel_mult{1} = [4]; +% % DO NOT USE; MISSING DATA: vel_fn{2}{1} = fullfile(vel_fn_dir,'greenland_vel_mosaic200_2014_2015_vv_v02.1.tif'; +% vel_fn{2}{1} = fullfile(vel_fn_dir,'greenland_vel_mosaic200_2015_2016_vv_v02.1.tif'); +% vel_mult{2} = [2]; +% vel_fn{3}{1} = fullfile(vel_fn_dir,'greenland_vel_mosaic200_2016_2017_vv_v02.1.tif'); +% vel_fn{3}{2} = fullfile(vel_fn_dir,'greenland_vel_mosaic200_2017_2018_vv_v02.1.tif'); +% vel_mult{3} = [1 1]; + +%% Automated Section +% ========================================================================= + +% Load multipass.multipass output +load(fn,'pass','param_multipass'); -vel_fn{1}{1} = '/cresis/snfs1/scratch/cbarnett/year_greenland_vv/greenland_vel_mosaic500_2012_2013_vv_v02.1.tif'; -vel_mult{1} = [4]; -% DO NOT USE; MISSING DATA: vel_fn{2}{1} = '/cresis/snfs1/scratch/cbarnett/year_greenland_vv/greenland_vel_mosaic200_2014_2015_vv_v02.1.tif'; -vel_fn{2}{1} = '/cresis/snfs1/scratch/cbarnett/year_greenland_vv/greenland_vel_mosaic200_2015_2016_vv_v02.1.tif'; -vel_mult{2} = [2]; -vel_fn{3}{1} = '/cresis/snfs1/scratch/cbarnett/year_greenland_vv/greenland_vel_mosaic200_2016_2017_vv_v02.1.tif'; -vel_fn{3}{2} = '/cresis/snfs1/scratch/cbarnett/year_greenland_vv/greenland_vel_mosaic200_2017_2018_vv_v02.1.tif'; -vel_mult{3} = [1 1]; +vel_param = []; vel_param.velocity_coregister = []; +if ~isfield(vel_param.velocity_coregister,'debug_plots') || isempty(vel_param.velocity_coregister.debug_plots) + vel_param.velocity_coregister.debug_plots = {'visible'}; +end +enable_visible_plot = any(strcmp('visible',vel_param.velocity_coregister.debug_plots)); +if ~isempty(vel_param.velocity_coregister.debug_plots) + h_fig = get_figures(4,enable_visible_plot); +end -load(fn); +%% Interpolate velocity +% ========================================================================= +% Interpolate the surface velocity maps for each pass. We do not +% interpolate the last pass in the list because this function assumes it +% to be the last pass in time and therefore does not need any velocity +% correction. All other passes should have a list of velocity maps that +% allow the velocity to be integrated over time to correct for velocity +% from when the particular pass was taken to the last pass. -figure(1); clf; +figure(h_fig(1)); clf; baseline_master_idx = param_multipass.multipass.baseline_master_idx; for pass_idx = 1:length(pass) @@ -86,7 +117,7 @@ % Debug: plot velocity geotiff if 0 - figure(1); + figure(h_fig(1)); imagesc(xaxis/1e3, yaxis/1e3, vel); set(gca,'YDir','normal'); colorbar; @@ -101,24 +132,27 @@ end end -%% -figure(2); clf; -figure(3); clf; -figure(4); clf; +%% Plot velocity corrected layers +% ========================================================================= +% Plot the integrated velocity and the surface velocity corrected ice surface +% and bottom layers for each pass. +figure(h_fig(2)); clf; +figure(h_fig(3)); clf; +figure(h_fig(4)); clf; leg_str = {}; h_vel = []; h_plot = []; for pass_idx = 1:length(pass) [year month day] = datevec(epoch_to_datenum(pass(pass_idx).gps_time(1))); - figure(2); + figure(h_fig(2)); h_vel(pass_idx) = plot(pass(baseline_master_idx).along_track/1e3, pass(pass_idx).vel); grid on; hold on; xlabel('Along-track (km)'); ylabel('Horizontal movement (m)'); - figure(3); + figure(h_fig(3)); lay_idx = 1; alongtrack = pass(baseline_master_idx).along_track + pass(pass_idx).vel; h_plot(pass_idx) = plot(alongtrack/1e3, pass(pass_idx).layers(lay_idx).layer_elev); @@ -132,7 +166,7 @@ xlabel('Along-track (km)'); ylabel('Elevation (m,WGS-84)'); - figure(4); + figure(h_fig(4)); lay_idx = 1; alongtrack = pass(baseline_master_idx).along_track; h_plot(pass_idx) = plot(alongtrack/1e3, pass(pass_idx).layers(lay_idx).layer_elev); diff --git a/cresis-toolbox/+sim/doa.m b/cresis-toolbox/+sim/doa.m index d8e74767..3117831e 100644 --- a/cresis-toolbox/+sim/doa.m +++ b/cresis-toolbox/+sim/doa.m @@ -20,6 +20,11 @@ param.src.ft_wind = @boxcar; end +if ~isfield(param.src,'sv_dielectric') || isempty(param.src.sv_dielectric) + param.src.sv_dielectric = 1; +end + + %% Setup % ========================================================================= diff --git a/cresis-toolbox/+sim/doa_example_wax.m b/cresis-toolbox/+sim/doa_example_wax.m index 53e89fb2..af3c4ee2 100644 --- a/cresis-toolbox/+sim/doa_example_wax.m +++ b/cresis-toolbox/+sim/doa_example_wax.m @@ -15,7 +15,7 @@ param_override.cluster.type = 'torque'; % param_override.cluster.type = 'debug'; -param_override.cpu_time = 100; +param_override.cpu_time = 50; param_override.mem = 1e9; param_override.cluster.ppn_fixed = 4; @@ -38,6 +38,8 @@ % Source parameters fc = 195e6; BW = 30e6; +% fc = 312.5e6; +% BW = 1e6; param.src.f0 = fc-BW/2; param.src.f1 = fc+BW/2; param.src.ft_wind = @hanning; @@ -60,12 +62,18 @@ % One source fixed at 0 deg, another fixed at 20 deg % Snapshots fixed at 10 param.monte.SNR = repmat(linspace(10,25,16).', [1 2]); + if 0 + param.monte.SNR = repmat(linspace(10,40,20).', [1 2]); + end num_tests = size(param.monte.SNR,1); param.monte.DOA = repmat([0 20],[num_tests 1]); - param.monte.Nsnap = repmat(11,[num_tests 1]); + param.monte.Nsnap = repmat(10,[num_tests 1]); + param.monte.Nsnap = repmat(10,[num_tests 1]); param.monte.runs = 1000; param.monte.random_seed_offset = 0; + + %% Fig 2: Run the simulation results = sim.doa(param,param_override); @@ -101,7 +109,7 @@ end %% Fig 3 Wax and Ziskind 1988 -% ======================================================================= +% ======================================;================================= if fig_to_plot == 3 %% Fig 3: Setup simulation parameters physical_constants; @@ -135,8 +143,9 @@ param.monte.Nsnap = round(logspace(log10(10),log10(1000),21).'); num_tests = size(param.monte.Nsnap,1); param.monte.SNR = repmat([20 20] - 10*log10(3),[num_tests 1]); +% param.monte.SNR = repmat([20 20],[num_tests 1]); param.monte.DOA = repmat([0 20],[num_tests 1]); - param.monte.runs = 100; + param.monte.runs = 1000; param.monte.random_seed_offset = 0; %% Fig 3: Run the simulation diff --git a/cresis-toolbox/+tomo/+c3d_rnn_models/c3d.pth b/cresis-toolbox/+tomo/+c3d_rnn_models/c3d.pth deleted file mode 100644 index 0010892d..00000000 Binary files a/cresis-toolbox/+tomo/+c3d_rnn_models/c3d.pth and /dev/null differ diff --git a/cresis-toolbox/+tomo/+c3d_rnn_models/rnn.pth b/cresis-toolbox/+tomo/+c3d_rnn_models/rnn.pth deleted file mode 100644 index fb99c9b9..00000000 Binary files a/cresis-toolbox/+tomo/+c3d_rnn_models/rnn.pth and /dev/null differ diff --git a/cresis-toolbox/+tomo/+parameter_tuning/+helper_functions/cluster_kernel_viterbi_2D.m b/cresis-toolbox/+tomo/+parameter_tuning/+helper_functions/cluster_kernel_viterbi_2D.m index 2927bd1a..36893c84 100644 --- a/cresis-toolbox/+tomo/+parameter_tuning/+helper_functions/cluster_kernel_viterbi_2D.m +++ b/cresis-toolbox/+tomo/+parameter_tuning/+helper_functions/cluster_kernel_viterbi_2D.m @@ -76,7 +76,7 @@ end % Load frames file - load(ct_filename_support(param,'','frames')); + frames = frames_load(param); if isempty(param.cmd.frms) param.cmd.frms = 1:length(frames.frame_idxs); diff --git a/cresis-toolbox/+tomo/LSMObject_tuning.m b/cresis-toolbox/+tomo/LSMObject_tuning.m index cce05ab4..cb397b62 100644 --- a/cresis-toolbox/+tomo/LSMObject_tuning.m +++ b/cresis-toolbox/+tomo/LSMObject_tuning.m @@ -58,6 +58,7 @@ function setLSMOptions(this, varargin) out.alpha = this.lsmArgs.alpha; out.mu = this.lsmArgs.mu; out.innerIter = this.lsmArgs.innerIter; + out.storeIter = this.lsmArgs.storeIter; out.outerIter = this.lsmArgs.outerIter; out.timestep = this.lsmArgs.timestep; out.narrowBand = this.lsmArgs.narrowBand; @@ -77,7 +78,8 @@ function setLSMOptions(this, varargin) end function showContours(this, k,s) - [img, ~, ftype]=readImage(this.imds{k}); + %[img, ~, ftype]=readImage(this.imds{k}); + img = double(this.imds{k}); ce=this.contours{k,s}; %returns the contours for image k showImage(img, ftype) %show the image hold on @@ -88,39 +90,57 @@ function showContours(this, k,s) end function [fname,ftype]=disp(this,k) - [fname,ftype]=getFileName(this.imds{k}); - this.showContours(k,this.lastContour); + if isfield(this,'imds') + [fname,ftype]=getFileName(this.imds{k}); + end + if isfield(this,'lastContour') + this.showContours(k,this.lastContour); + end end function [flag, top, bot, matrix_x, matrix_y] = runLSM(this) - - h = figure('Visible', 'off'); - flag = ones(1,16); + + flag = ones(1,length(this.lsmArgs.storeIter)); % Read the image - [img_orig,~,~] = readImage(this.imds{1}); + %[img_orig,~,~] = readImage(this.imds{1}); + img_orig = double(this.imds{1}); img = imresize(img_orig, this.resizeRate); - matrix_x = ones(2, size(img_orig, 2), 16); - matrix_y = ones(2, size(img_orig, 2), 16); + matrix_x = ones(2, size(img_orig, 2), length(this.lsmArgs.storeIter)); + matrix_y = ones(2, size(img_orig, 2), length(this.lsmArgs.storeIter)); g=indicateEdge(img); this.phi=initializeLSM(img, this.initiArgs); this.contours{1,1}= getLSF(this.phi, 1/this.resizeRate); % LSM - ctr = 1; + last_fprintf_time = -inf; for n=1:this.lsmArgs.outerIter + if now > last_fprintf_time+30/86400 + fprintf(' Iteration %.0f of %.0f (%s)\n', n, this.lsmArgs.outerIter, datestr(now)); + last_fprintf_time = now; + end + this.phi=lsmReg(this.phi, g, this.lsmArgs); m=this.lsmArgs.innerIter*n; if rem(m,10)==0 this.contours{1,m/10+1}= getLSF(this.phi, 1/this.resizeRate); end - - if rem(n, 25) == 0 - c = contour(this.phi, [0,0], 'r'); + match_idx = find(n==this.lsmArgs.storeIter); + if any(match_idx) + c = contourc(this.phi, [0,0]); s = tomo.contourdata(c); + % Ensure proper order of layers + if 0 + % Debug plot to show contours + figure; + c = contour(this.phi, [0,0], 'r'); + end try + if sum(s(1).ydata) > sum(s(2).ydata) + s = s([2 1]); + end top.x = (1 / this.resizeRate) * imresize(s(1).xdata, [size(img_orig, 2) 1]); top.y = (1 / this.resizeRate) * imresize(s(1).ydata, [size(img_orig, 2) 1]); bot.x = (1 / this.resizeRate) * imresize(s(2).xdata, [size(img_orig, 2) 1]); @@ -135,24 +155,23 @@ function showContours(this, k,s) keyboard end end - matrix_x(1, :, ctr) = top.x'; - matrix_x(2, :, ctr) = bot.x'; - matrix_y(1, :, ctr) = top.y'; - matrix_y(2, :, ctr) = bot.y'; + matrix_x(1, :, match_idx) = top.x'; + matrix_x(2, :, match_idx) = bot.x'; + matrix_y(1, :, match_idx) = top.y'; + matrix_y(2, :, match_idx) = bot.y'; if any(any(isnan(this.phi))) flag(ctr) = 0; - matrix_x(1, :, ctr) = ''; - matrix_x(2, :, ctr) = ''; - matrix_y(1, :, ctr) = ''; - matrix_y(2, :, ctr) = ''; + matrix_x(1, :, match_idx) = ''; + matrix_x(2, :, match_idx) = ''; + matrix_y(1, :, match_idx) = ''; + matrix_y(2, :, match_idx) = ''; end - ctr = ctr + 1; -% Use below code to get echogram images with different number of iterations (n) -% if(n==25||n==100||n==200||n==300||n==350) -% figure;imagesc(img_orig);colormap(1-gray(256));hold on; plot(top.x,top.y);plot(bot.x,bot.y); -% end - end + % Use below code to get echogram images with different number of iterations (n) + % if(n==25||n==100||n==200||n==300||n==350) + % figure;imagesc(img_orig);colormap(1-gray(256));hold on; plot(top.x,top.y);plot(bot.x,bot.y); + % end + end end end end @@ -222,6 +241,7 @@ function showContours(this, k,s) defaultMu = 0.1; defaultInnerIter = 2; defaultOuterIter = 400; +defaultStoreIter = defaultOuterIter; defaultTimestep = 2; defaultViewSnaps = 100; defaultNarrowBand = false; @@ -231,6 +251,7 @@ function showContours(this, k,s) p.addParameter('mu', defaultMu, @numeric); p.addParameter('innerIter', defaultInnerIter, @isPositiveInteger); p.addParameter('outerIter', defaultOuterIter, @isPositiveInteger); +p.addParameter('storeIter', defaultStoreIter, @isPositiveInteger); p.addParameter('timestep', defaultTimestep, @isPositiveInteger); p.addParameter('snapshots', defaultViewSnaps, @isPositiveInteger); p.addParameter('narrowBand', defaultNarrowBand, @islogical); @@ -257,6 +278,7 @@ function showContours(this, k,s) lsmArgs.alpha = results.alpha; lsmArgs.mu = results.mu; lsmArgs.innerIter = results.innerIter; +lsmArgs.storeIter = results.storeIter; lsmArgs.outerIter = results.outerIter; lsmArgs.timestep = results.timestep; lsmArgs.snapshots = results.snapshots; @@ -272,8 +294,8 @@ function showContours(this, k,s) function tf = isPositiveInteger(x) -isPositive = x>0; -isInteger = isreal(x) && isnumeric(x) && all(mod(x,1)==0); +isPositive = all(x>0); +isInteger = all(isreal(x)) && all(isnumeric(x)) && all(mod(x,1)==0); tf = isPositive && isInteger; end @@ -357,7 +379,7 @@ function showContours(this, k,s) [fname,ftype]=getFileName(filepath); if ftype=='.mat' in = load(filepath, 'Data'); -% img=lp(in.Data); + % img=lp(in.Data); img = 20*log10(in.Data); elseif ftype=='.png' img=imread(filepath); diff --git a/cresis-toolbox/+tomo/Layer_tracking_2D_parameters_Matrix.mat b/cresis-toolbox/+tomo/Layer_tracking_2D_parameters_Matrix.mat deleted file mode 100644 index 0499bed3..00000000 Binary files a/cresis-toolbox/+tomo/Layer_tracking_2D_parameters_Matrix.mat and /dev/null differ diff --git a/cresis-toolbox/+tomo/Layer_tracking_3D_parameters_Matrix.mat b/cresis-toolbox/+tomo/Layer_tracking_3D_parameters_Matrix.mat deleted file mode 100644 index 0deac28c..00000000 Binary files a/cresis-toolbox/+tomo/Layer_tracking_3D_parameters_Matrix.mat and /dev/null differ diff --git a/cresis-toolbox/+tomo/add_icemask_surfacedem.m b/cresis-toolbox/+tomo/add_dem_icemask.m similarity index 93% rename from cresis-toolbox/+tomo/add_icemask_surfacedem.m rename to cresis-toolbox/+tomo/add_dem_icemask.m index afefecfe..b3be9a01 100644 --- a/cresis-toolbox/+tomo/add_icemask_surfacedem.m +++ b/cresis-toolbox/+tomo/add_dem_icemask.m @@ -1,5 +1,5 @@ -function mdata = add_icemask_surfacedem(param, mdata) -% mdata = tomo.add_icemask_surfacedem(param, mdata) +function mdata = add_dem_icemask(param, mdata) +% mdata = tomo.add_dem_icemask(param, mdata) % % Description: Usually this function is called from tomo.collate_task. % Using a surface DEM and an ice mask, this function adds an aligned @@ -19,7 +19,7 @@ % NONE % % See also: tomo.run_collate, tomo.collate, tomo_collate_task, -% tomo.fuse_images, tomo.add_icemask_surfacedem, tomo.create_surfData, +% tomo.fuse_images, tomo.add_dem_icemask, tomo.create_surfData, % % Author: John Paden, Jordan Sprick, and Mingze Xu @@ -85,12 +85,10 @@ physical_constants; [latb,lonb] = bufferm(mdata.Latitude(dec_idxs),mdata.Longitude(dec_idxs),param.tomo_collate.dem_guard/WGS84.semimajor*180/pi); gdem_str = sprintf('%s:%s:%s_%03d',param.radar_name,param.season_name,param.day_seg,param.load.frm); -if ~strcmpi(gdem_str,gdem.name) - gdem.set_vector(latb,lonb,gdem_str); -end - gdem.set_vector(latb,lonb,gdem_str); + [DEM,msl,ocean_mask,proj,DEM_x,DEM_y] = gdem.get_vector_mosaic(100); +% fprintf('The dem_class found that %.f%% of the area is ocean. If this is incorrect set param.tomo_collate.ocean_mask to 0. Default is NaN which automates the mask') DEM(ocean_mask) = msl(ocean_mask); [mdata.x,mdata.y] = projfwd(proj,mdata.Latitude,mdata.Longitude); @@ -147,11 +145,16 @@ end if all(all(isnan(DEM))) - warning('Input DEM contains all NaN data for Frame %d.',param.proc.frm); + warning('Input DEM contains all NaN data for Frame %d.',param.load.frm); twtt(:,:) = NaN; Nx = 0; end - + +if param.tomo_collate.ground_based_flag == true + ice_mask = ones(Nsv,Nx); + twtt = zeros(Nsv,Nx); + +else DEM_coverage_warning = false; for rline = 1:Nx if ~mod(rline-1,10^floor(log10(Nx)-1)) @@ -165,7 +168,7 @@ DEM_idxs = find(DEM_mask); if numel(DEM_idxs)==0 - warning('Range Line %d of Frame %d is not spanned by DEM.',rline,param.proc.frm); + warning('Range Line %d of Frame %d is not spanned by DEM.',rline,param.load.frm); end if 0 @@ -180,12 +183,12 @@ physical_constants; [DEM_ecef_x,DEM_ecef_y,DEM_ecef_z] = geodetic2ecef(single(DEM_lat)/180*pi,single(DEM_lon)/180*pi,single(DEM_elev),WGS84.ellipsoid); - origin = mdata.param_array.array_proc.fcs{1}{1}.origin(:,rline); + origin = mdata.param_array.array_proc.fcs.origin(:,rline); % Convert from ECEF to FCS/SAR - Tfcs_ecef = [mdata.param_array.array_proc.fcs{1}{1}.x(:,rline), ... - mdata.param_array.array_proc.fcs{1}{1}.y(:,rline), ... - mdata.param_array.array_proc.fcs{1}{1}.z(:,rline)]; + Tfcs_ecef = [mdata.param_array.array_proc.fcs.x(:,rline), ... + mdata.param_array.array_proc.fcs.y(:,rline), ... + mdata.param_array.array_proc.fcs.z(:,rline)]; Tecef_fcs = inv(Tfcs_ecef); tmp = Tecef_fcs * [DEM_ecef_x.'-origin(1); DEM_ecef_y.'-origin(2); DEM_ecef_z.'-origin(3)]; @@ -250,7 +253,7 @@ twtt(:,rline) = NaN; end - if exist('ice_mask_all','var') + if exist('ice_mask_all','var') && ~isempty(ice_mask_all) if exist('intersection','var') % Convert from FCS/SAR to ECEF intersection_ecef = Tfcs_ecef * intersection; @@ -297,6 +300,7 @@ end end +end if 0 %% DEBUG diff --git a/cresis-toolbox/+tomo/collate.m b/cresis-toolbox/+tomo/collate.m index 1d13d59e..7f15f21f 100644 --- a/cresis-toolbox/+tomo/collate.m +++ b/cresis-toolbox/+tomo/collate.m @@ -25,64 +25,88 @@ %% Input Checks % ===================================================================== -% Load frames file -load(ct_filename_support(param,'','frames')); - % Remove frames that do not exist from param.cmd.frms list -load(ct_filename_support(param,'','frames')); % Load "frames" variable -if ~isfield(param.cmd,'frms') || isempty(param.cmd.frms) - param.cmd.frms = 1:length(frames.frame_idxs); -end -[valid_frms,keep_idxs] = intersect(param.cmd.frms, 1:length(frames.frame_idxs)); -if length(valid_frms) ~= length(param.cmd.frms) - bad_mask = ones(size(param.cmd.frms)); - bad_mask(keep_idxs) = 0; - warning('Nonexistent frames specified in param.cmd.frms (e.g. frame "%g" is invalid), removing these', ... - param.cmd.frms(find(bad_mask,1))); - param.cmd.frms = valid_frms; +frames = frames_load(param); +param.cmd.frms = frames_param_cmd_frms(param,frames); +if isempty(param.cmd.frms) + % No valid frames to process + warning('No valid frames were listed in param.cmd.frms. Skipping this segment.'); + return; end % sar.* fields % ------------------------------------------------------------------------- -% param.sar gets used to create the defaults of several fields so we make -% sure the field is available -if ~isfield(param,'sar') || isempty(param.sar) - param.sar = []; -end -if ~isfield(param.sar,'sigma_x') || isempty(param.sar.sigma_x) - error('The param.sar.sigma_x field must be set to calculate cpu time and memory requirements.'); -end - -% array.* fields +% tomo_collate.* fields % ------------------------------------------------------------------------- - -if ~isfield(param.array,'dline') || isempty(param.array.dline) - error('param.array.dline must be specified.'); +if ~isfield(param.tomo_collate,'frm_types') || isempty(param.tomo_collate.frm_types) + param.tomo_collate.frm_types = {-1,-1,-1,-1,-1}; end -if ~isfield(param.array,'method') || isempty(param.array.method) - param.array.method = 'music'; +if ~isfield(param.tomo_collate,'gt') || isempty(param.tomo_collate.gt) + param.tomo_collate.gt = []; +end +if ~isfield(param.tomo_collate.gt,'en') || isempty(param.tomo_collate.gt.en) + param.tomo_collate.gt.en = false; +end +if ~isfield(param.tomo_collate.gt,'path') || isempty(param.tomo_collate.gt.path) + param.tomo_collate.gt.path = 'surf'; +end +if ~isfield(param.tomo_collate.gt,'range') || isempty(param.tomo_collate.gt.range) + param.tomo_collate.gt.range = 5; +end +if ~isfield(param.tomo_collate.gt,'surf_name') || isempty(param.tomo_collate.gt.surf_name) + param.tomo_collate.gt.surf_name = 'bottom gt'; end -% tomo_collate.* fields -% ------------------------------------------------------------------------- -if ~isfield(param.tomo_collate,'frm_types') || isempty(param.tomo_collate.frm_types) - param.tomo_collate.frm_types = {-1,-1,-1,-1,-1}; +if ~isfield(param.tomo_collate,'ground_based_flag') || isempty(param.tomo_collate.ground_based_flag) + param.tomo_collate.ground_based_flag = false; end if ~isfield(param.tomo_collate,'in_path') || isempty(param.tomo_collate.in_path) param.tomo_collate.in_path = 'music3D'; end -if ~isfield(param.tomo_collate,'surf_out_path') || isempty(param.tomo_collate.surf_out_path) - param.tomo_collate.surf_out_path = 'surfData'; -end - if ~isfield(param.tomo_collate,'out_path') || isempty(param.tomo_collate.out_path) param.tomo_collate.out_path = param.tomo_collate.in_path; end +if ~isfield(param.tomo_collate,'surfData_mode') || isempty(param.tomo_collate.surfData_mode) + param.tomo_collate.surfData_mode = 'append'; +end + +if ~isfield(param.tomo_collate,'surf_out_path') || isempty(param.tomo_collate.surf_out_path) + param.tomo_collate.surf_out_path = 'surfData'; +end + +% Name of the top surface (usually set to 'top' when tracking surfaces below +% the ice-surface and set to an empty string, '', when tracking the +% ice-surface. +if ~isfield(param.tomo_collate,'top_name') || isempty(param.tomo_collate.top_name) + param.tomo_collate.top_name = 'top'; +end + +if ~isfield(param.tomo_collate,'array_manifold_cal_flag') || isempty(param.tomo_collate.array_manifold_cal_flag) + param.tomo_collate.array_manifold_cal_flag = false; +end + +if ~isfield(param.tomo_collate,'suppress_surf_flag') || isempty(param.tomo_collate.suppress_surf_flag) + if param.tomo_collate.array_manifold_cal_flag + param.tomo_collate.suppress_surf_flag = false; + else + param.tomo_collate.suppress_surf_flag = false; + end +end + +if ~isfield(param.tomo_collate,'suppress_surf_peak_val') || isempty(param.tomo_collate.suppress_surf_peak_val) + param.tomo_collate.suppress_surf_peak_val = 30; +end + +if ~isfield(param.tomo_collate,'suppress_surf_window') || isempty(param.tomo_collate.suppress_surf_window) + param.tomo_collate.suppress_surf_window = 100; +end + + %% Setup Processing % ===================================================================== @@ -90,11 +114,7 @@ [~,~,radar_name] = ct_output_dir(param.radar_name); % Load records file -records_fn = ct_filename_support(param,'','records'); -if ~exist(records_fn) - error('You must run create the records file before running anything else:\n %s', records_fn); -end -records = load(records_fn); +records = records_load(param); % Along-track along_track_approx = geodetic_to_along_track(records.lat,records.lon,records.elev); @@ -123,27 +143,37 @@ ctrl = cluster_new_batch(param); cluster_compile({'tomo_collate_task.m'},ctrl.cluster.hidden_depend_funs,ctrl.cluster.force_compile,ctrl); +% Load input array.m processed data product parameters +frm = param.cmd.frms(1); +in_fn = fullfile(out_path_dir, sprintf('Data_img_%02d_%s_%03d.mat', ... + param.tomo_collate.imgs{1}(1), param.day_seg, frm)); +load(in_fn,'param_array'); +array_proc_methods; % This script assigns the integer values for each method + +% Calculate the size of the images to determine cpu time and memory +% requirements total_num_sam = 0; total_img = 0; -if param.array.tomo_en - if any(strcmpi(param.array.method,{'music'})) - Nsv = param.array.Nsv; - elseif any(strcmpi(param.array.method,{'mle'})) - Nsv = 2*param.array.Nsrc; +if param_array.array.tomo_en + if any(param_array.array.method == MUSIC_METHOD) + Nsv = param_array.array.Nsv; + elseif any(param_array.array.method == MLE_METHOD) + Nsv = 2*param_array.array.Nsrc; else - error('Unsupported param.array.method %s\n', param.array.method); + error('Unsupported param_array.array.method %s\n', param_array.array.method); end else - error('param.array.tomo_en is false, tomography should be enabled when running tomo.collate.'); + error('param_array.array.tomo_en is false for the input file, tomography should be enabled during array process in order to run tomo.collate.'); end -[wfs,~] = data_load_wfs(setfield(param,'load',struct('imgs',{param.array.imgs})),records); +param.load.imgs = param_array.array.imgs; +[wfs,~] = data_load_wfs(param,records); if any(strcmpi(radar_name,{'acords','hfrds','hfrds2','mcords','mcords2','mcords3','mcords4','mcords5','mcords6','mcrds','seaice','accum2','accum3'})) for v_img = 1:length(param.tomo_collate.imgs) for h_img = 1:length(param.tomo_collate.imgs{v_img}) img = param.tomo_collate.imgs{v_img}(h_img); wf = param.tomo_collate.imgs{v_img}(1,1); if h_img == 1 - total_num_sam = total_num_sam + wfs(wf).Nt_pc; + total_num_sam = total_num_sam + wfs(wf).Nt; end total_img = total_img + 1; end @@ -235,7 +265,7 @@ % CPU Time and Memory estimates: % Nx*total_num_sam*K where K is some manually determined multiplier. - Nx = round(frm_dist / param.sar.sigma_x / param.array.dline); + Nx = round(frm_dist / param.sar.sigma_x / param_array.array.dline); dparam.cpu_time = 10 + Nx*Nsv*total_num_sam*cpu_time_mult; dparam.mem = 250e6 + Nx*Nsv*total_num_sam*mem_mult; ctrl = cluster_new_task(ctrl,sparam,dparam,'dparam_save',0); diff --git a/cresis-toolbox/+tomo/compare_layers_2D.m b/cresis-toolbox/+tomo/compare_layers_2D.m index b34b78d8..0c745a3c 100644 --- a/cresis-toolbox/+tomo/compare_layers_2D.m +++ b/cresis-toolbox/+tomo/compare_layers_2D.m @@ -81,7 +81,7 @@ param = merge_structs(param,gRadar); % Load frames file - load(ct_filename_support(param,'','frames')); + frames = frames_load(param); if isempty(param.cmd.frms) param.cmd.frms = 1:length(frames.frame_idxs); diff --git a/cresis-toolbox/+tomo/compare_surfdata.m b/cresis-toolbox/+tomo/compare_surfdata.m index 98dbc555..31227dd2 100644 --- a/cresis-toolbox/+tomo/compare_surfdata.m +++ b/cresis-toolbox/+tomo/compare_surfdata.m @@ -63,8 +63,12 @@ fprintf('%s: %s (%s)\n', dbstack_info(1).name, param.day_seg, datestr(now,'HH:MM:SS')); fprintf('=====================================================================\n'); +if ~isfield(param.compare,'mode') | isempty(param.compare.mode) + param.compare.method = 'twtt'; +end + % Load frames file -load(ct_filename_support(param,'','frames')); +frames = frames_load(param); if isempty(param.cmd.frms) param.cmd.frms = 1:length(frames.frame_idxs); @@ -110,13 +114,19 @@ fn_A = fullfile(ct_filename_out(param,surfdata_ref,''),sprintf('Data_%s_%03d.mat',param.day_seg,frm)); A = tomo.surfdata(fn_A); - + for comp_idx = 1:length(surfdata_other) fn_B = fullfile(ct_filename_out(param,surfdata_other{comp_idx},''),sprintf('Data_%s_%03d.mat',param.day_seg,frm)); B = tomo.surfdata(fn_B); - [rmse,mean_diff,median_diff,min_diff,max_diff,surf_diff] ... - = A.compare(surf_name_ref, B,surf_name_other{comp_idx},DOA_trim); + switch param.compare.mode + case 'twtt' + [rmse,mean_diff,median_diff,min_diff,max_diff,surf_diff] ... + = A.compare(surf_name_ref, B,surf_name_other{comp_idx},DOA_trim); + case 'doa' + [rmse,mean_diff,median_diff,min_diff,max_diff,surf_diff] ... + = A.compare_doa(surf_name_ref, B,surf_name_other{comp_idx},DOA_trim,param.radar.fs); + end rmse_f{comp_idx}(frame_idx) = rmse; mean_f{comp_idx}(frame_idx) = mean_diff; diff --git a/cresis-toolbox/+tomo/create_movie.m b/cresis-toolbox/+tomo/create_movie.m index 88687e38..132ae957 100644 --- a/cresis-toolbox/+tomo/create_movie.m +++ b/cresis-toolbox/+tomo/create_movie.m @@ -39,7 +39,7 @@ function create_movie (params, options) end % Load frames file - load(ct_filename_support(param,'','frames')); + frames = frames_load(param); if isempty(param.cmd.frms) param.cmd.frms = 1:length(frames.frame_idxs); @@ -59,8 +59,7 @@ function create_movie (params, options) try param.topDEM = load(fullfile(ct_filename_out(param, options.DEM_source, ''), sprintf('%s_%03d_top',param.day_seg, frm_idx))); param.bottomDEM = load(fullfile(ct_filename_out(param, options.DEM_source, ''), sprintf('%s_%03d_bottom',param.day_seg, frm_idx))); - records_fn = ct_filename_support(param,'','records'); - records = load(records_fn,'gps_time'); + records = records_load(param,[],'gps_time'); catch ME warning('Attention: DEM for reference frame %s not found. Skipping this frame.', sprintf('%s_%03d',param.day_seg, frm_idx)); continue; diff --git a/cresis-toolbox/+tomo/cross_track_slope_est.m b/cresis-toolbox/+tomo/cross_track_slope_est.m index c7284aeb..80bc0b9d 100644 --- a/cresis-toolbox/+tomo/cross_track_slope_est.m +++ b/cresis-toolbox/+tomo/cross_track_slope_est.m @@ -36,7 +36,7 @@ last_good_bin = 450; min_valid_value = 100; -elseif 1 +elseif 0 if ispc else % Summit @@ -49,6 +49,17 @@ last_good_bin = 1036; min_valid_value = 70; end + +elseif 1 + if ispc + else + % EGIG + fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_multipass/egig_2011_2012_2014_2018_allwf_2014.mat'; + + out_fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_multipass/egig_2011_2012_2014_2018_allwf.mat'; + last_good_bin = 970; + min_valid_value = 70; + end end %% Automated Section @@ -97,7 +108,8 @@ slope = nan_fir_dec(slope,ones(1,21)/21,1); slope = nan_fir_dec(slope.',ones(1,11)/11,1).'; slope = interp_finite(slope); -slope = medfilt2(slope,[31 31]); +slope = medfilt2(slope,[21 21]); +slope = echo_filt(slope,[31 31]); % Convert the results to direction of arrival slope = interp1(1:length(Tomo.theta),Tomo.theta,slope); diff --git a/cresis-toolbox/+tomo/fuse_images.m b/cresis-toolbox/+tomo/fuse_images.m index 5b225913..f38afdb7 100644 --- a/cresis-toolbox/+tomo/fuse_images.m +++ b/cresis-toolbox/+tomo/fuse_images.m @@ -108,7 +108,7 @@ % to copy all the parameters and support variables over). We will update % the 3D image and time fields in the file at the end of fusing. combined_fn = fullfile(in_dir,sprintf('Data_%s_%03.0f.mat',param.day_seg,param.load.frm)); - fprintf('Creating %s (%s)\n', combined_fn, datestr(now)); + fprintf('Copying\n%s\n to\n%s (%s)\n', fns{1}, combined_fn, datestr(now)); copyfile(fns{1}, combined_fn); end @@ -213,8 +213,8 @@ Time = Time(1:last_idx); Data = Data(1:last_idx,:); Tomo.img = Tomo.img(1:last_idx,:,:); - Tomo.theta = Tomo.theta(1:last_idx,:,:); if doa_method_flag + Tomo.theta = Tomo.theta(1:last_idx,:,:); Tomo.cost = Tomo.cost(1:last_idx,:,:); Tomo.hessian = Tomo.hessian(1:last_idx,:,:); end @@ -289,6 +289,7 @@ % Second row of img_bins indicates the end of the blend-region img_bins(2,:) = img_bins(1,:) + 10; + img_bins(2,img_bins(2,:)>length(Time)) = length(Time); difference = 10^(-0/10); @@ -348,6 +349,7 @@ end % Append new fused image to the combined file +fprintf('Saving %s (%s)\n', combined_fn, datestr(now)); if param.ct_file_lock file_version = '1L'; else @@ -355,6 +357,7 @@ end save(combined_fn,'-append','Time','Data','Tomo','file_version'); +% Create output argument which contains combined image mdata = mdata{1}; mdata.Time = Time; mdata.Data = Data; diff --git a/cresis-toolbox/+tomo/geostat_2D_DIM_and_AlongTrack.m b/cresis-toolbox/+tomo/geostat_2D_DIM_and_AlongTrack.m index f9e4ddd4..a8f493c5 100644 --- a/cresis-toolbox/+tomo/geostat_2D_DIM_and_AlongTrack.m +++ b/cresis-toolbox/+tomo/geostat_2D_DIM_and_AlongTrack.m @@ -49,7 +49,7 @@ continue; end % Load frames file - load(ct_filename_support(param,'','frames')); + frames = frames_load(param); if isempty(param.cmd.frms) param.cmd.frms = 1:length(frames.frame_idxs); end diff --git a/cresis-toolbox/+tomo/geostat_3D_AlongTrack.m b/cresis-toolbox/+tomo/geostat_3D_AlongTrack.m index d3c0ee42..44d5b2ff 100644 --- a/cresis-toolbox/+tomo/geostat_3D_AlongTrack.m +++ b/cresis-toolbox/+tomo/geostat_3D_AlongTrack.m @@ -33,7 +33,7 @@ end % Load frames file - load(ct_filename_support(param,'','frames')); + frames = frames_load(param); if isempty(param.cmd.frms) param.cmd.frms = 1:length(frames.frame_idxs); diff --git a/cresis-toolbox/+tomo/geostat_3D_CrossTrack.m b/cresis-toolbox/+tomo/geostat_3D_CrossTrack.m index 5d49fb57..5530891f 100644 --- a/cresis-toolbox/+tomo/geostat_3D_CrossTrack.m +++ b/cresis-toolbox/+tomo/geostat_3D_CrossTrack.m @@ -38,7 +38,7 @@ end % Load frames file - load(ct_filename_support(param,'','frames')); + frames = frames_load(param); if isempty(param.cmd.frms) param.cmd.frms = 1:length(frames.frame_idxs); diff --git a/cresis-toolbox/+tomo/geostat_3D_DIM.m b/cresis-toolbox/+tomo/geostat_3D_DIM.m index f91c031e..971be392 100644 --- a/cresis-toolbox/+tomo/geostat_3D_DIM.m +++ b/cresis-toolbox/+tomo/geostat_3D_DIM.m @@ -53,7 +53,7 @@ end % Load frames file - load(ct_filename_support(param,'','frames')); + frames = frames_load(param); if isempty(param.cmd.frms) param.cmd.frms = 1:length(frames.frame_idxs); diff --git a/cresis-toolbox/+tomo/lsm_tracker_2D.m b/cresis-toolbox/+tomo/lsm_tracker_2D.m deleted file mode 100644 index 608ec967..00000000 --- a/cresis-toolbox/+tomo/lsm_tracker_2D.m +++ /dev/null @@ -1,140 +0,0 @@ -function labels = lsm_tracker_2D (params, options, data_struct) -% function lsm_tracker_2D (params, options) -% -% See also: run_tracker_2D.m -% -% Authors: Victor Berger - -%% Automated Section -% ===================================================================== -% Create param structure array -% ===================================================================== - -global gRadar; -clear('param_override'); - -for param_idx = 1:length(params) - param = params(param_idx); - if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic - continue; - end - - % Load frames file - load(ct_filename_support(param,'','frames')); - - if isempty(param.cmd.frms) - param.cmd.frms = 1:length(frames.frame_idxs); - end - - % Remove frames that do not exist from param.cmd.frms list - [valid_frms,keep_idxs] = intersect(param.cmd.frms, 1:length(frames.frame_idxs)); - if length(valid_frms) ~= length(param.cmd.frms) - bad_mask = ones(size(param.cmd.frms)); - bad_mask(keep_idxs) = 0; - warning('Nonexistent frames specified in param.cmd.frms (e.g. frame "%g" is invalid), removing these', ... - param.cmd.frms(find(bad_mask,1))); - param.cmd.frms = valid_frms; - end - - data_fn_dir = ct_filename_out(param, options.name, ''); - for frm = param.cmd.frms - - fprintf('\nLSM: Running frame %s_%03d (%s)\n',param.day_seg, frm, datestr(now,'HH:MM:SS')); - - lsm_tic = tic; - - try - data = data_struct.(sprintf('data_%s_%03d',param.day_seg,frm)); - catch ME - fprintf('\nProblem with frame %s_%03d, verify.\n' ,param.day_seg,frm); - continue; - end - - data_fn_name = sprintf('Data_%s_%03d.mat',param.day_seg,frm); - data_fn = fullfile(data_fn_dir, data_fn_name); - imds = imageDatastore(data_fn, 'FileExtensions', '.mat'); - obj = tomo.LSMObject(imds.Files); - obj.setLSMOptions('y', 240, 'dy', 10, 'outerIter', options.lsm.numOuterIter); - - [flag, Labels.top, Labels.bot] = obj.runLSM(); - - if flag == 0 - warning('LSM failed to converge to two layers after %d inner iterations. Trying again with %d inner iterations...', 2*options.lsm.numOuterIter, 2*options.lsm.maxOuterIter); - obj = tomo.LSMObject(imds.Files); - obj.setLSMOptions('y', 240, 'dy', 10, 'outerIter', options.lsm.maxOuterIter); - [flag, Labels.top, Labels.bot] = obj.runLSM(); - fprintf('\nDone: second runLSM() call. '); - end - - Labels.top.y = interp1(Labels.top.x, Labels.top.y, 1:length(data.Bottom), 'linear', 'extrap'); - Labels.bot.y = interp1(Labels.bot.x, Labels.bot.y, 1:length(data.Bottom), 'linear', 'extrap')'; - - lsm_toc = toc(lsm_tic); - - if options.debug - figure; imagesc(lp(data.Data)); colormap(1 - gray(256)); hold on; - plot(Labels.top.y, 'g'); plot(Labels.bot.y, 'r'); - legend('Ice-surface', 'Ice-bottom'); - keyboard - end - - if options.ops_write - warning('off') - %% Set write options - % Load labels into OPS using opsCopyLayers - copy_param = []; - copy_param.layer_source.existence_check = false; - copy_param.layer_dest.existence_check = false; - - % Set the source - copy_param.layer_source.source = 'custom'; - copy_param.layer_dest.source = options.layer_dest_source; - - if strcmp(copy_param.layer_dest.source, 'layerdata') - copy_param.layer_dest.layerdata_source = options.layer_dest_layerdata_source; - copy_param.layer_dest.echogram_source = options.layer_dest_echogram_source; - end - - copy_param.copy_method = 'overwrite'; - - copy_param.gaps_fill.method = 'preserve_gaps'; - copy_param.gaps_fill.method_args = [40 20]; - - param = merge_structs(param,gRadar); - - %% Write surface layer - % Interpolate from row number to TWTT - data.TWTT = interp1(1:length(data.Time), data.Time, Labels.top.y); - copy_param.layer_source.gps_time = data.GPS_time; - copy_param.layer_source.twtt = data.TWTT; - - % Load labels into OPS using opsCopyLayers - copy_param.layer_dest.name = options.lsm.lyrtop; - fprintf('\nopsCopyLayers %s (%s) [TOP]', param.day_seg, datestr(now)); - opsCopyLayers(param,copy_param); - - %% Write bottom layer - % Interpolate from row number to TWTT - data.TWTT = interp1(1:length(data.Time), data.Time, Labels.bot.y); - copy_param.layer_source.gps_time = data.GPS_time; - copy_param.layer_source.twtt = data.TWTT; - - % Load labels into OPS using opsCopyLayers - copy_param.layer_dest.name = options.lsm.lyrbot; - fprintf('\nopsCopyLayers %s (%s) [BOTTOM]', param.day_seg, datestr(now)); - opsCopyLayers(param,copy_param); - - fprintf('\n Complete (%s)\n', datestr(now)); - warning('on'); - end - - labels.(sprintf('layer_%s_%03d',param.day_seg,frm)).top = ... - Labels.top.y; - labels.(sprintf('layer_%s_%03d',param.day_seg,frm)).bot = ... - Labels.bot.y; - labels.(sprintf('layer_%s_%03d',param.day_seg,frm)).toc = ... - lsm_toc; - - end -end -end \ No newline at end of file diff --git a/cresis-toolbox/+tomo/mcmc_tracker_2D.m b/cresis-toolbox/+tomo/mcmc_tracker_2D.m deleted file mode 100644 index 406faa02..00000000 --- a/cresis-toolbox/+tomo/mcmc_tracker_2D.m +++ /dev/null @@ -1,208 +0,0 @@ -function labels = mcmc_tracker_2D (params, options, data_struct) -% function mcmc_tracker_2D (params, options, data_struct) -% -% See also: run_tracker_2D.m -% -% Authors: Victor Berger - -%% Automated Section -% ===================================================================== -% Create param structure array -% ===================================================================== - -global gRadar; -clear('param_override'); - -% Input checking -if ~exist('params','var') - error('Use run_tracker_2D: A struct array of parameters must be passed in\n'); -end -if exist('param_override','var') - param_override = merge_structs(gRadar, param_override); -else - param_override = gRadar; -end - -if isfield(options.mcmc, 'top_smooth') - top_smooth = options.mcmc.top_smooth; -else - top_smooth = 1000; -end - -if isfield(options.mcmc, 'bottom_smooth') - bottom_smooth = options.mcmc.bottom_smooth; -else - bottom_smooth = 1000; -end - -if isfield(options.mcmc, 'top_peak') - top_peak = options.mcmc.top_peak; -else - top_peak = 0.5; -end - -if isfield(options.mcmc, 'bottom_peak') - bottom_peak = options.mcmc.bottom_peak; -else - bottom_peak = 0.5; -end - -if isfield(options.mcmc, 'repulsion') - repulsion = options.mcmc.repulsion; -else - repulsion = 10; -end - -labels = {}; - -for param_idx = 1:length(params) - param = params(param_idx); - if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic - continue; - end - - % Load frames file - load(ct_filename_support(param,'','frames')); - - if isempty(param.cmd.frms) - param.cmd.frms = 1:length(frames.frame_idxs); - end - - % Remove frames that do not exist from param.cmd.frms list - [valid_frms,keep_idxs] = intersect(param.cmd.frms, 1:length(frames.frame_idxs)); - if length(valid_frms) ~= length(param.cmd.frms) - bad_mask = ones(size(param.cmd.frms)); - bad_mask(keep_idxs) = 0; - warning('Nonexistent frames specified in param.cmd.frms (e.g. frame "%g" is invalid), removing these', ... - param.cmd.frms(find(bad_mask,1))); - param.cmd.frms = valid_frms; - end - - for frm = param.cmd.frms - - mcmc_tic = tic; - - fprintf('\nMCMC: Running frame %s_%03d (%s)\n',param.day_seg, frm, datestr(now,'HH:MM:SS')); - - try - big_matrix = data_struct.(sprintf('data_%s_%03d',param.day_seg,frm)); - catch ME - fprintf('\nProblem with frame %s_%03d, verify.\n' ,param.day_seg,frm); - continue; - end - - big_matrix.Data = lp(big_matrix.Data); - big_matrix.Data = 255 * ((big_matrix.Data - min(big_matrix.Data(:))) ... - ./ (max(big_matrix.Data(:) - min(big_matrix.Data(:))))); - big_matrix.Data = uint8(repmat(big_matrix.Data, [1 1 3])); - - if strcmp(options.mcmc.alg, 'MCMC') - try - tic - [big_matrix.Labels, big_matrix.lower, big_matrix.upper] = ... - tomo.RJ_MCMC(double(big_matrix.Data(:,:,1))); - toc - catch ME - try - mex -largeArrayDims RJ_MCMC.cpp - tic - [big_matrix.Labels, big_matrix.lower, big_matrix.upper] = ... - tomo.RJ_MCMC(double(big_matrix.Data(:,:,1))); - toc - catch ME - fprintf('\nProblem executing RJ_MCMC.cpp file. Verify.\n'); - keyboard - end - end - elseif strcmp(options.mcmc.alg, 'HMM') - opts = [top_smooth bottom_smooth top_peak bottom_peak repulsion]; - try - tic - [~, big_matrix.Labels] = tomo.stereo(1, double(big_matrix.Data(:,:,1)), opts); - toc - catch ME - try - mex -largeArrayDims stereo.cpp - tic - [~, big_matrix.Labels] = tomo.stereo(1, double(big_matrix.Data(:,:,1)), opts); - toc - catch ME - fprintf('\nProblem executing stereo.cpp file. Verify.\n'); - keyboard - end - end - else - fprintf('\nUnrecognized algorithm (must be ''MCMC'' or ''HMM'')'); - keyboard - end - - mcmc_toc = toc(mcmc_tic); - - if options.debug - figure; imshow(big_matrix.Data); hold on; - plot(big_matrix.Labels(1, :), 'g'); plot(big_matrix.Labels(2, :), 'r'); - legend('Ice-surface', 'Ice-bottom'); - keyboard - end - - if options.ops_write - warning('off'); - %% Set write options - % Load labels into OPS using opsCopyLayers - copy_param = []; - copy_param.layer_source.existence_check = false; - copy_param.layer_dest.existence_check = false; - - % Set the source - copy_param.layer_source.source = 'custom'; - copy_param.layer_dest.source = options.layer_dest_source; - - if strcmp(copy_param.layer_dest.source, 'layerdata') - copy_param.layer_dest.layerdata_source = options.layer_dest_layerdata_source; - copy_param.layer_dest.echogram_source = options.layer_dest_echogram_source; - end - - copy_param.copy_method = 'fillgaps'; - - copy_param.gaps_fill.method = 'preserve_gaps'; - copy_param.gaps_fill.method_args = [40 20]; - - param = merge_structs(param,gRadar); - - %% Write surface layer - % Interpolate from row number to TWTT - big_matrix.TWTT = interp1(1:length(big_matrix.Time), big_matrix.Time, big_matrix.Labels(1,:)); - copy_param.layer_source.gps_time = big_matrix.GPS_time; - copy_param.layer_source.twtt = big_matrix.TWTT; - - % Load labels into OPS using opsCopyLayers - copy_param.layer_dest.name = options.mcmc.lyrtop; - fprintf('\nopsCopyLayers %s (%s) [TOP]', param.day_seg, datestr(now)); - opsCopyLayers(param,copy_param); - fprintf('\n Complete (%s)\n', datestr(now)); - - %% Write bottom layer - % Interpolate from row number to TWTT - big_matrix.TWTT = interp1(1:length(big_matrix.Time), big_matrix.Time, big_matrix.Labels(2,:)); - copy_param.layer_source.gps_time = big_matrix.GPS_time; - copy_param.layer_source.twtt = big_matrix.TWTT; - - % Load labels into OPS using opsCopyLayers - copy_param.layer_dest.name = options.mcmc.lyrbot; - fprintf('\nopsCopyLayers %s (%s) [BOTTOM]', param.day_seg, datestr(now)); - opsCopyLayers(param,copy_param); - - fprintf('\n Complete (%s)\n', datestr(now)); - warning('on'); - end - - labels.(sprintf('layer_%s_%03d',param.day_seg,frm)).top = ... - big_matrix.Labels(1,:); - labels.(sprintf('layer_%s_%03d',param.day_seg,frm)).bot = ... - big_matrix.Labels(2,:); - labels.(sprintf('layer_%s_%03d',param.day_seg,frm)).toc = ... - mcmc_toc; - - end -end -end \ No newline at end of file diff --git a/cresis-toolbox/+tomo/run_LSM_tuning.m b/cresis-toolbox/+tomo/run_LSM_tuning.m index 458d2ab9..204ec5af 100644 --- a/cresis-toolbox/+tomo/run_LSM_tuning.m +++ b/cresis-toolbox/+tomo/run_LSM_tuning.m @@ -47,7 +47,7 @@ end % Load frames file - load(ct_filename_support(param,'','frames')); + frames = frames_load(param); if isempty(param.cmd.frms) param.cmd.frms = 1:length(frames.frame_idxs); end diff --git a/cresis-toolbox/+tomo/run_collate.m b/cresis-toolbox/+tomo/run_collate.m index 1327b4d4..312f7676 100644 --- a/cresis-toolbox/+tomo/run_collate.m +++ b/cresis-toolbox/+tomo/run_collate.m @@ -27,8 +27,8 @@ % .in_path: ct_filename_out directory to use at input, fused image will be stored here. tomo_collate.in_path = 'test_music3D'; - % .surf_out_path: ct_filename_out directory to use at output for surfData - tomo_collate.surf_out_path = 'surfData'; + % .surf_out_path: ct_filename_out directory to use at output for surfdata + tomo_collate.surf_out_path = 'surf'; % .imgs: list of images II to use from .in_path (Data_img_II*.mat). These % should be listed from left most beam to right most beam for @@ -71,6 +71,11 @@ % DEM points needed to properly represent the surface. tomo_collate.dem_per_slice_guard = 240; + % .ground_based_flag: logical default is false, if true data are treated + % as ground based and surface twtt and ice_mask are set to zero and one + % for all pixels respectively. + tomo_collate.ground_based_flag = false; + % .bounds_relative: DOA bins and along-track slices to trim off from each edge [top bottom left right] tomo_collate.bounds_relative = [3 2 0 0]; @@ -113,8 +118,8 @@ % .in_path: ct_filename_out directory to use at input, fused image will be stored here. tomo_collate.in_path = 'test_music3D'; - % .surf_out_path: ct_filename_out directory to use at output for surfData - tomo_collate.surf_out_path = 'surfData'; + % .surf_out_path: ct_filename_out directory to use at output for surfdata + tomo_collate.surf_out_path = 'surf'; % .imgs: list of images II to use from .in_path (Data_img_II*.mat). These % should be listed from left most beam to right most beam for @@ -160,6 +165,11 @@ % DEM points needed to properly represent the surface. tomo_collate.dem_per_slice_guard = 240; + % .ground_based_flag: logical default is false, if true data are treated + % as ground based and surface twtt and ice_mask are set to zero and one + % for all pixels respectively. + tomo_collate.ground_based_flag = false; + % .bounds_relative: DOA bins and along-track slices to trim off from each edge [top bottom left right] tomo_collate.bounds_relative = [3 2 0 0]; @@ -235,11 +245,17 @@ elseif strcmpi(example_setup,'grid') %% Grid multiwaveform fuse example % params = read_param_xls(ct_filename_param('rds_param_2011_Greenland_P3.xls'),'','post'); - params = read_param_xls(ct_filename_param('rds_param_2018_Greenland_Polar6_paden.xls')); +% params = read_param_xls(ct_filename_param('rds_param_2018_Greenland_Polar6_paden.xls')); + params = read_param_xls(ct_filename_param('rds_param_2018_Greenland_P3.xls')); + + cmd_method = 'array'; + param_override.array.out_path = 'music3D'; + rds_settings; -% params = ct_set_params(params,'cmd.generic',0); -% params = ct_set_params(params,'cmd.generic',1,'day_seg','20110317_03'); -% params = ct_set_params(params,'cmd.frms',[1]); + params = ct_set_params(params,'cmd.generic',0); + params = ct_set_params(params,'cmd.generic',1,'day_seg','20180406_01'); + params = ct_set_params(params,'cmd.frms',[1 2]); + % % % params = ct_set_params(params,'array.imgs',{[ones(15,1) (2:16)'],[2*ones(15,1) (2:16)']}); % params = ct_set_params(params,'array.imgs',{[ones(7,1) (2:8)'],[2*ones(7,1) (2:8)']}); @@ -250,13 +266,18 @@ % params = ct_set_params(params,'array.tomo_en',1); % params = ct_set_params(params,'array.method','mle'); -% Each cell array represent one horizontal image, which may contain more than -% one vertical image. For example, {[1 2],[1]} means there are 2 horizontal -% images: the first one contains 2 vertical images and the second contains -% just 1 vertical image. - tomo_collate.imgs = {[1],[2]}; % Vertical fusing +% Each cell array represent one horizontal (potentially combined) image, +% which may contain more than one vertical image. For example, {[1 2],[1]} +% means there are 2 horizontal images: the first one contains 2 vertical +% images and the second contains just 1 vertical image. + tomo_collate.imgs = {[1]}; % No fusing (single image) +% tomo_collate.imgs = {[1],[2]}; % Vertical fusing % tomo_collate.imgs = {[1 2]}; % Horizontal fusing - tomo_collate.img_comb = [3e-06 -Inf 1e-06 1e-05 -Inf 3e-06]; + +% .img_comb: vertical (fast-time) combining times (see img_combine.m +% "img_comb" description). + tomo_collate.img_comb = []; +% tomo_collate.img_comb = [3e-06 -Inf 1e-06 1e-05 -Inf 3e-06]; % for param_idx = 1:length(params) % if params(param_idx).cmd.generic @@ -288,19 +309,18 @@ % tomo_collate.in_path = 'test_music3D'; % tomo_collate.in_path = 'test_music3D_Nsv128_Nc15'; % tomo_collate.in_path = 'test_music3D_Nsv64_Nc7'; - tomo_collate.in_path = 'music_lr'; + tomo_collate.in_path = 'music3D'; - % .surf_out_path: ct_filename_out directory to use at output for surfData -% tomo_collate.surf_out_path = 'test_surfData_Nsv128_Nc15'; - tomo_collate.surf_out_path = 'surfData'; + % .surf_out_path: ct_filename_out directory to use at output for surfdata + tomo_collate.surf_out_path = 'surf'; % .imgs: list of images II to use from .in_path (Data_img_II*.mat). These % should be listed from left most beam to right most beam for % horizontal fusing and top to bottom for vertical fusing. - % .img_comb: Same as get_heights and combine worksheets. This is - % used for vertical using only. For N images, + % .img_comb: Same as qlook and array worksheets. This is + % used for vertical/fast-time using only. For N images, % there will be (N-1) combines performed. Each combine is described by % three numbers so that there should be (N-1)*3 elements in this % vector. Each set of three numbers indicates the following: @@ -308,7 +328,7 @@ % 2nd element: Minimum time after ice surface to begin combine % 3rd element: Time at end of the preceeding waveform to not use - % .fuse_columns: aligns with .imgs, each entry should contain 2xN-1 + % .fuse_columns: aligns with .imgs, each entry should contain 2x(N-1) % entries where N is the length of the corresponding cell in .imgs. Each % column of 2 numbers represents the start/stop columns to blend with. % There should be one column per interface between two images that needs @@ -316,13 +336,16 @@ % the horizontal dimension, the entry should be 2x2. If blending 2 % images, the entry should be 2x1. If there is only one image for a % particular vertical index, then fuse_columns should be empty. - tomo_collate.fuse_columns = {[], [], [32, 33]}; + %tomo_collate.fuse_columns = {[], [], [32, 33]}; + tomo_collate.fuse_columns = {[]}; % .sv_cal_fn: filename containing steering vector calibration, leave empty to not use tomo_collate.sv_cal_fn = ''; % .ice_mask_fn: filename of ice mask, leave empty to not use tomo_collate.ice_mask_fn = 'greenland/IceMask/GimpIceMask_90m_v1.1.tif'; + %tomo_collate.ice_mask_fn = ct_filename_gis([],'canada/ice_mask/03_rgi50_ArcticCanadaNorth/03_rgi50_ArcticCanadaNorth.mat'); + % .dem_guard: additional region in meters around flight line to search for DEM points % Setting too high slows the process down, setting too low will miss @@ -334,15 +357,25 @@ % DEM points needed to properly represent the surface. tomo_collate.dem_per_slice_guard = 240; + % .ground_based_flag: logical default is false, if true data are treated + % as ground based and surface twtt and ice_mask are set to zero and one + % for all pixels respectively. + tomo_collate.ground_based_flag = false; + % .bounds_relative: DOA bins and along-track slices to trim off from each edge [top bottom left right] tomo_collate.bounds_relative = [3 2 0 0]; % .layer_params: parameter structure for opsLoadLayer (first layer should - % be ice top and second layer should be ice bottom) + % be ice top and second layer should be ice bottom), these are used for + % the initial ground truth for tracking. tomo_collate.layer_params = struct('name','surface','source','layerdata'); tomo_collate.layer_params(2).name = 'bottom'; tomo_collate.layer_params(2).source = 'layerdata'; + % .tomo_params: parameters to constrain the range-bins of the 3D tracker + %tomo_collate.tomo_params = struct('name','surface','source','layerdata','eval',struct('cmd','s=s-1e-6;')); + %tomo_collate.tomo_params(2) = struct('name','bottom','source','layerdata','eval',struct('cmd','s=s+5e-6;')); + % surfData_mode: surfData mode ('overwrite','fillgaps', or 'append', note that append with the % same surface name as an existing surface will overwrite that surface whereas fillgaps % will leave the surface untouched if it already exists) @@ -350,14 +383,17 @@ % surfdata_cmds: surfdata commands to run tomo_collate.surfdata_cmds = []; -% tomo_collate.surfdata_cmds(end+1).cmd = 'trws'; -% tomo_collate.surfdata_cmds(end).surf_names = {'bottom trws','bottom'}; - - tomo_collate.surfdata_cmds(end+1).surf_names = {'bottom'}; - tomo_collate.surfdata_cmds(end).cmd = 'doa'; % surf_names should be set to 'bottom' always + % TRW-S (Tree Reweighted Sequential algorithm) + tomo_collate.surfdata_cmds(end+1).cmd = 'trws'; + tomo_collate.surfdata_cmds(end).surf_names = {'bottom trws','bottom'}; tomo_collate.surfdata_cmds(end).visible = true; -% tomo_collate.surfdata_cmds(end).max_loops = 10; + tomo_collate.surfdata_cmds(end).smooth_weight = [22 22]; + tomo_collate.surfdata_cmds(end).smooth_var = 32; + tomo_collate.surfdata_cmds(end).max_loops = 50; + + %param.tomo_collate.top_name = 'top'; + param.tomo_collate.top_name = ''; % .fuse_images_flag: runs fuse_images.m when true tomo_collate.fuse_images_flag = true; diff --git a/cresis-toolbox/+tomo/run_surfdata.m b/cresis-toolbox/+tomo/run_surfdata.m index fd72e076..1138ac99 100644 --- a/cresis-toolbox/+tomo/run_surfdata.m +++ b/cresis-toolbox/+tomo/run_surfdata.m @@ -1,221 +1,216 @@ %% Setup -% Create filename which will be used to demonstrate surfdata class -param = struct('radar_name','mcords','season_name','2009_Antarctica_TO','day_seg','20091224_01'); -frm = 16; -fn_original = fullfile(ct_filename_out(param,'surfData',''),sprintf('Data_%s_%03d.mat',param.day_seg,frm)); - -fn = [tempname '.mat']; -copyfile(fn_original,fn); - - -%% Constructor - -% No arguments -empty_surfdata = tomo.surfdata() - -% Filename argument -my_surfdata = tomo.surfdata(fn); -my_surfdata - -% Invalid filename example -try - my_surfdata = tomo.surfdata(1); % error; 1 is a number not a string -catch ME - warning(ME.getReport); -end - -%% load_surfdata -my_surfdata.load_surfdata(fn); - -% Invalid filename example -try - my_surfdata.load_surf(1); -catch ME - warning(ME.getReport); -end - -%% get surf -my_surfdata.get_surf('bottom') % get the bottom surface -my_surfdata.get_surf({'top','bottom'}) % get the top and bottom surfaces -my_surfdata.get_surf([1 2 3]) -try - my_surfdata.get_surf('123') % surface does not exist; -catch ME - warning(ME.getReport); -end -try - my_surfdata.get_surf(20) % invalid index -catch ME - warning(ME.getReport); -end -try - my_surfdata.get_surf(0) % invalid index -catch ME - warning(ME.getReport); -end - -%% insert_surf - -% Create a test surf structure from an existing surface -test_surf = my_surfdata.get_surf('bottom'); -test_surf.name = 'my_test_surf'; - -% Insert test surface -my_surfdata.insert_surf(test_surf); -my_surfdata - -% Insert multiple layers -for surf_idx = 1:length(my_surfdata.surf) - test_surf = my_surfdata.surf(surf_idx); - empty_surfdata.insert_surf(test_surf); -end -empty_surfdata - -% create a test_surf structure where (size of x) != (size of y) -test_surf = my_surfdata.get_surf('bottom'); -test_surf.x = test_surf.x(:,1:300); -test_surf.name = 'test'; -try - my_surfdata.insert_surf(test_surf); -catch ME - warning(ME.getReport); -end - -% create a test_surf which does not match existing surf x and y sizes -test_surf = my_surfdata.get_surf('bottom'); -test_surf.x = test_surf.x(1:32,1:300); -test_surf.y = test_surf.y(1:32,1:300); -test_surf.name = 'test'; -try - my_surfdata.insert_surf(test_surf); -catch ME - warning(ME.getReport); -end - -% insert an invalid struct -test_surf = struct(); -try - my_surfdata.insert_surf(test_surf); -catch ME - warning(ME.getReport); -end - -% insert surf struct with invalid name -test_surf = my_surfdata.surf(1); -test_surf.name = 123; -try - my_surfdata.insert_surf(test_surf); -catch ME - warning(ME.getReport); +% run_example_str = 'demo_new_file'; +% run_example_str = 'demo_exist_file'; +run_example_str = 'demo_update_file'; + +if strcmpi(run_example_str,'demo_new_file') + % Demonstrate creation of a new surf file + param = read_param_xls(ct_filename_param('rds_param_2019_Antarctica_Ground.xls'),'20200107_01'); + mdata = echo_load(param,'music3D_paden',1); + surf = tomo.surfdata(mdata,'surfData_paden'); end -% insert a duplicate -test_surf = my_surfdata.surf(2); -try - my_surfdata.insert_surf(test_surf); -catch ME - warning(ME.getReport); -end - -%% set_name -my_surfdata.set('bottom', 'name', 'bottom2'); -my_surfdata.get_surf('bottom2') -my_surfdata.set('bottom2', 'name', 'bottom'); - -try - my_surfdata.set('123','name','1234'); % surf not exist -catch ME - warning(ME.getReport); +if strcmpi(run_example_str,'demo_exist_file') + % Demonstrate loading an existing surf file and manipulating it + + param = read_param_xls(ct_filename_param('rds_param_2009_Antarctica_TO.xls'),'20091224_01'); frm = 1; + fn_original = fullfile(ct_filename_out(param,'surfData',''),sprintf('Data_%s_%03d.mat',param.day_seg,frm)); + fn_new = [tempname '.mat']; + % Ensure that example file is using the most up to date format + echogram_fn = fullfile(ct_filename_out(param,'music3D',''),sprintf('Data_%s_%03d.mat',param.day_seg,frm)); + tomo.surfdata.update_file(fn_original,fn_new,echogram_fn); + + surf = tomo.surfdata(fn_new); + + %% get surf + surf.get_surf('bottom') % get the bottom surface + surf.get_surf({'top','bottom'}) % get the top and bottom surfaces + surf.get_surf([1 2 3]) + try + surf.get_surf('123') % surface does not exist; + catch ME + warning(ME.getReport); + end + try + surf.get_surf(20) % invalid index + catch ME + warning(ME.getReport); + end + try + surf.get_surf(0) % invalid index + catch ME + warning(ME.getReport); + end + + %% insert_surf + + % Create a test surf structure from an existing surface + test_surf = surf.get_surf('bottom'); + test_surf.name = 'my_test_surf'; + + % Insert test surface + surf.insert_surf(test_surf); + surf + + % create a test_surf structure where (size of x) != (size of y) + test_surf = surf.get_surf('bottom'); + test_surf.x = test_surf.x(:,1:round(end/2)); + test_surf.name = 'test'; + try + surf.insert_surf(test_surf); + catch ME + warning(ME.getReport); + end + + % create a test_surf which does not match existing surf x and y sizes + test_surf = surf.get_surf('bottom'); + test_surf.x = test_surf.x(1:round(end/2),1:round(end/2)); + test_surf.y = test_surf.y(1:round(end/2),1:round(end/2)); + test_surf.name = 'test'; + try + surf.insert_surf(test_surf); + catch ME + warning(ME.getReport); + end + + % insert an invalid struct + test_surf = struct(); + try + surf.insert_surf(test_surf); + catch ME + warning(ME.getReport); + end + + % insert surf struct with invalid name + test_surf = surf.surf(1); + test_surf.name = 123; + try + surf.insert_surf(test_surf); + catch ME + warning(ME.getReport); + end + + % insert a duplicate + test_surf = surf.surf(2); + try + surf.insert_surf(test_surf); + catch ME + warning(ME.getReport); + end + + %% set_name + surf.set('bottom', 'name', 'bottom2'); + surf.get_surf('bottom2') + surf.set('bottom2', 'name', 'bottom'); + + try + surf.set('123','name','1234'); % surf not exist + catch ME + warning(ME.getReport); + end + + %% set_surf and clear_references + test_surf = surf.get_surf('bottom') + test_surf = surf.clear_references(test_surf) + + surf.set_surf(test_surf); + surf.get_surf('bottom') % after; check result + + % set a surf that is not valid + try + surf.set_surf(struct()); + catch ME + warning(ME.getReport); + end + + % set a surface where the surface name does not match an existing surface + try + test_surf = surf.get_surf('bottom'); + test_surf.name = 'other'; + surf.set_surf(test_surf); + catch ME + warning(ME.getReport); + end + + %% remove_surf + test_surf = surf.get_surf('bottom'); + test_surf.name = 'test'; + surf.insert_surf(test_surf); + surf + surf.remove_surf('test'); + surf + + % See the index changes + v_names = {'name', 'index', 'top', 'active', 'mask', 'gt', 'quality'}; + + surf_name = {surf.surf.name}'; + indices = (1:length(surf.surf))'; + surf_top = {surf.surf.top}'; + surf_active = {surf.surf.active}'; + surf_mask = {surf.surf.mask}'; + surf_gt = {surf.surf.gt}'; + surf_quality = {surf.surf.quality}'; + + table(surf_name, indices, surf_top, ... + surf_active, surf_mask, ... + surf_gt, surf_quality, ... + 'VariableNames', v_names) + + surf.remove_surf('bottom'); + + surf_name = {surf.surf.name}'; + indices = (1:length(surf.surf))'; + surf_top = {surf.surf.top}'; + surf_active = {surf.surf.active}'; + surf_mask = {surf.surf.mask}'; + surf_gt = {surf.surf.gt}'; + surf_quality = {surf.surf.quality}'; + + table(surf_name, indices, surf_top, ... + surf_active, surf_mask, ... + surf_gt, surf_quality, ... + 'VariableNames', v_names) + + test_surf.name = 'bottom'; + surf.insert_surf(test_surf); + surf.set({'bottom','ice mask','bottom gt','bottom quality'}, ... + 'top','top','active','bottom','mask','ice mask','gt','bottom gt','quality','bottom quality'); + + surf_name = {surf.surf.name}'; + indices = (1:length(surf.surf))'; + surf_top = {surf.surf.top}'; + surf_active = {surf.surf.active}'; + surf_mask = {surf.surf.mask}'; + surf_gt = {surf.surf.gt}'; + surf_quality = {surf.surf.quality}'; + + table(surf_name, indices, surf_top, ... + surf_active, surf_mask, ... + surf_gt, surf_quality, ... + 'VariableNames', v_names) + + try; + surf.remove_surf('123'); % error + catch ME + warning(ME.getReport) + end + + %% save_surfdata + surf.save_surfdata(fn_new); end -%% set_surf and clear_references -test_surf = my_surfdata.get_surf('bottom') -test_surf = my_surfdata.clear_references(test_surf) +% Demonstrate updating a file to the newest format +if strcmpi(run_example_str,'demo_update_file') + + param = read_param_xls(ct_filename_param('rds_param_2009_Antarctica_TO.xls'),'20091224_01'); + fn_original = fullfile(ct_filename_out(param,'surfData',''),sprintf('Data_%s_%03d.mat',param.day_seg,frm)); + fn_new = [tempname '.mat']; + + echogram_fn = fullfile(ct_filename_out(param,'music3D',''),sprintf('Data_%s_%03d.mat',param.day_seg,frm)); + + tomo.surfdata.update_file(fn_original,fn_new,echogram_fn); + + surf = tomo.surfdata(fn_new); -my_surfdata.set_surf(test_surf); -my_surfdata.get_surf('bottom') % after; check result - -% set a surf that is not valid -try - my_surfdata.set_surf(struct()); -catch ME - warning(ME.getReport); -end - -% set a surface where the surface name does not match an existing surface -try - test_surf = my_surfdata.get_surf('bottom'); - test_surf.name = 'other'; - my_surfdata.set_surf(test_surf); -catch ME - warning(ME.getReport); end - -%% remove_surf -my_surfdata.load_surfdata(fn); -test_surf = my_surfdata.get_surf('bottom'); -test_surf.name = 'test'; -my_surfdata.insert_surf(test_surf); -my_surfdata -my_surfdata.remove_surf('test'); -my_surfdata - -% See the index changes -v_names = {'name', 'index', 'top', 'active', 'mask', 'gt', 'quality'}; - -my_surfdata_name = {my_surfdata.surf.name}'; -indices = (1:length(my_surfdata.surf))'; -my_surfdata_top = {my_surfdata.surf.top}'; -my_surfdata_active = {my_surfdata.surf.active}'; -my_surfdata_mask = {my_surfdata.surf.mask}'; -my_surfdata_gt = {my_surfdata.surf.gt}'; -my_surfdata_quality = {my_surfdata.surf.quality}'; - -table(my_surfdata_name, indices, my_surfdata_top, ... - my_surfdata_active, my_surfdata_mask, ... - my_surfdata_gt, my_surfdata_quality, ... - 'VariableNames', v_names) - -my_surfdata.remove_surf('bottom'); - -my_surfdata_name = {my_surfdata.surf.name}'; -indices = (1:length(my_surfdata.surf))'; -my_surfdata_top = {my_surfdata.surf.top}'; -my_surfdata_active = {my_surfdata.surf.active}'; -my_surfdata_mask = {my_surfdata.surf.mask}'; -my_surfdata_gt = {my_surfdata.surf.gt}'; -my_surfdata_quality = {my_surfdata.surf.quality}'; - -table(my_surfdata_name, indices, my_surfdata_top, ... - my_surfdata_active, my_surfdata_mask, ... - my_surfdata_gt, my_surfdata_quality, ... - 'VariableNames', v_names) - -test_surf.name = 'bottom'; -my_surfdata.insert_surf(test_surf); -my_surfdata.set({'bottom','ice mask','bottom gt','bottom quality'}, ... - 'top','top','active','bottom','mask','ice mask','gt','bottom gt','quality','bottom quality'); - -my_surfdata_name = {my_surfdata.surf.name}'; -indices = (1:length(my_surfdata.surf))'; -my_surfdata_top = {my_surfdata.surf.top}'; -my_surfdata_active = {my_surfdata.surf.active}'; -my_surfdata_mask = {my_surfdata.surf.mask}'; -my_surfdata_gt = {my_surfdata.surf.gt}'; -my_surfdata_quality = {my_surfdata.surf.quality}'; - -table(my_surfdata_name, indices, my_surfdata_top, ... - my_surfdata_active, my_surfdata_mask, ... - my_surfdata_gt, my_surfdata_quality, ... - 'VariableNames', v_names) - -try; - my_surfdata.remove_surf('123'); % error -catch ME - warning(ME.getReport) -end - -%% save_surfdata -my_surfdata.save_surfdata(fn); - diff --git a/cresis-toolbox/+tomo/run_surfdata_modify.m b/cresis-toolbox/+tomo/run_surfdata_modify.m index 6552212e..5ce5f65c 100644 --- a/cresis-toolbox/+tomo/run_surfdata_modify.m +++ b/cresis-toolbox/+tomo/run_surfdata_modify.m @@ -1,4 +1,4 @@ -% script tomo.run_surfData_modify.m +% script tomo.run_surfdata_modify.m % % Example script for running surfData_modify.m. Demonstrates a few of the % most common operations to be performed with surfData_modify. @@ -12,7 +12,7 @@ % params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls')); params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls'),'20140325_07','post'); params.cmd.generic = 1; -params.cmd.frms = [1 2]; +params.cmd.frms = []; surfdata_source = 'paden_surfData'; @@ -21,18 +21,19 @@ %% Example for updating fields of a particular layer % layers = [1 7]; % args = []; -% args{1} = 'quality_layer'; +% args{1} = 'quality'; % args{2} = 8; layers = [2 3 4 5 6]; args = []; - args{1} = 'quality_layer'; + args{1} = 'quality'; args{2} = 9; elseif strcmpi(SURFDATA_MODIFY_EXAMPLE,'add_new_layer') %% Example for adding a new layer from ops in -% echo_source = 'paden_music'; -% ops_layer_name = ''; % Leave empty to not use ops layer +% echo_source = 'music3D'; +% echo_source_img = 1; +% layer_params = []; % Leave empty to not use opsLoadLayer at nadir % default_value = 1; % Default value to use when layer values not provided % new_layer = 8; % args = []; @@ -40,21 +41,22 @@ % args{end+1} = {'color','red','marker','x'}; % args{end+1} = 'name'; % args{end+1} = 'surface quality'; -% args{end+1} = 'surf_layer'; +% args{end+1} = 'top'; % args{end+1} = []; -% args{end+1} = 'active_layer'; +% args{end+1} = 'active'; % args{end+1} = 1; -% args{end+1} = 'mask_layer'; +% args{end+1} = 'mask'; % args{end+1} = []; -% args{end+1} = 'control_layer'; +% args{end+1} = 'gt'; % args{end+1} = 7; -% args{end+1} = 'quality_layer'; +% args{end+1} = 'quality'; % args{end+1} = 8; % args{end+1} = 'visible'; % args{end+1} = true; - echo_source = 'paden_music'; - ops_layer_name = ''; % Leave empty to not use ops layer + echo_source = 'music3D'; + echo_source_img = 1; + layer_params = []; % Leave empty to not use opsLoadLayer at nadir default_value = 1; % Default value to use when layer values not provided new_layer = 9; args = []; @@ -62,21 +64,22 @@ args{end+1} = {'color','red','marker','^'}; args{end+1} = 'name'; args{end+1} = 'bottom quality'; - args{end+1} = 'surf_layer'; + args{end+1} = 'top'; args{end+1} = 1; - args{end+1} = 'active_layer'; + args{end+1} = 'active'; args{end+1} = 2; - args{end+1} = 'mask_layer'; + args{end+1} = 'mask'; args{end+1} = 3; - args{end+1} = 'control_layer'; + args{end+1} = 'gt'; args{end+1} = 4; - args{end+1} = 'quality_layer'; + args{end+1} = 'quality'; args{end+1} = 9; args{end+1} = 'visible'; args{end+1} = true; -% echo_source = 'paden_music'; -% ops_layer_name = 'surface'; % Leave empty to not use ops layer +% echo_source = 'music3D'; +% echo_source_img = 1; +% layer_params = struct('name','surface'); % Use opsLoadLayer at nadir % default_value = NaN; % Default value to use when layer values not provided % new_layer = 7; % args = []; @@ -84,15 +87,15 @@ % args{end+1} = {'color','magenta','marker','^'}; % args{end+1} = 'name'; % args{end+1} = 'surface gt'; -% args{end+1} = 'surf_layer'; +% args{end+1} = 'top'; % args{end+1} = []; -% args{end+1} = 'active_layer'; +% args{end+1} = 'active'; % args{end+1} = 1; -% args{end+1} = 'mask_layer'; +% args{end+1} = 'mask'; % args{end+1} = []; -% args{end+1} = 'control_layer'; +% args{end+1} = 'gt'; % args{end+1} = 7; -% args{end+1} = 'quality_layer'; +% args{end+1} = 'quality'; % args{end+1} = 8; % args{end+1} = 'visible'; % args{end+1} = true; @@ -132,46 +135,37 @@ param = merge_structs(param,gRadar); % Determine which frames to process - load(ct_filename_support(param,'','frames')); - - if isempty(param.cmd.frms) - param.cmd.frms = 1:length(frames.frame_idxs); - end - % Remove frames that do not exist from param.cmd.frms list - [valid_frms,keep_idxs] = intersect(param.cmd.frms, 1:length(frames.frame_idxs)); - if length(valid_frms) ~= length(param.cmd.frms) - bad_mask = ones(size(param.cmd.frms)); - bad_mask(keep_idxs) = 0; - warning('Nonexistent frames specified in param.cmd.frms (e.g. frame "%g" is invalid), removing these', ... - param.cmd.frms(find(bad_mask,1))); - param.cmd.frms = valid_frms; - end + frames = frames_load(param); + param.cmd.frms = frames_param_cmd_frms(param,frames); %% Only do one frame at a time all_frms = param.cmd.frms; for frm = all_frms(:).' param.cmd.frms = frm; echo_fn_dir = ct_filename_out(param,echo_source); - echo_fn = fullfile(echo_fn_dir,sprintf('Data_%s_%03d.mat',param.day_seg, frm)); + if echo_source_img == 0 + echo_fn = fullfile(echo_fn_dir,sprintf('Data_%s_%03d.mat', param.day_seg, frm)); + else + echo_fn = fullfile(echo_fn_dir,sprintf('Data_img_%02d_%s_%03d.mat', echo_source_img, param.day_seg, frm)); + end + if ~exist(echo_fn,'file') + warning('File does not exist: %s', echo_fn); + continue; + end - mdata = load(echo_fn,'Surface','Time','GPS_time','Latitude','Longitude','Elevation','twtt'); + mdata = load(echo_fn,'Surface','Time','GPS_time','Latitude','Longitude','Elevation','Tomo'); args{end+1} = 'x'; - args{end+1} = repmat((1:size(mdata.twtt,1)).',[1 size(mdata.twtt,2)]); + args{end+1} = repmat(mdata.Tomo.theta(:,1),[1 size(mdata.Tomo.img,3)]); args{end+1} = 'y'; - surf_layer = default_value*ones(size(mdata.twtt)); + top = default_value*ones(size(mdata.Tomo.img,2), size(mdata.Tomo.img,3)); - if ~isempty(ops_layer_name) + if ~isempty(layer_params) % Query OPS for surface and bottom information param_load_layers = param; param_load_layers.cmd.frms = round([-1,0,1] + frm); - layer_params = []; - idx = 0; - idx = idx + 1; - layer_params(idx).name = ops_layer_name; - layer_params(idx).source = 'ops'; layers = opsLoadLayers(param_load_layers,layer_params); % Interpolate surface and bottom information to mdata @@ -194,10 +188,10 @@ end % Add surface information to surfData file - Surface = layers(1).twtt_ref; - surf_layer(floor(size(mdata.twtt,1)/2)+1,:) = interp1(mdata.Time, 1:length(mdata.Time), Surface); + [~,nadir_idx] = min(abs(mdata.Tomo.theta(:,1))); + top(nadir_idx,:) = layers(1).twtt_ref; end - args{end+1} = surf_layer; + args{end+1} = top; fprintf('surfData_modify %s_%03d\n', param.day_seg, frm); tomo.surfData_modify(param,surfdata_source,new_layer,args{:}); diff --git a/cresis-toolbox/+tomo/run_surfdata_update.m b/cresis-toolbox/+tomo/run_surfdata_update.m deleted file mode 100644 index 67e4f6e8..00000000 --- a/cresis-toolbox/+tomo/run_surfdata_update.m +++ /dev/null @@ -1,66 +0,0 @@ -%% Setup - -params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls'),'','post'); -% params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls'),'20140401_03','post'); -% params.cmd.generic = 1; -% params.cmd.frms = 22; -ct_set_params(params,'cmd.generic',0); -ct_set_params(params,'cmd.generic',1,'20140401_03|20140506_01|20140325_05|20140325_06|20140325_07'); -param_override.update.input = 'CSA_music_surfData_no_QC'; -param_override.update.output = 'surfData_v2_no_MC'; -param_override.update.echogram = 'CSA_music'; - -% params = read_param_xls(ct_filename_param('rds_param_2009_Antarctica_TO.xls'),'','post'); -% params.cmd.generic = 1; -% params.cmd.frms = []; -% param_override.update.input = 'surfData'; -% param_override.update.output = 'surfData_v2'; -% param_override.update.echogram = 'music3D'; - -%% Automated section -global gRadar; - -% Input checking -if exist('param_override','var') - param_override = merge_structs(gRadar,param_override); -else - param_override = gRadar; -end - -for param_idx = 1:length(params) - param = params(param_idx); - if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic - continue; - end - - param = merge_structs(param, param_override); - - - % Load frames file - load(ct_filename_support(param,'','frames')); - - if isempty(param.cmd.frms) - param.cmd.frms = 1:length(frames.frame_idxs); - end - % Remove frames that do not exist from param.cmd.frms list - [valid_frms,keep_idxs] = intersect(param.cmd.frms, 1:length(frames.frame_idxs)); - if length(valid_frms) ~= length(param.cmd.frms) - bad_mask = ones(size(param.cmd.frms)); - bad_mask(keep_idxs) = 0; - warning('Nonexistent frames specified in param.cmd.frms (e.g. frame "%g" is invalid), removing these', ... - param.cmd.frms(find(bad_mask,1))); - param.cmd.frms = valid_frms; - end - - for frm = param.cmd.frms - - fn = fullfile(ct_filename_out(param,param.update.input,''),sprintf('Data_%s_%03d.mat',param.day_seg,frm)); - fn_cur_ver = fullfile(ct_filename_out(param,param.update.output,''),sprintf('Data_%s_%03d.mat',param.day_seg,frm)); - echogram_fn = fullfile(ct_filename_out(param,param.update.echogram,''),sprintf('Data_%s_%03d.mat',param.day_seg,frm)); - - fprintf('Convert\n %s\n %s\n %s\n', fn, fn_cur_ver,echogram_fn); - tomo.surfdata.update_file(fn,fn_cur_ver,echogram_fn); - end - -end - diff --git a/cresis-toolbox/+tomo/run_tracker_2D.m b/cresis-toolbox/+tomo/run_tracker_2D.m index 896ed13d..ec9ae707 100644 --- a/cresis-toolbox/+tomo/run_tracker_2D.m +++ b/cresis-toolbox/+tomo/run_tracker_2D.m @@ -123,7 +123,7 @@ continue; end % Load frames file - load(ct_filename_support(param,'','frames')); + frames = frames_load(param); if isempty(param.cmd.frms) param.cmd.frms = 1:length(frames.frame_idxs); end @@ -161,7 +161,7 @@ continue; end % Load frames file - load(ct_filename_support(param,'','frames')); + frames = frames_load(param); if isempty(param.cmd.frms) param.cmd.frms = 1:length(frames.frame_idxs); end diff --git a/cresis-toolbox/+tomo/surfdata.m b/cresis-toolbox/+tomo/surfdata.m index 5c0fb136..9d327369 100644 --- a/cresis-toolbox/+tomo/surfdata.m +++ b/cresis-toolbox/+tomo/surfdata.m @@ -1,318 +1,281 @@ classdef surfdata < handle -% tomo.surfdata(fn) < handle -% -% A class that manages surf data in a frame. -% -% fn: optional string containing a surfdata filename to load, default -% is to create an empty class -% -% Example of a filepath: -% X:\ct_data\rds\2014_Greenland_P3\CSARP_surfData\20140401_03\Data_20140401_03_037.mat -% -% Example of how to generate a filepath: -% param.radar_name = 'rds'; -% param.season_name = '2014_Greenland_P3'; -% param.day_seg = '20140401_03' -% surfdata_ref = 'surfData'; -% frm = 37; -% fn = fullfile(ct_filename_out(param,surfdata_ref,''),... -% sprintf('Data_%s_%03d.mat',param.day_seg,frm)); -% -% Author: Shane Chu, John Paden -% -% See also: run_surfdata.m, surfdata.m + % tomo.surfdata(fn) < handle + % + % A class that manages surf data in a frame. + % + % fn: optional string containing a surfdata filename to load, default + % is to create an empty class + % + % Example of a filepath: + % X:\ct_data\rds\2014_Greenland_P3\CSARP_surfData\20140401_03\Data_20140401_03_037.mat + % + % Example of how to generate a filepath: + % param.radar_name = 'rds'; + % param.season_name = '2014_Greenland_P3'; + % param.day_seg = '20140401_03' + % surfdata_ref = 'surfData'; + % frm = 37; + % fn = fullfile(ct_filename_out(param,surfdata_ref,''),... + % sprintf('Data_%s_%03d.mat',param.day_seg,frm)); + % + % Author: Shane Chu, John Paden + % + % See also: run_surfdata.m, surfdata.m properties (Constant) - current_version = 2.0; + current_version = 3.0; end properties + surfdata_source + + % fcs: flight (SAR) coordinate system for each column in surf.[xy] + % .origin: origin, 3 by Nx + % .x: unit x-axis vector, 3 by Nx (points along-track) + % .y: unit y-axis vector, 3 by Nx (points left, but equal to cross(x,z)) + % .z: unit z-axis vector, 3 by Nx (points up, but orthogonal to x) + fcs + + % gps_source: String containing the gps source + gps_source + + % gps_time: GPS time of each column in surf.[xy] + gps_time + + % param: parameter structure + % param.day_seg: string containing the segment + % param.load.frm: integer scalar containing the frame number + % param.radar.lever_arm_fh: lever arm function handle used to create trajectories + % param.radar_name: string containing the radar name + % param.records.gps.time_offset: Time offset used when syncing radar records to GPS + % param.season_name: string containing the season name + % param.sw_version: software version structure from current_software_version.m + param + % surf: A structure array; each element holds a single surface - % .name: string containing the name of the surface (e.g. 'top', - % 'bottom') - % .x: Nsv by Nx array containing the x-coordinates of the surface points - % .y: Nsv by Nx array containing the y-coordinates of the surface points - % .visible: Logical scalar indicating whether or not the surface should - % be plotted. - % .top: Index into surf structure which indicates the "top" - % surface to use when this surface is selected. % .active: Index into surf structure which indicates the "active" - % surface to use when this surface is selected. + % surface to use when this surface is selected. Automated tracking + % results go to this surface. + % .gt: Index into surf structure which indicates the ground truth + % surface to use when this surface is selected. Manual labels or + % ground truth go to this surface. % .mask: Index into surf structure which indicates the "ice mask" % surface to use when this surface is selected. - % .control: Index into surf structure which indicates the "control" - % surface to use when this surface is selected. + % .name: string containing the name of the surface (e.g. 'top', + % 'bottom') + % .plot_name_values: Plot (name,value) pairs stored in a cell array + % which will be passed in when ever this surface is plotted. These + % use the Matlab "plot" function and allow specifying colors, + % marker, and so on. % .quality: Index into surf structure which indicates the "quality" % surface to use when this surface is selected. - % .plot_name_values: Plot (name,value) pairs stored in a cell array - % which will be passed in when ever this surface is plotted. + % .top: Index into surf structure which indicates the "top" + % surface to use when this surface is selected. + % .visible: Logical scalar indicating whether or not the surface should + % be plotted. + % .x: Nsv by Nx array containing the elevation angle of each surface + % point, a NaN value indicates no data at this point + % .y: Nsv by Nx array containing the twtt of each surface point, a NaN + % value indicates no data at this point surf - % radar_name: string containing the radar name - radar_name - % season_name: string containing the season name - season_name - % day_seg: string containing the segment - day_seg - % frm: integer scalar containing the frame number - frm - - % gps_time: GPS time of each column in surf.[xy] - gps_time + theta % Support legacy "bins" unit format + time % Support legacy "bins" unit format - % theta: Nsv by 1 matrix containing DOA for each row in surf.[xy] - % (zero theta is -z, theta increases toward positive y) - theta - - % time: fast (range) time which allows surf.y values (which are in - % range bins) to be converted to two way travel time - time - - % FCS: flight (SAR) coordinate system for each column in surf.[xy] - % .origin: origin, 3 by Nx - % .x: unit x-axis vector, 3 by Nx (points along-track) - % .y: unit y-axis vector, 3 by Nx (points left, but equal to cross(x,z)) - % .z: unit z-axis vector, 3 by Nx (points up, but orthogonal to x) - FCS + unit_type end - methods - %% constructor - function obj = surfdata(fn,param,frm,gps_time,theta,FCS) - % no arguments => create an empty surf struct array - if nargin == 0 - obj.surf = []; - obj.radar_name = []; - obj.season_name = []; - obj.day_seg = []; - obj.frm = []; - obj.gps_time = []; - obj.theta = []; - obj.time = []; - obj.FCS = []; - - % one argument => load file and create struct array accordingly - else - % Check to see if this is a direction of arrival method or a beam - % forming method file. - if ~exist('param','var') || ~isfield(param,'doa_method_flag') - doa_method_flag = false; - else - doa_method_flag = param.doa_method_flag; - end - try - obj.surf = []; - obj.load_surfdata(fn,doa_method_flag); - - catch ME - error('Failed to load filename (%s):\n %s', fn, ME.getReport()); - end - end - end + %% PUBLIC METHODS ========== + methods - %% load_surf - function load_surfdata(obj, fn,doa_method_flag) - % obj.load_surfdata(fn) - % - % Loads a new surf struct into the object. If fn is a filename, - % it should be a surfData file. Note that this function will overwrite - % the existing surf struct array. - % - % Input: - % fn: The file path OR a new struct array with all the fields in a - % surfData file. - % doa_method_flag: logical that specifies is this file stores DOA - % data or beam forming data. Default is false. + function obj = surfdata(source,surfdata_source) + %% surfdata constructor - if ~exist('doa_method_flag','var') - doa_method_flag = false; - end - - if ischar(fn) - if obj.version(fn) < obj.current_version - error('File "%s" is old. Run tomo.surfdata.update on the file.', fn); - end + if nargin < 2 || isempty(surfdata_source) + surfdata_source = 'surf'; % CSARP_surf directory end + obj.surfdata_source = surfdata_source; - old_surf = obj.surf; - try - if ischar(fn) - tmp = load(fn); - else - tmp = fn; + file_type = file_type_get(source); + if strcmp(file_type,'array') + % Input source is array echogram file: create a new surfdata + % ----------------------------------------------------------------- + if ischar(source) + % If a filename, load into a structure. + source = echo_load(source); end - obj.surf = tmp.surf; - for surf_idx = 1:length(obj.surf) - obj.valid_surf(obj.surf(surf_idx),doa_method_flag); + tmp_param = echo_param(source); + + obj.fcs.origin = tmp_param.array_proc.fcs.origin; + obj.fcs.x = tmp_param.array_proc.fcs.x; + obj.fcs.y = tmp_param.array_proc.fcs.y; + obj.fcs.z = tmp_param.array_proc.fcs.z; + + obj.gps_source = source.param_records.gps_source; + + obj.gps_time = source.GPS_time; + + obj.param.day_seg = tmp_param.day_seg; + obj.param.load.frm = tmp_param.load.frm; + obj.param.radar.lever_arm_fh = tmp_param.radar.lever_arm_fh; + obj.param.radar_name = tmp_param.radar_name; + obj.param.records.gps.time_offset = tmp_param.records.gps.time_offset; + obj.param.season_name = tmp_param.season_name; + obj.param.sw_version = current_software_version; + + obj.surf = struct('active',{}, ... + 'gt',{}, ... + 'mask',{}, ... + 'name',{}, ... + 'plot_name_values',{}, ... + 'quality',{}, ... + 'top',{}, ... + 'visible',{}, ... + 'x',{}, ... + 'y',{} ); + + elseif strcmp(file_type,'surf') + % Input source is surf file: load existing surfdata + % ----------------------------------------------------------------- + if ischar(source) + % If a filename, load into a structure. + source = load(source); end - obj.set_metadata(tmp,doa_method_flag); + obj.fcs.origin = source.fcs.origin; + obj.fcs.x = source.fcs.x; + obj.fcs.y = source.fcs.y; + obj.fcs.z = source.fcs.z; - catch ME - obj.surf = old_surf; - if ischar(fn) - error('Failed to load %s:\n %s', fn, ME.getReport()); - else - error('Failed to load struct:\n %s', ME.getReport()); + obj.gps_source = source.gps_source; + + obj.gps_time = source.gps_time; + + obj.param.day_seg = source.param.day_seg; + obj.param.load.frm = source.param.load.frm; + obj.param.radar.lever_arm_fh = source.param.radar.lever_arm_fh; + obj.param.radar_name = source.param.radar_name; + obj.param.records.gps.time_offset = source.param.records.gps.time_offset; + obj.param.season_name = source.param.season_name; + obj.param.sw_version = current_software_version; + + if isempty(source.surf) + source.surf = struct(); end + obj.surf = orderfields(source.surf); end - end - - function set_metadata(obj, md, doa_method_flag) - % obj.set_metadata(md) - % - % Function for validating all the metadata fields - % - % md: struct containing gps_time, theta, FCS, radar_name, - % season_name, day_seg, and frm. - % doa_method_flag: logical that specifies is this file stores DOA - % data or beam forming data. Default is false. - - obj.valid_metadata(md,doa_method_flag); - obj.radar_name = md.radar_name; - obj.season_name = md.season_name; - obj.day_seg = md.day_seg; - obj.frm = md.frm; - obj.gps_time = md.gps_time; - obj.theta = md.theta; - obj.time = md.time; - obj.FCS = md.FCS; + obj.unit_type = 'standard'; end - function valid_metadata(obj, md,doa_method_flag) - % obj.valid_metadata(md) - % - % Function for validating all the metadata fields + function adjust_surf(obj, dbin, surface_name_string) + %% adjust_surf + % obj.adjust_surf(dbin) % - % md: struct containing gps_time, theta, FCS, radar_name, - % season_name, day_seg, and frm. - % doa_method_flag: logical that specifies is this file stores DOA - % data or beam forming data. Default is false. - - if ~ischar(md.radar_name) - error('radar_name must be a string'); - end - - if ~ischar(md.season_name) - error('season_name must be a string'); - end + % Input: + % dbin: Number of bins to offset the surface + % + % surface_name_string: A cell array of strings that match the name + % of the surface in the surf struct array of the object. If empty + % or undefined, all surfaces will be updated. + % + % Result: + % 1. surf struct .y field will be adjusted by dbin - if ~ischar(md.day_seg) - error('day_seg must be a string'); + if ~exist('surface_name_string','var') + surface_name_string = []; end - - if ~isnumeric(md.frm) - error('frm must be a positive integer'); + for surf_idx = 1:length(obj.surf) + if isempty(surface_name_string) ... + || ((ischar(surface_name_string) || iscell(surface_name_string)) ... + && any(strcmpi(obj.surf(surf_idx).name,surface_name_string))) + imagesc(obj.surf(surf_idx).y) + colorbar; + obj.surf(surf_idx).name + obj.surf(surf_idx).y(1) + keyboard + % obj.surf(surf_idx).y = obj.surf(surf_idx).y + dbin; + end end - Nx = size(md.gps_time,2); - - if size(md.gps_time,1) ~= 1 - error('gps_time must be a row vector.'); - end + end + + function surf_idx = get_index(obj, surf_name, error_on_fail) + %% get_index + % obj.get_index(surf_name, error_on_fail) + % + % Input: + % surf_name: A string that matches the name of a surface in the + % surf struct array in the object OR a cell array of similar + % strings OR an index array of indices into the surf structure + % error_on_fail: Logical scalar indicating whether or not an error + % should be thrown if the surface is not found. Default is false. + % + % Return: + % 1. The index of that surf struct in the surf + % struct array in the object, if it is found. + % 2. Throws an error if it is not found and error_on_fail is true + % or if an array of indices were input and contain an invalid + % index. - if ~doa_method_flag - if ~isempty(obj.surf) && size(obj.surf(1).x,2) ~= Nx - error('gps_time must have same number of columns as surf.x'); + if isnumeric(surf_name) + % Just check that indices are valid + valid_list = intersect(unique(surf_name),1:length(obj.surf)); + if length(valid_list) ~= length(unique(surf_name)) + error('Some of the indices in surf_name are invalid.'); end + surf_idx = surf_name; - Nsv = size(md.theta,1); + else + if ~exist('error_on_fail','var') || isempty(error_on_fail) + error_on_fail = false; + end - if size(md.theta,2) ~= 1 - error('theta must be a column vector.'); + if ischar(surf_name) + surf_name = {surf_name}; + elseif ~iscell(surf_name) + error('Input type of surf_name is not valid.'); end - if ~isempty(obj.surf) && size(obj.surf(1).x,1) ~= Nsv - error('theta must have same number of rows as surf.x'); + for idx = 1:length(surf_name) + new_surf_idx = find(strcmp(surf_name{idx},{obj.surf.name})); + if error_on_fail && isempty(new_surf_idx) + error('Surface "%s" is not in the list of surfaces.', surf_name{idx}); + end + surf_idx(idx) = new_surf_idx; end end - - if size(md.time,2) ~= 1 - error('time must be a column vector of numbers'); - end - - if ~isnumeric(md.time) - error('time must be a column vector of numbers'); - end - - if size(md.FCS.origin,2) ~= Nx - error('FCS.origin must have same number of columns as gps_time.'); - end - - if size(md.FCS.x,2) ~= Nx - error('FCS.x must have same number of columns as gps_time.'); - end - - if size(md.FCS.y,2) ~= Nx - error('FCS.y must have same number of columns as gps_time.'); - end - - if size(md.FCS.z,2) ~= Nx - error('FCS.z must have same number of columns as gps_time.'); - end - - if size(md.FCS.origin,1) ~= 3 - error('FCS.origin must have 3 rows.'); - end - - if size(md.FCS.x,1) ~= 3 - error('FCS.x must have 3 rows.'); - end - - if size(md.FCS.y,1) ~= 3 - error('FCS.y must have 3 rows.'); - end - - if size(md.FCS.z,1) ~= 3 - error('FCS.z must have 3 rows.'); - end - end - - %% insert_surf - function insert_surf(obj, surf_struct,doa_method_flag) - % obj.insert_surf(surf_struct) + + function surf_names = get_names(obj) + %% get_names + % surf_names = get_names(obj) % - % Input: - % surf_struct: A surf structure. Normally, the indexing fields - % such as top, active, mask, gt, quality should be cleared. - % doa_method_flag: logical that specifies is this file stores DOA - % data or beam forming data. Default is false. + % Returns all the surface names in a cell array % - % Result: - % Adds a surf structure, surf_A, into the "surf" array of the object + % No inputs % - % See also: surfdata.clear_references - - if ~exist('doa_method_flag','var') - doa_method_flag = false; - end - % check if it is a valid surf structure - obj.valid_surf(surf_struct,doa_method_flag); + % surf_names: cell array of surface names - % check if name of the surface already existed in the class - if isempty(obj.surf) - obj.surf = [obj.surf surf_struct]; - elseif any(strcmpi(surf_struct.name,{obj.surf.name})) - error('This surface is already inserted.'); + if ~isempty(obj.surf) + surf_names = {obj.surf.name}; else - obj.surf = [obj.surf surf_struct]; + surf_names = {}; end - end - %% get_surf + function surf = get_surf(obj, surf_name) + %% get_surf % surf = obj.get_surf(surf_name) % - % Input: + % Input: % surf_name: Must be one of the following: % 1. a string containing a valid surface name % 2. a cell array of strings containing valid surface names - % 3. a numeric array of surface indices + % 3. a numeric array of surface indices % - % Return: + % Return: % A surf structure array matching surf_name if isa(surf_name, 'char') @@ -344,23 +307,104 @@ function insert_surf(obj, surf_struct,doa_method_flag) else error('Input must be either a string containing a valid surface name, a cell array of strings containing valid surface names, or a numeric array of surface indices.'); - end + end + end + + function insert_surf(obj, surf_struct) + %% insert_surf + % obj.insert_surf(surf_struct) + % + % Input: + % surf_struct: A surf structure. Normally, the indexing fields + % such as top, active, mask, gt, quality should be cleared. + % + % Result: + % Adds a surf structure, surf_A, into the "surf" array of the object + % + % See also: surfdata.clear_references + + % check if it is a valid surf structure + obj.valid_surf(surf_struct); + + % check if name of the surface already existed in the class + if isempty(obj.surf) + obj.surf = [obj.surf orderfields(surf_struct)]; + elseif any(strcmpi(surf_struct.name,{obj.surf.name})) + error('This surface is already inserted.'); + else + obj.surf = [obj.surf orderfields(surf_struct)]; + end + + end + + function remove_surf(obj, surface_name) + %% remove_surf + % obj.remove_surf(surface_name_string) + % + % Input: + % surface_name_string: A string that matches + % the name of the surface in the surf struct + % array of the object. + % + % Result: + % 1. Removes surf struct in the surf struct array + % of the object. + % 2. Throws an error if the surface_name is not + % in any of the surf struct in the surf struct array + % of the object. + + match_idx = obj.get_index(surface_name,true); + + for surf_idx = 1:length(obj.surf) + + if obj.surf(surf_idx).top == match_idx + obj.surf(surf_idx).top = []; + elseif obj.surf(surf_idx).top > match_idx + obj.surf(surf_idx).top = obj.surf(surf_idx).top -1; + end + + if obj.surf(surf_idx).active == match_idx + obj.surf(surf_idx).active = []; + elseif obj.surf(surf_idx).active > match_idx + obj.surf(surf_idx).active = obj.surf(surf_idx).active -1; + end + + if obj.surf(surf_idx).mask == match_idx + obj.surf(surf_idx).mask = []; + elseif obj.surf(surf_idx).mask > match_idx + obj.surf(surf_idx).mask = obj.surf(surf_idx).mask -1; + end + + if obj.surf(surf_idx).gt == match_idx + obj.surf(surf_idx).gt = []; + elseif obj.surf(surf_idx).gt > match_idx + obj.surf(surf_idx).gt = obj.surf(surf_idx).gt -1; + end + + if obj.surf(surf_idx).quality == match_idx + obj.surf(surf_idx).quality = []; + elseif obj.surf(surf_idx).quality > match_idx + obj.surf(surf_idx).quality = obj.surf(surf_idx).quality -1; + end + end + + obj.surf = [obj.surf(1:match_idx-1) obj.surf(match_idx+1:end)]; end - %% set function set(obj, surf_name, varargin) + %% set % obj.set(surf_name, NAME_VALUE_PAIRS) % - % Input: + % Input: % surf_name: surface name or cell array of surface names % NAME_VALUE_PAIRS: Pairs of input arguments where the first % argument in the pair is the property to change and the second % argument in the pair is the value. % - % Return: + % Return: % 1. Nothing on success % 2. Throws an error if surface_old_name is not found. - + surf_idxs = obj.get_index(surf_name,true); for arg_idx = 3:2:nargin @@ -375,7 +419,7 @@ function set(obj, surf_name, varargin) error('The new name matches an existing name for another surface.'); end obj.surf(surf_idxs).name = new_name; - + case {'top','active','mask','gt','quality'} new_surf_idx = obj.get_index(varargin{arg_idx-1}); if length(new_surf_idx) ~= 1 @@ -384,7 +428,7 @@ function set(obj, surf_name, varargin) for surf_idx = surf_idxs obj.surf(surf_idx).(field_name) = new_surf_idx; end - + otherwise error('Invalid field %s', varargin{arg_idx-2}); @@ -394,262 +438,259 @@ function set(obj, surf_name, varargin) end - %% set_surf - function obj = set_surf(obj, surf_struct) - % obj.set_surf(surf_struct) + function set_metadata(obj, md) + %% set_metadata(md) % - % Input: - % surf_struct: A surf structure. + % Function for validating all the metadata fields % - % Result: - % 1. Replace the surf structure if the name of - % surf_struct mathes with another surf struct - % in the surf struct array in the object. - % 2. Throws an error if the surf struct is not valid. + % md: struct containing gps_time, fcs, radar_name, param. May contain + % theta and time fields. + obj.valid_metadata(md); - % check if it is a valid surf structure - obj.valid_surf(surf_struct); - - % Sets an existing surface to the new field values - match_idx = obj.get_index(surf_struct.name,true); - obj.surf(match_idx) = surf_struct; - end - - %% remove_surf - function remove_surf(obj, surface_name) - % obj.remove_surf(surface_name_string) - % - % Input: - % surface_name_string: A string that matches - % the name of the surface in the surf struct - % array of the object. - % - % Result: - % 1. Removes surf struct in the surf struct array - % of the object. - % 2. Throws an error if the surface_name is not - % in any of the surf struct in the surf struct array - % of the object. + obj.param = md.param; + obj.gps_time = md.gps_time; + obj.fcs = md.fcs; - match_idx = obj.get_index(surface_name,true); - - for surf_idx = 1:length(obj.surf) - - if obj.surf(surf_idx).top == match_idx - obj.surf(surf_idx).top = []; - elseif obj.surf(surf_idx).top > match_idx - obj.surf(surf_idx).top = obj.surf(surf_idx).top -1; - end - - if obj.surf(surf_idx).active == match_idx - obj.surf(surf_idx).active = []; - elseif obj.surf(surf_idx).active > match_idx - obj.surf(surf_idx).active = obj.surf(surf_idx).active -1; - end - - if obj.surf(surf_idx).mask == match_idx - obj.surf(surf_idx).mask = []; - elseif obj.surf(surf_idx).mask > match_idx - obj.surf(surf_idx).mask = obj.surf(surf_idx).mask -1; - end - - if obj.surf(surf_idx).gt == match_idx - obj.surf(surf_idx).gt = []; - elseif obj.surf(surf_idx).gt > match_idx - obj.surf(surf_idx).gt = obj.surf(surf_idx).gt -1; - end - - if obj.surf(surf_idx).quality == match_idx - obj.surf(surf_idx).quality = []; - elseif obj.surf(surf_idx).quality > match_idx - obj.surf(surf_idx).quality = obj.surf(surf_idx).quality -1; - end + if isfield(md,'theta') + obj.theta = md.theta; + else + obj.theta = []; end - - obj.surf = [obj.surf(1:match_idx-1) obj.surf(match_idx+1:end)]; + if isfield(md,'time') + obj.time = md.time; + else + obj.time = []; + end + end - %% adjust_surf - function adjust_surf(obj, dbin, surface_name_string) - % obj.adjust_surf(dbin) - % - % Input: - % dbin: Number of bins to offset the surface - % - % surface_name_string: A cell array of strings that match the name - % of the surface in the surf struct array of the object. If empty - % or undefined, all surfaces will be updated. - % - % Result: - % 1. surf struct .y field will be adjusted by dbin - - if ~exist('surface_name_string','var') - surface_name_string = []; - end - for surf_idx = 1:length(obj.surf) - if isempty(surface_name_string) ... - || ((ischar(surface_name_string) || iscell(surface_name_string)) ... - && any(strcmpi(obj.surf(surf_idx).name,surface_name_string))) - imagesc(obj.surf(surf_idx).y) - colorbar; - obj.surf(surf_idx).name - obj.surf(surf_idx).y(1) - keyboard -% obj.surf(surf_idx).y = obj.surf(surf_idx).y + dbin; - end - end + function obj = set_surf(obj, surf_struct) + %% set_surf + % obj.set_surf(surf_struct) + % + % Input: + % surf_struct: A surf structure. + % + % Result: + % 1. Replace the surf structure if the name of + % surf_struct mathes with another surf struct + % in the surf struct array in the object. + % 2. Throws an error if the surf struct is not valid. + + % check if it is a valid surf structure + obj.valid_surf(surf_struct); + + % Sets an existing surface to the new field values + match_idx = obj.get_index(surf_struct.name,true); + obj.surf(match_idx) = surf_struct; end - %% save - function [] = save_surfdata(obj, fn,doa_method_flag) + function [] = save_surfdata(obj, fn) + %% save_surfdata % obj.save_surfdata(fn) % - % Input: + % Input: % fn: A string that will be the name of the saved surfData file. - % doa_method_flag: logical that specifies is this file stores DOA - % data or beam forming data. Default is false. % % Result: - % Saves the surf struct array in the object + % Saves the surf struct array in the object % as a .mat file. Creates directories as needed. - if ~exist('doa_method_flag','var') - doa_method_flag = false; + % Ensure surface fields are all correct + for surf_idx = 1:length(obj.surf) + obj.valid_surf(obj.surf(surf_idx)); + end + valid_metadata(obj,obj); + + old_units = obj.unit_type; + if ~strcmp(old_units,'standard') + obj.units('standard'); end + surf = []; for surf_idx = 1:length(obj.surf) - obj.valid_surf(obj.surf(surf_idx),doa_method_flag); - end - surf = obj.surf; - version = obj.current_version; - try - surf = rmfield(surf,'h_plot'); - end - valid_metadata(obj,obj,doa_method_flag); - radar_name = obj.radar_name; - season_name = obj.season_name; - day_seg = obj.day_seg; - frm = obj.frm; + surf(surf_idx).name = obj.surf(surf_idx).name; + surf(surf_idx).x = obj.surf(surf_idx).x; + surf(surf_idx).y = obj.surf(surf_idx).y; + surf(surf_idx).plot_name_values = obj.surf(surf_idx).plot_name_values; + surf(surf_idx).visible = obj.surf(surf_idx).visible; + surf(surf_idx).top = obj.surf(surf_idx).top; + surf(surf_idx).active = obj.surf(surf_idx).active; + surf(surf_idx).mask = obj.surf(surf_idx).mask; + surf(surf_idx).gt = obj.surf(surf_idx).gt; + surf(surf_idx).quality = obj.surf(surf_idx).quality; + end + + if ~strcmp(old_units,'standard') + obj.units(old_units); + end + + param = obj.param; + gps_source = obj.gps_source; gps_time = obj.gps_time; - theta = obj.theta; - time = obj.time; - FCS = obj.FCS; + fcs = obj.fcs; fn_dir = fileparts(fn); if ~exist(fn_dir,'dir') mkdir(fn_dir); end - file_version = '1L'; - save(fn, 'surf', 'version', 'gps_time', 'theta', 'time', 'FCS', ... - 'radar_name', 'season_name', 'day_seg', 'frm', 'file_version', '-v7.3'); + file_version = sprintf('%dL', obj.current_version); + file_type = 'surf'; + ct_save(fn, 'fcs', 'file_type', 'file_version', 'gps_source', 'gps_time', ... + 'param', 'surf', '-v7.3'); end - - %% get_names - function surf_names = get_names(obj) - % surf_names = get_names(obj) + + function units(obj, unit_type) + %% units(unit_type) % - % Returns all the surface names in a cell array + % Function for converting units of x and y in surf structure array. % - % No inputs + % unit_type: string 'bins' or 'standard'. Standard is elevation angle in deg and + % twtt in seconds) % - % surf_names: cell array of surface names + % Fields "obj.theta" and "obj.time" must be set before calling this function. - if ~isempty(obj.surf) - surf_names = {obj.surf.name}; - else - surf_names = {}; + if strcmp(unit_type,'standard') && strcmp(obj.unit_type,'bins') + % Switch x,y from elevation angle bins, range bins to elevation angle, twtt + for surf_idx = 1:length(obj.surf) + obj.surf(surf_idx).x = interp1(1:length(obj.theta),obj.theta,obj.surf(surf_idx).x); + if ~all(obj.surf(surf_idx).y(:) == 0 | obj.surf(surf_idx).y(:) == 1) + obj.surf(surf_idx).y = interp1(1:length(obj.time),obj.time,obj.surf(surf_idx).y); + end + end + obj.unit_type = 'standard'; + + elseif strcmp(unit_type,'bins') && strcmp(obj.unit_type,'standard') + % Switch x,y to elevation angle bins, range bins from elevation angle, twtt + for surf_idx = 1:length(obj.surf) + obj.surf(surf_idx).x = interp1(obj.theta,1:length(obj.theta),obj.surf(surf_idx).x); + if ~all(obj.surf(surf_idx).y(:) == 0 | obj.surf(surf_idx).y(:) == 1) + obj.surf(surf_idx).y = interp1(obj.time,1:length(obj.time),obj.surf(surf_idx).y); + end + end + obj.unit_type = 'bins'; end end - %% get_index - function surf_idx = get_index(obj, surf_name, error_on_fail) - % obj.get_index(surf_name, error_on_fail) - % - % Input: - % surf_name: A string that matches the name of a surface in the - % surf struct array in the object OR a cell array of similar - % strings OR an index array of indices into the surf structure - % error_on_fail: Logical scalar indicating whether or not an error - % should be thrown if the surface is not found. Default is false. + function valid_metadata(obj, md) + %% valid_metadata(md) % - % Return: - % 1. The index of that surf struct in the surf - % struct array in the object, if it is found. - % 2. Throws an error if it is not found and error_on_fail is true - % or if an array of indices were input and contain an invalid - % index. + % Function for validating all the metadata fields + % + % md: struct containing gps_time, theta, fcs, param.radar_name, + % param.season_name, param.day_seg, and param.load.frm. - if isnumeric(surf_name) - % Just check that indices are valid - valid_list = intersect(unique(surf_name),1:length(obj.surf)); - if length(valid_list) ~= length(unique(surf_name)) - error('Some of the indices in surf_name are invalid.'); - end - surf_idx = surf_name; - - else - if ~exist('error_on_fail','var') || isempty(error_on_fail) - error_on_fail = false; - end - - if ischar(surf_name) - surf_name = {surf_name}; - elseif ~iscell(surf_name) - error('Input type of surf_name is not valid.'); - end - - for idx = 1:length(surf_name) - new_surf_idx = find(strcmp(surf_name{idx},{obj.surf.name})); - if error_on_fail && isempty(new_surf_idx) - error('Surface "%s" is not in the list of surfaces.', surf_name{idx}); - end - surf_idx(idx) = new_surf_idx; - end + if ~ischar(md.param.day_seg) + error('param.day_seg must be a string'); + end + + if ~isnumeric(md.param.load.frm) + error('parma.load.frm must be a positive integer'); + end + + if ~isa(md.param.radar.lever_arm_fh,'function_handle') + error('param.radar.lever_arm_fh must be a function handle'); + end + + if ~ischar(md.param.radar_name) + error('param.radar_name must be a string'); + end + + if ~isnumeric(md.param.records.gps.time_offset) + error('param.records.gps.time_offset must be a positive integer'); + end + + if ~ischar(md.param.season_name) + error('param.season_name must be a string'); + end + + if ~ischar(md.gps_source) + error('gps_source must be a string'); + end + + if size(md.gps_source,1) ~= 1 + error('gps_source must be a row vector.'); + end + + Nx = size(md.gps_time,2); + + if size(md.gps_time,1) ~= 1 + error('gps_time must be a row vector.'); + end + + if ~isempty(obj.surf) && size(obj.surf(1).x,2) ~= Nx + error('gps_time must have same number of columns as surf.x'); + end + + if size(md.fcs.origin,2) ~= Nx + error('fcs.origin must have same number of columns as gps_time.'); + end + + if size(md.fcs.x,2) ~= Nx + error('fcs.x must have same number of columns as gps_time.'); + end + + if size(md.fcs.y,2) ~= Nx + error('fcs.y must have same number of columns as gps_time.'); + end + + if size(md.fcs.z,2) ~= Nx + error('fcs.z must have same number of columns as gps_time.'); + end + + if size(md.fcs.origin,1) ~= 3 + error('fcs.origin must have 3 rows.'); + end + + if size(md.fcs.x,1) ~= 3 + error('fcs.x must have 3 rows.'); + end + + if size(md.fcs.y,1) ~= 3 + error('fcs.y must have 3 rows.'); + end + + if size(md.fcs.z,1) ~= 3 + error('fcs.z must have 3 rows.'); end + end - - %% valid_surf - function [] = valid_surf(obj, surf_struct,doa_method_flag) + function [] = valid_surf(obj, surf_struct) + %% valid_surf % obj.valid_surf(surf_struct) % % Input: % surf_struct: A surf structure. - % doa_method_flag: logical that specifies is this file stores DOA - % data or beam forming data. Default is false. Field not used. % - % Result: + % Result: % 1. Nothing, if the surf structure is valid. - % 2. Throws an error if any of the condition + % 2. Throws an error if any of the condition % is not satisfied. - if ~exist('doa_method_flag','var') - doa_method_flag = false; - end - % check if it's a structure if ~isstruct(surf_struct) error('The input type is not a structure'); - end + end % check if all the fields exist - if ~isfield(surf_struct, {'x', 'y', 'plot_name_values', ... - 'name', 'top', 'active', ... - 'mask', 'gt', ... - 'quality','visible'}) - error('Struct missing field(s) that are necessary for a surface struct.'); - end + if ~isfield(surf_struct, {'x', 'y', 'plot_name_values', ... + 'name', 'top', 'active', ... + 'mask', 'gt', ... + 'quality','visible'}) + error('Struct missing field(s) that are necessary for a surface struct.'); + end % type check - if ~isa(surf_struct.x, 'double') - error('Invalid field type for the field x (Should be type double)'); - end - - if ~isa(surf_struct.y, 'double') && (~isa(surf_struct.y, 'logical')) - error('Invalid field type for the field y (Should be type double or logical)'); - end - + if ~isa(surf_struct.x, 'double') + error('Invalid field type for the field x (Should be type double)'); + end + + if ~isa(surf_struct.y, 'double') && (~isa(surf_struct.y, 'logical')) + error('Invalid field type for the field y (Should be type double or logical)'); + end + if ~isa(surf_struct.plot_name_values, 'cell') error('Invalid field type for the field plot_name_values (Should be type cell).'); end @@ -657,11 +698,11 @@ function adjust_surf(obj, dbin, surface_name_string) if ~isa(surf_struct.name, 'char') error('Invalid field type for the field name (Should be type char).'); end - + if ~isa(surf_struct.top, 'double') error('Invalid field type for the field top (Should be type double).'); end - + if ~isa(surf_struct.active, 'double') error('Invalid field type for the field active (Should be type double).'); end @@ -673,7 +714,7 @@ function adjust_surf(obj, dbin, surface_name_string) if ~isa(surf_struct.gt, 'double') error('Invalid field type for the field gt (Should be type double).'); end - + if ~isa(surf_struct.quality, 'double') error('Invalid field type for the field quality (Should be type double).'); end @@ -681,11 +722,11 @@ function adjust_surf(obj, dbin, surface_name_string) if ~isa(surf_struct.visible, 'logical') error('Invalid field type for the field visible (Should be type logical).'); end - + % Check that size of x and y match each other - if ~isequal(size(surf_struct.x), size(surf_struct.y)) - error('Size of the field x and field y do not match each other.'); - end + if ~isequal(size(surf_struct.x), size(surf_struct.y)) + error('Size of the field x and field y do not match each other.'); + end % check if all the surface_struct in the object has the same dimension in x and y if ~isempty(obj.surf) && ... ~all(arrayfun(@(arr) isequal(size(surf_struct.x), size(arr.x)), obj.surf)) @@ -732,7 +773,7 @@ function adjust_surf(obj, dbin, surface_name_string) % sd_other: another surfdata object to compare to (can also be this % surfdata object) % other: layer to compare (get_surf's surf_name argument) - % DOA_trim: + % DOA_trim: % % rmse: root mean squared error of difference % mean_diff: mean of the absolute value of the difference @@ -748,23 +789,23 @@ function adjust_surf(obj, dbin, surface_name_string) other = sd_other.get_surf(other); if 1 - quality_mtx_ref = obj.surf(other.quality).y; - quality_mtx_other = sd_other.surf(other.quality).y; - - bad_idx_ref = find(quality_mtx_ref==0); - bad_idx_other = find(quality_mtx_other==0); - - ref.y(bad_idx_ref) = NaN; - other.y(bad_idx_ref) = NaN; - - ref.y(bad_idx_other) = NaN; - other.y(bad_idx_other) = NaN; -% if any(isnan(ref.y(:))) || any(isnan(other.y(:))) -% keyboard -% end + quality_mtx_ref = obj.surf(other.quality).y; + quality_mtx_other = sd_other.surf(other.quality).y; + + bad_idx_ref = find(quality_mtx_ref==0); + bad_idx_other = find(quality_mtx_other==0); + + ref.y(bad_idx_ref) = NaN; + other.y(bad_idx_ref) = NaN; + + ref.y(bad_idx_other) = NaN; + other.y(bad_idx_other) = NaN; + % if any(isnan(ref.y(:))) || any(isnan(other.y(:))) + % keyboard + % end end - surf_diff = abs(other.y(1+DOA_trim(1):end-DOA_trim(end)+1,:) ... + surf_diff = (other.y(1+DOA_trim(1):end-DOA_trim(end)+1,:) ... - ref.y(1+DOA_trim(1):end-DOA_trim(end)+1,:)); rmse = sqrt(nanmean(abs(surf_diff(:)).^2)); mean_diff = nanmean(surf_diff(:)); @@ -772,11 +813,133 @@ function adjust_surf(obj, dbin, surface_name_string) min_diff = nanmin(surf_diff(:)); max_diff = nanmax(surf_diff(:)); end - + + function [rmse,mean_diff,median_diff,min_diff,max_diff,surf_diff] ... + = compare_doa(obj, ref, sd_other, other, DOA_trim,fs) + % [rmse,mean_diff,median_diff,min_diff,max_diff,surf_diff] ... + % = tomo.compare(ref, sd_other, other) + % + % Compares one of the surfaces in this object to a surface in another + % surfdata object. + % + % ref: reference layer (get_surf's surf_name argument) + % sd_other: another surfdata object to compare to (can also be this + % surfdata object) + % other: layer to compare (get_surf's surf_name argument) + % DOA_trim: + % + % rmse: root mean squared error of difference + % mean_diff: mean of the absolute value of the difference + % median_diff: median of the absolute value of the difference + % min_diff: minimum of the absolute value of the difference + % max_diff: maximum of the absolute value of the difference + % surf_diff: ansolute vlaue difference matrix + % + % See also: tomo.run_compare_surfdata, tomo.compare_surfdata, + % tomo.surfdata + last_fprintf_time = -inf; + + ref = obj.get_surf(ref); + other = sd_other.get_surf(other); + + if 1 + quality_mtx_ref = obj.surf(other.quality).y; + quality_mtx_other = sd_other.surf(other.quality).y; + + bad_doa_mask_ref = abs(ref.x) > 80*pi/180; + bad_doa_mask_other = abs(other.x) > 80*pi/180; + + bad_idx_ref = find(quality_mtx_ref==0); + bad_idx_other = find(quality_mtx_other==0); + + ref.y(bad_idx_ref) = NaN; + other.y(bad_idx_ref) = NaN; + ref.y(bad_doa_mask_ref) = NaN; + other.y(bad_doa_mask_other) = NaN; + + ref.y(bad_idx_other) = NaN; + other.y(bad_idx_other) = NaN; + % if any(isnan(ref.y(:))) || any(isnan(other.y(:))) + % keyboard + % end + end + + Td_max = max([max(ref.y(:)), max(other.y(:))]); + Nt = ceil(Td_max*fs); + [Nsv,Nx] = size(ref.y); + + twtt = (0:Nt-1)./fs; + + doa_error = []; + doa_val = []; + for rline = 1:Nx + + if now > last_fprintf_time+60/86400 + fprintf(' Along track: %2.1f (%.0f of %.0f) (%s)\n', rline, rline, Nx, datestr(now)); + last_fprintf_time = now; + last_fprintf_time_bin = now; + end + + ref_theta = nan(Nt,0); + other_theta = nan(Nt,0); + + x1_ref = ref.x(:,rline); + y1_ref = ref.y(:,rline); + + x1_other = other.x(:,rline); + y1_other = other.y(:,rline); + + x1_ref = x1_ref(:); + y1_ref = y1_ref(:).'; + + + x1_other = x1_other(:); + y1_other = y1_other(:).'; + + for rbin_idx = 1:length(twtt) + x2_ref = x1_ref; + y2_ref = twtt(rbin_idx)*ones(size(y1_ref)); + x2_other = x1_other; + y2_other = twtt(rbin_idx)*ones(size(y1_other)); + [bin_theta_ref,y0_ref,~,~] = intersections(x1_ref,y1_ref,x2_ref,y2_ref); + [bin_theta_other,y0_other,~,~] = intersections(x1_other,y1_other,x2_other,y2_other); + + nsrc_ref = length(bin_theta_ref(~isnan(bin_theta_ref))); + nsrc_other = length(bin_theta_other(~isnan(bin_theta_other))); + + if nsrc_ref == nsrc_other & all([nsrc_ref, nsrc_other]) + + good_ref = sort(bin_theta_ref(~isnan(bin_theta_ref))); + good_other = sort(bin_theta_other(~isnan(bin_theta_ref))); + + er_doa = good_ref - good_other; + doa_truth = good_ref; + + doa_error = [doa_error, er_doa(:).']; + doa_val = [doa_val, doa_truth(:).']; + end + + end + + + + % surf_diff = abs(other.y(1+DOA_trim(1):end-DOA_trim(end)+1,:) ... + % - ref.y(1+DOA_trim(1):end-DOA_trim(end)+1,:)); + % rmse = sqrt(nanmean(abs(surf_diff(:)).^2)); + % mean_diff = nanmean(surf_diff(:)); + % median_diff = nanmedian(surf_diff(:)); + % min_diff = nanmin(surf_diff(:)); + % max_diff = nanmax(surf_diff(:)); + end + keyboard + end + end + %% STATIC METHODS ========== methods(Static) function surf = clear_references(surf) + %% clear_references for idx = 1:length(surf) surf(idx).top = []; surf(idx).active = []; @@ -785,17 +948,14 @@ function adjust_surf(obj, dbin, surface_name_string) surf(idx).quality = []; end end - - function surf = empty_surf(fn) - % tomo.surfdata.empty_surf(fn) + + function surf = empty_surf() + %% empty_surf + % tomo.surfdata.empty_surf() % % Returns an empty surf structure - if exist('fn','var') && islogical(fn) && fn - surf.theta = []; - else - surf.x = []; - surf.y = []; - end + surf.x = []; + surf.y = []; surf.plot_name_values = {'color','blue','marker','^'}; surf.name = ''; surf.top = []; @@ -804,151 +964,338 @@ function adjust_surf(obj, dbin, surface_name_string) surf.gt= []; surf.quality = []; surf.visible = true; - end - function surfdata_ver = version(fn) - % tomo.version(fn) + function modify(param,fn,layers,varargin) + %% modify + % tomo.surfdata.modify(param,fn,layers,varargin) % - % Returns the version of the file contained in fn. + % Function for modifying surfData files. Can be used to create new layers + % by specifying a layer that does not already exist. + % + % param: parameter spreadsheet structure + % .radar_name: determines which radar to modify + % .season_name: determines which season to modify + % .day_seg: determines which segment to modify + % .cmd.frms: determines which frames to modify + % fn: filename to ct_filename_out(), leave empty for default 'surfData' + % varargin: arbitrary list of name,value pairs. + % third, fifth, etc. arguments are a string containing the name of the + % property to modify + % fourth, sixth, etc. arguments are the new value to use for the + % corresponding named property + % + % Author: John Paden - tmp = load(fn); - if isfield(tmp,'version') - if tmp.version == 2.0 - surfdata_ver = tmp.version; - return; - else - error('Invalid file version: %d.\n', tmp.version); + % Determine which frames to process + frames = frames_load(param); + + if isempty(param.cmd.frms) + param.cmd.frms = 1:length(frames.frame_idxs); + end + % Remove frames that do not exist from param.cmd.frms list + [valid_frms,keep_idxs] = intersect(param.cmd.frms, 1:length(frames.frame_idxs)); + if length(valid_frms) ~= length(param.cmd.frms) + bad_mask = ones(size(param.cmd.frms)); + bad_mask(keep_idxs) = 0; + warning('Nonexistent frames specified in param.cmd.frms (e.g. frame "%g" is invalid), removing these', ... + param.cmd.frms(find(bad_mask,1))); + param.cmd.frms = valid_frms; + end + + % out_path: output surfData directory + out_path = ct_filename_out(param, fn, 'CSARP_surfData'); + + for frm = param.cmd.frms + fn = fullfile(out_path,sprintf('Data_%s_%03d.mat', param.day_seg, frm)); + + % Load "surf" variable + load(fn); + for layer = layers(:).' + for idx = 1:2:length(varargin) + surf(layer).(varargin{idx}) = varargin{idx+1}; + end end + ct_save(fn,'-append','surf'); + end + end + + function run_update_file() + %% run_update_file + + params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls'),''); + % params = ct_set_params(params,'cmd.generic',0); + params = ct_set_params(params,'cmd.generic',1,'day_seg','20140506_01'); + param_override.cmd.frms = [2 3 4]; + + % params = read_param_xls(ct_filename_param('rds_param_2009_Antarctica_TO.xls')); + % params = ct_set_params(params,'cmd.generic',0); + % params = ct_set_params(params,'cmd.generic',1,'day_seg','20091224_01'); + % param_override.cmd.frms = []; + param_override.update.input = 'surfData_sar'; + param_override.update.output = 'surf_sar'; + param_override.update.echogram = 'standard_air'; + % param_override.update.echogram = 'CSARP_post/standard'; + + % params = read_param_xls(ct_filename_param('rds_param_2019_Antarctica_Ground.xls')); + % params = ct_set_params(params,'cmd.generic',0); + % params = ct_set_params(params,'cmd.generic',1,'day_seg','20200107_01'); + % param_override.cmd.frms = []; + % param_override.update.input = 'surfData_paden'; + % param_override.update.output = 'surfData_paden2'; + % param_override.update.echogram = 'music3D_paden'; + % + global gRadar; + + % Input checking + if exist('param_override','var') + param_override = merge_structs(gRadar,param_override); else - surfdata_ver = 1.0; - return; + param_override = gRadar; end + for param_idx = 1:length(params) + param = params(param_idx); + if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic + continue; + end + + param = merge_structs(param, param_override); + + % Load frames file + frames = frames_load(param); + param.cmd.frms = frames_param_cmd_frms(param,frames); + + for frm = param.cmd.frms + + fn = fullfile(ct_filename_out(param,param.update.input,''),sprintf('Data_%s_%03d.mat',param.day_seg,frm)); + fn_cur_ver = fullfile(ct_filename_out(param,param.update.output,''),sprintf('Data_%s_%03d.mat',param.day_seg,frm)); + echogram_fn = fullfile(ct_filename_out(param,param.update.echogram,''),sprintf('Data_img_01_%s_%03d.mat',param.day_seg,frm)); + % echogram_fn = fullfile(ct_filename_out(param,param.update.echogram,''),sprintf('Data_%s_%03d.mat',param.day_seg,frm)); + fprintf('Update\n %s\n %s\n', fn, echogram_fn); + tomo.surfdata.update_file(fn,fn_cur_ver,echogram_fn); + end + + end end function surf = update_file(fn,fn_new,echogram_fn) - % tomo.update_file(fn,fn_new,echogram_fn) + %% update_file + % surf = tomo.surfdata.update_file(fn,fn_new,echogram_fn) % % Updates filename fn to latest version of surfdata and stores the % result in fn_new. The updated surf structure is returned. % - % echogram_fn: the echoram filename is only required when converting - % from version 1.0 files. The echogram filename fields must match - % the surfData file or the updating will not work properly. + % echogram_fn: the echogram filename is only required when converting + % from version 1.0 and 2.0 files to the newest format. The echogram + % filename fields must match the surfData file or the updating will + % not work properly. fn_version = tomo.surfdata.version(fn); if fn_version == 1.0 - updated = load(fn); - for surf_idx = 1:length(updated.surf) - if strcmpi(updated.surf(surf_idx).name,'ice surface') - updated.surf(surf_idx).name = 'top'; - elseif strcmpi(updated.surf(surf_idx).name,'surface gt') - updated.surf(surf_idx).name = 'top gt'; - elseif strcmpi(updated.surf(surf_idx).name,'surface quality') - updated.surf(surf_idx).name = 'top quality'; + %% update_file: v1.0 + if ~isempty(whos('-file',echogram_fn,'param_array')) + tmp = load(echogram_fn,'param_array','param_records','GPS_time'); + tmp.param = tmp.param_array; + else + tmp = load(echogram_fn,'param_combine','param_records','GPS_time'); + tmp.param = tmp.param_combine; + tmp.param.radar.lever_arm_fh = tmp.param.csarp.lever_arm_fh; + tmp.param_records.records.gps.time_offset = tmp.param_records.vectors.gps.time_offset; + tmp.param.array_param.fcs.origin = tmp.param_combine.array_param.fcs{1}{1}.origin; + tmp.param.array_param.fcs.x = tmp.param_combine.array_param.fcs{1}{1}.x; + tmp.param.array_param.fcs.y = tmp.param_combine.array_param.fcs{1}{1}.y; + tmp.param.array_param.fcs.z = tmp.param_combine.array_param.fcs{1}{1}.z; + end + + surf_old = load(fn); + surf_new = []; + + surf_new.fcs.origin = tmp.param.array_param.fcs.origin; + surf_new.fcs.x = tmp.param.array_param.fcs.x; + surf_new.fcs.y = tmp.param.array_param.fcs.y; + surf_new.fcs.z = tmp.param.array_param.fcs.z; + + surf_new.gps_source = tmp.param_records.gps_source; + + surf_new.gps_time = tmp.GPS_time; + + surf_new.param.day_seg = surf_old.day_seg; + surf_new.param.load.frm = surf_old.frm; + surf_new.param.radar.lever_arm_fh = tmp.param.radar.lever_arm_fh; + surf_new.param.radar_name = surf_old.radar_name; + surf_new.param.records.gps.time_offset = tmp.param_records.records.gps.time_offset; + surf_new.param.season_name = surf_old.season_name; + surf_new.param.sw_version = current_software_version(); + + % Convert x,y from elevation angle bins, range bins to elevation angle, twtt + for surf_idx = 1:length(surf_new.surf) + surf_new.surf(surf_idx).active = surf_old.surf(surf_idx).active_layer; + surf_new.surf(surf_idx).gt = surf_old.surf(surf_idx).control_layer; + surf_new.surf(surf_idx).mask = surf_old.surf(surf_idx).mask_layer; + if strcmpi(surf_old.surf(surf_idx).name,'ice surface') + surf_new.surf(surf_idx).name = 'top'; + elseif strcmpi(surf_old.surf(surf_idx).name,'surface gt') + surf_new.surf(surf_idx).name = 'top gt'; + elseif strcmpi(surf_old.surf(surf_idx).name,'surface quality') + surf_new.surf(surf_idx).name = 'top quality'; + else + surf_new.surf(surf_idx).name = surf_old.surf(surf_idx).name; end - updated.surf(surf_idx).top = updated.surf(surf_idx).surf_layer; - updated.surf(surf_idx).active = updated.surf(surf_idx).active_layer; - updated.surf(surf_idx).mask = updated.surf(surf_idx).mask_layer; - updated.surf(surf_idx).gt = updated.surf(surf_idx).control_layer; - updated.surf(surf_idx).quality = updated.surf(surf_idx).quality_layer; - if ischar(updated.surf(surf_idx).visible) - if strcmpi(updated.surf(surf_idx).visible,'on') - updated.surf(surf_idx).visible = true; + surf_new.surf(surf_idx).plot_name_values = {}; + surf_new.surf(surf_idx).quality = surf_old.surf(surf_idx).quality_layer; + surf_new.surf(surf_idx).top = surf_old.surf(surf_idx).surf_layer; + if ischar(surf_old.surf(surf_idx).visible) + if strcmpi(surf_old.surf(surf_idx).visible,'on') + surf_old.surf(surf_idx).visible = true; else - updated.surf(surf_idx).visible = false; + surf_old.surf(surf_idx).visible = false; end end + surf_new.surf(surf_idx).x = interp1(1:length(surf_old.theta),surf_old.theta,surf_new.surf(surf_idx).x); + if all(surf_new.surf(surf_idx).y(:) == 0 | surf_new.surf(surf_idx).y(:) == 1) + surf_new.surf(surf_idx).y = surf_new.surf(surf_idx).y; + else + surf_new.surf(surf_idx).y = interp1(1:length(surf_old.time),surf_old.time,surf_new.surf(surf_idx).y); + end + end - updated.surf = rmfield(updated.surf,{'surf_layer','active_layer','mask_layer','control_layer','quality_layer'}); - updated.version = 2.0; - - tmp = load(echogram_fn,'GPS_time','theta','Time','param_combine'); - updated.radar_name = tmp.param_combine.radar_name; - updated.season_name = tmp.param_combine.season_name; - updated.day_seg = tmp.param_combine.day_seg; - updated.frm = tmp.param_combine.combine.frm; - updated.gps_time = tmp.GPS_time; - updated.theta = tmp.theta(:); % Make a column vector - updated.time = tmp.Time(:); % Make a column vector - updated.FCS.origin = tmp.param_combine.array_param.fcs{1}{1}.origin; - updated.FCS.x = tmp.param_combine.array_param.fcs{1}{1}.x; - updated.FCS.y = tmp.param_combine.array_param.fcs{1}{1}.y; - updated.FCS.z = tmp.param_combine.array_param.fcs{1}{1}.z; + surfdata_updated = true; elseif fn_version == 2.0 - updated = load(fn); + %% update_file: v2.0 + if ~isempty(whos('-file',echogram_fn,'param_array')) + tmp = load(echogram_fn,'param_array','param_records'); + tmp.param = tmp.param_array; + else + tmp = load(echogram_fn,'param_combine','param_records'); + tmp.param = tmp.param_combine; + tmp.param.radar.lever_arm_fh = tmp.param.csarp.lever_arm_fh; + tmp.param_records.records.gps.time_offset = tmp.param_records.vectors.gps.time_offset; + end + + surf_old = load(fn); + surf_new = []; + + surf_new.fcs = surf_old.FCS; + + surf_new.gps_source = tmp.param_records.gps_source; + + surf_new.gps_time = surf_old.gps_time; + + surf_new.param.day_seg = surf_old.day_seg; + surf_new.param.load.frm = surf_old.frm; + surf_new.param.radar.lever_arm_fh = tmp.param.radar.lever_arm_fh; + surf_new.param.radar_name = surf_old.radar_name; + surf_new.param.records.gps.time_offset = tmp.param_records.records.gps.time_offset; + surf_new.param.season_name = surf_old.season_name; + surf_new.param.sw_version = current_software_version(); + surf_new.surf = surf_old.surf; + + % Convert x,y from elevation angle bins, range bins to elevation angle, twtt + for surf_idx = 1:length(surf_new.surf) + surf_new.surf(surf_idx).x = interp1(1:length(surf_old.theta),surf_old.theta,surf_new.surf(surf_idx).x); + if all(surf_new.surf(surf_idx).y(:) == 0 | surf_new.surf(surf_idx).y(:) == 1) + surf_new.surf(surf_idx).y = surf_new.surf(surf_idx).y; + elseif all(~isfinite(surf_new.surf(surf_idx).y(:)) | surf_new.surf(surf_idx).y(:) < 1 ) + % Assume y-values are twtt (this should never happen) + surf_new.surf(surf_idx).y = surf_new.surf(surf_idx).y; + else + surf_new.surf(surf_idx).y = interp1(1:length(surf_old.time),surf_old.time,surf_new.surf(surf_idx).y); + end + end + surfdata_updated = true; + + elseif fn_version == 3.0 + surf_new = load(fn); + surf_new.param.sw_version = current_software_version(); + surfdata_updated = false; + + % Ensure x,y converted from elevation angle bins, range bins to elevation angle, twtt + for surf_idx = 1:length(surf_new.surf) + if any(surf_new.surf(surf_idx).y(isfinite(surf_new.surf(surf_idx).y)) > 1) + warning('HACK: Fixing temporary file format. Requires echogram_fn input argument. This will overwrite the existing surfdata file. Run "dbcont" to continue or "dbquit" to cancel.'); + keyboard; + if ~surfdata_updated + mdata = load(echogram_fn,'Time','Tomo'); + surfdata_updated = true; + end + if any(surf_new.surf(surf_idx).x(:) > pi) + surf_new.surf(surf_idx).x = interp1(1:length(mdata.Tomo.theta(:,1)),mdata.Tomo.theta(:,1),surf_new.surf(surf_idx).x); + end + if all(surf_new.surf(surf_idx).y(:) == 0 | surf_new.surf(surf_idx).y(:) == 1) + surf_new.surf(surf_idx).y = surf_new.surf(surf_idx).y; + else + surf_new.surf(surf_idx).y = interp1(1:length(mdata.Time),mdata.Time,surf_new.surf(surf_idx).y); + end + end + end else - error('Invalid file version: %d.\n', fn_version); + error('Invalid file version: %g.\n', fn_version); end if nargin >= 2 - obj = tomo.surfdata; - obj.load_surfdata(updated); - obj.save_surfdata(fn_new); + surf = tomo.surfdata(surf_new); + if ~strcmp(fn,fn_new) || surfdata_updated + fprintf(' Saving updated file: %s\n', fn_new); + surf.save_surfdata(fn_new); + end end - surf = updated.surf; - end - function [] = run_add_surf_from_dem() - %% run add surf from dem - % script run_add_surf_from_dem - % - % script for running tomo.surfdata.add_surf_from_dem - % - % Author: Theresa Moore + function surfdata_ver = version(fn) + %% version + % tomo.surfdata.version(fn) % - % See also: tomo.surfdata - + % Returns the version of the file contained in fn. - % User Setup - % ========================================================================= - param_override = []; - params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls')); - params = ct_set_params(params,'cmd.generic',0); - params = ct_set_params(params,'cmd.generic',1,'day_seg','20140506_01'); - params = ct_set_params(params,'cmd.frms',[2 3 4]); -% params = ct_set_params(params,'add_surf_from_dem.ice_mask_fn',fullfile('greenland','IceMask','GimpIceMask_90m_v1.1.tif'));%'antarctica\DEM\BEDMAP2\original_data\bedmap2_tiff\bedmap2_icemask_grounded_and_shelves.tif'; - params = ct_set_params(params,'add_surf_from_dem.ice_mask_fn','canada/ice_mask/03_rgi50_ArcticCanadaNorth/03_rgi50_ArcticCanadaNorth.mat');%'antarctica\DEM\BEDMAP2\original_data\bedmap2_tiff\bedmap2_icemask_grounded_and_shelves.tif'; -% params = ct_set_params(params,'add_surf_from_dem.ice_mask_fn',ct_filename_gis(params,fullfile('greenland','IceMask','GimpIceMask_90m_v1.1.tif')));%'antarctica\DEM\BEDMAP2\original_data\bedmap2_tiff\bedmap2_icemask_grounded_and_shelves.tif'; -% param.add_surf_from_dem.ice_mask_fn = ct_filename_gis(param,fullfile('greenland','IceMask','GimpIceMask_90m_v1.1.tif'));%'antarctica\DEM\BEDMAP2\original_data\bedmap2_tiff\bedmap2_icemask_grounded_and_shelves.tif'; -% param.add_surf_from_dem.ice_mask_fn = ct_filename_gis([],'canada/ice_mask/03_rgi50_ArcticCanadaNorth/03_rgi50_ArcticCanadaNorth.mat'); - - params = ct_set_params(params, 'add_surf_from_dem.dem_guard', 30e3); -% params = ct_set_params(params, 'add_surf_from_dem.dem_per_slice_guard', 500e3); + tmp = whos('-file',fn); + file_type_idx = find(strcmp('file_type',{tmp.name})); + if ~isempty(file_type_idx) + load(fn,'file_type'); + if ~strcmpi(file_type,'surf') + error('fn is not a surf file. fn: %s', fn); + end + end - % Automated Section - % ========================================================================= - % Input checking - global gRadar; - if exist('param_override','var') - param_override = merge_structs(gRadar,param_override); - else - param_override = gRadar; + version_idx = find(strcmp('version',{tmp.name})); + if ~isempty(version_idx) + load(fn,'version'); + if version == 2.0 + surfdata_ver = version; + return; + else + error('Invalid file version: %d.\n', version); + end end - % Process each of the segments - ctrl_chain = {}; - for param_idx = 1:length(params) - param = params(param_idx); - if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic - continue; + file_version_idx = find(strcmp('file_version',{tmp.name})); + if ~isempty(file_version_idx) + load(fn,'file_version'); + if file_version(1) == '3' + surfdata_ver = 3.0; + return; + else + error('Invalid file version: %s.\n', file_version); end - tomo.surfdata.add_surf_from_dem(param,param_override); end + + surfdata_ver = 1.0; + end function surf = add_surf_from_dem(param,param_override) - %% add surf from dem - % tomo.add_surf_from_dem(param,param_override) + %% add_surf_from_dem + % tomo.surfdata.add_surf_from_dem(param,param_override) % % param: struct with processing parameters or function handle to % script with processing parameters + % .add_surf_from_dem.delta_at = along track decimation factor + % .add_surf_from_dem.theta_vec + % .add_surf_from_dem.dem_res % % param_override: parameters in this struct override parameters in % param. This struct must also contain the gRdar fields. @@ -978,7 +1325,7 @@ function adjust_surf(obj, dbin, surface_name_string) % ================================================================= % Load frames file - load(ct_filename_support(param, '', 'frames')); + frames = load(ct_filename_support(param, '', 'frames')); % If no frames specified, then do all frames if isempty(param.cmd.frms) @@ -995,29 +1342,32 @@ function adjust_surf(obj, dbin, surface_name_string) param.cmd.frms = valid_frms; end - if ~isfield(param,'add_surf_from_dem') param.add_surf_from_dem = []; end + if ~isfield(param.add_surf_from_dem,'method') || isempty(param.add_surf_from_dem.method) + error('No valid method. User must specify either ''sar'' or ''surf''.'); + end + if ~isfield(param.add_surf_from_dem,'dem_res') || isempty(param.add_surf_from_dem.dem_res) - param.add_surf_from_dem.dem_res = 10; + param.add_surf_from_dem.dem_res = 10; % Arctic DEM resolution end - if ~isfield(param.add_surf_from_dem,'theta_vec') || isempty(param.add_surf_from_dem.theta_vec) - param.add_surf_from_dem.theta_vec = -85:85; + if ~isfield(param.add_surf_from_dem,'theta_bins') || isempty(param.add_surf_from_dem.theta_bins) + param.add_surf_from_dem.theta_bins = [-88:88].'; end if ~isfield(param.add_surf_from_dem,'dem_guard') || isempty(param.add_surf_from_dem.dem_guard) param.add_surf_from_dem.dem_guard = 12e3; end - if ~isfield(param.add_surf_from_dem,'ice_mask_fn') || isempty(param.add_surf_from_dem.ice_mask_fn) - param.add_surf_from_dem.ice_mask_fn = []; + if ~isfield(param.add_surf_from_dem,'delta_at') || isempty(param.add_surf_from_dem.delta_at) + param.add_surf_from_dem.delta_at = 10; end - if ~isfield(param.add_surf_from_dem,'sv_cal_fn') || isempty(param.add_surf_from_dem.sv_cal_fn) - param.add_surf_from_dem.sv_cal_fn = [ct_filename_ct_tmp(param,'','add_surf_from_dem','theta_cal') '.mat']; + if ~isfield(param.add_surf_from_dem,'ice_mask_fn') || isempty(param.add_surf_from_dem.ice_mask_fn) + error('User must specify an ice mask') end % .dem_per_slice_guard: additional region in meters around each slice to search for DEM points @@ -1031,90 +1381,152 @@ function adjust_surf(obj, dbin, surface_name_string) param.add_surf_from_dem.surfdata_mode.overwrite = true; end - - if ~isfield(param.add_surf_from_dem,'surf_out_path') || isempty(param.add_surf_from_dem.surf_out_path) - param.add_surf_from_dem.surf_out_path = ''; - end + if regexp(param.add_surf_from_dem.method,'surf') + % Output path is equal to the input path + param.add_surf_from_dem.surf_out_path = param.add_surf_from_dem.in_path; + param.add_surf_from_dem.delta_at = 1; + param.add_surf_from_dem.surfdata_mode.overwrite = true; + + if ~isfield(param.add_surf_from_dem,'ice_mask_surf_name') || isempty(param.add_surf_from_dem.ice_mask_surf_name) + param.add_surf_from_dem.ice_mask_surf_name = 'ice mask'; + end - % Set doa_method_flag to false (always) - doa_method_flag = false; + if ~isfield(param.add_surf_from_dem,'ref_surf_name') || isempty(param.add_surf_from_dem.ref_surf_name) + param.add_surf_from_dem.ref_surf_name = 'top'; + end + + elseif regexp(param.add_surf_from_dem.method,'sar') + if ~isfield(param.add_surf_from_dem,'surf_out_path') || isempty(param.add_surf_from_dem.surf_out_path) + param.add_surf_from_dem.surf_out_path = 'surf_sar'; + end + end - - %% Setup processing + + + %% add_surf_from_dem: Setup % ===================================================================== - % Load in sar coordinates file - % Currently supports default path only -- FIX THIS?? - sar_coord_fn = fullfile(ct_filename_out(param,'sar',''),'sar_coord.mat'); - sar_coord = load(sar_coord_fn); - - % Load records file - records_fn = ct_filename_support(param,'','records'); - records = load(records_fn); + if regexp(param.add_surf_from_dem.method,'sar') + % Load in sar coordinates file + sar_coord_fn = fullfile(ct_filename_out(param,param.add_surf_from_dem.in_path,''),'sar_coord.mat') + sar_coord = load(sar_coord_fn); + + % Load records file + records = records_load(param); + + % Load frames + frames = frames_load(param); + end + + % Input and output directories + out_dir = ct_filename_out(param,param.add_surf_from_dem.surf_out_path); + in_dir = ct_filename_out(param,param.add_surf_from_dem.in_path); + % Physical constants + physical_constants; % Loop over frames for frm_idx = 1:length(param.cmd.frms) + frm = param.cmd.frms(frm_idx); - fprintf('Creating surface data object for %s_%03d \n', param.day_seg,frm); +% fprintf('Creating surface data object for %s_%03d \n', param.day_seg,frm); - % recs: Determine the records for this frame - if frm < length(frames.frame_idxs) - recs = [frames.frame_idxs(frm), frames.frame_idxs(frm+1)-1]; - else - recs = [frames.frame_idxs(frm), length(records.gps_time)]; + if strcmp(param.add_surf_from_dem.method,'sar') + + fprintf('Creating surf sar data object for %s_%03d \n', param.day_seg,frm); + + % recs: Determine the records for this frame + if frm < length(frames.frame_idxs) + recs = [frames.frame_idxs(frm), frames.frame_idxs(frm+1)-1]; + else + recs = [frames.frame_idxs(frm), length(records.gps_time)]; + end + + % Find gps at frame boundaries and use this to mask out FCS of a + % frame + frm_gps_start = records.gps_time(recs(1)); + frm_gps_stop = records.gps_time(recs(end)); + + % Mask used to isolate FCS for a single frame + gps_mask = sar_coord.gps_time >= frm_gps_start & ... + sar_coord.gps_time <= frm_gps_stop; + + % GPS time, origin, Xpos, Ypos, Zpos for the frame + frm_gps_time = sar_coord.gps_time(gps_mask); + frm_origin = sar_coord.origin(:,gps_mask); + frm_x = sar_coord.x(:,gps_mask); + frm_z = sar_coord.z(:,gps_mask); + frm_y = cross(frm_z,frm_x); + + % Create the surfdata structure with necessary fields + delta_at = param.add_surf_from_dem.delta_at; + + % Create surface data structure with necessary fields + sd = []; + sd.fcs.x = frm_x(:,1:delta_at:end); + sd.fcs.z = frm_z(:,1:delta_at:end); + sd.fcs.y = frm_y(:,1:delta_at:end); + sd.fcs.origin = frm_origin(:,1:delta_at:end); + sd.gps_time = frm_gps_time(1:delta_at:end); + sd.gps_source = sar_coord.gps_source; + sd.param.day_seg = param.day_seg; + sd.param.load.frm = frm; + sd.param.radar.lever_arm_fh = param.radar.lever_arm_fh; + sd.param.radar_name = param.radar_name; + sd.param.records.gps.time_offset = param.records.gps.time_offset; + sd.param.season_name = param.season_name; + sd.param.sw_version = current_software_version; + sd.file_type = 'surf'; + sd.surf(1) = tomo.surfdata.empty_surf(); + sd.surf(2) = tomo.surfdata.empty_surf(); + sd = tomo.surfdata(sd,param.add_surf_from_dem.surf_out_path); + + elseif strcmp(param.add_surf_from_dem.method,'surf') + gps_time = []; + fcs = []; + surf_fn_name = sprintf('Data_%s_%03.0f.mat',param.day_seg,frm); + surf_fn = fullfile(in_dir,surf_fn_name); + + fprintf('Updating file with ice mask: %s \n', surf_fn); + + load(surf_fn,'fcs','gps_time') + sar_coord = fcs; + sd = []; + sd = tomo.surfdata(surf_fn); + + ice_mask_idx = []; + if ~isempty(find(strcmp(param.add_surf_from_dem.ice_mask_surf_name,{sd.surf.name}))) + ice_mask_idx = sd.get_index(param.add_surf_from_dem.ice_mask_surf_name); + else + ice_mask_idx = []; + end + + if ~isempty(find(strcmp(param.add_surf_from_dem.ref_surf_name,{sd.surf.name}))) + ref_surf_idx = sd.get_index(param.add_surf_from_dem.ref_surf_name); + param.add_surf_from_dem.theta_bins = sd.surf(ref_surf_idx).x(:,1); + else + ref_surf_idx = []; + param.add_surf_from_dem.theta_bins = sd.surf(ice_mask_idx).x(:,1); + end + + if isempty(ice_mask_idx) + error('Surfdata does not contain an ice mask surface layer') + end + +% param.add_surf_from_dem.theta_bins = sd.surf(ref_surf_idx).x(:,1); end - - - % The following is for Debug only - % res(2) = recs(1) + 20000; - - % Find gps at frame boundaries and use this to mask out FCS of a - % frame - frm_gps_start = records.gps_time(recs(1)); - frm_gps_stop = records.gps_time(recs(end)); - - % Mask used to isolate FCS for a single frame - gps_mask = sar_coord.gps_time >= frm_gps_start & ... - sar_coord.gps_time <= frm_gps_stop; - - % GPS time, origin, Xpos, Ypos, Zpos for the frame - frm_gps_time = sar_coord.gps_time(gps_mask); - frm_origin = sar_coord.origin(:,gps_mask); - frm_x = sar_coord.x(:,gps_mask); - frm_z = sar_coord.z(:,gps_mask); - frm_y = cross(frm_z,frm_x); - - % Along track decimation factor - delta_at = 10; - - % Create surfdata - sd = tomo.surfdata(); - sd.radar_name = param.radar_name; - sd.season_name = param.season_name; - sd.day_seg = param.day_seg; - sd.frm = frm; - sd.gps_time = frm_gps_time(1:delta_at:end); - sd.FCS.origin = frm_origin(:,1:delta_at:end); - sd.FCS.x = frm_x(:,1:delta_at:end); - sd.FCS.z = frm_z(:,1:delta_at:end); - sd.FCS.y = frm_y(:,1:delta_at:end); - sd.theta = param.add_surf_from_dem.theta_vec(:); - sd.time = zeros(0,1); - clear frm_gps_time frm_origin frm_x frm_y frm_z - %% Load Geotiff and Ice Mask + %% add_surf_from_dem: Geotiff and Ice Mask % ================================================================= - global gdem; if isempty(gdem) || ~isa(gdem,'dem_class') || ~isvalid(gdem) gdem = dem_class(param,param.add_surf_from_dem.dem_res); end gdem.set_res(param.add_surf_from_dem.dem_res); - % Load ice mask + % Load ice mask if isfield(param.add_surf_from_dem,'ice_mask_fn') && ~isempty(param.add_surf_from_dem.ice_mask_fn) ice_mask_fn = ct_filename_gis(param,param.add_surf_from_dem.ice_mask_fn); [~,ice_mask_fn_name,ice_mask_fn_ext] = fileparts(ice_mask_fn); @@ -1130,271 +1542,472 @@ function adjust_surf(obj, dbin, surface_name_string) ice_mask_all = []; end - % sv_cal_fn: steering vector calibration filename - sv_cal_fn = param.add_surf_from_dem.sv_cal_fn; - if ~exist(sv_cal_fn,'file') - sv_cal_fn = []; - end - - - %% Create DEM - % Convert decimated origin coordinates from ECEF to geodetic and - % use these to define boundaries of the dem - Nx = size(sd.FCS.x,2); - dec_idxs = round(linspace(1,Nx,min(Nx,200))); - [frm_lat_dec, frm_lon_dec, frm_elev_dec] = ecef2geodetic(sd.FCS.origin(1,dec_idxs),sd.FCS.origin(2,dec_idxs), sd.FCS.origin(3,dec_idxs),WGS84.ellipsoid); - frm_lat_dec = frm_lat_dec*180/pi; - frm_lon_dec = frm_lon_dec*180/pi; - - [latb,lonb] = bufferm(frm_lat_dec,frm_lon_dec,param.add_surf_from_dem.dem_guard/WGS84.semimajor*180/pi); - gdem_str = sprintf('%s:%s:%s_%03d',param.radar_name,param.season_name,param.day_seg,frm); - - if ~strcmpi(gdem_str,gdem.name) - gdem.set_vector(latb,lonb,gdem_str); - end - - % Generated a non-decimated version of the lat,lon for the frame - - % used below - [frm_lat, frm_lon, frm_elev] = ecef2geodetic(sd.FCS.origin(1,:),sd.FCS.origin(2,:), sd.FCS.origin(3,:),WGS84.ellipsoid); - frm_lat = frm_lat * 180/pi; - frm_lon = frm_lon * 180/pi; - - gdem.set_vector(latb,lonb,gdem_str); - [DEM,msl,ocean_mask,proj,DEM_x,DEM_y] = gdem.get_vector_mosaic(100); - DEM(ocean_mask) = msl(ocean_mask); - [frm_x,frm_y] = projfwd(proj,frm_lat,frm_lon); - - %% Interpolate at all of the bad value locations using the good data - bad_idxs = find(isnan(DEM)); - good_idxs = find(~isnan(DEM)); - x_idxs = repmat(1:size(DEM,2),[size(DEM,1) 1]); - y_idxs = repmat((1:size(DEM,1))',[1 size(DEM,2)]); - x_vals = x_idxs(good_idxs); - y_vals = y_idxs(good_idxs); - z_vals = DEM(good_idxs); - x_out = x_idxs(bad_idxs); - y_out = y_idxs(bad_idxs); - z_out = single(griddata(x_vals,y_vals,double(z_vals),x_out,y_out)); - if ~isempty(z_out) - DEM(bad_idxs) = z_out; - end - - if 0 - figure(1);clf; - imagesc(DEM_x,DEM_y,DEM); - hold on; - plot(frm_x,frm_y,'r'); - keyboard; - end - - DEM_x_mesh = repmat(DEM_x,[size(DEM,1) 1]); - DEM_y_mesh= repmat(DEM_y,[1 size(DEM,2)]); - - - %% For every position along the aperture, add theta dependent TWTT to DEM from origin of the FCS - Nx = size(sd.FCS.x,2); - Nsv = length(param.add_surf_from_dem.theta_vec); - twtt = zeros(Nsv,Nx); - ice_mask = NaN(Nsv,Nx); - - % Handle case where the DEM is all NaNs - if all(all(isnan(DEM))) - warning('Input DEM contains all NaN data for Frame %d.',param.proc.frm); - twtt(:,:) = NaN; - Nx = 0; - end - - DEM_coverage_warning = false; - - for rline = 1:Nx - if ~mod(rline-1,10^floor(log10(Nx)-1)) - fprintf(' %s %d of %d (%s)\n', mfilename, rline, Nx, datestr(now)); + if strcmp(param.add_surf_from_dem.method,'surf') && ~isempty(ref_surf_idx) + y_active = sin(sd.surf(ref_surf_idx).x) .* sd.surf(ref_surf_idx).y * c/2; + z_active = -cos(sd.surf(ref_surf_idx).x) .* sd.surf(ref_surf_idx).y * c/2; + + % Convert from radar FCS to ECEF + x_plane = zeros(size(y_active)); + y_plane = zeros(size(y_active)); + z_plane = zeros(size(y_active)); + for rline = 1:size(y_active,2) + x_plane(:,rline) = sd.fcs.origin(1,rline) ... + + sd.fcs.y(1,rline) * y_active(:,rline) ... + + sd.fcs.z(1,rline) * z_active(:,rline); + y_plane(:,rline) = sd.fcs.origin(2,rline) ... + + sd.fcs.y(2,rline) * y_active(:,rline) ... + + sd.fcs.z(2,rline) * z_active(:,rline); + z_plane(:,rline) = sd.fcs.origin(3,rline) ... + + sd.fcs.y(3,rline) * y_active(:,rline) ... + + sd.fcs.z(3,rline) * z_active(:,rline); end - dem_guard = param.add_surf_from_dem.dem_guard; + % Convert from ECEF to geodetic + [points.lat,points.lon,points.elev] = ecef2geodetic(x_plane,y_plane,z_plane,WGS84.ellipsoid); + points.lat = points.lat * 180/pi; + points.lon = points.lon * 180/pi; - DEM_mask = DEM_x_mesh > frm_x(rline)-dem_guard & DEM_x_mesh < frm_x(rline)+dem_guard ... - & DEM_y_mesh > frm_y(rline)-dem_guard & DEM_y_mesh < frm_y(rline)+dem_guard ... - & ~isnan(DEM); - DEM_idxs = find(DEM_mask); + % Convert from geodetic to projection + [points.x,points.y] = projfwd(ice_mask_all.proj,points.lat,points.lon); + intersection_x = interp_finite(points.x); + intersection_y = interp_finite(points.y); - if numel(DEM_idxs)==0 - warning('Range Line %d of Frame %d is not spanned by DEM.',rline,frm); - end + x_mesh = repmat(ice_mask_all.X,[size(ice_mask_all.mask,1) 1]); + y_mesh= repmat(ice_mask_all.Y(:),[1 size(ice_mask_all.mask,2)]); +% +% ice_mask_aligned = zeros(size(points.x)); +% ice_mask_aligned(nan_mask) = nan; +% ice_mask_q = interp2(x_mesh,y_mesh,ice_mask_all.mask,intersection_x_good, intersection_y_good,'nearest'); +% ice_mask_aligned(~nan_mask) = ice_mask_q; +% ice_mask = ice_mask_aligned; +% twtt = sd.surf(ref_surf_idx).y; + ice_mask_eval = double(ice_mask_all.mask); - if 0 - set(h_img,'AlphaData',DEM_mask); - end + ice_mask_q = interp2(x_mesh,y_mesh, ice_mask_eval, intersection_x, intersection_y,'nearest'); + nan_mask = isnan(ice_mask_q); + ice_mask_q(nan_mask) = 1; + ice_mask = logical(round(ice_mask_q)); + + twtt = sd.surf(ref_surf_idx).y; + else - % Convert from projection to geodetic (lat,lon,elev) - [DEM_lat,DEM_lon] = projinv(proj,DEM_x_mesh(DEM_idxs),DEM_y_mesh(DEM_idxs)); - DEM_elev = DEM(DEM_idxs); + %% add_surf_from_dem: DEM + % Convert decimated origin coordinates from ECEF to geodetic and + % use these to define boundaries of the dem + Nx = size(sd.fcs.x,2); + dec_idxs = round(linspace(1,Nx,min(Nx,200))); + [frm_lat_dec, frm_lon_dec, frm_elev_dec] = ecef2geodetic(sd.fcs.origin(1,dec_idxs),sd.fcs.origin(2,dec_idxs), sd.fcs.origin(3,dec_idxs),WGS84.ellipsoid); + frm_lat_dec = frm_lat_dec*180/pi; + frm_lon_dec = frm_lon_dec*180/pi; - % Convert from geodetic (lat,lon,elev) to ECEF (x,y,z) - physical_constants; - [DEM_ecef_x,DEM_ecef_y,DEM_ecef_z] = geodetic2ecef(single(DEM_lat)/180*pi,single(DEM_lon)/180*pi,single(DEM_elev),WGS84.ellipsoid); + [latb,lonb] = bufferm(frm_lat_dec,frm_lon_dec,param.add_surf_from_dem.dem_guard/WGS84.semimajor*180/pi); + gdem_str = sprintf('%s:%s:%s_%03d',param.radar_name,param.season_name,param.day_seg,frm); - origin = sd.FCS.origin(:,rline); + if ~strcmpi(gdem_str,gdem.name) + gdem.set_vector(latb,lonb,gdem_str); + end + + % Generated a non-decimated version of the lat,lon for the frame - + % used below. Geocode platform positions after SAR and project + % points to align with arctic dem + [frm_lat, frm_lon, frm_elev] = ecef2geodetic(sd.fcs.origin(1,:),sd.fcs.origin(2,:), sd.fcs.origin(3,:),WGS84.ellipsoid); + frm_lat = frm_lat * 180/pi; + frm_lon = frm_lon * 180/pi; - % Define matrices to change coordinate systems from FCS to ECEF - % and vice versa - Tfcs_ecef = [sd.FCS.x(:,rline), sd.FCS.y(:,rline), sd.FCS.z(:,rline)]; - Tecef_fcs = inv(Tfcs_ecef); + gdem.set_vector(latb,lonb,gdem_str); + [DEM,msl,ocean_mask,proj,DEM_x,DEM_y] = gdem.get_vector_mosaic(100); + DEM(ocean_mask) = msl(ocean_mask); + [frm_x,frm_y] = projfwd(proj,frm_lat,frm_lon); - % Convert DEM coordinates from ECEF to FCS - tmp = Tecef_fcs * [DEM_ecef_x.'-origin(1); DEM_ecef_y.'-origin(2); DEM_ecef_z.'-origin(3)]; - DEM_fcs_x = tmp(1,:); - DEM_fcs_y = tmp(2,:); - DEM_fcs_z = tmp(3,:); + %% add_surf_from_dem: Interpolate + % Fill bad values using the good data + bad_idxs = find(isnan(DEM)); + good_idxs = find(~isnan(DEM)); + x_idxs = repmat(1:size(DEM,2),[size(DEM,1) 1]); + y_idxs = repmat((1:size(DEM,1))',[1 size(DEM,2)]); + x_vals = x_idxs(good_idxs); + y_vals = y_idxs(good_idxs); + z_vals = DEM(good_idxs); + x_out = x_idxs(bad_idxs); + y_out = y_idxs(bad_idxs); + z_out = single(griddata(x_vals,y_vals,double(z_vals),x_out,y_out)); + if ~isempty(z_out) + DEM(bad_idxs) = z_out; + end if 0 - imagesc(reshape(DEM_fcs_x,[200 200])) - colorbar; + figure(10);clf; + % Convert DEM ecef points to geodetic + imagesc(DEM_x.*1e-3,DEM_y.*1e-3,DEM.*1e-3); + hold on; + plot(frm_x.*1e-3,frm_y.*1e-3,'rx','LineWidth',2); + set(gca,'TickLabelInterpreter','latex') + set(gca,'FontSize',14) + xlabel('X (km)','Interpreter','latex','FontSize',14) + ylabel('Y (km)','Interpreter','latex','FontSize',14) + title('ArcticDEM 20140325 07 002','FontSize',18,'Interpreter','latex') + xlim([-880 -810]); + ylim([-830 -750]); + % out_fn1 = 'arctic_dem_20140325_07_002.fig'; + % savefig(10,out_fn1); - imagesc(reshape(DEM_fcs_y,[200 200])) - colorbar; - - imagesc(reshape(DEM_fcs_z,[200 200])) - colorbar; + figure(2);clf; + % Convert DEM ecef points to geodetic + imagesc(ice_mask_all.X.*1e-3,ice_mask_all.Y.*1e-3,ice_mask_all.mask); + hold on; + plot(frm_x.*1e-3,frm_y.*1e-3,'rx','LineWidth',2); + set(gca,'TickLabelInterpreter','latex') + set(gca,'FontSize',14) + xlabel('X (km)','Interpreter','latex','FontSize',14) + ylabel('Y (km)','Interpreter','latex','FontSize',14) + title('Ice Mask 20140325 07 002','FontSize',18,'Interpreter','latex') + xlim([-880 -810]); + ylim([-830 -750]); + % out_fn2 = 'ice_mask_20140325_07_002.fig'; + % savefig(2,out_fn2); end - slice_mask = DEM_fcs_x > -param.add_surf_from_dem.dem_per_slice_guard & DEM_fcs_x < param.add_surf_from_dem.dem_per_slice_guard; + % Make x,y mesh from arctic dem bounds + DEM_x_mesh = repmat(DEM_x,[size(DEM,1) 1]); + DEM_y_mesh= repmat(DEM_y,[1 size(DEM,2)]); + + + %% add_surf_from_dem: twtt + % For every position along the aperture, add theta dependent TWTT to DEM from origin of the FCS + Nx = size(sd.fcs.x,2); + Nsv = length(param.add_surf_from_dem.theta_bins); + twtt = zeros(Nsv,Nx); + ice_mask_tmp = NaN(Nsv,Nx); + + % Handle case where the DEM is all NaNs + if all(all(isnan(DEM))) + warning('Input DEM contains all NaN data for Frame %d.',param.proc.frm); + twtt(:,:) = NaN; + Nx = 0; + end - x = DEM_fcs_x(slice_mask); - y = DEM_fcs_y(slice_mask); - z = DEM_fcs_z(slice_mask); + DEM_coverage_warning = false; - if (numel(x)>=3) - faces = delaunay(double(x),double(y)); - vertices = [double(x).' double(y).' double(z).']; % vertices stored as Nx3 matrix - vert1 = vertices(faces(:,1),:); - vert2 = vertices(faces(:,2),:); - vert3 = vertices(faces(:,3),:); + for rline = 1:Nx + % for rline = 1:1 + if ~mod(rline-1,10^floor(log10(Nx)-1)) + fprintf(' %s %d of %d (%s)\n', mfilename, rline, Nx, datestr(now)); + end + + dem_guard = param.add_surf_from_dem.dem_guard; + + DEM_mask = DEM_x_mesh > frm_x(rline)-dem_guard & DEM_x_mesh < frm_x(rline)+dem_guard ... + & DEM_y_mesh > frm_y(rline)-dem_guard & DEM_y_mesh < frm_y(rline)+dem_guard ... + & ~isnan(DEM); + DEM_idxs = find(DEM_mask); - orig = [0 0 0]; + if numel(DEM_idxs)==0 + warning('Range Line %d of Frame %d is not spanned by DEM.',rline,frm); + end + + if 0 + set(h_img,'AlphaData',DEM_mask); + end + + % Convert arctic dem mesh from projection to geodetic (lat,lon,elev) + [DEM_lat,DEM_lon] = projinv(proj,DEM_x_mesh(DEM_idxs),DEM_y_mesh(DEM_idxs)); + DEM_elev = DEM(DEM_idxs); - theta_rline = param.add_surf_from_dem.theta_vec*(pi/180); - - intersection = zeros(3,Nsv); + % Convert from geodetic (lat,lon,elev) to ECEF (x,y,z) + physical_constants; + [DEM_ecef_x,DEM_ecef_y,DEM_ecef_z] = geodetic2ecef(single(DEM_lat)/180*pi,single(DEM_lon)/180*pi,single(DEM_elev),WGS84.ellipsoid); - for theta_idx = 1:length(theta_rline) - dir = [0 sin(theta_rline(theta_idx)) -cos(theta_rline(theta_idx))]; - - [Intersect, t] = TriangleRayIntersection(orig, dir, vert1, vert2, vert3); + origin = sd.fcs.origin(:,rline); + + % Define matrices to change coordinate systems from FCS to ECEF + % and vice versa + Tfcs_ecef = [sd.fcs.x(:,rline), sd.fcs.y(:,rline), sd.fcs.z(:,rline)]; + Tecef_fcs = inv(Tfcs_ecef); + + % Convert DEM coordinates from ECEF to FCS + tmp = Tecef_fcs * [DEM_ecef_x.'-origin(1); DEM_ecef_y.'-origin(2); DEM_ecef_z.'-origin(3)]; + DEM_fcs_x = tmp(1,:); + DEM_fcs_y = tmp(2,:); + DEM_fcs_z = tmp(3,:); + + if 0 + imagesc(reshape(DEM_fcs_x,[200 200])) + colorbar; - intersect_idx = find(Intersect); - if isempty(intersect_idx) - twtt(theta_idx,rline) = NaN; - intersection(:,theta_idx) = NaN; - else - twtt(theta_idx,rline) = t(intersect_idx(1))/(3e8/2); - intersection(:,theta_idx) = mean([vert1(intersect_idx(1),:);vert2(intersect_idx(1),:);vert3(intersect_idx(1),:)],1); - end + imagesc(reshape(DEM_fcs_y,[200 200])) + colorbar; -% if 0 -% figure(999);clf -% D = c*twtt(theta_idx,rline)/2; -% Dvec = D*dir; -% Dx = Dvec(1); -% Dy = Dvec(2); -% Dz = Dvec(3); -% h = trisurf(faces,x,y,z, Intersect*1.0, 'FaceAlpha',0.9); -% set(h,'edgecolor','none') -% hold on -% line('XData',orig(1) + [0 Dx], 'YData',orig(2) + [0 Dy], 'ZData', orig(3) + [0 Dz], 'Color', 'r', 'LineWidth',3) -% xlabel('X_F_C_S (m)') -% ylabel('Y_F_C_S (m)') -% zlabel('Z_F_C_S (m)') -% -% end - end - else - if ~DEM_coverage_warning - DEM_coverage_warning = true; - warning('DEM dem_per_slice_guard too small.'); + imagesc(reshape(DEM_fcs_z,[200 200])) + colorbar; end - clear intersection; - twtt(:,rline) = NaN; - end - - - if exist('ice_mask_all','var') && ~isempty(ice_mask_all) - if exist('intersection','var') - % Convert from FCS/SAR to ECEF - intersection_ecef = Tfcs_ecef * intersection; - intersection_ecef_x = intersection_ecef(1,:).' + origin(1); - intersection_ecef_y = intersection_ecef(2,:).' + origin(2); - intersection_ecef_z = intersection_ecef(3,:).' + origin(3); - % Convert from ECEF to geodetic - [intersection_lat,intersection_lon,tri_h] = ecef2geodetic(intersection_ecef_x,intersection_ecef_y,intersection_ecef_z,WGS84.ellipsoid); - intersection_lat = intersection_lat*180/pi; - intersection_lon = intersection_lon*180/pi; - % Convert from geodetic to projection - [intersection_x,intersection_y] = projfwd(ice_mask_all.proj,intersection_lat,intersection_lon); - % Get mask coordinates nearest triangle center coordinates - intersection_x_idx = interp1(ice_mask_all.X,1:length(ice_mask_all.X),intersection_x,'nearest'); - intersection_y_idx = interp1(ice_mask_all.Y,1:length(ice_mask_all.Y),intersection_y,'nearest'); - % Find nan values and set to integer value - nidx = find(isnan(intersection_x_idx)); - intersection_x_idx(nidx) = 1; - intersection_y_idx(nidx) = 1; - % Convert triangle mask coordinates to matrix indices - mask_idx = (intersection_x_idx-1)*length(ice_mask_all.Y) + intersection_y_idx; - % Find ice mask for triangle coordinates - ice_mask(:,rline) = ice_mask_all.mask(mask_idx); - % Set previously nan valued coordinates to 0 mask -% ice_mask(nidx,rline) = 0; - else - % for ice_mask_idx = 1:length(mask_idx) - % ice_mask(theta_rline_row_idx(ice_mask_idx),theta_rline_col_idx(ice_mask_idx),rline) = ice_mask_all.mask(mask_idx(ice_mask_idx)); - % end - % % Set previously nan valued coordinates to 0 mask - % ice_mask(theta_rline_row_idx(nidx),theta_rline_col_idx(nidx),rline) = 0; - % end + + slice_mask = DEM_fcs_x > -param.add_surf_from_dem.dem_per_slice_guard & DEM_fcs_x < param.add_surf_from_dem.dem_per_slice_guard; + + x = DEM_fcs_x(slice_mask); + y = DEM_fcs_y(slice_mask); + z = DEM_fcs_z(slice_mask); + + if (numel(x)>=3) + faces = delaunay(double(x),double(y)); + vertices = [double(x).' double(y).' double(z).']; % vertices stored as Nx3 matrix + vert1 = vertices(faces(:,1),:); + vert2 = vertices(faces(:,2),:); + vert3 = vertices(faces(:,3),:); + + orig = [0 0 0]; + + theta_rline = param.add_surf_from_dem.theta_bins*(pi/180); + intersection = zeros(3,Nsv); + + for theta_idx = 1:length(theta_rline) + dir = [0 sin(theta_rline(theta_idx)) -cos(theta_rline(theta_idx))]; + + [Intersect, t] = TriangleRayIntersection(orig, dir, vert1, vert2, vert3); + + intersect_idx = find(Intersect); + if isempty(intersect_idx) + twtt(theta_idx,rline) = NaN; + intersection(:,theta_idx) = NaN; + else + range_intersections = t(intersect_idx); + [min_range_val,min_range_idx] = min(range_intersections); + prop_delay = min_range_val./(c/2); + good_intersect_idx = intersect_idx(min_range_idx); + + twtt(theta_idx,rline) = prop_delay; + intersection(:,theta_idx) = mean([vert1(good_intersect_idx,:);vert2(good_intersect_idx,:);vert3(good_intersect_idx,:)],1); + end + + end + else + if ~DEM_coverage_warning + DEM_coverage_warning = true; + warning('DEM dem_per_slice_guard too small.'); + end + clear intersection; + twtt(:,rline) = NaN; end + + + if exist('ice_mask_all','var') && ~isempty(ice_mask_all) + if exist('intersection','var') + % Convert from FCS/SAR to ECEF + intersection_ecef = Tfcs_ecef * intersection; + intersection_ecef_x = intersection_ecef(1,:).' + origin(1); + intersection_ecef_y = intersection_ecef(2,:).' + origin(2); + intersection_ecef_z = intersection_ecef(3,:).' + origin(3); + % Convert from ECEF to geodetic + [intersection_lat,intersection_lon,tri_h] = ecef2geodetic(intersection_ecef_x,intersection_ecef_y,intersection_ecef_z,WGS84.ellipsoid); + intersection_lat = intersection_lat*180/pi; + intersection_lon = intersection_lon*180/pi; + % Convert from geodetic to projection (align DEM to the ice + % mask) + [intersection_x,intersection_y] = projfwd(ice_mask_all.proj,intersection_lat,intersection_lon); + % Get mask coordinates nearest triangle center coordinates + intersection_x_idx = interp1(ice_mask_all.X,1:length(ice_mask_all.X),intersection_x,'nearest'); + intersection_y_idx = interp1(ice_mask_all.Y,1:length(ice_mask_all.Y),intersection_y,'nearest'); + % Find nan values and set to integer value + nidx = find(isnan(intersection_x_idx)); + intersection_x_idx(nidx) = 1; + intersection_y_idx(nidx) = 1; + % Convert triangle mask coordinates to matrix indices + mask_idx = (intersection_x_idx-1)*length(ice_mask_all.Y) + intersection_y_idx; + % Find ice mask for triangle coordinates + ice_mask_tmp(:,rline) = ice_mask_all.mask(mask_idx); + % Set previously nan valued coordinates to 0 mask + % ice_mask(nidx,rline) = 0; + else + % for ice_mask_idx = 1:length(mask_idx) + % ice_mask(theta_rline_row_idx(ice_mask_idx),theta_rline_col_idx(ice_mask_idx),rline) = ice_mask_all.mask(mask_idx(ice_mask_idx)); + % end + % % Set previously nan valued coordinates to 0 mask + % ice_mask_tmp(theta_rline_row_idx(nidx),theta_rline_col_idx(nidx),rline) = 0; + % end + + end + end + end end - %% Insert Surfaces - - % Insert TWTT of top surface - surf = tomo.surfdata.empty_surf(); - surf.x = repmat((1:Nsv).',[1 size(twtt,2)]); % Nsv x Nx + if regexp(param.add_surf_from_dem.method,'sar') + ice_mask = ice_mask_tmp; + end - surf.y = twtt; - surf.plot_name_values = {'color','black','marker','x'}; - surf.name = 'top twtt'; - sd.insert_surf(surf); - - % Insert icemask - surf = tomo.surfdata.empty_surf(); - surf.x = repmat((1:Nsv).',[1 size(twtt,2)]); % Nsv x Nx - surf.y = ice_mask; - surf.plot_name_values = {'color','white','marker','x'}; - surf.name = 'ice_mask'; - sd.insert_surf(surf); - - %% Create output filename - out_dir = ct_filename_out(param,param.add_surf_from_dem.surf_out_path,'surfData_sar'); - - if ~isdir(out_dir) - mkdir(out_dir); + %% add_surf_from_dem: Insert Surfaces + + if regexp(param.add_surf_from_dem.method,'surf') + if param.add_surf_from_dem.surfdata_mode.overwrite + % Get existing ice mask surface layer + surf_a = sd.get_surf(param.add_surf_from_dem.ice_mask_surf_name); + surf_a.x = repmat(param.add_surf_from_dem.theta_bins, [1 size(twtt,2)]); + surf_a.y = ice_mask; + + % If the current surface contains the ice mask surface layer, + % remove it + if any(strcmp(sd.get_names,param.add_surf_from_dem.ice_mask_surf_name)) + sd.remove_surf(param.add_surf_from_dem.ice_mask_surf_name); + end + + % Insert ice mask surface + sd.insert_surf(surf_a); + + if isempty(ref_surf_idx) + surf_b = sd.empty_surf; + surf_b.x = repmat(param.add_surf_from_dem.theta_bins, [1 size(twtt,2)]); + surf_b.y = twtt; + surf_b.name = 'top twtt'; + sd.insert_surf(surf_b); + end + end + + elseif regexp(param.add_surf_from_dem.method,'sar') + % Insert TWTT of top surface + sd.surf(1).name = 'top twtt'; + sd.surf(1).x = zeros(Nsv,length(sd.gps_time)); + sd.surf(1).y = zeros(Nsv,length(sd.gps_time)); + sd.surf(1).x = repmat(param.add_surf_from_dem.theta_bins,[1 size(twtt,2)]); % Nsv x Nx + sd.surf(1).y = twtt; + + sd.surf(2).name = 'ice mask'; + sd.surf(2).x = zeros(Nsv,length(sd.gps_time)); + sd.surf(2).y = zeros(Nsv,length(sd.gps_time)); + sd.surf(2).x= repmat(param.add_surf_from_dem.theta_bins,[1 size(twtt,2)]); % Nsv x Nx + sd.surf(2).y = ice_mask; + sd.surf(2).plot_name_values = {'color','white','marker','x'}; + end + %% add_surf_from_dem: Save output + out_fn_dir = ct_filename_out(param,param.add_surf_from_dem.surf_out_path); + if ~isdir(out_fn_dir) + mkdir(out_fn_dir); + end out_fn_name = sprintf('Data_%s_%03.0f.mat',param.day_seg,frm); - out_fn = fullfile(out_dir,out_fn_name); - sd.save_surfdata(out_fn,doa_method_flag); - fprintf('Done (%s)\n', datestr(now)); - + out_fn = fullfile(out_fn_dir,out_fn_name); + sd.save_surfdata(out_fn); + fprintf('Done (%s)\n\n', datestr(now)); end + end + + function [] = run_add_surf_from_dem() + %% run_add_surf_from_dem + % tomo.surfdata.run_add_surf_from_dem + % + % script for running tomo.surfdata.add_surf_from_dem + % + % Author: Theresa Moore + % + % See also: tomo.surfdata + + + % User Setup + % ========================================================================= + param_override = []; + params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls')); + params = ct_set_params(params,'cmd.generic',0); + + % params = ct_set_params(params,'cmd.generic',1,'day_seg','20140325_07'); + % params = ct_set_params(params,'cmd.frms',[1:5]); + + % params = ct_set_params(params,'cmd.generic',1,'day_seg','20140325_01'); + % params = ct_set_params(params,'cmd.frms',[1:3]); + + % params = ct_set_params(params,'cmd.generic',1,'day_seg','20140325_04'); + % params = ct_set_params(params,'cmd.frms',[1:26]); + + % params = ct_set_params(params,'cmd.generic',1,'day_seg','20140325_02'); + % params = ct_set_params(params,'cmd.frms',[1:7]); + % params = ct_set_params(params,'cmd.generic',1,'day_seg','20140401_03'); + % params = ct_set_params(params,'cmd.frms',[1:48]); + % params = ct_set_params(params,'cmd.generic',1,'day_seg','20140506_01'); + % params = ct_set_params(params,'cmd.frms',[1:46]); + % params = ct_set_params(params,'cmd.generic',1,'day_seg','20140325_07'); + % params = ct_set_params(params,'cmd.frms',[2]); - + % params = ct_set_params(params,'cmd.generic',1,'day_seg','20140401_03'); + % params = ct_set_params(params,'cmd.frms',[4,5,34,35]); + % params = ct_set_params(params,'cmd.generic',1,'day_seg','20140401_03'); + % params = ct_set_params(params,'cmd.frms',[19, 41, 42, 45]); + params = ct_set_params(params,'cmd.generic',1,'day_seg','20140401_03'); +% params = ct_set_params(params,'cmd.frms',[4, 5, 13, 15, 16, 19, 20, 34, 35, 41, 42, 45]); +% params = ct_set_params(params,'cmd.frms',[13]); +% params = ct_set_params(params,'cmd.frms',[34 35 38]); +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20140401_03'); +% params = ct_set_params(params,'cmd.frms',[41 42 45]); + % params = ct_set_params(params,'cmd.generic',1,'day_seg','20140401_03'); + % params = ct_set_params(params,'cmd.frms',[45]); + params = ct_set_params(params,'cmd.generic',1,'day_seg','20140401_03'); + params = ct_set_params(params,'cmd.frms',[20, 34, 35, 38]); + % params = ct_set_params(params,'cmd.generic',1,'day_seg','20140401_03'); + % params = ct_set_params(params,'cmd.frms',[ 13, 15:16, 38, 41, 42, 45]); + + % params = ct_set_params(params,'cmd.generic',1,'day_seg','20140506_01'); + % params = ct_set_params(params,'cmd.frms',[41,42,44]); + + % params = ct_set_params(params,'add_surf_from_dem.ice_mask_fn',fullfile('greenland','IceMask','GimpIceMask_90m_v1.1.tif'));%'antarctica\DEM\BEDMAP2\original_data\bedmap2_tiff\bedmap2_icemask_grounded_and_shelves.tif'; + params = ct_set_params(params,'add_surf_from_dem.ice_mask_fn','canada/ice_mask/03_rgi50_ArcticCanadaNorth/03_rgi50_ArcticCanadaNorth.mat');%'antarctica\DEM\BEDMAP2\original_data\bedmap2_tiff\bedmap2_icemask_grounded_and_shelves.tif'; + % params = ct_set_params(params,'add_surf_from_dem.ice_mask_fn',ct_filename_gis(params,fullfile('greenland','IceMask','GimpIceMask_90m_v1.1.tif')));%'antarctica\DEM\BEDMAP2\original_data\bedmap2_tiff\bedmap2_icemask_grounded_and_shelves.tif'; + % param.add_surf_from_dem.ice_mask_fn = ct_filename_gis(param,fullfile('greenland','IceMask','GimpIceMask_90m_v1.1.tif'));%'antarctica\DEM\BEDMAP2\original_data\bedmap2_tiff\bedmap2_icemask_grounded_and_shelves.tif'; + % param.add_surf_from_dem.ice_mask_fn = ct_filename_gis([],'canada/ice_mask/03_rgi50_ArcticCanadaNorth/03_rgi50_ArcticCanadaNorth.mat'); + + params = ct_set_params(params, 'add_surf_from_dem.dem_guard', 30e3); + params = ct_set_params(params, 'add_surf_from_dem.delta_at',10); + + params = ct_set_params(params, 'add_surf_from_dem.method','surf'); + % param_override.add_surf_from_dem.in_path = 'surf_tgrs2021_evd_20140401_03_lut'; + % param_override.add_surf_from_dem.in_path = 'surf_tgrs2021_evd_20140325_07_lut'; + % param_override.add_surf_from_dem.in_path = 'surf_tgrs2021_evd_20140506_01_lut'; + % + % param_override.add_surf_from_dem.in_path = 'surf_tgrs2021_evd_20140401_03_lut_test'; + % param_override.add_surf_from_dem.in_path = 'surf_tgrs2021_evd_20140325_07_lut_test'; + % param_override.add_surf_from_dem.in_path = 'surf_tgrs2021_evd_20140506_01_lut_test'; + +% param_override.add_surf_from_dem.in_path = 'surf_tgrs2021_evd_20140401_03_lut'; +% param_override.add_surf_from_dem.in_path = 'surf_tgrs2021_evd_20140325_07_lut'; +% param_override.add_surf_from_dem.in_path = 'surf_tgrs2021_evd_20140506_01_lut'; +% param_override.add_surf_from_dem.in_path = 'surf_nominal'; + +% param_override.add_surf_from_dem.in_path = 'surf_em_model_20140325_07_lut'; +% param_override.add_surf_from_dem.in_path = 'surf_em_model_20140506_01_lut'; + param_override.add_surf_from_dem.in_path = 'surf_pseudoinverse_evd_20140506_01_lut_test'; + % Use these options for create surfdata used to generate + % snapshots in array proc +% params = ct_set_params(params, 'add_surf_from_dem.method','sar'); +% param_override.add_surf_from_dem.in_path = 'sar_air'; + + + % param_override.add_surf_from_dem.in_path = 'sar_air'; + + + % param_override.update.output = 'surf_sar'; + % param_override.update.echogram = 'standard_air'; + + % Automated Section + % ========================================================================= + % Input checking + global gRadar; + if exist('param_override','var') + param_override = merge_structs(gRadar,param_override); + else + param_override = gRadar; + end + + % Process each of the segments + ctrl_chain = {}; + for param_idx = 1:length(params) + param = params(param_idx); + if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic + continue; + end + tomo.surfdata.add_surf_from_dem(param,param_override); + end end end diff --git a/cresis-toolbox/+tomo/surfdata_modify.m b/cresis-toolbox/+tomo/surfdata_modify.m deleted file mode 100644 index 50cfe963..00000000 --- a/cresis-toolbox/+tomo/surfdata_modify.m +++ /dev/null @@ -1,51 +0,0 @@ -function surfData_modify(param,fn,layers,varargin) -% tomo.surfData_modify(param,fn,layers,varargin) -% -% Function for modifying surfData files. Can be used to create new layers -% by specifying a layer that does not already exist. -% -% param: parameter spreadsheet structure -% .radar_name: determines which radar to modify -% .season_name: determines which season to modify -% .day_seg: determines which segment to modify -% .cmd.frms: determines which frames to modify -% fn: filename to ct_filename_out(), leave empty for default 'surfData' -% varargin: arbitrary list of name,value pairs. -% third, fifth, etc. arguments are a string containing the name of the -% property to modify -% fourth, sixth, etc. arguments are the new value to use for the -% corresponding named property -% -% Author: John Paden - -% Determine which frames to process -load(ct_filename_support(param,'','frames')); - -if isempty(param.cmd.frms) - param.cmd.frms = 1:length(frames.frame_idxs); -end -% Remove frames that do not exist from param.cmd.frms list -[valid_frms,keep_idxs] = intersect(param.cmd.frms, 1:length(frames.frame_idxs)); -if length(valid_frms) ~= length(param.cmd.frms) - bad_mask = ones(size(param.cmd.frms)); - bad_mask(keep_idxs) = 0; - warning('Nonexistent frames specified in param.cmd.frms (e.g. frame "%g" is invalid), removing these', ... - param.cmd.frms(find(bad_mask,1))); - param.cmd.frms = valid_frms; -end - -% out_path: output surfData directory -out_path = ct_filename_out(param, fn, 'CSARP_surfData'); - -for frm = param.cmd.frms - fn = fullfile(out_path,sprintf('Data_%s_%03d.mat', param.day_seg, frm)); - - % Load "surf" variable - load(fn); - for layer = layers(:).' - for idx = 1:2:length(varargin) - surf(layer).(varargin{idx}) = varargin{idx+1}; - end - end - save(fn,'-v7','surf'); -end diff --git a/cresis-toolbox/+tomo/surfdata_to_DEM.m b/cresis-toolbox/+tomo/surfdata_to_DEM.m index 56893126..cb3cffcb 100644 --- a/cresis-toolbox/+tomo/surfdata_to_DEM.m +++ b/cresis-toolbox/+tomo/surfdata_to_DEM.m @@ -46,6 +46,23 @@ function surfdata_to_DEM(param,param_override) end param = merge_structs(param, param_override); +dbstack_info = dbstack; +fprintf('=====================================================================\n'); +fprintf('%s: %s (%s)\n', dbstack_info(1).name, param.day_seg, datestr(now,'HH:MM:SS')); +fprintf('=====================================================================\n'); + +%% Input Checks: cmd +% ===================================================================== + +% Remove frames that do not exist from param.cmd.frms list +frames = frames_load(param); +param.cmd.frms = frames_param_cmd_frms(param,frames); + +%% Input Checks: dem +% ===================================================================== + +physical_constants; + if ~isfield(param.dem,'doa_method_flag') || isempty(param.dem.doa_method_flag) doa_method_flag = false; else @@ -58,13 +75,43 @@ function surfdata_to_DEM(param,param_override) doa_limits = param.dem.doa_limits*pi/180; end +% dem.er_ice: the relative dielectric of ice (defaults to 3.15 from +% physical_constants.m which is generally the setting for creating digital +% elevation models of the ice bottom). Set this to 1 for air (e.g. when +% producing a dem for an ice surface that is not the param.dem.ice_top). +% Note that this setting does not have any effect for the surface indicated by +% param.dem.ice_top (i.e. the value does not matter). +if ~isfield(param.dem,'er_ice') || isempty(param.dem.er_ice) + param.dem.er_ice = er_ice; +end -dbstack_info = dbstack; -fprintf('=====================================================================\n'); -fprintf('%s: %s (%s)\n', dbstack_info(1).name, param.day_seg, datestr(now,'HH:MM:SS')); -fprintf('=====================================================================\n'); +% dem.enforce_below_top: forces the surface to be below the top surface +% (i.e. the ice surface). Defaults to true which is the setting for +% creating DEMs of the ice bottom. Set to false to generate DEMs of the ice +% surface. Note that this setting does not have any effect for the surface +% indicated by param.dem.ice_top (i.e. can be true or false). +if ~isfield(param.dem,'enforce_below_top') || isempty(param.dem.enforce_below_top) + param.dem.enforce_below_top = true; +end -physical_constants; +if ~isfield(param.dem,'ice_mask') || isempty(param.dem.ice_mask) + param.dem.ice_mask = []; +end + +if ~isfield(param.dem,'ice_mask_flag') || isempty(param.dem.ice_mask_flag) + param.dem.ice_mask_flag = false; +end + +if ~isfield(param.dem,'array_manifold_mask_flag') || isempty(param.dem.array_manifold_mask_flag) + param.dem.array_manifold_mask_flag = false; +end + +if param.dem.array_manifold_mask_flag + param.dem.array_manifold_mask_flag = true; + if isempty(param.dem.ice_mask) + error('User must specify an ice mask surface to derive the array manifold mask') + end +end surface_names = param.dem.surface_names; figure_dots_per_km = param.dem.figure_dots_per_km; @@ -74,23 +121,8 @@ function surfdata_to_DEM(param,param_override) mkdir(out_dir); end -% Load frames file -load(ct_filename_support(param,'','frames')); - -if isempty(param.cmd.frms) - param.cmd.frms = 1:length(frames.frame_idxs); -end -% Remove frames that do not exist from param.cmd.frms list -[valid_frms,keep_idxs] = intersect(param.cmd.frms, 1:length(frames.frame_idxs)); -if length(valid_frms) ~= length(param.cmd.frms) - bad_mask = ones(size(param.cmd.frms)); - bad_mask(keep_idxs) = 0; - warning('Nonexistent frames specified in param.cmd.frms (e.g. frame "%g" is invalid), removing these', ... - param.cmd.frms(find(bad_mask,1))); - param.cmd.frms = valid_frms; -end - %% Loop through each frame and create data products +% ===================================================================== for frm_idx = 1:length(param.cmd.frms) frm = param.cmd.frms(frm_idx); @@ -101,7 +133,7 @@ function surfdata_to_DEM(param,param_override) surfdata_fn = fullfile(ct_filename_out(param,param.dem.surfdata_source),sprintf('Data_%s_%03d.mat',param.day_seg,frm)); data_fn = fullfile(ct_filename_out(param,param.dem.input_dir_name),sprintf('Data_%s_%03d.mat',param.day_seg,frm)); - %% Load surfData, echogram, steering vector calibration + %% Load surfdata, echogram, steering vector calibration and handle error conditions % Load surface data (usually top and bottom) fprintf(' %s\n', surfdata_fn); @@ -115,18 +147,36 @@ function surfdata_to_DEM(param,param_override) % Find the index to the ice top ice_top_idx = sd.get_index(param.dem.ice_top); + if ~isempty(find(strcmp(param.dem.ice_mask,{sd.surf.name}))) && param.dem.ice_mask_flag + ice_mask_idx = sd.get_index(param.dem.ice_mask); + ice_mask = sd.surf(ice_mask_idx).y; + array_manifold_mask = ice_mask; + array_manifold_mask = cumsum(array_manifold_mask,1,'reverse') >= 1 | cumsum(array_manifold_mask,1,'forward') >= 1; + else + ice_mask_idx =[]; + ice_mask = []; + array_manifold_mask = []; + end + + if isempty(ice_mask) && param.dem.array_manifold_mask_en + error('No valid ice mask found in surfdata file') + end + + if isempty(ice_mask) && param.dem.ice_mask + error('No valid ice mask found in surfdata file') + end + % Load the echogram fprintf(' %s\n', data_fn); mdata = load(data_fn); % Load the steering vector calibration file if specified - if(~isfield(mdata,'theta')) + if(~isfield(mdata.Tomo,'theta')) global gRadar theta = load(fullfile(gRadar.out_path,'rds','2014_Greenland_P3','CSARP_CSA_music','20140401_03','Data_20140401_03_037'),'theta'); - mdata.theta = theta.theta; + mdata.Tomo.theta = theta.theta; end - %% Setup for convert doa,twtt to radar FCS for each surface % FCS: flight coordinate system (aka SAR coordinate system) @@ -146,7 +196,7 @@ function surfdata_to_DEM(param,param_override) % Convert top from range bins to twtt (the top is needed with each % surface to handle refraction) - ice_top = interp1(1:length(mdata.Time), mdata.Time, sd.surf(ice_top_idx).y); + ice_top = sd.surf(ice_top_idx).y; % If no top defined, then assume top is a zero time (i.e. % ground based radar operation with antenna at surface) if all(all(isnan(ice_top))) @@ -165,11 +215,14 @@ function surfdata_to_DEM(param,param_override) % Determine which surface index in surfData file we are working with surface_idx = sd.get_index(surface_names{surface_names_idx}); - % Ensure non-negative ice thickness - sd.surf(surface_idx).y(sd.surf(surface_idx).y= tmp.ref.layers(1).twtt(rline)*2+multiple_guard_band(1) ... + & tmp.ref.time <= tmp.ref.layers(1).twtt(rline)*2+multiple_guard_band(2); + mask(multiple_bins,rline_idx) = false; +end + +if 0 + %% Debug + time = tmp.ref.time; + + figure(1); clf; + imagesc([],time,lp(insar1)); + hold on; + plot(tmp.ref.layers(1).twtt); + plot(tmp.ref.layers(1).twtt*2); + plot(tmp.ref.layers(2).twtt); + + figure(2); clf; + imagesc([],time,lp(insar2)); + hold on; + plot(tmp.ref.layers(1).twtt); + plot(tmp.ref.layers(1).twtt*2); + plot(tmp.ref.layers(2).twtt); + + figure(3); clf; + imagesc([],time,angle(insar_data)) + hold on; + plot(tmp.ref.layers(1).twtt*2); + plot(tmp.ref.layers(1).twtt*2); + + figure(4); clf; + imagesc([],time,mask); + + link_figures([1 2 3 4]); + +end + %% Surface flattening % Remove surface variations by moving all bins down so that the surface % aligns for every range line and the surface bin is the max surface bin @@ -114,12 +151,18 @@ Nt = size(insar_data,1); freq_bin = 1i*2*pi/Nt * ifftshift( -floor(Nt/2) : floor((Nt-1)/2) ).'; insar_data = ifft(fft(insar_data) .* exp(bsxfun(@times,dbin,freq_bin))); +for rline_idx = 1:length(rlines) + rline = rlines(rline_idx); + mask(:,rline_idx) = circshift(mask(:,rline_idx),-dbin(rline_idx)); +end % Remove bad circular deconvolution bins from surface flattening [max_dbin,max_dbin_rline] = max(abs(dbin)); max_dbin = ceil(max_dbin); insar_data = insar_data(1+max_dbin:end,:); rbins = rbins(1+max_dbin:end); +mask = mask(1+max_dbin:end,:); +time = tmp.ref.time(1+max_dbin:end); %% Baseline GPS/phase error correction % baseline_correction = fir_dec(mean(insar_data(1:30,:),1),ones(1,51)/51,1); @@ -135,7 +178,7 @@ % displacement = fir_dec(displacement - polyval(pp,range),ones(1,5)/5,1); - figure(3000); + figure(6000);clf; displacement(240:287) = nan; displacement_single = displacement; thickness = nanmean(tmp.ref.layers(2).twtt-tmp.ref.layers(1).twtt)*3e8/2/1.78; @@ -154,7 +197,7 @@ xlabel(sprintf('Vertical velocity\n(mm/year)')); ylabel('Height above bed (m)'); - figure(3001);clf; + figure(6001);clf; plot(thickness-range, displacement_single, '.','Color','red'); hold on; plot(thickness-range, displacement, 'LineWidth', 2, 'Color', 'blue'); @@ -188,9 +231,74 @@ return; end +%% Example plots for EGIG, Greenland +if 0 + range = (tmp.ref.time(rbins) - tmp.ref.time(max_surface_bin)) * c/2 / sqrt(er_ice); + relative_velocity_offset = 425; + displacement = relative_velocity_offset+unwrap(angle(mean(insar_data,2))) * c/(4*pi*195e6) / 2 / 1.78 * 1000; + pp = polyfit(range(325:650),displacement(325:650),2); + max_vel = 450; + + % displacement = fir_dec(displacement - polyval(pp,range),ones(1,5)/5,1); + + figure(6000);clf; +% displacement(240:287) = nan; + displacement_single = displacement; + thickness = nanmean(tmp.ref.layers(2).twtt-tmp.ref.layers(1).twtt)*3e8/2/1.78; + plot(displacement_single, thickness-range, '.','Color','red'); + hold on; +% displacement(240:287) = nan; + displacement = polyval(pp,range(:).') + nan_fir_dec(displacement(:).' - polyval(pp,range(:).'),ones(1,51)/51,1); + displacement = displacement(:).'; + range = range(:).'; + % displacement = displacement; + plot(displacement, thickness-range, 'LineWidth', 3, 'Color', 'blue'); + hold on; + grid on; + xlim([-20 max_vel]) + ylim([0 3082]); + xlabel(sprintf('Vertical velocity\n(mm/year)')); + ylabel('Height above bed (m)'); + + figure(6001);clf; + plot(thickness-range, displacement_single, '.','Color','red'); + hold on; + plot(thickness-range, displacement, 'LineWidth', 2, 'Color', 'blue'); + grid on; + ylim([-20 max_vel]) + xlim([0 3082]); + ylabel(sprintf('Vertical velocity\n(mm/year)')); + xlabel('Height above bed (m)'); + plot([3082 3082], [-20 max_vel],'Color',[0.5 0.5 0.5],'linewidth',3); + h=text(3082-100,50,'Surface','Rotation',90); + plot([0 0], [-20 max_vel],'Color',[0.5 0.5 0.5],'linewidth',3); + text(100,50,'Bottom','Rotation',90); + legend('Measured','Smoothed','location','best') + + % + displacement(1:100) = nan; + displacement(235:292) = nan; + displacement(850:end) = nan; + strain_rate = nan_fir_dec(diff(displacement) ./ diff(range), ones(1,3)/3,1); + figure(6002); clf; + plot(-strain_rate/1000, thickness-range(2:end), '.','Color','red'); + hold on + strain_rate = fir_dec(strain_rate,ones(1,101)/101,1); + plot(-strain_rate/1000, thickness-range(2:end), 'LineWidth', 3, 'Color', 'blue'); + grid on; + xlabel(sprintf('Strain rate\n(m\\cdotm^{-1}\\cdota^{-1})')); + ylabel('Height above bed (m)'); + ylim([0 3082]); + xlim([0 0.3]/1000); + + return; +end + %% Estimate vertical velocity range = (tmp.ref.time(rbins) - tmp.ref.time(max_surface_bin)) * c/2 / sqrt(er_ice); +insar_data(~mask) = NaN; + insar_data_filt = fir_dec(fir_dec(insar_data,ones(1,31)/31,1).',ones(1,3)/3).'; meter_units = true; @@ -238,21 +346,25 @@ set(3000,'Position',[56 470 867 426]); caxis([0 1]); -[fn_dir,fn_name] = fileparts(fn); -fn_insar_phase = fullfile(fn_dir,[fn_name '_phase.fig']); -fprintf('Saving %s (%s)\n', fn_insar_phase, datestr(now)); -saveas(3001,fn_insar_phase); -fn_insar_phase = fullfile(fn_dir,[fn_name '_phase.png']); -fprintf('Saving %s (%s)\n', fn_insar_phase, datestr(now)); -saveas(3001,fn_insar_phase); - -[fn_dir,fn_name] = fileparts(fn); -fn_insar_coherence = fullfile(fn_dir,[fn_name '_coherence.fig']); -fprintf('Saving %s (%s)\n', fn_insar_coherence, datestr(now)); -saveas(3000,fn_insar_coherence); -fn_insar_coherence = fullfile(fn_dir,[fn_name '_coherence.png']); -fprintf('Saving %s (%s)\n', fn_insar_coherence, datestr(now)); -saveas(3000,fn_insar_coherence); +%% Save Figures +% ========================================================================= +if 0 + [fn_dir,fn_name] = fileparts(fn); + fn_insar_phase = fullfile(fn_dir,[fn_name '_phase.fig']); + fprintf('Saving %s (%s)\n', fn_insar_phase, datestr(now)); + saveas(3001,fn_insar_phase); + fn_insar_phase = fullfile(fn_dir,[fn_name '_phase.png']); + fprintf('Saving %s (%s)\n', fn_insar_phase, datestr(now)); + saveas(3001,fn_insar_phase); + + [fn_dir,fn_name] = fileparts(fn); + fn_insar_coherence = fullfile(fn_dir,[fn_name '_coherence.fig']); + fprintf('Saving %s (%s)\n', fn_insar_coherence, datestr(now)); + saveas(3000,fn_insar_coherence); + fn_insar_coherence = fullfile(fn_dir,[fn_name '_coherence.png']); + fprintf('Saving %s (%s)\n', fn_insar_coherence, datestr(now)); + saveas(3000,fn_insar_coherence); +end link_figures([3000 3001]); @@ -261,7 +373,9 @@ figure(3003); clf; Nt = size(insar_data_filt,1); Mt = 200; -dd=abs(fftshift(fft(insar_data_filt,Mt*Nt),1)).^2; +insar_data_filt_mask = insar_data_filt; +insar_data_filt_mask(~isfinite(insar_data_filt_mask)) = 0; +dd=abs(fftshift(fft(insar_data_filt_mask,Mt*Nt),1)).^2; along_track_filt = 2001; dd=fir_dec(dd,ones(1,along_track_filt)/along_track_filt,20); along_track_dec=fir_dec(along_track,ones(1,along_track_filt)/along_track_filt,20); diff --git a/cresis-toolbox/+tomo/tomo_quick_loader.m b/cresis-toolbox/+tomo/tomo_quick_loader.m index 6a69d0b4..6522837f 100644 --- a/cresis-toolbox/+tomo/tomo_quick_loader.m +++ b/cresis-toolbox/+tomo/tomo_quick_loader.m @@ -14,36 +14,37 @@ % Example of multipass datasets if 0 % Camp Century Multipass - % fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_insar/rds_thule_20110502_02_032_wf1_music.mat'; - % fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_insar/rds_thule_20110506_01_004_wf1_music.mat'; - % fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_insar/rds_thule_20110509_01_004_wf1_music.mat'; - % fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_insar/rds_thule_20110509_02_034_wf1_music.mat'; - - % fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_insar/rds_thule_20130419_01_004_wf1_music.mat'; - % fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_insar/rds_thule_20130426_01_004_wf1_music.mat'; - - % fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_insar/rds_thule_20140429_01_005_wf2_music.mat'; - % fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_insar/rds_thule_20140429_01_067_wf1_music.mat'; - % fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_insar/rds_thule_20140429_01_067_wf2_music.mat'; - % fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_insar/rds_thule_20140429_01_067_wf3_music.mat'; - - fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_insar/rds_thule_combine_wf2_music.mat'; - % fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_insar/rds_thule_combine_wf2_singlepass_music.mat'; + fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_multipass/camp_century_2011_2012_2013_2014_music.mat'; rbin = [250]; % Start bin rbins = [200]; % Number of bins -elseif 1 +elseif 0 + % Herc + fn = '/cresis/snfs1/dataproducts/ct_data/rds/2019_Antarctica_Ground/CSARP_music3D/20200107_01/Data_img_03_20200107_01_001.mat'; + + % Multipass + rbin = [200]; % Start bin + rbins = [500]; % Number of bins + +elseif 0 % Summit Multipass - % fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_insar/summit_2012_2014_wf2_music.mat'; - % fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_insar/summit_2012_2014_wf2_2012_music.mat'; - % fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_insar/summit_2012_2014_wf2_2014_music.mat'; fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_multipass/summit_2012_2014_allwf_2012_music.mat'; % Multipass rbin = [200]; % Start bin rbins = [500]; % Number of bins +elseif 1 + % EGIG Multipass + fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_multipass/egig_2011_2012_2014_2018_allwf_2014_music.mat'; + + % Multipass + rbin = [60]; % Start bin + rbins = [940]; % Number of bins + rbin = [100]; % Start bin + rbins = [400]; % Number of bins + elseif 0 % CAA 3D fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_music3D/20140325_07/Data_img_02_20140325_07_004.mat'; @@ -51,7 +52,6 @@ rbin = [300]; % Start bin rbins = [250]; % Number of bins - elseif 0 % Eqip 3D fn ='/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_music3D_Nsrc3_rbins3/20140414_02/Data_img_02_20140414_02_012.mat'; @@ -64,6 +64,15 @@ rbin = [250]; % Start bin rbins = [200]; % Number of bins + +elseif 0 + % 2018 Greenland P3 + fn ='/N/dcwan/projects/cresis/output/rds/2018_Greenland_P3/CSARP_music3D/20180406_01/Data_img_01_20180406_01_001.mat'; + fn ='/N/dcwan/projects/cresis/output/rds/2018_Greenland_P3/CSARP_music3D/20180406_01/Data_img_01_20180406_01_002.mat'; + + rbin = [600]; % Start bin + rbins = [1000]; % Number of bins + end %% Automated Section diff --git a/cresis-toolbox/+tomo/create_surfdata.m b/cresis-toolbox/+tomo/track_surface.m similarity index 57% rename from cresis-toolbox/+tomo/create_surfdata.m rename to cresis-toolbox/+tomo/track_surface.m index 5317467e..666694cb 100644 --- a/cresis-toolbox/+tomo/create_surfdata.m +++ b/cresis-toolbox/+tomo/track_surface.m @@ -1,5 +1,5 @@ -function create_surfdata(param,mdata) -% tomo.create_surfdata(param,mdata) +function track_surface(param,mdata) +% tomo.track_surface(param,mdata) % % Description: Usually this function is called from tomo_collate_task. % Using a surface DEM and an ice mask, this function adds an aligned @@ -7,7 +7,7 @@ function create_surfdata(param,mdata) % % Inputs: % param: struct from parameter spreadsheet -% .tomo_collate: struct which control create_surfdata +% .tomo_collate: struct which control track_surface % .out_dir: ct_filename_out directory to which output % surfData will be exported. Default is surfData. % .layer_params: opsLoadLayers parameter struct array. The first element @@ -48,50 +48,49 @@ function create_surfdata(param,mdata) % mdata: 3D data file struct (Data_YYYYMMDD_SS_FFF.mat with Tomo) % field) % +% .twtt: an Nsv by Nx double matrix of two way travel times to each point +% on the surface, default value is a matrix with twtt's set to equal the +% 2D surface twtt read in with opsLoadLayers. +% +% .ice_mask: an Nsv by Nx logical matrix representing whether or not ice +% is present for a given point of the surface (1 == ice, 0 == no ice). +% Default is all true (ice). +% % Outputs: % NONE % % See also: tomo.run_collate, tomo.collate, tomo_collate_task, -% tomo.fuse_images, tomo.add_icemask_surfacedem, tomo.create_surfdata, +% tomo.fuse_images, tomo.add_icemask_surfacedem, tomo.track_surface, % % Author: John Paden, Jordan Sprick, Mingze Xu, and Victor Berger +physical_constants; + if ~isfield(param.tomo_collate,'surf_out_path') || isempty(param.tomo_collate.surf_out_path) - param.tomo_collate.surf_out_path = 'surfData'; + param.tomo_collate.surf_out_path = 'surf'; end -% If DOA method is used, set doa_method_flag = true -array_proc_methods; % This script assigns the integer values for each method -if ischar(param.array.method) - % Convert array method string to integer - method_integer = []; - if regexpi(param.array.method,'music_doa') - method_integer(end+1) = MUSIC_DOA_METHOD; - end - if regexpi(param.array.method,'mle') - method_integer(end+1) = MLE_METHOD; - end - if regexpi(param.array.method,'dcm') - method_integer(end+1) = DCM_METHOD; - end - % if regexpi(param.array.method,'pf') - % method_integer(end+1) = PF_METHOD; - % end -end -method_integer = intersect(method_integer, ... - [MUSIC_DOA_METHOD MLE_METHOD DCM_METHOD PF_METHOD], 'stable'); -if ~isempty(method_integer) - doa_method_flag = true; -else - doa_method_flag = false; +% tomo_params: opsLoadLayers layer structure for top and bottom constraint +% layers. +% Typical: param.tomo_collate.tomo_params = struct('name',{'tomo_top','tomo_bottom'}); +if ~isfield(param.tomo_collate,'tomo_params') || isempty(param.tomo_collate.tomo_params) + param.tomo_collate.tomo_params = []; end if ~isfield(param.tomo_collate,'merge_bottom_above_top') ... || isempty(param.tomo_collate.merge_bottom_above_top) - param.tomo_collate.merge_bottom_above_top = 1; + param.tomo_collate.merge_bottom_above_top = true; end merge_bottom_above_top = param.tomo_collate.merge_bottom_above_top; +%% Input data prep +data = single(10*log10(mdata.Tomo.img)); +Nt = size(data,1); +Nsv = size(data,2); +Nx = size(data,3); +theta = mdata.Tomo.theta(:,1); +[~,nadir_idx] = min(abs(theta)); + %% Load surface and bottom information param_load_layers = param; param_load_layers.cmd.frms = round([-1,0,1] + param.load.frm); @@ -128,23 +127,41 @@ function create_surfdata(param,mdata) Surface = mdata.Surface; end -%% Interpolate Bottom, mdata.twtt from twtt to bins -if ~doa_method_flag - Bottom_bin = interp1(mdata.Time, 1:length(mdata.Time), Bottom); - Bottom_bin(isnan(Bottom_bin)) = -1; +if ~isempty(param.tomo_collate.tomo_params) + %% Load tomo_top and tomo_bottom information + tomo_layers = opsLoadLayers(param_load_layers,param.tomo_collate.tomo_params); + + %% Interpolate tomo_layers information to mdata + for lay_idx = 1:length(tomo_layers) + ops_layer = []; + ops_layer{1}.gps_time = tomo_layers(lay_idx).gps_time; + + ops_layer{1}.type = tomo_layers(lay_idx).type; + ops_layer{1}.quality = tomo_layers(lay_idx).quality; + ops_layer{1}.twtt = tomo_layers(lay_idx).twtt; + ops_layer{1}.type(isnan(ops_layer{1}.type)) = 2; + ops_layer{1}.quality(isnan(ops_layer{1}.quality)) = 1; + lay = opsInterpLayersToMasterGPSTime(master,ops_layer,[300 60]); + tomo_layers(lay_idx).twtt_ref = lay.layerData{1}.value{2}.data; + end + tomo_top_bin = uint32(round(interp_finite(interp1(mdata.Time,1:length(mdata.Time),tomo_layers(1).twtt_ref),-inf))); + tomo_bottom_bin = uint32(round(interp_finite(interp1(mdata.Time,1:length(mdata.Time),tomo_layers(2).twtt_ref),inf))); end + +%% Interpolate Bottom, mdata.twtt from twtt to bins +Bottom(~isfinite(Bottom)) = NaN; +Bottom_bin = interp1(mdata.Time, 1:length(mdata.Time), Bottom); +Bottom_bin(isnan(Bottom_bin)) = -1; +% ice_mask: an Nsv by Nx matrix, set to equal the 2D surface twtt if +% missing if ~isfield(mdata,'twtt') - mdata.twtt = layers(1).twtt; + mdata.twtt = repmat(layers(1).twtt_ref, [Nsv 1]); end +% ice_mask: an Nsv by Nx matrix, set to all ones if missing if isfield(mdata,'ice_mask') ice_mask = mdata.ice_mask; else - ice_mask = ones(size(mdata.twtt)); -end - -%% Surface tracking prep: Convert img to double and log-scale -if ~doa_method_flag - data = 10*log10(double(mdata.Tomo.img)); + ice_mask = true(size(mdata.twtt)); end %% Surface tracking prep @@ -154,12 +171,26 @@ function create_surfdata(param,mdata) % within mu_length of the top/bottom of the range line, so we truncate % surface to ensure this never happens. mu_length = 11; +dt = mdata.Time(2)-mdata.Time(1); +twtt_bin(mdata.twtt < mdata.Time(1)+(mu_length+1)*dt) = 1+mu_length; twtt_bin(isnan(twtt_bin) | twtt_bin > length(mdata.Time)-mu_length) = length(mdata.Time)-mu_length; -if doa_method_flag - twtt_bin(isnan(mdata.twtt) | (mdata.twtt==0)) = NaN; + +%% Open Ground Truth Surf File +if param.tomo_collate.gt.en + gt_dir = ct_filename_out(param,param.tomo_collate.gt.path,''); + gt_fn_name = sprintf('Data_%s_%03.0f.mat',param.day_seg,param.load.frm); + gt_fn = fullfile(gt_dir,gt_fn_name); + fprintf(' Loading ground truth %s\n', gt_fn); + gt_sd = tomo.surfdata(gt_fn); + gt_surf = gt_sd.get_surf(param.tomo_collate.gt.surf_name); + gt_theta = gt_sd.theta; + gt_time = gt_sd.time; + delete(gt_sd); + % Convert twtt to range bins + gt_bins = interp1(mdata.Time, 1:length(mdata.Time), gt_surf.y); end -%% Create output filename +%% Open/Create Surf File out_dir = ct_filename_out(param,param.tomo_collate.surf_out_path,''); if ~isdir(out_dir) mkdir(out_dir); @@ -176,9 +207,7 @@ function create_surfdata(param,mdata) sd = tomo.surfdata(out_fn); catch ME % Output file is not good, so we need to create - warning('Output surfData file exists, but could not be loaded. Run "dbcont" to overwrite the file.'); - keyboard - param.tomo_collate.surfData_mode = 'overwrite'; + error('Output surfData file exists, but could not be loaded. Fix, move or delete the file: %s.', out_fn); end end elseif ~strcmpi(param.tomo_collate.surfData_mode,'overwrite') @@ -186,395 +215,144 @@ function create_surfdata(param,mdata) end if strcmpi(param.tomo_collate.surfData_mode,'overwrite') - %% Create surfData - sd = tomo.surfdata(); - sd.radar_name = mdata.param_array.radar_name; - sd.season_name = mdata.param_array.season_name; - sd.day_seg = mdata.param_array.day_seg; - sd.frm = mdata.param_array.load.frm; - sd.gps_time = mdata.GPS_time; - if ~doa_method_flag - sd.theta = mdata.Tomo.theta(:,1); - else - sd.theta = mdata.Tomo.theta; - end - sd.time = mdata.Time(:); % Make a column vector - sd.FCS.origin = mdata.param_array.array_proc.fcs{1}{1}.origin; - sd.FCS.x = mdata.param_array.array_proc.fcs{1}{1}.x; - sd.FCS.y = mdata.param_array.array_proc.fcs{1}{1}.y; - sd.FCS.z = mdata.param_array.array_proc.fcs{1}{1}.z; -end - -if ~doa_method_flag - Nsv = size(mdata.Tomo.img,2); -else - Nx = size(mdata.Tomo.theta,3); - for Nx_idx = 1:Nx - theta_tmp = mdata.Tomo.theta(:,:,Nx_idx); - theta_tmp = theta_tmp(~isnan(theta_tmp)); - max_Nsv(Nx_idx) = length(theta_tmp); - end - Nsv = max(max_Nsv); + sd = tomo.surfdata(mdata); end -if ~doa_method_flag - % Beamforming method - try - surf = sd.get_surf('top'); - if strcmpi(param.tomo_collate.surfData_mode,'overwrite') - surf.y = twtt_bin; - sd.set_surf(surf); - end - catch ME - surf = tomo.surfdata.empty_surf(); - surf.x = repmat((1:Nsv).',[1 size(mdata.twtt,2)]); - surf.y = twtt_bin; - surf.plot_name_values = {'color','black','marker','x'}; - surf.name = 'top'; - sd.insert_surf(surf); - end -else - % DOA method - try - surf = sd.get_surf('top'); - if strcmpi(param.tomo_collate.surfData_mode,'overwrite') - surf.y = twtt_bin; - sd.set_surf(surf); - end - catch ME - surf = tomo.surfdata.empty_surf(); - surf.y = twtt_bin; - surf.x = NaN(size(twtt_bin)); - for rline_idx = 1:Nx - theta_rline = mdata.Tomo.theta(:,:,rline_idx); - if ~all(isnan(theta_rline(:))) - theta_rline = theta_rline(~isnan(theta_rline)); - surf.x(1:length(theta_rline),rline_idx) = theta_rline; - end - end - % Sort DOA min to max (and, accordingly, range-bins). But surf.y is - % already sorted inside add_icemask_surfacedem - [surf.x x_idx] = sort(surf.x*180/pi,1,'ascend'); -% for rline_idx = 1:Nx -% surf.y(:,rline_idx) = surf.y(x_idx(:,rline_idx),rline_idx); -% end - - surf.plot_name_values = {'color','black','marker','*'}; % 'x' - surf.name = 'top'; - sd.insert_surf(surf); +% Insert top surface +% ------------------------------------------------------------------------- +try + surf = sd.get_surf('top'); + if strcmpi(param.tomo_collate.surfData_mode,'overwrite') + surf.y = mdata.twtt; + sd.set_surf(surf); end - ice_top.x = surf.x; - ice_top.y = surf.y; +catch ME + surf = tomo.surfdata.empty_surf(); + surf.x = repmat(theta,[1 Nx]); + surf.y = mdata.twtt; + surf.plot_name_values = {'color','black','marker','x'}; + surf.name = 'top'; + sd.insert_surf(surf); end - -if ~doa_method_flag - % Beamforming method - try - surf = sd.get_surf('bottom'); - if strcmpi(param.tomo_collate.surfData_mode,'overwrite') - surf.y = NaN(size(twtt_bin)); - sd.set_surf(surf); - end - catch ME - surf = tomo.surfdata.empty_surf(); - surf.x = repmat((1:Nsv).',[1 size(mdata.twtt,2)]); +% Insert bottom surface +% ------------------------------------------------------------------------- +try + surf = sd.get_surf('bottom'); + if strcmpi(param.tomo_collate.surfData_mode,'overwrite') surf.y = NaN(size(twtt_bin)); - surf.plot_name_values = {'color','blue','marker','^'}; - surf.name = 'bottom'; - sd.insert_surf(surf); - end -else - % DOA method - try - surf = sd.get_surf('bottom'); - if strcmpi(param.tomo_collate.surfData_mode,'overwrite') - surf.y = NaN(size(twtt_bin)); - sd.set_surf(surf); - end - catch ME - surf = tomo.surfdata.empty_surf(); - surf.x = NaN(size(twtt_bin)); - for rline_idx = 1:Nx - theta_rline = mdata.Tomo.theta(:,:,rline_idx); - theta_rline = theta_rline(~isnan(theta_rline)); - if ~all(isnan(theta_rline(:))) - surf.x(1:length(theta_rline),rline_idx) = theta_rline; - end - end - surf.y = NaN(size(twtt_bin)); % Will be created later in this script - % Sort DOA min to max (and, accordingly, range-bins) - [surf.x x_idx] = sort(surf.x*180/pi,1,'ascend'); - for rline_idx = 1:Nx - surf.y(:,rline_idx) = surf.y(x_idx(:,rline_idx),rline_idx); - end - % Ensure non-negative ice thickness - if merge_bottom_above_top && exist('ice_top','var') && isfield(ice_top,'y') && ~isempty(ice_top.y) - surf.y(surf.y= 1 + % Set the bins above the ground truth in the nadir column to -inf + data(1:Bottom_bin(rline)-gt_range,nadir_col,rline) = -inf; + % Set the bins below the ground truth in the nadir column to -inf + data(Bottom_bin(rline)+gt_range:end,nadir_col,rline) = -inf; + end + end + % Optional ground truth + if param.tomo_collate.gt.en + for rline = 1:Nx + % Interpolate ground truth surf data elevation angle (steering + % vector) axis to 3D image file elevation angle axis + gt_bins_rline = round(interp1(gt_surf.x(:,rline), gt_bins(:,rline), theta)); + for sv_idx = 1:Nsv + if ~isnan(gt_bins_rline(sv_idx)) + data([1:gt_bins_rline(sv_idx)-param.tomo_collate.gt.range, gt_bins_rline(sv_idx)+param.tomo_collate.gt.range:end], sv_idx, rline) = -inf; + end + end + end + end - if length(transition_mu) ~= Nsv - transition_mu = imresize(transition_mu, [1 Nsv]); + if param.tomo_collate.suppress_surf_flag + repulsion_val = -1e6; + Bpval = param.tomo_collate.suppress_surf_peak_val; + surf_window = param.tomo_collate.suppress_surf_window; + % Bpval = 30; + % surf_window = 110; + + indexes = 0:surf_window-1; + indexes = indexes(:); + suppress_trend = Bpval*0.5*(1 - ((indexes./surf_window) - ((surf_window - indexes)./surf_window))); + suppress_trend = suppress_trend(:); + + for rline = 1:Nx + slice = squeeze(data(:,:,rline)); + twtt_rline = twtt_bin(:,rline); + ice_mask_rline = ice_mask(:,rline); + for doa_idx = 1:Nsv + slice_line = slice(:,doa_idx); + surf_bin = twtt_rline(doa_idx); + data(1:surf_bin - 1,doa_idx,rline) = repulsion_val; % Suppress everything above the surface (and including?) + surf_row_indexes = surf_bin +indexes; + if surf_row_indexes(end) > Nt + mask = surf_row_indexes <= Nt; + trunc_indexes = surf_row_indexes(mask); + trunc_suppress_trend = suppress_trend(mask); + data(trunc_indexes(:),doa_idx,rline) = data(trunc_indexes(:),doa_idx,rline) - trunc_suppress_trend; + else + data(surf_row_indexes(:),doa_idx,rline) = data(surf_row_indexes(:),doa_idx,rline) - suppress_trend; + end + if ~ice_mask_rline(doa_idx) + data(surf_bin+1:end,doa_idx,rline) = repulsion_val; + end + end + end + end - if length(transition_sigma) ~= Nsv - transition_sigma = imresize(transition_sigma, [1 Nsv]); + if 0 + rline = 2295; + figure(32);clf;imagesc(1:Nsv,mdata.Time,squeeze(data(:,:,rline)));hold on + caxis([0 30]) + plot(1:Nsv,mdata.twtt(:,rline),'Color',[1 1 1],'Marker','x','MarkerSize',8) end - if length(RLINE_transition_sigma) ~= Nsv - RLINE_transition_sigma = imresize(RLINE_transition_sigma, [1 Nsv]); + dr = dt*c/2; + at_slope = single([diff(mdata.Elevation / dr) 0]); + + H = interp_finite(Surface,500)*c/2; + T = interp_finite(Bottom,1000)*c/2/sqrt(er_ice); + R = 1./cos(theta) * H + 1./cos(theta_ice) * T; + ct_slope = single([zeros(1,Nx); diff(R)/dr]+[diff(R)/dr; zeros(1,Nx)]); + ct_slope(2:end-1,:) = ct_slope(2:end-1,:)/2; + ct_slope = interp_finite(ct_slope); + + ct_weight = 1./(3+mean(abs(ct_slope),2)); + ct_weight = ct_weight_max*ct_weight./max(ct_weight); + ct_weight = single(ct_weight); + + if isempty(param.tomo_collate.tomo_params) + bounds = uint32([ones(1,Nx); Nt*ones(1,Nx)]); + else + bounds = [tomo_top_bin; tomo_bottom_bin]; + bounds(bounds<1) = 1; + bounds(bounds>Nt) = Nt; + bounds = bounds - 1; end - % Visualization of mean and variance vectors + + % if no ice match to surface + if 0 - figure; (plot(transition_mu)); hold on; - plot(transition_sigma); xlim([1 64]) - legend('Mean', 'Variance', 'Location', 'northwest'); - xlabel('DoA bins'); + %% TRW-S: Test Code (stop here) + tmp_data = data; + tmp_at_slope = at_slope; + tmp_ct_slope = ct_slope; + tmp_bounds = bounds; + Bottom_bin = round(Bottom_bin); + keyboard + %% TRW-S: Test Code (run by hand) + % rbins = 1000:1300; + rbins = 1:size(tmp_data,1); + % rlines = 1:100; + rlines = 1:size(tmp_data,3); + % Subset data to make the runtime shorter + data = tmp_data(rbins,:,rlines); + + at_weight = single(0.01); + ct_weight_max = 0.01; + max_loops = uint32(50); + + at_slope = tmp_at_slope(rlines); + ct_slope = tmp_ct_slope(:,rlines); + bounds = tmp_bounds(:,rlines); + bounds = bounds - rbins(1) + 1; + ct_weight = single(ct_weight_max * ct_weight / max(ct_weight)); + + tic + trws_surface = tomo.trws2(data,at_slope,at_weight,ct_slope,ct_weight,max_loops,bounds); + toc + + figure(1); clf; + for rline = 1:10:size(data,3) + imagesc(data(:,:,rline)) + hold on; plot(trws_surface(:,rline),'rx') + title(sprintf('%d',rline)); + pause + end + + figure(2); clf; + imagesc(trws_surface); colorbar; end - smooth_weight = 0.08 .* smooth_weight; - gt = zeros(3, length(Bottom_bin)); - gt(1, :) = 1 : length(Bottom_bin); - gt(2, :) = round(Nsv ./ 2) * ones(1, length(Bottom_bin)); - gt(3, :) = Bottom_bin(:) + 0.5; - - tic; - trws_surface = tomo.trws(double(data), ... - double(twtt_bin), double(Bottom_bin), double(gt), double(ice_mask), ... - double(mu), double(sigma), double(smooth_weight), double(smooth_var), ... - double(smooth_slope), double([]), double(max_loops), int64(bounds), ... - double(mask_dist), double(DIM_costmatrix), ... - double(transition_mu), double(transition_sigma), ... - double(RLINE_transition_sigma)); - toc; - trws_surface = reshape(trws_surface,size(mdata.Tomo.img,2), ... - size(mdata.Tomo.img,3)); + trws_surface = tomo.trws2(data,at_slope,at_weight,ct_slope,ct_weight,max_loops,bounds); + + trws_surface = double(reshape(trws_surface,size(mdata.Tomo.img,2), ... + size(mdata.Tomo.img,3))); for surf_name_idx = 1:length(surf_names) surf_name = surf_names{surf_name_idx}; try surf = sd.get_surf(surf_name); if ~strcmpi(param.tomo_collate.surfData_mode,'fillgaps') - surf.y = trws_surface; + surf.y = interp1(1:length(mdata.Time), mdata.Time, trws_surface); surf.plot_name_values = plot_name_values; surf.visible = visible; sd.set_surf(surf); end catch ME surf = tomo.surfdata.empty_surf(); - surf.x = repmat((1:Nsv).',[1 size(mdata.twtt,2)]); - surf.y = trws_surface; + surf.x = repmat(theta,[1 Nx]); + surf.y = interp1(1:length(mdata.Time), mdata.Time, trws_surface); surf.name = surf_name; surf.plot_name_values = plot_name_values; surf.visible = visible; @@ -1117,6 +990,7 @@ function create_surfdata(param,mdata) end elseif strcmpi(cmd,'c3d_rnn') + %% NN c3d_rnn.dwnsammat_dir = fullfile(ct_filename_out(param, 'C3D_RNN_temporary_resources'), ''); c3d_rnn.dwnsamnpy_dir = fullfile(ct_filename_out(param, 'C3D_RNN_temporary_resources'), ''); temp_str = strfind(c3d_rnn.dwnsammat_dir, filesep); @@ -1149,7 +1023,7 @@ function create_surfdata(param,mdata) end fprintf('\nFinished down-sampling and saving MAT files for %s_%03.0f.\nPath: %s\n\n',param.day_seg,param.proc.frm, out_dir); - %% Search for pre-trained model files (c3d.pth and rnn.pth) + %% NN: Search for pre-trained model files (c3d.pth and rnn.pth) c3d_rnn.pth_path = fullfile(param.path, '+tomo', 'c3d_rnn_models'); c3d_rnn.c3d_pth_path = fullfile(c3d_rnn.pth_path, 'c3d.pth'); c3d_rnn.rnn_pth_path = fullfile(c3d_rnn.pth_path, 'rnn.pth'); @@ -1160,7 +1034,7 @@ function create_surfdata(param,mdata) keyboard end - %% Convert from MAT to NPY and run C3D_RNN + %% NN: Convert from MAT to NPY and run C3D_RNN % Calls shell script fprintf('Executing shell script to run Python scripts...\n\n'); try @@ -1193,7 +1067,7 @@ function create_surfdata(param,mdata) c3d_rnn.result_bottom = ones(size(mdata.Topography.img,2), size(mdata.Topography.img,3)); sl_idx = 1; - %% Get surface and bottom vectors from generated text file + %% NN: Get surface and bottom vectors from generated text file fid = fopen(c3d_rnn.outtext_dir); tline = ''; @@ -1230,15 +1104,15 @@ function create_surfdata(param,mdata) try surf = sd.get_surf(surf_name); if ~strcmpi(param.tomo_collate.surfData_mode,'fillgaps') - surf.y = c3d_rnn.result_matrix(:, :, surf_name_idx); + surf.y = interp1(1:length(mdata.Time), mdata.Time, c3d_rnn.result_matrix(:, :, surf_name_idx)); surf.plot_name_values = plot_name_values; surf.visible = visible; sd.set_surf(surf); end catch ME surf = tomo.surfdata.empty_surf(); - surf.x = repmat((1:Nsv).',[1 size(mdata.twtt,2)]); - surf.y = c3d_rnn.result_matrix(:, :, surf_name_idx); + surf.x = repmat(theta,[1 Nx]); + surf.y = interp1(1:length(mdata.Time), mdata.Time, c3d_rnn.result_matrix(:, :, surf_name_idx)); surf.name = surf_name; surf.plot_name_values = plot_name_values; surf.visible = visible; @@ -1247,63 +1121,10 @@ function create_surfdata(param,mdata) sd.set(surf_name,'top','top','active','bottom','mask','ice mask', ... 'gt','bottom gt','quality','bottom quality'); end - elseif strcmpi(cmd,'doa') - %% DOA method: Only 'bottom' is supported at this point - doa_surface.x = NaN(size(twtt_bin)); - doa_surface.y = NaN(size(twtt_bin)); - for rline_idx = 1:Nx - theta_rline = mdata.Tomo.theta(:,:,rline_idx); - if ~all(isnan(theta_rline(:))) - [theta_rline_r theta_rline_c] = find(~isnan(theta_rline)); - theta_rline = theta_rline(~isnan(theta_rline)); - % Sort DOA min to max (and, accordingly, range-bins) -% doa_surface.x(1:length(theta_rline),rline_idx) = theta_rline; - [doa_surface.x(1:length(theta_rline),rline_idx), x_idx] = sort(theta_rline*180/pi,'ascend'); - doa_surface.y(x_idx,rline_idx) = theta_rline_r; - end - end - - % Sort DOA min to max (and, accordingly, range-bins) -% [doa_surface.x x_idx] = sort(doa_surface.x*180/pi,1,'ascend'); -% for rline_idx = 1:Nx -% doa_surface.y(:,rline_idx) = doa_surface.y(x_idx(:,rline_idx),rline_idx); -% end - for surf_name_idx = 1:length(surf_names) - surf_name = surf_names{surf_name_idx}; - try - surf = sd.get_surf(surf_name); - if ~strcmpi(param.tomo_collate.surfData_mode,'fillgaps') - surf.y = doa_surface.y; - % Ensure non-negative ice thickness - if merge_bottom_above_top && exist('ice_top','var') && isfield(ice_top,'y') && ~isempty(ice_top.y) - surf.y(surf.y +#include +#include +using namespace std; +#include "mex.h" +#include "trws2.h" + +void print_time(void) +{ + char time_str[256]; + time_t curr_time; + tm *curr_tm; + + time(&curr_time); + curr_tm = localtime(&curr_time); + strftime(time_str,256,"%T",curr_tm); + mexPrintf("%s\n", time_str); + mexEvalString("keyboard;"); +} + + +class TRWS { +public: + // mNt: Fast-time dimension (number of rows/first-dimension in mImage) + size_t mNt; + // mNsv: Cross-track dimension (number of columns/second-dimension in mImage) + size_t mNsv; + // mNsv_center: Starting index of TRW-S iteration in cross-track dimension + size_t mNsv_center; + // mNx: Along-track dimension (number of range lines/third-dimension in mImage) + size_t mNx; + // mMax_Loops: Number of message passing loops/iterations to run + const unsigned int mMax_Loops; + // mImage: 3D mNt*mNsv*mNx mImage + const float *mImage; + // message_DIRECTION: mNt*mNsv*mNx message matrixes. These are like an inbox: + // they represent messages sent from other nodes to the node with the + // corresponding index. + float *mMessage_Left; // Message from node to the left + float *mMessage_Up; // Message from node below + float *mMessage_Right; // Message from node to the right + float *mMessage_Down; // Message from node above + + // mAT_Slope: along-track expected slope (should generally compensate for radar platform elevation changes) + const float *mAT_Slope; + // mAT_Weight: along-track binary/transition/slope weight + const float *mAT_Weight; + // mCT_Slope: cross-track binary/transition/slope coefficients (first row: range, second row: theta) + const float *mCT_Slope; + // mCT_Weight: cross-track binary/transition/slope weight + const float *mCT_Weight; + // mBounds: fast-time dimension bounds + const unsigned int *mBounds; + + // Result + unsigned int *mResult; + + TRWS(const float *image, const size_t *dim_image, const float *at_slope, const float *at_weight, + const float *ct_slope, const float *ct_weight, const unsigned int max_loops, + const unsigned int *bounds, unsigned int *result) + : mImage(image), mAT_Slope(at_slope), mAT_Weight(at_weight), mCT_Slope(ct_slope), mCT_Weight(ct_weight), + mMax_Loops(max_loops), mBounds(bounds), mResult(result) { + // Set dimensions + mNt = dim_image[0]; + mNsv = dim_image[1]; + mNx = dim_image[2]; + mNsv_center = round(mNsv/2); + + // Allocate message inboxes + // calloc sets the values to zero which corresonds to floating point value of 0 + mMessage_Left = (float*)calloc(mNt*mNsv*mNx,sizeof(float)); + mMessage_Right = (float*)calloc(mNt*mNsv*mNx,sizeof(float)); + mMessage_Up = (float*)calloc(mNt*mNsv*mNx,sizeof(float)); + mMessage_Down = (float*)calloc(mNt*mNsv*mNx,sizeof(float)); + } + + ~TRWS() { + free(mMessage_Left); + free(mMessage_Right); + free(mMessage_Up); + free(mMessage_Down); + } + + // Set mResult + void set_result(); + // Extract surface + void solve(); +}; + +void TRWS::set_result() { + + size_t result_idx = 0; + for (size_t w = 0; w < mNx; w++) { + for (size_t h = 0; h < mNsv; h++) { + float min_val = INFINITY; + size_t best_result = mNx; + + for (size_t d = mBounds[2*w], message_idx = h*mNt + w*mNsv*mNt+mBounds[2*w]; d <= mBounds[2*w+1]; d++, message_idx++) { + // Unary cost + float temp = -mImage[message_idx]; + // Binary costs (up,down,left,right) + temp += mMessage_Left[message_idx]; + temp += mMessage_Up[message_idx]; + temp += mMessage_Right[message_idx]; + temp += mMessage_Down[message_idx]; + + // Check to see if this d is the minimum + if (temp < min_val) { + min_val = temp; + best_result = d; + } + } + + mResult[result_idx] = (int)(best_result+1); + result_idx = result_idx + 1; + } + } +} + +void TRWS::solve() { + // mNsvs_array: array of indices that allow iteration to go from the + // midpoint outward + // + // Propagate h/mNsv from the midpoint out (mNsv_center) and let it be + // biased by the current loop's mResults. This means the starting point + // can have a large affect on the result because propagation away from + // the starting point is much more effective and far reaching. In our + // case, the midpoint always has ground truth so it is a good. + float mNsvs_array[mNsv]; + for (int i=0; i<=mNsv_center; i++) { + mNsvs_array[i] = mNsv_center-i; // From center to left + } + for (int i=mNsv_center+1; i 0) { + mexPrintf("\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b", loop); + time_t curr_time; + time(&curr_time); + mexPrintf(" loop: %2d of %2d, time per loop: %5.0f seconds\n", loop+1, mMax_Loops, double(curr_time-start_time)/double(loop)); + } else { + mexPrintf(" loop: %2d of %2d \n", loop+1, mMax_Loops); + } + + // Loop through along-track dimension + for (size_t w_idx = 0; w_idx < mNx; w_idx++) { + size_t w; + if (loop%2) { + // Iterating right to left + w = mNx-1-w_idx; + } else { + // Iterating left to right + w = w_idx; + } + //mexPrintf(" w: %5d of %5d\n", w+1, mNx); + // To allow the Matlab program to continue to take GUI inputs while + // TRWS in running, call drawnow. + mexEvalString("drawnow;"); + + // Loop through cross-track dimension + for (size_t h_idx = 0; h_idx < mNsv; h_idx++) { + // Iterate from the center always + size_t h = mNsvs_array[h_idx]; + // Index to start of current range line in along-track/elevation angle in cross-track + size_t cur_message_idx = (h + w*mNsv)*mNt; + int cur_rbin_start = mBounds[2*w]; + int cur_rbin_stop = mBounds[2*w+1]; + + // Message to node on the left of current + // Sum all input messages and create the message for the node on the right + // Messages the current node sends that node is called "mMessage_Right" for that node + // Messages from the node to the right are stored in this current node's mMessage_Left + // ---------------------------------------------------------------- + for (size_t d = cur_rbin_start, message_idx = cur_message_idx+cur_rbin_start; d <= cur_rbin_stop; d++, message_idx++) { + message_sum[d] = mMessage_Up[message_idx] + mMessage_Right[message_idx] + mMessage_Down[message_idx] - mImage[message_idx]; + // message_in for node on the right (w-1) excludes mMessage_Left which came from the node on the right + } + if (w > 0) { + // Message destination index + size_t msg_dest_idx = (h + (w-1)*mNsv)*mNt; + int dest_rbin_start = mBounds[2*w-2]; + int dest_rbin_stop = mBounds[2*w-1]; + // Add binary cost to message being sent to node on the right + dt(message_sum, &(mMessage_Right[msg_dest_idx]), cur_rbin_start, cur_rbin_stop, dest_rbin_start, dest_rbin_stop, *mAT_Weight, -mAT_Slope[w]); + // Normalize message so smallest message has a cost of zero + float min_val = INFINITY; + for (size_t d = dest_rbin_start, message_idx = msg_dest_idx+dest_rbin_start; d <= dest_rbin_stop; d++, message_idx++) { + if (mMessage_Right[message_idx] < min_val) { + min_val = mMessage_Right[message_idx]; + } + } + for (size_t d = dest_rbin_start, message_idx = msg_dest_idx+dest_rbin_start; d <= dest_rbin_stop; d++, message_idx++) { + mMessage_Right[message_idx] = mMessage_Right[message_idx]-min_val; + } + } + + // Message to node above the current + // ---------------------------------------------------------------- + // Input message + for (size_t d = cur_rbin_start, message_idx = cur_message_idx+cur_rbin_start; d <= cur_rbin_stop; d++, message_idx++) { + // Add in the missing left message + message_sum[d] = message_sum[d] + mMessage_Left[message_idx]; + // message_in for node above (h-1) the current excludes mMessage_Up + message_in[d] = message_sum[d] - mMessage_Up[message_idx]; + } + if (h > 0) { + size_t msg_dest_idx = (h-1 + w*mNsv)*mNt; + int dest_rbin_start = mBounds[2*w]; + int dest_rbin_stop = mBounds[2*w+1]; + // Binary cost + dt(message_in, &(mMessage_Down[msg_dest_idx]), cur_rbin_start, cur_rbin_stop, dest_rbin_start, dest_rbin_stop, mCT_Weight[h], mCT_Slope[h+w*mNsv]); + // Normalize message so smallest message has a cost of zero + float min_val = INFINITY; + for (size_t d = dest_rbin_start, message_idx = msg_dest_idx+dest_rbin_start; d <= dest_rbin_stop; d++, message_idx++) { + if (mMessage_Down[message_idx] < min_val) { + min_val = mMessage_Down[message_idx]; + } + } + for (size_t d = dest_rbin_start, message_idx = msg_dest_idx+dest_rbin_start; d <= dest_rbin_stop; d++, message_idx++) { + mMessage_Down[message_idx] = mMessage_Down[message_idx]-min_val; + } + } + + // Message to node on the right of current + // ---------------------------------------------------------------- + for (size_t d = cur_rbin_start, message_idx = cur_message_idx+cur_rbin_start; d <= cur_rbin_stop; d++, message_idx++) { + message_in[d] = message_sum[d] - mMessage_Right[message_idx]; + } + if (w < mNx-1) { + size_t msg_dest_idx = (h + (w+1)*mNsv)*mNt; + int dest_rbin_start = mBounds[2*w+2]; + int dest_rbin_stop = mBounds[2*w+3]; + dt(message_in, &(mMessage_Left[msg_dest_idx]), cur_rbin_start, cur_rbin_stop, dest_rbin_start, dest_rbin_stop, *mAT_Weight, mAT_Slope[w]); + // Normalize message so smallest message has a cost of zero + float min_val = INFINITY; + for (size_t d = dest_rbin_start, message_idx = msg_dest_idx+dest_rbin_start; d <= dest_rbin_stop; d++, message_idx++) { + if (mMessage_Left[message_idx] < min_val) { + min_val = mMessage_Left[message_idx]; + } + } + for (size_t d = dest_rbin_start, message_idx = msg_dest_idx+dest_rbin_start; d <= dest_rbin_stop; d++, message_idx++) { + mMessage_Left[message_idx] = mMessage_Left[message_idx]-min_val; + } + } + + // Message to node below the current + // ---------------------------------------------------------------- + for (size_t d = cur_rbin_start, message_idx = cur_message_idx+cur_rbin_start; d <= cur_rbin_stop; d++, message_idx++) { + message_in[d] = message_sum[d] - mMessage_Down[message_idx]; + } + if (h < mNsv-1) { + size_t msg_dest_idx = (h+1 + w*mNsv)*mNt; + int dest_rbin_start = mBounds[2*w]; + int dest_rbin_stop = mBounds[2*w+1]; + dt(message_in, &(mMessage_Up[msg_dest_idx]), cur_rbin_start, cur_rbin_stop, dest_rbin_start, dest_rbin_stop, mCT_Weight[h], -mCT_Slope[h+w*mNsv]); + // Normalize message so smallest message has a cost of zero + float min_val = INFINITY; + for (size_t d = dest_rbin_start, message_idx = msg_dest_idx+dest_rbin_start; d <= dest_rbin_stop; d++, message_idx++) { + if (mMessage_Up[message_idx] < min_val) { + min_val = mMessage_Up[message_idx]; + } + } + for (size_t d = dest_rbin_start, message_idx = msg_dest_idx+dest_rbin_start; d <= dest_rbin_stop; d++, message_idx++) { + mMessage_Up[message_idx] = mMessage_Up[message_idx]-min_val; + } + } + + } + } + } + set_result(); +} + +// MATLAB FUNCTION START +void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]) { + if (nrhs != 7 || nlhs != 1) { + mexErrMsgTxt("Usage: uint32 labels = trws(single image, single at_slope, single at_weight, single ct_slope, single ct_weight, uint32 max_loops, uint32 bounds)\n\n size(image) is [Nt,Nsv,Nx]\n mean along-track slope numel(at_slope) is Nx (last element not used)\n along-track slope weight numel(at_weight) is 1\n cross-track slope coefficients numel(ct_slope) is Nsv (last element not used)\n cross-track slope weight numel(ct_weight) is Nsv (last element not used)\n numel(max_loops) is 1"); + } + + // image ================================================================ + if (!mxIsSingle(prhs[0])) { + mexErrMsgTxt("usage: image must be type single"); + } + if (mxGetNumberOfDimensions(prhs[0]) != 3) { + mexErrMsgTxt("usage: image must be a 3D matrix [rows=Nt, columns=Ndoa, slices=Nx]"); + } + const size_t *dim_image = mxGetDimensions(prhs[0]); + float *image = (float *)mxGetData(prhs[0]); + // dim_image[0]: Nt rows of one slice (fast-time: the hidden state we are trying to estimate) + // dim_image[1]: Nsv cols of one slice (cross-track dimension) + // dim_image[2]: Nx number of slices (along-track dimension) + + // at_slope ============================================================= + if (!mxIsSingle(prhs[1])) { + mexErrMsgTxt("usage: at_slope must be type single (float32)"); + } + if (mxGetNumberOfElements(prhs[1]) != dim_image[2]) { + mexErrMsgTxt("usage: at_slope must have numel equal to size(image,3)"); + } + float *at_slope = (float *)mxGetData(prhs[1]); + + // at_weight ============================================================ + if (!mxIsSingle(prhs[2])) { + mexErrMsgTxt("usage: at_weight must be type single (float32)"); + } + if (mxGetNumberOfElements(prhs[2]) != 1) { + mexErrMsgTxt("usage: at_weight must have numel equal to 1"); + } + float *at_weight = (float *)mxGetData(prhs[2]); + + // ct_slope ============================================================= + if (!mxIsSingle(prhs[3])) { + mexErrMsgTxt("usage: ct_slope must be type single (float32)"); + } + const size_t *dim_ct_slope = mxGetDimensions(prhs[3]); + if (dim_ct_slope[0] != dim_image[1] || dim_ct_slope[1] != dim_image[2]) { + mexErrMsgTxt("usage: ct_slope must have size(ct_slope,1)=size(image,2) and size(ct_slope,2)=size(image,3)"); + } + float *ct_slope = (float *)mxGetData(prhs[3]); + + // ct_weight ============================================================ + if (!mxIsSingle(prhs[4])) { + mexErrMsgTxt("usage: ct_weight must be type single (float32)"); + } + if (mxGetNumberOfElements(prhs[4]) != dim_image[1]) { + mexErrMsgTxt("usage: ct_weight must have numel equal to size(image,2)"); + } + float *ct_weight = (float *)mxGetData(prhs[4]); + + // max_loops =========================================================== + if (!mxIsClass(prhs[5],"uint32")) { + mexErrMsgTxt("usage: max_loops must be type unsigned int32"); + } + if (mxGetNumberOfElements(prhs[5]) != 1) { + mexErrMsgTxt("usage: max_loops must have numel equal to 1"); + } + unsigned int *max_loops = (unsigned int *)mxGetData(prhs[5]); + + // bounds =============================================================== + if (!mxIsClass(prhs[6],"uint32")) { + mexErrMsgTxt("usage: bounds must be type unsigned int32"); + } + const size_t *dim_bounds = mxGetDimensions(prhs[6]); + if (dim_bounds[0] != 2 || dim_bounds[1] != dim_image[2]) { + mexErrMsgTxt("usage: bounds must have size(bounds,1)=2 and size(bounds,2)=size(image,3)"); + } + unsigned int *bounds = (unsigned int *)mxGetData(prhs[6]); + + // ==================================================================== + + // Allocate output + plhs[0] = mxCreateNumericMatrix(dim_image[1], dim_image[2], mxUINT32_CLASS, mxREAL); + unsigned int *result = (unsigned int *)mxGetData(plhs[0]); + + // Run TRWS algorithm + TRWS obj(image, dim_image, at_slope, at_weight, ct_slope, ct_weight, *max_loops, bounds, result); + + obj.solve(); +} diff --git a/cresis-toolbox/+tomo/trws2.h b/cresis-toolbox/+tomo/trws2.h new file mode 100644 index 00000000..8f1bfb19 --- /dev/null +++ b/cresis-toolbox/+tomo/trws2.h @@ -0,0 +1,40 @@ +// trws.h: Some very helpful variables and functions. +// By Mingze Xu, November 2016 +// +// Minor style changes to comply with new C++ standards: Victor Berger 2018 +// +#ifndef _INSTANCES_H_ +#define _INSTANCES_H_ +#define _USE_MATH_DEFINES + +#include +#include + +// Compute square value +template inline T sqr(T x) { return x*x; } + +// THE CODE BELOW THIS POINT WAS TAKEN FROM DAVID CRANDALL +// Distance transform +// -- Every index from d1 to d2 will be set in dst +// -- dst will contain the minimum value for that destination +// -- dst_ind will contain the minimum source index for that destination +void dt(const float *src, float *dst, int s1, int s2, + int d1, int d2, float scale, int off = 0) { + + int d = (d1 + d2) >> 1, s = s1; // Find the midpoint of the destination + for (int p = s1; p <= s2; p++) { // Search through all the sources and find the minimum + if (src[s] + sqr(s-d-off) * scale > src[p] + sqr(p-d-off) * scale) { + s = p; + } + } + dst[d] = src[s] + sqr(s-d-off) * scale; // Minimum value to the midpoint + if(d2 >= d + 1) { // Recursive call, binary search (top half of destinations) + dt(src, dst, s, s2, d+1, d2, scale, off); + } + if(d-1 >= d1) { // Recursive call, binary search (bottom half of destinations) + dt(src, dst, s1, s, d1, d-1, scale, off); + } +} +// END CODE FROM DAVID CRANDALL + +#endif diff --git a/cresis-toolbox/+tomo/trws2.mexa64 b/cresis-toolbox/+tomo/trws2.mexa64 new file mode 100755 index 00000000..b9596186 Binary files /dev/null and b/cresis-toolbox/+tomo/trws2.mexa64 differ diff --git a/cresis-toolbox/+tomo/trws_sim.m b/cresis-toolbox/+tomo/trws_sim.m new file mode 100644 index 00000000..c46719c6 --- /dev/null +++ b/cresis-toolbox/+tomo/trws_sim.m @@ -0,0 +1,73 @@ +num_nodes = 20; +num_states = 10; +num_loops = 2; +I = ones(num_states,num_nodes); +I = 1*randn(num_states,num_nodes); +I(1:5,1:3) = I(1:5,1:3) + 3.5; +I(6,[1]) = I(6,[1]) + [10]; +I(7,[4 10]) = I(7,[4 10]) + [5 7]; +I(5,[13 15]) = I(5,[13 15]) + [5 7]; +I(4,[19]) = I(4,[19]) + [4]; +msg_left = zeros(num_states,num_nodes); +msg_right = zeros(num_states,num_nodes); +old_msg_left = zeros(num_states,num_nodes); +old_msg_right = zeros(num_states,num_nodes); +trws_en = true; +normalize = false; +loop_dir = 2; +figure(1); clf; +imagesc(I) +figure(2); clf; +for loop = 0:num_loops-1 + fprintf('loop %d\n', loop); + for idx = 1:num_nodes + %fprintf(' idx %d\n', idx); + if mod(floor(bitshift(loop,-log2(loop_dir/2))),2) == 0 + node_idx = idx; + else + node_idx = num_nodes+1-idx; + end + if node_idx < num_nodes + if 0 + msg_left(:,node_idx+1) = old_msg_left(:,node_idx) + I(:,node_idx); + else + for state = 1:num_states + msg_left(state,node_idx+1) = max(I(:,node_idx) + old_msg_left(:,node_idx) - (state-(1:10).').^2); + end + if normalize + msg_left(:,node_idx+1) = msg_left(:,node_idx+1) - max(msg_left(:,node_idx+1)); + end + end + end + if node_idx > 1 + if 0 + msg_right(:,node_idx-1) = old_msg_right(:,node_idx) + I(:,node_idx); + else + for state = 1:num_states + msg_right(state,node_idx-1) = max(I(:,node_idx) + old_msg_right(:,node_idx) - (state-(1:10).').^2); + end + if normalize + msg_right(:,node_idx-1) = msg_right(:,node_idx-1) - max(msg_right(:,node_idx-1)); + end + end + end; + if trws_en + old_msg_right = msg_right; + old_msg_left = msg_left; + end + imagesc(msg_left); + imagesc(msg_right); + + imagesc(I + msg_left + msg_right); + end + + old_msg_right = msg_right; + old_msg_left = msg_left; + [~,result] = max(I + msg_left + msg_right) + clf; + imagesc(I + msg_left + msg_right); + hold on + plot(result) +end + +[confidence,result] = max(I + msg_left + msg_right) diff --git a/cresis-toolbox/+tomo/trws_sim_2D.m b/cresis-toolbox/+tomo/trws_sim_2D.m new file mode 100644 index 00000000..7ac795d9 --- /dev/null +++ b/cresis-toolbox/+tomo/trws_sim_2D.m @@ -0,0 +1,162 @@ +% Not complete, but started work on 2D MRF +num_nodes = [32 24]; +num_states = 14; +num_loops = 16; +rng(0); +if 0 + I = ones(num_states,num_nodes(1),num_nodes(2)); +else + I = 1*randn(num_states,num_nodes(1),num_nodes(2)); + % Add in surface points + I(1:5,1:3,:) = I(1:5,1:3,:) + 3.5; + I(6,[1],:) = I(6,[1],:) + [10]; + I(7,[4 10],:) = bsxfun(@plus, I(7,[4 10],:),[5 7]); + I(5,[13 15],:) = bsxfun(@plus, I(5,[13 15],:),[5 7]); + I(4,[19],:) = I(4,[19],:) + [4]; + % Remove first 8 rows of every column + I(:,17:32,:) = 1*randn(num_states,16,num_nodes(2)); + % Artifact + I(12,26:28,14:16) = I(12,26:28,14:16) + 1000; +end +msg_up = zeros(num_states,num_nodes(1),num_nodes(2)); +msg_down = zeros(num_states,num_nodes(1),num_nodes(2)); +msg_left = zeros(num_states,num_nodes(1),num_nodes(2)); +msg_right = zeros(num_states,num_nodes(1),num_nodes(2)); +old_msg_up = zeros(num_states,num_nodes(1),num_nodes(2)); +old_msg_down = zeros(num_states,num_nodes(1),num_nodes(2)); +old_msg_left = zeros(num_states,num_nodes(1),num_nodes(2)); +old_msg_right = zeros(num_states,num_nodes(1),num_nodes(2)); +trws_en = true; +normalize = true; +smooth_scale = 1; + +loop_dir = 2; +row_loop_dir = 2; +figure(1); clf; +imagesc(I(:,:,1)) +figure(2); clf; +for loop = 0:num_loops-1 + fprintf('loop %d\n', loop); + for idx = 1:prod(num_nodes) + %fprintf(' idx %d\n', idx); + if mod(floor(bitshift(loop,-log2(loop_dir/2))),2) == 0 + node_idx = idx; + else + node_idx = prod(num_nodes)+1-idx; + end + if mod(floor(bitshift(loop,-log2(row_loop_dir/2))),2) == 0 + if mod(floor(bitshift(loop,-log2(loop_dir/2))),2) == 0 + row_idx = 1 + mod((node_idx-1),num_nodes(1)); + else + row_idx = num_nodes(1) - mod((node_idx-1),num_nodes(1)); + end + else + if mod(floor(bitshift(loop,-log2(loop_dir/2))),2) == 0 + row_idx = num_nodes(1) - mod((node_idx-1),num_nodes(1)); + else + row_idx = 1 + mod((node_idx-1),num_nodes(1)); + end + end + col_idx = 1 + floor((node_idx-1)/num_nodes(1)); + if 0 + fprintf('Loop %d\t%d\t%d\n', loop, row_idx, col_idx) + end + if row_idx < num_nodes(1) + if 0 + msg_up(:,row_idx+1,col_idx) = old_msg_up(:,row_idx,col_idx) + I(:,row_idx,col_idx); + else + for state = 1:num_states + msg_up(state,row_idx+1,col_idx) = max(I(:,row_idx,col_idx) + old_msg_left(:,row_idx,col_idx) + old_msg_right(:,row_idx,col_idx) + old_msg_up(:,row_idx,col_idx) - smooth_scale*(state-(1:num_states).').^2); + end + if normalize + msg_up(:,row_idx+1,col_idx) = msg_up(:,row_idx+1,col_idx) - max(msg_up(:,row_idx+1,col_idx)); + end + end + end + if row_idx > 1 + if 0 + msg_down(:,row_idx-1,col_idx) = old_msg_down(:,row_idx,col_idx) + I(:,row_idx,col_idx); + else + for state = 1:num_states + msg_down(state,row_idx-1,col_idx) = max(I(:,row_idx,col_idx) + old_msg_left(:,row_idx,col_idx) + old_msg_right(:,row_idx,col_idx) + old_msg_down(:,row_idx,col_idx) - smooth_scale*(state-(1:num_states).').^2); + end + if normalize + msg_down(:,row_idx-1,col_idx) = msg_down(:,row_idx-1,col_idx) - max(msg_down(:,row_idx-1,col_idx)); + end + end + end; + if col_idx < num_nodes(2) + if 0 + msg_left(:,row_idx,col_idx+1) = old_msg_left(:,row_idx,col_idx) + I(:,row_idx,col_idx); + else + for state = 1:num_states + msg_left(state,row_idx,col_idx+1) = max(I(:,row_idx,col_idx) + old_msg_up(:,row_idx,col_idx) + old_msg_down(:,row_idx,col_idx) + old_msg_left(:,row_idx,col_idx) - smooth_scale*(state-(1:num_states).').^2); + end + if normalize + msg_left(:,row_idx,col_idx+1) = msg_left(:,row_idx,col_idx+1) - max(msg_left(:,row_idx,col_idx+1)); + end + end + end + if col_idx > 1 + if 0 + msg_right(:,col_idx-1) = old_msg_right(:,col_idx) + I(:,col_idx); + else + for state = 1:num_states + msg_right(state,row_idx,col_idx-1) = max(I(:,row_idx,col_idx) + old_msg_up(:,row_idx,col_idx) + old_msg_down(:,row_idx,col_idx) + old_msg_right(:,row_idx,col_idx) - smooth_scale*(state-(1:num_states).').^2); + end + if normalize + msg_right(:,row_idx,col_idx-1) = msg_right(:,row_idx,col_idx-1) - max(msg_right(:,row_idx,col_idx-1)); + end + end + end; + if trws_en + old_msg_up = msg_up; + old_msg_down = msg_down; + old_msg_right = msg_right; + old_msg_left = msg_left; + end + + if 0 + figure(1); clf; imagesc(msg_left(:,:,1)); + figure(2); clf; imagesc(msg_right(:,:,1)); + figure(3); clf; imagesc(msg_up(:,:,1)); + figure(4); clf; imagesc(msg_down(:,:,1)); + elseif 0 + imagesc(I(:,:,1) + msg_left(:,:,1) + msg_right(:,:,1) + msg_up(:,:,1) + msg_down(:,:,1)); + end + end + + old_msg_up = msg_up; + old_msg_down = msg_down; + old_msg_right = msg_right; + old_msg_left = msg_left; + [~,result] = max(I + msg_left + msg_right + msg_up + msg_down); + if 0 + clf; + for col = 1:num_nodes(2) + imagesc(I(:,:,col) + msg_left(:,:,col) + msg_right(:,:,col) + msg_up(:,:,col) + msg_down(:,:,col)); + hold on + plot(result(1,:,col),'k') + title(sprintf('column %d',col)); + pause + end + drawnow; + end +end + +[confidence,result] = max(I + msg_left + msg_right + msg_up + msg_down); + +%% Print result +clf; +for col = 1:num_nodes(2) + figure(1); clf; + imagesc(I(:,:,col)) + hold on + plot(result(1,:,col),'k') + figure(2); clf; + imagesc(I(:,:,col) + msg_left(:,:,col) + msg_right(:,:,col) + msg_up(:,:,col) + msg_down(:,:,col)); + hold on + plot(result(1,:,col),'k') + title(sprintf('column %d',col)); + pause +end diff --git a/cresis-toolbox/+tomo/twtt_doa_to_yz.m b/cresis-toolbox/+tomo/twtt_doa_to_yz.m index b08c3033..9eae3a4d 100644 --- a/cresis-toolbox/+tomo/twtt_doa_to_yz.m +++ b/cresis-toolbox/+tomo/twtt_doa_to_yz.m @@ -1,4 +1,4 @@ -function [y,z] = twtt_doa_to_yz(doa,theta,surface,er_ice,twtt,doa_method_flag,doa_limits) +function [y,z] = twtt_doa_to_yz(doa,theta,surface,er_surf,er_ice,twtt,doa_method_flag,doa_limits) % [y,z] = tomo.twtt_doa_to_yz(doa,theta,surface,er_ice,twtt) % % Converts twtt,doa coordinates (the standard synthetic aperture radar @@ -16,6 +16,7 @@ % Direction of arrival vector corresponding to the surface matrix % surface: Nsv by Nx array or 1 by Nx [seconds] % The two way travel time to the ice top. +% er_surf: the relative dielectric of the medium of initial transmission % er_ice: scalar containing relative dielectric of ice % twtt: Nsig by Nx array [seconds] % The two way travel to the target (top or bottom) @@ -30,7 +31,7 @@ % theta = mdata.param_combine.array_param.theta; theta = reshape(theta,[Nsig 1]); % [y,z] = tomo.twtt_doa_to_yz(mdata.Time,repmat(theta,[1 Nx]),theta,mdata.twtt,3.15); % -% Author: John Paden +% Author: John Paden, Nick Holschuh % Load in standard constants like speed of light, c physical_constants; @@ -60,14 +61,14 @@ % Determine the time to the surface and the position of incidence twtt_to_surf = interp1(theta, surface(:,rline), doa(:,rline)); for doa_idx = 1:Nsig - inc_y = sin(doa(doa_idx,rline)) * twtt_to_surf(doa_idx) * c/2; - inc_z = -cos(doa(doa_idx,rline)) * twtt_to_surf(doa_idx) * c/2; + inc_y = sin(doa(doa_idx,rline)) * twtt_to_surf(doa_idx) * c/2/sqrt(er_surf); + inc_z = -cos(doa(doa_idx,rline)) * twtt_to_surf(doa_idx) * c/2/sqrt(er_surf); % For each time position up to the surface, finding y,z is a cylindrical to % cartesian coordinate conversion. if twtt(doa_idx,rline) <= twtt_to_surf(doa_idx) - y(doa_idx,rline) = sin(doa(doa_idx,rline)) * twtt(doa_idx,rline) * c/2; - z(doa_idx,rline) = -cos(doa(doa_idx,rline)) * twtt(doa_idx,rline) * c/2; + y(doa_idx,rline) = sin(doa(doa_idx,rline)) * twtt(doa_idx,rline) * c/2/sqrt(er_surf); + z(doa_idx,rline) = -cos(doa(doa_idx,rline)) * twtt(doa_idx,rline) * c/2/sqrt(er_surf); elseif twtt(doa_idx,rline) > twtt_to_surf(doa_idx) @@ -89,7 +90,7 @@ if abs(theta_inc) >= pi/2 continue; end - theta_tx = asin(sin(theta_inc)/sqrt(er_ice)); + theta_tx = asin(sin(theta_inc)*sqrt(er_surf)/sqrt(er_ice)); y(doa_idx,rline) = inc_y + sin(theta_tx) ... * (twtt(doa_idx,rline)-twtt_to_surf(doa_idx)) * c/2/sqrt(er_ice); @@ -124,8 +125,8 @@ % Nsig = length(tmp_twtt); % Nsv = Nsig; % Convert surface for this along-track slope to y,z coordinates - surf_y = sin(tmp_theta) .* tmp_surface * c/2; - surf_z = -cos(tmp_theta) .* tmp_surface * c/2; + surf_y = sin(tmp_theta) .* tmp_surface * c/2/sqrt(er_surf); + surf_z = -cos(tmp_theta) .* tmp_surface * c/2/sqrt(er_surf); p = polyfit(surf_y,surf_z,1); % Determine the time to the surface and the position of incidence @@ -160,14 +161,14 @@ for tmp_doa_idx = 1:Nsig doa_idx = dood_theta_idx(tmp_doa_idx); % tmp_doa(doa_idx) * 180/pi - inc_y = sin(tmp_doa(doa_idx)) * twtt_to_surf(doa_idx) * c/2; - inc_z = -cos(tmp_doa(doa_idx)) * twtt_to_surf(doa_idx) * c/2; + inc_y = sin(tmp_doa(doa_idx)) * twtt_to_surf(doa_idx) * c/2/sqrt(er_surf); + inc_z = -cos(tmp_doa(doa_idx)) * twtt_to_surf(doa_idx) * c/2/sqrt(er_surf); % For each time position up to the surface, finding y,z is a cylindrical to % cartesian coordinate conversion. if tmp_twtt(doa_idx) <= twtt_to_surf(doa_idx) - y(doa_idx,rline) = sin(tmp_doa(doa_idx)) * tmp_twtt(doa_idx) * c/2; - z(doa_idx,rline) = -cos(tmp_doa(doa_idx)) * tmp_twtt(doa_idx) * c/2; + y(doa_idx,rline) = sin(tmp_doa(doa_idx)) * tmp_twtt(doa_idx) * c/2/sqrt(er_surf); + z(doa_idx,rline) = -cos(tmp_doa(doa_idx)) * tmp_twtt(doa_idx) * c/2/sqrt(er_surf); elseif tmp_twtt(doa_idx) > twtt_to_surf(doa_idx) @@ -189,7 +190,7 @@ if abs(theta_inc) >= pi/2 continue; end - theta_tx = asin(sin(theta_inc)/sqrt(er_ice)); + theta_tx = asin(sin(theta_inc)*sqrt(er_surf)/sqrt(er_ice)); y(doa_idx,rline) = inc_y + sin(theta_tx) ... * (tmp_twtt(doa_idx)-twtt_to_surf(doa_idx)) * c/2/sqrt(er_ice); diff --git a/cresis-toolbox/+tomo/viterbi.cpp b/cresis-toolbox/+tomo/viterbi.cpp index 081b777e..7e3934d2 100644 --- a/cresis-toolbox/+tomo/viterbi.cpp +++ b/cresis-toolbox/+tomo/viterbi.cpp @@ -10,388 +10,555 @@ // // Changes to cost function (shifted exponential decay): Victor Berger and John Paden 2018 // Changes to cost function (geostatistical analysis): Victor Berger and John Paden 2019 +// Various changes throughout (purge dead code, fix bugs, add more weights): Reece Mathews 2020 // // See also: viterbi.h // // mex -v -largeArrayDims viterbi.cpp + #include "viterbi.h" // Used to define unary cost of target at position x, y -double viterbi::unary_cost(int x, int y) { - - // Merge layers when no ice exists - if (f_mask[x] == 0 && f_sgt[x] > t && y + t != f_sgt[x]) { - return LARGE; - } +double viterbi::unary_cost(int x, int y) +{ - // Set cost to large if bottom is above surface - if (y + t + 1 < f_sgt[x]) { - return LARGE; - } + double sgt = get_y(0, x); // Surface layer is always layer 0 + // Merge layers when no ice exists + if (f_mask[x] == 0 && y != sgt) + { + return LARGE; + } + + // Set cost to large if bottom is above surface + if (!mxIsNaN(sgt) && y < sgt) + { + return LARGE; + } - // Set cost to large if far from center ground truth (if present) - if ((f_bgt != -1) && (x == f_mid) && (y + t < f_bgt - 20 || y + t > f_bgt + 20)) { + double cost = 0; + + // Increase cost if far from layer ground truth + for (int layer_num = 0; layer_num < f_num_layers; layer_num++) { + double gt = get_y(layer_num, x); + if (is_valid(gt)) + { + double cutoff = get_y_cutoff(layer_num, x); + int dist = abs((int)gt - (int)y); + if (is_valid(cutoff) && dist > cutoff) + { + // Must be within cutoff range return LARGE; + } + else + { + double layer_cost = get_y_cost(layer_num, x); + if (!mxIsNaN(layer_cost)) { + if (layer_cost > 0) { + // Positive cost decreases cost further away -- Repel from point + cost += layer_cost * exp(-sqr(dist)*.001); + } + else { + // Negative cost increases cost further away -- Attract to point + cost += layer_cost * exp(-sqr(dist)*.001) - layer_cost; + } + } + } } + } - double cost = 0; - - // Increase cost if far from extra ground truth - for (int f = 0; f < (f_num_extra_tr / 2); ++f) { - if (f_egt_x[f] == x) { - cost += f_weight_points[x] * 10 * sqr(((int)f_egt_y[f] - (int)(t + y)) / f_egt_weight); - break; - } + // Increase cost if near surface or surface multiple bin + double multiple_cost = 0; + if (!mxIsNaN(sgt)) { + const int travel_time = sgt - f_zero_bin; // Between multiples + int dist_to_surf = abs((int)y - (int)sgt); + int multiple_bin = travel_time == 0 ? -1 : floor((y - sgt) / travel_time) - 1; + int dist_to_bin = travel_time == 0 ? dist_to_surf : dist_to_surf % travel_time; + // If closer to next multiple, use that distance instead + double current_mult_weight = f_mult_weight; + if (dist_to_bin > travel_time / 2) + { + dist_to_bin = travel_time - dist_to_bin; + multiple_bin++; + current_mult_weight *= .75; // Return is not as strong above multiple + } + if (multiple_bin < 0) + { + // multiple bin -1 is surface, others are multiples + multiple_bin = 0; + current_mult_weight = 0; // Surface repulsion handled above as cost } + multiple_cost = current_mult_weight; + multiple_cost *= pow(f_mult_weight_decay, multiple_bin); + multiple_cost *= pow(f_mult_weight_local_decay, dist_to_bin); + + multiple_cost = multiple_cost < 0 ? 0 : multiple_cost; + cost += multiple_cost; + // Distance from nearest ice-mask // Probabilistic measurement - int DIM = (std::isinf(f_mask_dist[x]) || f_mask_dist[x] >= f_costmatrix_Y) ? f_costmatrix_Y - 1 : f_mask_dist[x]; - cost += f_costmatrix[f_costmatrix_X * DIM + y + t + 1 - f_sgt[x]]; + int DIM = (std::isinf(f_mask_dist[x]) || f_mask_dist[x] >= f_costmatrix_Y) ? f_costmatrix_Y - 1 : f_mask_dist[x]; + int matrix_idx = f_costmatrix_X * DIM + y - sgt; + int matrix_final_idx = f_costmatrix_X * f_costmatrix_Y - 1; + if (matrix_idx >= 0 && matrix_idx <= matrix_final_idx) + { + // Index within bounds of costmatrix + cost += f_costmatrix[matrix_idx]; + } + else if (f_costmatrix_X > 0) + { + // costmatrix provided but index outside bounds + if (matrix_idx < 0) + { + // Use first cost in matrix when index is before matrix start + cost += f_costmatrix[0]; + } + else + { + // Use last cost in matrix when index is after matrix end + cost += f_costmatrix[matrix_final_idx]; + } + } + } + // Image magnitude + cost -= f_image[encode(x, y)] * f_img_mag_weight; - // Image magnitude correlation - double tmp_cost = 0; - for (size_t i = 0; i < f_ms; i++) { - cost -= f_image[encode(x, y + i)] * f_mu[i] / f_sigma[i]; - } - - return cost; + return cost; } // Returns Viterbi solution of optimal path -double* viterbi::find_path(void) { - start_col = f_bounds[0]; - end_col = f_bounds[1]; - t = (f_ms - 1) / 2; - depth = f_row - f_ms; - num_col_vis = end_col - start_col; - - int *path = new int[depth * (num_col_vis + 2)]; - double path_prob[depth], path_prob_next[depth], index[depth]; - - for (int k = 0; k < f_col; ++k) - f_result[k] = 0; - - for (int k = 0; k < depth * (num_col_vis + 2); ++k) { - path[k] = 0; - } - - for (int k = 0; k < depth; ++k) { - path_prob[k] = 0; - path_prob_next[k] = 0; - index[k] = 0; - } +double *viterbi::find_path(void) +{ + start_col = f_hori_bounds[0]; + end_col = f_hori_bounds[1]; + num_col_vis = end_col - start_col + 1; - viterbi_right(path, path_prob, path_prob_next, index); - - int encode; - int viterbi_index = calculate_best(path_prob); - int idx = end_col; - f_result[end_col - 1] = (f_mask[end_col - 1] == 1 || std::isinf(f_mask[end_col - 1])) ? viterbi_index + t : f_sgt[end_col - 1]; - - // Set result vector - for (int k = start_col + 1; k <= end_col; ++k) { - encode = vic_encode(viterbi_index, num_col_vis + start_col - k); - viterbi_index = path[encode]; - f_result[idx - 2] = viterbi_index + t; - --idx; - if (encode < 0 || idx < 2) { - break; - } - } - - delete[] path; - return f_result; + int *path = new int[f_row * num_col_vis]; + double path_prob[f_row], path_prob_next[f_row], index[f_row]; + + for (int k = 0; k < f_col; ++k) + f_result[k] = NAN; + + for (int k = 0; k < f_row * num_col_vis; ++k) + { + path[k] = 0; + } + + for (int k = 0; k < f_row; ++k) + { + path_prob[k] = 0; + path_prob_next[k] = 0; + index[k] = 0; + } + + viterbi_right(path, path_prob, path_prob_next, index); + + int encode; + int viterbi_index = calculate_best(path_prob); + int idx = end_col; + f_result[end_col] = (f_mask[end_col] == 1 || std::isinf(f_mask[end_col])) ? viterbi_index + 1 : get_y(0, end_col) + 1; + + // Set result vector + for (int k = start_col + 1; k <= end_col; ++k) + { + encode = vic_encode(viterbi_index, num_col_vis + start_col - k); + viterbi_index = path[encode]; + f_result[idx - 1] = viterbi_index + 1; // Account for matlab 1-indexing + --idx; + if (encode < 0 || idx < 1) + { + break; + } + } + + delete[] path; + return f_result; } // Select path with lowest overall cost -int viterbi::calculate_best(double *path_prob) { - double min = LARGE; - int viterbi_index = 0; - for (int k = 0; k < depth; ++k) { - if (path_prob[k] < min) { - min = path_prob[k]; - viterbi_index = k; - } - } - return viterbi_index; +int viterbi::calculate_best(double *path_prob) +{ + double min = LARGE; + int viterbi_index = 0; + for (int k = 0; k < f_row; ++k) + { + if (path_prob[k] < min) + { + min = path_prob[k]; + viterbi_index = k; + } + } + return viterbi_index; } // Perform viterbi to the right -void viterbi::viterbi_right(int *path, double *path_prob, double *path_prob_next, double *index) { - int idx = 0; - bool next = 0; - double norm = 0; - - for (int col = start_col; col <= end_col + 1; ++col) { - if (idx >= depth * (num_col_vis + 2) || col >= f_col || col < 0) { - continue; - } - if (f_transition_mu[col] < 0 && f_transition_sigma[col] < 0) { - norm = norm_pdf(col, (double)f_mid, f_smooth_var, f_smooth_weight); - } - else if (f_transition_mu[col] < 0) { - norm = norm_pdf(col, path[col], f_transition_sigma[col], f_smooth_weight); - norm = norm < f_smooth_weight ? f_smooth_weight : norm; - } - else { - norm = norm_pdf(col, f_transition_mu[col], f_transition_sigma[col], f_smooth_weight); - } - if (!next) { - dt_1d(path_prob, norm, path_prob_next, index, 0, depth, f_smooth_slope[col-1]); - } - else { - dt_1d(path_prob_next, norm, path_prob, index, 0, depth, f_smooth_slope[col-1]); - } - for (int row = 0; row < depth; ++row) { - path[idx] = index[row]; - if(!next) { - path_prob_next[row] += unary_cost(col, row); - } - else { - path_prob[row] += unary_cost(col, row); - } - ++idx; - } - next = !next; - } -} +void viterbi::viterbi_right(int *path, double *path_prob, double *path_prob_next, double *index) +{ + int idx = 0; + bool next = 0; -// MATLAB FUNCTION START -void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]) { - if (nrhs != 19 || nlhs != 1) { - mexErrMsgTxt("Usage: [labels] = viterbi(input_img, surface_gt, bottom_gt, extra_gt, ice_mask, mean, var, egt_weight, smooth_weight, smooth_var, smooth_slope, bounds, viterbi_weight, repulsion, ice_bin_thr, CF_sensory_distance, CF_max_cost, CF_lambda)\n"); - } - - // Input checking - // input image ======================================================== - if (!mxIsDouble(prhs[0])) { - mexErrMsgTxt("usage: image must be type double"); - } - if (mxGetNumberOfDimensions(prhs[0]) != 2) { - mexErrMsgTxt("usage: image must be a 2D matrix"); - } - const int _row = mxGetM(prhs[0]); - const int _col = mxGetN(prhs[0]); - const double *_image = mxGetPr(prhs[0]); - - // surface ground truth =============================================== - if (!mxIsDouble(prhs[1])) { - mexErrMsgTxt("usage: sgt must be type double"); - } - if (mxGetNumberOfElements(prhs[1]) != _col) { - mexErrMsgTxt("usage: sgt must have size(sgt,1)=size(image,2)"); - } - const double *_surf_tr = mxGetPr(prhs[1]); - - // bottom ground truth ================================================ - if (!mxIsDouble(prhs[2]) || mxGetNumberOfElements(prhs[2]) != 1) { - mexErrMsgTxt("usage: bgt must be a scalar of type double"); - } - const double *t_bott_tr = mxGetPr(prhs[2]); - - // extra ground truth ================================================= - if (!mxIsDouble(prhs[3])) { - mexErrMsgTxt("usage: egt must be type double"); - } - const int _num_extra_tr = mxGetNumberOfElements(prhs[3]); - if (_num_extra_tr % 2 != 0) { - mexErrMsgTxt("usage: egt size must be a multiple of 2"); - } - if (mxGetNumberOfElements(prhs[3]) > 0) { - if (mxGetNumberOfDimensions(prhs[3]) != 2) { - mexErrMsgTxt("usage: egt must be a 2xN array"); - } - } - const double *t_egt = mxGetPr(prhs[3]); - - // mask =============================================================== - if (!mxIsDouble(prhs[4])) { - mexErrMsgTxt("usage: mask must be type double"); - } - if (mxGetNumberOfElements(prhs[4]) != _col) { - mexErrMsgTxt("usage: mask must have size(mask,1)=size(image,2)"); - } - const double *_mask = mxGetPr(prhs[4]); - - // mu (mean) ========================================================== - if (!mxIsDouble(prhs[5])) { - mexErrMsgTxt("usage: mean must be type double"); + for (int col = start_col; col <= end_col; ++col) + { + if (idx >= f_row * num_col_vis || col >= f_col || col < 0) + { + continue; + } + // Have to add unary cost to first column before calculating best prev index for next column + for (int row = 0; row < f_row; ++row) + { + if (col > start_col) + { + path[idx] = index[row]; + } + if (next) + { + path_prob_next[row] += unary_cost(col, row); + } + else + { + path_prob[row] += unary_cost(col, row); + } + ++idx; } - const size_t _ms = mxGetNumberOfElements(prhs[5]); - const double *_mu = mxGetPr(prhs[5]); - - // sigma (variance) =================================================== - if (!mxIsDouble(prhs[6])) { - mexErrMsgTxt("usage: variance must be type double"); - } - if (_ms != mxGetNumberOfElements(prhs[6])) { - mexErrMsgTxt("usage: variance must have numel=numel(variance)"); - } - const double *_sigma = mxGetPr(prhs[6]); - - // extra ground truth weight ========================================== - if (!mxIsDouble(prhs[7])) { - mexErrMsgTxt("usage: extra_ground_truth must be type double"); - } - if (mxGetNumberOfElements(prhs[7]) != 1) { - mexErrMsgTxt("usage: extra_ground_truth must be a scalar"); - } - const double *t_egt_weight = mxGetPr(prhs[7]); - const double _egt_weight = t_egt_weight && t_egt_weight < 0 ? EGT_WEIGHT : t_egt_weight[0]; - - // smooth_weight ====================================================== - if (!mxIsDouble(prhs[8])) { - mexErrMsgTxt("usage: smooth_weight must be type double"); - } - if (mxGetNumberOfElements(prhs[8]) != 1) { - mexErrMsgTxt("usage: smooth_weight must be a scalar"); - } - const double *t_smooth_weight = mxGetPr(prhs[8]); - const double _smooth_weight = t_smooth_weight[0] < 0 ? SCALE : t_smooth_weight[0]; - - // smooth_var ========================================================= - if (!mxIsDouble(prhs[9])) { - mexErrMsgTxt("usage: smooth_var must be type double"); - } - if (mxGetNumberOfElements(prhs[9]) != 1) { - mexErrMsgTxt("usage: smooth_var must be a scalar"); - } - const double *t_smooth_var = mxGetPr(prhs[9]); - const double _smooth_var = t_smooth_var[0] < 0 ? SIGMA : t_smooth_var[0]; - - // smooth_slope ======================================================= - if (!mxIsDouble(prhs[10])) { - mexErrMsgTxt("usage: smooth_slope must be type double"); - } - if (_col-1 != mxGetNumberOfElements(prhs[10])) { - mexErrMsgTxt("usage: smooth_slope must have numel=size(image,2)-1"); - } - const double *_smooth_slope = mxGetPr(prhs[10]); - - // bounds ============================================================= - ptrdiff_t _bounds[2]; - if (nrhs >= 13 && mxGetNumberOfElements(prhs[11])) { - if (!mxIsInt64(prhs[11])) { - mexErrMsgTxt("Usage: bounds must be type int64"); - } - if (mxGetNumberOfElements(prhs[11]) != 2) { - mexErrMsgTxt("Usage: bounds must be a 2 element vector"); - } - ptrdiff_t *tmp = (ptrdiff_t*)mxGetPr(prhs[11]); - _bounds[0] = tmp[0]; - _bounds[1] = tmp[1]; - if (_bounds[0] < 0) { - _bounds[0] = 0; - } - if (_bounds[1] < 0) { - _bounds[1] = _col; - } - if (_bounds[0] > _col) { - mexErrMsgTxt("Usage: bounds[0] <= size(input,2)"); - } - if(_bounds[1] > _col) { - mexErrMsgTxt("Usage: bounds[1] <= size(input,2)"); - } - if(_bounds[1] < _bounds[0]) { - mexErrMsgTxt("Usage: bounds[1] must be greater than bounds[0]"); + + if (col >= end_col) + { + // Allow addition of unary cost to final column but do not + // add binary cost for next column since there are no more columns + + // Overwrite path_prob with path_prob_next if that is more recent + // path_prob is always searched for the best index after the viterbi_right call + if (next) + { + for (int i = 0; i < f_row; i++) + { + path_prob[i] = path_prob_next[i]; } + } + + continue; } - else { - // Default setting is to process all columns - _bounds[0] = 0; - _bounds[1] = _col; - } - - // weight_points ====================================================== - if (!mxIsDouble(prhs[12])) { - mexErrMsgTxt("usage: weight_points must be type double"); - } - if (mxGetNumberOfElements(prhs[12]) != _col) { - mexErrMsgTxt("usage: weight_points must have size(mask,1)=size(image,2)"); - } - const double *_weight_points = mxGetPr(prhs[12]); - - // repulsion ========================================================== - if (!mxIsDouble(prhs[13])) { - mexErrMsgTxt("usage: repulsion must be type double"); - } - if (mxGetNumberOfElements(prhs[13]) != 1) { - mexErrMsgTxt("usage: repulsion must be a scalar"); - } - const double *t_repulsion = mxGetPr(prhs[13]); - const double _repulsion = t_repulsion[0] < 0 ? REPULSION : t_repulsion[0]; - - // ice_bin_thr ======================================================== - if (!mxIsDouble(prhs[14])) { - mexErrMsgTxt("usage: ice_bin_thr must be type double"); - } - if (mxGetNumberOfElements(prhs[14]) != 1) { - mexErrMsgTxt("usage: ice_bin_thr must be a scalar"); - } - const double *t_ice_bin_thr = mxGetPr(prhs[14]); - const double _ice_bin_thr = t_ice_bin_thr[0] < 0 ? ICE_BIN_THR : t_ice_bin_thr[0]; - - // mask_dist ========================================================== - if (!mxIsDouble(prhs[15])) { - mexErrMsgTxt("usage: mask_dist must be type double"); - } - if (mxGetNumberOfElements(prhs[15]) != _col) { - mexErrMsgTxt("usage: mask_dist must have size(mask,1)=size(image,2)"); - } - const double *_mask_dist = mxGetPr(prhs[15]); - - // costmatrix ========================================================= - if (!mxIsDouble(prhs[16])) { - mexErrMsgTxt("usage: costmatrix must be type double"); - } - const double *_costmatrix = mxGetPr(prhs[16]); - const int _costmatrix_X = mxGetM(prhs[16]); - const int _costmatrix_Y = mxGetN(prhs[16]); - - // transition_mu ====================================================== - if (!mxIsDouble(prhs[17])) { - mexErrMsgTxt("usage: transition_mu must be type double"); - } - if (mxGetNumberOfElements(prhs[17]) != _col) { - mexErrMsgTxt("usage: transition_mu must have size(mask,1)=size(image,2)"); - } - const double *_transition_mu = mxGetPr(prhs[17]); - - // transition_sigma =================================================== - if (!mxIsDouble(prhs[18])) { - mexErrMsgTxt("usage: transition_sigma must be type double"); - } - if (mxGetNumberOfElements(prhs[18]) != _col) { - mexErrMsgTxt("usage: transition_sigma must have size(mask,1)=size(image,2)"); - } - const double *_transition_sigma = mxGetPr(prhs[18]); - - // ==================================================================== - - // Initialize surface layer array - int _sgt[_col]; - for (int k = 0; k < _col; ++k) { - _sgt[k] = (int)_surf_tr[k]; + + if (!next) + { + dt_1d(path_prob, f_transition_weights[col], path_prob_next, index, 0, f_row-1, f_smooth_slope[col], f_max_slope); } - - // Initialize variables to default values if temporary values not set - const int _mid = floor(_col / 2); - const int _bgt = ((t_bott_tr) ? (t_bott_tr[0] > 0 ? round(t_bott_tr[0]) : -1) : -1); - - double _egt_x[(_num_extra_tr / 2)], _egt_y[(_num_extra_tr / 2)]; - for (int p = 0; p < (_num_extra_tr / 2); ++p) { - _egt_x[p] = t_egt[(p * 2)]; - _egt_y[p] = t_egt[(p * 2) + 1]; + else + { + dt_1d(path_prob_next, f_transition_weights[col], path_prob, index, 0, f_row-1, f_smooth_slope[col], f_max_slope); } - - // Allocate output - plhs[0] = mxCreateDoubleMatrix(1, _col, mxREAL); - double *_result = mxGetPr(plhs[0]); - viterbi obj(_row, _col, _image, _sgt, _bgt, _mask, _mu, _sigma, _mid, - _egt_weight, _smooth_weight, _smooth_var, _smooth_slope, _bounds, - _ms, _num_extra_tr, _egt_x, _egt_y, _weight_points, _repulsion, - _ice_bin_thr, _mask_dist, _costmatrix, _costmatrix_X, _costmatrix_Y, - _transition_mu, _transition_sigma, _result); + next = !next; + } +} + +// MATLAB FUNCTION START +void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]) +{ + int max_args = 17; // All args including optional + int min_args = max_args - 1; // Number of required arguments + int arg = 0; + + if (nrhs < min_args || nrhs > max_args || nlhs != 1) + { + mexErrMsgTxt("Usage: [labels] = viterbi(input_img, layers, layer_costs, layer_cutoffs, ice_mask, img_mag_weight, smooth_slope, max_slope, horizontal bounds, vertical bounds, mask_dist, costmatrix, transition_weights, mult_weight, mult_weight_decay, mult_weight_local_decay, [zero_bin])\n"); + } + + // Input checking + // input image ======================================================== + if (!mxIsDouble(prhs[arg])) + { + mexErrMsgTxt("usage: image must be type double"); + } + if (mxGetNumberOfDimensions(prhs[arg]) != 2) + { + mexErrMsgTxt("usage: image must be a 2D matrix"); + } + const int _row = mxGetM(prhs[arg]); + const int _col = mxGetN(prhs[arg]); + const double *_image = mxGetPr(prhs[arg]); + + // layers =============================================== + arg++; + if (!mxIsDouble(prhs[arg])) // Must be double to allow for NaN + { + mexErrMsgTxt("usage: layers must be type double"); + } + const int _num_layers = mxGetM(prhs[arg]); + const int _layer_length = mxGetN(prhs[arg]); + if (_layer_length != _col) + { + mexErrMsgTxt("usage: layers must have size(layers,2)=size(image,2)"); + } + double *_layers = (double *)mxGetPr(prhs[arg]); + + // layer costs =============================================== + arg++; + if (!mxIsDouble(prhs[arg])) + { + mexErrMsgTxt("usage: layer costs must be type double"); + } + const int _num_layer_costs = mxGetM(prhs[arg]); + const int _layer_costs_length = mxGetN(prhs[arg]); + if (_layer_costs_length != _col) + { + mexErrMsgTxt("usage: layer costs must have size(layer costs,2)=size(image,2)"); + } + if (_num_layer_costs != _num_layers) + { + mexErrMsgTxt("usage: layer costs must have size(layer costs,1)=size(layers,1)"); + } + const double *_layer_costs = (double *)mxGetPr(prhs[arg]); + + // layer cutoffs =============================================== + arg++; + if (!mxIsDouble(prhs[arg])) + { + mexErrMsgTxt("usage: layer cutoffs must be type double"); + } + const int _num_layer_cutoffs = mxGetM(prhs[arg]); + const int _layer_cutoffs_length = mxGetN(prhs[arg]); + if (_layer_cutoffs_length != _col) + { + mexErrMsgTxt("usage: layer cutoffs must have size(layer cutoffs,2)=size(image,2)"); + } + if (_num_layer_cutoffs != _num_layers) + { + mexErrMsgTxt("usage: layer cutoffs must have size(layer cutoffs,1)=size(layers,1)"); + } + const double *_layer_cutoffs = (double *)mxGetPr(prhs[arg]); + + // mask =============================================================== + arg++; + if (!mxIsDouble(prhs[arg])) + { + mexErrMsgTxt("usage: mask must be type double"); + } + if (mxGetNumberOfElements(prhs[arg]) != _col) + { + mexErrMsgTxt("usage: mask must have numel(mask)=size(image,2)"); + } + const double *_mask = mxGetPr(prhs[arg]); + + // img_mag_weight ========================================================== + arg++; + if (!mxIsDouble(prhs[arg]) || mxGetNumberOfElements(prhs[arg]) != 1) + { + mexErrMsgTxt("usage: img_mag_weight must be scalar double"); + } + const double _img_mag_weight = *(double *)mxGetPr(prhs[arg]); + + // smooth_slope ======================================================= + arg++; + if (!mxIsDouble(prhs[arg])) + { + mexErrMsgTxt("usage: smooth_slope must be type double"); + } + if (_col - 1 != mxGetNumberOfElements(prhs[arg])) + { + mexErrMsgTxt("usage: smooth_slope must have numel(smooth_slope)=size(image,2)-1"); + } + const double *_smooth_slope = mxGetPr(prhs[arg]); + + // max_slope ======================================================= + arg++; + if (!mxIsDouble(prhs[arg]) || mxGetNumberOfElements(prhs[arg]) != 1) + { + mexErrMsgTxt("usage: max_slope must be type scalar double"); + } + const double _max_slope = *(double *)mxGetPr(prhs[arg]); + + // Horizontal bounds ============================================================= + arg++; + ptrdiff_t _hori_bounds[2]; + if (mxGetNumberOfElements(prhs[arg]) != 0) + { + if (!mxIsInt64(prhs[arg])) + { + mexErrMsgTxt("Usage: horizontal bounds must be type int64"); + } + if (mxGetNumberOfElements(prhs[arg]) != 2) + { + mexErrMsgTxt("Usage: horizontal bounds must be a 2 element vector"); + } + ptrdiff_t *tmp = (ptrdiff_t *)mxGetPr(prhs[arg]); + _hori_bounds[0] = tmp[0] - 1; // Account for matlab 1-indexing + _hori_bounds[1] = tmp[1] - 1; + if (_hori_bounds[0] < 0) + { + _hori_bounds[0] = 0; + } + if (_hori_bounds[1] < 0) + { + _hori_bounds[1] = _col - 1; + } + if (_hori_bounds[0] > _col) + { + mexErrMsgTxt("Usage: horizontal bounds[0] <= size(input,2)"); + } + if (_hori_bounds[1] > _col) + { + mexErrMsgTxt("Usage: horizontal bounds[1] <= size(input,2)"); + } + if (_hori_bounds[1] < _hori_bounds[0]) + { + mexErrMsgTxt("Usage: horizontal bounds[1] must be greater than horizontal bounds[0]"); + } + } + else + { + // Default setting is to process all columns + _hori_bounds[0] = 0; + _hori_bounds[1] = _col - 1; + } + + // Veritcal bounds ============================================================= + arg++; + ptrdiff_t _vert_bounds[2]; + if (mxGetNumberOfElements(prhs[arg]) != 0) + { + if (!mxIsInt64(prhs[arg])) + { + mexErrMsgTxt("Usage: veritcal bounds must be type int64"); + } + if (mxGetNumberOfElements(prhs[arg]) != 2) + { + mexErrMsgTxt("Usage: veritcal bounds must be a 2 element vector"); + } + ptrdiff_t *tmp = (ptrdiff_t *)mxGetPr(prhs[arg]); + _vert_bounds[0] = tmp[0] - 1; // Account for matlab 1-indexing + _vert_bounds[1] = tmp[1] - 1; + if (_vert_bounds[0] < 0) + { + _vert_bounds[0] = 0; + } + if (_vert_bounds[1] < 0) + { + _vert_bounds[1] = _row - 1; + } + if (_vert_bounds[0] > _row) + { + mexErrMsgTxt("Usage: veritcal bounds[0] <= size(input,2)"); + } + if (_vert_bounds[1] > _row) + { + mexErrMsgTxt("Usage: veritcal bounds[1] <= size(input,2)"); + } + if (_vert_bounds[1] < _vert_bounds[0]) + { + mexErrMsgTxt("Usage: veritcal bounds[1] must be greater than veritcal bounds[0]"); + } + } + else + { + // Default setting is to process all rows + _vert_bounds[0] = 0; + _vert_bounds[1] = _row - 1; + } + + // mask_dist ========================================================== + arg++; + if (!mxIsDouble(prhs[arg])) + { + mexErrMsgTxt("usage: mask_dist must be type double"); + } + if (mxGetNumberOfElements(prhs[arg]) != _col) + { + mexErrMsgTxt("usage: mask_dist must have numel(mask_dist)=size(image,2)"); + } + const double *_mask_dist = mxGetPr(prhs[arg]); + + // costmatrix ========================================================= + arg++; + if (!mxIsDouble(prhs[arg])) + { + mexErrMsgTxt("usage: costmatrix must be type double"); + } + const double *_costmatrix = mxGetPr(prhs[arg]); + const int _costmatrix_X = mxGetM(prhs[arg]); + const int _costmatrix_Y = mxGetN(prhs[arg]); + + // transition_weights =================================================== + arg++; + if (!mxIsDouble(prhs[arg])) + { + mexErrMsgTxt("usage: transition_weights must be type double"); + } + if (mxGetNumberOfElements(prhs[arg]) != _col - 1) + { + mexErrMsgTxt("usage: transition_weights have numel(transition_weights)=size(image,2)-1"); + } + const double *_transition_weights = mxGetPr(prhs[arg]); + + // mult_weight =================================================== + arg++; + if (!mxIsDouble(prhs[arg]) || mxGetNumberOfElements(prhs[arg]) != 1) + { + mexErrMsgTxt("usage: mult_weight must be type scalar double"); + } + double _mult_weight = *(double *)mxGetPr(prhs[arg]); + if (_mult_weight == -1) + { + _mult_weight = MULT_WEIGHT; + } + + // mult_weight_decay =================================================== + arg++; + if (!mxIsDouble(prhs[arg]) || mxGetNumberOfElements(prhs[arg]) != 1) + { + mexErrMsgTxt("usage: mult_weight_decay must be type scalar double"); + } + double _mult_weight_decay = *(double *)mxGetPr(prhs[arg]); + if (_mult_weight_decay == -1) + { + _mult_weight_decay = MULT_WEIGHT_DECAY; + } + + // mult_weight_local_decay =================================================== + arg++; + if (!mxIsDouble(prhs[arg]) || mxGetNumberOfElements(prhs[arg]) != 1) + { + mexErrMsgTxt("usage: mult_weight_local_decay must be type scalar double"); + } + double _mult_weight_local_decay = *(double *)mxGetPr(prhs[arg]); + if (_mult_weight_local_decay == -1) + { + _mult_weight_local_decay = MULT_WEIGHT_LOCAL_DECAY; + } + + // zero bin =================================================== + arg++; + int _zero_bin = 0; + if (nrhs >= min_args + 1) + { + if (!mxIsInt64(prhs[arg])) + { + mexErrMsgTxt("usage: zero bin must be type int64"); + } + _zero_bin = *(int *)mxGetPr(prhs[arg]) - 1; // Account for matlab 1-indexing + } + + // ==================================================================== + + // Assert arg == max_args + if (arg != max_args - 1) + { + mexErrMsgTxt("BUG: Viterbi.cpp mex function args incorrectly ordered."); + } + + // Account for Matlab 1-indexing + double _layers_indexed[_col * _num_layers]; + for (int k = 0; k < _col * _num_layers; k++) + { + _layers_indexed[k] = round(_layers[k]) - 1; + } + + // Allocate output + plhs[0] = mxCreateDoubleMatrix(1, _col, mxREAL); + double *_result = mxGetPr(plhs[0]); + viterbi obj(_row, _col, _image, _num_layers, _layers_indexed, _layer_costs, + _layer_cutoffs, _mask, _img_mag_weight, _smooth_slope, _max_slope, _hori_bounds, + _vert_bounds, _mask_dist, _costmatrix, _costmatrix_X, _costmatrix_Y, + _transition_weights, _mult_weight, _mult_weight_decay, _mult_weight_local_decay, + _zero_bin, _result); } diff --git a/cresis-toolbox/+tomo/viterbi.h b/cresis-toolbox/+tomo/viterbi.h index c17835e9..515b9c66 100644 --- a/cresis-toolbox/+tomo/viterbi.h +++ b/cresis-toolbox/+tomo/viterbi.h @@ -15,147 +15,134 @@ #include #include -// Default mu (mean) of smoothness -const int MID = 33; -// Default sigma (variance) of smoothness -const double SIGMA = 24; -// Scaling for smoothness constraint -const double SCALE = 12; -// Relative weight of extra GT -const double EGT_WEIGHT = 10; -// Icemask proximity scan threshold -const double ICE_BIN_THR = 3; -// Repulsion from surface -const double REPULSION = 150000; -// Mass conservation weight -const double MC_WEIGHT = 10; +// Weight for being in first multiple bin +const double MULT_WEIGHT = 100; +// Weight decay for subsequent multiples +const double MULT_WEIGHT_DECAY = .4; +// Weight decay for subsequent bins from nearest multiple +const double MULT_WEIGHT_LOCAL_DECAY = .8; // Large cost -const double LARGE = 1000000000; +const double LARGE = 1000000000; -class viterbi { +class viterbi +{ public: - viterbi( const int d_row, - const int d_col, - const double *d_image, - const int *d_sgt, - const int d_bgt, - const double *d_mask, - const double *d_mu, - const double *d_sigma, - const int d_mid, - const double d_egt_weight, - const double d_smooth_weight, - const double d_smooth_var, - const double *d_smooth_slope, - const ptrdiff_t *d_bounds, - const size_t d_ms, - const int d_num_extra_tr, - const double *d_egt_x, - const double *d_egt_y, - const double *d_weight_points, - const double d_repulsion, - const double d_ice_bin_thr, - const double *d_mask_dist, - const double *d_costmatrix, - const int d_costmatrix_X, - const int d_costmatrix_Y, - const double *d_transition_mu, - const double *d_transition_sigma, - double *d_result - ) : - f_row(d_row), - f_col(d_col), - f_image(d_image), - f_sgt(d_sgt), - f_bgt(d_bgt), - f_mask(d_mask), - f_mu(d_mu), - f_sigma(d_sigma), - f_mid(d_mid), - f_egt_weight(d_egt_weight), - f_smooth_weight(d_smooth_weight), - f_smooth_var(d_smooth_var), - f_smooth_slope(d_smooth_slope), - f_bounds(d_bounds), - f_ms(d_ms), - f_num_extra_tr(d_num_extra_tr), - f_egt_x(d_egt_x), - f_egt_y(d_egt_y), - f_weight_points(d_weight_points), - f_repulsion(d_repulsion), - f_ice_bin_thr(d_ice_bin_thr), - f_mask_dist(d_mask_dist), - f_costmatrix(d_costmatrix), - f_costmatrix_X(d_costmatrix_X), - f_costmatrix_Y(d_costmatrix_Y), - f_transition_mu(d_transition_mu), - f_transition_sigma(d_transition_sigma), - f_result(d_result) - { - find_path(); - } + viterbi(const int d_row, + const int d_col, + const double *d_image, + const int d_num_layers, + const double *d_layers, + const double *d_layer_costs, + const double *d_layer_cutoffs, + const double *d_mask, + const double d_img_mag_weight, + const double *d_smooth_slope, + const double d_max_slope, + const ptrdiff_t *d_hori_bounds, + const ptrdiff_t *d_vert_bounds, + const double *d_mask_dist, + const double *d_costmatrix, + const int d_costmatrix_X, + const int d_costmatrix_Y, + const double *d_transition_weights, + const double d_mult_weight, + const double d_mult_weight_decay, + const double d_mult_weight_local_decay, + const int d_zero_bin, + double *d_result) : f_row(d_row), + f_col(d_col), + f_image(d_image), + f_num_layers(d_num_layers), + f_layers(d_layers), + f_layer_costs(d_layer_costs), + f_layer_cutoffs(d_layer_cutoffs), + f_mask(d_mask), + f_img_mag_weight(d_img_mag_weight), + f_smooth_slope(d_smooth_slope), + f_max_slope(d_max_slope), + f_hori_bounds(d_hori_bounds), + f_vert_bounds(d_vert_bounds), + f_mask_dist(d_mask_dist), + f_costmatrix(d_costmatrix), + f_costmatrix_X(d_costmatrix_X), + f_costmatrix_Y(d_costmatrix_Y), + f_transition_weights(d_transition_weights), + f_mult_weight(d_mult_weight), + f_mult_weight_decay(d_mult_weight_decay), + f_mult_weight_local_decay(d_mult_weight_local_decay), + f_zero_bin(d_zero_bin), + f_result(d_result) + { + find_path(); + } - // VARIABLES - const int f_row, f_col, f_mid, f_bgt, *f_sgt, f_num_extra_tr, f_costmatrix_X, f_costmatrix_Y; - const double *f_image, *f_mask, *f_mu, *f_sigma, f_egt_weight, *f_smooth_slope, - *f_egt_x, *f_egt_y, *f_weight_points, f_smooth_weight, - f_smooth_var, f_repulsion, f_ice_bin_thr, *f_mask_dist, - *f_costmatrix, *f_transition_mu, *f_transition_sigma; - const ptrdiff_t *f_bounds; - const size_t f_ms; - double *f_result, *f_cost; + // VARIABLES + const int f_row, f_col, f_num_layers, f_costmatrix_X, f_costmatrix_Y, f_zero_bin; + const double *f_image, *f_layers, *f_layer_costs, *f_layer_cutoffs, + *f_mask, f_img_mag_weight, *f_smooth_slope, f_max_slope, *f_mask_dist, + f_mult_weight, f_mult_weight_decay, f_mult_weight_local_decay, *f_costmatrix, + *f_transition_weights; + const ptrdiff_t *f_hori_bounds, *f_vert_bounds; + double *f_result, *f_cost; - int depth, num_col_vis, t, start_col, end_col; + int num_col_vis, start_col, end_col; - // METHODS - int calculate_best(double *path_prob); - double unary_cost(int x, int y), *find_path(void); - void viterbi_right(int *path, double *path_prob, double *path_prob_next, double *index); + // METHODS + int calculate_best(double *path_prob); + double unary_cost(int x, int y), *find_path(void); + void viterbi_right(int *path, double *path_prob, double *path_prob_next, double *index); - // Dynamic smoothness - double norm_pdf(int x, double mu = MID, double sigma = SIGMA, double scale = SCALE) - { - if(sigma != std::numeric_limits::infinity()) { - return scale * (1.0/(sigma*sqrt(2*M_PI))) * exp(-0.5*sqr((x-mu)/sigma)); - } - return scale; - } - - // Compute square value - template inline T sqr(T x) { return x*x; } - - int encode(int x, int y) { return x * f_row + y; } - - int vic_encode(int row, int col) { return depth * col + row; } + // Compute square value + template + inline T sqr(T x) { return x * x; } + + int encode(int x, int y) { return x * f_row + y; } + + int layer_encode(int layer_num, int x) {return x*f_num_layers + layer_num; } + double get_y(int layer_num, int x) { return f_layers[layer_encode(layer_num, x)]; } + double get_y_cost(int layer_num, int x) { return f_layer_costs[layer_encode(layer_num, x)]; } + double get_y_cutoff(int layer_num, int x) { return f_layer_cutoffs[layer_encode(layer_num, x)]; } + bool is_valid(double value) { return !mxIsNaN(value) && value >= 0; } - // THE CODE BELOW THIS POINT WAS TAKEN FROM DAVID CRANDALL - // Distance transform - // -- Every index from d1 to d2 will be set in dst and dst_ind - // -- dst will contain the minimum value for that destination - // -- dst_ind will contain the minimum source index for that destination - void dt(const double *src, double *dst, double *dst_ind, int s1, int s2, - int d1, int d2, double scale, int off = 0) - { - - int d = (d1 + d2) >> 1, s = s1; // Find the midpoint of the destination - for (int p = s1; p <= s2; p++) { // Search through all the sources and find the minimum - if (src[s] + sqr(s-d-off) * scale > src[p] + sqr(p-d-off) * scale) { - s = p; - } - } - dst[d] = src[s] + sqr(s-d-off) * scale; // Minimum value to the midpoint - dst_ind[d] = s; // Minimum source index for the midpoint - if(d2 >= d + 1) { // Recursive call, binary search (top half of destinations) - dt(src, dst, dst_ind, s, s2, d+1, d2, scale, off); - } - if(d-1 >= d1) { // Recursive call, binary search (bottom half of destinations) - dt(src, dst, dst_ind, s1, s, d1, d-1, scale, off); - } + int vic_encode(int row, int col) { return f_row * col + row; } + + // THE CODE BELOW THIS POINT WAS TAKEN FROM DAVID CRANDALL + // With minor adjustments by Reece Mathews (max slope and transition_weight) + // Distance transform + // -- Every index from d1 to d2 will be set in dst and dst_ind + // -- dst will contain the minimum value for that destination + // -- dst_ind will contain the minimum source index for that destination + void dt(const double *src, double *dst, double *dst_ind, int s1, int s2, + int d1, int d2, double transition_weight, int off = 0, int max_slope = -1) + { + + int d = (d1 + d2) >> 1, s = d - off; // Find the midpoint of the destination + for (int p = s1; p <= s2; p++) + { // Search through all the sources and find the minimum + if (abs(p - d - off) > max_slope && max_slope > -1) { + continue; + } + if (src[s] + sqr(s - d - off) * transition_weight > src[p] + sqr(p - d - off) * transition_weight) + { + s = p; + } + } + dst[d] = src[s] + sqr(s - d - off) * transition_weight; // Minimum value to the midpoint + dst_ind[d] = s; // Minimum source index for the midpoint + if (d2 >= d + 1) + { // Recursive call, binary search (top half of destinations) + dt(src, dst, dst_ind, s, s2, d + 1, d2, transition_weight, off, max_slope); } - void dt_1d(const double *f, double scale, double *result, double *dst_ind, - int beg, int end, int off = 0) { - dt(f, result, dst_ind, beg, end-1, beg, end-1, scale, off); + if (d - 1 >= d1) + { // Recursive call, binary search (bottom half of destinations) + dt(src, dst, dst_ind, s1, s, d1, d - 1, transition_weight, off, max_slope); } - // END CODE FROM DAVID CRANDALL + } + void dt_1d(const double *f, double transition_weight, double *result, double *dst_ind, + int beg, int end, int off = 0, int max_slope = -1) + { + dt(f, result, dst_ind, beg, end, beg, end, transition_weight, off, max_slope); + } + // END CODE FROM DAVID CRANDALL }; #endif diff --git a/cresis-toolbox/+tomo/viterbi.mexa64 b/cresis-toolbox/+tomo/viterbi.mexa64 index ed672e42..aec0efa1 100644 Binary files a/cresis-toolbox/+tomo/viterbi.mexa64 and b/cresis-toolbox/+tomo/viterbi.mexa64 differ diff --git a/cresis-toolbox/+tomo/viterbi.mexw64 b/cresis-toolbox/+tomo/viterbi.mexw64 index 09d7ec67..912e3b3c 100644 Binary files a/cresis-toolbox/+tomo/viterbi.mexw64 and b/cresis-toolbox/+tomo/viterbi.mexw64 differ diff --git a/cresis-toolbox/+tomo/viterbi2.cpp b/cresis-toolbox/+tomo/viterbi2.cpp new file mode 100644 index 00000000..7b99d112 --- /dev/null +++ b/cresis-toolbox/+tomo/viterbi2.cpp @@ -0,0 +1,268 @@ +/* +viterbi2.cpp + +Layer-tracking program based on the Viterbi algorithm + for MUSIC-processed 2D and 3D data + +Authors: Victor Berger and John Paden + Center for Remote Sensing of Ice Sheets + 2017-2018 + Adapted from original code by Mingze Xu and David Crandall + +Changes to cost function (shifted exponential decay): Victor Berger and John Paden 2018 +Changes to cost function (geostatistical analysis): Victor Berger and John Paden 2019 +Various changes throughout (purge dead code, fix bugs, reduce inputs, remove unary cost): Reece Mathews 2020 + +See also: viterbi2.h + +mex -v -largeArrayDims viterbi2.cpp +*/ + +#include "viterbi2.h" + +// Returns Viterbi solution of optimal path +float *viterbi2::find_path(void) +{ + + int *path = new int[f_row * (f_col - 1)]; + float *path_prob = new float[f_row]; + float *path_prob_next = new float[f_row]; + float *index = new float[f_row]; + + for (int k = 0; k < f_col; ++k) + f_result[k] = NAN; + + for (int k = 0; k < f_row * (f_col - 1); ++k) + { + path[k] = 0; + } + + for (int k = 0; k < f_row; ++k) + { + path_prob[k] = 0; + path_prob_next[k] = 0; + index[k] = 0; + } + + viterbi_right(path, path_prob, path_prob_next, index); + + int encode; + int viterbi_index = calculate_best(path_prob); + f_result[f_col - 1] = viterbi_index + 1; // Account for matlab 1-indexing + + // Set result vector + for (int k = f_col - 2; k >= 0; k--) + { + encode = vic_encode(viterbi_index, k); + viterbi_index = path[encode]; // Follow path backwards + f_result[k] = viterbi_index + 1; // Account for matlab 1-indexing + } + + delete[] path; + delete[] path_prob; + delete[] path_prob_next; + delete[] index; + + return f_result; +} + +// Select path-terminal with lowest overall cost +int viterbi2::calculate_best(float *path_prob) +{ + float min = INF; + int viterbi_index = f_upper_bounds[f_col - 1]; + // Only search within bounds of last column + for (int k = viterbi_index; k <= f_lower_bounds[f_col - 1]; k++) + { + if (path_prob[k] < min) + { + min = path_prob[k]; + viterbi_index = k; + } + } + return viterbi_index; +} + +// Perform viterbi to the right +void viterbi2::viterbi_right(int *path, float *& path_prob, float *& path_prob_next, float *index) +{ + bool next = false; + + for (int col = 0; col < f_col; col++) + { + for (int row = f_upper_bounds[col]; row <= f_lower_bounds[col]; row++) + { + // Path points to best index in previous column -- not defined for first column + + // Add unary cost + if (next) + { + path_prob_next[row] -= f_image[vic_encode(row, col)]; + } + else + { + path_prob[row] -= f_image[vic_encode(row, col)]; + } + } + // No binary cost for last col -- added between columns + if (col == f_col - 1) + { + break; + } + + // Add binary cost from current to next + if (next) + { + // Upper bounds are smaller than lower bounds (smallest -> largest, top -> bottom) + dt(path_prob_next, path_prob, index, f_upper_bounds[col], f_lower_bounds[col], + f_upper_bounds[col + 1], f_lower_bounds[col + 1], f_along_track_weight, + f_along_track_slope[col]); + } + else + { + dt(path_prob, path_prob_next, index, f_upper_bounds[col], f_lower_bounds[col], + f_upper_bounds[col + 1], f_lower_bounds[col + 1], f_along_track_weight, + f_along_track_slope[col]); + } + next = !next; + + // Update path with updated index post-binary calculation. + // Must use bounds from this column, cannot condense path update into next outer loop iteration + for (int row = f_upper_bounds[col + 1]; row <= f_lower_bounds[col + 1]; row++) + { + path[vic_encode(row, col)] = index[row]; + } + } + + // path_prob is checked for best index but path_prob_next is more recent, + // update path_prob to reflect path_prob_next + if (next) + { + // path_prob must be passed by reference to alter target of outer path_prob pointer + // path_prob_next must be as well to swap the values to allow deletion of both arrays + float *temp = path_prob; + path_prob = path_prob_next; + path_prob_next = temp; + } +} + +/* Check bounds for invalid indices. */ +void verify_bounds(unsigned int *bounds_dest, const mxArray *mx_bounds, const char *bounds_name, int default_bound, int total_cols) +{ + char err_str[200]; + if (mxGetNumberOfElements(mx_bounds) != 0) + { + // Must be float or double for NaNs + if (!(mxIsSingle(mx_bounds) || mxIsDouble(mx_bounds))) + { + sprintf(err_str, "Usage: %s must be type floating-point", bounds_name); + mexErrMsgIdAndTxt("MATLAB:inputError", err_str); + } + if (mxGetNumberOfElements(mx_bounds) != total_cols) + { + sprintf(err_str, "Usage: numel(%s)=size(image,2)", bounds_name); + mexErrMsgIdAndTxt("MATLAB:inputError", err_str); + } + + for (int i = 0; i < total_cols; i++) + { + if (mxGetPr(mx_bounds)[i] <= 0) + { + sprintf(err_str, "Usage: %s must be positive (displaying with matlab 1-indexing: index %d of %s is %d)", bounds_name, i + 1, bounds_name, mxGetPr(mx_bounds)[i] + 1); + mexErrMsgIdAndTxt("MATLAB:inputError", err_str); + } + else if (mxIsNaN(mxGetPr(mx_bounds)[i])) + { + sprintf(err_str, "Usage: %s must be finite (displaying with matlab 1-indexing: index %d is %d)", bounds_name, i + 1, mxGetPr(mx_bounds)[i]); + mexErrMsgIdAndTxt("MATLAB:inputError", err_str); + } + bounds_dest[i] = static_cast(mxGetPr(mx_bounds)[i]) - 1; // Account for matlab 1-indexing + } + } + else + { + // Default setting if empty array passed is to process all rows + for (int i = 0; i < total_cols; i++) + { + bounds_dest[i] = default_bound; + } + } +} + +// MATLAB FUNCTION START +void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]) +{ + int arg = 0; + + if (nrhs != 5 || nlhs != 1) + { + mexErrMsgIdAndTxt("MATLAB:inputError", "Usage: [labels] = viterbi2(single image, single along_track_slope, single along_track_weight, single upper_bounds, single lower_bounds)\n"); + } + + // Input checking + // input image ======================================================== + if (!mxIsSingle(prhs[arg])) + { + mexErrMsgIdAndTxt("MATLAB:inputError", "usage: image must be type single"); + } + if (mxGetNumberOfDimensions(prhs[arg]) != 2) + { + mexErrMsgIdAndTxt("MATLAB:inputError", "usage: image must be a 2D matrix"); + } + const int _row = mxGetM(prhs[arg]); + const int _col = mxGetN(prhs[arg]); + const float *_image = reinterpret_cast(mxGetPr(prhs[arg])); + + // along_track_slope ======================================================= + arg++; + if (!(mxIsSingle(prhs[arg]) || mxIsDouble(prhs[arg]))) + { + mexErrMsgIdAndTxt("MATLAB:inputError", "usage: along_track_slope must be floating-point"); + } + if (_col - 1 != mxGetNumberOfElements(prhs[arg])) + { + mexErrMsgIdAndTxt("MATLAB:inputError", "usage: along_track_slope must have numel(along_track_slope)=size(image,2)-1"); + } + float *_along_track_slope = new float[_col - 1]; + // Cast all values to float (in case of double) + for (int i = 0; i < _col - 1; i++) { + _along_track_slope[i] = static_cast(mxGetPr(prhs[arg])[i]); + } + + // along_track_weight ========================================================== + arg++; + if (!(mxIsSingle(prhs[arg]) || mxIsDouble(prhs[arg])) || mxGetNumberOfElements(prhs[arg]) != 1) + { + mexErrMsgIdAndTxt("MATLAB:inputError", "usage: along_track_weight must be scalar floating-point"); + } + const float _along_track_weight = static_cast(*mxGetPr(prhs[arg])); + + // Upper bounds ============================================================= + arg++; + unsigned int _upper_bounds[_col]; + verify_bounds(_upper_bounds, prhs[arg], "upper_bounds", 0, _col); + + // Lower bounds ============================================================= + arg++; + unsigned int _lower_bounds[_col]; + verify_bounds(_lower_bounds, prhs[arg], "lower_bounds", _row - 1, _col); + + // Verify lower_bounds below upper_bounds + char err_str[200]; + for (int i = 0; i < _col; i++) + { + if (_lower_bounds[i] < _upper_bounds[i]) + { + sprintf(err_str, "Usage: lower_bounds must be lower (i.e. >=) than upper_bounds (displaying with matlab 1-indexing: index %d of lower_bounds is %d and corresponding upper_bound is %d)", i + 1, _lower_bounds[i] + 1, _upper_bounds[i] + 1); + mexErrMsgIdAndTxt("MATLAB:inputError", err_str); + } + } + + // Allocate output + const mwSize dims[]={1, static_cast(_col)}; + plhs[0] = mxCreateNumericArray(2, dims, mxSINGLE_CLASS, mxREAL); + float *_result = reinterpret_cast(mxGetPr(plhs[0])); + viterbi2 obj(_row, _col, _image, _along_track_slope, _along_track_weight, _upper_bounds, _lower_bounds, _result); + + delete[] _along_track_slope; +} \ No newline at end of file diff --git a/cresis-toolbox/+tomo/viterbi2.h b/cresis-toolbox/+tomo/viterbi2.h new file mode 100644 index 00000000..3851b75d --- /dev/null +++ b/cresis-toolbox/+tomo/viterbi2.h @@ -0,0 +1,95 @@ +// viterbi2.h +// +// Layer-tracking program based on the Viterbi algorithm +// for MUSIC-processed 2D and 3D data +// +// Adapted from original code by Mingze Xu, David Crandall, and John Paden +// +// Author: Victor Berger +// See also: viterbi2.cpp + +#ifndef _VITERBI_H_ +#define _VITERBI_H_ + +#include "mex.h" +#include +#include + + +float INF = std::numeric_limits::infinity(); + +class viterbi2 +{ +public: + viterbi2(const int d_row, + const int d_col, + const float *d_image, + const float *d_along_track_slope, + const float d_along_track_weight, + const unsigned int *d_upper_bounds, + const unsigned int *d_lower_bounds, + float *d_result) : f_row(d_row), + f_col(d_col), + f_image(d_image), + f_along_track_slope(d_along_track_slope), + f_along_track_weight(d_along_track_weight), + f_upper_bounds(d_upper_bounds), + f_lower_bounds(d_lower_bounds), + f_result(d_result) + { + find_path(); + } + + // VARIABLES + const int f_row, f_col; + const unsigned int *f_upper_bounds, *f_lower_bounds; + const float *f_image; + const float *f_along_track_slope, f_along_track_weight; + float *f_result; + + // METHODS + int calculate_best(float *path_prob); + float *find_path(void); + void viterbi_right(int *path, float *& path_prob, float *& path_prob_next, float *index); + + // Compute square value + template + inline T sqr(T x) { return x * x; } + + int encode(int x, int y) { return x * f_row + y; } + int vic_encode(int row, int col) { return f_row * col + row; } + + // THE CODE BELOW THIS POINT WAS TAKEN FROM DAVID CRANDALL + // With minor adjustments by Reece Mathews (transition_weight) + // Distance transform + // -- Every index from d1 to d2 will be set in dst and dst_ind + // -- Every index from s1 to s2 will be searched for minimum cost to each destination + // -- dst will contain the minimum cost for that destination + // -- dst_ind will contain the corresponding source index which yields the minimum cost for that destination + void dt(const float *src, float *dst, float *dst_ind, int s1, int s2, + int d1, int d2, float transition_weight, int off = 0) + { + // Destination and default source index set to midpoints of destination-space and source-space + int d = (d1 + d2) >> 1, s = (s1 + s2) >> 1; + for (int p = s1; p <= s2; p++) + { // Search through all the sources and find the index with minimum cost to the destination + // potential minimum cost < current minimum cost + if (src[p] + sqr(p - d - off) * transition_weight < src[s] + sqr(s - d - off) * transition_weight) + { + s = p; + } + } + dst[d] = src[s] + sqr(s - d - off) * transition_weight; // Minimum cost to the destination + dst_ind[d] = s; // Corresponding source index for the destination + if (d2 >= d + 1) + { // Recursive call, binary search (top half of destinations) + dt(src, dst, dst_ind, s, s2, d + 1, d2, transition_weight, off); + } + if (d - 1 >= d1) + { // Recursive call, binary search (bottom half of destinations) + dt(src, dst, dst_ind, s1, s, d1, d - 1, transition_weight, off); + } + } + // END CODE FROM DAVID CRANDALL +}; +#endif diff --git a/cresis-toolbox/+tomo/viterbi2.mexa64 b/cresis-toolbox/+tomo/viterbi2.mexa64 new file mode 100644 index 00000000..8454cb67 Binary files /dev/null and b/cresis-toolbox/+tomo/viterbi2.mexa64 differ diff --git a/cresis-toolbox/+tomo/viterbi2.mexmaci64 b/cresis-toolbox/+tomo/viterbi2.mexmaci64 new file mode 100644 index 00000000..584077d8 Binary files /dev/null and b/cresis-toolbox/+tomo/viterbi2.mexmaci64 differ diff --git a/cresis-toolbox/+tomo/viterbi2.mexw64 b/cresis-toolbox/+tomo/viterbi2.mexw64 new file mode 100644 index 00000000..7119f168 Binary files /dev/null and b/cresis-toolbox/+tomo/viterbi2.mexw64 differ diff --git a/cresis-toolbox/+tomo/viterbi_costs.md b/cresis-toolbox/+tomo/viterbi_costs.md new file mode 100644 index 00000000..82697a9a --- /dev/null +++ b/cresis-toolbox/+tomo/viterbi_costs.md @@ -0,0 +1,50 @@ +# Viterbi Costs + +Documentation of the extent to which various factors influence binary and unary cost calculation used in finding the best path in [viterbi.cpp](viterbi.cpp). + +## Context + +The *unary cost* is calculated for a given (x, y) point without regard for surrounding points (except in the case of image magnitude correlation cost). This occurs within `double* viterbi::find_path(void)`. + +The *binary cost* is calculated within [viterbi.h](viterbi.h) inside `void dt(...)` based on the distance from a given point to another. + +The *cummulative cost* is stored in the arrays `path_prob` and `path_prob_next` and accumulates the unary and binary cost to each row for the current and previous columns. + +Finding the best path is done by determining which row in the previous column to a given row in the current column yields the lowest cummulative cost. This is done for every row in the current column thus producing an array of lowest cost paths to every row in the current column. This is repeated for the next column and so on, only having to consider the previous column in each step (as the rows of the previous column are already known to hold the lowest cummulative cost possible to that point). + +## Trace + +Simplified steps taken in the Viterbi implementation. + +- `viterbi(...)` + - calls `find_path()` + - initializes `path`, `path_prob`, `path_prob_next`, and `index` to 0 arrays of size `depth` + - calls `viterbi_right(...)` + - for every column: + - appends `index` array to `path` matrix if after first column + - adds `unary_cost(col, row)` to every row + - calculates scale (`norm`) if not last column + - calls `dt_1d(...)` (distance transform -- binary cost calculation) if not last column + - calls `dt(...)` with initial recursion conditions + - recursively populates `path_prob_next` with minimum cummulative cost to each row from the previous + - recursively populates `index` with previous row index corresponding to these costs + - swaps `path_prob` with `path_prob_next` to use current column costs as previous for next column + - calls `calculate_best(path_prob)` and stores in `viterbi_index` + - returns index of minimum cummulative cost in last column + - populates `f_result` array back-to-front by following path of best indices backward from `viterbi_index` to the first column in `path` + - returns `f_result` + +## Cost Factors + +Upper and lower ends represent extreme costs calculated on tested frame `2011_Greenland_P3:20110331_02_019` from point (rangeline, rangebin) `(23.36, 425)` to point `(1624.88, 429)` in picker and reflect only cost added by the factor (except `LARGE` returns which do not add). The path returned by Viterbi for this input aligns fairly well with expected path. + +Factor | Influence | Conditions | Implementation | Notes | Upper end | Lower end | Average +---|---|---|---|---|---|---|--- +No Ice | Constant | No ice and y not in surface bin | `return LARGE`; | Condition not met on tested frame. | `LARGE` | `0` | `0` +Above Surface | Constant | `y + t + 1 < f_sgt[x]` | `return LARGE`; | | `LARGE` | `0` | `0` +Far from Center GT | Constant | Center GT exists, x is at center, and y is not within 20 bins of center GT | `return LARGE`; | Condition not met on tested frame. | `LARGE` | `0` | `0` +Far from Extra GT | Quadratic | extra GT present at x | `cost += f_weight_points[x] * 10 * sqr(((int)f_egt_y[f] - (int)(t + y)) / f_egt_weight)` | | `187142.4` | `0` | `49166.21` +Near Surface or Multiple Bin | Exponential | | `cost += max(0, (BIN_WEIGHT+base) * base^(-dist/(MAX_DIST+1) - multiple_bin/(MAX_NUM+1)) - base)`| [Explanation of Formula](https://www.geogebra.org/3d/zy3f6mde) | `10` | `0` | `.11823` +Far from Ice Mask | Linear | | `cost += f_costmatrix[f_costmatrix_X * DIM + y + t + 1 - f_sgt[x]]` | | `3.33362` | `0.00099` | `1.24319` +Image Magnitude | Linear | | `cost += -f_image[encode(x, y)]` | Able to decrease cost | `25.09595` | `-34.72310` | `-.00769` +Binary Cost (dt in [viterbi.h](viterbi.h)) | Quadratic | | `src[s] + sqr(s-d-off) * scale` | `src[s]` represents cummulative cost as `cost` does above | `31212` | `0` | `378.72577` diff --git a/cresis-toolbox/+tomo/viterbi_tests.m b/cresis-toolbox/+tomo/viterbi_tests.m new file mode 100644 index 00000000..7d51866c --- /dev/null +++ b/cresis-toolbox/+tomo/viterbi_tests.m @@ -0,0 +1,134 @@ +% Standalone test environment for viterbi implementation +% Author: Reece Mathews + +function viterbi_tests() + global matrix layer layers; + + % CONSTANTS + rows = 20; + cols = 15; + surf = 5; + mult = 10; + grnd = 17; + matrix = zeros(rows, cols); + matrix(surf, :) = ones(1, cols) * 30; % Surface + matrix(mult, :) = ones(1, cols) * 20; % Mult 1 + matrix(grnd, :) = ones(1, cols) * 10; % Bottom + + surf_layer = ones(1, cols) * surf; + gt_layer = nan(1, cols); + gt_layer([5 10]) = [16 16]; + + surf_costs = ones(1, cols) * 100; + gt_costs = nan(1, cols); + gt_costs(~isnan(gt_layer)) = 0; + + surf_cutoffs = nan(1, cols); + gt_cutoffs = nan(1, cols); + gt_cutoffs(~isnan(gt_layer)) = 2; + + layers = [surf_layer; gt_layer]; + layer_costs = [surf_costs; gt_costs]; + layer_cutoffs = [surf_cutoffs; gt_cutoffs]; + + % Viterbi params + transition_weights = ones(1, cols-1); + img_mag_weight = 1; + + mult_weight = 10; + mult_weight_decay = .3; + mult_weight_local_decay = .7; + + zero_bin = 1; + + % Other environment vars + mask = inf*ones(1, cols); + slope = round(diff(layers(1, :))); + max_slope = -1; + bounds = [5 10]; + mask_dist = inf(1, cols); + cost_matrix = ones(rows,cols); + + matrix = echo_norm(matrix,struct('scale',[-40 90])); + + % RUN + layer = tomo.viterbi(matrix, layers, layer_costs, layer_cutoffs, mask, ... + img_mag_weight, slope, max_slope, int64(bounds), [], mask_dist, cost_matrix, ... + transition_weights, mult_weight, mult_weight_decay, mult_weight_local_decay, ... + int64(zero_bin)); + hfig = setup(); + resize(hfig); + +end + +function plot_viterbi() + global layer layers; + + hold on; + + idxs = find(~isnan(layers(2, :))); + scatter(idxs, layers(2, idxs), 'rx'); + + plot(layer, 'g'); + + hold off; +end + + +function hfig = setup() + if ishandle(1) + hfig = figure(1); + else + hfig = figure('SizeChangedFcn', @resize); + end +end + + +function resize(src,~) + global matrix; + + r = size(matrix, 1); + c = size(matrix, 2); + + % Plot matrix + clf; + colormap(1-gray); + imagesc(matrix); + + % X-axis + xticks(.5:(c + .5)); + xlim([.5 (c + .5)]); + xticklabels(''); + + % Y-axis + yticks(.5:(r + .5)); + ylim([.5 (r + .5)]); + yticklabels(''); + grid; + + w = src.Position(3); + h = src.Position(4); + axes_pos = get(gca, 'Position'); + + left = axes_pos(1) * w; + bottom = axes_pos(2) * h; + + xmargin = axes_pos(3)*w/c; + ymargin = axes_pos(4)*h/r; + dimensions = [r c]; + for axis = 1:2 + for i = 1:dimensions(axis) + char = uicontrol('Style', 'text', 'String', sprintf('%d | %d', i-1, i), 'FontSize', 10); + char_pos = get(char, 'Extent'); + char_width = char_pos(3); + char_height = char_pos(4); + if axis == 2 + set(char, 'Position', [left + xmargin*i - xmargin/2 - char_width/2, bottom - char_height * 1.25, char_width, char_height]); + else + set(char, 'Position', [left - char_width * 1.25, bottom + ymargin*(r-i) + ymargin/2 - char_height/2, char_width, char_height]); + end + end + end + + plot_viterbi(); +end diff --git a/cresis-toolbox/+tomo/viterbi_tests2.m b/cresis-toolbox/+tomo/viterbi_tests2.m new file mode 100644 index 00000000..b1328399 --- /dev/null +++ b/cresis-toolbox/+tomo/viterbi_tests2.m @@ -0,0 +1,102 @@ +% Standalone test environment for viterbi implementation +% Author: Reece Mathews + +function viterbi_tests2() + global matrix layer elevation; + + % CONSTANTS + rows = 20; + cols = 20; + flatness = 100; + along_track_weight = 100; + down_shift = rows; + matrix = zeros(rows, cols); + shift_matrix = 1; + shift_elev = 1; + for i = 1:cols + matrix(floor(abs(sin(i)/flatness)*rows+1 - shift_matrix*(i/rows)*down_shift) + down_shift, i) = 30; % Layer + end + + elevation = (down_shift - (rows:-1:1) + 1) * shift_elev; + + along_track_slope = round(diff(elevation)); + upper_bounds = ones(1, cols); + lower_bounds = ones(1, cols)*size(matrix, 1); + + matrix = echo_norm(matrix,struct('scale',[-40 90])); + + % RUN + layer = tomo.viterbi2(single(matrix), along_track_slope, along_track_weight, upper_bounds, lower_bounds); + hfig = setup(); + resize(hfig); +end + +function plot_viterbi() + global layer elevation; + + hold on; + + plot(layer, 'g'); + plot(elevation); + + hold off; +end + + +function hfig = setup() + if ishandle(1) + hfig = figure(1); + else + hfig = figure('SizeChangedFcn', @resize); + end +end + + +function resize(src,~) + global matrix; + + r = size(matrix, 1); + c = size(matrix, 2); + + % Plot matrix + clf; + colormap(1-gray); + imagesc(matrix); + + % X-axis + xticks(.5:(c + .5)); + xlim([.5 (c + .5)]); + xticklabels(''); + + % Y-axis + yticks(.5:(r + .5)); + ylim([.5 (r + .5)]); + yticklabels(''); + grid; + + w = src.Position(3); + h = src.Position(4); + axes_pos = get(gca, 'Position'); + + left = axes_pos(1) * w; + bottom = axes_pos(2) * h; + + xmargin = axes_pos(3)*w/c; + ymargin = axes_pos(4)*h/r; + dimensions = [r c]; + for axis = 1:2 + for i = 1:dimensions(axis) + char = uicontrol('Style', 'text', 'String', sprintf('%d | %d', i-1, i), 'FontSize', 10); + char_pos = get(char, 'Extent'); + char_width = char_pos(3); + char_height = char_pos(4); + if axis == 2 + set(char, 'Position', [left + xmargin*i - xmargin/2 - char_width/2, bottom - char_height * 1.25, char_width, char_height]); + else + set(char, 'Position', [left - char_width * 1.25, bottom + ymargin*(r-i) + ymargin/2 - char_height/2, char_width, char_height]); + end + end + end + + plot_viterbi(); +end diff --git a/cresis-toolbox/+tomo/viterbi_tracker_2D.m b/cresis-toolbox/+tomo/viterbi_tracker_2D.m deleted file mode 100644 index 4cf623d8..00000000 --- a/cresis-toolbox/+tomo/viterbi_tracker_2D.m +++ /dev/null @@ -1,485 +0,0 @@ -function viterbi_tracker_2D (param, options, data_struct) -% function viterbi_tracker_2D (params, options, data_struct) -% -% Builds a large matrix containing multiple segments of 2D data and applies -% the Viterbi layer-tracking program on it. -% Loads crossovers from OPS to use as ground truth for the tracking. -% Other functionality includes multiple supression and -% a simple detrending technique. -% -% See also: run_tracker_2D.m -% -% Authors: Victor Berger - -%% Automated Section -% ===================================================================== -% Create param structure array -% ===================================================================== - -global gRadar; -param = merge_structs(param,gRadar); -physical_constants; - -%% Check parameters and set to default if needed -if isfield(options.viterbi, 'bottom_bin') && ~isempty(options.viterbi.bottom_bin) - bottom_bin = options.viterbi.bottom_bin; -else - bottom_bin = -1; -end - -if isfield(options.viterbi, 'egt_weight') && ~isempty(options.viterbi.egt_weight) - egt_weight = options.viterbi.egt_weight; -else - egt_weight = -1; -end - -if isfield(options.viterbi, 'ice_bin_thr') && ~isempty(options.viterbi.ice_bin_thr) - ice_bin_thr = options.viterbi.ice_bin_thr; -else - ice_bin_thr = 10; -end - -if isfield(options.viterbi, 'mu_size') && ~isempty(options.viterbi.mu_size) - mu_size = options.viterbi.mu_size; -else - mu_size = 31; -end - -if isfield(options.viterbi, 'mu_thr') && ~isempty(options.viterbi.mu_thr) - mu_thr = options.viterbi.mu_thr; -else - mu_thr = -30; -end - -if isfield(options.viterbi, 'mu') && ~isempty(options.viterbi.mu) - mu = options.viterbi.mu; -else - mu = log10(exp(-(-(mu_size-1)/2 : (mu_size-1)/2).^4/1)); - mu(mu < mu_thr) = mu_thr; - mu = mu - mean(mu); -end - -if isfield(options.viterbi, 'repulsion') && ~isempty(options.viterbi.repulsion) - repulsion = options.viterbi.repulsion; -else - repulsion = 150000; -end - -if isfield(options.viterbi, 'sigma') && ~isempty(options.viterbi.sigma) - sigma = options.viterbi.sigma; -else - sigma = sum(abs(mu))/10*ones(1, mu_size); -end - -if isfield(options.viterbi, 'smooth_var') && ~isempty(options.viterbi.smooth_var) - smooth_var = options.viterbi.smooth_var; -else - smooth_var = Inf; -end - -if isfield(options.viterbi, 'smooth_weight') && ~isempty(options.viterbi.smooth_weight) - smooth_weight = options.viterbi.smooth_weight; -else - smooth_weight = 5; -end - -if isfield(options.viterbi, 'surf_layer_params') && ~isempty(options.viterbi.surf_layer_params) - surf_layer_params = options.viterbi.surf_layer_params; -else - surf_layer_params = []; -end - -if ~isfield(surf_layer_params, 'name') || ~isempty(surf_layer_params.name) - surf_layer_params.name = 'surface'; -end - -%% Distance-to-Ice-Margin model -if ~isfield(options.viterbi, 'DIM_matrix') || isempty(options.viterbi.DIM_matrix) - prob.DIM_means = [6.908407 14.603709 22.077745 29.333980 36.375879 43.206908 49.830531 56.250214 62.469423 68.491622 74.320277 79.958853 85.410815 90.679629 95.768760 100.681673 105.421833 109.992706 114.397756 118.640450 122.724252 126.652627 130.429042 134.056960 137.539848 140.881171 144.084393 147.152981 150.090399 152.900113 155.585588 158.150289 160.597681 162.931230 165.154401 167.270659 169.283470 171.196299 173.012610 174.735870 176.369543 177.917095 179.381991 180.767696 182.077676 183.315395 184.484320 185.587915 186.629645 187.612977 188.541374 189.418303 190.247228 191.031615 191.774929 192.480636 193.152200 193.793087 194.406763 194.996691 195.566338 196.119170 196.658650 197.188245 197.711419 198.231639 198.752368 199.277073 199.809219 200.352271 200.909693 201.484953 202.081514 202.702842 203.352402 204.033660 204.750081 205.505130 206.302272 207.144972 208.036696 208.980910 209.981077 211.040664 212.163136 213.351958 214.610596 215.942514 217.351178 218.840053 220.412604 222.072297 223.822597 225.666969 227.608878 229.651790 231.799170 234.054483 236.421194 238.902770]; - prob.DIM_vars = [19.032807 25.055615 28.640839 32.753438 36.144273 39.329838 42.688930 45.511082 49.036818 52.177650 55.028380 57.516627 60.519048 63.217206 65.211203 67.459337 69.609678 71.543557 73.182822 74.615772 75.628159 77.127086 78.155483 79.447090 80.011376 81.108576 81.618789 82.287856 82.979740 83.561585 84.281769 84.648076 85.290095 85.566969 86.052342 86.487424 86.675812 86.959733 87.181337 87.641261 87.674246 87.947628 87.895269 87.286380 87.202972 86.878606 87.151259 87.477659 88.049960 88.587946 88.515276 89.070799 88.756636 88.345201 87.754785 87.689382 87.240118 86.800999 86.164340 86.085916 85.803664 85.356194 85.831974 85.264038 85.222428 84.898093 84.652262 84.332790 84.249144 83.871931 83.552786 83.233334 82.842279 82.658637 82.008042 81.694151 81.421515 80.901673 80.885452 81.070003 80.524210 80.776716 80.320438 80.445820 80.085639 79.751146 79.557559 78.923447 78.522063 77.525973 77.426494 76.624448 76.855826 77.277564 76.777165 76.716292 75.970217 77.149291 76.900846 76.890210]; - gauss = @(x, mean, var)((1 / (sqrt(2 * pi * (var.^2)))) * (exp(-((x - mean).^2)/(2 * var.^ 2)))); - costm = ones(50, 100); - for t = 1:50 - for i = 1:100 - costm(t,i) = 10 * exp(-0.01 .* t) - 10 * exp(-0.01 * 50); - end - end - alpha = 0.025; - beta = 0.05; - - DIM_costmatrix = ones(1000, 100); - for DIM = 1 : 100; - for T = 1 : 1000 - if alpha / gauss(T, prob.DIM_means(DIM), DIM * beta * prob.DIM_vars(DIM)) >= 200 - DIM_costmatrix(T, DIM) = 200; - else - DIM_costmatrix(T, DIM) = alpha/ gauss(T, prob.DIM_means(DIM), DIM * beta * prob.DIM_vars(DIM)); - end - end - end - - DIM_costmatrix = DIM_costmatrix(15:end,:); - - for k = 1:100 - DIM_costmatrix(1:50, k) = DIM_costmatrix(1:50, k) + k * costm(1:50, k); - end -else - clear DIM DIM_costmatrix; - DIM = load(fullfile(gRadar.path, options.viterbi.DIM_matrix)); - DIM_costmatrix = DIM.Layer_tracking_2D_parameters; - DIM_costmatrix = DIM_costmatrix .* (200 ./ max(DIM_costmatrix(:))); -end - -labels = {}; -big_matrix = {}; -big_matrix.Data = []; -big_matrix.GPS_time = []; -big_matrix.Lat = []; -big_matrix.Lon = []; -good_frms = param.cmd.frms; - -try - Surface = opsLoadLayers(param,surf_layer_params); -catch ME - warning(ME.getReport); - keyboard -end - -for frm = param.cmd.frms - if options.debug - fprintf('\nRunning frame %s_%03d.\n',param.day_seg,frm); - end - - try - cur_matrix = data_struct.(sprintf('data_%s_%03d',param.day_seg,frm)); - catch ME - good_frms = good_frms(good_frms ~= frm); - continue; - end - - big_matrix.GPS_time = horzcat(big_matrix.GPS_time, cur_matrix.GPS_time); - big_matrix.Lat = horzcat(big_matrix.Lat, cur_matrix.Latitude); - big_matrix.Lon = horzcat(big_matrix.Lon, cur_matrix.Longitude); - big_matrix.Time = cur_matrix.Time; - - %% Image combine - if ~options.viterbi.custom_combine - big_matrix.Data = horzcat(big_matrix.Data, cur_matrix.Data); - else - try - mode = 'get_heights'; - param.(mode).out_path = options.name; - param.load.frm = frm; - param.(mode).img_comb = param.combine.img_comb; - param.(mode).img_comb_mult = 1.15; - param.(mode).img_comb_bins = 50; - [Data, big_matrix.Time] = img_combine(param, mode, Surface); - big_matrix.Data = horzcat(big_matrix.Data, Data); - catch ME - if options.debug - fprintf('\nProblem during custom image combining'); - end - big_matrix.Data = horzcat(big_matrix.Data, cur_matrix.Data); - end - end -end - -if options.debug && options.viterbi.custom_combine - fprintf('\nDone: image combine (%s)', datestr(now,'HH:MM:SS')); -end - -% Catch error with layer not having correctly loaded from OPS -if(isempty(Surface.gps_time) || any(isnan(Surface.gps_time)) || any(isnan(Surface.twtt))) - fprintf('\nProblem processing frame %s', param.day_seg); - return; -end - -if options.debug - fprintf('\nDone: opsLoadLayers (%s)', datestr(now,'HH:MM:SS')); -end - -%% - -big_matrix.Surface = interp_finite(interp1(Surface.gps_time,Surface.twtt,big_matrix.GPS_time)); - -if options.debug - fprintf('\nDone: opsAuthenticate, opsGetSegmentInfo, time-range bin interpolation (%s)', datestr(now,'HH:MM:SS')); -end - -big_matrix.Data = lp(big_matrix.Data); -surf_bins = round(interp1(big_matrix.Time, 1:length(big_matrix.Time), big_matrix.Surface)); - -%% Top suppression -if options.viterbi.top_sup - topbuffer = 5; - botbuffer = 15; - filtvalue = 50; - for rline = 1 : size(big_matrix.Data, 2) - column_chunk = big_matrix.Data(round(surf_bins(rline) - topbuffer) : ... - round(surf_bins(rline) + botbuffer), rline); - big_matrix.Data(round(surf_bins(rline) - topbuffer) : ... - round(surf_bins(rline) + botbuffer), rline) = imgaussfilt(column_chunk, filtvalue); - end - - if options.debug - fprintf('\nDone: top suppression (%s)', datestr(now,'HH:MM:SS')); - end -end - -%% Multiple suppression -if options.viterbi.mult_sup - topbuffer = 12; - botbuffer = 12; - filtvalue = 30; - - % Step one: Determine the bins associated with the surface multiple - surf_mult_twtt = 2*big_matrix.Surface; - surf_mult_bins = round(interp1(big_matrix.Time, 1:length(big_matrix.Time), surf_mult_twtt)); - - % Step two: Filter the whole image - filt_img = fir_dec(fir_dec(big_matrix.Data,ones(1,31),1).',ones(1,31),1).'; - - % Step three: replace the surface multiple bins with the filtered image - for rline = 1 : size(big_matrix.Data, 2) - try - big_matrix.Data(round(surf_mult_bins(rline) - topbuffer) : ... - round(surf_mult_bins(rline) + botbuffer), rline) = ... - filt_img(round(surf_mult_bins(rline) - topbuffer) : ... - round(surf_mult_bins(rline) + botbuffer), rline); - catch ME - continue; - end - end -end - -Nx = size(big_matrix.Data, 2); -slope = round(diff(big_matrix.Surface)); -viterbi_weight = ones([1 Nx]); - -%% Ice mask calculation -if options.binary_icemask - mask = load(options.ice_mask_mat_fn,'R','X','Y','proj'); - [fid,msg] = fopen(options.icemask_fn,'r'); - if fid < 1 - fprintf('Could not open file %s\n', options.ice_mask_bin_fn); - error(msg); - end - mask.maskmask = logical(fread(fid,[length(mask.Y),length(mask.X)],'uint8')); - fclose(fid); -else - [mask.maskmask,mask.R,~] = geotiffread(options.icemask_fn); - mask.proj = geotiffinfo(options.icemask_fn); -end -[mask.x, mask.y] = projfwd(mask.proj, big_matrix.Lat, big_matrix.Lon); -mask.X = mask.R(3,1) + mask.R(2,1)*(1:size(mask.maskmask,2)); -mask.Y = mask.R(3,2) + mask.R(1,2)*(1:size(mask.maskmask,1)); -[mask.X,mask.Y] = meshgrid(mask.X,mask.Y); -ice_mask.mask = round(interp2(mask.X, mask.Y, double(mask.maskmask), mask.x, mask.y)); -ice_mask.mask(isnan(ice_mask.mask)) = 1; - -if options.debug - fprintf('\nDone: ice mask calculation (%s)', datestr(now,'HH:MM:SS')); -end - -%% Crossover loading -if options.viterbi.crossoverload - opsAuthenticate(param,false); - layer_name = 'bottom'; - sys = ct_output_dir(param.radar_name); - ops_param = struct('properties',[]); - ops_param.properties.season = param.season_name; - ops_param.properties.segment = param.day_seg; - [~,ops_frames] = opsGetSegmentInfo(sys,ops_param); - - ops_param = struct('properties',[]); - ops_param.properties.location = param.post.ops.location; - ops_param.properties.season = param.season_name; - ops_param.properties.start_gps_time = ops_frames.properties.start_gps_time(1); - ops_param.properties.stop_gps_time = ops_frames.properties.stop_gps_time(end); - ops_param.properties.nativeGeom = true; - [~,ops_data] = opsGetPath(sys,ops_param); - - query = sprintf('SELECT rds_segments.id FROM rds_seasons,rds_segments where rds_seasons.name=''%s'' and rds_seasons.id=rds_segments.season_id and rds_segments.name=''%s''',param.season_name,param.day_seg); - [~,tables] = opsQuery(query); - segment_id = tables{1}; - - ops_param = struct('properties',[]); - ops_param.properties.location = param.post.ops.location; - ops_param.properties.lyr_name = layer_name; - ops_param.properties.frame = ops_frames.properties.frame; - ops_param.properties.segment_id = ones(size(ops_param.properties.frame)) ... - *double(segment_id); - - [~,data] = opsGetCrossovers(sys,ops_param); - - %% Load and align crossovers - rline = []; - rows = []; - cols = []; - gps_time = []; - season_name = {}; - for i = 1 : length(data.properties.source_point_path_id) - if ~isnan(data.properties.twtt(i)) - new_rline = find(ops_data.properties.id == data.properties.source_point_path_id(i)); - new_gps_time = ops_data.properties.gps_time(new_rline); - new_season_name = data.properties.season_name{i}; - if big_matrix.GPS_time(1) <= new_gps_time ... - && big_matrix.GPS_time(end) >= new_gps_time %... - %&& str2double(new_season_name(1:4)) >= 2006 - rline(end+1) = new_rline; - gps_time(end+1) = new_gps_time; - season_name{end+1} = new_season_name; - [~, cols(end+1)] = min(abs(big_matrix.GPS_time - ops_data.properties.gps_time(rline(end)))); - twtt = data.properties.twtt(i); - twtt = twtt + (data.properties.source_elev(i)-data.properties.cross_elev(i))/(c/2); - rows(end+1) = round(interp1(big_matrix.Time, 1:length(big_matrix.Time), twtt)); - end - end - end - gt = [cols(:).'; rows(:).']; - if options.debug - fprintf('\nDone: opsGetCrossovers (%s)', datestr(now,'HH:MM:SS')); - end -else - gt = ''; -end - -%% Set variable echogram tracking parameters -bounds = [0 Nx]; -ice_mask.mask_dist = round(bwdist(ice_mask.mask == 0)); -transition_mu = -1 * ones(1, size(big_matrix.Data, 2)); -transition_sigma = 0.3759 * ones(1, size(big_matrix.Data, 2)); - -%% Detrending routine -if options.viterbi.detrending - detect_data = big_matrix.Data; - % Along track filtering - detect_data = fir_dec(detect_data,ones(1,5)/5,1); - % Estimate noise level - noise_value = mean(mean(detect_data(end-80:end-60,:))); - % Estimate trend - trend = mean(detect_data,2); - trend(trend insert_param.max_dist^2) = NaN; end copy_param = []; diff --git a/cresis-toolbox/OPS-MATLAB/input/opsInsertLayerFromGrid.m b/cresis-toolbox/OPS-MATLAB/input/opsInsertLayerFromGrid.m deleted file mode 100644 index 29278843..00000000 --- a/cresis-toolbox/OPS-MATLAB/input/opsInsertLayerFromGrid.m +++ /dev/null @@ -1,494 +0,0 @@ -function opsInsertLayerFromGrid(grid_param,layers) -% opsInsertLayerFromGrid(grid_param,layers) -% -% Makes layer for OPS from grid map using interp2. This function calls -% -% Input: -% grid_param: a structure the controls how to create layers -% .param_fn = file name of the parameter spreadsheets for a field season, -% set the generic column of the cmd worksheet to 1 for the -% selected segments and frames. -% .save_sources = string array containing layer source to update. Can be 'echogram', -% 'layerData', or 'ops'. -% .echogram_source = string to source for echogram source option. -% For example 'CSARP_post/qlook', 'standard', etc -% .layerdata_source = string to source for layerData source option. -% For example 'CSARP_post/layerData', 'layerData', etc -% .plot_en = set to true for displaying echograms with layers, set to -% false for creating layers faster -% proj: the projection structure of the grid map -% save_mode: string containing 'overwrite' or 'merge' (merge makes all -% NaN in the new data source be unchanged) -% layers: structure array with layer information -% .username = string containing username for this layer; -% .name = string with layer name; -% .group = string with layer group; -% .ref_name = string with reference layer name (optional), -% often set to 'surface' -% .ref_eval = string with command to evaluate that combines new layer -% with reference. Variables available to the command are: -% data: this is the data passed in after it is interpolated to flightline -% ref_data: this is the reference layer data (twtt, sec) -% elev: platform elevation (m) -% Examples: -% ref_eval = 'data = ref_data + data;' -% ref_eval = 'data = data - (elev - ref_data*149896229)/84456957 + ref_data;' -% not used. If not empty, eval(layers(lyr_idx).ref_data) will be run -% where ref_data contains ref layer data and data contains grid data -% interpolated to the line. -% .echogram_layer_name = string containing the variable name to store -% into the echogram file (usually 'Surface' or 'Bottom') -% .layerdata_layer_name = string that indicates which layer to save the -% data to. For example, 'surface' (idx 1), 'bottom' (idx 2). -% .x = x-vector for data matrix -% .y = y-vector for data matrix -% .data = twtt data matrix (or data matrix which will be used by ref_data) -% -% Output: layer points in OPS -% -% Examples: -% Examples at bottom of this file -% -% Author: Jilu Li, John Paden -% -% See also: opsCreateLayerPoints.m, opsInsertLayerFromPointCloud.m - -%% Load param worksheet -params = read_param_xls(grid_param.param_fn,[],'post'); - -param.day_seg = []; -for param_idx = 1:length(params) - param = params(param_idx); - - %% Determine if this segment should be processed or not - if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic - continue; - end - - % Load frames file - frames = load(ct_filename_support(param,'','frames')); - - %% Determine which frames need to be processed - if isempty(param.cmd.frms) - param.cmd.frms = 1:length(frames.frame_idxs); - end - % Remove frames that do not exist from param.cmd.frms list - [valid_frms,keep_idxs] = intersect(param.cmd.frms, 1:length(frames.frame_idxs)); - if length(valid_frms) ~= length(param.cmd.frms) - bad_mask = ones(size(param.cmd.frms)); - bad_mask(keep_idxs) = 0; - warning('Nonexistent frames specified in param.cmd.frms (e.g. frame "%g" is invalid), removing these', ... - param.cmd.frms(find(bad_mask,1))); - param.cmd.frms = valid_frms; - end - - if any(strcmpi(grid_param.save_sources,'ops')) - %% Get all the frames for this segment - sys = ct_output_dir(param.radar_name); - ops_param = struct('properties',[]); - ops_param.properties.season = param.season_name; - ops_param.properties.segment = param.day_seg; - [status,ops_seg_data] = opsGetSegmentInfo(sys,ops_param); - end - - %% Update each of the frames - for frm_idx = 1:length(param.cmd.frms) - frm = param.cmd.frms(frm_idx); - - fprintf(' Updating %s frame %03d (%d of %d) (%s)\n', param.day_seg, ... - frm, frm_idx, length(param.cmd.frms), datestr(now,'HH:MM:SS')); - data_fn = fullfile(ct_filename_out(param,grid_param.echogram_source,''), ... - sprintf('Data_%s_%03d.mat', param.day_seg, frm)); - if grid_param.plot_en - plot_vars = load(data_fn,'Time','Data'); - end - - if any(strcmpi(grid_param.save_sources,'echogram')) - %% 1. Open the specific echogram data file - mdata = load(data_fn,'GPS_time','Latitude','Longitude','Elevation'); - - %% 2. Interpolate the grid onto this echogram - [x,y] = projfwd(grid_param.proj,mdata.Latitude,mdata.Longitude); - - %% 3. Save the layer back to the file under grid_param.echogram_layer_name - for layer_idx = 1:length(layers) - data = interp2(reshape(layers(layer_idx).x,[1 numel(layers(layer_idx).x)]), ... - reshape(layers(layer_idx).y,[numel(layers(layer_idx).y) 1]),layers(layer_idx).data,x,y); - if isfield(layers(layer_idx),'ref_name') - tmp = load(data_fn,layers(layer_idx).ref_name); - ref_data = tmp.(layers(layer_idx).ref_name); - ref_data = interp_finite(ref_data); - elev = mdata.Elevation; - eval(layers(layer_idx).ref_eval); - end - - if strcmpi(grid_param.save_mode,'merge') - data(isnan(data)) = mdata.(layers(layer_idx).echogram_layer_name)(isnan(data)); - end - mdata.(layers(layer_idx).echogram_layer_name) = data; - - if grid_param.plot_en - figure(1); clf; - imagesc([],plot_vars.Time,lp(plot_vars.Data)); - colormap(1-gray(256)); - hold on; - plot(mdata.(layers(layer_idx).echogram_layer_name)); - hold off; - keyboard - end - - save(data_fn,'-append','-struct','mdata',layers(layer_idx).echogram_layer_name); - end - end - - if any(strcmpi(grid_param.save_sources,'layerdata')) - %% 1. Open the specific layer data file - layer_fn = fullfile(ct_filename_out(param,grid_param.layerdata_source,''), ... - sprintf('Data_%s_%03d.mat', param.day_seg, frm)); - if ~exist(layer_fn,'file') - error('Layer file %s does not exist', layer_fn); - end - % Load the layerData file - lay = load(layer_fn); - - %% 2. Interpolate the grid onto this layerData - [x,y] = projfwd(grid_param.proj,lay.Latitude,lay.Longitude); - - for layer_idx = 1:length(layers) - data = interp2(reshape(layers(layer_idx).x,[1 numel(layers(layer_idx).x)]), ... - reshape(layers(layer_idx).y,[numel(layers(layer_idx).y) 1]),layers(layer_idx).data,x,y); - if isfield(layers(layer_idx),'ref_name') - if strcmpi(layers(layer_idx).ref_name,'surface') - ref_idx = 1; - elseif strcmpi(layers(layer_idx).ref_name,'bottom') - ref_idx = 2; - else - found = false; - for ref_idx = 1:length(lay.layerData) - if isfield(lay.layerData{ref_idx},'name') ... - && strcmpi(lay.layerData{ref_idx}.name,layers(layer_idx).ref_name) - found = true; - break; - end - end - if ~found - error('Reference layer %s not found\n', layers(layer_idx).ref_name); - end - end - ref_data = lay.layerData{ref_idx}.value{2}.data; - ref_data = interp_finite(ref_data); - elev = lay.Elevation; - eval(layers(layer_idx).ref_eval); - end - - % Determine which index into lay.layerData needs to be updated - if strcmpi(layers(layer_idx).name,'surface') - lay_idx = 1; - found = true; - elseif strcmpi(layers(layer_idx).name,'bottom') - lay_idx = 2; - found = true; - else - found = false; - for lay_idx = 1:length(lay.layerData) - if isfield(lay.layerData{lay_idx},'name') ... - && strcmpi(lay.layerData{lay_idx}.name,layers(layer_idx).name) - found = true; - break; - end - end - if ~found - % Add a new layer if layer does not exist - lay_idx = length(lay.layerData) + 1; - % Create manual points - lay.layerData{lay_idx}.value{1}.data = NaN*zeros(size(data)); - % Create auto points - lay.layerData{lay_idx}.value{2}.data = NaN*zeros(size(data)); - % Set quality to good - lay.layerData{lay_idx}.quality.data = 1*ones(size(data)); - end - end - - % Update automated points - if strcmpi(grid_param.save_mode,'merge') - data(isnan(data)) = lay.layerData{lay_idx}.value{2}.data(isnan(data)); - end - lay.layerData{lay_idx}.value{2}.data = data; - - if grid_param.plot_en - figure(1); clf; - imagesc([],plot_vars.Time,lp(plot_vars.Data)); - colormap(1-gray(256)); - hold on; - plot(lay.layerData{lay_idx}.value{2}.data); - hold off; - keyboard - end - - %% 3. Save the layer back to the file under grid_param.layerdata_layer_name - % If layer names do not exist, then assume 'surface' is 1, 'bottom' - % is 2, and if a layer does not exist then a new layer is created. - save(layer_fn,'-append','-struct','lay','layerData'); - end - end - - if any(strcmpi(grid_param.save_sources,'ops')) - for layer_idx = 1:length(layers) - start_gps = ops_seg_data.properties.start_gps_time(frm); - stop_gps = ops_seg_data.properties.stop_gps_time(frm); - - %% 1. Check to see if layer exists - [status,data] = opsGetLayers(sys); - if ~any(strcmpi(data.properties.lyr_name,layers(layer_idx).name)) - % Create the layer if it does not exist - ops_param = []; - ops_param.properties.lyr_name = layers(layer_idx).name; - ops_param.properties.lyr_group_name = layers(layer_idx).group; - ops_param.properties.lyr_description = layers(layer_idx).description; - ops_param.properties.public = true; - - [status,ops_data] = opsCreateLayer(sys,ops_param); - end - - %% 2. OPS query to get all the point path ID's - ops_param = struct('properties',[]); - ops_param.properties.location = param.post.ops.location; - ops_param.properties.season = param.season_name; - ops_param.properties.start_gps_time = start_gps; - ops_param.properties.stop_gps_time = stop_gps; - ops_param.properties.nativeGeom = true; - [status,ops_data] = opsGetPath(sys,ops_param); - - %% 3. Interpolate the data onto the point path ID's - [x,y] = projfwd(grid_param.proj,ops_data.properties.Y,ops_data.properties.X); - data = interp2(reshape(layers(layer_idx).x,[1 numel(layers(layer_idx).x)]), ... - reshape(layers(layer_idx).y,[numel(layers(layer_idx).y) 1]),layers(layer_idx).data,x,y); - quality = round(interp2(reshape(layers(layer_idx).x,[1 numel(layers(layer_idx).x)]), ... - reshape(layers(layer_idx).y,[numel(layers(layer_idx).y) 1]),layers(layer_idx).quality,x,y)); - - %% 4. Load reference data - if isfield(layers(layer_idx),'ref_name') - ops_param = struct('properties',[]); - ops_param.properties.location = param.post.ops.location; - ops_param.properties.season = param.season_name; - ops_param.properties.segment = param.day_seg; - ops_param.properties.start_gps_time = start_gps; - ops_param.properties.stop_gps_time = stop_gps; - ops_param.properties.lyr_name = layers(layer_idx).ref_name; - [status,ref_data] = opsGetLayerPoints(sys,ops_param); - % Interpolate ref data onto point path - [ref_data.properties.gps_time,sort_idxs] = sort(ref_data.properties.gps_time); - ref_data.properties.twtt = ref_data.properties.twtt(sort_idxs); - ref_data = interp1(ref_data.properties.gps_time,ref_data.properties.twtt, ... - ops_data.properties.gps_time); - ref_data = interp_finite(ref_data); - elev = ops_data.properties.elev; - eval(layers(layer_idx).ref_eval); - end - - if strcmpi(grid_param.save_mode,'merge') - % Remove NaN layer points - ops_data.id = ops_data.id(~isnan(data)); - quality = quality(~isnan(data)); - data = data(~isnan(data)); - end - - ops_param = struct('properties',[]); - ops_param.properties.point_path_id = ops_data.properties.id; - ops_param.properties.twtt = double(data); - ops_param.properties.type = 2*ones(size(ops_param.properties.twtt)); - ops_param.properties.quality = double(quality); - ops_param.properties.lyr_name = layers(layer_idx).name; - - if grid_param.plot_en - figure(1); clf; - imagesc([],plot_vars.Time,lp(plot_vars.Data)); - colormap(1-gray(256)); - hold on; - plot(ops_param.properties.twtt); - hold off; - figure(2); clf; - imagesc(layers.x,layers.y,layers.data); set(gca,'ydir','normal'); - hold on; - plot(x,y,'k') - keyboard - end - - opsCreateLayerPoints(sys,ops_param); - end - end - - end - -end - -return; - -%% Example 1: mass conservation grid -% Use parameters spreadsheet to select segment and frame list for creating layers -% Set the generic to 1 for the selected segments and frames - -param = []; -param.param_fn = ct_filename_param('rds_param_2009_Greenland_TO.xls'); - -% grid_fn = '/cresis/snfs1/scratch/paden/mass_conservation/HelheimStream-2014-11-18.nc'; -grid_fn = '/cresis/snfs1/scratch/paden/mass_conservation/KangerdlugssuaqStream-2014-11-18.nc'; - -% Create geotiff projection structure -% 1. Grab from a geotiff file with the same projection -%param.proj = geotiffinfo(ct_filename_gis([],'greenland\Landsat-7\Greenland_natural_90m.tif')); -% 2. Create an mstruct with the same projection -%[x,y,mstruct] = geodetic_to_stereographic(70,-45); -% 3. Hand construction of geotiff projection OR -param.proj = []; -param.proj.CornerCoords = []; -param.proj.Ellipsoid = []; -param.proj.PM = []; -param.proj.PMLongToGreenwich = []; -param.proj.Zone = []; -geoid = almanac('earth',ncreadatt(grid_fn,'polar_stereographic','ellipsoid'),'m'); -param.proj.SemiMajor = geoid; -param.proj.SemiMinor = minaxis(geoid); -param.proj.ProjParm = zeros(7,1); -param.proj.ProjParm(1) = ncreadatt(grid_fn,'polar_stereographic','standard_parallel'); -param.proj.ProjParm(2) = ncreadatt(grid_fn,'polar_stereographic','straight_vertical_longitude_from_pole'); -param.proj.ProjParm(3) = 0; -param.proj.ProjParm(4) = 0; -param.proj.ProjParm(5) = 1; -param.proj.ProjParm(6) = ncreadatt(grid_fn,'polar_stereographic','false_easting'); -param.proj.ProjParm(7) = ncreadatt(grid_fn,'polar_stereographic','false_northing'); - -param.proj.CTProjection = 'CT_PolarStereographic'; % Choose from list in toolbox/map/mapproj/private/projcode -param.proj.GeoTIFFCodes.CTProjection = 15; % CT_PolarStereographic from toolbox/map/mapproj/private/projcode - -% Default GeoTIFFCode values -param.proj.GeoTIFFCodes.Model = int16(1); -param.proj.GeoTIFFCodes.PCS = int16(32767); -param.proj.GeoTIFFCodes.GCS = int16(32767); -param.proj.GeoTIFFCodes.UOMAngle = int16(32767); -param.proj.GeoTIFFCodes.Datum = int16(32767); -param.proj.GeoTIFFCodes.PM = int16(32767); -param.proj.GeoTIFFCodes.ProjCode = int16(32767); -param.proj.GeoTIFFCodes.Projection= int16(32767); -param.proj.GeoTIFFCodes.MapSys = int16(32767); - -% Default units to meters for now -param.proj.GeoTIFFCodes.UOMLength = int16(9001); - -layers = []; -layers(1).username = 'paden'; -layers(1).name = 'mc_bottom'; -layers(1).group = 'standard'; -layers(1).description = 'Mass conservation ice bottom'; -layers(1).ref_name = 'surface'; -layers(1).ref_eval = 'data = ref_data + data;'; -layers(1).echogram_layer_name = 'mc_bottom'; -layers(1).layerdata_layer_name = 'mc_bottom'; -layers(1).x = ncread(grid_fn,'x'); -layers(1).y = ncread(grid_fn,'y'); -physical_constants; -layers(1).data = ncread(grid_fn,'thickness').' / (c/2/sqrt(er_ice)); - -param.plot_en = false; -param.echogram_source = 'mvdr'; -param.layerdata_source = 'layerData'; -% save_sources: cell array of strings indicating which layer sources -% should be updated (options are ops, layerData, and echogram) -param.save_sources = {'ops'}; -param.save_mode = 'overwrite'; -opsInsertLayerFromGrid(param,layers); - -%% Example 2: Gogineni's Jakobshavn grid -% Use parameters spreadsheet to select segment and frame list for creating layers -% Set the generic to 1 for the selected segments and frames - -param = []; -param.param_fn = ct_filename_param('rds_param_2009_Greenland_TO.xls'); - -grid_fn = '/cresis/snfs1/scratch/paden/mass_conservation/Jakobshavn_2006_2009_Composite/Jakobshavn_2006_2009_Composite/grids/jakobshavn_2006_2009_composite_thickness.txt'; - -% Load grid -fid = fopen(grid_fn,'r'); -fseek(fid,0,-1); -grid = []; -for idx=1:6 - fields = textscan(fid,'%s%f',1); - grid.(fields{1}{1}) = fields{2}; -end -thickness = textscan(fid,'%f'); -fclose(fid); -thickness = reshape(thickness{1},[grid.ncols grid.nrows]).'; -thickness(thickness == grid.NODATA_value) = NaN; -imagesc(thickness); -x_axis = grid.xllcorner + grid.cellsize*(0:grid.ncols-1); -y_axis = grid.yllcorner + grid.cellsize*(0:grid.nrows-1); -y_axis = y_axis(end:-1:1); - -% Create geotiff projection structure -% 1. Grab from a geotiff file with the same projection -param.proj = geotiffinfo(ct_filename_gis([],'greenland\Landsat-7\Greenland_natural_90m.tif')); - -layers = []; -layers(1).username = 'paden'; -layers(1).name = 'gogineni2014'; -layers(1).group = 'standard'; -layers(1).description = 'Gogineni JofG 2014 grid'; -layers(1).ref_name = 'surface'; -layers(1).ref_eval = 'data = ref_data + data;'; -layers(1).echogram_layer_name = 'gogineni2014'; -layers(1).layerdata_layer_name = 'gogineni2014'; -layers(1).x = x_axis; -layers(1).y = y_axis; -physical_constants; -layers(1).data = thickness / (c/2/sqrt(er_ice)); - -param.plot_en = false; -param.echogram_source = 'mvdr'; -param.layerdata_source = 'layerData'; -% save_sources: cell array of strings indicating which layer sources -% should be updated (options are ops, layerData, and echogram) -param.save_sources = {'ops'}; -param.save_mode = 'overwrite'; -opsInsertLayerFromGrid(param,layers); - -%% Example 3: Joe Plummer's Jakobshavn grid -% Use parameters spreadsheet to select segment and frame list for creating layers -% Set the generic to 1 for the selected segments and frames - -param = []; -param.param_fn = ct_filename_param('rds_param_2009_Greenland_TO.xls'); - -grid_fn = ct_filename_gis([],'greenland/plummer_jakobshavn/jak_grid.tif'); - -param.proj = geotiffinfo(grid_fn); - -% Load the grid -proj = geotiffinfo(grid_fn); -[bt_elev,R] = geotiffread(grid_fn); -bt_elev(bt_elev == min(min(bt_elev))) = NaN; -X = R.XLimWorld(1) + [R.XLimIntrinsic(1):R.XLimIntrinsic(2)-1]'*R.DeltaX; -Y = R.YLimWorld(2) + [R.YLimIntrinsic(1):R.YLimIntrinsic(2)-1]'*R.DeltaY; - -layers = []; -layers(1).username = 'paden'; -layers(1).name = 'plummer'; -layers(1).group = 'standard'; -layers(1).description = 'Plummer Jakobshavn Grid'; -layers(1).ref_name = 'surface'; -layers(1).ref_eval = sprintf('data = (elev - ref_data*%.0f)/%.0f - data + ref_data;',c/2,c/2/sqrt(er_ice)); -layers(1).echogram_layer_name = 'plummer'; -layers(1).layerdata_layer_name = 'plummer'; -layers(1).x = X; -layers(1).y = Y; -physical_constants; -layers(1).data = bt_elev/(c/2/sqrt(er_ice)); - -param.plot_en = false; -param.echogram_source = 'mvdr'; -param.layerdata_source = 'layerData'; -% save_sources: cell array of strings indicating which layer sources -% should be updated (options are ops, layerData, and echogram) -param.save_sources = {'ops'}; -param.save_mode = 'overwrite'; -opsInsertLayerFromGrid(param,layers); - diff --git a/cresis-toolbox/OPS-MATLAB/input/opsInsertLayerFromPointCloud.m b/cresis-toolbox/OPS-MATLAB/input/opsInsertLayerFromPointCloud.m deleted file mode 100644 index 2a8340b6..00000000 --- a/cresis-toolbox/OPS-MATLAB/input/opsInsertLayerFromPointCloud.m +++ /dev/null @@ -1,478 +0,0 @@ -function opsInsertLayerFromPointCloud(grid_param,layers) -% opsInsertLayerFromPointCloud(grid_param,layers) -% -% Makes layer for OPS from point cloud using Delaunay triangulization. -% -% Input: -% grid_param: a structure the controls how to create layers -% .param_fn = file name of the parameter spreadsheets for a field season, -% set the generic column of the cmd worksheet to 1 for the -% selected segments and frames. -% .save_sources = string array containing layer source to update. Can be 'echogram', -% 'layerData', or 'ops'. -% .echogram_source = string to source for echogram source option. -% For example 'CSARP_post/qlook', 'standard', etc -% .layerdata_source = string to source for layerData source option. -% For example 'CSARP_post/layerData', 'layerData', etc -% .plot_en = set to true for displaying echograms with layers, set to -% false for creating layers faster -% .proj: the projection structure of the grid map -% .save_mode: string containing 'overwrite' or 'merge' (merge makes all -% NaN in the new data source be unchanged) -% layers: structure array with layer information -% .username = string containing username for this layer; -% .name = string with layer name; -% .group = string with layer group; -% .ref_name = string with reference layer name (optional), -% often set to 'surface' -% .ref_eval = string with command to evaluate that combines new layer -% with reference. Variables available to the command are: -% data: this is the data passed in after it is interpolated to flightline -% ref_data: this is the reference layer data (twtt, sec) -% elev: platform elevation (m) -% Examples: -% ref_eval = 'data = ref_data + data;' -% ref_eval = 'data = data - (elev - ref_data*149896229)/84456957 + ref_data;' -% not used. If not empty, eval(layers(lyr_idx).ref_data) will be run -% where ref_data contains ref layer data and data contains grid data -% interpolated to the line. -% .echogram_layer_name = string containing the variable name to store -% into the echogram file (usually 'Surface' or 'Bottom') -% .layerdata_layer_name = string that indicates which layer to save the -% data to. For example, 'surface' (idx 1), 'bottom' (idx 2). -% .x = x-vector for data matrix -% .y = y-vector for data matrix -% .data = twtt-vector (or data vector which will be used by ref_data) -% .interp_method: optional string containing TriScatteredInterp method ('natural', -% 'linear', or 'nearest') with 'linear' as default -% -% Output: layer points in OPS -% -% Examples: -% Examples at bottom of this file -% -% Author: Jilu Li, John Paden -% -% See also: opsCreateLayerPoints.m, opsInsertLayerFromGrid.m - -%% Load param worksheet -params = read_param_xls(grid_param.param_fn,[],'post'); - -param.day_seg = []; -for param_idx = 1:length(params) - param = params(param_idx); - - %% Determine if this segment should be processed or not - if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic - continue; - end - - % Load frames file - frames = load(ct_filename_support(param,'','frames')); - - %% Determine which frames need to be processed - if isempty(param.cmd.frms) - param.cmd.frms = 1:length(frames.frame_idxs); - end - % Remove frames that do not exist from param.cmd.frms list - [valid_frms,keep_idxs] = intersect(param.cmd.frms, 1:length(frames.frame_idxs)); - if length(valid_frms) ~= length(param.cmd.frms) - bad_mask = ones(size(param.cmd.frms)); - bad_mask(keep_idxs) = 0; - warning('Nonexistent frames specified in param.cmd.frms (e.g. frame "%g" is invalid), removing these', ... - param.cmd.frms(find(bad_mask,1))); - param.cmd.frms = valid_frms; - end - - if any(strcmpi(grid_param.save_sources,'ops')) - %% Get all the frames for this segment - sys = ct_output_dir(param.radar_name); - ops_param = struct('properties',[]); - ops_param.properties.season = param.season_name; - ops_param.properties.segment = param.day_seg; - [status,ops_seg_data] = opsGetSegmentInfo(sys,ops_param); - end - - for layer_idx = 1:length(layers) - % Remove non-unique points - [dtri_pnts dtri_idxs] = unique([layers(layer_idx).x layers(layer_idx).y],'rows'); - % Make a function handle for a function that applies triangularization - dtri = DelaunayTri(dtri_pnts(:,1),dtri_pnts(:,2)); - if ~isfield(layers(layer_idx),'interp_method') || isempty(layers(layer_idx).interp_method) - layers(layer_idx).interp_method = 'linear'; - end - layers(layer_idx).interp_fh = TriScatteredInterp(dtri,layers(layer_idx).data(dtri_idxs),layers(layer_idx).interp_method); - end - - %% Update each of the frames - for frm_idx = 1:length(param.cmd.frms) - frm = param.cmd.frms(frm_idx); - - fprintf(' Updating %s frame %03d (%d of %d) (%s)\n', param.day_seg, ... - frm, frm_idx, length(param.cmd.frms), datestr(now,'HH:MM:SS')); - data_fn = fullfile(ct_filename_out(param,grid_param.echogram_source,''), ... - sprintf('Data_%s_%03d.mat', param.day_seg, frm)); - if grid_param.plot_en - plot_vars = load(data_fn,'Time','Data'); - end - - if any(strcmpi(grid_param.save_sources,'echogram')) - %% 1. Open the specific echogram data file - mdata = load(data_fn,'GPS_time','Latitude','Longitude','Elevation'); - - %% 2. Interpolate the grid onto this echogram - [x,y] = projfwd(grid_param.proj,mdata.Latitude,mdata.Longitude); - - %% 3. Save the layer back to the file under grid_param.echogram_layer_name - for layer_idx = 1:length(layers) - - % Interpolate point cloud onto track - data = layers(layer_idx).interp_fh(x,y); - - if isfield(layers(layer_idx),'ref_name') - tmp = load(data_fn,layers(layer_idx).ref_name); - ref_data = tmp.(layers(layer_idx).ref_name); - ref_data = interp_finite(ref_data); - elev = mdata.Elevation; - eval(layers(layer_idx).ref_eval); - end - - if strcmpi(grid_param.save_mode,'merge') - data(isnan(data)) = mdata.(layers(layer_idx).echogram_layer_name)(isnan(data)); - end - mdata.(layers(layer_idx).echogram_layer_name) = data; - - if grid_param.plot_en - figure(1); clf; - imagesc([],plot_vars.Time,lp(plot_vars.Data)); - colormap(1-gray(256)); - hold on; - plot(mdata.(layers(layer_idx).echogram_layer_name)); - hold off; - keyboard - end - - save(data_fn,'-append','-struct','mdata',layers(layer_idx).echogram_layer_name); - end - end - - if any(strcmpi(grid_param.save_sources,'layerdata')) - %% 1. Open the specific layer data file - layer_fn = fullfile(ct_filename_out(param,grid_param.layerdata_source,''), ... - sprintf('Data_%s_%03d.mat', param.day_seg, frm)); - if ~exist(layer_fn,'file') - error('Layer file %s does not exist', layer_fn); - end - % Load the layerData file - lay = load(layer_fn); - - %% 2. Interpolate the grid onto this layerData - [x,y] = projfwd(grid_param.proj,lay.Latitude,lay.Longitude); - - for layer_idx = 1:length(layers) - % Interpolate point cloud onto track - data = layers(layer_idx).interp_fh(x,y); - if isfield(layers(layer_idx),'ref_name') - if strcmpi(layers(layer_idx).ref_name,'surface') - ref_idx = 1; - elseif strcmpi(layers(layer_idx).ref_name,'bottom') - ref_idx = 2; - else - found = false; - for ref_idx = 1:length(lay.layerData) - if isfield(lay.layerData{ref_idx},'name') ... - && strcmpi(lay.layerData{ref_idx}.name,layers(layer_idx).ref_name) - found = true; - break; - end - end - if ~found - error('Reference layer %s not found\n', layers(layer_idx).ref_name); - end - end - ref_data = lay.layerData{ref_idx}.value{2}.data; - ref_data = interp_finite(ref_data); - elev = lay.Elevation; - eval(layers(layer_idx).ref_eval); - end - - % Determine which index into lay.layerData needs to be updated - if strcmpi(layers(layer_idx).name,'surface') - lay_idx = 1; - found = true; - elseif strcmpi(layers(layer_idx).name,'bottom') - lay_idx = 2; - found = true; - else - found = false; - for lay_idx = 1:length(lay.layerData) - if isfield(lay.layerData{lay_idx},'name') ... - && strcmpi(lay.layerData{lay_idx}.name,layers(layer_idx).name) - found = true; - break; - end - end - if ~found - % Add a new layer if layer does not exist - lay_idx = length(lay.layerData) + 1; - % Create manual points - lay.layerData{lay_idx}.value{1}.data = NaN*zeros(size(data)); - % Create auto points - lay.layerData{lay_idx}.value{2}.data = NaN*zeros(size(data)); - % Set quality to good - lay.layerData{lay_idx}.quality.data = 1*ones(size(data)); - end - end - - % Update automated points - if strcmpi(grid_param.save_mode,'merge') - data(isnan(data)) = lay.layerData{lay_idx}.value{2}.data(isnan(data)); - end - lay.layerData{lay_idx}.value{2}.data = data; - - if grid_param.plot_en - figure(1); clf; - imagesc([],plot_vars.Time,lp(plot_vars.Data)); - colormap(1-gray(256)); - hold on; - plot(lay.layerData{lay_idx}.value{2}.data); - hold off; - keyboard - end - - %% 3. Save the layer back to the file under grid_param.layerdata_layer_name - % If layer names do not exist, then assume 'surface' is 1, 'bottom' - % is 2, and if a layer does not exist then a new layer is created. - save(layer_fn,'-append','-struct','lay','layerData'); - end - end - - if any(strcmpi(grid_param.save_sources,'ops')) - for layer_idx = 1:length(layers) - start_gps = ops_seg_data.properties.start_gps_time(frm); - stop_gps = ops_seg_data.properties.stop_gps_time(frm); - - %% 1. Check to see if layer exists - [status,data] = opsGetLayers(sys); - if ~any(strcmpi(data.properties.lyr_name,layers(layer_idx).name)) - % Create the layer if it does not exist - ops_param = []; - ops_param.properties.lyr_name = layers(layer_idx).name; - ops_param.properties.lyr_group_name = layers(layer_idx).group; - ops_param.properties.lyr_description = layers(layer_idx).description; - ops_param.properties.public = true; - - [status,ops_data] = opsCreateLayer(sys,ops_param); - end - - %% 2. OPS query to get all the point path ID's - ops_param = struct('properties',[]); - ops_param.properties.location = param.post.ops.location; - ops_param.properties.season = param.season_name; - ops_param.properties.start_gps_time = start_gps; - ops_param.properties.stop_gps_time = stop_gps; - ops_param.properties.nativeGeom = true; - [status,ops_data] = opsGetPath(sys,ops_param); - - %% 3. Interpolate the data onto the point path ID's - [x,y] = projfwd(grid_param.proj,ops_data.properties.Y,ops_data.properties.X); - data = layers(layer_idx).interp_fh(x,y); - - %% 4. Load reference data - if isfield(layers(layer_idx),'ref_name') - ops_param = struct('properties',[]); - ops_param.properties.location = param.post.ops.location; - ops_param.properties.season = param.season_name; - ops_param.properties.segment = param.day_seg; - ops_param.properties.start_gps_time = start_gps; - ops_param.properties.stop_gps_time = stop_gps; - ops_param.properties.lyr_name = layers(layer_idx).ref_name; - [status,ref_data] = opsGetLayerPoints(sys,ops_param); - % Interpolate ref data onto point path - [ref_data.properties.gps_time,sort_idxs] = sort(ref_data.properties.gps_time); - ref_data.properties.twtt = ref_data.properties.twtt(sort_idxs); - ref_data = interp1(ref_data.properties.gps_time,ref_data.properties.twtt, ... - ops_data.properties.gps_time); - ref_data = interp_finite(ref_data); - elev = ops_data.properties.elev; - eval(layers(layer_idx).ref_eval); - end - - if strcmpi(grid_param.save_mode,'merge') - % Remove NaN layer points - ops_data.id = ops_data.id(~isnan(data)); - data = data(~isnan(data)); - end - - ops_param = struct('properties',[]); - ops_param.properties.point_path_id = ops_data.properties.id; - ops_param.properties.twtt = data; - ops_param.properties.type = 2*ones(size(ops_param.properties.twtt)); - ops_param.properties.quality = 1*ones(size(ops_param.properties.twtt)); - ops_param.properties.lyr_name = layers(layer_idx).name; - - if grid_param.plot_en - figure(1); clf; - imagesc([],plot_vars.Time,lp(plot_vars.Data)); - colormap(1-gray(256)); - hold on; - plot(ops_param.properties.twtt); - hold off; - keyboard - end - - opsCreateLayerPoints(sys,ops_param); - end - end - - end - -end - -return; - -%% Example 1: Gogineni's Jakobshavn points -% Use parameters spreadsheet to select segment and frame list for creating layers -% Set the generic to 1 for the selected segments and frames - -param = []; -param.param_fn = ct_filename_param('rds_param_2008_Greenland_TO.xls'); - -points_fn = '/cresis/snfs1/scratch/paden/mass_conservation/Jakobshavn_2006_2009_Composite/Jakobshavn_2006_2009_Composite/flightlines/Jakobshavn_2006_2009_Composite_Flightlines.txt'; - -% Load CSV points file -fid = fopen(points_fn,'r'); -first_line = fgets(fid); -headers = textscan(first_line,'%s','Delimiter',','); -data = textscan(fid,'%f%f%f%s%s%f%f%f%f%f%f%f%f%f%f','Delimiter',','); -fclose(fid); -points = []; -for header_idx = 1:length(headers{1}) - points.(headers{1}{header_idx}) = data{header_idx}; -end - -% Create geotiff projection structure -% 1. Grab from a geotiff file with the same projection -param.proj = geotiffinfo(ct_filename_gis([],'greenland\Landsat-7\Greenland_natural_90m.tif')); -[points.x,points.y] = projfwd(param.proj,points.LAT,points.LON); - -layers = []; -layers(1).username = 'paden'; -layers(1).name = 'gogineni2014_pnt'; -layers(1).group = 'standard'; -layers(1).description = 'Gogineni JofG 2014 grid'; -layers(1).ref_name = 'surface'; -layers(1).ref_eval = 'data = ref_data + data;'; -layers(1).echogram_layer_name = 'gogineni2014_pnt'; -layers(1).layerdata_layer_name = 'gogineni2014_pnt'; -layers(1).x = points.x; -layers(1).y = points.y; -physical_constants; -layers(1).data = (points.A_SURF-points.A_BED) / (c/2/sqrt(er_ice)); - -param.plot_en = false; -param.echogram_source = 'mvdr'; -param.layerdata_source = 'layerData'; -% save_sources: cell array of strings indicating which layer sources -% should be updated (options are ops, layerData, and echogram) -param.save_sources = {'ops'}; -param.save_mode = 'overwrite'; -opsInsertLayerFromPointCloud(param,layers); - -%% Example 2: ATM Ramp Pass Data -% Use parameters spreadsheet to select segment and frame list for creating layers -% Set the generic to 1 for the selected segments and frames - -param = []; -param.param_fn = ct_filename_param('kuband_param_2014_Greenland_P3.xls'); - -ramp_fn = '/scratch/metadata/2014_Greenland_P3/130407_truk_l12_kangramp_anthtrem_nopark.txt'; -[ramp.lat,ramp.lon,ramp.elev] = read_ramp_pass(ramp_fn); - -ins_param.update_ops_en = false; -ins_param.layer_name = 'Bottom'; -ins_param.er_ice = 1; -geotiff_fn = fullfile(gRadar.gis_path,'greenland','Landsat-7','mzl7geo_90m_lzw.tif'); -ins_param.proj = geotiffinfo(geotiff_fn); - - -% Create geotiff projection structure -% 1. Grab from a geotiff file with the same projection -param.proj = geotiffinfo(ct_filename_gis([],'greenland\Landsat-7\Greenland_natural_90m.tif')); - -[ramp.x,ramp.y] = projfwd(param.proj,ramp.lat,ramp.lon); - -layers = []; -layers(1).username = 'paden'; -layers(1).name = 'kanger_ramp'; -layers(1).group = 'standard'; -layers(1).description = 'ATM ramp pass Kangerlussuaq 2014'; -layers(1).ref_name = 'surface'; -layers(1).ref_eval = sprintf('data = (elev - data)/%.0f;',c/2); -layers(1).echogram_layer_name = 'kanger_ramp'; -layers(1).layerdata_layer_name = 'kanger_ramp'; -layers(1).x = ramp.x; -layers(1).y = ramp.y; -physical_constants; -layers(1).data = ramp; - -param.plot_en = false; -param.echogram_source = 'qlook'; -param.layerdata_source = 'layerData'; -% save_sources: cell array of strings indicating which layer sources -% should be updated (options are ops, layerData, and echogram) -param.save_sources = {'ops'}; -param.save_mode = 'overwrite'; -opsInsertLayerFromPointCloud(param,layers); - -%% Example 3: Insert 3-D data -% Use parameters spreadsheet to select segment and frame list for creating layers -% Set the generic to 1 for the selected segments and frames - -param = []; -param.param_fn = ct_filename_param('rds_param_2009_Greenland_TO.xls'); - -load('jakob_frm1.mat','points'); -frm2 = load('jakob_frm2.mat','points'); -points.lat = cat(2,points.lat,frm2.points.lat); -points.lon = cat(2,points.lon,frm2.points.lon); -points.elev = cat(2,points.elev,frm2.points.elev); -frm3 = load('jakob_2009frm1.mat','points'); -points.lat = cat(2,points.lat,frm3.points.lat); -points.lon = cat(2,points.lon,frm3.points.lon); -points.elev = cat(2,points.elev,frm3.points.elev); - -% Create geotiff projection structure -% 1. Grab from a geotiff file with the same projection -param.proj = geotiffinfo(ct_filename_gis([],'greenland\Landsat-7\Greenland_natural_90m.tif')); - -[points.x,points.y] = projfwd(param.proj,points.lat,points.lon); -good_mask = ~isnan(points.elev); -points.x = points.x(good_mask); -points.y = points.y(good_mask); -points.elev = points.elev(good_mask); - -layers = []; -layers(1).username = 'paden'; -layers(1).name = 'jakobshavn3D'; -layers(1).group = 'standard'; -layers(1).description = 'Basic 3D results for 20060530_08'; -layers(1).ref_name = 'surface'; -physical_constants; -layers(1).ref_eval = sprintf('data = (elev - ref_data*%.0f)/%.0f - data + ref_data;',c/2,c/2/sqrt(er_ice)); -layers(1).echogram_layer_name = 'jakobshavn3D'; -layers(1).layerdata_layer_name = 'jakobshavn3D'; -layers(1).x = points.x; -layers(1).y = points.y; -layers(1).data = points.elev/(c/2/sqrt(er_ice)); - -param.plot_en = false; -param.echogram_source = 'mvdr'; -param.layerdata_source = 'layerData'; -% save_sources: cell array of strings indicating which layer sources -% should be updated (options are ops, layerData, and echogram) -param.save_sources = {'ops'}; -param.save_mode = 'overwrite'; -opsInsertLayerFromPointCloud(param,layers); - - diff --git a/cresis-toolbox/OPS-MATLAB/input/runOpsInsertLayer.m b/cresis-toolbox/OPS-MATLAB/input/runOpsInsertLayer.m index 8badb536..4540b896 100644 --- a/cresis-toolbox/OPS-MATLAB/input/runOpsInsertLayer.m +++ b/cresis-toolbox/OPS-MATLAB/input/runOpsInsertLayer.m @@ -6,13 +6,14 @@ % Authors: John Paden %% Example Operations (just choose one) +example_str = 'mass_conservation'; -if 0 - %% Example 1: Gogineni's Jakobshavn points +if strcmpi(example_str,'gogineni_jakobshavn_point_cloud') + %% Example 1: Gogineni's Jakobshavn Point Cloud % ===================================================================== % ===================================================================== % Use parameters spreadsheet to select segment and frame list for creating layers - % Set the generic to 1 for the selected segments and frames + % Set the cmd.generic to 1 and cmd.frms for the selected segments and frames physical_constants; insert_param = []; @@ -40,10 +41,10 @@ insert_param.eval.ref_source.name = 'surface'; insert_param.eval.ref_source.source = 'ops'; insert_param.eval.ref_gaps_fill.method = 'interp_finite'; - insert_param.eval.cmd = 'source = ref.twtt + source;'; + insert_param.eval.cmd = 's = ref.twtt + s;'; insert_param.x = points.x; insert_param.y = points.y; - insert_param.data = (points.A_SURF-points.A_BED) / (c/2/sqrt(er_ice));; + insert_param.data = (points.A_SURF-points.A_BED) / (c/2/sqrt(er_ice)); insert_param.type = 'point'; % Point data insert_param.layer_dest.name = 'gogineni2014_pnt'; @@ -56,12 +57,12 @@ insert_param.gaps_fill.method_args = [300 60]; opsInsertLayer(params, insert_param); -elseif 0 +elseif strcmpi(example_str,'atm_ramp_pass') %% Example 2: ATM Ramp Pass Data % ===================================================================== % ===================================================================== % Use parameters spreadsheet to select segment and frame list for creating layers - % Set the generic to 1 for the selected segments and frames + % Set the cmd.generic to 1 and cmd.frms for the selected segments and frames physical_constants; insert_param = []; @@ -80,7 +81,7 @@ insert_param.eval.ref_source.name = 'surface'; insert_param.eval.ref_source.source = 'ops'; insert_param.eval.ref_gaps_fill.method = 'interp_finite'; - insert_param.eval.cmd = 'source = (elev - source)/(c/2);'; + insert_param.eval.cmd = 's = (elev - s)/(c/2);'; insert_param.x = points.x; insert_param.y = points.y; insert_param.data = points.elev; @@ -96,12 +97,12 @@ insert_param.gaps_fill.method_args = [300 60]; opsInsertLayer(params, insert_param); -elseif 0 - %% Example 3: Insert 3-D data +elseif strcmpi(example_str,'tomography_point_cloud') + %% Example 3: Insert 3-D Tomography Swath Point Cloud % ===================================================================== % ===================================================================== % Use parameters spreadsheet to select segment and frame list for creating layers - % Set the generic to 1 for the selected segments and frames + % Set the cmd.generic to 1 and cmd.frms for the selected segments and frames physical_constants; insert_param = []; @@ -131,7 +132,7 @@ insert_param.eval.ref_source.name = 'surface'; insert_param.eval.ref_source.source = 'ops'; insert_param.eval.ref_gaps_fill.method = 'interp_finite'; - insert_param.eval.cmd = 'source = (elev - ref.twtt*c/2)/(c/2/sqrt(er_ice)) - source + ref.twtt;'; + insert_param.eval.cmd = 's = (elev - ref.twtt*c/2)/(c/2/sqrt(er_ice)) - s + ref.twtt;'; insert_param.x = points.x; insert_param.y = points.y; insert_param.data = points.elev/(c/2/sqrt(er_ice)); @@ -147,12 +148,12 @@ insert_param.gaps_fill.method_args = [300 60]; opsInsertLayer(params, insert_param); -elseif 1 - %% Example 4: Merge Geoid data onto another (e.g. GIMP) layer +elseif strcmpi(example_str,'EGM96') + %% Example 4: Insert EGM96 Geoid data % ===================================================================== % ===================================================================== % Use parameters spreadsheet to select segment and frame list for creating layers - % Set the generic to 1 for the selected segments and frames + % Set the cmd.generic to 1 and cmd.frms for the selected segments and frames physical_constants; insert_param = []; @@ -178,7 +179,7 @@ points.y = points.y(good_mask); points.elev = points.elev(good_mask); - insert_param.eval.cmd = 'source = (elev - source)/(c/2);'; + insert_param.eval.cmd = 's = (elev - s)/(c/2);'; insert_param.x = points.x; insert_param.y = points.y; insert_param.data = points.elev; @@ -194,86 +195,82 @@ insert_param.gaps_fill.method = 'interp_finite'; opsInsertLayer(params, insert_param); -elseif 0 +elseif strcmpi(example_str,'mass_conservation') %% Example 5: mass conservation grid % ===================================================================== % ===================================================================== % Use parameters spreadsheet to select segment and frame list for creating layers - % Set the generic to 1 for the selected segments and frames - + % Set the cmd.generic to 1 and cmd.frms for the selected segments and frames + physical_constants; insert_param = []; - params = read_param_xls(ct_filename_param('rds_param_2017_Greenland_P3.xls'),'','post'); - - grid_fn = ct_filename_gis('greenland/mass_conservation/BedMachineGreenland-2017-09-20.nc'); - - % Create geotiff projection structure - % 1. Grab from a geotiff file with the same projection - % insert_param.proj = geotiffinfo(ct_filename_gis([],'greenland\Landsat-7\Greenland_natural_90m.tif')); - % 2. Create an mstruct with the same projection - %[x,y,mstruct] = geodetic_to_stereographic(70,-45); - % 3. Hand construction of geotiff projection OR - insert_param.proj = []; - insert_param.proj.CornerCoords = []; - insert_param.proj.Ellipsoid = []; - insert_param.proj.PM = []; - insert_param.proj.PMLongToGreenwich = []; - insert_param.proj.Zone = []; - grid_ellipsoid = referenceEllipsoid('WGS84'); - insert_param.proj.SemiMajor = grid_ellipsoid.SemimajorAxis; - insert_param.proj.SemiMinor = grid_ellipsoid.SemiminorAxis; - insert_param.proj.ProjParm = zeros(7,1); - insert_param.proj.ProjParm(1) = ncreadatt(grid_fn,'mapping','standard_parallel'); - insert_param.proj.ProjParm(2) = ncreadatt(grid_fn,'mapping','straight_vertical_longitude_from_pole'); - insert_param.proj.ProjParm(3) = 0; - insert_param.proj.ProjParm(4) = 0; - insert_param.proj.ProjParm(5) = 1; - insert_param.proj.ProjParm(6) = ncreadatt(grid_fn,'mapping','false_easting'); - insert_param.proj.ProjParm(7) = ncreadatt(grid_fn,'mapping','false_northing'); - - insert_param.proj.CTProjection = 'CT_PolarStereographic'; % Choose from list in toolbox/map/mapproj/private/projcode - insert_param.proj.GeoTIFFCodes.CTProjection = 15; % CT_PolarStereographic from toolbox/map/mapproj/private/projcode - - % Default GeoTIFFCode values - insert_param.proj.GeoTIFFCodes.Model = int16(1); - insert_param.proj.GeoTIFFCodes.PCS = int16(32767); - insert_param.proj.GeoTIFFCodes.GCS = int16(32767); - insert_param.proj.GeoTIFFCodes.UOMAngle = int16(32767); - insert_param.proj.GeoTIFFCodes.Datum = int16(32767); - insert_param.proj.GeoTIFFCodes.PM = int16(32767); - insert_param.proj.GeoTIFFCodes.ProjCode = int16(32767); - insert_param.proj.GeoTIFFCodes.Projection= int16(32767); - insert_param.proj.GeoTIFFCodes.MapSys = int16(32767); - - % Default units to meters for now - insert_param.proj.GeoTIFFCodes.UOMLength = int16(9001); + % params = read_param_xls(ct_filename_param('rds_param_2018_Greenland_P3.xls'),''); + params = read_param_xls(ct_filename_param('accum_param_2019_Antarctica_TObas.xls'),''); + params = ct_set_params(params,'cmd.generic',1); + params = ct_set_params(params,'cmd.generic',0,'cmd.notes','do not process'); + params = ct_set_params(params,'cmd.frms',[]); + % params = ct_set_params(params,'cmd.generic',0); + % params = ct_set_params(params,'cmd.generic',1,'day_seg','20200127_01'); + % params = ct_set_params(params,'cmd.frms',[28:33]); + + proj_load_standard; + if 0 + % Greenland Mass Conservation + grid_fn = ct_filename_gis(fullfile('greenland','mass_conservation','BedMachineGreenland-2017-09-20.nc')); + insert_param.proj = arctic_proj; + else + % Antarctica Mass Conservation + grid_fn = ct_filename_gis(fullfile('antarctica','mass_conservation','BedMachineAntarctica_2020-07-15_v02.nc')); + insert_param.proj = antarctic_proj; + end insert_param.eval.ref_source.name = 'surface'; - insert_param.eval.ref_source.source = 'ops'; + insert_param.eval.ref_source.source = 'layerdata'; + insert_param.eval.ref_source.layerdata_source = 'layer'; insert_param.eval.ref_gaps_fill.method = 'interp_finite'; - insert_param.eval.cmd = 'source = ref.twtt + source;'; + insert_param.eval.cmd = 's = ref.twtt + s;'; insert_param.x = double(ncread(grid_fn,'x')); insert_param.y = double(ncread(grid_fn,'y')); - insert_param.data = double(ncread(grid_fn,'thickness')).' / (c/2/sqrt(er_ice)); insert_param.type = 'raster'; % Raster data - insert_param.layer_dest.name = 'mc_bottom'; - insert_param.layer_dest.source = 'ops'; + insert_param.layer_dest.source = 'layerdata'; + insert_param.layer_dest.layerdata_source = 'layer'; insert_param.layer_dest.username = 'paden'; % For OPS layer_dest source insert_param.layer_dest.group = 'standard'; % For OPS layer_dest source insert_param.layer_dest.description = 'Mass conservation ice bottom'; % For OPS layer_dest source insert_param.layer_dest.existence_check = false; % Create layer if it does not exist insert_param.copy_method = 'overwrite'; insert_param.gaps_fill.method = 'interp_finite'; - opsInsertLayer(params, insert_param); -elseif 0 - %% Example 6: Gogineni's Jakobshavn grid + % ENABLE EACH DESIRED LAYER + if 1 + % Mass conservation estimate + insert_param.data = max(0,(double(ncread(grid_fn,'thickness'))).' / (c/2/sqrt(er_ice))); + insert_param.layer_dest.name = 'bottom_mc'; + opsInsertLayer(params, insert_param); + end + + if 1 + % Mass conservation estimate, lower (minimum elevation) bound + insert_param.data = max(0,(double(ncread(grid_fn,'thickness')) - ncread(grid_fn,'errbed')).' / (c/2/sqrt(er_ice))); % bottom_mc_top + insert_param.layer_dest.name = 'bottom_mc_top'; + opsInsertLayer(params, insert_param); + end + + if 1 + % Mass conservation estimate, upper (maximum elevation) bound + insert_param.data = max(0,(double(ncread(grid_fn,'thickness')) + ncread(grid_fn,'errbed')).' / (c/2/sqrt(er_ice))); % bottom_mc_bottom + insert_param.layer_dest.name = 'bottom_mc_bot'; + opsInsertLayer(params, insert_param); + end + +elseif strcmpi(example_str,'gogineni_jakobshavn_grid') + %% Example 6: Gogineni's Jakobshavn Grid % ===================================================================== % ===================================================================== % Use parameters spreadsheet to select segment and frame list for creating layers - % Set the generic to 1 for the selected segments and frames + % Set the cmd.generic to 1 and cmd.frms for the selected segments and frames physical_constants; insert_param = []; @@ -302,11 +299,11 @@ % Create geotiff projection structure % 1. Grab from a geotiff file with the same projection insert_param.proj = geotiffinfo(ct_filename_gis([],'greenland\Landsat-7\Greenland_natural_90m.tif')); - + insert_param.eval.ref_source.name = 'surface'; insert_param.eval.ref_source.source = 'ops'; insert_param.eval.ref_gaps_fill.method = 'interp_finite'; - insert_param.eval.cmd = 'source = ref.twtt + source;'; + insert_param.eval.cmd = 's = ref.twtt + s;'; insert_param.x = x_axis; insert_param.y = y_axis; insert_param.data = thickness / (c/2/sqrt(er_ice)); @@ -320,14 +317,14 @@ insert_param.layer_dest.existence_check = false; % Create layer if it does not exist insert_param.copy_method = 'overwrite'; insert_param.gaps_fill.method = 'interp_finite'; - opsInsertLayer(params, insert_param); + opsInsertLayer(params, insert_param); -elseif 0 - %% Example 7: Joe Plummer's Jakobshavn grid +elseif strcmpi(example_str,'joel_plummer_jakobshavn_grid') + %% Example 7: Joe Plummer's Jakobshavn Grid % ===================================================================== % ===================================================================== % Use parameters spreadsheet to select segment and frame list for creating layers - % Set the generic to 1 for the selected segments and frames + % Set the cmd.generic to 1 and cmd.frms for the selected segments and frames physical_constants; insert_param = []; @@ -343,11 +340,11 @@ bt_elev(bt_elev == min(min(bt_elev))) = NaN; x_axis = R.XLimWorld(1) + [R.XLimIntrinsic(1):R.XLimIntrinsic(2)-1]'*R.DeltaX; y_axis = R.YLimWorld(2) + [R.YLimIntrinsic(1):R.YLimIntrinsic(2)-1]'*R.DeltaY; - + insert_param.eval.ref_source.name = 'surface'; insert_param.eval.ref_source.source = 'ops'; insert_param.eval.ref_gaps_fill.method = 'interp_finite'; - insert_param.eval.cmd = 'source = (elev - ref.twtt*c/2)/(c/2/sqrt(er_ice)) - source + ref.twtt;'; + insert_param.eval.cmd = 's = (elev - ref.twtt*c/2)/(c/2/sqrt(er_ice)) - s + ref.twtt;'; insert_param.x = x_axis; insert_param.y = y_axis; insert_param.data = bt_elev/(c/2/sqrt(er_ice)); @@ -363,18 +360,18 @@ insert_param.gaps_fill.method = 'interp_finite'; opsInsertLayer(params, insert_param); -elseif 0 - %% Example 8: GIMP grid +elseif strcmpi(example_str,'gimp_grid') + %% Example 8: GIMP Grid % ===================================================================== % ===================================================================== % Use parameters spreadsheet to select segment and frame list for creating layers - % Set the generic to 1 for the selected segments and frames + % Set the cmd.generic to 1 and cmd.frms for the selected segments and frames physical_constants; insert_param = []; %params = read_param_xls(ct_filename_param('snow_param_2015_Greenland_Polar6.xls')); - params = read_param_xls(ct_filename_param('rds_param_2016_Greenland_G1XB.xls'),'','post'); + params = read_param_xls(ct_filename_param('rds_param_2016_Greenland_G1XB.xls'),''); grid_fn = ct_filename_gis([],'greenland/DEM/GIMP/gimpdem_90m.tif'); @@ -387,7 +384,7 @@ insert_param.proj = geotiffinfo(grid_fn); - insert_param.eval.cmd = 'source = (elev - source)/(c/2);'; + insert_param.eval.cmd = 's = (elev - s)/(c/2);'; insert_param.x = x_axis; insert_param.y = y_axis; insert_param.data = points.elev; @@ -404,44 +401,279 @@ insert_param.gaps_fill.method = 'interp_finite'; opsInsertLayer(params, insert_param); -elseif 1 - %% Example 9: BEDMAP2 grid +elseif strcmpi(example_str,'bedmap2_grid') + %% Example 9: BEDMAP2 Grid % ===================================================================== % ===================================================================== % Use parameters spreadsheet to select segment and frame list for creating layers - % Set the generic to 1 for the selected segments and frames + % Set the cmd.generic to 1 and cmd.frms for the selected segments and frames physical_constants; insert_param = []; - params = read_param_xls(ct_filename_param('rds_param_2009_Antarctica_TO_ndh_targetframes.xls'),'','post'); + params = read_param_xls(ct_filename_param('rds_param_2018_Antarctica_Ground.xls'),''); + params = ct_set_params(params,'cmd.generic',0); + %params = ct_set_params(params,'cmd.generic',1,'day_seg','20181224_03'); - grid_fn = ct_filename_gis([],'antarctica/DEM/BEDMAP2/original_data/bedmap2_tiff/bedmap2_surface.tif'); + grid_fn = ct_filename_gis([],fullfile('antarctica','DEM','BEDMAP2','original_data','bedmap2_tiff','bedmap2_surface.tif')); + geoid_fn = ct_filename_gis([],fullfile('antarctica','DEM','BEDMAP2','original_data','bedmap2_tiff','gl04c_geiod_to_WGS84.tif')); % Load the grid points = []; [points.elev,R] = geotiffread(grid_fn); - points.elev = double(points.elev); + [geoid.elev,R] = geotiffread(geoid_fn); + points.elev = double(points.elev) + double(geoid.elev); x_axis = R.XLimWorld(1) + [R.XLimIntrinsic(1):R.XLimIntrinsic(2)-1]'*R.DeltaX; y_axis = R.YLimWorld(2) + [R.YLimIntrinsic(1):R.YLimIntrinsic(2)-1]'*R.DeltaY; insert_param.proj = geotiffinfo(grid_fn); - insert_param.eval.cmd = 'source = (elev - source)/(c/2);'; + insert_param.eval.cmd = 's = (elev - s)/(c/2);'; insert_param.x = x_axis; insert_param.y = y_axis; insert_param.data = points.elev; insert_param.type = 'raster'; % Raster data insert_param.layer_dest.name = 'BEDMAP_surface'; - insert_param.layer_dest.source = 'ops'; + insert_param.layer_dest.desc = 'BEDMAP 2 Surface'; % For OPS layer_dest source + insert_param.layer_dest.source = 'layerdata'; insert_param.layer_dest.username = 'paden'; % For OPS layer_dest source - insert_param.layer_dest.group = 'standard'; % For OPS layer_dest source - insert_param.layer_dest.description = 'BEDMAP 2 Surface'; % For OPS layer_dest source - insert_param.layer_dest.layerdata_source = 'layerData'; % For layerData layer_dest source + insert_param.layer_dest.group_name = ''; % For OPS layer_dest source + insert_param.layer_dest.layerdata_source = 'layer'; % For layerData layer_dest source + insert_param.layer_dest.existence_check = false; % Create layer if it does not exist + insert_param.copy_method = 'overwrite'; + insert_param.gaps_fill.method = 'interp_finite'; + opsInsertLayer(params, insert_param); + +elseif strcmpi(example_str,'south_dakota_grid') + %% Example 10: South Dakota Grid + % ===================================================================== + % ===================================================================== + % Use parameters spreadsheet to select segment and frame list for creating layers + % Set the cmd.generic to 1 and cmd.frms for the selected segments and frames + + physical_constants; + insert_param = []; + + params = read_param_xls(ct_filename_param('snow_param_2019_SouthDakota_N1KU.xls'),''); + params = ct_set_params(params,'cmd.generic',0); + %params = ct_set_params(params,'cmd.generic',1,'day_seg','20200129_01'); + + grid_fn = ct_filename_gis([],fullfile('usa','DEM','NED','National_Elevation_Data_DEM_10m.tif')); + + % Load the grid + points = []; + [points.elev,R] = geotiffread(grid_fn); + points.elev = double(points.elev); + x_axis = R.XLimWorld(1) + [R.XLimIntrinsic(1):R.XLimIntrinsic(2)]'*R.DeltaX; + y_axis = R.YLimWorld(2) + [R.YLimIntrinsic(1):R.YLimIntrinsic(2)]'*R.DeltaY; + + insert_param.proj = geotiffinfo(grid_fn); + + insert_param.eval.cmd = 's = (elev - s)/(c/2);'; + insert_param.x = x_axis; + insert_param.y = y_axis; + insert_param.data = points.elev; + + insert_param.type = 'raster'; % Raster data + insert_param.layer_dest.name = 'USGS_surface'; + insert_param.layer_dest.desc = 'USGS Surface'; % For OPS layer_dest source + insert_param.layer_dest.source = 'layerdata'; + insert_param.layer_dest.username = 'paden'; % For OPS layer_dest source + insert_param.layer_dest.group_name = ''; % For OPS layer_dest source + insert_param.layer_dest.layerdata_source = 'layer'; % For layerData layer_dest source + insert_param.layer_dest.existence_check = false; % Create layer if it does not exist + insert_param.copy_method = 'overwrite'; + insert_param.gaps_fill.method = 'interp_finite'; + opsInsertLayer(params, insert_param); + +elseif strcmpi(example_str,'other_season_points') + %% Example 11: Other Season Points + % ===================================================================== + % ===================================================================== + % Use parameters spreadsheet to select segment and frame list for creating layers + % Set the cmd.generic to 1 and cmd.frms for the selected segments and frames + + global gRadar; + physical_constants; + insert_param = []; + + params = read_param_xls(ct_filename_param('accum_param_2019_Antarctica_TObas.xls'),''); + params = ct_set_params(params,'cmd.generic',0); + params = ct_set_params(params,'cmd.generic',1,'day_seg','20191225_01'); + params = ct_set_params(params,'cmd.generic',1,'day_seg','20191226_01'); + params = ct_set_params(params,'cmd.generic',1,'day_seg','20200127_01'); + + % Setup the points + proj_load_standard; + insert_param.proj = arctic_proj; + insert_param.x = []; + insert_param.y = []; + insert_param.data = []; + insert_param.eval.ref_source.name = 'surface'; + insert_param.eval.ref_source.source = 'layerdata'; + insert_param.eval.ref_gaps_fill.method = 'interp_finite'; + % Convert "layer twtt below the surface" to "total twtt" + insert_param.eval.cmd = 's = s + ref.twtt;'; + insert_param.interp_method = 'nearest'; + + % Load points from other segments + param = read_param_xls(ct_filename_param('accum_param_2018_Antarctica_TObas.xls'),'20190201_01','post'); + layer_params = struct('name',{'surface','bottom'},'source','ops'); + layers = opsLoadLayers(merge_structs(param,gRadar), layer_params); + [layers(2).x,layers(2).y] = projfwd(insert_param.proj, layers(2).lat, layers(2).lon); + insert_param.x = layers(2).x(:); + insert_param.y = layers(2).y(:); + % Convert "total twtt" to "twtt below the surface" + insert_param.data = layers(2).twtt-interp1(layers(1).gps_time,layers(1).twtt,layers(2).gps_time); + insert_param.data = insert_param.data(:); + + insert_param.type = 'point'; % Point data + insert_param.layer_dest.name = 'bottom_old'; + insert_param.layer_dest.desc = 'Layer from other segments'; % For OPS layer_dest source + insert_param.layer_dest.source = 'layerdata'; + insert_param.layer_dest.username = 'paden'; % For OPS layer_dest source + insert_param.layer_dest.group_name = ''; % For OPS layer_dest source + insert_param.layer_dest.layerdata_source = 'layer'; % For layerData layer_dest source insert_param.layer_dest.existence_check = false; % Create layer if it does not exist insert_param.copy_method = 'overwrite'; insert_param.gaps_fill.method = 'interp_finite'; opsInsertLayer(params, insert_param); - + +elseif strcmpi(example_str,'snotel') + %% Example 12: SNOTEL + % ===================================================================== + % ===================================================================== + % Use parameters spreadsheet to select segment and frame list for creating layers + % Set the cmd.generic to 1 and cmd.frms for the selected segments and frames + + global gRadar; + physical_constants; + insert_param = []; + + params = read_param_xls(ct_filename_param('snow_param_2019_SouthDakota_N1KU.xls'),''); + params = ct_set_params(params,'cmd.generic',1); + params = ct_set_params(params,'cmd.generic',0,'cmd.notes','do not process'); + +% params = ct_set_params(params,'cmd.generic',0); +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20200128_03'); +% params = ct_set_params(params,'cmd.frms',27); + + % Setup the points + proj_load_standard; + insert_param.proj = arctic_proj; + insert_param.gaps_fill.method = 'preserve_gaps'; + insert_param.gaps_fill.method_args = [300 60]; + insert_param.eval.ref_source.name = 'surface'; + insert_param.eval.ref_source.source = 'layerdata'; + insert_param.eval.ref_gaps_fill.method = 'interp_finite'; + insert_param.interp_method = 'nearest'; + insert_param.type = 'point'; % Point data + insert_param.layer_dest.source = 'layerdata'; + insert_param.layer_dest.username = 'paden'; % For OPS layer_dest source + insert_param.layer_dest.group_name = ''; % For OPS layer_dest source + insert_param.layer_dest.layerdata_source = 'layer'; % For layerData layer_dest source + insert_param.layer_dest.existence_check = false; % Create layer if it does not exist + insert_param.copy_method = 'overwrite'; + + % max_time_diff_sec: maximum number of days between measurement and segment + max_time_diff_sec = inf; + % insert_param.max_dist: maximum distance from point to interpolate (NaN + % beyond this distance) + insert_param.max_dist = 1500; + + % Load Snotel files + snotel_fns = get_filenames(fullfile(gRadar.data_support_path, params(1).season_name,'snotel'),'','','.csv'); + snotel_data = []; + for snotel_fns_idx = 1:length(snotel_fns) + snotel_fn = snotel_fns{snotel_fns_idx}; + [fid,msg] = fopen(snotel_fn,'r'); + if fid<0 + error('Failed to open file: %s', msg); + end + % Expected file format: + % GPS_Date,GPS_Time,Station Id,Station Name,Latitude,Longitude,Snow Water Equivalent (in) Start of Day Values,Snow Depth (in) Start of Day Values + % 1/26/2020,14:00,920,North Rapid Creek,44.20617,-103.78758,4,20 + raw_data = textscan(fid,'%s %s %s %s %f %f %f %f','headerlines',1','delimiter',','); + fclose(fid); + if length(raw_data) == 8 + % Use the length of the last field since an incomplete last record + % should be ignored + N = length(raw_data{8}); + % date_str: Combine date and time fields + date_str = cellfun(@(x,y) cat(2,x,' ',y),raw_data{1},raw_data{2},'UniformOutput',false); + if isempty(snotel_data) + % Initialize variables on the first good data + snotel_data = struct('gps_time',datenum_to_epoch(datenum(date_str))); + snotel_data.lat = raw_data{5}(1:N); + snotel_data.lon = raw_data{6}(1:N); + snotel_data.SWE = raw_data{7}(1:N)*2.54/100; + snotel_data.depth = raw_data{8}(1:N)*2.54/100; + else + % Append variables + snotel_data.gps_time(end+(1:N)) = datenum_to_epoch(datenum(date_str)); + snotel_data.lat(end+(1:N)) = raw_data{5}(1:N); + snotel_data.lon(end+(1:N)) = raw_data{6}(1:N); + snotel_data.SWE(end+(1:N)) = raw_data{7}(1:N)*2.54/100; + snotel_data.depth(end+(1:N)) = raw_data{8}(1:N)*2.54/100; + end + end + end + [snotel_data.x,snotel_data.y] = projfwd(insert_param.proj,snotel_data.lat,snotel_data.lon); + + % Since different points need to be used depending on the date, we only + % do this one segment at a time + for param_idx = 1:length(params) + param = params(param_idx); + if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic + continue; + end + records = records_load(param); + % done_mask: when true it means the snotel point will no longer be + % considered + done_mask = snotel_data.gps_time < records.gps_time(1)-max_time_diff_sec ... + | snotel_data.gps_time > records.gps_time(end)+max_time_diff_sec; + insert_param.x = []; + insert_param.y = []; + insert_param.data = []; + while ~all(done_mask) + idx = find(~done_mask,1); + % Only use one data value at each location + cur_idxs = find(snotel_data.x(idx) == snotel_data.x & snotel_data.y(idx) == snotel_data.y); + done_mask(cur_idxs) = true; + % Find the closest point in time at this location + [~,match_idx] = min(snotel_data.gps_time(cur_idxs)); + % Append the new point + insert_param.x(end+1) = snotel_data.x(cur_idxs(match_idx)); + insert_param.y(end+1) = snotel_data.y(cur_idxs(match_idx)); + % Convert SWE and snow depth into density + density = 0.917 * snotel_data.SWE(cur_idxs(match_idx)) / snotel_data.depth(cur_idxs(match_idx)); + % Use Tiuri et al. 1984 to turn density into relative permitivity + er_snow = 1 + 1.7*density + 0.7*density^2; + % Turn depth and permitivity into twtt through snow layer + insert_param.data(end+1) = snotel_data.depth(cur_idxs(match_idx)) * sqrt(er_snow) * 2/c; + end + + if length(insert_param.data) < 3 + % Not enough points to triangulate so create a small triangle around + % the last point to satisfy the opsInsertLayer's triangulation + % interpolation + insert_param.x(end+(1:2)) = insert_param.x(end) + [1; 0]; + insert_param.y(end+(1:2)) = insert_param.y(end) + [0; 1]; + insert_param.data(end+(1:2)) = insert_param.data(end); + end + + insert_param.layer_dest.name = 'snotel_surface'; + insert_param.layer_dest.desc = 'SNOTEL snow surface'; % For OPS layer_dest source + % Convert "layer twtt in air" to "total twtt" + insert_param.eval.cmd = 's = ref.twtt;'; + opsInsertLayer(param, insert_param); + + insert_param.layer_dest.name = 'snotel'; + insert_param.layer_dest.desc = 'SNOTEL snow bottom'; % For OPS layer_dest source + % Convert "layer twtt in air" to "total twtt" + insert_param.eval.cmd = 's = s + ref.twtt;'; + opsInsertLayer(param, insert_param); + end + end diff --git a/cresis-toolbox/OPS-MATLAB/output/opsGetLayerPoints.m b/cresis-toolbox/OPS-MATLAB/output/opsGetLayerPoints.m index eb8cbaf3..3e161f45 100644 --- a/cresis-toolbox/OPS-MATLAB/output/opsGetLayerPoints.m +++ b/cresis-toolbox/OPS-MATLAB/output/opsGetLayerPoints.m @@ -14,7 +14,9 @@ % properties.segment = string % % OR INSTEAD OF THE ABOVE -% properties.point_path_id: integer array +% properties.location = string ('arctic' or 'antarctic') +% properties.point_path_id: integer array as row vector +% (also requires location, season, segment_id currently...) % % OPTIONAL: % properties.start_gps_time = double diff --git a/cresis-toolbox/OPS-MATLAB/output/opsGetPath.m b/cresis-toolbox/OPS-MATLAB/output/opsGetPath.m index faaf760b..932421e6 100644 --- a/cresis-toolbox/OPS-MATLAB/output/opsGetPath.m +++ b/cresis-toolbox/OPS-MATLAB/output/opsGetPath.m @@ -13,6 +13,10 @@ % properties.stop_gps_time = double % (optional) properties.nativeGeom = true (returns wgs1984 coordinates) % +% OR INSTEAD OF THE ABOVE +% properties.location = string ('arctic' or 'antarctic') +% properties.point_path_id: integer array as row vector +% % Output: % status: integer (0:Error,1:Success) % data: structure with fields (or error message) diff --git a/cresis-toolbox/OPS-MATLAB/output/opsLoadLayers.m b/cresis-toolbox/OPS-MATLAB/output/opsLoadLayers.m index 57557271..f7c500d1 100644 --- a/cresis-toolbox/OPS-MATLAB/output/opsLoadLayers.m +++ b/cresis-toolbox/OPS-MATLAB/output/opsLoadLayers.m @@ -6,56 +6,78 @@ % * The source of the layer data can be records, echogram files, layerData files, % ATM or AWI Lidar, OPS. % * Controlled from param spreadsheet -% * Use opsInsertLayerFromGrid and opsInsertLayerFromPointCloud to compare +% * Use opsInsertLayer to compare % layers to grids and point clouds % * Use runOpsCopyLayers to copy layers from one radar to another % -% param = param spreadsheet structure -% layer_params = N element struct array indicating which layers are to be loaded -% and which source to use for each layer -% .name: string (e.g. 'surface', 'Surface', 'bottom', 'atm', etc) -% .source: string -% 'records': Loads layer data from records file -% 'echogram': Loads layer data from echogram files -% 'layerdata': Loads layer data from layer data files -% 'lidar': Loads (ATM, AWI, or DTU) lidar data -% 'ops': Loads layer data from Open Polar Server -% .echogram_source = string containing ct_filename_out argument if using -% 'echogram' source -% (e.g. 'qlook', 'mvdr', 'CSARP_post/standard') -% .layerdata_source = string containing ct_filename_out argument if using -% 'echogram' source -% (e.g. 'layerData', 'CSARP_post/layerData') -% .lidar_source = string containing 'atm', 'awi', or 'dtu' if using lidar source -% .existence_check = boolean, default is true and causes an error to be -% thrown if the layer does not exist. If false, no data points are -% returned when the layer does not exist and only a warning is given. -% .fix_broken_layerdata: logical, default is false and causes any layerdata -% errors to be fixed while loading by setting the layer for the -% particular data frame that has the error to the default values (NaN -% for twtt, quality 1, and type 2). If true, an error is thrown if -% layerdata errors exist. -% .debug = default is false +% Inputs +% ========================================================================= +% +% param: param spreadsheet structure +% +% layer_params: N element struct array indicating which layers are to be +% loaded and which source to use for each layer. Fields: +% +% .age: used when creating a nonexistent layerdata layer +% +% .age_source: used when creating a nonexistent layerdata layer +% +% .desc: used when creating a nonexistent layerdata layer +% +% .echogram_source: string containing ct_filename_out argument if using +% 'echogram' source (e.g. 'qlook', 'mvdr', 'CSARP_post/standard') +% % .eval: Optional structure for performing operations on the layer % .cmd: Command string that will be passed to eval % .$(custom): Custom fields to be accessed in cmd as es.$(custom) % Variables available are: % physical_constants -% "gps_time" (sec) -% "along_track" (m) +% "time" (ANSI-C GPS time, seconds since Jan 1, 1970) +% "at" (along-track, m) % "lat" (deg) % "lon" (deg) % "elev" (m) -% "source" (twtt in sec) +% "s" (two way travel time, twtt, in sec) % "es" (this eval structure passed in by the user) -% The cmd string should generally update "source" variable. For example: -% '[B,A] = butter(0.1,2); source = filtfilt(B,A,source);' % Filter -% 'source = source + 0.1;' % Apply a twtt shift -% 'source = source*2;' % Surface multiple -% .frms: This field overrides the param.cmd.frms field, but must be the -% same for all elements of the layer_params struct array since only the -% first element will be used. +% The cmd string should generally update "s" variable. For example: +% '[B,A] = butter(0.1,2); s = filtfilt(B,A,s);' % Filter +% 's = s + 0.1;' % Apply a twtt shift +% 's = s*2;' % Surface multiple +% Add ";" to the end of each command to suppress console output. +% +% .existence_check: boolean, default is true and causes an error to be +% thrown if the layer does not exist. If false, no data points are +% returned when the layer does not exist and only a warning is given. +% +% .fix_broken_layerdata: logical, default is false and causes any +% layerdata errors to be fixed while loading by setting the layer for the +% particular data frame that has the error to the default values (NaN for +% twtt, quality 1, and type 2). If true, an error is thrown if layerdata +% errors exist. +% +% .group_name: used when creating a nonexistent layerdata layer +% +% .layerdata_source: string containing ct_filename_out argument of where +% the layerdata is (default is "layer") (e.g. 'layer', 'CSARP_post/layer') +% +% .lidar_source: string containing 'atm', 'awi', or 'dtu' if using lidar source % +% .name: string containing the layer name (e.g. 'surface', 'Surface', 'bottom', 'atm', etc), default +% is "surface". Leave empty if using regular expression. +% +% .regexp: string containing a regular expression, all layers matching the +% regular expression will be loaded. Default is empty/not defined. +% +% .source: string +% 'custom': Custom layer source similar to opsCopyLayers +% 'echogram': Loads layer data from echogram files +% 'records': Loads layer data from records file +% 'layerdata': Loads layer data from layer data files (default) +% 'lidar': Loads (ATM, AWI, or DTU) lidar data +% 'ops': Loads layer data from Open Polar Server +% +% Outputs +% ========================================================================= % layers: N element struct array with layer information % .gps_time % .lat @@ -66,6 +88,10 @@ % .twtt % .point_path_id database key (only filled if source is ops) % .frm: numeric vector of frame ids (1 to 999) +% .name: layer name string (only filled if source is ops or layerdata) +% .group_name: layer group name string (only filled if source is ops or layerdata) +% .desc: layer description string (only filled if source is ops or layerdata) +% .age: layer age (only filled if source is layerdata) % % Authors: John Paden % @@ -77,24 +103,39 @@ param.records = []; end +if ~exist('layer_params','var') + layer_params = []; +end + % Check layers defined by regular expression regexp layer_idx = 1; while layer_idx <= length(layer_params) - if ~isfield(layer_params,'name') || isempty(layer_params(layer_idx).name) + layer_names = {}; + if isfield(layer_params(layer_idx),'name') && iscell(layer_params(layer_idx).name) + layer_names = layer_params(layer_idx).name; + elseif ~isfield(layer_params,'name') || isempty(layer_params(layer_idx).name) % Name is not specified, so check regular expression field if ~isfield(layer_params,'regexp') || isempty(layer_params(layer_idx).regexp) % Default is surface warning('No name specified for layer %d, defaulting to use layer "surface.', layer_idx); layer_params(layer_idx).name = 'surface'; else - error('regexp not supported yet. Just need to get list of layer names that match.'); - layer_params = layer_params([1:layer_idx-1 layer_idx*ones(1,length(layer_names)) layer_idx+1:end]); - for layer_regexp_idx = 1:length(layer_names) - layer_params(layer_idx+layer_regexp_idx-1) = layer_params(layer_idx); - layer_params(layer_idx+layer_regexp_idx-1).name = layer_names{layer_regexp_idx}; - layer_params(layer_idx+layer_regexp_idx-1).regexp = ''; + if ~isfield(layer_params,'layerdata_source') || isempty(layer_params(layer_idx).layerdata_source) + % Default is layerData + layer_params(layer_idx).layerdata_source = 'layer'; end + tmp_layers = layerdata(param, layer_params(layer_idx).layerdata_source); + layer_names = tmp_layers.get_layer_names(layer_params(layer_idx).regexp); + end + end + if ~isempty(layer_names) + layer_params = layer_params([1:layer_idx-1 layer_idx*ones(1,length(layer_names)) layer_idx+1:end]); + for new_idx = 1:length(layer_names) + layer_params(layer_idx+new_idx-1).name = layer_names{new_idx}; + layer_params(layer_idx+new_idx-1).regexp = ''; end + layer_idx = layer_idx+new_idx-1; + end layer_idx = layer_idx + 1; end @@ -119,12 +160,22 @@ % Default is layerData layer_params(layer_idx).layerdata_source = 'layer'; end + if ~isfield(layer_params,'read_only') || isempty(layer_params(layer_idx).read_only) + % Default is layerData + layer_params(layer_idx).read_only = true; + end if ~isfield(layer_params,'lever_arm_en') || isempty(layer_params(layer_idx).lever_arm_en) - layer_params(layer_idx).lever_arm_en = false; + layer_params(layer_idx).lever_arm_en = true; + end + if ~isfield(layer_params,'lidar_max_gap') || isempty(layer_params(layer_idx).lidar_max_gap) + layer_params(layer_idx).lidar_max_gap = 300; end if ~isfield(layer_params(layer_idx),'existence_check') || isempty(layer_params(layer_idx).existence_check) layer_params(layer_idx).existence_check = true; end + if ~isfield(layer_params(layer_idx),'existence_warning') || isempty(layer_params(layer_idx).existence_warning) + layer_params(layer_idx).existence_warning = true; + end switch lower(layer_params(layer_idx).source) case 'ops' ops_en = true; @@ -138,6 +189,8 @@ echogram_en = true; case 'lidar' lidar_layer_idx(end+1) = layer_idx; + otherwise + error('Invalid layer source specified: layer_params(%d).source == %s is not a valid source. Must be ops, layerdata, records, echogram, or lidar.', layer_idx, layer_params(layer_idx).source); end end @@ -153,24 +206,10 @@ if ~isempty(layerdata_sources) || records_en || echogram_en || ~isempty(lidar_layer_idx) % Load frames file frames = frames_load(param); + param.cmd.frms = frames_param_cmd_frms(param,frames); - if records_en || ~isempty(lidar_layer_idx) || ~isfield(frames,'frame_idxs') - records_fn = ct_filename_support(param,'','records'); - records = load(records_fn); - end - - % Determine which frames need to be processed - if isempty(param.cmd.frms) - param.cmd.frms = 1:length(frames.frame_idxs); - end - % Remove frames that do not exist from param.cmd.frms list - [valid_frms,keep_idxs] = intersect(param.cmd.frms, 1:length(frames.frame_idxs)); - if length(valid_frms) ~= length(param.cmd.frms) - bad_mask = ones(size(param.cmd.frms)); - bad_mask(keep_idxs) = 0; - warning('Nonexistent frames specified in param.cmd.frms (e.g. frame "%g" is invalid), removing these', ... - param.cmd.frms(find(bad_mask,1))); - param.cmd.frms = valid_frms; + if records_en || ~isempty(lidar_layer_idx) + records = records_load(param); end end @@ -232,22 +271,31 @@ lidar = read_lidar_dtu(lidar_fns,param); + elseif any(strcmpi('las',{layer_params.lidar_source})) + lidar_fns = get_filenames_lidar(param, 'las', ... + datenum_to_epoch(datenum(param.day_seg(1:8),'yyyymmdd'))); + + lidar = read_lidar_las(lidar_fns,param); + else error('Invalid LIDAR source %s', layer_params.lidar_source); end - % Remove NAN's from LIDAR Data - good_lidar_idxs = ~isnan(lidar.gps_time); - lidar.gps_time = lidar.gps_time(good_lidar_idxs); - lidar.surface = lidar.surface(good_lidar_idxs); - lidar.lat = lidar.lat(good_lidar_idxs); - lidar.lon = lidar.lon(good_lidar_idxs); - if ~isempty(isnan(lidar.surface)) - good_lidar_idxs = ~isnan(lidar.surface); + if ~any(strcmpi('las',{layer_params.lidar_source})) + % If not LAS lidar type, then remove gps_time that are NAN's from LIDAR + % Data + good_lidar_idxs = ~isnan(lidar.gps_time); lidar.gps_time = lidar.gps_time(good_lidar_idxs); lidar.surface = lidar.surface(good_lidar_idxs); lidar.lat = lidar.lat(good_lidar_idxs); lidar.lon = lidar.lon(good_lidar_idxs); + if ~isempty(isnan(lidar.surface)) + good_lidar_idxs = ~isnan(lidar.surface); + lidar.gps_time = lidar.gps_time(good_lidar_idxs); + lidar.surface = lidar.surface(good_lidar_idxs); + lidar.lat = lidar.lat(good_lidar_idxs); + lidar.lon = lidar.lon(good_lidar_idxs); + end end % Find reference trajectory @@ -258,48 +306,49 @@ % IMU elevation and not the radar phase center elevation. lidar.elev = interp1(records.gps_time,records.elev,lidar.gps_time); else - % Create reference trajectory (rx_path == 0, tx_weights = []). Update - % the records field with this information. - trajectory_param = struct('gps_source',records.gps_source, ... - 'season_name',param.season_name,'radar_name',param.radar_name,'rx_path', 0, ... - 'tx_weights', [], 'lever_arm_fh', param.radar.lever_arm_fh); - records = trajectory_with_leverarm(records,trajectory_param); - - % Find the closest lidar point for each point along the reference - % trajectory. Set the lidar.gps_time and lidar.elev fields to match - % this reference point's gps_time and elev so that interpolation later - % with the gps_time fields will work properly. - % Convert coordinates to ECEF - [lidar_x,lidar_y,lidar_z] = geodetic2ecef(lidar.lat/180*pi,lidar.lon/180*pi,zeros(size(lidar.lat)),WGS84.ellipsoid); - [records_x,records_y,records_z] = geodetic2ecef(records.lat/180*pi,records.lon/180*pi,zeros(size(records.lat)),WGS84.ellipsoid); - start_idx = 1; - stop_idx = 1; - lidar.elev = zeros(size(lidar.lat)); - for lidar_idx = 1:length(lidar.lat) - while start_idx < length(records.gps_time) && records.gps_time(start_idx) < lidar.gps_time(lidar_idx)-10 - start_idx = start_idx + 1; - end - while stop_idx < length(records.gps_time) && records.gps_time(stop_idx) < lidar.gps_time(lidar_idx)+10 - stop_idx = stop_idx + 1; + if isempty(lidar.lat) + warning('No lidar data exists.'); + lidar.elev = []; + else + records = records_reference_trajectory_load(param,records); + + % Project to map coordinates + proj_load_standard; + if strcmpi(param.post.ops.location,'antarctic') + proj = antarctic_proj; + else % if strcmpi(param.post.ops.location,'arctic') + proj = arctic_proj; end - if abs(records.gps_time(start_idx) - lidar.gps_time(lidar_idx)) < 10 - recs = start_idx:stop_idx; - dist = abs(lidar_x(lidar_idx)-records_x(recs)).^2 + abs(lidar_y(lidar_idx)-records_y(recs)).^2 + abs(lidar_z(lidar_idx)-records_z(recs)).^2; - [min_dist,min_idx] = min(dist); - lidar.gps_time(lidar_idx) = records.gps_time(recs(min_idx)); - lidar.elev(lidar_idx) = records.elev(recs(min_idx)); - else - lidar.gps_time(lidar_idx) = NaN; - lidar.elev(lidar_idx) = NaN; + [records_x,records_y] = projfwd(proj,records.lat,records.lon); + [lidar_x,lidar_y] = projfwd(proj,lidar.lat,lidar.lon); + lidar_pnts = [lidar_x.',lidar_y.']; + lidar_pnts = unique(lidar_pnts,'rows','stable'); + + % Find the closest point for each record + if 0 + % Slowest method (517 sec) + T = delaunayn(lidar_pnts); + [xi,dist] = dsearchn(lidar_pnts,T,[records_x.' records_y.']); + elseif 1 + % Second slowest method (69 sec) + dt = delaunayTriangulation(lidar_pnts); + [xi,dist] = nearestNeighbor(dt, [records_x.' records_y.']); + clear dt; + elseif 0 + % Fastest method but requires toolbox (29 sec) + [xi,dist] = knnsearch(lidar_pnts,[records_x.' records_y.']); end + + % Remove records which are too far from closest lidar data point + mask = dist < layer_params(lidar_layer_idx).lidar_max_gap; + lidar.gps_time = records.gps_time; + lidar.lat = records.lat; + lidar.lon = records.lon; + lidar.surface = lidar.surface(xi); + lidar.surface(~mask) = NaN; + lidar.elev = records.elev; end - % Remove NAN's from LIDAR Data - good_lidar_idxs = ~isnan(lidar.gps_time); - lidar.gps_time = lidar.gps_time(good_lidar_idxs); - lidar.surface = lidar.surface(good_lidar_idxs); - lidar.lat = lidar.lat(good_lidar_idxs); - lidar.lon = lidar.lon(good_lidar_idxs); - lidar.elev = lidar.elev(good_lidar_idxs); + end end @@ -314,6 +363,11 @@ layers(layer_idx).type = []; layers(layer_idx).quality = []; layers(layer_idx).point_path_id = []; + layers(layer_idx).age = []; + layers(layer_idx).age_source = []; + layers(layer_idx).desc = ''; + layers(layer_idx).group_name = ''; + layers(layer_idx).name = layer_params(layer_idx).name; end %% Load each of the frames (lidar, echogram, records file layer sources) @@ -377,7 +431,9 @@ if layer_param.existence_check error('Echogram file %s does not exist', data_fn); else - warning('Echogram file %s does not exist', data_fn); + if layer_param.existence_warning + warning('Echogram file %s does not exist', data_fn); + end continue; end end @@ -436,7 +492,7 @@ for layer_idx = 1:length(layer_params) layer_param = layer_params(layer_idx); - if ~strcmpi(layer_param.source,'layerdata') + if ~strcmpi(layer_param.source,'layerdata') || ~strcmpi(layer_param.layerdata_source,layerdata_source) continue; end @@ -444,20 +500,40 @@ layers(layer_idx).lat = tmp_layers.lat(param.cmd.frms); layers(layer_idx).lon = tmp_layers.lon(param.cmd.frms); layers(layer_idx).elev = tmp_layers.elev(param.cmd.frms); - id = tmp_layers.get_id(layer_param.name); + [id,name,group_name,desc,age,age_source] = tmp_layers.get_id(layer_param.name); if isempty(id) if layer_param.existence_check - error('Layer %s does not exist in %s.', layer_param.name, tmp_layers.layer_organizer_fn()); + error('layer_param.existence_check is true and layer %s does not exist in %s. Set to false to still load layers even if they do not exist yet.', layer_param.name, tmp_layers.layer_organizer_fn()); + end + if ~ischar(layer_param.name) + error('Layer name must be a string because it does not exist in %s and a new layer must be inserted for which a name is required.', tmp_layers.layer_organizer_fn()); end layer_organizer = []; layer_organizer.lyr_name = {layer_param.name}; + if isfield(layer_param,'age') + layer_organizer.lyr_age = layer_param.age; + end + if isfield(layer_param,'age_source') + layer_organizer.lyr_age_source = {layer_param.age_source}; + end + if isfield(layer_param,'desc') + layer_organizer.lyr_desc = {layer_param.desc}; + end if isfield(layer_param,'group_name') layer_organizer.lyr_group_name = {layer_param.group_name}; end tmp_layers.insert_layers(layer_organizer); + [id,name,group_name,desc,age,age_source] = tmp_layers.get_id(layer_param.name); end + layers(layer_idx).age = age; + layers(layer_idx).age_source = age_source; + layers(layer_idx).desc = desc; + layers(layer_idx).group_name = group_name; + layers(layer_idx).name = name; [layers(layer_idx).twtt,layers(layer_idx).quality,layers(layer_idx).type] = tmp_layers.get_layer(param.cmd.frms,layer_param.name); - tmp_layers.save(); + if ~layer_param.read_only + tmp_layers.save(); + end end end @@ -474,19 +550,20 @@ stop_gps = ops_seg_data.properties.stop_gps_time(max(param.cmd.frms)); end - found = true; - if ~layer_param.existence_check - % If the layer does not exist, we need to determine this before - % we call opsGetLayerPoints, otherwise we will get an error in - % that function. - [status,data] = opsGetLayers(sys); - if ~any(strcmpi(data.properties.lyr_name,layer_param.name)) - found = false; - warning('Layer %s does not exist in OPS.', layer_param.name); + % Get layer information and find out if layer exists or not + [status,data] = opsGetLayers(sys); + match_idx = find(strcmpi(data.properties.lyr_name,layer_param.name)); + if isempty(match_idx) + if layer_param.existence_check + error('Layer %s does not exist in OPS.', layer_param.name); + else + if layer_param.existence_warning + warning('Layer %s does not exist in OPS.', layer_param.name); + end end - end - - if found + else + layers(layer_idx).group_name = data.properties.lyr_group_name{match_idx}; + ops_param = struct('properties',[]); ops_param.tmp_path = param.tmp_path; ops_param.properties.location = param.post.ops.location; @@ -498,23 +575,14 @@ ops_param.properties.return_geom = 'geog'; [status,data] = opsGetLayerPoints(sys,ops_param); - [data.properties.gps_time,sort_idxs] = sort(data.properties.gps_time); - data.properties.twtt = data.properties.twtt(sort_idxs); - data.properties.elev = data.properties.elev(sort_idxs); - data.properties.lat = data.properties.lat(sort_idxs); - data.properties.lon = data.properties.lon(sort_idxs); - data.properties.type = data.properties.type(sort_idxs); - data.properties.quality = data.properties.quality(sort_idxs); - data.properties.point_path_id = data.properties.point_path_id(sort_idxs); - - layers(layer_idx).gps_time = data.properties.gps_time; - layers(layer_idx).twtt = data.properties.twtt; - layers(layer_idx).elev = data.properties.elev; - layers(layer_idx).lat = data.properties.lat; - layers(layer_idx).lon = data.properties.lon; - layers(layer_idx).type = data.properties.type; - layers(layer_idx).quality = data.properties.quality; - layers(layer_idx).point_path_id = data.properties.point_path_id; + [layers(layer_idx).gps_time,sort_idxs] = sort(data.properties.gps_time); + layers(layer_idx).twtt = data.properties.twtt(sort_idxs); + layers(layer_idx).elev = data.properties.elev(sort_idxs); + layers(layer_idx).lat = data.properties.lat(sort_idxs); + layers(layer_idx).lon = data.properties.lon(sort_idxs); + layers(layer_idx).type = data.properties.type(sort_idxs); + layers(layer_idx).quality = data.properties.quality(sort_idxs); + layers(layer_idx).point_path_id = data.properties.point_path_id(sort_idxs); end end end @@ -559,12 +627,12 @@ if frm == 1 start_gps_time = 0; else - start_gps_time = records.gps_time(frames.frame_idxs(frm)); + start_gps_time = frames.gps_time(frm); end if frm == length(frames.frame_idxs) stop_gps_time = inf; else - stop_gps_time = records.gps_time(frames.frame_idxs(frm+1)); + stop_gps_time = frames.gps_time(frm+1); end layers(layer_idx).frm(layers(layer_idx).gps_time >= start_gps_time ... & layers(layer_idx).gps_time < stop_gps_time) = frm; diff --git a/cresis-toolbox/OPS-MATLAB/output/runOpsLoadLayers.m b/cresis-toolbox/OPS-MATLAB/output/runOpsLoadLayers.m index 9fcff49d..c6891ebb 100644 --- a/cresis-toolbox/OPS-MATLAB/output/runOpsLoadLayers.m +++ b/cresis-toolbox/OPS-MATLAB/output/runOpsLoadLayers.m @@ -10,10 +10,12 @@ %% User Settings % ===================================================================== -params = read_param_xls(ct_filename_param('rds_param_2019_Antarctica_Ground.xls')); +% params = read_param_xls(ct_filename_param('rds_param_2012_Greenland_P3.xls')); +params = read_param_xls(ct_filename_param('snow_param_2012_Greenland_P3.xls')); params = ct_set_params(params,'cmd.generic',0); -params = ct_set_params(params,'cmd.generic',1,'day_seg','20200107_01'); +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20120330_03'); +params = ct_set_params(params,'cmd.generic',1,'day_seg','20120330_04'); layer_params = []; idx = 0; @@ -29,10 +31,18 @@ layer_params(idx).echogram_source = 'qlook'; elseif 1 - %% Load a single layer from the layerData file + %% Load two layers from the layerData file ref_idx = 1; idx = idx + 1; - layer_params(idx).name = 'surface'; + layer_params(idx).name = {'surface','bottom'}; + layer_params(idx).source = 'layerdata'; + layer_params(idx).layerdata_source = 'layer'; + +elseif 0 + %% Load two layers from the layerData file + ref_idx = 1; + idx = idx + 1; + layer_params(idx).name = {'surface','layer.*'}; layer_params(idx).source = 'layerdata'; layer_params(idx).layerdata_source = 'layer'; @@ -51,7 +61,7 @@ layer_params(idx).name = 'surface'; layer_params(idx).source = 'records'; -elseif 1 +elseif 0 %% Compare echogram and custom layers in layerData ref_idx = 1; idx = idx + 1; @@ -92,6 +102,36 @@ idx = idx + 1; layer_params(idx).name = 'surface'; layer_params(idx).source = 'lidar'; +elseif 0 + %% load surface, bottom and MacGregor layers + ref_idx = 1; + idx = idx + 1; + layer_params(idx).name = 'surface'; + layer_params(idx).source = 'layerdata'; + layer_params(idx).layerdata_source = 'layer'; + idx = idx + 1; + layer_params(idx).name = 'bottom'; + layer_params(idx).source = 'layerdata'; + layer_params(idx).layerdata_source = 'layer'; + idx = idx + 1; + layer_params(idx).regexp = 'L'; + layer_params(idx).source = 'layerdata'; + layer_params(idx).layerdata_source = 'layer_MacGregor'; +else + %% load surface, bottom and snow layers + ref_idx = 1; + idx = idx + 1; + layer_params(idx).name = 'surface'; + layer_params(idx).source = 'layerdata'; + layer_params(idx).layerdata_source = 'layer'; + idx = idx + 1; + layer_params(idx).name = 'bottom'; + layer_params(idx).source = 'layerdata'; + layer_params(idx).layerdata_source = 'layer'; + idx = idx + 1; + layer_params(idx).regexp = 'snow'; + layer_params(idx).source = 'layerdata'; + layer_params(idx).layerdata_source = 'layer'; end % ===================================================================== @@ -103,6 +143,8 @@ %% Load each of the day segments layers = {}; day_seg = {}; +params_list = {}; +new_layer_params = []; for param_idx = 1:length(params) param = params(param_idx); if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) ... @@ -115,6 +157,7 @@ fprintf('opsLoadLayers %s\n', param.day_seg); [layers{end+1},new_layer_params] = opsLoadLayers(param,layer_params); day_seg{end+1} = param.day_seg; + params_list{end+1} = param; end layer_params = new_layer_params; @@ -162,28 +205,30 @@ || isempty(layer_params(ref_idx).twtt_offset) layer_params(ref_idx).twtt_offset = 0; end + figure(1); clf; h_axes = axes('Parent',1); - h_plot = plot(layers{seg_idx}(ref_idx).gps_time, ... + [~,ref_frm,~] = get_frame_id(params_list{seg_idx},layers{seg_idx}(ref_idx).gps_time,struct('segment_id_num',true)); + h_plot = plot(ref_frm, ... layers{seg_idx}(ref_idx).twtt + layer_params(ref_idx).twtt_offset, ... 'k.','Parent',h_axes); title(sprintf('%s', day_seg{seg_idx}),'Interpreter','none','Parent',h_axes); legend_strs = {sprintf('Ref %d', ref_idx)}; hold(h_axes,'on'); grid(h_axes,'on'); - xlabel('GPS time (sec)','Parent',h_axes); + xlabel('Frame','Parent',h_axes); ylabel('TWTT (us)','Parent',h_axes); figure(2); clf(2); h_axes_comp = axes('Parent',2); - h_plot_comp = plot(layers{seg_idx}(ref_idx).gps_time, ... + h_plot_comp = plot(ref_frm, ... layers{seg_idx}(ref_idx).twtt_ref + layer_params(ref_idx).twtt_offset ... - (layers{seg_idx}(ref_idx).twtt_ref + layer_params(ref_idx).twtt_offset), ... 'k.','Parent',h_axes_comp); title(sprintf('%s', day_seg{seg_idx}),'Interpreter','none','Parent',h_axes_comp); hold(h_axes_comp,'on'); grid(h_axes_comp,'on'); - xlabel('GPS time (sec)','Parent',h_axes_comp); + xlabel('Frame','Parent',h_axes_comp); ylabel('TWTT Difference (us)','Parent',h_axes_comp); fprintf('Layer Index\tMedian Offset\tMean Offset\n'); @@ -194,12 +239,13 @@ layer_params(lay_idx).twtt_offset = 0; end - h_plot(end+1) = plot(layers{seg_idx}(lay_idx).gps_time, ... + [~,lay_frm,~] = get_frame_id(params_list{seg_idx},layers{seg_idx}(lay_idx).gps_time,struct('segment_id_num',true)); + h_plot(end+1) = plot(lay_frm, ... layers{seg_idx}(lay_idx).twtt + layer_params(lay_idx).twtt_offset, ... plot_modes{mod(lay_idx-1,length(plot_modes))+1},'Parent',h_axes); legend_strs{end+1} = sprintf('Layer %d', lay_idx); - h_plot_comp(end+1) = plot(layers{seg_idx}(ref_idx).gps_time, ... + h_plot_comp(end+1) = plot(ref_frm, ... layers{seg_idx}(lay_idx).twtt_ref + layer_params(lay_idx).twtt_offset ... - (layers{seg_idx}(ref_idx).twtt_ref + layer_params(ref_idx).twtt_offset), ... plot_modes{mod(lay_idx-1,length(plot_modes))+1},'Parent',h_axes_comp); @@ -233,7 +279,7 @@ for seg_idx = 1:length(layers) figure(1); clf; - plot(layers{seg_idx}(2).gps_time, layers{seg_idx}(ref_idx).twtt, 'k.'); + plot(layers{seg_idx}(ref_idx).gps_time, layers{seg_idx}(ref_idx).twtt, 'k.'); title(sprintf('%s', day_seg{seg_idx}),'Interpreter','none'); hold on; for lay_idx = lay_idxs @@ -243,7 +289,7 @@ h_axis = gca; grid on xlabel('GPS time (sec)'); - ylabel('TWTT ({\mieu}s)'); + ylabel('TWTT ({\mu}s)','interpreter','tex'); figure(2); clf; plot(layers{seg_idx}(ref_idx).gps_time, layers{seg_idx}(ref_idx).twtt, '.'); @@ -256,7 +302,7 @@ end h_axis(2) = gca; xlabel('GPS time (sec)'); - ylabel('TWTT ({\mieu}s)'); + ylabel('TWTT ({\mu}s)','interpreter','tex'); figure(3); clf; for lay_idx = lay_idxs @@ -275,7 +321,7 @@ hold off; h_axis(3) = gca; xlabel('GPS time (sec)'); - ylabel('TWTT ({\mieu}s)'); + ylabel('TWTT ({\mu}s)','interpreter','tex'); fprintf('\n'); linkaxes(h_axis,'x'); diff --git a/cresis-toolbox/OPS-MATLAB/utility/opsBulkInsert.m b/cresis-toolbox/OPS-MATLAB/utility/opsBulkInsert.m index 4fd92cb2..0888b28d 100644 --- a/cresis-toolbox/OPS-MATLAB/utility/opsBulkInsert.m +++ b/cresis-toolbox/OPS-MATLAB/utility/opsBulkInsert.m @@ -2,47 +2,39 @@ function opsBulkInsert(settings) % % opsBulkInsert(settings) % -% Loads data in bulk from the CReSIS filesystem to the OPS. +% Loads flight paths in bulk from records files into OPS. % -% Author: Kyle W. Purdon +% Author: Kyle W. Purdon, John Paden % -% see also runOpsBulkInsert +% See also: runOpsBulkInsert -%% SETUP DEFAULTS AND OTHER INPUTS +%% Setup defaults, perform input checks -global gRadar; opsCmd; -% SET THE LOG BASE DIRECTORY -if settings.logsOn - logsBaseFn = gRadar.tmp_path; - fprintf('Logs will be saved to: %s\n',logsBaseFn); -end +% Convenience variable naming +params = settings.params; +radar_name = params(1).radar_name; +season_name = params(1).season_name; +sys = ct_output_dir(params(1).radar_name); -% LAYER FILTER DEFAULTS -if ~isfield(settings,'layerFilter') || isempty(settings.layerFilter) - settings.layerFilter = inline('~isempty(regexp(x,''.*''))'); +% Set settings.path_spacing +if ~isfield(settings,'path_spacing') || isempty(settings.path_spacing) + if any(strcmpi(sys,{'kuband','snow'})) + settings.path_spacing = 5; + else + settings.path_spacing = 15; + end end -% PATH SPACING DEFAULT -if ~isfield(settings,'pathSpacing') || isempty(settings.pathSpacing) - settings.pathSpacing = 15; +% Set settings.season_group +if ~isfield(settings,'season_group') || isempty(settings.season_group) + % settings.season_group = 'cresis_private'; + settings.season_group = 'cresis_public'; end -% GET A BOOLEAN VALUE FOR EACH RUN OPTION -insertPathCmd = any(settings.runType == [1,4,5]); -insertLayerCmd = any(settings.runType == [2,4,5]); -insertAtmCmd = any(settings.runType == [3,5]); - -%% LOAD THE PARAM SPREADSHEET AND SAVE LOCAL VARIABLES FROM THE SETTINGS - -params = read_param_xls(ct_filename_param(settings.paramFn)); -settings.radarName = params(1).radar_name; -settings.seasonName = params(1).season_name; - -%% CHECK FILE INPUTS - -fprintf('Checking file inputs ...\n'); +% Checking to see if any selected segments contain "do not process" in +% cmd.notes for param_idx = 1:length(params) param = params(param_idx); if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic @@ -50,545 +42,161 @@ function opsBulkInsert(settings) end if ~isempty(regexpi(param.cmd.notes,'do not process')) - warning('You have enabled a segment with ''do not process'' in the cmd.notes, dbcont to continue'); + warning('You have enabled %s, a segment with ''do not process'' in the cmd.notes, dbcont to continue', param.day_seg); keyboard end - recordsFn = ct_filename_support(param,'','records'); - framesFn = ct_filename_support(param,'','frames'); - layerDir = ct_filename_out(param,settings.layerDataPath,''); - if ~exist(recordsFn,'file') && insertPathCmd - error(' %s: missing %s\n', param.day_seg, recordsFn); - elseif ~exist(framesFn,'file') && insertPathCmd - error(' %s: missing %s\n', param.day_seg, framesFn); - elseif ~exist(layerDir,'dir') && insertLayerCmd - warning(' %s: missing %s. Layer data directory for the particular segment is missing. If this is expected, run "dbcont".\n', param.day_seg, layerDir); - keyboard - else - fprintf(' %s checked\n', param.day_seg); - end end -%% USER CONFIRMATION +%% User Confirmation -switch settings.runType - case 1 - runStr = 'path'; - case 2 - runStr = 'layer'; - case 3 - runStr = 'atm'; - case 4 - runStr = 'path,layer'; - case 5 - runStr = 'path,layer,atm'; -end +confirm_params = {sprintf('SERVER: \t %s',gOps.serverUrl),'',sprintf('SEASON NAME: \t %s',season_name),'',... + sprintf('RADAR NAME: \t %s',radar_name),'',... + sprintf('PATH SPACING: \t %0.2f meters',settings.path_spacing),''}; -confirmParams = {sprintf('SERVER: \t %s',gOps.serverUrl),'',sprintf('SEASON NAME: \t %s',settings.seasonName),'',... - sprintf('RADAR NAME: \t %s',settings.radarName),'',sprintf('RUN TYPE: \t %s',runStr),'',... - sprintf('SYSTEM NAME: \t %s',settings.sysName),'',sprintf('PATH SPACING: \t %0.2f meters',settings.pathSpacing),'',... - sprintf('LOCATION: \t %s',settings.location),''}; +confirm_button = questdlg(confirm_params,'Confirm Settings','YES:LOAD','NO:QUIT','NO:QUIT'); -confirmButton = questdlg(confirmParams,'Confirm Settings','YES:LOAD','NO:QUIT','NO:QUIT'); - -switch confirmButton +switch confirm_button case 'NO:QUIT' error('PROCESS STOPPED BY USER.'); end -%% PATH INSERTION +%% Path Insert Loop -if insertPathCmd - - failedSegments = {}; % STORE SEGMENTS THAT FAIL - - % START LOGGING - if settings.logsOn - pathLogFn = fullfile(logsBaseFn,strcat('OPS_PATHINSERT_',datestr(now,'yyyy.mm.dd'),'_',datestr(now,'HH.MM.SS'),'.txt')); - diary(pathLogFn); - end - - % FOR EACH SEGMENT PROCESS THE INPUT AND PUSH THE DATA TO THE SERVER - for paramIdx = 1:length(params) - - try - - % CONFIRM THAT GENERIC IS NOT FLAGGED - param = params(paramIdx); - if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic - continue; - end - - fprintf('Loading path for segment %s ... ',param.day_seg); - - start = tic; % SET UP TIMING - - % GET THE RECORDS AND FRAMES FILENAMES - recordsFn = ct_filename_support(param,'','records'); - framesFn = ct_filename_support(param,'','frames'); - - frames = load(framesFn); % LOAD FRAMES FILE - records = load(recordsFn); % LOAD RECORDS FILE - if isfield(records,'records') - records = records.records; % SUPPORT OLD RECORDS FORMAT - end - - % GET THE START GPS TIME OF EACH FRAME - frameStartGpsTime = zeros(1,length(frames.frame_idxs)); - for frmIdx = 1:length(frames.frame_idxs) - if frmIdx == length(frames.frame_idxs) - frameStartGpsTime(1,frmIdx) = records.gps_time(frames.frame_idxs(frmIdx)); % ADD THE END SEGMENT - else - frameStartGpsTime(1,frmIdx) = records.gps_time(frames.frame_idxs(frmIdx)); % ADD 1:N-1 SEGMENTS - end - end - - % INTERPOLATE RECORDS GPS TIME ONTO THE GIVEN SPACING (DEFAULT = 15m) - alongTrack = geodetic_to_along_track(records.lat,records.lon,records.elev); - newAlongTrack = 0:settings.pathSpacing:alongTrack(end); - positive_idxs = [1, 1+find(diff(alongTrack) > 0)]; - outGpsTime = interp1(alongTrack(positive_idxs),records.gps_time(positive_idxs),newAlongTrack,'pchip'); - - % INTERPOLATE RECORDS VALUES ONTO NEW GPS TIME - outLon = interp1(records.gps_time,records.lon,outGpsTime,'pchip'); - outLat = interp1(records.gps_time,records.lat,outGpsTime,'pchip'); - outElev = interp1(records.gps_time,records.elev,outGpsTime,'pchip'); - outRoll = interp1(records.gps_time,records.roll,outGpsTime,'pchip'); - outPitch = interp1(records.gps_time,records.pitch,outGpsTime,'pchip'); - outHeading = interp1(records.gps_time,records.heading,outGpsTime,'pchip'); - - % ERROR CHECK OUTPUT DATA - if any(find(outHeading>(pi*2))) || any(find(outHeading<(-pi*2))) - warning('OUTPUT HEADING OUT OF BOUND 2pi <> -2pi'); - keyboard; - end - if any(find(outPitch>(pi*2))) || any(find(outPitch<(-pi*2))) - warning('OUTPUT PITCH OUT OF BOUND 2pi <> -2pi'); - keyboard; - end - if any(find(outRoll>(pi*2))) || any(find(outRoll<(-pi*2))) - warning('OUTPUT ROLL OUT OF BOUND 2pi <> -2pi'); - keyboard; - end - if any(find(outLon>180)) || any(find(outLon<-180)) - warning('OUTPUT LONGITUDE OUT OF BOUND 180 <> -180'); - keyboard; - end - if any(find(outLat>90)) || any(find(outLat<-90)) - warning('OUTPUT LATITUDE OUT OF BOUND 90 <> -90'); - keyboard; - end - - % BUILD STRUCTURE FOR opsCreatePath() - outData.geometry.coordinates = [outLon' outLat']; - outData.properties.location = settings.location; - outData.properties.season = settings.seasonName; - outData.properties.season_group = settings.seasonGroup; - outData.properties.radar = settings.radarName; - outData.properties.segment = param.day_seg; - outData.properties.gps_time = outGpsTime; - outData.properties.elev = outElev; - outData.properties.roll = outRoll; - outData.properties.pitch = outPitch; - outData.properties.heading = outHeading; - outData.properties.frame_count = length(frames.frame_idxs); - outData.properties.frame_start_gps_time = frameStartGpsTime; - - clear records frames - - mstop = toc(start); % RECORD MATLAB COMPUTATION TIME - - % PUSH DATA TO THE SERVER - [status,message] = opsCreatePath(settings.sysName,outData); - - pstop = toc(start)-mstop; % RECORD PYTHON COMPUTATION TIME - - if status ~= 1 - fprintf('\n'); - warning(message); - failedSegments{end+1} = param.day_seg; % STORE THE SEGMENT IF THE SERVER PUSH FAILED - end - - % REPORT TIMING AND STATUS - fprintf('\n\t-> Time: Matlab %2.2fs Python %2.2fs\n',mstop,pstop); - fprintf('\t-> Status: %s\n',message); - - catch ME - - diary OFF - ME.getReport() - failedSegments{end+1} = param.day_seg; % STORE THE SEGMENT IF ANYTHING FAILED - - end - end - - % REPORT FAILED SEGMENTS - if ~isempty(failedSegments) - fprintf('\n\nThere were issues loading paths for segments:\n'); - for failIdx = 1:length(failedSegments) - fprintf('\t%s\n',failedSegments{failIdx}); - end - fprintf('No layer points will be loaded for these segments.\n\n'); - end - - diary OFF % STOP LOGGING - -end +failed_segments = {}; % STORE SEGMENTS THAT FAIL -%% LAYER INSERTION +% START LOGGING +log_fn = fullfile(settings.tmp_path,mfilename,sys,strcat(season_name,'_',datestr(now,'yyyymmdd_HHMMSS'),'.txt')); +log_fn_dir = fileparts(log_fn); +if ~exist(log_fn_dir,'dir') + mkdir(log_fn_dir); +end +diary(log_fn); +fprintf('Logs will be saved to: %s\n',log_fn); -if insertLayerCmd - - mstart = tic; - - failedFrames = {}; % STORE FRAMES THAT FAIL - failedLayers = {}; % STORE LAYERS THAT FAIL - frameName = ''; - segDayObj = []; - opsLayerDataSub = []; - - if ~exist('failedSegments','var') - failedSegments = {}; - end - - % START LOGGING - if settings.logsOn - pathLogFn = fullfile(logsBaseFn,strcat('OPS_LAYERINSERT_',datestr(now,'yyyy.mm.dd'),'_',datestr(now,'HH.MM.SS'),'.txt')); - diary(pathLogFn); - end +% FOR EACH SEGMENT PROCESS THE INPUT AND PUSH THE DATA TO THE SERVER +for param_idx = 1:length(params) - % BUILD THE DAY / DAYSEG LIST FOR PROCESSING - for paramIdx = 1:length(params) + try - % CONFIRM THAT GENERIC IS NOT FLAGGED - param = params(paramIdx); + %% Path Insert: Setup + % CONFIRM THAT GENERIC IS FLAGGED + param = params(param_idx); if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic continue; end - % DO NOT LOAD LAYERS FOR FAILTED SEGMENTS - if any(strcmp(param.day_seg,failedSegments)) - fprintf('Skipping layers for segment %s, path loading failed for this segment.\n',param.day_seg); - continue; - end + fprintf('Loading path for segment %s ... ',param.day_seg); - load(ct_filename_support(param,'','frames')); % LOAD FRAMES FILE + start = tic; % SET UP TIMING - % CROSS VERIFY FRAMES FILE WITH PARAM.CMD.FRMS LIST - if isempty(param.cmd.frms) - param.cmd.frms = 1:length(frames.frame_idxs); - end - [validFrms,keepIdxs] = intersect(param.cmd.frms, 1:length(frames.frame_idxs)); - if length(validFrms) ~= length(param.cmd.frms) - badMask = ones(size(param.cmd.frms)); - badMask(keepIdxs) = 0; - warning('Nonexistent frames in param.cmd.frms (e.g. frame "%g" is invalid). These will be removed.',param.cmd.frms(find(badMask,1))); - param.cmd.frms = validFrms; + if ~any(strcmp(param.post.ops.location,{'arctic','antarctic'})) + error('param.post.ops.location must be set to ''arctic'' or ''antarctic''. Current value is %s.', param.post.ops.location); end + fprintf(' Loading into location %s\n', param.post.ops.location); - for frmId = 1:length(param.cmd.frms) - - % BUILD UP DAY/DAYSEG/DAYSEGFRM OBJECT - segDayObj = cat(1,segDayObj,{param.day_seg(1:end-3) param.day_seg sprintf('%s_%03d',param.day_seg,frmId)}); - - end + % Load records and frames files + frames = frames_load(param); + records = records_load(param); - end - - % GET UNIQUE LIST OF DAYS - segDays = unique(segDayObj(:,1)); - - % PROCESS EACH DAY - for segDayIdx = 1:length(segDays) - - curDay = segDays{segDayIdx}; - fprintf('Processing data for day %s ...\n',curDay); + % Apply reference lever arm to records trajectory + trajectory_param = struct('gps_source',records.gps_source, ... + 'season_name',param.season_name,'radar_name',param.radar_name,'rx_path', 0, ... + 'tx_weights', [], 'lever_arm_fh', param.radar.lever_arm_fh); + records = trajectory_with_leverarm(records,trajectory_param); - % LOAD ALL THE LAYERDATA FOR THE CURRENT DAY - layerBaseDir = ct_filename_out(param,settings.layerDataPath,'',true); - dayLayerData = opsMergeLayerData(layerBaseDir,curDay); - if isempty(dayLayerData.GPS_time) - error('WARNING: NO LAYER DATA EXIST'); - end - [opsLayerData,opsLayerDataGpsTime] = layerDataToOps(dayLayerData,settings); - - if isfield(settings,'layer_name_map') - for layerIdx = 1:length(opsLayerData) - match_idx = find(strcmpi(opsLayerData(layerIdx).properties.lyr_name, settings.layer_name_map.source_names)); - if ~isempty(match_idx) - opsLayerData(layerIdx).properties.lyr_name = settings.layer_name_map.dest_names{match_idx}; - end + % Get the start time of each frame + frm_start_gps_time = zeros(1,length(frames.frame_idxs)); + for frmIdx = 1:length(frames.frame_idxs) + if frmIdx == length(frames.frame_idxs) + frm_start_gps_time(1,frmIdx) = records.gps_time(frames.frame_idxs(frmIdx)); % ADD THE END SEGMENT + else + frm_start_gps_time(1,frmIdx) = records.gps_time(frames.frame_idxs(frmIdx)); % ADD 1:N-1 SEGMENTS end end - % CHECK FOR EMPTY LAYERS - emptyLayerIdxs = []; - for layerIdx = 1:length(opsLayerData) - if isempty(opsLayerData(layerIdx).properties.twtt) - emptyLayerIdxs(end+1) = layerIdx; - end - end - opsLayerData(emptyLayerIdxs) = []; % SET THE EMPTY STRUCTURES - if isempty(opsLayerData) - message = 'WARNING: NO LAYERS TO INSERT, LAYERS ARE EMPTY'; % SAVE MESSAGE IF ALL ARE EMPTY - fprintf('\n\t\t\t-> Time: Matlab %2.2fs Python %2.2fs\n',0.00,0.00); - fprintf('\t\t\t-> Status: %s\n',message); - failedFrames{end+1} = frameName; - end + %% Path Insert: Interpolate - % GET A LIST OF SEGMENTS FOR THE CURRENT DAY - segList = unique(segDayObj(find(strcmp(segDayObj(:,1),curDay)),2)); + % INTERPOLATE RECORDS GPS TIME ONTO THE GIVEN SPACING (DEFAULT = 15m) + along_track = geodetic_to_along_track(records.lat,records.lon,records.elev); + new_along_track = 0:settings.path_spacing:along_track(end); + positive_idxs = [1, 1+find(diff(along_track) > 0)]; + out_gps_time = interp1(along_track(positive_idxs),records.gps_time(positive_idxs),new_along_track,'pchip'); - % PROCESS EACH SEGMENT - for segIdx = 1:length(segList) - - curSeg = segList{segIdx}; - fprintf('Loading layers for segment %s ...\n',curSeg); - - segInfoParam.properties.segment = curSeg; - segInfoParam.properties.season = settings.seasonName; - [~,segData] = opsGetSegmentInfo(settings.sysName,segInfoParam); - - mstop = toc(mstart); % RECORD MATLAB COMPUTATION TIME - - if settings.layerRelativeSurface - param_idx = strmatch(curSeg,{params.day_seg}); - layer_params = []; - layer_params.name = 'surface'; - layer_params.source = 'layerdata'; - layers = opsLoadLayers(params(param_idx),layer_params); - end - - % PROCESS EACH FRAME - for frmIdx = 1:length(segData.properties.frame) - - try - curFrame = segData.properties.frame{frmIdx}; - if ~any(strcmp(segDayObj(:,3),curFrame)) - continue; - end - - fprintf('\tLoading frame %s ...\n',curFrame); - - % PROCESS EACH LAYER - for layerIdx = 1:length(opsLayerData) - - curLayer = opsLayerData(layerIdx).properties.lyr_name; - - try - - ptic = tic; - fprintf('\t\tLoading layer %s ...',curLayer); - - % GET POINT PATH IDS FOR FRAME TIME RANGE - pathParam.properties.location = settings.location; - pathParam.properties.season = settings.seasonName; - pathParam.properties.start_gps_time = segData.properties.start_gps_time(frmIdx); - pathParam.properties.stop_gps_time = segData.properties.stop_gps_time(frmIdx); - [~,pathData] = opsGetPath(settings.sysName,pathParam); - [~,keepIdxs] = ismember(pathData.properties.id,opsLayerData(layerIdx).properties.point_path_id); - keepIdxs = keepIdxs(find(keepIdxs~=0)); - - % SUBSET OPSLAYERDATA TO FRAME - opsLayerDataSub.properties.point_path_id = opsLayerData(layerIdx).properties.point_path_id(keepIdxs); - opsLayerDataSub.properties.twtt = opsLayerData(layerIdx).properties.twtt(keepIdxs); - opsLayerDataSub.properties.type = opsLayerData(layerIdx).properties.type(keepIdxs); - opsLayerDataSub.properties.quality = opsLayerData(layerIdx).properties.quality(keepIdxs); - opsLayerDataSub.properties.lyr_name = opsLayerData(layerIdx).properties.lyr_name; - - if settings.layerRelativeSurface - if 0 - % DEBUG CODE - figure(101); clf; - plot(layers.gps_time,layers.twtt); - hold on; - plot(opsLayerDataGpsTime{layerIdx}(keepIdxs),opsLayerDataSub.properties.twtt) - end - - new_surf = interp1(layers.gps_time,layers.twtt, ... - opsLayerDataGpsTime{layerIdx}(keepIdxs),'linear'); - new_surf = interp_finite(new_surf,NaN); - old_surf = interp1(dayLayerData.GPS_time,dayLayerData.layerData{1}.value{2}.data, ... - opsLayerDataGpsTime{layerIdx}(keepIdxs),'linear'); - old_surf = interp_finite(old_surf,NaN); - twtt_correction = new_surf-old_surf; - opsLayerDataSub.properties.twtt = opsLayerDataSub.properties.twtt + twtt_correction; - - if 0 - % DEBUG CODE - plot(opsLayerDataGpsTime{layerIdx}(keepIdxs),opsLayerDataSub.properties.twtt) - end - end - - % PUSH DATA TO THE SERVER - [status,message] = opsCreateLayerPoints(settings.sysName,opsLayerDataSub); - pstop = toc(ptic); - - opsLayerDataSub = []; % RESET SUBSET STRUCTURE - - if status ~= 1 - - % REPORT A FAILED LAYER - failedLayers{end+1} = {curFrame,opsLayerData(layerIdx).properties.lyr_name}; - fprintf('\n'); - warning(message); - - else - - fprintf('\n\t\t\t-> Time: Matlab %2.2fs Python %2.2fs\n',mstop,pstop); - fprintf('\t\t\t-> Status: %s\n',message); - - end - - catch ME - - % REPORT A FAILED FRAME - fprintf('\n'); - warning(sprintf('%s at line %d in file %s.',ME.message,ME.stack(1).line,ME.stack(1).name)); - failedLayers{end+1} = {curFrame,opsLayerData(layerIdx).properties.lyr_name}; % STORE THE LAYER NAME IF ANYTHING FAILED - continue - - end - end - - catch ME - - fprintf('\n'); - warning(sprintf('%s at line %d in file %s.',ME.message,ME.stack(1).line,ME.stack(1).name)); - failedFrames{end+1} = curFrame; % STORE THE FRAME NAME IF ANYTHING FAILED - continue - - end - end - mstart = tic; % RESTART MATLAB TIMING (END OF SEGMENT LOAD) + % INTERPOLATE RECORDS VALUES ONTO NEW GPS TIME + outLon = gps_interp1(records.gps_time,records.lon/180*pi,out_gps_time,'pchip')*180/pi; + outLat = interp1(records.gps_time,records.lat,out_gps_time,'pchip'); + out_elev = interp1(records.gps_time,records.elev,out_gps_time,'pchip'); + out_roll = interp1(records.gps_time,records.roll,out_gps_time,'pchip'); + out_pitch = interp1(records.gps_time,records.pitch,out_gps_time,'pchip'); + out_heading = gps_interp1(records.gps_time,records.heading,out_gps_time,'pchip'); + + %% Path Insert: ERROR CHECK OUTPUT DATA + if any(find(out_heading>(pi*2))) || any(find(out_heading<(-pi*2))) + warning('OUTPUT HEADING OUT OF BOUND 2pi <> -2pi'); + keyboard; end - end - - % REPORT FAILED FRAMES - if ~isempty(failedFrames) - fprintf('\n\nThere were issues loading data for frames:\n'); - for failIdx = 1:length(failedFrames) - fprintf('\t%s\n',failedFrames{failIdx}); + if any(find(out_pitch>(pi*2))) || any(find(out_pitch<(-pi*2))) + warning('OUTPUT PITCH OUT OF BOUND 2pi <> -2pi'); + keyboard; end - end - - % REPORT FAILED LAYERS - if ~isempty(failedLayers) - fprintf('\n\nThere were issues loading data for layers:\n'); - for failIdx = 1:length(failedLayers) - fprintf('\t%s %s\n',failedLayers{failIdx}{1},failedLayers{failIdx}{2}); + if any(find(out_roll>(pi*2))) || any(find(out_roll<(-pi*2))) + warning('OUTPUT ROLL OUT OF BOUND 2pi <> -2pi'); + keyboard; end - end - - diary OFF -end - -%% ATM INSERTION - -if insertAtmCmd - - atmFailed=false; - segDayObj = []; - - if ~exist('failedSegments','var') - failedSegments = {}; - end - - % START LOGGING - if settings.logsOn - pathLogFn = fullfile(logsBaseFn,strcat('OPS_ATMINSERT_',datestr(now,'yyyy.mm.dd'),'_',datestr(now,'HH.MM.SS'),'.txt')); - diary(pathLogFn); - end - - start = tic; % SET UP TIMING - - % BUILD THE DAY / DAYSEG LIST FOR PROCESSING - for paramIdx = 1:length(params) - - % CONFIRM THAT GENERIC IS NOT FLAGGED - param = params(paramIdx); - if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic - continue; + if any(find(outLon>180)) || any(find(outLon<-180)) + warning('OUTPUT LONGITUDE OUT OF BOUND 180 <> -180'); + keyboard; end - - % DO NOT LOAD LAYERS FOR FAILTED SEGMENTS - if any(strcmp(param.day_seg,failedSegments)) - fprintf('Skipping layers for segment %s, path loading failed for this segment.\n',param.day_seg); - continue; + if any(find(outLat>90)) || any(find(outLat<-90)) + warning('OUTPUT LATITUDE OUT OF BOUND 90 <> -90'); + keyboard; end - % BUILD UP DAY/DAYSEG/DAYSEGFRM OBJECT - segDayObj = cat(1,segDayObj,{param.day_seg(1:end-3) param.day_seg}); + %% Path Insert: opsCreatePath + out_data.geometry.coordinates = [outLon' outLat']; + out_data.properties.location = param.post.ops.location; + out_data.properties.season = season_name; + out_data.properties.season_group = settings.season_group; + out_data.properties.radar = radar_name; + out_data.properties.segment = param.day_seg; + out_data.properties.gps_time = out_gps_time; + out_data.properties.elev = out_elev; + out_data.properties.roll = out_roll; + out_data.properties.pitch = out_pitch; + out_data.properties.heading = out_heading; + out_data.properties.frame_count = length(frames.frame_idxs); + out_data.properties.frame_start_gps_time = frm_start_gps_time; - end - - % CREATE PARAM FOR ATM LAYER - opsAtmLayerParam.properties.lyr_name = 'atm'; - opsAtmLayerParam.properties.lyr_group_name = 'lidar'; - opsAtmLayerParam.properties.lyr_description = 'atm l2 lidar surface'; - opsAtmLayerParam.properties.public = true; - - % PUSH DATA TO THE SERVER - try - [~,~] = opsCreateLayer(settings.sysName,opsAtmLayerParam); - catch ME - fprintf('\t-> Layer ''atm'' exists, layer points will be added to layer.\n'); - end - - % GET UNIQUE LIST OF DAYS - segDays = unique(segDayObj(:,1)); - - % PROCESS EACH DAY - for segDayIdx = 1:length(segDays) + mstop = toc(start); % RECORD MATLAB COMPUTATION TIME - try - curDay = segDays{segDayIdx}; - fprintf('Loading ATM data for day %s ...\n',curDay); - - % GET ATM L2 FILENAMES - atmFns = get_filenames_atm(settings.location,curDay,settings.data_support_path); - if isempty(atmFns) - warning('No atm data. Skipping segment %s',param.day_seg); - continue; - end - - % CONVERT ATM L2 TO OPS FORMAT - opsAtmData = atmToOps(atmFns,settings); - - mstop = toc(start); % RECORD MATLAB COMPUTATION TIME - ptic = tic; - - % PUSH DATA TO THE SERVER - [status,message] = opsCreateLayerPoints(settings.sysName,opsAtmData); - - pstop = toc(ptic); - - if status ~= 1 - - atmFailed = true; - fprintf('\n'); - warning(message); - - else - - fprintf('\t-> Time: Matlab %2.2fs Python %2.2fs\n',mstop,pstop); - fprintf('\t-> Status: %s\n',message); - - end - - catch ME + [status,message] = opsCreatePath(sys,out_data); + + pstop = toc(start)-mstop; % RECORD PYTHON COMPUTATION TIME + + if status ~= 1 fprintf('\n'); - warning(sprintf('%s at line %d in file %s.',ME.message,ME.stack(1).line,ME.stack(1).name)); - atmFailed = true; - continue + warning(message); + failed_segments{end+1} = param.day_seg; % STORE THE SEGMENT IF THE SERVER PUSH FAILED end + %% Path Insert: Report timing/status + fprintf('\n\t-> Time: Matlab %2.2fs Python %2.2fs\n',mstop,pstop); + fprintf('\t-> Status: %s\n',message); + + catch ME + + ME.getReport() + failed_segments{end+1} = param.day_seg; % STORE THE SEGMENT IF ANYTHING FAILED + end - - % REPORT FAILED ATM - if atmFailed - fprintf('\n\nThere were issues loading atm data.\n'); +end + +%% REPORT FAILED SEGMENTS +if ~isempty(failed_segments) + fprintf('\n\nThere were issues loading paths for segments:\n'); + for failIdx = 1:length(failed_segments) + fprintf('\t%s\n',failed_segments{failIdx}); end - - diary OFF - -end \ No newline at end of file +end + +diary OFF % STOP LOGGING diff --git a/cresis-toolbox/OPS-MATLAB/utility/opsCopyLayers.m b/cresis-toolbox/OPS-MATLAB/utility/opsCopyLayers.m index 13fc54a1..e8386fe0 100644 --- a/cresis-toolbox/OPS-MATLAB/utility/opsCopyLayers.m +++ b/cresis-toolbox/OPS-MATLAB/utility/opsCopyLayers.m @@ -3,19 +3,25 @@ % % Copies contents from one layer into another layer. The copy method can % be merge, overwrite, or fill gaps. If the destination layer does not -% exist, it will be created. The source and destination for the data can be +% exist, it will be created. The source and destination for the data can be % OPS, layerData, echogram, or records. Additionally, the source can be % lidar (ATM) or a custom GPS-time/two-way-travel-time. % % An operation on the source data using the eval field can be added. More -% complicated operations can be done using opsLoadLayers, applying the +% complicated operations can be done using opsLoadLayers, applying the % complex operation on that result and then using the custom input. % % Input: % param: Parameter structure from read_param_xls parameter spreadsheet % copy_param: Structure which controls copying process % .layer_source: structure specifying the source layer -% .name: string (e.g. 'surface', 'Surface', 'bottom', 'atm', etc) +% +% .name: cell array of layer names or a string with a single layer +% name in it; layer names must be unique even if the group name is +% different (e.g. 'surface', 'Surface', 'bottom', 'atm', etc. are all +% different layers); the same layer can be copied multiple times in a +% single call +% % .source: string (e.g. 'records', 'echogram', 'layerdata', 'lidar', % 'custom', or 'ops') % .echogram_source: used only with "echogram" source, string @@ -30,18 +36,40 @@ % (e.g. 'layerData', 'CSARP_post/layerData') % .existence_check: used only with ops source, set to false to allow % layers that do not exist (default is true) -% .gps_time: used only with custom source, Nx by 1 vector, -% GPS time (ANSI-C standard: seconds since Jan 1, 1970) -% .twtt: used only with custom source, Nx by 1 vector corresponding -% to gps_time field containing two way travel time to layer -% .type: used only with custom source, Nx by 1 vector corresponding -% to gps_time field containing type (1=manual, 2=auto), 2 is -% default when not specified or NaN -% .quality: used only with custom source, Nx by 1 vector corresponding -% to gps_time field containing quality (1=good, 2=moderate, -% 3=derived/poor), 1 is default when not specified or NaN +% ------------------------------------------------------------------- +% CUSTOM SOURCE FIELDS (used only with layer_source.source = 'custom') +% * Each field is a cell array +% * Each cell array should be the same length as the cell array +% layer_source.name +% * Entries in these cell arrays align with the corresponding entry +% in layer_source.name. +% * The length of each vector, Nx, in the cell array only needs to +% be the same for a particular layer. Different layers can have +% different values of Nx. +% * Legacy Support: if only one layer specified in +% layer_source.name, then each of these fields only needs one +% cell entry and no cell array was used (just an Nx by 1 vector +% was passed in). +% .gps_time: used only with custom source, cell array of Nx by 1 +% vectors corresponding to GPS time (ANSI-C standard: seconds since +% Jan 1, 1970) +% .twtt: used only with custom source, cell array of Nx by 1 vectors +% corresponding to gps_time field containing two way travel time to +% layer +% .type: used only with custom source, cell array of Nx by 1 vectors +% corresponding to gps_time field containing type (1=manual, 2=auto), +% 2 is default when not specified or NaN +% .quality: used only with custom source, cell array of Nx by 1 +% vectors corresponding to gps_time field containing quality (1=good, +% 2=moderate, 3=derived/poor), 1 is default when not specified or NaN +% END CUSTOM SOURCE FIELDS +% ------------------------------------------------------------------- +% % .layer_dest = structure specifying the destination layer -% .name: string (e.g. 'surface', 'Surface', 'bottom', 'atm', etc) +% .name: cell array of layer names or a string with a single layer +% name in it. This is the destination of the layers specified in +% source so layer_dest.name should list the same number of names as +% layer_source.name % .source: string (e.g. 'records', 'echogram', 'layerdata', or 'ops') % .echogram_source: used only with echogram source, string % containing file path argument to ct_filename_out.m @@ -53,32 +81,56 @@ % layers that do not exist (default is true). If false, the layer % will be created if it does not exist. If true, an error will be % thrown if the layer does not exist. -% .group: used only with ops source and only needed when the layer -% does not already exist. Should be a string containing the group -% name that the layer should be added to. Leave blank to use the -% standard group. -% .description: used only with ops source and only needed when the layer -% does not already exist. Should be a string containing a -% description of the layer contents. +% +% .group_name (or .group to support legacy code): Optional argument. +% This field is for overriding the group_name of the source. It +% should be a cell array with the same dimensions as .name if .name +% is a cell array or a string if .name is a string. Leave undefined +% or an empty matrix to use the default. If a cell has an empty +% matrix in it, [], then the default is used for that layer. Note +% that an empty string is a valid entry and will not trigger the +% default to be used. The strings should contain the group_name of +% the corresponding layer in .name. The default value is the value +% from the destination layer if it already exists. If the +% destination layer does not exist, then the layer source value is +% used. If the layer source does not have a value set, then the +% value will be the string "standard" for each layer. +% +% .desc (or .description to support legacy code): Has the same +% behavior as .group_name except the default value is an empty +% string when no other value is available +% +% .age: Has the same behavior as .group_name except the fields are +% double scalars. The default value is a NaN when no other value is +% available. +% +% .age_source: Has the same behavior as .group_name except the +% fields are all structure arrays. The default value is an empty +% structure array when no other value is available. Note that to +% override the age_source field to remove any age sources, care must +% be taken to ensure that an empty structure is passed in as +% "struct()" and not "[]" since the latter will cause the default +% value to be used. +% % .eval: Optional structure for performing operations on the source % before it is written to the destination. % .cmd: Command string that will be passed to eval % .$(custom): Custom fields % Variables available are: % physical_constants -% "gps_time" (sec) -% "along_track" (m) +% "time" (ANSI-C GPS time, seconds since Jan 1, 1970) +% "at" (along-track, m) % "lat" (deg) % "lon" (deg) % "elev" (m) -% "source" (twtt in sec) +% "s" (two way travel time, twtt, in sec) % "eval_struct" (the eval structure passed in by the user) % The cmd string should generally update "source" variable. For example: % '[B,A] = butter(0.1,2); source = filtfilt(B,A,source);' % Filter % 'source = source + 0.1;' % Apply a twtt shift % 'source = source*2;' % Surface multiple % 'source = interp1(gps_time+15,source,gps_time);' % Apply a GPS time shift -% .quality: struct controlling the quality level +% .quality: struct controlling the quality level % .mode: string specifying one of these methods: % 'overwrite': Overwrites the quality level with ".quality" % 'preserve': Quality level preserved from source @@ -110,7 +162,7 @@ end if ~any(strcmpi(copy_param.gaps_fill.method,{'preserve_gaps','interp_finite'})) - error('Invalid gap_fill.method %s', copy_param.gap_fill.method); + error('Invalid gap_fill.method %s', copy_param.gaps_fill.method); end if strcmpi(copy_param.gaps_fill.method,'preserve_gaps') ... @@ -131,13 +183,101 @@ error('Invalid quality value %d', copy_param.quality.value); end +% Legacy format used .group and .description, update these if needed +if ~isfield(copy_param.layer_dest,'desc') && isfield(copy_param.layer_dest,'description') + copy_param.layer_dest.desc = copy_param.layer_dest.description; +end +if ~isfield(copy_param.layer_dest,'group_name') && isfield(copy_param.layer_dest,'group') + copy_param.layer_dest.group_name = copy_param.layer_dest.group; +end + +% copy_param.layer_dest.age: optional field, overrides age field in the +% corresponding output fields. If a single scalar numeric value, then all +% layers will be written with this value. If a cell array, then each cell +% corresponds to the layer_source layer at the same index. If the contents +% of the cell are [] or if the cell does not exist, then the default value +% is used. The default value first looks at the destination layer. If not +% defined, then the default value pulls from the source layer. If not +% defined, then the value is NaN. +if ~isfield(copy_param.layer_dest,'age') + copy_param.layer_dest.age = {}; +end +if iscell(copy_param.layer_dest.age) + for idx = 1:length(copy_param.layer_dest.age) + if ~isnumeric(copy_param.layer_dest.age{idx}) + error('copy_param.layer_dest.age{%d} must be 1) a numeric scalar or 2) an empty matrix [].', idx); + end + end +else + if ~isempty(copy_param.layer_dest.age) && ~isnumeric(copy_param.layer_dest.age) + error('copy_param.layer_dest.age must be 1) a cell array of numeric scalars, 2) a numeric scalar, 3) an empty numeric matrix [], or 4) undefined.'); + end +end + +% copy_param.layer_dest.age_source: Same as .age field except instead of +% numeric values it is a structure array. If not defined elsewhere, then +% the value is struct() instead of NaN. +if ~isfield(copy_param.layer_dest,'age_source') + copy_param.layer_dest.age_source = {}; +end +if iscell(copy_param.layer_dest.age_source) + for idx = 1:length(copy_param.layer_dest.age_source) + if ~isstruct(copy_param.layer_dest.age_source{idx}) + error('copy_param.layer_dest.age_source{%d} must be 1) a struct array or 2) an empty matrix [].', idx); + end + end +else + if ~isempty(copy_param.layer_dest.age_source) && ~isstruct(copy_param.layer_dest.age_source) + error('copy_param.layer_dest.age_source must be 1) a cell array of structure arrays, 2) a structure array, 3) an empty numeric matrix [], or 4) undefined.'); + end +end + +% copy_param.layer_dest.group_name: Same as .age field except instead of +% numeric values it is a string. If not defined elsewhere, then +% the value is 'standard' instead of NaN. +if ~isfield(copy_param.layer_dest,'group_name') + copy_param.layer_dest.group_name = {}; +end +if iscell(copy_param.layer_dest.group_name) + for idx = 1:length(copy_param.layer_dest.group_name) + if ~ischar(copy_param.layer_dest.group_name{idx}) + error('copy_param.layer_dest.group_name{%d} must be 1) a string or 2) an empty matrix [].', idx); + end + end +else + if ~isempty(copy_param.layer_dest.group_name) && ~ischar(copy_param.layer_dest.group_name) + error('copy_param.layer_dest.group_name must be 1) a cell array of strings, 2) a string, 3) an empty numeric matrix [], or 4) undefined.'); + end +end + +% copy_param.layer_dest.desc: Same as .age field except instead of +% numeric values it is a string. If not defined elsewhere, then +% the value is '' (empty string) instead of NaN. +if ~isfield(copy_param.layer_dest,'desc') + copy_param.layer_dest.desc = {}; +end +if iscell(copy_param.layer_dest.desc) + for idx = 1:length(copy_param.layer_dest.desc) + if ~ischar(copy_param.layer_dest.desc{idx}) + error('copy_param.layer_dest.desc{%d} must be 1) a string or 2) an empty matrix [].', idx); + end + end +else + if ~isempty(copy_param.layer_dest.desc) && ~ischar(copy_param.layer_dest.desc) + error('copy_param.layer_dest.desc must be 1) a cell array of strings, 2) a string, 3) an empty numeric matrix [], or 4) undefined.'); + end +end + if ~isfield(copy_param.layer_dest,'layerdata_source') || isempty(copy_param.layer_dest.layerdata_source) - % Default is a combined file Data_YYYYMMDD_SS.mat + % Default is the CSARP_layer directory copy_param.layer_dest.layerdata_source = 'layer'; end +% For the opsLoadLayers to write the files if they do not already exist +copy_param.layer_dest.read_only = false; + if ~isfield(copy_param.layer_source,'layerdata_source') || isempty(copy_param.layer_source.layerdata_source) - % Default is a combined file Data_YYYYMMDD_SS.mat + % Default is the CSARP_layer directory copy_param.layer_source.layerdata_source = 'layer'; end @@ -148,44 +288,56 @@ %% Load "frames" file frames = frames_load(param); - -%% Determine which frames to be processed -if isempty(param.cmd.frms) - param.cmd.frms = 1:length(frames.frame_idxs); -end -% Remove frames that do not exist from param.cmd.frms list -[valid_frms,keep_idxs] = intersect(param.cmd.frms, 1:length(frames.frame_idxs)); -if length(valid_frms) ~= length(param.cmd.frms) - bad_mask = ones(size(param.cmd.frms)); - bad_mask(keep_idxs) = 0; - warning('Nonexistent frames specified in param.cmd.frms (e.g. frame "%g" is invalid), removing these', ... - param.cmd.frms(find(bad_mask,1))); - param.cmd.frms = valid_frms; -end +param.cmd.frms = frames_param_cmd_frms(param,frames); %% Load source layer data and existing destination layer data for this segment % Load all frames (for better edge interpolation) load_param = param; load_param.cmd.frms = max(1,min(param.cmd.frms)-1) : min(length(frames.frame_idxs),max(param.cmd.frms)+1); +layer_dest = opsLoadLayers(load_param,copy_param.layer_dest); if strcmpi(copy_param.layer_source.source,'custom') - layer_source.gps_time = copy_param.layer_source.gps_time; - layer_source.twtt = copy_param.layer_source.twtt; - if isfield(copy_param.layer_source,'type') ... - && ~isempty(copy_param.layer_source.type) - layer_source.type = copy_param.layer_source.type; - else - layer_source.type = 2*ones(size(layer_source.gps_time)); - end - if isfield(copy_param.layer_source,'quality') ... - && ~isempty(copy_param.layer_source.quality) - layer_source.quality = copy_param.layer_source.quality; - else - layer_source.quality = ones(size(layer_source.gps_time)); + layer_source = []; + for layer_idx = 1:length(layer_dest) + % gps_time copy + if ~iscell(copy_param.layer_source.gps_time) + copy_param.layer_source.gps_time = {copy_param.layer_source.gps_time}; + end + layer_source(layer_idx).gps_time = copy_param.layer_source.gps_time{layer_idx}; + % twtt copy + if ~iscell(copy_param.layer_source.twtt) + copy_param.layer_source.twtt = {copy_param.layer_source.twtt}; + end + layer_source(layer_idx).twtt = copy_param.layer_source.twtt{layer_idx}; + % type + if isfield(copy_param.layer_source,'type') ... + && ~isempty(copy_param.layer_source.type) + if ~iscell(copy_param.layer_source.type) + copy_param.layer_source.type = {copy_param.layer_source.type}; + end + layer_source(layer_idx).type = copy_param.layer_source.type{layer_idx}; + else + layer_source(layer_idx).type = 2*ones(size(layer_source(layer_idx).gps_time)); + end + % quality + if isfield(copy_param.layer_source,'quality') ... + && ~isempty(copy_param.layer_source.quality) + if ~iscell(copy_param.layer_source.quality) + copy_param.layer_source.quality = {copy_param.layer_source.quality}; + end + layer_source(layer_idx).quality = copy_param.layer_source.quality{layer_idx}; + else + layer_source(layer_idx).quality = ones(size(layer_source(layer_idx).gps_time)); + end + % age, age_source, desc, group_name not set for custom sources, these + % can be set using the corresponding destination override fields + layer_source(layer_idx).age = []; + layer_source(layer_idx).age_source = []; + layer_source(layer_idx).desc = []; + layer_source(layer_idx).group_name = []; end else layer_source = opsLoadLayers(load_param,copy_param.layer_source); end -layer_dest = opsLoadLayers(load_param,copy_param.layer_dest); %% Load framing information (to determine start/stop gps times of each frame) if strcmpi(copy_param.layer_dest.source,'ops') @@ -258,6 +410,7 @@ % .ids: only if ops is the destination (database point IDs) % .twtt: the current twtt for each point (NaN means does not exist) % .twtt_interp: the new twtt for each point +all_points = []; if strcmpi(copy_param.layer_dest.source,'ops') % OPS source only returns points with picks from opsLoadLayers. So to get the % points that do not have picks, we have to load the flight lines. @@ -269,237 +422,410 @@ ops_param.properties.nativeGeom = true; %ops_param.properties.lyr_id = ops_data.properties.id; [~,ops_data] = opsGetPath(sys,ops_param); - - all_points.gps_time = ops_data.properties.gps_time; - all_points.lat = ops_data.properties.Y; - all_points.lon = ops_data.properties.X; - all_points.elev = ops_data.properties.elev; - - % Runs through each data point. At each ops_data point id, if there is - % no layer_dest point id that matches it, that point is assigned NaN - all_points.ids = ops_data.properties.id; - all_points.twtt = zeros(size(ops_data.properties.id)); - - for point_idx = 1:length(all_points.ids) - match_idx = find(all_points.ids(point_idx) == layer_dest.point_path_id); - if isempty(match_idx) - all_points.twtt(point_idx) = NaN; - all_points.quality(point_idx) = 1; - all_points.type(point_idx) = 2; - else - all_points.twtt(point_idx) = layer_dest.twtt(match_idx); - all_points.quality(point_idx) = layer_dest.quality(match_idx); - all_points.type(point_idx) = layer_dest.type(match_idx); + + for layer_idx = 1:length(layer_dest) + all_points(layer_idx).gps_time = ops_data.properties.gps_time; + all_points(layer_idx).lat = ops_data.properties.Y; + all_points(layer_idx).lon = ops_data.properties.X; + all_points(layer_idx).elev = ops_data.properties.elev; + + % Runs through each data point. At each ops_data point id, if there is + % no layer_dest point id that matches it, that point is assigned NaN + all_points(layer_idx).ids = ops_data.properties.id; + all_points(layer_idx).twtt = zeros(size(ops_data.properties.id)); + + for point_idx = 1:length(all_points(layer_idx).ids) + match_idx = find(all_points(layer_idx).ids(point_idx) == layer_dest(layer_idx).point_path_id); + if isempty(match_idx) + all_points(layer_idx).twtt(point_idx) = NaN; + all_points(layer_idx).quality(point_idx) = 1; + all_points(layer_idx).type(point_idx) = 2; + else + all_points(layer_idx).twtt(point_idx) = layer_dest(layer_idx).twtt(match_idx); + all_points(layer_idx).quality(point_idx) = layer_dest(layer_idx).quality(match_idx); + all_points(layer_idx).type(point_idx) = layer_dest(layer_idx).type(match_idx); + end end end - + else % Non-OPS sources automatically include all points since NaN are used to % fill in missing points - all_points.gps_time = layer_dest.gps_time; - all_points.lat = layer_dest.lat; - all_points.lon = layer_dest.lon; - all_points.elev = layer_dest.elev; - all_points.ids = layer_dest.point_path_id; - all_points.type = layer_dest.type; - all_points.twtt = layer_dest.twtt; - all_points.quality = layer_dest.quality; + for layer_idx = 1:length(layer_dest) + all_points(layer_idx).gps_time = layer_dest(layer_idx).gps_time; + all_points(layer_idx).lat = layer_dest(layer_idx).lat; + all_points(layer_idx).lon = layer_dest(layer_idx).lon; + all_points(layer_idx).elev = layer_dest(layer_idx).elev; + all_points(layer_idx).ids = layer_dest(layer_idx).point_path_id; + all_points(layer_idx).type = layer_dest(layer_idx).type; + all_points(layer_idx).twtt = layer_dest(layer_idx).twtt; + all_points(layer_idx).quality = layer_dest(layer_idx).quality; + end end % Check to make sure there are any points to copy -if isempty(all_points.gps_time) - error('Destination has no GPS time points. No work to do. This might be an error caused by echogram destination source not finding any echograms.'); +if any(cellfun(@isempty,{all_points.gps_time})) + error('Destination layer(s) have no GPS time points. No work to do. This might be an error caused by echogram destination source not finding any echograms.'); end -% Set invalid types to "auto" or 2 -layer_source.type(~isfinite(layer_source.type)) = 2; -% Set invalid quality levels to "good" or 1 -layer_source.quality(~isfinite(layer_source.quality)) = 1; -layer_source.quality(layer_source.quality ~= 1 & layer_source.quality ~= 2 & layer_source.quality ~= 3) = 1; +for layer_idx = 1:length(layer_source) + % Set invalid types to "auto" or 2 + layer_source(layer_idx).type(~isfinite(layer_source(layer_idx).type)) = 2; + % Set invalid quality levels to "good" or 1 + layer_source(layer_idx).quality(~isfinite(layer_source(layer_idx).quality)) = 1; + layer_source(layer_idx).quality(layer_source(layer_idx).quality ~= 1 & layer_source(layer_idx).quality ~= 2 & layer_source(layer_idx).quality ~= 3) = 1; +end %% Apply evaluation operation to source if isfield(copy_param,'eval') && ~isempty(copy_param.eval) - source = layer_source.twtt; - gps_time = layer_source.gps_time; - lat = layer_source.lat; - lon = layer_source.lon; - elev = layer_source.elev; - along_track = geodetic_to_along_track(lat,lon,elev); - eval_struct = copy_param.eval; - eval(copy_param.eval.cmd); - layer_source.twtt = source; + for layer_idx = 1:length(layer_source) + s = layer_source(layer_idx).twtt; + time = layer_source(layer_idx).gps_time; + lat = layer_source(layer_idx).lat; + lon = layer_source(layer_idx).lon; + elev = layer_source(layer_idx).elev; + at = geodetic_to_along_track(lat,lon,elev); + eval_struct = copy_param.eval; + eval(copy_param.eval.cmd); + layer_source(layer_idx).twtt = s; + end end %% Interpolate two way travel time (twtt) from layer_source to all_points if strcmpi(copy_param.gaps_fill.method,'preserve_gaps') % interpolation preserves_gaps which preserves gaps in the data - master = []; - master.GPS_time = all_points.gps_time; - master.Latitude = all_points.lat; - master.Longitude = all_points.lon; - master.Elevation = all_points.elev; - ops_layer = []; - ops_layer{1}.gps_time = layer_source.gps_time; - ops_layer{1}.type = layer_source.type; - ops_layer{1}.quality = layer_source.quality; - ops_layer{1}.twtt = layer_source.twtt; - if length(master.GPS_time) < 2 - error('Too few points in destination to interpolate onto.'); + for layer_idx = 1:length(layer_source) + master = []; + master.GPS_time = all_points(layer_idx).gps_time; + master.Latitude = all_points(layer_idx).lat; + master.Longitude = all_points(layer_idx).lon; + master.Elevation = all_points(layer_idx).elev; + ops_layer = []; + ops_layer{1}.gps_time = layer_source(layer_idx).gps_time; + ops_layer{1}.type = layer_source(layer_idx).type; + ops_layer{1}.quality = layer_source(layer_idx).quality; + ops_layer{1}.twtt = layer_source(layer_idx).twtt; + if length(master.GPS_time) < 2 + error('Too few points in destination to interpolate onto.'); + end + lay = opsInterpLayersToMasterGPSTime(master,ops_layer,copy_param.gaps_fill.method_args); + all_points(layer_idx).twtt_interp = lay.layerData{1}.value{2}.data; + % All points automated (type 2) by default + all_points(layer_idx).type_interp = 2*ones(size(all_points(layer_idx).twtt_interp)); + % Overwrite manual points (type 1) + manual_mask = isfinite(lay.layerData{1}.value{1}.data); + all_points(layer_idx).type_interp(manual_mask) = 1; + all_points(layer_idx).twtt_interp(manual_mask) = lay.layerData{1}.value{1}.data(manual_mask); end - lay = opsInterpLayersToMasterGPSTime(master,ops_layer,copy_param.gaps_fill.method_args); - all_points.twtt_interp = lay.layerData{1}.value{2}.data; - % All points automated (type 2) by default - all_points.type_interp = 2*ones(size(all_points.twtt_interp)); - % Overwrite manual points (type 1) - manual_mask = isfinite(lay.layerData{1}.value{1}.data); - all_points.type_interp(manual_mask) = 1; - all_points.twtt_interp(manual_mask) = lay.layerData{1}.value{1}.data(manual_mask); elseif strcmpi(copy_param.gaps_fill.method,'interp_finite') % interpolation interp_finite which interpolates through gaps - % Interpolate source onto destination points using linear interpolation - if length(layer_source.gps_time) >= 2 - all_points.twtt_interp = interp1(layer_source.gps_time, layer_source.twtt, all_points.gps_time); - else - all_points.twtt_interp = NaN(size(all_points.gps_time)); + for layer_idx = 1:length(layer_source) + % Interpolate source onto destination points using linear interpolation + if length(layer_source(layer_idx).gps_time) >= 2 + all_points(layer_idx).twtt_interp = interp1(layer_source(layer_idx).gps_time, layer_source(layer_idx).twtt, all_points(layer_idx).gps_time); + else + all_points(layer_idx).twtt_interp = NaN(size(all_points(layer_idx).gps_time)); + end + % Fill in NaN gaps using interp_finite (default is 0 twtt if no good data + % exists) + all_points(layer_idx).twtt_interp = interp_finite(all_points(layer_idx).twtt_interp,0); + + % All points automated (type 2) by default + all_points(layer_idx).type_interp = 2*ones(size(all_points(layer_idx).twtt_interp)); + % Overwrite manual points (type 1) + manual_idxs = find(layer_source(layer_idx).type == 1); + manual_idxs_map = interp1_robust(all_points(layer_idx).gps_time, 1:length(all_points(layer_idx).gps_time), ... + layer_source(layer_idx).gps_time(manual_idxs),'nearest'); + manual_idxs = manual_idxs(~isnan(manual_idxs_map)); + manual_idxs_map = manual_idxs_map(~isnan(manual_idxs_map)); + all_points(layer_idx).twtt_interp(manual_idxs_map) ... + = layer_source(layer_idx).twtt(manual_idxs); + all_points(layer_idx).type_interp(manual_idxs_map) = 1; end - % Fill in NaN gaps using interp_finite (default is 0 twtt if no good data - % exists) - all_points.twtt_interp = interp_finite(all_points.twtt_interp,0); - - % All points automated (type 2) by default - all_points.type_interp = 2*ones(size(all_points.twtt_interp)); - % Overwrite manual points (type 1) - manual_idxs = find(layer_source.type == 1); - manual_idxs_map = interp1_robust(all_points.gps_time, 1:length(all_points.gps_time), ... - layer_source.gps_time(manual_idxs),'nearest'); - manual_idxs = manual_idxs(~isnan(manual_idxs_map)); - manual_idxs_map = manual_idxs_map(~isnan(manual_idxs_map)); - all_points.twtt_interp(manual_idxs_map) ... - = layer_source.twtt(manual_idxs); - all_points.type_interp(manual_idxs_map) = 1; end -%% Combine the newly interpolated result with the current values -if strcmpi(copy_param.copy_method,'fillgaps') - update_mask = isnan(all_points.twtt) & ~isnan(all_points.twtt_interp); -elseif strcmpi(copy_param.copy_method,'merge') - update_mask = ~isnan(all_points.twtt_interp); -elseif strcmpi(copy_param.copy_method,'overwrite') - update_mask = logical(ones(size(all_points.twtt))); -end - -%% For nonselected frames, set update_mask to zero for all points in those frames -all_frms = 1:length(frames.frame_idxs); -% If the user selects all frames in the segment on the params -% spreadsheet, then the update mask will not change. -% Otherwise we will create a new mask where each point within the start -% and stop gps times of the selected frames is equal to 1 and all other -% points are equal to 0. -% -% We divide up the segment into frames so that we avoid rounding issues and -% to every point is included in a frame. -frms_mask = zeros(size(update_mask)); -if strcmpi(copy_param.layer_dest.source,'ops') - for frm = param.cmd.frms - if frm == 1 - if frm == length(ops_frames.properties.start_gps_time) - frms_mask(:) = 1; +update_mask = {}; +for layer_idx = 1:length(layer_source) + + %% Combine the newly interpolated result with the current values + if strcmpi(copy_param.copy_method,'fillgaps') + update_mask{layer_idx} = isnan(all_points(layer_idx).twtt) & ~isnan(all_points(layer_idx).twtt_interp); + elseif strcmpi(copy_param.copy_method,'merge') + update_mask{layer_idx} = ~isnan(all_points(layer_idx).twtt_interp); + elseif strcmpi(copy_param.copy_method,'overwrite') + update_mask{layer_idx} = logical(ones(size(all_points(layer_idx).twtt))); + end + + %% For nonselected frames, set update_mask{layer_idx} to zero for all points in those frames + all_frms = 1:length(frames.frame_idxs); + % If the user selects all frames in the segment on the params + % spreadsheet, then the update mask will not change. + % Otherwise we will create a new mask where each point within the start + % and stop gps times of the selected frames is equal to 1 and all other + % points are equal to 0. + % + % We divide up the segment into frames so that we avoid rounding issues and + % to every point is included in a frame. + frms_mask = zeros(size(update_mask{layer_idx})); + if strcmpi(copy_param.layer_dest.source,'ops') + for frm = param.cmd.frms + if frm == 1 + if frm == length(ops_frames.properties.start_gps_time) + frms_mask(:) = 1; + else + frms_mask(all_points(layer_idx).gps_time < ops_frames.properties.start_gps_time(frm+1)) = 1; + end + elseif frm < length(ops_frames.properties.start_gps_time) + frms_mask(all_points(layer_idx).gps_time >= ops_frames.properties.start_gps_time(frm)... + & all_points(layer_idx).gps_time < ops_frames.properties.start_gps_time(frm+1)) = 1; else - frms_mask(all_points.gps_time < ops_frames.properties.start_gps_time(frm+1)) = 1; + frms_mask(all_points(layer_idx).gps_time >= ops_frames.properties.start_gps_time(frm)) = 1; end - elseif frm < length(ops_frames.properties.start_gps_time) - frms_mask(all_points.gps_time >= ops_frames.properties.start_gps_time(frm)... - & all_points.gps_time < ops_frames.properties.start_gps_time(frm+1)) = 1; - else - frms_mask(all_points.gps_time >= ops_frames.properties.start_gps_time(frm)) = 1; end - end - -else - for frm = param.cmd.frms - if frm == 1 - if frm == length(frames.frame_idxs) - frms_mask(:) = 1; + + else + for frm = param.cmd.frms + if frm == 1 + if frm == length(frames.frame_idxs) + frms_mask(:) = 1; + else + frms_mask(all_points(layer_idx).gps_time < frames.gps_time(frm+1)) = 1; + end + elseif frm < length(frames.frame_idxs) + frms_mask(all_points(layer_idx).gps_time >= frames.gps_time(frm)... + & all_points(layer_idx).gps_time < frames.gps_time(frm+1)) = 1; else - frms_mask(all_points.gps_time < frames.gps_time(frm+1)) = 1; + frms_mask(all_points(layer_idx).gps_time >= frames.gps_time(frm)) = 1; end - elseif frm < length(frames.frame_idxs) - frms_mask(all_points.gps_time >= frames.gps_time(frm)... - & all_points.gps_time < frames.gps_time(frm+1)) = 1; - else - frms_mask(all_points.gps_time >= frames.gps_time(frm)) = 1; end end -end - -update_mask = frms_mask & update_mask; - -%% Overwrite quality level -if isempty(layer_source.gps_time) - all_points.quality_interp = NaN*zeros(size(all_points.gps_time)); -elseif length(layer_source.gps_time) == 1 - all_points.quality_interp = layer_source.quality; -else - all_points.quality_interp = interp1(layer_source.gps_time,layer_source.quality,all_points.gps_time,'nearest'); -end -if strcmpi(copy_param.quality.mode,'overwrite') - all_points.quality_interp(:) = copy_param.quality.value; -end - -%% Write the new layer data to the destination -surface = all_points.twtt; -surface(update_mask) = all_points.twtt_interp(update_mask); -quality = all_points.quality; -quality(update_mask) = all_points.quality_interp(update_mask); -layer_type = all_points.type; -layer_type(update_mask) = all_points.type_interp(update_mask); - -if 0 - % Debug code - figure(1); clf; - plot(all_points.twtt); - hold on; - plot(surface(update_mask),'rx'); - grid on; - hold off; - drawnow; - pause(1); + + update_mask{layer_idx} = frms_mask & update_mask{layer_idx}; + + %% Quality level overwrite + if isempty(layer_source(layer_idx).gps_time) + all_points(layer_idx).quality_interp = NaN*zeros(size(all_points(layer_idx).gps_time)); + elseif length(layer_source(layer_idx).gps_time) == 1 + all_points(layer_idx).quality_interp = layer_source(layer_idx).quality; + else + all_points(layer_idx).quality_interp = interp1(layer_source(layer_idx).gps_time,layer_source(layer_idx).quality,all_points(layer_idx).gps_time,'nearest'); + end + if strcmpi(copy_param.quality.mode,'overwrite') + all_points(layer_idx).quality_interp(:) = copy_param.quality.value; + end + + %% Mask the output + all_points(layer_idx).twtt_final = all_points(layer_idx).twtt; + all_points(layer_idx).twtt_final(update_mask{layer_idx}) = all_points(layer_idx).twtt_interp(update_mask{layer_idx}); + all_points(layer_idx).quality_final = all_points(layer_idx).quality; + all_points(layer_idx).quality_final(update_mask{layer_idx}) = all_points(layer_idx).quality_interp(update_mask{layer_idx}); + all_points(layer_idx).type_final = all_points(layer_idx).type; + all_points(layer_idx).type_final(update_mask{layer_idx}) = all_points(layer_idx).type_interp(update_mask{layer_idx}); + + if 0 + % Debug code + figure(1); clf; + plot(all_points(layer_idx).twtt); + hold on; + plot(surface(update_mask{layer_idx}),'rx'); + grid on; + hold off; + drawnow; + pause(1); + end + end if strcmpi(copy_param.layer_dest.source,'ops') - %% Check to see if layer exists + %% Save OPS + + % Check to see if layer exists + % ----------------------------------------------------------------------- [status,data] = opsGetLayers(sys); - if ~any(strcmpi(data.properties.lyr_name,copy_param.layer_dest.name)) - % Create the layer if it does not exist - ops_param = []; - ops_param.properties.lyr_name = copy_param.layer_dest.name; - ops_param.properties.lyr_group_name = copy_param.layer_dest.group; - ops_param.properties.lyr_description = copy_param.layer_dest.description; - ops_param.properties.public = true; + for layer_idx = 1:length(layer_dest) + if ~any(strcmpi(data.properties.lyr_name,layer_dest(layer_idx).name)) + % Create the layer if it does not exist + ops_param = []; + ops_param.properties.lyr_name = layer_dest(layer_idx).name; + + if ischar(copy_param.layer_dest.group_name) + ops_param.properties.lyr_group_name = copy_param.layer_dest.group_name; + + elseif length(copy_param.layer_dest.group_name) >= layer_idx && ischar(copy_param.layer_dest.group_name{layer_idx}) + ops_param.properties.lyr_group_name = copy_param.layer_dest.group_name{layer_idx}; + + elseif ischar(layer_dest(layer_idx).group_name) + ops_param.properties.lyr_group_name = layer_dest(layer_idx).group_name; + + elseif ischar(layer_source(layer_idx).group_name) + ops_param.properties.lyr_group_name = layer_source(layer_idx).group_name; + + else + ops_param.properties.lyr_group_name = 'standard'; + end + + if ischar(copy_param.layer_dest.desc) + ops_param.properties.lyr_description = copy_param.layer_dest.desc; + + elseif length(copy_param.layer_dest.desc) >= layer_idx && ischar(copy_param.layer_dest.desc{layer_idx}) + ops_param.properties.lyr_description = copy_param.layer_dest.desc{layer_idx}; + + elseif ischar(layer_dest(layer_idx).group_name) + ops_param.properties.lyr_description = layer_dest(layer_idx).desc; + + elseif ischar(layer_source(layer_idx).desc) + ops_param.properties.lyr_description = layer_source(layer_idx).desc; + + else + ops_param.properties.lyr_description = 'standard'; + end + + ops_param.properties.public = true; + + [status,ops_data] = opsCreateLayer(sys,ops_param); + end + + % Remove all_points that are not in the selected frames + % Use update_mask to exclude all points that are not getting updated + % ----------------------------------------------------------------------- + ops_param.properties.point_path_id = all_points(layer_idx).ids(update_mask{layer_idx}); + ops_param.properties.twtt = all_points(layer_idx).twtt_final(update_mask{layer_idx}); + ops_param.properties.type = all_points(layer_idx).type_final(update_mask{layer_idx}); + ops_param.properties.quality = all_points(layer_idx).quality_final(update_mask{layer_idx}); + ops_param.properties.lyr_name = layer_dest(layer_idx).name; - [status,ops_data] = opsCreateLayer(sys,ops_param); + % Update these points + % ----------------------------------------------------------------------- + opsCreateLayerPoints(sys,ops_param); end - %% Remove all_points that are not in the selected frames - % Use update_mask to exclude all points that are not getting updated - ops_param.properties.point_path_id = all_points.ids(update_mask); - ops_param.properties.twtt = surface(update_mask); - ops_param.properties.type = layer_type(update_mask); - ops_param.properties.quality = quality(update_mask); - ops_param.properties.lyr_name = copy_param.layer_dest.name; - - %% Update these points - opsCreateLayerPoints(sys,ops_param); - elseif strcmpi(copy_param.layer_dest.source,'layerdata') + %% Save layerdata layers = layerdata(param, copy_param.layer_dest.layerdata_source); - id = layers.get_id(copy_param.layer_dest.name); - if isempty(id) && copy_param.layer_dest.existence_check - error('Reference layer %s not found in layer organizer file %s. Set copy_param.layer_dest.existence_check to false to ignore this error.\n', copy_param.layer_dest.name, layers.layer_organizer_fn()); + for layer_idx = 1:length(layer_dest) + id = layers.get_id(layer_dest(layer_idx).name); + if isempty(id) + if copy_param.layer_dest.existence_check + error('Layer %s not found in layer organizer file %s. Set copy_param.layer_dest.existence_check to false to ignore this error and opsCopyLayers will create layers that do not exist already.\n', layer_dest(layer_idx).name, layers.layer_organizer_fn()); + else + layer_organizer = []; + + if ischar(copy_param.layer_dest.group_name) + layer_organizer.lyr_group_name = {copy_param.layer_dest.group_name}; + + elseif length(copy_param.layer_dest.group_name) >= layer_idx && ischar(copy_param.layer_dest.group_name{layer_idx}) + layer_organizer.lyr_group_name = {copy_param.layer_dest.group_name{layer_idx}}; + + elseif ischar(layer_dest(layer_idx).group_name) + layer_organizer.lyr_group_name = {layer_dest(layer_idx).group_name}; + + elseif ischar(layer_source(layer_idx).group_name) + layer_organizer.lyr_group_name = {layer_source(layer_idx).group_name}; + + else + ops_param.properties.lyr_group_name = 'standard'; + end + + layer_organizer.lyr_name = {layer_dest(layer_idx).name}; + id = layers.insert_layers(layer_organizer); + end + end + % Update "desc" field + if ischar(copy_param.layer_dest.desc) + layers.set_desc(id,copy_param.layer_dest.desc); + + elseif length(copy_param.layer_dest.desc) >= layer_idx && ischar(copy_param.layer_dest.desc{layer_idx}) + layers.set_desc(id,copy_param.layer_dest.desc{layer_idx}); + + elseif ischar(layer_dest(layer_idx).desc) + layers.set_desc(id,layer_dest(layer_idx).desc); + + elseif ischar(layer_source(layer_idx).desc) + layers.set_desc(id,layer_source(layer_idx).desc); + + else + layers.set_desc(id,NaN); + end + % Update "age" field + if isnumeric(copy_param.layer_dest.age) + layers.set_age(id,copy_param.layer_dest.age); + + elseif length(copy_param.layer_dest.age) >= layer_idx && isnumeric(copy_param.layer_dest.age{layer_idx}) + layers.set_age(id,copy_param.layer_dest.age{layer_idx}); + + elseif isnumeric(layer_dest(layer_idx).age) + layers.set_age(id,layer_dest(layer_idx).age); + + elseif isnumeric(layer_source(layer_idx).age) + layers.set_age(id,layer_source(layer_idx).age); + + else + layers.set_age(id,'standard'); + end + % Update "age_source" field + if isstruct(copy_param.layer_dest.age_source) + layers.set_age_source(id,copy_param.layer_dest.age_source); + + elseif length(copy_param.layer_dest.age_source) >= layer_idx && isstruct(copy_param.layer_dest.age_source{layer_idx}) + layers.set_age_source(id,copy_param.layer_dest.age_source{layer_idx}); + + elseif isstruct(layer_dest(layer_idx).age_source) + layers.set_age_source(id,layer_dest(layer_idx).age_source); + + elseif isstruct(layer_source(layer_idx).age_source) + layers.set_age_source(id,layer_source(layer_idx).age_source); + + else + layers.set_age_source(id,struct()); + end + + % Update layer vectors + layers.update_layer(param.cmd.frms, id, all_points(layer_idx).gps_time, ... + all_points(layer_idx).twtt_final,all_points(layer_idx).quality_final,all_points(layer_idx).type_final); end - layers.update_layer(param.cmd.frms, id, all_points.gps_time,surface,quality,layer_type); layers.save(); +elseif strcmpi(copy_param.layer_dest.source,'echogram') + %% Save echogram + for frm = param.cmd.frms + % Create file name for each frame + if copy_param.layer_source.echogram_source_img == 0 + echo_fn = fullfile(ct_filename_out(param,copy_param.layer_dest.echogram_source,''), ... + sprintf('Data_%s_%03d.mat', param.day_seg, frm)); + else + echo_fn = fullfile(ct_filename_out(param,copy_param.layer_dest.echogram_source,''), ... + sprintf('Data_img_%02d_%s_%03d.mat', copy_param.layer_source.echogram_source_img, param.day_seg, frm)); + end + if ~exist(echo_fn,'file') + warning('Echogram data file does not exist: %s\n', echo_fn); + continue; + end + + % Load echogram + echo = load(echo_fn,'GPS_time'); + + layers = layerdata(param, copy_param.layer_dest.layerdata_source); + for layer_idx = 1:length(layer_dest) + + % Create automated points: + if strcmpi(copy_param.layer_dest.name,'surface') + echo.Surface = interp1(all_points(layer_idx).gps_time,all_points(layer_idx).twtt_final,echo.GPS_time); + % Append the new results back to the echogram file + ct_save(echo_fn,'-append','-struct','echo','Surface'); + elseif strcmpi(copy_param.layer_dest.name,'bottom') + echo.Bottom = interp1(all_points(layer_idx).gps_time,all_points(layer_idx).twtt_final,echo.GPS_time); + % Append the new results back to the echogram file + ct_save(echo_fn,'-append','-struct','echo','Bottom'); + else + echo.(copy_param.layer_dest.name) = interp1(all_points(layer_idx).gps_time,all_points(layer_idx).twtt_final,echo.GPS_time); + % Append the new results back to the echogram file + ct_save(echo_fn,'-append','-struct','echo',copy_param.layer_dest.name); + end + + end + + end + end diff --git a/cresis-toolbox/OPS-MATLAB/utility/opsInterpLayersToMasterGPSTime.m b/cresis-toolbox/OPS-MATLAB/utility/opsInterpLayersToMasterGPSTime.m index b88eabd1..056391e2 100644 --- a/cresis-toolbox/OPS-MATLAB/utility/opsInterpLayersToMasterGPSTime.m +++ b/cresis-toolbox/OPS-MATLAB/utility/opsInterpLayersToMasterGPSTime.m @@ -70,13 +70,26 @@ tie_idx = find(slave >= lay.GPS_time(1) ... & slave <= lay.GPS_time(end),2); % Interpolate the points onto the echogram GPS time - if length(tie_idx) < 2 + if isempty(tie_idx) % Not enough points to interpolate, so set to all NaN lay.layerData{layer_idx}.value{2}.data ... = NaN*zeros(size(lay.GPS_time)); lay.layerData{layer_idx}.quality ... = NaN*zeros(size(lay.GPS_time)); + elseif length(tie_idx) == 1 + lay.layerData{layer_idx}.value{2}.data ... + = interp1(ops_layer{layer_idx}.gps_time(auto_idxs), ... + ops_layer{layer_idx}.twtt(auto_idxs), lay.GPS_time, 'nearest'); + lay.layerData{layer_idx}.quality ... + = interp1(ops_layer{layer_idx}.gps_time(auto_idxs), ... + ops_layer{layer_idx}.quality(auto_idxs), lay.GPS_time, 'nearest'); + slave_along_track = interp1(lay.GPS_time, master_along_track, slave, 'linear', 'extrap'); + gap_data_idxs = data_gaps_check_mex(master_along_track, slave_along_track, ... + gaps_dist(1), gaps_dist(2)); + lay.layerData{layer_idx}.value{2}.data(gap_data_idxs) = NaN; + lay.layerData{layer_idx}.quality(gap_data_idxs) = NaN; + else lay.layerData{layer_idx}.value{2}.data ... = interp1(ops_layer{layer_idx}.gps_time(auto_idxs), ... diff --git a/cresis-toolbox/OPS-MATLAB/utility/runOpsBulkInsert.m b/cresis-toolbox/OPS-MATLAB/utility/runOpsBulkInsert.m index 2bcb29f9..98a42b89 100644 --- a/cresis-toolbox/OPS-MATLAB/utility/runOpsBulkInsert.m +++ b/cresis-toolbox/OPS-MATLAB/utility/runOpsBulkInsert.m @@ -1,129 +1,40 @@ -% ========================================================================= -% OPS BULK DATA LOADER +% script runOpsBulkInsert % -% Loads data in bulk from the CReSIS filesystem to the OPS. +% Loads flight path data in bulk from the CReSIS filesystem (records and +% frames files) to the OPS. % % To use this script manually edit the input fileds under USER INPUT. % % Data inputs required: % path: records and frames files -% layerData: layerData files or echogram files -% atm: atm L2 nadir interpolated files % % Multi-Layer data is supported, the layerData files must contain a name % field with a unique name identifying each layer. % % Authors: Kyle W. Purdon, Trey Stafford, John Paden % -% see also opsBulkInsert.m opsCreatePath.m opsCreaLayerPoints.m layerDataToOps.m -% -% ========================================================================= +% See also: opsBulkInsert.m, opsCreatePath.m %% USER INPUT % Initialize settings structure settings = []; -% ---------------------------------------------------------------- -% runType: THE TYPE OF DATA INSERT TO COMPLETE -% 1: INSERTS PATHS ONLY -% 2: INSERTS LAYERS ONLY -% 3: INSERT ATM LAYERS ONLY -% 4: (1) AND (2) -% 5: (1), (2), AND (3) -settings.runType = 4; - % ---------------------------------------------------------------- % paramFn: FILENAME.xls OF EXCEL CReSIS PARAMS SHEET -settings.paramFn = 'rds_param_2018_Antarctica_DC8.xls'; +settings.params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls')); -% ---------------------------------------------------------------- -% location: LOCATION NAME ('arctic' OR 'antarctic') -% settings.location = 'arctic'; -settings.location = 'antarctic'; +settings.params = ct_set_params(settings.params, 'cmd.generic', 1); +settings.params = ct_set_params(settings.params, 'cmd.generic', 0, 'cmd.notes', 'do not process'); -% ---------------------------------------------------------------- -% sysName: SYSTEM NAME ('rds','snow','accum','kuband') -settings.sysName = 'rds'; - -% ---------------------------------------------------------------- -% layerDataPath: PATH TO LAYERDATA FILES -% 'layerData': normal layerData path -% 'qlook': normal layerData path for using echogram data -% 'fullfile('CSARP_post','layerData')': posted layerData path -% 'fullfile('CSARP_post','qlook')': posted layerData qlook path -settings.layerDataPath = 'layerData'; -% settings.layerDataPath = 'standard'; -% settings.layerDataPath = fullfile('CSARP_post','layerData'); -% settings.layerDataPath = fullfile('CSARP_post','qlook'); - -% ---------------------------------------------------------------- -% layer_name_map: OPTIONAL LAYER NAME REMAPPING -% This allows layers to be renamed when inserted into OPS. -% Normally this should be commented out so that layers are not renamed. -% settings.layer_name_map.source_names = {'surface','bottom'}; -% settings.layer_name_map.dest_names = {'surface_old','bottom_old'}; - -% ---------------------------------------------------------------- -% layerFilter: REGULAR EXPRESSION OF LAYER NAMES TO INSERT -% inline('~isempty(regexp(x,''(^surface$|^bottom$)''))');: only surface and bottom layers -% inline('~isempty(regexp(x,''^lm.*''))');: only layers starting with 'lm' -settings.layerFilter = inline('~isempty(regexp(x,''(^surface$|^bottom$)''))'); -% settings.layerFilter = inline('~isempty(regexp(x,''(^bottom$)''))'); -% settings.layerFilter = inline('~isempty(regexp(x,''^lm.*''))'); - -% ---------------------------------------------------------------- -% layerRelativeSurface: LOGICAL THAT IF TRUE ADDS ALL NEW LAYERS RELATIVE -% TO THE EXISTING SURFACE -settings.layerRelativeSurface = false; - -% ---------------------------------------------------------------- -% gaps_dist: TWO ELEMENTS THAT CONTROL HOW DATA IS INTERPOLATED ACROSS GAPS -% [300 60] is typical, arguments passed to data_gaps_check_mex.cpp -settings.gaps_dist = [300 60]; - -%% OPTIONAL USER INPUT (COMMON DEFAULT PROPERTIES) - -% ---------------------------------------------------------------- -% pathSpacing: DISTANCE IN METERS TO SPACE THE PATH/LAYER POINTS (DEFAULT = 15m) -% settings.pathSpacing = 5; % 5 m for snow and kuband -settings.pathSpacing = 15; % 15 m for accum and rds - -% ---------------------------------------------------------------- -% seasonGroup: STRING name of the group for the season (DEFAULT = 'cresis_private') -% settings.seasonGroup = 'cresis_private'; -settings.seasonGroup = 'cresis_public'; - -% ---------------------------------------------------------------- -% NO LONGER USED. SET seaonGroup to 'cresis_public'. -% autoReleaseSeason: BOOLEAN, SHOULD THE SEASON AUTOMATICALLY BE PUBLIC? (DEFAULT = false) -% settings.autoReleaseSeason = false; - -% ---------------------------------------------------------------- -% logsOn: BOOLEAN, SHOULD THE COMMAND WINDOW BE LOGGED TO A TXT FILE? -settings.logsOn = false; +%settings.params = ct_set_params(settings.params, 'cmd.generic', 0); +%settings.params = ct_set_params(settings.params, 'cmd.generic', 1, 'day_seg', '20120416_01'); %% AUTOMATED SECTION -% ERROR CHECK THE INPUT -if ~any(strcmp(settings.location,{'arctic','antarctic'})); - tmp = questdlg({'Location must be ''arctic'' or ''antarctic''','',sprintf('You entered: %s',settings.location),''},'INVALID LOCATION','OK','OK'); - error('INVALID LOCATION'); -elseif ~any(strcmp(settings.sysName,{'rds','snow','accum','kuband'})); - tmp = questdlg({'System must be ''rds'',''snow'',''accum'' or ''kuband''','',sprintf('You entered: %s',settings.sysName),''},'INVALID SYSTEM','OK','OK'); - error('INVALID SYSTEM'); -end - -% GET THE CReSIS GLOBAL +% GET THE CReSIS GLOBAL settings global gRadar; -if isvarname('gRadar') - settings = mergestruct(settings,gRadar); -else - warning('gRadar IS NOT A GLOBAL VARIABLE.') - fprintf('Type ''dbcont'' to run startup.m and continue\n or ''dbquit'' to fix this yourself ...\n'); - keyboard; - startup -end +settings = merge_structs(gRadar,settings); % RUN OPS BULK INSERT -opsBulkInsert(settings); \ No newline at end of file +opsBulkInsert(settings); diff --git a/cresis-toolbox/OPS-MATLAB/utility/runOpsCopyLayers.m b/cresis-toolbox/OPS-MATLAB/utility/runOpsCopyLayers.m index 92f36309..a87219a0 100644 --- a/cresis-toolbox/OPS-MATLAB/utility/runOpsCopyLayers.m +++ b/cresis-toolbox/OPS-MATLAB/utility/runOpsCopyLayers.m @@ -12,7 +12,7 @@ % Load the parameter spreadsheet params = read_param_xls(ct_filename_param('rds_param_2019_Antarctica_Ground.xls')); params = ct_set_params(params,'cmd.generic',0); -params = ct_set_params(params,'cmd.generic',1,'day_seg','20191230_01'); +params = ct_set_params(params,'cmd.generic',1,'day_seg','20200107_01'); params = ct_set_params(params,'cmd.frms',[1]); % Set the operation to run (just choose one operation) @@ -31,10 +31,10 @@ copy_param.layer_source.existence_check = false; copy_param.layer_dest.existence_check = false; - % Set the layer name for the source (e.g. 'surface', 'bottom') + % Set the layer name(s) for the source (e.g. 'surface', 'bottom', {'surface','bottom'}) copy_param.layer_source.name = 'surface'; - % Set the layer name for the destination (e.g. 'surface', 'bottom') + % Set the layer name(s) for the destination (e.g. 'surface', 'bottom', {'surface','bottom'}) copy_param.layer_dest.name = 'surface'; % Set the source (choose one) @@ -55,13 +55,20 @@ elseif 1 copy_param.layer_source.source = 'layerdata'; copy_param.layer_source.layerdata_source = 'layer'; + elseif 0 + copy_param.layer_source.source = 'custom'; + copy_param.layer_source.gps_time = {[0 1e20]}; + copy_param.layer_source.twtt = {[1e-6 1e-6]}; + copy_param.layer_source.type = {[2 2]}; + copy_param.layer_source.quality = {[1 1]}; else copy_param.layer_source.source = 'lidar'; copy_param.layer_source.lidar_source = 'awi'; + copy_param.layer_source.lever_arm_en = true; end if 1 - copy_param.copy_method = 'overwrite'; + copy_param.copy_method = 'overwrite'; % default elseif 0 copy_param.copy_method = 'fillgaps'; else @@ -84,7 +91,7 @@ if twtt_offset ~= 0 || gps_time_offset ~= 0 warning('You have set a nonzero twtt_offset(%.12g) or gps_time_offset(%.3g). Normally these are both zero. Please verify that this is what you want to do before running "dbcont" to continue.\n', twtt_offset, gps_time_offset); keyboard - copy_param.eval.cmd = sprintf('source = interp1(gps_time+%.3g,source + %.12g,gps_time);',gps_time_offset,twtt_offset); + copy_param.eval.cmd = sprintf('s = interp1(time+%.3g,s + %.12g,time);',gps_time_offset,twtt_offset); end % Set overwrite quality level (e.g. []: do not overwrite, 1: good, 2: medium, 3: bad) @@ -115,6 +122,9 @@ copy_param.layer_dest.source = 'layerdata'; copy_param.layer_dest.layerdata_source = 'layer'; end + + % Usually the group name is standard + copy_param.layer_dest.group_name = 'standard'; end %% Copy surface from mcords records to snow layerdata @@ -154,15 +164,36 @@ %% Automated Section % ===================================================================== -%% Load each of the day segments global gRadar; + +% Input checking +if exist('param_override','var') + param_override = merge_structs(gRadar,param_override); +else + param_override = gRadar; +end + +%% Copy layers for each of the enabled segments +failed_segments = []; for param_idx = 1:length(params) param = params(param_idx); if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic continue; end - param = merge_structs(param,gRadar); - fprintf('opsCopyLayers %s (%s)\n', param.day_seg, datestr(now)); - opsCopyLayers(param,copy_param); + param = merge_structs(param,param_override); + fprintf('opsCopyLayers %s from %s:%s to %s:%s (%s)\n', param.day_seg, copy_param.layer_source.source, copy_param.layer_source.name, copy_param.layer_dest.source, copy_param.layer_dest.name, datestr(now)); + try + opsCopyLayers(param,copy_param); + catch ME + failed_segments(end+1).param_idx = param_idx; + failed_segments(end).report = ME.getReport; + failed_segments(end).message = ME.message; + %keyboard + end fprintf(' Complete (%s)\n', datestr(now)); end + +for failed_idx = 1:length(failed_segments) + fprintf('%s: %s\n', params(failed_segments(failed_idx).param_idx).day_seg, ... + failed_segments(failed_idx).message); +end diff --git a/cresis-toolbox/OPS-MATLAB/utility/run_all_ops_copy_layers.m b/cresis-toolbox/OPS-MATLAB/utility/run_all_ops_copy_layers.m new file mode 100644 index 00000000..4f76dafc --- /dev/null +++ b/cresis-toolbox/OPS-MATLAB/utility/run_all_ops_copy_layers.m @@ -0,0 +1,166 @@ +% script runOpsCopyLayers.m +% +% Example script for running opsCopyLayers.m. Demonstrates a few of the +% most common operations to be performed with opsCopyLayers. +% +% Authors: John Paden + +%% User Settings +% ========================================================================= + +% Select seasons in run_all: +run_all; + +copy_param = []; +copy_param.layer_source.existence_check = false; +copy_param.layer_dest.existence_check = false; + +% Set the layer name(s) for the source (e.g. 'surface', 'bottom', {'surface','bottom'}) +copy_param.layer_source.name = {'surface'}; + +% Set the layer name(s) for the destination (e.g. 'surface', 'bottom', {'surface','bottom'}) +copy_param.layer_dest.name = {'surface'}; + +% Set the source (choose one) +if 0 + copy_param.layer_source.source = 'ops'; +elseif 0 + copy_param.layer_source.source = 'records'; +elseif 1 + copy_param.layer_source.source = 'echogram'; + % Set the echogram source if using echogram + if 1 + copy_param.layer_source.echogram_source = 'CSARP_post/qlook'; + elseif 0 + copy_param.layer_source.echogram_source = 'deconv'; + else + copy_param.layer_source.echogram_source = 'standard'; + end +elseif 0 + copy_param.layer_source.source = 'layerdata'; + copy_param.layer_source.layerdata_source = 'layer'; +elseif 0 + copy_param.layer_source.source = 'custom'; + copy_param.layer_source.gps_time = {[0 1e20]}; + copy_param.layer_source.twtt = {[1e-6 1e-6]}; + copy_param.layer_source.type = {[2 2]}; + copy_param.layer_source.quality = {[1 1]}; +else + copy_param.layer_source.source = 'lidar'; + copy_param.layer_source.lidar_source = 'awi'; +end + +if 1 + copy_param.copy_method = 'overwrite'; +elseif 0 + copy_param.copy_method = 'fillgaps'; +else + copy_param.copy_method = 'merge'; +end + +if strcmpi(copy_param.layer_dest.name,'surface') + copy_param.gaps_fill.method = 'interp_finite'; +else + copy_param.gaps_fill.method = 'preserve_gaps'; + copy_param.gaps_fill.method_args = [40 20]; +end + +% Set the twtt offset (for positive offset layer shifts down) +twtt_offset = 0e-6; + +% Set the GPS time offset (for positive offset layer shifts right) +gps_time_offset = 0; + +if twtt_offset ~= 0 || gps_time_offset ~= 0 + warning('You have set a nonzero twtt_offset(%.12g) or gps_time_offset(%.3g). Normally these are both zero. Please verify that this is what you want to do before running "dbcont" to continue.\n', twtt_offset, gps_time_offset); + keyboard + copy_param.eval.cmd = sprintf('s = interp1(time+%.3g,s + %.12g,time);',gps_time_offset,twtt_offset); +end + +% Set overwrite quality level (e.g. []: do not overwrite, 1: good, 2: medium, 3: bad) +quality = []; +if ~isempty(quality) && any(quality == [1 2 3]) + warning('You have set quality to %d. Normally it should be []. Please verify that you want to overwrite the quality level before running "dbcont" to continue.\n', quality); + keyboard + copy_param.quality.mode = 'overwrite'; + copy_param.quality.value = quality; +end + +% Set the destination (choose one): it can be the same as the source +if 0 + copy_param.layer_dest.source = 'ops'; +elseif 0 + copy_param.layer_dest.source = 'records'; +elseif 0 + copy_param.layer_dest.source = 'echogram'; + % Set the echogram source if using echogram + if 1 + copy_param.layer_dest.echogram_source = 'qlook'; + elseif 0 + copy_param.layer_dest.echogram_source = 'deconv'; + else + copy_param.layer_dest.echogram_source = 'standard'; + end +elseif 1 + copy_param.layer_dest.source = 'layerdata'; + copy_param.layer_dest.layerdata_source = 'layer'; +end + +% Usually the group name is standard +copy_param.layer_dest.group_name = 'standard'; + +% ===================================================================== +%% Automated Section +% ===================================================================== + +global gRadar; + +%% Loop to process each season +for param_idx = 1:length(param_fns) + + % Read in parameter spreadsheet + param_fn = ct_filename_param(param_fns{param_idx}); + fprintf('Reading %s\n', param_fn); + params = read_param_xls(param_fn,''); + + if isempty(params) + continue; + end + + % Run all segments (except "do not process") + if 0 + params = ct_set_params(params,'cmd.generic',1); + params = ct_set_params(params,'cmd.generic',0,'cmd.notes','do not process'); + params = ct_set_params(params,'cmd.generic',0,'day_seg','20120330_04'); + else + % HACK!!! + keyboard +% params = ct_set_params(params,'cmd.generic',0); +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20120330_04'); + params = ct_set_params(params,'cmd.generic',1); + params = ct_set_params(params,'cmd.generic',0,'cmd.notes','do not process'); + params = ct_set_params(params,'cmd.generic',0,'day_seg','20120330_04'); + end + + %% Process each segment + for param_idx = 1:length(params) + param = params(param_idx); + + if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic + fprintf('%s\tdo not process\n', param.day_seg); + continue; + end + + global gRadar; + param = merge_structs(param,gRadar); + + try + fprintf('opsCopyLayers %s (%s)\n', param.day_seg, datestr(now)); + opsCopyLayers(param,copy_param); + fprintf(' Complete (%s)\n', datestr(now)); + catch ME + fprintf('%s\terror!!!\t%s\n', param.day_seg, ME.getReport); + continue; + end + end +end diff --git a/cresis-toolbox/accum/convert_accum_2007_Greenland_Ground_to_new_format.m b/cresis-toolbox/accum/convert_accum_2007_Greenland_Ground_to_new_format.m index a9c00a9f..41e5e668 100644 --- a/cresis-toolbox/accum/convert_accum_2007_Greenland_Ground_to_new_format.m +++ b/cresis-toolbox/accum/convert_accum_2007_Greenland_Ground_to_new_format.m @@ -133,7 +133,7 @@ along_track = geodetic_to_along_track(gps.lat,gps.lon,gps.elev); rlines = get_equal_alongtrack_spacing_idxs(along_track,10); - physical_constants; + physical_constants; % Load WGS84.spheroid for rline_idx = 1:length(rlines) rline = rlines(rline_idx); if rline_idx < length(rlines) @@ -141,14 +141,14 @@ else rline_end = length(along_track); end - [origin(1),origin(2),origin(3)] = geodetic2ecef(gps.lat(rline)/180*pi,gps.lon(rline)/180*pi,gps.elev(rline),WGS84.ellipsoid); - [heading(1),heading(2),heading(3)] = geodetic2ecef(gps.lat(rline_end)/180*pi,gps.lon(rline_end)/180*pi,gps.elev(rline_end),WGS84.ellipsoid); + [origin(1),origin(2),origin(3)] = geodetic2ecef(WGS84.spheroid,gps.lat(rline),gps.lon(rline),gps.elev(rline)); + [heading(1),heading(2),heading(3)] = geodetic2ecef(WGS84.spheroid,gps.lat(rline_end),gps.lon(rline_end),gps.elev(rline_end)); heading = heading - origin; % Determine east vector - [east(1) east(2) east(3)] = lv2ecef(1,0,0,gps.lat(rline)/180*pi,gps.lon(rline)/180*pi,gps.elev(rline),WGS84.ellipsoid); + [east(1) east(2) east(3)] = enu2ecef(1,0,0,gps.lat(rline),gps.lon(rline),gps.elev(rline),WGS84.spheroid); east = east - origin; % Determine north vector - [north(1) north(2) north(3)] = lv2ecef(0,1,0,gps.lat(rline)/180*pi,gps.lon(rline)/180*pi,gps.elev(rline),WGS84.ellipsoid); + [north(1) north(2) north(3)] = enu2ecef(0,1,0,gps.lat(rline),gps.lon(rline),gps.elev(rline),WGS84.spheroid); north = north - origin; % Determine heading gps.heading(rline:rline_end) = atan2(dot(east,heading),dot(north,heading)); diff --git a/cresis-toolbox/cluster/cluster_chain_stage.m b/cresis-toolbox/cluster/cluster_chain_stage.m index 6ac971aa..89aa8ccd 100644 --- a/cresis-toolbox/cluster/cluster_chain_stage.m +++ b/cresis-toolbox/cluster/cluster_chain_stage.m @@ -19,11 +19,18 @@ % % Authors: John Paden % -% See also: cluster_chain_stage, cluster_cleanup, cluster_compile -% cluster_exec_job, cluster_get_batch, cluster_get_batch_list, -% cluster_hold, cluster_job, cluster_new_batch, cluster_new_task, -% cluster_print, cluster_run, cluster_submit_batch, cluster_submit_task, -% cluster_update_batch, cluster_update_task +% See also: cluster_chain_stage.m, cluster_cleanup.m, cluster_compile.m, +% cluster_cpu_affinity.m, cluster_error_mask.m, cluster_exec_task.m, +% cluster_file_success.m, cluster_get_batch_list.m, cluster_get_batch.m, +% cluster_get_chain_list.m, cluster_hold.m, cluster_job_check.m, +% cluster_job.m, cluster_job.sh, cluster_load_chain.m, cluster_new_batch.m, +% cluster_new_task.m, cluster_print_chain.m, cluster_print.m, +% cluster_reset.m, cluster_run.m, cluster_save_chain.m, +% cluster_save_dparam.m, cluster_save_sparam.m, cluster_set_chain.m, +% cluster_set_dparam.m, cluster_set_sparam.m, cluster_stop.m, +% cluster_submit_batch.m, cluster_submit_job.m, cluster_update_batch.m, +% cluster_update_task.m + %% Traverse chain list active_stage = ones(size(ctrl_chain)); diff --git a/cresis-toolbox/cluster/cluster_cleanup.m b/cresis-toolbox/cluster/cluster_cleanup.m index a6b4336c..b1e4c80b 100644 --- a/cresis-toolbox/cluster/cluster_cleanup.m +++ b/cresis-toolbox/cluster/cluster_cleanup.m @@ -1,10 +1,15 @@ function cluster_cleanup(ctrl_chain,mode) % cluster_cleanup(ctrl_chain,mode) % -% Similar to cluster_stop, but also removes chain and batch files. +% Similar to cluster_stop, but also removes chain and batch files. Run with +% no arguments to clean everything. % % Inputs: -% ctrl_chain = Several options to specify +% +% ctrl_chain: Optional argument. If no arguments are passed to +% cluster_cleanup you are given the option to remove all chains and +% batches. Otherwise, the first argument must be one of the following +% options: % 1. Vector of chain IDs or batch IDs, if batch ID, then mode must be set to 'batch' % 2. A ctrl structure identifying a batch % 3. A chain (cell array of ctrl) @@ -14,15 +19,23 @@ function cluster_cleanup(ctrl_chain,mode) % if mode is 'batch'. % % Examples: +% cluster_cleanup % And then answer "y" +% % cluster_cleanup(1) % % Author: John Paden % -% See also: cluster_chain_stage, cluster_cleanup, cluster_compile -% cluster_exec_job, cluster_get_batch, cluster_get_batch_list, -% cluster_hold, cluster_job, cluster_new_batch, cluster_new_task, -% cluster_print, cluster_run, cluster_submit_batch, cluster_submit_task, -% cluster_update_batch, cluster_update_task +% See also: cluster_chain_stage.m, cluster_cleanup.m, cluster_compile.m, +% cluster_cpu_affinity.m, cluster_error_mask.m, cluster_exec_task.m, +% cluster_file_success.m, cluster_get_batch_list.m, cluster_get_batch.m, +% cluster_get_chain_list.m, cluster_hold.m, cluster_job_check.m, +% cluster_job.m, cluster_job.sh, cluster_load_chain.m, cluster_new_batch.m, +% cluster_new_task.m, cluster_print_chain.m, cluster_print.m, +% cluster_reset.m, cluster_run.m, cluster_save_chain.m, +% cluster_save_dparam.m, cluster_save_sparam.m, cluster_set_chain.m, +% cluster_set_dparam.m, cluster_set_sparam.m, cluster_stop.m, +% cluster_submit_batch.m, cluster_submit_job.m, cluster_update_batch.m, +% cluster_update_task.m %% Input check if nargin == 0 || isempty(ctrl_chain) diff --git a/cresis-toolbox/cluster/cluster_compile.m b/cresis-toolbox/cluster/cluster_compile.m index 3be28dff..2bc65139 100644 --- a/cresis-toolbox/cluster/cluster_compile.m +++ b/cresis-toolbox/cluster/cluster_compile.m @@ -43,11 +43,17 @@ function cluster_compile(fun,hidden_depend_funs,force_compile,ctrl) % % Author: John Paden % -% See also: cluster_chain_stage, cluster_cleanup, cluster_compile -% cluster_exec_job, cluster_get_batch, cluster_get_batch_list, -% cluster_hold, cluster_job, cluster_new_batch, cluster_new_task, -% cluster_print, cluster_run, cluster_submit_batch, cluster_submit_task, -% cluster_update_batch, cluster_update_task +% See also: cluster_chain_stage.m, cluster_cleanup.m, cluster_compile.m, +% cluster_cpu_affinity.m, cluster_error_mask.m, cluster_exec_task.m, +% cluster_file_success.m, cluster_get_batch_list.m, cluster_get_batch.m, +% cluster_get_chain_list.m, cluster_hold.m, cluster_job_check.m, +% cluster_job.m, cluster_job.sh, cluster_load_chain.m, cluster_new_batch.m, +% cluster_new_task.m, cluster_print_chain.m, cluster_print.m, +% cluster_reset.m, cluster_run.m, cluster_save_chain.m, +% cluster_save_dparam.m, cluster_save_sparam.m, cluster_set_chain.m, +% cluster_set_dparam.m, cluster_set_sparam.m, cluster_stop.m, +% cluster_submit_batch.m, cluster_submit_job.m, cluster_update_batch.m, +% cluster_update_task.m % Find dependencies that are directly called @@ -196,6 +202,13 @@ function cluster_compile(fun,hidden_depend_funs,force_compile,ctrl) cmd = [cmd ' ' hidden_depend_funs{dep_idx}{1}]; end + for input_idx = 1:length(fun) + % Add in functions in case they are not in the hidden dependency list + if isempty(regexpi(cmd, fun{input_idx})) + cmd = [cmd ' ' fun{input_idx}]; + end + end + % Check to make sure the working directory is not a package or class % directory. This messes up the compiler if it uses functions from those % packages or classes. diff --git a/cresis-toolbox/cluster/cluster_cpu_affinity.m b/cresis-toolbox/cluster/cluster_cpu_affinity.m index 2d1888be..9353f7bc 100644 --- a/cresis-toolbox/cluster/cluster_cpu_affinity.m +++ b/cresis-toolbox/cluster/cluster_cpu_affinity.m @@ -13,11 +13,17 @@ function cluster_cpu_affinity(num_my_processors,process_list,force) % % Author: John Paden % -% See also: cluster_chain_stage, cluster_cleanup, cluster_compile -% cluster_exec_job, cluster_get_batch, cluster_get_batch_list, -% cluster_hold, cluster_job, cluster_new_batch, cluster_new_task, -% cluster_print, cluster_run, cluster_submit_batch, cluster_submit_task, -% cluster_update_batch, cluster_update_task +% See also: cluster_chain_stage.m, cluster_cleanup.m, cluster_compile.m, +% cluster_cpu_affinity.m, cluster_error_mask.m, cluster_exec_task.m, +% cluster_file_success.m, cluster_get_batch_list.m, cluster_get_batch.m, +% cluster_get_chain_list.m, cluster_hold.m, cluster_job_check.m, +% cluster_job.m, cluster_job.sh, cluster_load_chain.m, cluster_new_batch.m, +% cluster_new_task.m, cluster_print_chain.m, cluster_print.m, +% cluster_reset.m, cluster_run.m, cluster_save_chain.m, +% cluster_save_dparam.m, cluster_save_sparam.m, cluster_set_chain.m, +% cluster_set_dparam.m, cluster_set_sparam.m, cluster_stop.m, +% cluster_submit_batch.m, cluster_submit_job.m, cluster_update_batch.m, +% cluster_update_task.m if ~exist('process_list','var') process_list = ''; diff --git a/cresis-toolbox/cluster/cluster_error_mask.m b/cresis-toolbox/cluster/cluster_error_mask.m index 37e87f47..20f848c7 100644 --- a/cresis-toolbox/cluster/cluster_error_mask.m +++ b/cresis-toolbox/cluster/cluster_error_mask.m @@ -12,11 +12,17 @@ function cluster_error_mask(error_mask) % % Author: John Paden % -% See also: cluster_chain_stage, cluster_cleanup, cluster_compile -% cluster_exec_job, cluster_get_batch, cluster_get_batch_list, -% cluster_hold, cluster_job, cluster_new_batch, cluster_new_task, -% cluster_print, cluster_run, cluster_submit_batch, cluster_submit_task, -% cluster_update_batch, cluster_update_task +% See also: cluster_chain_stage.m, cluster_cleanup.m, cluster_compile.m, +% cluster_cpu_affinity.m, cluster_error_mask.m, cluster_exec_task.m, +% cluster_file_success.m, cluster_get_batch_list.m, cluster_get_batch.m, +% cluster_get_chain_list.m, cluster_hold.m, cluster_job_check.m, +% cluster_job.m, cluster_job.sh, cluster_load_chain.m, cluster_new_batch.m, +% cluster_new_task.m, cluster_print_chain.m, cluster_print.m, +% cluster_reset.m, cluster_run.m, cluster_save_chain.m, +% cluster_save_dparam.m, cluster_save_sparam.m, cluster_set_chain.m, +% cluster_set_dparam.m, cluster_set_sparam.m, cluster_stop.m, +% cluster_submit_batch.m, cluster_submit_job.m, cluster_update_batch.m, +% cluster_update_task.m if nargin == 0 assignin('caller','out_fn_exist_error',1); @@ -33,6 +39,7 @@ function cluster_error_mask(error_mask) assignin('caller','walltime_exceeded_error',2048); assignin('caller','max_mem_exceeded_error',4096); assignin('caller','insufficient_mcr_space',8192); + assignin('caller','matlab_task_out_of_memory',16384); assignin('caller','critical_error',2^10-1); elseif error_mask == 0 @@ -83,4 +90,7 @@ function cluster_error_mask(error_mask) if bitand(error_mask,insufficient_mcr_space) fprintf('insufficient_mcr_space: Job failed because there was insufficient space for the Matlab Compiler Runtime files.\n'); end + if bitand(error_mask,matlab_task_out_of_memory) + fprintf('matlab_task_out_of_memory: Matlab returned out of memory error for this task.\n'); + end end diff --git a/cresis-toolbox/cluster/cluster_exec_task.m b/cresis-toolbox/cluster/cluster_exec_task.m index ad458575..fa71684f 100644 --- a/cresis-toolbox/cluster/cluster_exec_task.m +++ b/cresis-toolbox/cluster/cluster_exec_task.m @@ -106,6 +106,9 @@ function cluster_exec_task(ctrl,task_ids,run_mode) fprintf(' %s: %s\n', mfilename, param.notes); fprintf(' %s: Eval %s\n', mfilename, eval_cmd); + global gRadar; + gRadar.cluster.is_cluster_job = false; + if ctrl.cluster.dbstop_if_error dbstop_if_error = false; breakpoints = dbstatus; @@ -151,14 +154,14 @@ function cluster_exec_task(ctrl,task_ids,run_mode) system([ctrl.cluster.cluster_job_fn ' 0 @@ -676,18 +695,19 @@ try % Memory requested in megabytes idx = regexp(result,'Max Mem'); - tmp_result = result(idx:end); + tmp_result = result(idx(end):end); idx = find(tmp_result==':',1); tmp_result = tmp_result(idx+2:end); idx = find(tmp_result==10|tmp_result==13,1); tmp_result = tmp_result(1:idx); [mem,~,~,idx] = sscanf(tmp_result,'%d'); - info(id_idx).mem_actual = mem/1e3; - + info(id_idx).mem = mem/1e3; + end + try if isempty(info(id_idx).exec_node) % Exec host idx = regexp(result,'hostname:'); - tmp_result = result(idx:end); + tmp_result = result(idx(end):end); idx = find(tmp_result==':',1); tmp_result = tmp_result(idx+2:end); idx = find(tmp_result==' '); diff --git a/cresis-toolbox/cluster/cluster_print_chain.m b/cresis-toolbox/cluster/cluster_print_chain.m index 270b8913..86499b6c 100644 --- a/cresis-toolbox/cluster/cluster_print_chain.m +++ b/cresis-toolbox/cluster/cluster_print_chain.m @@ -24,11 +24,17 @@ % % Author: John Paden % -% See also: cluster_chain_stage, cluster_cleanup, cluster_compile -% cluster_exec_job, cluster_get_batch, cluster_get_batch_list, -% cluster_hold, cluster_job, cluster_new_batch, cluster_new_task, -% cluster_print, cluster_run, cluster_submit_batch, cluster_submit_task, -% cluster_update_batch, cluster_update_task +% See also: cluster_chain_stage.m, cluster_cleanup.m, cluster_compile.m, +% cluster_cpu_affinity.m, cluster_error_mask.m, cluster_exec_task.m, +% cluster_file_success.m, cluster_get_batch_list.m, cluster_get_batch.m, +% cluster_get_chain_list.m, cluster_hold.m, cluster_job_check.m, +% cluster_job.m, cluster_job.sh, cluster_load_chain.m, cluster_new_batch.m, +% cluster_new_task.m, cluster_print_chain.m, cluster_print.m, +% cluster_reset.m, cluster_run.m, cluster_save_chain.m, +% cluster_save_dparam.m, cluster_save_sparam.m, cluster_set_chain.m, +% cluster_set_dparam.m, cluster_set_sparam.m, cluster_stop.m, +% cluster_submit_batch.m, cluster_submit_job.m, cluster_update_batch.m, +% cluster_update_task.m if ~exist('force_check','var') || isempty(force_check) force_check = false; @@ -122,12 +128,10 @@ elseif isnumeric(ctrl_chain) if strcmpi(mode,'batch') ctrl = cluster_get_batch(ctrl_chain,false,0); - ctrl_chain = cluster_print_chain({{ctrl}},force_check); + [ctrl_chain,stats] = cluster_print_chain({{ctrl}},force_check); else ctrl_chain = cluster_load_chain(ctrl_chain); - ctrl_chain = cluster_print_chain(ctrl_chain,force_check); + [ctrl_chain,stats] = cluster_print_chain(ctrl_chain,force_check); end end - -return diff --git a/cresis-toolbox/cluster/cluster_reset.m b/cresis-toolbox/cluster/cluster_reset.m index 887070ce..d40b7fba 100644 --- a/cresis-toolbox/cluster/cluster_reset.m +++ b/cresis-toolbox/cluster/cluster_reset.m @@ -7,8 +7,20 @@ % 2. The retries are reset to zero for these tasks. % 3. The status is set to 'T' % 4. The tasks are added to the submission queue -% +% % Author: John Paden +% +% See also: cluster_chain_stage.m, cluster_cleanup.m, cluster_compile.m, +% cluster_cpu_affinity.m, cluster_error_mask.m, cluster_exec_task.m, +% cluster_file_success.m, cluster_get_batch_list.m, cluster_get_batch.m, +% cluster_get_chain_list.m, cluster_hold.m, cluster_job_check.m, +% cluster_job.m, cluster_job.sh, cluster_load_chain.m, cluster_new_batch.m, +% cluster_new_task.m, cluster_print_chain.m, cluster_print.m, +% cluster_reset.m, cluster_run.m, cluster_save_chain.m, +% cluster_save_dparam.m, cluster_save_sparam.m, cluster_set_chain.m, +% cluster_set_dparam.m, cluster_set_sparam.m, cluster_stop.m, +% cluster_submit_batch.m, cluster_submit_job.m, cluster_update_batch.m, +% cluster_update_task.m if iscell(ctrl_chain) %% Traverse chain list diff --git a/cresis-toolbox/cluster/cluster_run.m b/cresis-toolbox/cluster/cluster_run.m index b7a7f7b3..dbb1f48b 100644 --- a/cresis-toolbox/cluster/cluster_run.m +++ b/cresis-toolbox/cluster/cluster_run.m @@ -1,4 +1,4 @@ -function ctrl_chain = cluster_run(ctrl_chain,cluster_run_mode) +function [ctrl_chain,cluster_run_mode] = cluster_run(ctrl_chain,cluster_run_mode) % ctrl_chain = cluster_run(ctrl_chain,cluster_run_mode) % % Submits jobs in a list of batch chains. Each chain in the list runs in @@ -34,11 +34,17 @@ % % Author: John Paden % -% See also: cluster_chain_stage, cluster_cleanup, cluster_compile -% cluster_exec_job, cluster_get_batch, cluster_get_batch_list, -% cluster_hold, cluster_job, cluster_new_batch, cluster_new_task, -% cluster_print, cluster_run, cluster_submit_batch, cluster_submit_task, -% cluster_update_batch, cluster_update_task +% See also: cluster_chain_stage.m, cluster_cleanup.m, cluster_compile.m, +% cluster_cpu_affinity.m, cluster_error_mask.m, cluster_exec_task.m, +% cluster_file_success.m, cluster_get_batch_list.m, cluster_get_batch.m, +% cluster_get_chain_list.m, cluster_hold.m, cluster_job_check.m, +% cluster_job.m, cluster_job.sh, cluster_load_chain.m, cluster_new_batch.m, +% cluster_new_task.m, cluster_print_chain.m, cluster_print.m, +% cluster_reset.m, cluster_run.m, cluster_save_chain.m, +% cluster_save_dparam.m, cluster_save_sparam.m, cluster_set_chain.m, +% cluster_set_dparam.m, cluster_set_sparam.m, cluster_stop.m, +% cluster_submit_batch.m, cluster_submit_job.m, cluster_update_batch.m, +% cluster_update_task.m %% Input checking if ~exist('cluster_run_mode','var') || isempty(cluster_run_mode) @@ -54,6 +60,8 @@ if iscell(ctrl_chain) %% Traverse chain list + global gctrl_chain; + gctrl_chain = ctrl_chain; active_stage = ones(numel(ctrl_chain),1); first_run = ones(numel(ctrl_chain),1); while any(isfinite(active_stage)) @@ -82,14 +90,19 @@ pause(ctrl.cluster.stat_pause); end - % 3. Submit jobs from the active stage for each parallel control structure + % 3. Update ctrl_chain + ctrl_chain{chain}{active_stage(chain)} = ctrl; + gctrl_chain = ctrl_chain; + + % 4. Submit jobs from the active stage for each parallel control structure % ctrl.max_active_jobs. - ctrl = cluster_run(ctrl,cluster_run_mode); + [ctrl,cluster_run_mode] = cluster_run(ctrl,cluster_run_mode); - % 4. Update ctrl_chain + % 5. Update ctrl_chain ctrl_chain{chain}{active_stage(chain)} = ctrl; + gctrl_chain = ctrl_chain; - % 5. If all jobs completed in a batch and: + % 6. If all jobs completed in a batch and: % If no errors, move to the next stage % If errors, stop chain % Note: cluster_update_task guarantees that a task that still has retries left will never have 'C' status if it had errors @@ -113,7 +126,7 @@ end end - % 6. Check to see if a hold has been placed on this batch + % 7. Check to see if a hold has been placed on this batch if cluster_run_mode >= 0 && exist(ctrl.hold_fn,'file') warning('This batch has a hold. Run cluster_hold(ctrl) to remove. Run "cluster_run_mode=-1" to stop cluster_run.m now in a clean way. Either way, run dbcont to continue.\n'); keyboard @@ -157,7 +170,13 @@ if strcmpi(ctrl.cluster.type,'none') return; end - + + % Ensure that submission_queue and job_status=='T' match + if ~isempty(setxor(ctrl.submission_queue,find(ctrl.job_status=='T'))) + warning('submission_queue and job_status are not in sync. Resyncing.') + ctrl.submission_queue = find(ctrl.job_status=='T'); + end + % Sort submission queue tasks based on memory usage: this is done to % increase the chance that tasks with similar memory usage will be % grouped together in jobs to make the cluster memory request more @@ -179,7 +198,25 @@ ctrl.cluster.max_time_per_job, ctrl.batch_id, task_id, task_cpu_time); keyboard; end - if ctrl.cluster.max_mem_per_job < task_mem + if ctrl.cluster.desired_time_per_job < job_cpu_time + task_cpu_time && ~isempty(job_tasks) + [ctrl,new_job_id] = cluster_submit_job(ctrl,job_tasks,job_cpu_time,job_mem); + fprintf('Submitted %d tasks in cluster job (%d): (%s)\n %s\n %d: %d', length(job_tasks), ... + new_job_id, datestr(now), ctrl.notes{job_tasks(1)}, ctrl.batch_id, job_tasks(1)); + if length(job_tasks) > 1 + fprintf(', %d', job_tasks(2:end)); + end + fprintf('\n'); + job_tasks = []; + job_cpu_time = 0; + job_mem = 0; + pause(ctrl.cluster.submit_pause); + end + if task_mem <= ctrl.cluster.max_mem_per_job + job_tasks(end+1) = task_id; + job_cpu_time = job_cpu_time + task_cpu_time; + job_mem = max(job_mem, task_mem); + ctrl.submission_queue = ctrl.submission_queue(2:end); + else warning('ctrl.cluster.max_mem_per_job (%.1f GB) is less than task %d:%d''s requested mem: %.1f GB', ... ctrl.cluster.max_mem_per_job/1e9, ctrl.batch_id, task_id, task_mem/1e9); if ~isempty(regexpi(ctrl.cluster.max_mem_mode,'debug')) @@ -195,32 +232,22 @@ keyboard; end if ~isempty(regexpi(ctrl.cluster.max_mem_mode,'local')) + % Run the task now locally cur_cluster_type = ctrl.cluster.type; ctrl.cluster.type = 'debug'; - ctrl = cluster_submit_job(ctrl,job_tasks,job_cpu_time,job_mem); + ctrl = cluster_submit_job(ctrl,task_id,task_cpu_time,task_mem); ctrl.cluster.type = cur_cluster_type; end if ~isempty(regexpi(ctrl.cluster.max_mem_mode,'truncate')) + % Run the task anyway, but truncate the memory request to the + % maximum allowed. task_mem = ctrl.cluster.max_mem_per_job; + job_tasks(end+1) = task_id; + job_cpu_time = job_cpu_time + task_cpu_time; + job_mem = max(job_mem, task_mem); end + ctrl.submission_queue = ctrl.submission_queue(2:end); end - if ctrl.cluster.desired_time_per_job < job_cpu_time + task_cpu_time && ~isempty(job_tasks) - [ctrl,new_job_id] = cluster_submit_job(ctrl,job_tasks,job_cpu_time,job_mem); - fprintf('Submitted %d tasks in cluster job (%d): (%s)\n %s\n %d: %d', length(job_tasks), ... - new_job_id, datestr(now), ctrl.notes{job_tasks(1)}, ctrl.batch_id, job_tasks(1)); - if length(job_tasks) > 1 - fprintf(', %d', job_tasks(2:end)); - end - fprintf('\n'); - job_tasks = []; - job_cpu_time = 0; - job_mem = 0; - pause(ctrl.cluster.submit_pause); - end - job_tasks(end+1) = task_id; - job_cpu_time = job_cpu_time + task_cpu_time; - job_mem = max(job_mem, task_mem); - ctrl.submission_queue = ctrl.submission_queue(2:end); % Check to see if a hold has been placed on this batch if exist(ctrl.hold_fn,'file') diff --git a/cresis-toolbox/cluster/cluster_save_chain.m b/cresis-toolbox/cluster/cluster_save_chain.m index 3a7a0a3b..aec79937 100644 --- a/cresis-toolbox/cluster/cluster_save_chain.m +++ b/cresis-toolbox/cluster/cluster_save_chain.m @@ -20,11 +20,17 @@ % % Author: John Paden % -% See also: cluster_chain_stage, cluster_cleanup, cluster_compile -% cluster_exec_job, cluster_get_batch, cluster_get_batch_list, -% cluster_hold, cluster_job, cluster_new_batch, cluster_new_task, -% cluster_print, cluster_run, cluster_submit_batch, cluster_submit_task, -% cluster_update_batch, cluster_update_task +% See also: cluster_chain_stage.m, cluster_cleanup.m, cluster_compile.m, +% cluster_cpu_affinity.m, cluster_error_mask.m, cluster_exec_task.m, +% cluster_file_success.m, cluster_get_batch_list.m, cluster_get_batch.m, +% cluster_get_chain_list.m, cluster_hold.m, cluster_job_check.m, +% cluster_job.m, cluster_job.sh, cluster_load_chain.m, cluster_new_batch.m, +% cluster_new_task.m, cluster_print_chain.m, cluster_print.m, +% cluster_reset.m, cluster_run.m, cluster_save_chain.m, +% cluster_save_dparam.m, cluster_save_sparam.m, cluster_set_chain.m, +% cluster_set_dparam.m, cluster_set_sparam.m, cluster_stop.m, +% cluster_submit_batch.m, cluster_submit_job.m, cluster_update_batch.m, +% cluster_update_task.m %% Input Checks good_chain_idx = []; diff --git a/cresis-toolbox/cluster/cluster_save_dparam.m b/cresis-toolbox/cluster/cluster_save_dparam.m index fe5d9f90..6dcb3bc5 100644 --- a/cresis-toolbox/cluster/cluster_save_dparam.m +++ b/cresis-toolbox/cluster/cluster_save_dparam.m @@ -1,4 +1,24 @@ function ctrl = cluster_save_dparam(ctrl) +% ctrl = cluster_save_dparam(ctrl) +% +% Function to save dparam all at once. Saving dparam as the tasks are being +% generated can be very slow when there are many tasks. It is better to set +% 'dparam_save' to false in cluster_new_task.m and then call +% cluster_save_dparam at the end when many tasks could be created. +% +% Author: John Paden +% +% See also: cluster_chain_stage.m, cluster_cleanup.m, cluster_compile.m, +% cluster_cpu_affinity.m, cluster_error_mask.m, cluster_exec_task.m, +% cluster_file_success.m, cluster_get_batch_list.m, cluster_get_batch.m, +% cluster_get_chain_list.m, cluster_hold.m, cluster_job_check.m, +% cluster_job.m, cluster_job.sh, cluster_load_chain.m, cluster_new_batch.m, +% cluster_new_task.m, cluster_print_chain.m, cluster_print.m, +% cluster_reset.m, cluster_run.m, cluster_save_chain.m, +% cluster_save_dparam.m, cluster_save_sparam.m, cluster_set_chain.m, +% cluster_set_dparam.m, cluster_set_sparam.m, cluster_stop.m, +% cluster_submit_batch.m, cluster_submit_job.m, cluster_update_batch.m, +% cluster_update_task.m if isfield(ctrl,'dparam') dynamic_in_fn = fullfile(ctrl.in_fn_dir,'dynamic.mat'); diff --git a/cresis-toolbox/cluster/cluster_save_sparam.m b/cresis-toolbox/cluster/cluster_save_sparam.m new file mode 100644 index 00000000..4c792910 --- /dev/null +++ b/cresis-toolbox/cluster/cluster_save_sparam.m @@ -0,0 +1,25 @@ +function ctrl = cluster_save_sparam(ctrl,sparam) +% ctrl = cluster_save_sparam(ctrl,sparam) +% +% Function to save sparam separately from creating new tasks. Sometimes +% useful when the full sparam structure is not known until all the tasks +% have already been created. +% +% Author: John Paden +% +% See also: cluster_chain_stage.m, cluster_cleanup.m, cluster_compile.m, +% cluster_cpu_affinity.m, cluster_error_mask.m, cluster_exec_task.m, +% cluster_file_success.m, cluster_get_batch_list.m, cluster_get_batch.m, +% cluster_get_chain_list.m, cluster_hold.m, cluster_job_check.m, +% cluster_job.m, cluster_job.sh, cluster_load_chain.m, cluster_new_batch.m, +% cluster_new_task.m, cluster_print_chain.m, cluster_print.m, +% cluster_reset.m, cluster_run.m, cluster_save_chain.m, +% cluster_save_dparam.m, cluster_save_sparam.m, cluster_set_chain.m, +% cluster_set_dparam.m, cluster_set_sparam.m, cluster_stop.m, +% cluster_submit_batch.m, cluster_submit_job.m, cluster_update_batch.m, +% cluster_update_task.m + +static_in_fn = fullfile(ctrl.in_fn_dir,'static.mat'); +cluster_new_task_sparam_check; +static_param = sparam; +robust_save(static_in_fn,ctrl.cluster.file_version,'static_param'); diff --git a/cresis-toolbox/cluster/cluster_set_chain.m b/cresis-toolbox/cluster/cluster_set_chain.m index 385279ec..ab0780d8 100644 --- a/cresis-toolbox/cluster/cluster_set_chain.m +++ b/cresis-toolbox/cluster/cluster_set_chain.m @@ -4,6 +4,20 @@ % ctrl_chain = cluster_set_chain(ctrl_chain,'cluster.cpu_time_mult',2,'cluster.mem_mult',1); % ctrl_chain = cluster_set_chain(ctrl_chain,'cluster.type','debug'); % ctrl_chain = cluster_set_chain(ctrl_chain,'cluster.desired_time_per_job',24*60*60); +% +% Author: John Paden +% +% See also: cluster_chain_stage.m, cluster_cleanup.m, cluster_compile.m, +% cluster_cpu_affinity.m, cluster_error_mask.m, cluster_exec_task.m, +% cluster_file_success.m, cluster_get_batch_list.m, cluster_get_batch.m, +% cluster_get_chain_list.m, cluster_hold.m, cluster_job_check.m, +% cluster_job.m, cluster_job.sh, cluster_load_chain.m, cluster_new_batch.m, +% cluster_new_task.m, cluster_print_chain.m, cluster_print.m, +% cluster_reset.m, cluster_run.m, cluster_save_chain.m, +% cluster_save_dparam.m, cluster_save_sparam.m, cluster_set_chain.m, +% cluster_set_dparam.m, cluster_set_sparam.m, cluster_stop.m, +% cluster_submit_batch.m, cluster_submit_job.m, cluster_update_batch.m, +% cluster_update_task.m if isnumeric(ctrl_chain) ctrl_chain = cluster_load_chain(ctrl_chain); diff --git a/cresis-toolbox/cluster/cluster_set_dparam.m b/cresis-toolbox/cluster/cluster_set_dparam.m index 234b5497..5df0a306 100644 --- a/cresis-toolbox/cluster/cluster_set_dparam.m +++ b/cresis-toolbox/cluster/cluster_set_dparam.m @@ -6,6 +6,18 @@ % cluster_set_dparam(ctrl,'cpu_time',100); % % Author: John Paden +% +% See also: cluster_chain_stage.m, cluster_cleanup.m, cluster_compile.m, +% cluster_cpu_affinity.m, cluster_error_mask.m, cluster_exec_task.m, +% cluster_file_success.m, cluster_get_batch_list.m, cluster_get_batch.m, +% cluster_get_chain_list.m, cluster_hold.m, cluster_job_check.m, +% cluster_job.m, cluster_job.sh, cluster_load_chain.m, cluster_new_batch.m, +% cluster_new_task.m, cluster_print_chain.m, cluster_print.m, +% cluster_reset.m, cluster_run.m, cluster_save_chain.m, +% cluster_save_dparam.m, cluster_save_sparam.m, cluster_set_chain.m, +% cluster_set_dparam.m, cluster_set_sparam.m, cluster_stop.m, +% cluster_submit_batch.m, cluster_submit_job.m, cluster_update_batch.m, +% cluster_update_task.m if iscell(ctrl_chain) %% Traverse chain lists diff --git a/cresis-toolbox/cluster/cluster_set_sparam.m b/cresis-toolbox/cluster/cluster_set_sparam.m index a376526f..74ef7c98 100644 --- a/cresis-toolbox/cluster/cluster_set_sparam.m +++ b/cresis-toolbox/cluster/cluster_set_sparam.m @@ -6,6 +6,18 @@ % cluster_set_sparam(ctrl,'argsin{1}',20); % % Author: John Paden +% +% See also: cluster_chain_stage.m, cluster_cleanup.m, cluster_compile.m, +% cluster_cpu_affinity.m, cluster_error_mask.m, cluster_exec_task.m, +% cluster_file_success.m, cluster_get_batch_list.m, cluster_get_batch.m, +% cluster_get_chain_list.m, cluster_hold.m, cluster_job_check.m, +% cluster_job.m, cluster_job.sh, cluster_load_chain.m, cluster_new_batch.m, +% cluster_new_task.m, cluster_print_chain.m, cluster_print.m, +% cluster_reset.m, cluster_run.m, cluster_save_chain.m, +% cluster_save_dparam.m, cluster_save_sparam.m, cluster_set_chain.m, +% cluster_set_dparam.m, cluster_set_sparam.m, cluster_stop.m, +% cluster_submit_batch.m, cluster_submit_job.m, cluster_update_batch.m, +% cluster_update_task.m if iscell(ctrl_chain) %% Traverse chain lists diff --git a/cresis-toolbox/cluster/cluster_stop.m b/cresis-toolbox/cluster/cluster_stop.m index 832fc627..bc036711 100644 --- a/cresis-toolbox/cluster/cluster_stop.m +++ b/cresis-toolbox/cluster/cluster_stop.m @@ -20,11 +20,17 @@ function cluster_stop(ctrl_chain,mode) % % Author: John Paden % -% See also: cluster_chain_stage, cluster_cleanup, cluster_compile -% cluster_exec_job, cluster_get_batch, cluster_get_batch_list, -% cluster_hold, cluster_job, cluster_new_batch, cluster_new_task, -% cluster_print, cluster_run, cluster_submit_batch, cluster_submit_task, -% cluster_update_batch, cluster_update_task +% See also: cluster_chain_stage.m, cluster_cleanup.m, cluster_compile.m, +% cluster_cpu_affinity.m, cluster_error_mask.m, cluster_exec_task.m, +% cluster_file_success.m, cluster_get_batch_list.m, cluster_get_batch.m, +% cluster_get_chain_list.m, cluster_hold.m, cluster_job_check.m, +% cluster_job.m, cluster_job.sh, cluster_load_chain.m, cluster_new_batch.m, +% cluster_new_task.m, cluster_print_chain.m, cluster_print.m, +% cluster_reset.m, cluster_run.m, cluster_save_chain.m, +% cluster_save_dparam.m, cluster_save_sparam.m, cluster_set_chain.m, +% cluster_set_dparam.m, cluster_set_sparam.m, cluster_stop.m, +% cluster_submit_batch.m, cluster_submit_job.m, cluster_update_batch.m, +% cluster_update_task.m %% Input check if nargin == 0 || isempty(ctrl_chain) diff --git a/cresis-toolbox/cluster/cluster_submit_batch.m b/cresis-toolbox/cluster/cluster_submit_batch.m index 748df6e2..f9411c7b 100644 --- a/cresis-toolbox/cluster/cluster_submit_batch.m +++ b/cresis-toolbox/cluster/cluster_submit_batch.m @@ -1,16 +1,28 @@ -function out = cluster_submit_batch(fun,block,argsin,num_args_out,cpu_time) -% out = cluster_submit_batch(fun,block,argsin,num_args_out,notes,cpu_time) +function out = cluster_submit_batch(fun,block,argsin,num_args_out,cpu_time,mem) +% out = cluster_submit_batch(fun,block,argsin,num_args_out,notes,cpu_time,mem) % % Runs an arbitrary job called "fun" with inputs "argsin". Introduces % about 30 seconds of overhead as long as a compile is not necessary. % % INPUTS: -% fun: string containing the function name (e.g. 'hanning') that the task -% will run +% +% fun: String containing the function name (e.g. 'hanning') that the task +% will run. This should NOT include '.m'. Note that if there are any +% function dependencies to this function that are not explicit, these must +% be added to gRadar.cluster.hidden_depend_funs in startup.m. This happens +% when passing in a function handle or function name string as an input or +% creating commands that run via eval because the cluster_compile function +% cannot detect these implicit dependencies since the code must be +% evaluated to see them. +% % argsin: cell vector of input arguments to the task (e.g. {10,'arg2'}) +% % num_args_out: number of output arguments +% % notes: string containing notes about the task +% % cpu_time: expected maximum cpu time in seconds +% % mem: expected maximum memory usage in bytes % % OUTPUTS: @@ -19,10 +31,10 @@ % % EXAMPLES: % % Blocking example: -% out = cluster_submit_batch('hanning',true,{10},1,60) +% out = cluster_submit_batch('hanning',true,{10},1,60,1500e6) % % % Non-blocking example: -% ctrl = cluster_submit_batch('hanning',false,{10},1,60); +% ctrl = cluster_submit_batch('hanning',false,{10},1,60,1500e6); % ctrl_chain = {{ctrl}}; % [~,chain_id] = cluster_save_chain(ctrl_chain); % @@ -52,12 +64,13 @@ ctrl = cluster_new_batch; -cluster_compile(fun,[],0,ctrl); +cluster_compile({[fun '.m']},[],0,ctrl); param.task_function = fun; param.argsin = argsin; param.num_args_out = num_args_out; param.cpu_time = cpu_time; +param.mem = mem; repeat_task = 1; for tmp = 1:repeat_task ctrl = cluster_new_task(ctrl,param,[]); diff --git a/cresis-toolbox/cluster/cluster_submit_job.m b/cresis-toolbox/cluster/cluster_submit_job.m index f64845ad..2a36c7a7 100644 --- a/cresis-toolbox/cluster/cluster_submit_job.m +++ b/cresis-toolbox/cluster/cluster_submit_job.m @@ -21,11 +21,17 @@ % % Author: John Paden % -% See also: cluster_chain_stage, cluster_cleanup, cluster_compile -% cluster_exec_job, cluster_get_batch, cluster_get_batch_list, -% cluster_hold, cluster_job, cluster_new_batch, cluster_new_task, -% cluster_print, cluster_run, cluster_submit_batch, cluster_submit_task, -% cluster_update_batch, cluster_update_task +% See also: cluster_chain_stage.m, cluster_cleanup.m, cluster_compile.m, +% cluster_cpu_affinity.m, cluster_error_mask.m, cluster_exec_task.m, +% cluster_file_success.m, cluster_get_batch_list.m, cluster_get_batch.m, +% cluster_get_chain_list.m, cluster_hold.m, cluster_job_check.m, +% cluster_job.m, cluster_job.sh, cluster_load_chain.m, cluster_new_batch.m, +% cluster_new_task.m, cluster_print_chain.m, cluster_print.m, +% cluster_reset.m, cluster_run.m, cluster_save_chain.m, +% cluster_save_dparam.m, cluster_save_sparam.m, cluster_set_chain.m, +% cluster_set_dparam.m, cluster_set_sparam.m, cluster_stop.m, +% cluster_submit_batch.m, cluster_submit_job.m, cluster_update_batch.m, +% cluster_update_task.m %% Create the temporary file names in_fn = ctrl.in_fn_dir; diff --git a/cresis-toolbox/cluster/cluster_update_batch.m b/cresis-toolbox/cluster/cluster_update_batch.m index 23ed21a7..6e17e196 100644 --- a/cresis-toolbox/cluster/cluster_update_batch.m +++ b/cresis-toolbox/cluster/cluster_update_batch.m @@ -41,11 +41,17 @@ % % Author: John Paden % -% See also: cluster_chain_stage, cluster_cleanup, cluster_compile -% cluster_exec_job, cluster_get_batch, cluster_get_batch_list, -% cluster_hold, cluster_job, cluster_new_batch, cluster_new_task, -% cluster_print, cluster_run, cluster_submit_batch, cluster_submit_task, -% cluster_update_batch, cluster_update_task +% See also: cluster_chain_stage.m, cluster_cleanup.m, cluster_compile.m, +% cluster_cpu_affinity.m, cluster_error_mask.m, cluster_exec_task.m, +% cluster_file_success.m, cluster_get_batch_list.m, cluster_get_batch.m, +% cluster_get_chain_list.m, cluster_hold.m, cluster_job_check.m, +% cluster_job.m, cluster_job.sh, cluster_load_chain.m, cluster_new_batch.m, +% cluster_new_task.m, cluster_print_chain.m, cluster_print.m, +% cluster_reset.m, cluster_run.m, cluster_save_chain.m, +% cluster_save_dparam.m, cluster_save_sparam.m, cluster_set_chain.m, +% cluster_set_dparam.m, cluster_set_sparam.m, cluster_stop.m, +% cluster_submit_batch.m, cluster_submit_job.m, cluster_update_batch.m, +% cluster_update_task.m if ~exist('force_check','var') || isempty(force_check) force_check = false; diff --git a/cresis-toolbox/cluster/cluster_update_task.m b/cresis-toolbox/cluster/cluster_update_task.m index 5eb92ac3..bbbc69a6 100644 --- a/cresis-toolbox/cluster/cluster_update_task.m +++ b/cresis-toolbox/cluster/cluster_update_task.m @@ -12,6 +12,13 @@ % .out = up-to-Nx1 vector of cells containing the outputs for each % job as they complete (read in from the output.mat files) % job_id: the job id to check up on +% update_mode: scalar integer (-1, 0, 1, or 2). Default is 1. +% -1: populates cpu_time and like fields and returns without checking +% ==0: is not subject to cluster_hold +% >0: check stdout/stderr +% >0: print error information +% >0: cpu time and memory warnings +% >0: retry failed jobs % % Outputs: % ctrl = updated ctrl structure @@ -28,11 +35,17 @@ % % Author: John Paden % -% See also: cluster_chain_stage, cluster_cleanup, cluster_compile -% cluster_exec_job, cluster_get_batch, cluster_get_batch_list, -% cluster_hold, cluster_job, cluster_new_batch, cluster_new_task, -% cluster_print, cluster_run, cluster_submit_batch, cluster_submit_task, -% cluster_update_batch, cluster_update_task +% See also: cluster_chain_stage.m, cluster_cleanup.m, cluster_compile.m, +% cluster_cpu_affinity.m, cluster_error_mask.m, cluster_exec_task.m, +% cluster_file_success.m, cluster_get_batch_list.m, cluster_get_batch.m, +% cluster_get_chain_list.m, cluster_hold.m, cluster_job_check.m, +% cluster_job.m, cluster_job.sh, cluster_load_chain.m, cluster_new_batch.m, +% cluster_new_task.m, cluster_print_chain.m, cluster_print.m, +% cluster_reset.m, cluster_run.m, cluster_save_chain.m, +% cluster_save_dparam.m, cluster_save_sparam.m, cluster_set_chain.m, +% cluster_set_dparam.m, cluster_set_sparam.m, cluster_stop.m, +% cluster_submit_batch.m, cluster_submit_job.m, cluster_update_batch.m, +% cluster_update_task.m if ~exist('update_mode','var') || isempty(update_mode) update_mode = 1; @@ -178,6 +191,13 @@ if ~isempty(regexp(error_file_str,'PBS: job killed: walltime')) error_mask = bitor(error_mask,walltime_exceeded_error); end + % slurmstepd: error: *** JOB 15269436 ON prod-0105 CANCELLED AT 2022-05-15T23:32:22 DUE TO TIME LIMIT *** + if ~isempty(regexp(error_file_str,'CANCELLED')) + error_mask = bitor(error_mask,cluster_killed_error); + end + if ~isempty(regexp(error_file_str,'DUE TO TIME LIMIT')) + error_mask = bitor(error_mask,walltime_exceeded_error); + end end end end @@ -225,7 +245,13 @@ if isfield(out,'errorstruct') if ~isempty(out.errorstruct) - error_mask = bitor(error_mask,errorstruct_contains_error); + if regexp(out.errorstruct.message,'Out of memory') + % Warning: Out of memory. Type "help memory" for your options. + error_mask = bitor(error_mask,max_mem_exceeded_error); + error_mask = bitor(error_mask,matlab_task_out_of_memory); + else + error_mask = bitor(error_mask,errorstruct_contains_error); + end end else error_mask = bitor(error_mask,errorstruct_exist_error); @@ -278,7 +304,8 @@ end if update_mode && ctrl.error_mask(task_id) - warning(' Job Error %d:%d/%d (lead task %d)\n', ctrl.batch_id, task_id, job_id, task_id_out); + fprintf('%s\n','='*ones(1,80)); + warning(' Job Error %d:%d/%d (lead task %d) (%s)\n', ctrl.batch_id, task_id, job_id, task_id_out, datestr(now)); if any(strcmpi(ctrl.cluster.type,{'torque','slurm'})) fprintf(' hostname:%s attempt:%d max_attempts:%d\n', hostname, attempt, max_attempts); end @@ -301,7 +328,7 @@ fprintf(' errorstruct contains an error:\n'); warning('%s',out.errorstruct.getReport); if ctrl.cluster.stop_on_error - fprintf('\nctrl.cluster.stop_on_error is enabled which causes the cluster running process to stop whenever there is a Matlab coding error. To disable this for this batch, you can run "ctrl.cluster.stop_on_error=false". Fix the coding bug printed above which might require running cluster_compile.m if you change cluster task code and then run "dbcont".\n'); + fprintf('\nctrl.cluster.stop_on_error is enabled which causes the cluster running process to stop whenever there is a Matlab coding error. To disable this for this batch, you can run "ctrl.cluster.stop_on_error=false". Fix the coding bug printed above which might require running cluster_compile.m if you change cluster task code and then run "dbcont". You may also run "cluster_run_mode=-1" to stop cluster_run.m in a clean way (it will complete updating this branch and then exit). You can test the function by running "cluster_exec_task(ctrl,task_id);". If the task completes successfully, then the error mask can be set to zero by running "ctrl.error_mask(task_id)=0;".\n'); keyboard end end @@ -309,27 +336,28 @@ fprintf(' Max memory potentially exceeded\n'); fprintf(' Job max_mem used is %.1f GB\n', max_mem/1e9); fprintf(' Task id %d:%d\n', ctrl.batch_id, task_id); - fprintf(' Task memory requested %.1f GB\n', ctrl.mem(task_id)/1e9); + fprintf(' Task memory requested %.1f*%.1f = %.1f GB\n', ctrl.mem(task_id)/1e9, ctrl.cluster.mem_mult, ctrl.mem(task_id)*ctrl.cluster.mem_mult/1e9); fprintf(' Job''s last executed task id %d\n', last_task_id); end - if bitand(ctrl.error_mask(task_id),max_mem_exceeded_error) && task_id == last_task_id + if bitand(ctrl.error_mask(task_id),matlab_task_out_of_memory) ... + || bitand(ctrl.error_mask(task_id),max_mem_exceeded_error) && task_id == last_task_id fprintf(' Task max memory exceeded.\n'); - if ~isempty(regexpi(ctrl.cluster.max_mem_mode,'debug')) - ctrl.cluster.max_mem_mode - fprintf(' task memory (%.1f GB) exceeded the maximum memory requested (%.1f GB):\n', max_mem/1e9, ctrl.mem(task_id)*ctrl.cluster.mem_mult/1e9); + if ~isempty(regexpi(ctrl.cluster.mem_mult_mode,'debug')) + ctrl.cluster.mem_mult_mode + fprintf(' task memory (%.1f GB) exceeded the maximum memory requested (%.1f*%.1f = %.1f GB):\n', max_mem/1e9, ctrl.mem(task_id)/1e9, ctrl.cluster.mem_mult, ctrl.mem(task_id)*ctrl.cluster.mem_mult/1e9); fprintf('%s\n',ones(1,80)*'='); fprintf('Options:\n'); fprintf(' 1. Increase ctrl.mem(task_id) or ctrl.cluster.mem_mult\n'); fprintf(' 2. Run job locally by running cluster_exec_task(ctrl,task_id);\n'); fprintf(' Be sure to run ctrl.error_mask(task_id) = 0 after successfully\n'); fprintf(' running task.\n'); - fprintf(' 3. Set ctrl.cluster.max_mem_mode = ''auto'';\n'); + fprintf(' 3. Set ctrl.cluster.mem_mult_mode = ''auto''; to automatically increase the memory requested.\n'); fprintf('After making changes, run dbcont to continue.\n'); keyboard end - if ~isempty(regexpi(ctrl.cluster.max_mem_mode,'auto')) - fprintf(' Automatically doubling memory request for this task.\n'); - ctrl.mem(task_id) = max_mem*1.5/ctrl.cluster.mem_mult; + if ~isempty(regexpi(ctrl.cluster.mem_mult_mode,'auto')) + fprintf(' Automatically 1.5x the memory request for this task.\n'); + ctrl.mem(task_id) = max(max_mem/ctrl.cluster.mem_mult,ctrl.mem(task_id))*1.5; end end if bitand(ctrl.error_mask(task_id),insufficient_mcr_space) @@ -346,6 +374,8 @@ end if bitand(ctrl.error_mask(task_id),walltime_exceeded_error) fprintf(' Cluster killed this job due to wall time. This means the job requested too little cpu time. cluster.cpu_time_mult should be increased.\n'); + fprintf(' Automatically doubling wall time request for this task.\n'); + ctrl.cpu_time(task_id) = ctrl.cpu_time(task_id)*2/ctrl.cluster.cpu_time_mult; end if bitand(ctrl.error_mask(task_id),file_success_error) fprintf(' File success check failed (missing files)\n'); @@ -362,11 +392,11 @@ if update_mode if ctrl.cluster.cpu_time_mult*ctrl.cpu_time(task_id)*0.9 < ctrl.cpu_time_actual(task_id) - warning(' %d:%d/%d: CPU time actual (%.0f sec) is more than 90%% of estimated time (%.0f sec). Consider revising estimates.', ... - ctrl.batch_id, task_id, job_id, ctrl.cpu_time_actual(task_id), ctrl.cluster.cpu_time_mult*ctrl.cpu_time(task_id)); + warning(' %d:%d/%d: CPU time actual (%.0f sec) is more than 90%% of estimated time (%.0f sec). Consider revising estimates. (%s)', ... + ctrl.batch_id, task_id, job_id, ctrl.cpu_time_actual(task_id), ctrl.cluster.cpu_time_mult*ctrl.cpu_time(task_id), datestr(now)); elseif ctrl.cpu_time_actual(task_id)>0 && ctrl.cluster.cpu_time_mult*ctrl.cpu_time(task_id)*0.2 > ctrl.cpu_time_actual(task_id) - warning(' %d:%d/%d: CPU time actual (%.0f sec) is less than 20%% of estimated time (%.0f sec). Consider revising estimates.', ... - ctrl.batch_id, task_id, job_id, ctrl.cpu_time_actual(task_id), ctrl.cluster.cpu_time_mult*ctrl.cpu_time(task_id)); + warning(' %d:%d/%d: CPU time actual (%.0f sec) is less than 20%% of estimated time (%.0f sec). Consider revising estimates. (%s)', ... + ctrl.batch_id, task_id, job_id, ctrl.cpu_time_actual(task_id), ctrl.cluster.cpu_time_mult*ctrl.cpu_time(task_id), datestr(now)); end end diff --git a/cresis-toolbox/ct_support/@layerdata/callback_closePB.m b/cresis-toolbox/ct_support/@layerdata/callback_closePB.m new file mode 100644 index 00000000..c23c6221 --- /dev/null +++ b/cresis-toolbox/ct_support/@layerdata/callback_closePB.m @@ -0,0 +1,3 @@ +function callback_closePB(obj,status,event) + +delete(obj); diff --git a/cresis-toolbox/ct_support/@layerdata/callback_importPB.m b/cresis-toolbox/ct_support/@layerdata/callback_importPB.m new file mode 100644 index 00000000..22e56ad9 --- /dev/null +++ b/cresis-toolbox/ct_support/@layerdata/callback_importPB.m @@ -0,0 +1,23 @@ +function callback_importPB(obj,status,event) + +%% Get import directory +% ========================================================================= +folder_name = uigetdir(ct_filename_out(obj.param,obj.layerdata_source,[],1),'Choose layer data folder'); +if isnumeric(folder_name) + % User canceled in uigetdir + return; +end + +%% Load layers +% ========================================================================= +obj.gui_layers{end+1} = layerdata(obj.param,folder_name); +obj.gui_layers{end}.check_layer_organizer(); +obj.gui_layers{end}.check_all_frames(); + +%% Set auto-merge settings +% ========================================================================= + + +%% Update user interface +% ========================================================================= +obj.update_ui(); \ No newline at end of file diff --git a/cresis-toolbox/ct_support/@layerdata/callback_layersSB.m b/cresis-toolbox/ct_support/@layerdata/callback_layersSB.m new file mode 100644 index 00000000..1ee2f2c5 --- /dev/null +++ b/cresis-toolbox/ct_support/@layerdata/callback_layersSB.m @@ -0,0 +1,7 @@ +function callback_layersSB(obj,status,event) + +% Populate Metadata window + +% Order by twtt + +% Other context menu options \ No newline at end of file diff --git a/cresis-toolbox/ct_support/@layerdata/callback_plotCB.m b/cresis-toolbox/ct_support/@layerdata/callback_plotCB.m new file mode 100644 index 00000000..57e20903 --- /dev/null +++ b/cresis-toolbox/ct_support/@layerdata/callback_plotCB.m @@ -0,0 +1,5 @@ +function callback_plotCB(obj,status,event) + +status + +event diff --git a/cresis-toolbox/ct_support/@layerdata/callback_presetPB.m b/cresis-toolbox/ct_support/@layerdata/callback_presetPB.m new file mode 100644 index 00000000..db593671 --- /dev/null +++ b/cresis-toolbox/ct_support/@layerdata/callback_presetPB.m @@ -0,0 +1,7 @@ +function callback_presetPB(obj,status,event) + +% Update the merge settings + +% Update user interface +obj.update_ui(); + diff --git a/cresis-toolbox/ct_support/@layerdata/callback_savePB.m b/cresis-toolbox/ct_support/@layerdata/callback_savePB.m new file mode 100644 index 00000000..3be93792 --- /dev/null +++ b/cresis-toolbox/ct_support/@layerdata/callback_savePB.m @@ -0,0 +1,6 @@ +function callback_savePB(obj,status,event) + +% Save layers + +% Delete layer object and close GUI +delete(obj); diff --git a/cresis-toolbox/ct_support/@layerdata/callback_save_metadataPB.m b/cresis-toolbox/ct_support/@layerdata/callback_save_metadataPB.m new file mode 100644 index 00000000..0c8eed31 --- /dev/null +++ b/cresis-toolbox/ct_support/@layerdata/callback_save_metadataPB.m @@ -0,0 +1,5 @@ +function callback_save_metadataPB(obj,status,event) + +status + +event diff --git a/cresis-toolbox/ct_support/@layerdata/create_ui.m b/cresis-toolbox/ct_support/@layerdata/create_ui.m new file mode 100644 index 00000000..8db72c74 --- /dev/null +++ b/cresis-toolbox/ct_support/@layerdata/create_ui.m @@ -0,0 +1,438 @@ +function create_ui(obj) +% layerdata.create_ui(obj) +% +% Function for creating the graphical user interface + +if ~isempty(obj.h_fig) + % GUI already created, so just bring to the foreground and return + set(obj.h_fig,'Visible','on'); + set(obj.h_fig_twtt,'Visible','on'); + figure(obj.h_fig_twtt); + figure(obj.h_fig); + return; +end + +%% Layer Figures +% ========================================================================= +obj.h_fig = figure; +set(obj.h_fig,'Units','Points','Position',[35.2500 300.7500 420.0000 315.0000]); +set(obj.h_fig,'DockControls','off') +set(obj.h_fig,'NumberTitle','off'); +if strcmpi(class(obj.h_fig),'double') + set(obj.h_fig,'Name',sprintf('%d: layerdata %s',obj.h_fig, obj.param.day_seg)); +else + set(obj.h_fig,'Name',sprintf('%d: layerdata %s',obj.h_fig.Number, obj.param.day_seg)); +end +set(obj.h_fig,'ToolBar','none'); +set(obj.h_fig,'MenuBar','none'); +set(obj.h_fig,'CloseRequestFcn',@obj.callback_closePB); + +obj.h_fig_twtt = figure; +obj.h_axes_twtt = axes('parent',obj.h_fig_twtt); +set(obj.h_fig_twtt,'DockControls','off') +set(obj.h_fig_twtt,'NumberTitle','off'); +if strcmpi(class(obj.h_fig_twtt),'double') + set(obj.h_fig_twtt,'Name',sprintf('%d: twtt layerdata %s',obj.h_fig_twtt, obj.param.day_seg)); +else + set(obj.h_fig_twtt,'Name',sprintf('%d: twtt layerdata %s',obj.h_fig_twtt.Number, obj.param.day_seg)); +end +set(obj.h_fig_twtt,'ToolBar','none'); +set(obj.h_fig_twtt,'MenuBar','none'); +set(obj.h_fig_twtt,'CloseRequestFcn',@obj.callback_closePB); + +obj.h_fig_metadata = figure; +set(obj.h_fig_metadata,'Units','Points','Position',[35.2500 638.2500 420.0000 201.0000]); +set(obj.h_fig_metadata,'DockControls','off') +set(obj.h_fig_metadata,'NumberTitle','off'); +if strcmpi(class(obj.h_fig_metadata),'double') + set(obj.h_fig_metadata,'Name',sprintf('%d: metadata layerdata %s',obj.h_fig_metadata, obj.param.day_seg)); +else + set(obj.h_fig_metadata,'Name',sprintf('%d: metadata layerdata %s',obj.h_fig_metadata.Number, obj.param.day_seg)); +end +set(obj.h_fig_metadata,'ToolBar','none'); +set(obj.h_fig_metadata,'MenuBar','none'); +set(obj.h_fig_metadata,'CloseRequestFcn',@obj.callback_closePB); + +%% Main Figure: Create the widgets +% ========================================================================= + +% Layer selection class (populate later from preference file) +obj.h_gui.h_layers = selectionbox(obj.h_fig,'Layers',[],1); +set(obj.h_gui.h_layers.h_list_available,'TooltipString','Layers to delete (double or right click to keep).'); +set(obj.h_gui.h_layers.h_list_selected,'TooltipString','Layers to keep (double or right click to delete).'); +obj.h_gui.h_layers.set_enable(true); + +uimenu(obj.h_gui.h_layers.h_list_availableCM, 'Label', 'Merge: diff frames', 'Callback', @obj.callback_layers_SB); +uimenu(obj.h_gui.h_layers.h_list_availableCM, 'Label', 'Merge: overwrite', 'Callback', @obj.callback_layers_SB); +uimenu(obj.h_gui.h_layers.h_list_availableCM, 'Label', 'Merge: NaN', 'Callback', @obj.callback_layers_SB); +uimenu(obj.h_gui.h_layers.h_list_availableCM, 'Label', '-', 'Callback', @obj.callback_layers_SB); +uimenu(obj.h_gui.h_layers.h_list_availableCM, 'Label', 'Order by twtt', 'Callback', @obj.callback_layers_SB); +uimenu(obj.h_gui.h_layers.h_list_availableCM, 'Label', 'Sequence layer names', 'Callback', @obj.callback_layers_SB); +uimenu(obj.h_gui.h_layers.h_list_availableCM, 'Label', 'Set name based on twtt matches', 'Callback', @obj.callback_layers_SB); +uimenu(obj.h_gui.h_layers.h_list_availableCM, 'Label', 'Undo name changes', 'Callback', @obj.callback_layers_SB); + +% Plot Deleted Button +obj.h_gui.plot_deletedCB = uicontrol('Parent',obj.h_fig); +set(obj.h_gui.plot_deletedCB,'Style','CheckBox'); +set(obj.h_gui.plot_deletedCB,'Value',1); +set(obj.h_gui.plot_deletedCB,'String','Plot Delete Layers'); +set(obj.h_gui.plot_deletedCB,'Callback',@obj.callback_plotCB); +set(obj.h_gui.plot_deletedCB,'TooltipString','Check to plot layers in the to be deleted listbox. Layers are plotted as solid blue lines.'); + +% Plot Keep Button +obj.h_gui.plot_keepCB = uicontrol('Parent',obj.h_fig); +set(obj.h_gui.plot_keepCB,'Style','CheckBox'); +set(obj.h_gui.plot_keepCB,'Value',1); +set(obj.h_gui.plot_keepCB,'String','Plot Keep Layers'); +set(obj.h_gui.plot_keepCB,'Callback',@obj.callback_plotCB); +set(obj.h_gui.plot_keepCB,'TooltipString','Check to plot layers in the keep listbox. Layers are plotted as dashed red lines.'); + +% Import Button +obj.h_gui.importPB = uicontrol('Parent',obj.h_fig); +set(obj.h_gui.importPB,'Style','PushButton'); +set(obj.h_gui.importPB,'String','Import'); +set(obj.h_gui.importPB,'Callback',@obj.callback_importPB); +set(obj.h_gui.importPB,'TooltipString','Check to plot layers in the to be deleted listbox.'); + +% Merge Preset Button +obj.h_gui.merge_presetPB = uicontrol('Parent',obj.h_fig); +set(obj.h_gui.merge_presetPB,'Style','PushButton'); +set(obj.h_gui.merge_presetPB,'String','Merge Preset'); +set(obj.h_gui.merge_presetPB,'Callback',@obj.callback_presetPB); +set(obj.h_gui.merge_presetPB,'TooltipString','Check to plot layers in the to be deleted listbox.'); + +% Save Button +obj.h_gui.savePB = uicontrol('Parent',obj.h_fig); +set(obj.h_gui.savePB,'Style','PushButton'); +set(obj.h_gui.savePB,'String','Save and Close'); +set(obj.h_gui.savePB,'Callback',@obj.callback_savePB); +set(obj.h_gui.savePB,'TooltipString','Save and close.'); + +% Close Button +obj.h_gui.closePB = uicontrol('Parent',obj.h_fig); +set(obj.h_gui.closePB,'Style','PushButton'); +set(obj.h_gui.closePB,'String','Close without Saving'); +set(obj.h_gui.closePB,'Callback',@obj.callback_closePB); +set(obj.h_gui.closePB,'TooltipString','Close without saving.'); + +%% Main Figure: Create the table +% ========================================================================= +obj.h_gui.table.ui=obj.h_fig; +obj.h_gui.table.width_margin = NaN*zeros(30,30); % Just make these bigger than they have to be +obj.h_gui.table.height_margin = NaN*zeros(30,30); +obj.h_gui.table.false_width = NaN*zeros(30,30); +obj.h_gui.table.false_height = NaN*zeros(30,30); +obj.h_gui.table.offset = [0 0]; + +row = 1; col = 1; +obj.h_gui.table.handles{row,col} = obj.h_gui.h_layers.h_text; +obj.h_gui.table.width(row,col) = inf; +obj.h_gui.table.height(row,col) = 20; +obj.h_gui.table.width_margin(row,col) = 1; +obj.h_gui.table.height_margin(row,col) = 1; + +col = 2; +obj.h_gui.table.handles{row,col} = []; +obj.h_gui.table.width(row,col) = inf; +obj.h_gui.table.height(row,col) = 20; +obj.h_gui.table.width_margin(row,col) = 1; +obj.h_gui.table.height_margin(row,col) = 1; + +row = row + 1; col = 1; +obj.h_gui.table.handles{row,col} = obj.h_gui.h_layers.h_list_available; +obj.h_gui.table.width(row,col) = inf; +obj.h_gui.table.height(row,col) = inf; +obj.h_gui.table.width_margin(row,col) = 1; +obj.h_gui.table.height_margin(row,col) = 1; + +col = 2; +obj.h_gui.table.handles{row,col} = obj.h_gui.h_layers.h_list_selected; +obj.h_gui.table.width(row,col) = inf; +obj.h_gui.table.height(row,col) = inf; +obj.h_gui.table.width_margin(row,col) = 1; +obj.h_gui.table.height_margin(row,col) = 1; + +row = row + 1; col = 1; +obj.h_gui.table.handles{row,col} = obj.h_gui.plot_deletedCB; +obj.h_gui.table.width(row,col) = inf; +obj.h_gui.table.height(row,col) = 25; +obj.h_gui.table.width_margin(row,col) = 1; +obj.h_gui.table.height_margin(row,col) = 1; + +col = 2; +obj.h_gui.table.handles{row,col} = obj.h_gui.plot_keepCB; +obj.h_gui.table.width(row,col) = inf; +obj.h_gui.table.height(row,col) = 25; +obj.h_gui.table.width_margin(row,col) = 1; +obj.h_gui.table.height_margin(row,col) = 1; + +row = row + 1; col = 1; +obj.h_gui.table.handles{row,col} = obj.h_gui.importPB; +obj.h_gui.table.width(row,col) = inf; +obj.h_gui.table.height(row,col) = 25; +obj.h_gui.table.width_margin(row,col) = 1; +obj.h_gui.table.height_margin(row,col) = 1; + +col = 2; +obj.h_gui.table.handles{row,col} = obj.h_gui.merge_presetPB; +obj.h_gui.table.width(row,col) = inf; +obj.h_gui.table.height(row,col) = 25; +obj.h_gui.table.width_margin(row,col) = 1; +obj.h_gui.table.height_margin(row,col) = 1; + +row = row + 1; col = 1; +obj.h_gui.table.handles{row,col} = obj.h_gui.savePB; +obj.h_gui.table.width(row,col) = inf; +obj.h_gui.table.height(row,col) = 25; +obj.h_gui.table.width_margin(row,col) = 1; +obj.h_gui.table.height_margin(row,col) = 1; + +col = 2; +obj.h_gui.table.handles{row,col} = obj.h_gui.closePB; +obj.h_gui.table.width(row,col) = inf; +obj.h_gui.table.height(row,col) = 25; +obj.h_gui.table.width_margin(row,col) = 1; +obj.h_gui.table.height_margin(row,col) = 1; + +clear row col +table_draw(obj.h_gui.table); + + +%% Metadata Figure: Create the widgets +% ========================================================================= + +% Layer age label +obj.h_gui_metadata.age_text = uicontrol('Parent',obj.h_fig_metadata); +set(obj.h_gui_metadata.age_text,'Style','Text'); +set(obj.h_gui_metadata.age_text,'String','Layer age:'); +tooltip_str = 'Layer age scalar. Age represents when the layer was formed on the ice surface in Julian years.'; +set(obj.h_gui_metadata.age_text,'TooltipString',tooltip_str); + +% Layer age edit +obj.h_gui_metadata.age_edit = uicontrol('Parent',obj.h_fig_metadata); +set(obj.h_gui_metadata.age_edit,'Style','Edit'); +set(obj.h_gui_metadata.age_edit,'String',''); +set(obj.h_gui_metadata.age_edit,'TooltipString',tooltip_str); + +% Layer age source label +obj.h_gui_metadata.age_source_text = uicontrol('Parent',obj.h_fig_metadata); +set(obj.h_gui_metadata.age_source_text,'Style','Text'); +set(obj.h_gui_metadata.age_source_text,'String','Layer age source:'); +tooltip_str = 'Layer age scalar. Age represents when the layer was formed on the ice surface in Julian years.'; +set(obj.h_gui_metadata.age_source_text,'TooltipString',tooltip_str); + +% Layer age source edit +obj.h_gui_metadata.age_source_edit = uicontrol('Parent',obj.h_fig_metadata); +set(obj.h_gui_metadata.age_source_edit,'Style','Edit'); +set(obj.h_gui_metadata.age_source_edit,'String',''); +set(obj.h_gui_metadata.age_source_edit,'TooltipString',tooltip_str); + +% Layer age source age label +obj.h_gui_metadata.age_source_age_text = uicontrol('Parent',obj.h_fig_metadata); +set(obj.h_gui_metadata.age_source_age_text,'Style','Text'); +set(obj.h_gui_metadata.age_source_age_text,'String','Layer age source age:'); +tooltip_str = 'Layer age scalar. Age represents when the layer was formed on the ice surface in Julian years.'; +set(obj.h_gui_metadata.age_source_age_text,'TooltipString',tooltip_str); + +% Layer age source age edit +obj.h_gui_metadata.age_source_age_edit = uicontrol('Parent',obj.h_fig_metadata); +set(obj.h_gui_metadata.age_source_age_edit,'Style','Edit'); +set(obj.h_gui_metadata.age_source_age_edit,'String',''); +set(obj.h_gui_metadata.age_source_age_edit,'TooltipString',tooltip_str); + +% Layer age source type label +obj.h_gui_metadata.age_source_type_text = uicontrol('Parent',obj.h_fig_metadata); +set(obj.h_gui_metadata.age_source_type_text,'Style','Text'); +set(obj.h_gui_metadata.age_source_type_text,'String','Layer age source type:'); +tooltip_str = 'Layer age scalar. Age represents when the layer was formed on the ice surface in Julian years.'; +set(obj.h_gui_metadata.age_source_type_text,'TooltipString',tooltip_str); + +% Layer age source type edit +obj.h_gui_metadata.age_source_type_edit = uicontrol('Parent',obj.h_fig_metadata); +set(obj.h_gui_metadata.age_source_type_edit,'Style','Edit'); +set(obj.h_gui_metadata.age_source_type_edit,'String',''); +set(obj.h_gui_metadata.age_source_type_edit,'TooltipString',tooltip_str); + +% Layer description label +obj.h_gui_metadata.description_text = uicontrol('Parent',obj.h_fig_metadata); +set(obj.h_gui_metadata.description_text,'Style','Text'); +set(obj.h_gui_metadata.description_text,'String','Layer description:'); +tooltip_str = 'String containing the layer description.'; +set(obj.h_gui_metadata.description_text,'TooltipString',tooltip_str); + +% Layer description edit +obj.h_gui_metadata.description_edit = uicontrol('Parent',obj.h_fig_metadata); +set(obj.h_gui_metadata.description_edit,'Style','Edit'); +set(obj.h_gui_metadata.description_edit,'String',''); +set(obj.h_gui_metadata.description_edit,'TooltipString',tooltip_str); + +% Layer group name label +obj.h_gui_metadata.group_name_text = uicontrol('Parent',obj.h_fig_metadata); +set(obj.h_gui_metadata.group_name_text,'Style','Text'); +set(obj.h_gui_metadata.group_name_text,'String','Layer group name:'); +tooltip_str = 'String containing the group name.'; +set(obj.h_gui_metadata.group_name_text,'TooltipString',tooltip_str); + +% Layer group name edit +obj.h_gui_metadata.group_name_edit = uicontrol('Parent',obj.h_fig_metadata); +set(obj.h_gui_metadata.group_name_edit,'Style','Edit'); +set(obj.h_gui_metadata.group_name_edit,'String',''); +set(obj.h_gui_metadata.group_name_edit,'TooltipString',tooltip_str); + +% Layer name label +obj.h_gui_metadata.name_text = uicontrol('Parent',obj.h_fig_metadata); +set(obj.h_gui_metadata.name_text,'Style','Text'); +set(obj.h_gui_metadata.name_text,'String','Layer name:'); +tooltip_str = 'String containing the layer name. Layer names must be unique.'; +set(obj.h_gui_metadata.name_text,'TooltipString',tooltip_str); + +% Layer name edit +obj.h_gui_metadata.name_edit = uicontrol('Parent',obj.h_fig_metadata); +set(obj.h_gui_metadata.name_edit,'Style','Edit'); +set(obj.h_gui_metadata.name_edit,'String',''); +set(obj.h_gui_metadata.name_edit,'TooltipString',tooltip_str); + +% Save Metadata Button +obj.h_gui_metadata.save_metadataPB = uicontrol('Parent',obj.h_fig_metadata); +set(obj.h_gui_metadata.save_metadataPB,'Style','PushButton'); +set(obj.h_gui_metadata.save_metadataPB,'String','Save Metadata'); +set(obj.h_gui_metadata.save_metadataPB,'Callback',@obj.callback_save_metadataPB); +set(obj.h_gui_metadata.save_metadataPB,'TooltipString','Press "Save Metadata" to update the metadata for the selected layer.'); + +%% Metadata Figure: Create the table +% ========================================================================= +obj.h_gui_metadata.table.ui=obj.h_fig_metadata; +obj.h_gui_metadata.table.width_margin = NaN*zeros(30,30); % Just make these bigger than they have to be +obj.h_gui_metadata.table.height_margin = NaN*zeros(30,30); +obj.h_gui_metadata.table.false_width = NaN*zeros(30,30); +obj.h_gui_metadata.table.false_height = NaN*zeros(30,30); +obj.h_gui_metadata.table.offset = [0 0]; + +row = 1; col = 1; +obj.h_gui_metadata.table.handles{row,col} = obj.h_gui_metadata.age_text; +obj.h_gui_metadata.table.width(row,col) = inf; +obj.h_gui_metadata.table.height(row,col) = 25; +obj.h_gui_metadata.table.width_margin(row,col) = 1; +obj.h_gui_metadata.table.height_margin(row,col) = 1; + +col = 2; +obj.h_gui_metadata.table.handles{row,col} = obj.h_gui_metadata.age_edit; +obj.h_gui_metadata.table.width(row,col) = inf; +obj.h_gui_metadata.table.height(row,col) = 25; +obj.h_gui_metadata.table.width_margin(row,col) = 1; +obj.h_gui_metadata.table.height_margin(row,col) = 1; + +row = row + 1; col = 1; +obj.h_gui_metadata.table.handles{row,col} = obj.h_gui_metadata.age_source_text; +obj.h_gui_metadata.table.width(row,col) = inf; +obj.h_gui_metadata.table.height(row,col) = 25; +obj.h_gui_metadata.table.width_margin(row,col) = 1; +obj.h_gui_metadata.table.height_margin(row,col) = 1; + +col = 2; +obj.h_gui_metadata.table.handles{row,col} = obj.h_gui_metadata.age_source_edit; +obj.h_gui_metadata.table.width(row,col) = inf; +obj.h_gui_metadata.table.height(row,col) = 25; +obj.h_gui_metadata.table.width_margin(row,col) = 1; +obj.h_gui_metadata.table.height_margin(row,col) = 1; + +row = row + 1; col = 1; +obj.h_gui_metadata.table.handles{row,col} = obj.h_gui_metadata.age_source_age_text; +obj.h_gui_metadata.table.width(row,col) = inf; +obj.h_gui_metadata.table.height(row,col) = 25; +obj.h_gui_metadata.table.width_margin(row,col) = 1; +obj.h_gui_metadata.table.height_margin(row,col) = 1; + +col = 2; +obj.h_gui_metadata.table.handles{row,col} = obj.h_gui_metadata.age_source_age_edit; +obj.h_gui_metadata.table.width(row,col) = inf; +obj.h_gui_metadata.table.height(row,col) = 25; +obj.h_gui_metadata.table.width_margin(row,col) = 1; +obj.h_gui_metadata.table.height_margin(row,col) = 1; + +row = row + 1; col = 1; +obj.h_gui_metadata.table.handles{row,col} = obj.h_gui_metadata.age_source_type_text; +obj.h_gui_metadata.table.width(row,col) = inf; +obj.h_gui_metadata.table.height(row,col) = 25; +obj.h_gui_metadata.table.width_margin(row,col) = 1; +obj.h_gui_metadata.table.height_margin(row,col) = 1; + +col = 2; +obj.h_gui_metadata.table.handles{row,col} = obj.h_gui_metadata.age_source_type_edit; +obj.h_gui_metadata.table.width(row,col) = inf; +obj.h_gui_metadata.table.height(row,col) = 25; +obj.h_gui_metadata.table.width_margin(row,col) = 1; +obj.h_gui_metadata.table.height_margin(row,col) = 1; + +row = row + 1; col = 1; +obj.h_gui_metadata.table.handles{row,col} = obj.h_gui_metadata.description_text; +obj.h_gui_metadata.table.width(row,col) = inf; +obj.h_gui_metadata.table.height(row,col) = 25; +obj.h_gui_metadata.table.width_margin(row,col) = 1; +obj.h_gui_metadata.table.height_margin(row,col) = 1; + +col = 2; +obj.h_gui_metadata.table.handles{row,col} = obj.h_gui_metadata.description_edit; +obj.h_gui_metadata.table.width(row,col) = inf; +obj.h_gui_metadata.table.height(row,col) = 25; +obj.h_gui_metadata.table.width_margin(row,col) = 1; +obj.h_gui_metadata.table.height_margin(row,col) = 1; + +row = row + 1; col = 1; +obj.h_gui_metadata.table.handles{row,col} = obj.h_gui_metadata.group_name_text; +obj.h_gui_metadata.table.width(row,col) = inf; +obj.h_gui_metadata.table.height(row,col) = 25; +obj.h_gui_metadata.table.width_margin(row,col) = 1; +obj.h_gui_metadata.table.height_margin(row,col) = 1; + +col = 2; +obj.h_gui_metadata.table.handles{row,col} = obj.h_gui_metadata.group_name_edit; +obj.h_gui_metadata.table.width(row,col) = inf; +obj.h_gui_metadata.table.height(row,col) = 25; +obj.h_gui_metadata.table.width_margin(row,col) = 1; +obj.h_gui_metadata.table.height_margin(row,col) = 1; + +row = row + 1; col = 1; +obj.h_gui_metadata.table.handles{row,col} = obj.h_gui_metadata.name_text; +obj.h_gui_metadata.table.width(row,col) = inf; +obj.h_gui_metadata.table.height(row,col) = 25; +obj.h_gui_metadata.table.width_margin(row,col) = 1; +obj.h_gui_metadata.table.height_margin(row,col) = 1; + +col = 2; +obj.h_gui_metadata.table.handles{row,col} = obj.h_gui_metadata.name_edit; +obj.h_gui_metadata.table.width(row,col) = inf; +obj.h_gui_metadata.table.height(row,col) = 25; +obj.h_gui_metadata.table.width_margin(row,col) = 1; +obj.h_gui_metadata.table.height_margin(row,col) = 1; + +row = row + 1; col = 1; +obj.h_gui_metadata.table.handles{row,col} = []; +obj.h_gui_metadata.table.width(row,col) = inf; +obj.h_gui_metadata.table.height(row,col) = 25; +obj.h_gui_metadata.table.width_margin(row,col) = 1; +obj.h_gui_metadata.table.height_margin(row,col) = 1; + +col = 2; +obj.h_gui_metadata.table.handles{row,col} = obj.h_gui_metadata.save_metadataPB; +obj.h_gui_metadata.table.width(row,col) = inf; +obj.h_gui_metadata.table.height(row,col) = 25; +obj.h_gui_metadata.table.width_margin(row,col) = 1; +obj.h_gui_metadata.table.height_margin(row,col) = 1; + +clear row col +table_draw(obj.h_gui_metadata.table); + +%% Load layers +% ========================================================================= +obj.gui_layers{end+1} = obj; +obj.gui_layers{end}.check_layer_organizer(); +obj.gui_layers{end}.check_all_frames(); + +%% Set auto-merge settings +% ========================================================================= + + +%% Update user interface +% ========================================================================= +obj.update_ui(); diff --git a/cresis-toolbox/ct_support/layerdata.m b/cresis-toolbox/ct_support/@layerdata/layerdata.m similarity index 65% rename from cresis-toolbox/ct_support/layerdata.m rename to cresis-toolbox/ct_support/@layerdata/layerdata.m index 3a4353a1..7d49458c 100644 --- a/cresis-toolbox/ct_support/layerdata.m +++ b/cresis-toolbox/ct_support/@layerdata/layerdata.m @@ -42,13 +42,23 @@ sample_spacing_method % 'records' or 'along_track' (default) along_track_spacing % either 5 m (non-rds) or 15 m (rds) by default record_spacing % 200 by default - + + % GUI + h_fig + h_fig_twtt + h_fig_metadata + h_axes_twtt + h_gui + h_gui_metadata + gui_layers + end - %% Private Methods + %% Private Methods ========== methods (Access = private) %% create: create layer file + % NOTE: just creates the file structure, but does not save it function create(obj,frm) obj.check_records(); @@ -91,6 +101,8 @@ function create(obj,frm) obj.layer{frm}.quality = zeros(0,size(1,Nx),'uint8'); obj.layer{frm}.type = zeros(0,size(1,Nx),'uint8'); obj.layer_modified(frm) = true; + % Ensure alphabetical ordering so that fields can be concatenated efficiently + obj.layer{frm} = orderfields(obj.layer{frm}); end %% create_layer_organizer: create layer organizer if it does not exist @@ -111,9 +123,87 @@ function create_layer_organizer(obj,frm) obj.layer_organizer_modified = true; end + + %% load: load layer + % DO NOT CALL THIS LOAD FUNCTION DIRECTLY: call check.m + function load(obj,frm) + layer_fn = fullfile(ct_filename_out(obj.param,obj.layerdata_source,''),sprintf('Data_%s_%03d.mat',obj.param.day_seg,frm)); + if ~exist(layer_fn,'file') + warning('Layer file does not exist. Creating %s\n', layer_fn); + obj.create(frm); + else + obj.layer{frm} = load(layer_fn); + obj.layer_modified(frm) = false; + obj.fix(frm); + end + end + + %% load_frames: load frames + % DO NOT CALL THIS LOAD FUNCTION DIRECTLY: call check_frames.m + function load_frames(obj) + obj.frames = frames_load(obj.param); + end + + %% load_layer_organizer: load layer organizer + % DO NOT CALL THIS LOAD FUNCTION DIRECTLY: call check_layer_organizer.m + function load_layer_organizer(obj) + layer_organizer_fn = fullfile(ct_filename_out(obj.param,obj.layerdata_source,''),sprintf('layer_%s.mat',obj.param.day_seg)); + if ~exist(layer_organizer_fn,'file') + obj.create_layer_organizer(); + else + obj.layer_organizer = load(layer_organizer_fn); + obj.layer_organizer_modified = false; + obj.fix_layer_organizer(); + end + % If old files, then we need to check one frames file (this code + % assumes all frames are the same format) to properly fill the + % layer_organizer field + obj.check(1); + end + + %% load_records: load records and frames + % DO NOT CALL THIS LOAD FUNCTION DIRECTLY: call check_records.m + function load_records(obj,records) + + if ~exist('records','var') || isempty(records) + % Load records file + obj.records = records_load(obj.param); + end + + % Ensure frames are loaded + obj.check_frames(); + + % Load/create-if-needed reference trajectory + obj.records = records_reference_trajectory_load(obj.param,obj.records); + obj.param.records.gps.time_offset = obj.records.param_records.records.gps.time_offset; + obj.param.radar.lever_arm_fh = obj.records.param_records.radar.lever_arm_fh; + + obj.records.along_track = geodetic_to_along_track(obj.records.lat,obj.records.lon); + + if isempty(obj.along_track_spacing) + sys = ct_output_dir(obj.param.radar_name); + switch (sys) + case {'rds','accum'} + obj.along_track_spacing = 15; + otherwise + obj.along_track_spacing = 5; + end + end + obj.along_track = [0:obj.along_track_spacing:obj.records.along_track(end)]; + + if isempty(obj.sample_spacing_method) + if length(obj.along_track) < 2 && length(obj.records.gps_time) > obj.record_spacing + obj.sample_spacing_method = 'records'; + else + obj.sample_spacing_method = 'along_track'; + end + end + + end + end - %% Public Methods + %% Public Methods ========== methods %% constructor function obj = layerdata(param,layerdata_source) @@ -121,6 +211,7 @@ function create_layer_organizer(obj,frm) layerdata_source = 'layer'; end obj.param = param; + obj.param.radar_name = ct_output_dir(obj.param.radar_name); obj.layerdata_source = layerdata_source; obj.frames = []; obj.records = []; @@ -133,23 +224,56 @@ function create_layer_organizer(obj,frm) obj.sample_spacing_method = ''; obj.along_track_spacing = []; obj.record_spacing = 200; + + % GUI + obj.h_fig = []; + obj.h_fig_twtt = []; + obj.h_fig_metadata = []; + obj.h_axes_twtt = []; + obj.h_gui = []; + obj.h_gui_metadata = []; + obj.gui_layers = []; + end + + %% destructor + function delete(obj) + try + delete(obj.h_fig); + end + try + delete(obj.h_fig_twtt); + end + try + delete(obj.h_fig_metadata); + end + for idx = 1:length(obj.gui_layers) + try + delete(obj.gui_layers{idx}); + end + end end - %% check: check that layer is loaded - function check(obj,frm) - if length(obj.layer) < frm || isempty(obj.layer{frm}) - % Load layer file - obj.load(frm); + %% check: check that layers are loaded for a particular set of frames + function check(obj,frms) + for frm = frms(:).' + if length(obj.layer) < frm || isempty(obj.layer{frm}) + % Load layer file + obj.load(frm); + end end end - %% check_all: check to make sure layer organizer and all layers loaded + %% check_all: check that records, frames, layer organizer and layers loaded function check_all(obj) obj.check_records(); obj.check_layer_organizer(); - for frm = 1:length(obj.frames.frame_idxs) - obj.check(frm); - end + obj.check(1:length(obj.frames.frame_idxs)); + end + + %% check: check that layers are loaded for all frames + function check_all_frames(obj) + obj.check_frames(); + obj.check(1:length(obj.frames.frame_idxs)); end %% check_layer_organizer: check that layer organizer is loaded @@ -159,6 +283,13 @@ function check_layer_organizer(obj) end end + %% check_frames: check that frames are loaded + function check_frames(obj) + if isempty(obj.frames) + obj.load_frames(); + end + end + %% check_records: check that records and frames are loaded % records: optionally records may be passed in from external source function check_records(obj,records) @@ -172,7 +303,7 @@ function check_records(obj,records) %% delete: delete layer file function delete_layer_file(obj,frms) for frm = frms(:).' - layer_fn = fullfile(ct_filename_out(obj.param,'',obj.layerdata_source),sprintf('Data_%s_%03d.mat',obj.param.day_seg,frm)); + layer_fn = fullfile(ct_filename_out(obj.param,obj.layerdata_source,''),sprintf('Data_%s_%03d.mat',obj.param.day_seg,frm)); try delete(layer_fn); end @@ -188,7 +319,7 @@ function delete_all(obj) %% delete_layer_organizer: delete layer organizer file function delete_layer_organizer(obj) - layer_organizer_fn = fullfile(ct_filename_out(obj.param,'',obj.layerdata_source),sprintf('layer_%s.mat',obj.param.day_seg)); + layer_organizer_fn = fullfile(ct_filename_out(obj.param,obj.layerdata_source,''),sprintf('layer_%s.mat',obj.param.day_seg)); try delete(layer_organizer_fn); end @@ -197,7 +328,7 @@ function delete_layer_organizer(obj) %% elev: get elev function elev = elev(obj,frms) elev = []; - for frm = 1:frms(:).' + for frm = frms(:).' obj.check(frm); elev(end+(1:length(obj.layer{frm}.elev))) = obj.layer{frm}.elev; end @@ -209,7 +340,8 @@ function fix(obj,frm) if ~isfield(obj.layer{frm},'file_version') % Old file format - warning('Old layer file format frame %d\n', frm); + layer_fn = fullfile(ct_filename_out(obj.param,obj.layerdata_source,''),sprintf('Data_%s_%03d.mat',obj.param.day_seg,frm)); + warning('Old layer file format frame %d: %s\n', frm, layer_fn); lay = obj.layer{frm}; obj.check_records(); obj.layer_modified(frm) = true; @@ -390,7 +522,7 @@ function fix(obj,frm) obj.layer_modified(frm) = true; end if ~isa(obj.layer{frm}.twtt,'double') - obj.layer{frm}.quality = double(obj.layer{frm}.quality); + obj.layer{frm}.twtt = double(obj.layer{frm}.twtt); obj.layer_modified(frm) = true; end if ~isa(obj.layer{frm}.quality,'uint8') @@ -464,12 +596,21 @@ function fix(obj,frm) end % Ensure all values are valid - mask = ~isfinite(obj.layer{frm}.quality < 1 | obj.layer{frm}.quality > 3); + mask = obj.layer{frm}.quality < 1 | obj.layer{frm}.quality > 3; if any(mask) obj.layer{frm}.quality(mask) = 1; obj.layer_modified(frm) = true; end - mask = ~isfinite(obj.layer{frm}.type < 1 | obj.layer{frm}.type > 4); + mask = isinf(obj.layer{frm}.twtt); + if any(mask) + obj.layer{frm}.twtt(mask) = NaN; + obj.layer_modified(frm) = true; + end + mask = abs(obj.layer{frm}.twtt) > 1; + if any(mask) + warning('abs(twtt) of some layers is > 1 which is probably incorrect. Manual correction is required if this is an error.'); + end + mask = obj.layer{frm}.type < 1 | obj.layer{frm}.type > 4; if any(mask) obj.layer{frm}.type(mask) = 2; obj.layer_modified(frm) = true; @@ -546,6 +687,24 @@ function fix(obj,frm) obj.layer_modified(frm) = true; end + % Check that GPS times are monotonically increasing (it should never + % not be monotonic, but in case files were corrupted or old files + % with problems) + [new_gps_time,new_idx] = unique(obj.layer{frm}.gps_time); + if ~isequal(new_gps_time,obj.layer{frm}.gps_time) + obj.layer_modified(frm) = true; + obj.layer{frm}.gps_time = new_gps_time; + obj.layer{frm}.lat = obj.layer{frm}.lat(new_idx); + obj.layer{frm}.lon = obj.layer{frm}.lon(new_idx); + obj.layer{frm}.elev = obj.layer{frm}.elev(new_idx); + obj.layer{frm}.quality = obj.layer{frm}.quality(:,new_idx); + obj.layer{frm}.twtt = obj.layer{frm}.twtt(:,new_idx); + obj.layer{frm}.type = obj.layer{frm}.type(:,new_idx); + end + + % Ensure alphabetical ordering so that fields can be concatenated efficiently + obj.layer{frm} = orderfields(obj.layer{frm}); + end %% fix_layer_organizer: fix any problems with layer organizer @@ -759,32 +918,57 @@ function fix_layer_organizer(obj) end end - %% get_id: get layer id from layer name - function id = get_id(obj,name) + %% get_id: get layer id and other layer information from layer id or name + function [id,name,group_name,desc,age,age_source] = get_id(obj,name) obj.check_layer_organizer(); - match_idx = find(strcmp(name,obj.layer_organizer.lyr_name)); - id = obj.layer_organizer.lyr_id(match_idx); + if ischar(name) + match_idx = find(strcmp(name,obj.layer_organizer.lyr_name),1); + else + match_idx = find(name == obj.layer_organizer.lyr_id,1); + end + if isempty(match_idx) + id = []; + age = []; + age_source = []; + desc = []; + group_name = []; + name = []; + else + id = obj.layer_organizer.lyr_id(match_idx); + age = obj.layer_organizer.lyr_age(match_idx); + age_source = obj.layer_organizer.lyr_age_source{match_idx}; + desc = obj.layer_organizer.lyr_desc{match_idx}; + group_name = obj.layer_organizer.lyr_group_name{match_idx}; + name = obj.layer_organizer.lyr_name{match_idx}; + end end %% get_layer: get layer twtt, quality, type function [twtt,quality,type] = get_layer(obj,frms,id) obj.check_layer_organizer(); + + % Get/check the layer id + % ------------------------------------------------------------------- if ischar(id) % name passed in rather than id - id = find(strcmpi(id,obj.layer_organizer.lyr_name)); - if isempty(id) - error('Layer does not exist in layer organizer. Run insert_layer() first.'); + match_idx = find(strcmpi(id,obj.layer_organizer.lyr_name)); + if isempty(match_idx) + error('Layer does not exist in layer organizer. Run insert_layers() first.'); end + id = obj.layer_organizer.lyr_id(match_idx); else % id passed in if all(obj.layer_organizer.lyr_id ~= id) - error('Layer does not exist in layer organizer. Run insert_layer() first.'); + error('Layer does not exist in layer organizer. Run insert_layers() first.'); end end + + % Load layer data + % ------------------------------------------------------------------- twtt = []; quality = []; type = []; - for frm = 1:frms(:).' + for frm = frms(:).' obj.check(frm); lay_idx = find(obj.layer{frm}.id == id); Nx = length(obj.layer{frm}.gps_time); @@ -802,10 +986,110 @@ function fix_layer_organizer(obj) end end + %% get_layer_by_gps_time: get layer twtt, quality, type + function [twtt,quality,type] = get_layer_by_gps_time(obj,query_gps_time,id) + obj.check_layer_organizer(); + + % Get/check the layer id + % ------------------------------------------------------------------- + if ischar(id) + % name passed in rather than id + match_idx = find(strcmpi(id,obj.layer_organizer.lyr_name)); + if isempty(match_idx) + error('Layer does not exist in layer organizer. Run insert_layers() first.'); + end + id = obj.layer_organizer.lyr_id(match_idx); + else + % id passed in + if all(obj.layer_organizer.lyr_id ~= id) + error('Layer does not exist in layer organizer. Run insert_layers() first.'); + end + end + + % Determine which frames to load + % ------------------------------------------------------------------- + obj.check_frames(); + + if isstruct(query_gps_time) + % Assume a echo structure (qlook.m/array.m output) such as + % CSARP_qlook or CSARP_standard + if ~isfield(query_gps_time,'GPS_time') + error('If query_gps_time is a struct, it must contain a field "GPS_time".'); + end + query_gps_time = query_gps_time.GPS_time; + end + + first_frm = find(min(query_gps_time) < obj.frames.gps_time(1:end-1),1); + first_frm = max(1,first_frm-2); + if isempty(first_frm) + first_frm = length(obj.frames.frame_idxs); + end + + last_frm = find(max(query_gps_time) < obj.frames.gps_time(1:end-1),1); + if isempty(last_frm) + last_frm = length(obj.frames.frame_idxs); + end + + frms = first_frm:last_frm; + + % Load layer data + % ------------------------------------------------------------------- + gps_time = []; + twtt = []; + quality = []; + type = []; + for frm = frms(:).' + obj.check(frm); + lay_idx = find(obj.layer{frm}.id == id); + Nx = length(obj.layer{frm}.gps_time); + gps_time(end+(1:Nx)) = obj.layer{frm}.gps_time; + if isempty(lay_idx) + % Layer does not exist in file, set to defaults + twtt(end+(1:Nx)) = NaN; + quality(end+(1:Nx)) = 1; + type(end+(1:Nx)) = 2; + else + % Layer exists, get values + twtt(end+(1:Nx)) = obj.layer{frm}.twtt(lay_idx,:); + quality(end+(1:Nx)) = obj.layer{frm}.quality(lay_idx,:); + type(end+(1:Nx)) = obj.layer{frm}.type(lay_idx,:); + end + end + + % Interpolate onto the gps_time provided + % ------------------------------------------------------------------- + twtt = interp1(gps_time,twtt,query_gps_time); + quality = interp1(gps_time,quality,query_gps_time,'nearest'); + type = interp1(gps_time,type,query_gps_time,'nearest'); + end + + %% get_layer_names: get available layer names + function layer_names = get_layer_names(obj,regexp_str) + check_layer_organizer(obj); + if nargin < 2 + % If no regular expression string is given, then return all layers + layer_names = obj.layer_organizer.lyr_name; + lyr_order = obj.layer_organizer.lyr_order; + else + layer_names = {}; + lyr_order = []; + idx = 0; + for lyr_idx = 1:length(obj.layer_organizer.lyr_name) + if regexp(obj.layer_organizer.lyr_name{lyr_idx},regexp_str) + idx = idx + 1; + layer_names{idx} = obj.layer_organizer.lyr_name{lyr_idx}; + lyr_order(idx) = obj.layer_organizer.lyr_order(lyr_idx); + end + end + end + [~,sort_idxs] = sort(lyr_order); + layer_names = layer_names(sort_idxs); + end + %% gps_time: get gps_time function gps_time = gps_time(obj,frms) gps_time = []; - for frm = 1:frms(:).' + for frm = frms(:).' obj.check(frm); gps_time(end+(1:length(obj.layer{frm}.gps_time))) = obj.layer{frm}.gps_time; end @@ -815,53 +1099,59 @@ function fix_layer_organizer(obj) function ids = insert_layers(obj,layer_organizer) obj.check_layer_organizer(); - Nlayers = length(layer_organizer.lyr_name); - if isempty(obj.layer_organizer.lyr_id) - ids = 1:Nlayers; - else - ids = max(obj.layer_organizer.lyr_id) + (1:Nlayers); - end - obj.layer_organizer.lyr_id(end+(1:Nlayers)) = ids; - obj.layer_organizer.lyr_name(end+(1:Nlayers)) = layer_organizer.lyr_name; - - if isfield(layer_organizer,'lyr_age') && length(layer_organizer.lyr_age) == Nlayers - obj.layer_organizer.lyr_age(end+(1:Nlayers)) = layer_organizer.lyr_age; - else - obj.layer_organizer.lyr_age(end+(1:Nlayers)) = NaN; - end - if isfield(layer_organizer,'lyr_age_source') && length(layer_organizer.lyr_age_source) == Nlayers - obj.layer_organizer.lyr_age_source(end+(1:Nlayers)) = layer_organizer.lyr_age_source; - else - obj.layer_organizer.lyr_age_source(end+(1:Nlayers)) = cellfun(@struct,cell(1,Nlayers),'UniformOutput',false); - end - if isfield(layer_organizer,'lyr_desc') && length(layer_organizer.lyr_desc) == Nlayers - obj.layer_organizer.lyr_desc(end+(1:Nlayers)) = layer_organizer.lyr_desc; - else - obj.layer_organizer.lyr_desc(end+(1:Nlayers)) = cellfun(@char,cell(1,Nlayers),'UniformOutput',false); - end - if isfield(layer_organizer,'lyr_group_name') && length(layer_organizer.lyr_group_name) == Nlayers - obj.layer_organizer.lyr_group_name(end+(1:Nlayers)) = layer_organizer.lyr_group_name; - else - obj.layer_organizer.lyr_group_name(end+(1:Nlayers)) = cellfun(@char,cell(1,Nlayers),'UniformOutput',false); - end - if isfield(layer_organizer,'lyr_order') && length(layer_organizer.lyr_order) == Nlayers - obj.layer_organizer.lyr_order(end+(1:Nlayers)) = layer_organizer.lyr_order; - else - if isempty(obj.layer_organizer.lyr_order) - new_orders = 1:Nlayers; + old_layer_organizer = obj.layer_organizer; + try + Nlayers = length(layer_organizer.lyr_name); + if isempty(obj.layer_organizer.lyr_id) + ids = 1:Nlayers; else - new_orders = max(obj.layer_organizer.lyr_order) + (1:Nlayers); + ids = max(obj.layer_organizer.lyr_id) + (1:Nlayers); + end + obj.layer_organizer.lyr_id(end+(1:Nlayers)) = ids; + obj.layer_organizer.lyr_name(end+(1:Nlayers)) = layer_organizer.lyr_name; + + if isfield(layer_organizer,'lyr_age') && length(layer_organizer.lyr_age) == Nlayers + obj.layer_organizer.lyr_age(end+(1:Nlayers)) = layer_organizer.lyr_age; + else + obj.layer_organizer.lyr_age(end+(1:Nlayers)) = NaN; + end + if isfield(layer_organizer,'lyr_age_source') && length(layer_organizer.lyr_age_source) == Nlayers + obj.layer_organizer.lyr_age_source(end+(1:Nlayers)) = layer_organizer.lyr_age_source; + else + obj.layer_organizer.lyr_age_source(end+(1:Nlayers)) = cellfun(@struct,cell(1,Nlayers),'UniformOutput',false); + end + if isfield(layer_organizer,'lyr_desc') && length(layer_organizer.lyr_desc) == Nlayers + obj.layer_organizer.lyr_desc(end+(1:Nlayers)) = layer_organizer.lyr_desc; + else + obj.layer_organizer.lyr_desc(end+(1:Nlayers)) = cellfun(@char,cell(1,Nlayers),'UniformOutput',false); end - obj.layer_organizer.lyr_order(end+(1:Nlayers)) = new_orders; + if isfield(layer_organizer,'lyr_group_name') && length(layer_organizer.lyr_group_name) == Nlayers + obj.layer_organizer.lyr_group_name(end+(1:Nlayers)) = layer_organizer.lyr_group_name; + else + obj.layer_organizer.lyr_group_name(end+(1:Nlayers)) = cellfun(@char,cell(1,Nlayers),'UniformOutput',false); + end + if isfield(layer_organizer,'lyr_order') && length(layer_organizer.lyr_order) == Nlayers + obj.layer_organizer.lyr_order(end+(1:Nlayers)) = layer_organizer.lyr_order; + else + if isempty(obj.layer_organizer.lyr_order) + new_orders = 1:Nlayers; + else + new_orders = max(obj.layer_organizer.lyr_order) + (1:Nlayers); + end + obj.layer_organizer.lyr_order(end+(1:Nlayers)) = new_orders; + end + obj.fix_layer_organizer(); + catch ME + warning('insert_layers failed, reverting to the original layer_organizer. Error: %s: ', ME.getReport); + obj.layer_oranizer = old_layer_organizer; end - obj.fix_layer_organizer(); obj.layer_organizer_modified = true; end %% lat: get lat function lat = lat(obj,frms) lat = []; - for frm = 1:frms(:).' + for frm = frms(:).' obj.check(frm); lat(end+(1:length(obj.layer{frm}.lat))) = obj.layer{frm}.lat; end @@ -869,89 +1159,31 @@ function fix_layer_organizer(obj) %% layer_fn: get layer filename function fn = layer_fn(obj,frm) - fn = fullfile(ct_filename_out(obj.param,'',obj.layerdata_source),sprintf('Data_%s_%03d.mat',obj.param.day_seg,frm)); + fn = fullfile(ct_filename_out(obj.param,obj.layerdata_source,''),sprintf('Data_%s_%03d.mat',obj.param.day_seg,frm)); end %% layer_organizer_fn: get layer organizer filename function fn = layer_organizer_fn(obj,frm) - fn = fullfile(ct_filename_out(obj.param,'',obj.layerdata_source),sprintf('layer_%s.mat',obj.param.day_seg)); - end - - %% load: load layer - function load(obj,frm) - layer_fn = fullfile(ct_filename_out(obj.param,'',obj.layerdata_source),sprintf('Data_%s_%03d.mat',obj.param.day_seg,frm)); - if ~exist(layer_fn,'file') - obj.create(frm); - else - obj.layer{frm} = load(layer_fn); - obj.layer_modified(frm) = false; - obj.fix(frm); - end - end - - %% load_layer_organizer: load layer organizer - function load_layer_organizer(obj) - layer_organizer_fn = fullfile(ct_filename_out(obj.param,'',obj.layerdata_source),sprintf('layer_%s.mat',obj.param.day_seg)); - if ~exist(layer_organizer_fn,'file') - obj.create_layer_organizer(); - else - obj.layer_organizer = load(layer_organizer_fn); - obj.layer_organizer_modified = false; - obj.fix_layer_organizer(); - end - end - - %% load_records: load records and frames - function load_records(obj,records) - - if ~exist('records','var') || isempty(records) - % Load records file - records_fn = ct_filename_support(obj.param,'','records'); - obj.records = load(records_fn); - end - - % Load frames file - obj.frames = frames_load(obj.param); - - % Create reference trajectory (rx_path == 0, tx_weights = []). Update - % the records field with this information. - trajectory_param = struct('gps_source',obj.records.gps_source, ... - 'season_name',obj.param.season_name,'radar_name',obj.param.radar_name,'rx_path', 0, ... - 'tx_weights', [], 'lever_arm_fh', obj.param.radar.lever_arm_fh); - obj.records = trajectory_with_leverarm(obj.records,trajectory_param); - - obj.records.along_track = geodetic_to_along_track(obj.records.lat,obj.records.lon); - - if isempty(obj.along_track_spacing) - sys = ct_output_dir(obj.param.radar_name); - switch (sys) - case {'rds','accum'} - obj.along_track_spacing = 15; - otherwise - obj.along_track_spacing = 5; - end - end - obj.along_track = [0:obj.along_track_spacing:obj.records.along_track(end)]; - - if isempty(obj.sample_spacing_method) - if length(obj.along_track) < 2 && length(obj.records.gps_time) > obj.record_spacing - obj.sample_spacing_method = 'records'; - else - obj.sample_spacing_method = 'along_track'; - end - end - + fn = fullfile(ct_filename_out(obj.param,obj.layerdata_source,''),sprintf('layer_%s.mat',obj.param.day_seg)); end %% lon: get lon function lon = lon(obj,frms) lon = []; - for frm = 1:frms(:).' + for frm = frms(:).' obj.check(frm); lon(end+(1:length(obj.layer{frm}.lon))) = obj.layer{frm}.lon; end end + %% print_layer_names: print available layer names + function print_layer_names(obj) + fprintf('Available layers:\n'); + for idx=1:length(obj.layer_organizer.lyr_name); + fprintf(' %s\n', obj.layer_organizer.lyr_name{idx}.'); + end; + end + %% save: save changes to layers function save(obj, layerdata_source) if exist('layerdata_source','var') && ~isempty(layerdata_source) @@ -962,7 +1194,7 @@ function save(obj, layerdata_source) end if obj.layer_organizer_modified == true - layer_organizer_fn = fullfile(ct_filename_out(obj.param,'',obj.layerdata_source),sprintf('layer_%s.mat',obj.param.day_seg)); + layer_organizer_fn = fullfile(ct_filename_out(obj.param,obj.layerdata_source,''),sprintf('layer_%s.mat',obj.param.day_seg)); layer_organizer = obj.layer_organizer; fprintf('Saving %s\n', layer_organizer_fn); ct_save(layer_organizer_fn,'-struct','layer_organizer'); @@ -971,7 +1203,7 @@ function save(obj, layerdata_source) for frm = 1:length(obj.layer_modified) if obj.layer_modified(frm) == true - layer_fn = fullfile(ct_filename_out(obj.param,'',obj.layerdata_source),sprintf('Data_%s_%03d.mat',obj.param.day_seg,frm)); + layer_fn = fullfile(ct_filename_out(obj.param,obj.layerdata_source,''),sprintf('Data_%s_%03d.mat',obj.param.day_seg,frm)); layer = obj.layer{frm}; fprintf('Saving %s\n', layer_fn); ct_save(layer_fn,'-struct','layer'); @@ -980,9 +1212,34 @@ function save(obj, layerdata_source) end end + %% set_age: set age for a layer + function set_age(obj,id,age) + obj.check_layer_organizer(); + match_idx = find(id == obj.layer_organizer.lyr_id,1); + obj.layer_organizer.lyr_age(match_idx) = age; + obj.layer_organizer_modified = true; + end + + %% set_age_source: set age_source for a layer + function set_age_source(obj,id,age_source) + obj.check_layer_organizer(); + match_idx = find(id == obj.layer_organizer.lyr_id,1); + obj.layer_organizer.lyr_age_source{match_idx} = age_source; + obj.layer_organizer_modified = true; + end + + %% set_desc: set desc for a layer + function set_desc(obj,id,desc) + obj.check_layer_organizer(); + match_idx = find(id == obj.layer_organizer.lyr_id,1); + obj.layer_organizer.lyr_desc{match_idx} = desc; + obj.layer_organizer_modified = true; + end + %% update_gps: update gps for a frame function update_gps(obj,frms) obj.check_records(); + obj.check(frms); for frm = frms(:).' if strcmp(obj.sample_spacing_method,'records') @@ -996,7 +1253,7 @@ function update_gps(obj,frms) obj.layer{frm}.lat = obj.records.lat(recs); obj.layer{frm}.lon = obj.records.lon(recs); else - if frm < length(obj.frame_idxs) + if frm < length(obj.frames.frame_idxs) frm_mask = find(obj.along_track >= obj.records.along_track(obj.frames.frame_idxs(frm)) ... & obj.along_track < obj.records.along_track(obj.frames.frame_idxs(frm+1))); else @@ -1029,15 +1286,15 @@ function update_layer(obj,frms,id,gps_time,twtt,quality,type) % name passed in rather than id id = find(strcmpi(id,obj.layer_organizer.lyr_name)); if isempty(id) - error('Layer does not exist in layer organizer. Run insert_layer() first.'); + error('Layer does not exist in layer organizer. Run insert_layers() first.'); end else if all(obj.layer_organizer.lyr_id ~= id) - error('Layer does not exist in layer organizer. Run insert_layer() first.'); + error('Layer does not exist in layer organizer. Run insert_layers() first.'); end end - for frm = 1:frms(:).' + for frm = frms(:).' obj.check(frm); lay_idx = find(obj.layer{frm}.id == id); if isempty(lay_idx) @@ -1061,7 +1318,173 @@ function update_layer(obj,frms,id,gps_time,twtt,quality,type) obj.layer_modified(frm) = true; end end + + %% GUI function declarations + create_ui(obj); + update_ui(obj); + callback_layersSB(obj,status,event); + callback_plotCB(obj,status,event); + callback_importPB(obj,status,event); + callback_presetPB(obj,status,event); + callback_savePB(obj,status,event); + callback_closePB(obj,status,event); + callback_save_metadataPB(obj,status,event); % metadata window okay button + + end % Methods + + %% Static Methods ========== + methods(Static) + %% interp: interpolate layer data from opsLoadLayers to a common GPS time + % Input: + % layers.twtt, layers.quality, layers.type + % layers.gps_time, layers.lat, layers.lon, and layers.elev + % Output: + % layers.twtt_ref, layers.type_ref, layers.quality_ref + % + % + % param = read_param_xls(ct_filename_param('rds_param_2019_Antarctica_Ground.xls'),'20200107_01'); + % layers = opsLoadLayers(param,struct('name','surface')); + % mdata = echo_load(param,'standard',1); + % layers = layerdata.interp(mdata,layers); + function layers = interp(mdata,layers) + master = []; + if isfield(mdata,'gps_time') + master.GPS_time = mdata.gps_time; + else + master.GPS_time = mdata.GPS_time; + end + if isfield(mdata,'lat') + master.Latitude = mdata.lat; + else + master.Latitude = mdata.Latitude; + end + if isfield(mdata,'lon') + master.Longitude = mdata.lon; + else + master.Longitude = mdata.Longitude; + end + if isfield(mdata,'elev') + master.Elevation = mdata.elev; + else + master.Elevation = mdata.Elevation; + end + if 0 + % New Method: if opsLoadLayers is updated to always return + % uniformly sampled data where NaN are used to represent gaps in + % the layers (rather than just not including any points), simple + % interpolation works fine. Currently opsLoadLayers for source ops, + % does not do this... so "Old Method" must be used. + for lay_idx = length(layers) + layers(lay_idx).twtt_ref = interp1(layers(lay_idx).gps_time, layers(lay_idx).twtt, master.GPS_time, 'spline'); + layers(lay_idx).quality_ref = interp1(layers(lay_idx).gps_time, layers(lay_idx).quality, master.GPS_time, 'nearest'); + layers(lay_idx).type_ref = interp1(layers(lay_idx).gps_time, layers(lay_idx).type, master.GPS_time, 'nearest'); + end + else + % Old Method: this is necessary for layer data loaded from OPS + % where gaps are represented by no data points in those spots. + % A special interpolation is then required to preserve gaps + % properly. + for lay_idx = length(layers) + ops_layer = []; + ops_layer{1}.gps_time = layers(lay_idx).gps_time; + ops_layer{1}.type = layers(lay_idx).type; + ops_layer{1}.quality = layers(lay_idx).quality; + ops_layer{1}.twtt = layers(lay_idx).twtt; + ops_layer{1}.type(isnan(ops_layer{1}.type)) = 2; + ops_layer{1}.quality(isnan(ops_layer{1}.quality)) = 1; + lay = opsInterpLayersToMasterGPSTime(master,ops_layer,[300 60]); + layers(lay_idx).twtt_ref = lay.layerData{1}.value{2}.data; + end + end + end - end + %% load_layers: load list of layers + % varargout = load_layers(mdata,layerdata_source,varargin) + % + % mdata: echogram structure (only GPS_time used) + % + % layerdata_source: optional string containing the directory of the + % layerdata files (default is "layer" for directory CSARP_layer). + % + % varargin: strings containing layer names + % + % fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_standard/20140512_01/Data_20140512_01_018.mat'; + % mdata = load_L1B(fn); + % [surface,bottom] = layerdata.load_layers(mdata,'','surface','bottom'); + % [layer_list{1:N}] = layerdata.load_layers(mdata,'',layer_names{1:N}); + function varargout = load_layers(mdata,layerdata_source,varargin) + layers = layerdata(echo_param(mdata),layerdata_source); + + try + for idx = 1:nargin-2 + varargout{idx} = layers.get_layer_by_gps_time(mdata.GPS_time,varargin{idx}); + end + catch ME + warning('load_layers failed during get_layer_by_gps_time: %s', ME.getReport); + end + delete(layers); + end + + %% profile: define layerdata opsLoadLayers profiles + % layer_params = profile(layer_profile) + % + % This function converts profile strings into standard opsLoadLayers + % layer_params structures. + % + % Inputs: + % ===================================================================== + % + % layer_profile: Structure or string. If string, then should contain + % the name of a layer parameter profile to load into the output + % layer_params. If structure then layer_params is set equal to + % layer_profile and nothing else is done. + % + % Outputs: + % ===================================================================== + % + % layer_params: layer parameter structure of which layers to load and how + % + % Example: + % + % layer_params = profile('rds_ops_layer'); + function layer_params = profile(layer_profile) + if isempty(layer_profile) + layer_params = []; + elseif isstruct(layer_profile) + layer_params = layer_profile; + elseif ischar(layer_profile) + + if strcmpi(layer_profile, 'accum_ops_layers') + layer_params = struct('source', 'ops', 'name',{'surface'}); + + elseif strcmpi(layer_profile, 'accum_layers') + layer_params = struct('source', 'layerdata', 'name', {'surface'}); + + elseif strcmpi(layer_profile, 'rds_ops_layers') + layer_params = struct('source', 'ops', 'name',{'surface', 'bottom'}); + + elseif strcmpi(layer_profile, 'rds_layers') + layer_params = struct('source', 'layerdata', 'name', {'surface', 'bottom'}); + + elseif strcmpi(layer_profile, 'snow_ops_layers') + layer_params = struct('source', 'ops', 'name',{'surface'}); + + elseif strcmpi(layer_profile, 'snow_layers') + layer_params = struct('source', 'layerdata', 'name', {'surface'}); + + else + error('Profile string specified does not exist.'); + end + else + error('Invalid type for layer_profile which must be empty, a layer profile string, or layer_params structure.'); + end + end -end + + % Functions for editing layerdata properties or merging two separate + % layerdata directories + run_merge(); + merge(param,param_override); + + end % Static methods +end % layerdata class diff --git a/cresis-toolbox/ct_support/@layerdata/merge.m b/cresis-toolbox/ct_support/@layerdata/merge.m new file mode 100644 index 00000000..adb8594d --- /dev/null +++ b/cresis-toolbox/ct_support/@layerdata/merge.m @@ -0,0 +1,82 @@ +function merge(param,param_override) +% merge(param,param_override) +% +% 1. Loads coincident LIDAR data if it exists +% 2. Loads DTU sea surface DEM and arctic/antarctica land DEM, combines +% these two DEMS taking land DEM over sea surface DEM. +% 3. Combines LIDAR data and DEM data, taking LIDAR data over DEM data. +% Uses elevation to interpolate where data are not available. +% 4. Estimates Tadc_adjust or t_ref error by comparing radar surface from +% the specified layer source and the LIDAR/DEM combination. +% * The error is calculated as the correction that needs to be applied. In +% other words if the radar surface twtt is too large, then the error is +% reported as a negative number. +% * This error should be added to param.radar.wfs.Tadc_adjust for pulsed systems. +% * This error should be added to param.radar.wfs.t_ref for deramp systems. +% 5. Estimates GPS offset by comparing radar surface and LIDAR/DEM. This offset +% should be added to param.records.gps.time_offset. +% 6. For deramp systems, uses the LIDAR/DEM data to determine the Nyquist +% zone and sets the records.settings.nyquist_zone based on this. +% The second decimal mask in frames.proc_mode is also set to one for +% frames that will be outside max_nyquist_zone. +% +% See run_check_surface.m for how to run. +% +% cat /N/dcwan/projects/cresis/output/ct_tmp/check_surface/snow/2017_Greenland_P3/*.txt +% +% +% Author: John Paden + +% ===================================================================== +%% General Setup +% ===================================================================== +param = merge_structs(param, param_override); + +fprintf('=====================================================================\n'); +fprintf('%s: %s (%s)\n', mfilename, param.day_seg, datestr(now)); +fprintf('=====================================================================\n'); + +% ===================================================================== +%% Input checks +% ===================================================================== + +%% Input Checks: layerdata_merge +if ~isfield(param,'layerdata_merge') || isempty(param.layerdata_merge) + param.layerdata_merge = []; +end + +if ~isfield(param.layerdata_merge,'new') || isempty(param.layerdata_merge.new) + param.layerdata_merge.new = ''; +end + +if ~isfield(param.layerdata_merge,'old') || isempty(param.layerdata_merge.old) + param.layerdata_merge.old = 'layer'; +end + +% ===================================================================== +%% layerdata object +% ===================================================================== + +% Create the layerdata object for this segment +layers = layerdata(param,param.layerdata_merge.old); + +% ===================================================================== +%% GUI +% ===================================================================== + +% Create or open the GUI +layers.create_ui(); + +%% Load layers +% ========================================================================= +layers.gui_layers{end+1} = layerdata(layers.param,param.layerdata_merge.new); +layers.gui_layers{end}.check_layer_organizer(); +layers.gui_layers{end}.check_all_frames(); + +%% Set auto-merge settings +% ========================================================================= + + +%% Update user interface +% ========================================================================= +layers.update_ui(); diff --git a/cresis-toolbox/ct_support/@layerdata/run_merge.m b/cresis-toolbox/ct_support/@layerdata/run_merge.m new file mode 100644 index 00000000..5b8b4b3c --- /dev/null +++ b/cresis-toolbox/ct_support/@layerdata/run_merge.m @@ -0,0 +1,44 @@ +function run_merge() +% layerdata.run_merge() +% +% Runs layerdata.merge.m +% +% Author: John Paden + +%% User Settings +param_override = []; + +params = read_param_xls(ct_filename_param('snow_param_2012_Greenland_P3.xls')); + +params = ct_set_params(params,'cmd.generic',0); +params = ct_set_params(params,'cmd.generic',1,'day_seg','20120330_04'); +% params = ct_set_params(params,'cmd.generic',1); +% params = ct_set_params(params,'cmd.generic',0,'cmd.notes','Do not process'); + +param_override.layerdata_merge.old = 'layer'; +param_override.layerdata_merge.new = 'layer_test'; + +%% Automated Section +% ===================================================================== + +% Input checking +global gRadar; +if exist('param_override','var') + param_override = merge_structs(gRadar,param_override); +else + param_override = gRadar; +end + +% Process each of the segments +ctrl_chain = {}; +first_time = true; +for param_idx = 1:length(params) + param = params(param_idx); + if isfield(param.cmd,'generic') && ~iscell(param.cmd.generic) && ~ischar(param.cmd.generic) && param.cmd.generic + if ~first_time + fprintf('Run dbcont when ready to open the next layerdata GUI.\n'); + keyboard + end + layerdata.merge(param,param_override); + end +end diff --git a/cresis-toolbox/ct_support/@layerdata/update_ui.m b/cresis-toolbox/ct_support/@layerdata/update_ui.m new file mode 100644 index 00000000..4af1255c --- /dev/null +++ b/cresis-toolbox/ct_support/@layerdata/update_ui.m @@ -0,0 +1,91 @@ +function update_ui(obj) +% layerdata.update_ui(obj) +% +% Function for updating the graphical user interface + +%% Populate layers listbox +% ========================================================================= + +lyr_name = {}; +lyr_id = []; +lyr_order = []; +lyr_idx = []; + +for gui_layers_idx = 1:length(obj.gui_layers) + + import_name = ''; + fn_dir = ct_filename_out(obj.param, obj.gui_layers{gui_layers_idx}.layerdata_source,[],1); + [fn_dir,fn_name] = fileparts(fn_dir); + while strncmp(fn_name,'CSARP_',6) + if isempty(import_name) + import_name = fn_name(7:end); + else + import_name = [import_name '/' fn_name(7:end)]; + end + [fn_dir,fn_name] = fileparts(fn_dir); + end + + if isempty(import_name) + import_name = sprintf('/%d',length(obj.gui_layers)+1); + end + + for lyr_name_idx = 1:length(obj.gui_layers{gui_layers_idx}.layer_organizer.lyr_name) + lyr_name{end+1} = [import_name '/' obj.gui_layers{gui_layers_idx}.layer_organizer.lyr_name{lyr_name_idx}]; + % Add the automated layer match information + % Add the merge information + end + if gui_layers_idx == 1 + lyr_id(end+(1:length(obj.gui_layers{gui_layers_idx}.layer_organizer.lyr_id))) = obj.gui_layers{gui_layers_idx}.layer_organizer.lyr_id; + lyr_order(end+(1:length(obj.gui_layers{gui_layers_idx}.layer_organizer.lyr_order))) = obj.gui_layers{gui_layers_idx}.layer_organizer.lyr_order; + else + lyr_id(end+(1:length(obj.gui_layers{gui_layers_idx}.layer_organizer.lyr_id))) = max(lyr_id)+1 + obj.gui_layers{gui_layers_idx}.layer_organizer.lyr_id; + lyr_order(end+(1:length(obj.gui_layers{gui_layers_idx}.layer_organizer.lyr_order))) = max(lyr_order)+1 + obj.gui_layers{gui_layers_idx}.layer_organizer.lyr_order; + end + lyr_idx(end+(1:length(obj.gui_layers{gui_layers_idx}.layer_organizer.lyr_id))) = gui_layers_idx * ones(size(obj.gui_layers{gui_layers_idx}.layer_organizer.lyr_id)); +end + +obj.h_gui.h_layers.set_list(lyr_name, lyr_id, lyr_order); + +%% Plot layers +% ========================================================================= + +%% Update metadata window +% ========================================================================= + +%% Notes +% Example listbox contents +% ========================================================================= +% layer/surface +% post/layer/surface merge into surface NaN +% post/layer/bottom +% /2/lay_001 +% /2/bottom merge diff frames into bottom +% /3/lay_001 rename lay_002 +% /3/bottom merge diff frames into bottom + +% Context Menu Commands: (check in menu depends on first selected entry) +% ========================================================================= +% THIS can be set by param.layerdata_merge.automerge_merge_mode +% Auto-merge diff frames +% Merge diff frames into... (opens dialog to determine which base layer to merge into) +% Auto-merge NaN +% Merge NaN into... (opens dialog to determine which layers to merge into) +% Overwrite + +% THIS can be set by param.layerdata_merge.automerge_match_mode +% Set auto-merge based on twtt and name (may cause metadata rename) +% Set auto-merge based only on name + +% Sequence layer names (may cause metadata renames) + +% Merge Preset: just reruns import without adding any new layerdata +% sources, update_gui called with no arguments +% create_gui: update_gui called with no arguments +% import: import directory dialog and then call update_gui + +% On Save: +% ========================================================================= +% 1. Run through the listbox for each item (while idx < end) +% 2. For each item search ahead for other layers that match and apply each +% merge command and then mark the mask for each item that is applied +% 3. idx increased until the next entry that has not been handled yet. diff --git a/cresis-toolbox/ct_support/basic_load.m b/cresis-toolbox/ct_support/basic_load.m index 15e99353..a2bea5ec 100644 --- a/cresis-toolbox/ct_support/basic_load.m +++ b/cresis-toolbox/ct_support/basic_load.m @@ -380,6 +380,7 @@ hdr.file_version = hdr.file_version(1:rline_out); hdr.num_waveforms = hdr.num_waveforms(1:rline_out); hdr.num_adc = hdr.num_adc(1:rline_out); +hdr.complex_flag = hdr.complex_flag(1:rline_out); hdr.nyquist_zone = hdr.nyquist_zone(1:rline_out); hdr.bit_shifts = hdr.bit_shifts(1:rline_out); hdr.start_idx = hdr.start_idx(1:rline_out); diff --git a/cresis-toolbox/ct_support/check_raw_files.m b/cresis-toolbox/ct_support/check_raw_files.m index d091edab..1213b334 100644 --- a/cresis-toolbox/ct_support/check_raw_files.m +++ b/cresis-toolbox/ct_support/check_raw_files.m @@ -66,7 +66,7 @@ function check_raw_files(param,param_override) records_fn = ct_filename_support(param,'','records'); if exist(records_fn,'file') - records = load(records_fn); + records = records_load(param); else records = []; end diff --git a/cresis-toolbox/ct_support/ct_filename_ct_tmp.m b/cresis-toolbox/ct_support/ct_filename_ct_tmp.m index 534a5b08..7d67f113 100644 --- a/cresis-toolbox/ct_support/ct_filename_ct_tmp.m +++ b/cresis-toolbox/ct_support/ct_filename_ct_tmp.m @@ -45,8 +45,13 @@ else % Generate the default filename [tmp name ext] = fileparts(filename); - fn = fullfile(param.ct_tmp_path, type, output_dir, ... - param.season_name, sprintf('%s_%s%s', name, param.day_seg, ext)); + if isempty(name) && isempty(ext) + fn = fullfile(param.ct_tmp_path, type, output_dir, ... + param.season_name, param.day_seg); + else + fn = fullfile(param.ct_tmp_path, type, output_dir, ... + param.season_name, sprintf('%s_%s%s', name, param.day_seg, ext)); + end end elseif fn(1) == filesep || (ispc && (~isempty(strfind(fn,':\')) || ~isempty(strfind(fn,':/')))) % This is already an absolute path diff --git a/cresis-toolbox/ct_support/ct_filename_out.m b/cresis-toolbox/ct_support/ct_filename_out.m index 04681451..84091426 100644 --- a/cresis-toolbox/ct_support/ct_filename_out.m +++ b/cresis-toolbox/ct_support/ct_filename_out.m @@ -25,6 +25,9 @@ % no effect. % generic_data_flag: if enabled, the segment ID is excluded from the path % +% ct_filename_out(param,'qlook') +% ct_filename_out(param,'standard') +% % Author: John Paden % % See also: ct_filename_data, ct_filename_out, ct_filename_support, diff --git a/cresis-toolbox/ct_support/ct_filename_support.m b/cresis-toolbox/ct_support/ct_filename_support.m index 390740b5..59b440d8 100644 --- a/cresis-toolbox/ct_support/ct_filename_support.m +++ b/cresis-toolbox/ct_support/ct_filename_support.m @@ -11,16 +11,21 @@ % standardized path/directory structure is not used (base_fn is used) % - base_fn is empty: the param.support_path and standardized path are used % -% param = control structure to data processor -% .radar_name (e.g. mcords) -% .season_name (e.g. 2009_antarctica_DC8) -% .day_seg (e.g. 20091020_03) -% fn = parameter filename provided (leave blank to get the default file -% path) -% type = string containing the type of data (e.g. vectors, gps, records, -% frames, radar_config) -% generic_data_flag = if enabled, the radar name and segment are excluded. -% This field is optional. Default is false. +% param: control structure to data processor +% * .radar_name (e.g. mcords): See ct_output_dir.m for valid options. Only +% required if fn not set. +% * .season_name (e.g. 2009_antarctica_DC8). Only required if fn not set. +% * .day_seg (e.g. 20091020_03). Used when fn not set. Optional. +% +% fn: parameter filename provided (leave blank to get the default file +% path) +% +% type: string containing the type of data (e.g. vectors, gps, records, +% frames, radar_config). Only required if fn not set. +% +% generic_data_flag: if enabled, the radar name and specific segment are +% excluded from the output file path. This field is optional. Default is +% false. % % Author: John Paden % @@ -34,8 +39,6 @@ generic_data_flag = 0; end -[output_dir,radar_type] = ct_output_dir(param.radar_name); - if strcmpi(type,'records') && isfield(param,'records') && isfield(param.records,'records_fn') && isempty(fn) fn = param.records.records_fn; end @@ -45,6 +48,16 @@ end if isempty(fn) + if ~isfield(param,'radar_name') || isempty(param.radar_name) + error('ct_filename_support requires that param.radar_name exist and be nonempty if input fn is not set.'); + end + if ~isfield(param,'season_name') || isempty(param.season_name) + error('ct_filename_support requires that param.season_name exist and be nonempty if input fn is not set.'); + end +end + +if isempty(fn) + [output_dir,radar_type] = ct_output_dir(param.radar_name); if ~isfield(param,'day_seg') || isempty(param.day_seg) % Generate the default path if generic_data_flag diff --git a/cresis-toolbox/ct_support/ct_filename_tmp.m b/cresis-toolbox/ct_support/ct_filename_tmp.m index 61d99f39..90382c95 100644 --- a/cresis-toolbox/ct_support/ct_filename_tmp.m +++ b/cresis-toolbox/ct_support/ct_filename_tmp.m @@ -50,8 +50,13 @@ else % Generate the default filename [tmp name ext] = fileparts(filename); - fn = fullfile(param.tmp_path, type, output_dir, ... - param.season_name, sprintf('%s_%s%s', name, param.day_seg, ext)); + if isempty(name) && isempty(ext) + fn = fullfile(param.tmp_path, type, output_dir, ... + param.season_name, param.day_seg); + else + fn = fullfile(param.tmp_path, type, output_dir, ... + param.season_name, sprintf('%s_%s%s', name, param.day_seg, ext)); + end end elseif fn(1) == filesep || (ispc && (~isempty(strfind(fn,':\')) || ~isempty(strfind(fn,':/')))) % This is already an absolute path diff --git a/cresis-toolbox/ct_support/ct_generic_en.m b/cresis-toolbox/ct_support/ct_generic_en.m new file mode 100644 index 00000000..d983ec9a --- /dev/null +++ b/cresis-toolbox/ct_support/ct_generic_en.m @@ -0,0 +1,7 @@ +function en = ct_generic_en(param) + +if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic + en = 0; +else + en = 1; +end diff --git a/cresis-toolbox/ct_support/ct_get_frame_list.m b/cresis-toolbox/ct_support/ct_get_frame_list.m new file mode 100644 index 00000000..37acee36 --- /dev/null +++ b/cresis-toolbox/ct_support/ct_get_frame_list.m @@ -0,0 +1,66 @@ +function frm_strs = ct_get_frame_list(params,get_mode) +% frm_strs = ct_get_frame_list(params,get_mode) +% +% Returns a cell array of frame strings {'YYYYMMDD_SS_FFF', ...} +% +% params: this may be a struct or a filename. If a struct, it should be the +% parameter structure returned from read_param_xls. If a filename, it +% should be the parameter spreadsheet filename. +% +% get_mode: scalar numeric. Default is 0. If 0, then all segments without +% "do not process" in cmd.notes are returned. All frames will be included. +% If 1, then all segments are returned. All frames will be included. If 2, +% then only segments with cmd.generic enabled are listed. Only frames that +% are selected with cmd.frms. +% +% frm_strs: cell vector of frame strings {'YYYYMMDD_SS_FFF', ...} +% +% Examples: +% frm_strs = ct_get_frame_list(ct_filename_param('kuband_param_2011_Greenland_P3.xls')); +% +% params = read_param_xls(ct_filename_param('kuband_param_2011_Greenland_P3.xls')); +% frm_strs = ct_get_frame_list(params); +% frm_strs = ct_get_frame_list(params,1); +% params = ct_set_params(params,'cmd.generic',1,'cmd.notes','do not process'); +% frm_strs = ct_get_frame_list(params,2); +% +% Author: John Paden +% +% See also: ct_get_segment_list, ct_get_frame_list + +if ~exist('get_mode','var') || isempty(get_mode) + get_mode = 0; +end + +if ischar(params) + % Parameter spreadsheet filename + params = read_param_xls(params); +end + +frm_strs = {}; + +for param_idx = 1:length(params) + frames = frames_load(params(param_idx)); + if get_mode == 0 + if ~isfield(params(param_idx),'cmd') || ~isfield(params(param_idx).cmd,'notes') ... + || isempty(regexpi(params(param_idx).cmd.notes,'do not process')) + for frm = 1:length(frames.frame_idxs) + frm_strs{end+1} = sprintf('%s_%03d',params(param_idx).day_seg, frm); + end + end + elseif get_mode == 1 + day_segs{end+1} = params(param_idx).day_seg; + for frm = 1:length(frames.frame_idxs) + frm_strs{end+1} = sprintf('%s_%03d',params(param_idx).day_seg, frm); + end + else + if ct_generic_en(params(param_idx)) + day_segs{end+1} = params(param_idx).day_seg; + frms = frames_param_cmd_frms(params(param_idx),frames); + for frm = frms + frm_strs{end+1} = sprintf('%s_%03d',params(param_idx).day_seg, frm); + end + end + end + +end diff --git a/cresis-toolbox/ct_support/ct_get_segment_list.m b/cresis-toolbox/ct_support/ct_get_segment_list.m new file mode 100644 index 00000000..7729d40a --- /dev/null +++ b/cresis-toolbox/ct_support/ct_get_segment_list.m @@ -0,0 +1,54 @@ +function day_segs = ct_get_segment_list(params,get_mode) +% day_segs = ct_get_segment_list(params,get_mode) +% +% Returns a cell array of frame strings {'YYYYMMDD_SS_FFF', ...} +% +% params: this may be a struct or a filename. If a struct, it should be the +% parameter structure returned from read_param_xls. If a filename, it +% should be the parameter spreadsheet filename. +% +% get_mode: scalar numeric. Default is 0. If 0, then all segments without "do +% not process" in cmd.notes are returned. If 1, then all segments are +% returned. If 2, then only segments with cmd.generic enabled are listed. +% +% day_segs: cell vector of segment name strings {'YYYYMMDD_SS', ...} +% +% Examples: +% day_segs = ct_get_segment_list(ct_filename_param('kuband_param_2011_Greenland_P3.xls')) +% +% params = read_param_xls(ct_filename_param('kuband_param_2011_Greenland_P3.xls')); +% day_segs = ct_get_segment_list(params) +% day_segs = ct_get_segment_list(params,1) +% params = ct_set_params(params,'cmd.generic',1,'cmd.notes','do not process'); +% day_segs = ct_get_segment_list(params,2) +% +% Author: John Paden +% +% See also: ct_get_segment_list, ct_get_frame_list + +if ~exist('get_mode','var') || isempty(get_mode) + get_mode = 0; +end + +if ischar(params) + % Parameter spreadsheet filename + params = read_param_xls(params); +end + +day_segs = {}; + +for param_idx = 1:length(params) + if get_mode == 0 + if ~isfield(params(param_idx),'cmd') || ~isfield(params(param_idx).cmd,'notes') ... + || isempty(regexpi(params(param_idx).cmd.notes,'do not process')) + day_segs{end+1} = params(param_idx).day_seg; + end + elseif get_mode == 1 + day_segs{end+1} = params(param_idx).day_seg; + else + if ct_generic_en(params(param_idx)) + day_segs{end+1} = params(param_idx).day_seg; + end + end + +end diff --git a/cresis-toolbox/ct_support/ct_param_path_update.m b/cresis-toolbox/ct_support/ct_param_path_update.m new file mode 100644 index 00000000..b111a0d8 --- /dev/null +++ b/cresis-toolbox/ct_support/ct_param_path_update.m @@ -0,0 +1,43 @@ +function param = ct_param_path_update(param,param_override) +% param = ct_param_path_update(param,param_override) +% +% Function to update the paths of a parameter structure with another +% parameter structure. This is useful when a parameter structure is read in +% from a file that was processed on a different system and therefore may +% have paths that do no work with the current system, but we want to use +% the parameter structure. This function allows the paths to be easily +% updated with the current systems paths. +% +% Inputs: +% +% param: parameter spreadsheet structure +% +% param_override: override parameters. If set to [] or not defined, then +% the global variable gRadar will be used. +% +% Outputs: +% +% param: updated parameter spreadsheet structure +% +% Example: +% +% echo_fn = '/N/dcwan/projects/cresis/output/rds/2018_Greenland_P3/CSARP_qlook/20180406_01/Data_20180406_01_001.mat'; +% mdata = load_L1B(echo_fn); +% param = ct_param_path_update(mdata.param_qlook); +% +% Author: John Paden + +if ~exist('param_override','var') || ~isstruct(param_override) + global gRadar; + param_override = gRadar; +end + +field_names = {'cluster','path','path_override','param_path','tmp_path', ... + 'ct_tmp_path','data_path','data_support_path','support_path','out_path', ... + 'gis_path','ct_file_lock_check','ct_file_lock','slurm_jobs_path','ops'}; + +for idx = 1:length(field_names) + if isfield(param_override,field_names{idx}) + param.(field_names{idx}) = param_override.(field_names{idx}); + end +end diff --git a/cresis-toolbox/ct_support/ct_proc_frame.m b/cresis-toolbox/ct_support/ct_proc_frame.m index bf5c9b75..6d788533 100644 --- a/cresis-toolbox/ct_support/ct_proc_frame.m +++ b/cresis-toolbox/ct_support/ct_proc_frame.m @@ -14,7 +14,7 @@ % Entering a negative number into a cell means that particular mask is % ignored. The 5 elements are: % R1: SAR/Doppler domain controls -% R2: 0 = good frame/process, 1 = process but do not post, 2 = post only +% R2: 0 = good frame/process, 1 = process but do not post, 2 = do not process or post % R3: Reserved % R4: Reserved % U: User defined diff --git a/cresis-toolbox/ct_support/ct_remove_toolboxes.m b/cresis-toolbox/ct_support/ct_remove_toolboxes.m new file mode 100644 index 00000000..afda5b7c --- /dev/null +++ b/cresis-toolbox/ct_support/ct_remove_toolboxes.m @@ -0,0 +1,78 @@ +function rmpath_strs = ct_remove_toolboxes(toolbox_strs) +% rmpath_strs = ct_remove_toolboxes(toolbox_strs) +% +% Function for removing toolboxes from the path in order to test functions +% to verify that they use or do not use a toolbox. +% +% Run startup again to recover toolbox path or path(pathdef) to recover +% default matlab path. +% +% Inputs: +% +% toolbox_strs: cell array of strings containing the directory name for the toolboxes +% that are to be removed +% +% Outputs: +% +% rmpath_strs: colon delimited/separated paths that are removed +% +% Examples: +% +% rmpath_strs = ct_remove_toolboxes({'compiler','globaloptim','images','map','optim','signal'}) +% +% path([path ':' rmpath_strs]); +% +% Author: John Paden +% +% See also: pathdef + +% Return the Matlab root installation directory. +% e.g. /cresis/snfs1/sw/matlab/2020a/ +path_root = matlabroot; + +% Parses path into cell array +path_list = strsplit(path,':'); + +rmpath_strs = ''; + +if any(strcmpi(toolbox_strs,'images')) + for idx_path = 1:length(path_list) + if ~isempty(regexp(path_list{idx_path},fullfile(path_root,'toolbox','images'))) + fprintf('Removing:\t%s\n', path_list{idx_path}); + rmpath(path_list{idx_path}); + if isempty(rmpath_strs) + rmpath_strs = path_list{idx_path}; + else + rmpath_strs = [rmpath_strs, ':', path_list{idx_path}]; + end + end + end +end + +if any(strcmpi(toolbox_strs,'map')) + for idx_path = 1:length(path_list) + if ~isempty(regexp(path_list{idx_path},fullfile(path_root,'toolbox','map'))) + fprintf('Removing:\t%s\n', path_list{idx_path}); + rmpath(path_list{idx_path}); + if isempty(rmpath_strs) + rmpath_strs = path_list{idx_path}; + else + rmpath_strs = [rmpath_strs, ':', path_list{idx_path}]; + end + end + end +end + +if any(strcmpi(toolbox_strs,'signal')) + for idx_path = 1:length(path_list) + if ~isempty(regexp(path_list{idx_path},fullfile(path_root,'toolbox','signal'))) + fprintf('Removing:\t%s\n', path_list{idx_path}); + rmpath(path_list{idx_path}); + if isempty(rmpath_strs) + rmpath_strs = path_list{idx_path}; + else + rmpath_strs = [rmpath_strs, ':', path_list{idx_path}]; + end + end + end +end diff --git a/cresis-toolbox/ct_support/ct_save.m b/cresis-toolbox/ct_support/ct_save.m index 02fe2198..e10bbfbc 100644 --- a/cresis-toolbox/ct_support/ct_save.m +++ b/cresis-toolbox/ct_support/ct_save.m @@ -49,7 +49,20 @@ function ct_save(varargin) % Run the save command if ~isempty(varargin) - cmd = sprintf('save(''%s''%s,''%s'')',fn,sprintf(',''%s''',varargin{:}),mat_file_version); + append_field_mask = ~strcmp('-append',varargin); + if exist(fn,'file') + if any(append_field_mask) + % If file exists then we cannot set the file version if we are + % appending + cmd = sprintf('save(''%s''%s)',fn,sprintf(',''%s''',varargin{:})); + else + cmd = sprintf('save(''%s''%s,''%s'')',fn,sprintf(',''%s''',varargin{:}),mat_file_version); + end + else + % If the file does not exist, then we cannot append. + varargin = varargin(append_field_mask); + cmd = sprintf('save(''%s''%s,''%s'')',fn,sprintf(',''%s''',varargin{:}),mat_file_version); + end else cmd = sprintf('save(''%s'',''%s'')',fn,mat_file_version); end diff --git a/cresis-toolbox/ct_support/ct_smooth.m b/cresis-toolbox/ct_support/ct_smooth.m index 680ccdd2..70bfaf7b 100644 --- a/cresis-toolbox/ct_support/ct_smooth.m +++ b/cresis-toolbox/ct_support/ct_smooth.m @@ -3,6 +3,7 @@ N = round(1/cutoff/2)*2+1; H = tukeywin(N,0.5).'; +H = H ./ sum(H); data = fir_dec(data,H,1); diff --git a/cresis-toolbox/ct_support/ct_whos.m b/cresis-toolbox/ct_support/ct_whos.m new file mode 100644 index 00000000..bab30989 --- /dev/null +++ b/cresis-toolbox/ct_support/ct_whos.m @@ -0,0 +1,17 @@ +function size_bytes = ct_whos(field) +% size_bytes = ct_whos(field) +% +% Convenience function to get the size in bytes of a variable. Useful in +% general to save a couple of lines of code. +% +% Example: +% S.A = ones(1,3); +% S.B = ones(4,4); +% ct_whos(S) +% ct_whos(S.A) +% ct_whos(S.B) +% +% Author: John Paden + +s = whos('field'); +size_bytes = s.bytes; diff --git a/cresis-toolbox/ct_support/explore_raw_data.m b/cresis-toolbox/ct_support/explore_raw_data.m new file mode 100644 index 00000000..0f381c20 --- /dev/null +++ b/cresis-toolbox/ct_support/explore_raw_data.m @@ -0,0 +1,482 @@ +% script explore_raw_data +% +% Use this script to explore raw radar data files + +if 0 + % ========================================================================= + % Used for 2020_South_Dakota_N1KU digital system file test (Jan 27, 2021) + % ========================================================================= + % EPRI 10824: 16646 0x00004106 b00000000000000000100000100000110 16646 0 + % fraction 10825: 11261135 0x00abd4cf b00000000101010111101010011001111 -11057 171 + % counter 10826: 3951368227 0xeb851823 b11101011100001010001100000100011 6179 -5243 + % zeros 10827: 0 0x00000000 b00000000000000000000000000000000 0 0 + % start|stop 10828: 6585100 0x00647b0c b00000000011001000111101100001100 31500 100 + % zeros 10829: 0 0x00000000 b00000000000000000000000000000000 0 0 + + % Estimate fraction/counter increment record to record + fs = 125000000; + prf = 3900; + presums = 16; + fs/prf * presums + fs/512800 * presums + + % Fraction field checks: + 12261135-11761135 + 11761135-11261135 + + % Counter field check: + 35761135-35261135 + + % Record size check + (1549718-1518312)*4 + (1518312-1486906)*4 + (31500-100)*8 + 48 +end + +% ========================================================================= +% Start +% ========================================================================= +clc; + +% Read in file +% Files 0,1,2 Noise only. All channels disconnected +% fn = '/cresis/snfs1/projects/cessna_test/Multi_channel_wGPS/data_v11_00_20210126_183415_0001.bin'; +% Files 3,4,5 Ch0 connected +% fn = '/cresis/snfs1/projects/cessna_test/Multi_channel_wGPS/data_v11_00_20210126_183440_0004.bin'; +% Files 6,7,8 Ch0, Ch1 connected +% fn = '/cresis/snfs1/projects/cessna_test/Multi_channel_wGPS/data_v11_00_20210126_183505_0007.bin'; +% Files 9, 10,11 Ch0, Ch1, Ch2 connected +% fn = '/cresis/snfs1/projects/cessna_test/Multi_channel_wGPS/data_v11_00_20210126_183530_0010.bin'; +% Files 12,13,14 Ch0, Ch1, Ch2, Ch3 connected +% fn = '/cresis/snfs1/projects/cessna_test/Multi_channel_wGPS/data_v11_00_20210126_183555_0013.bin'; + +% File 0-4 8 presums +% fn = '/cresis/snfs1/projects/cessna_test/01282021/data_v11_00_20210128_004010_0002.bin'; +% File 5-8, 15 presums +% fn = '/cresis/snfs1/projects/cessna_test/01282021/data_v11_00_20210128_004209_0006.bin'; + +% PRF is 3900 Hz. I recorded data with 16 presums. +% fn = '/cresis/snfs1/projects/cessna_test/01282021b/data_v11_00_20210128_210219_0002.bin'; + +% PRF is 3125 Hz. I recorded data with 16 presums. +% fn = '/cresis/snfs1/projects/cessna_test/64bitcheck/data_v11_00_20210129_190854_0002.bin'; + +% PRF is 3125 Hz. 8 presums. 600 to 31900 samples. +% fn = '/cresis/snfs1/projects/Field_Experiments/2021_Winter_Cessna172/Lab_tests/recording_tests/loopback_wGPS/seg1/data_v11_00_20210130_031200_0002.bin'; + +% PRF is 3125 Hz. 16 presums. 600 to 31900 samples. +fn = '/cresis/snfs1/projects/Field_Experiments/2021_Winter_Cessna172/Lab_tests/recording_tests/loopback_wGPS/seg2/data_v11_00_20210130_031607_0002.bin'; + + +[fid,msg] = fopen(fn,'rb','ieee-be'); + +[A,cnt] = fread(fid,inf,'uint32=>uint32'); + +fclose(fid); + +% Print record boundaries in file + +% Use zero fields to find record boundaries +if 0 + burst_idxs = find(A == 0); +else + burst_idxs = find(A == hex2dec('1acffc1d')); +end + +% Restrict to 100 records max +burst_idxs = burst_idxs(1:min(100,end)); + +figure(1); clf; +fprintf('Field 1: File index (uint32 blocks)\n'); +fprintf('Field 2: Decimal value\n'); +fprintf('Field 3: Hex value\n'); +fprintf('Field 4: Binary value\n'); +fprintf('Field 5: int16 samples (two per uint32 that is read in)\n'); +old_burst_idx = -inf; +for burst_idxs_idx = 1:length(burst_idxs) + burst_idx = burst_idxs(burst_idxs_idx); + + if 0 + start_offset = 3; + stop_offset = 2; + else + start_offset = 0; + stop_offset = 11; + end + if burst_idx >= start_offset+1 && burst_idx > old_burst_idx + 50; + idxs = burst_idx + (-start_offset:stop_offset); + + fprintf('============================================================\n'); + for idxs_idx = 1:length(idxs) + idx = idxs(idxs_idx); + fprintf('%10d: %16d 0x%08x b%32s %10d %10d\n', idx, A(idx), A(idx), dec2bin(A(idx),32), typecast(A(idx),'int16')); + end + + if 1 && isfinite(old_burst_idx) + % Plot as uint32 + %plot(A(old_burst_idx-start_offset:burst_idx-start_offset-1)); + % Plot as int16 + plot(typecast(A(old_burst_idx+stop_offset:burst_idx-start_offset-1),'int16')); + hold on + end + old_burst_idx = burst_idx; + end +end + +% Example output: +% +% Field 1: File index (uint32 blocks) +% Field 2: Decimal value +% Field 3: Hex value +% Field 4: Binary value +% Field 5: int16 samples (two per uint32 that is read in) +% ============================================================ +% 10824: 16646 0x00004106 b00000000000000000100000100000110 +% 10825: 11261135 0x00abd4cf b00000000101010111101010011001111 +% 10826: 3951368227 0xeb851823 b11101011100001010001100000100011 +% 10827: 0 0x00000000 b00000000000000000000000000000000 +% 10828: 6585100 0x00647b0c b00000000011001000111101100001100 +% 10829: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 42230: 16647 0x00004107 b00000000000000000100000100000111 +% 42231: 11761135 0x00b375ef b00000000101100110111010111101111 +% 42232: 3951868227 0xeb8cb943 b11101011100011001011100101000011 +% 42233: 0 0x00000000 b00000000000000000000000000000000 +% 42234: 6585100 0x00647b0c b00000000011001000111101100001100 +% 42235: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 73636: 16648 0x00004108 b00000000000000000100000100001000 +% 73637: 12261135 0x00bb170f b00000000101110110001011100001111 +% 73638: 3952368227 0xeb945a63 b11101011100101000101101001100011 +% 73639: 0 0x00000000 b00000000000000000000000000000000 +% 73640: 6585100 0x00647b0c b00000000011001000111101100001100 +% 73641: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 105042: 16649 0x00004109 b00000000000000000100000100001001 +% 105043: 12761135 0x00c2b82f b00000000110000101011100000101111 +% 105044: 3952868227 0xeb9bfb83 b11101011100110111111101110000011 +% 105045: 0 0x00000000 b00000000000000000000000000000000 +% 105046: 6585100 0x00647b0c b00000000011001000111101100001100 +% 105047: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 136448: 16650 0x0000410a b00000000000000000100000100001010 +% 136449: 13261135 0x00ca594f b00000000110010100101100101001111 +% 136450: 3953368227 0xeba39ca3 b11101011101000111001110010100011 +% 136451: 0 0x00000000 b00000000000000000000000000000000 +% 136452: 6585100 0x00647b0c b00000000011001000111101100001100 +% 136453: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 167854: 16651 0x0000410b b00000000000000000100000100001011 +% 167855: 13761135 0x00d1fa6f b00000000110100011111101001101111 +% 167856: 3953868227 0xebab3dc3 b11101011101010110011110111000011 +% 167857: 0 0x00000000 b00000000000000000000000000000000 +% 167858: 6585100 0x00647b0c b00000000011001000111101100001100 +% 167859: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 199260: 16652 0x0000410c b00000000000000000100000100001100 +% 199261: 14261135 0x00d99b8f b00000000110110011001101110001111 +% 199262: 3954368227 0xebb2dee3 b11101011101100101101111011100011 +% 199263: 0 0x00000000 b00000000000000000000000000000000 +% 199264: 6585100 0x00647b0c b00000000011001000111101100001100 +% 199265: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 230666: 16653 0x0000410d b00000000000000000100000100001101 +% 230667: 14761135 0x00e13caf b00000000111000010011110010101111 +% 230668: 3954868227 0xebba8003 b11101011101110101000000000000011 +% 230669: 0 0x00000000 b00000000000000000000000000000000 +% 230670: 6585100 0x00647b0c b00000000011001000111101100001100 +% 230671: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 262072: 16654 0x0000410e b00000000000000000100000100001110 +% 262073: 15261135 0x00e8ddcf b00000000111010001101110111001111 +% 262074: 3955368227 0xebc22123 b11101011110000100010000100100011 +% 262075: 0 0x00000000 b00000000000000000000000000000000 +% 262076: 6585100 0x00647b0c b00000000011001000111101100001100 +% 262077: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 293478: 16655 0x0000410f b00000000000000000100000100001111 +% 293479: 15761135 0x00f07eef b00000000111100000111111011101111 +% 293480: 3955868227 0xebc9c243 b11101011110010011100001001000011 +% 293481: 0 0x00000000 b00000000000000000000000000000000 +% 293482: 6585100 0x00647b0c b00000000011001000111101100001100 +% 293483: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 324884: 16656 0x00004110 b00000000000000000100000100010000 +% 324885: 16261135 0x00f8200f b00000000111110000010000000001111 +% 324886: 3956368227 0xebd16363 b11101011110100010110001101100011 +% 324887: 0 0x00000000 b00000000000000000000000000000000 +% 324888: 6585100 0x00647b0c b00000000011001000111101100001100 +% 324889: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 356290: 16657 0x00004111 b00000000000000000100000100010001 +% 356291: 16761135 0x00ffc12f b00000000111111111100000100101111 +% 356292: 3956868227 0xebd90483 b11101011110110010000010010000011 +% 356293: 0 0x00000000 b00000000000000000000000000000000 +% 356294: 6585100 0x00647b0c b00000000011001000111101100001100 +% 356295: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 387696: 16658 0x00004112 b00000000000000000100000100010010 +% 387697: 17261135 0x0107624f b00000001000001110110001001001111 +% 387698: 3957368227 0xebe0a5a3 b11101011111000001010010110100011 +% 387699: 0 0x00000000 b00000000000000000000000000000000 +% 387700: 6585100 0x00647b0c b00000000011001000111101100001100 +% 387701: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 419102: 16659 0x00004113 b00000000000000000100000100010011 +% 419103: 17761135 0x010f036f b00000001000011110000001101101111 +% 419104: 3957868227 0xebe846c3 b11101011111010000100011011000011 +% 419105: 0 0x00000000 b00000000000000000000000000000000 +% 419106: 6585100 0x00647b0c b00000000011001000111101100001100 +% 419107: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 450508: 16660 0x00004114 b00000000000000000100000100010100 +% 450509: 18261135 0x0116a48f b00000001000101101010010010001111 +% 450510: 3958368227 0xebefe7e3 b11101011111011111110011111100011 +% 450511: 0 0x00000000 b00000000000000000000000000000000 +% 450512: 6585100 0x00647b0c b00000000011001000111101100001100 +% 450513: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 481914: 16661 0x00004115 b00000000000000000100000100010101 +% 481915: 18761135 0x011e45af b00000001000111100100010110101111 +% 481916: 3958868227 0xebf78903 b11101011111101111000100100000011 +% 481917: 0 0x00000000 b00000000000000000000000000000000 +% 481918: 6585100 0x00647b0c b00000000011001000111101100001100 +% 481919: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 513320: 16662 0x00004116 b00000000000000000100000100010110 +% 513321: 19261135 0x0125e6cf b00000001001001011110011011001111 +% 513322: 3959368227 0xebff2a23 b11101011111111110010101000100011 +% 513323: 0 0x00000000 b00000000000000000000000000000000 +% 513324: 6585100 0x00647b0c b00000000011001000111101100001100 +% 513325: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 544726: 16663 0x00004117 b00000000000000000100000100010111 +% 544727: 19761135 0x012d87ef b00000001001011011000011111101111 +% 544728: 3959868227 0xec06cb43 b11101100000001101100101101000011 +% 544729: 0 0x00000000 b00000000000000000000000000000000 +% 544730: 6585100 0x00647b0c b00000000011001000111101100001100 +% 544731: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 576132: 16664 0x00004118 b00000000000000000100000100011000 +% 576133: 20261135 0x0135290f b00000001001101010010100100001111 +% 576134: 3960368227 0xec0e6c63 b11101100000011100110110001100011 +% 576135: 0 0x00000000 b00000000000000000000000000000000 +% 576136: 6585100 0x00647b0c b00000000011001000111101100001100 +% 576137: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 607538: 16665 0x00004119 b00000000000000000100000100011001 +% 607539: 20761135 0x013cca2f b00000001001111001100101000101111 +% 607540: 3960868227 0xec160d83 b11101100000101100000110110000011 +% 607541: 0 0x00000000 b00000000000000000000000000000000 +% 607542: 6585100 0x00647b0c b00000000011001000111101100001100 +% 607543: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 638944: 16666 0x0000411a b00000000000000000100000100011010 +% 638945: 21261135 0x01446b4f b00000001010001000110101101001111 +% 638946: 3961368227 0xec1daea3 b11101100000111011010111010100011 +% 638947: 0 0x00000000 b00000000000000000000000000000000 +% 638948: 6585100 0x00647b0c b00000000011001000111101100001100 +% 638949: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 670350: 16667 0x0000411b b00000000000000000100000100011011 +% 670351: 21761135 0x014c0c6f b00000001010011000000110001101111 +% 670352: 3961868227 0xec254fc3 b11101100001001010100111111000011 +% 670353: 0 0x00000000 b00000000000000000000000000000000 +% 670354: 6585100 0x00647b0c b00000000011001000111101100001100 +% 670355: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 701756: 16668 0x0000411c b00000000000000000100000100011100 +% 701757: 22261135 0x0153ad8f b00000001010100111010110110001111 +% 701758: 3962368227 0xec2cf0e3 b11101100001011001111000011100011 +% 701759: 0 0x00000000 b00000000000000000000000000000000 +% 701760: 6585100 0x00647b0c b00000000011001000111101100001100 +% 701761: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 733162: 16669 0x0000411d b00000000000000000100000100011101 +% 733163: 22761135 0x015b4eaf b00000001010110110100111010101111 +% 733164: 3962868227 0xec349203 b11101100001101001001001000000011 +% 733165: 0 0x00000000 b00000000000000000000000000000000 +% 733166: 6585100 0x00647b0c b00000000011001000111101100001100 +% 733167: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 764568: 16670 0x0000411e b00000000000000000100000100011110 +% 764569: 23261135 0x0162efcf b00000001011000101110111111001111 +% 764570: 3963368227 0xec3c3323 b11101100001111000011001100100011 +% 764571: 0 0x00000000 b00000000000000000000000000000000 +% 764572: 6585100 0x00647b0c b00000000011001000111101100001100 +% 764573: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 795974: 16671 0x0000411f b00000000000000000100000100011111 +% 795975: 23761135 0x016a90ef b00000001011010101001000011101111 +% 795976: 3963868227 0xec43d443 b11101100010000111101010001000011 +% 795977: 0 0x00000000 b00000000000000000000000000000000 +% 795978: 6585100 0x00647b0c b00000000011001000111101100001100 +% 795979: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 827380: 16672 0x00004120 b00000000000000000100000100100000 +% 827381: 24261135 0x0172320f b00000001011100100011001000001111 +% 827382: 3964368227 0xec4b7563 b11101100010010110111010101100011 +% 827383: 0 0x00000000 b00000000000000000000000000000000 +% 827384: 6585100 0x00647b0c b00000000011001000111101100001100 +% 827385: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 858786: 16673 0x00004121 b00000000000000000100000100100001 +% 858787: 24761135 0x0179d32f b00000001011110011101001100101111 +% 858788: 3964868227 0xec531683 b11101100010100110001011010000011 +% 858789: 0 0x00000000 b00000000000000000000000000000000 +% 858790: 6585100 0x00647b0c b00000000011001000111101100001100 +% 858791: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 890192: 16674 0x00004122 b00000000000000000100000100100010 +% 890193: 25261135 0x0181744f b00000001100000010111010001001111 +% 890194: 3965368227 0xec5ab7a3 b11101100010110101011011110100011 +% 890195: 0 0x00000000 b00000000000000000000000000000000 +% 890196: 6585100 0x00647b0c b00000000011001000111101100001100 +% 890197: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 921598: 16675 0x00004123 b00000000000000000100000100100011 +% 921599: 25761135 0x0189156f b00000001100010010001010101101111 +% 921600: 3965868227 0xec6258c3 b11101100011000100101100011000011 +% 921601: 0 0x00000000 b00000000000000000000000000000000 +% 921602: 6585100 0x00647b0c b00000000011001000111101100001100 +% 921603: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 953004: 16676 0x00004124 b00000000000000000100000100100100 +% 953005: 26261135 0x0190b68f b00000001100100001011011010001111 +% 953006: 3966368227 0xec69f9e3 b11101100011010011111100111100011 +% 953007: 0 0x00000000 b00000000000000000000000000000000 +% 953008: 6585100 0x00647b0c b00000000011001000111101100001100 +% 953009: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 984410: 16677 0x00004125 b00000000000000000100000100100101 +% 984411: 26761135 0x019857af b00000001100110000101011110101111 +% 984412: 3966868227 0xec719b03 b11101100011100011001101100000011 +% 984413: 0 0x00000000 b00000000000000000000000000000000 +% 984414: 6585100 0x00647b0c b00000000011001000111101100001100 +% 984415: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 1015816: 16678 0x00004126 b00000000000000000100000100100110 +% 1015817: 27261135 0x019ff8cf b00000001100111111111100011001111 +% 1015818: 3967368227 0xec793c23 b11101100011110010011110000100011 +% 1015819: 0 0x00000000 b00000000000000000000000000000000 +% 1015820: 6585100 0x00647b0c b00000000011001000111101100001100 +% 1015821: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 1047222: 16679 0x00004127 b00000000000000000100000100100111 +% 1047223: 27761135 0x01a799ef b00000001101001111001100111101111 +% 1047224: 3967868227 0xec80dd43 b11101100100000001101110101000011 +% 1047225: 0 0x00000000 b00000000000000000000000000000000 +% 1047226: 6585100 0x00647b0c b00000000011001000111101100001100 +% 1047227: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 1078628: 16680 0x00004128 b00000000000000000100000100101000 +% 1078629: 28261135 0x01af3b0f b00000001101011110011101100001111 +% 1078630: 3968368227 0xec887e63 b11101100100010000111111001100011 +% 1078631: 0 0x00000000 b00000000000000000000000000000000 +% 1078632: 6585100 0x00647b0c b00000000011001000111101100001100 +% 1078633: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 1110034: 16681 0x00004129 b00000000000000000100000100101001 +% 1110035: 28761135 0x01b6dc2f b00000001101101101101110000101111 +% 1110036: 3968868227 0xec901f83 b11101100100100000001111110000011 +% 1110037: 0 0x00000000 b00000000000000000000000000000000 +% 1110038: 6585100 0x00647b0c b00000000011001000111101100001100 +% 1110039: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 1141440: 16682 0x0000412a b00000000000000000100000100101010 +% 1141441: 29261135 0x01be7d4f b00000001101111100111110101001111 +% 1141442: 3969368227 0xec97c0a3 b11101100100101111100000010100011 +% 1141443: 0 0x00000000 b00000000000000000000000000000000 +% 1141444: 6585100 0x00647b0c b00000000011001000111101100001100 +% 1141445: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 1172846: 16683 0x0000412b b00000000000000000100000100101011 +% 1172847: 29761135 0x01c61e6f b00000001110001100001111001101111 +% 1172848: 3969868227 0xec9f61c3 b11101100100111110110000111000011 +% 1172849: 0 0x00000000 b00000000000000000000000000000000 +% 1172850: 6585100 0x00647b0c b00000000011001000111101100001100 +% 1172851: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 1204252: 16684 0x0000412c b00000000000000000100000100101100 +% 1204253: 30261135 0x01cdbf8f b00000001110011011011111110001111 +% 1204254: 3970368227 0xeca702e3 b11101100101001110000001011100011 +% 1204255: 0 0x00000000 b00000000000000000000000000000000 +% 1204256: 6585100 0x00647b0c b00000000011001000111101100001100 +% 1204257: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 1235658: 16685 0x0000412d b00000000000000000100000100101101 +% 1235659: 30761135 0x01d560af b00000001110101010110000010101111 +% 1235660: 3970868227 0xecaea403 b11101100101011101010010000000011 +% 1235661: 0 0x00000000 b00000000000000000000000000000000 +% 1235662: 6585100 0x00647b0c b00000000011001000111101100001100 +% 1235663: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 1267064: 16686 0x0000412e b00000000000000000100000100101110 +% 1267065: 31261135 0x01dd01cf b00000001110111010000000111001111 +% 1267066: 3971368227 0xecb64523 b11101100101101100100010100100011 +% 1267067: 0 0x00000000 b00000000000000000000000000000000 +% 1267068: 6585100 0x00647b0c b00000000011001000111101100001100 +% 1267069: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 1298470: 16687 0x0000412f b00000000000000000100000100101111 +% 1298471: 31761135 0x01e4a2ef b00000001111001001010001011101111 +% 1298472: 3971868227 0xecbde643 b11101100101111011110011001000011 +% 1298473: 0 0x00000000 b00000000000000000000000000000000 +% 1298474: 6585100 0x00647b0c b00000000011001000111101100001100 +% 1298475: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 1329876: 16688 0x00004130 b00000000000000000100000100110000 +% 1329877: 32261135 0x01ec440f b00000001111011000100010000001111 +% 1329878: 3972368227 0xecc58763 b11101100110001011000011101100011 +% 1329879: 0 0x00000000 b00000000000000000000000000000000 +% 1329880: 6585100 0x00647b0c b00000000011001000111101100001100 +% 1329881: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 1361282: 16689 0x00004131 b00000000000000000100000100110001 +% 1361283: 32761135 0x01f3e52f b00000001111100111110010100101111 +% 1361284: 3972868227 0xeccd2883 b11101100110011010010100010000011 +% 1361285: 0 0x00000000 b00000000000000000000000000000000 +% 1361286: 6585100 0x00647b0c b00000000011001000111101100001100 +% 1361287: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 1392688: 16690 0x00004132 b00000000000000000100000100110010 +% 1392689: 33261135 0x01fb864f b00000001111110111000011001001111 +% 1392690: 3973368227 0xecd4c9a3 b11101100110101001100100110100011 +% 1392691: 0 0x00000000 b00000000000000000000000000000000 +% 1392692: 6585100 0x00647b0c b00000000011001000111101100001100 +% 1392693: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 1424094: 16691 0x00004133 b00000000000000000100000100110011 +% 1424095: 33761135 0x0203276f b00000010000000110010011101101111 +% 1424096: 3973868227 0xecdc6ac3 b11101100110111000110101011000011 +% 1424097: 0 0x00000000 b00000000000000000000000000000000 +% 1424098: 6585100 0x00647b0c b00000000011001000111101100001100 +% 1424099: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 1455500: 16692 0x00004134 b00000000000000000100000100110100 +% 1455501: 34261135 0x020ac88f b00000010000010101100100010001111 +% 1455502: 3974368227 0xece40be3 b11101100111001000000101111100011 +% 1455503: 0 0x00000000 b00000000000000000000000000000000 +% 1455504: 6585100 0x00647b0c b00000000011001000111101100001100 +% 1455505: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 1486906: 16693 0x00004135 b00000000000000000100000100110101 +% 1486907: 34761135 0x021269af b00000010000100100110100110101111 +% 1486908: 3974868227 0xecebad03 b11101100111010111010110100000011 +% 1486909: 0 0x00000000 b00000000000000000000000000000000 +% 1486910: 6585100 0x00647b0c b00000000011001000111101100001100 +% 1486911: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 1518312: 16694 0x00004136 b00000000000000000100000100110110 +% 1518313: 35261135 0x021a0acf b00000010000110100000101011001111 +% 1518314: 3975368227 0xecf34e23 b11101100111100110100111000100011 +% 1518315: 0 0x00000000 b00000000000000000000000000000000 +% 1518316: 6585100 0x00647b0c b00000000011001000111101100001100 +% 1518317: 0 0x00000000 b00000000000000000000000000000000 +% ============================================================ +% 1549718: 16695 0x00004137 b00000000000000000100000100110111 +% 1549719: 35761135 0x0221abef b00000010001000011010101111101111 +% 1549720: 3975868227 0xecfaef43 b11101100111110101110111101000011 +% 1549721: 0 0x00000000 b00000000000000000000000000000000 +% 1549722: 6585100 0x00647b0c b00000000011001000111101100001100 +% 1549723: 0 0x00000000 b00000000000000000000000000000000 diff --git a/cresis-toolbox/ct_support/file_type_get.m b/cresis-toolbox/ct_support/file_type_get.m new file mode 100644 index 00000000..e6073027 --- /dev/null +++ b/cresis-toolbox/ct_support/file_type_get.m @@ -0,0 +1,104 @@ +function file_type = file_type_get(source) +% file_type = file_type_get(source) +% +% source: if a string, then it specifies a file path and the +% "file_type" field will be loaded from that file. +% if a struct, then it returns the "file_type" field from the struct +% For old file formats (including structures loaded from old files), it +% will try to determine the file type based on the fields that exist in the +% file or in the structure. +% +% file_type: string containing the file type + +if ischar(source) + % source is a filename + file_vars = whos('-file',source); + file_type_idx = find(strcmp('file_type',{file_vars.name})); + if ~isempty(file_type_idx) + % Done! + load(source,'file_type'); + return; + end + + % This is an old file with no file_type field. We determine the file type + % now by looking at its contents: + + field_idx = find(strcmp('param_qlook',{file_vars.name})); + if ~isempty(field_idx) + file_type = 'qlook'; + return; + end + + field_idx = find(strcmp('param_get_heights',{file_vars.name})); + if ~isempty(field_idx) + file_type = 'qlook'; + return; + end + + field_idx = find(strcmp('param_array',{file_vars.name})); + if ~isempty(field_idx) + file_type = 'array'; + return; + end + + field_idx = find(strcmp('param_combine',{file_vars.name})); + if ~isempty(field_idx) + file_type = 'array'; + return; + end + + field_idx = find(strcmp('records',{file_vars.name})); + if ~isempty(field_idx) + file_type = 'param'; + return; + end + + field_idx = find(strcmp('surf',{file_vars.name})); + if ~isempty(field_idx) + file_type = 'surf'; + return; + end + + error('File type is not currently supported by file_type_get.'); + +elseif isstruct(source) + if isfield(source,'file_type') + % Done! + file_type = source.file_type; + return; + end + + % This is an old file struct with no file_type field. We determine the + % file type now by looking at its contents: + + if isfield(source,'param_qlook') + file_type = 'qlook'; + return; + end + + if isfield(source,'param_get_heights') + file_type = 'qlook'; + return; + end + + if isfield(source,'param_array') + file_type = 'array'; + return; + end + + if isfield(source,'param_combine') + file_type = 'array'; + return; + end + + if isfield(source,'records') + file_type = 'param'; + return; + end + + if isfield(source,'surf') + file_type = 'surf'; + return; + end + +end \ No newline at end of file diff --git a/cresis-toolbox/ct_support/frames_autogenerate.m b/cresis-toolbox/ct_support/frames_autogenerate.m index 7b5f96b3..69456ee9 100644 --- a/cresis-toolbox/ct_support/frames_autogenerate.m +++ b/cresis-toolbox/ct_support/frames_autogenerate.m @@ -54,7 +54,7 @@ function autogenerate_frames(param,param_override) return; end -records = load(records_fn); +records = records_load(param); frames = []; %% Mode 2: fixed along-track spacing diff --git a/cresis-toolbox/ct_support/frames_create.m b/cresis-toolbox/ct_support/frames_create.m index b07f8412..7410d2e4 100644 --- a/cresis-toolbox/ct_support/frames_create.m +++ b/cresis-toolbox/ct_support/frames_create.m @@ -25,7 +25,11 @@ %% Creator: General Setup % ===================================================================== - obj.param = merge_structs(param, param_override); + if exist('param_override','var') + obj.param = merge_structs(param, param_override); + else + obj.param = param; + end fprintf('=====================================================================\n'); fprintf('%s: %s (%s)\n', mfilename, obj.param.day_seg, datestr(now)); @@ -56,19 +60,16 @@ frames_fn = ct_filename_support(obj.param,'','frames'); - records_fn = ct_filename_support(obj.param,'','records'); - obj.records = load(records_fn,'lat','lon'); + obj.records = records_load(obj.param,'gps_time','lat','lon'); if exist(frames_fn,'file') - tmp = load(frames_fn,'frames'); - obj.frames = tmp.frames; + obj.frames = frames_load(obj.param); if any(obj.frames.frame_idxs > length(obj.records.lat)) warning('Frames file %s\ncontains indices past the end of the records file. These indices will be ignored.', frames_fn); obj.frames.frame_idxs = obj.frames.frame_idxs(obj.frames.frame_idxs <= length(obj.records.lat)); end else obj.frames.frame_idxs = 1; - obj.frames.nyquist_zone = NaN; obj.frames.proc_mode = 0; end @@ -355,8 +356,8 @@ hold(obj.h_geotiff.h_axes,'on'); if isempty(obj.h_geotiff.proj) - x = records.lon; - y = records.lat; + x = obj.records.lon; + y = obj.records.lat; else [x,y] = projfwd(obj.h_geotiff.proj,obj.records.lat,obj.records.lon); x = x/1e3; @@ -454,7 +455,6 @@ function deletePB_callback(obj,hObj,event) mask = mask{1}; mask(1) = 0; obj.frames.frame_idxs = obj.frames.frame_idxs(~mask); - obj.frames.nyquist_zone = obj.frames.nyquist_zone(~mask); obj.frames.proc_mode = obj.frames.proc_mode(~mask); obj.h_geotiff.delete_pnt(1,find(mask)); @@ -522,9 +522,6 @@ function autoGenPB_callback(obj,hObj,event) rec = rec + 1; end - obj.frames.nyquist_zone(frm+1+num:end+num) = obj.frames.nyquist_zone(frm+1:end); - obj.frames.nyquist_zone(frm+(1:num)) = NaN; - obj.frames.proc_mode(frm+1+num:end+num) = obj.frames.proc_mode(frm+1:end); new_proc_mode = str2double(get(obj.h_gui.fig.ctrl_panel.procTB,'String')); obj.frames.proc_mode(frm+(1:num)) = new_proc_mode; @@ -560,27 +557,27 @@ function savePB_callback(obj,hObj,event) mkdir(frames_fn_dir); end - frames.gps_time = [obj.records.gps_time(frames.frame_idxs), obj.records.gps_time(end)]; - Nfrms = length(frames.frame_idxs); - frames.notes = cell(1,Nfrms); - if ~isfield(frames,'quality') - frames.quality = zeros(1,Nfrms); + obj.frames.gps_time = [obj.records.gps_time(obj.frames.frame_idxs), obj.records.gps_time(end)]; + Nfrms = length(obj.frames.frame_idxs); + obj.frames.notes = cell(1,Nfrms); + if ~isfield(obj.frames,'quality') + obj.frames.quality = zeros(1,Nfrms); end - if ~isfield(frames,'proc_mode') - frames.proc_mode = zeros(1,Nfrms); + if ~isfield(obj.frames,'proc_mode') + obj.frames.proc_mode = zeros(1,Nfrms); end - frames.Nx = length(obj.records.gps_time); - frames.param.day_seg = param.day_seg; - frames.param.season_name = param.season_name; - frames.param.radar_name = param.radar_name; - frames.param.sw_version = param.sw_version; + obj.frames.Nx = length(obj.records.gps_time); + obj.frames.param.day_seg = obj.param.day_seg; + obj.frames.param.season_name = obj.param.season_name; + obj.frames.param.radar_name = obj.param.radar_name; + obj.frames.param.sw_version = obj.param.sw_version; if obj.param.ct_file_lock - frames.file_version = '1L'; + obj.frames.file_version = '1L'; else - frames.file_version = '1'; + obj.frames.file_version = '1'; end - frames.file_type = 'frames'; + obj.frames.file_type = 'frames'; ct_file_lock_check(frames_fn,3); fprintf('Saving %s\n', frames_fn); frames = obj.frames; @@ -651,8 +648,6 @@ function update_frame_gui(obj) end obj.frames.frame_idxs(frm+1:end+1) = obj.frames.frame_idxs(frm:end); obj.frames.frame_idxs(frm) = min_idx; - obj.frames.nyquist_zone(frm+1:end+1) = obj.frames.nyquist_zone(frm:end); - obj.frames.nyquist_zone(frm) = NaN; obj.frames.proc_mode(frm+1:end+1) = obj.frames.proc_mode(frm:end); obj.frames.proc_mode(frm) = str2double(get(obj.h_gui.fig.ctrl_panel.procTB,'String')); segment.lat = obj.records.lat(min_idx); diff --git a/cresis-toolbox/ct_support/frames_id_parse.m b/cresis-toolbox/ct_support/frames_id_parse.m new file mode 100644 index 00000000..426092f9 --- /dev/null +++ b/cresis-toolbox/ct_support/frames_id_parse.m @@ -0,0 +1,57 @@ +function [day_seg,frm] = frames_id_parse(frm_id) +% [day_seg,frm] = frames_id_parse(frm_id) +% +% Function for parsing a frame ID string ('YYYYMMDD_SS_FFF') into day +% segment and frame number. +% +% frm_id: string in the form 'YYYYMMDD_SS_FFF'. +% +% day_seg: string containing the day_seg string ('YYYYMMDD_SS') +% frm: numeric value for FFF part of string. If frame FFF part of string +% is missing, then the frm number returned in NaN. +% +% Example: +% [day_seg,frm] = frames_id_parse('20140310_02_023') +% [day_seg,frm] = frames_id_parse('20140310_02_023.mat') +% [day_seg,frm] = frames_id_parse('20140310_02_02345') +% [day_seg,frm] = frames_id_parse('20140310_02_2') +% [day_seg,frm] = frames_id_parse('20140310_02') +% [day_seg,frm] = frames_id_parse({'20140310_02_023','20140310_02_025'}) +% [day_seg,frm] = frames_id_parse({'20140310_02_023','20140310_0'}) % Error test +% +% Author: John Paden + +if ischar(frm_id) + if length(frm_id) < 11 + error('frame_id_parse:frm_id_too_short','The frm_id (%s) string is too short. Length 11 (''YYYYMMDD_SS'') is minimum length.', frm_id); + end + day_seg = frm_id(1:11); + if length(frm_id) < 13 + frm = NaN; + else + frm = sscanf(frm_id(13:end),'%d'); + end + +% elseif isstruct(frm_id) + % Assume that this is a parameter structure + +% elseif isnumeric(frm_id) + +elseif iscell(frm_id) + day_seg = cell(size(frm_id)); + frm = nan(size(frm_id)); + for idx = 1:length(frm_id) + if length(frm_id{idx}) < 11 + error('frame_id_parse:frm_id_too_short','The frm_id{%d} (%s) string is too short. Length 11 (''YYYYMMDD_SS'') is minimum length.', idx, frm_id{idx}); + end + day_seg{idx} = frm_id{idx}(1:11); + if length(frm_id{idx}) < 13 + frm(idx) = NaN; + else + frm(idx) = sscanf(frm_id{idx}(13:end),'%d'); + end + end + +else + error('frame_id_parse:invalid_frm_id_type','Invalid type of frm_id'); +end diff --git a/cresis-toolbox/ct_support/frames_load.m b/cresis-toolbox/ct_support/frames_load.m index 7b40f4d5..8ee2ebf8 100644 --- a/cresis-toolbox/ct_support/frames_load.m +++ b/cresis-toolbox/ct_support/frames_load.m @@ -1,8 +1,40 @@ function frames = frames_load(param) +% frames = frames_load(param) +% +% Loads frames file. Handles old file formats. +% +% param: parameter spreadsheet structure +% +% frames: structure containing all of the frames file fields +% +% Example +% param = read_param_xls(ct_filename_param('rds_param_2019_Antarctica_Ground.xls'),'20191231_04'); +% frames = frames_load(param); +% +% Author: John Paden + +if isempty(param) + error('param is empty and should contain a radar parameter spreadsheet structure or a frames filename.'); +end + +if ischar(param) + frames_fn = param; + param = []; +elseif isstruct(param) + frames_fn = ct_filename_support(param,'','frames'); +else + error('param must be a string or struct.'); +end -frames_fn = ct_filename_support(param,'','frames'); frames = load(frames_fn); if ~isfield(frames,'frame_idxs') + if isempty(param) + if isfield(frames,'param') + param = frames.param; + else + error('A param structure must be passed in instead of frames filename to use the old file format. After updating, the filename may be used with frames_load.'); + end + end warning('Old frames file format. frames_update.m being run to update frames file.'); frames_update(param,[]); frames = load(ct_filename_support(param,'','frames')); % Load "frames" variable diff --git a/cresis-toolbox/ct_support/frames_param_cmd_frms.m b/cresis-toolbox/ct_support/frames_param_cmd_frms.m new file mode 100644 index 00000000..5409c892 --- /dev/null +++ b/cresis-toolbox/ct_support/frames_param_cmd_frms.m @@ -0,0 +1,28 @@ +function frms = frames_param_cmd_frms(param,frames) +% frms = frames_param_cmd_frms(param,frames) +% +% Checks param.cmd.frms and removes any nonexistent frames. If +% param.cmd.frms is empty, then it is set to be all frames. +% +% param: radar parameter spreadsheet structure usually read from +% read_param_xls.m. Only the param.cmd.frms field is used +% frames: frames file structure usually read with frames_load.m. Only the +% frames.frame_idxs field is used. +% +% Example: +% frames = frames_load(param); +% param.cmd.frms = frames_param_cmd_frms(param,frames); + +if ~isfield(param,'cmd') || ~isfield(param.cmd,'frms') || isempty(param.cmd.frms) + param.cmd.frms = 1:length(frames.frame_idxs); +end +[valid_frms,keep_idxs] = intersect(param.cmd.frms, 1:length(frames.frame_idxs)); +if length(valid_frms) ~= length(param.cmd.frms) + bad_mask = ones(size(param.cmd.frms)); + bad_mask(keep_idxs) = 0; + warning('Nonexistent frames specified in param.cmd.frms (e.g. frame "%g" is invalid), removing these', ... + param.cmd.frms(find(bad_mask,1))); + param.cmd.frms = valid_frms; +end + +frms = param.cmd.frms; diff --git a/cresis-toolbox/ct_support/frames_update.m b/cresis-toolbox/ct_support/frames_update.m index e798d274..7e527863 100644 --- a/cresis-toolbox/ct_support/frames_update.m +++ b/cresis-toolbox/ct_support/frames_update.m @@ -5,6 +5,8 @@ function frames_update(param,param_override) % primary purpose is to update an old frames file to the new file format. % % param: parameter spreadsheet structure array +% param_override: usually contains gRadar contents plus any additional +% parameters to override. % % Examples: See run_all_frames_update.m % @@ -21,67 +23,146 @@ function frames_update(param,param_override) %% Input checks frames_fn = ct_filename_support(param,'','frames'); +if cluster_job_check() + error('frames_update may not be called from cluster_job (gRadar.cluster.is_cluster_job is currently set to true). To remove this error, run frames_update on: %s', frames_fn); +end + +if ~isfield(param,'frames_update') || isempty(param.frames_update) + param.frames_update = []; +end + +% param.frames_update.force_update: scalar logical. Default is true. If +% true, the frames file will be updated (e.g. GPS times from records file +% will be updated). If false, the file will be checked for being in the +% newest format. If it is, then nothing is done. +if ~isfield(param.frames_update,'force_update') || isempty(param.frames_update.force_update) + param.frames_update.force_update = true; +end -% Check file format +%% Update according to file format var_list = whos('-file',frames_fn); if any(strcmpi('frames',{var_list.name})) - % Old format had a variable "frames" which the new format does not have - load(frames_fn,'frames'); + % Old format put all the variables into a single struct variable + % "frames". The new format puts all these variables at the base level in + % the file. + param.frames_update.force_update = true; + old_format = load(frames_fn,'frames'); % Load records file - records_fn = ct_filename_support(param,'','records'); - records = load(records_fn); + records = records_load(param,'gps_time'); + frames = []; + frames.frame_idxs = old_format.frames.frame_idxs; frames.gps_time = [records.gps_time(frames.frame_idxs), records.gps_time(end)]; Nfrms = length(frames.frame_idxs); frames.notes = cell(1,Nfrms); - if ~isfield(frames,'quality') + if isfield(old_format.frames,'quality') + frames.quality = old_format.frames.quality; + else frames.quality = zeros(1,Nfrms); end - if ~isfield(frames,'proc_mode') + if isfield(old_format.frames,'proc_mode') + frames.proc_mode = old_format.frames.proc_mode; + else frames.proc_mode = zeros(1,Nfrms); end frames.Nx = length(records.gps_time); frames.param.day_seg = param.day_seg; frames.param.season_name = param.season_name; frames.param.radar_name = param.radar_name; - frames.param.sw_version = param.sw_version; - - if param.ct_file_lock - frames.file_version = '1L'; - else - frames.file_version = '1'; + if ~isfield(param,'sw_version') + param.sw_version = current_software_version; end - frames.file_type = 'frames'; + frames.param.sw_version = param.sw_version; - fprintf('Saving updated frames file: %s\n', frames_fn); - ct_save(frames_fn,'-struct','frames'); else % New format frames = load(frames_fn); - % Load records file - records_fn = ct_filename_support(param,'','records'); - records = load(records_fn); - - frames.gps_time = [records.gps_time(frames.frame_idxs), records.gps_time(end)]; - Nfrms = length(frames.frame_idxs); - if ~isfield(frames,'notes') - frames.notes = cell(1,Nfrms); - end - if ~isfield(frames,'quality') - frames.quality = zeros(1,Nfrms); - end - if ~isfield(frames,'proc_mode') - frames.proc_mode = zeros(1,Nfrms); + if ~param.frames_update.force_update + + if ~isfield(frames,'frame_idxs') + error('Frames file is missing frame_idxs field. This is an incomplete frames file which cannot be updated without this essential information.'); + end + + if ~isfield(frames,'Nx') || length(frames.Nx) ~= 1 + param.frames_update.force_update = true; + end + + if ~isfield(frames,'file_type') || ~strcmp(frames.file_type,'frames') + param.frames_update.force_update = true; + end + + if ~isfield(frames,'file_version') || ~any(frames.file_version == '1') + param.frames_update.force_update = true; + end + + if ~isfield(frames,'gps_time') || length(frames.gps_time)-1 ~= length(frames.frame_idxs) + param.frames_update.force_update = true; + end + + if ~isfield(frames,'notes') || length(frames.notes) ~= length(frames.frame_idxs) + param.frames_update.force_update = true; + end + + if ~isfield(frames,'param') + param.frames_update.force_update = true; + end + + if ~isfield(frames.param,'day_seg') || ~strcmp(frames.param.day_seg,param.day_seg) + param.frames_update.force_update = true; + end + + if ~isfield(frames.param,'season_name') || ~strcmp(frames.param.season_name,param.season_name) + param.frames_update.force_update = true; + end + + if ~isfield(frames.param,'radar_name') || ~strcmp(frames.param.radar_name,param.radar_name) + param.frames_update.force_update = true; + end + + if ~isfield(frames.param,'sw_version') + param.frames_update.force_update = true; + end + + if ~isfield(frames,'proc_mode') || length(frames.proc_mode) ~= length(frames.frame_idxs) + param.frames_update.force_update = true; + end + + if ~isfield(frames,'quality') || length(frames.quality) ~= length(frames.frame_idxs) + param.frames_update.force_update = true; + end end - frames.Nx = length(records.gps_time); - frames.param.day_seg = param.day_seg; - frames.param.season_name = param.season_name; - frames.param.radar_name = param.radar_name; - frames.param.sw_version = param.sw_version; - if param.ct_file_lock + if param.frames_update.force_update + frames = load(frames_fn); + + % Load records file + records = records_load(param,'gps_time'); + + frames.gps_time = [records.gps_time(frames.frame_idxs), records.gps_time(end)]; + Nfrms = length(frames.frame_idxs); + if ~isfield(frames,'notes') + frames.notes = cell(1,Nfrms); + end + if ~isfield(frames,'quality') + frames.quality = zeros(1,Nfrms); + end + if ~isfield(frames,'proc_mode') + frames.proc_mode = zeros(1,Nfrms); + end + frames.Nx = length(records.gps_time); + frames.param.day_seg = param.day_seg; + frames.param.season_name = param.season_name; + frames.param.radar_name = param.radar_name; + frames.param.sw_version = param.sw_version; + else + fprintf(' Frames file does not need to be updated.\n'); + end +end + +if param.frames_update.force_update + if isfield(param,'ct_file_lock') && param.ct_file_lock frames.file_version = '1L'; else frames.file_version = '1'; diff --git a/cresis-toolbox/ct_support/get_frame_id.m b/cresis-toolbox/ct_support/get_frame_id.m index bd1550d0..e9fdc1f9 100644 --- a/cresis-toolbox/ct_support/get_frame_id.m +++ b/cresis-toolbox/ct_support/get_frame_id.m @@ -17,6 +17,9 @@ % .days_after: default is 1, searches this many days after max gps_time % .segment_end_time_guard: default is 0 seconds, causes the end time of % each segment to be this many seconds later than it actually is +% .segment_id_num: default is 0 and day_seg is returned as a cell array of +% strings, if set to true then day_seg is returned as a numeric +% YYYYMMDDSS. % % day_seg: cell array the same size as gps_time filled with the day_seg of % the corresponding gps_time @@ -34,6 +37,7 @@ % [day_seg,frm_id,recs] = get_frame_id(param,[1303490781.93816 1303491582.05660]) % [day_seg,frm_id,recs] = get_frame_id(param,[1303837435.04928 1303838235.17157]) % [day_seg,frm_id,recs] = get_frame_id(param,datenum_to_epoch(datenum(2011,4,22,16,46,21))) +% [day_seg,frm_id,recs] = get_frame_id(param,datenum_to_epoch(datenum('2019-09-09 14:11:19.08'))) % % Author: John Paden % @@ -56,6 +60,13 @@ search_params.segment_end_time_guard = 1; end +% search_params.segment_id_num: logical scalar, default false, if false +% then day_seg is a cell array of strings, if true then day_seg is a +% numeric array which is much faster for large gps_time input vectors. +if ~isfield(search_params,'segment_id_num') || isempty(search_params.segment_id_num) + search_params.segment_id_num = false; +end + %% Setup % Preallocate outputs @@ -72,12 +83,12 @@ if ~isfield(param,'day_seg') % If the day-segment is not provided, then load all segments records_dir = ct_filename_support(param, '', 'records'); - fns = get_filenames(records_dir,'records','','.nc'); + fns = get_filenames(records_dir,'records','','.mat'); else % If the day-segment is provided, then just load that file records_fn = ct_filename_support(param, '', 'records'); [records_fn_dir,records_fn_name] = fileparts(records_fn); - fns = {fullfile(records_fn_dir,sprintf('%s.nc',records_fn_name))}; + fns = {fullfile(records_fn_dir,sprintf('%s.mat',records_fn_name))}; end if isempty(fns) @@ -111,24 +122,10 @@ first_gps_time = []; for file_idx = 1:length(fns) records_fn = fns{file_idx}; - [records_fn_dir records_fn_name] = fileparts(records_fn); - cdf_fn = fullfile(records_fn_dir, sprintf('%s.nc', records_fn_name)); - - try - ncid = netcdf.open(cdf_fn,'NOWRITE'); - catch ME - warning('Exception during file opening'); - ME - keyboard - end - var_idx = netcdf.inqVarID(ncid,'gps_time'); - first_gps_time(file_idx) = netcdf.getVar(ncid,var_idx,[0 0]); - - finfo = ncinfo(cdf_fn); - num_recs = finfo.Variables(find(strcmp('gps_time',{finfo.Variables.Name}))).Size(2); - - last_gps_time(file_idx) = netcdf.getVar(ncid,var_idx,[0 num_recs-1]); - netcdf.close(ncid); + records = records_load(records_fn,'gps_time'); + first_gps_time(file_idx) = records.gps_time(1); + last_gps_time(file_idx) = records.gps_time(end); + num_recs = length(records.gps_time); end keep_mask = isfinite(first_gps_time) & isfinite(last_gps_time); fns = fns(keep_mask); @@ -165,7 +162,11 @@ %% Determine the record and frame numbers and form outputs cur_fn_idx = NaN; -day_seg = cell(size(gps_time)); +if search_params.segment_id_num + day_seg = zeros(size(gps_time)); +else + day_seg = cell(size(gps_time)); +end frm_id = zeros(size(frm_id)); recs = zeros(size(recs)); num_recs = zeros(size(num_recs)); @@ -196,17 +197,23 @@ records_fn = fns{cur_fn_idx}; [records_fn_dir records_fn_name] = fileparts(records_fn); - records.gps_time = ncread(records_fn,'gps_time'); + records = records_load(records_fn,'gps_time'); param.day_seg = records_fn_name(9:19); - frames_fn = ct_filename_support(param,'','frames'); - load(frames_fn); + frames = frames_load(param); end end if ~isnan(cur_fn_idx) gps_idx = gps_idx + 1; recs(cur_gps_idx:gps_idx-1) = interp1(records.gps_time,1:length(records.gps_time),sort_gps_time(cur_gps_idx:gps_idx-1),'linear','extrap'); num_recs(cur_gps_idx:gps_idx-1) = length(records.gps_time); - [day_seg{cur_gps_idx:gps_idx-1}] = deal(param.day_seg); + if search_params.segment_id_num + % day_seg is numeric array + segment_id_num = 100*str2double(param.day_seg(1:8)) + str2double(param.day_seg(10:11)); + [day_seg(cur_gps_idx:gps_idx-1)] = segment_id_num; + else + % day_seg is cell array + [day_seg{cur_gps_idx:gps_idx-1}] = deal(param.day_seg); + end for offset_idx = cur_gps_idx:gps_idx-1 new_frm_id = find(frames.frame_idxs <= recs(offset_idx),1,'last'); if isempty(new_frm_id) diff --git a/cresis-toolbox/ct_support/get_raw_files.m b/cresis-toolbox/ct_support/get_raw_files.m index e2a61a87..a406b361 100644 --- a/cresis-toolbox/ct_support/get_raw_files.m +++ b/cresis-toolbox/ct_support/get_raw_files.m @@ -1,34 +1,81 @@ -function [load_info,gps_time,recs] = get_raw_files(param,frm_id,imgs,rec_range,rec_range_type,out_dir) +function [load_info,gps_time,recs] = get_raw_files(param,frm_id,imgs,rec_range,rec_range_type,out_dir,tape_list, small_file_archives) % [load_info,gps_time,recs]= get_raw_files(param,frm_id,imgs,rec_range,rec_range_type) % +% Get a list of raw data filenames for particular frames and images or +% record ranges. Also can copy files. +% +% Inputs +% ========================================================================= +% % param: Can be either a string with the parameter spreadsheet filename OR -% a struct containing radar, season, and optionally day_seg information. +% a struct containing radar, season, and optionally day_seg information. +% % .radar_name: string containing radar name (e.g. 'kuband2') +% % .season_name: string containing season name (e.g. '2012_Greenland_P3') +% % .day_seg: string containing the day segment (e.g. '20170412_01'). This -% is not required if frm_id is a string with the day_seg in it. -% imgs: a cell array of wf-adc pair lists -% frm_id: One of these options: +% is not required if frm_id is a string with the day_seg in it. +% +% frm_id: Indicates which frames to get information about. Must be one of +% these options: +% % 1. string containing frame id (e.g. '20120514_01_317') -% 2. an integer containing the frame number (param.day_seg must be passed -% in) -% rec_range: Specifies a range of records to load in. The units specified by -% rec_range_type. Only the first and last element of rec_range are used. +% +% 2. an integer array containing the frame numbers (param.day_seg must be +% passed in) +% +% 3. a cell array of strings containing frame ids (e.g. +% {'20120514_01_317','20120514_01_318'}) +% +% imgs: a cell array of wf-adc pair lists, leave undefined or empty to do +% all images +% +% rec_range: Specifies a range of records to load in. The units specified +% by rec_range_type. Only the first and last element of rec_range are used. +% % rec_range_type: String containing 'gps_time' or 'records'. Default is -% 'records'. If 'records' is used, either the param.day_seg field must be -% defined or the frm_id must be a string with the day_seg in it. -% out_dir: Optional. May be left empty or not defined. Specifies an -% output directory to copy raw files to. +% 'records'. If 'records' is used, either the param.day_seg field must be +% defined or the frm_id must be a string with the day_seg in it. +% +% out_dir: Optional. May be left empty or not defined. Specifies an output +% directory to copy raw files to. +% +% tape_list: The name of a file containing mappings between files and tapes. +% Or a matrix containing this mapping. The matrix must be presorted, formatted +% in the same manner as is done at the TAPE_LIST SORT comment below. See run_get_raw_files. +% If provided and not empty, load_info will contain a field, tapes, +% containing the name of the tape each corresponding file is present in as +% well as a field, stored_filenames, with the name of the file on the tape. +% +% small_file_archives: A mapping between directories and the name of the +% corresponding small file archive. See run_get_raw_files. +% +% Outputs +% ========================================================================= % % load_info: struct with file information for the range of data specified +% % .filenames: cell array of cells which contain the raw data filenames +% +% .tapes: cell array of tapes corresponding to the filenames when +% tape_list is given +% +% .stored_filenames: cell array of filenames on tapes corresponding to the +% filenames array when tape_list is given +% % .file_idx: cell array of integers which specify an index into .filenames -% for each record +% for each record +% % .offset: raw file byte offset to the beginning of each record +% % gps_time: prints out the start and stop GPS times and puts them here +% % recs: raw data records into csarp_support records file % -% Example: +% Examples +% ========================================================================= +% % % Get filename information for a range of records % [load_info,gps_time,recs] = get_raw_files(struct('radar_name','rds','season_name','2017_Antarctica_TObas'),'20170122_01',{},[25092 27499]) % @@ -40,7 +87,9 @@ % [load_info,gps_time,recs] = get_raw_files('accum_param_2017_Greenland_P3.xls','20170412_01_023',[],1.4920018728e9,'gps_time'); % % % Example copying files -% load_info = get_raw_files(struct('radar_name','mcrds','season_name','2008_Greenland_TO'),'20080627_06_001','/tmp/) +% load_info = get_raw_files(struct('radar_name','mcrds','season_name','2008_Greenland_TO'),'20080627_06_001','/tmp/') +% +% ========================================================================= % % Author: John Paden % @@ -51,20 +100,27 @@ if ischar(param) param_fn = ct_filename_param(param); clear param; - if ~ischar(frm_id) - error('param as a filename requires frm_id to be a frame ID string so the segment can be determined.'); + if ~ischar(frm_id) && ~iscell(frm_id) + error('param as a filename requires frm_id to be a frame ID string or cell array of frame ID strings so the segment can be determined.'); end end if ischar(frm_id) - param.day_seg = frm_id(1:11); + [param.day_seg] = frames_id_parse(frm_id); +elseif iscell(frm_id) && length(frm_id) >= 1 && ischar(frm_id{1}) + param.day_seg = frm_id{1}(1:11); elseif ~isfield(param,'day_seg') - error('param.day_seg or frm_id as a frame id string must be provided'); + error('param.day_seg or frm_id as a frame id string or cell array of frame ID strings must be provided'); end if ischar(frm_id) - frm = str2double(frm_id(end-2:end)); -else + [~,frm] = frames_id_parse(frm_id); % Extract frame number from frame ID +elseif iscell(frm_id) + [~,frm] = frames_id_parse(frm_id); % Extract frame numbers from frame IDs + frm_id = frm_id{1}; +elseif isnumeric(frm_id) && length(frm_id) >= 1 frm = frm_id; - frm_id = sprintf('%s_%03d', param.day_seg, frm); + frm_id = sprintf('%s_%03d', param.day_seg, frm(1)); +else + error('Invalid combination of input arguments for param and frm.'); end if ~isempty(param_fn) @@ -82,8 +138,16 @@ global gRadar; param = merge_structs(gRadar,param); +for wf = 1:length(param.radar.wfs) + if isfield(param.radar.wfs(wf),'rx_paths') && ~isempty(param.radar.wfs(wf).rx_paths) + param.radar.wfs(wf).rx_paths = param.radar.wfs(wf).rx_paths; + else + param.radar.wfs(wf).rx_paths = 1; + end +end + % Populate imgs cell array with all wf-adc pairs if not specified -if isempty(imgs) +if ~exist('imgs','var') || isempty(imgs) for wf = 1:length(param.radar.wfs) for adc = 1:length(param.radar.wfs(wf).rx_paths) imgs{wf}(adc,1:2) = [wf adc]; @@ -97,13 +161,10 @@ end % Load the records file -records_fn = ct_filename_support(param,'','records'); -% load(records_fn); -records = load(records_fn); +records = records_load(param); % Load the frames file -frames_fn = ct_filename_support(param,'','frames'); -load(frames_fn); +frames = frames_load(param); %% Get the records associated with the frm or record if exist('rec_range','var') && ~isempty(rec_range) @@ -129,25 +190,26 @@ stop_rec = good_recs(end); end end + recs = start_rec:stop_rec; else % Use the provided frame ID to determine the records to get - - if frm > length(frames.frame_idxs) - error('Frame %d > %d does not exist\n', frm, length(frames.frame_idxs)); - elseif frm == length(frames.frame_idxs) - % Special case that handles last frame in segment - start_rec = frames.frame_idxs(frm); - stop_rec = length(records.lat); - else - start_rec = frames.frame_idxs(frm); - stop_rec = frames.frame_idxs(frm+1)-1; + + param.cmd.frms = frm; + frms = frames_param_cmd_frms(param,frames); + recs = false(size(records.gps_time)); + for idx = 1:length(frms) + if frms(idx) == length(frames.frame_idxs) % handling the special case for the last frame + recs(frames.frame_idxs(frms(idx)):frames.Nx) = true; + else + recs(frames.frame_idxs(frms(idx)):frames.frame_idxs(frms(idx)+1)-1) = true; + end end + recs = find(recs); end -recs = start_rec:stop_rec; %% Get GPS Times -gps_time = records.gps_time(start_rec:stop_rec); +gps_time = records.gps_time(recs); %% Get filenames load_info = get_raw_files_sub(param,wf_adc_list,records,recs); @@ -176,3 +238,103 @@ end end +%% Find Tape Locations +% Determine the tape in which each file is present from the given tape_list + +% TAPE_LIST SORT +if exist('tape_list', 'var') && ~isempty(tape_list) && size(tape_list, 1) == 1 + % Given a string as input, load the matrix + tape_list = readmatrix(tape_list, 'Delimiter', ' ', 'OutputType', 'string'); + [~, file, ext] = fileparts(tape_list(:, 2)); + tape_list = [tape_list file + ext]; + tape_list = sortrows(tape_list, 3); +end + +if exist('tape_list', 'var') && ~isempty(tape_list) && size(tape_list, 1) > 1 + + % SMALL_FILE_ARCHIVES CONSTRUCTION + % Map directories to the corresponding small file archives + if ~exist('small_file_archives', 'var') || isempty(small_file_archives) + small_file_archives = string(); + for file_idx = 1:size(tape_list, 1) + filepath = tape_list{file_idx, 2}; + tapes = tape_list(file_idx, 1); + if endsWith(filepath, "small_file_archive.tar") + [parent, file_name, ext] = fileparts(filepath); + file_name = [file_name ext]; + archive_idx = size(small_file_archives, 1) + 1; + small_file_archives(archive_idx, 1) = tapes; + small_file_archives(archive_idx, 2) = convertCharsToStrings(parent); + small_file_archives(archive_idx, 3) = convertCharsToStrings(file_name); + end + end + small_file_archives = sortrows(small_file_archives, 2); + end + + % We have a matrix of tape locations, match to filenames + + load_info.stored_filenames = {}; + load_info.tapes = {}; + for filename_group_idx=1:length(load_info.filenames) + filename_group = load_info.filenames{filename_group_idx}; + load_info.stored_filenames{filename_group_idx} = {}; + load_info.tapes{filename_group_idx} = {}; + + for filename_idx=1:length(filename_group) + filepath = filename_group{filename_idx}; + [~, file, ext] = fileparts(filepath); + filename = [file ext]; + + % Perform binary search to find filename in sorted tape_list + [lia, locb] = ismember(filename, tape_list(:, 3)); + + if ~lia + load_info.stored_filenames{filename_group_idx}{filename_idx} = nan; + load_info.tapes{filename_group_idx}{filename_idx} = nan; + else + + % Check if next file in list has same filename and warn user that a duplicate exists + if strcmp(tape_list{locb + 1, 3}, filename) + disp 'duplicate filenames in tape_list'; + keyboard; + end + load_info.stored_filenames{filename_group_idx}{filename_idx} = tape_list{locb, 2}; + load_info.tapes{filename_group_idx}{filename_idx} = tape_list{locb, 1}; + end + end + + % Find corresponding small_file_archive + filepath = load_info.stored_filenames{filename_group_idx}{1}; % All files in group should have same parent + while true + % Iterate up the file path and see if any directory is in the small_file_archives mapping + [filepath, ~, ~] = fileparts(filepath); + if strcmp(filepath, "/") + break; + end + [~, locb] = ismember(convertCharsToStrings(filepath), small_file_archives(:, 2)); + if locb ~= 0 + load_info.stored_filenames{filename_group_idx}{end + 1} = fullfile(small_file_archives{locb, 2}, small_file_archives{locb, 3}); + load_info.tapes{filename_group_idx}{end + 1} = small_file_archives{locb, 1}; + + % Find original path + [~, parent , ~] = fileparts(small_file_archives{locb, 2}); + original_path = filename_group{filename_idx}; + found = false; + while ~found + [original_path, ~, ~] = fileparts(original_path); + if endsWith(original_path, parent) + found = true; + break; + end + end + if ~found + % Original path could not be determined + original_path = nan; + end + load_info.filenames{filename_group_idx}{end + 1} = fullfile(original_path, small_file_archives{locb, 3}); + + break; + end + end + end +end \ No newline at end of file diff --git a/cresis-toolbox/ct_support/get_raw_files_sub.m b/cresis-toolbox/ct_support/get_raw_files_sub.m index f70663b4..248c6bb1 100644 --- a/cresis-toolbox/ct_support/get_raw_files_sub.m +++ b/cresis-toolbox/ct_support/get_raw_files_sub.m @@ -3,13 +3,8 @@ % % Support function for get_raw_files -% adc_headers: the actual adc headers that were loaded -if ~isfield(param.records.file,'adc_headers') || isempty(param.records.file.adc_headers) - param.records.file.adc_headers = param.records.file.adcs; -end - % boards_headers: the boards that the actual adc headers were loaded from -[boards,board_idx,profile] = wf_adc_to_board(param,wf_adc_list); +[boards,board_idx,~] = wf_adc_to_board(param,wf_adc_list); % Populate load_info struct load_info = []; @@ -76,7 +71,7 @@ else for fn_idx = 1:length(load_info.filenames{idx}) % Get the file's name - fn_name = records.relative_filename{board_idx(idx)}{fn_idx}; + fn_name = load_info.filenames{idx}{fn_idx}; [fn_dir] = get_segment_file_list(param,board_idx(idx)); fn = fullfile(fn_dir,fn_name); %fprintf('%s\n', fn); diff --git a/cresis-toolbox/ct_support/get_raw_files_tape.m b/cresis-toolbox/ct_support/get_raw_files_tape.m new file mode 100644 index 00000000..81aae566 --- /dev/null +++ b/cresis-toolbox/ct_support/get_raw_files_tape.m @@ -0,0 +1,107 @@ +function get_raw_files_tape(load_info,retrieve_mode,varargin) +% get_raw_files_tape(load_info,retrieve_mode,varargin) +% +% load_info: result from get_raw_files.m +% +% retrieve_mode: string containing the mode which can be: +% * 'list' (default): lists the files to be copied and pasted +% * 'read': retrieves the files by reading the first byte of each file +% * 'copy': copies the files to a new location, this requires passing +% base_dir and new_base_dir strings as name value pairs (see example) +% +% % Get the file information +% [load_info,gps_time,recs] = get_raw_files(ct_filename_param('rds_param_2014_Greenland_P3.xls'),{'20140410_01_057','20140410_01_058'},{[1 1;1 5;1 9; 1 13]}); +% +% % List files +% get_raw_files_tape(load_info,'list'); +% +% % Tape or disk +% get_raw_files_tape(load_info,'tape_or_disk'); +% +% % Forces files to be retrieved by reading the first byte from each file +% get_raw_files_tape(load_info,'read'); +% +% % Copies files +% get_raw_files_tape(load_info,'copy','base_dir','/cresis/snfs1/data/MCoRDS/','new_base_dir','/tmp/') +% % Copies files output: +% /cresis/snfs1/data/MCoRDS/2014_Greenland_P3/20140410/board0/mcords3_0_20140410_105853_00_0002.bin +% /tmp/2014_Greenland_P3/20140410/board0/mcords3_0_20140410_105853_00_0002.bin (02-Feb-2021 15:05:01) +% +% Authors: John Paden + +if ~exist('retrieve_mode','var') || isempty(retrieve_mode) + retrieve_mode = 'list'; +end + +for name_val_idx = 1:2:nargin-2 + param.(varargin{name_val_idx}) = varargin{name_val_idx+1}; +end + +if ~isempty(regexpi(retrieve_mode,'list')) + fprintf('%s\n', repmat('=',[1 80])); + fprintf('Raw files to be retrieved from tape\n'); + fprintf('%s\n', repmat('=',[1 80])); + for board_idx = 1:length(load_info.filenames) + for fn_idx = 1:length(load_info.filenames{board_idx}) + fn = load_info.filenames{board_idx}{fn_idx}; + fprintf('%s\n', fn); + end + end +end + +if ~isempty(regexpi(retrieve_mode,'tape_or_disk')) + fprintf('%s\n', repmat('=',[1 80])); + fprintf('Raw files to be retrieved from tape\n'); + fprintf('%s\n', repmat('=',[1 80])); + for board_idx = 1:length(load_info.filenames) + for fn_idx = 1:length(load_info.filenames{board_idx}) + fn = load_info.filenames{board_idx}{fn_idx}; + cmd = sprintf('du -sk %s | awk ''{print $1}''',fn); + cmd2 = sprintf('du -sk --apparent-size %s | awk ''{print $1}''',fn); + [status,result] = system(cmd); + [status2,result2] = system(cmd2); + if status == 0 && status2 == 0 + size_kb = str2double(result); + size_kb2 = str2double(result2); + if 10*size_kb < size_kb2 + fprintf('%s\tTAPE\n', fn); + else + fprintf('%s\tDISK\n', fn); + end + else + fprintf('%s: COMMAND FAILED, NO ANSWER\n', fn); + end + end + end +end + +if ~isempty(regexpi(retrieve_mode,'read')) + fprintf('%s\n', repmat('=',[1 80])); + fprintf('Retrieving (reads first byte from each file)\n'); + fprintf('%s\n', repmat('=',[1 80])); + for board_idx = 1:length(load_info.filenames) + for fn_idx = 1:length(load_info.filenames{board_idx}) + fn = load_info.filenames{board_idx}{fn_idx}; + fprintf('%s (%s)\n', fn, datestr(now)); + fid = fopen(fn,'r'); + fread(fid,1); + fclose(fid); + end + end +end + +if ~isempty(regexpi(retrieve_mode,'copy')) + % Requires base_dir and new_base_dir to be assigned in name-value pair + % varargin + fprintf('%s\n', repmat('=',[1 80])); + fprintf('Copying\n'); + fprintf('%s\n', repmat('=',[1 80])); + for board_idx = 1:length(load_info.filenames) + for fn_idx = 1:length(load_info.filenames{board_idx}) + fn = load_info.filenames{board_idx}{fn_idx}; + new_fn = fullfile(param.new_base_dir,fn(1+length(param.base_dir):end)); + fprintf('%s\n %s (%s)\n', fn, new_fn, datestr(now)); + %copyfile(fn,new_fn); + end + end +end diff --git a/cresis-toolbox/ct_support/get_segment_file_list.m b/cresis-toolbox/ct_support/get_segment_file_list.m index ff7e7742..881271c2 100644 --- a/cresis-toolbox/ct_support/get_segment_file_list.m +++ b/cresis-toolbox/ct_support/get_segment_file_list.m @@ -6,16 +6,35 @@ % using run_get_segment_file_list.m. % % param: struct from param spreadsheet read in by read_param_xls +% % .records +% % .file -% .version: raw file version (see "raw file guide" on wiki) -% .board_folder_name: board folder name (%b in the filename will be -% replaced by the board number) +% % .base_dir: base directory -% .prefix: beginning filename search term -% .midfix: middle filename search term -% .regexp: additional regular expression to run after the initial file -% search +% +% .board_folder_name: board folder name (%b in the filename will be +% replaced by the corresponding string for the current board from the +% cell array of strings param.records.file.boards) +% +% .boards: cell array of strings containing the board folder names (e.g. +% {'board0','board1',...} or {'chan1','chan2',...}) +% +% .midfix: string containing middle filename search term. Default is +% empty string. +% +% .prefix: string containing beginning filename search term. Default is +% empty string. +% +% .regexp: optional regular expression string that runs after the +% initial file search if not empty. Default is empty string. +% +% .suffix: string containing end filename search term. Default is empty +% string. +% +% .version: integer scaler containing the raw file version (see "raw +% file guide" on wiki) +% % board: optional parameter used with some radars that have multiple adcs % % Author: John Paden @@ -51,7 +70,7 @@ end % Determine raw filename extension and check file version -if any(param.records.file.version == [1 9:10 101 103 401 412]) +if any(param.records.file.version == [1 9:10 101 103 401 412 415 420]) ext = '.dat'; elseif any(param.records.file.version == [2:8 11 102 402:408 411]) ext = '.bin'; @@ -144,9 +163,21 @@ % A stop index of -N says to include all but the last N files stop_idx = length(fns) + param.records.file.stop_idx; else - stop_idx = param.records.file.stop_idx(board_idx); + if board_idx > 1 && length(param.records.file.stop_idx) == 1 + % Old parameter spreadsheet format only contained a single entry for + % all boards in param.records.file.stop_idx + stop_idx = param.records.file.stop_idx; + else + stop_idx = param.records.file.stop_idx(board_idx); + end + end + if board_idx > 1 && length(param.records.file.start_idx) == 1 + % Old parameter spreadsheet format only contained a single entry for + % all boards in param.records.file.start_idx + file_idxs = param.records.file.start_idx:stop_idx; + else + file_idxs = param.records.file.start_idx(board_idx):stop_idx; end - file_idxs = param.records.file.start_idx(board_idx):stop_idx; if isempty(file_idxs) error('No files selected to load out of %i files', length(fns)); diff --git a/cresis-toolbox/ct_support/layer_tracker.m b/cresis-toolbox/ct_support/layer_tracker.m deleted file mode 100644 index feede56b..00000000 --- a/cresis-toolbox/ct_support/layer_tracker.m +++ /dev/null @@ -1,737 +0,0 @@ -function Surface = layer_tracker(param,param_override) -% Surface = layer_tracker(param,param_override) -% -% Tracks a layer using one of the layer tracking methods. -% -% track: Parameters used to control the tracker. Only general parameters -% used within update_surface_with_tracker are included here. For more -% details about tracker specific parameters, look at the help for: -% tracker_snake_simple, tracker_threshold, tracker_max -% -% track structure fields: -% ------------------------------------------------------------------------- -% .en: must be true for update_surface_with_tracker to run on a segment -% .medfilt: scalar which specifies the length of a median filter to apply -% to the tracked data. Similar to medfilt1, but it handles edge -% conditions and allows for a threshold parameter to be set. -% .medfilt_threshold: scalar used with medfilt. Median filter will only -% update in the difference of the median filter is more than this -% threshold. Default is 0 which means every pixel is updated. -% .feedthru: all values in the radar image which exceed the power mask set -% by the time and power_dB vectors will be set to zero power. To get -% these values a typical procedure is: -% plot(mdata.Time, lp(mean(mdata.Data,2))) -% [track.feedthru.time,track.feedthru.power_dB] = ginput; % Press enter to end the input -% .time: N length vector of two way travel times -% .power_dB: N length vector of power in dB -% .method: string containing the method to use for tracking. Options: -% 'threshold': runs tracker_threshold (generally the best for surface altimetry) -% 'snake': runs tracker_snake_simple -% 'max': tracker_max -% .max_diff: used by tracker routines (optional, default is inf) -% .min_bin: used by tracker routines -% .max_bin: used by tracker routines (optional, default is not used) -% .init.method: used by tracker routines (optional, default is not used) -% .init.dem_layer: dem layer struct to be used with opsLoadLayers -% (usually lidar surface) -% -% Surface: If there is an output argument, then param.cmd.frms must be set -% to a single frame and the surface for this frame will be returned and no -% outputs will be saved. -% -% For more details about tracker specific parameters: -% tracker_snake_simple, tracker_threshold, tracker_max -% -% Example: -% See run_layer_tracker.m to run in regular mode. -% See qlook_combine_task.m to run in single frame mode. -% -% Author: John Paden - - -%% General Setup -% ===================================================================== -param = merge_structs(param, param_override); - -%% Input Checks -% ===================================================================== - -physical_constants; - -if ~isfield(param,'layer_tracker') || isempty(param.layer_tracker) - param.layer_tracker = []; -end - -if ~isfield(param.layer_tracker,'debug_plots') || isempty(param.layer_tracker.debug_plots) - param.layer_tracker.debug_plots = {}; -end -if ~isempty(param.layer_tracker.debug_plots) - h_fig = get_figures(1,true); -end -enable_debug_plot = any(strcmp('debug',param.layer_tracker.debug_plots)); - -% echogram_img: To choose an image besides the base (0) image -if ~isfield(param.layer_tracker,'echogram_img') || isempty(param.layer_tracker.echogram_img) - param.layer_tracker.echogram_img = 0; -end -echogram_img = param.layer_tracker.echogram_img; - -% echogram_source: location of echogram data used for tracking -if ~isfield(param.layer_tracker,'echogram_source') || isempty(param.layer_tracker.echogram_source) - error('An echogram_source must be specified.'); -end -echogram_source = param.layer_tracker.echogram_source; - -if ~isfield(param.layer_tracker,'layer_params') || isempty(param.layer_tracker.layer_params) - param.layer_tracker.layer_params = []; -end -layer_params = param.layer_tracker.layer_params; - -%% General Setup Continued -% ===================================================================== - -if isstruct(echogram_source) - fprintf(' %s: %s (%s)\n', mfilename, 'param.layer_tracker.echogram_source', datestr(now)); -else - fprintf('=====================================================================\n'); - fprintf('%s: %s (%s)\n', mfilename, param.day_seg, datestr(now)); - fprintf('=====================================================================\n'); -end - -%% Input Checks: track field -% ===================================================================== - -if ~isfield(param.layer_tracker,'track') || isempty(param.layer_tracker.track) - param.layer_tracker.track = []; -end -track = merge_structs(param.qlook.surf,param.layer_tracker.track); - -% profile: default is no profile, otherwise loads preset configuration -if ~isfield(track,'profile') || isempty(track.profile) - track.profile = ''; -end -track = merge_structs(layer_tracker_profile(param,track.profile), track); - -if ~isfield(track,'en') || isempty(track.en) - % If true, tracking will be done on this segment. If false, then no - % tracking is done. Default is true. - track.en = true; -end -if ~track.en - return; -end - -if ~isfield(track,'data_noise_en') || isempty(track.data_noise_en) - % If true, then a matrix data_noise will be created which will go through - % all the same operations as data except no "sidelobe" and no "feedthru" - % masking will be done. This is sometimes necessary to enable when - % sidelobe or feedthru remove too much data so that the noise estimatation - % process in the tracker does not function properly. Currently only - % tracker_threshold makes use of data_noise. - track.data_noise_en = false; -end - -% debug_time_guard: Vertical band around layer in debug plot -if ~isfield(track,'debug_time_guard') || isempty(track.debug_time_guard) - track.debug_time_guard = 50e-9; -end -debug_time_guard = track.debug_time_guard; - -if ~isfield(track,'detrend') || (isempty(track.detrend) && ~ischar(track.detrend)) - track.detrend = 0; -end -if ischar(track.detrend) - % Load detrend file generated by run_get_echogram_stats - % e.g. ct_tmp/echogram_stats/snow/2011_Greenland_P3/stats_20110329_02.mat - if isempty(track.detrend) - track.detrend = [ct_filename_ct_tmp(param,'','echogram_stats','stats') '.mat']; - end - detrend = load(track.detrend,'dt','bins','min_means'); - detrend.time = detrend.dt*detrend.bins; -end - -if ~isfield(track,'feedthru') || isempty(track.feedthru) - track.feedthru = []; -end - -if ~isfield(track,'filter') || isempty(track.filter) - track.filter = [1 1]; -end -if length(track.filter) == 1 - warning('Deprecated surf.filter format. Should specify 2 element vector that specifies the multilooks in [cross-track along-track].'); - track.filter = [1 track.filter(1)]; -end -if any(mod(track.filter,2) == 0) - error('Surface filter lengths must be odd. layer_tracker.track.filter = [%d %d].', layer_tracker.track.filter); -end - -if ~isfield(track,'filter_trim') || isempty(track.filter_trim) - track.filter_trim = [0 0]; -end - -if ~isfield(track,'fixed_value') || isempty(track.fixed_value) - track.fixed_value = 0; -end - -if ~isfield(track,'init') || isempty(track.init) - track.init = []; -end -if ~isfield(track.init,'method') || isempty(track.init.method) - track.init.method = 'max'; -end -if ~isfield(track.init,'snake_rng') || isempty(track.init.snake_rng) - track.init.snake_rng = [-2e-7 2e-7]; -end -if ~isfield(track.init,'dem_layer') || isempty(track.init.dem_layer) - track.init.dem_layer = ''; -end -if ~any(strcmpi(track.init.method,{'max','nan','snake','medfilt','dem'})) - error('Unsupported surface init method %s. Options are max, nan, snake, medfilt, or dem. max is default.', track.init.method); -end -if ~isfield(track.init,'max_diff') || isempty(track.init.max_diff) - track.init.max_diff = inf; -end -if ~isfield(track.init,'max_diff_method') || isempty(track.init.max_diff_method) - if strcmpi(track.init.method,{'dem'}) || ~isempty(track.init.dem_layer) - % If the initial surface is from a dem or reference layer, the default - % method for outliers is to use merge_vectors to fill the outliers in. - track.init.max_diff_method = 'merge_vectors'; - else - % Otherwise the default method is just to interpolate between the good - % points that exist to fill the outliers in. - track.init.max_diff_method = 'interp_finite'; - end -end -if ~any(strcmpi(track.init.max_diff_method,{'merge_vectors','interp_finite'})) - error('Unsupported max diff method %s. Options are merge_vectors, interp_finite. The default is interp_finite unless a dem or reference layer is provided.', track.init.max_diff_method); -end - -if ~isfield(track,'max_bin') || isempty(track.max_bin) - track.max_bin = inf; -end - -if ~isfield(track,'max_rng') || isempty(track.max_rng) - track.max_rng = [0 0]; -end - -if ~isfield(track,'max_rng_units') || isempty(track.max_rng_units) - track.max_rng_units = 'time'; -end - -if ~isfield(track,'medfilt') || isempty(track.medfilt) - % Like medfilt1 except it handles the edges of the frame better - track.medfilt = 0; -end -if ~isfield(track,'medfilt_threshold') || isempty(track.medfilt_threshold) - % medfilt_threshold: the point is compared to the medfilt1 result and if - % the point is > medfilt_threshold from medfilt1, then the point is - % replaced with the medfilt1 result. The two extremes are: - % 0 makes medfilt act like medfilt1 - % inf effectively disables the medfilt operation - track.medfilt_threshold = 0; -end - -if ~isfield(track,'method') || isempty(track.method) - track.method = ''; -end - -if ~isfield(track,'min_bin') || isempty(track.min_bin) - track.min_bin = 0; -end - -if ~isfield(track,'prefilter_trim') || isempty(track.prefilter_trim) - track.prefilter_trim = [0 0]; -end - -if ~isfield(track,'sidelobe_rows') || isempty(track.sidelobe_rows) || ~isfield(track,'sidelobe_dB') || isempty(track.sidelobe_dB) - track.sidelobe_dB = []; - track.sidelobe_rows = []; -end - -if ~isfield(track,'snake_rng') || isempty(track.snake_rng) - track.snake_rng = [-2e-7 2e-7]; -end - -if ~isfield(track,'threshold') || isempty(track.threshold) - track.threshold = 15; -end - -if ~isfield(track,'threshold_noise_rng') || isempty(track.threshold_noise_rng) - track.threshold_noise_rng = [0 -inf -1]; -end - -%% Load in ocean mask, land DEM, and sea surface DEM -if isfield(track,'init') && strcmpi(track.init.method,'dem') - global gdem; - if isempty(gdem) || ~isa(gdem,'dem_class') || ~isvalid(gdem) - gdem = dem_class(param,500); - end - gdem.set_res(500); - gdem.ocean_mask_mode = 'l'; - - gdem_str = sprintf('%s:%s:%s',param.radar_name,param.season_name,param.day_seg); - if ~strcmpi(gdem_str,gdem.name) - % Load records file - records_fn = ct_filename_support(param,'','records'); - records = load(records_fn); - gdem.set_vector(records.lat,records.lon,gdem_str); - end -end - -%% Determine valid frames to process -if nargout ~= 1 - frames_fn = ct_filename_support(param,'','frames'); - if ~exist(frames_fn,'file') - warning('Cannot verify param.cmd.frms because frames file does not exist: %s.', frames_fn); - else - frames = load(frames_fn); - if isempty(param.cmd.frms) - param.cmd.frms = 1:length(frames.frame_idxs); - end - % Remove frames that do not exist from param.cmd.frms list - [valid_frms,keep_idxs] = intersect(param.cmd.frms, 1:length(frames.frame_idxs)); - if length(valid_frms) ~= length(param.cmd.frms) - bad_mask = ones(size(param.cmd.frms)); - bad_mask(keep_idxs) = 0; - warning('Nonexistent frames specified in param.cmd.frms (e.g. frame "%g" is invalid), removing these', ... - param.cmd.frms(find(bad_mask,1))); - param.cmd.frms = valid_frms; - end - end -end - -%% Load reference surface -if isfield(track,'init') && isfield(track.init,'dem_layer') ... - && ~isempty(track.init.dem_layer) - layers = opsLoadLayers(param,track.init.dem_layer); - - % Ensure that layer gps times are monotonically increasing (this should - % always be the case, but occasionally files are corrupt/have errors) - lay_idx = 1; - layers_fieldnames = fieldnames(layers(lay_idx)); - [~,unique_idxs] = unique(layers(lay_idx).gps_time); - for field_idx = 1:length(layers_fieldnames)-1 - if ~isempty(layers(lay_idx).(layers_fieldnames{field_idx})) - layers(lay_idx).(layers_fieldnames{field_idx}) = layers(lay_idx).(layers_fieldnames{field_idx})(unique_idxs); - end - end -end - -%% Track -orig_track = track; -for frm_idx = 1:length(param.cmd.frms) - %% Track: Load echogram data - frm = param.cmd.frms(frm_idx); - track = orig_track; - if isstruct(echogram_source) - % echogram_source is the data structure - mdata = echogram_source; - data_fn_name = 'param.layer_tracker.echogram_source'; - - else - % echogram_source is a file path indicator - if echogram_img == 0 - data_fn = fullfile(ct_filename_out(param,echogram_source,''), ... - sprintf('Data_%s_%03d.mat', param.day_seg, frm)); - else - data_fn = fullfile(ct_filename_out(param,echogram_source,''), ... - sprintf('Data_img_%02d_%s_%03d.mat', echogram_img, param.day_seg, frm)); - end - [~,data_fn_name] = fileparts(data_fn); - fprintf('%d of %d %s (%s)\n', frm_idx, length(param.cmd.frms), data_fn, datestr(now)); - - if ~exist(data_fn,'file') - warning(' Missing file\n'); - continue; - end - - if strcmpi(track.method,'') && ~enable_debug_plot - mdata = load_L1B(data_fn,'GPS_time','Latitude','Longitude','Elevation','Time'); - else - mdata = load_L1B(data_fn); - end - end - - data = lp(mdata.Data); - Nx = size(mdata.Data,2); - if size(mdata.Data,1) < 2 - Surface = nan(1,Nx); - new_quality = ones(1,Nx); - - else - - %% Track: Interpolate GIMP and Geoid - if strcmpi(track.init.method,'dem') - gdem.set_vector(mdata.Latitude,mdata.Longitude); - [land_dem,msl,ocean_mask] = gdem.get_vector_dem(); - - % Merge land surface and sea surface DEMs - track.dem = land_dem; - track.dem(ocean_mask) = msl(ocean_mask); - track.dem = (mdata.Elevation - track.dem) / (c/2); - track.dem = interp1(mdata.Time,1:length(mdata.Time),track.dem + track.init.dem_offset,'linear','extrap'); - track.dem = interp_finite(track.dem,1); - end - - %% Track: Prefilter trim - % Also set leading/following zeros or ~isfinite to NaN - for rline = 1:Nx - start_bin = find(isfinite(data(:,rline)),1); - if ~isempty(start_bin) - stop_bin = min(size(data,1), start_bin+track.prefilter_trim(1)-1); - data(1:stop_bin,rline) = NaN; - end - stop_bin = find(isfinite(data(:,rline)),1,'last'); - if ~isempty(stop_bin) - start_bin = max(1, stop_bin-track.prefilter_trim(2)+1); - data(start_bin:end,rline) = NaN; - end - end - - if track.data_noise_en - data_noise = data; - end - - %% Track: Feed through removal - if ~isempty(track.feedthru) - % Interpolate feed through power levels on to data time axis - feedthru_threshold = interp1(track.feedthru.time,track.feedthru.power_dB,mdata.Time); - feedthru_threshold = interp_finite(feedthru_threshold,-inf,[],@(x) ~isnan(x)); - - % Set all data to zero that does not exceed the feed through - % threshold power - for rline=1:Nx - data(data(:,rline) <= feedthru_threshold,rline) = NaN; - end - end - - %% Track: Detrend - if ischar(track.detrend) - % A detrend file was passed in - detrend_curve = interp_finite(interp1(detrend.time,interp_finite(detrend.min_means),mdata.Time),NaN); - if all(isnan(detrend_curve)) - error('Detrend curve is all NaN.'); - end - if 0 - % Debug - rline = 200; - figure(1); clf; - plot(data(:,rline)) - hold on - mean_power = nanmean(data,2); - plot(mean_power) - plot(detrend_curve); - keyboard - end - data = bsxfun(@minus,data,detrend_curve); - if track.data_noise_en - data_noise = bsxfun(@minus,data_noise,detrend_curve); - end - - elseif track.detrend > 0 - poly_x = (-size(data,1)/2+(1:size(data,1))).'; - mean_power = nanmean(data,2); - good_mask = isfinite(mean_power); - p = polyfit(poly_x(good_mask),mean_power(good_mask),track.detrend); - detrend_curve = polyval(p,poly_x); - detrend_curve(~good_mask) = NaN; - detrend_curve = interp_finite(detrend_curve,0); - if 0 - % Debug - rline = 200; - figure(1); clf; - plot(data(:,rline)) - hold on - plot(mean_power) - plot(detrend_curve); - keyboard - end - data = bsxfun(@minus,data,detrend_curve); - if track.data_noise_en - data_noise = bsxfun(@minus,data_noise,detrend_curve); - end - end - - %% Track: Sidelobe - if ~isempty(track.sidelobe_rows) - mask = sidelobe_mask_mex(single(data),int32(track.sidelobe_rows),single(track.sidelobe_dB)); - data(mask) = NaN; - end - - %% Track: Filter - if track.filter(1) ~= 1 - % Multilooking in cross-track/fast-time - data = lp(nan_fir_dec(10.^(data/10).',ones(1,track.filter(1))/track.filter(1),1,[],[],[],[],2.0).'); - if track.data_noise_en - data_noise = lp(nan_fir_dec(10.^(data_noise/10).',ones(1,track.filter(1))/track.filter(1),1,[],[],[],[],2.0).'); - end - end - if track.filter(2) ~= 1 - % Apply motion compensation - if track.filter_mocomp - dt = mdata.Time(2)-mdata.Time(1); - dbin = round((mdata.Elevation - median(mdata.Elevation)) / (c/2) / dt); - for rline = 1:size(data,2) - data(:,rline) = circshift(data(:,rline),[-dbin(rline) 0]); - end - end - % Multilooking in along-track - data = lp(nan_fir_dec(10.^(data/10),ones(1,track.filter(2))/track.filter(2),1,[],[],[],[],2.0)); - if track.data_noise_en - data_noise = lp(nan_fir_dec(10.^(data_noise/10),ones(1,track.filter(2))/track.filter(2),1,[],[],[],[],2.0)); - end - % Remove motion compensation - if track.filter_mocomp - for rline = 1:size(data,2) - data(:,rline) = circshift(data(:,rline),[dbin(rline) 0]); - end - end - end - - %% Track: Post-filter trim - for rline = 1:Nx - start_bin = find(isfinite(data(:,rline)),1); - if ~isempty(start_bin) - stop_bin = min(size(data,1), start_bin+track.filter_trim(1)-1); - data(start_bin:stop_bin,rline) = NaN; - if track.data_noise_en - data_noise(start_bin:stop_bin,rline) = NaN; - end - end - stop_bin = find(isfinite(data(:,rline)),1,'last'); - if ~isempty(stop_bin) - start_bin = max(1, stop_bin-track.filter_trim(2)+1); - data(start_bin:stop_bin,rline) = NaN; - if track.data_noise_en - data_noise(start_bin:stop_bin,rline) = NaN; - end - end - end - - %% Track: min_bin/max_bin + time to bins conversions - % Convert from two way travel time to bins - track.min_bin = find(mdata.Time >= orig_track.min_bin, 1); - track.max_bin = find(mdata.Time <= orig_track.max_bin, 1, 'last'); - dt = mdata.Time(2) - mdata.Time(1); - track.init.max_diff = orig_track.init.max_diff/dt; - if strcmpi(track.max_rng_units,'bins') - track.max_rng = orig_track.max_rng(1) : orig_track.max_rng(end); - else - track.max_rng = round(orig_track.max_rng(1)/dt) : round(orig_track.max_rng(end)/dt); - end - data = data(track.min_bin:track.max_bin,:); - if track.data_noise_en - data_noise = data_noise(track.min_bin:track.max_bin,:); - end - - %% Track: Create Initial Surface - if strcmpi(track.init.method,'dem') - % Correct for min_bin removal - track.dem = track.dem - track.min_bin + 1; - elseif strcmp(track.init.method,'snake') - track.init.search_rng = round(orig_track.init.snake_rng(1)/dt) : round(orig_track.init.snake_rng(2)/dt); - track.dem = tracker_snake_simple(data,track.init); - elseif strcmp(track.init.method,'nan') - track.dem = nan(1,Nx); - else - % max or medfilt - [~,track.dem] = max(data,[],1); - if strcmp(track.init.method,'medfilt') - track.dem = medfilt1(track.dem,track.init.medfilt); - end - end - - %% Track: Merge DEM and reference layer - if ~isempty(track.init.dem_layer) - % Interpolate reference layer onto radar GPS time - ref_interp_gaps_dist = [150 75]; - ops_layer = []; - ops_layer{1}.gps_time = layers(lay_idx).gps_time; - ops_layer{1}.type = layers(lay_idx).type; - ops_layer{1}.quality = layers(lay_idx).quality; - ops_layer{1}.twtt = layers(lay_idx).twtt; - ops_layer{1}.type(isnan(ops_layer{1}.type)) = 2; - ops_layer{1}.quality(isnan(ops_layer{1}.quality)) = 1; - lay = opsInterpLayersToMasterGPSTime(mdata,ops_layer,ref_interp_gaps_dist); - dem_layer = lay.layerData{1}.value{2}.data; - % Convert from twtt to bins - dem_layer = interp1(mdata.Time,1:length(mdata.Time),dem_layer + track.init.dem_layer_offset); - % Correct for min_bin removal - dem_layer = dem_layer - track.min_bin + 1; - % Merge ref layer over track.dem - track.dem = merge_vectors(dem_layer, track.dem); - end - - %% Track: Tracking - if strcmpi(track.method,'threshold') - track.threshold_noise_rng = round(orig_track.threshold_noise_rng/dt); - if track.data_noise_en - track.data_noise = data_noise; - else - track.data_noise = data; - end - [new_layer,new_quality] = tracker_threshold(data,track); - elseif strcmpi(track.method,'viterbi') - new_layer = tracker_viterbi(data,track); - new_quality = ones(1,Nx); - elseif strcmpi(track.method,'max') - new_layer = tracker_max(data,track); - new_quality = ones(1,Nx); - elseif strcmpi(track.method,'snake') - track.search_rng = round(orig_track.snake_rng(1)/dt) : round(orig_track.snake_rng(2)/dt); - new_layer = tracker_snake_simple(data,track); - new_quality = ones(1,Nx); - elseif strcmpi(track.method,'fixed') - new_layer = ones(size(mdata.GPS_time)) ... - * interp1(mdata.Time, 1:length(mdata.Time), track.fixed_value,'linear','extrap'); - new_quality = ones(1,Nx); - elseif isempty(track.method) - new_layer = track.dem; - new_quality = ones(1,Nx); - else - error('Not a supported layer tracking method.'); - end - - %% Track: max_diff - new_layer(abs(new_layer - track.dem) > track.init.max_diff) = NaN; - switch (track.init.max_diff_method) - case 'merge_vectors' - new_layer = merge_vectors(new_layer, track.dem); - case 'interp_finite' - new_layer = interp_finite(new_layer,NaN); - end - - %% Track: Median filtering - if isfield(track,'medfilt') && ~isempty(track.medfilt) - % This median filter is designed to only operate when the point in - % question is an outlier exceeding track.medfilt_threshold - for rline=1:Nx - rlines = rline + (-track.medfilt:track.medfilt); - rlines = rlines(rlines>=1 & rlines<=length(new_layer)); - if abs(new_layer(rline) - nanmedian(new_layer(rlines))) > track.medfilt_threshold - new_layer(rline) = nanmedian(new_layer(rlines)); - end - end - end - - %% Track: Max search - if (length(track.max_rng) > 1 || track.max_rng ~= 0) - % Find the next peak after the threshold - for rline = 1:Nx - search_bins = round(new_layer(rline)) + track.max_rng; - search_bins = search_bins(find(search_bins >= 1 & search_bins <= size(data,1))); - [~,offset] = max(data(search_bins,rline)); - if ~isempty(offset) - new_layer(rline) = search_bins(offset); - end - end - end - - %% Track: Convert bins to twtt - Surface = interp1(1:length(mdata.Time), mdata.Time, new_layer + track.min_bin - 1,'linear','extrap'); - - end - - % Some layer sources may not be "double", but we require that Surface be - % double: - Surface = double(Surface); - - %% Track: Debug plot - if enable_debug_plot - clf(h_fig(1)); - set(h_fig(1),'name',sprintf('layer_tracker %s',data_fn_name)); - h_axes(1) = axes('parent',h_fig(1)); - imagesc([],mdata.Time,lp(mdata.Data), 'parent', h_axes(1)); - colormap(h_axes(1), 1-gray(256)); - hold(h_axes(1),'on'); - plot(h_axes(1),find(new_quality==1),Surface(new_quality==1),'g.'); - plot(h_axes(1),find(new_quality==3),Surface(new_quality==3),'r.'); - if strcmpi(track.init.method,{'dem'}) || ~isempty(track.init.dem_layer) - plot(h_axes(1),interp1(1:length(mdata.Time),mdata.Time,track.dem+track.min_bin-1),'m--') - plot(h_axes(1),interp1(1:length(mdata.Time),mdata.Time, ... - track.dem+track.min_bin-1-track.init.max_diff),'r--') - plot(h_axes(1),interp1(1:length(mdata.Time),mdata.Time, ... - track.dem+track.min_bin-1+track.init.max_diff),'b--') - end - hold(h_axes(1),'off'); - if ~isempty(mdata.Time) - ylims = [max(mdata.Time(1),min(Surface)-debug_time_guard) min(mdata.Time(end),max(Surface)+debug_time_guard)]; - if ylims(end)>ylims(1) - ylim(h_axes(1),ylims); - end - end - title(h_axes(1),sprintf('%s',regexprep(data_fn_name,'_','\\_'))); - keyboard - end - - %% Track: Save - if nargout == 1 - return; - end - for layer_idx = 1:length(layer_params) - layer_param = layer_params(layer_idx); - - if strcmpi(layer_param.source,'echogram') - if isempty(layer_param.echogram_source) - layer_param.echogram_source = echogram_source; - end - - data_fn = fullfile(ct_filename_out(param,layer_param.echogram_source,''), ... - sprintf('Data_%s_%03d.mat', param.day_seg, frm)); - fprintf(' Saving %s (%s)\n', data_fn, datestr(now)); - save(data_fn,'-append','Surface'); - end - - if strcmpi(layer_param.source,'layerdata') - if ~exist('layers','var') || isempty(layers) - layers = layerdata(param,layer_param.layerdata_source); - id = layers.get_id(layer_param.name); - if isempty(id) - layer_organizer = []; - layer_organizer.lyr_name = {layer_param.name}; - if isfield(layer_param,'group_name') - layer_organizer.lyr_group_name = {layer_param.group_name}; - end - layers.insert_layers(layer_organizer); - end - layers.update_layer(frm,layer_param.name,mdata.GPS_time,Surface,new_quality,2*ones(size(mdata.GPS_time))); - fprintf(' Saving %s (%s)\n', layers.layer_fn(frm), datestr(now)); - layers.save(); - end - end - - if strcmpi(layer_param.source,'ops') - % Get all the frames for this segment - if any(strcmpi({layer_params.source},'ops')) - opsAuthenticate(param,false); - sys = ct_output_dir(param.radar_name); - ops_param = struct('properties',[]); - ops_param.properties.season = param.season_name; - ops_param.properties.segment = param.day_seg; - [status,ops_seg_data] = opsGetSegmentInfo(sys,ops_param); - end - - % OPS query to get the point path ID's - ops_param = struct('properties',[]); - ops_param.properties.location = param.post.ops.location; - ops_param.properties.season = param.season_name; - ops_param.properties.start_gps_time = ops_seg_data.properties.start_gps_time(frm); - ops_param.properties.stop_gps_time = ops_seg_data.properties.stop_gps_time(frm); - - sys = ct_output_dir(param.radar_name); - [status,data] = opsGetPath(sys,ops_param); - - % Write the new layer information to these point path ID's - ops_param = struct('properties',[]); - ops_param.properties.point_path_id = data.properties.id; - ops_param.properties.twtt = interp_finite(interp1(mdata.GPS_time,Surface,data.properties.gps_time)); - ops_param.properties.type = 2*ones(size(ops_param.properties.twtt)); - ops_param.properties.quality = interp1(mdata.GPS_time,new_quality,data.properties.gps_time,'nearest'); - ops_param.properties.lyr_name = layer_param.name; - - opsCreateLayerPoints(sys,ops_param); - end - - end - -end diff --git a/cresis-toolbox/ct_support/layer_tracker_profile.m b/cresis-toolbox/ct_support/layer_tracker_profile.m deleted file mode 100644 index bdca81de..00000000 --- a/cresis-toolbox/ct_support/layer_tracker_profile.m +++ /dev/null @@ -1,129 +0,0 @@ -function track = layer_tracker_profile(param,profile_str) -% track = layer_tracker_profile(param,profile_str) -% -% Returns a default set of tracking parameters for layer tracking with -% layer_tracker.m. -% -% param: parameter spreadsheet structure -% profile_str: optional string containing the profile name to load (default -% is "default" -% -% track: Parameters used to control the tracker. See layer_tracker.m. -% -% Example: -% Should only be used from layer_tracker.m -% -% Author: John Paden - -%% Check input arguments -if ~exist('profile_str','var') || isempty(profile_str) - profile_str = 'default'; -end -track.profile = profile_str; - -%% Default track settings -track.data_noise_en = false; -track.detrend = 0; -track.en = true; -track.init = []; -track.init.method = 'max'; -track.init.snake_rng = [-2e-7 2e-7]; -track.init.dem_layer = ''; -track.init.dem_offset = 0; -track.init.dem_layer_offset = 0; -track.init.max_diff = inf; -track.init.max_diff_method = 'interp_finite'; -track.filter_mocomp = true; -track.filter = [1 1]; -track.filter_trim = [0 0]; -track.fixed_value = 0; -track.min_bin = 0; -track.max_bin = inf; -track.max_rng = [0 0]; -track.max_rng_units = 'time'; -track.medfilt = 0; -track.medfilt_threshold = 0; -track.method = 'threshold'; -track.prefilter_trim = [0 0]; -track.sidelobe_dB = []; -track.sidelobe_rows = []; -track.snake_rng = [-2e-7 2e-7]; -track.threshold = 15; -track.threshold_noise_rng = [0 -inf -1]; - -if strcmpi(profile_str,'default') - %% Default profile - -elseif strcmpi(profile_str,'RDS_OIB') - %% RDS_OIB profile - track.debug_time_guard = 2e-6; - track.filter = [1 5]; - track.init.method = 'medfilt'; - track.init.medfilt = 11; - track.init.max_diff = 0.5e-6; - track.max_rng = [0 1]; - track.max_rng_units = 'bins'; - track.medfilt = 11; - track.medfilt_threshold = 30; - track.method = 'threshold'; - track.min_bin = 1.6e-6; - track.threshold = 10; - track.threshold_noise_rng = [0 -2e-6 -0.2e-6]; - track.threshold_rel_max = -9; - track.threshold_rng = 5; - -elseif strcmpi(profile_str,'ACCUM') - %% ACCUM profile - track.debug_time_guard = 2e-6; - track.filter = [3 7];%[3 3]; - track.filter_trim = [0 3];%[3 3]; - track.init.method = 'medfilt'; - track.init.medfilt = 11; - track.init.max_diff = 1e-6;%0.5e-6; - track.max_rng = [0 1];%[0 2]; - track.max_rng_units = 'bins'; - track.medfilt = 3;%11; - track.medfilt_threshold = 10;%30; - track.method = 'threshold'; - track.min_bin = 0;%1.6e-6; - track.threshold = 5;%10; - track.threshold_noise_rng = [0 -1e-6 -0.3e-6]; - track.threshold_rel_max = -9; - track.threshold_rng = 5; - -elseif strcmpi(profile_str,'SNOW_AWI') - %% SNOW_AWI profile - track.debug_time_guard = 50e-9; - track.min_bin = 0.1e-6; - track.prefilter_trim = [0 0]; - track.filter = [5 3]; - track.filter_trim = [10 10]; - track.init.method = 'medfilt'; - track.init.medfilt = 51; - track.init.max_diff = 0.3e-6; - track.max_rng = [0 9]; - track.max_rng_units = 'bins'; - track.medfilt = 11; - track.medfilt_threshold = 100; - track.method = 'threshold'; - track.threshold = 8; - track.threshold_noise_rng = [15e-9 -75e-9 -30e-9]; - track.threshold_rel_max = -9; - -elseif strcmpi(profile_str,'DEM_LIDAR') - %% DEM profile - track.debug_time_guard = 100e-9; - track.min_bin = 0e-6; - track.init.method = 'dem'; - track.init.dem_layer.name = 'surface'; - track.init.dem_layer.source = 'lidar'; - track.init.dem_layer.lidar_source = 'atm'; - track.init.dem_layer.lever_arm_en = true; - track.init.max_diff_method = 'merge_vectors'; - track.init.max_diff = 0e-6; % Force output layer to equal DEM - track.max_rng = [0 0]; - track.method = ''; % No data dependent tracking - -else - error('Invalid profile selected: %s\n', profile_str); -end diff --git a/cresis-toolbox/ct_support/layer_update.m b/cresis-toolbox/ct_support/layer_update.m index 82269f2c..d40b4e6c 100644 --- a/cresis-toolbox/ct_support/layer_update.m +++ b/cresis-toolbox/ct_support/layer_update.m @@ -21,18 +21,17 @@ %% Input checks +% param.layer_update.in_path: input path string, default is 'layer' if ~isfield(param.layer_update,'in_path') || isempty(param.layer_update.in_path) param.layer_update.in_path = 'layer'; end +% param.layer_update.out_path: output path string, default is for out_path to +% equal in_path if ~isfield(param.layer_update,'out_path') || isempty(param.layer_update.out_path) param.layer_update.out_path = param.layer_update.in_path; end -if ~isfield(param.layer_update,'frames_records_en') || isempty(param.layer_update.frames_records_en) - param.layer_update.frames_records_en = true; -end - %% Update layers layers = layerdata(param,param.layer_update.in_path); diff --git a/cresis-toolbox/ct_support/load_sar_data.m b/cresis-toolbox/ct_support/load_sar_data.m deleted file mode 100644 index 9418b2f7..00000000 --- a/cresis-toolbox/ct_support/load_sar_data.m +++ /dev/null @@ -1,301 +0,0 @@ -function [data,metadata] = load_sar_data(param) -% [data,metadata] = load_sar_data(param) -% -% Loads and concatenates sar data from a single frame. Currently requires -% all data to be in the same directory. Returns data, position, and header -% information associated with the sar data. -% -% param.load_sar_data: parameter structure controlling how data are loaded. -% See input arguments section for details -% -% Author: John Paden, Logan Smith -% -% See also: run_master.m, master.m, run_array.m, array.m, load_sar_data.m, -% array_proc.m, array_task.m, array_combine_task.m -% -% Also used in: run_load_sar_data.m - -%% Check input arguments -% ========================================================================= - -%% Input: sar param -% subap: list of subapertures to load (array of integers, default is 1) -if ~isfield(param.sar,'sub_aperture_steering') || isempty(param.sar.sub_aperture_steering) - % Single aperture which points broadside to SAR is default - param.sar.sub_aperture_steering = [0]; -end - -%% Input: load_sar_data param -% combine_channels: sum wf-adc pairs together in each image -if ~isfield(param.load_sar_data,'combine_channels') || isempty(param.load_sar_data.combine_channels) - param.load_sar_data.combine_channels = false; -end - -% combine_imgs: combine images together -if ~isfield(param.load_sar_data,'combine_imgs') || isempty(param.load_sar_data.combine_imgs) - param.load_sar_data.combine_imgs = false; -end - -% chunk: two element vector specifying the start and stop chunk to load -% (minimum value is 1 and maximum value is the number of chunks in the -% frame). If the stop chunk is specified, then the stop chunk is set to the -% number of chunks. The default is to load all chunks which is [1 inf]. -if ~isfield(param.load_sar_data,'chunk') || isempty(param.load_sar_data.chunk) - param.load_sar_data.chunk = [1 inf]; -end - -% debug_level -if ~isfield(param.load_sar_data,'debug_level') || isempty(param.load_sar_data.debug_level) - param.load_sar_data.debug_level = 1; -end - -% detrend: structure controlling detrending. Arguments are passed into -% local_detrend.m. Detrend is disabled by default (detrend.cmd == 0). -% Detrending only runs if combine_channels == true and incoherent == true. -if ~isfield(param.load_sar_data,'detrend') || isempty(param.load_sar_data.detrend) - param.load_sar_data.detrend.cmd = []; -end -if ~isfield(param.load_sar_data.detrend,'B_noise') || isempty(param.load_sar_data.detrend.B_noise) - param.load_sar_data.detrend.B_noise = [100 200]; -end -if ~isfield(param.load_sar_data.detrend,'B_sig') || isempty(param.load_sar_data.detrend.B_sig) - param.load_sar_data.detrend.B_sig = [10 20]; -end -if ~isfield(param.load_sar_data.detrend,'cmd') || isempty(param.load_sar_data.detrend.cmd) - param.load_sar_data.detrend.cmd = 0; -end -if ~isfield(param.load_sar_data.detrend,'minVal') || isempty(param.load_sar_data.detrend.minVal) - param.load_sar_data.detrend.minVal = -inf; -end - -% fn: Data directory. Default is "sar" which loads from "CSARP_sar". -if ~isfield(param.load_sar_data,'fn') || isempty(param.load_sar_data.fn) - param.load_sar_data.fn = 'sar'; -end - -% frm: Data frame to load -if ~isfield(param.load_sar_data,'frm') || isempty(param.load_sar_data.frm) - warning('The param.load_sar_data.frm field must be set to an integer indicating which frame to load. Defaulting to param.load_sar_data.frm = 1.'); - param.load_sar_data.frm = 1; -end - -% imgs: cell array of images to load. Each cell array contains a 2 by N -% waveform-adc pair list where N is the number of wf-adc pairs to load. -% Default is {[1 1]} which loads one image and this image is pulled from -% waveform = 1, adc = 1. -if ~isfield(param.load_sar_data,'imgs') || isempty(param.load_sar_data.imgs) - param.load_sar_data.imgs = {[1 1]}; -end - -% incoherent: logical scalar, default is false, if true, the data will be -% power detected on load with abs()^2. Incoherent only runs if -% combine_channels == true. -if ~isfield(param.load_sar_data,'incoherent') || isempty(param.load_sar_data.incoherent) - param.load_sar_data.incoherent = false; -end - -% sar_type: 'fk' or 'tdbp'. Default is 'fk'. -if ~isfield(param.load_sar_data,'sar_type') || isempty(param.load_sar_data.sar_type) - param.load_sar_data.sar_type = 'fk'; -end - -% subap: list of subapertures to load (array of integers, default is all -% subapertures specified in param.sar.sub_aperture_steering) -if ~isfield(param.load_sar_data,'subap') || isempty(param.load_sar_data.subap) - param.load_sar_data.subap = 1:length(param.sar.sub_aperture_steering); -end - -physical_constants; - -%% Load subapertures -% ========================================================================= - -% The base path for all the data -base_path = ct_filename_out(param,param.load_sar_data.fn,''); - -% Initialize memory for outputs -data = cell(length(param.load_sar_data.imgs),1); -metadata = []; - -for subap_idx = 1:length(param.load_sar_data.subap) - subap = param.load_sar_data.subap(subap_idx); - - in_path = fullfile(base_path, ... - sprintf('%s_data_%03d_%02d_01/',param.load_sar_data.sar_type,param.load_sar_data.frm, subap)); - - %% Determine which chunks are available for this subaperture - img = 1; - wf_adc_list = param.load_sar_data.imgs{img}; - wf_adc = 1; - wf = wf_adc_list(wf_adc,1); - adc = wf_adc_list(wf_adc,2); - fns = get_filenames(in_path,sprintf('wf_%02.0f_adc_%02.0f_chk_',wf,adc),'','.mat'); - valid_chks = []; - for fns_idx = 1:length(fns) - [~,fn_name] = fileparts(fns{fns_idx}); - valid_chks(end+1) = str2double(fn_name(end-2:end)); - end - param.load_sar_data.chunk(param.load_sar_data.chunk==inf) = max(valid_chks); - chks_to_load = param.load_sar_data.chunk(1):param.load_sar_data.chunk(end); - - % Remove chunks that do not exist from chunks_to_load list - [valid_chks,keep_idxs] = intersect(chks_to_load, valid_chks); - if length(valid_chks) ~= length(chks_to_load) - bad_mask = ones(size(chks_to_load)); - bad_mask(keep_idxs) = 0; - warning('Nonexistent chunks specified in chks_to_load (e.g. chunk "%g" is invalid), removing these', ... - chks_to_load(find(bad_mask,1))); - chks_to_load = valid_chks; - end - - %% Subapertures: Load chunks (blocks) of SAR data - cur_rline = 0; - for chunk = chks_to_load - - if param.load_sar_data.debug_level >= 1 - fprintf('Loading chunk %d (%s)\n', chunk, datestr(now,'HH:MM:SS')); - end - - % ===================================================================== - % Load data - - tx = 1; - for img = 1:length(param.load_sar_data.imgs) - wf_adc_list = param.load_sar_data.imgs{img}; - - % ------------------------------------------------------------------- - % Load data - for wf_adc = 1:size(wf_adc_list,1) - wf = wf_adc_list(wf_adc,1); - adc = wf_adc_list(wf_adc,2); - - sar_fn = fullfile(in_path,sprintf('wf_%02.0f_adc_%02.0f_chk_%03.0f.mat',wf,adc,chunk)); - if param.load_sar_data.debug_level >= 2 - fprintf(' %s (%s)\n', sar_fn, datestr(now,'HH:MM:SS')); - end - sar_data = load(sar_fn); - - % Only add in non-overlapping part of SAR image (this is to support - % legacy SAR data format since current format does not have - % overlap) - if img == 1 && wf_adc == 1 - if chunk == param.load_sar_data.chunk(1) - new_idxs = 1:length(sar_data.fcs.gps_time); - else - % 1e-3 added to avoid rounding errors... this is a hack - new_idxs = find(sar_data.fcs.gps_time > fcs{img}{wf_adc}.gps_time(end)+1e-3); - end - end - - % Get output image positions (not phase centers) - if img == 1 && wf_adc == 1 && subap_idx == 1 - if chunk == param.load_sar_data.chunk(1) - metadata.lat = sar_data.lat; - metadata.lon = sar_data.lon; - metadata.elev = sar_data.elev; - else - metadata.lat = [metadata.lat sar_data.lat(new_idxs)]; - metadata.lon = [metadata.lon sar_data.lon(new_idxs)]; - metadata.elev = [metadata.elev sar_data.elev(new_idxs)]; - end - end - - if subap_idx == 1 - if chunk == param.load_sar_data.chunk(1) - fcs{img}{wf_adc} = sar_data.fcs; - - else - fcs{img}{wf_adc}.gps_time = cat(2,fcs{img}{wf_adc}.gps_time, ... - sar_data.fcs.gps_time(new_idxs)); - fcs{img}{wf_adc}.x = cat(2,fcs{img}{wf_adc}.x, ... - sar_data.fcs.x(:,new_idxs)); - fcs{img}{wf_adc}.y = cat(2,fcs{img}{wf_adc}.y, ... - sar_data.fcs.y(:,new_idxs)); - fcs{img}{wf_adc}.z = cat(2,fcs{img}{wf_adc}.z, ... - sar_data.fcs.z(:,new_idxs)); - fcs{img}{wf_adc}.origin = cat(2,fcs{img}{wf_adc}.origin, ... - sar_data.fcs.origin(:,new_idxs)); - fcs{img}{wf_adc}.pos = cat(2,fcs{img}{wf_adc}.pos, ... - sar_data.fcs.pos(:,new_idxs)); - fcs{img}{wf_adc}.roll = cat(2,fcs{img}{wf_adc}.roll, ... - sar_data.fcs.roll(new_idxs)); - fcs{img}{wf_adc}.pitch = cat(2,fcs{img}{wf_adc}.pitch, ... - sar_data.fcs.pitch(new_idxs)); - fcs{img}{wf_adc}.heading = cat(2,fcs{img}{wf_adc}.heading, ... - sar_data.fcs.heading(new_idxs)); - fcs{img}{wf_adc}.surface = cat(2,fcs{img}{wf_adc}.surface, ... - sar_data.fcs.surface(new_idxs)); - fcs{img}{wf_adc}.bottom = cat(2,fcs{img}{wf_adc}.bottom, ... - sar_data.fcs.bottom(new_idxs)); - end - end - - Nt = size(sar_data.fk_data,1); - if ~param.load_sar_data.combine_channels - if subap_idx == 1 && wf_adc == 1 - % Allocate memory in a special way when loading 3D data - data{img}(size(sar_data.fk_data,1), ... - size(data{img},2)+length(new_idxs),size(wf_adc_list,1),length(param.load_sar_data.subap)) = single(0); - end - data{img}(:,cur_rline + (1:length(new_idxs)),wf_adc,subap_idx) = sar_data.fk_data(:,new_idxs); - else - % When combining channels, take the mean of the data as it - % is loaded in to reduce peak memory consumption. - if wf_adc == 1 - sar_data_data = sar_data.fk_data(:,new_idxs) / size(wf_adc_list,1); - else - sar_data_data = sar_data_data + sar_data.fk_data(:,new_idxs) / size(wf_adc_list,1); - end - end - - end - - if param.load_sar_data.combine_channels - if param.load_sar_data.incoherent - data{img} = [data{img} abs(sar_data_data).^2]; - else - data{img} = [data{img} sar_data_data]; - end - end - end - - cur_rline = cur_rline + length(new_idxs); - end -end - -% Create lat/lon fields in FCS for convenience -for img = 1:length(fcs) - for wf_adc = 1:length(fcs{img}) - [fcs{img}{wf_adc}.lat,fcs{img}{wf_adc}.lon,fcs{img}{wf_adc}.elev] ... - = ecef2geodetic(fcs{img}{wf_adc}.origin(1,:) + sum(fcs{img}{wf_adc}.x.*fcs{img}{wf_adc}.pos), ... - fcs{img}{wf_adc}.origin(2,:) + sum(fcs{img}{wf_adc}.y.*fcs{img}{wf_adc}.pos), ... - fcs{img}{wf_adc}.origin(3,:) + sum(fcs{img}{wf_adc}.z.*fcs{img}{wf_adc}.pos), ... - WGS84.ellipsoid); - fcs{img}{wf_adc}.lat = fcs{img}{wf_adc}.lat * 180/pi; - fcs{img}{wf_adc}.lon = fcs{img}{wf_adc}.lon * 180/pi; - end -end - -metadata.fcs = fcs; -metadata.wfs = sar_data.wfs; -metadata.param_records = sar_data.param_records; -metadata.param_sar = sar_data.param_sar; - -if param.load_sar_data.combine_channels && param.load_sar_data.combine_imgs - -end - -if param.load_sar_data.combine_channels && param.load_sar_data.incoherent ... - && param.load_sar_data.detrend.cmd - % Detrend data - detrend = param.load_sar_data.detrend; - if param.load_sar_data.combine_imgs - data = local_detrend(data, detrend.B_noise, ... - detrend.B_sig, detrend.cmd, detrend.minVal); - else - for img = 1:length(param.imgs) - data{img} = local_detrend(data{img}, detrend.B_noise, ... - detrend.B_sig, detrend.cmd, detrend.minVal); - end - end -end diff --git a/cresis-toolbox/ct_support/preprocess.m b/cresis-toolbox/ct_support/preprocess.m index 86cdf0e2..8cbd6796 100644 --- a/cresis-toolbox/ct_support/preprocess.m +++ b/cresis-toolbox/ct_support/preprocess.m @@ -36,144 +36,129 @@ sparam.num_args_out = 1; for config_idx = 1:numel(param.config.default) - cparam = merge_structs(param,param.config.default{config_idx}); - cparam.config.base_dir = param.config.base_dir{config_idx}; - cparam.config.config_folder_names = param.config.config_folder_names{config_idx}; - cparam.config.board_folder_names = param.config.board_folder_names{config_idx}; - cparam.config.date_str = param.config.date_strs{config_idx}; + cparam = param.config.default{config_idx}(); + cparam.config.default = param.config.default{config_idx}; + if isfield(param.config,'base_dir') + cparam.config.base_dir = param.config.base_dir{config_idx}; + cparam.config.config_folder_names = param.config.config_folder_names{config_idx}; + cparam.config.board_folder_names = param.config.board_folder_names{config_idx}; + cparam.config.date_str = param.config.date_str{config_idx}; + else + cparam.config.base_dir = ''; + cparam.config.config_folder_names = ''; + cparam.config.board_folder_names = '%b'; + cparam.config.date_str = datestr(now,'yyyymmdd'); + end if isfield(param.config,'regexp') cparam.config.file.regexp = param.config.regexp{config_idx}; end - cparam.config = rmfield(cparam.config,'date_strs'); - cparam.config = rmfield(cparam.config,'default'); + cparam = merge_structs(cparam, param_override); %% Input checks % ========================================================================= - if strcmpi(cparam.config.daq_type,'cresis') - % CReSIS DAQ only parameters - if ~isfield(cparam.config.cresis,'gps_file_mask') || isempty(cparam.config.cresis.gps_file_mask) - % File mask relative to param.config.config_folder_name for GPS files - % Leave empty if there are no GPS files to copy. - cparam.config.cresis.gps_file_mask = ''; - end + % .config.cpu_time_per_file: Positive double with units of seconds. + % Default is 10 seconds. Cluster cpu_time allotment per raw data file + if ~isfield(cparam.config,'cpu_time_per_file') || isempty(cparam.config.cpu_time_per_file) + cparam.config.cpu_time_per_file = 10; end - - if ~isfield(cparam.config,'date_strs') || isempty(cparam.config.date_strs) + if ~isfield(cparam.config,'date_str') || isempty(cparam.config.date_str) % Segment ID is created from the date of the config folder unless this % cell array is set. - cparam.config.date_strs = {}; + cparam.config.date_str = {}; end if ~isfield(cparam.config,'file') || isempty(cparam.config.file) cparam.config.file = []; end + % .config.file.prefix: get_filenames prefix argument. Leave empty to + % not use. Default is ''. if ~isfield(cparam.config.file,'prefix') || isempty(cparam.config.file.prefix) cparam.config.file.prefix = ''; end + % .config.file.midfix: get_filenames midfix argument. Leave empty to + % not use. Default is ''. if ~isfield(cparam.config.file,'midfix') || isempty(cparam.config.file.midfix) cparam.config.file.midfix = ''; end + % .config.file.suffix: get_filenames suffix argument. Leave empty to + % not use. Default is ''. if ~isfield(cparam.config.file,'suffix') || isempty(cparam.config.file.suffix) cparam.config.file.suffix = ''; end + % .config.file.regexp: get_filenames regular expression. Leave empty to + % not use. Default is ''. if ~isfield(cparam.config.file,'regexp') || isempty(cparam.config.file.regexp) cparam.config.file.regexp = ''; end - if ~isfield(cparam.config,'mat_or_bin_hdr_output') || isempty(cparam.config.mat_or_bin_hdr_output) - cparam.config.mat_or_bin_hdr_output = '.mat'; + % .config.gps_file_mask: File mask relative to + % param.config.config_folder_name for GPS files. Leave empty if there are + % no GPS files to copy. Default is an empty string. + if ~isfield(cparam.config,'gps_file_mask') || isempty(cparam.config.gps_file_mask) + cparam.config.gps_file_mask = ''; end if ~isfield(cparam.config,'max_time_gap') || isempty(cparam.config.max_time_gap) % Maximum time in seconds between two data records before forcing a segment break cparam.config.max_time_gap = 10; end + % .config.min_seg_size: Scalar integer. Default is 2. Minimum number of files in + % a segment (segments with less files will be discarded) if ~isfield(cparam.config,'min_seg_size') || isempty(cparam.config.min_seg_size) - % Minimum number of files in a segment (segments with less files will be - % discarded) cparam.config.min_seg_size = 2; end + % .config.online_mode: Scaler integer. Default is 0. Allowed values: + % * 0: not online mode + % * 1: online mode (process all files) + % * 2: online mode (process only the most recent file) if ~isfield(cparam.config,'online_mode') || isempty(cparam.config.online_mode) cparam.config.online_mode = 0; - % 0: not online mode - % 1: online mode (process all files) - % 2: online mode (process only the most recent file) end + % .config.param_fn: String containing the parameter spreadsheet filename. + % Default is RADARNAME_param_SEASONNAME.xls. if ~isfield(cparam.config,'param_fn') || isempty(cparam.config.param_fn) cparam.config.param_fn ... = ct_filename_param(sprintf('%s_param_%s.xls',ct_output_dir(cparam.radar_name),cparam.season_name)); end + % .config.reuse_tmp_files: Logical scaler. Default is true. If true, the + % function will use any existing header files in ct_tmp and not recreate + % them. if ~isfield(cparam.config,'reuse_tmp_files') || isempty(cparam.config.reuse_tmp_files) cparam.config.reuse_tmp_files = true; end + % .config.skip_files_end: Scaler integer. Default is 0. Number of files + % to ignore at the end of the segment if ~isfield(cparam.config,'skip_files_end') || isempty(cparam.config.skip_files_end) - % Number of files to ignore at the end of the segment cparam.config.skip_files_end = 0; end + % .config.skip_files_start: Scaler integer. Default is 0. Number of files + % to ignore at the beginning of the segment. if ~isfield(cparam.config,'skip_files_start') || isempty(cparam.config.skip_files_start) - % Number of files to ignore at the beginning of the segment cparam.config.skip_files_start = 0; end + % .config.tmp_load_mode: Scaler logical. Default is false. If true, then + % preprocess runs just for quick temporary loading of files. + if ~isfield(cparam.config,'tmp_load_mode') || isempty(cparam.config.tmp_load_mode) + cparam.config.tmp_load_mode = false; + end %% Setup preprocess task dparam = []; - dparam.argsin{1}.season_name = cparam.season_name; - dparam.argsin{1}.radar_name = cparam.radar_name; - dparam.argsin{1}.config = cparam.config; - dparam.argsin{1}.config.config_folder_name = cparam.config.config_folder_names; - dparam.argsin{1}.config.board_folder_name = cparam.config.board_folder_names; - dparam.argsin{1}.config.date_str = cparam.config.date_str; + dparam.argsin{1} = cparam; - if strcmpi(cparam.config.daq_type,'arena') - fns = get_filenames(fullfile(cparam.config.base_dir,cparam.config.config_folder_names),'','','.dat', ... - struct('recursive',true,'regexp','[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]')); - num_fns = length(fns); - - elseif strcmpi(cparam.config.daq_type,'cresis') - num_fns = 0; - for board_idx = 1:length(cparam.config.board_map) - board = cparam.config.board_map{board_idx}; - board_folder_name = cparam.config.board_folder_names; - board_folder_name = regexprep(board_folder_name,'%b',board); - - get_filenames_param = struct('regexp',cparam.config.file.regexp); - fns = get_filenames(fullfile(cparam.config.base_dir,board_folder_name), ... - cparam.config.file.prefix, cparam.config.file.midfix, ... - cparam.config.file.suffix, get_filenames_param); - num_fns = num_fns + length(fns); - end - - elseif strcmpi(cparam.config.daq_type,'utua') - num_fns = 0; - for board_idx = 1:length(cparam.config.board_map) - board = cparam.config.board_map{board_idx}; - board_folder_name = cparam.config.board_folder_names; - board_folder_name = regexprep(board_folder_name,'%b',board); - - get_filenames_param = struct('regexp',cparam.config.file.regexp); - fns = get_filenames(fullfile(cparam.config.base_dir,board_folder_name), ... - cparam.config.file.prefix, cparam.config.file.midfix, ... - cparam.config.file.suffix, get_filenames_param); - num_fns = num_fns + length(fns); - end - - elseif strcmpi(cparam.config.daq_type,'bas') - num_fns = 0; - board_idx = 1; - board = cparam.config.board_map{board_idx}; + num_fns = 0; + for board_idx = 1:length(cparam.records.file.boards) + board = cparam.records.file.boards{board_idx}; board_folder_name = cparam.config.board_folder_names; board_folder_name = regexprep(board_folder_name,'%b',board); - - get_filenames_param = struct('regexp',cparam.config.file.regexp,'recursive',true); + + get_filenames_param = struct('regexp',cparam.config.file.regexp); fns = get_filenames(fullfile(cparam.config.base_dir,board_folder_name), ... cparam.config.file.prefix, cparam.config.file.midfix, ... cparam.config.file.suffix, get_filenames_param); num_fns = num_fns + length(fns); - - else - error('cparam.config.daq_type = %s is not a supported type', cparam.config.daq_type); - end - dparam.cpu_time = 60 + 10*num_fns; + + dparam.cpu_time = 60 + cparam.config.cpu_time_per_file*num_fns; dparam.mem = 4e9; dparam.notes = sprintf('%s:%s:%s %d files', ... mfilename, cparam.config.base_dir, cparam.config.config_folder_names, num_fns); @@ -182,6 +167,4 @@ ctrl_chain = {{ctrl}}; -fprintf('Done %s\n', datestr(now)); - -return; +fprintf('%s: Done %s\n', mfilename, datestr(now)); diff --git a/cresis-toolbox/ct_support/create_segments.m b/cresis-toolbox/ct_support/preprocess_create_segments.m similarity index 67% rename from cresis-toolbox/ct_support/create_segments.m rename to cresis-toolbox/ct_support/preprocess_create_segments.m index 4d557c65..02679ef6 100644 --- a/cresis-toolbox/ct_support/create_segments.m +++ b/cresis-toolbox/ct_support/preprocess_create_segments.m @@ -1,10 +1,54 @@ -function [segs,stats] = create_segments(counters,file_idxs,day_wrap_offset,threshold) -% [segs,stats] = create_segments(counters,file_idxs,day_wrap_offset,threshold) +function [segs,stats] = preprocess_create_segments(counters,file_idxs,day_wrap_offset,threshold,segment_end_file_trim) +% [segs,stats] = preprocess_create_segments(counters,file_idxs,day_wrap_offset,threshold,segment_end_file_trim) % -% Support function for preprocess_task_cresis.m +% Support function for preprocess_task_cresis.m. Takes header information +% from raw data files and generates a list of segments by finding time gaps +% in the data. The minimum size of the gap and segment end effects are +% parameters. +% +% Inputs: +% ========================================================================= +% +% counters: cell array of counters vectors. One cell for each board. Each +% vector contains one entry per radar record. +% +% file_idxs: cell array of file_idxs vectors. One cell for each board. Each +% vector contains one entry per radar record. +% +% day_wrap_offset: cell array of day_wrap_offset vectors. One cell for each +% board. Each vector contains one entry per radar record. +% +% threshold: Positive scaler double. Default is 10. Specifies the duration +% in seconds that will cause a new segment to be formed. +% +% segment_end_file_trim: Optional. Positive scaler integer. Default is 0. +% Specifies the number of files at the end of a segment/radar-settings to +% ignore if there is a time gap that ocurs in these files. +% +% Outputs: +% ========================================================================= +% +% segs: +% +% stats: % % Author: John Paden +%% Input check + +% segment_end_file_trim: Positive scaler integer. Default is 0. Specifies +% the number of files at the end of a segment/radar-settings to ignore if +% there is a time gap that ocurs in these files. +if ~exist('segment_end_file_trim','var') || isempty(segment_end_file_trim) + segment_end_file_trim = 0; +end + +% threshold: Positive scaler double. Default is 10. Specifies the duration +% in seconds that will cause a new segment to be formed. +if ~exist('threshold','var') || isempty(threshold) + threshold = 10; +end + % Debug: Test Code debug_test_code = 0; if debug_test_code @@ -45,6 +89,8 @@ return; end dcounters = diff(all_counters); +% FOR UTIG: +%overall_mask = false(size(counters)); % segs_idxs: list of all the segments (gaps in counter >= threshold) segs_idxs = [0 find(dcounters(1:end-1) > threshold)]; for seg_idx = 1:length(segs_idxs) @@ -66,9 +112,11 @@ % Find the file range for each board for board_idx = 1:num_boards % Find files for this segment - mask = counters{board_idx} >= start_counter & counters{board_idx} <= stop_counter; + % FOR UTIG: + %mask = ~overall_mask & (counters{board_idx} >= start_counter & counters{board_idx} <= stop_counter); + mask = (counters{board_idx} >= start_counter & counters{board_idx} <= stop_counter); match_files = file_idxs{board_idx}(mask); - if isempty(match_files) + if isempty(match_files) || (seg_idx>1 && max(file_idxs{board_idx}) - min(match_files) < segment_end_file_trim) segs(seg_idx).start_idxs(board_idx) = 0; segs(seg_idx).stop_idxs(board_idx) = -1; segs(seg_idx).day_wrap_offset(board_idx) = 0; @@ -123,6 +171,9 @@ stats.board_time(seg_idx,board_idx) = counters_board(end)-counters_board(1); end + % FOR UTIG: + %overall_mask = overall_mask | mask; + % Debug: Test Code if debug_test_code fprintf(' %d: %d to %d\n', board_idx, start_idx, stop_idx) diff --git a/cresis-toolbox/ct_support/preprocess_task.m b/cresis-toolbox/ct_support/preprocess_task.m index 67a7a7fe..cab7309c 100644 --- a/cresis-toolbox/ct_support/preprocess_task.m +++ b/cresis-toolbox/ct_support/preprocess_task.m @@ -25,6 +25,8 @@ success = preprocess_task_utua(param); elseif strcmpi(param.config.daq_type,'bas') success = preprocess_task_bas(param); +elseif strcmpi(param.config.daq_type,'utig') + success = preprocess_task_utig(param); else error('Invalid param.config.daq_type %s\n', param.config.daq_type); end diff --git a/cresis-toolbox/ct_support/preprocess_task_arena.m b/cresis-toolbox/ct_support/preprocess_task_arena.m index d7ed7bcd..5708a726 100644 --- a/cresis-toolbox/ct_support/preprocess_task_arena.m +++ b/cresis-toolbox/ct_support/preprocess_task_arena.m @@ -19,9 +19,10 @@ % Pull in the inputs from param struct base_dir = fullfile(param.config.base_dir); -config_folder_name = fullfile(param.config.config_folder_name); +config_folder_name = fullfile(param.config.config_folder_names); reuse_tmp_files = param.config.reuse_tmp_files; -mat_or_bin_hdr_output = param.config.mat_or_bin_hdr_output; + +[~,defaults] = param.config.default(); %% Read each config/system XML file pair into a configs structure % ========================================================================= @@ -38,6 +39,7 @@ for config_idx = 1:length(config_fns) config_fn = config_fns{config_idx}; + fprintf('Reading\t%s\n', config_fn); try configs(config_idx) = read_arena_xml(config_fn,'',param.config.board_map,param.config.tx_map); catch ME @@ -66,20 +68,22 @@ for config_idx = 1:length(configs) %% Initialize variables arena_radar_header_type; % Load radar header types + last_bytes_m = []; last_bytes = zeros(64,1,'uint8'); last_bytes_len = int32(0); num_expected = int32(-1); pkt_counter = int32(-1); - if strcmpi(configs(config_idx).radar_name,'KUSnow') - radar_header_type = snow_radar_header_type; + if strcmpi(configs(config_idx).radar_name,'KUSnow') ... + || strcmpi(configs(config_idx).radar_name,'SnowRadar2') + radar_header_type = snow_radar_header_type; % arena_radar_header_type min_num_expected = int32(0); max_num_expected = int32(configs(config_idx).max_num_bins); default_num_expected = int32(512); num_header_fields = int32(9); length_field_offset = int32(68); elseif strcmpi(configs(config_idx).radar_name,'TOHFSounder') - radar_header_type = hf_sounder_radar_header_type; + radar_header_type = hf_sounder_radar_header_type; % arena_radar_header_type min_num_expected = int32(0); max_num_expected = int32(configs(config_idx).max_num_bins); default_num_expected = int32(512); @@ -91,8 +95,8 @@ default_num_expected = int32(512); num_header_fields = int32(33); length_field_offset = int32(260); - elseif strcmpi(configs(config_idx).radar_name,'ku0001') - radar_header_type = ku0001_radar_header_type; + elseif strcmpi(configs(config_idx).radar_name,'ku0001') || strcmpi(configs(config_idx).radar_name,'ku0002') || strcmpi(configs(config_idx).radar_name,'extsyncarena5xx') + radar_header_type = ku0001_radar_header_type; % arena_radar_header_type min_num_expected = int32(0); max_num_expected = int32(configs(config_idx).max_num_bins); default_num_expected = int32(512); @@ -100,14 +104,20 @@ length_field_offset = int32(68); else - error('Not a supported radar header type %d.', radar_header_type); + warning('Not a supported arena config XML radar name %s. Could indicate a file error. If a new system, then an entry may need to be added here. Run "dbcont" to continue and use default values.', configs(config_idx).radar_name); + keyboard + min_num_expected = int32(0); + max_num_expected = int32(configs(config_idx).max_num_bins); + default_num_expected = int32(512); + num_header_fields = int32(9); + length_field_offset = int32(68); end %% Get Files for each board for board_idx = 1:length(param.config.board_map) board = param.config.board_map(board_idx); - board_folder_name = fullfile(param.config.board_folder_name); + board_folder_name = fullfile(param.config.board_folder_names); % Replace all "%b" in board_folder_name with the board number board_folder_name = regexprep(board_folder_name,'%b',board); @@ -129,6 +139,10 @@ old_fn_dir = []; for fn_idx = 1:length(configs(config_idx).fns{board_idx}) fn = fullfile(configs(config_idx).fns{board_idx}{fn_idx}); + fn_info = dir(fn); + if fn_info.bytes == 0 + error('Zero length file found: %s\n Remove zero-length files first. For example: find INSERT_RAW_FILE_PATH_HERE -size 0 -iname "*.dat" -exec rm {} \\;\n', fn); + end [fn_dir,fn_name] = fileparts(fn); if ~strcmpi(fn_dir,old_fn_dir) @@ -147,11 +161,7 @@ fullfile(board_folder_name, fn_name)); [out_fn_dir,out_fn_name] = fileparts(out_fn); out_fn = fullfile(out_fn_dir,[out_fn_name,'.dat']); - if strcmpi(mat_or_bin_hdr_output,'.mat') - out_hdr_fn = fullfile(out_fn_dir,[out_fn_name,'.mat']); - else - out_hdr_fn = fullfile(out_fn_dir,[out_fn_name,'.hdr']); - end + out_hdr_fn = fullfile(out_fn_dir,[out_fn_name,'.mat']); % Print status fprintf('arena_pkt_strip %d/%d %d/%d %s (%s)\n %s\n', config_idx, ... @@ -198,7 +208,7 @@ if fn_idx == 1 && board_idx == 1 [out_config_fn_dir] = fileparts(configs(config_idx).config_fn); log_files = fullfile(out_config_fn_dir,'logs/*'); - out_log_dir = fullfile(param.data_support_path, param.season_name, param.config.config_folder_name); + out_log_dir = fullfile(param.data_support_path, param.season_name, param.config.config_folder_names); try fprintf('Copy %s\n %s\n', log_files, out_log_dir); if ~exist(out_log_dir,'dir') @@ -211,40 +221,99 @@ end % Check to see if outputs already exist - if reuse_tmp_files && exist(out_hdr_fn,'file') ... - && (strcmpi(configs(config_idx).datastream_type,'tcp') || exist(out_fn,'file')) - load(out_hdr_fn,'last_bytes','last_bytes_len','num_expected','pkt_counter'); - continue; + if reuse_tmp_files && exist(out_hdr_fn,'file') + % For older systems that used UDP (datastream_type == 'udp'), the + % UDP packet headers are in the raw data files and preprocess also + % creates a copy of the data files without the packet header. Look + % for this file in that case. Newer systems can override this check + % by setting defaults{end}.arena.daq.udp_packet_headers = false. + if ~strcmpi(configs(config_idx).datastream_type,'udp') ... + || exist(out_fn,'file') ... + || isfield(defaults{end}.arena,'daq') ... + && isfield(defaults{end}.arena.daq,'udp_packet_headers') ... + && ~defaults{end}.arena.daq.udp_packet_headers + % Load the file to ensure it is not corrupted: + % * If not corrupted, then execution continues onto the next + % file. + % * If the file is corrupted, matlab will produce an error here + % and stop. The corrupt file should be deleted and the + % preprocess run again. + load(out_hdr_fn,'last_bytes','last_bytes_len','num_expected','pkt_counter'); + continue; + end end %% Read in headers from data file and create network packet stripped data file - if strcmpi(configs(config_idx).datastream_type,'tcp') - [hdr,last_bytes_len,num_expected,pkt_counter] = arena_packet_strip_tcp_mex(fn,out_fn,last_bytes,last_bytes_len, ... + if strcmpi(configs(config_idx).datastream_type,'udp') ... + && (~isfield(defaults{end}.arena,'daq') ... + || ~isfield(defaults{end}.arena.daq,'udp_packet_headers') ... + || defaults{end}.arena.daq.udp_packet_headers) + % In old arena systems, choosing the UDP datastream created raw + % data files with UDP packet headers in them and a copy of the raw + % data file without the packet headers will be made in "out_fn". If + % using a new system, then the UDP packet headers are not in the + % files, but oparams{end}.arena.daq.udp_packet_headers must be set + % to true for preprocess to process them correctly. + [hdr,last_bytes_len,num_expected,pkt_counter] = arena_packet_strip_mex(fn,out_fn,last_bytes,last_bytes_len, ... num_expected,pkt_counter,min_num_expected,max_num_expected, ... default_num_expected,num_header_fields,length_field_offset); else - [hdr,last_bytes_len,num_expected,pkt_counter] = arena_packet_strip_mex(fn,out_fn,last_bytes,last_bytes_len, ... + % The "out_fn" input argument is ignored. These raw data files do + % not contain UDP packet headers. + [hdr,last_bytes_len,num_expected,pkt_counter] = arena_packet_strip_tcp_mex(fn,out_fn,last_bytes,last_bytes_len, ... num_expected,pkt_counter,min_num_expected,max_num_expected, ... default_num_expected,num_header_fields,length_field_offset); end %% Write header output file + mat_or_bin_hdr_output = '.mat'; if strcmpi(mat_or_bin_hdr_output,'.mat') if strcmpi(configs(config_idx).radar_name,'ku0001') offset = mod(hdr(1,:),2^32); mode_latch = mod(hdr(3,:),2^8); subchannel = mod(bitshift(hdr(3,:),-8),2^8); + profile = mod(bitshift(hdr(3,:),-16),2^8); wg_delay_latch = mod(hdr(4,:),2^16); rel_time_cntr_latch = double(hdr(5,:)); profile_cntr_latch = double(hdr(6,:)); pps_ftime_cntr_latch = double(hdr(7,:)); pps_cntr_latch = double(hdr(8,:)); + save(out_hdr_fn, 'offset','mode_latch','subchannel','profile','wg_delay_latch', ... + 'rel_time_cntr_latch','profile_cntr_latch','pps_ftime_cntr_latch','pps_cntr_latch', ... + 'last_bytes','last_bytes_len','num_expected','pkt_counter'); + + elseif strcmpi(configs(config_idx).radar_name,'ku0002') + offset = hdr{1}.frame_sync; + mode_latch = hdr{1}.mode; + subchannel = hdr{1}.subchannel; + wg_delay_latch = 0; + rel_time_cntr_latch = hdr{1}.rel_time_cntr_latch; + profile_cntr_latch = hdr{1}.profile_cntr_latch; + pps_ftime_cntr_latch =hdr{1}.pps_ftime_cntr_latch; + pps_cntr_latch = hdr{1}.pps_cntr_latch; + save(out_hdr_fn, 'offset','mode_latch','subchannel','wg_delay_latch', ... 'rel_time_cntr_latch','profile_cntr_latch','pps_ftime_cntr_latch','pps_cntr_latch', ... 'last_bytes','last_bytes_len','num_expected','pkt_counter'); - elseif strcmpi(configs(config_idx).radar_name,'KUSnow') + elseif strcmpi(configs(config_idx).radar_name,'extsyncarena5xx') + offset = mod(hdr(1,:),2^32); + mode_latch = mod(hdr(3,:),2^8); + subchannel = mod(bitshift(hdr(3,:),-8),2^8); + profile = mod(bitshift(hdr(3,:),-16),2^8); + wg_delay_latch = mod(hdr(4,:),2^16); + rel_time_cntr_latch = double(hdr(5,:)); + profile_cntr_latch = double(hdr(6,:)); + pps_ftime_cntr_latch = double(hdr(7,:)); + pps_cntr_latch = double(hdr(8,:)); + + save(out_hdr_fn, 'offset','mode_latch','subchannel','profile','wg_delay_latch', ... + 'rel_time_cntr_latch','profile_cntr_latch','pps_ftime_cntr_latch','pps_cntr_latch', ... + 'last_bytes','last_bytes_len','num_expected','pkt_counter'); + + elseif strcmpi(configs(config_idx).radar_name,'KUSnow') ... + || strcmpi(configs(config_idx).radar_name,'SnowRadar2') offset = mod(hdr(1,:),2^32); mode_latch = mod(hdr(3,:),2^8); subchannel = mod(bitshift(hdr(3,:),-8),2^8); @@ -327,6 +396,8 @@ %% Print out segments % ========================================================================= oparams = {}; +segment = 0; +last_day_seg = '00000000'; for config_idx = 1:length(configs) if isempty(configs(config_idx).fns{1}) @@ -337,22 +408,29 @@ % Determine which default parameters to use % ======================================================================= match_idx = []; - for default_idx = 1:length(param.config.defaults) - match = regexpi(configs(config_idx).psc.config_name, param.config.defaults{default_idx}.config_regexp); - if ~isempty(match) - match_idx = default_idx; - break; - end - end - if isempty(match_idx) - error('No match for psc config name %s.', configs(config_idx).psc.config_name); - end - oparams{end+1} = param.config.defaults{match_idx}; + default = default_radar_params_settings_match(defaults,configs(config_idx).psc.config_name); + default = merge_structs(param,default); + oparams{end+1} = default; + try; oparams{end} = rmfield(oparams{end},'config_regexp'); end; + try; oparams{end} = rmfield(oparams{end},'name'); end; + +% for default_idx = 1:length(param.config.defaults) +% match = regexpi(configs(config_idx).psc.config_name, param.config.defaults{default_idx}.config_regexp); +% if ~isempty(match) +% match_idx = default_idx; +% break; +% end +% end +% if isempty(match_idx) +% error('No match for psc config name %s.', configs(config_idx).psc.config_name); +% end +% oparams{end+1} = param.config.defaults{match_idx}; % Create map from wfs to board_idx, mode, subchannel, adc % ======================================================================= data_map = oparams{end}.records.data_map; board_idx_map = []; + profile_map = []; mode_map = []; subchannel_map = []; wfs_map = []; @@ -360,10 +438,22 @@ for board_idx = 1:length(data_map) for data_idx = 1:size(data_map{board_idx},1) board_idx_map(end+1) = board_idx; - mode_map(end+1) = data_map{board_idx}(data_idx,1); - subchannel_map(end+1) = data_map{board_idx}(data_idx,2); - wfs_map(end+1) = data_map{board_idx}(data_idx,3); - adc_map(end+1) = data_map{board_idx}(data_idx,4); + if size(data_map{board_idx},2) == 4 + % NON-PROFILE MODE + % data_map rows: [mode subchannel wf adc] + mode_map(end+1) = data_map{board_idx}(data_idx,1); + subchannel_map(end+1) = data_map{board_idx}(data_idx,2); + wfs_map(end+1) = data_map{board_idx}(data_idx,3); + adc_map(end+1) = data_map{board_idx}(data_idx,4); + else + % PROFILE + % data_map rows: [profile mode subchannel wf adc] + profile_map(end+1) = data_map{board_idx}(data_idx,1); + mode_map(end+1) = data_map{board_idx}(data_idx,2); + subchannel_map(end+1) = data_map{board_idx}(data_idx,3); + wfs_map(end+1) = data_map{board_idx}(data_idx,4); + adc_map(end+1) = data_map{board_idx}(data_idx,5); + end end end [wfs,unique_map] = unique(wfs_map); @@ -371,11 +461,22 @@ mode_wfs = mode_map(unique_map); subchannel_wfs = subchannel_map(unique_map); adc_wfs = adc_map(unique_map); + if size(data_map{board_idx},2) == 5 + profile_wfs = profile_map(unique_map); + end % Parameter spreadsheet % ======================================================================= [~,config_fn_name] = fileparts(configs(config_idx).config_fn); - oparams{end}.day_seg = sprintf('%s_%02d',config_fn_name(1:8),numel(oparams)); + if strcmp(last_day_seg(1:8),config_fn_name(1:8)) + % Same day + segment = segment + 1; + else + % New day + segment = 1; + end + oparams{end}.day_seg = sprintf('%s_%02d',config_fn_name(1:8),segment); + last_day_seg = oparams{end}.day_seg; oparams{end}.cmd.notes = configs(config_idx).psc.config_name(5:end); oparams{end}.records.file.version = 103; @@ -385,12 +486,17 @@ oparams{end}.records.file.start_idx(board_idx) = 1; oparams{end}.records.file.stop_idx(board_idx) = length(configs(config_idx).fns{board_idx}); end - if strcmpi(configs(config_idx).datastream_type,'tcp') - oparams{end}.records.file.base_dir = base_dir; - else + if strcmpi(configs(config_idx).datastream_type,'udp') ... + && (~isfield(defaults{end}.arena,'daq') ... + || ~isfield(defaults{end}.arena.daq,'udp_packet_headers') ... + || defaults{end}.arena.daq.udp_packet_headers) + % See earlier discussion on udp_packet_headers. The raw data files with + % the UDP packet headers removed are stored in ct_tmp. oparams{end}.records.file.base_dir = ct_filename_ct_tmp(param,'','headers',''); + else + oparams{end}.records.file.base_dir = base_dir; end - oparams{end}.records.file.board_folder_name = param.config.board_folder_name; + oparams{end}.records.file.board_folder_name = param.config.board_folder_names; if ~isnan(str2double(oparams{end}.records.file.board_folder_name)) oparams{end}.records.file.board_folder_name = ['/' oparams{end}.records.file.board_folder_name]; end @@ -398,9 +504,26 @@ oparams{end}.records.gps.time_offset = 0; oparams{end}.records.gps.en = 1; [~,config_fn_name] = fileparts(configs(config_idx).config_fn); - oparams{end}.records.config_fn = fullfile(param.config.config_folder_name, [config_fn_name '.xml']); + oparams{end}.records.config_fn = fullfile(param.config.config_folder_names, [config_fn_name '.xml']); - oparams{end}.radar.fs = configs(config_idx).adc{board_idx_wfs(1),1+mode_wfs(1),1+subchannel_wfs(1)}.sampFreq; + found_board = false; + adc_list_str = ''; + for search_idx = 1:size(configs(config_idx).adc,1) + if ~isfield(configs(config_idx).adc{search_idx},'name') + adc_list_str = [adc_list_str '"",']; + else + adc_list_str = [adc_list_str '"' configs(config_idx).adc{search_idx}.name '",']; + if strcmpi(configs(config_idx).adc{search_idx}.name,param.config.board_map{board_idx}) + adc_board_idx = search_idx; + found_board = true; + break; + end + end + end + if ~found_board + error('Board %s was not found in adc list (%s) from config file %s.',param.config.board_map{board_idx},adc_list_str(1:end-1),configs.config_fn); + end + oparams{end}.radar.fs = configs(config_idx).adc{adc_board_idx,1+mode_wfs(1),1+subchannel_wfs(1)}.sampFreq; oparams{end}.radar.prf = configs(config_idx).prf * configs(config_idx).total_presums; % Usually the default.radar.wfs structure only has one waveform @@ -415,15 +538,88 @@ board_idx = board_idx_wfs(wf_idx); mode_latch = mode_wfs(wf_idx); subchannel = subchannel_wfs(wf_idx); + if strcmpi(param.season_name,'2022_Greenland_P3') + % Delete This After AITT 2022 done <-- BROKEN NOW??? + configs(config_idx).dac{1,mode_latch+1}.wfs{1}.pulse{1}.centerFreq = 10e9; % TEMP FIX + configs(config_idx).dac{1,mode_latch+1}.wfs{1}.pulse{1}.bandwidth = 15e9; % TEMP FIX + configs(config_idx).dac{1,mode_latch+1}.sampFreq = 2400; % TEMP FIX + configs(config_idx).dac{1,mode_latch+1}.wfs{1}.pulse{1}.numPoints = 240e-6*configs(config_idx).dac{1,mode_latch+1}.sampFreq; % TEMP FIX + configs(config_idx).dac{1}.delay = 0; % TEMP FIX + configs(config_idx).dac{1,mode_latch+1}.wfs{1}.pulse{1}.alpha = 15e9; + for tx = 1:length(oparams{end}.radar.wfs(wf).tx_paths) + tx_idx = oparams{end}.radar.wfs(wf).tx_paths(tx); + if isfinite(oparams{end}.radar.wfs(wf).tx_paths(tx)) + configs(config_idx).dac{tx_idx,mode_latch+1}.wfs{1}.scale = 1; + else + configs(config_idx).dac{tx_idx,mode_latch+1}.wfs{1}.scale = 0; + end + end + else + % scale: Determines the transmit antenna array weights (e.g. which + % antenna phase centers were used and what the transmit weights were) + scale = []; + % use_tx_map_idx: Which transmit map index represents an active + % waveform. The assumption is that any non-zero scale output AWG will + % have the current Tpd, fc, BW, etc. + use_tx_map_idx = 1; + + for tx_map_idx = 1:length(param.config.tx_map) + for tx_paths_idx = 1:length(oparams{end}.radar.wfs(wf).tx_paths{tx_map_idx}) + if isfinite(oparams{end}.radar.wfs(wf).tx_paths{tx_map_idx}(tx_paths_idx)) + tx = oparams{end}.radar.wfs(wf).tx_paths{tx_map_idx}(tx_paths_idx); + if tx_paths_idx > length(configs(config_idx).dac{tx_map_idx,mode_latch+1}.wfs) ... + || isempty(configs(config_idx).dac{tx_map_idx,mode_latch+1}.wfs{tx_paths_idx}) + scale(tx) = 0; + else + scale(tx) = configs(config_idx).dac{tx_map_idx,mode_latch+1}.wfs{tx_paths_idx}.pulse{1}.scale; + if scale(tx) ~= 0 + use_tx_map_idx = tx_map_idx; + end + end + end + end + end + end + + % Explanation of how ADC and DAC map to cresis-toolbox values (e.g. + % wf, adc, rx_paths, tx_paths + + % param.config.board_map <-- maps board_idx to config entry + % multiple adc channels per board, handled by param.records.data_map - fc = configs(config_idx).dac{1,mode_latch+1}.wfs{1}.centerFreq*1e6; - BW = configs(config_idx).dac{1,mode_latch+1}.wfs{1}.bandwidth*1e6; - Nt = configs(config_idx).dac{1,mode_latch+1}.wfs{1}.numPoints; - fs = configs(config_idx).dac{1,mode_latch+1}.sampFreq*1e6; + % param.config.tx_map <-- maps tx_idx to config entry + % multiple transmit channels per tx, handled by + % param.records.tx_paths + % If mapping is simple with one DAC/tx_path per tx_map entry then the mapping should be automatic + % {entry 1--> tx_path 1, entry 2 --> tx_path 2, etc.} + % + % This can be assumed tx_path when tx_path not specified. + % tx_paths = {1,2,3,4,5,6,7,8} <-- POLAR 6 (default tx_paths) + % tx_paths = {[1 2 5 6]} <-- GHOST RDS + % tx_weights gets generated as [1 1 0 0 1 1] + % + % Eventually should allow tx_paths to change on a per wf basis + % tx_paths = {[1 2 3 4; 5 6 7 8]} <-- SWITCHING left/right for wf 1 + % and 2 and using the same transmitter for both left and right + % elements with an analog switch to choose which side + % + % Eventually should also have tx_pol and rx_pol with the same size as + % tx_paths and rx_paths except populated with 'H', 'V', 'L', 'R', 'U' + % for horizontal, vertical, left-circular, right-circular, unknown + + + fc = configs(config_idx).dac{use_tx_map_idx,mode_latch+1}.wfs{1}.pulse{1}.centerFreq; + BW = configs(config_idx).dac{use_tx_map_idx,mode_latch+1}.wfs{1}.pulse{1}.bandwidth; + if strcmpi(configs(config_idx).radar_name,'ku0002') + fc = fc*param.config.defaults{1}.radar.wfs(wf).fmult + param.config.defaults{1}.radar.wfs(wf).fLO; + BW = BW*param.config.defaults{1}.radar.wfs(wfs).fmult; + end + Nt = configs(config_idx).dac{use_tx_map_idx,mode_latch+1}.wfs{1}.pulse{1}.numPoints; + fs = configs(config_idx).dac{use_tx_map_idx,mode_latch+1}.wfs{1}.sampFreq; Tpd = Nt/fs; - t_dac = (configs(config_idx).dac{1}.delay) * 1e-6; + t_dac = (configs(config_idx).dac{use_tx_map_idx,1}.delay); - switch (configs(config_idx).adc{board_idx_wfs(1),1+mode_wfs(1),1+subchannel_wfs(1)}.adcMode) + switch (configs(config_idx).adc{adc_board_idx,1+mode_wfs(1),1+subchannel_wfs(1)}.adcMode) case 0 oparams{end}.radar.wfs(wf).DDC_dec = 1; case 1 @@ -433,59 +629,79 @@ end if subchannel_wfs(1) == 0 - switch (configs(config_idx).adc{board_idx_wfs(1),1+mode_wfs(1),1+subchannel_wfs(1)}.ddc0NcoMode) + switch (configs(config_idx).adc{adc_board_idx,1+mode_wfs(1),1+subchannel_wfs(1)}.ddc0NcoMode) case 0 oparams{end}.radar.wfs(wf).DDC_freq = 0; case 1 oparams{end}.radar.wfs(wf).DDC_freq = oparams{end}.radar.fs/4; case 2 - oparams{end}.radar.wfs(wf).DDC_freq = configs(config_idx).adc{board_idx_wfs(1),1+mode_wfs(1),1+subchannel_wfs(1)}.ddc0NcoFreq; + oparams{end}.radar.wfs(wf).DDC_freq = configs(config_idx).adc{adc_board_idx,1+mode_wfs(1),1+subchannel_wfs(1)}.ddc0NcoFreq; end elseif subchannel_wfs(1) == 1 - switch (configs(config_idx).adc{board_idx_wfs(1),1+mode_wfs(1),1+subchannel_wfs(1)}.ddc1NcoMode) + switch (configs(config_idx).adc{adc_board_idx,1+mode_wfs(1),1+subchannel_wfs(1)}.ddc1NcoMode) + case 0 + oparams{end}.radar.wfs(wf).DDC_freq = 0; + case 1 + oparams{end}.radar.wfs(wf).DDC_freq = oparams{end}.radar.fs/4; + case 2 + oparams{end}.radar.wfs(wf).DDC_freq = configs(config_idx).adc{adc_board_idx,1+mode_wfs(1),1+subchannel_wfs(1)}.ddc1NcoFreq; + end + elseif subchannel_wfs(1) == 2 + switch (configs(config_idx).adc{adc_board_idx,1+mode_wfs(1),1+subchannel_wfs(1)}.ddc2NcoMode) + case 0 + oparams{end}.radar.wfs(wf).DDC_freq = 0; + case 1 + oparams{end}.radar.wfs(wf).DDC_freq = oparams{end}.radar.fs/4; + case 2 + oparams{end}.radar.wfs(wf).DDC_freq = configs(config_idx).adc{adc_board_idx,1+mode_wfs(1),1+subchannel_wfs(1)}.ddc2NcoFreq; + end + elseif subchannel_wfs(1) == 3 + switch (configs(config_idx).adc{adc_board_idx,1+mode_wfs(1),1+subchannel_wfs(1)}.ddc3NcoMode) case 0 oparams{end}.radar.wfs(wf).DDC_freq = 0; case 1 oparams{end}.radar.wfs(wf).DDC_freq = oparams{end}.radar.fs/4; case 2 - oparams{end}.radar.wfs(wf).DDC_freq = configs(config_idx).adc{board_idx_wfs(1),1+mode_wfs(1),1+subchannel_wfs(1)}.ddc1NcoFreq; + oparams{end}.radar.wfs(wf).DDC_freq = configs(config_idx).adc{adc_board_idx,1+mode_wfs(1),1+subchannel_wfs(1)}.ddc3NcoFreq; end end oparams{end}.radar.wfs(wf).f0 = fc-BW/2; oparams{end}.radar.wfs(wf).f1 = fc+BW/2; - oparams{end}.radar.wfs(wf).tukey = configs(config_idx).dac{1,mode_latch+1}.wfs{1}.alpha; + oparams{end}.radar.wfs(wf).tukey = configs(config_idx).dac{1,mode_latch+1}.wfs{1}.pulse{1}.alpha; oparams{end}.radar.wfs(wf).BW_window = [fc-BW/2 fc+BW/2]; oparams{end}.radar.wfs(wf).Tpd = Tpd; - scale = []; - for tx = 1:length(oparams{end}.radar.wfs(wf).tx_paths) - if isfinite(oparams{end}.radar.wfs(wf).tx_paths(tx)) - tx_idx = oparams{end}.radar.wfs(wf).tx_paths(tx); - scale(tx) = configs(config_idx).dac{tx_idx,mode_latch+1}.wfs{1}.scale; - else - scale(tx) = 0; - end - end oparams{end}.radar.wfs(wf).tx_weights = scale; - oparams{end}.radar.wfs(wf).presums = configs(config_idx).adc{board_idx,mode_latch+1,subchannel+1}.presums; + oparams{end}.radar.wfs(wf).presums = configs(config_idx).adc{adc_board_idx,mode_latch+1,subchannel+1}.presums; adc_idxs = find(wfs_map == wf); for adc_idx = adc_idxs adc = adc_map(adc_idx); adc_board_idx = board_idx_map(adc_idx); adc_mode = mode_map(adc_idx); adc_subchannel = subchannel_map(adc_idx); - oparams{end}.radar.wfs(wf).bit_shifts(adc) = configs(config_idx).adc{adc_board_idx,adc_mode+1,adc_subchannel+1}.shiftLSB - 2 - param.config.arena.adc(adc_board_idx).gain_dB(adc_subchannel+1)/6; + if isfield(oparams{end}.radar.wfs(wf), 'bit_shifts') + % User bit shifts override + oparams{end}.radar.wfs(wf).bit_shifts(adc) = oparams{end}.radar.wfs(wf).bit_shifts(adc); + else + % Bit shifts from digital radar config file + oparams{end}.radar.wfs(wf).bit_shifts(adc) = configs(config_idx).adc{adc_board_idx,adc_mode+1,adc_subchannel+1}.shiftLSB; + end + % Note that receiver gain from radar config file should possibly be + % included: + %oparams{end}.arena.adc(adc_board_idx).gain_dB(adc_subchannel+1) end - oparams{end}.radar.wfs(wf).Tadc = sscanf(configs(config_idx).adc{board_idx,mode_latch+1,subchannel+1}.rg,'%d') ... + oparams{end}.radar.wfs(wf).Tadc = sscanf(configs(config_idx).adc{adc_board_idx,mode_latch+1,subchannel+1}.rg,'%d') ... / oparams{end}.radar.fs*oparams{end}.radar.wfs(wf).DDC_dec ... - - param.config.arena.param.ADC_time_delay - t_dac; + - oparams{end}.arena.param.ADC_time_delay - t_dac; end end %% Print out segments % ========================================================================= -if ~isempty(param.config.param_fn) +if ~exist(param.config.param_fn,'file') + warning('Could not find parameter spreadsheet file so not printing parameter values.\n param.config.param_fn = ''%s''', param.config.param_fn); +else % Print parameter spreadsheet values to stdout and param_txt_fn % ========================================================================= fid = 1; @@ -503,8 +719,20 @@ read_param_xls_print(param.config.param_fn,'radar',oparams,fid); fprintf(fid,'%s\n','='*ones(1,80)); fprintf(fid,' post\n'); fprintf(fid,'%s\n','='*ones(1,80)); read_param_xls_print(param.config.param_fn,'post',oparams,fid); - fprintf(fid,'%s\n','='*ones(1,80)); fprintf(fid,' analysis\n'); fprintf(fid,'%s\n','='*ones(1,80)); - read_param_xls_print(param.config.param_fn,'analysis',oparams,fid); + % Other sheets + try + warning off MATLAB:xlsfinfo:ActiveX + [status, sheets] = xlsfinfo(param.config.param_fn); + warning on MATLAB:xlsfinfo:ActiveX + for sheet_idx = 1:length(sheets) + if ~any(strcmpi(sheets{sheet_idx},{'cmd','records','qlook','sar','array','radar','post'})) + fprintf(fid,'%s\n','='*ones(1,80)); fprintf(fid,' %s\n', sheets{sheet_idx}); fprintf(fid,'%s\n','='*ones(1,80)); + read_param_xls_print(param.config.param_fn,sheets{sheet_idx},oparams,fid); + end + end + catch ME + ME.getReport + end fprintf(fid,'\n'); param_txt_fn = ct_filename_ct_tmp(param,'','param', [param.config.date_str,'.txt']); @@ -538,8 +766,20 @@ fprintf(fid,'%s\n','='*ones(1,80)); fprintf(fid,' post\n'); fprintf(fid,'%s\n','='*ones(1,80)); read_param_xls_print(param.config.param_fn,'post',oparams,fid); fprintf(fid,'\n'); - fprintf(fid,'%s\n','='*ones(1,80)); fprintf(fid,' analysis\n'); fprintf(fid,'%s\n','='*ones(1,80)); - read_param_xls_print(param.config.param_fn,'analysis',oparams,fid); + % Other sheets + try + warning off MATLAB:xlsfinfo:ActiveX + [status, sheets] = xlsfinfo(param.config.param_fn); + warning on MATLAB:xlsfinfo:ActiveX + for sheet_idx = 1:length(sheets) + if ~any(strcmpi(sheets{sheet_idx},{'cmd','records','qlook','sar','array','radar','post'})) + fprintf(fid,'%s\n','='*ones(1,80)); fprintf(fid,' %s\n', sheets{sheet_idx}); fprintf(fid,'%s\n','='*ones(1,80)); + read_param_xls_print(param.config.param_fn,sheets{sheet_idx},oparams,fid); + end + end + catch ME + ME.getReport + end fprintf(fid,'\n'); fclose(fid); end diff --git a/cresis-toolbox/ct_support/preprocess_task_bas.m b/cresis-toolbox/ct_support/preprocess_task_bas.m index 718bb428..2cca1780 100644 --- a/cresis-toolbox/ct_support/preprocess_task_bas.m +++ b/cresis-toolbox/ct_support/preprocess_task_bas.m @@ -124,7 +124,7 @@ file_idxs{board_idx} = board_hdrs{board_idx}.file_idxs; day_wrap_offset{board_idx} = board_hdrs{board_idx}.day_wrap_offset; end -[segs,stats] = create_segments(counters,file_idxs,day_wrap_offset,param.config.max_time_gap); +[segs,stats] = preprocess_create_segments(counters,file_idxs,day_wrap_offset,param.config.max_time_gap); if 1 % Debug: Test Code diff --git a/cresis-toolbox/ct_support/preprocess_task_cresis.m b/cresis-toolbox/ct_support/preprocess_task_cresis.m index c9f25d2f..dcf8f4c6 100644 --- a/cresis-toolbox/ct_support/preprocess_task_cresis.m +++ b/cresis-toolbox/ct_support/preprocess_task_cresis.m @@ -22,6 +22,10 @@ param.config.field_time_gap = 'utc_time_sod'; end +if ~isfield(param.config,'segment_end_file_trim') || isempty(param.config.segment_end_file_trim) + param.config.segment_end_file_trim = 1; +end + if ~isfield(param.config,'plots_visible') || isempty(param.config.plots_visible) param.config.plots_visible = 1; end @@ -31,11 +35,11 @@ % Save concatenated temporary file fn_board_hdrs = ct_filename_ct_tmp(param,'','headers', fullfile(param.config.date_str,'board_hdrs.mat')); -num_board_to_load = numel(param.config.board_map); +num_board_to_load = numel(param.records.file.boards); board_hdrs = cell(1,num_board_to_load); failed_load = cell(1,num_board_to_load); fns_list = cell(1,num_board_to_load); -if param.config.reuse_tmp_files && exist(fn_board_hdrs,'file') +if param.config.reuse_tmp_files && exist(fn_board_hdrs,'file') && ~param.config.tmp_load_mode try fprintf('Found %s\n Trying to load...\n', fn_board_hdrs); load(fn_board_hdrs,'board_hdrs','fns_list','failed_load'); @@ -47,13 +51,13 @@ for board_idx = 1:num_board_to_load %% Read Headers: Filenames - board = param.config.board_map{board_idx}; - board_folder_name = param.config.board_folder_name; + board = param.records.file.boards{board_idx}; + board_folder_name = param.config.board_folder_names; board_folder_name = regexprep(board_folder_name,'%b',board); get_filenames_param = struct('regexp',param.config.file.regexp); fns = get_filenames(fullfile(param.config.base_dir,board_folder_name), ... - param.config.file.prefix, param.config.file.midfix, ... + param.records.file.prefix, param.config.file.midfix, ... param.config.file.suffix, get_filenames_param); if param.config.online_mode == 2 @@ -62,8 +66,8 @@ fns_list{board_idx} = fns; % Copy Log Files - if board_idx == 1 && ~isempty(param.config.cresis.gps_file_mask) - log_files = fullfile(param.config.base_dir,param.config.config_folder_name,param.config.cresis.gps_file_mask); + if board_idx == 1 && ~isempty(param.config.gps_file_mask) + log_files = fullfile(param.config.base_dir,param.config.config_folder_names,param.config.gps_file_mask); out_log_dir = fullfile(param.data_support_path, param.season_name, param.config.date_str); fprintf('Copy %s\n %s\n', log_files, out_log_dir); try @@ -78,7 +82,7 @@ if isempty(fns) warning('No files found matching %s*%s*%s', ... - fullfile(param.config.base_dir,board_folder_name,param.config.file.prefix), ... + fullfile(param.config.base_dir,board_folder_name,param.records.file.prefix), ... param.config.file.midfix, param.config.file.suffix); fprintf('%s done %s\n', mfilename, datestr(now)); success = true; @@ -91,12 +95,12 @@ % ACORDS filenames are not in chronological order, resort by their % extension (.1, .2, .3, ..., .100, etc.) - if any(param.config.file.version == [405 406]) + if any(param.records.file.version == [405 406]) basenames = {}; file_idxs = []; new_fns = {}; finfo_param.hnum = 1; - finfo_param.preprocess.file.version = param.config.file.version; + finfo_param.preprocess.file.version = param.records.file.version; for fidx = 1:length(fns) fname = fname_info_acords(fns{fidx},finfo_param); new_fns{fidx} = [fname.basename sprintf('.%03d',fname.file_idx)]; @@ -107,69 +111,74 @@ %% Read Headers: Header Info hdr_param = struct('file_mode','ieee-be'); - if any(param.config.file.version == [1]) + if any(param.records.file.version == [1]) hdr_param.frame_sync = uint32(hex2dec('DEADBEEF')); hdr_param.field_offsets = int32([4 16 20]); % epri seconds fractions hdr_param.field_types = {uint32(1) uint32(1) uint32(1)}; - elseif any(param.config.file.version == [2]) + elseif any(param.records.file.version == [2]) hdr_param.frame_sync = uint32(hex2dec('BADA55E5')); hdr_param.field_offsets = int32([4 8 12 24]); % epri seconds fractions loopback/nyquist-zone hdr_param.field_types = {uint32(1) uint32(1) uint32(1) uint32(1)}; - elseif any(param.config.file.version == [4]) + elseif any(param.records.file.version == [4]) hdr_param.frame_sync = uint32(hex2dec('BADA55E5')); hdr_param.field_offsets = int32([4 8 12 16]); % epri sec1 sec2 fractions hdr_param.field_types = {uint32(1) uint32(1) uint32(1) uint64(1)}; - elseif any(param.config.file.version == [3 5]) + elseif any(param.records.file.version == [3 5]) hdr_param.frame_sync = uint32(hex2dec('BADA55E5')); hdr_param.field_offsets = int32(4*[1 2 3 9 10 11]); % epri seconds fractions start/stop-index DDCfield1 DDCfield2 hdr_param.field_types = {uint32(1) uint32(1) uint32(1) uint32(1) uint32(1) uint32(1)}; - elseif any(param.config.file.version == [8]) + elseif any(param.records.file.version == [8]) hdr_param.frame_sync = uint32(hex2dec('BADA55E5')); hdr_param.field_offsets = int32([4 8 12 16 33 36 38 40]); % epri seconds fractions counter nyquist-zone waveform-ID hdr_param.field_types = {uint32(1) uint32(1) uint32(1) uint64(1) uint8(1) uint16(1) uint16(1) uint64(1)}; - elseif any(param.config.file.version == [101]) + elseif any(param.records.file.version == [101]) hdr_param.frame_sync = uint32(hex2dec('DEADBEEF')); hdr_param.field_offsets = int32([4 8 12]); % epri seconds fractions hdr_param.field_types = {uint32(1) uint32(1) uint32(1)}; - elseif any(param.config.file.version == [102]) + elseif any(param.records.file.version == [102]) hdr_param.frame_sync = uint32(hex2dec('1ACFFC1D')); hdr_param.field_offsets = int32(4*[1 3 4 5 6]); % epri seconds fractions hdr_param.field_types = {uint32(1) uint32(1) uint32(1) uint32(1) uint32(1)}; hdr_param.frame_sync = hex2dec('1ACFFC1D'); - elseif any(param.config.file.version == [401]) + elseif any(param.records.file.version == [401]) hdr_param.frame_sync = uint32(hex2dec('DEADBEEF')); hdr_param.field_offsets = int32([16 8 12]); % epri seconds fractions hdr_param.field_types = {uint32(1) uint32(1) uint32(1)}; - elseif any(param.config.file.version == [402 403]) + elseif any(param.records.file.version == [402 403]) hdr_param.frame_sync = uint32(hex2dec('BADA55E5')); hdr_param.field_offsets = int32([4 8 12 16]); % epri seconds fraction counter hdr_param.field_types = {uint32(1) uint32(1) uint32(1) uint64(1)}; - elseif any(param.config.file.version == [404]) + elseif any(param.records.file.version == [404]) hdr_param.frame_sync = uint32(hex2dec('1ACFFC1D')); hdr_param.field_offsets = int32([4 16 20 24]); % epri seconds fraction counter hdr_param.field_types = {uint32(1) uint32(1) uint32(1) uint64(1)}; - elseif any(param.config.file.version == [405 406]) + elseif any(param.records.file.version == [405 406]) hdr_param.file_mode = 'ieee-le'; hdr_param.frame_sync = uint32(0); hdr_param.field_offsets = int32([0 4]); % epri seconds fractions hdr_param.field_types = {uint32(1) uint32(1)}; - elseif any(param.config.file.version == [7 11 407 408]) + elseif any(param.records.file.version == [7 11 407 408]) hdr_param.frame_sync = uint32(hex2dec('1ACFFC1D')); - + + elseif any(param.records.file.version == [420]) + hdr_param.frame_sync = uint32(1212568132); + hdr_param.field_offsets = int32([4 8 12]); % epri seconds fraction + hdr_param.field_types = {uint32(1) uint32(1) uint32(1)}; + hdr_param.file_mode = 'ieee-le'; else - error('Unsupported file version %d (%s)', param.config.file.version, param.radar_name); + error('Unsupported file version %d (%s)', param.records.file.version, param.radar_name); end %% Read Headers: File Loop @@ -188,7 +197,7 @@ % Create temporary filename that will store the header information for % this file. fn = fns{fn_idx}; - if any(param.config.file.version == [405 406]) + if any(param.records.file.version == [405 406]) [~,fn_name,ext] = fileparts(fn); fn_name = [fn_name,ext]; else @@ -208,7 +217,7 @@ if param.config.reuse_tmp_files && exist(tmp_hdr_fn,'file') % Try to load temporary file try - if any(param.config.file.version == [101]) + if any(param.records.file.version == [101]) hdr = load(tmp_hdr_fn); board_hdrs{board_idx}.unknown ... = cat(2,board_hdrs{board_idx}.unknown,reshape(hdr.unknown,[1 length(hdr.unknown)])); @@ -216,13 +225,13 @@ = cat(2,board_hdrs{board_idx}.seconds,reshape(hdr.seconds,[1 length(hdr.seconds)])); board_hdrs{board_idx}.fraction ... = cat(2,board_hdrs{board_idx}.fraction,reshape(hdr.fraction,[1 length(hdr.fraction)])); - elseif any(param.config.file.version == [102]) + elseif any(param.records.file.version == [102]) hdr = load(tmp_hdr_fn); board_hdrs{board_idx}.radar_time ... = cat(2,board_hdrs{board_idx}.radar_time,hdr.radar_time); board_hdrs{board_idx}.radar_time_1pps ... = cat(2,board_hdrs{board_idx}.radar_time_1pps,hdr.radar_time_1pps); - elseif any(param.config.file.version == [405 406]) + elseif any(param.records.file.version == [405 406]) hdr = load(tmp_hdr_fn); hdr_log = [hdr_log,hdr.hdr]; hdr_raw = [hdr_raw fn_idx*ones(1,length(hdr.hdr))]; @@ -237,9 +246,9 @@ = cat(2,board_hdrs{board_idx}.seconds,reshape(hdr.seconds,[1 length(hdr.seconds)])); board_hdrs{board_idx}.fraction ... = cat(2,board_hdrs{board_idx}.fraction,reshape(hdr.fraction,[1 length(hdr.fraction)])); - if any(param.config.file.version == [8]) + if any(param.records.file.version == [8]) board_hdrs{board_idx}.waveform_ID = cat(2,board_hdrs{board_idx}.waveform_ID,reshape(hdr.waveform_ID,[1 length(hdr.waveform_ID)])); - elseif any(param.config.file.version == [403 407 408]) + elseif any(param.records.file.version == [403 407 408]) board_hdrs{board_idx}.counter = cat(2,board_hdrs{board_idx}.counter,reshape(hdr.counter,[1 length(hdr.counter)])); end end @@ -266,51 +275,51 @@ end try - if any(param.config.file.version == [1]) - hdr = basic_load_fmcw(fn, struct('file_version',param.config.file.version,'clk',param.config.cresis.clk)); + if any(param.records.file.version == [1]) + hdr = basic_load_fmcw(fn, struct('file_version',param.records.file.version,'clk',param.records.file.clk)); wfs = hdr.wfs; - elseif any(param.config.file.version == [2]) - hdr = basic_load_fmcw2(fn, struct('file_version',param.config.file.version,'clk',param.config.cresis.clk)); + elseif any(param.records.file.version == [2]) + hdr = basic_load_fmcw2(fn, struct('file_version',param.records.file.version,'clk',param.records.file.clk)); wfs = hdr.wfs; - elseif any(param.config.file.version == [4]) - hdr = basic_load_fmcw2(fn, struct('file_version',param.config.file.version,'clk',param.config.cresis.clk)); + elseif any(param.records.file.version == [4]) + hdr = basic_load_fmcw2(fn, struct('file_version',param.records.file.version,'clk',param.records.file.clk)); wfs = hdr.wfs; - elseif any(param.config.file.version == [3 5]) - hdr = basic_load_fmcw3(fn, struct('file_version',param.config.file.version,'clk',param.config.cresis.clk)); + elseif any(param.records.file.version == [3 5]) + hdr = basic_load_fmcw3(fn, struct('file_version',param.records.file.version,'clk',param.records.file.clk)); wfs = struct('presums',hdr.presums); - elseif any(param.config.file.version == [6]) - hdr = basic_load_fmcw4(fn, struct('file_version',param.config.file.version,'clk',param.config.cresis.clk)); + elseif any(param.records.file.version == [6]) + hdr = basic_load_fmcw4(fn, struct('file_version',param.records.file.version,'clk',param.records.file.clk)); wfs = struct('presums',hdr.presums); - elseif any(param.config.file.version == [7 11]) - hdr = basic_load(fn, struct('file_version',param.config.file.version,'clk',param.config.cresis.clk)); + elseif any(param.records.file.version == [7 11]) + hdr = basic_load(fn, struct('file_version',param.records.file.version,'clk',param.records.file.clk)); wfs = hdr.wfs; hdr_param.field_offsets = int32([4 8 12 16]); % epri seconds fractions counter - elseif any(param.config.file.version == [8]) - hdr = basic_load_fmcw8(fn, struct('file_version',param.config.file.version,'clk',param.config.cresis.clk)); + elseif any(param.records.file.version == [8]) + hdr = basic_load_fmcw8(fn, struct('file_version',param.records.file.version,'clk',param.records.file.clk)); wfs = struct('presums',hdr.presums); - elseif any(param.config.file.version == [101]) - hdr = basic_load_accum(fn, struct('file_version',param.config.file.version,'clk',param.config.cresis.clk)); + elseif any(param.records.file.version == [101]) + hdr = basic_load_accum(fn, struct('file_version',param.records.file.version,'clk',param.records.file.clk)); wfs = hdr.wfs; - elseif any(param.config.file.version == [102]) - hdr = basic_load_accum2(fn, struct('file_version',param.config.file.version,'clk',param.config.cresis.clk)); + elseif any(param.records.file.version == [102]) + hdr = basic_load_accum2(fn, struct('file_version',param.records.file.version,'clk',param.records.file.clk)); wfs = hdr.wfs; - elseif any(param.config.file.version == [401]) - hdr = basic_load_mcords(fn, struct('file_version',param.config.file.version,'clk',param.config.cresis.clk)); + elseif any(param.records.file.version == [401]) + hdr = basic_load_mcords(fn, struct('file_version',param.records.file.version,'clk',param.records.file.clk)); wfs = hdr.wfs; - elseif any(param.config.file.version == [402]) - hdr = basic_load_mcords2(fn, struct('file_version',param.config.file.version,'clk',param.config.cresis.clk)); + elseif any(param.records.file.version == [402]) + hdr = basic_load_mcords2(fn, struct('file_version',param.records.file.version,'clk',param.records.file.clk)); wfs = hdr.wfs; - elseif any(param.config.file.version == [403]) - hdr = basic_load_mcords3(fn, struct('file_version',param.config.file.version,'clk',param.config.cresis.clk)); + elseif any(param.records.file.version == [403]) + hdr = basic_load_mcords3(fn, struct('file_version',param.records.file.version,'clk',param.records.file.clk)); wfs = hdr.wfs; - elseif any(param.config.file.version == [404]) - hdr = basic_load_mcords4(fn, struct('file_version',param.config.file.version,'clk',param.config.cresis.clk)); + elseif any(param.records.file.version == [404]) + hdr = basic_load_mcords4(fn, struct('file_version',param.records.file.version,'clk',param.records.file.clk)); wfs = hdr.wfs; - elseif any(param.config.file.version == [405 406]) + elseif any(param.records.file.version == [405 406]) % Load header information that never changes % You need to get the record sizes clear hdr wfs - hdr = basic_load_acords(fn,struct('datatype',0,'file_version',param.config.file.version,'verbose',0)); + hdr = basic_load_acords(fn,struct('datatype',0,'file_version',param.records.file.version,'verbose',0)); for hidx = 1:length(hdr) wfs{1}.num_samp(hidx) = hdr(hidx).num_samp; wfs{2}.num_samp(hidx) = hdr(hidx).num_samp; @@ -334,23 +343,26 @@ wfs{2}.tx_win(hidx) = hdr(hidx).tx_win; wfs{1}.t0(hidx) = hdr(hidx).samp_win_delay; wfs{2}.t0(hidx) = hdr(hidx).samp_win_delay; - if param.config.file.version == 406 + if param.records.file.version == 406 wfs{1}.elem_slots(hidx,:) = [hdr(hidx).elem_1 hdr(hidx).elem_2 hdr(hidx).elem_3 hdr(hidx).elem_4]; wfs{2}.elem_slots(hidx,:) = [hdr(hidx).elem_1 hdr(hidx).elem_2 hdr(hidx).elem_3 hdr(hidx).elem_4]; wfs{1}.blank(hidx) = hdr(hidx).rx_blank_end; wfs{1}.adc_gains(hidx,:) = 10.^((44-hdr(hidx).low_gain_atten).*ones(1,hdr(hidx).num_elem+1)./20); wfs{2}.blank(hidx) = hdr(hidx).hg_blank_end; wfs{2}.adc_gains(hidx,:) = 10.^((80-hdr(hidx).high_gain_atten).*ones(1,hdr(hidx).num_elem+1)./20); - elseif param.config.file.version == 405 + elseif param.records.file.version == 405 wfs{1}.blank(hidx) = hdr(hidx).rx_blank_end; wfs{1}.adc_gains(hidx,:) = 10.^(44-hdr(hidx).low_gain_atten./20); wfs{2}.blank(hidx) = hdr(hidx).hg_blank_end; wfs{2}.adc_gains(hidx,:) = 10.^(80-hdr(hidx).high_gain_atten./20); end end - elseif any(param.config.file.version == [407 408]) + elseif any(param.records.file.version == [407 408]) try - hdr = basic_load_mcords5(fn,struct('presum_bug_fixed',param.config.cresis.presum_bug_fixed)); + if ~isfield(param.records,'presum_mode') + error('param.records.presum_mode must be set. 1 for the old DDS and 0 for the new Arena waveform generator.'); + end + hdr = basic_load_mcords5(fn,struct('presum_mode',param.records.presum_mode)); hdr_param.frame_sync = uint32(hex2dec('1ACFFC1D')); if hdr.file_version == 407 hdr_param.field_offsets = int32([4 16 20 24]); % epri seconds fractions counter @@ -361,11 +373,11 @@ catch ME if 1 fprintf('Warning HACK NOT enabled for mcords5 without frame sync field. Enabling may fix this problem.\n'); - error(ME); + rethrow(ME); else fprintf('Warning HACK enabled for mcords5 without frame sync field\n'); fn_hack = '/mnt/HDD10/1805101801/UWB/chan6/mcords5_06_20180510_112936_00_0000.bin'; - hdr = basic_load_mcords5(fn_hack,struct('presum_bug_fixed',param.config.cresis.presum_bug_fixed)); + hdr = basic_load_mcords5(fn_hack,struct('presum_mode',param.records.presum_mode)); hdr_param.frame_sync = uint32(hex2dec('01600558')); % Used for 20180510 Greenland Polar6 recovery hdr_param.field_offsets = int32([4 16 20 24]-36); % epri seconds fractions counter % Used for 20180511 Greenland Polar6 recovery hdr_param.field_types = {uint32(1) uint32(1) uint32(1) uint64(1)}; @@ -373,6 +385,9 @@ end wfs = hdr.wfs; for wf=1:length(wfs); wfs(wf).file_version = hdr.file_version; end; + elseif any(param.records.file.version == [420]) + hdr = hfrds.basic_load_vapor(fn, struct('file_version',param.records.file.version,'clk',param.config.cresis.clk)); + wfs = hdr.wfs; end catch ME ME.getReport @@ -381,7 +396,7 @@ continue; end - if any(param.config.file.version == [1]) + if any(param.records.file.version == [1]) [file_size offset epri seconds fraction] = basic_load_hdr_mex(fn,hdr_param.frame_sync,hdr_param.field_offsets,hdr_param.field_types,hdr_param.file_mode); % Find bad records by checking their size (i.e. the distance between @@ -399,7 +414,7 @@ save(tmp_hdr_fn,'offset','epri','seconds','fraction','wfs'); - elseif any(param.config.file.version == [2]) + elseif any(param.records.file.version == [2]) [file_size offset epri seconds fraction tmp] = basic_load_hdr_mex(fn,hdr_param.frame_sync,hdr_param.field_offsets,hdr_param.field_types,hdr_param.file_mode); loopback = mod(floor(tmp/2^16),2^2) - 1; nyquist_zone = mod(tmp,2^3) - 1; @@ -421,7 +436,7 @@ save(tmp_hdr_fn,'offset','epri','seconds','fraction','loopback','nyquist_zone','wfs'); - elseif any(param.config.file.version == [4]) + elseif any(param.records.file.version == [4]) [file_size offset epri sec1 sec2 fraction] = basic_load_hdr_mex(fn,hdr_param.frame_sync,hdr_param.field_offsets,hdr_param.field_types,hdr_param.file_mode); % Convert seconds from NMEA ASCII string @@ -446,23 +461,23 @@ save(tmp_hdr_fn,'offset','epri','seconds','fraction','wfs'); - elseif any(param.config.file.version == [3 5 6]) + elseif any(param.records.file.version == [3 5 6]) [file_size offset epri seconds fraction hdr9 hdr10 hdr11] = basic_load_hdr_mex(fn,hdr_param.frame_sync,hdr_param.field_offsets,hdr_param.field_types,hdr_param.file_mode); start_idx = floor(hdr9/2^16); stop_idx = mod(hdr9,2^16); NCO_freq_step = mod(hdr10,2^16); - if param.config.file.version == 6 + if param.records.file.version == 6 nadir_or_sidelooking_select = mod(floor(hdr11/2^24),2^8); else nyquist_zone = mod(floor(hdr11/2^24),2^8); end - if param.config.file.version == 3 + if param.records.file.version == 3 DDC_filter_select = mod(floor(hdr11/2^16),2^8) + 1; else DDC_filter_select = mod(floor(hdr11/2^16),2^8); end - if param.config.file.version == 3 || param.config.file.version == 6 + if param.records.file.version == 3 || param.records.file.version == 6 DDC_or_raw_select = mod(hdr11,2^8); else DDC_or_raw_select = mod(hdr11,2^8); @@ -495,7 +510,7 @@ 'start_idx','stop_idx','DDC_filter_select','DDC_or_raw_select', ... 'num_sam','nyquist_zone','NCO_freq_step'); - elseif any(param.config.file.version == [8]) + elseif any(param.records.file.version == [8]) [file_size offset epri seconds fraction counter nyquist_zone start_idx stop_idx waveform_ID] = basic_load_hdr_mex(fn,hdr_param.frame_sync,hdr_param.field_offsets,hdr_param.field_types,hdr_param.file_mode); HEADER_SIZE = 48; @@ -524,7 +539,7 @@ save(tmp_hdr_fn,'offset','epri','seconds','fraction','wfs', ... 'counter','nyquist_zone','start_idx','stop_idx','waveform_ID'); - elseif any(param.config.file.version == [101]) + elseif any(param.records.file.version == [101]) [file_size offset unknown seconds fraction] = basic_load_hdr_mex(fn,hdr_param.frame_sync,hdr_param.field_offsets,hdr_param.field_types,hdr_param.file_mode); % Find bad records by checking their size (i.e. the distance between @@ -542,21 +557,21 @@ save(tmp_hdr_fn,'offset','unknown','seconds','fraction'); - elseif any(param.config.file.version == [102]) + elseif any(param.records.file.version == [102]) [file_size offset radar_time_ms radar_time_ls radar_time_1pps_ms radar_time_1pps_ls] ... = basic_load_hdr_mex(fn,hdr_param.frame_sync,hdr_param.field_offsets,hdr_param.field_types,hdr_param.file_mode); - radar_time = (hdr_data(3,:)*2^32 + hdr_data(4,:)) / (param.config.cresis.clk/100); - radar_time_1pps = (hdr_data(5,:)*2^32 + hdr_data(6,:)) / (param.config.cresis.clk/100); + radar_time = (hdr_data(3,:)*2^32 + hdr_data(4,:)) / (param.records.file.clk/100); + radar_time_1pps = (hdr_data(5,:)*2^32 + hdr_data(6,:)) / (param.records.file.clk/100); save(tmp_hdr_fn,'offset','radar_time','radar_time_1pps','wfs'); - elseif any(param.config.file.version == [401]) + elseif any(param.records.file.version == [401]) error('Not supported'); - elseif any(param.config.file.version == [402]) + elseif any(param.records.file.version == [402]) error('Not supported'); - elseif any(param.config.file.version == [403]) + elseif any(param.records.file.version == [403]) [file_size offset epri seconds fraction counter] = basic_load_hdr_mex(fn,hdr_param.frame_sync,hdr_param.field_offsets,hdr_param.field_types,hdr_param.file_mode); seconds = BCD_to_seconds(seconds); @@ -579,7 +594,7 @@ save(tmp_hdr_fn,'offset','epri','seconds','fraction','counter','wfs'); - elseif any(param.config.file.version == [404]) + elseif any(param.records.file.version == [404]) [file_size offset epri seconds fraction counter] = basic_load_hdr_mex(fn,hdr_param.frame_sync,hdr_param.field_offsets,hdr_param.field_types,hdr_param.file_mode); seconds = BCD_to_seconds(seconds); @@ -602,7 +617,7 @@ save(tmp_hdr_fn,'offset','epri','seconds','fraction','counter','wfs'); - elseif any(param.config.file.version == [405 406]) + elseif any(param.records.file.version == [405 406]) % Load header information that can change on every record AND % is required for records generation (radar time) % radar_time = @@ -610,20 +625,20 @@ offset = []; seconds = []; % Get header timestamps and offsets - [hdr htime hoffset] = basic_load_acords(fn,struct('datatype',0,'file_version',param.config.file.version,'verbose',0)); + [hdr htime hoffset] = basic_load_acords(fn,struct('datatype',0,'file_version',param.records.file.version,'verbose',0)); raw_file_time = htime(1); % Get data records timestamps and offsets - [data seconds offset] = basic_load_acords(fn,struct('datatype',2,'file_version',param.config.file.version,'verbose',0)); + [data seconds offset] = basic_load_acords(fn,struct('datatype',2,'file_version',param.records.file.version,'verbose',0)); fractions = zeros(size(seconds)); save(tmp_hdr_fn,'offset','seconds','hdr','hoffset','htime','wfs','raw_file_time'); - elseif any(param.config.file.version == [7 11 407 408]) + elseif any(param.records.file.version == [7 11 407 408]) hdr_param.field_types = {uint32(1) uint32(1) uint32(1) uint64(1)}; [file_size offset epri seconds fraction counter] = basic_load_hdr_mex(fn,hdr_param.frame_sync,hdr_param.field_offsets,hdr_param.field_types,hdr_param.file_mode); seconds = BCD_to_seconds(seconds); % Find bad records by checking their size - if any(param.config.file.version == [407 408]) + if any(param.records.file.version == [407 408]) % The distance between frame syncs should be constant expected_rec_size = median(diff(offset)); meas_rec_size = diff(offset); @@ -633,7 +648,7 @@ % the next file to see if the complete record is there) bad_mask(end+1) = false; - elseif any(param.config.file.version == [7 11]) + elseif any(param.records.file.version == [7 11]) % User must supply the valid record sizes if ~isfield(param.config.cresis,'expected_rec_sizes') || isempty(param.config.cresis.expected_rec_sizes) fprintf('Record sizes found in this file:\n'); @@ -661,11 +676,29 @@ counter = double(counter(~bad_mask)); save(tmp_hdr_fn,'offset','epri','seconds','fraction','counter','wfs'); + + elseif any(param.records.file.version == [420]) + [file_size offset epri seconds fraction] = basic_load_hdr_mex(fn,hdr_param.frame_sync,hdr_param.field_offsets,hdr_param.field_types,hdr_param.file_mode); + seconds = BCD_to_seconds(seconds,1); + + % Find bad records by checking their size (i.e. the distance between + % frame syncs which should be constant). + expected_rec_size = median(diff(offset)); + meas_rec_size = diff(offset); + bad_mask = meas_rec_size ~= expected_rec_size; + bad_mask(end+1) = file_size < offset(end) + expected_rec_size; + + % Remove bad records (i.e. ones with sizes that are not expected + offset = double(offset(~bad_mask)); + epri = double(epri(~bad_mask)); + seconds = double(seconds(~bad_mask)); + fraction = double(fraction(~bad_mask)); + save(tmp_hdr_fn,'offset','epri','seconds','fraction','wfs'); end % Load and concatenate temporary file - if any(param.config.file.version == [101]) + if any(param.records.file.version == [101]) hdr = load(tmp_hdr_fn); board_hdrs{board_idx}.unknown ... = cat(2,board_hdrs{board_idx}.unknown,reshape(hdr.unknown,[1 length(hdr.unknown)])); @@ -673,13 +706,13 @@ = cat(2,board_hdrs{board_idx}.seconds,reshape(hdr.seconds,[1 length(hdr.seconds)])); board_hdrs{board_idx}.fraction ... = cat(2,board_hdrs{board_idx}.fraction,reshape(hdr.fraction,[1 length(hdr.fraction)])); - elseif any(param.config.file.version == [102]) + elseif any(param.records.file.version == [102]) hdr = load(tmp_hdr_fn); board_hdrs{board_idx}.radar_time ... = cat(2,board_hdrs{board_idx}.radar_time,hdr.radar_time); board_hdrs{board_idx}.radar_time_1pps ... = cat(2,board_hdrs{board_idx}.radar_time_1pps,hdr.radar_time_1pps); - elseif any(param.config.file.version == [405 406]) + elseif any(param.records.file.version == [405 406]) hdr = load(tmp_hdr_fn); hdr_log = [hdr_log,hdr.hdr]; hdr_raw = [hdr_raw fn_idx*ones(1,length(hdr.hdr))]; @@ -694,9 +727,9 @@ = cat(2,board_hdrs{board_idx}.seconds,reshape(hdr.seconds,[1 length(hdr.seconds)])); board_hdrs{board_idx}.fraction ... = cat(2,board_hdrs{board_idx}.fraction,reshape(hdr.fraction,[1 length(hdr.fraction)])); - if any(param.config.file.version == [8]) + if any(param.records.file.version == [8]) board_hdrs{board_idx}.waveform_ID = cat(2,board_hdrs{board_idx}.waveform_ID,reshape(hdr.waveform_ID,[1 length(hdr.waveform_ID)])); - elseif any(param.config.file.version == [403 407 408]) + elseif any(param.records.file.version == [403 407 408]) board_hdrs{board_idx}.counter = cat(2,board_hdrs{board_idx}.counter,reshape(hdr.counter,[1 length(hdr.counter)])); end end @@ -714,9 +747,9 @@ %% List bad files % ========================================================================= -for board_idx = 1:numel(param.config.board_map) +for board_idx = 1:numel(param.records.file.boards) if any(failed_load{board_idx}) - warning('Some files failed to load, consider deleting these to avoid problems.'); + warning('Some files failed to load, consider deleting these to avoid problems:'); for fn_idx = find(failed_load{board_idx}) fprintf(' %s\n', fns_list{board_idx}{fn_idx}); end @@ -726,7 +759,7 @@ %% waveform_ID Debug % ========================================================================= if 0 - % waveform_ID decoding for: any(param.config.file.version == [8]) + % waveform_ID decoding for: any(param.records.file.version == [8]) keyboard waveform_ID_unique = unique(waveform_ID) waveform_ID_char = char(reshape(typecast(waveform_ID_unique,'int8'),[8 length(waveform_ID_unique)]).') @@ -786,7 +819,7 @@ % ========================================================================= if param.config.online_mode - for board_idx = 1:numel(param.config.board_map) + for board_idx = 1:numel(param.records.file.boards) epri_jumps = diff(double(board_hdrs{board_idx}.epri)); fprintf('Board %d: List of up to 10 last EPRI jumps of >100 records:\n', board_idx); bad_jumps = epri_jumps(abs(epri_jumps) > 100); @@ -794,7 +827,7 @@ fprintf(' %.0f', bad_jumps(max(1,end-9):end)); fprintf(' record jumps\n'); - utc_time_sod = double(board_hdrs{board_idx}.seconds) + double(board_hdrs{board_idx}.fraction) / param.config.cresis.clk; + utc_time_sod = double(board_hdrs{board_idx}.seconds) + double(board_hdrs{board_idx}.fraction) / param.records.file.clk; fprintf('Board %d: UTC time SOD jumps of >0.5 sec:\n', board_idx); utc_time_sod_jumps = diff(utc_time_sod); bad_jumps = utc_time_sod_jumps(abs(utc_time_sod_jumps) > 0.5); @@ -808,10 +841,10 @@ %% Correct time variable % ========================================================================= -for board_idx = 1:numel(param.config.board_map) +for board_idx = 1:numel(param.records.file.boards) - if any(param.config.file.version == [1 2 3 4 5 6 7 8 11 101 403 404 407 408]) - utc_time_sod = double(board_hdrs{board_idx}.seconds) + double(board_hdrs{board_idx}.fraction) / param.config.cresis.clk; + if any(param.records.file.version == [1 2 3 4 5 6 7 8 11 101 403 404 407 408 420]) + utc_time_sod = double(board_hdrs{board_idx}.seconds) + double(board_hdrs{board_idx}.fraction) / param.records.file.clk; epri = double(board_hdrs{board_idx}.epri); if 0 @@ -839,6 +872,8 @@ % 1: EPRI good % 2: Time-generated EPRI good % 3: EPRI and time-generated EPRI good + % "EPRI good" means that depri == 1 + % "Time-generated EPRI good" means 0.9 < dtime_epri < 1.1 dtime_epri_threshold = 0.1; % Allow for 10% PRI error mask = (depri == 1) + (2*(abs(dtime_epri-1) < dtime_epri_threshold)); % If the EPRI's both indicate the same number of skipped records, @@ -898,7 +933,7 @@ set(h_fig(1),'Name',sprintf('%d: UTC Time',h_fig(1).Number)); plot(h_axes,utc_time_sod); hold(h_axes,'on'); - plot(h_axes,utc_time_sod_new,'r'); + plot(h_axes,find(good_out_mask), utc_time_sod_new(good_out_mask),'r.'); hold(h_axes,'off'); xlabel(h_axes,'Record number'); ylabel(h_axes,'UTC Time SOD (sec)'); @@ -909,7 +944,7 @@ clf(h_fig(2)); h_axes = axes('parent',h_fig(2)); set(h_fig(2),'NumberTitle','off'); set(h_fig(2),'Name',sprintf('%d: Time Correction',h_fig(1).Number)); - plot(h_axes,utc_time_sod - utc_time_sod_new); + plot(h_axes,find(good_out_mask), utc_time_sod(good_out_mask) - utc_time_sod_new(good_out_mask),'.'); xlabel(h_axes,'Record number'); ylabel(h_axes,'Time correction (sec)'); ylim(h_axes,[-UTC_MAX_ERROR UTC_MAX_ERROR]); @@ -952,6 +987,8 @@ saveas(h_fig(3),fn_fig); utc_time_sod = utc_time_sod_new; + utc_time_sod(~good_out_mask) = NaN; + utc_time_sod = interp_finite(utc_time_sod,NaN); % Check for day wraps in the UTC time seconds of day day_wrap_idxs = find(diff(utc_time_sod) < -50000); @@ -969,7 +1006,7 @@ board_hdrs{board_idx}.utc_time_sod = utc_time_sod; board_hdrs{board_idx}.epri = epri; - elseif any(param.config.file.version == [405 406]) + elseif any(param.records.file.version == [405 406]) utc_time_sod = board_hdrs{board_idx}.seconds; % this is actually comp_time but doesn't need to % be converted to actual utc_time_sod since it's only looking at gaps in % the data @@ -995,9 +1032,9 @@ hold off; names = fieldnames(hdr_log(1)); - if param.config.file.version == 406 + if param.records.file.version == 406 change_fields = [2 5 6 7 8 9 10 11 12 17 18 19 20 21]; - elseif param.config.file.version == 405 + elseif param.records.file.version == 405 change_fields = [2 5 6 7 8 9 10 11 12]; end @@ -1082,23 +1119,23 @@ %% Create Segments % ========================================================================= -if any(param.config.file.version == [403 404 407 408]) +if any(param.records.file.version == [403 404 407 408]) %% Create Segments: Read XML settings % NI XML settings files available, break segments based on settings files % and header information - xml_version = param.config.daq.xml_version; + xml_version = param.config.cresis.config_version; cresis_xml_mapping; - settings_fn_dir = fullfile(param.config.base_dir,param.config.config_folder_name); + settings_fn_dir = fullfile(param.config.base_dir,param.config.config_folder_names); fprintf('\nSettings Directory: %s\n\n', settings_fn_dir); % Read XML files in this directory - [settings,settings_enc] = read_ni_xml_directory(settings_fn_dir,xml_file_prefix,false); + [settings,~] = read_ni_xml_directory(settings_fn_dir,xml_file_prefix,false); % Get the date information out of the filename fn_datenums = {}; - for board_idx = 1:numel(param.config.board_map) + for board_idx = 1:numel(param.records.file.boards) fn_datenums{board_idx} = []; for data_fn_idx = 1:length(fns_list{board_idx}) fname = fname_info_mcords2(fns_list{board_idx}{data_fn_idx}); @@ -1108,6 +1145,7 @@ %% Create Segments: Print settings oparams = {}; + [~,defaults] = param.config.default(); for set_idx = 1:length(settings) % Print out settings [~,settings_fn_name] = fileparts(settings(set_idx).fn); @@ -1128,7 +1166,7 @@ fprintf(' WF %d Len:', wf); fprintf(' %.1f us', 1e6*settings(set_idx).(config_var).Base_Len*settings(set_idx).(config_var).Waveforms(wf).Len_Mult); fprintf('\n'); end - for board_idx = 1:numel(param.config.board_map) + for board_idx = 1:numel(param.records.file.boards) if set_idx < length(settings) settings(set_idx).file_matches{board_idx} = find(fn_datenums{board_idx} >= settings(set_idx).datenum & fn_datenums{board_idx} < settings(set_idx+1).datenum); else @@ -1137,27 +1175,28 @@ end % Associate default parameters with each settings - default = default_radar_params_settings_match(param.config.defaults,settings(set_idx)); + default = default_radar_params_settings_match(defaults,settings(set_idx)); + default = merge_structs(param,default); oparams{end+1} = default; - oparams{end} = rmfield(oparams{end},'config_regexp'); - oparams{end} = rmfield(oparams{end},'name'); + try; oparams{end} = rmfield(oparams{end},'config_regexp'); end; + try; oparams{end} = rmfield(oparams{end},'name'); end; % Parameter spreadsheet % ======================================================================= oparams{end}.cmd.notes = default.name; oparams{end}.records.file.base_dir = param.config.base_dir; - oparams{end}.records.file.board_folder_name = param.config.board_folder_name; + oparams{end}.records.file.board_folder_name = param.config.board_folder_names; if ~isempty(oparams{end}.records.file.board_folder_name) ... && oparams{end}.records.file.board_folder_name(1) ~= filesep % Ensures that board_folder_name is not a text number which Excel % will misinterpret as a numeric type oparams{end}.records.file.board_folder_name = ['/' oparams{end}.records.file.board_folder_name]; end - oparams{end}.records.file.boards = param.config.board_map; - oparams{end}.records.file.version = param.config.file.version; - oparams{end}.records.file.prefix = param.config.file.prefix; - oparams{end}.records.file.clk = param.config.cresis.clk; + oparams{end}.records.file.boards = param.records.file.boards; + oparams{end}.records.file.version = param.records.file.version; + oparams{end}.records.file.prefix = param.records.file.prefix; + oparams{end}.records.file.clk = param.records.file.clk; oparams{end}.radar.prf = settings(set_idx).(config_var).(prf_var); % Usually the default.radar.wfs structure only has one waveform @@ -1172,7 +1211,7 @@ oparams{end}.radar.wfs(wf).f1 = settings(set_idx).(config_var).Waveforms(wf).Stop_Freq(1); oparams{end}.radar.wfs(wf).tukey = settings(set_idx).(config_var).RAM_Taper; % Transmit weights - if any(param.config.file.version == [403 407 408]) + if any(param.records.file.version == [403 407 408]) tx_mask_inv = fliplr(~(dec2bin(double(settings(set_idx).(config_var).Waveforms(wf).TX_Mask),8) - '0')); tx_weights = double(settings(set_idx).(config_var).(ram_var)) .* tx_mask_inv ./ param.config.max_tx.*param.config.max_tx_voltage; else @@ -1186,11 +1225,15 @@ % ADC Gains atten = double(settings(set_idx).(config_var).Waveforms(wf).Attenuator_1(1)) ... + double(settings(set_idx).(config_var).Waveforms(wf).Attenuator_2(1)); - oparams{end}.radar.wfs(wf).adc_gains_dB = param.config.cresis.rx_gain_dB - atten(1)*ones(1,length(oparams{end}.radar.wfs(wf).rx_paths)); + if isfield(oparams{end}.radar.wfs(wf),'rx_paths') + oparams{end}.radar.wfs(wf).adc_gains_dB = param.config.cresis.rx_gain_dB - atten(1)*ones(1,length(oparams{end}.radar.wfs(wf).rx_paths)); + else + oparams{end}.radar.wfs(wf).adc_gains_dB = param.config.cresis.rx_gain_dB - atten(1)*ones(1,length(oparams{end}.radar.rx_paths)); + end % DDC mode and frequency if isfield(settings(set_idx), 'DDC_Ctrl') - oparams{end}.radar.wfs(wf).DDC_dec = 2^(2+settings(set_idx).DDC_Ctrl.DDC_sel.Val); + oparams{end}.radar.wfs(wf).DDC_dec = 2^(1+settings(set_idx).DDC_Ctrl.DDC_sel.Val); oparams{end}.radar.wfs(wf).DDC_freq = settings(set_idx).DDC_Ctrl.(NCO_freq)*1e6; else oparams{end}.radar.wfs(wf).DDC_dec = 1; @@ -1200,7 +1243,7 @@ counters = {}; file_idxs = {}; - for board_idx = 1:numel(param.config.board_map) + for board_idx = 1:numel(param.records.file.boards) counters{board_idx} = double(board_hdrs{board_idx}.(param.config.field_time_gap)); file_idxs{board_idx} = board_hdrs{board_idx}.file_idxs; day_wrap_offset{board_idx} = board_hdrs{board_idx}.day_wrap_offset; @@ -1216,7 +1259,7 @@ day_wrap_offset{board_idx} = day_wrap_offset{board_idx}(mask); end end - [segs,stats] = create_segments(counters,file_idxs,day_wrap_offset,param.config.max_time_gap); + [segs,stats] = preprocess_create_segments(counters,file_idxs,day_wrap_offset,param.config.max_time_gap,param.config.segment_end_file_trim); if 1 % Debug: Test Code @@ -1252,12 +1295,12 @@ oparams{end}.day_seg = sprintf('%s_%02d',param.config.date_str,length(oparams)); oparams{end}.records.file.start_idx = segment.start_idxs; oparams{end}.records.file.stop_idx = segment.stop_idxs; - oparams{end}.records.gps.time_offset = default.records.gps.time_offset + segment.day_wrap_offset; + oparams{end}.records.gps.time_offset = param.records.gps.time_offset + segment.day_wrap_offset; end end end -elseif any(param.config.file.version == [410]) +elseif any(param.records.file.version == [410]) % MCRDS headers available, break segments based on filenames else @@ -1266,12 +1309,12 @@ % param.config.max_time_gap determine which field and gap size to use). counters = {}; file_idxs = {}; - for board_idx = 1:numel(param.config.board_map) + for board_idx = 1:numel(param.records.file.boards) counters{board_idx} = double(board_hdrs{board_idx}.(param.config.field_time_gap)); file_idxs{board_idx} = board_hdrs{board_idx}.file_idxs; day_wrap_offset{board_idx} = board_hdrs{board_idx}.day_wrap_offset; end - [segs,stats] = create_segments(counters,file_idxs,day_wrap_offset,param.config.max_time_gap); + [segs,stats] = preprocess_create_segments(counters,file_idxs,day_wrap_offset,param.config.max_time_gap); if 1 % Debug: Test Code @@ -1298,27 +1341,29 @@ % Create the parameters to output oparams = {}; + [~,defaults] = param.config.default(); for segment_idx = 1:length(segs) segment = segs(segment_idx); % Determine which default parameters to use % ======================================================================= match_idx = 1; - oparams{end+1} = param.config.defaults{match_idx}; + + oparams{end+1} = defaults{match_idx}; oparams{end} = rmfield(oparams{end},'config_regexp'); oparams{end} = rmfield(oparams{end},'name'); % Parameter spreadsheet % ======================================================================= oparams{end}.day_seg = sprintf('%s_%02d',param.config.date_str,segment_idx); - oparams{end}.cmd.notes = param.config.defaults{match_idx}.name; + oparams{end}.cmd.notes = defaults{match_idx}.name; oparams{end}.records.file.start_idx = segment.start_idxs; oparams{end}.records.file.stop_idx = segment.stop_idxs; - oparams{end}.records.gps.time_offset = param.config.defaults{match_idx}.records.gps.time_offset + segment.day_wrap_offset; + oparams{end}.records.gps.time_offset = param.records.gps.time_offset + segment.day_wrap_offset; oparams{end}.records.file.base_dir = param.config.base_dir; - oparams{end}.records.file.board_folder_name = param.config.board_folder_name; + oparams{end}.records.file.board_folder_name = param.config.board_folder_names; if ~isempty(oparams{end}.records.file.board_folder_name) ... && oparams{end}.records.file.board_folder_name(1) ~= filesep % Ensures that board_folder_name is not a text number which Excel @@ -1328,17 +1373,19 @@ if ~isnan(str2double(oparams{end}.records.file.board_folder_name)) oparams{end}.records.file.board_folder_name = ['/' oparams{end}.records.file.board_folder_name]; end - oparams{end}.records.file.boards = param.config.board_map; - oparams{end}.records.file.version = param.config.file.version; - oparams{end}.records.file.prefix = param.config.file.prefix; - oparams{end}.records.file.clk = param.config.cresis.clk; + oparams{end}.records.file.boards = param.records.file.boards; + oparams{end}.records.file.version = param.records.file.version; + oparams{end}.records.file.prefix = param.records.file.prefix; + oparams{end}.records.file.clk = param.records.file.clk; end end %% Print out segments % ========================================================================= -if ~isempty(param.config.param_fn) +if ~exist(param.config.param_fn,'file') + warning('Could not find parameter spreadsheet file so not printing parameter values.\n param.config.param_fn = %s', param.config.param_fn); +else % Print parameter spreadsheet values to stdout and param_txt_fn % ========================================================================= fid = 1; @@ -1356,8 +1403,16 @@ read_param_xls_print(param.config.param_fn,'radar',oparams,fid); fprintf(fid,'%s\n','='*ones(1,80)); fprintf(fid,' post\n'); fprintf(fid,'%s\n','='*ones(1,80)); read_param_xls_print(param.config.param_fn,'post',oparams,fid); - fprintf(fid,'%s\n','='*ones(1,80)); fprintf(fid,' analysis\n'); fprintf(fid,'%s\n','='*ones(1,80)); - read_param_xls_print(param.config.param_fn,'analysis',oparams,fid); + % Other sheets + warning off MATLAB:xlsfinfo:ActiveX + [status, sheets] = xlsfinfo(param.config.param_fn); + warning on MATLAB:xlsfinfo:ActiveX + for sheet_idx = 1:length(sheets) + if ~any(strcmpi(sheets{sheet_idx},{'cmd','records','qlook','sar','array','radar','post'})) + fprintf(fid,'%s\n','='*ones(1,80)); fprintf(fid,' %s\n', sheets{sheet_idx}); fprintf(fid,'%s\n','='*ones(1,80)); + read_param_xls_print(param.config.param_fn,sheets{sheet_idx},oparams,fid); + end + end fprintf(fid,'\n'); param_txt_fn = ct_filename_ct_tmp(param,'','param', [param.config.date_str,'.txt']); @@ -1391,8 +1446,16 @@ fprintf(fid,'%s\n','='*ones(1,80)); fprintf(fid,' post\n'); fprintf(fid,'%s\n','='*ones(1,80)); read_param_xls_print(param.config.param_fn,'post',oparams,fid); fprintf(fid,'\n'); - fprintf(fid,'%s\n','='*ones(1,80)); fprintf(fid,' analysis\n'); fprintf(fid,'%s\n','='*ones(1,80)); - read_param_xls_print(param.config.param_fn,'analysis',oparams,fid); + % Other sheets + warning off MATLAB:xlsfinfo:ActiveX + [status, sheets] = xlsfinfo(param.config.param_fn); + warning on MATLAB:xlsfinfo:ActiveX + for sheet_idx = 1:length(sheets) + if ~any(strcmpi(sheets{sheet_idx},{'cmd','records','qlook','sar','array','radar','post'})) + fprintf(fid,'%s\n','='*ones(1,80)); fprintf(fid,' %s\n', sheets{sheet_idx}); fprintf(fid,'%s\n','='*ones(1,80)); + read_param_xls_print(param.config.param_fn,sheets{sheet_idx},oparams,fid); + end + end fprintf(fid,'\n'); fclose(fid); end diff --git a/cresis-toolbox/ct_support/preprocess_task_utig.m b/cresis-toolbox/ct_support/preprocess_task_utig.m new file mode 100644 index 00000000..071f1d07 --- /dev/null +++ b/cresis-toolbox/ct_support/preprocess_task_utig.m @@ -0,0 +1,321 @@ +function success = preprocess_task_utig(param) +% success = preprocess_task_utig(param) +% +% Strips data out of utig packets and creates: +% 1. Files of just radar data +% 2. Files with just header data +% a. binary flat file .hdr +% b. Matlab .mat format (if supported) +% NOTE: This script only works on files with fixed header lengths. More +% specifically, every header must have the payload length field in the same spot. +% +% Example: +% Called from run_utig_packet_strip.m +% +% Author: John Paden +% +% See also: basic_load_utig.m, run_utig_packet_strip.m, +% utig_packet_strip.m, utig_packet_strip_task.m + +% Pull in the inputs from param struct +base_dir = fullfile(param.config.base_dir); +config_folder_name = fullfile(param.config.config_folder_names); +reuse_tmp_files = param.config.reuse_tmp_files; + +[~,defaults] = param.config.default(); + +%% Read each config/system XML file pair into a configs structure +% ========================================================================= + +% Not applicable yet + +%% Process each file of data +% ========================================================================= + +board = param.config.board_map(1); + +board_folder_name = fullfile(param.config.board_folder_names); +config_folder_name = fullfile(param.config.config_folder_names); + +% Replace all "%b" in board_folder_name with the board number +board_folder_name = regexprep(board_folder_name,'%b',board); + +% ========================================================================= +%% Get Data File list for this board +get_filenames_param = struct('regexp',param.config.file.regexp); +fns = get_filenames(fullfile(param.config.base_dir,board_folder_name), ... + param.config.file.prefix, param.config.file.midfix, ... + param.config.file.suffix, get_filenames_param); +fns_datenum = zeros(size(fns)); +for fn_idx = 1:length(fns) + config_fname_info = fname_info_utig(fns{fn_idx}); + fns_datenum(fn_idx) = config_fname_info.datenum; +end + +%% Iterate packet_strip through file list +old_fn_dir = []; +board_hdrs{1}.radar_time = []; +board_hdrs{1}.comp_time = []; +board_hdrs{1}.file_idxs = []; +for fn_idx = 1:length(fns) + fn = fullfile(fns{fn_idx}); + + [fn_dir,fn_name] = fileparts(fn); + if ~strcmpi(fn_dir,old_fn_dir) + % New data directory: assume that this is from a different utig 313 + % board and state vectors should be reset. + last_bytes_m = []; + last_bytes = zeros(64,1,'uint8'); + last_bytes_len = int32(0); + num_expected = int32(-1); + pkt_counter = int32(-1); + old_fn_dir = fn_dir; + end + + % Create output filenames + out_fn = ct_filename_ct_tmp(param,'','headers', ... + fullfile(board_folder_name, fn_name)); + [out_fn_dir,out_fn_name] = fileparts(out_fn); + out_fn = fullfile(out_fn_dir,[out_fn_name,'.dat']); + out_hdr_fn = fullfile(out_fn_dir,[out_fn_name,'.mat']); + + % Print status + fprintf('utig_pkt_strip %d/%d %s (%s)\n %s\n', fn_idx, ... + length(fns), fn, datestr(now), out_fn); + + % Check to make sure output directory exists + if ~exist(out_fn_dir,'dir') + mkdir(out_fn_dir); + end + + % Copy GPS (UTIG ELSA) Files + if fn_idx == 1 + out_config_fn_dir = fullfile(fn_dir,config_folder_name); + gps_files = fullfile(out_config_fn_dir,'serial*'); + out_log_dir = fullfile(param.data_support_path, param.season_name, param.config.date_str, 'ELSA'); + try + fprintf('Copy %s\n %s\n', gps_files, out_log_dir); + if ~exist(out_log_dir,'dir') + mkdir(out_log_dir) + end + copyfile(gps_files, out_log_dir); + catch ME + warning('Error while copying log files:\n%s\n', ME.getReport); + end + end + + finfo = fname_info_utig(fn); + + % Check to see if outputs already exist + if reuse_tmp_files && exist(out_hdr_fn,'file') + load(out_hdr_fn,'radar_time','comp_time'); + board_hdrs{1}.radar_time(end+(1:length(radar_time))) = radar_time; + board_hdrs{1}.comp_time(end+(1:length(radar_time))) = comp_time; + %board_hdrs{1}.file_idxs(end+(1:length(radar_time))) = finfo.file_idx*ones([1 length(radar_time)]); + board_hdrs{1}.file_idxs(end+(1:length(radar_time))) = fn_idx*ones([1 length(radar_time)]); + continue; + end + + hdr = basic_load_utig(fn); + if isempty(hdr) + continue; + end + + %% Write header output file + radar_time = hdr{1}.ct_time; + comp_time = hdr{1}.ct_clk; + offset = zeros(size(hdr{1}.offset)); + for idx = 1:length(hdr{1}.offset) + if hdr{1}.rseq(idx) ~= hdr{3}.rseq(idx) + keyboard + end + offset(idx) = min(hdr{1}.offset(idx),hdr{3}.offset(idx)); + end + + for chan=1:length(hdr) + pri{chan}.rseq = hdr{chan}.rseq; + end + + save(out_hdr_fn, 'offset', 'radar_time', 'comp_time'); + + board_hdrs{1}.radar_time(end+(1:length(radar_time))) = radar_time; + board_hdrs{1}.comp_time(end+(1:length(radar_time))) = comp_time; + %board_hdrs{1}.file_idxs(end+(1:length(radar_time))) = finfo.file_idx*ones([1 length(radar_time)]); + board_hdrs{1}.file_idxs(end+(1:length(radar_time))) = fn_idx*ones([1 length(radar_time)]); + + if 0 + %% Debug outputs + % load(out_tmp_fn); + end + +end + +% Convert comp_time from Matlab's datenum format to ANSI C seconds since +% Jan 1 1970 epoch +board_hdrs{1}.comp_time = datenum_to_epoch(board_hdrs{1}.comp_time); + +% No heading information, break segments based on time, epri, or radar +% counter information (param.config.field_time_gap and +% param.config.max_time_gap determine which field and gap size to use). +counters = {}; +file_idxs = {}; +param.config.field_time_gap = 'comp_time'; +for board_idx = 1:numel(param.records.file.boards) + counters{board_idx} = double(board_hdrs{board_idx}.(param.config.field_time_gap)); + file_idxs{board_idx} = board_hdrs{board_idx}.file_idxs; + day_wrap_offset{board_idx} = zeros(size(board_hdrs{board_idx}.file_idxs)) +end +[segs,stats] = preprocess_create_segments(counters,file_idxs,day_wrap_offset,param.config.max_time_gap); + +if 1 + % Debug: Test Code + for seg_idx = 1:length(segs) + fprintf('Segment %d\n', seg_idx); + disp(segs(seg_idx)) + end + + fprintf('On time: %g\n', sum(stats.on_time)); + fprintf('Seg\tOn%%\tOn'); + for board_idx = 1:size(stats.board_time,2) + fprintf('\t%d%%\t%d', board_idx, board_idx); + end + fprintf('\n'); + + for seg_idx = 1:length(segs) + fprintf('%d\t%.0f%%\t%.1g', seg_idx, stats.on_time(seg_idx)/sum(stats.on_time)*100, stats.on_time(seg_idx)); + for board_idx = 1:size(stats.board_time,2) + fprintf('\t%.0f%%\t%.1g', stats.board_time(seg_idx,board_idx)/stats.on_time(seg_idx)*100, stats.board_time(seg_idx,board_idx)); + end + fprintf('\n'); + end +end + +% Create the parameters to output +oparams = {}; +for segment_idx = 1:length(segs) + segment = segs(segment_idx); + + % Determine which default parameters to use + % ======================================================================= + match_idx = 1; + oparams{end+1} = merge_structs(param,defaults{match_idx}); + oparams{end} = rmfield(oparams{end},'config_regexp'); + oparams{end} = rmfield(oparams{end},'name'); + + % Parameter spreadsheet + % ======================================================================= + oparams{end}.day_seg = sprintf('%s_%02d',param.config.date_str,segment_idx); + oparams{end}.cmd.notes = defaults{match_idx}.name; + + oparams{end}.records.file.start_idx = segment.start_idxs; + oparams{end}.records.file.stop_idx = segment.stop_idxs; + defaults{match_idx}.records.gps.time_offset = 0; + oparams{end}.records.gps.time_offset = defaults{match_idx}.records.gps.time_offset + segment.day_wrap_offset; + + oparams{end}.records.file.base_dir = param.config.base_dir; + oparams{end}.records.file.board_folder_name = param.config.board_folder_names; + if ~isempty(oparams{end}.records.file.board_folder_name) ... + && oparams{end}.records.file.board_folder_name(1) ~= filesep + % Ensures that board_folder_name is not a text number which Excel + % will misinterpret as a numeric type + oparams{end}.records.file.board_folder_name = ['/' oparams{end}.records.file.board_folder_name]; + end + if ~isnan(str2double(oparams{end}.records.file.board_folder_name)) + oparams{end}.records.file.board_folder_name = ['/' oparams{end}.records.file.board_folder_name]; + end + oparams{end}.records.file.boards = param.records.file.boards; + oparams{end}.records.file.version = param.records.file.version; + oparams{end}.records.file.prefix = param.records.file.prefix; + oparams{end}.records.file.clk = param.records.file.clk; +end + +%% Print out segments +% ========================================================================= +if ~isempty(param.config.param_fn) + % Print parameter spreadsheet values to stdout and param_txt_fn + % ========================================================================= + fid = 1; + fprintf(fid,'%s\n','='*ones(1,80)); fprintf(fid,' cmd\n'); fprintf(fid,'%s\n','='*ones(1,80)); + read_param_xls_print(param.config.param_fn,'cmd',oparams,fid); + fprintf(fid,'%s\n','='*ones(1,80)); fprintf(fid,' records\n'); fprintf(fid,'%s\n','='*ones(1,80)); + read_param_xls_print(param.config.param_fn,'records',oparams,fid); + fprintf(fid,'%s\n','='*ones(1,80)); fprintf(fid,' qlook\n'); fprintf(fid,'%s\n','='*ones(1,80)); + read_param_xls_print(param.config.param_fn,'qlook',oparams,fid); + fprintf(fid,'%s\n','='*ones(1,80)); fprintf(fid,' sar\n'); fprintf(fid,'%s\n','='*ones(1,80)); + read_param_xls_print(param.config.param_fn,'sar',oparams,fid); + fprintf(fid,'%s\n','='*ones(1,80)); fprintf(fid,' array\n'); fprintf(fid,'%s\n','='*ones(1,80)); + read_param_xls_print(param.config.param_fn,'array',oparams,fid); + fprintf(fid,'%s\n','='*ones(1,80)); fprintf(fid,' radar\n'); fprintf(fid,'%s\n','='*ones(1,80)); + read_param_xls_print(param.config.param_fn,'radar',oparams,fid); + fprintf(fid,'%s\n','='*ones(1,80)); fprintf(fid,' post\n'); fprintf(fid,'%s\n','='*ones(1,80)); + read_param_xls_print(param.config.param_fn,'post',oparams,fid); + % Other sheets + try + warning off MATLAB:xlsfinfo:ActiveX + [status, sheets] = xlsfinfo(param.config.param_fn); + warning on MATLAB:xlsfinfo:ActiveX + for sheet_idx = 1:length(sheets) + if ~any(strcmpi(sheets{sheet_idx},{'cmd','records','qlook','sar','array','radar','post'})) + fprintf(fid,'%s\n','='*ones(1,80)); fprintf(fid,' %s\n', sheets{sheet_idx}); fprintf(fid,'%s\n','='*ones(1,80)); + read_param_xls_print(param.config.param_fn,sheets{sheet_idx},oparams,fid); + end + end + catch ME + ME.getReport + end + fprintf(fid,'\n'); + + param_txt_fn = ct_filename_ct_tmp(param,'','param', [param.config.date_str,'.txt']); + fprintf('Writing %s\n\n', param_txt_fn); + param_txt_fn_dir = fileparts(param_txt_fn); + if ~exist(param_txt_fn_dir,'dir') + mkdir(param_txt_fn_dir); + end + [fid,msg] = fopen(param_txt_fn,'wb'); + if fid<0 + error('Could not write to %s: %s\n', param_txt_fn, msg); + end + fprintf(fid,'%s\n','='*ones(1,80)); fprintf(fid,' cmd\n'); fprintf(fid,'%s\n','='*ones(1,80)); + read_param_xls_print(param.config.param_fn,'cmd',oparams,fid); + fprintf(fid,'\n'); + fprintf(fid,'%s\n','='*ones(1,80)); fprintf(fid,' records\n'); fprintf(fid,'%s\n','='*ones(1,80)); + read_param_xls_print(param.config.param_fn,'records',oparams,fid); + fprintf(fid,'\n'); + fprintf(fid,'%s\n','='*ones(1,80)); fprintf(fid,' qlook\n'); fprintf(fid,'%s\n','='*ones(1,80)); + read_param_xls_print(param.config.param_fn,'qlook',oparams,fid); + fprintf(fid,'\n'); + fprintf(fid,'%s\n','='*ones(1,80)); fprintf(fid,' sar\n'); fprintf(fid,'%s\n','='*ones(1,80)); + read_param_xls_print(param.config.param_fn,'sar',oparams,fid); + fprintf(fid,'\n'); + fprintf(fid,'%s\n','='*ones(1,80)); fprintf(fid,' array\n'); fprintf(fid,'%s\n','='*ones(1,80)); + read_param_xls_print(param.config.param_fn,'array',oparams,fid); + fprintf(fid,'\n'); + fprintf(fid,'%s\n','='*ones(1,80)); fprintf(fid,' radar\n'); fprintf(fid,'%s\n','='*ones(1,80)); + read_param_xls_print(param.config.param_fn,'radar',oparams,fid); + fprintf(fid,'\n'); + fprintf(fid,'%s\n','='*ones(1,80)); fprintf(fid,' post\n'); fprintf(fid,'%s\n','='*ones(1,80)); + read_param_xls_print(param.config.param_fn,'post',oparams,fid); + fprintf(fid,'\n'); + % Other sheets + try + warning off MATLAB:xlsfinfo:ActiveX + [status, sheets] = xlsfinfo(param.config.param_fn); + warning on MATLAB:xlsfinfo:ActiveX + for sheet_idx = 1:length(sheets) + if ~any(strcmpi(sheets{sheet_idx},{'cmd','records','qlook','sar','array','radar','post'})) + fprintf(fid,'%s\n','='*ones(1,80)); fprintf(fid,' %s\n', sheets{sheet_idx}); fprintf(fid,'%s\n','='*ones(1,80)); + read_param_xls_print(param.config.param_fn,sheets{sheet_idx},oparams,fid); + end + end + catch ME + ME.getReport + end + fprintf(fid,'\n'); + fclose(fid); +end + +%% Exit task +% ========================================================================= +fprintf('%s done %s\n', mfilename, datestr(now)); + +success = true; diff --git a/cresis-toolbox/ct_support/preprocess_task_utua.m b/cresis-toolbox/ct_support/preprocess_task_utua.m index 2ea103b4..8560e48c 100644 --- a/cresis-toolbox/ct_support/preprocess_task_utua.m +++ b/cresis-toolbox/ct_support/preprocess_task_utua.m @@ -289,7 +289,7 @@ day_wrap_offset{board_idx} = day_wrap_offset{board_idx}(mask); end end - [segs,stats] = create_segments(counters,file_idxs,day_wrap_offset,param.config.max_time_gap); + [segs,stats] = preprocess_create_segments(counters,file_idxs,day_wrap_offset,param.config.max_time_gap); if 1 % Debug: Test Code diff --git a/cresis-toolbox/ct_support/quality_control.m b/cresis-toolbox/ct_support/quality_control.m index c8889975..8d8ea292 100644 --- a/cresis-toolbox/ct_support/quality_control.m +++ b/cresis-toolbox/ct_support/quality_control.m @@ -1,26 +1,26 @@ -% script update_frames +% script quality_control % -% Creates frames files for FMCW and accumulation radar. Requires -% that the records file be created. +% Updates the frames file's proc_mode or quality field. This is necessary +% for kuband radar and snow radar sea ice data where artifacts need to be +% indicated for NSIDC data products. The proc_mode field can be used during +% processing to identify specific frames to process. To run this script, +% the images and the records and frames files should be available. +% records file be created. % -% See run_update_frames.m to see how to run. +% See run_quality_control.m to see how to run. % % Prompt -% F(FFF:PPPP:N[D]): +% F(FFF:PPPP): % FFF = frame ID -% PPPP = processing type -% N = nyquist zone -% D = default nyquist zone if 'D' present +% PPPP = field to update (user specifies quality or proc_mode) % % Commands (case-insensitive except where noted) -% empty or 'n' +% empty % proceed to next frame % 'p' % proceed to previous frame % # % sets frame processing type to # and proceeds to next frame -% n# -% sets nyquist zone to # (N# proceeds to next frames after setting) % c or k % runs Matlab eval command in debug prompt % d @@ -37,6 +37,8 @@ % quits without saving % % Author: John Paden +% +% See also: quality_control.m, run_quality_control.m %% Automated Section @@ -48,8 +50,6 @@ error('Invalid field type update_field_type=%s', update_field_type); end -records_fn = ct_filename_support(param,'','records'); - img_type = param.img_type; num_img = length(param.image_out_dir); @@ -66,10 +66,6 @@ end linkaxes(h_axes,'xy'); -fprintf('Loading records file %s\n',records_fn); -records = load(records_fn,'relative_filename','relative_rec_num'); -[records_fn_dir,records_fn_name] = fileparts(records_fn); -param.day_seg = records_fn_name(9:end); for img=1:num_img image_fn_dir{img} = ct_filename_out(param,param.image_out_dir{img}{:}); @@ -77,25 +73,8 @@ end frames_fn = ct_filename_support(param,'','frames'); +frames = frames_load(param); -if exist(frames_fn,'file') - fprintf('Loading frames file %s\n', frames_fn); - frames = load(frames_fn); - if isfield(frames,'nyquist_zone') - warning('Detected old file format with nyquist_zone field. This field will be removed when frames file is saved.'); - end - if ~isfield(frames,'file_version') - warning('Detected old file format with no file_version field. This file will be updated to version ''1'' when frames file is saved.'); - frames.file_version = '1'; - end - file_version = frames.file_version; - frames = frames.frames; - if ~isfield(frames,update_field) - frames.(update_field) = NaN*zeros(size(frames.frame_idxs)); - end -else - error('No frames files exists yet\n'); -end old_frames = frames; frm_idx = 1; @@ -215,7 +194,7 @@ else set(h_plot(img),'Color','black') for rline=1:size(mdata.Data,2) - mdata.Data(lp(mdata.Data(:,rline)) threshold),5)) figure(2); clf; @@ -316,14 +297,65 @@ keyboard elseif strcmpi(val,'d') - diff_frms = find((frames.(update_field)~=old_frames.(update_field) ... - & ~(isnan(frames.(update_field)) & isnan(old_frames.(update_field))))); - fprintf('%-5s\t%-7s\t%-7s\t%7s\t%7s\n', 'Frm', 'Old', 'New'); - for cur_frm = diff_frms - fprintf('%03.0f \t%04.0f \t%04.0f\n', cur_frm, ... - old_frames.(update_field)(cur_frm), frames.(update_field)(cur_frm)); + fprintf('\n'); + diff_frms = find((frames.(update_field)~=old_frames.(update_field) ... + & ~(isnan(frames.(update_field)) & isnan(old_frames.(update_field))))); + fprintf('%s\t%s\t%s\t%s\t%s\n', 'Frm', 'Old_Value', 'New_Value', 'Old_Binary', 'New_Binary'); + for cur_frm = diff_frms + if isnan(old_frames.(update_field)(cur_frm)) + old_dec2bin_str = 'NaN'; + else + old_dec2bin_str = dec2bin(old_frames.(update_field)(cur_frm),8); + end + if isnan(frames.(update_field)(cur_frm)) + dec2bin_str = 'NaN'; + else + dec2bin_str = dec2bin(frames.(update_field)(cur_frm),8); + end + fprintf('%3.0f\t%9.0f\t%9.0f\t%10s\t%10s\n', cur_frm, ... + old_frames.(update_field)(cur_frm), frames.(update_field)(cur_frm), ... + old_dec2bin_str, dec2bin_str); + end + fprintf('\n'); + + elseif strcmpi(val,'f') + nan_frms = find(isnan(frames.(update_field))); + fprintf('\nThere are %d frames with isnan(frames.%s).\n', length(nan_frms), update_field); + if ~isempty(nan_frms) + fprintf('These frames are: %s\n\n', mat2str_generic(nan_frms)); + else + fprintf('\n'); end + zero_frms = find(frames.(update_field) == 0); + fprintf('There are %d frames with frames.%s == 0.\n', length(zero_frms), update_field); + if ~isempty(zero_frms) + fprintf('These frames are: %s\n\n', mat2str_generic(zero_frms)); + else + fprintf('\n'); + end + + fprintf('Frames which are ~isnan and ~=0:\n'); + fprintf('%s\t%s\t%s\t%s\t%s\n', 'Frm', 'Old_Value', 'New_Value', 'Old_Binary', 'New_Binary'); + for cur_frm = 1:length(frames.(update_field)) + if ~isnan(frames.(update_field)(cur_frm)) && frames.(update_field)(cur_frm) ~= 0 + if isnan(old_frames.(update_field)(cur_frm)) + old_dec2bin_str = 'NaN'; + else + old_dec2bin_str = dec2bin(old_frames.(update_field)(cur_frm),8); + end + if isnan(frames.(update_field)(cur_frm)) + dec2bin_str = 'NaN'; + else + dec2bin_str = dec2bin(frames.(update_field)(cur_frm),8); + end + fprintf('%3.0f\t%9.0f\t%9.0f\t%10s\t%10s\n', cur_frm, ... + old_frames.(update_field)(cur_frm), frames.(update_field)(cur_frm), ... + old_dec2bin_str, dec2bin_str); + end + end + fprintf('\n'); + elseif strcmp(val,'i') if strcmpi(param.img_type,img_type) img_type = 'mat'; @@ -368,6 +400,11 @@ frames.(update_field)(frm) = num_val; frm_idx = frm_idx + 1; + elseif strcmpi(val,'n') + fprintf(' Changing frames.%s(%d) to %d\n', update_field, frm, NaN); + frames.(update_field)(frm) = NaN; + frm_idx = frm_idx + 1; + elseif strcmpi(val,'p') frm_idx = frm_idx - 1; @@ -382,16 +419,16 @@ mkdir(out_dir); end fprintf(' Saving frames file %s\n',frames_fn); - ct_save(frames_fn,'-v7.3','frames','file_version'); + ct_save(frames_fn,'-struct','frames'); old_frames = frames; elseif strcmpi(val,'?') fprintf(' : go to next frame\n'); fprintf(' p: previous frame\n'); - fprintf(' n#: set nyquist zone to # (e.g. "n3")\n'); fprintf(' i: swap image type between mat and %s (currently %s)\n', param.img_type, img_type); fprintf(' I: toggle false color and y-limits (currently %d)\n', fmcw_img_debug_mode); - fprintf(' d: print out changes to frames so far\n'); + fprintf(' d: print out differences or changes to frames so far\n'); + fprintf(' f: print out status of frames\n'); fprintf(' j: jump to frame\n'); fprintf(' c or k: enter matlab command\n'); fprintf(' s: save frames file\n'); @@ -410,15 +447,27 @@ end %% Print out frames that changed +% Code is copied from (d)iff function above +fprintf('\n'); diff_frms = find((frames.(update_field)~=old_frames.(update_field) ... & ~(isnan(frames.(update_field)) & isnan(old_frames.(update_field))))); -if ~isempty(diff_frms) - fprintf('%-5s\t%-7s\t%-7s\n', 'Frm', 'Old', 'New'); - for cur_frm = diff_frms - fprintf('%03.0f \t%04.0f \t%04.0f \t%7.0f\t%7.0f\n', cur_frm, ... - old_frames.(update_field)(cur_frm), frames.(update_field)(cur_frm)); +fprintf('%s\t%s\t%s\t%s\t%s\n', 'Frm', 'Old_Value', 'New_Value', 'Old_Binary', 'New_Binary'); +for cur_frm = diff_frms + if isnan(old_frames.(update_field)(cur_frm)) + old_dec2bin_str = 'NaN'; + else + old_dec2bin_str = dec2bin(old_frames.(update_field)(cur_frm),8); end + if isnan(frames.(update_field)(cur_frm)) + dec2bin_str = 'NaN'; + else + dec2bin_str = dec2bin(frames.(update_field)(cur_frm),8); + end + fprintf('%3.0f\t%9.0f\t%9.0f\t%10s\t%10s\n', cur_frm, ... + old_frames.(update_field)(cur_frm), frames.(update_field)(cur_frm), ... + old_dec2bin_str, dec2bin_str); end +fprintf('\n'); if ~isempty(diff_frms) %% Save Output @@ -429,10 +478,8 @@ mkdir(out_dir); end fprintf(' Saving frames file %s\n',frames_fn); - ct_save(frames_fn,'-v7.3','frames','file_version'); + ct_save(frames_fn,'-struct','frames'); else - fprintf(' Not saving (can still manually save by pasting commands)\n'); + warning('Not saving (can still manually save by pasting save commands above this warning message).'); end end - -return; diff --git a/cresis-toolbox/ct_support/read_param_xls.m b/cresis-toolbox/ct_support/read_param_xls.m index 350c744b..b565801f 100644 --- a/cresis-toolbox/ct_support/read_param_xls.m +++ b/cresis-toolbox/ct_support/read_param_xls.m @@ -27,7 +27,22 @@ % Author: Brady Maasen, John Paden % % See also: ct_set_params, master, read_param_xls +% +% See also for spreadsheet cell loading: +% read_param_xls_boolean.m, read_param_xls_general.m, +% read_param_xls_text.m +% +% See also for worksheet loading: +% read_param_xls_generic.m, read_param_xls_radar.m: +% +% See also for printing out spreadsheet to stdout: +% read_param_xls_print, read_param_xls_print_headers.m +if isstruct(param_fn) + day_seg_filter = param_fn.day_seg; + param_fn = ct_filename_param(sprintf('%s_param_%s.xls', ct_output_dir(param_fn.radar_name), param_fn.season_name)); +end + %% Load standard worksheets warning('off','MATLAB:xlsread:Mode'); [params] = read_param_xls_radar(param_fn); diff --git a/cresis-toolbox/ct_support/read_param_xls_boolean.m b/cresis-toolbox/ct_support/read_param_xls_boolean.m index 60d14219..3cf1dcea 100644 --- a/cresis-toolbox/ct_support/read_param_xls_boolean.m +++ b/cresis-toolbox/ct_support/read_param_xls_boolean.m @@ -6,6 +6,21 @@ % Support function for read_param_xls_* % % Author: John Paden, Brady Maasen +% +% See also: ct_set_params, master, read_param_xls +% +% See also for spreadsheet cell loading: +% read_param_xls_boolean.m, read_param_xls_general.m, +% read_param_xls_text.m +% +% See also for worksheet loading: +% read_param_xls_generic.m, read_param_xls_radar.m: +% +% See also for printing out spreadsheet to stdout: +% read_param_xls_print, read_param_xls_print_headers.m + +% +% PRINT OUT XLS: read_param_xls_print if size(num,1) < row || size(num,2) < col || isnan(num(row,col)) if size(txt,1) < row || size(txt,2) < col || isempty(txt{row,col}) diff --git a/cresis-toolbox/ct_support/read_param_xls_general.m b/cresis-toolbox/ct_support/read_param_xls_general.m index 3a6fbd45..caa7c4b4 100644 --- a/cresis-toolbox/ct_support/read_param_xls_general.m +++ b/cresis-toolbox/ct_support/read_param_xls_general.m @@ -6,6 +6,18 @@ % Support function for read_param_xls_* % % Author: John Paden, Brady Maasen +% +% See also: ct_set_params, master, read_param_xls +% +% See also for spreadsheet cell loading: +% read_param_xls_boolean.m, read_param_xls_general.m, +% read_param_xls_text.m +% +% See also for worksheet loading: +% read_param_xls_generic.m, read_param_xls_radar.m: +% +% See also for printing out spreadsheet to stdout: +% read_param_xls_print, read_param_xls_print_headers.m if size(num,1) < row || size(num,2) < col if size(txt,1) >= row && size(txt,2) >= col && ~isempty(txt{row,col}) diff --git a/cresis-toolbox/ct_support/read_param_xls_generic.m b/cresis-toolbox/ct_support/read_param_xls_generic.m index fb4b1f23..9b826f6b 100644 --- a/cresis-toolbox/ct_support/read_param_xls_generic.m +++ b/cresis-toolbox/ct_support/read_param_xls_generic.m @@ -12,7 +12,17 @@ % % Author: Theresa Stumpf, John Paden % -% See also: read_param_xls +% See also: ct_set_params, master, read_param_xls +% +% See also for spreadsheet cell loading: +% read_param_xls_boolean.m, read_param_xls_general.m, +% read_param_xls_text.m +% +% See also for worksheet loading: +% read_param_xls_generic.m, read_param_xls_radar.m: +% +% See also for printing out spreadsheet to stdout: +% read_param_xls_print, read_param_xls_print_headers.m %% Input checks and setup % ======================================================================= @@ -28,7 +38,6 @@ %% Cell input: Recurse to load a list of worksheets % ======================================================================= if iscell(generic_ws) - warning('off','MATLAB:xlsread:Mode'); for idx = 1:size(generic_ws,1) tmp = read_param_xls_generic(param_fn,generic_ws{idx,1},params,read_param); if size(generic_ws,2) > 1 @@ -38,7 +47,6 @@ params = tmp; end end - warning('on','MATLAB:xlsread:Mode'); return; end @@ -46,16 +54,46 @@ % ======================================================================= sheet_name = generic_ws; fprintf('Reading sheet %s of xls file: %s\n', sheet_name, param_fn); -warning off MATLAB:xlsfinfo:ActiveX -[status, sheets] = xlsfinfo(param_fn); -warning on MATLAB:xlsfinfo:ActiveX + +[~,~,param_fn_ext] = fileparts(param_fn); +matlab_ver = ver('matlab'); +use_read_table = str2double(matlab_ver.Version) >= 9.10 || any(strcmpi(param_fn_ext,{'xlsx','ods'})); +clear('matlab_ver'); +if use_read_table + sheets = sheetnames(param_fn); +else + warning off MATLAB:xlsfinfo:ActiveX + [status, sheets] = xlsfinfo(param_fn); + warning on MATLAB:xlsfinfo:ActiveX +end + sheet_idx = strmatch(generic_ws,sheets,'exact'); if isempty(sheet_idx) fprintf(' Sheet not found\n'); return end -[num txt] = xlsread(param_fn,sheet_name,'','basic'); +if use_read_table + read_table_opts = detectImportOptions(param_fn,'NumHeaderLines',0,'Sheet',sheet_name,'ReadVariableNames',false); + read_table_opts.DataRange = 'A1'; + for var_idx = 1:length(read_table_opts.VariableTypes) + read_table_opts.VariableTypes{var_idx} = 'char'; + end + read_table_output = readtable(param_fn,read_table_opts); + txt = table2cell(read_table_output); + num = nan(size(txt)); + for element_idx = 1:numel(txt) + num(element_idx) = str2double(txt{element_idx}); + if ~isnan(num(element_idx)) + txt{element_idx} = ''; + end + end + clear('read_table_opts','read_table_output','var_idx','element_idx'); +else + warning('off','MATLAB:xlsread:Mode'); + [num, txt] = xlsread(param_fn,sheet_name,'','basic'); + warning('on','MATLAB:xlsread:Mode'); +end num_header_rows = find(strcmp( 'Date' , txt(1:end,1) )) + 1; if isempty(num_header_rows) @@ -130,6 +168,12 @@ % ======================================================================= for idx = 1:rows row = idx + num_header_rows; + if row > size(num,1) + error('xls row %d error: There is content on this or a later row, but there is no day segment. No content should be placed below the last day segment row.', row); + end + if size(num,2) < 2 || ~isfinite(num(row,1)) || ~isfinite(num(row,2)) + error('xls row %d error: The day segment is not formed properly on this row.', row); + end day_seg = sprintf('%08.0f_%02.0f',num(row,1),num(row,2)); if strcmpi(generic_ws,'cmd') params(idx).day_seg = day_seg; @@ -141,7 +185,7 @@ % Not found, so we skip this row continue; elseif length(idx) > 1 - error('xls row %d error: More than one matching segment to "%s" found in input params.', day_seg); + error('xls row %d error: More than one matching segment to "%s" found in input params.', row, day_seg); end else error('The date segment order of sheets cmd and %s do not match at row %d in the excel spreadsheet. Each sheet must have the same date and segment list.',sheet_name,row); diff --git a/cresis-toolbox/ct_support/read_param_xls_print.m b/cresis-toolbox/ct_support/read_param_xls_print.m index 1e337c69..54ed0cae 100644 --- a/cresis-toolbox/ct_support/read_param_xls_print.m +++ b/cresis-toolbox/ct_support/read_param_xls_print.m @@ -10,7 +10,20 @@ function read_param_xls_print(param_fn, generic_ws, params, fid) % % Author: John Paden % -% See also: read_param_xls +% See also: ct_set_params, master, read_param_xls +% +% See also for spreadsheet cell loading: +% read_param_xls_boolean.m, read_param_xls_general.m, +% read_param_xls_text.m +% +% See also for worksheet loading: +% read_param_xls_generic.m, read_param_xls_radar.m: +% +% See also for printing out spreadsheet to stdout: +% read_param_xls_print.m, read_param_xls_print_headers.m +% +% See also for printing matlab values: +% mat2str_generic.m if ~exist('fid','var') fid = 1; % stdout by default @@ -22,7 +35,7 @@ function read_param_xls_print(param_fn, generic_ws, params, fid) % Get the headers from the parameter spreadsheet try - [headers] = read_param_xls_headers(param_fn, generic_ws); + [headers] = read_param_xls_print_headers(param_fn, generic_ws); catch ME warning('Could not read worksheet: %s', ME.getReport); return; diff --git a/cresis-toolbox/ct_support/read_param_xls_headers.m b/cresis-toolbox/ct_support/read_param_xls_print_headers.m similarity index 91% rename from cresis-toolbox/ct_support/read_param_xls_headers.m rename to cresis-toolbox/ct_support/read_param_xls_print_headers.m index db06db63..0b60b564 100644 --- a/cresis-toolbox/ct_support/read_param_xls_headers.m +++ b/cresis-toolbox/ct_support/read_param_xls_print_headers.m @@ -1,5 +1,5 @@ -function [headers] = read_param_xls_headers(param_fn, generic_ws) -% [headers] = read_param_xls_headers(param_fn, generic_ws) +function [headers] = read_param_xls_print_headers(param_fn, generic_ws) +% [headers] = read_param_xls_print_headers(param_fn, generic_ws) % % Support function for read_param_xls_print, generic worksheet printer. % @@ -8,7 +8,13 @@ % % Author: John Paden % -% See also: read_param_xls_print +% See also: ct_set_params, master, read_param_xls +% +% SUPPORT FUNCTIONS: read_param_xls_boolean.m, read_param_xls_general.m, +% read_param_xls_generic.m, read_param_xls_headers.m, +% read_param_xls_radar.m read_param_xls_text.m +% +% PRINT OUT XLS: read_param_xls_print %% Setup % ======================================================================% diff --git a/cresis-toolbox/ct_support/read_param_xls_radar.m b/cresis-toolbox/ct_support/read_param_xls_radar.m index 1f41e4db..468ae21e 100644 --- a/cresis-toolbox/ct_support/read_param_xls_radar.m +++ b/cresis-toolbox/ct_support/read_param_xls_radar.m @@ -6,21 +6,38 @@ % % Author: Brady Maasen, John Paden % -% See also: read_param_xls +% See also: ct_set_params, master, read_param_xls +% +% See also for spreadsheet cell loading: +% read_param_xls_boolean.m, read_param_xls_general.m, +% read_param_xls_text.m +% +% See also for worksheet loading: +% read_param_xls_generic.m, read_param_xls_radar.m: +% +% See also for printing out spreadsheet to stdout: +% read_param_xls_print, read_param_xls_print_headers.m cell_boolean = @read_param_xls_boolean; cell_text = @read_param_xls_text; cell_read = @read_param_xls_general; -warning('off','MATLAB:xlsread:Mode'); - %% Create Command Parameters % ======================================================================= fprintf('Reading command/cmd of xls file: %s\n', param_fn); -warning off MATLAB:xlsfinfo:ActiveX -[status, sheets] = xlsfinfo(param_fn); -warning on MATLAB:xlsfinfo:ActiveX +[~,~,param_fn_ext] = fileparts(param_fn); +matlab_ver = ver('matlab'); +use_read_table = str2double(matlab_ver.Version) >= 9.10 || any(strcmpi(param_fn_ext,{'xlsx','ods'})); +clear('matlab_ver'); +if use_read_table + sheets = sheetnames(param_fn); +else + warning off MATLAB:xlsfinfo:ActiveX + [status, sheets] = xlsfinfo(param_fn); + warning on MATLAB:xlsfinfo:ActiveX +end + cmd_sheet_name = 'cmd'; sheet_idx = strmatch(cmd_sheet_name,sheets,'exact'); if isempty(sheet_idx) @@ -31,7 +48,30 @@ end end -[num, txt] = xlsread(param_fn,cmd_sheet_name,'','basic'); +if use_read_table + read_table_opts = detectImportOptions(param_fn,'NumHeaderLines',0,'Sheet',cmd_sheet_name,'ReadVariableNames',false); + read_table_opts.DataRange = 'A1'; + for var_idx = 1:length(read_table_opts.VariableTypes) + read_table_opts.VariableTypes{var_idx} = 'char'; + end + read_table_output = readtable(param_fn,read_table_opts); + txt = table2cell(read_table_output); + num = nan(size(txt)); + for element_idx = 1:numel(txt) + num(element_idx) = str2double(txt{element_idx}); + if ~isnan(num(element_idx)) + txt{element_idx} = ''; + end + end + % read_param_xls_radar only correction to param_file_version: + txt{1,2} = read_table_output{1,2}{1}; + num(1,2) = NaN; + clear('read_table_opts','read_table_output','var_idx','element_idx'); +else + warning('off','MATLAB:xlsread:Mode'); + [num, txt] = xlsread(param_fn,cmd_sheet_name,'','basic'); + warning('on','MATLAB:xlsread:Mode'); +end param_file_version = cell_text(1,2,num,txt); radar_name = cell_text(2,2,num,txt); @@ -50,6 +90,7 @@ error('Could not find version field text in row 1, column 2 of the first worksheet.'); end if version < 4 + warning('Using an old version of the parameter spreadsheet. Please update the parameter spreadsheet to the newest version format.'); num_header_rows = 5; rows = max(size(num,1), size(txt,1)) - num_header_rows; @@ -61,19 +102,37 @@ row = idx + num_header_rows; params(idx).day_seg = sprintf('%08.0f_%02.0f',num(row,1),num(row,2)); col = 3; - params(idx).cmd.frms = cell_read(row,col,num,txt); col = col + 1; - params(idx).cmd.vectors = cell_boolean(row,col,num,txt); col = col + 1; - params(idx).cmd.records = cell_boolean(row,col,num,txt); col = col + 1; - params(idx).cmd.frames = cell_boolean(row,col,num,txt); col = col + 1; - params(idx).cmd.qlook = cell_boolean(row,col,num,txt); col = col + 1; - params(idx).cmd.sar = cell_boolean(row,col,num,txt); col = col + 1; - params(idx).cmd.array = cell_boolean(row,col,num,txt); col = col + 1; - params(idx).cmd.generic = cell_read(row,col,num,txt); col = col + 1; - if isempty(params(idx).cmd.generic) - params(idx).cmd.generic = 0; + + try + fieldname = 'frms'; + params(idx).cmd.frms = cell_read(row,col,num,txt); col = col + 1; + fieldname = 'vectors'; + params(idx).cmd.vectors = cell_boolean(row,col,num,txt); col = col + 1; + fieldname = 'records'; + params(idx).cmd.records = cell_boolean(row,col,num,txt); col = col + 1; + fieldname = 'frames'; + params(idx).cmd.frames = cell_boolean(row,col,num,txt); col = col + 1; + fieldname = 'qlook'; + params(idx).cmd.qlook = cell_boolean(row,col,num,txt); col = col + 1; + fieldname = 'sar'; + params(idx).cmd.sar = cell_boolean(row,col,num,txt); col = col + 1; + fieldname = 'array'; + params(idx).cmd.array = cell_boolean(row,col,num,txt); col = col + 1; + fieldname = 'generic'; + params(idx).cmd.generic = cell_read(row,col,num,txt); col = col + 1; + if isempty(params(idx).cmd.generic) + params(idx).cmd.generic = 0; + end + fieldname = 'mission_names'; + params(idx).cmd.mission_names = cell_text(row,col,num,txt); col = col + 1; + fieldname = 'notes'; + params(idx).cmd.notes = cell_text(row,col,num,txt); col = col + 1; + catch ME + if ~strcmp(ME.identifier,'read_param_xls_radar:eval_error') + fprintf(' Error in row %d, column %s/%d (field %s)\n', row, char(65+mod(col-1,26)), col, fieldname); + end + rethrow(ME) end - params(idx).cmd.mission_names = cell_text(row,col,num,txt); col = col + 1; - params(idx).cmd.notes = cell_text(row,col,num,txt); col = col + 1; params(idx).sw_version = sw_version; params(idx).param_file_version = param_file_version; end diff --git a/cresis-toolbox/ct_support/read_param_xls_text.m b/cresis-toolbox/ct_support/read_param_xls_text.m index e45b08e6..bbdaf4a2 100644 --- a/cresis-toolbox/ct_support/read_param_xls_text.m +++ b/cresis-toolbox/ct_support/read_param_xls_text.m @@ -5,7 +5,17 @@ % % Support function for read_param_xls_* % -% Author: John Paden, Brady Maasen +% See also: ct_set_params, master, read_param_xls +% +% See also for spreadsheet cell loading: +% read_param_xls_boolean.m, read_param_xls_general.m, +% read_param_xls_text.m +% +% See also for worksheet loading: +% read_param_xls_generic.m, read_param_xls_radar.m: +% +% See also for printing out spreadsheet to stdout: +% read_param_xls_print, read_param_xls_print_headers.m if size(num,1) >= row && size(num,2) >= col && ~isnan(num(row,col)) error('Cell (%d,%d)/(%d,%s) must be text (use '' before the number)\n', row, col, row, 'A'+col-1); diff --git a/cresis-toolbox/ct_support/records_bit_mask.m b/cresis-toolbox/ct_support/records_bit_mask.m index 0430f559..f2e91534 100644 --- a/cresis-toolbox/ct_support/records_bit_mask.m +++ b/cresis-toolbox/ct_support/records_bit_mask.m @@ -1,8 +1,10 @@ function records_bit_mask(param,param_override) % -% GPR profiles usually contain stops, 90+ deg sharp turns, etc that may be -% undesirable for SAR processing. Use this script to help find the -% bad data records associated with these maneuvers and remove them. +% GPR profiles usually contain stops, 270+ deg loop turns, etc that may be +% undesirable for SAR processing. Use this script to help find the data +% records associated with these maneuvers and mask them for SAR processing. +% Usually it is not necessary to do this unless there are long sections of +% stationary data which can cause the SAR processor to run out of memory. % % param = struct with processing parameters % param_override = parameters in this struct will override parameters @@ -35,14 +37,18 @@ function records_bit_mask(param,param_override) end bad_heading_diff_threshold = param.records_bit_mask.bad_heading_diff_threshold; -if ~isfield(param.records_bit_mask,'manual_masking') || isempty(param.records_bit_mask.manual_masking) - param.records_bit_mask.manual_masking = 1; -end - if ~isfield(param.records_bit_mask,'debug_plot') || isempty(param.records_bit_mask.debug_plot) param.records_bit_mask.debug_plot = 1; end +if ~isfield(param.records_bit_mask,'manual_masking_en') || isempty(param.records_bit_mask.manual_masking_en) + param.records_bit_mask.manual_masking_en = 1; +end + +if ~isfield(param.records_bit_mask,'mode') || isempty(param.records_bit_mask.mode) + param.records_bit_mask.mode = 1; +end + %% Load records and background geotiff for GUI geotiff_fn = ct_filename_gis(param,param.records.frames.geotiff_fn); @@ -53,11 +59,11 @@ function records_bit_mask(param,param_override) [x,y] = projfwd(proj,records.lat,records.lon); -%% Fabricating a heading now (pulled from gps_make.m) +%% Fabricating a heading now (code copied from gps_create.m) gps = records; along_track = geodetic_to_along_track(gps.lat,gps.lon); rlines = get_equal_alongtrack_spacing_idxs(along_track,0.75); -physical_constants; +physical_constants; % Load WGS84.spheroid est_heading = size(gps.heading); clear origin heading east north; for rline_idx = 1:length(rlines) @@ -67,91 +73,139 @@ function records_bit_mask(param,param_override) else rline_end = length(along_track); end - [origin(1),origin(2),origin(3)] = geodetic2ecef(gps.lat(rline)/180*pi,gps.lon(rline)/180*pi,gps.elev(rline),WGS84.ellipsoid); - [heading(1),heading(2),heading(3)] = geodetic2ecef(gps.lat(rline_end)/180*pi,gps.lon(rline_end)/180*pi,gps.elev(rline_end),WGS84.ellipsoid); + [origin(1),origin(2),origin(3)] = geodetic2ecef(WGS84.spheroid, gps.lat(rline),gps.lon(rline),gps.elev(rline)); + [heading(1),heading(2),heading(3)] = geodetic2ecef(WGS84.spheroid, gps.lat(rline_end),gps.lon(rline_end),gps.elev(rline_end)); heading = heading - origin; % Determine east vector - [east(1) east(2) east(3)] = lv2ecef(1,0,0,gps.lat(rline)/180*pi,gps.lon(rline)/180*pi,gps.elev(rline),WGS84.ellipsoid); + [east(1) east(2) east(3)] = enu2ecef(1,0,0,gps.lat(rline),gps.lon(rline),gps.elev(rline),WGS84.spheroid); east = east - origin; % Determine north vector - [north(1) north(2) north(3)] = lv2ecef(0,1,0,gps.lat(rline)/180*pi,gps.lon(rline)/180*pi,gps.elev(rline),WGS84.ellipsoid); + [north(1) north(2) north(3)] = enu2ecef(0,1,0,gps.lat(rline),gps.lon(rline),gps.elev(rline),WGS84.spheroid); north = north - origin; % Determine heading est_heading(rline:rline_end) = atan2(dot(east,heading),dot(north,heading)); end % Slow velocity mask -vel = diff(along_track) ./ diff(records.gps_time); -vel_mask = vel < bad_vel_threshold; -vel_mask(end+1) = 0; +speed = [inf diff(along_track) ./ diff(records.gps_time)]; +speed_mask = speed < bad_vel_threshold; % Fast heading change mask -heading_diff = diff(est_heading); +heading_diff = [0 diff(est_heading)]; heading_mask = abs(heading_diff) > bad_heading_diff_threshold; -heading_mask(end+1) = 0; + +clip_vectors_param = []; +clip_vectors_param.fh_button_motion = @records_bit_mask_button_motion; +clip_vectors_param.fh_close_win = @records_bit_mask_close_window; +clip_vectors_param.user_data.param = param; +clip_vectors_param.user_data.x = x; +clip_vectors_param.user_data.y = y; +clip_vectors_param.user_data.records = records; +clip_vectors_param.user_data.speed = speed; +clip_vectors_param.user_data.speed_mask = speed_mask; +clip_vectors_param.user_data.heading_diff = heading_diff; +clip_vectors_param.user_data.heading_mask = heading_mask; % This either initiates the manual removal of records or does it % automatically based on thresholds. -if param.records_bit_mask.manual_masking == 1 +if param.records_bit_mask.manual_masking_en == 1 % Plot records - figure; clf; + h_fig = figure; + h_axes = axes('parent',h_fig); h_plots = []; - h_plots(end+1) = plot(x,y); + switch (param.records_bit_mask.mode) + case 0 + h_plots(end+1) = plot(x/1e3,y/1e3,'parent',h_axes); + case 2 + x_cur = x; + x_cur(speed_mask | heading_mask) = NaN; + y_cur = y; + y_cur(speed_mask | heading_mask) = NaN; + h_plots(end+1) = plot(x_cur/1e3,y_cur/1e3,'parent',h_axes); + otherwise % case 1 + x_cur = x; + x_cur(logical(bitand(records.bit_mask(1,:),2))) = NaN; + y_cur = y; + y_cur(logical(bitand(records.bit_mask(1,:),2))) = NaN; + h_plots(end+1) = plot(x_cur/1e3,y_cur/1e3,'parent',h_axes); + end hold on; - fprintf('\nRed is slow velocity records\n'); - h_plots(end+1) = plot(x(vel_mask),y(vel_mask),'r.'); - fprintf('\nGreen is fast heading change records\n'); + fprintf('\nRed indicates slow speed records\n'); + h_plots(end+1) = plot(x(speed_mask)/1e3,y(speed_mask)/1e3,'r.','parent',h_axes); + fprintf('\nGreen indicates fast heading change records\n'); if ~any(heading_mask) - h_plots(end+1) = plot(NaN,NaN,'g.'); + h_plots(end+1) = plot(NaN,NaN,'g.','parent',h_axes); else - h_plots(end+1) = plot(x(heading_mask),y(heading_mask),'g.'); + h_plots(end+1) = plot(x(heading_mask)/1e3,y(heading_mask)/1e3,'g.','parent',h_axes); end % Manual tool for removing records - clip_vectors(h_plots); - - fprintf('\nRemove bad records (press F1 in plot for help). Once done, type dbcont.\n'); - keyboard + h_clip_vectors = clip_vectors(h_plots,clip_vectors_param); + set(h_clip_vectors.sf.h_fig,'Visible','off'); % Selection filter not useful so hide it + fprintf('\nRemove bad records (press F1 in plot for help). Once done, close the window and the changes will be saved to records.bit_mask.\n'); + h_clip_vectors.key_press([],struct('Modifier',{{}},'Key','f1')); +else + % Accept the automatically determined stationary/fast-heading change + % records without opening the manual GUI. + records_bit_mask_close_window(clip_vectors_param); +end + +end + +%% records_bit_mask_button_motion +function records_bit_mask_button_motion(h_clip_vector,src,event) + +mouse_pos = get(h_clip_vector.h_axes,'CurrentPoint'); +x = mouse_pos(1,1)*1e3; +y = mouse_pos(1,2)*1e3; + +[~,idx] = min((h_clip_vector.user_data.x-x).^2+(h_clip_vector.user_data.y-y).^2); +h=title(sprintf('%8d: (%+5.0f km,%+5.0f km) %3.1f m/s %4.1f deg heading change', idx, ... + round(h_clip_vector.user_data.x(idx)/1e3), ... + round(h_clip_vector.user_data.y(idx)/1e3), ... + round(h_clip_vector.user_data.speed(idx)*10)/10, ... + abs(round(h_clip_vector.user_data.heading_diff(idx)*180/pi*10)/10) ),'FontName','FixedWidth','FontSize',8,'parent',h_clip_vector.h_axes); + +end + +%% records_bit_mask_close_window +function records_bit_mask_close_window(h_clip_vector) + +param = h_clip_vector.user_data.param; +records = h_clip_vector.user_data.records; +heading_mask = h_clip_vector.user_data.heading_mask; +speed_mask = h_clip_vector.user_data.speed_mask; + +if isa(h_clip_vector,'clip_vectors') % Find the bad records - ydata = get(h_plots(1),'YData'); + ydata = get(h_clip_vector.h_plots(1),'YData'); good_mask = ~isnan(ydata); - else - good_mask = [~heading_mask & ~vel_mask]; + good_mask = [~heading_mask & ~speed_mask]; end if param.records_bit_mask.debug_plot == 1 - figure; clf; - plot(records.lon(good_mask),records.lat(good_mask)); + h_fig = figure; h_axes = axes('parent',h_fig); + plot(records.lon(good_mask),records.lat(good_mask),'parent',h_axes); title([num2str(sum(good_mask)),' of ',num2str(length(good_mask)),' remaining']); - fprintf('Check to make sure you are happy with the results. Once done, type dbcont.\n'); + fprintf('Check to make sure you are happy with the results. Once done, run "dbcont" or press F5 to continue. If not, run "dbquit" and start over.\n'); keyboard end -%%%%%%%% Here we use the good_mask info to produce records.bit_mask. -%%%%%%%% This preserves data already masked for bad headers, but adds masks -%%%%%%%% to stationary or incorrectly moving records. - +% Here we use the good_mask info to produce records.bit_mask. +% This preserves data already masked for bad headers (bit_mask bit 0 set), but adds masks +% to stationary or incorrectly moving records. if ~isfield(records,'bit_mask') - records.bit_mask = zeros(size(records.offset)); + records.bit_mask = zeros(size(records.offset),'uint8'); end - for board_idx = 1:size(records.offset,1) - % For records that are good and are considered bad by this analysis, set - % them as bad. - mask = [records.bit_mask(board_idx,:) == 0 & ~good_mask]; - records.bit_mask(board_idx,mask) = 2; - - % For records that are bad and are now considered good by this analysis, - % set them as good. - mask = [records.bit_mask(board_idx,:) == 2 & good_mask]; - records.bit_mask(board_idx,mask) = 0; + records.bit_mask(board_idx,:) ... + = bitand(records.bit_mask(board_idx,:),bin2dec('11111101')) + 2*~good_mask; end records_fn = ct_filename_support(param,'','records'); fprintf(' Saving records %s\n', records_fn); ct_save(records_fn,'-struct','records'); -records_aux_files_create(records_fn); - +end diff --git a/cresis-toolbox/ct_support/records_load.m b/cresis-toolbox/ct_support/records_load.m index 872ddaeb..45db17e3 100644 --- a/cresis-toolbox/ct_support/records_load.m +++ b/cresis-toolbox/ct_support/records_load.m @@ -1,15 +1,185 @@ -function records = records_load(param,recs,field_names) +function records = records_load(param,varargin) +% records = records_load(param,varargin) +% +% Loads records file. Handles old file formats. Three Modes: +% +% MODE 1 +% ------------------------------------------------------------------------- +% records = records_load(param) +% Loads records file. +% +% param: parameter spreadsheet structure +% +% records: struct with all the fields from the frames file are loaded +% +% MODE 2 +% ------------------------------------------------------------------------- +% records = records_load(param,recs) +% Loads only the specified records from the records file. +% +% param: parameter spreadsheet structure +% recs: 2x1 vector specifying the start/stop records [start_record, stop_record] +% Records are one-indexed (first records is record "1") +% Set stop record to inf to read to end of file +% "inf" can be used for start or stop record to mean last record +% +% records: struct with all the fields from the frames file are loaded +% +% MODE 3 +% ------------------------------------------------------------------------- +% records: records_load(param,FIELD_STRING_ARGS ...) +% Loads only the specified records from the records file. This +% functionality is slower, but reduces memory usage. +% +% param: parameter spreadsheet structure +% FIELD_STRING_ARGS: Argument list of strings containing specific fields to load from the records file. +% +% records: struct with fields listed in FIELD_STRING_ARGS are loaded from +% the records file +% +% Example +% param = read_param_xls(ct_filename_param('rds_param_2019_Antarctica_Ground.xls'),'20191231_04'); +% records = records_load(param); +% +% records = records_load(param,[1 100]); +% +% records = records_load(param,'lat','lon'); +% +% Author: John Paden -if ~exist('field_names','var') || isempty(field_names) - %% Load all fields +if ischar(param) + records_fn = param; + records = load(records_fn,'param_records'); + param = records.param_records; + global gRadar; + param = merge_structs(param,gRadar); +else + if ~isfield(param,'day_seg') || isempty(param.day_seg) + error('records_load requires that param.day_seg exist and be nonempty'); + end records_fn = ct_filename_support(param,'','records'); + if ~exist(records_fn,'file') + error('Records file does not exist: %s (%s).', records_fn, datestr(now)); + end +end + +%% Correct old files +% ========================================================================= +update_records_flag = false; +records = load(records_fn,'settings'); +if isfield(records,'settings') && isfield(records.settings,'wfs') && isfield(records.settings.wfs,'wfs') + update_records_flag = true; +end +if isfield(records,'settings') && isfield(records.settings,'nyquist_zone_hw') + update_records_flag = true; +end +if isfield(records,'settings') && isfield(records.settings,'nyquist_zone') + update_records_flag = true; +end + +mat_vars = whos('-file',records_fn); + +if any(strcmp('vectors',{mat_vars.name})) + update_records_flag = true; +end + +if any(strcmp('surface',{mat_vars.name})) +% update_records_flag = true; +end + +if ~update_records_flag + try + records = load(records_fn,'param_records'); + delta_offset = max(param.records.gps.time_offset) - max(records.param_records.records.gps.time_offset); + % Often param.records.gps.time_offset is NaN (unknown), if this is the + % case, then we do not do the delta_offset check + if isfinite(param.records.gps.time_offset) && delta_offset ~= 0 + update_records_flag = true; + end + end +end + +if update_records_flag + records_update(param,[]); +end + +if nargin == 1 + %% Load all fields + % ======================================================================= records = load(records_fn); - % Add new field "bit_mask" if missing if ~isfield(records,'bit_mask') - records.bit_mask = zeros(size(records.offset)); + records.bit_mask = zeros(size(records.offset),'uint8'); + end + +elseif nargin == 2 && isnumeric(varargin{1}) + %% Load specific records + % ======================================================================= + + recs = varargin{1}; + % Find the gps_time field in the records file + match_idx = find(strcmp('gps_time',{mat_vars.name})); + % Determine the number of records which is equal to the length of num_recs + num_recs = mat_vars(match_idx).size(2); + if recs(2) == inf + recs(2) = num_recs; end - % Remove deprecated field "surface" if present - if isfield(records,'surface') - records = rmfield(records,'surface'); + if recs(2) > num_recs + % Some functions (like qlook_task) will ask for more records than are + % available; this is allowed but the number of records returned is + % truncated in this case and a warning printed. + warning('Requested records beyond the end of the records file: recs(2)=%d > num_recs=%d. There are %d records in the records file: %s.', recs(2), num_recs, num_recs, records_fn); + recs(2) = num_recs; + end + if recs(2) < recs(1) + error('Requested records beyond the end of the records file or requested zero records: recs(1)=%d > recs(2)=%d. There are %d records in the records file: %s.', recs(1), recs(2), num_recs, records_fn); + end + + records = load(records_fn,'gps_source','param_records','relative_filename','relative_rec_num','settings'); + + records_mat = matfile(records_fn); + records.elev = records_mat.elev(:,recs(1):recs(2)); + records.gps_time = records_mat.gps_time(:,recs(1):recs(2)); + records.heading = records_mat.heading(:,recs(1):recs(2)); + records.lat = records_mat.lat(:,recs(1):recs(2)); + records.lon = records_mat.lon(:,recs(1):recs(2)); + records.offset = records_mat.offset(:,recs(1):recs(2)); + records.pitch = records_mat.pitch(:,recs(1):recs(2)); + records.roll = records_mat.roll(:,recs(1):recs(2)); + + if any(strcmp('bit_mask',{mat_vars.name})) + records.bit_mask = records_mat.bit_mask(:,recs(1):recs(2)); + else + records.bit_mask = zeros(size(records.offset),'uint8'); end -end \ No newline at end of file + + if any(strcmp('phase_correction',{mat_vars.name})) + records.phase_correction = records_mat.phase_correction(:,recs(1):recs(2)); + else + records.phase_correction = nan(size(records.gps_time)); + end + + if any(strcmp('Tadc_correction',{mat_vars.name})) + records.Tadc_correction = records_mat.Tadc_correction(:,recs(1):recs(2)); + else + records.Tadc_correction = nan(size(records.gps_time)); + end + + if any(strcmp('nyquist_zone_sig',{mat_vars.name})) + records.nyquist_zone_sig = records_mat.nyquist_zone_sig(:,recs(1):recs(2)); + else + records.nyquist_zone_sig = nan(size(records.gps_time)); + end + + if any(strcmp('nyquist_zone_hw',{mat_vars.name})) + records.nyquist_zone_hw = records_mat.nyquist_zone_hw(:,recs(1):recs(2)); + else + records.nyquist_zone_hw = nan(size(records.gps_time)); + end + +elseif nargin > 1 && all(cellfun(@ischar,varargin)) + %% Load specific fields + records = load(records_fn,varargin{:}); + +else + error('Invalid arguments to records_load.'); +end diff --git a/cresis-toolbox/ct_support/records_update.m b/cresis-toolbox/ct_support/records_update.m index e643a849..f84d28cd 100644 --- a/cresis-toolbox/ct_support/records_update.m +++ b/cresis-toolbox/ct_support/records_update.m @@ -20,17 +20,16 @@ function records_update(param,param_override) fprintf('%s: %s (%s)\n', dbstack_info(1).name, param.day_seg, datestr(now,'HH:MM:SS')); fprintf('=====================================================================\n'); -%% Input checks - -% save_changes: Logical, For debugging purposes, you can turn the file save on/off -save_changes = true; - %% Prep (load records and gps files) records_fn = ct_filename_support(param,'','records'); +if cluster_job_check() + error('records_update may not be called from cluster_job (gRadar.cluster.is_cluster_job is currently set to true). To remove this error, run records_update on: %s', records_fn); +end if ~exist(records_fn,'file') warning('Records file does not exist: %s (%s).\n', records_fn, datestr(now)); return; end + records = load(records_fn); if isfield(records,'settings') && isfield(records.settings,'wfs') && isfield(records.settings.wfs,'wfs') warning('Old records.settings format with "settings.wfs.wfs" field found in records file. Updating format.'); @@ -39,6 +38,16 @@ function records_update(param,param_override) records.settings = rmfield(records.settings,'wfs_records'); end end +if isfield(records,'settings') && isfield(records.settings,'nyquist_zone_hw') + warning('Old records.settings format with "settings.nyquist_zone_hw" field found in records file. Updating format.'); + records.nyquist_zone_hw = records.settings.nyquist_zone_hw; + records.settings = rmfield(records.settings,'nyquist_zone_hw'); +end +if isfield(records,'settings') && isfield(records.settings,'nyquist_zone') + warning('Old records.settings format with "settings.nyquist_zone" field found in records file. Updating format.'); + records.nyquist_zone_sig = records.settings.nyquist_zone; + records.settings = rmfield(records.settings,'nyquist_zone'); +end if isfield(records.param_records,'vectors') warning('Old parameter format with "vectors" field found in records file. Updating format.'); records.param_records.records.file.start_idx = records.param_records.vectors.file.start_idx; @@ -135,6 +144,22 @@ function records_update(param,param_override) records.gps_source = gps.gps_source; else % Determine time offset delta and apply to radar time + if ~isfinite(param.records.gps.time_offset) + warning('param.records.gps.time_offset must be finite. Trying to load value from spreadsheet.'); + try + new_param = read_param_xls(param); + param.records.gps.time_offset = new_param.records.gps.time_offset; + catch ME + error('param.records.gps.time_offset must be finite. Call records_update with a finite value.'); + end + if ~isfinite(param.records.gps.time_offset) + error('param.records.gps.time_offset must be finite. Call records_update with a finite value.'); + end + end + if ~isfinite(records.param_records.records.gps.time_offset) + warning('records.param_records.records.gps.time_offset must be finite. Assuming that it is equal to param.records.gps.time_offset.'); + records.param_records.records.gps.time_offset = param.records.gps.time_offset; + end delta_offset = max(param.records.gps.time_offset) - max(records.param_records.records.gps.time_offset); records.param_records.records.gps.time_offset = param.records.gps.time_offset; records.gps_time = records.gps_time + delta_offset; @@ -156,17 +181,21 @@ function records_update(param,param_override) records = rmfield(records,'surface'); end -if save_changes - % Save outputs - fprintf(' Saving records %s\n', records_fn); - if param.ct_file_lock - records.file_version = '1L'; - else - records.file_version = '1'; - end - records.file_type = 'records'; - ct_save(records_fn,'-v7.3','-struct','records'); - records_aux_files_create(records_fn); +%% Save outputs +fprintf(' Saving records %s\n', records_fn); +if isfield(param,'ct_file_lock') && param.ct_file_lock + records.file_version = '1L'; else - fprintf(' Not saving information (TEST MODE)\n'); + records.file_version = '1'; +end +records.file_type = 'records'; +ct_save(records_fn,'-v7.3','-struct','records'); + +%% Update reference trajectory +ref_fn_dir = ct_filename_out(param,'reference_trajectory','',1); +ref_fn = fullfile(ref_fn_dir,sprintf('ref_%s.mat', param.day_seg)); +% Only update the file if it exists +if exist(ref_fn,'file') + fprintf(' Updating reference trajectory %s\n', ref_fn); + records_reference_trajectory(param,param_override); end diff --git a/cresis-toolbox/ct_support/run_all.m b/cresis-toolbox/ct_support/run_all.m new file mode 100644 index 00000000..655a17e1 --- /dev/null +++ b/cresis-toolbox/ct_support/run_all.m @@ -0,0 +1,160 @@ +% script run_all +% +% Used by run_all scripts to select parameter spreadsheets to run + +param_fns = {}; + +%% CReSIS Accumulation Radar +% param_fns{end+1} = 'accum_param_2010_Greenland_P3.xls'; +% param_fns{end+1} = 'accum_param_2011_Greenland_P3.xls'; +% param_fns{end+1} = 'accum_param_2012_Greenland_P3.xls'; +% param_fns{end+1} = 'accum_param_2013_Greenland_P3.xls'; +% param_fns{end+1} = 'accum_param_2013_Antarctica_P3.xls'; +% param_fns{end+1} = 'accum_param_2013_Antarctica_Ground.xls'; % Sridar Anandakrishnan Ice Stream A/B +% param_fns{end+1} = 'accum_param_2014_Greenland_P3.xls'; +% param_fns{end+1} = 'accum_param_2015_Greenland_Ground.xls'; % Bo Vinther Renland Ice Cap +% param_fns{end+1} = 'accum_param_2015_Antarctica_Ground.xls'; % Howard Conway Crary Ice Rise +% param_fns{end+1} = 'accum_param_2017_Greenland_P3.xls'; +% param_fns{end+1} = 'accum_param_2018_Greenland_P3.xls'; +% param_fns{end+1} = 'accum_param_2018_Antarctica_TObas.xls'; +% param_fns{end+1} = 'accum_param_2019_Antarctica_TObas.xls'; +% param_fns{end+1} = 'accum_param_2022_Greenland_Ground.xls'; +% param_fns{end+1} = 'accum_param_2022_Antarctica_Ground.xls'; +% param_fns{end+1} = 'accum_param_2022_Antarctica_BaslerMKB.xls'; +% param_fns{end+1} = 'accum_param_2023_Greenland_Ground.xls'; + +%% CReSIS Kuband Radar +% param_fns{end+1} = 'kuband_param_2009_Antarctica_DC8.xls'; +% param_fns{end+1} = 'kuband_param_2010_Greenland_DC8.xls'; +% param_fns{end+1} = 'kuband_param_2010_Greenland_P3.xls'; +% param_fns{end+1} = 'kuband_param_2010_Antarctica_DC8.xls'; +% param_fns{end+1} = 'kuband_param_2011_Greenland_P3.xls'; +% param_fns{end+1} = 'kuband_param_2011_Antarctica_DC8.xls'; +% param_fns{end+1} = 'kuband_param_2011_Antarctica_TO.xls'; +% param_fns{end+1} = 'kuband_param_2012_Greenland_P3.xls'; +% param_fns{end+1} = 'kuband_param_2012_Antarctica_DC8.xls'; +% param_fns{end+1} = 'kuband_param_2013_Greenland_P3.xls'; +% param_fns{end+1} = 'kuband_param_2013_Antarctica_P3.xls'; +% param_fns{end+1} = 'kuband_param_2013_Antarctica_Basler.xls'; +% param_fns{end+1} = 'kuband_param_2014_Antarctica_DC8.xls'; +% param_fns{end+1} = 'kuband_param_2014_Greenland_P3.xls'; +% param_fns{end+1} = 'kuband_param_2015_Greenland_C130.xls'; +% param_fns{end+1} = 'kuband_param_2016_Greenland_P3.xls'; +% param_fns{end+1} = 'kuband_param_2016_Antarctica_DC8.xls'; +% param_fns{end+1} = 'kuband_param_2019_Greenland_TO.xls'; + +%% CReSIS Radar Depth Sounder (RDS) +% param_fns{end+1} = 'rds_param_2009_Antarctica_DC8.xls'; +% param_fns{end+1} = 'rds_param_2009_Antarctica_TO.xls'; +% param_fns{end+1} = 'rds_param_2010_Greenland_DC8.xls'; +% param_fns{end+1} = 'rds_param_2010_Greenland_P3.xls'; +% param_fns{end+1} = 'rds_param_2010_Antarctica_DC8.xls'; +% param_fns{end+1} = 'rds_param_2011_Antarctica_DC8.xls'; +% param_fns{end+1} = 'rds_param_2011_Antarctica_TO.xls'; +% param_fns{end+1} = 'rds_param_2011_Greenland_P3.xls'; +% param_fns{end+1} = 'rds_param_2012_Greenland_P3.xls'; +% param_fns{end+1} = 'rds_param_2012_Antarctica_DC8.xls'; +% param_fns{end+1} = 'rds_param_2013_Greenland_P3.xls'; +% param_fns{end+1} = 'rds_param_2013_Antarctica_P3.xls'; +% param_fns{end+1} = 'rds_param_2013_Antarctica_Basler.xls'; +% param_fns{end+1} = 'rds_param_2014_Greenland_P3.xls'; +% param_fns{end+1} = 'rds_param_2014_Antarctica_DC8.xls'; +% param_fns{end+1} = 'rds_param_2015_Greenland_C130.xls'; +% param_fns{end+1} = 'rds_param_2016_Greenland_P3.xls'; +% param_fns{end+1} = 'rds_param_2016_Greenland_G1XB.xls'; +% param_fns{end+1} = 'rds_param_2016_Greenland_Polar6.xls'; +% param_fns{end+1} = 'rds_param_2016_Greenland_TOdtu.xls'; +% param_fns{end+1} = 'rds_param_2016_Antarctica_DC8.xls'; +% param_fns{end+1} = 'rds_param_2017_Greenland_P3.xls'; +% param_fns{end+1} = 'rds_param_2017_Antarctica_P3.xls'; +% param_fns{end+1} = 'rds_param_2017_Antarctica_Basler.xls'; +% param_fns{end+1} = 'rds_param_2018_Greenland_P3.xls'; +% param_fns{end+1} = 'rds_param_2018_Antarctica_DC8.xls'; +% param_fns{end+1} = 'rds_param_2018_Antarctica_Ground.xls'; +% param_fns{end+1} = 'rds_param_2019_Antarctica_Ground.xls'; +% param_fns{end+1} = 'rds_param_2019_Antarctica_GV.xls'; +% param_fns{end+1} = 'rds_param_2019_Greenland_P3.xls'; +% param_fns{end+1} = 'rds_param_2022_Antarctica_GroundGHOST.xls'; +% param_fns{end+1} = 'rds_param_2022_Antarctica_BaslerMKB.xls'; + +%% CReSIS Snow Radar +% param_fns{end+1} = 'snow_param_2009_Antarctica_DC8.xls'; +% param_fns{end+1} = 'snow_param_2009_Greenland_P3.xls'; +% param_fns{end+1} = 'snow_param_2010_Antarctica_DC8.xls'; +% param_fns{end+1} = 'snow_param_2010_Greenland_DC8.xls'; +% param_fns{end+1} = 'snow_param_2010_Greenland_P3.xls'; +% param_fns{end+1} = 'snow_param_2011_Antarctica_DC8.xls'; +% param_fns{end+1} = 'snow_param_2011_Greenland_P3.xls'; +% param_fns{end+1} = 'snow_param_2012_Antarctica_DC8.xls'; +% param_fns{end+1} = 'snow_param_2012_Greenland_P3.xls'; +% param_fns{end+1} = 'snow_param_2013_Greenland_P3.xls'; +% param_fns{end+1} = 'snow_param_2013_Antarctica_P3.xls'; +% param_fns{end+1} = 'snow_param_2013_Antarctica_Basler.xls'; +% param_fns{end+1} = 'snow_param_2014_Antarctica_DC8.xls'; +% param_fns{end+1} = 'snow_param_2014_Greenland_P3.xls'; +% param_fns{end+1} = 'snow_param_2015_Greenland_C130.xls'; +% param_fns{end+1} = 'snow_param_2016_Greenland_P3.xls'; +% param_fns{end+1} = 'snow_param_2016_Antarctica_DC8.xls'; +% param_fns{end+1} = 'snow_param_2017_Greenland_P3.xls'; +% param_fns{end+1} = 'snow_param_2017_Antarctica_P3.xls'; +% param_fns{end+1} = 'snow_param_2018_Greenland_P3.xls'; +% param_fns{end+1} = 'snow_param_2018_Alaska_SO.xls'; +% param_fns{end+1} = 'snow_param_2018_Antarctica_DC8.xls'; +% param_fns{end+1} = 'snow_param_2019_Greenland_P3.xls'; +% param_fns{end+1} = 'snow_param_2019_Arctic_GV.xls'; +% param_fns{end+1} = 'snow_param_2019_Antarctica_GV.xls'; +% param_fns{end+1} = 'snow_param_2019_SouthDakota_N1KU.xls'; +% param_fns{end+1} = 'snow_param_2020_SouthDakota_N1KU.xls'; +% param_fns{end+1} = 'snow_param_2022_Greenland_P3.xls'; +% param_fns{end+1} = 'snow_param_2022_Antarctica_GroundGHOST.xls'; + +%% CReSIS RDS (data products only) +% param_fns{end+1} = 'rds_param_2009_Greenland_TO_wise.xls'; % Data outputs only +% param_fns{end+1} = 'rds_param_2010_Greenland_TO_wise.xls'; % Data outputs only + +%% CReSIS Accumulation Radar (not complete) + +%% CReSIS Kuband (not complete) +% param_fns{end+1} = 'kuband_param_2014_Alaska_TOnrl.xls'; % Season not complete/NOT CREATED + +%% CReSIS RDS (not complete) +% param_fns{end+1} = 'rds_param_1993_Greenland_P3.xls'; % ICARDS not supported +% param_fns{end+1} = 'rds_param_1995_Greenland_P3.xls'; % ICARDS not supported +% param_fns{end+1} = 'rds_param_1996_Greenland_P3.xls'; % ICARDS not supported +% param_fns{end+1} = 'rds_param_1997_Greenland_P3.xls'; % ICARDS not supported +% param_fns{end+1} = 'rds_param_1998_Greenland_P3.xls'; % ICARDS not supported +% param_fns{end+1} = 'rds_param_1999_Greenland_P3.xls'; % ICARDS not supported +% param_fns{end+1} = 'rds_param_2001_Greenland_P3.xls'; % ICARDS not supported +% param_fns{end+1} = 'rds_param_2002_Antarctica_P3chile.xls'; % ICARDS not supported +% param_fns{end+1} = 'rds_param_2002_Greenland_P3.xls'; % ICARDS not supported +% param_fns{end+1} = 'rds_param_2003_Greenland_P3.xls'; % ACORDS not supported +% param_fns{end+1} = 'rds_param_2004_Antarctica_P3chile.xls'; % ACORDS not supported +% param_fns{end+1} = 'rds_param_2005_Greenland_TO.xls'; % ACORDS not supported/NOT CREATED +% param_fns{end+1} = 'rds_param_2006_Greenland_TO.xls'; % MCRDS not supported +% param_fns{end+1} = 'rds_param_2007_Greenland_P3.xls'; % MCRDS not supported/NOT CREATED +% param_fns{end+1} = 'rds_param_2008_Greenland_Ground.xls'; % SAR not supported +% param_fns{end+1} = 'rds_param_2008_Greenland_TO.xls'; % MCRDS not supported +% param_fns{end+1} = 'rds_param_2009_Greenland_TO.xls'; % MCRDS not supported +% param_fns{end+1} = 'rds_param_2011_Greenland_TO.xls'; % Season not completed + +%% CReSIS Snow Radar (not complete) +% param_fns{end+1} = 'snow_param_2013_Greenland_Ground.xls'; % Season not complete/NOT CREATED +% param_fns{end+1} = 'snow_param_2014_Alaska_TOnrl.xls'; % Season not complete/NOT CREATED +% param_fns{end+1} = 'snow_param_2015_Alaska_TOnrl.xls'; % Season not complete/NOT CREATED +% param_fns{end+1} = 'snow_param_2016_Alaska_TOnrl.xls'; % Season not complete/NOT CREATED +% param_fns{end+1} = 'snow_param_2021_Alaska_SO.xls'; % Season not complete/NOT CREATED + +%% AWI Radar Depth Sounder +% rds_param_2015_Greenland_Polar6 % No good data from this season +% rds_param_2016_Greenland_Polar6 % Listed under CReSIS RDS +% param_fns{end+1} = 'rds_param_2017_Antarctica_Polar6.xls'; +% param_fns{end+1} = 'rds_param_2018_Greenland_Polar6.xls'; +% param_fns{end+1} = 'rds_param_2019_Antarctica_Polar6.xls'; +% param_fns{end+1} = 'rds_param_2021_Greenland_Polar5.xls'; +% param_fns{end+1} = 'rds_param_2022_Greenland_Polar5.xls'; + +%% AWI Snow Radar +% param_fns{end+1} = 'snow_param_2016_Greenland_Polar6.xls'; % Season not completed +% param_fns{end+1} = 'snow_param_2017_Arctic_Polar5.xls'; +% param_fns{end+1} = 'snow_param_2019_Arctic_Polar6.xls'; +% param_fns{end+1} = 'snow_param_2020_Arctic_Polar6.xls'; diff --git a/cresis-toolbox/ct_support/run_all_frames_update.m b/cresis-toolbox/ct_support/run_all_frames_update.m index 0fc47fb9..d6a8358b 100644 --- a/cresis-toolbox/ct_support/run_all_frames_update.m +++ b/cresis-toolbox/ct_support/run_all_frames_update.m @@ -10,68 +10,11 @@ %% User Settings % ========================================================================= -%% User Settings: Select seasons -% ------------------------------------------------------------------------- -param_fns = {}; -% param_fns{end+1} = 'accum_param_2015_Antarctica_Ground.xls'; -% param_fns{end+1} = 'accum_param_2018_Antarctica_TObas.xls'; -% param_fns{end+1} = 'accum_param_2019_Antarctica_TObas.xls'; -% param_fns{end+1} = 'rds_param_1993_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_1995_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_1996_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_1997_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_1998_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_1999_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2001_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2002_Antarctica_P3chile.xls'; % May not work -% param_fns{end+1} = 'rds_param_2002_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2003_Greenland_P3.xls'; % May not work -% param_fns{end+1} = 'rds_param_2004_Antarctica_P3chile.xls'; % May not work -% param_fns{end+1} = 'rds_param_2005_Greenland_TO.xls'; % May not work -% param_fns{end+1} = 'rds_param_2006_Greenland_TO.xls'; -% param_fns{end+1} = 'rds_param_2007_Greenland_P3.xls'; % May not work -% param_fns{end+1} = 'rds_param_2008_Greenland_Ground.xls'; -% param_fns{end+1} = 'rds_param_2008_Greenland_TO.xls'; -% param_fns{end+1} = 'rds_param_2009_Antarctica_DC8.xls'; -% param_fns{end+1} = 'rds_param_2009_Antarctica_TO.xls'; -% param_fns{end+1} = 'rds_param_2009_Greenland_TO.xls'; -% param_fns{end+1} = 'rds_param_2009_Greenland_TO_wise.xls'; -% param_fns{end+1} = 'rds_param_2010_Antarctica_DC8.xls'; -% param_fns{end+1} = 'rds_param_2010_Greenland_DC8.xls'; -% param_fns{end+1} = 'rds_param_2010_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2010_Greenland_TO_wise.xls'; -% param_fns{end+1} = 'rds_param_2011_Antarctica_DC8.xls'; -% param_fns{end+1} = 'rds_param_2011_Antarctica_TO.xls'; -% param_fns{end+1} = 'rds_param_2011_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2011_Greenland_TO.xls'; -% param_fns{end+1} = 'rds_param_2012_Antarctica_DC8.xls'; -% param_fns{end+1} = 'rds_param_2012_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2013_Antarctica_Basler.xls'; -% param_fns{end+1} = 'rds_param_2013_Antarctica_P3.xls'; -% param_fns{end+1} = 'rds_param_2013_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2014_Antarctica_DC8.xls'; -% param_fns{end+1} = 'rds_param_2014_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2015_Greenland_C130.xls'; -% param_fns{end+1} = 'rds_param_2016_Antarctica_DC8.xls'; -% param_fns{end+1} = 'rds_param_2016_Greenland_G1XB.xls'; -% param_fns{end+1} = 'rds_param_2016_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2016_Greenland_Polar6.xls'; -% param_fns{end+1} = 'rds_param_2016_Greenland_TOdtu.xls'; -% param_fns{end+1} = 'rds_param_2017_Antarctica_Basler.xls'; -% param_fns{end+1} = 'rds_param_2017_Antarctica_P3.xls'; -% param_fns{end+1} = 'rds_param_2017_Antarctica_Polar6.xls'; -% param_fns{end+1} = 'rds_param_2017_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2018_Antarctica_DC8.xls'; -% param_fns{end+1} = 'rds_param_2018_Antarctica_Ground.xls'; -% param_fns{end+1} = 'rds_param_2018_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2018_Greenland_Polar6.xls'; -% param_fns{end+1} = 'rds_param_2019_Greenland_P3.xls'; -param_fns{end+1} = 'rds_param_2019_Antarctica_Ground.xls'; -% param_fns{end+1} = 'rds_param_2019_Antarctica_GV.xls'; -% param_fns{end+1} = 'snow_param_2012_Greenland_P3.xls'; -% param_fns{end+1} = 'snow_param_2019_SouthDakota_CESSNA.xls'; +% Select seasons in run_all: +run_all; param_override = []; +param_override.frames_update.force_update = false; %% Automated Section % ========================================================================= diff --git a/cresis-toolbox/ct_support/run_all_layer_update.m b/cresis-toolbox/ct_support/run_all_layer_update.m index ab2c72f1..c5f17ce4 100644 --- a/cresis-toolbox/ct_support/run_all_layer_update.m +++ b/cresis-toolbox/ct_support/run_all_layer_update.m @@ -1,7 +1,9 @@ % script run_all_layer_update % run_all_layer_update % -% Run layer_update on all seasons. +% Run layer_update on all seasons. Consider copying from OPS to CSARP_layer if +% OPS may have newer layer information. You may want to move the existing +% CSARP_layer files before doing this. % % Author: John Paden % @@ -10,71 +12,18 @@ %% User Settings % ========================================================================= -%% User Settings: Select seasons -% ------------------------------------------------------------------------- -param_fns = {}; -% param_fns{end+1} = 'accum_param_2015_Antarctica_Ground.xls'; -% param_fns{end+1} = 'accum_param_2018_Antarctica_TObas.xls'; -% param_fns{end+1} = 'accum_param_2019_Antarctica_TObas.xls'; -% param_fns{end+1} = 'rds_param_1993_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_1995_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_1996_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_1997_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_1998_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_1999_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2001_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2002_Antarctica_P3chile.xls'; % May not work -% param_fns{end+1} = 'rds_param_2002_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2003_Greenland_P3.xls'; % May not work -% param_fns{end+1} = 'rds_param_2004_Antarctica_P3chile.xls'; % May not work -% param_fns{end+1} = 'rds_param_2005_Greenland_TO.xls'; % May not work -% param_fns{end+1} = 'rds_param_2006_Greenland_TO.xls'; -% param_fns{end+1} = 'rds_param_2007_Greenland_P3.xls'; % May not work -% param_fns{end+1} = 'rds_param_2008_Greenland_Ground.xls'; -% param_fns{end+1} = 'rds_param_2008_Greenland_TO.xls'; -% param_fns{end+1} = 'rds_param_2009_Antarctica_DC8.xls'; -% param_fns{end+1} = 'rds_param_2009_Antarctica_TO.xls'; -% param_fns{end+1} = 'rds_param_2009_Greenland_TO.xls'; -% param_fns{end+1} = 'rds_param_2009_Greenland_TO_wise.xls'; -% param_fns{end+1} = 'rds_param_2010_Antarctica_DC8.xls'; -% param_fns{end+1} = 'rds_param_2010_Greenland_DC8.xls'; -% param_fns{end+1} = 'rds_param_2010_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2010_Greenland_TO_wise.xls'; -% param_fns{end+1} = 'rds_param_2011_Antarctica_DC8.xls'; -% param_fns{end+1} = 'rds_param_2011_Antarctica_TO.xls'; -% param_fns{end+1} = 'rds_param_2011_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2011_Greenland_TO.xls'; -% param_fns{end+1} = 'rds_param_2012_Antarctica_DC8.xls'; -param_fns{end+1} = 'rds_param_2012_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2013_Antarctica_Basler.xls'; -% param_fns{end+1} = 'rds_param_2013_Antarctica_P3.xls'; -% param_fns{end+1} = 'rds_param_2013_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2014_Antarctica_DC8.xls'; -% param_fns{end+1} = 'rds_param_2014_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2015_Greenland_C130.xls'; -% param_fns{end+1} = 'rds_param_2016_Antarctica_DC8.xls'; -% param_fns{end+1} = 'rds_param_2016_Greenland_G1XB.xls'; -% param_fns{end+1} = 'rds_param_2016_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2016_Greenland_Polar6.xls'; -% param_fns{end+1} = 'rds_param_2016_Greenland_TOdtu.xls'; -% param_fns{end+1} = 'rds_param_2017_Antarctica_Basler.xls'; -% param_fns{end+1} = 'rds_param_2017_Antarctica_P3.xls'; -% param_fns{end+1} = 'rds_param_2017_Antarctica_Polar6.xls'; -% param_fns{end+1} = 'rds_param_2017_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2018_Antarctica_DC8.xls'; -% param_fns{end+1} = 'rds_param_2018_Antarctica_Ground.xls'; -% param_fns{end+1} = 'rds_param_2018_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2018_Greenland_Polar6.xls'; -% param_fns{end+1} = 'rds_param_2019_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2019_Antarctica_Ground.xls'; -% param_fns{end+1} = 'rds_param_2019_Antarctica_GV.xls'; -% param_fns{end+1} = 'snow_param_2012_Greenland_P3.xls'; -% param_fns{end+1} = 'snow_param_2019_SouthDakota_CESSNA.xls'; +% Select seasons in run_all: +run_all; param_override = []; -param_override.layer_update.in_path = 'layerData'; -param_override.layer_update.out_path = 'layer'; -param_override.layer_update.frames_records_en = true; + +if 0 + % Example to convert old files to new format + param_override.layer_update.in_path = 'layerData'; + param_override.layer_update.out_path = 'layer'; +elseif 0 + % Example to ensure files are in the newest format (this is the default setting) +end %% Automated Section % ========================================================================= diff --git a/cresis-toolbox/ct_support/run_all_records_update.m b/cresis-toolbox/ct_support/run_all_records_update.m index 0b3a0d7b..63f220d8 100644 --- a/cresis-toolbox/ct_support/run_all_records_update.m +++ b/cresis-toolbox/ct_support/run_all_records_update.m @@ -10,66 +10,8 @@ %% User Settings % ========================================================================= -%% User Settings: Select seasons -% ------------------------------------------------------------------------- -param_fns = {}; -% param_fns{end+1} = 'accum_param_2015_Antarctica_Ground.xls'; -% param_fns{end+1} = 'accum_param_2018_Antarctica_TObas.xls'; -% param_fns{end+1} = 'accum_param_2019_Antarctica_TObas.xls'; -% param_fns{end+1} = 'rds_param_1993_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_1995_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_1996_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_1997_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_1998_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_1999_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2001_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2002_Antarctica_P3chile.xls'; % May not work -% param_fns{end+1} = 'rds_param_2002_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2003_Greenland_P3.xls'; % May not work -% param_fns{end+1} = 'rds_param_2004_Antarctica_P3chile.xls'; % May not work -% param_fns{end+1} = 'rds_param_2005_Greenland_TO.xls'; % May not work -% param_fns{end+1} = 'rds_param_2006_Greenland_TO.xls'; -% param_fns{end+1} = 'rds_param_2007_Greenland_P3.xls'; % May not work -% param_fns{end+1} = 'rds_param_2008_Greenland_Ground.xls'; -% param_fns{end+1} = 'rds_param_2008_Greenland_TO.xls'; -% param_fns{end+1} = 'rds_param_2009_Antarctica_DC8.xls'; -% param_fns{end+1} = 'rds_param_2009_Antarctica_TO.xls'; -% param_fns{end+1} = 'rds_param_2009_Greenland_TO.xls'; -% param_fns{end+1} = 'rds_param_2009_Greenland_TO_wise.xls'; -% param_fns{end+1} = 'rds_param_2010_Antarctica_DC8.xls'; -% param_fns{end+1} = 'rds_param_2010_Greenland_DC8.xls'; -% param_fns{end+1} = 'rds_param_2010_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2010_Greenland_TO_wise.xls'; -% param_fns{end+1} = 'rds_param_2011_Antarctica_DC8.xls'; -% param_fns{end+1} = 'rds_param_2011_Antarctica_TO.xls'; -% param_fns{end+1} = 'rds_param_2011_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2011_Greenland_TO.xls'; -% param_fns{end+1} = 'rds_param_2012_Antarctica_DC8.xls'; -% param_fns{end+1} = 'rds_param_2012_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2013_Antarctica_Basler.xls'; -% param_fns{end+1} = 'rds_param_2013_Antarctica_P3.xls'; -% param_fns{end+1} = 'rds_param_2013_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2014_Antarctica_DC8.xls'; -% param_fns{end+1} = 'rds_param_2014_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2015_Greenland_C130.xls'; -% param_fns{end+1} = 'rds_param_2016_Antarctica_DC8.xls'; -% param_fns{end+1} = 'rds_param_2016_Greenland_G1XB.xls'; -% param_fns{end+1} = 'rds_param_2016_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2016_Greenland_Polar6.xls'; -% param_fns{end+1} = 'rds_param_2016_Greenland_TOdtu.xls'; -% param_fns{end+1} = 'rds_param_2017_Antarctica_Basler.xls'; -% param_fns{end+1} = 'rds_param_2017_Antarctica_P3.xls'; -% param_fns{end+1} = 'rds_param_2017_Antarctica_Polar6.xls'; -% param_fns{end+1} = 'rds_param_2017_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2018_Antarctica_DC8.xls'; -% param_fns{end+1} = 'rds_param_2018_Antarctica_Ground.xls'; -% param_fns{end+1} = 'rds_param_2018_Greenland_P3.xls'; -% param_fns{end+1} = 'rds_param_2018_Greenland_Polar6.xls'; -% param_fns{end+1} = 'rds_param_2019_Greenland_P3.xls'; -param_fns{end+1} = 'rds_param_2019_Antarctica_Ground.xls'; -% param_fns{end+1} = 'rds_param_2019_Antarctica_GV.xls'; -% param_fns{end+1} = 'snow_param_2012_Greenland_P3.xls'; -% param_fns{end+1} = 'snow_param_2019_SouthDakota_CESSNA.xls'; +% Select seasons in run_all: +run_all; param_override = []; diff --git a/cresis-toolbox/ct_support/run_frames_create.m b/cresis-toolbox/ct_support/run_frames_create.m index c477572d..ee91aaac 100644 --- a/cresis-toolbox/ct_support/run_frames_create.m +++ b/cresis-toolbox/ct_support/run_frames_create.m @@ -17,7 +17,8 @@ % params = ct_set_params(params,'cmd.generic',1,'cmd.notes','do not process'); % params = ct_set_params(params,'cmd.generic',1,'day_seg','20180929_05'); -param_override.ct_file_lock = 1; +% param_override.records.frames.mode = 2; +% param_override.ct_file_lock = 1; %% Automated Section % ===================================================================== @@ -37,11 +38,12 @@ continue; end % Create frames - if param.records.frames.mode == 0 + test_param = merge_structs(param,param_override); + if test_param.records.frames.mode == 0 fprintf('param.records.frames.mode == 0. Skipping.\n'); - elseif param.records.frames.mode == 1 + elseif test_param.records.frames.mode == 1 frames_create(param,param_override); - elseif param.records.frames.mode >= 2 - autogenerate_frames(param,param_override); + elseif test_param.records.frames.mode >= 2 + frames_autogenerate(param,param_override); end end diff --git a/cresis-toolbox/processing/run_make_layer_files.m b/cresis-toolbox/ct_support/run_frames_update.m similarity index 50% rename from cresis-toolbox/processing/run_make_layer_files.m rename to cresis-toolbox/ct_support/run_frames_update.m index 9fcae4cf..aa525dd1 100644 --- a/cresis-toolbox/processing/run_make_layer_files.m +++ b/cresis-toolbox/ct_support/run_frames_update.m @@ -1,32 +1,28 @@ -% script run_make_layer_files +% script run_frames_update % -% Calls the make_layer_files function +% This script sets up the parameters and calls frames_update. Rename to +% a local copy of the file in your personal folder before making changes. % % Author: John Paden % -% See also: make_layer_files +% See also: run_all_frames_update, run_frames_update, frames_update %% User Settings % ========================================================================= param_override = []; % Parameters spreadsheet to use for updating +% 1. Segment and frame list are taken from the parameter sheet +% 2. For GPS update, GPS time offsets are pulled from the parameter sheet params = read_param_xls(ct_filename_param('rds_param_2019_Antarctica_Ground.xls')); params = ct_set_params(params,'cmd.generic',0); -params = ct_set_params(params,'cmd.generic',1,'day_seg','20191231'); -params = ct_set_params(params,'cmd.frms',[]); +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20191231'); +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20200107'); +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20200108'); -% .out_path: string containing file path where layer files will be stored -% to; string is passed to ct_filename_out to form the file path. -% Default is 'layer'. -param_override.make_layer_files.out_path = 'layer'; -% param_override.make_layer_files.out_path = 'CSARP_post/layer'; - -% Uncomment to delete all layer files and start over: -param_override.make_layer_files.update_mode = 1; - -% Uncomment to even update files that already exist: -% param_override.make_layer_files.update_mode = 2; +% param.frames_update.force_update: required if only updating gps_time from +% records file. +% param_override.frames_update.force_update = true; %% Automated section % ========================================================================= @@ -46,6 +42,5 @@ continue; end - make_layer_files(param,param_override); - %make_layer_files + frames_update(param,param_override); end diff --git a/cresis-toolbox/ct_support/run_get_raw_files.m b/cresis-toolbox/ct_support/run_get_raw_files.m new file mode 100644 index 00000000..7f5d87ee --- /dev/null +++ b/cresis-toolbox/ct_support/run_get_raw_files.m @@ -0,0 +1,72 @@ +% script run_get_raw_files +% +% Script for running get_raw_files. +% +% Creates an output file containing the tape locations of +% all the files comprising the given frames. +% +% Authors: Reece Mathews +% +% See also: get_raw_files.m + +%% USER INPUT + +seasons = {'accum_param_2010_Greenland_P3', 'accum_param_2011_Greenland_P3', 'accum_param_2012_Greenland_P3', 'accum_param_2013_Greenland_P3', 'accum_param_2014_Greenland_P3', 'accum_param_2017_Greenland_P3', 'accum_param_2018_Greenland_P3'}; +flight_lines = { + {'20100508_01_112', '20100508_01_113', '20100508_01_114', '20100508_01_115', '20100508_01_116', '20100508_01_117', '20100508_01_118'}, ... + {'20110419_01_006', '20110419_01_007', '20110419_01_008', '20110419_01_009', '20110419_01_010', '20110419_01_011', '20110419_01_012'}, ... + {'20120418_01_127', '20120418_01_128', '20120418_01_129', '20120418_01_130', '20120418_01_131', '20120418_01_132'}, ... + {'20130405_01_163', '20130405_01_164', '20130405_01_165', '20130405_01_166', '20130405_01_167', '20130405_01_168', '20130405_01_169'}, ... + {'20140424_01_001', '20140424_01_002', '20140424_01_003', '20140424_01_004', '20140424_01_005', '20140424_01_005'}, ... + {'20170422_01_167', '20170422_01_168', '20170422_01_169', '20170422_01_170', '20170422_01_168', '20170422_01_172'}, ... + {'20180427_01_168', '20180427_01_169', '20180427_01_170', '20180427_01_171', '20180427_01_172', '20180427_01_173'}, ... +}; + +tape_list_path = '~/RAS_files_tapes_table.txt'; +output_file = 'tapes.txt'; + +%% AUTOMATED PART + +% Produce tape_list sorted by filenames +tape_list = readmatrix(tape_list_path, 'Delimiter', ' ', 'OutputType', 'string'); +[~, file, ext] = fileparts(tape_list(:, 2)); +tape_list = [tape_list file + ext]; +tape_list = sortrows(tape_list, 3); + +% Map directories to the corresponding small file archives +small_file_archives = string(); +for file_idx = 1:size(tape_list, 1) + filepath = tape_list{file_idx, 2}; + tapes = tape_list(file_idx, 1); + if endsWith(filepath, "small_file_archive.tar") + [parent, file_name, ext] = fileparts(filepath); + file_name = [file_name ext]; + archive_idx = size(small_file_archives, 1) + 1; + small_file_archives(archive_idx, 1) = tapes; + small_file_archives(archive_idx, 2) = convertCharsToStrings(parent); + small_file_archives(archive_idx, 3) = convertCharsToStrings(file_name); + end +end +small_file_archives = sortrows(small_file_archives, 2); + + +output_fid = fopen(output_file,'w'); + +for season_idx=1:length(seasons) + season_name = ct_filename_param(seasons{season_idx}); + frame_idxs = flight_lines{season_idx}; + + [load_info,gps_time,recs] = get_raw_files(season_name, frame_idxs, {}, {}, {}, {}, tape_list, small_file_archives); + + fprintf(output_fid, '%s\n', seasons{season_idx}); + for file_list=1:length(load_info.filenames) + fprintf(output_fid, 'Filelist: %d\n', file_list); + fprintf(output_fid, 'tapes filename stored_filename\n'); + for file_idx=1:length(load_info.filenames{file_list}) + fprintf(output_fid, '%s %s %s\n', load_info.tapes{file_list}{file_idx}, load_info.filenames{file_list}{file_idx}, load_info.stored_filenames{file_list}{file_idx}); + end + end + fprintf(output_fid, '\n'); +end + +fclose(output_fid); \ No newline at end of file diff --git a/cresis-toolbox/ct_support/run_layer_tracker.m b/cresis-toolbox/ct_support/run_layer_tracker.m deleted file mode 100644 index 66a9cef2..00000000 --- a/cresis-toolbox/ct_support/run_layer_tracker.m +++ /dev/null @@ -1,187 +0,0 @@ -% script run_layer_tracker -% -% Runs layer_tracker.m -% -% Author: John Paden - -%% User Settings -% ---------------------------------------------------------------------- -param_override = []; - -% params = read_param_xls(ct_filename_param('rds_param_2018_Greenland_P3.xls')); -params = read_param_xls(ct_filename_param('snow_param_2017_Arctic_Polar5.xls')); - -params = ct_set_params(params,'cmd.generic',0); -params = ct_set_params(params,'cmd.generic',1,'day_seg','20170330_01'); -params = ct_set_params(params,'cmd.frms',[1]); % Specify specific frames (or leave empty/undefined to do all frames) - -param_override.layer_tracker.debug_plots = {'debug'}; - -param_override.layer_tracker.echogram_img = 0; % To choose an image besides the base (0) image -% echogram_source: location of echogram data used for tracking -% param_override.layer_tracker.echogram_source = 'CSARP_post/qlook'; -param_override.layer_tracker.echogram_source = 'qlook'; - -% layer_params: structure of layer references of where to store the output -param_override.layer_tracker.layer_params = []; idx = 0; -% idx = idx + 1; -% param_override.layer_tracker.layer_params(idx).name = 'surface'; -% param_override.layer_tracker.layer_params(idx).source = 'echogram'; -% param_override.layer_tracker.layer_params(idx).echogram_source = 'deconv'; -idx = idx + 1; -param_override.layer_tracker.layer_params(idx).name = 'surface'; -param_override.layer_tracker.layer_params(idx).source = 'layerdata'; -param_override.layer_tracker.layer_params(idx).layerdata_source = 'layerData'; -param_override.layer_tracker.layer_params(idx).echogram_source = 'qlook'; -% idx = idx + 1; -% param_override.layer_tracker.layer_params(idx).name = 'surface'; -% param_override.layer_tracker.layer_params(idx).source = 'ops'; - -%% Enable one set of parameters -track_override = []; -track_override.en = true; -switch ct_output_dir(params(1).radar_name) - case 'rds' - % RDS - track_override.profile = 'rds_OIB'; - - % Override default filter settings for low AGL - if 0 - track_override.min_bin = 0.75e-6; - end - - % Override default filter settings for broad bandwidth - if 0 - track_override.max_rng = [0 10]; - track_override.max_rng_units = 'bins'; - end - - % Override default filter settings for rapidly changing elevation - if 0 - track_override.filter = [5 1]; - end - - % Override default filter settings - if 0 - track_override.filter = [3 3]; - track_override.filter_trim = [3 3]; - track_override.threshold = 10; - track_override.max_rng = [0 2]; - end - - % Use sidelobe rejection - if 0 - % run_get_echogram_stats output - sidelobe = load('/N/dcwan/projects/cresis/output/ct_tmp/echogram_stats/rds/2018_Greenland_P3/stats_20180421_01.mat','sidelobe_rows','sidelobe_dB','sidelobe_vals'); - track_override.sidelobe_rows = [sidelobe.sidelobe_rows(75:98)]; - track_override.sidelobe_dB = -(sidelobe.sidelobe_dB(75:98,1)-max(sidelobe.sidelobe_dB(:,1))+21); - track_override.sidelobe_dB(track_override.sidelobe_dB<9) = 9; - track_override.threshold_rel_max = -max(track_override.sidelobe_dB); - track_override.data_noise_en = true; - end - - % Use feedthrough rejection - if 0 - % run_get_echogram_stats output - feedthru = load('/N/dcwan/projects/cresis/output/ct_tmp/echogram_stats/rds/2018_Greenland_P3/stats_20180421_01.mat'); - track_override.feedthru.time = feedthru.dt*feedthru.bins; - track_override.feedthru.power_dB = feedthru.min_means+20; - bin_mask = track_override.feedthru.time<2e-6; - track_override.feedthru.time = track_override.feedthru.time(bin_mask); - track_override.feedthru.power_dB = track_override.feedthru.power_dB(bin_mask); - track_override.feedthru.power_dB(end) = -inf; - track_override.min_bin = 0.5e-6; - track_override.data_noise_en = true; - end - - % Override default init method - if 0 - track_override.init.method = 'dem'; - track_override.init.dem_offset = 0; - track_override.init.dem_layer.name = 'surface'; - track_override.init.dem_layer.source = 'lidar'; - track_override.init.dem_layer.lidar_source = 'atm'; - track_override.init.max_diff = 1e-6; - track_override.init.max_diff_method = 'merge_vectors'; - elseif 0 - track_override.init.method = 'snake'; - track_override.init.snake_rng = [-0.5e-6 0.5e-6]; - track_override.init.max_diff = 0.5e-6; - end - - % Override default method - if 0 - track_override.method = 'snake'; - track_override.snake_rng = [-0.15e-6 0.15e-6]; - end - - case 'accum' - % ACCUM - track_override.profile = 'ACCUM'; - - % Override default init method - if 0 - track_override.init.method = 'dem'; - track_override.init.dem_offset = 0; - track_override.init.dem_layer.name = 'surface'; - track_override.init.dem_layer.source = 'lidar'; - track_override.init.dem_layer.lidar_source = 'atm'; - track_override.init.max_diff = 0.3e-6; - track_override.init.max_diff_method = 'merge_vectors'; - end - - % Override default method - if 0 - track_override.method = 'snake'; - track_override.snake_rng = [-0.15e-6 0.15e-6]; - end - - case {'snow','kuband','kaband'} - % FMCW - track_override.profile = 'snow_AWI'; - - % Use sidelobe rejection - if 0 - % run_get_echogram_stats output - sidelobe = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/echogram_stats/snow/2011_Greenland_P3/stats_20110329_01.mat','sidelobe_rows','sidelobe_dB','sidelobe_vals'); - track_override.sidelobe_rows = [sidelobe.sidelobe_rows(1:194)]; - track_override.sidelobe_dB = [-sidelobe.sidelobe_dB(1:194,end)]-sidelobe.sidelobe_vals(end)+4.5; - track_override.threshold_rel_max = -max(track_override.sidelobe_dB); - end - - % Override default init method - if 0 - track_override.init.method = 'dem'; - track_override.init.dem_offset = 0; - track_override.init.dem_layer.name = 'surface'; - track_override.init.dem_layer.source = 'lidar'; - track_override.init.dem_layer.lidar_source = 'atm'; - track_override.init.max_diff = 0.3e-6; - track_override.init.max_diff_method = 'merge_vectors'; - elseif 0 - track_override.init.method = 'snake'; - track_override.init.snake_rng = [-15e-9 15e-9]; - track_override.init.max_diff = 0.3e-6; - end - -end -param_override.layer_tracker.track = track_override; - -%% Automated Section -% ---------------------------------------------------------------------- - -% Input checking -global gRadar; -if exist('param_override','var') - param_override = merge_structs(gRadar,param_override); -else - param_override = gRadar; -end - -% Process each of the segments -for param_idx = 1:length(params) - param = params(param_idx); - if isfield(param.cmd,'generic') && ~iscell(param.cmd.generic) && ~ischar(param.cmd.generic) && param.cmd.generic - layer_tracker(param,param_override); - end -end diff --git a/cresis-toolbox/ct_support/run_load_sar_data.m b/cresis-toolbox/ct_support/run_load_sar_data.m deleted file mode 100644 index 6f455efb..00000000 --- a/cresis-toolbox/ct_support/run_load_sar_data.m +++ /dev/null @@ -1,92 +0,0 @@ -% script run_load_sar_data -% -% Load SAR processed data (e.g. output of csarp stage) -% -% Author: John Paden, Logan Smith - -warning('This is an example file, copy to personal directory and remove this warning/return to use'); -%return; - -% ======================================================================= -% User Settings -% ======================================================================= - -param = read_param_xls(ct_filename_param('rds_param_2019_Antarctica_Ground.xls'),'20191231_04'); - -% param.load_sar_data.fn = ''; % Leave empty for default 'sar' - -% Start and stop chunk to load (inf for second element loads to the end) -% param.load_sar_data.chunk = []; % Leave empty for all chunks - -% param.load_sar_data.sar_type = ''; % Leave empty for default 'fk' - -param.load_sar_data.frm = 1; % Specify data frame to load - -% param.load_sar_data.subap = []; % Leave empty for default (all subapertures) - -% {Images} with [wf,adc] pairs to load -% param.load_sar_data.imgs = {[1 1; 1 2; 1 3; 1 4; 1 5; 1 6]}; -% param.load_sar_data.imgs = {[2 2; 2 3; 2 4; 2 5; 2 6; 2 7; 2 9; 2 10; 2 11; 2 12; 2 13; 2 14]}; -param.load_sar_data.imgs = {[1*ones(8,1) (1:8)'],[2*ones(8,1) (1:8)'],[3*ones(8,1) (1:8)']}; - -% Debug level (1 = default) -param.load_sar_data.debug_level = 2; - -% Combine receive channels -% param.load_sar_data.combine_channels = 0; - -% Combine waveforms parameters -% param.load_sar_data.wf_comb = 10e-6; - -% Take abs()^2 of the data (only runs if combine_channels runs) -% param.load_sar_data.incoherent = 0; - -% Combine waveforms (only runs if incoherent runs) -% param.load_sar_data.combine_imgs = 0; - -% Parameters for local_detrend (cmd == 5 disables, only runs if incoherent runs) -% param.load_sar_data.detrend.cmd = 3; -% param.load_sar_data.detrend.B_noise = [100 200]; -% param.load_sar_data.detrend.B_sig = [1 10]; -% param.load_sar_data.detrend.minVal = -inf; - -[data,metadata] = load_sar_data(param); - -return - - -%% Examples of manipulating multilook data -good_mask = metadata.wfs(1).time > 0e-6 & metadata.wfs(1).time < 30e-6; - -% Step through multilooks -subap = 0; -figure(1); clf; -while 1 - subap = mod(subap,size(data{1},4))+1; - imagesc([],metadata.wfs(1).time(good_mask),lp(mean(data{1}(good_mask,:,:,subap),3)),[-150 -23]) - title(sprintf('%.0f',subap)); - if subap == 1 - beep - end - pause; % pause for keystroke advance or pause(0.1) for animation -end - - -figure(2); clf; -imagesc([],metadata.wfs(1).time(good_mask),lp(local_detrend(mean(abs(mean(data{1}(good_mask,:,:,[6:16]),3)).^2,4),[40 200],[1 3]))) - -figure(2); clf; -imagesc([],metadata.wfs(1).time(good_mask),lp(mean(abs(mean(data{1}(good_mask,:,:,[6:16]),3)).^2,4)),[-150 -23]) - -figure(2); clf; -imagesc([],metadata.wfs(1).time(good_mask),lp(mean(abs(mean(data{1}(good_mask,:,:,[1:6]),3)).^2,4)),[-150 -23]) - -figure(3); clf; -imagesc([],metadata.wfs(1).time(good_mask),lp(mean(abs(mean(data{1}(good_mask,:,:,[end-5:end]),3)).^2,4)),[-150 -23]) - - - - - - - diff --git a/cresis-toolbox/ct_support/run_preprocess.m b/cresis-toolbox/ct_support/run_preprocess.m index cea7188a..f4897c8b 100644 --- a/cresis-toolbox/ct_support/run_preprocess.m +++ b/cresis-toolbox/ct_support/run_preprocess.m @@ -14,7 +14,7 @@ % ========================================================================= param_override = []; param = []; -run_preprocess_BAS; % REPLACE THIS LINE WITH CORRECT SETUP SCRIPT +run_preprocess_settings_BAS; % REPLACE THIS LINE WITH CORRECT SETTINGS SCRIPT dbstop if error; param_override.config.reuse_tmp_files = true; diff --git a/cresis-toolbox/ct_support/run_quality_control.m b/cresis-toolbox/ct_support/run_quality_control.m new file mode 100644 index 00000000..6ff50181 --- /dev/null +++ b/cresis-toolbox/ct_support/run_quality_control.m @@ -0,0 +1,103 @@ +% script run_quality_control +% +% Script for running "quality_control.m" +% +% Author: John Paden +% +% See also: quality_control.m, run_quality_control.m + +clear param; + +if 0 + param.radar_name = 'rds'; + + param.day_seg = '20140514_01'; + + param.season_name = '2014_Greenland_P3'; + param.image_out_dir = { + {ct_filename_out(rmfield(param,'day_seg'),'','post/images',1)}}; + param.mat_out_dir = {{'/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_post/CSARP_standard'}}; + param.mat_out_img = {0}; + + param.img_type = 'echo.jpg'; + + % RDS image debugging mode (for snow, kuband radars) + fmcw_img_debug_mode = false; + noise_time_buffer = 250e-9; + noise_time_duration = 45e-9; + img_sidelobe = -40; + noise_threshold_offset_dB = 5.2; + +elseif 1 + param.radar_name = 'snow'; + + param.day_seg = '20170407_02'; + + param.season_name = '2017_Greenland_P3'; + param.image_out_dir = {{ct_filename_out(rmfield(param,'day_seg'),'','post/images',1)}, + {ct_filename_out(rmfield(param,'day_seg'),'','post_uwb/images',1)}, + {ct_filename_out(rmfield(param,'day_seg'),'','post_deconv/images',1)}, + {ct_filename_out(rmfield(param,'day_seg'),'','post_kuband/images',1)}}; + param.mat_out_dir = {{'/N/dcwan/projects/cresis/output/snow/2017_Greenland_P3/CSARP_qlook'}, + {'/N/dcwan/projects/cresis/output/snow/2017_Greenland_P3/CSARP_qlook_uwb'}, + {'/N/dcwan/projects/cresis/output/snow/2017_Greenland_P3/CSARP_deconv'}, + {'/N/dcwan/projects/cresis/output/snow/2017_Greenland_P3/CSARP_qlook_kuband'}}; + param.mat_out_img = {0,0,0,0}; + + param.img_type = 'echo.jpg'; + + % FMCW image debugging mode (for snow, kuband radars) + fmcw_img_debug_mode = true; + noise_time_buffer = 250e-9; + noise_time_duration = 45e-9; + img_sidelobe = -40; + noise_threshold_offset_dB = 5.2; +end + +% update_field: string containing the field in frames file to update. +% Currently there are two options: +% ('proc_mode', 'quality') +if 0 + update_field = 'proc_mode'; update_field_type = 'double'; +else + update_field = 'quality'; update_field_type = 'mask'; + update_field_mask = {'turn all masks off','coherent noise','deconvolution artifact', ... + 'raised noise floor/vertical stripes','missing data','no good data','low SNR','unclassified','land or iceberg'}; + % 8-bit mask: + % + % 1: coherent noise -- horizontal lines in "i" view which plots against + % two way travel. In the posted image view which uses WGS-84, coherent + % noise is not horizontal but can be distinguished by parallel lines. If + % unusual line present, switch to "i" view that plots against two way + % travel time to verify this as coherent noise. Mark this bit even if a + % part of frame has them. Also, only mark as noise if the lines exceed + % the color threshold in the "I" view which shows gray scale + HSV scale. + % + % 2: deconvolution artifact -- rising edge sidelobe levels to high, use + % "i" mode to verify the sidelobe exceeds the threshold (use the "I" mode + % which shows a gray scale + HSV scale and mark sidelobes that are in + % color), ignore first sidelobe if very close in to peak as the first + % hanning sidelobe is expected to exceed 40 dB + % + % 3: raised noise floor/vertical stripes -- vertical stripes when noise + % level increases suddenly + % + % 4: missing data -- see a white space in posted image + % 5: no good data -- salt and pepper noise type (no obvious signal present) + % + % 6: low SNR -- barely visible sea ice or surface, usually caused by weak + % signal (e.g. caused by high altitude, vertical stripe noise, or very + % fast altitude changes) + % + % 7: unclassified -- any other kind of artifact that does not fit into the other categories + % 8: land or iceberg -- if any land or iceberg is present in the frame (iceberg > 3 m above average surface height) + % All bits are independent of each other. A frame with missing data+coherent noise+deconv artifact will be 124 instead of just 4 +end + +audio_tone_for_nonzero_nonisnan = true; +audio_tone_check_code = '~isnan(frames.(update_field)(frm)) && frames.(update_field)(frm) ~= 0'; + +%update_field_match = [1 2 3]; +update_field_match = []; + +quality_control; diff --git a/cresis-toolbox/ct_support/run_records_bit_mask.m b/cresis-toolbox/ct_support/run_records_bit_mask.m index 45877f33..af077aa8 100644 --- a/cresis-toolbox/ct_support/run_records_bit_mask.m +++ b/cresis-toolbox/ct_support/run_records_bit_mask.m @@ -10,15 +10,16 @@ param_override = []; params = read_param_xls(ct_filename_param('rds_param_2019_Antarctica_Ground.xls'),''); -% params = ct_set_params(params,'cmd.generic',0); -% params = ct_set_params(params,'cmd.generic',1,'day_seg','20200108_02'); -params = ct_set_params(params,'cmd.generic',1); -params = ct_set_params(params,'cmd.generic',0,'cmd.notes','do not process'); +params = ct_set_params(params,'cmd.generic',0); +params = ct_set_params(params,'cmd.generic',1,'day_seg','20200107_01'); +% params = ct_set_params(params,'cmd.generic',1); +% params = ct_set_params(params,'cmd.generic',0,'cmd.notes','do not process'); -param_override.records_bit_mask.manual_masking = 0; -param_override.records_bit_mask.debug_plot = 0; -param_override.records_bit_mask.bad_vel_threshold = 0.1; % <== OFTEN CHANGED (0.25 default) +param_override.records_bit_mask.bad_vel_threshold = 0.25; % <== OFTEN CHANGED (0.25 default) param_override.records_bit_mask.bad_heading_diff_threshold = Inf; % <== OFTEN CHANGED (1 default) +param_override.records_bit_mask.manual_masking_en = true; % <== Set to true to enable manual GUI (true default) +param_override.records_bit_mask.debug_plot = false; +param_override.records_bit_mask.mode = 1; % Default is 1: start with current bit_mask settings (0: resets bit_mask, 2: starts with automated mask) %% Automated Section % ===================================================================== @@ -39,4 +40,3 @@ end records_bit_mask(param,param_override); end - diff --git a/cresis-toolbox/ct_support/run_records_update.m b/cresis-toolbox/ct_support/run_records_update.m index 308fe1ea..23b8d17c 100644 --- a/cresis-toolbox/ct_support/run_records_update.m +++ b/cresis-toolbox/ct_support/run_records_update.m @@ -14,9 +14,11 @@ % Parameters spreadsheet to use for updating % 1. Segment and frame list are taken from the parameter sheet % 2. For GPS update, GPS time offsets are pulled from the parameter sheet -params = read_param_xls(ct_filename_param('snow_param_2018_Greenland_P3.xls')); +params = read_param_xls(ct_filename_param('rds_param_2019_Antarctica_Ground.xls')); params = ct_set_params(params,'cmd.generic',0); -params = ct_set_params(params,'cmd.generic',1,'day_seg','20180315'); +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20191231'); +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20200107'); +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20200108'); %% Automated section % ========================================================================= diff --git a/cresis-toolbox/ct_support/run_sar_load.m b/cresis-toolbox/ct_support/run_sar_load.m new file mode 100644 index 00000000..e52d1b68 --- /dev/null +++ b/cresis-toolbox/ct_support/run_sar_load.m @@ -0,0 +1,96 @@ +% script run_load_sar_data +% +% Example for running sar_load.m +% +% Author: John Paden, Logan Smith + +warning('This is an example file, copy to personal directory, rename, and remove this warning/return to use'); + +% ======================================================================= +% User Settings +% ======================================================================= + +% param = read_param_xls(ct_filename_param('rds_param_2019_Antarctica_Ground.xls'),'20200107_01'); +param = read_param_xls(ct_filename_param('rds_param_2018_Greenland_P3.xls'),'20180501_01'); + +% param.sar_load.in_path = ''; % Leave empty for default 'sar' + +% param.sar_load.chunk: One cell entry per frame; leave empty for all +% chunks Each entry is two positive integers specifying the start and stop +% chunk to load (inf for second element loads to the end) +% param.sar_load.chunk = {}; + +% param.sar_load.sar_type = ''; % Leave empty for default 'fk' + +param.sar_load.frms = [51 52]; % Specify data frames to load + +% param.sar_load.subap = []; % Leave empty for default (all subapertures) + +% {Images} with [wf,adc] pairs to load +param.sar_load.imgs = {[1 1]}; +% param.sar_load.imgs = {[1 1; 1 2; 1 3; 1 4; 1 5; 1 6]}; +% param.sar_load.imgs = {[2 2; 2 3; 2 4; 2 5; 2 6; 2 7; 2 9; 2 10; 2 11; 2 12; 2 13; 2 14]}; +% param.sar_load.imgs = {[1*ones(8,1) (1:8)'],[2*ones(8,1) (1:8)'],[3*ones(8,1) (1:8)']}; +% param.sar_load.imgs = {[1*ones(8,1) (1:8)'],[2*ones(8,1) (1:8)'],[[3*ones(8,1) (1:8)'];[4*ones(8,1) (1:8)']]}; + +% Debug level (1 = default) +param.sar_load.debug_level = 2; + +% Combine receive channels +% param.sar_load.combine_channels = 0; + +% Combine waveforms parameters +% param.sar_load.wf_comb = 10e-6; + +% Take abs()^2 of the data (only runs if combine_channels runs) +% param.sar_load.incoherent = 0; + +% Combine waveforms (only runs if incoherent runs) +% param.sar_load.combine_imgs = 0; + +% Parameters for local_detrend (cmd == 5 disables, only runs if incoherent runs) +% param.sar_load.detrend.cmd = 3; +% param.sar_load.detrend.B_noise = [100 200]; +% param.sar_load.detrend.B_sig = [1 10]; +% param.sar_load.detrend.minVal = -inf; + +[data,metadata] = sar_load(param); + +return + + +%% Examples of manipulating multilook data +good_mask = metadata.wfs(1).time > 0e-6 & metadata.wfs(1).time < 30e-6; + +% Step through multilooks +subap = 0; +figure(1); clf; +while 1 + subap = mod(subap,size(data{1},4))+1; + imagesc([],metadata.wfs(1).time(good_mask),lp(mean(data{1}(good_mask,:,:,subap),3)),[-150 -23]) + title(sprintf('%.0f',subap)); + if subap == 1 + beep + end + pause; % pause for keystroke advance or pause(0.1) for animation +end + + +figure(2); clf; +imagesc([],metadata.wfs(1).time(good_mask),lp(local_detrend(mean(abs(mean(data{1}(good_mask,:,:,[6:16]),3)).^2,4),[40 200],[1 3]))) + +figure(2); clf; +imagesc([],metadata.wfs(1).time(good_mask),lp(mean(abs(mean(data{1}(good_mask,:,:,[6:16]),3)).^2,4)),[-150 -23]) + +figure(2); clf; +imagesc([],metadata.wfs(1).time(good_mask),lp(mean(abs(mean(data{1}(good_mask,:,:,[1:6]),3)).^2,4)),[-150 -23]) + +figure(3); clf; +imagesc([],metadata.wfs(1).time(good_mask),lp(mean(abs(mean(data{1}(good_mask,:,:,[end-5:end]),3)).^2,4)),[-150 -23]) + + + + + + + diff --git a/cresis-toolbox/ct_support/run_update_frames.m b/cresis-toolbox/ct_support/run_update_frames.m deleted file mode 100644 index 0ba0af5b..00000000 --- a/cresis-toolbox/ct_support/run_update_frames.m +++ /dev/null @@ -1,70 +0,0 @@ -% script run_update_frames -% -% Script for running "update_frames.m" - -clear param; - -if 1 - param.radar_name = 'rds'; - - param.day_seg = '20140514_01'; - - param.season_name = '2014_Greenland_P3'; - param.image_out_dir = { - {ct_filename_out(rmfield(param,'day_seg'),'','post/images',1)}}; - param.mat_out_dir = {{'/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_post/CSARP_standard'}}; - param.mat_out_img = {0}; - - param.img_type = 'echo.jpg'; - - % RDS image debugging mode (for snow, kuband radars) - fmcw_img_debug_mode = false; - noise_time_buffer = 250e-9; - noise_time_duration = 45e-9; - img_sidelobe = -40; - noise_threshold_offset_dB = 5.2; - -elseif 0 - param.radar_name = 'snow'; - - param.day_seg = '20170330_01'; - - param.season_name = '2017_Greenland_P3'; - param.image_out_dir = {{ct_filename_out(rmfield(param,'day_seg'),'','post/images',1)}, - {ct_filename_out(rmfield(param,'day_seg'),'','post/images_uwb',1)}, - {ct_filename_out(rmfield(param,'day_seg'),'','post/images_kuband',1)}, - {ct_filename_out(rmfield(param,'day_seg'),'','post/images_deconv',1)}}; - param.mat_out_dir = {{'dummy_string'}, - {'/cresis/snfs1/dataproducts/ct_data/snow/2017_Greenland_P3/CSARP_post/CSARP_qlook_uwb'}, - {'/cresis/snfs1/dataproducts/ct_data/snow/2017_Greenland_P3/CSARP_post/CSARP_qlook_kuband'}, - {'/cresis/snfs1/dataproducts/ct_data/snow/2017_Greenland_P3/CSARP_post/CSARP_deconv'}}; - param.mat_out_img = {0,0,0,0}; - - param.img_type = 'echo.jpg'; - - % FMCW image debugging mode (for snow, kuband radars) - fmcw_img_debug_mode = true; - noise_time_buffer = 250e-9; - noise_time_duration = 45e-9; - img_sidelobe = -40; - noise_threshold_offset_dB = 5.2; -end - -% update_field: string containing the field in frames file to update. -% Currently there are two options: -% ('proc_mode', 'quality') -if 1 - update_field = 'proc_mode'; update_field_type = 'double'; -else - update_field = 'quality'; update_field_type = 'mask'; - update_field_mask = {'turn all masks off','coherent noise','deconvolution artifact', ... - 'raised noise floor/vertical stripes','missing data','no good data','low SNR','unclassified','land or iceberg','sea ice'}; -end - -audio_tone_for_nonzero_nonisnan = true; -audio_tone_check_code = '~isnan(frames.(update_field)(frm)) && frames.(update_field)(frm) ~= 0'; - -%update_field_match = [1 2 3]; -update_field_match = []; - -update_frames; diff --git a/cresis-toolbox/ct_support/run_update_surface_twtt_delta.m b/cresis-toolbox/ct_support/run_update_surface_twtt_delta.m index 46887779..5603d612 100644 --- a/cresis-toolbox/ct_support/run_update_surface_twtt_delta.m +++ b/cresis-toolbox/ct_support/run_update_surface_twtt_delta.m @@ -6,28 +6,30 @@ %% User Settings % ---------------------------------------------------------------------- +params = read_param_xls(ct_filename_param('accum_param_2019_Antarctica_TObas.xls')); % params = read_param_xls(ct_filename_param('snow_param_2017_Greenland_P3.xls')); -params = read_param_xls(ct_filename_param('rds_param_2018_Greenland_P3.xls')); +% params = read_param_xls(ct_filename_param('rds_param_2018_Greenland_P3.xls')); -params = ct_set_params(params,'cmd.generic',0); -params = ct_set_params(params,'cmd.generic',1,'day_seg','20180404_02'); +% params = ct_set_params(params,'cmd.generic',0); +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20180404_02'); % params = ct_set_params(params,'cmd.frms',[]); -% params = ct_set_params(params,'cmd.generic',1); -% params = ct_set_params(params,'cmd.generic',0,'cmd.notes','Do not process'); +params = ct_set_params(params,'cmd.generic',1); +params = ct_set_params(params,'cmd.generic',0,'cmd.notes','Do not process'); -params = ct_set_params(params,'update_surface_twtt_delta.data_types',{'music_imgs4_Nsig2'}); -% params = ct_set_params(params,'update_surface_twtt_delta.data_types',{'qlook','deconv','qlook_uwb','qlook_kuband'}); -params = ct_set_params(params,'update_surface_twtt_delta.imgs',[0 1 2 3 4]); -params = ct_set_params(params,'update_surface_twtt_delta.update_adc_gains_dB',0); +params = ct_set_params(params,'update_surface_twtt_delta.data_types',{'qlook'}); +% params = ct_set_params(params,'update_surface_twtt_delta.data_types',{'qlook','deconv','qlook_uwb','qlook_kuband'}); % Snow Radar +params = ct_set_params(params,'update_surface_twtt_delta.imgs',[0 1 2]); % 0: combined img, 1+: img_II files +params = ct_set_params(params,'update_surface_twtt_delta.update_radiometric',false); -for param_idx = 1:length(params) - %param = params(param_idx); - for wf = 1:length(params(param_idx).radar.wfs) - params(param_idx).radar.wfs(wf).Tsys = [0.46 -4.66 0.14 -1.77 0 -2.63 -3.38 -69.66 -75.57 -75.45 -80.42 -80.49 -75.71 -77.69 -70.53]/1e9; - params(param_idx).radar.wfs(wf).chan_equal_dB = [6.8 -0.6 3 0.1 0 3.5 3.9 7 3.3 4.8 6.1 6.2 4.6 3.1 6.2]; - params(param_idx).radar.wfs(wf).chan_equal_deg = [-166.2 -142.7 177 -95.9 0 -25.9 -86.5 -27.4 128.1 41.6 -46.8 43 90.7 121.3 31.6]; - end -end +% Example to override parameters +% for param_idx = 1:length(params) +% %param = params(param_idx); +% for wf = 1:length(params(param_idx).radar.wfs) +% params(param_idx).radar.wfs(wf).Tsys = [0.46 -4.66 0.14 -1.77 0 -2.63 -3.38 -69.66 -75.57 -75.45 -80.42 -80.49 -75.71 -77.69 -70.53]/1e9; +% params(param_idx).radar.wfs(wf).chan_equal_dB = [6.8 -0.6 3 0.1 0 3.5 3.9 7 3.3 4.8 6.1 6.2 4.6 3.1 6.2]; +% params(param_idx).radar.wfs(wf).chan_equal_deg = [-166.2 -142.7 177 -95.9 0 -25.9 -86.5 -27.4 128.1 41.6 -46.8 43 90.7 121.3 31.6]; +% end +% end %% Automated Section % ===================================================================== diff --git a/cresis-toolbox/ct_support/sar_load.m b/cresis-toolbox/ct_support/sar_load.m new file mode 100644 index 00000000..0c1a5841 --- /dev/null +++ b/cresis-toolbox/ct_support/sar_load.m @@ -0,0 +1,324 @@ +function [data,metadata] = sar_load(param) +% [data,metadata] = sar_load(param) +% +% Loads and concatenates sar data from a single frame. Currently requires +% all data to be in the same directory. Returns data, position, and header +% information associated with the sar data. +% +% param.sar_load: parameter structure controlling how data are loaded. +% See input arguments section for details +% +% Author: John Paden, Logan Smith +% +% See also: run_master.m, master.m, run_array.m, array.m, run_sar_equal.m, +% run_sar_load.m, sar_equal.m, sar_load.m, array_proc.m, array_task.m, +% array_combine_task.m + +%% Check input arguments +% ========================================================================= + +%% Input: sar param +% subap: list of subapertures to load (array of integers, default is 1) +if ~isfield(param.sar,'sub_aperture_steering') || isempty(param.sar.sub_aperture_steering) + % Single aperture which points broadside to SAR is default + param.sar.sub_aperture_steering = [0]; +end + +%% Input: sar_load param +% combine_channels: Logical scaler. Sum wf-adc pairs together coherently in +% each image to form a single image. +if ~isfield(param.sar_load,'combine_channels') || isempty(param.sar_load.combine_channels) + param.sar_load.combine_channels = false; +end + +% combine_imgs: Logical scaler. Combine images together in fast-time +% (requires wf_comb to be set). Often used to combine low gain and high +% gain images and short and long pulse duration images. +if ~isfield(param.sar_load,'combine_imgs') || isempty(param.sar_load.combine_imgs) + param.sar_load.combine_imgs = false; +end + +% debug_level +if ~isfield(param.sar_load,'debug_level') || isempty(param.sar_load.debug_level) + param.sar_load.debug_level = 1; +end + +% detrend: structure controlling detrending. Arguments are passed into +% local_detrend.m. Detrend is disabled by default (detrend.cmd == 0). +% Detrending only runs if combine_channels == true and incoherent == true. +if ~isfield(param.sar_load,'detrend') || isempty(param.sar_load.detrend) + param.sar_load.detrend.cmd = []; +end +if ~isfield(param.sar_load.detrend,'B_noise') || isempty(param.sar_load.detrend.B_noise) + param.sar_load.detrend.B_noise = [100 200]; +end +if ~isfield(param.sar_load.detrend,'B_sig') || isempty(param.sar_load.detrend.B_sig) + param.sar_load.detrend.B_sig = [10 20]; +end +if ~isfield(param.sar_load.detrend,'cmd') || isempty(param.sar_load.detrend.cmd) + param.sar_load.detrend.cmd = 0; +end +if ~isfield(param.sar_load.detrend,'minVal') || isempty(param.sar_load.detrend.minVal) + param.sar_load.detrend.minVal = -inf; +end + +% fn: Data directory. Default is "sar" which loads from "CSARP_sar". +if ~isfield(param.sar_load,'in_path') || isempty(param.sar_load.in_path) + param.sar_load.in_path = 'sar'; +end + +% frm: Data frames to load +if isfield(param.sar_load,'frm') + warning('Legacy input field param.sar_load.frm is ignored and is now called param.sar_load.frms since sar_load.m now supports loading multiple frames at once.'); +end +if ~isfield(param.sar_load,'frms') || isempty(param.sar_load.frms) + warning('The param.sar_load.frms field must be set to an integer array indicating which frames to load. Defaulting to param.sar_load.frms = 1.'); + param.sar_load.frms = 1; +end +% chunk: two element vector specifying the start and stop chunk to load +% (minimum value is 1 and maximum value is the number of chunks in the +% frame). If the stop chunk is specified, then the stop chunk is set to the +% number of chunks. The default is to load all chunks which is [1 inf]. All +% chunks for a single frame in param.sar_load.frms is {[1 inf]}. +if ~isfield(param.sar_load,'chunk') || isempty(param.sar_load.chunk) + param.sar_load.chunk = {}; +elseif isnumeric(param.sar_load.chunk) && length(param.sar_load.chunk) == 2 + % Legacy: support old single frame format + param.sar_load.chunk = {param.sar_load.chunk}; +end +for frm_idx = 1:length(param.sar_load.frms) + if length(param.sar_load.chunk) < frm_idx || isempty(param.sar_load.chunk{frm_idx}) + % Load all chunks if chunk range is not specified for a frame + param.sar_load.chunk{frm_idx} = [1 inf]; + end +end + +% imgs: cell array of images to load. Each cell array contains a 2 by N +% waveform-adc pair list where N is the number of wf-adc pairs to load. +% Default is {[1 1]} which loads one image and this image is pulled from +% waveform = 1, adc = 1. +if ~isfield(param.sar_load,'imgs') || isempty(param.sar_load.imgs) + param.sar_load.imgs = {[1 1]}; +end + +% incoherent: logical scalar, default is false, if true, the data will be +% power detected on load with abs()^2. Incoherent only runs if +% combine_channels == true. +if ~isfield(param.sar_load,'incoherent') || isempty(param.sar_load.incoherent) + param.sar_load.incoherent = false; +end + +% sar_type: 'fk' or 'tdbp'. Default is 'fk'. +if ~isfield(param.sar_load,'sar_type') || isempty(param.sar_load.sar_type) + param.sar_load.sar_type = 'fk'; +end + +% subap: list of subapertures to load (array of integers, default is all +% subapertures specified in param.sar.sub_aperture_steering) +if ~isfield(param.sar_load,'subap') || isempty(param.sar_load.subap) + param.sar_load.subap = 1:length(param.sar.sub_aperture_steering); +end + +physical_constants; + +%% Load subapertures +% ========================================================================= + +% The base path for all the data +base_path = ct_filename_out(param,param.sar_load.in_path,''); + +% Initialize memory for outputs +data = cell(length(param.sar_load.imgs),1); +metadata = []; +metadata.lat = []; + +for subap_idx = 1:length(param.sar_load.subap) + subap = param.sar_load.subap(subap_idx); + + %% Frames: Load frames of SAR data + cur_rline = 0; + for frm_idx = 1:length(param.sar_load.frms) + frm = param.sar_load.frms(frm_idx); + + in_path = fullfile(base_path, ... + sprintf('%s_data_%03d_%02d_01',param.sar_load.sar_type,frm,subap)); + + %% Determine which chunks are available for this subaperture + img = 1; + wf_adc_list = param.sar_load.imgs{img}; + wf_adc = 1; + wf = wf_adc_list(wf_adc,1); + adc = wf_adc_list(wf_adc,2); + fns = get_filenames(in_path,sprintf('wf_%02.0f_adc_%02.0f_chk_',wf,adc),'','.mat'); + valid_chks = []; + for fns_idx = 1:length(fns) + [~,fn_name] = fileparts(fns{fns_idx}); + valid_chks(end+1) = str2double(fn_name(end-2:end)); + end + if isempty(valid_chks) + error('No valid sar chunks/blocks exist for the requested data.'); + end + param.sar_load.chunk{frm_idx}(param.sar_load.chunk{frm_idx}==inf) = max(valid_chks); + chks_to_load = param.sar_load.chunk{frm_idx}(1):param.sar_load.chunk{frm_idx}(end); + + % Remove chunks that do not exist from chunks_to_load list + [valid_chks,keep_idxs] = intersect(chks_to_load, valid_chks); + if length(valid_chks) ~= length(chks_to_load) + bad_mask = ones(size(chks_to_load)); + bad_mask(keep_idxs) = 0; + warning('Nonexistent chunks specified in chks_to_load (e.g. chunk "%g" is invalid), removing these', ... + chks_to_load(find(bad_mask,1))); + chks_to_load = valid_chks; + end + + %% Subapertures: Load chunks (blocks) of SAR data for frame + for chunk = chks_to_load + + if param.sar_load.debug_level >= 1 + fprintf('Loading chunk %d (%s)\n', chunk, datestr(now,'HH:MM:SS')); + end + + % ===================================================================== + % Load data + + tx = 1; + for img = 1:length(param.sar_load.imgs) + wf_adc_list = param.sar_load.imgs{img}; + + % ------------------------------------------------------------------- + % Load data + for wf_adc = 1:size(wf_adc_list,1) + wf = wf_adc_list(wf_adc,1); + adc = wf_adc_list(wf_adc,2); + + sar_fn = fullfile(in_path,sprintf('wf_%02.0f_adc_%02.0f_chk_%03.0f.mat',wf,adc,chunk)); + if param.sar_load.debug_level >= 2 + fprintf(' %s (%s)\n', sar_fn, datestr(now,'HH:MM:SS')); + end + sar_data = load(sar_fn); + + % Only add in non-overlapping part of SAR image (this is to support + % legacy SAR data format since current format does not have + % overlap) + if img == 1 && wf_adc == 1 + if frm_idx == 1 && chunk == param.sar_load.chunk{frm_idx}(1) + new_idxs = 1:length(sar_data.fcs.gps_time); + else + % 1e-6 added to avoid rounding errors... this is a hack + new_idxs = find(sar_data.fcs.gps_time > fcs{img}{wf_adc}.gps_time(end)+1e-6); + end + end + + % Get output image positions (not phase centers) + if img == 1 && wf_adc == 1 && subap_idx == 1 + if isempty(metadata.lat) + metadata.lat = sar_data.lat; + metadata.lon = sar_data.lon; + metadata.elev = sar_data.elev; + else + metadata.lat = [metadata.lat sar_data.lat(new_idxs)]; + metadata.lon = [metadata.lon sar_data.lon(new_idxs)]; + metadata.elev = [metadata.elev sar_data.elev(new_idxs)]; + end + end + + if subap_idx == 1 + if frm_idx == 1 && chunk == param.sar_load.chunk{frm_idx}(1) + fcs{img}{wf_adc} = sar_data.fcs; + + else + fcs{img}{wf_adc}.gps_time = cat(2,fcs{img}{wf_adc}.gps_time, ... + sar_data.fcs.gps_time(new_idxs)); + fcs{img}{wf_adc}.x = cat(2,fcs{img}{wf_adc}.x, ... + sar_data.fcs.x(:,new_idxs)); + fcs{img}{wf_adc}.y = cat(2,fcs{img}{wf_adc}.y, ... + sar_data.fcs.y(:,new_idxs)); + fcs{img}{wf_adc}.z = cat(2,fcs{img}{wf_adc}.z, ... + sar_data.fcs.z(:,new_idxs)); + fcs{img}{wf_adc}.origin = cat(2,fcs{img}{wf_adc}.origin, ... + sar_data.fcs.origin(:,new_idxs)); + fcs{img}{wf_adc}.pos = cat(2,fcs{img}{wf_adc}.pos, ... + sar_data.fcs.pos(:,new_idxs)); + fcs{img}{wf_adc}.roll = cat(2,fcs{img}{wf_adc}.roll, ... + sar_data.fcs.roll(new_idxs)); + fcs{img}{wf_adc}.pitch = cat(2,fcs{img}{wf_adc}.pitch, ... + sar_data.fcs.pitch(new_idxs)); + fcs{img}{wf_adc}.heading = cat(2,fcs{img}{wf_adc}.heading, ... + sar_data.fcs.heading(new_idxs)); + fcs{img}{wf_adc}.surface = cat(2,fcs{img}{wf_adc}.surface, ... + sar_data.fcs.surface(new_idxs)); + fcs{img}{wf_adc}.bottom = cat(2,fcs{img}{wf_adc}.bottom, ... + sar_data.fcs.bottom(new_idxs)); + end + end + + Nt = size(sar_data.fk_data,1); + if ~param.sar_load.combine_channels + if subap_idx == 1 && wf_adc == 1 + % Allocate memory in a special way when loading 3D data + data{img}(size(sar_data.fk_data,1), ... + size(data{img},2)+length(new_idxs),size(wf_adc_list,1),length(param.sar_load.subap)) = single(0); + end + data{img}(:,cur_rline + (1:length(new_idxs)),wf_adc,subap_idx) = sar_data.fk_data(:,new_idxs); + else + % When combining channels, take the mean of the data as it + % is loaded in to reduce peak memory consumption. + if wf_adc == 1 + sar_data_data = sar_data.fk_data(:,new_idxs) / size(wf_adc_list,1); + else + sar_data_data = sar_data_data + sar_data.fk_data(:,new_idxs) / size(wf_adc_list,1); + end + end + + end + + if param.sar_load.combine_channels + if param.sar_load.incoherent + data{img} = [data{img} abs(sar_data_data).^2]; + else + data{img} = [data{img} sar_data_data]; + end + end + end + + cur_rline = cur_rline + length(new_idxs); + end + end +end + +% Create lat/lon fields in FCS for convenience +for img = 1:length(fcs) + for wf_adc = 1:length(fcs{img}) + [fcs{img}{wf_adc}.lat,fcs{img}{wf_adc}.lon,fcs{img}{wf_adc}.elev] ... + = ecef2geodetic(fcs{img}{wf_adc}.origin(1,:) + sum(fcs{img}{wf_adc}.x.*fcs{img}{wf_adc}.pos), ... + fcs{img}{wf_adc}.origin(2,:) + sum(fcs{img}{wf_adc}.y.*fcs{img}{wf_adc}.pos), ... + fcs{img}{wf_adc}.origin(3,:) + sum(fcs{img}{wf_adc}.z.*fcs{img}{wf_adc}.pos), ... + WGS84.ellipsoid); + fcs{img}{wf_adc}.lat = fcs{img}{wf_adc}.lat * 180/pi; + fcs{img}{wf_adc}.lon = fcs{img}{wf_adc}.lon * 180/pi; + end +end + +metadata.fcs = fcs; +metadata.wfs = sar_data.wfs; +metadata.param_records = sar_data.param_records; +metadata.param_sar = sar_data.param_sar; + +if param.sar_load.combine_channels && param.sar_load.combine_imgs + +end + +if param.sar_load.combine_channels && param.sar_load.incoherent ... + && param.sar_load.detrend.cmd + % Detrend data + detrend = param.sar_load.detrend; + if param.sar_load.combine_imgs + data = local_detrend(data, detrend.B_noise, ... + detrend.B_sig, detrend.cmd, detrend.minVal); + else + for img = 1:length(param.imgs) + data{img} = local_detrend(data{img}, detrend.B_noise, ... + detrend.B_sig, detrend.cmd, detrend.minVal); + end + end +end diff --git a/cresis-toolbox/ct_support/sidelobe_mask_mex.mexa64 b/cresis-toolbox/ct_support/sidelobe_mask_mex.mexa64 old mode 100755 new mode 100644 diff --git a/cresis-toolbox/ct_support/update_surface_from_other_data_source.m b/cresis-toolbox/ct_support/update_surface_from_other_data_source.m deleted file mode 100644 index e4e66782..00000000 --- a/cresis-toolbox/ct_support/update_surface_from_other_data_source.m +++ /dev/null @@ -1,229 +0,0 @@ -% script update_surface_from_other_data_source -% -% Converts layers from one radar or ATM lidar and applies them to another -% radar. Uses param spreadsheet to determine which frames and segments -% to update. -% -% Author: John Paden - -% ===================================================================== -%% User Settings -% ===================================================================== - -params = read_param_xls(ct_filename_param('snow_param_2014_Greenland_P3.xls')); - -sync_param.surface_type = 'radar'; % 'radar' or 'lidar' -sync_param.location = 'arctic'; % 'arctic' or 'antarctic' - -sync_param.lidar.maxgap = 500; - -sync_param.radar.radar_name = 'kuband'; -sync_param.radar.data_type = 'qlook'; -sync_param.manual = false; -sync_param.output_type = 'qlook'; -sync_param.echo_type = 'qlook'; -sync_param.range_offset = 0; - -% ===================================================================== -%% Automated Section -% ===================================================================== - -fprintf('=======================================================\n'); -physical_constants; - -for param_idx = 1:length(params) - param = params(param_idx); - if ischar(param.cmd.generic) || ~param.cmd.generic - continue; - end - - fprintf('Updating surface %s\n', param.day_seg); - - %% Load the surface data source specified by sync_param.surface_type - % surf.gps_time: seconds since Jan 1, 1970 - % surf.surface: surface height in meters referenced to WGS-84 - - if strcmp(sync_param.surface_type,'lidar') - %% Load LIDAR surface to replace with - atm_dir = fullfile(sync_param.lidar.base_dir,param.season_name); - - global gRadar; - atm_fns = get_filenames_atm(sync_param.location,param.day_seg(1:8),gRadar.data_support_path); - %atm_fns = get_filenames(atm_dir,param.day_seg(3:8),'smooth_nadir','seg',struct('recursive',1)); - - lidar = read_lidar_atm(atm_fns); - - % Remove NAN's from LIDAR Data - good_lidar_idxs = ~isnan(lidar.gps_time); - lidar.gps_time = lidar.gps_time(good_lidar_idxs); - lidar.surface = lidar.surface(good_lidar_idxs); - lidar.lat = lidar.lat(good_lidar_idxs); - lidar.lon = lidar.lon(good_lidar_idxs); - if ~isempty(isnan(lidar.surface)) - good_lidar_idxs = ~isnan(lidar.surface); - lidar.gps_time = lidar.gps_time(good_lidar_idxs); - lidar.surface = lidar.surface(good_lidar_idxs); - lidar.lat = lidar.lat(good_lidar_idxs); - lidar.lon = lidar.lon(good_lidar_idxs); - end - - % ============= FIND LARGE GAPS IN LIDAR ============= - - - % Find Gaps Based on Along_Track Distance - lidar.along_track = geodetic_to_along_track(lidar.lat,lidar.lon,lidar.surface); - lidar.lidar_gaps = (diff(lidar.along_track) > sync_param.lidar.maxgap); - - % Prepare Data for Interp1 if gaps exist - if ~isempty(lidar.lidar_gaps) - % Find Start Indexes of Gaps - lidar.lidar_gap_start_idxs = strfind([0 lidar.lidar_gaps 0],[0 1]); - lidar.lidar_gap_start_idxs = lidar.lidar_gap_start_idxs-1; - if lidar.lidar_gap_start_idxs(1) == 0 - lidar.lidar_gap_start_idxs(1) = 1; - end - - % Find End Indexes of Gaps - lidar.lidar_gap_end_idxs = strfind([0 lidar.lidar_gaps 0],[1 0]); - if lidar.lidar_gap_end_idxs(end) > length(lidar.lidar_gaps) - lidar.lidar_gap_end_idxs(end) = length(lidar.lidar_gaps); - end - - % Add NaN's to lidar at Gap_IDXS (Start & End) - lidar.surface([lidar.lidar_gap_start_idxs lidar.lidar_gap_end_idxs]) = NaN; - end - - % ============= SET UP RADAR DATA FOR SYNC ============= - - %Test for and remove duplicates in LIDAR - if length(unique(lidar.gps_time)) < length(lidar.gps_time) - [lidar.gps_time lidar.unique_idxs] = unique(lidar.gps_time); - lidar.surface = lidar.surface(lidar.unique_idxs); - end - - surf = lidar; - - elseif strcmp(sync_param.surface_type,'radar') - %% Load RADAR surface to replace with - - % Get all the layer files from the corresponding day - sync_param.radar.season_name = param.season_name; - seg_dirs = get_filenames(ct_filename_out(sync_param.radar,sync_param.radar.data_type,'',1),param.day_seg(1:8),'','',struct('type','d')); - - %% Load each of the altimeter data files, store Elevation, Surface, GPS_time - surf.surface = []; - surf.gps_time = []; - for dir_idx = 1:length(seg_dirs) - fprintf('Loading segment %s\n', seg_dirs{dir_idx}); - % Get layer files in this segment directory - layer_files = get_filenames(seg_dirs{dir_idx},'Data_','','.mat'); - - %% Loop through each of the layer files and load it - for layer_idx=1:length(layer_files) - % fprintf(' File %i of %i\n', layer_idx, length(layer_files)); - warning off; - layer = load(layer_files{layer_idx},'layerData','Surface','GPS_time','Elevation'); - warning on; - if isfield(layer,'layerData') - surf.gps_time = cat(2,surf.gps_time,layer.GPS_time); - surf.surface = cat(2,surf.surface,layer.Elevation - layer.layerData{1}.value{2}.data*c/2); - else - surf.gps_time = cat(2,surf.gps_time,layer.GPS_time); - surf.surface = cat(2,surf.surface,layer.Elevation - reshape(layer.Surface,[1 length(layer.Surface)])*c/2); - end - end - - end - - %% REALLY need to check for gaps in the data here like we do for lidar data source - - end - - load(ct_filename_support(param,'','frames')); - - if isempty(param.cmd.frms) - param.cmd.frms = 1:length(frames.frame_idxs); - end - % Remove frames that do not exist from param.cmd.frms list - [valid_frms,keep_idxs] = intersect(param.cmd.frms, 1:length(frames.frame_idxs)); - if length(valid_frms) ~= length(param.cmd.frms) - bad_mask = ones(size(param.cmd.frms)); - bad_mask(keep_idxs) = 0; - warning('Nonexistent frames specified in param.cmd.frms (e.g. frame "%g" is invalid), removing these', ... - param.cmd.frms(find(bad_mask,1))); - param.cmd.frms = valid_frms; - end - - %% Loop through each frame and correct its corresponding layer file - for frm = param.cmd.frms - fprintf('Processing frame %s%03d %s\n', param.day_seg, frm, datestr(now)); - - layer_fn = fullfile(ct_filename_out(param,sync_param.output_type),sprintf('Data_%s_%03d.mat',param.day_seg,frm)); - - warning off; - layer = load(layer_fn,'layerData','Surface','GPS_time','Elevation'); - warning on; - - if isfield(layer,'layerData') - old_surface = layer.layerData{1}.value{2}.data; - else - old_surface = layer.Surface; - end - - %% Interpolate new surface onto old surface GPS time stamps - warning off; - new_Surface = interp1(surf.gps_time,surf.surface,layer.GPS_time,'linear'); - warning on; - new_Surface = (layer.Elevation - new_Surface + sync_param.range_offset) / (c/2); - - % Get the indexes of newsurface to replace. - newsurface_replace_idxs = isnan(new_Surface); - - % Recommended param.range_offset value - recommended_offset = sync_param.range_offset + median(new_Surface(~isnan(new_Surface)) - old_surface(~isnan(new_Surface)))*3e8/2 - - % Fill in the newsurface at the replacement points with radar surface. - new_Surface(newsurface_replace_idxs) = interp1(layer.GPS_time,old_surface,... - layer.GPS_time(newsurface_replace_idxs)); - - %% Save the output if the user wants to - if sync_param.manual - - %% Load the echogram and plotting - echo_fn = fullfile(ct_filename_out(param,sync_param.echo_type),sprintf('Data_%s_%03d.mat',param.day_seg,frm)) - - echo = load(echo_fn); - figure(1); clf; - imagesc(echo.GPS_time,echo.Time*1e6,lp(echo.Data)); - colormap(1-gray(256)); - hold on; - plot(layer.GPS_time,old_surface*1e6,'b','LineWidth',2); - - % ============= INSERT RADAR INTO LIDAR GAPS ============= - - plot(layer.GPS_time,new_Surface*1e6,'r--','LineWidth',2); - hold off; - ylabel('propagation time (us)') - xlabel('GPS time (sec)'); - - cmd = input(' (1) Skip, (2) Correct layer data: '); - else - % Automated, just update all files - cmd = 2; - end - if cmd == 2 - fprintf(' Correcting layer data\n'); - if isfield(layer,'layerData') - layer.layerData{1}.value{2}.data = new_Surface; - save(layer_fn,'-append','-struct','layer','layerData'); - else - layer.Surface = new_Surface; - save(layer_fn,'-append','-struct','layer','Surface'); - end - end - - end - -end - -return diff --git a/cresis-toolbox/ct_support/update_surface_twtt_delta.m b/cresis-toolbox/ct_support/update_surface_twtt_delta.m index eff7dd25..ac5e7b09 100644 --- a/cresis-toolbox/ct_support/update_surface_twtt_delta.m +++ b/cresis-toolbox/ct_support/update_surface_twtt_delta.m @@ -12,18 +12,23 @@ function update_surface_twtt_delta(param,param_override) % param_{qlook,sar}radar.wfs(wf).Tsys, % param_{qlook,sar}radar.wfs(wf).t_ref, Surface, and Bottom variables. % -% Optionally updates adc_gains_dB. If enabled and there has been a change -% in the adc_gains_dB field, this will cause -% param_{qlook,sar}radar.wfs(wf).adc_gains_dB and Data to be updated. +% Optionally updates radiometric fields adc_gains_dB, system_dB, and +% radiometric_corr_dB. If enabled and there has been a change in the +% radiometric field, this will cause these fields to be updated: +% param_{qlook,sar}.radar.wfs(wf).adc_gains_dB +% param_{qlook,sar}.radar.wfs(wf).system_dB +% param_{qlook,array}.radiometric_corr_dB +% Data % -% Note that only the param_{qlook,sar} field that matters gets updated. For -% example, param_records and param_combine will not be used or updated -% because Tadc, Tadc_adjust and Tsys are not used during these processes -% and so their value does not matter when those processes are applied. +% Note that only the param_{qlook,sar,array} field that matters gets +% updated. For example, param_records and param_array will not be used or +% updated because Tadc, Tadc_adjust and Tsys are not used during these +% processes and so their value does not matter when those processes are +% applied. % -% Tsys, t_ref, Tadc, Tadc_adjust, and adc_gains_dB changes can only be -% corrected when the same change is applied to all waveform-adc pairs used -% in the data product. +% Tsys, t_ref, Tadc, Tadc_adjust, adc_gains_dB, system_dB, and +% radiometric_corr_dB changes can only be corrected when the same change is +% applied to all waveform-adc pairs used in the data product. % % Note that Tadc, Tadc_adjust and t_ref are opposite sign to Tsys % - Tsys is subtracted away from time @@ -35,6 +40,7 @@ function update_surface_twtt_delta(param,param_override) % Author: John Paden %% Setup +param = merge_structs(param, param_override); fprintf('=====================================================================\n'); fprintf('%s: %s (%s)\n', mfilename, param.day_seg, datestr(now)); @@ -48,9 +54,14 @@ function update_surface_twtt_delta(param,param_override) return; end -if ~isfield(param.update_surface_twtt_delta,'update_adc_gains_dB') ... - || isempty(param.update_surface_twtt_delta.update_adc_gains_dB) - param.update_surface_twtt_delta.update_adc_gains_dB = false; +if ~isfield(param.update_surface_twtt_delta,'update_radiometric') ... + || isempty(param.update_surface_twtt_delta.update_radiometric) + param.update_surface_twtt_delta.update_radiometric = false; +end + +if ~isfield(param.update_surface_twtt_delta,'update_Tsys') ... + || isempty(param.update_surface_twtt_delta.update_Tsys) + param.update_surface_twtt_delta.update_Tsys = true; end if ~isfield(param.update_surface_twtt_delta,'data_types') ... @@ -60,7 +71,7 @@ function update_surface_twtt_delta(param,param_override) end % Remove frames that do not exist from param.cmd.frms list -load(ct_filename_support(param,'','frames')); +frames = frames_load(param); if isempty(param.cmd.frms) param.cmd.frms = 1:length(frames.frame_idxs); end @@ -111,8 +122,10 @@ function update_surface_twtt_delta(param,param_override) if isfield(mdata,'param_qlook') param_field = 'param_qlook'; + echo_param_field = 'qlook'; elseif isfield(mdata,'param_sar') param_field = 'param_sar'; + echo_param_field = 'array'; else error('Data file contained neither param_qlook or param_sar, but should contain one or the other.'); end @@ -129,10 +142,15 @@ function update_surface_twtt_delta(param,param_override) delta_offset_Tadc_mask = logical([]); delta_offset_adc_gains_dB = []; delta_offset_adc_gains_dB_mask = logical([]); + delta_offset_system_dB = []; + delta_offset_system_dB_mask = logical([]); + delta_offset_radiometric_corr_dB = []; + delta_offset_radiometric_corr_dB_mask = logical([]); for img = 1:length(mdata.(param_field).(param_field(7:end)).imgs) for wf_adc_pair = 1:size(mdata.(param_field).(param_field(7:end)).imgs{img},1) wf = abs(mdata.(param_field).(param_field(7:end)).imgs{img}(wf_adc_pair,1)); adc = abs(mdata.(param_field).(param_field(7:end)).imgs{img}(wf_adc_pair,2)); + rx_path = mdata.(param_field).radar.wfs(wf).rx_paths(adc); % t_ref correction if ~isfield(param.radar.wfs(wf),'t_ref') ... @@ -150,16 +168,18 @@ function update_surface_twtt_delta(param,param_override) % Tsys correction if ~isfield(param.radar.wfs(wf),'Tsys') ... || numel(param.radar.wfs(wf).Tsys) < adc - param.radar.wfs(wf).Tsys(adc) = 0; + param.radar.wfs(wf).Tsys(rx_path) = 0; end if ~isfield(mdata.(param_field).radar.wfs(wf),'Tsys') ... || numel(mdata.(param_field).radar.wfs(wf).Tsys) < adc - mdata.(param_field).radar.wfs(wf).Tsys(adc) = 0; + mdata.(param_field).radar.wfs(wf).Tsys(rx_path) = 0; end - delta_offset_Tsys(img,wf_adc_pair) = param.radar.wfs(wf).Tsys(adc) ... - - mdata.(param_field).radar.wfs(wf).Tsys(adc); - delta_offset_Tsys_mask(img,wf_adc_pair) = true; + if param.update_surface_twtt_delta.update_Tsys + delta_offset_Tsys(img,wf_adc_pair) = param.radar.wfs(wf).Tsys(rx_path) ... + - mdata.(param_field).radar.wfs(wf).Tsys(rx_path); + delta_offset_Tsys_mask(img,wf_adc_pair) = true; + end % Tadc correction if ~isfield(param.radar.wfs(wf),'Tadc') ... @@ -187,8 +207,8 @@ function update_surface_twtt_delta(param,param_override) - mdata.(param_field).radar.wfs(wf).Tadc_adjust; delta_offset_Tadc_adjust_mask(wf) = true; - % adc_gains_dB correction - if param.update_surface_twtt_delta.update_adc_gains_dB + if param.update_surface_twtt_delta.update_radiometric + % adc_gains_dB correction if ~isfield(param.radar.wfs(wf),'adc_gains_dB') ... || numel(param.radar.wfs(wf).adc_gains_dB) < adc param.radar.wfs(wf).adc_gains_dB(adc) = 0; @@ -200,6 +220,33 @@ function update_surface_twtt_delta(param,param_override) delta_offset_adc_gains_dB(img,wf_adc_pair) = param.radar.wfs(wf).adc_gains_dB(adc) ... - mdata.(param_field).radar.wfs(wf).adc_gains_dB(adc); delta_offset_adc_gains_dB_mask(img,wf_adc_pair) = true; + + % system_dB correction + if ~isfield(param.radar.wfs(wf),'system_dB') ... + || numel(param.radar.wfs(wf).system_dB) < rx_path + param.radar.wfs(wf).system_dB(rx_path) = 0; + end + if ~isfield(mdata.(param_field).radar.wfs(wf),'system_dB') ... + || numel(mdata.(param_field).radar.wfs(wf).system_dB) < rx_path + mdata.(param_field).radar.wfs(wf).system_dB(rx_path) = 0; + end + delta_offset_system_dB(img,wf_adc_pair) = param.radar.wfs(wf).system_dB(rx_path) ... + - mdata.(param_field).radar.wfs(wf).system_dB(rx_path); + delta_offset_system_dB_mask(img,wf_adc_pair) = true; + + % radiometric_corr_dB correction + if ~isfield(param.(echo_param_field),'radiometric_corr_dB') ... + || isempty(param.(echo_param_field).radiometric_corr_dB) + param.(echo_param_field).radiometric_corr_dB = NaN; + end + if ~isfield(mdata.(['param_' echo_param_field]).(echo_param_field),'radiometric_corr_dB') ... + || isempty(mdata.(['param_' echo_param_field]).(echo_param_field).radiometric_corr_dB) + mdata.(['param_' echo_param_field]).(echo_param_field).radiometric_corr_dB = NaN; + end + delta_offset_radiometric_corr_dB(img,wf_adc_pair) = param.(echo_param_field).radiometric_corr_dB ... + - mdata.(['param_' echo_param_field]).(echo_param_field).radiometric_corr_dB; + delta_offset_radiometric_corr_dB_mask(img,wf_adc_pair) = true; + end end end @@ -212,13 +259,15 @@ function update_surface_twtt_delta(param,param_override) end delta_offset_t_ref = delta_offset_t_ref(first_idx); - first_idx = find(delta_offset_Tsys_mask,1); - delta_offset_Tsys(~delta_offset_Tsys_mask) = NaN; - if ~all(delta_offset_Tsys(delta_offset_Tsys_mask) == delta_offset_Tsys(first_idx)) - delta_offset_Tsys - error('Different Tsys delta offsets for each waveform-adc-pair, cannot proceed: reprocess data.'); + if param.update_surface_twtt_delta.update_Tsys + first_idx = find(delta_offset_Tsys_mask,1); + delta_offset_Tsys(~delta_offset_Tsys_mask) = NaN; + if ~all(delta_offset_Tsys(delta_offset_Tsys_mask) == delta_offset_Tsys(first_idx)) + delta_offset_Tsys + error('Different Tsys delta offsets for each waveform-adc-pair, cannot proceed: reprocess data.'); + end + delta_offset_Tsys = delta_offset_Tsys(first_idx); end - delta_offset_Tsys = delta_offset_Tsys(first_idx); first_idx = find(delta_offset_Tadc_mask,1); delta_offset_Tadc(~delta_offset_Tadc_mask) = NaN; @@ -236,7 +285,7 @@ function update_surface_twtt_delta(param,param_override) end delta_offset_Tadc_adjust = delta_offset_Tadc_adjust(first_idx); - if param.update_surface_twtt_delta.update_adc_gains_dB + if param.update_surface_twtt_delta.update_radiometric first_idx = find(delta_offset_adc_gains_dB_mask,1); delta_offset_adc_gains_dB(~delta_offset_adc_gains_dB_mask) = NaN; if ~all(delta_offset_adc_gains_dB(delta_offset_adc_gains_dB_mask) == delta_offset_adc_gains_dB(first_idx)) @@ -244,36 +293,67 @@ function update_surface_twtt_delta(param,param_override) error('Different adc_gains_dB delta offsets for each waveform, cannot proceed: reprocess data.'); end delta_offset_adc_gains_dB = delta_offset_adc_gains_dB(first_idx); + + first_idx = find(delta_offset_system_dB_mask,1); + delta_offset_system_dB(~delta_offset_system_dB_mask) = NaN; + if ~all(delta_offset_system_dB(delta_offset_system_dB_mask) == delta_offset_system_dB(first_idx)) + delta_offset_system_dB + error('Different system_dB delta offsets for each waveform, cannot proceed: reprocess data.'); + end + delta_offset_system_dB = delta_offset_system_dB(first_idx); + + first_idx = find(delta_offset_radiometric_corr_dB_mask,1); + delta_offset_radiometric_corr_dB(~delta_offset_radiometric_corr_dB_mask) = NaN; + if ~all(delta_offset_radiometric_corr_dB(delta_offset_radiometric_corr_dB_mask) == delta_offset_radiometric_corr_dB(first_idx)) ... + && any(~isnan(delta_offset_radiometric_corr_dB(delta_offset_radiometric_corr_dB_mask))) + delta_offset_radiometric_corr_dB + error('Different radiometric_corr_dB delta offsets for each waveform, cannot proceed: reprocess data.'); + end + delta_offset_radiometric_corr_dB = delta_offset_radiometric_corr_dB(first_idx); end for img = 1:length(mdata.(param_field).(param_field(7:end)).imgs) for wf_adc_pair = 1:size(mdata.(param_field).(param_field(7:end)).imgs{img},1) wf = abs(mdata.(param_field).(param_field(7:end)).imgs{img}(wf_adc_pair,1)); adc = abs(mdata.(param_field).(param_field(7:end)).imgs{img}(wf_adc_pair,2)); + rx_path = mdata.(param_field).radar.wfs(wf).rx_paths(adc); mdata.(param_field).radar.wfs(wf).t_ref = param.radar.wfs(wf).t_ref; - mdata.(param_field).radar.wfs(wf).Tsys(adc) = param.radar.wfs(wf).Tsys(adc); + mdata.(param_field).radar.wfs(wf).Tsys(rx_path) = param.radar.wfs(wf).Tsys(rx_path); mdata.(param_field).radar.wfs(wf).Tadc = param.radar.wfs(wf).Tadc; mdata.(param_field).radar.wfs(wf).Tadc_adjust = param.radar.wfs(wf).Tadc_adjust; - if param.update_surface_twtt_delta.update_adc_gains_dB + if param.update_surface_twtt_delta.update_radiometric mdata.(param_field).radar.wfs(wf).adc_gains_dB(adc) = param.radar.wfs(wf).adc_gains_dB(adc); + mdata.(param_field).radar.wfs(wf).system_dB(rx_path) = param.radar.wfs(wf).system_dB(rx_path); + mdata.(['param_' echo_param_field]).(echo_param_field).radiometric_corr_dB = param.(echo_param_field).radiometric_corr_dB; end end end %% Update Data File - if delta_offset_t_ref ~= 0 || delta_offset_Tsys ~= 0 ... + if delta_offset_t_ref ~= 0 ... + || (param.update_surface_twtt_delta.update_Tsys ... + && delta_offset_Tsys ~= 0) ... || delta_offset_Tadc ~= 0 || delta_offset_Tadc_adjust ~= 0 ... - || (param.update_surface_twtt_delta.update_adc_gains_dB && delta_offset_adc_gains_dB ~= 0) + || (param.update_surface_twtt_delta.update_radiometric ... + && (delta_offset_adc_gains_dB ~= 0 || delta_offset_system_dB ~= 0 || delta_offset_radiometric_corr_dB ~= 0)) fprintf(' t_ref Offset %g %s (%s)\n', delta_offset_t_ref, echo_fn, datestr(now,'HH:MM:SS')); - fprintf(' Tsys Offset %g %s (%s)\n', delta_offset_Tsys, echo_fn, datestr(now,'HH:MM:SS')); + if param.update_surface_twtt_delta.update_Tsys + fprintf(' Tsys Offset %g %s (%s)\n', delta_offset_Tsys, echo_fn, datestr(now,'HH:MM:SS')); + end fprintf(' Tadc Offset %g %s (%s)\n', delta_offset_Tadc, echo_fn, datestr(now,'HH:MM:SS')); fprintf(' Tadc_adjust Offset %g %s (%s)\n', delta_offset_Tadc_adjust, echo_fn, datestr(now,'HH:MM:SS')); - if param.update_surface_twtt_delta.update_adc_gains_dB + if param.update_surface_twtt_delta.update_radiometric fprintf(' adc_gains_dB Offset %g %s (%s)\n', delta_offset_adc_gains_dB, echo_fn, datestr(now,'HH:MM:SS')); - if delta_offset_adc_gains_dB ~= 0 + fprintf(' system_dB Offset %g %s (%s)\n', delta_offset_system_dB, echo_fn, datestr(now,'HH:MM:SS')); + fprintf(' radiometric_corr Offset %g %s (%s)\n', delta_offset_radiometric_corr_dB, echo_fn, datestr(now,'HH:MM:SS')); + if delta_offset_adc_gains_dB + delta_offset_system_dB + delta_offset_radiometric_corr_dB ~= 0 tmp = load(echo_fn,'Data'); - tmp.Data = tmp.Data * 10^(-delta_offset_adc_gains_dB/10); + if isfinite(delta_offset_radiometric_corr_dB) + tmp.Data = tmp.Data * 10^((-delta_offset_adc_gains_dB + delta_offset_system_dB + delta_offset_radiometric_corr_dB)/10); + else + tmp.Data = tmp.Data * 10^((-delta_offset_adc_gains_dB + delta_offset_system_dB)/10); + end mdata.Data = tmp.Data; fields_to_update{end+1} = 'Data'; end @@ -282,7 +362,11 @@ function update_surface_twtt_delta(param,param_override) % Note that Tadc_adjust is opposite sign to Tsys % - Tsys is subtracted away % - Tadc_adjust is added on - delta_offset = delta_offset_Tsys - delta_offset_Tadc - delta_offset_Tadc_adjust - delta_offset_t_ref; + if param.update_surface_twtt_delta.update_Tsys + delta_offset = delta_offset_Tsys - delta_offset_Tadc - delta_offset_Tadc_adjust - delta_offset_t_ref; + else + delta_offset = -delta_offset_Tadc - delta_offset_Tadc_adjust - delta_offset_t_ref; + end mdata.Time = mdata.Time - delta_offset; if isfield(mdata,'Surface') diff --git a/cresis-toolbox/display/add_aoa_labels.m b/cresis-toolbox/display/add_aoa_labels.m deleted file mode 100644 index 9e1f3046..00000000 --- a/cresis-toolbox/display/add_aoa_labels.m +++ /dev/null @@ -1,26 +0,0 @@ -function add_aoa_labels(ha,theta) -% function add_aoa_labels(ha,theta) -% -% Add angle of arrival x-axis labels for range-Doppler -% -% Author: Logan Smith - -stheta = fftshift(theta); - -add_theta = 0; -deg_labels = []; -while add_theta < max(theta) - deg_labels(end+1) = add_theta; - add_theta = add_theta + 10; -end -deg_labels(end+1) = max(theta); -deg_labels = [-fliplr(deg_labels(2:end)) deg_labels]; -for tidx=1:length(deg_labels) - if sign(deg_labels(tidx)) < 0 - xtick_locs(tidx) = find(stheta <= deg_labels(tidx),1,'last'); - else - xtick_locs(tidx) = find(stheta >= deg_labels(tidx),1); - end -end -set(ha,'XTick',xtick_locs) -set(ha,'XTickLabel',round(stheta(xtick_locs))) \ No newline at end of file diff --git a/cresis-toolbox/display/copy_fig.m b/cresis-toolbox/display/copy_fig.m deleted file mode 100644 index c5335cbc..00000000 --- a/cresis-toolbox/display/copy_fig.m +++ /dev/null @@ -1,27 +0,0 @@ -function copy_fig(fh,fig_num) -% function copy_fig(fh,fig_num) -% -% Copy all properties of a figure window into a new figure window. -% -% Inputs: -% fh: figure handle of figure to be copied -% fig_num (optional): figure number to be copied to -% -% Examples: -% 1) copy_fig(gcf) -% 2) fig1 = figure(1); -% copy_fig(fig1,2) -% -% Author: Logan Smith - -error(nargchk(1,2,nargin,'struct')); -h1=fh; -if nargin > 2 - h2=figure(fig_num); clf; -else - h2=figure; -end -objects=allchild(h1); -copyobj(get(h1,'children'),h2); - -end \ No newline at end of file diff --git a/cresis-toolbox/display/detrending.m b/cresis-toolbox/display/detrending.m deleted file mode 100644 index 04c27b9c..00000000 --- a/cresis-toolbox/display/detrending.m +++ /dev/null @@ -1,21 +0,0 @@ -function Data = detrending(data) - -xx=double(data); -xx1=10*log10(xx(:,:)); -[M N]=size(xx1); -xy=zeros(M,N); -%xy_before_denoise=zeros(M,N); -xinput=zeros(2^nextpow2(M),1); -win=151; -for ii=1:N; - [detrended_data, x_trend] = detrending_method(xx1(:,ii), win, 2); - %xy_before_denoise(:,ii)=detrended_data; - - xinput(1:M,1)=detrended_data; - - [sigDEN,wDEC] = func_denoise_sw1d(xinput); - xy(:,ii)=sigDEN(1:M); - end - -Data=xy; -return \ No newline at end of file diff --git a/cresis-toolbox/display/detrending_coeff.m b/cresis-toolbox/display/detrending_coeff.m deleted file mode 100644 index 2c021fe8..00000000 --- a/cresis-toolbox/display/detrending_coeff.m +++ /dev/null @@ -1,11 +0,0 @@ -function [coeff_output, A] = detrending_coeff(win_len, order) -%win_len = 51; -%order = 2; -n = (win_len - 1) / 2; -A = ones(win_len, order + 1); -x = -n : n; -for j = 0 : order - A(:, j + 1) = x.^j; -end - -coeff_output = inv(A' * A) * A'; \ No newline at end of file diff --git a/cresis-toolbox/display/detrending_method.m b/cresis-toolbox/display/detrending_method.m deleted file mode 100644 index b3b0197a..00000000 --- a/cresis-toolbox/display/detrending_method.m +++ /dev/null @@ -1,273 +0,0 @@ -% clear; close all; clc; warning off; - -function [detrended_data, record_y] = detrending_method(data, seg_len, fit_order) - -if size(data, 1) < size(data, 2) - data = data'; -end - -% seg_len = 1001; % odd num -nonoverlap_len = (seg_len - 1)/2; - -% load brod2005.txt -% data = brod2005; -data_len = length(data); - -% calculate the coefficient, given a window size and fitting order -[coeff_output, A] = detrending_coeff(seg_len, fit_order); -A_coeff = A * coeff_output; - -for seg_index = 1 - % left trend - % seg_index = 1; - xi = 1 + (seg_index - 1) * (seg_len - 1) : seg_index * (seg_len - 1) + 1; - xi_left = xi; - seg_data = data(xi); - %slope = polyfit(xi, seg_data', fit_order); - %left_trend1 = polyval(slope, xi); - % to be replaced by pre-calculated coefficients - left_trend = (A_coeff * seg_data)'; - - % mid trend - if seg_index * (seg_len - 1) + 1 + nonoverlap_len > data_len - xi = 1 + (seg_index - 1) * (seg_len - 1) + nonoverlap_len : length(data); - xi_mid = xi; - seg_data = data(xi); - %slope = polyfit(xi, seg_data', fit_order); - %mid_trend = polyval(slope, xi); - if (length(seg_data) < seg_len) - [coeff_output1, A1] = detrending_coeff(length(seg_data), fit_order); - A_coeff1 = A1 * coeff_output1; - mid_trend = (A_coeff1 * seg_data)'; - else - mid_trend = (A_coeff * seg_data)'; - end - - xx1 = left_trend((seg_len + 1)/2:seg_len); - xx2 = mid_trend(1:(seg_len + 1)/2); - w = (0 : nonoverlap_len)/nonoverlap_len; - xx_left = xx1 .* (1 - w) + xx2 .* w; - - record_x = xi_left(1:nonoverlap_len); - record_y = left_trend(1:nonoverlap_len); - - mid_start_index = find(xi_mid == (xi_left(end) + 1)); - if (length(mid_start_index) == 0) - record_x = [record_x xi_left((end+3)/2 : end)]; - record_y = [record_y xx_left(2 : end)]; - else - record_x = [record_x xi_left((end+1)/2 : end) xi_mid(mid_start_index : end)]; - record_y = [record_y xx_left(1 : end) mid_trend((seg_len + 3)/2:end)]; - end - - detrended_data = data - record_y'; - return; - else - xi = 1 + (seg_index - 1) * (seg_len - 1) + nonoverlap_len : seg_index * (seg_len - 1) + 1 + nonoverlap_len; - xi_mid = xi; - seg_data = data(xi); - %slope = polyfit(xi, seg_data', fit_order); - %mid_trend = polyval(slope, xi); - mid_trend = (A_coeff * seg_data)'; - - % right trend - if (seg_index + 1) * (seg_len - 1) + 1 > data_len % not enough data points available in the right part - xi = seg_index * (seg_len - 1) + 1 : data_len; - xi_right = xi; - seg_data = data(xi); - %slope = polyfit(xi, seg_data', fit_order); - %right_trend = polyval(slope, xi); - if (length(seg_data) < seg_len) - % need to re-calculate coefficient and A - [coeff_output1, A1] = detrending_coeff(length(seg_data), fit_order); - A_coeff1 = A1 * coeff_output1; - right_trend = (A_coeff1 * seg_data)'; - else - right_trend = (A_coeff * seg_data)'; - end - - xx1 = left_trend((seg_len + 1)/2:seg_len); - xx2 = mid_trend(1:(seg_len + 1)/2); - w = (0 : nonoverlap_len)/nonoverlap_len; - xx_left = xx1 .* (1 - w) + xx2 .* w; - - xx1 = mid_trend((seg_len + 1)/2:seg_len); - xx2 = right_trend(1:(seg_len + 1)/2); - w = (0 : nonoverlap_len)/nonoverlap_len; - xx_right = xx1 .* (1 - w) + xx2 .* w; - - record_x = xi_left(1:nonoverlap_len); - record_y = left_trend(1:nonoverlap_len); - - record_x = [record_x xi_left((end+1)/2 : end) xi_mid((end+1)/2 + 1 : end)]; - record_y = [record_y xx_left(1 : end) xx_right(2:end)]; - - right_start_index = find(xi_right == xi_mid(end) + 1); - record_x = [record_x xi_right(right_start_index:end)]; - record_y = [record_y right_trend(right_start_index:end)]; - detrended_data = data - record_y'; - return; - else - xi = seg_index * (seg_len - 1) + 1 : (seg_index + 1) * (seg_len - 1) + 1; - xi_right = xi; - seg_data = data(xi); - %slope = polyfit(xi, seg_data', fit_order); - %right_trend = polyval(slope, xi); - right_trend = (A * coeff_output * seg_data)'; - - xx1 = left_trend((seg_len + 1)/2:seg_len); - xx2 = mid_trend(1:(seg_len + 1)/2); - w = (0 : nonoverlap_len)/nonoverlap_len; - xx_left = xx1 .* (1 - w) + xx2 .* w; - % plot(xi_left(end/2 : end), xx_left, 'r', 'LineWidth', 2) - - xx1 = mid_trend((seg_len + 1)/2:seg_len); - xx2 = right_trend(1:(seg_len + 1)/2); - w = (0 : nonoverlap_len)/nonoverlap_len; - xx_right = xx1 .* (1 - w) + xx2 .* w; - % plot(xi_mid(end/2:end), xx_right, 'r', 'LineWidth', 2) - - record_x = xi_left(1:nonoverlap_len); - record_y = left_trend(1:nonoverlap_len); - - record_x = [record_x xi_left((end+1)/2 : end) xi_mid((end+1)/2 + 1 : end)]; - record_y = [record_y xx_left(1 : end) xx_right(2:end)]; - end - end -end - -for seg_index = 2 : floor((data_len-1)/(seg_len-1)) - 1 - % left trend - % seg_index = 1; - xi = 1 + (seg_index - 1) * (seg_len - 1) : seg_index * (seg_len - 1) + 1; - xi_left = xi; - seg_data = data(xi); - %slope = polyfit(xi_left, seg_data', fit_order); - %left_trend = polyval(slope, xi); - left_trend = (A_coeff * seg_data)'; - - % mid trend - xi = 1 + (seg_index - 1) * (seg_len - 1) + nonoverlap_len : seg_index * (seg_len - 1) + 1 + nonoverlap_len; - xi_mid = xi; - seg_data = data(xi); - %slope = polyfit(xi, seg_data', fit_order); - %mid_trend = polyval(slope, xi); - mid_trend = (A_coeff * seg_data)'; - - % right trend - xi = seg_index * (seg_len - 1) + 1 : (seg_index + 1) * (seg_len - 1) + 1; - xi_right = xi; - seg_data = data(xi); - %slope = polyfit(xi, seg_data', fit_order); - %right_trend = polyval(slope, xi); - right_trend = (A_coeff * seg_data)'; - - xx1 = left_trend((seg_len + 1)/2:seg_len); - xx2 = mid_trend(1:(seg_len + 1)/2); - w = (0 : nonoverlap_len)/nonoverlap_len; - xx_left = xx1 .* (1 - w) + xx2 .* w; -% plot(xi_left(end/2 : end), xx_left, 'r', 'LineWidth', 2) - - xx1 = mid_trend((seg_len + 1)/2:seg_len); - xx2 = right_trend(1:(seg_len + 1)/2); - w = (0 : nonoverlap_len)/nonoverlap_len; - xx_right = xx1 .* (1 - w) + xx2 .* w; -% plot(xi_mid(end/2:end), xx_right, 'r', 'LineWidth', 2) - - record_x = [record_x xi_left((end+3)/2 : end) xi_mid((end+1)/2 + 1 : end)]; - record_y = [record_y xx_left(2 : end) xx_right(2:end)]; -end - -% last part of data -for seg_index = floor((data_len-1)/(seg_len-1)) - % left trend - % seg_index = 1; - xi = 1 + (seg_index - 1) * (seg_len - 1) : seg_index * (seg_len - 1) + 1; - xi_left = xi; - seg_data = data(xi); - %slope = polyfit(xi_left, seg_data', fit_order); - %left_trend = polyval(slope, xi); - left_trend = (A_coeff * seg_data)'; - - % mid trend - if (seg_index * (seg_len - 1) + 1 + nonoverlap_len > length(data)) - xi = 1 + (seg_index - 1) * (seg_len - 1) + nonoverlap_len : length(data); - xi_mid = xi; - seg_data = data(xi); - %slope = polyfit(xi, seg_data', fit_order); - %mid_trend = polyval(slope, xi); - - % may need to re-calcualte coefficient and A, since the length of - % 'seg_data' may be smaller than seg_len - if (length(seg_data) < seg_len) - % need to re-calculate coefficient and A - [coeff_output1, A1] = detrending_coeff(length(seg_data), fit_order); - A_coeff1 = A1 * coeff_output1; - mid_trend = (A_coeff1 * seg_data)'; - else - mid_trend = (A_coeff * seg_data)'; - end - - xx1 = left_trend((seg_len + 1)/2:seg_len); - xx2 = mid_trend(1:(seg_len + 1)/2); - w = (0 : nonoverlap_len)/nonoverlap_len; - xx_left = xx1 .* (1 - w) + xx2 .* w; - - mid_start_index = find(xi_mid == (xi_left(end) + 1)); - if (length(mid_start_index) == 0) - record_x = [record_x xi_left((end+3)/2 : end)]; - record_y = [record_y xx_left(2 : end)]; - else - record_x = [record_x xi_left((end+3)/2 : end) xi_mid(mid_start_index : end)]; - record_y = [record_y xx_left(2 : end) mid_trend((seg_len + 3)/2:end)]; - end - - break; - else - xi = 1 + (seg_index - 1) * (seg_len - 1) + nonoverlap_len : seg_index * (seg_len - 1) + 1 + nonoverlap_len; - xi_mid = xi; - seg_data = data(xi); - %slope = polyfit(xi, seg_data', fit_order); - %mid_trend = polyval(slope, xi); - mid_trend = (A_coeff * seg_data)'; - end -% plot(xi, mid_trend); -% hold on; - - % right trend - xi = seg_index * (seg_len - 1) + 1 : data_len; - xi_right = xi; - seg_data = data(xi); - %slope = polyfit(xi, seg_data', fit_order); - %right_trend = polyval(slope, xi); - - if (length(seg_data) < seg_len) - % need to re-calculate coefficient and A - [coeff_output1, A1] = detrending_coeff(length(seg_data), fit_order); - A_coeff1 = A1 * coeff_output1; - right_trend = (A_coeff1 * seg_data)'; - else - right_trend = (A_coeff * seg_data)'; - end - - xx1 = left_trend((seg_len + 1)/2:seg_len); - xx2 = mid_trend(1:(seg_len + 1)/2); - w = (0 : nonoverlap_len)/nonoverlap_len; - xx_left = xx1 .* (1 - w) + xx2 .* w; -% plot(xi_left(end/2 : end), xx_left, 'r', 'LineWidth', 2) - - xx1 = mid_trend((seg_len + 1)/2:seg_len); - xx2 = right_trend(1:(seg_len + 1)/2); - w = (0 : nonoverlap_len)/nonoverlap_len; - xx_right = xx1 .* (1 - w) + xx2 .* w; -% plot(xi_mid(end/2:end), xx_right, 'r', 'LineWidth', 2) - - record_x = [record_x xi_left((end+3)/2 : end) xi_mid((end+1)/2 + 1 : end)]; - record_y = [record_y xx_left(2 : end) xx_right(2:end)]; - - right_start_index = find(xi_right == xi_mid(end) + 1); - record_x = [record_x xi_right(right_start_index:end)]; - record_y = [record_y right_trend(right_start_index:end)]; -end - -detrended_data = data - record_y'; \ No newline at end of file diff --git a/cresis-toolbox/display/echo_concatenate.m b/cresis-toolbox/display/echo_concatenate.m new file mode 100644 index 00000000..199ba85e --- /dev/null +++ b/cresis-toolbox/display/echo_concatenate.m @@ -0,0 +1,73 @@ +function cat_data = echo_concatenate(cat_data, data) +% cat_data = echo_concatenate(cat_data, data) +% +% Concatenates echogram structures together. +% +% INPUTS: +% +% cat_data: concatenated data structure (empty if no data concatenated yet) +% +% data: a struct with a field "Data" with a linear +% +% OUTPUTS: +% cat_data: cat_data with data concatenated to the end +% +% Example: +% +% param = read_param_xls(ct_filename_param('snow_param_2016_Greenland_P3.xls'),'20160519_01'); +% mdata = echo_load(param,'CSARP_post/qlook',1); +% mdata = echo_concatenate(mdata,echo_load(param,'CSARP_post/qlook',2)); +% +% Author: John Paden +% +% See also: echo_detrend, echo_filt, echo_mult_suppress, echo_noise, +% echo_norm, echo_param, echo_stats, echo_stats_layer, echo_xcorr, +% echo_xcorr_profile + +if ischar(data) + % data should be a filename to load + data = load_L1B(data); +end + +if isempty(cat_data) + cat_data = data; + return; +end + +% Concatenate the metadata +good_mask = data.GPS_time > cat_data.GPS_time(end); +Nx = sum(good_mask); +cat_data.Elevation(end+(1:Nx)) = data.Elevation(good_mask); +cat_data.GPS_time(end+(1:Nx)) = data.GPS_time(good_mask); +cat_data.Heading(end+(1:Nx)) = data.Heading(good_mask); +cat_data.Latitude(end+(1:Nx)) = data.Latitude(good_mask); +cat_data.Longitude(end+(1:Nx)) = data.Longitude(good_mask); +cat_data.Pitch(end+(1:Nx)) = data.Pitch(good_mask); +cat_data.Roll(end+(1:Nx)) = data.Roll(good_mask); +cat_data.Surface(end+(1:Nx)) = data.Surface(good_mask); + +% Expand original "cat_data" time axis to accomodate earlier and later times +% that are only available in "data". +dt = cat_data.Time(2)-cat_data.Time(1); +Npre = floor((cat_data.Time(1) - data.Time(1))/dt); +if Npre > 0 + cat_data.Time = [cat_data.Time(1)+dt*(-Npre:-1).'; cat_data.Time]; + cat_data.Data = [nan(Npre,size(cat_data.Data,2)); cat_data.Data]; +end +Npost = floor((data.Time(end) - cat_data.Time(end))/dt); +if Npost > 0 + cat_data.Time = [cat_data.Time; cat_data.Time(end)+dt*(1:Npost).']; + cat_data.Data = [cat_data.Data; nan(Npost,size(cat_data.Data,2))]; +end + +% Concatenate the data (interpolate the new data to the old time axis and +% only concatenate new data) +cat_data.Data = [cat_data.Data, interp1(data.Time,data.Data(:,good_mask),cat_data.Time)]; + +try + cat_param_str = echo_param(cat_data,1); + param_str = echo_param(data,1); + + cat_data.(cat_param_str).load.frm ... + = unique([cat_data.(cat_param_str).load.frm data.(param_str).load.frm]); +end diff --git a/cresis-toolbox/display/echo_detrend.m b/cresis-toolbox/display/echo_detrend.m new file mode 100644 index 00000000..e8afab87 --- /dev/null +++ b/cresis-toolbox/display/echo_detrend.m @@ -0,0 +1,344 @@ +function data = echo_detrend(mdata, param) +% data = echo_detrend(mdata, param) +% +% The trend of the data is estimated using various methods and this trend +% is removed from the data. +% +% INPUTS: +% +% mdata: 2D input data matrix (log power) or echogram structure with "Data" +% field. "Time" and "Roll" may also be used depending on param settings. +% +% param: struct controlling how the detrending is done +% .units: units of the data, string containing 'l' (linear power) or 'd' (dB +% log power) +% +% .method: string indicating the method. The methods each have their own +% parameters +% +% 'file': NOT SUPPORTED +% +% 'local': takes a local mean to determine the trend +% .filt_len: 2 element vector of positive integers indicating the size of +% the trend estimation filter where the first element is the boxcar +% filter length in fast-time (row) and the second element is the boxcar +% filter length in slow-time (columns). Elements must be odd since +% fir_dec used. Elements can be greater than the corresponding +% dimensions Nt or Nx in which case an ordinary mean is used in that +% dimension. +% +% 'mean': takes the mean in the along-track and averages this in the +% cross-track. This is the default method. +% +% 'polynomial': polynomial fit to mean power of data between two layers, +% outside of the region between the two layers, the trend is found using +% nearest neighborhood interpolation to this polynomial clipped to the +% two layers. +% +% .layer_bottom: 1 by Nx vector of bottom layer twtt or range bins (see +% "units" field) +% +% .layer_top: 1 by Nx vector of top layer twtt or range bins (see +% "units" field) +% +% .order: nonnegative integer scalar indicating polynomial order. +% Default is 7 +% .units: scalar char, either "s" for seconds or "b" for bins. If +% any(layer_bottom < 1) then "s" is default, otherwise "b". The same +% check is done for layer_top. +% +% 'tonemap': uses Matlab's tonemap command +% +% .roll_comp_en: logical scalar, enables +% +% .roll_comp_data: structure loaded from roll compensation file generated +% by FILLTHISIN.m. Should have two fields: +% +% .all_roll: roll axis vector + +% .all_pwr: power axis vector correspondign to all_roll vector +% +% OUTPUTS: +% +% data: detrended input (log power) +% +% Examples: +% +% fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_standard/20140512_01/Data_20140512_01_018.mat'; +% mdata = load(fn); mdata.Data = 10*log10(mdata.Data); +% +% imagesc(echo_detrend(mdata)); +% +% [surface,bottom] = layerdata.load_layers(mdata,'','surface','bottom'); +% imagesc(echo_detrend(mdata, struct('method','polynomial','layer_top',surface,'layer_bottom',bottom))); +% +% imagesc(echo_detrend(mdata, struct('method','tonemap'))); +% +% imagesc(echo_detrend(mdata, struct('method','local','filt_len',[51 101]))); +% +% param = struct('method','polynomial','roll_comp_en',true); +% param.roll_comp_data = load('/cresis/snfs1/scratch/ibikunle/ct_user_tmp/roll_compensation.mat'); +% [param.layer_top,param.layer_bottom] = layerdata.load_layers(mdata,'','surface','bottom'); +% imagesc(echo_detrend(mdata, param)); +% +% Author: John Paden +% +% See also: echo_detrend, echo_filt, echo_mult_suppress, echo_noise, +% echo_norm, echo_param, echo_stats, echo_stats_layer, echo_xcorr, +% echo_xcorr_profile + +if ~exist('param','var') || isempty(param) + param = []; +end + +if ~isfield(param,'method') || isempty(param.method) + param.method = 'mean'; +end + +if ~isfield(param,'roll_comp_en') || isempty(param.roll_comp_en) + param.roll_comp_en = false; +end + +if ~isfield(param,'detrend_offset') || isempty(param.detrend_offset) + param.detrend_offset = 0; +end + +if isstruct(mdata) + data = mdata.Data; + if param.roll_comp_en && isfield(mdata,'Roll') + roll = mdata.Roll; + end +else + data = mdata; + if param.roll_comp_en + roll = zeros(1,size(data,2)); + end +end + +switch param.method + case 'eval' + time = mdata.Time; % Variable to be available to param.eval_cmd + for rline = 1:size(data,2) + surf = mdata.Surface(rline); % Variable to be available to param.eval_cmd + data(:,rline) = data(:,rline) + eval(param.eval_cmd); + end + + case 'file' + error('Not supported yet...'); + % Load detrend file generated by run_echo_stats + % e.g. ct_tmp/echogram_stats/snow/2011_Greenland_P3/stats_20110329_02.mat + if isempty(track.detrend) + track.detrend = [ct_filename_ct_tmp(param,'','echogram_stats','stats') '.mat']; + end + detrend = load(track.detrend,'dt','bins','min_means'); + detrend.time = detrend.dt*detrend.bins; + + track.detrend_struct = detrend; + detrend_curve = interp_finite(interp1(detrend.time,interp_finite(detrend.min_means),mdata.Time),NaN); + if all(isnan(detrend_curve)) + error('Detrend curve is all NaN.'); + end + if 0 + % Debug + rline = 200; + figure(1); clf; + plot(data(:,rline)) + hold on + mean_power = nanmean(data,2); + plot(mean_power) + plot(detrend_curve); + keyboard + end + data = bsxfun(@minus,data,detrend_curve); + if track.data_noise_en + data_noise = bsxfun(@minus,data_noise,detrend_curve); + end + + case 'local' + if ~isfield(param,'filt_len') || isempty(param.filt_len) + param.filt_len = [21 51]; + end + Nt = size(data,1); + Nx = size(data,2); + data = 10.^(data/10); + if param.filt_len(1) < Nt + trend = nan_fir_dec(data.',ones(1,param.filt_len(1))/param.filt_len(1),1).'; + else + trend = repmat(nan_mean(data,1), [Nt 1]); + end + if param.filt_len(2) < Nx + trend = nan_fir_dec(trend,ones(1,param.filt_len(2))/param.filt_len(2),1); + else + trend = repmat(nan_mean(trend,2), [Nx 1]); + end + data = 10*log10(data ./ trend); + + case 'mean' + data = 10.^(data/10); + data = 10*log10(bsxfun(@times,data,1./mean(data,2))); + + case 'polynomial' + Nt = size(data,1); + Nx = size(data,2); + + if param.roll_comp_en + % Roll compensation data interpolated to specific roll values for + % this echogram + roll_comp = interp1(param.roll_comp_data.all_roll,param.roll_comp_data.all_pwr,roll); + end + + if ~isfield(param,'order') || isempty(param.order) + param.order = 7; + end + if all(isnan(param.layer_bottom)) + param.layer_bottom(:) = Nt; + elseif any(param.layer_bottom < 1) + % layer is two way travel time + if isstruct(mdata) + param.layer_bottom = round(interp_finite(interp1(mdata.Time,1:length(mdata.Time),param.layer_bottom,'linear','extrap'),Nt)); + else + error('mdata should be an echogram struct with a "Time" field since layer_bottom is specific in two way travel time and needs to be converted to range bins.'); + end + else + % layer is in range bins + param.layer_bottom = round(interp_finite(param.layer_bottom,1)); + end + if all(isnan(param.layer_top)) + param.layer_top(:) = 1; + elseif any(param.layer_top < 1) + % layer is two way travel time + if isstruct(mdata) + param.layer_top = round(interp_finite(interp1(mdata.Time,1:length(mdata.Time),param.layer_top,'linear','extrap'),1)); + else + error('mdata should be an echogram struct with a "Time" field since layer_top is specific in two way travel time and needs to be converted to range bins.'); + end + else + % layer is in range bins + param.layer_top = round(interp_finite(param.layer_top,1)); + end + + mask = false(size(data)); + x_axis = nan(size(data)); + for rline = 1:Nx + top_bin = param.layer_top(rline); + bottom_bin = param.layer_bottom(rline); + if top_bin < 1 && bottom_bin >= 1 + top_bin = 1; + end + if top_bin <= Nt && bottom_bin > Nt + bottom_bin = Nt; + end + if top_bin > Nt || bottom_bin < 1 + continue; + end + bins = round(top_bin):round(bottom_bin); + + if length(bins) >= 2 + mask(bins,rline) = isfinite(data(bins,rline)); + x_axis(bins,rline) = (bins - param.layer_top(rline)) / (param.layer_bottom(rline) - param.layer_top(rline)); + end + end + + % Find the polynomial coefficients + detrend_poly = polyfit(x_axis(mask),data(mask), param.order); + if sum(mask(:))-Nx < 3*param.order + % Insufficient data to estimate polynomial + return; + end + + trend = zeros(Nt,1); + for rline = 1:Nx + % Section 2: Evaluate the polynomial + top_bin = round(param.layer_top(rline)); + bottom_bin = round(param.layer_bottom(rline)); + if top_bin < 1 && bottom_bin >= 1 + top_bin = 1; + end + if top_bin <= Nt && bottom_bin > Nt + bottom_bin = Nt; + end + + if top_bin > Nt + trend(:) = polyval(detrend_poly,0); + [min_val,~] = min(trend); + if param.roll_comp_en + trend = trend - roll_comp(rline) ; + end + + elseif bottom_bin < 1 + trend(:) = polyval(detrend_poly,1); + [min_val,~] = min(trend); + if param.roll_comp_en + trend = trend - roll_comp(rline) ; + end + + elseif top_bin >= bottom_bin + trend(1:top_bin) = polyval(detrend_poly,0); + trend(top_bin+1:end) = polyval(detrend_poly,1); + [min_val,~] = min(trend (top_bin+1:end) ); + + if param.roll_comp_en + trend(top_bin+1:end) = trend(top_bin+1:end) - roll_comp(rline) ; + end + + else + bins = top_bin:bottom_bin; + + % Section 1: Polynomial fit + trend(bins) = polyval(detrend_poly,x_axis(bins,rline)); + + % Section 2: Constant above surface + trend(1:bins(1)-1) = trend(bins(1)); + + % Section 3: Constant below bottom + [min_val,min_loc] = min(trend(bins)); + min_loc = min_loc + top_bin; + + trend(min_loc:end) = min(trend(bins)); + + if param.roll_comp_en + % Apply roll compensation + trend(top_bin:min_loc) = trend(top_bin:min_loc) - roll_comp(rline); + end + + end + + trend = trend - param.detrend_offset; + trend(trend < min_val) = min_val; + + if 0 + %% Debug plots + figure(1); clf; + plot(data(:,rline)); + if param.roll_comp_en + title(sprintf('Rangeline %d Roll= %.3f',rline,roll(rline) ),'Interpreter','none'); + else + title(sprintf('Rangeline %d',rline),'Interpreter','none'); + end + hold on; + plot(trend); + hold on; plot(data(:,rline) - trend); + grid on; grid minor + pause(0.5) + end + + data(:,rline) = data(:,rline) - trend; + end + + case 'tonemap' + tmp = []; + min_data = min(data(isfinite(data))); + if isempty(min_data) + % No finite data, so set min_data to 0 + min_data = 0; + end + tmp(:,:,1) = data - min_data; + tmp(:,:,2) = data - min_data; + tmp(:,:,3) = data - min_data; + % for generating synthetic HDR images + data = tonemap(tmp, 'AdjustLightness', [0.1 1], 'AdjustSaturation', 1.5); + data = single(data(:,:,2)); + + otherwise + error('Invalid param.method %s.', param.method); +end diff --git a/cresis-toolbox/display/echo_filename.m b/cresis-toolbox/display/echo_filename.m new file mode 100644 index 00000000..1a3b58aa --- /dev/null +++ b/cresis-toolbox/display/echo_filename.m @@ -0,0 +1,55 @@ +function [echo_fn,echo_fn_dir] = echo_filename(param,ct_filename_out_fn,frm,img) +% [echo_fn,echo_fn_dir] = echo_filename(param,ct_filename_out_fn,frm,img) +% +% Returns the standard filename for an echogram. +% +% param: parameter spreadsheet structure +% +% ct_filename_out_fn: string containing the ct_filename_out "fn" argument. +% For example "qlook", "standard", etc. Default is "qlook". +% +% frm: positive integer, default is 1 +% +% img: nonnegative scalar integer, default is 0, 0 for combined file, >0 +% +% OUTPUTS: +% +% echo_fn: echogram filename string +% +% echo_fn_dir: echogram filename string +% +% param = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls'),'20140506_01'); +% [echo_fn,echo_fn_dir] = echo_filename(param,'standard',1,0); +% if ~isdir(echo_fn_dir) +% mkdir(echo_fn_dir); +% end +% +% Author: John Paden +% +% See also: echo_detrend, echo_filt, echo_mult_suppress, echo_noise, +% echo_norm, echo_param, echo_stats, echo_stats_layer, echo_xcorr, +% echo_xcorr_profile + +if nargin < 1 + error('Must pass in param which is a parameter spreadsheet structure.'); +end + +if ~exist('ct_filename_out_fn','var') + ct_filename_out_fn = 'qlook'; +end + +if ~exist('frm','var') + frm = 1; +end + +if ~exist('img','var') + img = 0; +end + +echo_fn_dir = ct_filename_out(param,ct_filename_out_fn); + +if img == 0 + echo_fn = fullfile(echo_fn_dir,sprintf('Data_%s_%03d.mat',param.day_seg,frm)); +else + echo_fn = fullfile(echo_fn_dir,sprintf('Data_img_%02d_%s_%03d.mat',img,param.day_seg,frm)); +end diff --git a/cresis-toolbox/display/echo_filt.m b/cresis-toolbox/display/echo_filt.m new file mode 100644 index 00000000..f3d66a82 --- /dev/null +++ b/cresis-toolbox/display/echo_filt.m @@ -0,0 +1,68 @@ +function data = echo_filt(data, filt_len) +% data = echo_filt(data, filt_len) +% +% Along-track and cross-track multilook filtering. +% +% INPUTS: +% +% data: linear power echogram or a struct with a field "Data" with a linear +% power echogram in it +% +% filt_len: optional parameter, default is 5, should be a one or two +% element vector containing positive odd integers (1, 3, 5, ...) +% numel(filt_len)==1 means that along-track filtering will be applied +% with a boxcar filter equal in length to filt_len(1). +% numel(filt_len)==2 means that along-track filtering will be applied +% with a boxcar filter equal in length to filt_len(2) and cross-track +% filtering will be applied with a boxcar filter equal in length to +% filt_len(1). +% +% OUTPUTS: +% data: input data after filtering, will be the same size +% +% Example: +% +% fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_standard/20140512_01/Data_20140512_01_018.mat'; +% mdata = load(fn); +% +% imagesc(lp(echo_filt(mdata))); % Default 1x5 filter +% imagesc(lp(echo_filt(mdata,11))); % 1x11 filter +% imagesc(lp(echo_filt(mdata,[3 5]))); % 3x5 filter +% +% imagesc(lp(echo_filt(mdata.Data,[3 5]))); % 3x5 filter, called with data matrix +% +% Author: John Paden +% +% See also: echo_detrend, echo_filt, echo_mult_suppress, echo_noise, +% echo_norm, echo_param, echo_stats, echo_stats_layer, echo_xcorr, +% echo_xcorr_profile + +if isstruct(data) + if isfield(data,'Data') + data = data.Data; + else + error('Is data is a struct, then it must contain a field Data with the echogram image.'); + end +end + +if ~exist('filt_len','var') || isempty(filt_len) + filt_len = 5; +end + +if numel(filt_len) == 1 + filt_len = [1 filt_len]; +end + +% Ensure positive odd integer filter lengths +filt_len = 1+2*round((filt_len-1)/2); +filt_len = max(1,filt_len); + +% Filter along first dimension (fast-time) +if filt_len(1) ~= 1 + data = nan_fir_dec(data.',ones(1,filt_len(1))/filt_len(1),1,[],[],[],[],2.0).'; + +end +% Filter along second dimension (slow-time) +if filt_len(2) ~= 1 + data = nan_fir_dec(data,ones(1,filt_len(2))/filt_len(2),1,[],[],[],[],2.0); +end diff --git a/cresis-toolbox/display/echo_flatten.m b/cresis-toolbox/display/echo_flatten.m new file mode 100644 index 00000000..91721c25 --- /dev/null +++ b/cresis-toolbox/display/echo_flatten.m @@ -0,0 +1,263 @@ +function [data,resample_field] = echo_flatten(mdata,resample_field,inverse_flag,interp_method,ref_col,trim_nan_en) +% [data,resample_field] = echo_flatten(mdata,resample_field,inverse_flag,interp_method,ref_col,trim_nan_en) +% +% Shifts columns up/down in echogram using interpolation in order to +% flatten one or more 1D layers in the data or to flatten a 2D slope field. +% Defaults to flattening the surface stored in CSARP_layer if no +% resample_field given. +% +% There are two interpolations done by echo_flatten: the first ons is to +% interpolate to find the shift at each point in the data matrix. The +% second interpolation is used to apply the shift to the data and linear or +% Fourier based interpolation may be used. +% +% For layers provided by the user, interp_finite is used to fill in any +% gaps. +% +% For the shift field, spline is use on the interior points and linear +% exterpolation is used. +% +% Since interpolation is used for the shifts, oversampling (e.g. 2x) in the +% fast-time domain is recommended for best results when the shifts are +% non-integer. +% +% The shift can also be inverted so that a flattened echogram can be +% unflattened. +% +% The reference column is used as the reference that all shifts are made +% relative to and is left unchanged except for padding at the start or end. +% The reference column defaults to column 1, but the reference column may +% be specified. +% +% To compensate for aircraft elevation, just pass in a custom resample_field +% similar to opsCopyLayers and set the twtt = elev*c/2. +% +% INPUTS: +% +% mdata: echogram structure from an echogram file (e.g. returned from +% echo_load) or a data matrix. If a data matrix, then resample_field must +% be a numeric matrix. +% +% resample_field: input resample field information; several types are +% supported: +% +% opsLoadLayers struct array or empty matrix: an array of opsLoadLayers to +% layers that will be flattened. When an empty matrix, default layer is +% "surface". +% +% string: slope field file path (ct_filename_out will be used to determine +% filepath) Should have a 'resample_field' field containing layer_bins to +% flatten. If an empty string, then set to "slope" for CSARP_slope. +% +% numeric array: double, assumed to be layer_bins (one indexed so valid +% range is from 1 to Nt) +% +% interp_method: string containing 'circular_convolution', 'linear' or +% 'sinc'. The default is 'linear'. sinc is not recommended unless +% the data are Nyquist sampled. Note that power detection doubles the +% required sampling rate; for example 30 MHz data must be sampled at 60 MHz +% before power detection occurs. +% +% inverse_flag: logical scalar, default is false. If true, the shift will be +% inverted (undoes the flattening operation). +% +% ref_col: scalar positive integer indicating the column to use as a +% reference +% +% trim_nan: logical scalar. Default is true. Removes NaN at the start and +% end of the record. +% +% OUTPUTS: +% +% data: flattened data matrix +% +% resample_field: resampling matrix that was used (can be used to undo the +% resampling using the inverse_flag and to interpret layers in the +% flattened domain) +% +% Examples: +% +% param = read_param_xls(ct_filename_param('snow_param_2012_Greenland_P3.xls'),'20120330_04'); +% mdata = echo_load(param,'CSARP_post/qlook',3); +% +% figure(1); clf; imagesc(lp(mdata.Data)); +% [mdata.Data,resample_field] = echo_flatten(mdata); % Flatten to surface in layerdata-CSARP_layer-surface +% figure(2); clf; imagesc(lp(mdata.Data)); +% mdata.Data = echo_flatten(mdata,[],true); % Unflatten +% figure(3); clf; imagesc(lp(mdata.Data)); +% +% physical_constants; mdata = echo_flatten(mdata, interp1(mdata.Time,... +% 1:length(mdata.Time), mdata.Elevation*c/2)); % Flatten elevation +% mdata = echo_flatten(mdata, 'slope'); % Flatten to slope-field in CSARP_slope +% +% % Load "layer_param" with a list of opsLoadLayers layer_param structures +% mdata = echo_flatten(mdata, resample_field); % Flatten to surface in layerdata-CSARP_layer-surface +% +% Author: John Paden, Reece Mathews +% +% See also: echo_detrend, echo_filt, echo_mult_suppress, echo_noise, +% echo_norm, echo_param, echo_stats, echo_stats_layer, echo_xcorr, +% echo_xcorr_profile + +%% Input checks + +if ~exist('resample_field','var') + resample_field = []; +end +if isempty(resample_field) + if ischar(resample_field) + % slope file path + resample_field = 'slope'; + else + % opsLoadLayers structure + resample_field.name = 'surface'; + end +end + +if ~exist('interp_method','var') || isempty(interp_method) + interp_method = 'linear'; +end + +if ~exist('inverse_flag','var') || isempty(inverse_flag) + inverse_flag = false; +end + +if ~exist('ref_col','var') || isempty(ref_col) + ref_col = 1; +end + +if ~exist('trim_nan_en','var') || isempty(trim_nan_en) + trim_nan_en = true; +end + +if isnumeric(mdata) + mdata = struct('Data',mdata); +else + param = echo_param(mdata); +end + +Nx = size(mdata.Data, 2); + +%% Load resample_field +if isstruct(resample_field) + %% Load resample_field: opsLoadLayers layer_params + + % Check inputs + if ~isfield(mdata,'Time') + error('When opsLoadLayers resample_field are passed in, mdata.Time must be defined.'); + end + if ~isfield(mdata,'GPS_time') + error('When opsLoadLayers resample_field are passed in, mdata.GPS_time must be defined.'); + end + + % Load layers + ops_param = param; + ops_param.cmd.frms = [param.load.frm(1)-1 : param.load.frm(end)+1]; + layers = opsLoadLayers(ops_param, resample_field); + + % Interpolate layers onto mdata.GPS_time and convert from twtt to + % range-bins (rows) + resample_field = nan(length(layers), Nx); + for lay_idx = length(layers) + resample_field(lay_idx,:) = interp1(mdata.Time, 1:length(mdata.Time), ... + interp_finite(interp1(layers(lay_idx).gps_time, layers(lay_idx).twtt, mdata.GPS_time),0),'linear','extrap'); + end + +elseif isnumeric(resample_field) + %% Load resample_field: passed in directly + % Passed layer in units of range-bins (rows) + +elseif ischar(resample_field) + %% Load resample_field: along-track slope file + + % Check inputs + if ~isfield(mdata,'Time') + error('When opsLoadLayers resample_field are passed in, mdata.Time must be defined.'); + end + dt = mdata.Time(2)-mdata.Time(1); + if ~isfield(mdata,'GPS_time') + error('When opsLoadLayers resample_field are passed in, mdata.GPS_time must be defined.'); + end + + % Interpolate layers onto mdata.GPS_time + frms = [param.load.frm(1)-1 param.load.frm param.load.frm(end)+1]; + for frm_idx = 1:length(frms) + frm = frms(frm_idx); + slope_fn = fullfile(ct_filename_out(param,'',resample_field), ... + sprintf('slope_%s_%03d.mat', param.day_seg, frm)); + if frm_idx == 1 + load(slope_fn, 'gps_time', 'resample_field'); + else + error('Combining slope field files across frames not supported since overlap is required to do the interpolation properly.'); + tmp = load(slope_fn, 'gps_time', 'resample_field'); + gps_time = [gps_time tmp.gps_time]; + resample_field = [resample_field tmp.resample_field]; + end + end + resample_field = interp_finite(interp1(gps_time, resample_field.', mdata.GPS_time)).'/dt; + +else + error('resample_field must be a struct or numeric matrix.'); +end + +%% Interpolate resample_field + +% Find relative shifts to the reference (fixed, master) column +ref_axis = resample_field(:,ref_col); +resample_field = bsxfun(@minus,resample_field,ref_axis); +Nt = size(mdata.Data,1); +Nt_resample_init = size(resample_field,1); + +if inverse_flag + resample_axis = (1-round(max(-resample_field(:))) : Nt - round(min(-resample_field(:)))).'; +else + resample_axis = (1-round(max(resample_field(:))) : Nt - round(min(resample_field(:)))).'; +end +Nt_resample = length(resample_axis); + +for col = 1:Nx + if Nt_resample_init == 1 + resample_field(1:Nt_resample,col) = resample_field(1,col); + else + resample_field(1:Nt_resample,col) = interp1(resample_axis,(1:Nt_resample_init).',resample_field(1:Nt_resample_init,col),'spline'); + end +end +if inverse_flag + resample_field = bsxfun(@plus,-interp_finite(resample_field,0),resample_axis); +else + resample_field = bsxfun(@plus,interp_finite(resample_field,0),resample_axis); +end + +%% Interpolate mdata.Data + +if strcmpi(interp_method,'linear') + data = zeros(Nt_resample,Nx); + for col = 1:Nx + data(:, col) = interp1(1:Nt, mdata.Data(1:Nt, col), resample_field(:,col), 'linear'); + end +elseif strcmpi(interp_method,'sinc') + error('interp_method = sinc not supported.'); +else + error('resample_field must be a struct or numeric matrix.'); +end + +%% Trim NaN from start/end of record + +if trim_nan_en + start_bin = nan(1,Nx); + stop_bin = nan(1,Nx); + for rline = 1:Nx + start_bin_tmp = find(~isnan(data(:,rline)),1); + if ~isempty(start_bin_tmp) + start_bin(rline) = start_bin_tmp; + end + stop_bin_tmp = find(~isnan(data(:,rline)),1,'last'); + if ~isempty(stop_bin_tmp) + stop_bin(rline) = stop_bin_tmp; + end + end + start_bin = min(start_bin); + stop_bin = max(stop_bin); + data = data(start_bin:stop_bin,:); + resample_field = resample_field(start_bin:stop_bin,:); +end diff --git a/cresis-toolbox/display/echo_flatten_test.m b/cresis-toolbox/display/echo_flatten_test.m new file mode 100644 index 00000000..55abdc46 --- /dev/null +++ b/cresis-toolbox/display/echo_flatten_test.m @@ -0,0 +1,147 @@ +GENERATE = true; +USE_ELEVATION = false; + +physical_constants; +vel_air = c/2; + + +if ~GENERATE + param = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls'),'20140516_01'); + mdata = echo_load(param,'CSARP_post/standard',31); + + mdata.Data = real(10*log10(mdata.Data)); + + max_val = max(max(mdata.Data)); + + if USE_ELEVATION + slope_type = 'Elevation with mirrored test layer'; + elevation = mdata.Elevation; + time = mdata.Time; + + dt = time(2) - time(1); + drange = dt * vel_air; + bins = elevation / drange; + + % Insert test layer which mirrors elevation + for i = 1:length(bins) + mdata.Data(round(max(bins) - bins(i) + min(bins)), i) = max_val; + end + else + slope_type = 'Surface'; + bins = interp1(mdata.Time, 1:length(mdata.Time), mdata.Surface); + end + + + along_track_slope = zeros(1, size(mdata.Data, 2) - 1); + along_track_weight = 1; + upper_bounds = ones(1, size(mdata.Data, 2)); + lower_bounds = ones(1, size(mdata.Data, 2)) * size(mdata.Data, 1); + + figure(1); + clf; + hold on; + imagesc(mdata.Data); + colormap(1-gray); + y_new = tomo.viterbi2(single(mdata.Data), along_track_slope, along_track_weight, upper_bounds, lower_bounds); + plot(y_new); + plot(bins); + legend({'Viterbi', slope_type}); + title('Echo before shift'); + axis ij; + + bin_struct.mirror = USE_ELEVATION; + bin_struct.slope = bins; + + flattened = echo_flatten(mdata, bin_struct); + upper_bounds = flattened.Vertical_Bounds(1, :); + lower_bounds = flattened.Vertical_Bounds(2, :); + + figure(2); + clf; + hold on; + imagesc(flattened.Data); + colormap(1-gray); + y_new = tomo.viterbi2(single(flattened.Data), along_track_slope, along_track_weight, upper_bounds, lower_bounds); + plot(y_new); + plot(bins); + plot(upper_bounds, '--'); + plot(lower_bounds, '--'); + legend({'Viterbi', sprintf('Original %s', slope_type), 'Upper Bounds', 'Lower Bounds'}); + title('Echo after shift'); + axis ij; + +else + rows = 20; + cols = 100; + + % Create default matrix + matrix = magic(max([rows cols])); + matrix = matrix(1:rows, 1:cols); + matrix = matrix ./ max(max(matrix)) .* 5; + + % Create borders + matrix(1, : ) = 10; + matrix(end, : ) = 10; + + slope = zeros(1, cols); + third_width = floor(cols/3); + half_height = rows/2; + for i = 1:cols + if i <= third_width + per = (i-1)/third_width; + row = per * half_height + half_height; + elseif i < third_width * 2 + per = (i-third_width)/third_width; + row = rows-per*rows + 1; + else + per = (i-third_width*2)/third_width; + row = per*half_height + 1; + end + bin = [round(row), i]; + slope(i) = bin(1); + matrix(bin(1), bin(2)) = 30; + end + + slope_struct.mirror = false; + if USE_ELEVATION + slope_struct.mirror = true; + slope = -slope + slope(1)*2; + end + + along_track_slope = zeros(1, size(matrix, 2) - 1); + along_track_weight = 100; + upper_bounds = ones(1, size(matrix, 2)); + lower_bounds = ones(1, size(matrix, 2)) * size(matrix, 1); + + figure(1); + clf; + hold on; + imagesc(matrix); + colormap(1-gray); + y_new = tomo.viterbi2(single(matrix), along_track_slope, along_track_weight, upper_bounds, lower_bounds); + plot(y_new); + plot(slope); + legend({'Viterbi', 'Slope'}); + title('Shifted layer in echogram'); + axis ij; + + slope_struct.slope = slope; + flattened = echo_flatten(matrix, slope_struct); + upper_bounds = flattened.Vertical_Bounds(1, :); + lower_bounds = flattened.Vertical_Bounds(2, :); + + figure(2); + clf; + hold on; + imagesc(flattened.Data); + colormap(1-gray); + y_new = tomo.viterbi2(single(flattened.Data), along_track_slope, along_track_weight, upper_bounds, lower_bounds); + plot(y_new); + plot(upper_bounds, '--'); + plot(lower_bounds, '--'); + legend({'Viterbi', 'Upper Bounds', 'Lower Bounds'}); + title('Echogram shifted to account for shift in layer'); + axis ij; +end + + diff --git a/cresis-toolbox/display/echo_load.m b/cresis-toolbox/display/echo_load.m new file mode 100644 index 00000000..1b2335fa --- /dev/null +++ b/cresis-toolbox/display/echo_load.m @@ -0,0 +1,89 @@ +function [mdata,echo_fn] = echo_load(param,echogram_source,frms,img) +% [mdata,echo_fn] = echo_load(param,echogram_source,frms,img) +% +% The trend of the data is estimated using various methods and this trend +% is removed from the data. +% +% INPUTS: +% +% param: radar parameter spreadsheet structure +% +% echogram_source: 'standard' +% +% frms: List of frames to load, default is +% +% img: Scalar nonnegative integer indicating which image to load. Default +% is zero and loads the combined file Data_YYYYMMDD_SS_FFF.mat. Positive +% integers II load the file Data_img_II_YYYYMMDD_SS_FFF.mat +% +% OUTPUTS: +% +% mdata: echogram structure +% echo_fn: string containing the filename loaded +% +% Examples: +% +% param = read_param_xls(ct_filename_param('rds_param_2019_Antarctica_Ground.xls'),'20200107_01'); +% mdata = echo_load(param,'standard',1); +% mdata = echo_load(param,'standard',1:3); +% +% param = read_param_xls(ct_filename_param('snow_param_2016_Greenland_P3.xls'),'20160519_01'); +% mdata = echo_load(param,'CSARP_post/qlook',1:4); +% +% Author: John Paden +% +% See also: echo_detrend, echo_filt, echo_mult_suppress, echo_noise, +% echo_norm, echo_param, echo_stats, echo_stats_layer, echo_xcorr, +% echo_xcorr_profile + +if ischar(param) + % param is a filename (second and third arguments are ignored) + echo_fn = param; + mdata = load_L1B(echo_fn); + return; + +elseif isstruct(param) + % param is a parameter spreadsheet structure + if length(param) > 1 + error('echo_load:param','A parameter structure array with more than one element was passed in. length(param) must be 1.'); + end + + % Override default echogram source if second argument defined + if ~exist('echogram_source','var') || isempty(echogram_source) + echogram_source = 'standard'; + end + + % Override default img if third argument defined + if ~exist('img','var') || isempty(img) + % img == 0 is combined + img = 0; + end + + % Override frames field if third argument defined + if ~exist('frms','var') && ~isempty(frms) + frms = 1; + end + + for frm_idx = 1:length(frms) + if img == 0 + % Combined file + echo_fn{frm_idx} = fullfile(ct_filename_out(param,echogram_source),sprintf('Data_%s_%03d.mat',param.day_seg,frms(frm_idx))); + else + % Image "_img_II" file + echo_fn{frm_idx} = fullfile(ct_filename_out(param,echogram_source),sprintf('Data_img_%02d_%s_%03d.mat',img, param.day_seg,frms(frm_idx))); + end + end + +elseif iscell(param) + % param is a cell array of filenames (second and third arguments are ignored) + echo_fn = param; + +else + error('Invalid type for param.'); + +end + +mdata = []; +for frm_idx = 1:length(echo_fn) + mdata = echo_concatenate(mdata, echo_fn{frm_idx}); +end diff --git a/cresis-toolbox/display/echo_mult_suppress.m b/cresis-toolbox/display/echo_mult_suppress.m new file mode 100644 index 00000000..7bf1bee6 --- /dev/null +++ b/cresis-toolbox/display/echo_mult_suppress.m @@ -0,0 +1,318 @@ +function data = echo_mult_suppress(mdata, layer, param) +% data = echo_mult_suppress(mdata, layer, param) +% +% Suppress surface multiple using the surface layer location. +% +% INPUTS: +% +% mdata = echogram struct from qlook.m or array.m. Echogram should be +% linear power. Struct must include .Data, .Roll, and .Time fields. If layer not +% passed in, then it must include the param field (param_qlook or +% param_array) and .GPS_time. mdata.Data should be linear power. +% +% param: struct controlling how the surface multiple suppression is done +% +% .est_value_filter: Estimated surface multiple filter. Default is: +% param.est_value_filter = [0 1 2 2 2 2 1]; +% 20 * param.est_value_filter/sum(param.est_value_filter); +% +% .guard_dB: threshold above noise floor that signal must remain to not +% get removed. Default is 12. +% +% .max_filt_len: Length of max_filt1 on surface values. Default is 11. +% +% .multiple_loss_dB: Extra loss of surface multiple relative to surface +% multiple. Default is 52. +% +% .noise_bins: Bins relative to surface multiple to estimate background +% power. Default is 20. +% +% .noise_threshold_dB: threshold above noise floor for which surface +% multiple corrected data will be set to NaN. Default is 5. +% +% .window_units: scalar char either "s" for seconds or "b" for bins. +% Default is "s". +% +% OUTPUTS: +% +% data: surface multiple suppressed input (2D matrix the same size as +% mdata.Data), linear power +% +% Examples: +% +% +% fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_post/CSARP_standard/20140313_08/Data_20140313_08_001.mat'; +% mdata = load(fn); +% fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_standard/20140512_01/Data_20140512_01_018.mat'; +% mdata = load(fn); +% +% imagesc(lp(echo_mult_suppress(mdata))); +% +% Author: John Paden +% +% See also: echo_detrend, echo_filt, echo_mult_suppress, echo_noise, +% echo_norm, echo_param, echo_stats, echo_stats_layer, echo_xcorr, +% echo_xcorr_profile + +%% Input checks + +data = mdata.Data; + +if ~exist('layer','var') || isempty(layer) + if isstruct(mdata) + layers = layerdata(echo_param(mdata)); + layer = layers.get_layer_by_gps_time(mdata.GPS_time,'surface'); + else + error('Layer must be defined if mdata is not an echogram struct since this echogram struct is used to load the default surface layer.'); + end +end + +if layer > 1 + error('Layer must have units of two way travel time (seconds). Values greater than 1 occur.'); +end + +surf_bins = round(interp1(mdata.Time,1:length(mdata.Time),layer)); +mult_bins = round(interp1(mdata.Time,1:length(mdata.Time),layer*2)); + +if ~exist('param','var') || isempty(param) + param = []; +end + +% .est_value_filter: Estimated surface multiple filter +if ~isfield(param,'est_value_filter') || isempty(param.est_value_filter) + param.est_value_filter = [0 1 2 2 2 2 1]; + %param.est_value_filter = [linspace(0,2,3+5) 2 2 2 1 zeros(1,5)]; % Longer tail + param.est_value_filter = 20 * param.est_value_filter/sum(param.est_value_filter); +end +est_value_filter = param.est_value_filter; + +% .guard_dB: threshold above noise floor that signal must remain to not +% get removed +if ~isfield(param,'guard_dB') || isempty(param.guard_dB) + param.guard_dB = 12; +end +guard_dB = param.guard_dB; + +% .max_filt_len: Length of max_filt1 on surface values +if ~isfield(param,'max_filt_len') || isempty(param.max_filt_len) + param.max_filt_len = 11; +end +max_filt_len = param.max_filt_len; + +% .multiple_loss_dB: Extra loss of surface multiple relative to surface +% multiple +if ~isfield(param,'multiple_loss_dB') || isempty(param.multiple_loss_dB) + param.multiple_loss_dB = 52; +end +multiple_loss_dB = param.multiple_loss_dB; + +% .noise_bins: Bins relative to surface multiple to estimate background +% power +if ~isfield(param,'noise_bins') || isempty(param.noise_bins) + param.noise_bins = [20]; +end +noise_bins = param.noise_bins; + +% .noise_threshold_dB: threshold above noise floor for which surface +% multiple corrected data will be set to NaN +if ~isfield(param,'noise_threshold_dB') || isempty(param.noise_threshold_dB) + param.noise_threshold_dB = 5; +end +noise_threshold = 10.^(param.noise_threshold_dB/10); + +% .window_units: scalar char either "s" for seconds or "b" for bins +if ~isfield(param,'window_units') || isempty(param.window_units) + param.window_units = 's'; +end + +% Convert param.window into bins +if param.window_units == 's' + dt = mdata.Time(2) - mdata.Time(1); + % window: two elements vector of window relative to layer to obtain + % statistics + if ~isfield(param,'window') || isempty(param.window) + param.window = [-5*dt; 11*dt]; + end + bins = round(param.window / dt); + bins = bins(1):bins(end); +else + if ~isfield(param,'window') || isempty(param.window) + param.window = [-5; 11]; + end + bins = param.window(1):param.window(end); +end + +% Get surface and surface multiple layer statistics +param.inc_dec = 1; +[surf_value,~,~,~,surf_wfs] = echo_stats_layer(mdata,layer,param); +[mult_value,~,~,neighbor_power_mult,mult_wfs] = echo_stats_layer(mdata,layer*2,param); +surf_value = 10*log10(surf_value); +mult_value = 10*log10(mult_value); + +Nt = size(data,1); +Nx = size(data,2); + +%% Spherical spreading range loss (relative dB) +surf_spherical_spreading_loss_dB = 25*log10(layer); +surf_spherical_spreading_loss_dB = surf_spherical_spreading_loss_dB - min(surf_spherical_spreading_loss_dB); + +surf_value_corr = surf_value + surf_spherical_spreading_loss_dB; +mult_value_corr = mult_value + 2*surf_spherical_spreading_loss_dB; + +if 0 + % Check surface and multiple tracking + figure(1); clf; + imagesc(10*log10(data)); + hold on; + plot(surf_bins.','r'); + plot(mult_bins.','g'); +end + +%% Estimate surface roll correction +mask = isfinite(mult_value_corr); +p_roll_surf_corr=polyfit(mdata.Roll(mask)*180/pi, surf_value_corr(mask),15); +if 0 + figure(2); clf; + plot(mdata.Roll*180/pi, surf_value_corr,'.') + hold on + plot(mdata.Roll*180/pi, polyval(p_roll_surf_corr,mdata.Roll*180/pi),'r.'); + xlabel('Roll (deg)'); + ylabel('Surface relative power (dB)') + grid on; + p_roll_surf_corr(end) = 0; + fprintf('%.14g ',p_roll_surf_corr); fprintf('\n'); +end +p_roll_surf_corr(end) = 0; +if any(~isfinite(p_roll_surf_corr)) + p_roll_surf_corr = mean(surf_value_corr(mask)); +end + +%% Estimate surface multiple roll correction +mask = isfinite(mult_value_corr); +p_roll_corr=polyfit(mdata.Roll(mask)*180/pi, mult_value_corr(mask),15); +if 0 + figure(3); clf; + plot(mdata.Roll*180/pi, mult_value_corr,'.') + hold on + plot(mdata.Roll*180/pi, polyval(p_roll_corr,mdata.Roll*180/pi),'r.'); + xlabel('Roll (deg)'); + ylabel('Surface multiple relative power (dB)') + grid on; + p_roll_corr(end) = 0; + fprintf('%.14g ',p_roll_corr); fprintf('\n'); +end +p_roll_corr(end) = 0; +if any(~isfinite(p_roll_corr)) + p_roll_corr = mean(mult_value_corr(mask)); +end + +% p_roll_surf_corr = [2.2255958717854e-15 -1.3572630870759e-13 2.3527709210261e-12 2.124281696932e-11 -1.1228041256249e-09 6.0543207183435e-09 1.6645234883187e-07 -1.6766045475499e-06 -1.0712601140495e-05 0.0001376158654882 0.0003367560066867 -0.0042097661445678 -0.0054248806810891 -0.018969197708196 -0.23904092116243 0]; +roll_surf_loss_dB = polyval(p_roll_surf_corr, mdata.Roll*180/pi); +% roll_surf_loss_dB(mdata.Roll*180/pi < -11.673888571906199) = polyval(p_roll_surf_corr, -11.673888571906199); +% roll_surf_loss_dB(mdata.Roll*180/pi > 18.274466278841704) = polyval(p_roll_surf_corr, 18.274466278841704); + +% p_roll_corr = [3.2936805253613e-14 -8.0792025960357e-13 -3.1275394496497e-11 9.2958491383191e-10 6.9811704932971e-09 -3.2389776773949e-07 -2.2783931893159e-07 5.1833633310267e-05 -8.3842235360433e-05 -0.0041934948578307 0.0093679210573843 0.16172483912271 -0.30900350768736 -2.1963125631644 1.9972031515517 0]; +roll_loss_dB = polyval(p_roll_corr, mdata.Roll*180/pi); +% roll_loss_dB(mdata.Roll*180/pi < -11.673888571906199) = polyval(p_roll_corr, -11.673888571906199); +% roll_loss_dB(mdata.Roll*180/pi > 18.274466278841704) = polyval(p_roll_corr, 18.274466278841704); + +surf_value_corr = surf_value + surf_spherical_spreading_loss_dB ... + - roll_surf_loss_dB; +mult_value_corr = mult_value + 2*surf_spherical_spreading_loss_dB ... + - roll_loss_dB; + +%% Estimate surface multiple power from surface power +system_dB = -51.89; +system_dB = -70.36; +surf_value_corr_filt = max_filt1(surf_value_corr,max_filt_len); +est_value = system_dB + 2*(surf_value_corr_filt - system_dB) - 2*surf_spherical_spreading_loss_dB - multiple_loss_dB + roll_loss_dB; + +if 0 + % Search for the system_dB value + figure(2); clf; + system_dB_list = -80:0.01:-45; + est_error = zeros(size(system_dB_list)); + for idx = 1:length(system_dB_list) + system_dB = system_dB_list(idx); + est_mult_value = system_dB + 2*(surf_value_corr - system_dB) - 2*surf_spherical_spreading_loss_dB - multiple_loss_dB + roll_loss_dB; + est_error(idx) = mean(abs(mult_value - est_mult_value)); + end + plot(system_dB_list, est_error); + return; +end + +if 0 + % Compare corrections and estimated values + figure(2); clf; + plot(surf_value) + hold on + plot(surf_value_corr) + plot(mult_value,'.'); + plot(mult_value_corr); + plot(est_value,'o'); + return +end + +%% Adjust surface multiple in image +for rline = 1:Nx + % Select the surface bins + sbins = surf_bins(rline)+bins; + % Select which bins to modify + mbins = mult_bins(rline)+bins; + % Remove invalid bins + mask = find(~(~isfinite(sbins) | sbins<1 | sbins>Nt | ~isfinite(mbins) | mbins<1 | mbins>Nt)); + % Estimate the value for these bins + est_value = system_dB + 2*(10*log10(data(sbins(mask),rline))-roll_surf_loss_dB(rline) - system_dB) - 2*surf_spherical_spreading_loss_dB(rline) - multiple_loss_dB + roll_loss_dB(rline) + guard_dB + surf_value_corr_filt(rline) - surf_value_corr(rline); + + roll_spread = round(abs(mdata.Roll(rline)*180/pi))*2; + if roll_spread>0 + if 0 + Hwind = hanning(roll_spread)/(roll_spread/2+0.5) + else + Hwind = exp(-(1:roll_spread)/roll_spread*8); + Hwind = Hwind / sum(Hwind); + end + est_value_filter_tmp = fliplr(filter(Hwind,1,fliplr([zeros(1,roll_spread) est_value_filter zeros(1,roll_spread)]))); + else + est_value_filter_tmp = est_value_filter; + end + %est_value_filter_tmp = [ones(1,roll_spread) est_value_filter zeros(1,roll_spread)]; + if ~isequal(est_value_filter_tmp,1) + est_value = 10*log10(fir_dec(10.^(est_value(:).'/10),est_value_filter_tmp,1).'); + end + + noise_fill = linspace(neighbor_power_mult(1,rline),neighbor_power_mult(2,rline), length(mbins)).'; + noise = min(neighbor_power_mult(:,rline)) * ones(size(mbins)).'; + + orig_val = data(:,rline); + if 1 + new_val = data(mbins(mask),rline); + new_val(new_val<10.^(est_value/10)) = NaN; + data(mbins(mask),rline) = new_val; + data(:,rline) = 10.^(interp_finite(10*log10(data(:,rline)))/10); + elseif 0 + new_val = data(mbins(mask),rline) - 10.^(est_value/10); + noise_mask = new_val=2324 + figure(1); clf; + plot(10*log10(orig_val)); + hold on; + plot(10*log10(data(:,rline)),'r.'); + plot(sbins(mask), 10*log10(data(sbins(mask),rline)),'go'); + plot(surf_bins(rline), 10*log10(data(surf_bins(rline),rline)),'ko'); + plot(mbins(mask), est_value,'go'); + hold off; +% xlim([350 470]) + keyboard + end +end diff --git a/cresis-toolbox/display/echo_noise.m b/cresis-toolbox/display/echo_noise.m new file mode 100644 index 00000000..52d6702a --- /dev/null +++ b/cresis-toolbox/display/echo_noise.m @@ -0,0 +1,141 @@ +function noise = echo_noise(mdata, param) +% noise = echo_noise(mdata, param) +% +% Estimate the noise on each range line based on the parameters +% +% data: linear power echogram or an echogram struct with a field "Data" +% with a linear power echogram in it. If an echogram struct, the field +% "Time" may be required. +% +% param: structure defining how the noise is estimated +% .method_fh: function handle to method for estimating noise in the +% window +% +% .valid_dB: 2 element numeric vector; specifies the valid range for the +% noise-estimate; default is [-inf inf] which effectively disables this +% valid noise-estimate range constraint; +% +% .window: 2x1 or 2xN element numeric array; specifies the valid range +% for the noise-estimate; default is [-inf; inf] which effectively +% disables this valid noise-estimate range constraint; This parameter can +% be specified for each range line by passing in a 2 x Nx array where +% each column restricts the noise bins to be estimated +% +% .window_units: string which indicates the units used for the window. +% Default is 's' for seconds. If window is finite, then mdata must be a +% structure with the Time variable and not just an image. '%' means +% percentage ratio from 0 (start of fast-time) to 1 (end of fast-time). +% 'b' means range bins (rows). +% +% Example: +% +% fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_standard/20140512_01/Data_20140512_01_018.mat'; +% mdata = load(fn); +% +% % Default [-inf inf] window, median fast-time and mean slow-time +% noise = echo_noise(mdata); plot(lp(noise)); +% +% % Example using the bottom layer: +% bottom = layerdata.load_layers(mdata,'','bottom'); +% noise = echo_noise(mdata, struct('window',[bottom+5e-6; inf(size(bottom))])); +% plot(lp(noise)); +% +% % Example using the surface layer: +% surface = layerdata.load_layers(mdata,'','surface'); +% noise = echo_noise(mdata, struct('window',[surface-100e-9; surface-75e-9])); +% plot(lp(noise)); +% +% % Example using a time window: +% noise = echo_noise(mdata, struct('window',[-inf; inf])); plot(lp(noise)); +% +% % Example using a percentage window 80-90%: +% noise = echo_noise(mdata, struct('window',[0.8; 0.9],'window_units','%')); plot(lp(noise)); +% +% Author: John Paden +% +% See also: echo_detrend, echo_filt, echo_mult_suppress, echo_noise, +% echo_norm, echo_param, echo_stats, echo_stats_layer, echo_xcorr, +% echo_xcorr_profile + +%% Input checks + +if isstruct(mdata) + data = mdata.Data; +else + data = mdata; +end + +if ~exist('param','var') || isempty(param) + param = []; +end + +% method_fh: estimation method (e.g. @nanmean, @nanmedian, @mean or @median), default is @nanmedian +if ~isfield(param,'method_fh') || isempty(param.method_fh) + param.method_fh = @nanmedian; +end + +% window_units: string which indicates the units used for the window. Default is +% 's' for seconds. If window is finite, then mdata must be a structure with +% the Time variable and not just an image. '%' means percentage ratio from +% 0 (start of fast-time) to 1 (end of fast-time). +if ~isfield(param,'window_units') || isempty(param.window_units) + param.window_units = 's'; +end + +% valid_dB: 2 element numeric vector; specifies the valid +% range for the noise-estimate; default is [-inf inf] which effectively +% disables this valid noise-estimate range constraint; +if ~isfield(param,'valid_dB') || isempty(param.valid_dB) + param.valid_dB = [-inf inf]; +end + +% window: 2x1 or 2xN element numeric array; specifies the valid +% range for the noise-estimate; default is [-inf; inf] which effectively +% disables this valid noise-estimate range constraint; This parameter can +% be specified for each range line by passing in a 2 x Nx array where each +% column restricts the noise bins to be estimated +if ~isfield(param,'window') || isempty(param.window) + param.window = [-inf; inf]; +end +window = param.window; + +Nt = size(data,1); +Nx = size(data,2); + +if size(window,1) ~= 2 || (size(window,2) ~= 1 && size(window,2) ~= Nx) + error('window must be a 2 x 1 or 2 x Nx matrix (where Nx is the number of columns in the echogram).'); +end + +%% Convert window to range bins +if param.window_units == '%' || isequal(window,[-inf; inf]) + window(1,:) = max(1,min(Nt,round(window(1,:)*Nt))); + window(2,:) = max(window(1,:),min(Nt,round(window(2)*Nt))); + +elseif param.window_units == 's' + if ~isstruct(mdata) + error('If param.window_units is "s", mdata must be an echogram struct since this echogram struct is used to determine the window.'); + end + window(1,:) = min(Nt,max(1, interp1(mdata.Time,1:Nt,window(1,:),'nearest','extrap') )); + window(2,:) = max(window(1,:),min(Nt, interp1(mdata.Time,1:Nt,window(2,:),'nearest','extrap') )); + +elseif param.window_units == 'b' + window = max(1,min(Nt,round(window))); + +else + error('param.window_units must be "s" for seconds, "b" for bins (rows), or "%" for percentage ratio 0 to 1.'); +end + +if size(window,2) == 1 + noise = param.method_fh(data(window(1):window(2),:)); + +else + noise = nan(1,Nx); + for rline = 1:Nx + rbins = window(1,rline):window(2,rline); + if ~isempty(rbins) + noise(rline) = param.method_fh(data(rbins,rline)); + end + end +end + +noise = min(param.valid_dB(2), max(param.valid_dB(1), noise)); diff --git a/cresis-toolbox/display/echo_norm.m b/cresis-toolbox/display/echo_norm.m new file mode 100644 index 00000000..ea9a4197 --- /dev/null +++ b/cresis-toolbox/display/echo_norm.m @@ -0,0 +1,70 @@ +function data = echo_norm(mdata, param) +% data = echo_norm(mdata, param) +% +% The trend of the data is estimated using various methods and this trend +% is removed from the data. +% +% INPUTS: +% +% mdata: 2D input data matrix (log power) or echogram file structure +% +% param: struct controlling how the normalization is done +% +% OUTPUTS: +% +% data: normalized input (log power normalized) +% +% Examples: +% +% fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_standard/20140512_01/Data_20140512_01_018.mat'; +% mdata = load(fn); mdata.Data = 10*log10(mdata.Data); +% +% imagesc(echo_norm(mdata)); colorbar; caxis([0 1]); +% +% imagesc(echo_norm(mdata,struct('scale',[25 255]))); colorbar; caxis([0 255]); +% +% Author: John Paden +% +% See also: echo_detrend, echo_filt, echo_mult_suppress, echo_noise, +% echo_norm, echo_param, echo_stats, echo_stats_layer, echo_xcorr, +% echo_xcorr_profile + +if isstruct(mdata) + data = mdata.Data; +else + data = mdata; +end + +if ~exist('param','var') || isempty(param) + param = []; +end + +% valid_max_range_dB: 2 element numeric vector; specifies the valid range +% for the max value; default is [-inf inf]. Use [-inf inf] to effectively +% disable this valid max value range constraint; +if ~isfield(param,'valid_max_range_dB') || isempty(param.valid_max_range_dB) + param.valid_max_range_dB = [-inf inf]; +end + +if ~isfield(param,'scale') || isempty(param.scale) + param.scale = [0.1 0.9]; +end + + +% Estimate the noise +if isstruct(mdata) + mdata.Data = 10.^(mdata.Data/10); + noise = echo_noise(mdata, param); + noise = 10*log10(mean(noise(isfinite(noise)))); +else + noise = echo_noise(10.^(mdata/10), param); + noise = 10*log10(mean(noise(isfinite(noise)))); +end + +% Determine max of data +max_scalar = min(noise+param.valid_max_range_dB(2), ... + max(noise+param.valid_max_range_dB(1), ... + max(data(:)))); + +% Scale and offset data +data = param.scale(1) + (data-noise) / (max_scalar - noise) * (param.scale(2)-param.scale(1)); diff --git a/cresis-toolbox/display/echo_param.m b/cresis-toolbox/display/echo_param.m new file mode 100644 index 00000000..2390d6a8 --- /dev/null +++ b/cresis-toolbox/display/echo_param.m @@ -0,0 +1,82 @@ +function param = echo_param(mdata,mode) +% param = echo_param(mdata,mode) +% +% Returns the processing parameters structure used to create the data +% mdata. mdata is loaded from qlook.m or array.m echogram outputs. gRadar +% is merged with this structure to ensure paths are updated to the current +% environment. +% +% mdata: structure loaded from echogram file (e.g. CSARP_qlook or +% CSARP_standard) or a filename string that can be loaded to obtain the +% structure. +% +% mode: scalar numeric, default is zero. mode 0 returns parameter +% structure, mode 1 returns the string +% +% OUTPUTS: +% +% param: mode 0 returns parameter spreadsheet structure that was used to +% create echo_param, mode 1 returns the string containing the fieldname to +% this param structure +% +% fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_standard/20140512_01/Data_20140512_01_018.mat'; +% mdata = load(fn); +% param = echo_param(mdata); +% +% Author: John Paden +% +% See also: echo_detrend, echo_filt, echo_mult_suppress, echo_noise, +% echo_norm, echo_param, echo_stats, echo_stats_layer, echo_xcorr, +% echo_xcorr_profile + +if ~exist('mode','var') || isempty(mode) + mode = 0; +end + +if ischar(mdata) + % Assume mdata contains a filename string and load it + mdata = load(mdata); +end + +if mode == 0 + if isfield(mdata,'param') + param = mdata.param; + elseif isfield(mdata,'param_array') + param = mdata.param_array; + elseif isfield(mdata,'param_qlook') + param = mdata.param_qlook; + elseif isfield(mdata,'param_combine') + param = mdata.param_combine; + elseif isfield(mdata,'param_combine_wf_chan') + param = mdata.param_csarp; + elseif isfield(mdata,'param_get_heights') + param = mdata.param_get_heights; + else + error('There is no param_array, param_qlook, param_combine, param_combine_wf_chan, or param_get_heights field in mdata.'); + end + param = ct_param_path_update(param); + +else + if isfield(mdata,'param') + param = 'param'; + elseif isfield(mdata,'param_array') + param = 'param_array'; + elseif isfield(mdata,'param_qlook') + param = 'param_qlook'; + elseif isfield(mdata,'param_combine') + param = 'param_combine'; + elseif isfield(mdata,'param_combine_wf_chan') + param = 'param_combine_wf_chan'; + elseif isfield(mdata,'param_get_heights') + param = 'param_get_heights'; + else + error('There is no param_array, param_qlook, param_combine, param_combine_wf_chan, or param_get_heights field in mdata.'); + end + return +end + +% Merge the gRadar structure into the param structure so that the file +% paths match the current environment rather than the file paths from when +% it was processed. +global gRadar; +param = merge_structs(param,gRadar); diff --git a/cresis-toolbox/display/echo_param_update.m b/cresis-toolbox/display/echo_param_update.m new file mode 100644 index 00000000..94d782cc --- /dev/null +++ b/cresis-toolbox/display/echo_param_update.m @@ -0,0 +1,36 @@ +function mdata = echo_param_update(mdata,param_override) +% mdata = echo_param_update(mdata,param_override) +% +% Function to update the paths of each parameter structure in an echogram +% structure. Relies on ct_param_path_update. +% +% Inputs: +% +% mdata: echogram structure +% +% param_override: override parameters. If set to [] or not defined, then +% the global variable gRadar will be used. +% +% Outputs: +% +% param: updated echogram structure +% +% Example: +% +% echo_fn = '/N/dcwan/projects/cresis/output/rds/2018_Greenland_P3/CSARP_qlook/20180406_01/Data_20180406_01_001.mat'; +% mdata = load_L1B(echo_fn); +% mdata = echo_param_update(mdata); +% +% Author: John Paden + +if ~exist('param_override','var') || ~isstruct(param_override) + global gRadar; + param_override = gRadar; +end + +field_names = fieldnames(mdata); +for idx=1:length(field_names) + if strncmp(field_names{idx},'param',5) + mdata.(field_names{idx}) = ct_param_path_update(mdata.(field_names{idx}),param_override); + end +end diff --git a/cresis-toolbox/display/echo_remove_overlap.m b/cresis-toolbox/display/echo_remove_overlap.m new file mode 100644 index 00000000..65fb05ed --- /dev/null +++ b/cresis-toolbox/display/echo_remove_overlap.m @@ -0,0 +1,55 @@ +function mdata = echo_remove_overlap(mdata) +% mdata = echo_remove_overlap(mdata) +% +% Old data files contained data from neighboring frames so that the frame +% data overlapped from frame to frame. This function removes this +% overlapped data. +% +% INPUTS: +% +% mdata: structure loaded from echogram file (e.g. CSARP_qlook or +% CSARP_standard) +% +% OUTPUTS: +% +% mdata: structure loaded from echogram file (e.g. CSARP_qlook or +% CSARP_standard) with overlapped data removed from all fields (Data, +% GPS_time, Latitude, Longitude, Elevation, Roll, Pitch, Heading, Surface, +% Bottom) +% +% Examples: +% +% fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_standard/20140512_01/Data_20140512_01_018.mat'; +% mdata = load(fn); +% mdata = echo_remove_overlap(mdata); +% +% Author: John Paden +% +% See also: echo_detrend, echo_filt, echo_mult_suppress, echo_noise, +% echo_norm, echo_param, echo_stats, echo_stats_layer, echo_xcorr, +% echo_xcorr_profile + +param = echo_param(mdata); + +frames = frames_load(param); + +% Mask data from this frame only +mask_valid = mdata.GPS_time >= frames.gps_time(param.load.frm) & mdata.GPS_time < frames.gps_time(param.load.frm+1); + +mdata.GPS_time = mdata.GPS_time(mask_valid); +mdata.Latitude = mdata.Latitude(mask_valid); +mdata.Longitude = mdata.Longitude(mask_valid); +mdata.Elevation = mdata.Elevation(mask_valid); +mdata.Roll = mdata.Roll(mask_valid); +mdata.Pitch = mdata.Pitch(mask_valid); +mdata.Heading = mdata.Heading(mask_valid); + +mdata.Data = mdata.Data(:,mask_valid); + +if isfield(mdata,'Surface') + mdata.Surface = mdata.Surface(mask_valid); +end + +if isfield(mdata,'Bottom') + mdata.Bottom = mdata.Bottom(mask_valid); +end diff --git a/cresis-toolbox/processing/get_echogram_stats.m b/cresis-toolbox/display/echo_stats.m similarity index 77% rename from cresis-toolbox/processing/get_echogram_stats.m rename to cresis-toolbox/display/echo_stats.m index 0f9c64ab..efd89236 100644 --- a/cresis-toolbox/processing/get_echogram_stats.m +++ b/cresis-toolbox/display/echo_stats.m @@ -1,7 +1,11 @@ -function get_echogram_stats(param,param_override) -% get_echogram_stats(param,param_override) +function echo_stats(param,param_override) +% echo_stats(param,param_override) % % Author: John Paden +% +% See also: echo_detrend, echo_filt, echo_mult_suppress, echo_noise, +% echo_norm, echo_param, echo_stats, echo_stats_layer, echo_xcorr, +% echo_xcorr_profile %% General Setup % ===================================================================== @@ -14,11 +18,10 @@ function get_echogram_stats(param,param_override) %% Input arguments check and setup % ========================================================================= -echogram_fn_dir = ct_filename_out(param,param.get_echogram_stats.data_type); +echogram_fn_dir = ct_filename_out(param,param.echo_stats.data_type); % Load frames associated with this segment -frames_fn = ct_filename_support(param,'','frames'); -load(frames_fn); +frames = frames_load(param); % Load layer information layer_params = []; @@ -28,8 +31,8 @@ function get_echogram_stats(param,param_override) surf = opsLoadLayers(param,layer_params); -noise_bins = param.get_echogram_stats.noise_bins; -signal_bins = param.get_echogram_stats.signal_bins; +noise_bins = param.echo_stats.noise_bins; +signal_bins = param.echo_stats.signal_bins; %% Echogram stats @@ -50,11 +53,11 @@ function get_echogram_stats(param,param_override) peak_wfs = []; for frm = 1:length(frames.frame_idxs) %% Echogram stats: Load echogram - if param.get_echogram_stats.echogram_img == 0 + if param.echo_stats.echogram_img == 0 echogram_fn = fullfile(echogram_fn_dir,sprintf('Data_%s_%03d.mat',param.day_seg,frm)); else echogram_fn = fullfile(echogram_fn_dir, ... - sprintf('Data_img_%02d_%s_%03d.mat',param.get_echogram_stats.echogram_img,param.day_seg,frm)); + sprintf('Data_img_%02d_%s_%03d.mat',param.echo_stats.echogram_img,param.day_seg,frm)); end fprintf('%d of %d: %s\n', frm, length(frames.frame_idxs), echogram_fn); mdata = load_L1B(echogram_fn); @@ -94,7 +97,7 @@ function get_echogram_stats(param,param_override) end data = data_nomask; data(mask) = NaN; - data(data>param.get_echogram_stats.detrend_threshold) = NaN; + data(data>param.echo_stats.detrend_threshold) = NaN; %% Echogram stats: Average bin value % Mean value results @@ -132,21 +135,21 @@ function get_echogram_stats(param,param_override) first_bin = find(sums_frm~=0,1) + 0; last_bin = find(sums_frm~=0,1,'last') - 0; means_frm = sums_frm ./ counts_frm; - idxs = find(counts_frm > param.get_echogram_stats.sum_threshold); + idxs = find(counts_frm > param.echo_stats.sum_threshold); idxs = idxs(idxs>=first_bin & idxs <= last_bin); min_means(bin_start + idxs) = min(min_means(bin_start + idxs),means_frm(idxs)); %% Echogram stats: Peak waveforms [max_vals,max_idxs] = max(data_nomask,[],1); - peak_wfs_idxs = 1:length(param.get_echogram_stats.peak_wfs_bins); + peak_wfs_idxs = 1:length(param.echo_stats.peak_wfs_bins); cur_rline = size(peak_wfs,2); - peak_wfs = [peak_wfs, nan(length(param.get_echogram_stats.peak_wfs_bins),Nx)]; + peak_wfs = [peak_wfs, nan(length(param.echo_stats.peak_wfs_bins),Nx)]; for rline = 1:Nx if ~isnan(max_vals(rline)) - test = max_idxs(rline)+param.get_echogram_stats.peak_wfs_bins; + test = max_idxs(rline)+param.echo_stats.peak_wfs_bins; good_mask = test >= 1 & test <= Nt; - peak_wfs(peak_wfs_idxs(good_mask),cur_rline+rline) = data_nomask(max_idxs(rline)+param.get_echogram_stats.peak_wfs_bins(good_mask),rline); + peak_wfs(peak_wfs_idxs(good_mask),cur_rline+rline) = data_nomask(max_idxs(rline)+param.echo_stats.peak_wfs_bins(good_mask),rline); end end @@ -185,7 +188,7 @@ function get_echogram_stats(param,param_override) end %% Save Results -output_dir = fileparts(ct_filename_ct_tmp(param,'','echogram_stats','')); +output_dir = fileparts(ct_filename_ct_tmp(param,'','echo_stats','')); if ~exist(output_dir,'dir') fprintf('Creating directory %s\n', output_dir); mkdir(output_dir); @@ -201,10 +204,10 @@ function get_echogram_stats(param,param_override) grid(h_axes,'on'); xlabel(h_axes,'Two way travel time (\mus)'); ylabel(h_axes,'Relative power (dB)'); -fig_fn = [ct_filename_ct_tmp(param,'','echogram_stats','detrend') '.fig']; +fig_fn = [ct_filename_ct_tmp(param,'','echo_stats','detrend') '.fig']; fprintf('Saving %s\n', fig_fn); saveas(h_fig,fig_fn); -fig_fn = [ct_filename_ct_tmp(param,'','echogram_stats','detrend') '.jpg']; +fig_fn = [ct_filename_ct_tmp(param,'','echo_stats','detrend') '.jpg']; fprintf('Saving %s\n', fig_fn); saveas(h_fig,fig_fn); close(h_fig); @@ -224,8 +227,8 @@ function get_echogram_stats(param,param_override) h_axes = axes('parent',h_fig); plot_color = colormap(jet(length(SNR_bins))); for idx = 1:length(SNR_bins) - %scatter(h_axes,param.get_echogram_stats.peak_wfs_bins,SNR_wfs_norm(:,idx),[],SNR_bins(idx)*ones(size(SNR_wfs,1),1),'.'); - plot(h_axes,param.get_echogram_stats.peak_wfs_bins,SNR_wfs_norm(:,idx),'Color', plot_color(idx,:)); + %scatter(h_axes,param.echo_stats.peak_wfs_bins,SNR_wfs_norm(:,idx),[],SNR_bins(idx)*ones(size(SNR_wfs,1),1),'.'); + plot(h_axes,param.echo_stats.peak_wfs_bins,SNR_wfs_norm(:,idx),'Color', plot_color(idx,:)); hold(h_axes,'on'); end xlabel(h_axes,'Range bin'); @@ -236,10 +239,10 @@ function get_echogram_stats(param,param_override) h_color = colorbar(h_axes); set(get(h_color,'ylabel'),'string','SNR (dB)') grid(h_axes,'on'); -fig_fn = [ct_filename_ct_tmp(param,'','echogram_stats','peak_wfs') '.fig']; +fig_fn = [ct_filename_ct_tmp(param,'','echo_stats','peak_wfs') '.fig']; fprintf('Saving %s\n', fig_fn); saveas(h_fig,fig_fn); -fig_fn = [ct_filename_ct_tmp(param,'','echogram_stats','peak_wfs') '.jpg']; +fig_fn = [ct_filename_ct_tmp(param,'','echo_stats','peak_wfs') '.jpg']; fprintf('Saving %s\n', fig_fn); saveas(h_fig,fig_fn); close(h_fig); @@ -253,7 +256,7 @@ function get_echogram_stats(param,param_override) [max_vals_rounded,unique_idxs] = unique(max_vals_rounded); sidelobe_dB = sidelobe_dB(:,unique_idxs); sidelobe_vals = min(max_vals_rounded) : max(max_vals_rounded); -sidelobe_rows = param.get_echogram_stats.peak_wfs_bins; +sidelobe_rows = param.echo_stats.peak_wfs_bins; if isempty(max_vals_rounded) sidelobe_dB = nan(size(sidelobe_rows)); sidelobe_dB = sidelobe_dB(:); @@ -279,16 +282,16 @@ function get_echogram_stats(param,param_override) ylabel(h_axes,'Relative power (dB)'); grid(h_axes,'on'); title(sprintf('%s',regexprep(param.day_seg,'_','\\_'))); -fig_fn = [ct_filename_ct_tmp(param,'','echogram_stats','SNR') '.fig']; +fig_fn = [ct_filename_ct_tmp(param,'','echo_stats','SNR') '.fig']; fprintf('Saving %s\n', fig_fn); saveas(h_fig,fig_fn); -fig_fn = [ct_filename_ct_tmp(param,'','echogram_stats','SNR') '.jpg']; +fig_fn = [ct_filename_ct_tmp(param,'','echo_stats','SNR') '.jpg']; fprintf('Saving %s\n', fig_fn); saveas(h_fig,fig_fn); close(h_fig); % Matlab file -mat_fn = [ct_filename_ct_tmp(param,'','echogram_stats','stats') '.mat']; +mat_fn = [ct_filename_ct_tmp(param,'','echo_stats','stats') '.mat']; fprintf('Saving %s\n', mat_fn); -param_get_echogram_stats = param; -save(mat_fn,'-v7.3','param_get_echogram_stats','dt','gps_time','bins','sums','counts','min_means','all_bins','all_sums','all_counts','signal_max_vals','signal_mean_vals','noise_max_vals','noise_mean_vals','peak_wfs','SNR_wfs','SNR_bins','sidelobe_vals','sidelobe_rows','sidelobe_dB'); +param_echo_stats = param; +save(mat_fn,'-v7.3','param_echo_stats','dt','gps_time','bins','sums','counts','min_means','all_bins','all_sums','all_counts','signal_max_vals','signal_mean_vals','noise_max_vals','noise_mean_vals','peak_wfs','SNR_wfs','SNR_bins','sidelobe_vals','sidelobe_rows','sidelobe_dB'); diff --git a/cresis-toolbox/display/echo_stats_layer.m b/cresis-toolbox/display/echo_stats_layer.m new file mode 100644 index 00000000..7714443a --- /dev/null +++ b/cresis-toolbox/display/echo_stats_layer.m @@ -0,0 +1,258 @@ +function [max_power,aggregate_power,aggregate_bins,neighbor_power,waveforms] = echo_stats_layer(mdata,layer,param) +% [max_power,aggregate_power,aggregate_bins,neighbor_power,waveforms] = echo_stats_layer(mdata,layer,param) +% +% Returns statistics for layer in echogram. +% +% INPUTS: +% +% data: Assumed to be log power image of size Nt by Nx +% +% layer: 1 by Nx vector of a layer. If all values are less than 1, then +% layer is assumed to be two way travel time with units of seconds. If not, +% layer is assumed to be in range bins (rows) so that layer(col) indicates +% the location of the layer for column col in the echogram. +% +% param: structure defining how the layer statistics are estimated +% .inc_dec: scalar indicating the length of boxcar filter and decimation +% rate of incoherent "data" before statistics are extracted (echogram +% columns are circularly shifted to flatten the layer before averaging) +% +% .neighbor_window: scalar numeric indicating the size of the neighborhood +% power estimate (this window is relative to the start and end of the +% "window" that is used to get the layer power). +% +% .window: 2 x 1 vector of window relative to layer to obtain +% statistics. Default is [-1; 1] (i.e. a length three window which starts +% one bin before the layer and ends one bin after the layer). If the +% window_units are seconds, then the default is [-dt dt] where dt is the +% fast-time-sample spacing of the echogram. Generally this should be a +% negative integer (starting before the layer) followed by a positive +% integer (ending after the layer). +% +% .window_mode: string with either 'f' (fixed) or 'a' (adaptive) in it. +% Adaptive mode should have significant multilooking (e.g. inc_dec >> 1) +% to work properly. Default is fixed. +% +% .window_threshold: numeric scalar indicating the adaptive window +% threshold relative to the neighbor_window power. The default is 2 (3 dB +% above the neighborhood power). For example, if the neighborhood power is +% -100 dB and window_threshold is set to 2 (3 dB), then the adaptive +% window will end when the power drops below -97 dB. If the power never +% drops below the threshold, then all bins are used. The adaptive window +% grows outward from the layer, so the start bin is when the threshold is +% passed towards negative time starting at the layer and the stop bin is +% when the threshold is passed towards positive time again starting at the +% layer. +% +% .window_units: string with either 'b' (bins) or 's' (seconds) in it. +% Default is 'b' or bins. +% +% OUTPUTS: +% +% Nx is the new dimension of data after inc_dec operation. +% +% max_power: 1 by Nx vector of the maximum power in the window for each +% column +% +% aggregate_power: 1 by Nx vector of the sum of the power in the window. If +% fixed window_mode, then every bin in the window is used. If adaptive +% window_mode is used, then the bin +% +% aggregate_bins: 1 by Nx vector of the number of bins included in the +% aggregate_power sum. This only varies when window_mode is adaptive. +% +% neighbor_power: 2 by Nx vector of the power in the neighborhood around +% the window. The first row gives the power estimate before the window. The +% second row gives the power estimate after the window. The size of the +% neighborhood power window is set by neighbor_window and includes the bins +% preceding and succeeding the window. +% +% waveforms: Nt_window by Nx array of the extracted waveform from each +% column of the data +% +% EXAMPLE: +% +% % Example getting surface and surface multiple statistics +% fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_standard/20140512_01/Data_20140512_01_018.mat'; +% mdata = load(fn); +% layer = layerdata.load_layers(mdata,'','surface'); +% +% param.window = [-3 11]; +% param.window_units = 'b'; % bins +% param.window_mode = 'f'; % fixed +% param.inc_dec = 1; +% [max_power,~,~,neighbor_power,~] = echo_stats_layer(mdata,layer,param); +% [max_power_mult,~,~,neighbor_power_mult,~] = echo_stats_layer(mdata,layer*2,param); +% figure; +% plot(lp(max_power),'b','LineWidth',2); +% hold on; +% plot(lp(max_power_mult),'r','LineWidth',2); +% plot(lp(neighbor_power.'),'b'); +% plot(lp(neighbor_power_mult.'),'r'); +% +% % Example bottom statistics +% fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_standard/20140512_01/Data_20140512_01_018.mat'; +% mdata = load(fn); +% layer = layerdata.load_layers(mdata,'','bottom'); +% +% param.window = [-1.5e-6 4e-6]; +% param.window_units = 's'; % seconds +% param.window_mode = 'a'; % adaptive +% param.inc_dec = 100; +% [max_power,aggregate_power,aggregate_bins,neighbor_power,waveforms] = echo_stats_layer(mdata,layer,param); +% figure; +% plot(lp(max_power)); +% hold on; +% plot(lp(aggregate_power.')); +% plot(lp(neighbor_power.')); +% legend('max','aggregate','neighbor start','neighbor stop'); +% figure; +% imagesc(lp(waveforms)); +% +% Author: John Paden +% +% See also: echo_detrend, echo_filt, echo_mult_suppress, echo_noise, +% echo_norm, echo_param, echo_stats, echo_stats_layer, echo_xcorr, +% echo_xcorr_profile + +%% Input Check + +if isstruct(mdata) + data = mdata.Data; +else + data = mdata; +end + +if ~exist('layer','var') || isempty(layer) + if isstruct(mdata) + layers = layerdata(echo_param(mdata)); + layer = layers.get_layer_by_gps_time(mdata.GPS_time,'surface'); + else + error('Layer must be defined if mdata is not an echogram struct since this echogram struct is used to load the default surface layer.'); + end +end + +if any(layer<1) + % layer is two way travel time + if isstruct(mdata) + layer = interp1(mdata.Time,1:length(mdata.Time),layer); + end +end + +if ~exist('param','var') || isempty(param) + param = []; +end + +% inc_dec: filtering and decimation of incoherent "data" before statistics +% are extracted (echogram columns are circularly shifted to flatten the +% layer before averaging) +if ~isfield(param,'inc_dec') || isempty(param.inc_dec) + param.inc_dec = 1; +end + +% window_mode: string with either 'f' (fixed) or 'a' (adaptive) in it. +% Adaptive mode should have significant multilooking (e.g. inc_dec >> 1) to +% work properly. +if ~isfield(param,'window_mode') || isempty(param.window_mode) + param.window_mode = 'f'; +end + +if ~isfield(param,'window_threshold') || isempty(param.window_threshold) + param.window_threshold = 2; % 3 dB +end + +% window_units: string with either 'b' (bins) or 's' (seconds) in it +if ~isfield(param,'window_units') || isempty(param.window_units) + param.window_units = 'b'; +end + +% Convert param.window into bins +if param.window_units == 's' + if isstruct(mdata) + dt = mdata.Time(2) - mdata.Time(1); + % window: two elements vector of window relative to layer to obtain + % statistics + if ~isfield(param,'window') || isempty(param.window) + param.window = [-dt; dt]; + end + bins = round(param.window / dt); + bins = bins(1):bins(end); + % neighbor_window: scalar numeric indicating the size of the + % neighborhood power estimate (this window is relative to the start and + % end of the "window" that is used to get the layer power). + if ~isfield(param,'neighbor_window') || isempty(param.neighbor_window) + param.neighbor_window = dt; + end + neighbor_bins = 1 : param.neighbor_window / dt; + else + error('If param.window_units is "s", mdata must be an echogram struct since this echogram struct is used to determine the window.'); + end +else + if ~isfield(param,'window') || isempty(param.window) + param.window = [-1; 1]; + end + bins = param.window(1):param.window(end); + if ~isfield(param,'neighbor_window') || isempty(param.neighbor_window) + param.neighbor_window = 1; + end + neighbor_bins = 1 : param.neighbor_window; +end + +% Round layer +layer = round(interp_finite(layer)); + +Nbins = length(bins); + +%% Setup +Nt = size(data,1); +Nx = size(data,2); + +%% Flatten the layer +mid_point = round(Nt/2); +max_shift = max(abs(mid_point-layer)); +data = [data; nan(max_shift,Nx)]; +for rline = 1:Nx + data(:,rline) = circshift(data(:,rline), mid_point-layer(rline)); +end + +%% Apply along-track filtering +data = fir_dec(data, param.inc_dec); +Nx = size(data,2); + +%% Estimate statistics +max_power = max(data(mid_point+bins,:),[],1); +neighbor_power(1,:) = mean(data(mid_point+bins(1)-neighbor_bins,:),1); +neighbor_power(2,:) = mean(data(mid_point+bins(end)+neighbor_bins,:),1); +waveforms = data(mid_point+bins,:); + +if param.window_mode == 'f' + aggregate_power = sum(data(mid_point+bins,:),1); + aggregate_bins = (bins(end)-bins(1))*ones(1,Nx); +else + aggregate_power = nan(1,Nx); + aggregate_bins = zeros(1,Nx); + for rline = 1:Nx + % Get the bins for this range line + start_bin = find(data(mid_point+ (-1:-1:bins(1)),rline) < param.window_threshold*neighbor_power(1,rline),1); + if isempty(start_bin) + start_bin = bins(1); + end + stop_bin = find(data(mid_point+ (1:bins(end)),rline) < param.window_threshold*neighbor_power(2,rline),1); + if isempty(stop_bin) + stop_bin = bins(end); + end + if 0 + %% Debug plot + figure(1); clf; + plot(lp(data(:,rline))); + hold on; + plot(mid_point-start_bin, lp(data(mid_point-start_bin,rline)),'x'); + plot(mid_point+stop_bin, lp(data(mid_point+stop_bin,rline)),'x'); + plot(mid_point + bins(1) - 1, lp(neighbor_power(1,rline)),'o'); + plot(mid_point + bins(end) - 1, lp(neighbor_power(2,rline)),'o'); + grid on; + end + aggregate_bins(rline) = stop_bin-start_bin; + aggregate_power(rline) = sum(data(mid_point+(-start_bin:stop_bin),rline)); + end +end diff --git a/cresis-toolbox/display/echo_xcorr.m b/cresis-toolbox/display/echo_xcorr.m new file mode 100644 index 00000000..5e01c11e --- /dev/null +++ b/cresis-toolbox/display/echo_xcorr.m @@ -0,0 +1,68 @@ +function data = echo_xcorr(data, param) +% data = echo_xcorr(data, param) +% +% Fast-time cross correlation. +% +% INPUTS: +% +% data = 2D input data matrix (log power) +% +% param: struct controlling how the cross correlation is done +% +% OUTPUTS: +% +% data: detrended input (log power) +% +% Examples: +% +% fn = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_standard/20140512_01/Data_20140512_01_018.mat'; +% mdata = load(fn); mdata.Data = 10*log10(mdata.Data); +% +% imagesc(echo_xcorr(mdata)); +% +% imagesc(echo_xcorr(mdata,echo_xcorr_profile('peaky'))); +% +% imagesc(echo_xcorr(mdata,echo_xcorr_profile('snow'))); +% +% imagesc(echo_xcorr(mdata,echo_xcorr_profile('short_unitstep'))); +% +% imagesc(echo_xcorr(mdata,echo_xcorr_profile('long_unitstep'))); +% +% Author: John Paden +% +% See also: echo_detrend, echo_filt, echo_mult_suppress, echo_noise, +% echo_norm, echo_param, echo_stats, echo_stats_layer, echo_xcorr, +% echo_xcorr_profile + +%% Input checks + +if isstruct(data) + data = data.Data; +else + data = data; +end + +if ~exist('param','var') || isempty(param) + param = []; +end + +% h_filt: vector of cross-correlation filter coefficients applied using +% filter.m +if ~isfield(param,'h_filt') || isempty(param.h_filt) + param.h_filt = fliplr([0.1*ones(1,6) 0.5*ones(1,7)]); +end +param.h_filt = param.h_filt-mean(param.h_filt); + +% h_filt_offset: integer scalar which indicates the delay compensation +% needed to remove the filter.m delay from h_filt. I.e. this is the delay +% offset in bins to apply after h_filt in order to align the filtered image +% with the desired peak location +if ~isfield(param,'h_filt_offset') || isempty(param.h_filt_offset) + param.h_filt_offset = -4; +end + +%% Fast-time correlation in log domain +data = filter(param.h_filt,1,data); +data = circshift(data,[param.h_filt_offset 0]); +data(1:length(param.h_filt),:) = NaN; +data(end-length(param.h_filt)+1:end,:) = NaN; diff --git a/cresis-toolbox/display/echo_xcorr_profile.m b/cresis-toolbox/display/echo_xcorr_profile.m new file mode 100644 index 00000000..7be708ee --- /dev/null +++ b/cresis-toolbox/display/echo_xcorr_profile.m @@ -0,0 +1,73 @@ +function param = echo_xcorr_profile(profile_name,varargin) +% param = echo_xcorr_profile(profile_name,varargin) +% +% Fast-time cross correlation parameter profiles. +% +% INPUTS: +% +% profile_name: string containing profile name +% +% varargin: name value pairs (even input arguments give the field to update such +% as "h_filt_offset" and odd input arguments give the value). +% +% OUTPUTS: +% +% param: parameters for the selected profile with the varargin updates +% applied +% +% Examples: +% +% See echo_xcorr.m +% +% Author: John Paden +% +% See also: echo_detrend, echo_filt, echo_mult_suppress, echo_noise, +% echo_norm, echo_param, echo_stats, echo_stats_layer, echo_xcorr, +% echo_xcorr_profile + +switch (profile_name) + case {'','short_unitstep'} % RDS + param.h_filt = fliplr([0.1*ones(1,6) 0.5*ones(1,7)]); + param.h_filt = param.h_filt-mean(param.h_filt); + param.h_filt_offset = -4; + case 'long_unitstep' % RDS long + param.h_filt = fliplr([0.1*ones(1,10) 0.5*ones(1,11)]); + param.h_filt = param.h_filt-mean(param.h_filt); + param.h_filt_offset = -8; + case 'xlong_unitstep' % Accum or UWB RDS + param.h_filt = fliplr([0.1*ones(1,20) 0.5*ones(1,21)]); + param.h_filt = param.h_filt-mean(param.h_filt); + param.h_filt_offset = -16; + case 'xxlong_unitstep' % Snow + param.h_filt = fliplr([0.1*ones(1,50) 0.5*ones(1,51)]); + param.h_filt = param.h_filt-mean(param.h_filt); + param.h_filt_offset = -40; + case 'snow' % Snow layer + param.h_filt = fliplr([0.1*ones(1,21) 4*ones(1,5) 0.5*ones(1,21)]); + param.h_filt = param.h_filt-mean(param.h_filt); + param.h_filt_offset = -20; %-48 + case 'peaky' % Enhance peaky as opposed to unit step responses + param.h_filt = [0.1 0.1 0.4 0.5 0.4 0.1 0.1]; + param.h_filt = param.h_filt-mean(param.h_filt); + param.h_filt_offset = -3; + case 'short_unitstep+peaky' % Enhance peaky with unit step responses + param.h_filt = fliplr([0.1*ones(1,6) 0.5*ones(1,7)]) + [0 0 0 0.1 0.1 0.4 0.5 0.4 0.1 0.1 0 0 0]; + param.h_filt = param.h_filt-mean(param.h_filt); + param.h_filt_offset = -4; + otherwise + error('Profile %s does not exist.', profile_name); +end + +% Assign custom name-value pairs +if nargin > 1 + for idx = 3:2:nargin + if strcmpi(varargin{idx-2},'h_filt_len') + % Handle special case to make generic h_filt from h_filt_len + % parameter + param.h_filt = fliplr([0.1*ones(1,varargin{idx-1}) 0.5*ones(1,varargin{idx-1}+1)]); + param.h_filt = param.h_filt-mean(param.h_filt); + else + param.(varargin{idx-2}) = varargin{idx-1}; + end + end +end \ No newline at end of file diff --git a/cresis-toolbox/display/func_denoise_sw1d.m b/cresis-toolbox/display/func_denoise_sw1d.m deleted file mode 100644 index 2444822d..00000000 --- a/cresis-toolbox/display/func_denoise_sw1d.m +++ /dev/null @@ -1,60 +0,0 @@ -function [sigDEN,wDEC] = func_denoise_sw1d(SIG) -% FUNC_DENOISE_SW1D Saved Denoising Process. -% SIG: vector of data -% ------------------- -% sigDEN: vector of denoised data -% wDEC: stationary wavelet decomposition - -% Auto-generated by Wavelet Toolbox on 10-Jun-2012 18:11:49 - -% Analysis parameters. -%--------------------- -wname = 'haar'; -level = 3; - -% Denoising parameters. -%---------------------- -% meth = 'sqtwolog'; -% scal_OR_alfa = mln; -sorh = 's'; % Specified soft or hard thresholding -thrSettings = {... - [... - 1.000000000000000 1024.000000000000000 2.905534542026620; ... - ]; ... - [... - 1.000000000000000 1024.000000000000000 4.492379955192392; ... - ]; ... - [... - 1.000000000000000 1024.000000000000000 5.751309758393311; ... - ]; ... - }; - -% Decompose using SWT. -%--------------------- -wDEC = swt(SIG,level,wname); - -% Denoise. -%--------- -len = length(SIG); -for k = 1:level - thr_par = thrSettings{k}; - if ~isempty(thr_par) - NB_int = size(thr_par,1); - x = [thr_par(:,1) ; thr_par(NB_int,2)]; - x = round(x); - x(x<1) = 1; - x(x>len) = len; - thr = thr_par(:,3); - for j = 1:NB_int - if j==1 , d_beg = 0; else d_beg = 1; end - j_beg = x(j)+d_beg; - j_end = x(j+1); - j_ind = (j_beg:j_end); - wDEC(k,j_ind) = wthresh(wDEC(k,j_ind),sorh,thr(j)); - end - end -end - -% Reconstruct the denoise signal using ISWT. -%------------------------------------------- -sigDEN = iswt(wDEC,wname); diff --git a/cresis-toolbox/display/hsv_plot.m b/cresis-toolbox/display/hsv_plot.m index c32e1cbb..de424511 100644 --- a/cresis-toolbox/display/hsv_plot.m +++ b/cresis-toolbox/display/hsv_plot.m @@ -26,6 +26,8 @@ % set(get(h_colorbar,'ylabel'),'string','angle (rad)') % % Author: John Paden +% +% See also: hsv_plot.m, hsv_plot_coherence.m if exist('val_limits','var') if length(val_limits) == 1 diff --git a/cresis-toolbox/display/hsv_plot_coherence.m b/cresis-toolbox/display/hsv_plot_coherence.m index b26c1ed9..dc9f166b 100644 --- a/cresis-toolbox/display/hsv_plot_coherence.m +++ b/cresis-toolbox/display/hsv_plot_coherence.m @@ -23,6 +23,8 @@ % set(get(h_colorbar,'ylabel'),'string','angle (rad)') % % Author: John Paden +% +% See also: hsv_plot.m, hsv_plot_coherence.m val = abs(coherence); @@ -39,3 +41,4 @@ val = val / coherence_limits(2); data_rgb = hsv2rgb(cat(3,hue,sat,val)); + diff --git a/cresis-toolbox/display/inc_filt.m b/cresis-toolbox/display/inc_filt.m deleted file mode 100644 index 227caee2..00000000 --- a/cresis-toolbox/display/inc_filt.m +++ /dev/null @@ -1,39 +0,0 @@ -function data = inc_filt(data,rows,cols,method) -% data = inc_filt(data,rows,cols,method) -% -% Incoherently averages data in row and column dimensions. Detects the -% type of data (complex or real) and then converts to log scale. -% It does not decimate the data. -% -% INPUTS: -% method: optional parameter, default is 1 for real data, 2 for complex -% 1: takes abs(data) -% 2: takes abs(data)^2 -% -% Example: -% -% imagesc(incfilt(data,4,10)) -% -% imagesc(incfilt(data,4,10,2)) -% -% Author: John Paden -% -% See also: lp.m - - -if ~exist('method','var') || isempty(method) - if isreal(data) && all(data(:) > 0) - method = 1; - else - method = 2; - end -end - -if method == 1 - data = 10*log10(filter2(ones(rows,cols)/(rows*cols),abs(data))); -else - data = 10*log10(filter2(ones(rows,cols)/(rows*cols),abs(data).^2)); -end - -return; - diff --git a/cresis-toolbox/display/local_detrend.m b/cresis-toolbox/display/local_detrend.m deleted file mode 100644 index 7913e023..00000000 --- a/cresis-toolbox/display/local_detrend.m +++ /dev/null @@ -1,117 +0,0 @@ -function data = local_detrend(data, B_noise, B_sig, cmd, minVal) -% data = local_detrend(data, B_noise, B_sig, cmd, minVal) -% -% data = 2D input data matrix (linear power) -% B_noise = 2x1 vector giving size of noise window -% B_noise(1) = row/first-dim size -% B_noise(2) = col/second-dim size -% B_sig = 2x1 vector giving size of signal window -% B_sig(1) = row/first-dim size -% B_sig(2) = col/second-dim size -% cmd = scalar command to combine estimates -% 0: does nothing at all (no filtering or power detection) -% 1: combine with max -% 2: combine with min -% 3: use just method 1 [default] -% 4: use just method 2 -% 5: use no detrend, but filters -% minVal = scalar minimum value of detrend [-inf default] -% Units of dB (so 10*log10(data)) -% -% This local detrend function combines to noise power -% estimates: -% 1. average power for each bin (mean on second dimension) -% 2. average power from noise window (mean from filter2) -% The signal power is then scaled relative to the noise power. -% -% imagesc(10*log10(local_detrend(data,[80 20], [4 2]))); -% -% imagesc(10*log10(local_detrend(data,[80 20], [4 2], , 220))); -% -% Look into: -% Data = tonemap(repmat(Data,[1 1 3]), 'AdjustLightness', [0.1 1], 'AdjustSaturation', 1.5); -% Data = Data(:,:,2); - -debug_level = 1; - -if ~exist('cmd','var') || isempty(cmd) - cmd = 3; -end -if ~exist('minVal','var') || isempty(minVal) - minVal = -inf; -end -if ~exist('B_noise','var') || isempty(B_noise) - B_noise = [1 1]; -end -if ~exist('B_sig','var') || isempty(B_sig) - B_sig = [1 10]; -end - -if cmd == 0 - return; -end - -if cmd == 5 - data = filter2(ones(B_sig(1),B_sig(2))/(B_sig(1)*B_sig(2)),data); - return; -end - -% Prevent log function from returning -inf -data(data==0) = min(data(data>0)); - -% Detrend method 1 -detrend1 = mean(data,2); -filter_length = ceil(size(data,1)/25); -detrend1_tmp = 10*log10(filter(exp(-(linspace(-2,2,filter_length)).^2),1,detrend1)); -detrend1(1:ceil(filter_length/2)) = detrend1_tmp(filter_length); -detrend1(floor(filter_length/2)+1:end-ceil(filter_length/2)) = detrend1_tmp(filter_length+1:end); -detrend1(end-ceil(filter_length/2)+1:end) = detrend1_tmp(end); - -% [B,A] = butter(2,1/(0.05*size(data,1))); -% detrend1 = filtfilt(B,A,10*log10(detrend1)); -if debug_level == 2 - figure(1); clf; - plot(detrend1); -end -detrend1 = repmat(detrend1,[1 size(data,2)]); - -if cmd == 1 || cmd == 2 || cmd == 4 - % Detrend method 2 - detrend2 = filter2(ones(B_noise(1),B_noise(2))/(B_noise(1)*B_noise(2)),data); - %detrend2 = medfilt2(data,[B_noise(1),B_noise(2)]); - detrend2 = 10*log10(detrend2); - if debug_level == 2 - figure(1); clf; - imagesc(detrend2); - colorbar; - end -end - -% Detrend methods combined -switch cmd - case 1 - detrendc = max(detrend1,detrend2); - case 2 - detrendc = min(detrend1,detrend2); - case 3 - detrendc = detrend1; - case 4 - detrendc = detrend2; - otherwise - warning('Invalid command (using default of 1)'); - detrendc = max(detrend1,detrend2); -end - -% The detrend normalizes the data, this limits the -% normalization (noise powers below the threshold -% won't be normalized more than the threshold) -detrendc(detrendc depth(1) % % Example: +% ----------------------------------------------------------------------- % [depth,er] = summitPerm(195e6); % [TWtime,gain] = genPropProfileFromPerm(depth,er,195e6); % plot(depth(2:end),10*log10(gain)) % +% ----------------------------------------------------------------------- +% freq = [150]*1e6; +% depth = [1 2 3 4]; +% er = [1 1.7 1.9 2.1 2.5 3.15]'; +% [TWtime,gain] = genPropProfileFromPerm(depth,er,freq); +% +% ----------------------------------------------------------------------- +% freq = [120:10:160]*1e6; +% depth = [1 2 3 4]; +% er = [1*ones(1,5) +% 1.7*ones(1,5) +% 2*ones(1,5) +% 3.15*ones(1,5)]; +% [TWtime,gain] = genPropProfileFromPerm(depth,er,freq); +% +% ----------------------------------------------------------------------- +% freq = 150e6; +% [depth,er] = gisp2Perm(freq); +% [TWtime,gain] = genPropProfileFromPerm(depth,er,freq); +% % Author: John Paden physical_constants; + +if length(depth) == 1 + depth = [0 1]; + TWtime = [0 depth(end)/c*2*sqrt(er)]; + gain = [1 1]; + return; +end + w = 2*pi*freq; % Preallocate memory @@ -93,27 +122,4 @@ gain(n,:) = gain(n-1,:).*exp(-4*alpha.*thick).*tau; end -return; - -% ------------------------------------------------------------------------------ -% Example -% ------------------------------------------------------------------------------ -% freq = [150]*1e6; -% depth = [1 2 3 4]; -% er = [1 1.7 1.9 2.1 2.5 3.15]'; -% [TWtime,gain] = genPropProfileFromPerm(depth,er,freq); -% -% freq = [120:10:160]*1e6; -% depth = [1 2 3 4]; -% er = [1*ones(1,5) -% 1.7*ones(1,5) -% 2*ones(1,5) -% 3.15*ones(1,5)]; -% [TWtime,gain] = genPropProfileFromPerm(depth,er,freq); -% -% freq = 150e6; -% [depth,er] = gisp2Perm(freq); -% [TWtime,gain] = genPropProfileFromPerm(depth,er,freq); - - - +TWtime = [0; TWtime]; diff --git a/cresis-toolbox/em_model/smb_to_layers.m b/cresis-toolbox/em_model/smb_to_layers.m new file mode 100644 index 00000000..453dbc8b --- /dev/null +++ b/cresis-toolbox/em_model/smb_to_layers.m @@ -0,0 +1,50 @@ +function layers = smb_to_layers(smb) +% layers = smb_to_layers(smb) +% +% Converts a list of annual surface mass balances at a location into a list +% of layer thicknesses. Accounts for how negative mass balance (e.g. due to +% melting) would affect previous accumulation. +% +% smb: Numeric array. Surface mass balance with the most recent year first. +% Units of meters. +% +% layers: Numeric array. Layer thicknesses associated with the surface mass +% balances. +% +% Example: +% smb = [1 -2 1 1 0.5 1 -2 2.1 1.5]; +% % This means the most recent year has 1 m of accumulation, the year +% % before had 2 m of melt, the year before that had 1 m of accum, etc. +% layers = smb_to_layers(smb) +% +% % Expected output explanation: +% % layers = [1 0 0 0 0.5 1 0 0.1 1.5] +% % The first -2 cancels out the two 1's that came before it. +% % The second -2 only partially cancels out the 2.1 that came before it. + +% How much melt has accumulated +cur_melt = 0; + +% Starting from the most recent year, progress back through time to +% reconstruct the layers +layers = zeros(size(smb)); +for idx = 1:length(smb) + if smb(idx) < 0 + % Negative smb, so layer has zero thickness and we need to cumulate the + % melt into cur_melt + cur_melt = cur_melt - smb(idx); + layers(idx) = 0; + elseif cur_melt > 0 + % Positive smb, but there is unaccounted for melting from a previous + % year + cur_melt = cur_melt - smb(idx); + if cur_melt < 0 + layers(idx) = -cur_melt; + cut_melt = 0; + end + else + % Simple case where there is no accumulated melt and the smb is + % positive. The layer thickness is equal to the smb. + layers(idx) = smb(idx); + end +end diff --git a/cresis-toolbox/example_startup.m b/cresis-toolbox/example_startup.m index f39dfcea..53b38273 100644 --- a/cresis-toolbox/example_startup.m +++ b/cresis-toolbox/example_startup.m @@ -73,9 +73,10 @@ profile(pidx).cluster.max_retries = 1; profile(pidx).cluster.submit_pause = 0; profile(pidx).cluster.stat_pause = 1; - - profile(pidx).ops.url = 'https://ops.cresis.ku.edu/'; % Read-only for outside of CReSIS - % profile(pidx).ops.url = 'http://ops.cresis.ku.edu/'; % Use from within CReSIS + + profile(pidx).ops.url = 'https://ops.cresis.ku.edu/'; + profile(pidx).ops.google_map_api_key = 'AIzaSyCNexiP6WcIda8ZEa2MnwznWrGotDoLu0w'; % Fill in with your Google API key + %% Profile Linux/Mac (PROFILE 2) % ---------------------------------------------------------------------- @@ -107,9 +108,10 @@ profile(pidx).cluster.submit_pause = 0.2; profile(pidx).cluster.stat_pause = 2; profile(pidx).cluster.file_check_pause = 4; - - profile(pidx).ops.url = 'https://ops.cresis.ku.edu/'; % Read-only for outside of CReSIS - % profile(pidx).ops.url = 'http://ops.cresis.ku.edu/'; % Use from within CReSIS + + profile(pidx).ops.url = 'https://ops.cresis.ku.edu/'; + profile(pidx).ops.google_map_api_key = 'AIzaSyCNexiP6WcIda8ZEa2MnwznWrGotDoLu0w'; % Fill in with your Google API key + %% KU Profile Linux (PROFILE 3) % ---------------------------------------------------------------------- @@ -131,7 +133,7 @@ profile(pidx).cluster.data_location = fullfile(profile(pidx).tmp_file_path,'cluster-temp'); - profile(pidx).cluster.type = 'torque'; + profile(pidx).cluster.type = 'slurm'; %profile(pidx).cluster.type = 'matlab'; %profile(pidx).cluster.type = 'debug'; profile(pidx).cluster.max_jobs_active = 96; @@ -145,17 +147,17 @@ profile(pidx).cluster.mem_to_ppn = 0.9 * 16e9 / 8; % profile(pidx).cluster.mem_to_ppn = 0.9 * 256e9 / 24; profile(pidx).cluster.max_ppn = 4; - profile(pidx).cluster.max_mem_per_job = 15e9; - profile(pidx).cluster.max_mem_mode = 'debug'; - %profile(pidx).cluster.qsub_submit_arguments = '-m n -q high -l nodes=1:ppn=%p,pmem=%m,walltime=%t'; - %profile(pidx).cluster.qsub_submit_arguments = '-m n -l nodes=1:ppn=%p,pmem=%m,walltime=%t'; + profile(pidx).cluster.max_mem_per_job = 16e9; + profile(pidx).cluster.mem_mult_mode = 'debug'; + profile(pidx).cluster.slurm_submit_arguments = '--partition=cresis -N 1 -n 1 --cpus-per-task=%p --mem=%m --time=%t'; %profile(pidx).cluster.cpu_time_mult = 2; %profile(pidx).cluster.mem_mult = 2; %profile(pidx).cluster.ppn_fixed = 4; - % profile(pidx).ops.url = 'https://ops.cresis.ku.edu/'; % Read-only for outside of CReSIS - profile(pidx).ops.url = 'http://ops.cresis.ku.edu/'; % Use from within CReSIS - + profile(pidx).ops.url = 'https://ops.cresis.ku.edu/'; + profile(pidx).ops.google_map_api_key = 'AIzaSyCNexiP6WcIda8ZEa2MnwznWrGotDoLu0w'; % Fill in with your Google API key + + %% KU Field Profile Linux (PROFILE 4) % ---------------------------------------------------------------------- pidx = 4; % profile index @@ -188,10 +190,10 @@ profile(pidx).cluster.max_ppn = 6; profile(pidx).cluster.job_complete_pause = 5; profile(pidx).cluster.max_mem_per_job = 126e9; - profile(pidx).cluster.max_mem_mode = 'debug'; + profile(pidx).cluster.mem_mult_mode = 'debug'; - profile(pidx).ops.url = 'https://ops.cresis.ku.edu/'; % Read-only for outside of CReSIS - % profile(pidx).ops.url = 'http://ops.cresis.ku.edu/'; % Use from within CReSIS + profile(pidx).ops.url = 'https://ops.cresis.ku.edu/'; + profile(pidx).ops.google_map_api_key = 'AIzaSyCNexiP6WcIda8ZEa2MnwznWrGotDoLu0w'; % Fill in with your Google API key %% KU Desktop Profile Windows (PROFILE 5) @@ -223,8 +225,8 @@ profile(pidx).cluster.submit_pause = 0; profile(pidx).cluster.stat_pause = 1; - % profile(pidx).ops.url = 'https://ops.cresis.ku.edu/'; % Read-only for outside of CReSIS - profile(pidx).ops.url = 'http://ops.cresis.ku.edu/'; % Use from within CReSIS + profile(pidx).ops.url = 'https://ops.cresis.ku.edu/'; + profile(pidx).ops.google_map_api_key = 'AIzaSyCNexiP6WcIda8ZEa2MnwznWrGotDoLu0w'; % Fill in with your Google API key %% IU Profile Linux (PROFILE 6) % ---------------------------------------------------------------------- @@ -235,7 +237,7 @@ profile(pidx).param_path = sprintf('%s/scripts/ct_params/',home_dir); username = getenv('USER'); - profile(pidx).tmp_file_path = sprintf('/N/dc2/scratch/%s/ct_user_tmp/',username); % scratch may be on dcwan or dc2 + profile(pidx).tmp_file_path = sprintf('/N/slate/%s/ct_user_tmp/',username); % scratch may be on dcwan or slate profile(pidx).data_path = '/N/dcwan/projects/cresis/'; profile(pidx).data_support_path = '/N/dcwan/projects/cresis/metadata/'; @@ -246,49 +248,52 @@ profile(pidx).cluster.data_location = fullfile(profile(pidx).tmp_file_path,'cluster-temp'); - profile(pidx).cluster.type = 'torque'; + %profile(pidx).cluster.type = 'slurm'; + %profile(pidx).cluster.type = 'torque'; %profile(pidx).cluster.type = 'matlab'; - %profile(pidx).cluster.type = 'debug'; - if 1 + profile(pidx).cluster.type = 'debug'; + if 0 profile(pidx).cluster.ssh_hostname = 'karst.uits.iu.edu'; profile(pidx).cluster.mem_to_ppn = 0.9 * 32e9 / 16; profile(pidx).cluster.max_ppn = 8; profile(pidx).cluster.max_mem_per_job = 30e9; - profile(pidx).cluster.max_mem_mode = 'debug'; + profile(pidx).cluster.mem_mult_mode = 'debug'; else - %profile(pidx).cluster.ssh_hostname = 'carbonate.uits.iu.edu'; profile(pidx).cluster.mem_to_ppn = 0.9 * 256e9 / 24; profile(pidx).cluster.max_ppn = 12; profile(pidx).cluster.max_mem_per_job = 250e9; - profile(pidx).cluster.max_mem_mode = 'debug'; + profile(pidx).cluster.mem_mult_mode = 'auto'; end - profile(pidx).cluster.max_jobs_active = 128; + profile(pidx).cluster.max_jobs_active = 512; profile(pidx).cluster.max_time_per_job = 4*86400; - profile(pidx).cluster.desired_time_per_job = 8*3600; + profile(pidx).cluster.desired_time_per_job = 2*3600; profile(pidx).cluster.max_retries = 2; profile(pidx).cluster.submit_pause = 1; profile(pidx).cluster.stat_pause = 10; profile(pidx).cluster.file_check_pause = 0; - profile(pidx).cluster.qsub_submit_arguments = '-m n -l nodes=1:ppn=%p:dcwan:dc2,pmem=%m,walltime=%t'; + profile(pidx).cluster.slurm_submit_arguments = '-N 1 -n 1 --cpus-per-task=%p --mem=%m --time=%t'; + %profile(pidx).cluster.slurm_submit_arguments = '-p debug -N 1 -n 1 --cpus-per-task=%p --mem=%m --time=%t'; - profile(pidx).ops.url = 'https://ops.cresis.ku.edu/'; % Read-only for outside of CReSIS - % profile(pidx).ops.url = 'http://ops.cresis.ku.edu/'; % Use from within CReSIS + profile(pidx).ops.url = 'https://ops.cresis.ku.edu/'; + profile(pidx).ops.google_map_api_key = 'AIzaSyCNexiP6WcIda8ZEa2MnwznWrGotDoLu0w'; % Fill in with your Google API key + %% AWI Profile Field Windows (PROFILE 7) % ---------------------------------------------------------------------- pidx = 7; % profile index - profile(pidx).path_override = 'C:\tmp\scripts\matlab\'; - profile(pidx).path = 'C:\tmp\scripts\cresis-toolbox\cresis-toolbox\'; - profile(pidx).param_path = 'C:\tmp\scripts\ct_params\'; - - profile(pidx).tmp_file_path = 'F:\ct_user_tmp\'; - - profile(pidx).data_path = 'D:\'; - profile(pidx).data_support_path = 'F:\metadata\'; - profile(pidx).support_path = 'F:\csarp_support\'; - profile(pidx).out_path = 'F:\'; - profile(pidx).gis_path = 'C:\tmp\GIS_data\'; + base_dir = 'S:\Scratch\'; + profile(pidx).path_override = fullfile(base_dir,'scripts','matlab'); + profile(pidx).path = fullfile(base_dir,'scripts','cresis-toolbox','cresis-toolbox'); + profile(pidx).param_path = fullfile(base_dir,'scripts','ct_params'); + + profile(pidx).tmp_file_path = fullfile(base_dir,'scripts','ct_user_tmp'); + + profile(pidx).data_path = fullfile(base_dir,'scripts'); + profile(pidx).data_support_path = fullfile(base_dir,'scripts','metadata'); + profile(pidx).support_path = fullfile(base_dir,'scripts','csarp_support'); + profile(pidx).out_path = fullfile(base_dir,'scripts'); + profile(pidx).gis_path = fullfile(base_dir,'scripts','GIS_data'); profile(pidx).ct_tmp_file_path = fullfile(profile(pidx).out_path,'ct_tmp'); profile(pidx).cluster.data_location = fullfile(profile(pidx).tmp_file_path,'cluster-temp'); @@ -302,8 +307,9 @@ profile(pidx).cluster.submit_pause = 0; profile(pidx).cluster.stat_pause = 1; - profile(pidx).ops.url = 'https://ops.cresis.ku.edu/'; % Read-only for outside of CReSIS - % profile(pidx).ops.url = 'http://ops.cresis.ku.edu/'; % Use from within CReSIS + profile(pidx).ops.url = 'https://ops.cresis.ku.edu/'; + profile(pidx).ops.google_map_api_key = 'AIzaSyCNexiP6WcIda8ZEa2MnwznWrGotDoLu0w'; % Fill in with your Google API key + %% AWI Profile Field Linux (PROFILE 8) % ---------------------------------------------------------------------- @@ -337,11 +343,12 @@ profile(pidx).cluster.mem_to_ppn = 0.9 * 64e9 / 8; profile(pidx).cluster.max_ppn = 4; profile(pidx).cluster.max_mem_per_job = 62e9; - profile(pidx).cluster.max_mem_mode = 'debug'; - - profile(pidx).ops.url = 'https://ops.cresis.ku.edu/'; % Read-only for outside of CReSIS - % profile(pidx).ops.url = 'http://ops.cresis.ku.edu/'; % Use from within CReSIS + profile(pidx).cluster.mem_mult_mode = 'debug'; + profile(pidx).ops.url = 'https://ops.cresis.ku.edu/'; + profile(pidx).ops.google_map_api_key = 'AIzaSyCNexiP6WcIda8ZEa2MnwznWrGotDoLu0w'; % Fill in with your Google API key + + %% AWI Profile Ollie (PROFILE 9) % ---------------------------------------------------------------------- pidx = 9; % profile index @@ -377,13 +384,14 @@ profile(pidx).cluster.mem_to_ppn = 0.9 * 64e9 / 36; profile(pidx).cluster.max_ppn = 18; profile(pidx).cluster.max_mem_per_job = 62e9; - profile(pidx).cluster.max_mem_mode = 'debug'; + profile(pidx).cluster.mem_mult_mode = 'debug'; profile(pidx).cluster.mcc = 'system_eval'; - profile(pidx).ops.url = 'https://ops.cresis.ku.edu/'; % Read-only for outside of CReSIS - % profile(pidx).ops.url = 'http://ops.cresis.ku.edu/'; % Use from within CReSIS - + profile(pidx).ops.url = 'https://ops.cresis.ku.edu/'; + profile(pidx).ops.google_map_api_key = 'AIzaSyCNexiP6WcIda8ZEa2MnwznWrGotDoLu0w'; % Fill in with your Google API key + + %% Startup code (Automated Section) % ===================================================================== @@ -450,6 +458,9 @@ % 2: check this file and its dependencies only if "fun" is not % specified in call to torque_compile.m (all functions called % by torque_compile) + % ======================================================================= + % REMINDER: Run cluster_compile after making changes to this list! + % ======================================================================= gRadar.cluster.hidden_depend_funs = {}; gRadar.cluster.hidden_depend_funs{end+1} = {'tomo_collate_task.m' 2}; gRadar.cluster.hidden_depend_funs{end+1} = {'analysis_task.m' 2}; @@ -464,6 +475,8 @@ gRadar.cluster.hidden_depend_funs{end+1} = {'array_combine_task.m' 2}; gRadar.cluster.hidden_depend_funs{end+1} = {'nsidc_delivery_script_task.m' 2}; gRadar.cluster.hidden_depend_funs{end+1} = {'preprocess_task.m' 2}; + gRadar.cluster.hidden_depend_funs{end+1} = {'layer_tracker_task.m' 2}; + gRadar.cluster.hidden_depend_funs{end+1} = {'layer_tracker_combine_task.m' 2}; gRadar.cluster.hidden_depend_funs{end+1} = {'sim_doa_task.m' 2}; gRadar.cluster.hidden_depend_funs{end+1} = {'hanning.m' 0}; gRadar.cluster.hidden_depend_funs{end+1} = {'hamming.m' 0}; @@ -477,6 +490,11 @@ gRadar.cluster.hidden_depend_funs{end+1} = {'array_proc_sv.m' 1}; gRadar.cluster.hidden_depend_funs{end+1} = {'lever_arm.m' 1}; gRadar.cluster.hidden_depend_funs{end+1} = {'doa_nonlcon.m' 1}; + gRadar.cluster.hidden_depend_funs{end+1} = {'burst_noise_corr.m' 1}; + gRadar.cluster.hidden_depend_funs{end+1} = {'burst_noise_bad_samples.m' 1}; + % ======================================================================= + % REMINDER: Run cluster_compile after making changes to this list! + % ======================================================================= % .path = used by the scheduler to build the file dependency path % typically this is the same at .path diff --git a/cresis-toolbox/fmcw/BW_window_gen.m b/cresis-toolbox/fmcw/BW_window_gen.m index 8237683f..7d3ee9d6 100644 --- a/cresis-toolbox/fmcw/BW_window_gen.m +++ b/cresis-toolbox/fmcw/BW_window_gen.m @@ -32,28 +32,48 @@ function BW_window_gen(param,param_override) % ===================================================================== % Load records -fn_records = ct_filename_support(param,'','records'); -records = load(fn_records); +records = records_load(param); % Check BW_window_gen parameters if ~isfield(param,'BW_window_gen') || isempty(param.BW_window_gen) param.BW_window_gen = []; end +% BW_guard: Two element vector which specifies the bandwidth guard to +% remove at the start and end of the bandwidth (e.g. if the signal is poor +% on the edges or you are selecting a subband; subband is normally done +% during deconvolution and not with BW_guard). Default is [1 1] and the +% units are set to '%' which means 1% of the bandwidth if removed at the +% start and end. if ~isfield(param.BW_window_gen,'BW_guard') || isempty(param.BW_window_gen.BW_guard) - param.BW_window_gen.BW_guard = [5 5]; + param.BW_window_gen.BW_guard = [1 1]; param.BW_window_gen.BW_guard_units = '%'; end +if length(param.BW_window_gen.BW_guard) ~= 2 || any(~param.BW_window_gen.BW_guard < 0) + error('param.BW_window_gen.BW_guard must be a 2 element vector with nonnegative numbers'); +end + +% BW_guard_units: String containing 'Hz' or '%' and affects how the +% BW_guard field is interpreted. '%' is a percentage of abs(f1-f0). if ~isfield(param.BW_window_gen,'BW_guard_units') || isempty(param.BW_window_gen.BW_guard_units) param.BW_window_gen.BW_guard_units = 'Hz'; end +% param.BW_window_gen.imgs: Which wf-adc pairs to calculate bandwidths for. +% Default is {[1 1]} if ~isfield(param.BW_window_gen,'imgs') || isempty(param.BW_window_gen.imgs) param.BW_window_gen.imgs = {[1 1]}; end param.load.imgs = param.BW_window_gen.imgs; +% nice_BW_ratio: Specify the range of bandwidth to search for a nicer round +% number for the bandwidth. This is a percentage of abs(f1-f0). Default is +% 1% or 0.01. +if ~isfield(param.BW_window_gen,'nice_BW_ratio') || isempty(param.BW_window_gen.nice_BW_ratio) + param.BW_window_gen.nice_BW_ratio = 0.01; +end + % stretch_deramp: LO is extended so that RF is fully supported if ~isfield(param.BW_window_gen,'stretch_deramp') || isempty(param.BW_window_gen.stretch_deramp) param.BW_window_gen.stretch_deramp = false; @@ -83,4 +103,24 @@ function BW_window_gen(param,param_override) % Ensure that the BW window is a multiple of df BW_window = round(BW_window(1)/df)*df + [0 round(diff(BW_window)/df)*df]; -fprintf('%s\t[%.12g %.12g]\n',param.day_seg, BW_window); +% Find BW_window_nice which is BW_window adjusted to choose round numbers +% without long non-zero decimals +last_char = []; +steps = [0 : round(diff(BW_window)*param.BW_window_gen.nice_BW_ratio/df)]; +for idx = 1:length(steps) + str = sprintf('%.12g', BW_window(1) + df*steps(idx)); + last_char(idx) = find(str ~= '0' & str ~= '.',1,'last'); +end +[~,idx] = min(last_char); +BW_window_nice(1) = BW_window(1) + steps(idx)*df; + +last_char = []; +steps = [0 : round(diff(BW_window)*param.BW_window_gen.nice_BW_ratio/df)]; +for idx = 1:length(steps) + str = sprintf('%.12g', BW_window(2) - df*steps(idx)); + last_char(idx) = find(str ~= '0' & str ~= '.',1,'last'); +end +[~,idx] = min(last_char); +BW_window_nice(2) = BW_window(2) - steps(idx)*df; + +fprintf('%s\t[%.3g %.3g]\t[%.12g %.12g]\t[%.12g %.12g]\t%.12g\n',param.day_seg, BW_window/1e9, BW_window, BW_window_nice, df); diff --git a/cresis-toolbox/fmcw/compress_echogram.m b/cresis-toolbox/fmcw/compress_echogram.m index 0e350e6a..e7cac919 100644 --- a/cresis-toolbox/fmcw/compress_echogram.m +++ b/cresis-toolbox/fmcw/compress_echogram.m @@ -128,21 +128,8 @@ function compress_echogram(param, param_override) truncate_data = param.compress_echogram.truncate_data; % Load the frames file -frames_fn = ct_filename_support(param,'','frames'); -load(frames_fn); - -if isempty(param.cmd.frms) - param.cmd.frms = 1:length(frames.frame_idxs); -end -% Remove frames that do not exist from param.cmd.frms list -[valid_frms,keep_idxs] = intersect(param.cmd.frms, 1:length(frames.frame_idxs)); -if length(valid_frms) ~= length(param.cmd.frms) - bad_mask = ones(size(param.cmd.frms)); - bad_mask(keep_idxs) = 0; - warning('Nonexistent frames specified in param.cmd.frms (e.g. frame "%g" is invalid), removing these', ... - param.cmd.frms(find(bad_mask,1))); - param.cmd.frms = valid_frms; -end +frames = frames_load(param); +param.cmd.frms = frames_param_cmd_frms(param,frames); %% Setup % ========================================================================= @@ -197,7 +184,7 @@ function compress_echogram(param, param_override) dBins = round(dRange / (c/2) / dt); end zero_pad_len = max(abs(dBins)); - tmp.Data = cat(1,tmp.Data,zeros(zero_pad_len,size(tmp.Data,2))); + tmp.Data = cat(1,tmp.Data,nan(zero_pad_len,size(tmp.Data,2))); if ~isempty(tmp.Time) tmp.Time = tmp.Time(1) + dt * (0:size(tmp.Data,1)-1).'; end @@ -240,9 +227,9 @@ function compress_echogram(param, param_override) % Truncating data adds four variables % 1)Truncate_Bins tmp.Truncate_Bins = good_bins; - tmp.Truncate_Median = NaN*zeros(1,size(tmp.Data,2)); - tmp.Truncate_Mean = NaN*zeros(1,size(tmp.Data,2)); - tmp.Truncate_Std_Dev = NaN*zeros(1,size(tmp.Data,2)); + tmp.Truncate_Median = nan(1,size(tmp.Data,2)); + tmp.Truncate_Mean = nan(1,size(tmp.Data,2)); + tmp.Truncate_Std_Dev = nan(1,size(tmp.Data,2)); % 2) Truncate_Median, Truncate_Mean, Truncate_Std_Dev % These will have a NaN for a range line when no valid bins were % truncated for that range line. Zero padded bins from elevation diff --git a/cresis-toolbox/fmcw/fmcw_compare_echograms.m b/cresis-toolbox/fmcw/fmcw_compare_echograms.m index 7adcc612..66aa7101 100644 --- a/cresis-toolbox/fmcw/fmcw_compare_echograms.m +++ b/cresis-toolbox/fmcw/fmcw_compare_echograms.m @@ -10,7 +10,7 @@ param = read_param_xls(ct_filename_param('snow_param_2012_Greenland_P3.xls'),'20120319_05'); param2 = read_param_xls(ct_filename_param('kuband_param_2012_Greenland_P3.xls'),'20120319_05'); -load(ct_filename_support(param,'','frames')); +frames = frames_load(param); fn = fullfile(gRadar.data_support_path,'kurtz_seaice_thickness','2012.03.19','IDCSI2_20120319.txt'); data = read_seaice_kurtz(fn); diff --git a/cresis-toolbox/fmcw/load_fmcw_data.m b/cresis-toolbox/fmcw/load_fmcw_data.m index a73b71b8..815d8f54 100644 --- a/cresis-toolbox/fmcw/load_fmcw_data.m +++ b/cresis-toolbox/fmcw/load_fmcw_data.m @@ -1209,7 +1209,7 @@ % If the frame is not found in the list, then closest_idx % will be empty and the deconv waveform that is closest in % time will be selected below (which is the default mode). - load(ct_filename_support(param,'','frames')); + frames = frames_load(param); frm = find(mean(param.load.recs) >= frames.frame_idxs,1,'last'); closest_idx = param.proc.deconv_enforce_wf_idx(find(param.proc.deconv_enforce_wf_idx(:,1) == frm),2); else diff --git a/cresis-toolbox/fmcw/run_BW_window_gen.m b/cresis-toolbox/fmcw/run_BW_window_gen.m index 4b0e8ac4..43625256 100644 --- a/cresis-toolbox/fmcw/run_BW_window_gen.m +++ b/cresis-toolbox/fmcw/run_BW_window_gen.m @@ -17,8 +17,14 @@ % params = read_param_xls(ct_filename_param('snow_param_2013_Greenland_P3.xls')); % params = ct_set_params(params,'radar.wfs.DDC_valid',[1 4 8 16 32]); -params = read_param_xls(ct_filename_param('snow_param_2014_Greenland_P3.xls')); -params = ct_set_params(params,'radar.wfs.DDC_valid',[1 2 4 8 16]); +% params = read_param_xls(ct_filename_param('snow_param_2014_Greenland_P3.xls')); +% params = ct_set_params(params,'radar.wfs.DDC_valid',[1 2 4 8 16]); + +% params = read_param_xls(ct_filename_param('snow_param_2017_Greenland_P3.xls')); +% params = ct_set_params(params,'radar.wfs.DDC_valid',1); + +params = read_param_xls(ct_filename_param('snow_param_2019_Arctic_GV.xls')); +params = ct_set_params(params,'radar.wfs.DDC_valid',1); % params = read_param_xls(ct_filename_param('snow_param_2019_Arctic_Polar6.xls')); % params = ct_set_params(params,'radar.wfs(1).DDC_valid',[1 2 4 8 16]); @@ -28,6 +34,18 @@ % params = ct_set_params(params,'cmd.generic',1,'day_seg','20120330_03'); % params = ct_set_params(params,'cmd.generic',1,'day_seg','20120330_04'); +% BW_guard: Two element vector which specifies the bandwidth guard to +% remove at the start and end of the bandwidth (e.g. if the signal is poor +% on the edges or you are selecting a subband; subband is normally done +% during deconvolution and not with BW_guard). Default is [1 1] and the +% BW_guard_units are set to '%' which means 1% of the bandwidth if removed +% at the start and end. The default units are 'Hz' if BW_guard is defined. +% param_override.BW_window_gen.BW_guard = [0.8e9 0.5e9]; + +% nice_BW_ratio: Specify the range of bandwidth to search for a nicer round +% number for the bandwidth. This is a percentage of abs(f1-f0). Default is +% 1% or 0.01. +% param_override.BW_window_gen.nice_BW_ratio = 0.005; warning off; @@ -44,7 +62,8 @@ % Process each of the segments -fprintf('%s\t%s\n', 'day_seg', 'BW_window'); +fprintf('Consider using BW_window_nice rather than BW_window to use more readable numbers.\n'); +fprintf('%s\t%s\t%s\t%s\t%s\n', 'day_seg', 'BW_window_GHz', 'BW_window', 'BW_window_nice', 'df'); for param_idx = 1:length(params) param = params(param_idx); if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic @@ -53,10 +72,10 @@ if isempty(regexpi(param.cmd.notes,'do not process')) try BW_window_gen(param,param_override); - catch - fprintf('%s\t\n', param.day_seg); + catch ME + fprintf('%s\tFAILED: %s\t\n', param.day_seg, regexprep(ME.getReport,'[\n\r]',' ')); end else - fprintf('%s\t\n', param.day_seg); + fprintf('%s\tDO NOT PROCESS\t\n', param.day_seg); end end diff --git a/cresis-toolbox/fmcw/uncompress_echogram.m b/cresis-toolbox/fmcw/uncompress_echogram.m index 7175839a..d2942173 100644 --- a/cresis-toolbox/fmcw/uncompress_echogram.m +++ b/cresis-toolbox/fmcw/uncompress_echogram.m @@ -51,7 +51,7 @@ end if isfield(mdata,'Data') % Some times the Data is not loaded, so we have a special check for this - mdata.Data = [zeros(Nz,size(mdata.Data,2)); mdata.Data]; + mdata.Data = [nan(Nz,size(mdata.Data,2)); mdata.Data]; for rline = 1:length(mdata.Elevation_Correction) mdata.Data(:,rline) = circshift(mdata.Data(:,rline),-mdata.Elevation_Correction(rline)); end @@ -71,5 +71,3 @@ mdata = rmfield(mdata,'Data'); end end - -return; diff --git a/cresis-toolbox/gis/bottom_dem_class.m b/cresis-toolbox/gis/bottom_dem_class.m new file mode 100644 index 00000000..9e673859 --- /dev/null +++ b/cresis-toolbox/gis/bottom_dem_class.m @@ -0,0 +1,699 @@ +% Class bottom_dem_class +% +% Class for loading and working with ice bottom digital elevation models. +% Normally stored in the global variable gbottom_dem to avoid multiple +% loads of the GIS files. +% +% Examples: +% See run_dem_class.m +% +% Author: John Paden + +classdef bottom_dem_class < handle + properties + % dem_info: Cell array of dem_info structures that each describe a DEM + dem_info + % param: gRadar/parameter spreadsheet parameter structure + param % only .gis_path used + + % DEM properties + dem_all + x_all + y_all + + % Region of interest fields + % Truncated DEM structure: dem, x, y + dem + + % Geographic vector + name % name of vector (usually day_seg) + x % vector query point x-value, N elements + y % vector query point y-value, N elements + di % DEM index for each (x,y) vector query point, N elements + lat % vector query point latitude-value, N elements + lon % vector query point longitude-value, N elements + min_x + max_x + min_y + max_y + min_lat + max_lat + mean_lon + max_lon + min_lon + + % Mode + dem_mode % string, either 'thickness' or 'bed'. Default is 'bed'. + end + + methods + %% constructor + function obj = bottom_dem_class(param) + + % Input checks + % =================================================================== + if ~exist('param','var') || isempty(param) || ~isfield(param,'gis_path') || isempty(param.gis_path) + error('param.gis_path must be defined.'); + end + obj.param = param; + + % Setup DEM List + % =================================================================== + obj.dem_info = {}; + proj_load_standard; % Load standard projections + + % Arctic Ice Bottom DEM, Bed Machine Greenland v3 + % ------------------------------------------------------------------- + % + try + new_dem_info.url = 'https://n5eil01u.ecs.nsidc.org/ICEBRIDGE/IDBMG4.003/1993.01.01/BedMachineGreenland-2017-09-20.nc'; + new_dem_info.res = [NaN]; + new_dem_info.res_str = {''}; + new_dem_info.tile_en = [0 0 0 0]; + new_dem_info.subtile_en = [0 0 0 0]; + new_dem_info.tile_fn_ext = ''; + new_dem_info.mosaic_fn_fh = @(res_str) ''; + new_dem_info.acknowledge = 'See citation. https://nsidc.org/data/idbmg4'; + new_dem_info.citation = sprintf('(1) Morlighem, M., C. Williams, E. Rignot, L. An, J. E. Arndt, J. Bamber, G. Catania, N. Chauché, J. A. Dowdeswell, B. Dorschel, I. Fenty, K. Hogan, I. Howat, A. Hubbard, M. Jakobsson, T. M. Jordan, K. K. Kjeldsen, R. Millan, L. Mayer, J. Mouginot, B. Noël, C. O''Cofaigh, S. J. Palmer, S. Rysgaard, H. Seroussi, M. J. Siegert, P. Slabon, F. Straneo, M. R. van den Broeke, W. Weinrebe, M. Wood, and K. Zinglersen. 2017. BedMachine v3: Complete bed topography and ocean bathymetry mapping of Greenland from multi-beam echo sounding combined with mass conservation, Geophysical Research Letters. 44. https://doi.org/10.1002/2017GL074954.\nMorlighem, M. et al. 2017, updated 2018. IceBridge BedMachine Greenland, Version 3. Boulder, Colorado USA. NASA National Snow and Ice Data Center Distributed Active Archive Center. doi: https://doi.org/10.5067/2CIX82HUV88Y.'); + new_dem_info.no_data = -9999; + new_dem_info.out_path = fullfile(param.gis_path,'greenland','mass_conservation'); + new_dem_info.proj = arctic_proj; % Loaded from proj_load_standard.m + obj.dem_info{end+1} = new_dem_info; + catch ME + warning(ME.getReport); + end + + % Antarctica Ice Bottom DEM, MEaSUREs BedMachine Antarctica, Version 2 + % ------------------------------------------------------------------- + try + new_dem_info.url = 'https://n5eil01u.ecs.nsidc.org/MEASURES/NSIDC-0756.002/1970.01.01/BedMachineAntarctica_2020-07-15_v02.nc'; + new_dem_info.res = [NaN]; + new_dem_info.res_str = {''}; + new_dem_info.tile_en = [0 0 0 0]; + new_dem_info.subtile_en = [0 0 0 0]; + new_dem_info.tile_fn_ext = ''; + new_dem_info.mosaic_fn_fh = @(res_str) ''; + new_dem_info.acknowledge = 'See citation. https://nsidc.org/data/nsidc-0756/versions/2'; + new_dem_info.citation = sprintf('(1) Morlighem, M. 2020. MEaSUREs BedMachine Antarctica, Version 2. Boulder, Colorado USA. NASA National Snow and Ice Data Center Distributed Active Archive Center. doi: https://doi.org/10.5067/E1QL9HFQ7A8M.\n(2) Morlighem, M., E. Rignot, T. Binder, D. D. Blankenship, R. Drews, G. Eagles, O. Eisen, F. Ferraccioli, R. Forsberg, P. Fretwell, V. Goel, J. S. Greenbaum, H. Gudmundsson, J. Guo, V. Helm, C. Hofstede, I. Howat, A. Humbert, W. Jokat, N. B. Karlsson, W. Lee, K. Matsuoka, R. Millan, J. Mouginot, J. Paden, F. Pattyn, J. L. Roberts, S. Rosier, A. Ruppel, H. Seroussi, E. C. Smith, D. Steinhage, B. Sun, M. R. van den Broeke, T. van Ommen, M. van Wessem, and D. A. Young. 2020. Deep glacial troughs and stabilizing ridges unveiled beneath the margins of the Antarctic ice sheet, Nature Geoscience. 13. 132-137. https://doi.org/10.1038/s41561-019-0510-8'); + new_dem_info.no_data = -9999; + new_dem_info.out_path = fullfile(param.gis_path,'antarctica','mass_conservation'); + new_dem_info.proj = antarctic_proj; % Loaded from proj_load_standard.m + obj.dem_info{end+1} = new_dem_info; + catch ME + warning(ME.getReport); + end + + % Prepare DEM fields + % ------------------------------------------------------------------- + obj.dem_mode = 'bed'; + obj.clear(); + obj.reset(); + end + + %% destructor + % ===================================================================== + function delete(obj) + end + + %% set_vector: Set Geographic Vector + % ===================================================================== + function obj = set_vector(obj,lat,lon,name) + + % Update class fields and determine which DEM each point belongs to + % ------------------------------------------------------------------- + if exist('name','var') + obj.name = name; + end + if isempty(lat) + return; + else + obj.lat = lat; + obj.lon = lon; + obj.x = nan(size(lat)); + obj.y = nan(size(lat)); + obj.di = nan(size(lat)); + % For each input determine the dem index to use + % Arctic Ice Bottom DEM, Bed Machine Greenland v3 + obj.di(lat>0) = 1; + % Antarctica Ice Bottom DEM, MEaSUREs BedMachine Antarctica, Version 2 + obj.di(lat<=0) = 1; + + for di = unique(obj.di) + mask = obj.di == di; + [obj.x(mask),obj.y(mask)] = projfwd(obj.dem_info{di}.proj,lat(mask),lon(mask)); + end + end + + % Determine bounds of region of interest + % ------------------------------------------------------------------- + obj.min_x = min(obj.x(:)); + obj.max_x = max(obj.x(:)); + obj.min_y = min(obj.y(:)); + obj.max_y = max(obj.y(:)); + + obj.min_lat = min(obj.lat(:)); + obj.max_lat = max(obj.lat(:)); + % Handle longitude in a special way because it wraps around. + obj.mean_lon = angle(mean(exp(1i*obj.lon(:)/180*pi)))*180/pi; + obj.max_lon = obj.mean_lon + max(angle(exp(1i*(obj.lon(:)-obj.mean_lon)/180*pi)))*180/pi; + obj.min_lon = obj.mean_lon + min(angle(exp(1i*(obj.lon(:)-obj.mean_lon)/180*pi)))*180/pi; + + % Load GIS data as needed + % ------------------------------------------------------------------- + + % If msl is not big enough, it is reloaded to fit vector + %obj.load_msl(); + + % If ocean is not big enough or has not been truncated yet, a new + % truncated version is created + %bj.load_ocean(); + + % If the whole region fits into a single tile, then this tile is + % loaded and truncated to the region of interest. If the region does + % not fit into a single tile, then the tiles will be loaded later on + % an as needed basis. + obj.load_dem(); + end + + %% load_msl: Load MSL + % ===================================================================== + function obj = load_msl(obj) + if isempty(obj.x) + return; + end + if ~isempty(obj.msl) + if obj.msl.lon(1) <= obj.min_lon && obj.msl.lon(end) >= obj.max_lon ... + && obj.msl.lat(1) <= obj.min_lat && obj.msl.lat(end) >= obj.max_lat + end + end + + % Mean Sea Level (msl) + % ------------------------------------------------------------------- + if 0 + % EGM-96 + obj.msl.fn = ct_filename_gis(obj.param,fullfile('world','egm96_geoid','WW15MGH.DAC')); + points = []; + [obj.msl.lat,obj.msl.lon,obj.msl.elev] = egm96_loader(obj.msl.fn); + obj.msl.lon = [obj.msl.lon 360]; + obj.msl.elev = [obj.msl.elev obj.msl.elev(:,1)]; + [obj.msl.lon,obj.msl.lat] = meshgrid(obj.msl.lon,obj.msl.lat); + else + % Load DTU mean sea level + obj.msl.fn = ct_filename_gis(obj.param,fullfile('world','dtu_meansealevel','DTU10MSS_1min.nc')); + obj.msl.lat = ncread(obj.msl.fn,'lat'); + obj.msl.lon = ncread(obj.msl.fn,'lon'); + dlat = obj.msl.lat(2)-obj.msl.lat(1); + lat_idxs = find(obj.msl.lat >= obj.min_lat-2*dlat & obj.msl.lat <= obj.max_lat+2*dlat); + dlon = obj.msl.lon(2)-obj.msl.lon(1); + rel_lon = obj.mean_lon + angle(exp(1i*(obj.msl.lon - obj.mean_lon)/180*pi))*180/pi; + lon_idxs = find(rel_lon >= obj.min_lon-2*dlon & rel_lon <= obj.max_lon+2*dlon); + break_idx = find(diff(lon_idxs)~=1); + obj.msl.lat = obj.msl.lat(lat_idxs); + obj.msl.lon = rel_lon(lon_idxs); + % Transpose elev because "x" axis (which is longitude) must be on the + % column dimension for interp2. + % Convert to single because interp2 requires single or double type + % and single is smaller yet has enough precision. + if isempty(break_idx) + obj.msl.elev = single(ncread(obj.msl.fn,'mss', ... + [lon_idxs(1) lat_idxs(1)],[length(lon_idxs) length(lat_idxs)]).'); + else + obj.msl.elev = single(ncread(obj.msl.fn,'mss', ... + [lon_idxs(break_idx+1) lat_idxs(1)],[length(lon_idxs)-break_idx length(lat_idxs)]).'); + obj.msl.elev = [obj.msl.elev, single(ncread(obj.msl.fn,'mss', ... + [1 lat_idxs(1)],[break_idx length(lat_idxs)]).')]; + obj.msl.lon = obj.msl.lon([break_idx+1:end,1:break_idx]); + end + [obj.msl.lon,unique_idxs] = unique(obj.msl.lon); + obj.msl.elev = obj.msl.elev(:,unique_idxs); + end + end + + %% load_ocean: Load ocean mask + % ===================================================================== + function obj = load_ocean(obj) + if isempty(obj.x) + return; + end + + % Load ocean mask shape file (-180 to +180 lon) + % ------------------------------------------------------------------- + if ~isfield(obj.ocean,'shp_all') + ocean_mask_fn = ct_filename_gis(obj.param,fullfile('world','land_mask','Land_Mask_IDL_jharbeck','GSHHS_f_L1.shp')); + ocean_mask_fn_antarctica = ct_filename_gis(obj.param,fullfile('world','land_mask','Land_Mask_IDL_jharbeck','GSHHS_f_L5.shp')); + warning off; + obj.ocean.shp_all = shaperead(ocean_mask_fn); + if isfield(obj.param,'post') && isfield(obj.param.post,'ops') ... + && isfield(obj.param.post.ops,'location') && strcmp(obj.param.post.ops.location,'antarctic') + shp_antarctica = shaperead(ocean_mask_fn_antarctica); + obj.ocean.shp_all = cat(1,obj.ocean.shp_all,shp_antarctica); + end + warning on; + obj.ocean.bb_all = [obj.ocean.shp_all(:).BoundingBox]; + end + + % Get all ocean shapes within the data segment bounding box. Shape is + % not included if any of the following holds: + % - Bottom of the shape is above the top of the segment, >max_lat + % - Top of the shape is below the bottom of the segment, max_lon + % - Right side of the shape is to the left of the segment, obj.max_lat | obj.ocean.bb_all(2,2:2:end)obj.max_lon | rel_max_lon 2000 + plot(obj.ocean.shp(idx).X(1:5:end),obj.ocean.shp(idx).Y(1:5:end)) + else + plot(obj.ocean.shp(idx).X,obj.ocean.shp(idx).Y) + end + hold on; + end + plot(obj.lon(1:100:end),obj.lat(1:100:end),'k.') + end + + end + + %% load_dem: Load land DEM + % ===================================================================== + function obj = load_dem(obj) + + physical_constants; + + for di = unique(obj.di) + mask = obj.di == di; + if all(~mask) + continue + end + ri = obj.get_ri(di); + + if ~obj.dem_info{di}.tile_en(ri) + if ~isempty(obj.dem.x{di}) ... + && obj.min_x >= min(obj.dem.x{di}) && obj.max_x <= max(obj.dem.x{di}) ... + && obj.min_y >= min(obj.dem.y{di}) && obj.max_y <= max(obj.dem.y{di}) + continue; + end + + % Single large DEM file + % --------------------------------------------------------------- + url = [obj.dem_info{di}.url, obj.dem_info{di}.mosaic_fn_fh(obj.dem_info{di}.res_str{ri})]; + [~,url_name,url_ext] = fileparts(url); + nc_fn = fullfile(obj.dem_info{di}.out_path, [url_name,url_ext]); + + if ~exist(nc_fn,'file') + cmd = sprintf('wget -P %s %s', obj.dem_info{di}.out_path, url); + fprintf(' %s\n', cmd); + system(cmd); + if ~exist(nc_fn,'file') + error('Failed to download file.'); + end + end + + if isempty(obj.x_all{di}) + % Load the DEM file + % --------------------------------------------------------------- + obj.x_all{di} = double(ncread(nc_fn,'x')); + obj.y_all{di} = double(ncread(nc_fn,'y')); + if strcmpi(obj.dem_mode,'bed') + obj.dem_all{di} = double(ncread(nc_fn,'bed').') / (c/2/sqrt(er_ice)); + elseif strcmpi(obj.dem_mode,'thickness') + obj.dem_all{di} = double(ncread(nc_fn,'thickness').') / (c/2/sqrt(er_ice)); + end + end + + % Truncate the DEM to region of interest + % --------------------------------------------------------------- + dx = abs(obj.x_all{di}(2) - obj.x_all{di}(1)); + dy = abs(obj.y_all{di}(2) - obj.y_all{di}(1)); + x_mask = obj.x_all{di} >= obj.min_x-2*dx & obj.x_all{di} <= obj.max_x+2*dx; + y_mask = obj.y_all{di} >= obj.min_y-2*dy & obj.y_all{di} <= obj.max_y+2*dy; + + obj.dem.dem{di} = obj.dem_all{di}(y_mask,x_mask); + obj.dem.dem{di}(obj.dem.dem{di}==obj.dem_info{di}.no_data) = NaN; + obj.dem.x{di} = obj.x_all{di}(x_mask); + obj.dem.y{di} = obj.y_all{di}(y_mask); + + if 0 + figure(1); clf; + imagesc(obj.dem.x{di}/1e3,obj.dem.y{di}/1e3,obj.dem.dem{di}); + set(gca,'YDir','normal') + hold on; + plot(obj.x/1e3,obj.y/1e3,'k','LineWidth',2); + cc = caxis; + end + end + end + end + + %% get_vector_dem: Get DEM corresponding to geographic vector + % ===================================================================== + function [land_dem,msl,ocean_mask] = get_vector_dem(obj) + % When a tile is accessed and it does not exist, then it is + % downloaded, loaded, and stored + + % MSL + % ------------------------------------------------------------------- + %msl = interp2(obj.msl.lon,obj.msl.lat,obj.msl.elev,obj.lon,obj.lat); + + % DEM and ocean_mask + % ------------------------------------------------------------------- + %ocean_mask = true(size(obj.lat)); + for di = unique(obj.di) + mask = obj.di == di; + if all(~mask) + continue + end + + % DEM + % ----------------------------------------------------------------- + ri = obj.get_ri(di); + if ~obj.dem_info{di}.tile_en(ri) + % Single DEM file + land_dem(mask) = interp2(obj.dem.x{di},obj.dem.y{di},obj.dem.dem{di},obj.x(mask),obj.y(mask)); + + else + % DEM is broken into multiple tile files + land_dem = nan(size(obj.x)); + + % Determine which tiles are needed + tile_x = obj.dem_info{di}.x_tile_origin + obj.x/obj.dem_info{di}.x_tile_size; + tile_y = obj.dem_info{di}.y_tile_origin + obj.y/obj.dem_info{di}.y_tile_size; + tiles = unique(floor([tile_x(:), tile_y(:)]),'rows'); + + % Download and untar files + % =============================================================== + for tiles_idx = 1:size(tiles,1) + url_list = {}; + xi_list = []; + yi_list = []; + if obj.dem_info{di}.subtile_en(ri) + for sub_x_idx = 1:2 + for sub_y_idx = 1:2 + if any(tile_x >= tiles(tiles_idx,1) & tile_x <= tiles(tiles_idx,1)+1 ... + & tile_y >= tiles(tiles_idx,2) & tile_y <= tiles(tiles_idx,2)+1) + % Create URL for subtile file and add to url_list to download + url_list{end+1} = [obj.dem_info{di}.url,obj.dem_info{di}.subtile_fn_fh(tiles(tiles_idx,1),tiles(tiles_idx,2),sub_x_idx,sub_y_idx,obj.dem_info{di}.res_str{ri})]; + xi_list(end+1) = 2*tiles(tiles_idx,1) + sub_x_idx - 2; + yi_list(end+1) = 2*tiles(tiles_idx,2) + sub_y_idx - 2; + x_tile_size = obj.dem_info{di}.x_subtile_size; + y_tile_size = obj.dem_info{di}.y_subtile_size; + x_tile_origin = obj.dem_info{di}.x_tile_origin - 1; + y_tile_origin = obj.dem_info{di}.y_tile_origin - 1; + end + end + end + else + % Create URL for tile file and add to url_list to download + url_list{end+1} = [obj.dem_info{di}.url,obj.dem_info{di}.tile_fn_fh(tiles(tiles_idx,1),tiles(tiles_idx,2),obj.dem_info{di}.res_str{ri})]; + xi_list(end+1) = tiles(tiles_idx,1); + yi_list(end+1) = tiles(tiles_idx,2); + x_tile_size = obj.dem_info{di}.x_tile_size; + y_tile_size = obj.dem_info{di}.y_tile_size; + x_tile_origin = obj.dem_info{di}.x_tile_origin; + y_tile_origin = obj.dem_info{di}.y_tile_origin; + end + + for url_idx = 1:length(url_list) + url = url_list{url_idx}; + xi = xi_list(url_idx); + yi = yi_list(url_idx); + fprintf('%s: %s\n', datestr(now), url); + + [~,url_name,url_ext1] = fileparts(url); + [~,url_name,url_ext2] = fileparts(url_name); + % fn: archive file to be downloaded (will be deleted after we are done extracting the tif file) + fn = fullfile(obj.dem_info{di}.out_path,[url_name,url_ext2,url_ext1]); + % nc_fn: tif file inside the archive file that we want + nc_fn = fullfile(obj.dem_info{di}.out_path,[url_name obj.dem_info{di}.tile_fn_ext]); + + % Check to see if download file AND tif file do not exist + if ~exist(fn,'file') && ~exist(nc_fn,'file') + cmd = sprintf('wget -P %s %s', obj.dem_info{di}.out_path, url); + fprintf(' %s\n', cmd); + system(cmd); + if ~exist(fn,'file') + fprintf(2,' Failed to download file. It may not exist.\n'); + continue; + end + end + + % + if ~exist(nc_fn,'file') + fprintf('Untar tar file %s\n to tif file: %s\n', fn, nc_fn); + untar(fn, obj.dem_info{di}.out_path); + if ~exist(nc_fn,'file') + error('Failed to find tif file after untar. If tar file is corrupted, try deleting the tar file and starting over so that it will be downloaded again.'); + end + end + + % Delete tar file, but keep tif file + if exist(fn,'file') + delete(fn); + end + + if size(obj.dem_all{di},1) < xi || size(obj.dem_all{di},2) < yi ... + || isempty(obj.dem_all{di}{xi,yi}) + % Adapted from NSIDC website: + % Retrieve the image info + GeoKeys = imfinfo(nc_fn); + N = GeoKeys.Width; + M = GeoKeys.Height; + dx = GeoKeys.ModelPixelScaleTag(1); + dy = GeoKeys.ModelPixelScaleTag(2); + minx = GeoKeys.ModelTiepointTag(4); + maxy = GeoKeys.ModelTiepointTag(5); + + % Generate obj.x/obj.y pixel location vectors + obj.x_all{di}{xi,yi} = minx + dx/2 + ((0:N-1).*dx); + obj.y_all{di}{xi,yi} = maxy - dy/2 - ((M -1:-1:0).*dy); + + % Read the image data + obj.dem_all{di}{xi,yi} = imread(nc_fn); + + % Set bad values to NaN + obj.dem_all{di}{xi,yi}(obj.dem_all{di}{xi,yi}==obj.dem_info{di}.no_data) = NaN; + + % Flip upside down if necessary + if GeoKeys.Orientation + obj.y_all{di}{xi,yi} ... + = obj.y_all{di}{xi,yi}(end:-1:1); + end + end + + x_min_tile = (xi-x_tile_origin) * x_tile_size; + x_max_tile = x_min_tile + x_tile_size; + y_min_tile = (yi-y_tile_origin) * y_tile_size; + y_max_tile = y_min_tile + y_tile_size; + tile_mask = obj.x >= x_min_tile & obj.x <= x_max_tile ... + & obj.y >= y_min_tile & obj.y <= y_max_tile; + land_dem(tile_mask) = interp2(obj.x_all{di}{xi,yi}, ... + obj.y_all{di}{xi,yi}, obj.dem_all{di}{xi,yi}, obj.x(tile_mask), obj.y(tile_mask),'*linear'); + + end + end + end + + % Ocean Mask + % ----------------------------------------------------------------- + % Use shapefile to determine ocean mask + + % min_x = min(obj.x); + % max_x = max(obj.x); + % min_y = min(obj.y); + % max_y = max(obj.y); + % + % min_lat = min(obj.lat(mask)); + % max_lat = max(obj.lat(mask)); + % % Handle longitude in a special way because it wraps around. + % mean_lon = angle(mean(exp(1i*obj.lon(mask)/180*pi)))*180/pi; + % max_lon = mean_lon + max(angle(exp(1i*(obj.lon(mask)-mean_lon)/180*pi)))*180/pi; + % min_lon = mean_lon + min(angle(exp(1i*(obj.lon(mask)-mean_lon)/180*pi)))*180/pi; + + % Restrict ocean mask features to our dataset (i.e. mask all features + % whose bounding boxes fall outside our limits. + if isempty(obj.ocean.shp) + ocean_shp = []; + else + rel_min_lon = obj.mean_lon + angle(exp(1i*(obj.ocean.bb(1,1:2:end) - obj.mean_lon)/180*pi))*180/pi; + rel_max_lon = obj.mean_lon + angle(exp(1i*(obj.ocean.bb(2,1:2:end) - obj.mean_lon)/180*pi))*180/pi; + bb_good_mask = ~(obj.ocean.bb(1,2:2:end)>obj.max_lat | obj.ocean.bb(2,2:2:end)obj.max_lon | rel_max_lon obj.min_x ... + && min(y) < obj.max_y && max(y)>obj.min_y + % add polygon + poly_x{end+1} = [x,nan]; + poly_y{end+1} = [y,nan]; + end + end + + % Create ocean mask to determine which points lie in the ocean + mask_idxs = find(mask); + for poly_idx = 1:length(poly_x) + % Decimate polygon except within the object's bounding box. This + % is done to speed up inpolygon + poly_mask = false(size(poly_x{poly_idx})); + poly_mask(1:obj.ocean_mask_dec:end) = true; + poly_mask(poly_x{poly_idx}<=obj.max_x&poly_x{poly_idx}>=obj.min_x ... + & poly_y{poly_idx}<=obj.max_y&poly_y{poly_idx}>=obj.min_y) = true; + + % Mask showing which DEM points are in polygon (on land) + land_mask_tmp = inpolygon(obj.x(mask),obj.y(mask),[poly_x{poly_idx}(poly_mask)],[poly_y{poly_idx}(poly_mask)]); + ocean_mask(mask_idxs(land_mask_tmp)) = false; + end + + if obj.ocean_mask_mode(1) == 'l' + % Also use land threshold to determine ocean mask + land_threshold = 5; + ocean_mask(land_dem-land_threshold > msl) = false; + end + + % Debug + % ----------------------------------------------------------------- + if 0 + x_out = obj.x(mask); + y_out = obj.y(mask); + dem_out = land_dem(mask); + dec_idxs = round(linspace(1,length(obj.x(mask)),1000)); + mask_dec_idxs = logical(interp1(1:length(obj.x(mask)), single(ocean_mask(mask)), dec_idxs, 'nearest', 'extrap')); + along_track = geodetic_to_along_track(obj.lat(mask),obj.lon(mask)); + + figure(2); clf; colormap(parula(256)); + scatter(along_track(dec_idxs),dem_out(dec_idxs),[],dem_out(dec_idxs)); + hold on; + scatter(along_track(dec_idxs),msl(dec_idxs),[],msl(dec_idxs)); + plot(along_track(dec_idxs(mask_dec_idxs)),msl(dec_idxs(mask_dec_idxs)),'k.'); + caxis([min(obj.dem.dem{di}(:)) max(obj.dem.dem{di}(:))]); + + figure(3); clf; colormap(parula(256)); + scatter(x_out(dec_idxs),y_out(dec_idxs),[],dem_out(dec_idxs)); + hold on; + plot(x_out(dec_idxs(mask_dec_idxs)),y_out(dec_idxs(mask_dec_idxs)),'k.'); + caxis([min(obj.dem.dem{di}(:)) max(obj.dem.dem{di}(:))]); + end + end + + end + + %% get_vector_mosaic: Get DEM mosaic around geographic vector + % ===================================================================== + function [land_dem,msl,ocean_mask,proj,x,y] = get_vector_mosaic(obj,res) + % When a tile is accessed and it does not exist, then it is + % downloaded, loaded, and stored + + % Find the first valid dem_info and use this dem's projection to + % define the proj, x, and y + proj = obj.dem_info{obj.di(1)}.proj; + + % Determine bounds of region of interest + min_x = round(min(obj.x)/res)*res; + max_x = round(max(obj.x)/res)*res; + min_y = round(min(obj.y)/res)*res; + max_y = round(max(obj.y)/res)*res; + + % Create output mosaic coordinates + x = min_x : res : max_x; + y = (min_y : res : max_y).'; + x_mesh = repmat(x,[size(y,1) 1]); + y_mesh = repmat(y,[1 size(x,2)]); + [lat_mesh,lon_mesh] = projinv(proj,x_mesh,y_mesh); + + obj.set_vector(lat_mesh,lon_mesh); + + [land_dem,msl,ocean_mask] = get_vector_dem(obj); + + end + + %% clear: Clear all loaded DEMs from memory + % ===================================================================== + function clear(obj) + obj.dem_all = cell(size(obj.dem_info)); + obj.x_all = cell(size(obj.dem_info)); + obj.y_all = cell(size(obj.dem_info)); + end + + %% reset: Reset to load a new segment + % ===================================================================== + function reset(obj) + if ~isempty(obj.di) + for di = unique(obj.di) + mask = obj.di == di; + if all(~mask) + continue + end + ri = obj.get_ri(di); + + if obj.dem_info{di}.tile_en(ri) + % Tiles were used, so clear all loaded DEMs since it is likely that + % they will not be reused + obj.clear(); + end + end + end + + % Geographic vector + obj.di = []; + obj.x = []; + obj.y = []; + obj.lat = []; + obj.lon = []; + + % Region of interest fields + obj.name = []; + obj.dem.dem = cell(size(obj.dem_info)); + obj.dem.x = cell(size(obj.dem_info)); + obj.dem.y = cell(size(obj.dem_info)); + end + + %% set_res: Set resolution + % ===================================================================== + function set_res(obj,res) + end + + %% get_ri: Get resolution index + % ===================================================================== + function ri = get_ri(obj,di) + ri = 1; + end + + %% set_dem_mode: Set to retrieve thickness or ice bottom + % ===================================================================== + function set_dem_mode(obj,dem_mode) + % dem_mode is either 'bed' or 'thickness' and controls which DEM product + % is returned + if ~isequal(dem_mode,obj.dem_mode) + obj.clear(); + obj.reset(); + obj.dem_mode = dem_mode; + end + end + + end + + +end diff --git a/cresis-toolbox/gis/coverage_maps_from_gps_or_records.m b/cresis-toolbox/gis/coverage_maps_from_gps_or_records.m index fec5e6f0..a4fe1019 100644 --- a/cresis-toolbox/gis/coverage_maps_from_gps_or_records.m +++ b/cresis-toolbox/gis/coverage_maps_from_gps_or_records.m @@ -23,24 +23,31 @@ dt = 1; % data_source: string 'gps' or 'records' (which files in csarp_support to use) -data_source = 'gps'; +data_source = 'records'; + +% South Dakota +% geotiff_fn = ct_filename_gis(gRadar,fullfile('SouthDakota','Model_Landsat_UTM.tif')); +% axis_limits = [567 622 4856 4907]; % Greenland -geotiff_fn = ct_filename_gis(gRadar,fullfile('arctic','NaturalEarth_Data','Arctic_NaturalEarth.tif')); % (use for OIB coverage maps) +% geotiff_fn = ct_filename_gis(gRadar,fullfile('arctic','NaturalEarth_Data','Arctic_NaturalEarth.tif')); % (use for OIB coverage maps) % geotiff_fn = ct_filename_gis(gRadar,fullfile('arctic','Landsat-7','arctic_natural_90m.tif')); % geotiff_fn = ct_filename_gis(gRadar,fullfile('greenland','Landsat-7','Greenland_natural.tif')); % geotiff_fn = ct_filename_gis(gRadar,fullfile('canada','Landsat-7','Canada_90m.tif')); -axis_limits = [-2300 1000 -3500 1500]; % All of Arctic (use for OIB coverage maps) +% axis_limits = [-2300 1000 -3500 1500]; % All of Arctic (use for OIB coverage maps) % Antarctica -% geotiff_fn = ct_filename_gis(gRadar,fullfile('antarctica','NaturalEarth_Data','Antarctica_NaturalEarth.tif')); % (use for OIB coverage maps) -% axis_limits = [-3000 1000 -1500 2500]; % All of Antarctica (use for OIB coverage maps) +geotiff_fn = ct_filename_gis(gRadar,fullfile('antarctica','NaturalEarth_Data','Antarctica_NaturalEarth.tif')); % (use for OIB coverage maps) +axis_limits = [-3000 1000 -1500 2500]; % All of Antarctica (use for OIB coverage maps) +% axis_limits = [808 873 996 1060]; % 2018 Antarctica Ground % Cell vector of param spreadsheet filenames -param_fns = {ct_filename_param('rds_param_2016_Greenland_P3.xls'),ct_filename_param('rds_param_2017_Greenland_P3.xls')}; +param_fns = {'rds_param_2018_Antarctica_Ground.xls'}; +% run_all % plot_args: cell array of argument to plot function (e.g. 'b.' for blue dots) -plot_args = {'LineWidth',1}; +% plot_args = {'LineWidth',2,'Color','black'}; +plot_args = {'LineWidth',2}; % along_track_sampling: desired along track sampling of plot (m) along_track_sampling = 100; @@ -49,12 +56,15 @@ % .day_seg: cell list of day segments to match % .frm: equal length cell vector to .day_seg with specific frames to label % .text: equal length cell vector to .day_seg with string for text label -label.day_seg = {}; -label.frm = {}; -label.text = {}; +label = []; +% label.day_seg = {'20140325_07','20140401_03','20140506_01','20140325_07'}; +% label.frm = {1,1,1,2}; +% label.text = {'20140325\_07','20140401\_03','20140506\_01','20140325\_07\_002'}; +% label.first_point_only = {true,true,true,false}; % label.day_seg = {'20091102_02','20091016_01','20091016_01','20091102_02','20091102_02'}; % label.frm = {8, 21, 26, 23, 32}; % label.text = {' 1',' 2',' 3',' 4',' 5'}; +% label.first_point_only = {false,false,false,false,false}; % figure_position: Set figure Position property (leave empty to use default/current figure size) figure_position = []; @@ -65,7 +75,7 @@ if exist(figure_position,'var') && ~isempty(figure_position) set(fig_h,'Position',figure_position); end -proj = plot_geotiff(geotiff_fn,[],[],fig_h); +proj = geotiff_plot(geotiff_fn,[],[],fig_h); axis normal; axis equal; if ~isempty(axis_limits) axis(axis_limits); @@ -78,7 +88,12 @@ param_fn = param_fns{param_fn_idx}; fprintf('Parameters %s\n', param_fn); - params = read_param_xls(param_fn); + params = read_param_xls(ct_filename_param(param_fn)); + + if 0 + % Specify segments + params = ct_set_params(params,'cmd.generic',1,'day_seg','20140325_07|20140401_03|20140506_01'); + end yyyymmdd_list = {}; for param_idx = 1:length(params) @@ -89,49 +104,42 @@ % Already done this segment continue; end - -% if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic -% continue; -% end + + if 0 + % Only plot specific segments + if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic + continue; + end + end if isempty(regexpi(param.cmd.notes,'do not process')) yyyymmdd_list{end+1} = param.yyyymmdd; if strcmp(data_source,'gps') gps_fn = ct_filename_support(param,[],'gps',1); - fprintf(' Processing %s: %s (%s)\n', param.day_seg, gps_fn, datestr(now,'HH:MM:SS')); - gps = load(gps_fn,'gps_time','lat','lon'); + %fprintf(' Processing %s: %s (%s)\n', param.day_seg, gps_fn, datestr(now,'HH:MM:SS')); + if ~exist(gps_fn) + fprintf('%s\t0\tMissing\n', param.day_seg(1:8)); + continue; + else + gps = load(gps_fn,'gps_time','lat','lon'); + along_track = geodetic_to_along_track(gps.lat,gps.lon); + fprintf('%s\t%g\t\n', param.day_seg, along_track(end)); + end elseif strcmp(data_source,'records') records_fn = ct_filename_support(param,[],'records'); if ~exist(records_fn,'file') - warning('No records file: %s\n', records_fn); + fprintf('%s\t0\tMissing\n', param.day_seg); continue; end - frames_fn = ct_filename_support(param,'','frames'); - if ~exist(frames_fn,'file') - warning('No frames file: %s\n', frames_fn); - continue; - end - - fprintf(' Processing %s: %s (%s)\n', param.day_seg, records_fn, datestr(now,'HH:MM:SS')); - gps = load(records_fn,'gps_time','lat','lon'); % Load frames file - load(frames_fn); + frames = frames_load(param); + param.cmd.frms = frames_param_cmd_frms(param,frames); - if isempty(param.cmd.frms) - param.cmd.frms = 1:length(frames.frame_idxs); - end - % Remove frames that do not exist from param.cmd.frms list - [valid_frms,keep_idxs] = intersect(param.cmd.frms, 1:length(frames.frame_idxs)); - if length(valid_frms) ~= length(param.cmd.frms) - bad_mask = ones(size(param.cmd.frms)); - bad_mask(keep_idxs) = 0; - warning('Nonexistent frames specified in param.cmd.frms (e.g. frame "%g" is invalid), removing these', ... - param.cmd.frms(find(bad_mask,1))); - param.cmd.frms = valid_frms; - end - % Force into a row vector - param.cmd.frms = param.cmd.frms(:).'; + %fprintf(' Processing %s: %s (%s)\n', param.day_seg, records_fn, datestr(now,'HH:MM:SS')); + gps = load(records_fn,'gps_time','lat','lon'); + along_track = geodetic_to_along_track(gps.lat,gps.lon); + fprintf('%s\t%g\t\n', param.day_seg, along_track(end)); end if strcmp(data_source,'gps') @@ -173,14 +181,28 @@ h = plot(x/1e3,y/1e3,plot_args{:},'UserData',param.day_seg); drawnow; - if isfield('label','day_seg') + if isfield(label,'day_seg') match_idxs = find(strcmpi(param.day_seg,label.day_seg(:).')); for match_idx = match_idxs for frm = label.frm{match_idx} first_idx = frames.frame_idxs(frm); - [x,y] = projfwd(proj,gps.lat(first_idx),gps.lon(first_idx)); - plot(x/1e3,y/1e3,'.','MarkerSize',12,'Color','Black'); - label.h_text(match_idx) = text(x/1e3,y/1e3,label.text{match_idx},'Color','black','FontWeight','bold','FontSize',14); + [x_first,y_first] = projfwd(proj,gps.lat(first_idx),gps.lon(first_idx)); + if label.first_point_only{match_idx} + % Just the first point of the frame in black + label.h_plot(match_idx) = plot(x_first/1e3,y_first/1e3,'.','MarkerSize',12,'Color','Black'); + else + % Whole frame in black + if frm == length(frames.frame_idxs) + last_idx = frames.Nx; + else + last_idx = frames.frame_idxs(frm+1); + end + along_track = geodetic_to_along_track(gps.lat(first_idx:last_idx),gps.lon(first_idx:last_idx)); + decim_idxs = get_equal_alongtrack_spacing_idxs(along_track,along_track_sampling); + [x,y] = projfwd(proj,gps.lat(decim_idxs+first_idx-1),gps.lon(decim_idxs+first_idx-1)); + label.h_plot(match_idx) = plot(x/1e3,y/1e3,'.','MarkerSize',12,'Color','Black'); + end + label.h_text(match_idx) = text(x_first/1e3,y_first/1e3,label.text{match_idx},'Color','black','FontWeight','bold','FontSize',14); end end end @@ -192,6 +214,11 @@ end +if isfield(label,'day_seg') + uistack(label.h_text,'top'); + uistack(label.h_plot,'top'); +end + return %% Below are common steps to customize the figure diff --git a/cresis-toolbox/gis/coverage_maps_master_fig_files.m b/cresis-toolbox/gis/coverage_maps_master_fig_files.m index 4ea2262b..d3bcb64e 100644 --- a/cresis-toolbox/gis/coverage_maps_master_fig_files.m +++ b/cresis-toolbox/gis/coverage_maps_master_fig_files.m @@ -190,7 +190,7 @@ set(1,'UserData',season_names); %% Save Matlab figure -image_master_fn = fullfile(out_dir,sprintf('%s_All_Seasonssssss.fig',location)); +image_master_fn = fullfile(out_dir,sprintf('%s_All_Seasons.fig',location)); saveas(1,image_master_fn); %% Save JPG figure diff --git a/cresis-toolbox/gis/coverage_maps_old_with_gps.m b/cresis-toolbox/gis/coverage_maps_old_with_gps.m deleted file mode 100644 index 9e81151c..00000000 --- a/cresis-toolbox/gis/coverage_maps_old_with_gps.m +++ /dev/null @@ -1,248 +0,0 @@ -% script coverage_maps_old_with_gps -% -% This script is called from coverage_maps -% -% It plots out the coverage maps and creates statistics files for -% all the old seasons that satisfy these confitions: -% 1. do not have parameter file -% 2. do have GPS files -% -% See also: coverage_maps, coverage_maps_old_without_gps, -% coverage_maps_with_param_file -% -% Author: Steven Foga, John Paden - -fprintf('===========================================================\n'); -fprintf('coverage_maps_old_with_gps\n\n'); - -% ------------------------------------------------------------------------ -% Plot coverage maps and create statistics files -% ------------------------------------------------------------------------ - -% Calculate statistics - Greenland -totalGood = {}; totalAll = {}; pctGood = {}; -clear gps_lat gps_lon; - -for season_idx = 1:length(param.stat_good) - segs = make_segment_list(param.rdr_season_type); - for seg_idx = 1:length(segs) - segs{seg_idx} = segs{seg_idx}(1:8); - end - radar_days = unique(segs); - flights = param.all_season_type; - clear totalOff totalOn; - for file_idx = 1 - segment_csv_fn = flights; - season_segments = fopen(segment_csv_fn,'r'); - % Read in CSV file (using field 3, GPSTIMESOD) - gps_time_sod = textscan(season_segments, '%f%f%f%f%f%s%f%f%f','headerlines',1,'delimiter',','); - gps_time_sod = gps_time_sod{3}; - if gps_time_sod > 0 - gpsDiff = diff(gps_time_sod); - gpsOff = gpsDiff >= 30; % If the gap is >= 30 seconds - % Sum up all the time gaps greater than 30 seconds - totalOff(file_idx) = (sum(gpsDiff(gpsOff))); - % Get the total time on - totalOn(file_idx) = max(gps_time_sod) - min(gps_time_sod); - else - totalOff(file_idx) = 0; - totalOn(file_idx) = 0; - fclose(season_segments); - end - end - gpsOffTotal = sum(totalOff); - pctOn{season_idx} = (sum(totalOn) / (gpsOffTotal + sum(totalOn)))*100; - - gre_good_fn = param.stat_good; - gre_all_fn = param.stat_all; - - fid = fopen(gre_good_fn,'r'); - % Scan csv_good file (only good bottom data) - tmp = textscan(fid, '%f%f%f%f%f%s%f%f%f','headerlines',1,'delimiter',','); - fclose(fid); - LAT_good{season_idx} = tmp{1}; - LON_good{season_idx} = tmp{2}; - BOTTOM_good{season_idx} = tmp{8}; - QUAL_good{season_idx} = tmp{9}; - - fid = fopen(gre_all_fn,'r'); - % Scan csv file (all data) - tmp = textscan(fid, '%f%f%f%f%f%s%f%f%f','headerlines',1,'delimiter',','); - LAT{season_idx} = tmp{1}; - LON{season_idx} = tmp{2}; - TIME{season_idx} = tmp{3}; - FRAME{season_idx} = tmp{6}; - BOTTOM{season_idx} = tmp{8}; - fclose(fid); - - clear tmp; - clear moderate_qual_idxs detrend_qual_idxs; - moderate_qual_idxs{season_idx} = find(QUAL_good{season_idx} == 2); - detrend_qual_idxs{season_idx} = find(QUAL_good{season_idx} == 3); - - totalGood{season_idx} = length(BOTTOM_good{season_idx}); - totalAll{season_idx} = length(BOTTOM{season_idx}); - pctGood{season_idx} = ((totalGood{season_idx}/totalAll{season_idx})*100); - - % -------------------------------------------------------------------- - % Get the lat/lon for the entire trajectory for every day the radar - % was turned on - % -------------------------------------------------------------------- - gps_path = param.gps_all; - gps_end_path = '.mat'; - % Load all the GPS files in a loop - gps_set{season_idx} = get_filenames(gps_path,'','',gps_end_path,'recursive'); - for gps_idx = 1:length(gps_set{season_idx}) - gps_all{season_idx}{gps_idx} = load(gps_set{season_idx}{gps_idx}); - if isempty(strmatch(datestr(epoch_to_datenum(gps_all{season_idx}{gps_idx}.UTC_time(1)),'yyyymmdd'), radar_days)) - % If this is not a day that the radar flew, then skip it - continue - else - gps_lat{season_idx}{gps_idx} = gps_all{season_idx}{gps_idx}.lat; - gps_lon{season_idx}{gps_idx} = gps_all{season_idx}{gps_idx}.lon; - end - end -end - -%% Plot Coverage Maps -fprintf('Statistics calculated (Greenland).\n'); - -% Download and load the geotiff based on the location selection. -fprintf('-----------------------------------------------------------------\n'); -fprintf('Loading and Plotting GeoTIFF. May take a few minutes ... '); -fprintf('%s', datestr(now,'HH:MM:SS\n')); -fprintf('Loading GeoTIFF ... '); - -% if run_antarctica -% clear fig fig_paper_position -% else -% end -for plot_idx = 1 -% for plot_idx = 1:length(param.stat_good); - fig(plot_idx) = figure(101); clf; - set(fig(plot_idx),'Position',fig_size); - for geotiff_fns_idx = 1 -% for geotiff_fns_idx = 1:length(geotiff_fns) - proj = geotiffinfo(geotiff_fns{geotiff_fns_idx}); - [RGB{geotiff_fns_idx}, R{geotiff_fns_idx}, tmp] = geotiffread(geotiff_fns{geotiff_fns_idx}); - R{geotiff_fns_idx} = R{geotiff_fns_idx}/1e3; - mapshow(RGB{geotiff_fns_idx}, R{geotiff_fns_idx}); - hold on; - end - hold on; - for geotiff_fns_idx = 1 -% for geotiff_fns_idx = 1:length(geotiff_fns) - mapshow(RGB{geotiff_fns_idx}, R{geotiff_fns_idx}); - end - axis([R{1}(3,1)+[0 R{1}(2,1)*(size(RGB{1},2)-1)] sort(R{1}(3,2)+[0 R{1}(1,2)*(size(RGB{1},1)-1)])]); - - fprintf('%s', datestr(now,'HH:MM:SS\n')); - fprintf('Loading Flightlines ...\n'); - - % Plot flightlines - % Display csv data (all data) - [X,Y] = projfwd(proj,LAT{plot_idx},LON{plot_idx}); - X = X/1e3; - Y = Y/1e3; - % Display csv_good data (bottom only data) - [X_good,Y_good] = projfwd(proj,LAT_good{plot_idx},LON_good{plot_idx}); - X_good = X_good/1e3; - Y_good = Y_good/1e3; - - clear mod_qual det_qual; - mod_qual = moderate_qual_idxs{plot_idx}; - det_qual = detrend_qual_idxs{plot_idx}; - - [X_mod,Y_mod] = projfwd(proj,LAT_good{plot_idx}(mod_qual),LON_good{plot_idx}(mod_qual)); - X_mod = X_mod/1e3; - Y_mod = Y_mod/1e3; - - [X_det,Y_det] = projfwd(proj,LAT_good{plot_idx}(det_qual),LON_good{plot_idx}(det_qual)); - X_det = X_det/1e3; - Y_det = Y_det/1e3; - - if ~exist(out_dir,'dir') - mkdir(out_dir); - end - stat_fn_name = sprintf('%s_stats.txt', param.season_name); - stat_fn_name(stat_fn_name==' ') = '_'; - stat_fn = fullfile(out_dir,stat_fn_name); - fid = fopen(stat_fn,'w'); - fprintf(fid,'%s\n',param.season_name); - stat_txt = sprintf(' %.2f%% of data has good ice bottom.\n',pctGood{plot_idx}); - fprintf(fid,'%s',stat_txt); - fprintf('%s',stat_txt); - stat_txt = sprintf(' %.2f%% of the time the radar was turned on.\n',pctOn{plot_idx}); - fprintf(fid,'%s',stat_txt); - fprintf('%s',stat_txt); - fclose(fid); - - % Display entire flight data - clear X_gps; clear Y_gps; - for gps_idx = 1:length(gps_lat{plot_idx}) - [X_gps{gps_idx},Y_gps{gps_idx}] = projfwd(proj,gps_lat{plot_idx}{gps_idx},gps_lon{plot_idx}{gps_idx}); - [len wid] = size(X_gps{gps_idx}); - if len > 1 - X_gps{gps_idx} = (X_gps{gps_idx}/1e3)'; - Y_gps{gps_idx} = (Y_gps{gps_idx}/1e3)'; - else - X_gps{gps_idx} = X_gps{gps_idx}/1e3; - Y_gps{gps_idx} = Y_gps{gps_idx}/1e3; - end - - figure(fig(plot_idx)); - h_gps = plot(X_gps{gps_idx},Y_gps{gps_idx},'-b'); - end - - figure(fig(plot_idx)); - h_all = plot(X,Y,'-r'); - - figure(fig(plot_idx)); - h_good = plot(X_good,Y_good,'.g'); - - figure(fig(plot_idx)); - h_mod = plot(X_mod,Y_mod,'.y'); - - figure(fig(plot_idx)); - h_det = plot(X_det,Y_det,'.r'); - -% Add single season plot labels - figure(fig(plot_idx)); - h_gps = plot(1,NaN,'-b'); - h_all = plot(1,NaN,'-r'); - h_good = plot(1,NaN,'g.'); - h_mod = plot(1,NaN,'y.'); - h_det = plot(1,NaN,'r.'); - h_legend = legend([h_gps h_all h_good h_mod h_det],'Entire Flight','All Radar Data', ... - 'Good Radar Data','Moderate Radar Data','Detrended Radar Data','Location','Best'); - if ~ispc % Linux fonts need to be enlarged for the legend - set(h_legend,'FontSize',18); - else - set(h_legend,'FontSize',6); - end - season_name = param.season_name; - season_name = strrep(season_name, '_',' '); - title(season_name); - - xlabel('X (km)'); - ylabel('Y (km)'); - hold off; - - fprintf('%s', datestr(now,'HH:MM:SS\n')); - - if ~exist(out_dir,'dir') - mkdir(out_dir); - end - - if strcmpi(location,'Canada') - image_fn_name = sprintf('%s_%s_coverage_map.jpg','Canada',param.season_name); - else % Else Antarctica or Greenland - image_fn_name = sprintf('%s_coverage_map.jpg', param.season_name); - end - - image_fn = fullfile(out_dir,image_fn_name); - - print(fig(plot_idx),'-djpeg','-r300',image_fn); -end - -return; diff --git a/cresis-toolbox/gis/coverage_maps_old_without_gps.m b/cresis-toolbox/gis/coverage_maps_old_without_gps.m deleted file mode 100644 index ada731f3..00000000 --- a/cresis-toolbox/gis/coverage_maps_old_without_gps.m +++ /dev/null @@ -1,175 +0,0 @@ -% script coverage_maps_old_without_gps.m -% -% Creates data coverage maps for ever year before 2001 (since these years -% do not have dedicated GPS files.) They are: 1993, 1995, 1996, 1997, 1998, -% 1999, 2001 and 2002_Greenland_P3. -% 1. Plots blue line for all GPS trajectory -% 2. Computes coverage rate which is the percentage of time that the radar -% is on starting from the time it is turned on to the time it is turned -% off. -% 3. Plots red line on top of this blue line of when radar was on. -% 4. Plots green dots on red line where ice bottom was detected. - -% For ever year before 1993 (since these years do not have dedicated GPS -% files.) They are: 1993, 1993, 1993, 1993, 1993, 1993, 1993, -% and 1993_Greenland_P3. - -% Author: Steven Foga - -%---------------------------------------- -% Script -%---------------------------------------- - -for param_idx = 1:length(param) - - fig(param_idx) = figure(101); clf; - set(fig(param_idx),'Position',fig_size); - for geotiff_fns_idx = 1:length(geotiff_fns) - proj = geotiffinfo(geotiff_fns{geotiff_fns_idx}); - [RGB{geotiff_fns_idx}, R{geotiff_fns_idx}, tmp] = geotiffread(geotiff_fns{geotiff_fns_idx}); - R{geotiff_fns_idx} = R{geotiff_fns_idx}/1e3; - mapshow(RGB{geotiff_fns_idx}, R{geotiff_fns_idx}); - hold on; - end - hold on; - - axis([R{1}(3,1)+[0 R{1}(2,1)*(size(RGB{1},2)-1)] sort(R{1}(3,2)+[0 R{1}(1,2)*(size(RGB{1},1)-1)])]); - coverage_image_fn = fullfile(out_dir,sprintf('coverage_%s_%s.jpg',param.season_name)); - file_end = '.csv'; - param(1,length(param)).recursive = 1; - flights = param.all_season_type; - - for file_idx = 1:length(flights) - data_all = fopen(flights,'r'); - C = textscan(data_all, '%f%f%f%f%f%s%f%f%f','headerlines',1,'delimiter',','); - clear lat; clear lon; clear bottom; clear utc_time_sod; clear frame_id; - clear year; clear month; clear day; clear utc_time; clear gps_time; - clear x_good; clear y_good; - lat = C{1}; - lon = C{2}; - utc_time_sod = C{3}; - fclose(data_all); - if utc_time_sod > 0 - gpsDiff = diff(utc_time_sod); - gpsOff = gpsDiff >= 30; % If the gap is >= 30 seconds - gpsOff2 = find(gpsOff==1); - totalOff(file_idx) = (sum(gpsDiff(gpsOff2))); - totalOn(file_idx) = max(utc_time_sod) - min(utc_time_sod); - - else - totalOff(file_idx) = 0; - totalOn(file_idx) = 0; - end - - gpsOffTotal = sum(totalOff); - coverage_stat = (sum(totalOn) / (gpsOffTotal + sum(totalOn)))*100; - [x,y] = projfwd(proj,lat,lon); - x = x / 1e3; - y = y / 1e3; - figure(fig(param_idx)); - h_all = plot(x,y,'-r'); - end - - % Good data pct - clear all_data_fn data_all_csv - all_data_fn = param(param_idx).all_season_type; - data_all_csv = fopen(all_data_fn,'r'); - F = textscan(data_all_csv, '%f%f%f%f%f%s%f%f%f','headerlines',1,'delimiter',','); - clear lat_all lon_all bottom_all good_idxs totalGood totalAll bottom_good_stat x_good y_good - lat_all = F{1}; - lon_all = F{2}; - bottom_all = F{8}; - qual_all = F{9}; - fclose(data_all_csv); - good_idxs = find(bottom_all ~= -9999); - totalGood = length(good_idxs); - totalAll = length(bottom_all); - bottom_good_stat = ((totalGood/totalAll)*100); - moderate_qual_idxs = find(qual_all == 2); - detrend_qual_idxs = find(qual_all == 3); - [x_good,y_good] = projfwd(proj,lat_all(good_idxs),lon_all(good_idxs)); - x_good = x_good/1e3; - y_good = y_good/1e3; - - [X_mod,Y_mod] = projfwd(proj,lat_all(moderate_qual_idxs),lon_all(moderate_qual_idxs)); - X_mod = X_mod/1e3; - Y_mod = Y_mod/1e3; - - [X_det,Y_det] = projfwd(proj,lat_all(detrend_qual_idxs),lon_all(detrend_qual_idxs)); - X_det = X_det/1e3; - Y_det = Y_det/1e3; - - disp(bottom_good_stat); disp(coverage_stat); - clear h_good - figure(fig(param_idx)); - h_good = plot(x_good,y_good,'.g'); - - figure(fig(param_idx)); - h_mod = plot(X_mod,Y_mod,'.y'); - - figure(fig(param_idx)); - h_det = plot(X_det,Y_det,'.r'); - - %-------------- - % Plot - %-------------- - if ~exist(out_dir,'dir') - mkdir(out_dir); - end - - stat_fn_name = sprintf('%s_stats.txt', param.season_name); - stat_fn_name(stat_fn_name==' ') = '_'; - stat_fn = fullfile(out_dir,stat_fn_name); - fid = fopen(stat_fn,'w'); - fprintf(fid,'%s\n',param.season_name); - stat_txt = sprintf(' %.2f%% of data has good ice bottom.\n',bottom_good_stat); - fprintf(fid,'%s',stat_txt); - fprintf('%s',stat_txt); - stat_txt = sprintf(' %.2f%% of the time the radar was turned on.\n',coverage_stat); - fprintf(fid,'%s',stat_txt); - fprintf('%s',stat_txt); - fclose(fid); - -% Add single-season plot labels -figure(fig(param_idx)); -h_all = plot(1,NaN,'-r'); -h_good = plot(1,NaN,'g.'); -h_mod = plot(1,NaN,'y.'); -h_det = plot(1,NaN,'r.'); -h_legend = legend([h_all h_good h_mod h_det],'All Radar Data','Good Radar Data',... - 'Moderate Radar Data','Detrended Radar Data','Location','Best'); -if ~ispc % Linux fonts need to be enlarged for the legend - set(h_legend,'FontSize',18); -else - set(h_legend,'FontSize',6); -end - -season_name = param.season_name; -season_name = strrep(season_name, '_',' '); - title(season_name); - - - xlabel('X (km)'); - ylabel('Y (km)'); - hold off; - - fprintf('%s', datestr(now,'HH:MM:SS\n')); - - if ~exist(out_dir,'dir') - mkdir(out_dir); - end - - if strcmpi(location,'Canada') - image_fn_name = sprintf('%s_%s_coverage_map.jpg','Canada',param.season_name); - else % Else Antarctica or Greenland - image_fn_name = sprintf('%s_coverage_map.jpg', param.season_name); - end - - image_fn = fullfile(out_dir,image_fn_name); - - print(fig(param_idx),'-djpeg','-r300',image_fn); - -end - - - diff --git a/cresis-toolbox/gis/coverage_maps_with_param_file.m b/cresis-toolbox/gis/coverage_maps_with_param_file.m deleted file mode 100644 index 8c7e8845..00000000 --- a/cresis-toolbox/gis/coverage_maps_with_param_file.m +++ /dev/null @@ -1,212 +0,0 @@ -% script coverage_maps_with_param_file -% -% Creates data coverage maps (called by coverage_maps.m, which includes -% user-defined settings.) -% 1. Plots blue line for all GPS trajectory -% 2. Computes coverage rate which is the percentage of time that the radar -% is on starting from the time it is turned on to the time it is turned -% off. -% 3. Plots red line on top of this blue line of when radar was on. -% 4. Plots green dots on red line where ice bottom was detected. -% -% Authors: John Paden -% Contributions by Steven Foga - -% ================================================================== -% Automated Section -% ================================================================== -for param_idx = 1:length(param) - clear gps_legend all_legend good_legend mod_legend det_legend; - fig(param_idx) = figure(101); clf; - set(fig(param_idx),'Position',fig_size); - for geotiff_fns_idx = 1:length(geotiff_fns) - proj = geotiffinfo(geotiff_fns{geotiff_fns_idx}); - [RGB{geotiff_fns_idx}, R{geotiff_fns_idx}, tmp] = geotiffread(geotiff_fns{geotiff_fns_idx}); - R{geotiff_fns_idx} = R{geotiff_fns_idx}/1e3; - mapshow(RGB{geotiff_fns_idx}, R{geotiff_fns_idx}); - hold on; - end - hold on; - axis([R{1}(3,1)+[0 R{1}(2,1)*(size(RGB{1},2)-1)] sort(R{1}(3,2)+[0 R{1}(1,2)*(size(RGB{1},1)-1)])]); - - post_dir = ct_filename_out(param(param_idx),'','CSARP_post',1); - csv_dir = fullfile(post_dir,'csv'); - - coverage_image_fn = fullfile(out_dir,sprintf('coverage_%s_%s.jpg',param(param_idx).radar_name,param.season_name)); - - for day_seg_idx = 1:length(param.segs) - days{day_seg_idx} = param(param_idx).segs{day_seg_idx}(1:8); - end - days = unique(days); - - clear good_pnts; clear all_pnts; clear all_pnts_sec; clear all_gps_sec; - good_pnts = 0; - all_pnts = 0; - all_pnts_sec = 0; - all_gps_sec = 0; - - for day_idx = 1:length(days) - day = days{day_idx}; - csv_fns = get_filenames(csv_dir, sprintf('Data_%s',day),'','.csv'); - - param(param_idx).day_seg = [day '_01']; - gps_path = (ct_filename_support(param(param_idx),'','gps',true)); - if isempty(dir(gps_path)) - continue - else - gps = load(gps_path); - end - [gps.gps_time good_idxs] = unique(gps.gps_time); - gps.lat = gps.lat(good_idxs); - gps.lon = gps.lon(good_idxs); - gps.elev = gps.elev(good_idxs); - % Decimation - along_track = geodetic_to_along_track(gps.lat,gps.lon,gps.elev); - decim_idxs = get_equal_alongtrack_spacing_idxs(along_track,50); - gps.lat = gps.lat(decim_idxs); - gps.lon = gps.lon(decim_idxs); - gps.elev = gps.elev(decim_idxs); - gps.gps_time = gps.gps_time(decim_idxs); - gps.mask = zeros(size(gps.lat)); - [gps.x,gps.y] = projfwd(proj,gps.lat,gps.lon); - gps.x = gps.x / 1000; - gps.y = gps.y / 1000; - figure(fig(param_idx)); - h_gps = plot(gps.x,gps.y,'-b'); - - for day_seg_idx = 1:length(csv_fns) - csv_fn = csv_fns{day_seg_idx}; - fid = fopen(csv_fn); - C = textscan(fid,'%f%f%f%f%f%s%f%f%f','Delimiter',',','Headerlines',1); - fclose(fid); - clear lat; clear lon; clear bottom elev; clear utc_time_sod; clear frame_id; - clear year; clear month; clear day; clear utc_time; clear gps_time; - clear x_good; clear y_good; clear qual moderate_qual_idxs detrend_qual_idxs; - clear along_track_csv decim_csv_idxs; - lat = C{1}; - lon = C{2}; - % Decimation - along_track_csv = geodetic_to_along_track(lat,lon,(zeros(size(lat)))); - decim_csv_idxs = get_equal_alongtrack_spacing_idxs(along_track_csv,50); - lat = lat(decim_csv_idxs); - lon = lon(decim_csv_idxs); - bottom = C{8}; - bottom = bottom(decim_csv_idxs); - qual = C{9}; - qual = qual(decim_csv_idxs); - utc_time_sod = C{3}; - utc_time_sod = utc_time_sod(decim_csv_idxs); - frame_id = C{6}; - frame_id = frame_id(decim_csv_idxs); - year = str2double(frame_id{1}(1:4)); - month = str2double(frame_id{1}(5:6)); - day = str2double(frame_id{1}(7:8)); - utc_time = datenum_to_epoch(datenum(year,month,day,0,0,utc_time_sod)); - gps_time = utc_time + utc_leap_seconds(utc_time(1)); - [x,y] = projfwd(proj,lat,lon); - x = x / 1e3; - y = y / 1e3; - good_idxs = find(bottom ~= -9999); - % For each segment, set the mask to 1 during this segment to indicate - % that the radar was turned on (i.e. "coverage" variable below) - gps.mask(gps_time(1) <= gps.gps_time & gps.gps_time <= gps_time(end)) = 1; - good_pnts = good_pnts + length(good_idxs); - all_pnts = all_pnts + length(bottom); - moderate_qual_idxs = find(qual == 2); - detrend_qual_idxs = find(qual == 3); - [x_good,y_good] = projfwd(proj,lat(good_idxs),lon(good_idxs)); - x_good = x_good/1e3; - y_good = y_good/1e3; - - [X_mod,Y_mod] = projfwd(proj,lat(moderate_qual_idxs),lon(moderate_qual_idxs)); - X_mod = X_mod/1e3; - Y_mod = Y_mod/1e3; - - [X_det,Y_det] = projfwd(proj,lat(detrend_qual_idxs),lon(detrend_qual_idxs)); - X_det = X_det/1e3; - Y_det = Y_det/1e3; - - first_idx = find(gps.mask == 1, 1); - last_idx = find(gps.mask == 1, 1, 'last'); - - % Interpolate to 1 second intervals from first measurement of the - % day to the last measurement of the day - gps.mask_interp = interp1(gps.gps_time, gps.mask, gps.gps_time(first_idx):gps.gps_time(last_idx)); - all_pnts_sec = all_pnts_sec + sum(gps.mask_interp); - all_gps_sec = all_gps_sec + length(gps.mask_interp); - - figure(fig(param_idx)); - h_all = plot(x,y,'-r'); - - figure(fig(param_idx)); - h_good = plot(x_good,y_good,'.g'); - - figure(fig(param_idx)); - h_mod = plot(X_mod,Y_mod,'.y'); - - figure(fig(param_idx)); - h_det = plot(X_det,Y_det,'.r'); - end - end - - clear coverage; clear bottom_good; - coverage = all_pnts_sec / all_gps_sec * 100; - - bottom_good = good_pnts / all_pnts * 100; - % ========================================================================= - % ========================================================================= - if ~exist(out_dir,'dir') - mkdir(out_dir); - end - - stat_fn_name = sprintf('%s_stats.txt', param.season_name); - stat_fn_name(stat_fn_name==' ') = '_'; - stat_fn = fullfile(out_dir,stat_fn_name); - fid = fopen(stat_fn,'w'); - fprintf(fid,'%s\n',param.season_name); - stat_txt = sprintf(' %.2f%% of data has good ice bottom.\n',bottom_good); - fprintf(fid,'%s',stat_txt); - fprintf('%s',stat_txt); - stat_txt = sprintf(' %.2f%% of the time the radar was turned on.\n',coverage); - fprintf(fid,'%s',stat_txt); - fprintf('%s',stat_txt); - fclose(fid); - -% Add legend and plot labels - figure(fig); - h_gps = plot(1,NaN,'-b'); - h_all = plot(1,NaN,'-r'); - h_good = plot(1,NaN,'g.'); - h_mod = plot(1,NaN,'y.'); - h_det = plot(1,NaN,'r.'); - h_legend = legend([h_gps h_all h_good h_mod h_det],'Entire Flight','All Radar Data', ... - 'Good Radar Data','Moderate Radar Data','Detrended Radar Data','Location','Best'); - if ~ispc % Linux fonts need to be enlarged for the legend - set(h_legend,'FontSize',18); - else - set(h_legend,'FontSize',6); - end - - season_name = param.season_name; - season_name = strrep(season_name, '_',' '); - title(season_name); - xlabel('X (km)'); - ylabel('Y (km)'); - hold off; - - fprintf('%s', datestr(now,'HH:MM:SS\n')); - - if ~exist(out_dir,'dir') - mkdir(out_dir); - end - - if strcmpi(location,'Canada') - image_fn_name = sprintf('%s_%s_coverage_map.jpg','Canada',param.season_name); - else % Else Antarctica or Greenland - image_fn_name = sprintf('%s_coverage_map.jpg', param.season_name); - end - - image_fn = fullfile(out_dir,image_fn_name); - print(fig(param_idx),'-djpeg','-r300',image_fn); -end -return; diff --git a/cresis-toolbox/gis/dem_class.m b/cresis-toolbox/gis/dem_class.m index c2e3fbaa..7fbf8af4 100644 --- a/cresis-toolbox/gis/dem_class.m +++ b/cresis-toolbox/gis/dem_class.m @@ -15,7 +15,10 @@ dem_info % res: Resolution (m) res - % ocean_mask_mode: 'shapefile' (default) or 'landdem' + % ocean_mask_mode: string containing 'shapefile' (default) or + % 'landdem'. shapefile uses NOAA coastline. landdem marks everything + % above 5 m as land; the latter may be necessary if the NOAA coastline + % filtering misses a land feature. ocean_mask_mode % ocean_mask_dec: default is 100, positive integer decimation rate ocean_mask_dec @@ -74,21 +77,25 @@ % Setup DEM List % =================================================================== obj.dem_info = {}; + proj_load_standard; % Load standard projections % Arctic DEM % ------------------------------------------------------------------- + % http://data.pgc.umn.edu/elev/dem/setsm/ArcticDEM/mosaic/v3.0/500m/arcticdem_mosaic_500m_v3.0.tif % http://data.pgc.umn.edu/elev/dem/setsm/ArcticDEM/mosaic/v3.0/10m/11_41/11_41_10m_v3.0.tar.gz % http://data.pgc.umn.edu/elev/dem/setsm/ArcticDEM/mosaic/v3.0/2m/48_63/48_63_1_1_2m_v3.0.tar.gz % http://data.pgc.umn.edu/elev/dem/setsm/ArcticDEM/mosaic/v3.0/2m/48_63/48_63_2_2_2m_v3.0.tar.gz + % try new_dem_info.url = 'http://data.pgc.umn.edu/elev/dem/setsm/ArcticDEM/mosaic/v3.0/'; new_dem_info.res = [2 10 32 100 500 1000]; new_dem_info.res_str = {'2m' '10m' '32m' '100m' '500m' '1km'}; new_dem_info.tile_en = [1 1 1 0 0 0]; new_dem_info.subtile_en = [1 0 0 0 0 0]; + new_dem_info.tile_fn_ext = '_reg_dem.tif'; new_dem_info.mosaic_fn_fh = @(res_str) sprintf('%s/arcticdem_mosaic_%s_v3.0.tif',res_str,res_str); - new_dem_info.tile_fn_fh = @(x,y,res_str) sprintf('%s/%d_%d/%d_%d_%s_v3.0.tar.gz',res_str,y,x,y,x,res_str); - new_dem_info.subtile_fn_fh = @(x,y,x_sub,y_sub,res_str) sprintf('%s/%d_%d/%d_%d_%d_%d_%s_v3.0.tar.gz',res_str,y,x,y,x,y_sub,x_sub,res_str); + new_dem_info.tile_fn_fh = @(x,y,res_str) sprintf('%s/%02d_%02d/%02d_%02d_%s_v3.0.tar.gz',res_str,y,x,y,x,res_str); + new_dem_info.subtile_fn_fh = @(x,y,x_sub,y_sub,res_str) sprintf('%s/%02d_%02d/%02d_%02d_%d_%d_%s_v3.0.tar.gz',res_str,y,x,y,x,y_sub,x_sub,res_str); new_dem_info.acknowledge = 'Geospatial support for this work provided by the Polar Geospatial Center under NSF-OPP awards 1043681 and 1559691. DEMs provided by the Polar Geospatial Center under NSF-OPP awards 1043681, 1559691, and 1542736.'; new_dem_info.citation = 'Porter, Claire; Morin, Paul; Howat, Ian; Noh, Myoung-Jon; Bates, Brian; Peterman, Kenneth; Keesey, Scott; Schlenk, Matthew; Gardiner, Judith; Tomko, Karen; Willis, Michael; Kelleher, Cole; Cloutier, Michael; Husby, Eric; Foga, Steven; Nakamura, Hitomi; Platson, Melisa; Wethington, Michael, Jr.; Williamson, Cathleen; Bauer, Gregory; Enos, Jeremy; Arnold, Galen; Kramer, William; Becker, Peter; Doshi, Abhijit; D?Souza, Cristelle; Cummens, Pat; Laurier, Fabien; Bojesen, Mikkel, 2018, "ArcticDEM", https://doi.org/10.7910/DVN/OHHUKH, Harvard Dataverse, V1, [Date Accessed]'; new_dem_info.y_tile_origin = 41; @@ -98,7 +105,7 @@ new_dem_info.y_subtile_size = 50e3; new_dem_info.x_subtile_size = 50e3; new_dem_info.no_data = -9999; - new_dem_info.proj = geotiffinfo(ct_filename_gis(obj.param,fullfile('greenland','DEM','GIMP','gimpdem_90m.tif'))); + new_dem_info.proj = arctic_proj; % Loaded from proj_load_standard.m new_dem_info.out_path = ct_filename_gis(obj.param,fullfile('arctic','ArcticDEM')); obj.dem_info{end+1} = new_dem_info; catch ME @@ -107,6 +114,7 @@ % REMA % ------------------------------------------------------------------- + % http://data.pgc.umn.edu/elev/dem/setsm/REMA/mosaic/v1.1/100m/REMA_100m_dem.tif % http://data.pgc.umn.edu/elev/dem/setsm/REMA/mosaic/v1.1/8m/09_38/09_38_8m.tar.gz try new_dem_info.url = 'http://data.pgc.umn.edu/elev/dem/setsm/REMA/mosaic/v1.1/'; @@ -114,8 +122,9 @@ new_dem_info.res_str = {'8m' '100m' '200m' '1km'}; new_dem_info.tile_en = [1 0 0 0]; new_dem_info.subtile_en = [0 0 0 0]; + new_dem_info.tile_fn_ext = '_dem.tif'; new_dem_info.mosaic_fn_fh = @(res_str) sprintf('%s/REMA_%s_dem.tif',res_str,res_str); - new_dem_info.tile_fn_fh = @(x,y,res_str) sprintf('%s/%d_%d/%d_%d_%s.tar.gz',res_str,y,x,y,x,res_str); + new_dem_info.tile_fn_fh = @(x,y,res_str) sprintf('%s/%02d_%02d/%02d_%02d_%s.tar.gz',res_str,y,x,y,x,res_str); new_dem_info.acknowledge = 'Geospatial support for this work provided by the Polar Geospatial Center under NSF-OPP awards 1043681 and 1559691. DEMs provided by the Byrd Polar and Climate Research Center and the Polar Geospatial Center under NSF-OPP awards 1543501, 1810976, 1542736, 1559691, 1043681, 1541332, 0753663, 1548562, 1238993 and NASA award NNX10AN61G. Computer time provided through a Blue Waters Innovation Initiative. DEMs produced using data from DigitalGlobe, Inc.'; new_dem_info.citation = 'Howat, Ian; Morin, Paul; Porter, Claire; Noh, Myong-Jong, 2018, "The Reference Elevation Model of Antarctica", https://doi.org/10.7910/DVN/SAIK8B, Harvard Dataverse, V1'; new_dem_info.y_tile_origin = 31; @@ -123,13 +132,45 @@ new_dem_info.y_tile_size = 100e3; new_dem_info.x_tile_size = 100e3; new_dem_info.no_data = -9999; - new_dem_info.proj = geotiffinfo(ct_filename_gis(obj.param,fullfile('antarctica','DEM','BEDMAP2','original_data','bedmap2_tiff','bedmap2_surface.tif'))); + new_dem_info.proj = antarctic_proj; % Loaded from proj_load_standard.m new_dem_info.out_path = ct_filename_gis(obj.param,fullfile('antarctica','DEM','REMA')); obj.dem_info{end+1} = new_dem_info; catch ME warning(ME.getReport); end + % /cresis/snfs1/dataproducts/GIS_data/usa/DEM/NED/National_Elevation_Data_DEM_10m.tif + % USGS National Elevation Dataset (NED) + % 1/3 arc-seconds: 10 m resolution + % https://www.usgs.gov/core-science-systems/ngp/3dep/data-tools + % https://prd-tnm.s3.amazonaws.com/StagedProducts/Elevation/13/TIFF/n44w104/USGS_13_n44w104.tif <-- I think this URL can be automatically generated to grab the tiles that are needed for a specific region + % + try + new_dem_info.url = ''; + new_dem_info.res = [10]; + new_dem_info.res_str = {'10m'}; + new_dem_info.tile_en = [0 0 0 0]; + new_dem_info.subtile_en = [0 0 0 0]; + new_dem_info.tile_fn_ext = ''; + new_dem_info.mosaic_fn_fh = @(res_str) sprintf('National_Elevation_Data_DEM_%s.tif',res_str); + new_dem_info.tile_fn_fh = @(x,y,res_str) sprintf('',res_str,y,x,y,x,res_str); + new_dem_info.acknowledge = 'USGS National Elevation Dataset (NED)'; + new_dem_info.citation = 'USGS National Elevation Dataset (NED)'; + new_dem_info.y_tile_origin = NaN; + new_dem_info.x_tile_origin = NaN; + new_dem_info.y_tile_size = NaN; + new_dem_info.x_tile_size = NaN; + new_dem_info.no_data = -9999; + new_dem_info.proj = usa_ned_proj; % Loaded from proj_load_standard.m + new_dem_info.out_path = ct_filename_gis(obj.param,fullfile('usa','DEM','NED')); + obj.dem_info{end+1} = new_dem_info; + catch ME + warning(ME.getReport); + end + + % NOAA Shoreline: + % Wessel, P., and W. H. F. Smith (1996), A global, self-consistent, hierarchical, high-resolution shoreline database, J. Geophys. Res., 101(B4), 8741-8743, doi:10.1029/96JB00104. + % Prepare DEM fields % ------------------------------------------------------------------- obj.clear(); @@ -158,9 +199,12 @@ function delete(obj) obj.x = nan(size(lat)); obj.y = nan(size(lat)); obj.di = nan(size(lat)); - % Northern hemisphere - obj.di(lat>0) = 1; - % Southern hemisphere + % For each input determine the dem index to use + % Northern hemisphere, ArcticDEM + obj.di(lat>49.0024) = 1; + % USGS NED Dem + obj.di(lat<=49.0024 & lat >= 25.4483) = 3; + % Southern hemisphere, REMA obj.di(lat<0) = 2; for di = unique(obj.di) @@ -194,7 +238,9 @@ function delete(obj) obj.load_ocean(); % If the whole region fits into a single tile, then this tile is - % loaded and truncated to the region of interest + % loaded and truncated to the region of interest. If the region does + % not fit into a single tile, then the tiles will be loaded later on + % an as needed basis. obj.load_dem(); end @@ -263,8 +309,14 @@ function delete(obj) % ------------------------------------------------------------------- if ~isfield(obj.ocean,'shp_all') ocean_mask_fn = ct_filename_gis(obj.param,fullfile('world','land_mask','Land_Mask_IDL_jharbeck','GSHHS_f_L1.shp')); + ocean_mask_fn_antarctica = ct_filename_gis(obj.param,fullfile('world','land_mask','Land_Mask_IDL_jharbeck','GSHHS_f_L5.shp')); warning off; obj.ocean.shp_all = shaperead(ocean_mask_fn); + if isfield(obj.param,'post') && isfield(obj.param.post,'ops') ... + && isfield(obj.param.post.ops,'location') && strcmp(obj.param.post.ops.location,'antarctic') + shp_antarctica = shaperead(ocean_mask_fn_antarctica); + obj.ocean.shp_all = cat(1,obj.ocean.shp_all,shp_antarctica); + end warning on; obj.ocean.bb_all = [obj.ocean.shp_all(:).BoundingBox]; end @@ -432,6 +484,7 @@ function delete(obj) for sub_y_idx = 1:2 if any(tile_x >= tiles(tiles_idx,1) & tile_x <= tiles(tiles_idx,1)+1 ... & tile_y >= tiles(tiles_idx,2) & tile_y <= tiles(tiles_idx,2)+1) + % Create URL for subtile file and add to url_list to download url_list{end+1} = [obj.dem_info{di}.url,obj.dem_info{di}.subtile_fn_fh(tiles(tiles_idx,1),tiles(tiles_idx,2),sub_x_idx,sub_y_idx,obj.dem_info{di}.res_str{ri})]; xi_list(end+1) = 2*tiles(tiles_idx,1) + sub_x_idx - 2; yi_list(end+1) = 2*tiles(tiles_idx,2) + sub_y_idx - 2; @@ -443,6 +496,7 @@ function delete(obj) end end else + % Create URL for tile file and add to url_list to download url_list{end+1} = [obj.dem_info{di}.url,obj.dem_info{di}.tile_fn_fh(tiles(tiles_idx,1),tiles(tiles_idx,2),obj.dem_info{di}.res_str{ri})]; xi_list(end+1) = tiles(tiles_idx,1); yi_list(end+1) = tiles(tiles_idx,2); @@ -460,9 +514,12 @@ function delete(obj) [~,url_name,url_ext1] = fileparts(url); [~,url_name,url_ext2] = fileparts(url_name); + % fn: archive file to be downloaded (will be deleted after we are done extracting the tif file) fn = fullfile(obj.dem_info{di}.out_path,[url_name,url_ext2,url_ext1]); - tif_fn = fullfile(obj.dem_info{di}.out_path,[url_name '_reg_dem.tif']); + % tif_fn: tif file inside the archive file that we want + tif_fn = fullfile(obj.dem_info{di}.out_path,[url_name obj.dem_info{di}.tile_fn_ext]); + % Check to see if download file AND tif file do not exist if ~exist(fn,'file') && ~exist(tif_fn,'file') cmd = sprintf('wget -P %s %s', obj.dem_info{di}.out_path, url); fprintf(' %s\n', cmd); @@ -473,14 +530,16 @@ function delete(obj) end end + % if ~exist(tif_fn,'file') - fprintf('Untar %s\n %s\n', fn, obj.dem_info{di}.out_path); + fprintf('Untar tar file %s\n to tif file: %s\n', fn, tif_fn); untar(fn, obj.dem_info{di}.out_path); if ~exist(tif_fn,'file') - error('Failed to untar file or file not found in tar archive.'); + error('Failed to find tif file after untar. If tar file is corrupted, try deleting the tar file and starting over so that it will be downloaded again.'); end end + % Delete tar file, but keep tif file if exist(fn,'file') delete(fn); end diff --git a/cresis-toolbox/gis/ecmwf_load.m b/cresis-toolbox/gis/ecmwf_load.m index 2eafcd98..b09f55ef 100644 --- a/cresis-toolbox/gis/ecmwf_load.m +++ b/cresis-toolbox/gis/ecmwf_load.m @@ -1,3 +1,6 @@ + +% cresis-toolbox\python\ECMWF\ecmwf_download.py +% D:\tmp\koenig_internal_layers\ ecmwf = ncinfo('erai_tp_e_79_17.nc'); % units: degrees_east @@ -54,22 +57,31 @@ fprintf('%s to %s\n', datestr(ecmwf.time(1)), datestr(ecmwf.time(end))) +xmask = ecmwf.lon > 360-70 & ecmwf.lon < 360-20; +ymask = ecmwf.lat > 60 & ecmwf.lat < 80; + +% Greenland meters SWE per year figure(1); clf; -imagesc(ecmwf.lon,ecmwf.lat,sum(ecmwf.tp,3)) -% imagesc(ecmwf.lon,ecmwf.lat,mean(ecmwf.tp,3)) -% imagesc(ecmwf.lon,ecmwf.lat,max(ecmwf.tp,[],3)) +imagesc(ecmwf.lon(xmask),ecmwf.lat(ymask),(mean(ecmwf.tp(ymask,xmask,:),3)+mean(ecmwf.e(ymask,xmask,:),3))*365.25) set(gca,'YDir','normal'); xlabel('Longitude (deg)'); ylabel('Latitude (deg)'); h = colorbar; -set(get(h,'YLabel'),'String','Total precipitation (m)'); +set(get(h,'YLabel'),'String','Mean SMB (m/year)'); figure(2); clf; -imagesc(ecmwf.lon,ecmwf.lat,sum(ecmwf.e,3)) +imagesc(ecmwf.lon,ecmwf.lat,mean(ecmwf.tp,3)) set(gca,'YDir','normal'); xlabel('Longitude (deg)'); ylabel('Latitude (deg)'); h = colorbar; -set(get(h,'YLabel'),'String','Evaporation (m)'); +set(get(h,'YLabel'),'String','Mean precipitation (m/day)'); +figure(3); clf; +imagesc(ecmwf.lon,ecmwf.lat,mean(ecmwf.e,3)) +set(gca,'YDir','normal'); +xlabel('Longitude (deg)'); +ylabel('Latitude (deg)'); +h = colorbar; +set(get(h,'YLabel'),'String','Mean Evaporation (m/day)'); diff --git a/cresis-toolbox/gis/egm96_loader.m b/cresis-toolbox/gis/egm96_loader.m index bf5449f4..60f33f32 100644 --- a/cresis-toolbox/gis/egm96_loader.m +++ b/cresis-toolbox/gis/egm96_loader.m @@ -22,7 +22,7 @@ % % fn = string containing input filename (path to WW15MGH.DAC file) % lat = Nx1 double vector, latitude (deg,N) -% lon = Mx1 double vector, longitude (deg,N) +% lon = Mx1 double vector, longitude (deg,E) % egm = NxM double matrix containing error from WGS-84 of actual sea level (m) % First axis (row): Latitude % Second axis (col): Longitude diff --git a/cresis-toolbox/gis/make_coverage_maps.m b/cresis-toolbox/gis/make_coverage_maps.m deleted file mode 100644 index 176685ab..00000000 --- a/cresis-toolbox/gis/make_coverage_maps.m +++ /dev/null @@ -1,216 +0,0 @@ -% make_coverage_maps.m -% -% Used to create individual maps of each flight season's radar depth -% sounder (RDS) coverage. This script produces: -% 1. individual JPEG maps (with Landsat GeoTIFF as background) -% 2. text file with statistics on: -% i. percent time the radar was turned on -% ii. percent of good radar data exists for the season. -% 3. a combined MATLAB figure (.fig) of all of the seasons. -% -% This script calls the individual scripts: -% coverage_maps_old_without_gps -% coverage_maps_old_with_gps -% coverage_maps_with_param_file -% -% Authors: Steven Foga, John Paden -% -% See also: coverage_maps_old_without_gps, coverage_maps_old_with_gps, -% coverage_maps_with_param_file, coverage_maps_master_fig_files - -fprintf('===========================================================\n'); -fprintf('coverage_maps\n\n'); - -global gRadar; - -% =================================================================== -% User Settings -% =================================================================== - -% location = 'Greenland'; -% location = 'Canada'; -location = 'Antarctica'; - -out_dir = '/cresis/scratch1/petan/coverage_maps/'; -% out_dir = 'Z:\sfoga\coverage_maps\'; - -if strcmpi(location,'Greenland') || strcmpi(location,'Canada') - season_names = {}; - season_names{end+1,1} = 'icards/1993_Greenland_P3'; -% season_names{end+1,1} = 'icards/1995_Greenland_P3'; -% season_names{end+1,1} = 'icards/1996_Greenland_P3'; -% season_names{end+1,1} = 'icards/1997_Greenland_P3'; -% season_names{end+1,1} = 'icards/1998_Greenland_P3'; -% season_names{end+1,1} = 'icards/1999_Greenland_P3'; -% season_names{end+1,1} = 'icards/2001_Greenland_P3'; -% season_names{end+1,1} = 'icards/2002_Greenland_P3'; -% season_names{end+1,1} = 'acords/2003_Greenland_P3'; -% season_names{end+1,1} = 'acords/2005_Greenland_TO'; -% season_names{end+1,1} = 'mcrds/2006_Greenland_TO'; -% season_names{end+1,1} = 'mcrds/2007_Greenland_P3'; -% season_names{end+1,1} = 'mcrds/2008_Greenland_Ground'; -% season_names{end+1,1} = 'mcrds/2008_Greenland_TO'; -% season_names{end+1,1} = 'mcrds/2009_Greenland_TO'; -% season_names{end+1,1} = 'mcords/2010_Greenland_DC8'; -% season_names{end+1,1} = 'mcords/2010_Greenland_P3'; -% season_names{end+1,1} = 'mcords/2011_Greenland_TO'; -% season_names{end+1,1} = 'mcords2/2011_Greenland_P3'; -% season_names{end+1,1} = 'mcords2/2012_Greenland_P3'; -elseif strcmpi(location,'Antarctica') - season_names = {}; - season_names{end+1,1} = 'icards/2002_Antarctica_P3chile'; - season_names{end+1,1} = 'acords/2004_Antarctica_P3chile'; - season_names{end+1,1} = 'mcords/2009_Antarctica_DC8'; - season_names{end+1,1} = 'mcords/2009_Antarctica_TO'; - season_names{end+1,1} = 'mcords/2010_Antarctica_DC8'; - season_names{end+1,1} = 'mcords/2011_Antarctica_DC8'; - season_names{end+1,1} = 'mcords2/2011_Antarctica_TO'; - season_names{end+1,1} = 'mcords2/2012_Antarctica_DC8'; -else - return -end - -% =================================================================== -% Automated Section -% =================================================================== -global gRadar - -if strcmpi(location,'Greenland') - - geotiff_fns = {}; - geotiff_fns{1} = ct_filename_gis(gRadar,'greenland/Landsat-7/Greenland_natural_250m.tif'); - - - fig_size = [50 50 600 800]; - fig_paper_position = [0.25 2.5 6 8]; - fig_legend_size = [0.5742 0.1341 0.3051 0.0992]; - -elseif strcmpi(location,'Canada') - - geotiff_fns = {}; - geotiff_fns{1} = ct_filename_gis(gRadar,'canada/Landsat-7/Canada_250m.tif'); - - fig_size = [50 50 560 420]; - fig_paper_position = [0.25 2.5 6 8]; - fig_legend_size = [100 25 0 0]; - -elseif strcmpi(location,'Antarctica') - - geotiff_fns = {}; - geotiff_fns{1} = ct_filename_gis(gRadar,'antarctica/Landsat-7/Antarctica_LIMA_480m.tif'); - - fig_size = [50 50 600 800]; - fig_legend_size = [235 0 0 0]; -else - return -end - - - for season_idx = 1:length(season_names) - fclose all; - fprintf('Processing season %s\n', season_names{season_idx}); - - filesep_idx = find(season_names{season_idx}=='/',1); - clear param; - param.radar_name = season_names{season_idx}(1:filesep_idx-1); - param.season_name = season_names{season_idx}(filesep_idx+1:end); - param.rdr_season_type = ct_filename_out(param,'','CSARP_post/csv/',true); - param.rdr_season_good = ct_filename_out(param,'','CSARP_post/csv_good/',true); - param.all_season_type = strcat(param.rdr_season_type,'Browse_',param.season_name,'.csv'); - - % Replace stat_good_gre with param.stat_good - param.stat_good = strcat(param.rdr_season_good,'Browse_',param.season_name,'.csv'); - - % Replace stat_all_gre with param.stat_all - param.stat_all = strcat(param.rdr_season_type,'Browse_',param.season_name,'.csv'); - - - % Navigation to 'params' toolbox (only needed to find param - % spreadsheets) - param.param_path = strrep(gRadar.path,'cresis-toolbox/',''); - param.param_path = strcat(param.param_path,'params/'); - - param.gps_all = []; - - % GPS or Param file paths - if strcmp(param.season_name,'2004_Antarctica_P3chile') - param.gps_all = '/cresis/scratch1/mdce/csarp_support/gps/ACORDS_2004_Antarctica_POS_GPS'; - elseif strcmp(param.season_name,'2003_Greenland_P3') - param.gps_all = '/cresis/scratch1/mdce/csarp_support/gps/ACORDS_2003_Greenland_POS_GPS'; - elseif strcmp(param.season_name,'2005_Greenland_TO') - param.gps_all = '/cresis/scratch1/mdce/csarp_support/gps/ACORDS_2005_Greenland_POS_GPS'; - elseif strcmp(param.season_name,'2006_Greenland_TO') - param.gps_all = '/cresis/scratch1/mdce/csarp_support/gps/MCRDS_2006_Greenland_POS_GPS'; - elseif strcmp(param.season_name,'2007_Greenland_P3') - param.gps_all = '/cresis/scratch1/mdce/csarp_support/gps/MCRDS_2007_Greenland_POS_GPS'; - elseif strcmp(param.season_name,'2008_Greenland_TO') - param.gps_all = '/cresis/scratch1/mdce/csarp_support/gps/MCRDS_2008_Greenland_POS_DGPS'; - elseif strcmp(param.season_name,'2008_Greenland_Ground') - param.gps_all = '/cresis/scratch1/mdce/csarp_support/gps/MCRDS_2008_Greenland_Ground_GPS'; - elseif strcmp(param.season_name,'2009_Greenland_TO') - param.gps_all = '/cresis/scratch1/mdce/csarp_support/gps/MCRDS_2009_Greenland_POS_GPS'; - elseif strcmp(param.season_name,'2009_Antarctica_DC8') - param.param_path = strcat(param.param_path,'mcords_param_',param.season_name,'.xls'); - param.segs = make_segment_list(param.param_path); - elseif strcmp(param.season_name,'2009_Antarctica_TO') - param.param_path = strcat(param.param_path,'mcords_param_',param.season_name,'.xls'); - param.segs = make_segment_list(param.param_path); - elseif strcmp(param.season_name,'2010_Antarctica_DC8') - param.param_path = strcat(param.param_path,'mcords_param_',param.season_name,'.xls'); - param.segs = make_segment_list(param.param_path); - elseif strcmp(param.season_name,'2011_Antarctica_DC8') - param.param_path = strcat(param.param_path,'mcords_param_',param.season_name,'.xls'); - param.segs = make_segment_list(param.param_path); - elseif strcmp(param.season_name,'2011_Antarctica_TO') - param.param_path = strcat(param.param_path,'mcords_param_',param.season_name,'.xls'); - param.segs = make_segment_list(param.param_path); - elseif strcmp(param.season_name,'2010_Greenland_P3') - param.param_path = strcat(param.param_path,'mcords_param_',param.season_name,'.xls'); - param.segs = make_segment_list(param.param_path); - elseif strcmp(param.season_name,'2010_Greenland_DC8') - param.param_path = strcat(param.param_path,'mcords_param_',param.season_name,'.xls'); - param.segs = make_segment_list(param.param_path); - elseif strcmp(param.season_name,'2011_Greenland_TO') - param.param_path = strcat(param.param_path,'mcords_param_',param.season_name,'.xls'); - param.segs = make_segment_list(param.param_path); - elseif strcmp(param.season_name,'2011_Greenland_P3') - param.param_path = strcat(param.param_path,'mcords_param_',param.season_name,'.xls'); - param.segs = make_segment_list(param.param_path); - elseif strcmp(param.season_name,'2012_Greenland_P3') - param.param_path = strcat(param.param_path,'mcords_param_','_param_',param.season_name,'.xls'); - param.segs = make_segment_list(param.param_path); - elseif strcmp(param.season_name,'2012_Antarctica_DC8') - param.param_path = strcat(param.param_path,'mcords_param_','_param_',param.season_name,'.xls'); - param.segs = make_segment_list(param.param_path); - else - fprintf('ERROR: No GPS files exist (2002 or before), or no matching season found.\n\n'); - end - - % No ct_filename exists for scratch1 (GPS file location), so - % filepaths conversion must be performed manually. - if ispc && ~isempty(param.gps_all) - param.gps_all = strrep(param.gps_all,'/cresis/scratch1/','Y:\'); - param.gps_all = strrep(param.gps_all,'/','\'); - else - end - - if strcmp(param.season_name,'2002_Antarctica_P3chile') || strcmp(param.season_name,'1993_Greenland_P3') || ... - strcmp(param.season_name,'1995_Greenland_P3') || strcmp(param.season_name,'1996_Greenland_P3') || ... - strcmp(param.season_name,'1997_Greenland_P3') || strcmp(param.season_name,'1998_Greenland_P3') || ... - strcmp(param.season_name,'1999_Greenland_P3') || strcmp(param.season_name,'2001_Greenland_P3') || ... - strcmp(param.season_name,'2002_Greenland_P3') || strcmp(param.season_name,'2002 Antarctica P3chile') - - coverage_maps_old_without_gps; - - elseif strcmp(param.season_name,'2003_Greenland_P3') || strcmp(param.season_name,'2005_Greenland_TO') ... - || strcmp(param.season_name,'2006_Greenland_TO') || strcmp(param.season_name,'2007_Greenland_P3') ... - || strcmp(param.season_name,'2008_Greenland_Ground') || strcmp(param.season_name,'2008_Greenland_TO') ... - || strcmp(param.season_name,'2009_Greenland_TO') || strcmp(param.season_name,'2004_Antarctica_P3chile') - - coverage_maps_old_with_gps; - - else % All missions flown after 2009_Greenland_TO - coverage_maps_with_param_file; - end - end - return; \ No newline at end of file diff --git a/cresis-toolbox/gis/project_locations.m b/cresis-toolbox/gis/project_locations.m index 5950575a..dcfd71ac 100644 --- a/cresis-toolbox/gis/project_locations.m +++ b/cresis-toolbox/gis/project_locations.m @@ -1,4 +1,12 @@ function loc = project_locations +% loc = project_locations +% +% Common locations of interest +% +% This function needs to be formalized better. It is a placeholder for a +% better function right now. +% +% See Also: physical_constants.m, proj_load_standard.m, project_locations.m loc.melt.gzds.lat = -(75+12.208/60); loc.melt.gzds.lon = -(104+49.850/60); @@ -28,3 +36,6 @@ loc.ngrip.lat = 75+1/60; loc.ngrip.lon = -(42+32/60); + +loc.egrip.lat = 75+38/60; +loc.egrip.lon = -(36+0/60); diff --git a/cresis-toolbox/gis/run_bottom_dem_class.m b/cresis-toolbox/gis/run_bottom_dem_class.m new file mode 100644 index 00000000..e65a5848 --- /dev/null +++ b/cresis-toolbox/gis/run_bottom_dem_class.m @@ -0,0 +1,43 @@ +% script run_bottom_dem_class +% +% Script for demonstrating use of the bottom_dem_class. +% +% Author: John Paden + +%% Typical Use +% ========================================================================= +% Check to see if class already exists +global gbottom_dem; +if isempty(gbottom_dem) || ~isa(gbottom_dem,'bottom_dem_class') || ~isvalid(gbottom_dem) + gbottom_dem = bottom_dem_class(gRadar); +end + +if ~exist('records','var') + records = records_load(struct('season_name','2018_Greenland_P3','radar_name','rds','day_seg','20180418_04')); +end +% Set the class up to look at this whole record +if ~strcmpi(gbottom_dem.name,'rds:2018_Greenland_P3:20180418_04') + % Name the vector so the second time the script is run, we don't setup + % the vector again unless the name is changed to a different segment. + gbottom_dem.set_vector(records.lat,records.lon,'rds:2018_Greenland_P3:20180418_04'); +end + +gbottom_dem.set_vector(records.lat(1:10000),records.lon(1:10000)); +[land_dem,msl,ocean_mask] = gbottom_dem.get_vector_dem(); + +return; + + +% This may be necessary to clear the class from memory when debugging +clear all; +startup; +dbstop if error; + + +% Delete object when you no longer need the dem and want to free memory +global gbottom_dem; +try + delete(gbottom_dem); +end + + diff --git a/cresis-toolbox/gis/run_dem_class.m b/cresis-toolbox/gis/run_dem_class.m index 1676641e..756f3c48 100644 --- a/cresis-toolbox/gis/run_dem_class.m +++ b/cresis-toolbox/gis/run_dem_class.m @@ -7,6 +7,7 @@ %% Typical Use % ========================================================================= % Check to see if class already exists +global gdem; if isempty(gdem) || ~isa(gdem,'dem_class') || ~isvalid(gdem) gdem = dem_class(gRadar); end @@ -16,7 +17,7 @@ gdem.set_res(10); if ~exist('records','var') - records = load('/cresis/snfs1/dataproducts/csarp_support/records/rds/2018_Greenland_P3/records_20180418_04.mat'); + records = load(fullfile(gRadar.support_path,'records','rds','2018_Greenland_P3','records_20180418_04.mat')); end % Set the class up to look at this whole record if ~strcmpi(gdem.name,'rds:2018_Greenland_P3:20180418_04') diff --git a/cresis-toolbox/gis/run_make_coverage_maps_master_fig.m b/cresis-toolbox/gis/run_make_coverage_maps_master_fig.m index bab3d152..3e0812aa 100644 --- a/cresis-toolbox/gis/run_make_coverage_maps_master_fig.m +++ b/cresis-toolbox/gis/run_make_coverage_maps_master_fig.m @@ -90,7 +90,7 @@ % --------- STEP 5: SELECT THE DESTINATION WHERE THE FIGURE IS TO BE SAVED AT --------- if user_variables.save_img == 1 - out_dir = 'Y:\rohan\coverage_maps_loader\'; + out_dir = fullfile(gRadar.out_path,'coverage_maps'); if ~exist(out_dir,'dir') mkdir(out_dir); diff --git a/cresis-toolbox/gis/run_make_coverage_maps_seg.m b/cresis-toolbox/gis/run_make_coverage_maps_seg.m index 69dcc1bf..76473f9b 100644 --- a/cresis-toolbox/gis/run_make_coverage_maps_seg.m +++ b/cresis-toolbox/gis/run_make_coverage_maps_seg.m @@ -30,8 +30,14 @@ params = {}; -params{end+1} = read_param_xls(ct_filename_param('rds_param_2010_Greenland_DC8.xls')); -params{end} = ct_set_params(params{end},'cmd.generic',1); +% params{end+1} = read_param_xls(ct_filename_param('rds_param_2010_Greenland_DC8.xls')); +% params{end} = ct_set_params(params{end},'cmd.generic',1); + +params{end+1} = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls')); +params{end} = ct_set_params(params{end},'cmd.generic',0); +params{end} = ct_set_params(params{end},'cmd.generic',1,'day_seg','20140325_07'); +params{end} = ct_set_params(params{end},'cmd.generic',1,'day_seg','20140401_03'); +params{end} = ct_set_params(params{end},'cmd.generic',1,'day_seg','20140506_01'); %params{end+1} = read_param_xls(ct_filename_param('rds_param_2011_Greenland_TO.xls')); %params{end} = ct_set_params(params{end},'cmd.generic',1); @@ -41,8 +47,8 @@ % --------- STEP 3 : SELECT THE GIS FILE --------- if strcmpi(params{1}(1).post.ops.location,'arctic') - user_variables.geotiff_fn = ct_filename_gis(gRadar,fullfile('arctic','Landsat-7','arctic_natural_90m.tif')); %Default - %user_variables.geotiff_fn = ct_filename_gis(gRadar,fullfile('arctic','NaturalEarth_Data','Arctic_NaturalEarth.tif')); + %user_variables.geotiff_fn = ct_filename_gis(gRadar,fullfile('arctic','Landsat-7','arctic_natural_90m.tif')); %Default + user_variables.geotiff_fn = ct_filename_gis(gRadar,fullfile('arctic','NaturalEarth_Data','Arctic_NaturalEarth.tif')); elseif strcmpi(params{1}(1).post.ops.location,'antarctic') user_variables.geotiff_fn = ct_filename_gis(gRadar,'antarctica/Landsat-7/Antarctica_LIMA_240m.tif'); %Default %user_variables.geotiff_fn = ct_filename_gis(gRadar,fullfile('antarctica','NaturalEarth_Data','Antarctica_NaturalEarth.tif')); @@ -53,8 +59,8 @@ % --------- STEP 4 : SELECT WHETHER TO SAVE THE FIGURE --------- % set save image flag -user_variables.save_img = 1; %to save, default -% user_variables.save_img = 0; % to not save +% user_variables.save_img = 1; %to save, default +user_variables.save_img = 0; % to not save @@ -70,7 +76,7 @@ % --------- STEP 6: SELECT THE DESTINATION WHERE THE FIGURE IS TO BE SAVED AT --------- if user_variables.save_img == 1 - user_variables.out_dir = 'H:\rohan\coverage_maps\selected_segments\'; + user_variables.out_dir = fullfile(gRadar.out_path,'coverage_maps'); end @@ -94,7 +100,7 @@ user_variables.along_track_sampling = 100; % plot_args: cell array of argument to plot function (e.g. 'b.' for blue dots) -user_variables.plot_args = {'LineWidth',1}; +user_variables.plot_args = {'LineWidth',2}; % label: Add text labels to the plots for specific frames % .day_seg: cell list of day segments to match @@ -103,6 +109,9 @@ label.day_seg = {}; label.frm = {}; label.text = {}; +label.day_seg = {'20140325_07','20140401_03','20140506_01'}; +label.frm = {[1],[13],[1]}; +label.text = {'20140325\_07','20140401\_03','20140506\_01'}; % label.day_seg = {'20091102_02','20091016_01','20091016_01','20091102_02','20091102_02'}; % label.frm = {8, 21, 26, 23, 32}; % label.text = {' 1',' 2',' 3',' 4',' 5'}; @@ -125,7 +134,7 @@ %Set the params according to what you want params{params_idx} = ct_set_params(params{params_idx},'cmd.generic',0,'cmd.notes','do not process'); user_variables.params_idx = params_idx; - user_variables.proj = plot_geotiff(user_variables.geotiff_fn,[],[], params_idx); + user_variables.proj = geotiff_plot(user_variables.geotiff_fn,[],[], params_idx); test_make_coverage_maps(params{params_idx}, user_variables); if ~exist(out_dir,'dir') diff --git a/cresis-toolbox/gis/test_make_coverage_maps.m b/cresis-toolbox/gis/test_make_coverage_maps.m index fc1fa552..d3daea26 100644 --- a/cresis-toolbox/gis/test_make_coverage_maps.m +++ b/cresis-toolbox/gis/test_make_coverage_maps.m @@ -23,7 +23,7 @@ function test_make_coverage_maps(params, user_variables) % .ext: string % '.m': MATLAB FIG-file and MATLAB code that opens figure % '.jpg': JPEG image -% '.fig': MATLAB FIG-file +% '.fig': MATLAB FIG-file % '.axis_limits': vector % - a vector of axis limits for the geotiff % '.along_track_sampling': integer @@ -85,7 +85,7 @@ function test_make_coverage_maps(params, user_variables) end %Plotting if not 'do not process' - if isempty(regexpi(parameter.cmd.notes,'do not process')) + if ct_generic_en(parameter) && isempty(regexpi(parameter.cmd.notes,'do not process')) yyyymmdd_list{end+1} = parameter.yyyymmdd; %Adding segment to the list of segments already covered %Load lat, lon depending on data source @@ -103,10 +103,10 @@ function test_make_coverage_maps(params, user_variables) warning('No frames file: %s\n', frames_fn); return; end - gps = load(records_fn,'gps_time','lat','lon'); + gps = records_load(parameter,'gps_time','lat','lon'); % Load frames file - load(frames_fn); + frames = frames_load(parameter); if isempty(parameter.cmd.frms) parameter.cmd.frms = 1:length(frames.frame_idxs); @@ -450,7 +450,7 @@ function test_make_coverage_maps(params, user_variables) % image_fn_name = sprintf('aggregate_greenland_master_coverage_map_%s', date); - image_fn_name = sprintf('modified201617test_%s', date); + image_fn_name = sprintf('coverage_map%s', date); image_fn = fullfile(user_variables.out_dir,image_fn_name); savefig(figure(1), image_fn, 'compact'); diff --git a/cresis-toolbox/gps/BCD_to_seconds.m b/cresis-toolbox/gps/BCD_to_seconds.m index e9b8f722..56790350 100644 --- a/cresis-toolbox/gps/BCD_to_seconds.m +++ b/cresis-toolbox/gps/BCD_to_seconds.m @@ -1,8 +1,31 @@ -function seconds = BCD_to_seconds(seconds_BCD) - -seconds_BCD = double(seconds_BCD); - -seconds = ... - 3600*(10*mod(floor(seconds_BCD/2^8),2^4) + mod(floor(seconds_BCD/2^12),2^4)) ... - + 60*(10*mod(floor(seconds_BCD/2^16),2^4) + mod(floor(seconds_BCD/2^20),2^4)) ... - + (10*mod(floor(seconds_BCD/2^24),2^4) + mod(floor(seconds_BCD/2^28),2^4)); +function seconds = BCD_to_seconds(seconds_BCD,BCD_order) +% seconds = BCD_to_seconds(seconds_BCD,BCD_order) +% +% Converts from 32 bit BCD encoded time to seconds of day +% +% BCD_order: Scalar integer. Default is -1. Defines the order of the +% decimal numbers in the 32 bit integer. -1 is reverse order. +1 is forward +% order. + +if ~exist('BCD_order','var') || isempty(BCD_order) + BCD_order = -1; +end + +seconds_BCD = double(seconds_BCD); + +if BCD_order == -1 + % BCD encoded into 32 bits as: + % SSMMHH-- + % (reverse order so that least significant decimal digit comes first) + seconds = ... + 3600*(10*mod(floor(seconds_BCD/2^8),2^4) + mod(floor(seconds_BCD/2^12),2^4)) ... + + 60*(10*mod(floor(seconds_BCD/2^16),2^4) + mod(floor(seconds_BCD/2^20),2^4)) ... + + (10*mod(floor(seconds_BCD/2^24),2^4) + mod(floor(seconds_BCD/2^28),2^4)); +else + % BCD encoded into 32 bits as: + % --HHMMSS + seconds = ... + 3600*(10*mod(floor(seconds_BCD/2^20),2^4) + mod(floor(seconds_BCD/2^16),2^4)) ... + + 60*(10*mod(floor(seconds_BCD/2^12),2^4) + mod(floor(seconds_BCD/2^8),2^4)) ... + + (10*mod(floor(seconds_BCD/2^4),2^4) + mod(floor(seconds_BCD/2^0),2^4)); +end \ No newline at end of file diff --git a/cresis-toolbox/gps/flight_tracker.m b/cresis-toolbox/gps/flight_tracker.m index fd691c51..92c4e299 100644 --- a/cresis-toolbox/gps/flight_tracker.m +++ b/cresis-toolbox/gps/flight_tracker.m @@ -1,72 +1,204 @@ % script flight_tracker % -% Flight tracker which reads in a NMEA stream (GPGGA) from a serial -% device and plots the result on a geotiff. +% Flight tracker which reads in a NMEA stream (GPGGA) from a serial device +% or a file that is being recorded (by the radar for example) and plots the +% result on a geotiff. % -% Optionally plots a KML flight line (this part is customized -% to NASA ATM's flightplan). Leave kml_fn empty to turn off -% this feature. +% Example of loading KML and old datasets given. % % Close plotting window or press ctrl-C (multiple times rapidly!) % to quit. Serial device is stored in global variable, -% flight_tracker_serial_dev, so that you can close it if the program +% gflight_tracker.serial_dev, so that you can close it if the program % fails to do so. % -% Requires Mapping Toolbox, read_xml.m, plot_geotiff.m +% Requires Mapping Toolbox, read_xml.m, geotiff_plot.m % % Examples: % flight_tracker % % Author: John Paden +global gRadar; +global gflight_tracker; +physical_constants; + +%% User Settings % ================================================================= -% User Settings -% ================================================================= -% geotiff_fn = 'C:\GIS_data\greenland\Landsat-7\mzl7geo_90m_lzw.tif'; -geotiff_fn = 'C:\GIS_data\arctic\NaturalEarth_Data\Arctic_NaturalEarth.tif'; -% geotiff_fn = '/scratch/GIS_data/greenland/Landsat-7/mzl7geo_90m_lzw.tif'; % For Land Ice -% geotiff_fn = '/scratch/GIS_data/arctic/NaturalEarth_Data/Arctic_NaturalEarth.tif'; % For Sea Ice - -gps_input_type = 'file_mcords'; % file_accum, or serial -%serial_dev = '/dev/ttyUSB0'; -%serial_dev = '/dev/ttyUSB1'; -% You may have to run from bash shell: "chmod a+rwx /dev/ttyS0" as root -serial_dev = '/dev/ttyS0'; -gps_input_fn_dir = 'E:\'; -gps_input_fn_start = 'GPS'; -% gps_input_fn_dir = '\\172.18.1.33\accum\'; -% gps_input_fn_dir = '/net/field1/landing/mcords/mcords5/'; -gps_input_fn_skip = false; % Enables skipping reading old data, sometimes - % need to do this if files contain errors that - % cause program to crash - -% For OIB, get kmz file from John Sonntag, then unzip the kmz file and use the "doc.kml" that is inside -kml_fn = 'C:\Users\administrator\Desktop\doc.kml'; -% kml_fn = ''; % Set this to empty if the file is not available -kml_mission_name = ''; - -enable_gps_record = false; -gps_fn_dir = '/scratch/metadata/2015_Greenland_LC130/'; +% geotiff_fn: String containing file path for geotiff file for map +% geotiff_fn = fullfile(gRadar.gis_path,'arctic','NaturalEarth_Data\Arctic_NaturalEarth.tif'); % For Sea Ice +% geotiff_fn = fullfile(gRadar.gis_path,'arctic','Landsat-7','arctic_natural_90m.tif'); % For Land Ice +% geotiff_fn = fullfile(gRadar.gis_path,'arctic','ArcticDEM','arcticdem_mosaic_500m_v3.0.tif'); % For Land Ice +geotiff_fn = fullfile(gRadar.gis_path,'antarctica','Landsat-7','Antarctica_LIMA.tif'); % For Land Ice + +% dem_fn: String containing file path for DEM +% dem_fn = fullfile(gRadar.gis_path,'arctic','ArcticDEM','arcticdem_mosaic_500m_v3.0.tif'); +dem_fn = fullfile(gRadar.gis_path,'antarctica','DEM','REMA','REMA_1km_dem_filled.tif'); + +% gps_input_type: String containing GPS source +gps_input_type = 'file_novatelraw'; % 'file_novatelraw', 'file_mcords', 'file_accum', or 'serial' + +% serial_dev: String containing the name of the serial device for 'serial' +% gps_input_type. +% +% On Linux, you may have to give file permissions from the bash terminal to +% allow the serial device to be opened: "sudo chmod a+rwx /dev/ttyS0" +serial_dev = '/dev/ttyUSB0'; +% serial_dev = '/dev/ttyUSB1'; +% serial_dev = '/dev/ttyS0'; + +% serial_serial_baud_rate: integer scaler containing BAUD rate for 'serial' +% gps_input_type. +serial_baud_rate = 115200; % e.g. 9600, 57600, 115200, etc. + +% gps_input_fn_dir: string containing the file path to the directory +% containing the GPS file for 'file_mcords' and 'file_accum' +% gps_input_type. +% gps_input_fn_dir = '\\192.168.1.100\d\awi'; +gps_input_fn_dir = '/data/'; +% gps_input_fn_dir = '\\ground1\O\UWB\20220607'; +gps_input_fn_prefix = 'GPS_Novatel_raw_aq-field22_20230124'; + +% gps_input_geoid_en: logical scaler. If true, the EGM-96 geoid is +% subtracted from GPS information that is read in. +gps_input_geoid_en = false; + +% gps_input_fn_skip: logical scaler for 'file_mcords' and 'file_accum' +% gps_input_type. Normally false. If true, all information in existing GPS +% files will be ignored and only new information will be used by +% flight_tracker.m. +% +% Enables skipping reading old data, sometimes need to do this if files +% contain errors that cause flight_tracker.m to crash. +gps_input_fn_skip = false; +% gps_record_en: Logical scaler. If true, then this program will record +% the GPS information it receives to a file. +gps_record_en = false; + +% gps_record_fn_dir: String containing the path to the directory of where +% to store the GPS information. Only used if gps_record_en is true. +gps_record_fn_dir = '/home/cresis1/tmp/'; + +% year,month,day: Usually do not need to modify this [year,month,day] = datevec(now); +gps_start_ref = datenum(year,month,day); -% ================================================================= -% Automated Section -% ================================================================= +% debug_start_line: positive integer scaler. Set to inf to disable debug +% mode, set to a finite number to load this many lines from the GPS source +% file. Only works with 'file_mcords' and 'file_accum' gps_input_type. +debug_start_line = inf; % <== Set to "inf" to disable debug mode -if isempty(kml_fn) - kml_lon = []; - kml_lat = []; -else - if 0 +%% User Settings: Desired flight track +% gps_desire.en: Logical scaler. If true, load desired flight track. Set to +% false if no desired flight track exists. +gps_desire = struct('en',true); +if gps_desire.en + gps_desire_source = 'sonntagsequence2'; % <== CHOOSE A DESIRED FLIGHT TRACK SOURCE + + if strcmpi(gps_desire_source,'records') + %% User Settings: Desired flight track: records + gps_desire = records_load(fullfile(gRadar.support_path,'records','rds','2018_Greenland_Polar6','records_20180509_01')); + gps_desire.along_track = geodetic_to_along_track(gps_desire.lat,gps_desire.lon,gps_desire.elev); + decim_idxs = get_equal_alongtrack_spacing_idxs(gps_desire.along_track,50); + gps_desire.lat = gps_desire.lat(decim_idxs); + gps_desire.lon = gps_desire.lon(decim_idxs); + gps_desire.elev = gps_desire.elev(decim_idxs); + [gps_desire.ecef_x,gps_desire.ecef_y,gps_desire.ecef_z] = geodetic2ecef(gps_desire.lat/180*pi,gps_desire.lon/180*pi,gps_desire.elev, WGS84.ellipsoid); + + elseif strcmpi(gps_desire_source,'sonntagsequence2') + gps_desire = read_gps_sonntagsequence2(fullfile(gRadar.data_support_path,'2022_Antarctica_BaslerMKB','F16','CLX_EAST02A_5HR.sequence2')); + gps_desire.along_track = geodetic_to_along_track(gps_desire.lat,gps_desire.lon,gps_desire.elev); + decim_idxs = get_equal_alongtrack_spacing_idxs(gps_desire.along_track,50); + gps_desire.lat = gps_desire.lat(decim_idxs); + gps_desire.lon = gps_desire.lon(decim_idxs); + gps_desire.elev = gps_desire.elev(decim_idxs); + + [gps_desire.ecef_x,gps_desire.ecef_y,gps_desire.ecef_z] = geodetic2ecef(gps_desire.lat/180*pi,gps_desire.lon/180*pi,gps_desire.elev, WGS84.ellipsoid); + + orig_lat = gps_desire.lat; + orig_lon = gps_desire.lon; + [gps_desire.lat,gps_desire.lon] = interpm(gps_desire.lat,gps_desire.lon,450/(2*pi*earthRadius),'gc'); + % Find original points + gps_desire.orig_pnt = false(size(gps_desire.lat)); + for idx = 1:length(orig_lat) + match_idx = find(gps_desire.lat == orig_lat(idx) & gps_desire.lon == orig_lon(idx)); + gps_desire.orig_pnt(match_idx) = true; + end + gps_desire.along_track = geodetic_to_along_track(gps_desire.lat,gps_desire.lon); + % decim_idxs = get_equal_alongtrack_spacing_idxs(gps_desire.along_track,50); + % gps_desire.along_track = gps_desire.along_track(decim_idxs); + % gps_desire.lat = gps_desire.lat(decim_idxs); + % gps_desire.lon = gps_desire.lon(decim_idxs); + + % gps_desire.constant_AGL_en: logical scaler. If true, then dem_fn will + % be used to set the elevation according to constant_AGL_ft. + constant_AGL_en = true; + % gps_desire.constant_AGL_ft: double scaler. The gps_desire.elev will + % be set to be this many feet above the dem_fn surface if + % constant_AGL_en is true. + constant_AGL_ft = 2000; + + elseif strcmpi(gps_desire_source,'kml_simple') + %% User Settings: Desired flight track: kml_simple + % NOTE: This KML example only works for some KML file types. + % Simple KML file + % kml_fn = 'C:\metadata\2022_Greenland_Polar5\flight_lines\AdditionalSounding1.kml'; + % kml_fn = 'C:\metadata\2022_Greenland_Polar5\flight_lines\Additional_Soundings_2_new.kml'; + kml_fn = 'C:\metadata\2022_Greenland_Polar5\flight_lines\Bons_A.kml'; xDoc = xmlread(kml_fn); document = read_xml(xDoc); - pos = textscan(document.kml{1}.Document{1}.Placemark{1}.LineString{1}.coordinates{1}.text{1}.node_val, ... - '%f%f%f','Delimiter',','); - kml_lon = pos{1}; - kml_lat = pos{2}; - else + if isfield(document.kml{1}.Document{1},'Placemark') + pos = textscan(document.kml{1}.Document{1}.Placemark{1}.LineString{1}.coordinates{1}.text{1}.node_val, ... + '%f%f%f','Delimiter',','); + elseif isfield(document.kml{1}.Document{1}.Folder{1},'Placemark') + if length(document.kml{1}.Document{1}.Folder{1}.Placemark) == 1 + pos = textscan(document.kml{1}.Document{1}.Folder{1}.Placemark{1}.LineString{1}.coordinates{1}.text{1}.node_val, ... + '%f%f','Delimiter',', '); + else + pos{1} = []; + pos{2} = []; + for idx = 1:length(document.kml{1}.Document{1}.Folder{1}.Placemark) + tmp_pos = textscan(document.kml{1}.Document{1}.Folder{1}.Placemark{idx}.Point{1}.coordinates{1}.text{1}.node_val, ... + '%f%f','Delimiter',', '); + pos{1}(end+1) = tmp_pos{1}; + pos{2}(end+1) = tmp_pos{2}; + end + end + else + error('Not able to find "Placemark" field that probably contains the LineString field that should be read.'); + end + gps_desire.lon = pos{1}; + gps_desire.lat = pos{2}; + clear xDoc document; + + orig_lat = gps_desire.lat; + orig_lon = gps_desire.lon; + [gps_desire.lat,gps_desire.lon] = interpm(gps_desire.lat,gps_desire.lon,50/(2*pi*earthRadius),'gc'); + % Find original points + gps_desire.orig_pnt = false(size(gps_desire.lat)); + for idx = 1:length(orig_lat) + match_idx = find(gps_desire.lat == orig_lat(idx) & gps_desire.lon == orig_lon(idx)); + gps_desire.orig_pnt(match_idx) = true; + end + gps_desire.along_track = geodetic_to_along_track(gps_desire.lat,gps_desire.lon); + % decim_idxs = get_equal_alongtrack_spacing_idxs(gps_desire.along_track,50); + % gps_desire.along_track = gps_desire.along_track(decim_idxs); + % gps_desire.lat = gps_desire.lat(decim_idxs); + % gps_desire.lon = gps_desire.lon(decim_idxs); + + % gps_desire.constant_AGL_en: logical scaler. If true, then dem_fn will + % be used to set the elevation according to constant_AGL_ft. + constant_AGL_en = true; + % gps_desire.constant_AGL_ft: double scaler. The gps_desire.elev will + % be set to be this many feet above the dem_fn surface if + % constant_AGL_en is true. + constant_AGL_ft = 1200; + + elseif strcmpi(gps_desire_source,'kml_complex') + %% User Settings: Desired flight track: kml_complex + % NOTE: This KML example only works for some KML file types. + % KML file with many named lines in it (allows for selecting one of the lines) kml_pos = kml_read_shapefile(kml_fn); if isempty(kml_mission_name) for idx=1:length(kml_pos) @@ -88,50 +220,218 @@ kml_lon = kml_pos(kml_mission_idx).X; kml_lat = kml_pos(kml_mission_idx).Y; end + clear kml_pos kml_mission_idx; end end +%% ======================================================================== +%% flight_tracker +% ========================================================================= + +%% gflight_tracker definition +% ========================================================================= +if isempty(gflight_tracker) + gflight_tracker = struct('dem_RGB',[],'dem_proj',[],'dem_x_axis',[],'dem_y_axis',[], ... + 'serial_device',[], ... + 'lat',[],'lon',[],'elev',[],'gps_time',[],'x',[],'y',[], ... + 'h_fig',[],'h_axes',[],'h_axes_yz',[],'h_plot_yz1',[],'h_plot_yz2',[],'h_plot_yz3',[],'h_axes_dem',[],'h_plot_dem1',[],'h_plot_dem2',[]); +end + +%% Geoid EGM96 Load +% ================================================================= +if gps_input_geoid_en + egm96_fn = ct_filename_gis([],'world\egm96_geoid\WW15MGH.DAC'); + [gflight_tracker.egm96_lat,gflight_tracker.egm96_lon,gflight_tracker.egm96_elev] = egm96_loader(egm96_fn); +end + +%% DEM load +% ================================================================= +[gflight_tracker.dem_RGB, dem_R, ~] = geotiffread(dem_fn); +gflight_tracker.dem_proj = geotiffinfo(dem_fn); +gflight_tracker.dem_x_axis = dem_R(3,1) + dem_R(2,1)*(1:size(gflight_tracker.dem_RGB,2)); +gflight_tracker.dem_y_axis = dem_R(3,2) + dem_R(1,2)*(1:size(gflight_tracker.dem_RGB,1)); + +[dem_x,dem_y] = projfwd(gflight_tracker.dem_proj,gps_desire.lat,gps_desire.lon); +gps_desire.elev_ground = double(interp2(gflight_tracker.dem_x_axis,gflight_tracker.dem_y_axis,gflight_tracker.dem_RGB,dem_x,dem_y)); +if constant_AGL_en + gps_desire.elev = gps_desire.elev_ground + constant_AGL_ft*12*2.54/100; +end +[gps_desire.ecef_x,gps_desire.ecef_y,gps_desire.ecef_z] = geodetic2ecef(gps_desire.lat/180*pi,gps_desire.lon/180*pi,gps_desire.elev, WGS84.ellipsoid); + +%% Get plot figure handles +% ========================================================================= +% h_fig(1): geotiff map with flightlines plotted on it (also displays the +% current status information) +% h_fig(2): Elevation plot +% h_fig(3): YZ-offset from line (plot of last 10 seconds, estimate of next +% 10 seconds based on current heading) +% KML used for YZ-offset (resampled to 100 m) or gps source (e.g. gps or +% records file from a previous flight) +gflight_tracker.h_fig = get_figures(3,true); +set(gflight_tracker.h_fig,'DockControls','off') +set(gflight_tracker.h_fig,'NumberTitle','off'); +set(gflight_tracker.h_fig,'MenuBar','none'); +set(gflight_tracker.h_fig(2:3),'ToolBar','none'); +clf(gflight_tracker.h_fig(2)); +clf(gflight_tracker.h_fig(3)); +set(gflight_tracker.h_fig(1),'ToolBar','figure'); +if strcmpi(class(gflight_tracker.h_fig(1)),'double') + set(gflight_tracker.h_fig(1),'Name',sprintf('%d: map',gflight_tracker.h_fig(1))); +else + set(gflight_tracker.h_fig(1),'Name',sprintf('%d: map',gflight_tracker.h_fig(1).Number)); +end +if strcmpi(class(gflight_tracker.h_fig(2)),'double') + set(gflight_tracker.h_fig(2),'Name',sprintf('%d: offset',gflight_tracker.h_fig(2))); +else + set(gflight_tracker.h_fig(2),'Name',sprintf('%d: offset',gflight_tracker.h_fig(2).Number)); +end +if strcmpi(class(gflight_tracker.h_fig(3)),'double') + set(gflight_tracker.h_fig(3),'Name',sprintf('%d: elev',gflight_tracker.h_fig(3))); +else + set(gflight_tracker.h_fig(3),'Name',sprintf('%d: elev',gflight_tracker.h_fig(3).Number)); +end + +gflight_tracker.h_axes_yz = axes('parent',gflight_tracker.h_fig(2)); +hold(gflight_tracker.h_axes_yz,'on'); +grid(gflight_tracker.h_axes_yz,'on'); +gflight_tracker.h_plot_yz1 = plot(NaN,NaN,'x-','parent',gflight_tracker.h_axes_yz,'linewidth',2,'color','white'); +gflight_tracker.h_plot_yz3 = plot(NaN,NaN,'o-','markersize',6,'linewidth',1,'parent',gflight_tracker.h_axes_yz,'color','green'); +gflight_tracker.h_plot_yz2 = plot(NaN,NaN,'o','markersize',12,'linewidth',4,'parent',gflight_tracker.h_axes_yz,'color','white'); +xlabel(gflight_tracker.h_axes_yz,'Horizontal (ft)') +ylabel(gflight_tracker.h_axes_yz,'Elevation (ft)') +plot(gflight_tracker.h_axes_yz,[-1e5 1e5],[0 0],'m','LineWidth',2); +plot(gflight_tracker.h_axes_yz,[0 0],[-1e5 1e5],'m','LineWidth',2); +xlims_old = 50; +ylims_old = 50; +set(gflight_tracker.h_axes_yz,'FontSize',14) + +gflight_tracker.h_axes_dem = subplot(2,1,1,'parent',gflight_tracker.h_fig(3)); +gflight_tracker.h_axes_dem_all = subplot(2,1,2,'parent',gflight_tracker.h_fig(3)); +hold(gflight_tracker.h_axes_dem,'on'); +grid(gflight_tracker.h_axes_dem,'on'); +gflight_tracker.h_plot_dem1 = plot(NaN,NaN,'x','parent',gflight_tracker.h_axes_dem); +gflight_tracker.h_plot_dem2 = plot(NaN,NaN,'x','parent',gflight_tracker.h_axes_dem); +xlabel(gflight_tracker.h_axes_dem,'Future (seconds)') +ylabel(gflight_tracker.h_axes_dem,'Elevation (ft)') +plot(gflight_tracker.h_axes_dem,[0 0],[-1e3 50e3],'w','LineWidth',2); +set(gflight_tracker.h_axes_dem,'FontSize',14) + +set(gflight_tracker.h_fig(2),'Color','k') +set(gflight_tracker.h_axes_yz,'Color','k') +set(gflight_tracker.h_axes_yz,'GridColor','w') +set(gflight_tracker.h_axes_yz,'MinorGridColor','w') +set(gflight_tracker.h_axes_yz,'GridAlpha',0.5) +set(gflight_tracker.h_axes_yz,'XColor','w') +set(gflight_tracker.h_axes_yz,'YColor','w') +set(gflight_tracker.h_fig(3),'Color','k') +set(gflight_tracker.h_axes_dem,'Color','k') +set(gflight_tracker.h_axes_dem,'GridColor','w') +set(gflight_tracker.h_axes_dem,'MinorGridColor','w') +set(gflight_tracker.h_axes_dem,'GridAlpha',0.5) +set(gflight_tracker.h_axes_dem,'XColor','w') +set(gflight_tracker.h_axes_dem,'YColor','w') + +plot(gflight_tracker.h_axes_dem_all, gps_desire.along_track/1852, gps_desire.elev_ground*100/12/2.54, 'r-','linewidth',2); +hold(gflight_tracker.h_axes_dem_all,'on'); +grid(gflight_tracker.h_axes_dem_all,'on'); +plot(gflight_tracker.h_axes_dem_all, gps_desire.along_track/1852, gps_desire.elev*100/12/2.54, 'm-', 'linewidth', 2); +ylim(gflight_tracker.h_axes_dem_all, ... + [min([gps_desire.elev_ground;gps_desire.elev])*100/12/2.54, ... + max([gps_desire.elev_ground;gps_desire.elev])*100/12/2.54]) +xlim(gflight_tracker.h_axes_dem_all, gps_desire.along_track([1 end])/1852); +gflight_tracker.h_plot_dem_all = plot(gflight_tracker.h_axes_dem_all, NaN, NaN, 'w-', 'linewidth', 2); +set(gflight_tracker.h_axes_dem_all,'Color','k') +set(gflight_tracker.h_axes_dem_all,'GridColor','w') +set(gflight_tracker.h_axes_dem_all,'MinorGridColor','w') +set(gflight_tracker.h_axes_dem_all,'GridAlpha',0.5) +set(gflight_tracker.h_axes_dem_all,'XColor','w') +set(gflight_tracker.h_axes_dem_all,'YColor','w') +set(gflight_tracker.h_axes_dem_all,'FontSize',14) +xlabel(gflight_tracker.h_axes_dem_all,'Along-track (nm)') +ylabel(gflight_tracker.h_axes_dem_all,'Elevation (ft)') + +%% Serial device open +% ========================================================================= if strcmpi(gps_input_type,'serial') - global flight_tracker_serial_dev; - if isempty(flight_tracker_serial_dev) + if isempty(gflight_tracker.serial_dev) fprintf('Getting serial device %s handle\n', serial_dev); - flight_tracker_serial_dev = serial(serial_dev); + gflight_tracker.serial_dev = serial(serial_dev,'BaudRate',serial_baud_rate); end - - if ~strcmpi(get(flight_tracker_serial_dev,'Status'),'open') + + if ~strcmpi(get(gflight_tracker.serial_dev,'Status'),'open') fprintf('Opening serial device %s\n', serial_dev); - fopen(flight_tracker_serial_dev); + fopen(gflight_tracker.serial_dev); else fprintf('Serial device %s already open, just going to start reading\n', ... serial_dev); end - - pos_buf = NaN*zeros(1e5,2); - time_buf = NaN*zeros(10,1); - lat_buf = NaN*zeros(length(time_buf),1); - lon_buf = NaN*zeros(length(time_buf),1); - elev_buf = NaN*zeros(length(time_buf),1); end +%% Geotiff +% ========================================================================= fprintf('Plotting geotiff\n'); -[proj,fig_h] = plot_geotiff(geotiff_fn, kml_lat, kml_lon, 1,'r'); -haxes = get(fig_h,'Children'); +[gflight_tracker.proj,gflight_tracker.h_fig(1),gflight_tracker.h_axes,gflight_tracker.h_image] ... + = geotiff_plot(geotiff_fn, [], [], gflight_tracker.h_fig(1),'r'); +set(gflight_tracker.h_fig(1),'color','black'); +set(gflight_tracker.h_axes,'color','black','xcolor','white','ycolor','white','fontsize',14); + +%% Desired flight track (e.g. from previous mission) +% ========================================================================= +if exist('gps_desire','var') + fprintf('Plotting desired flight track\n') + [gps_desire.x,gps_desire.y] = projfwd(gflight_tracker.proj,gps_desire.lat,gps_desire.lon); + % Convert from m to km: + gps_desire.x = gps_desire.x/1e3; + gps_desire.y = gps_desire.y/1e3; + hold(gflight_tracker.h_axes,'on'); + gflight_tracker.h_plot_gps_prev = plot(gps_desire.x, gps_desire.y,'parent',gflight_tracker.h_axes(1),'Color','magenta'); +end + +%% Plot existing flight track data +% ========================================================================= +if any(strcmpi(gps_input_type,{'file_novatelraw'})) + gflight_tracker.lat = []; + gflight_tracker.lon = []; + gflight_tracker.elev = []; + gflight_tracker.gps_time = []; + gflight_tracker.x = []; + gflight_tracker.y = []; + gps_in_fns = get_filenames(gps_input_fn_dir,gps_input_fn_prefix,'','.gps'); + gps_in_fn = gps_in_fns{end}; + gps_input = read_gps_novatelraw(gps_in_fn); + gflight_tracker.lat = gps_input.lat; + gflight_tracker.lon = gps_input.lon; + gflight_tracker.elev = gps_input.elev; + gflight_tracker.gps_time = gps_input.gps_time; + [gflight_tracker.x,gflight_tracker.y] = projfwd(gflight_tracker.proj, gflight_tracker.lat, gflight_tracker.lon); + gflight_tracker.x = gflight_tracker.x/1e3; + gflight_tracker.y = gflight_tracker.y/1e3; + + gps_in_fn_pos = dir(gps_in_fn); + gps_in_fn_pos = gps_in_fn_pos.bytes; + +end if any(strcmpi(gps_input_type,{'file_accum','file_mcords'})) + gflight_tracker.lat = []; + gflight_tracker.lon = []; + gflight_tracker.elev = []; + gflight_tracker.gps_time = []; + gflight_tracker.x = []; + gflight_tracker.y = []; % Look for, load, and plot all GPS files if strcmpi(gps_input_type,'file_accum') gps_input_fn_ext = '.gps'; else gps_input_fn_ext = '.txt'; end - + % Monitoring: % Look for the latest GPS file % If there is no file, skip steps % If newest file matches the current file, then keep current position % Load latest file and search for '$' if ~gps_input_fn_skip - gps = struct('lat',[],'lon',[]); gps_in_fns = get_filenames(gps_input_fn_dir,'','',gps_input_fn_ext); if isempty(gps_in_fns) warning('No GPS files in %s\n', gps_input_fn_dir); @@ -140,56 +440,50 @@ for gps_in_fn_idx = 1:length(gps_in_fns) gps_in_fn = gps_in_fns{gps_in_fn_idx}; fid = fopen(gps_in_fn,'r'); - while ~feof(fid) + debug_num_lines = 0; + while ~feof(fid) && debug_num_lines < debug_start_line line_input = fgets(fid); if feof(fid) break; end A = textscan(line_input,'%s%f%f%c%f%c%u%u%f%f%c%f%c%s%s%f%f%f%f','delimiter',', ','emptyvalue',NaN); try - if all(~cellfun(@isempty,A([1 4 5 6]))) && strcmp(A{1},'$GPGGA') && ~isnan(A{3}) && any(strcmpi(A{4},{'N','S'})) && ~isnan(A{5}) && any(strcmpi(A{6},{'W','E'})) - gps.lat(1,end+1) = ((A{4}=='N')*2-1) .* A{3}; - gps.lon(1,end+1) = ((A{6}=='E')*2-1) .* A{5}; + if all(~cellfun(@isempty,A([1 2 3 4 5 6 10]))) && strcmp(A{1},'$GPGGA') && ~isnan(A{3}) && any(strcmpi(A{4},{'N','S'})) && ~isnan(A{5}) && any(strcmpi(A{6},{'W','E'})) + hour = floor(A{2}/1e4); + minute = floor((A{2}-hour*1e4)/1e2); + sec= A{2}-hour*1e4-minute*1e2; + gflight_tracker.gps_time(1,end+1) = gps_start_ref + hour/24 + minute/1440 + sec/86400; + gflight_tracker.lat(1,end+1) = ((A{4}=='N')*2-1) .* A{3}; + gflight_tracker.lon(1,end+1) = ((A{6}=='E')*2-1) .* A{5}; + gflight_tracker.elev(1,end+1) = A{10}; end end + debug_num_lines = debug_num_lines + 1; + gps_in_fn_pos = ftell(fid); end fclose(fid); end - gps.lat = fix(gps.lat/100) + (gps.lat/100 - fix(gps.lat/100))./60*100; - gps.lon = fix(gps.lon/100) + (gps.lon/100 - fix(gps.lon/100))./60*100; + gflight_tracker.lat = fix(gflight_tracker.lat/100) + (gflight_tracker.lat/100 - fix(gflight_tracker.lat/100))./60*100; + gflight_tracker.lon = fix(gflight_tracker.lon/100) + (gflight_tracker.lon/100 - fix(gflight_tracker.lon/100))./60*100; + [gflight_tracker.x,gflight_tracker.y] = projfwd(gflight_tracker.proj, gflight_tracker.lat, gflight_tracker.lon); + gflight_tracker.x = gflight_tracker.x/1e3; + gflight_tracker.y = gflight_tracker.y/1e3; end end - - gps_in_fn = ''; - gps_in_fn_pos = -inf; - - pos_buf = NaN*zeros(1e5,2); - time_buf = NaN*zeros(10,1); - lat_buf = NaN*zeros(length(time_buf),1); - lon_buf = NaN*zeros(length(time_buf),1); - elev_buf = NaN*zeros(length(time_buf),1); - - if ~gps_input_fn_skip - [x,y] = projfwd(proj,gps.lat,gps.lon); - x = x/1e3; - y = y/1e3; - idx_to_use = max(1,length(x)-size(pos_buf,1)+1) : length(x); - pos_buf(1:length(idx_to_use),1) = fliplr(x(idx_to_use)); - pos_buf(1:length(idx_to_use),2) = fliplr(y(idx_to_use)); + if debug_start_line == inf; + gps_in_fn = ''; + gps_in_fn_pos = -inf; end end -hold on; -hline = plot(pos_buf(:,1),pos_buf(:,2),'b-','Parent',haxes); -hpos = plot(pos_buf(1,1),pos_buf(1,2),'rx','MarkerSize',10,'LineWidth',3,'Parent',haxes); -hold off; - -if enable_gps_record - gps_fn = fullfile(gps_fn_dir,sprintf('gps_%04d%02d%02d.csv',year,month,day)); +%% Plotting existing flight track data recorded by flight_tracker.m +% ========================================================================= +if gps_record_en + gps_fn = fullfile(gps_record_fn_dir,sprintf('gps_%04d%02d%02d.csv',year,month,day)); if exist(gps_fn,'file') try gps = read_gps_csv(gps_fn, struct('time_reference','utc')); - [x,y] = projfwd(proj,gps.lat,gps.lon); + [x,y] = projfwd(gflight_tracker.proj,gps.lat,gps.lon); x = x/1e3; y = y/1e3; idx_to_use = max(1,length(x)-size(pos_buf,1)+1) : length(x); @@ -204,206 +498,421 @@ catch end end - - % =========================================================== - %% Opening GPS log file + + %% Plotting existing flight track data: Open log file % =========================================================== fprintf('Opening GPS log file %s\n', gps_fn); - [fid_out,msg] = fopen(gps_fn,'a'); + [gflight_tracker.fid_out,msg] = fopen(gps_fn,'a'); if fid_out <= 0 error(msg); end - if ftell(fid_out) == 0 + if ftell(gflight_tracker.fid_out) == 0 % Write header line if file is empty fprintf(' New file, writing CSV header line\n'); - fprintf(fid_out,'year,month,day,UTC_sod,latNdeg,lonEdeg,elevm\n'); + fprintf(gflight_tracker.fid_out,'year,month,day,UTC_sod,latNdeg,lonEdeg,elevm\n'); end end -update_geotif_tstart = uint64(0); - -try - done = false; - while ~done - %% Update Map Axes - xlim_orig = xlim(haxes); - ylim_orig = ylim(haxes); - axis normal; - % Force axis to be equal - xlim_new = xlim(); - ylim_new = ylim(); - axes_pos = get(haxes,'position'); aspect_ratio = axes_pos(3)/axes_pos(4); - xlim_bigger = diff(xlim_new)/diff(ylim_new)/aspect_ratio > 1; - if xlim_bigger - ylim_new(1) = mean(ylim_new) - diff(xlim_new)/2*aspect_ratio; - ylim_new(2) = mean(ylim_new) + diff(xlim_new)/2*aspect_ratio; - else - xlim_new(1) = mean(xlim_new) - diff(ylim_new)/2/aspect_ratio; - xlim_new(2) = mean(xlim_new) + diff(ylim_new)/2/aspect_ratio; +hold(gflight_tracker.h_axes,'on'); +gflight_tracker.h_line= plot(gflight_tracker.x,gflight_tracker.y,'b-','Parent',gflight_tracker.h_axes); +gflight_tracker.h_plot = plot(gflight_tracker.x(end),gflight_tracker.y(end),'rx','MarkerSize',10,'LineWidth',3,'Parent',gflight_tracker.h_axes); +hold(gflight_tracker.h_axes,'off'); + +% update_geotiff_tstart: timer to force map to update XY axes every 30 +% seconds to the current position +update_geotiff_tstart = uint64(0); +update_plot_tstart = uint64(0); + +%% Loop for new data +% ========================================================================= +% try +done = false; +nmea_dollarsign_synced = false; +while ~done + %% Loop for new data: Load GPS data from serial + % ===================================================================== + if strcmpi(gps_input_type,'serial') + try + nmea_str = fscanf(gflight_tracker.serial_dev); + catch ME + pause(0.5); + continue; end - if toc(update_geotif_tstart) > 30 - update_geotif_tstart = tic; - if isfinite(pos_buf(1,1)) && isfinite(pos_buf(1,2)) - % Update to current platform position - xlim_new = xlim_new + pos_buf(1,1) - mean(xlim_new); - ylim_new = ylim_new + pos_buf(1,2) - mean(ylim_new); - end + if isempty(nmea_str) + fprintf('Empty string\n'); + pause(0.5); + continue; end - xlim(haxes,xlim_new); - ylim(haxes,ylim_new); - - %% Load GPS data from serial - if strcmpi(gps_input_type,'serial') - try - nmea_str = fscanf(flight_tracker_serial_dev); - catch ME - pause(0.5); - continue; + A = textscan(nmea_str,'%s%f%f%c%f%c%u%u%f%f%c%f%c%s%s','delimiter',',','emptyvalue',NaN); + clear nmea_str; + end + + %% Loop for new data: Load GPS data from file + % ===================================================================== + if any(strcmpi(gps_input_type,{'file_novatelraw'})) + % Load in the last BESTPOSB record in the file + gps_in_fns = get_filenames(gps_input_fn_dir,gps_input_fn_prefix,'','.gps'); + if ~isempty(gps_in_fns) + if ~strcmpi(gps_in_fn,gps_in_fns{end}) + gps_in_fn = gps_in_fns{end}; + gps_in_fn_pos = 0; end - if isempty(nmea_str) - fprintf('Empty string\n'); - pause(0.5); - continue; + gps = read_gps_novatelraw(gps_in_fn,struct('first_byte',gps_in_fn_pos)); + + if ~isempty(gps.gps_time) + gps_in_fn_pos = dir(gps_in_fn); + gps_in_fn_pos = gps_in_fn_pos.bytes; + gflight_tracker.lat(end+(1:length(gps.gps_time))) = gps.lat; + gflight_tracker.lon(end+(1:length(gps.gps_time))) = gps.lon; + gflight_tracker.elev(end+(1:length(gps.gps_time))) = gps.elev; + if gps_input_geoid_en + gflight_tracker.elev(end) = gflight_tracker.elev(end) ... + + interp2(gflight_tracker.egm96_lon,gflight_tracker.egm96_lat,gflight_tracker.egm96_elev, ... + mod(gflight_tracker.lon(end),360),gflight_tracker.lat(end)); + end + gflight_tracker.gps_time(end+(1:length(gps.gps_time))) = gps.gps_time; + [x,y] = projfwd(gflight_tracker.proj,gps.lat,gps.lon); + gflight_tracker.x(end+(1:length(gps.gps_time))) = x/1e3; + gflight_tracker.y(end+(1:length(gps.gps_time))) = y/1e3; + set(gflight_tracker.h_line,'XData',gflight_tracker.x,'YData',gflight_tracker.y); + set(gflight_tracker.h_plot,'XData',gflight_tracker.x(end),'YData',gflight_tracker.y(end)); + drawnow; end - A = textscan(nmea_str,'%s%f%f%c%f%c%u%u%f%f%c%f%c%s%s','delimiter',',','emptyvalue',NaN); - end - - %% Load GPS data from file - if any(strcmpi(gps_input_type,{'file_accum','file_mcords'})) - % Look for, load, and plot all GPS files - if strcmpi(gps_input_type,'file_accum') - gps_input_fn_ext = '.gps'; - else - gps_input_fn_ext = '.txt'; + + % if length(lat) > 10 + % along_track = geodetic_to_along_track(lat(end-9:end),lon(end-9:end),elev(end-9:end)); + % speed = along_track(end) / abs(utc_time(end)-utc_time(end-9)); + % end + % fprintf('%9.6f N %11.6f E | %6.1f m | %6.1f m AGL | %3.0f m/s %3.0f kn\n', lat, lon, elev, elev-dem_elev, speed, speed/0.5144444); + + if gps_record_en + fprintf(fid_out,'%04d,%02d,%02d,%f,%f,%f,%f\n', year, month, day, utc_sod, lat, lon, elev); end - % Check to see if we are looking at the most recent GPS file - gps_in_fns = get_filenames(gps_input_fn_dir,gps_input_fn_start,'',gps_input_fn_ext); - if ~isempty(gps_in_fns) - if ~strcmpi(gps_in_fn,gps_in_fns{end}) - % New GPS file - gps_in_fn = gps_in_fns{end}; - - % Find the last dollar sign '$' in the file - fid = fopen(gps_in_fn,'r'); - fseek(fid,0,1); - dollar_found = false; - while ~dollar_found - if ftell(fid) == 0 - % Last GPS file does not contain '$' - gps_in_fn_pos = -inf; - break; - end + end + end + + if any(strcmpi(gps_input_type,{'file_accum','file_mcords'})) + % Look for, load, and plot all GPS files + if strcmpi(gps_input_type,'file_accum') + gps_input_fn_ext = '.gps'; + else + gps_input_fn_ext = '.txt'; + end + % Check to see if we are looking at the most recent GPS file + gps_in_fns = get_filenames(gps_input_fn_dir,gps_input_fn_prefix,'',gps_input_fn_ext); + if ~isempty(gps_in_fns) + if ~strcmpi(gps_in_fn,gps_in_fns{end}) + % New GPS file so set the current gps file to the last in the + % list + gps_in_fn = gps_in_fns{end}; + nmea_dollarsign_synced = false; + + % Find the last dollar sign '$' in the file + fid = fopen(gps_in_fn,'r'); + fseek(fid,0,1); % Seek to end of file + gps_in_fn_pos = ftell(fid)-1; + while ~nmea_dollarsign_synced + if ftell(fid) == 0 + % Last GPS file does not contain '$', this may be because the + % file is just getting written to for the first time. Try to + % read the file again on the next time around. + break; + end + % Go back 1 character in the file + fseek(fid,-1,0); + % Read the character + A = fread(fid,1,'char'); + if A == '$' + % Found $ + gps_in_fn_pos = ftell(fid)-1; + nmea_dollarsign_synced = true; + break; + else + % Character is not $, so go back one more character fseek(fid,-1,0); - A = fread(fid,1,'char'); - if A == '$' - gps_in_fn_pos = ftell(fid)-1; - dollar_found = true; - else - fseek(fid,-1,0); - end end - fclose(fid); end - - if isfinite(gps_in_fn_pos) - % We have a valid file position handle - pause(0.5); - fid = fopen(gps_in_fn,'r'); - fseek(fid,gps_in_fn_pos,-1); - nmea_str = fgets(fid); - A = fread(fid,1,'char'); - if isempty(A) || A(1) ~= '$' + fclose(fid); + end + + fid = fopen(gps_in_fn,'r'); + fseek(fid,gps_in_fn_pos,-1); + if ~nmea_dollarsign_synced + % Search forward for a dollar sign character + while ~feof(fid) + test_char = fread(fid,1,'char'); + if test_char=='$' + nmea_dollarsign_synced = true; gps_in_fn_pos = ftell(fid)-1; + break; + end + end + end + if ~nmea_dollarsign_synced + pause(0.5); + else + % We have a valid file position handle pointing at the '$' + % character at the start of a NMEA string. + fseek(fid,gps_in_fn_pos,-1); + nmea_str = fgets(fid); + if nmea_str(end) ~= 10 && nmea_str(end) ~= 13 + % End of file since nmea_str is not terminated in a end of line + % character (10 or 13). + % End of file, try again on the next loop and maybe some new + % data has been written to the file by then. + A = {''}; + pause(0.5); + else + % fgets terminated in an end of line character + A = textscan(nmea_str,'%s%f%f%c%f%c%u%u%f%f%c%f%c%s%s%f%f%f%f','delimiter',',','emptyvalue',NaN); + A{1} = A{1}{1}; + + test_char = fread(fid,1,'char'); + if isempty(test_char) + % End of file, try again on the next loop and maybe some new + % data has been written to the file by then. + gps_in_fn_pos = ftell(fid); + nmea_dollarsign_synced = false; + pause(0.5); + elseif test_char ~= '$' + % Bad line, throw away previous result just in case. + gps_in_fn_pos = ftell(fid); + nmea_dollarsign_synced = false; A = {''}; else gps_in_fn_pos = ftell(fid)-1; - A = textscan(nmea_str,'%s%f%f%c%f%c%u%u%f%f%c%f%c%s%s%f%f%f%f','delimiter',',','emptyvalue',NaN); end - fclose(fid); - else - % We do not have valid data - A = {''}; end end + fclose(fid); end - - if strcmpi(A{1},'$GPGGA') - lat_deg = floor(A{3}/100); - lat_min = A{3}-lat_deg*100; - lat = lat_deg + lat_min/60; - if A{4} ~= 'N' - lat = -lat; - end - lon_deg = floor(A{5}/100); - lon_min = A{5}-lon_deg*100; - lon = lon_deg + lon_min/60; - if A{6} ~= 'E' - lon = -lon; - end - hour = floor(A{2}/1e4); - minute = floor((A{2}-hour*1e4)/1e2); - sec= A{2}-hour*1e4-minute*1e2; - - elev = A{10}; - - utc_time = datenum(year,month,day,hour,minute,sec); - day_start = datenum(year,month,day,0,0,0); - utc_sod = (utc_time - day_start)*86400; - - title(sprintf('UTC %02d:%02d:%05.2f, %.2f SOD, %.0f m/%.0f ft',hour,minute,sec, ... - utc_sod, elev, elev*100/2.54/12)); - - if isempty(lat) - lat = NaN; - end - if isempty(lon) - lon = NaN; - end - if isempty(elev) - elev = NaN; - end - time_buf(2:end) = time_buf(1:end-1); - time_buf(1) = utc_sod; - lat_buf(2:end) = lat_buf(1:end-1); - lat_buf(1) = lat; - lon_buf(2:end) = lon_buf(1:end-1); - lon_buf(1) = lon; - elev_buf(2:end) = elev_buf(1:end-1); - elev_buf(1) = elev; - along_track = geodetic_to_along_track(lat_buf([1 end]),lon_buf([1 end]),elev_buf([1 end])); - speed = along_track(2) / abs(diff(time_buf([1 end]))); - - fprintf('%7.1f | %9.6f N %11.6f E | %6.1f m | %3.0f m/s %3.0f kn\n', utc_sod, lat, lon, elev, speed, speed/0.5144444); - - if enable_gps_record - fprintf(fid_out,'%04d,%02d,%02d,%f,%f,%f,%f\n', year, month, day, utc_sod, lat, lon, elev); + end + + %% Loop for new data: Interpret NMEA string + % ===================================================================== + if ~strcmpi(gps_input_type,'file_novatelraw') ... + && length(A) >= 10 && strcmpi(A{1},'$GPGGA') + lat_deg = floor(A{3}/100); + lat_min = A{3}-lat_deg*100; + lat = lat_deg + lat_min/60; + if A{4} ~= 'N' + lat = -lat; + end + lon_deg = floor(A{5}/100); + lon_min = A{5}-lon_deg*100; + lon = lon_deg + lon_min/60; + if A{6} ~= 'E' + lon = -lon; + end + hour = floor(A{2}/1e4); + minute = floor((A{2}-hour*1e4)/1e2); + sec= A{2}-hour*1e4-minute*1e2; + + elev = A{10}; + + utc_time = datenum(year,month,day,hour,minute,sec); + day_start = datenum(year,month,day,0,0,0); + utc_sod = (utc_time - day_start)*86400; + + if ~isempty(lat) && ~isempty(lon) && ~isempty(elev) && ~isempty(utc_time) + gflight_tracker.lat(end+1) = lat; + gflight_tracker.lon(end+1) = lon; + gflight_tracker.elev(end+1) = elev; + if gps_input_geoid_en + gflight_tracker.elev(end) = gflight_tracker.elev(end) ... + + interp2(gflight_tracker.egm96_lon,gflight_tracker.egm96_lat,gflight_tracker.egm96_elev, ... + mod(gflight_tracker.lon(end),360),gflight_tracker.lat(end)); end - - [x,y] = projfwd(proj,lat,lon); - x = x/1e3; - y = y/1e3; - pos_buf(1,1) = x; - pos_buf(1,2) = y; - set(hline,'XData',pos_buf(:,1)); - set(hline,'YData',pos_buf(:,2)); - set(hpos,'XData',pos_buf(1,1)); - set(hpos,'YData',pos_buf(1,2)); + gflight_tracker.gps_time(end+1) = utc_time; + [x,y] = projfwd(gflight_tracker.proj,gflight_tracker.lat(end),gflight_tracker.lon(end)); + gflight_tracker.x(end+1) = x/1e3; + gflight_tracker.y(end+1) = y/1e3; + set(gflight_tracker.h_line,'XData',gflight_tracker.x,'YData',gflight_tracker.y); + set(gflight_tracker.h_plot,'XData',gflight_tracker.x(end),'YData',gflight_tracker.y(end)); drawnow; - pos_buf(2:end,:) = pos_buf(1:end-1,:); + end + + % if length(lat) > 10 + % along_track = geodetic_to_along_track(lat(end-9:end),lon(end-9:end),elev(end-9:end)); + % speed = along_track(end) / abs(utc_time(end)-utc_time(end-9)); + % end + % fprintf('%9.6f N %11.6f E | %6.1f m | %6.1f m AGL | %3.0f m/s %3.0f kn\n', lat, lon, elev, elev-dem_elev, speed, speed/0.5144444); + + if gps_record_en + fprintf(fid_out,'%04d,%02d,%02d,%f,%f,%f,%f\n', year, month, day, utc_sod, lat, lon, elev); + end + end + + %% Loop for new data: DEM profile plot and y-z offset plot + % ===================================================================== + if exist('gps_desire','var') + if isfinite(debug_start_line) + % finite debug_start_line implies we are in debug mode. Simulate + % the regular delay between GPS updates (1 second). + update_plot_delay = toc(update_plot_tstart); + update_plot_tstart = tic; + if update_plot_delay < 1 + fprintf('Delaying %.3f sec\n', update_plot_delay); + pause(1 - update_plot_delay); + end + end + num_pnts = 19; + gps_time = gflight_tracker.gps_time(end-num_pnts:end); + lat = gflight_tracker.lat(end-num_pnts:end); + lon = gflight_tracker.lon(end-num_pnts:end); + elev = gflight_tracker.elev(end-num_pnts:end); + + % Generate heading from last end-10:end-5 trajectory positions + % Generate heading from last end-5:end trajectory positions + [ecef_x,ecef_y,ecef_z] = geodetic2ecef(lat/180*pi,lon/180*pi,elev, WGS84.ellipsoid); + + future_x = [ecef_x(end-3:end-1), ecef_x(end) + (ecef_x(end)-ecef_x(end-1))*(0:num_pnts)/1]; + future_y = [ecef_y(end-3:end-1), ecef_y(end) + (ecef_y(end)-ecef_y(end-1))*(0:num_pnts)/1]; + future_z = [ecef_z(end-3:end-1), ecef_z(end) + (ecef_z(end)-ecef_z(end-1))*(0:num_pnts)/1]; + + % Find closest point on previous trajectory + dist = sqrt((gps_desire.ecef_x-ecef_x(end)).^2+(gps_desire.ecef_y-ecef_y(end)).^2+(gps_desire.ecef_z-ecef_z(end)).^2); + [min_dist,min_idx] = min(dist); + [up.ecef_x,up.ecef_y,up.ecef_z] = geodetic2ecef(gps_desire.lat(min_idx)/180*pi,gps_desire.lon(min_idx)/180*pi,gps_desire.elev(min_idx)+1, WGS84.ellipsoid); + + set(gflight_tracker.h_plot_dem_all,'XData',gps_desire.along_track(min_idx)/1852*[1 1], ... + 'YData', [-50e3 50e3]); + + % Generate FCS on previous trajectory using next N_lookahead positions + N_lookahead = 1; + fcs_x = [gps_desire.ecef_x(min_idx+N_lookahead) - gps_desire.ecef_x(min_idx); + gps_desire.ecef_y(min_idx+N_lookahead) - gps_desire.ecef_y(min_idx); + gps_desire.ecef_z(min_idx+N_lookahead) - gps_desire.ecef_z(min_idx)]; + fcs_z = [up.ecef_x(1) - gps_desire.ecef_x(min_idx); + up.ecef_y(1) - gps_desire.ecef_y(min_idx); + up.ecef_z(1) - gps_desire.ecef_z(min_idx)]; + fcs_x = fcs_x / sqrt(dot(fcs_x,fcs_x)); + fcs_z = fcs_z - fcs_x*dot(fcs_z,fcs_x); + fcs_z = fcs_z / sqrt(dot(fcs_z,fcs_z)); + fcs_y = cross(fcs_z, fcs_x); + T = [fcs_x fcs_y fcs_z]; + % Project current position into FCS x,y,z + offset = T \ [future_x - gps_desire.ecef_x(min_idx); + future_y - gps_desire.ecef_y(min_idx); + future_z - gps_desire.ecef_z(min_idx)]; + + set(gflight_tracker.h_plot_yz1,'XData',-offset(2,4:12)*100/12/2.54,'YData',offset(3,4:12)*100/12/2.54); + set(gflight_tracker.h_plot_yz3,'XData',-offset(2,1:4)*100/12/2.54,'YData',offset(3,1:4)*100/12/2.54); + set(gflight_tracker.h_plot_yz2,'XData',-offset(2,4)*100/12/2.54,'YData',offset(3,4)*100/12/2.54); + + speed = sqrt((ecef_x(end)-ecef_x(end-1)).^2+(ecef_y(end)-ecef_y(end-1)).^2+(ecef_z(end)-ecef_z(end-1)).^2) / (gps_time(end)-gps_time(end-1)) / 86400; + + distance_to_end = gps_desire.along_track(end) - gps_desire.along_track(min_idx); + distance_to_next = gps_desire.along_track(min_idx - 1 + find(gps_desire.orig_pnt(min_idx:end),1)) - gps_desire.along_track(min_idx); + + % if min_dist > 1000 + % title(sprintf('Minimum distance between desired and current trajectory: %g km\n', min_dist/1e3),'parent',gflight_tracker.h_axes_yz); + % else + % bearing: required change in bearing to get back on the line + % within 9 seconds + bearing = atan2d(offset(2,4)/9,speed) - atan2d(diff(-offset(2,3:4)),speed); + % climb_ft_min: required change in climb to get back on the line + % within 9 seconds + climb_ft_min = -offset(3,4)*100/2.54/12/9*60 - diff(offset(3,3:4))*100/2.54/12*60; + % ft/min + % max_climb_ft_min: maximum climb rate for the given prf, presums, + % and wavelength + prf = 10000; + presums = 38; + freq = 195e6; + max_climb_ft_min = 45/360 * 3e8/2/freq * (prf/presums)*100/12/2.54 * 60; + if abs(climb_ft_min) > max_climb_ft_min + climb_ft_min = max_climb_ft_min * sign(climb_ft_min); + end + title(sprintf('Bearing: %+5.1f deg\nClimb: %+5.1f ft/min\nY: %4.0f ft Z: %4.0f ft\n DTE %5.1f nm DTN: %5.1f nm', ... + bearing, climb_ft_min, -offset(2,4)*100/2.54/12, offset(3,4)*100/2.54/12, distance_to_end/1852, distance_to_next/1852), 'parent',gflight_tracker.h_axes_yz,'FontSize',24,'FontWeight','bold','color','white'); + % end + % Implement hysteresis on the autoscale of the y-z plot + xlims_tight = 50*2^(max(0,ceil(log2(max(abs(offset(2,4)))/50/0.5*100/2.54/12)))); + ylims_tight = 50*2^(max(0,ceil(log2(max(abs(offset(3,4)))/50/0.5*100/2.54/12)))); + xlims = 50*2^(max(0,ceil(log2(max(abs(offset(2,4)))/50/0.8*100/2.54/12)))); + ylims = 50*2^(max(0,ceil(log2(max(abs(offset(3,4)))/50/0.8*100/2.54/12)))); + if xlims_old > xlims_tight + xlims_old = xlims_tight; + elseif xlims_old < xlims + xlims_old = xlims; + end + if ylims_old > ylims_tight + ylims_old = ylims_tight; + elseif ylims_old < ylims + ylims_old = ylims; + end + xlim(gflight_tracker.h_axes_yz, xlims_old*[-1 1]); + ylim(gflight_tracker.h_axes_yz, ylims_old*[-1 1]); + + [future_lat,future_lon,future_elev] = ecef2geodetic(future_x,future_y,future_z,WGS84.ellipsoid); + future_lat = future_lat*180/pi; + future_lon = future_lon*180/pi; + + [future_dem_x,future_dem_y] = projfwd(gflight_tracker.dem_proj,future_lat,future_lon); + future_dem_elev = interp2(gflight_tracker.dem_x_axis,gflight_tracker.dem_y_axis,gflight_tracker.dem_RGB,future_dem_x,future_dem_y); + future_dem_elev_ft = future_dem_elev*100/2.54/12; + + future_elev_ft = future_elev*100/2.54/12; + future_elev_agl_ft = (future_elev_ft - future_dem_elev_ft); + if any(future_elev_agl_ft < 700) + set(gflight_tracker.h_plot_dem1,'linewidth',4,'color','red'); + set(gflight_tracker.h_plot_dem2,'linewidth',4,'color','white'); + else + set(gflight_tracker.h_plot_dem1,'linewidth',2,'color','white'); + set(gflight_tracker.h_plot_dem2,'linewidth',2,'color','white'); + end + set(gflight_tracker.h_plot_dem1,'XData',-3:num_pnts,'YData',future_elev_ft); + set(gflight_tracker.h_plot_dem2,'XData',-3:num_pnts,'YData',future_dem_elev_ft); + ylims(1) = min([future_elev_ft,future_dem_elev_ft]); + ylims(2) = max([future_elev_ft,future_dem_elev_ft]); + ylim(gflight_tracker.h_axes_dem, ylims); + + [year,month,day,hour,minute,sec] = datevec(epoch_to_datenum(gflight_tracker.gps_time(end) - utc_leap_seconds(gflight_tracker.gps_time(end)))); + title(sprintf('UTC %02d:%02d:%05.2f %.0f m/%.0f ft, %.0f m / %.0f ft AGL',hour,minute,sec, ... + elev(end), elev(end)*100/2.54/12, elev(end)-future_dem_elev(4), (elev(end)-future_dem_elev(4))*100/2.54/12),'parent',gflight_tracker.h_axes,'color','white'); + end + + %% Loop for new data: Update Map Axes to Show Current Position + % ===================================================================== + xlim_orig = xlim(gflight_tracker.h_axes); + ylim_orig = ylim(gflight_tracker.h_axes); + axis normal; + % Force axis to be equal + xlim_new = xlim(gflight_tracker.h_axes); + ylim_new = ylim(gflight_tracker.h_axes); + axes_pos = get(gflight_tracker.h_axes,'position'); aspect_ratio = axes_pos(3)/axes_pos(4); + xlim_bigger = diff(xlim_new)/diff(ylim_new)/aspect_ratio > 1; + if xlim_bigger + ylim_new(1) = mean(ylim_new) - diff(xlim_new)/2*aspect_ratio; + ylim_new(2) = mean(ylim_new) + diff(xlim_new)/2*aspect_ratio; + else + xlim_new(1) = mean(xlim_new) - diff(ylim_new)/2/aspect_ratio; + xlim_new(2) = mean(xlim_new) + diff(ylim_new)/2/aspect_ratio; + end + if toc(update_geotiff_tstart) > 0 + update_geotiff_tstart = tic; + if isfinite(gflight_tracker.x(end)) && isfinite(gflight_tracker.y(end)) + % Update to current platform position + xlim_new = xlim_new + gflight_tracker.x(end) - mean(xlim_new); + ylim_new = ylim_new + gflight_tracker.y(end) - mean(ylim_new); end end -catch ME - ME - ME.stack(1) + xlim(gflight_tracker.h_axes,xlim_new); + ylim(gflight_tracker.h_axes,ylim_new); + clear xlim_orig ylim_orig xlim_new ylim_new axes_pos xlim_bigger + + %% Force redraw + drawnow; end +% end +%% Cleanup if strcmpi(gps_input_type,'serial') fprintf('Closing serial device %s\n', serial_dev); - fclose(flight_tracker_serial_dev); + fclose(gflight_tracker.serial_dev); end -if enable_gps_record +if gps_record_en fprintf('Closing gps log file %s\n', gps_fn); fclose(fid_out); end - -return; diff --git a/cresis-toolbox/gps/geodetic_to_along_track.m b/cresis-toolbox/gps/geodetic_to_along_track.m index ad6f5864..870d49f3 100644 --- a/cresis-toolbox/gps/geodetic_to_along_track.m +++ b/cresis-toolbox/gps/geodetic_to_along_track.m @@ -1,168 +1,168 @@ -function [along_track,lat_filt,lon_filt,elev_filt] = geodetic_to_along_track(lat,lon,elev,spacing) -% [along_track,lat_filt,lon_filt,elev_filt] = geodetic_to_along_track(lat,lon,elev,spacing) -% -% Converts geodetic (lat,lon,elev) in WGS-84 into along-track. The distance -% between each point is cumulated to create the along_track vector. -% Used by functions like csarp.m. -% -% lat,lon in degrees -% elev in meters (optional, filled in with zeros if left empty or undefined) -% spacing: optional scalar argument. When specified and not empty, the geodetic -% coordinates are decimated to a minimum along track spacing specified -% by "spacing" using the get_equal_alongtrack_spacing_idxs.m function -% and then all the samples are filled in using these decimated indices. -% This is better for SAR processing and helps deal with the case when -% gps errors are on the order of the sample spacing (e.g. due to very -% slow moving trajectory). -% -% Author: John Paden -% -% See also: geodetic_to_along_track.m, get_equal_alongtrack_spacing_idxs.m - -% Load WGS84 ellipsoid -physical_constants; - -if isstruct(lat) && isfield(lat,'Latitude') - elev = lat.Elevation; - lon = lat.Longitude; - lat = lat.Latitude; -end -if isstruct(lat) && isfield(lat,'lat') - elev = lat.elev; - lon = lat.lon; - lat = lat.lat; -end - -if ~exist('elev','var') || isempty(elev) - elev = zeros(size(lat)); -end - -if isempty(lat) - along_track = []; - return; -elseif length(lat) == 1 - along_track = 0; - return; -end - -if ~exist('spacing','var') || isempty(spacing) - % Cumulative sum for each step of motion - [x,y,z] = geodetic2ecef(lat/180*pi,lon/180*pi,elev,WGS84.ellipsoid); - if size(x,1) > 1 - along_track = [0; cumsum(sqrt(diff(x).^2 + diff(y).^2 + diff(z).^2))]; - else - along_track = [0 cumsum(sqrt(diff(x).^2 + diff(y).^2 + diff(z).^2))]; - end - lat_filt = lat; - lon_filt = lon; - elev_filt = elev; -else - % Find the decimation indices - % Even indices are the center of the sections that we will use - % Odd indices define the boundaries of which points contribute to the - % even points during averaging - decim_idxs = get_equal_alongtrack_spacing_idxs( ... - struct('lat',lat,'lon',lon,'elev',elev),spacing/2); - physical_constants; - [ecef(1,:),ecef(2,:),ecef(3,:)] = geodetic2ecef(lat/180*pi,lon/180*pi,elev,WGS84.ellipsoid); - if length(decim_idxs) < 2 - decim_idxs = [1 length(lat)]; - end - % If odd length, then just truncate to even length - if mod(length(decim_idxs),2) - decim_idxs = decim_idxs(1:end-1); - end - - %% Preallocate outputs - along_track = zeros(size(lat)); - filt_ecef = zeros(3,numel(lat)); - - %% Find averaged positions for each section - pnt_ecef = zeros(3,length(decim_idxs)); - for decim_idxs_idx = 2:2:length(decim_idxs) - % Start index of averaging - start_idx = decim_idxs(decim_idxs_idx-1); - - % Stop index of averaging - if decim_idxs_idx >= length(decim_idxs)-1 - stop_idx = length(lat); - else - stop_idx = decim_idxs(decim_idxs_idx+1)-1; - end - - pnt_ecef(:,decim_idxs_idx) = mean(ecef(:,start_idx:stop_idx),2); - end - - %% Find along-track value for every point for each section - for decim_idxs_idx = 2:2:length(decim_idxs) - - % Determine range of along-track positions we will be filling in - if decim_idxs_idx == 2 - start_idx = 1; - else - start_idx = decim_idxs(decim_idxs_idx); - end - if decim_idxs_idx == length(decim_idxs) - stop_idx = length(lat); - else - stop_idx = decim_idxs(decim_idxs_idx+2); - end - - % Determine which points will be used for the line fit - if length(decim_idxs) >= 4 - if decim_idxs_idx == length(decim_idxs) - % Last point does not have a next point, so we use the previous - % point instead - start_pnt = decim_idxs_idx-2; - stop_pnt = decim_idxs_idx; - else - start_pnt = decim_idxs_idx; - stop_pnt = decim_idxs_idx+2; - end - % The origin and vector produce a line connecting two averaged positions - ref_orig = [pnt_ecef(1,decim_idxs_idx); ... - pnt_ecef(2,decim_idxs_idx); pnt_ecef(3,decim_idxs_idx)]; - ref_vector = [pnt_ecef(1,stop_pnt)-pnt_ecef(1,start_pnt); ... - pnt_ecef(2,stop_pnt)-pnt_ecef(2,start_pnt); pnt_ecef(3,stop_pnt)-pnt_ecef(3,start_pnt)]; - else - % This is a short track with only two points after decimation - % (i.e. length(decim_idxs) == 2) - - % For the origin, we use the averaged position - start_pnt = decim_idxs_idx; - ref_orig = [pnt_ecef(1,decim_idxs_idx); ... - pnt_ecef(2,decim_idxs_idx); pnt_ecef(3,decim_idxs_idx)]; - - % For the direction vector, we just use the first and last point - start_pnt = 1; - stop_pnt = length(lat); - ref_vector = [ecef(1,stop_pnt)-ecef(1,start_pnt); ... - ecef(2,stop_pnt)-ecef(2,start_pnt); ecef(3,stop_pnt)-ecef(3,start_pnt)]; - ref_vector = ref_vector./sqrt(dot(ref_vector,ref_vector)); - end - ref_vector = ref_vector./sqrt(dot(ref_vector,ref_vector)); - - % Find the closest point on the line joining the two ~closest reference points - % to the current point - % See equation 21.4.16 from section "Lines in Three Dimensions", Numerical Recipes - - % ref_orig (a) is origin of line joining two closest reference points - % ref_vector (v) is unit vector connecting two closest reference points - - idxs = start_idx:stop_idx; - offset_from_origin = dot([ecef(1,idxs)-ref_orig(1); ecef(2,idxs)-ref_orig(2); ecef(3,idxs)-ref_orig(3)], ... - repmat(ref_vector,[1 length(idxs)])); - filt_ecef(:,idxs) = bsxfun(@plus,ref_orig,bsxfun(@times,offset_from_origin,ref_vector)); - if start_idx == 1 - along_track(idxs) = offset_from_origin - offset_from_origin(1); - else - along_track(idxs) = along_track(start_idx) + offset_from_origin - offset_from_origin(1); - end - end - [lat_filt,lon_filt,elev_filt] = ecef2geodetic(filt_ecef(1,:),filt_ecef(2,:),filt_ecef(3,:),WGS84.ellipsoid); - lat_filt = lat_filt*180/pi; - lon_filt = lon_filt*180/pi; - -end - -return; +function [along_track,lat_filt,lon_filt,elev_filt] = geodetic_to_along_track(lat,lon,elev,spacing) +% [along_track,lat_filt,lon_filt,elev_filt] = geodetic_to_along_track(lat,lon,elev,spacing) +% +% Converts geodetic (lat,lon,elev) in WGS-84 into along-track. The distance +% between each point is cumulated to create the along_track vector. +% Used by functions like csarp.m. +% +% lat,lon in degrees +% elev in meters (optional, filled in with zeros if left empty or undefined) +% spacing: optional scalar argument. When specified and not empty, the geodetic +% coordinates are decimated to a minimum along track spacing specified +% by "spacing" using the get_equal_alongtrack_spacing_idxs.m function +% and then all the samples are filled in using these decimated indices. +% This is better for SAR processing and helps deal with the case when +% gps errors are on the order of the sample spacing (e.g. due to very +% slow moving trajectory). +% +% Author: John Paden +% +% See also: geodetic_to_along_track.m, get_equal_alongtrack_spacing_idxs.m + +% Load WGS84 ellipsoid +physical_constants; + +if isstruct(lat) && isfield(lat,'Latitude') + elev = lat.Elevation; + lon = lat.Longitude; + lat = lat.Latitude; +end +if isstruct(lat) && isfield(lat,'lat') + elev = lat.elev; + lon = lat.lon; + lat = lat.lat; +end + +if ~exist('elev','var') || isempty(elev) + elev = zeros(size(lat)); +end + +if isempty(lat) + along_track = []; + return; +elseif length(lat) == 1 + along_track = 0; + return; +end + +if ~exist('spacing','var') || isempty(spacing) + % Cumulative sum for each step of motion + [x,y,z] = geodetic2ecef(lat/180*pi,lon/180*pi,elev,WGS84.ellipsoid); + if size(x,1) > 1 + along_track = [0; cumsum(sqrt(diff(x).^2 + diff(y).^2 + diff(z).^2))]; + else + along_track = [0 cumsum(sqrt(diff(x).^2 + diff(y).^2 + diff(z).^2))]; + end + lat_filt = lat; + lon_filt = lon; + elev_filt = elev; +else + % Find the decimation indices + % Even indices are the center of the sections that we will use + % Odd indices define the boundaries of which points contribute to the + % even points during averaging + decim_idxs = get_equal_alongtrack_spacing_idxs( ... + struct('lat',lat,'lon',lon,'elev',elev),spacing/2); + physical_constants; + [ecef(1,:),ecef(2,:),ecef(3,:)] = geodetic2ecef(lat/180*pi,lon/180*pi,elev,WGS84.ellipsoid); + if length(decim_idxs) < 2 + decim_idxs = [1 length(lat)]; + end + % If odd length, then just truncate to even length + if mod(length(decim_idxs),2) + decim_idxs = decim_idxs(1:end-1); + end + + %% Preallocate outputs + along_track = zeros(size(lat)); + filt_ecef = zeros(3,numel(lat)); + + %% Find averaged positions for each section + pnt_ecef = zeros(3,length(decim_idxs)); + for decim_idxs_idx = 2:2:length(decim_idxs) + % Start index of averaging + start_idx = decim_idxs(decim_idxs_idx-1); + + % Stop index of averaging + if decim_idxs_idx >= length(decim_idxs)-1 + stop_idx = length(lat); + else + stop_idx = decim_idxs(decim_idxs_idx+1)-1; + end + + pnt_ecef(:,decim_idxs_idx) = mean(ecef(:,start_idx:stop_idx),2); + end + + %% Find along-track value for every point for each section + for decim_idxs_idx = 2:2:length(decim_idxs) + + % Determine range of along-track positions we will be filling in + if decim_idxs_idx == 2 + start_idx = 1; + else + start_idx = decim_idxs(decim_idxs_idx); + end + if decim_idxs_idx == length(decim_idxs) + stop_idx = length(lat); + else + stop_idx = decim_idxs(decim_idxs_idx+2); + end + + % Determine which points will be used for the line fit + if length(decim_idxs) >= 4 + if decim_idxs_idx == length(decim_idxs) + % Last point does not have a next point, so we use the previous + % point instead + start_pnt = decim_idxs_idx-2; + stop_pnt = decim_idxs_idx; + else + start_pnt = decim_idxs_idx; + stop_pnt = decim_idxs_idx+2; + end + % The origin and vector produce a line connecting two averaged positions + ref_orig = [pnt_ecef(1,decim_idxs_idx); ... + pnt_ecef(2,decim_idxs_idx); pnt_ecef(3,decim_idxs_idx)]; + ref_vector = [pnt_ecef(1,stop_pnt)-pnt_ecef(1,start_pnt); ... + pnt_ecef(2,stop_pnt)-pnt_ecef(2,start_pnt); pnt_ecef(3,stop_pnt)-pnt_ecef(3,start_pnt)]; + else + % This is a short track with only two points after decimation + % (i.e. length(decim_idxs) == 2) + + % For the origin, we use the averaged position + start_pnt = decim_idxs_idx; + ref_orig = [pnt_ecef(1,decim_idxs_idx); ... + pnt_ecef(2,decim_idxs_idx); pnt_ecef(3,decim_idxs_idx)]; + + % For the direction vector, we just use the first and last point + start_pnt = 1; + stop_pnt = length(lat); + ref_vector = [ecef(1,stop_pnt)-ecef(1,start_pnt); ... + ecef(2,stop_pnt)-ecef(2,start_pnt); ecef(3,stop_pnt)-ecef(3,start_pnt)]; + ref_vector = ref_vector./sqrt(dot(ref_vector,ref_vector)); + end + ref_vector = ref_vector./sqrt(dot(ref_vector,ref_vector)); + + % Find the closest point on the line joining the two ~closest reference points + % to the current point + % See equation 21.4.16 from section "Lines in Three Dimensions", Numerical Recipes + + % ref_orig (a) is origin of line joining two closest reference points + % ref_vector (v) is unit vector connecting two closest reference points + + idxs = start_idx:stop_idx; + offset_from_origin = dot([ecef(1,idxs)-ref_orig(1); ecef(2,idxs)-ref_orig(2); ecef(3,idxs)-ref_orig(3)], ... + repmat(ref_vector,[1 length(idxs)])); + filt_ecef(:,idxs) = bsxfun(@plus,ref_orig,bsxfun(@times,offset_from_origin,ref_vector)); + if start_idx == 1 + along_track(idxs) = offset_from_origin - offset_from_origin(1); + else + along_track(idxs) = along_track(start_idx) + offset_from_origin - offset_from_origin(1); + end + end + [lat_filt,lon_filt,elev_filt] = ecef2geodetic(filt_ecef(1,:),filt_ecef(2,:),filt_ecef(3,:),WGS84.ellipsoid); + lat_filt = lat_filt*180/pi; + lon_filt = lon_filt*180/pi; + +end + +return; diff --git a/cresis-toolbox/gps/geotiff_plot.m b/cresis-toolbox/gps/geotiff_plot.m index 6bf2b6b9..7f2d8d0e 100644 --- a/cresis-toolbox/gps/geotiff_plot.m +++ b/cresis-toolbox/gps/geotiff_plot.m @@ -1,5 +1,5 @@ -function [proj,h_fig] = plot_geotiff(geotiff_fn, lat, lon, h_fig, varargin) -% [proj,h_fig] = plot_geotiff(geotiff_fn, lat, lon, h_fig, varargin) +function [proj,h_fig,h_axes,h_image,h_plot,h_plot_start] = geotiff_plot(geotiff_fn, lat, lon, h_fig, varargin) +% [proj,h_fig,h_axes,h_image,h_plot,h_plot_start] = geotiff_plot(geotiff_fn, lat, lon, h_fig, varargin) % % Function for quickly plotting a geotiff and optionally plotting % some points/vectors on top of that. The first point is plotted @@ -17,13 +17,13 @@ % % Examples: % geotiff_fn = ct_filename_gis('greenland/Landsat-7/mzl7geo_90m_lzw.tif'); -% plot_geotiff(geotiff_fn); +% geotiff_plot(geotiff_fn); % % geotiff_fn = ct_filename_gis('world/NaturalEarth/NE2_HR_LC_SR_W_DR.tif'); -% plot_geotiff(geotiff_fn); +% geotiff_plot(geotiff_fn); % % geotiff_fn = ct_filename_gis('antarctica/Landsat-7/Antarctica_LIMA_480m.tif'); -% plot_geotiff(geotiff_fn,Latitude,Longitude,'b-'); +% geotiff_plot(geotiff_fn,Latitude,Longitude,'b-'); % % Author: John Paden @@ -73,12 +73,14 @@ % DEBUG RGB(RGB<0) = NaN; end -imagesc(R(3,1) + R(2,1)*(1:size(RGB,2)), R(3,2) + R(1,2)*(1:size(RGB,1)), RGB,'parent',h_axes); +h_image = imagesc(R(3,1) + R(2,1)*(1:size(RGB,2)), R(3,2) + R(1,2)*(1:size(RGB,1)), RGB,'parent',h_axes); if isa(RGB,'uint8') colormap(h_axes,gray(256)); end set(h_axes,'YDir','normal'); +h_plot = []; +h_plot_start = []; if exist('lat','var') && ~isempty(lat) switch lower(proj.ModelType) case {'modeltypeprojected'} @@ -91,11 +93,9 @@ end cur_hold = ishold(h_axes); hold(h_axes,'on'); - plot(h_axes,X,Y,varargin{:}); - plot(h_axes,X(1),Y(1),'bo'); + h_plot = plot(h_axes,X,Y,varargin{:}); + h_plot_start = plot(h_axes,X(1),Y(1),'bo'); if ~cur_hold hold(h_axes,'off'); end end - -return; diff --git a/cresis-toolbox/gps/google_map.m b/cresis-toolbox/gps/google_map.m index e0b02e29..510cdd13 100644 --- a/cresis-toolbox/gps/google_map.m +++ b/cresis-toolbox/gps/google_map.m @@ -1,16 +1,70 @@ classdef google_map % google_map class % + % ----------------------------------------------------------------------- + % Google API Key + % + % A key is required to access Google's API. Without a key, map requests + % cannot be completed. + % + % This class uses Google's "Maps Static API. Simple, embeddable map image + % with minimal code." + % + % To obtain a google API key: + % + % 1. https://developers.google.com/maps/ and login with a Google account. + % + % 2. Click "Get Started" + % + % 3. Answer all of the questions, but you will need to provide Google + % with a valid payment method (e.g. credit card). + % + % 4. At the end copy the "API Key". It should look something like this: + % AIzaSyCNexiP6WcIda8ZEa2MnwznWrGotDoLu0w + % If you need to find your key in the future, it should be under the + % "Credentials" tab. + % + % 5. If you keep your API key private, then you can manually ensure that + % you never go over the limit of free requests. Right now the limit is + % 100,000 requests per month. This was calculated based on current + % pricing. Google charges $2 per 1000 requests and also does not charge + % you unless there is $200 of usage in a month. + % + % 6. If you may not be able to keep your API key private or are using the + % API key in automated scripts that could potentially exceed 100000 + % requests in a month, then you can restrict the API key so that you + % never are charged in this way: + % + % 1. Go to "Credentials" tab + % + % 2. Select the specific API key + % + % 3. Set "API restrictions" to "Restrict key" to "Maps Static API" + % + % 4. Go to "Quotas" tab + % + % 5. Select the "Maps Static API" + % + % 6. For every category set "... requests ... per day" to 3000. By + % setting to 3225, you will ensure that you can never exceed 100000 + % requests in a month. NOTE: THIS IS NOT VERIFIED TO WORK AS INTENDED. + % + % ----------------------------------------------------------------------- % World coordinates description: - % x ranges from 0 to 256 and corresponds linearly to -180 to +180 - % longitude - % y ranges from 0 to 256 and corresponds to -85.051128779806632 to +85.051128779806632 + % + % x ranges from 0 to 256 and corresponds linearly to -180 to +180 + % longitude + % + % y ranges from 0 to 256 and corresponds to -85.051128779806632 to + % +85.051128779806632 + % + % https://developers.google.com/maps/documentation/javascript/coordinates#world-coordinates % % Authors: Rohan Choudhari, John Paden %% public properties properties - key = 'AIzaSyCNexiP6WcIda8ZEa2MnwznWrGotDoLu0w'; + key; tile_size = 256; w = 1280; h = 1280; @@ -24,8 +78,11 @@ methods %% google_map constructor - function obj = google_map(data) - obj.wms_obj = WebMapServer('http://maps.googleapis.com/maps/api'); + % key: key is a string containing the user's Google API key (see info + % above about obtaining a key) + function obj = google_map(key) + obj.wms_obj = WebMapServer('https://maps.googleapis.com/maps/api'); + obj.key = key; end %% google_map destructor @@ -116,7 +173,7 @@ function delete(obj) % Request PNG map (PNG is the default format) % - Only the first six digits of precision for lon and lat are used % by Google - url = sprintf('http://maps.googleapis.com/maps/api/staticmap?center=%0.6f,%0.6f&zoom=%d&scale=%d&size=%dx%d&maptype=satellite&key=%s', ... + url = sprintf('https://maps.googleapis.com/maps/api/staticmap?center=%0.6f,%0.6f&zoom=%d&scale=%d&size=%dx%d&maptype=satellite&key=%s', ... c_lat, c_lon, zoom, obj.scale, obj.w, obj.h, obj.key); A = obj.wms_obj.getMap(url); end @@ -151,6 +208,7 @@ function delete(obj) wc_x(idx) = 256*(0.5 + lon(idx)/360); wc_y(idx) = 256*(0.5 - log((1 + siny)/(1 - siny))/(4*pi)); end + wc_x = mod(wc_x,256); end %% world_to_latlon diff --git a/cresis-toolbox/gps/gps_check.m b/cresis-toolbox/gps/gps_check.m new file mode 100644 index 00000000..f14643d6 --- /dev/null +++ b/cresis-toolbox/gps/gps_check.m @@ -0,0 +1,341 @@ +function gps_check(param,param_override) +% gps_check(param,param_override) +% +% Checks the fields in the gps files for potential errors. +% Use run_all_gps_check to run on all seasons. +% +% To concatenate and look at all the outputs in Linux: +% grep "^" `find /cresis/snfs1/dataproducts/ct_data/ct_tmp/gps_check -iname "*output*" -print | sort` | less +% +% param = parameter structure indicating which radar and season to load. +% This causes the standard parameter spreadsheet filename to be created +% and loaded. Therefore it will not work if you don't have the parameter +% spreadsheet in the standard location with the standard name. +% ALTERNATE MODE: pass in a string containing the filename of a specific +% gps to load +% +% Example: +% +% % csarp_support/gps filename +% gps_check('/cresis/snfs1/dataproducts/csarp_support/gps/2019_Antarctica_Ground/gps_20200106.mat'); +% gps_check('/cresis/snfs1/dataproducts/csarp_support/gps/2019_Antarctica_Ground/gps_20200107.mat'); +% +% % Parameter structure +% param = []; +% param.radar_name = 'rds'; +% param.season_name = '2019_Antarctica_Ground'; +% gps_check(param) +% +% Author: John Paden +% +% See also: check_data_products, frames_check, gps_check, records_check +% gps_load, gps_create +% run_all_frames_check, run_all_gps_check, run_all_records_check + + +%% gps_check ============================================================== + +%% Input checks +global gRadar; +if exist('param_override','var') + param_override = merge_structs(gRadar,param_override); +else + param_override = gRadar; +end + +old_time = [-inf -inf]; +if ischar(param) + %% param is a string containing the filename + % ======================================================================= + gps_fn = param; + + gps = gps_load(gps_fn); + param = []; + param.season_name = gps.season_name; + param.gps_check.date_str = gps.date_str; + + fprintf('%d of %d: Checking gps %s\n', 1, 1, gps_fn); + gps_check_support_func(gps_fn,param,old_time); + +elseif isstruct(param) + %% param is a struct + % ======================================================================= + % struct indicates radar/season to check (checks all segments) + [output_dir,radar_type] = ct_output_dir(param.radar_name); + + param_fn = ct_filename_param(sprintf('%s_param_%s.xls', output_dir, param.season_name)); + params = read_param_xls(param_fn); + + for param_idx = 1:length(params) + param = params(param_idx); + + % Skip segment if the same day as the last segment or if set as "do not + % process" + if param_idx > 1 && strcmp(param.day_seg(1:8),params(param_idx-1).day_seg(1:8)) ... + || ~isempty(regexpi(param.cmd.notes,'do not process')) + continue; + end + + % DEBUG OPTION TO JUST CHECK GPS BASED ON GENERIC COLUMN IN SPREADSHEET + % Uses the generic column of the parameter spreadsheet to determine which segments + % to check. + %if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic + % continue; + %end + + if ~isfield(param.records.gps,'fn') + param.records.gps.fn = ''; + end + gps_fn = ct_filename_support(param,param.records.gps.fn,'gps',true); + + fprintf('%d of %d: Checking gps %s\n', param_idx, length(params), gps_fn); + + param = merge_structs(param,param_override); + + param.gps_check.date_str = param.day_seg(1:8); + + old_time = gps_check_support_func(gps_fn,param,old_time); + end + +else + error('Invalid argument') +end + +end + +%% gps_check_support_func ============================================= +function old_time = gps_check_support_func(gps_fn,param,old_time) +% gps_check_support_func(gps_fn,param,old_time) +% +% Support function which does the actual checking of the gps + +%% input checks +% ========================================================================= + +command_window_out_fn = ct_filename_ct_tmp(ct_rmfield(param,{'day_seg','radar_name'}),'','gps_check', sprintf('output_%s.txt',param.gps_check.date_str)); +command_window_out_fn_dir = fileparts(command_window_out_fn); +if ~exist(command_window_out_fn_dir,'dir') + mkdir(command_window_out_fn_dir); +end +fid = fopen(command_window_out_fn,'wb'); +fprintf(' Console output: %s\n', command_window_out_fn); +fprintf(fid, '%s\n', param.gps_check.date_str); + +% fixable_error: Set to true if a fixable error was found in which case the +% gps file will be updated at the end of this function. +fixable_error = false; + +% gps_source_check: logical scalar, default true, if true prints non-final GPS source error +if ~isfield(param.gps_check,'gps_source_check') || isempty(param.gps_check.gps_source_check) + param.gps_check.gps_source_check = true; +end + +% roll_limit: positive numeric scalar, default 25, units deg, threshold for +% bad roll +if ~isfield(param.gps_check,'roll_limit') || isempty(param.gps_check.roll_limit) + param.gps_check.roll_limit = 25; +end + +if ~exist(gps_fn,'file') + warning('Missing gps file %s\n', gps_fn); + fclose(fid); + return; +end + +gps = gps_load(gps_fn); + +if old_time(2) > gps.gps_time(1) + warning('gps out of order (meaning that the previous day has a gps time (%s to %s) that is greater than the start of this day (%s to %s), but the current day is later and must be listed chronologically)', ... + datestr(epoch_to_datenum(old_time(1))), datestr(epoch_to_datenum(old_time(end))), ... + datestr(epoch_to_datenum(gps.gps_time(1))), datestr(epoch_to_datenum(gps.gps_time(end)))); + fprintf(fid,' gps out of order (meaning that the previous day has a gps time (%s to %s) that is greater than the start of this day (%s to %s), but the current day is later and must be listed chronologically)', ... + datestr(epoch_to_datenum(old_time(1))), datestr(epoch_to_datenum(old_time(end))), ... + datestr(epoch_to_datenum(gps.gps_time(1))), datestr(epoch_to_datenum(gps.gps_time(end)))); +end + +% old_time: output argument used to ensure gps files are in chronological order +old_time = gps.gps_time([1 end]); + +%% gps_time and gps_source checks +% ========================================================================= + +% Print out start/stop times in several formats +% ------------------------------------------------------------------------- +gps_time_datenum = epoch_to_datenum(gps.gps_time); +[year month day hour minute sec] = datevec(gps_time_datenum(1)); +gps_sod = (gps_time_datenum - datenum([year month day 0 0 0])) * 86400; + +along_track = geodetic_to_along_track(gps.lat,gps.lon,gps.elev); + +fprintf(' Start: %16.14g %s %12.3f %12.3f m UTC is %3f sec\n', gps.gps_time(1), datestr(gps_time_datenum(1)), gps_sod(1), along_track(1), -utc_leap_seconds(gps.gps_time(1))); +fprintf(' Stop: %16.14g %s %12.3f %12.3f m\n', gps.gps_time(end), datestr(gps_time_datenum(end)), gps_sod(end), along_track(end)); +fprintf(fid,' Start: %16.14g %s %12.3f %12.3f m UTC is %3f sec\n', gps.gps_time(1), datestr(gps_time_datenum(1)), gps_sod(1), along_track(1), -utc_leap_seconds(gps.gps_time(1))); +fprintf(fid,' Stop: %16.14g %s %12.3f %12.3f m\n', gps.gps_time(end), datestr(gps_time_datenum(end)), gps_sod(end), along_track(end)); + +if ~isfield(gps,'gps_source') + fprintf('gps_source: field does not exist!!!\n'); + fprintf(fid,' gps_source: field does not exist!!!\n'); + gps.gps_source = ''; + fixable_error = true; +end + +if isempty(regexpi(gps.gps_source,'final')) + warning('GPS source is %s and not final\n', gps.gps_source); + fprintf(fid,' GPS source is %s and not final!!!\n', gps.gps_source); +end + +dgps_time = diff(gps.gps_time); + +nonmonotonic_idxs = find(dgps_time <= 0); +if ~isempty(nonmonotonic_idxs) + warning('%d indexes are nonmonotonically increasing with diff(gps_time) <= 0', length(nonmonotonic_idxs)); + fprintf(fid,' %d indexes are nonmonotonically increasing with diff(gps_time) <= 0!!!', length(nonmonotonic_idxs)); +end + +skip_idxs = find(dgps_time > 1.1); +if ~isempty(skip_idxs) + warning('%d indexes skip more than 1 second with diff(gps_time) <= 1.1. Largest is %g sec.', length(skip_idxs), max(dgps_time)); + fprintf(fid, ' %d indexes skip more than 1 second with diff(gps_time) <= 1.1. Largest is %g sec.', length(skip_idxs), max(dgps_time)); +end + +if any(isnan(gps.gps_time)) + warning('NaN in gps.gps_time'); + fprintf(fid,' NaN in gps.gps_time!!!'); +end + +if any(isnan(gps.lat) | isnan(gps.lon) | isnan(gps.elev) | isnan(gps.roll) | isnan(gps.pitch) | isnan(gps.heading)) + warning('NaN in gps trajectory/attitude fields'); + fprintf(fid,' NaN in gps trajectory/attitude fields!!!'); +end + +%% length of vectors +% ========================================================================= + +Nx = length(gps.gps_time); + +if length(gps.lat) ~= Nx + warning('Length of lat does not match gps_time.\n', length(gps.lat), Nx); + fprintf(fid,' Length of lat does not match gps_time.!!!\n', length(gps.lat), Nx); +end +if length(gps.lon) ~= Nx + warning('Length of lon does not match gps_time.\n', length(gps.lon), Nx); + fprintf(fid,' Length of lon does not match gps_time.!!!\n', length(gps.lon), Nx); +end +if length(gps.elev) ~= Nx + warning('Length of elev does not match gps_time.\n', length(gps.elev), Nx); + fprintf(fid,' Length of elev does not match gps_time!!!.\n', length(gps.elev), Nx); +end +if length(gps.roll) ~= Nx + warning('Length of roll does not match gps_time.\n', length(gps.roll), Nx); + fprintf(fid,' Length of roll does not match gps_time.!!!\n', length(gps.roll), Nx); +end +if length(gps.pitch) ~= Nx + warning('Length of pitch does not match gps_time.\n', length(gps.pitch), Nx); + fprintf(fid,' Length of pitch does not match gps_time.!!!\n', length(gps.pitch), Nx); +end +if length(gps.heading) ~= Nx + warning('Length of heading does not match gps_time.\n', length(gps.heading), Nx); + fprintf(fid,' Length of heading does not match gps_time.!!!\n', length(gps.heading), Nx); +end + +if isfield(gps,'comp_time') && length(gps.comp_time) ~= length(gps.sync_gps_time) + warning('Length of comp_time does not match sync_gps_time.\n', length(gps.comp_time), length(gps.sync_gps_time)); + fprintf(fid,' Length of comp_time does not match sync_gps_time.!!!\n', length(gps.comp_time), length(gps.sync_gps_time)); +end +if isfield(gps,'radar_time') && length(gps.radar_time) ~= length(gps.sync_gps_time) + warning('Length of radar_time does not match sync_gps_time.\n', length(gps.radar_time), length(gps.sync_gps_time)); + fprintf(fid,' Length of radar_time does not match sync_gps_time.!!!\n', length(gps.radar_time), length(gps.sync_gps_time)); +end +if isfield(gps,'sync_lat') && length(gps.sync_lat) ~= length(gps.sync_gps_time) + warning('Length of sync_lat does not match sync_gps_time.\n', length(gps.sync_lat), length(gps.sync_gps_time)); + fprintf(fid,' Length of sync_lat does not match sync_gps_time.!!!\n', length(gps.sync_lat), length(gps.sync_gps_time)); +end +if isfield(gps,'sync_lon') && length(gps.sync_lon) ~= length(gps.sync_gps_time) + warning('Length of sync_lon does not match sync_gps_time.\n', length(gps.sync_lon), length(gps.sync_gps_time)); + fprintf(fid,' Length of sync_lon does not match sync_gps_time.!!!\n', length(gps.sync_lon), length(gps.sync_gps_time)); +end +if isfield(gps,'sync_elev') && length(gps.sync_elev) ~= length(gps.sync_gps_time) + warning('Length of sync_elev does not match sync_gps_time.\n', length(gps.sync_elev), length(gps.sync_gps_time)); + fprintf(fid,' Length of sync_elev does not match sync_gps_time.!!!\n', length(gps.sync_elev), length(gps.sync_gps_time)); +end + +%% lat, lon, elev +% ========================================================================= +if any(gps.lat >= 90 | gps.lat <= -90) + warning('lat out of bounds'); + fprintf(fid,' lat out of bounds!!!'); +end + +if any(gps.lon >= 360 | gps.lon <= -360) + warning('lon out of bounds'); + fprintf(fid,' lon out of bounds!!!'); +end + +if any(gps.elev >= 40000 | gps.elev <= -10000) + warning('elev out of bounds'); + fprintf(fid,' elev out of bounds!!!'); +end + +%% roll, pitch, heading +% ========================================================================= +if any(gps.roll >= param.gps_check.roll_limit/180*pi | gps.roll <= -param.gps_check.roll_limit/180*pi) + warning('abs(roll) > %g deg: max %.1f min %.1f', param.gps_check.roll_limit, max(gps.roll)*180/pi, min(gps.roll)*180/pi); + fprintf(fid,' abs(roll) > %g deg: max %.1f min %.1f', param.gps_check.roll_limit, max(gps.roll)*180/pi, min(gps.roll)*180/pi); +end + +if any(gps.pitch >= 25/180*pi | gps.pitch <= -25/180*pi) + warning('abs(pitch) > 25 deg: max %.1f min %.1f', max(gps.pitch)*180/pi, min(gps.pitch)*180/pi); + fprintf(fid,' abs(pitch) > 25 deg: max %.1f min %.1f', max(gps.pitch)*180/pi, min(gps.pitch)*180/pi); +end + +if any(gps.heading >= 2*pi | gps.heading <= -2*pi) + warning('abs(heading) > 2*pi'); + fprintf(fid,' abs(heading) > 2*pi'); +end + +%% Speed +% ========================================================================= +speed = diff(along_track) ./ diff(gps.gps_time); + +%% Sync GPS Time Plot +% ========================================================================= +if isfield(gps,'sync_gps_time') + dgps_time = diff(gps.sync_gps_time); + + nonmonotonic_idxs = find(dgps_time <= 0); + if ~isempty(nonmonotonic_idxs) + warning('%d indexes are nonmonotonically increasing with diff(sync_gps_time) <= 0', length(nonmonotonic_idxs)); + fprintf(fid,' %d indexes are nonmonotonically increasing with diff(sync_gps_time) <= 0!!!', length(nonmonotonic_idxs)); + end + + skip_idxs = find(dgps_time > 1.1); + if ~isempty(skip_idxs) + warning('%d indexes skip more than 1 second with diff(sync_gps_time) <= 1.1. Largest is %g sec.', length(skip_idxs), max(dgps_time)); + fprintf(fid, ' %d indexes skip more than 1 second with diff(sync_gps_time) <= 1.1. Largest is %g sec.', length(skip_idxs), max(dgps_time)); + end +end + +%% radar_time check +% ========================================================================= +if isfield(gps,'radar_time') + dgps_time = diff(gps.radar_time); + + nonmonotonic_idxs = find(dgps_time <= 0); + if ~isempty(nonmonotonic_idxs) + warning('%d indexes are nonmonotonically increasing with diff(radar_time) <= 0', length(nonmonotonic_idxs)); + fprintf(fid,' %d indexes are nonmonotonically increasing with diff(radar_time) <= 0!!!', length(nonmonotonic_idxs)); + end + + skip_idxs = find(dgps_time > 1.1); + if ~isempty(skip_idxs) + warning('%d indexes skip more than 1 second with diff(radar_time) <= 1.1. Largest is %g sec.', length(skip_idxs), max(dgps_time)); + fprintf(fid, ' %d indexes skip more than 1 second with diff(radar_time) <= 1.1. Largest is %g sec.', length(skip_idxs), max(dgps_time)); + end +end + +%% Cleanup +% ========================================================================= + +fclose(fid); + +end diff --git a/cresis-toolbox/gps/gps_make.m b/cresis-toolbox/gps/gps_create.m similarity index 63% rename from cresis-toolbox/gps/gps_make.m rename to cresis-toolbox/gps/gps_create.m index da2d2b39..18b8ed0c 100644 --- a/cresis-toolbox/gps/gps_make.m +++ b/cresis-toolbox/gps/gps_create.m @@ -1,498 +1,522 @@ -% script gps_make -% -% Makes GPS files (called from each mission specific routine) -% Variables set up in gps_make_SEASON_SOURCE -% -% in_fn = 1 by N cell array of strings containing filenames -% Full path of input GPS file -% Can also do a cell array of strings in place of one of the strings and -% this will cause all the file to be concatenated into one output GPS file -% file_type = 1 by N cell array of strings -% File format for each cell in in_fn (e.g. 'Applanix', 'ATM', 'Litton', etc) -% Determines which read_gps_? function will be called -% Can also do a cell array of file type strings in place of one of the -% strings as long as the length matches the corresponding cell from in_fn. -% This allows the file type to be set on a per file basis. -% in_fn_ins = 1 by N cell array of strings containing filenames (optional) -% Full path of input INS file (optional since INS often in GPS file) -% Can also do a cell array of strings in place of one of the strings and -% this will cause all the file to be concatenated into one output INS file -% file_type_ins = 1 by N cell array of strings -% File format for each cell in in_fn_ins (e.g. 'Applanix', 'ATM', 'Litton', etc) -% Determines which read_gps_? or read_ins_? function will be called -% Can also do a cell array of file type strings in place of one of the -% strings as long as the length matches the corresponding cell from in_fn_ins. -% This allows the file type to be set on a per file basis. -% params = 1 by N cell array of structures -% Parameters for each instance -% If the option to pass a cell array of filename strings for a single output GPS -% file is used, then the params structure must also be cell array of -% structures which correspond to each of the files in the cell array of -% filename strings. -% gps_path = string -% Output directory -% out_fn = 1 by N cell array of strings -% Output file name for each instance -% gps_source = string describing GPS processing location and the -% version of the processed file. This is done by placing a hyphen -% after the location identifier. The location identifier is used -% with the season ID and the lever arm function to determine which -% lever arm to use. Examples: -% gravimeter-field: processed to gravimeter INS, field version of GPS/INS -% product -% atm-field: processed to ATM DGPS antenna, field version of GPS/INS -% product -% atm-final_20110826: processed to ATM DGPS antenna, final version of GPS/INS -% product as of 20110826 -% -% -% Author: John Paden - -% ====================================================================== -% Read and translate files according to user settings -% ====================================================================== - -for file_idx = 1:length(in_fns) - in_fn = in_fns{file_idx}; - if ischar(in_fn) - in_fn = {in_fn}; - end - if ~exist('sync_flag','var') || numel(sync_flag) < file_idx - sync_flag{file_idx} = 0; - end - if ~sync_flag{file_idx} - sync_fn = {}; - else - if ~exist('sync_fns','var') || file_idx > length(sync_fns) - error('sync_flag is true, but no files specified in sync_fns for this date.'); - else - sync_fn = sync_fns{file_idx}; - end - end - if ischar(sync_fn) - sync_fn = {sync_fn}; - end - out_fn = fullfile(gps_path,out_fns{file_idx}); - if isempty(in_fn) - error('No input GPS files found in in_fns{%d} variable. Usually this is because the file path is not setup.\n', file_idx); - end - - fprintf('=====================================================================\n'); - fprintf('%s: %s (%s)\n', mfilename, out_fn, datestr(now)); - fprintf('=====================================================================\n'); - - %% Load Radar Sync GPS files (mcrds, accum2, arena) - if sync_flag{file_idx} - if isstruct(sync_params{file_idx}) - sync_params{file_idx} = repmat({sync_params{file_idx}},size(sync_fn)); - end - clear sync_gps; - fprintf('Sync files (%.1f sec)\n', toc); - for sync_fn_idx = 1:length(sync_fn) - if ~exist('sync_file_type','var') || length(sync_file_type) < file_idx || isempty(sync_file_type{file_idx}) - % Assume nmea because of legacy support which did not require this field to be set - warning('Set sync_file_type{%d} since corresponding sync_flag{%d} is set to true. Assuming sync_file_type{%d} = ''nmea''.', file_idx, file_idx, file_idx); - sync_file_type{file_idx} = 'nmea'; - end - if iscell(sync_file_type{file_idx}) - cur_file_type = sync_file_type{file_idx}{sync_fn_idx}; - else - cur_file_type = sync_file_type{file_idx}; - end - gps_fh = gps_make_fh(cur_file_type); - fprintf(' %s\n', sync_fn{sync_fn_idx}); - gps_tmp = gps_fh(sync_fn{sync_fn_idx},sync_params{file_idx}{sync_fn_idx}); - - if sync_fn_idx == 1 - sync_gps = gps_tmp; - else - sync_gps.gps_time = [sync_gps.gps_time, gps_tmp.gps_time]; - sync_gps.lat = [sync_gps.lat, gps_tmp.lat]; - sync_gps.lon = [sync_gps.lon, gps_tmp.lon]; - sync_gps.elev = [sync_gps.elev, gps_tmp.elev]; - sync_gps.roll = [sync_gps.roll, gps_tmp.roll]; - sync_gps.pitch = [sync_gps.pitch, gps_tmp.pitch]; - sync_gps.heading = [sync_gps.heading, gps_tmp.heading]; - sync_gps.comp_time = [sync_gps.comp_time, gps_tmp.comp_time]; - if isfield(sync_gps,'radar_time') - sync_gps.radar_time = [sync_gps.radar_time, gps_tmp.radar_time]; - end - if isfield(sync_gps,'profileCntr') - sync_gps.profileCntr = [sync_gps.profileCntr, gps_tmp.profileCntr]; - end - end - end - - %% Remove records with NaN - if isfield(sync_gps,'radar_time') - good_mask = ~(isnan(sync_gps.gps_time) | isnan(sync_gps.radar_time)); - sync_gps.radar_time = sync_gps.radar_time(good_mask); - else - good_mask = ~(isnan(sync_gps.gps_time) | isnan(sync_gps.comp_time)); - end - sync_gps.gps_time = sync_gps.gps_time(good_mask); - sync_gps.comp_time = sync_gps.comp_time(good_mask); - sync_gps.lat = sync_gps.lat(good_mask); - sync_gps.lon = sync_gps.lon(good_mask); - sync_gps.elev = sync_gps.elev(good_mask); - sync_gps.roll = sync_gps.roll(good_mask); - sync_gps.pitch = sync_gps.pitch(good_mask); - sync_gps.heading = sync_gps.heading(good_mask); - - %% Check for day wraps - % Find jumps in the GPS time that are probably due to day interval - % 86400 seconds. - day_jumps = find(diff(sync_gps.gps_time) < -60000); - if ~isempty(day_jumps) - warning('Found a day wrap in sync gps... correcting'); - end - for jump_idx = day_jumps - sync_gps.gps_time(jump_idx+1:end) = sync_gps.gps_time(jump_idx+1:end) + 86400; - end - - %% Check/make the sync GPS data monotonic in time in case it is not - sync_gps = gps_make_monotonic(sync_gps); - end - - %% Load GPS files - if ischar(in_fn) - in_fn = {in_fn}; - end - if isstruct(params{file_idx}) - params{file_idx} = repmat({params{file_idx}},size(in_fn)); - end - clear gps; - separate_ins_data_flag = false; - fprintf('Input files (%.1f sec)\n', toc); - for in_fn_idx = 1:length(in_fn) - if iscell(file_type{file_idx}) - cur_file_type = file_type{file_idx}{in_fn_idx}; - plus_idx = regexp(cur_file_type,'+'); - if ~isempty(plus_idx) - warning('Deprecated GPS+INS file_type format using "+". Use file_type_ins instead of combining INS type with GPS file_type. For example "awi_netcdf+awi_netcdf" should be "awi_netcdf" in file_type and "awi_netcdf" in file_type_ins.'); - separate_ins_data_flag = true; - file_type_ins{file_idx}{in_fn_idx} = cur_file_type(plus_idx+1:end); - cur_file_type = cur_file_type(1:plus_idx-1); - end - else - cur_file_type = file_type{file_idx}; - plus_idx = regexp(cur_file_type,'+'); - if ~isempty(plus_idx) - warning('Deprecated GPS+INS file_type format using "+". Use file_type_ins instead of combining INS type with GPS file_type. For example "awi_netcdf+awi_netcdf" should be "awi_netcdf" in file_type and "awi_netcdf" in file_type_ins.'); - separate_ins_data_flag = true; - file_type_ins{file_idx} = cur_file_type(plus_idx+1:end); - cur_file_type = cur_file_type(1:plus_idx-1); - end - end - gps_fh = gps_make_fh(cur_file_type); - fn = in_fn{in_fn_idx}; - [fn_dir,fn_name] = fileparts(fn); - gps_tmp = gps_fh(fn,params{file_idx}{in_fn_idx}); - if ~isempty(gps_tmp.gps_time) - % Print out filename, start/stop GPS time, and duration in seconds - fprintf('-- %s\t%s\t%s\t%s\t%g\n', fn_dir,fn_name, ... - datestr(epoch_to_datenum(gps_tmp.gps_time(1))), ... - datestr(epoch_to_datenum(gps_tmp.gps_time(end))), diff(gps_tmp.gps_time([1 end]))); - else - % Print out filename only because there are no entries - fprintf('-- %s\t%s\t\t\t\n', fn_dir,fn_name); - end - - try - gps_tmp = rmfield(gps_tmp,'radar_time'); - end - - if in_fn_idx == 1 - gps.gps_time = gps_tmp.gps_time; - gps.lat = gps_tmp.lat; - gps.lon = gps_tmp.lon; - gps.elev = gps_tmp.elev; - gps.roll = gps_tmp.roll; - gps.pitch = gps_tmp.pitch; - gps.heading = gps_tmp.heading; - else - gps.gps_time = [gps.gps_time, gps_tmp.gps_time]; - gps.lat = [gps.lat, gps_tmp.lat]; - gps.lon = [gps.lon, gps_tmp.lon]; - gps.elev = [gps.elev, gps_tmp.elev]; - gps.roll = [gps.roll, gps_tmp.roll]; - gps.pitch = [gps.pitch, gps_tmp.pitch]; - gps.heading = [gps.heading, gps_tmp.heading]; - end - end - - if isempty(gps.gps_time) - file_list_str = sprintf(' %s\n', in_fn{:}); - error('No GPS data loaded, isempty(gps.gps_time) == true for files:\n%s', file_list_str); - end - - %% Remove records with NaN - good_mask = ~(isnan(gps.gps_time) | isnan(gps.lat) ... - | isnan(gps.lon) | isnan(gps.elev) ... - | isnan(gps.roll) | isnan(gps.pitch) | isnan(gps.heading)); - gps.gps_time = gps.gps_time(good_mask); - gps.lat = gps.lat(good_mask); - gps.lon = gps.lon(good_mask); - gps.elev = gps.elev(good_mask); - gps.roll = gps.roll(good_mask); - gps.pitch = gps.pitch(good_mask); - gps.heading = gps.heading(good_mask); - gps.gps_source = gps_source{file_idx}; - - %% Check for day wraps - % Find jumps in the GPS time that are probably due to day interval - % 86400 seconds. - day_jumps = find(diff(gps.gps_time) < -60000); - if ~isempty(day_jumps) - warning('Found a day wrap... correcting'); - end - for jump_idx = day_jumps - gps.gps_time(jump_idx+1:end) = gps.gps_time(jump_idx+1:end) + 86400; - end - - %% Check/make the GPS data monotonic in time in case it is not - gps = gps_make_monotonic(gps); - - %% Fabricating a heading now - along_track = geodetic_to_along_track(gps.lat,gps.lon,gps.elev); - rlines = get_equal_alongtrack_spacing_idxs(along_track,10); - physical_constants; - est_heading = size(gps.heading); - clear origin heading east north; - for rline_idx = 1:length(rlines) - rline = rlines(rline_idx); - if rline_idx < length(rlines) - rline_end = rlines(rline_idx+1); - else - rline_end = length(along_track); - end - [origin(1),origin(2),origin(3)] = geodetic2ecef(gps.lat(rline)/180*pi,gps.lon(rline)/180*pi,gps.elev(rline),WGS84.ellipsoid); - [heading(1),heading(2),heading(3)] = geodetic2ecef(gps.lat(rline_end)/180*pi,gps.lon(rline_end)/180*pi,gps.elev(rline_end),WGS84.ellipsoid); - heading = heading - origin; - % Determine east vector - [east(1) east(2) east(3)] = lv2ecef(1,0,0,gps.lat(rline)/180*pi,gps.lon(rline)/180*pi,gps.elev(rline),WGS84.ellipsoid); - east = east - origin; - % Determine north vector - [north(1) north(2) north(3)] = lv2ecef(0,1,0,gps.lat(rline)/180*pi,gps.lon(rline)/180*pi,gps.elev(rline),WGS84.ellipsoid); - north = north - origin; - % Determine heading - est_heading(rline:rline_end) = atan2(dot(east,heading),dot(north,heading)); - end - - %% Load INS data for special case where it is separate from GPS - if separate_ins_data_flag ... - || (exist('in_fns_ins','var') && length(in_fns_ins) >= file_idx && ~isempty(in_fns_ins{file_idx})) - if ischar(in_fns_ins{file_idx}) - in_fns_ins{file_idx} = {in_fns_ins{file_idx}}; - end - if isstruct(params_ins{file_idx}) - params_ins{file_idx} = repmat({params_ins{file_idx}},size(in_fns_ins{file_idx})); - end - clear ins; - if isempty(in_fns_ins{file_idx}) - error('No input INS files found in in_fns{%d} variable. Usually this is because the file path is not setup.\n', file_idx); - end - for in_fn_idx = 1:length(in_fns_ins{file_idx}) - if iscell(file_type_ins{file_idx}) - cur_file_type = file_type_ins{file_idx}{in_fn_idx}; - else - cur_file_type = file_type_ins{file_idx}; - end - fprintf(' INS file %s\n', in_fns_ins{file_idx}{in_fn_idx}); - if strcmpi(cur_file_type,'applanix') - ins_tmp = read_gps_applanix(in_fns_ins{file_idx}{in_fn_idx},params_ins{file_idx}{in_fn_idx}); - elseif strcmpi(cur_file_type,'Litton') - ins_tmp = read_ins_litton(in_fns_ins{file_idx}{in_fn_idx},params_ins{file_idx}{in_fn_idx}); - elseif strcmpi(cur_file_type,'Litton_DGPS') - ins_tmp = read_gps_litton(in_fns_ins{file_idx}{in_fn_idx},params_ins{file_idx}{in_fn_idx}); - elseif strcmpi(cur_file_type,'General_ASCII') - ins_tmp = read_gps_general_ascii(in_fns_ins{file_idx}{in_fn_idx},params_ins{file_idx}{in_fn_idx}); - elseif strcmpi(cur_file_type,'awi_netcdf') - ins_tmp = read_gps_netcdf(in_fns_ins{file_idx}{in_fn_idx},params_ins{file_idx}{in_fn_idx}); - else - error('Unknown INS type %s.', cur_file_type); - end - if in_fn_idx == 1 - ins = ins_tmp; - else - ins.gps_time = [ins.gps_time, ins_tmp.gps_time]; - ins.lat = [ins.lat, ins_tmp.lat]; - ins.lon = [ins.lon, ins_tmp.lon]; - ins.elev = [ins.elev, ins_tmp.elev]; - ins.roll = [ins.roll, ins_tmp.roll]; - ins.pitch = [ins.pitch, ins_tmp.pitch]; - ins.heading = [ins.heading, ins_tmp.heading]; - end - end - - bad_mask = ins.heading == 0; - if sum(bad_mask) > 0 - fprintf('Found %d bad records based on heading == 0\n', sum(bad_mask)); - ins.gps_time = ins.gps_time(~bad_mask); - ins.lat = ins.lat(~bad_mask); - ins.lon = ins.lon(~bad_mask); - ins.elev = ins.elev(~bad_mask); - ins.roll = ins.roll(~bad_mask); - ins.pitch = ins.pitch(~bad_mask); - ins.heading = ins.heading(~bad_mask); - end - - % Check for large time gaps in INS data and fill with level flight - if 0 - figure(1); clf; - bad_mask = diff(ins.gps_time) > 10; - bad_mask = grow(bad_mask,1); - time_skips = diff(ins.gps_time); - along_track = geodetic_to_along_track(gps.lat,gps.lon); - vel = diff(along_track) ./ diff(gps.gps_time); - plot(gps.gps_time(1:end-1),vel) - hold on - plot(ins.gps_time(bad_mask),time_skips(bad_mask),'r'); - hold off; - - figure(2); clf; - plot(ins.gps_time,ins.roll*180/pi,'.') - hold on - plot(ins.gps_time(bad_mask),time_skips(bad_mask),'r'); - hold off; - ylim([-20 20]) - grid on - - figure(3); clf; - plot(gps.gps_time,est_heading*180/pi,'g.'); - hold on - plot(ins.gps_time,ins.heading*180/pi,'.') - plot(ins.gps_time(bad_mask),time_skips(bad_mask),'r'); - hold off; - grid on - end - - bad_idxs = find(diff(ins.gps_time) > 20); - if ins.gps_time(1) - gps.gps_time(1) > 10 - bad_idxs = [0 bad_idxs]; - end - if gps.gps_time(end) - ins.gps_time(end) > 10 - bad_idxs = [bad_idxs length(ins.gps_time)]; - end - for bad_idxs_idx = 1:length(bad_idxs) - idx = bad_idxs(bad_idxs_idx); - - % Find gps indices in this gap - if idx == 0 - new_gps_times = gps.gps_time(1):ins.gps_time(idx+1)-10; - edge_gps_times = [gps.gps_time(1) ins.gps_time(idx+1)]; - est_heading_error = [0 ins.heading(idx+1) - interp1(gps.gps_time,est_heading,ins.gps_time(idx+1))]; - elseif idx == length(ins.gps_time) - new_gps_times = ins.gps_time(idx)+10:gps.gps_time(end); - edge_gps_times = [ins.gps_time(idx) gps.gps_time(end)]; - est_heading_error = [ins.heading(idx) - interp1(gps.gps_time,est_heading,ins.gps_time(idx)) 0]; - else - new_gps_times = ins.gps_time(idx)+10:ins.gps_time(idx+1)-10; - edge_gps_times = ins.gps_time(idx:idx+1); - est_heading_error = ins.heading(idx:idx+1) - interp1(gps.gps_time,est_heading,ins.gps_time(idx:idx+1)); - end - bad_idxs(bad_idxs_idx+1:end) = bad_idxs(bad_idxs_idx+1:end) + length(new_gps_times); - fprintf(' Inserting %d level flight records due to missing INS data\n', length(new_gps_times)); - - ins.roll = [ins.roll(1:idx) zeros(size(new_gps_times)) ins.roll(idx+1:end)]; - ins.pitch = [ins.pitch(1:idx) zeros(size(new_gps_times)) ins.pitch(idx+1:end)]; - ins.lat = [ins.lat(1:idx) NaN*zeros(size(new_gps_times)) ins.lat(idx+1:end)]; - ins.lon = [ins.lon(1:idx) NaN*zeros(size(new_gps_times)) ins.lon(idx+1:end)]; - ins.elev = [ins.elev(1:idx) NaN*zeros(size(new_gps_times)) ins.elev(idx+1:end)]; - - inserted_heading = interp1(gps.gps_time,est_heading,new_gps_times) ... - + interp1(edge_gps_times,est_heading_error,new_gps_times); - ins.heading = [ins.heading(1:idx) inserted_heading ins.heading(idx+1:end)]; - - ins.gps_time = [ins.gps_time(1:idx) new_gps_times ins.gps_time(idx+1:end)]; - - end - - if 0 - figure(4); clf; - plot(gps.gps_time,est_heading*180/pi,'g.'); - hold on - plot(ins.gps_time,ins.heading*180/pi,'.') - hold off; - grid on - keyboard - end - - % Insert filler points from gps (ensures that sparse ins data does not - % create a sparse final dataset when gps data is available) - new_gps_time = union(gps.gps_time,ins.gps_time); - ins.roll = interp1(ins.gps_time,ins.roll,new_gps_time); - ins.pitch = interp1(ins.gps_time,ins.pitch,new_gps_time); - ins.heading = interp1(ins.gps_time,ins.heading,new_gps_time); - if any(gps.lon >180) % 2018_Antarctica_DC8 ATM trajectory files - gps.lon(gps.lon>180) = gps.lon(gps.lon>180) -360; - end - interp_idxs = find(ins.gps_time >= gps.gps_time(1) & ins.gps_time <= gps.gps_time(end)); - ins.lat(interp_idxs) = interp1(gps.gps_time,gps.lat,ins.gps_time(interp_idxs)); - ins.lon(interp_idxs) = interp1(gps.gps_time,gps.lon,ins.gps_time(interp_idxs)); - ins.elev(interp_idxs) = interp1(gps.gps_time,gps.elev,ins.gps_time(interp_idxs)); - ins.lat = interp1(ins.gps_time,ins.lat,new_gps_time); - ins.lon = interp1(ins.gps_time,ins.lon,new_gps_time); - ins.elev = interp1(ins.gps_time,ins.elev,new_gps_time); - ins.gps_time = new_gps_time; - gps = ins; - - %% Remove records with NaN - good_mask = ~(isnan(gps.gps_time) | isnan(gps.lat) ... - | isnan(gps.lon) | isnan(gps.elev) ... - | isnan(gps.roll) | isnan(gps.pitch) | isnan(gps.heading)); - gps.gps_time = gps.gps_time(good_mask); - gps.lat = gps.lat(good_mask); - gps.lon = gps.lon(good_mask); - gps.elev = gps.elev(good_mask); - gps.roll = gps.roll(good_mask); - gps.pitch = gps.pitch(good_mask); - gps.heading = gps.heading(good_mask); - gps.gps_source = gps_source{file_idx}; - end - - %% Now that INS data may have been added, check/make the GPS data monotonic in time in case it is not - gps = gps_make_monotonic(gps); - - %% Fabricate a heading from the trajectory if it is all zeros - if all(gps.heading == 0) - warning('These input files have heading(:) == 0. Using the estimated heading.'); - gps.heading = est_heading; - end - - %% Add software revision information - gps.sw_version = current_software_version; - - %% Save output file - fprintf('Output file %s\n', out_fn); - gps.file_version = '1'; - if sync_flag{file_idx} - % Add the Radar Synchronization variables for mcrds, accum2, acords, - % arena - - % Synchronize computer time and radar time from NMEA file to gps time - gps.sync_gps_time = sync_gps.gps_time; - gps.sync_lat = sync_gps.lat; - gps.sync_lon = sync_gps.lon; - gps.sync_elev = sync_gps.elev; - gps.comp_time = sync_gps.comp_time; - if isfield(sync_gps,'radar_time') - gps.radar_time = sync_gps.radar_time; - end - - if isfield(gps,'radar_time') - ct_save(out_fn,'-v7.3','-STRUCT','gps','gps_time','lat','lon','elev','roll','pitch','heading','gps_source','sync_gps_time','sync_lat','sync_lon','sync_elev','comp_time','radar_time','sw_version','file_version'); - else - ct_save(out_fn,'-v7.3','-STRUCT','gps','gps_time','lat','lon','elev','roll','pitch','heading','gps_source','sync_gps_time','sync_lat','sync_lon','sync_elev','comp_time','sw_version','file_version'); - end - else - ct_save(out_fn,'-v7.3','-STRUCT','gps','gps_time','lat','lon','elev','roll','pitch','heading','gps_source','sw_version','file_version'); - end - - if debug_level >= 2 - gps_plot(out_fn); - end -end - -return; - +% script gps_create +% +% Makes GPS files (called from each mission specific routine) +% Variables set up in gps_create_SEASON_SOURCE +% +% in_fn = 1 by N cell array of strings containing filenames +% Full path of input GPS file +% Can also do a cell array of strings in place of one of the strings and +% this will cause all the file to be concatenated into one output GPS file +% file_type = 1 by N cell array of strings +% File format for each cell in in_fn (e.g. 'Applanix', 'ATM', 'Litton', etc) +% Determines which read_gps_? function will be called +% Can also do a cell array of file type strings in place of one of the +% strings as long as the length matches the corresponding cell from in_fn. +% This allows the file type to be set on a per file basis. +% in_fn_ins = 1 by N cell array of strings containing filenames (optional) +% Full path of input INS file (optional since INS often in GPS file) +% Can also do a cell array of strings in place of one of the strings and +% this will cause all the file to be concatenated into one output INS file +% file_type_ins = 1 by N cell array of strings +% File format for each cell in in_fn_ins (e.g. 'Applanix', 'ATM', 'Litton', etc) +% Determines which read_gps_? or read_ins_? function will be called +% Can also do a cell array of file type strings in place of one of the +% strings as long as the length matches the corresponding cell from in_fn_ins. +% This allows the file type to be set on a per file basis. +% params = 1 by N cell array of structures +% Parameters for each instance +% If the option to pass a cell array of filename strings for a single output GPS +% file is used, then the params structure must also be cell array of +% structures which correspond to each of the files in the cell array of +% filename strings. +% gps_path = string +% Output directory +% out_fn = 1 by N cell array of strings +% Output file name for each instance +% gps_source = string describing GPS processing location and the +% version of the processed file. This is done by placing a hyphen +% after the location identifier. The location identifier is used +% with the season ID and the lever arm function to determine which +% lever arm to use. Examples: +% gravimeter-field: processed to gravimeter INS, field version of GPS/INS +% product +% atm-field: processed to ATM DGPS antenna, field version of GPS/INS +% product +% atm-final_20110826: processed to ATM DGPS antenna, final version of GPS/INS +% product as of 20110826 +% date_str: 8 letter sequence containing the date 'YYYYMMDD' +% season_name: string containing season name YYYY_LOCATION_PLATFORM +% +% +% Author: John Paden + +% ====================================================================== +% Read and translate files according to user settings +% ====================================================================== + +for file_idx = 1:length(in_fns) + in_fn = in_fns{file_idx}; + if ischar(in_fn) + in_fn = {in_fn}; + end + if ~exist('sync_flag','var') || numel(sync_flag) < file_idx + sync_flag{file_idx} = 0; + end + if ~sync_flag{file_idx} + sync_fn = {}; + else + if ~exist('sync_fns','var') || file_idx > length(sync_fns) + error('sync_flag is true, but no files specified in sync_fns for this date.'); + else + sync_fn = sync_fns{file_idx}; + end + end + if ischar(sync_fn) + sync_fn = {sync_fn}; + end + out_fn = fullfile(gps_path,out_fns{file_idx}); + if isempty(in_fn) + error('No input GPS files found in in_fns{%d} variable. Usually this is because the file path is not setup.\n', file_idx); + end + + fprintf('=====================================================================\n'); + fprintf('%s:\t%s\t%s\n', mfilename, out_fn, datestr(now,'yyyymmdd_HHMMSS')); + fprintf('=====================================================================\n'); + + %% Load Radar Sync GPS files (mcrds, accum2, arena) + if sync_flag{file_idx} + if isstruct(sync_params{file_idx}) + sync_params{file_idx} = repmat({sync_params{file_idx}},size(sync_fn)); + end + clear sync_gps; + fprintf('Sync_files\t%s\n', datestr(now,'yyyymmdd_HHMMSS')); + for sync_fn_idx = 1:length(sync_fn) + if ~exist('sync_file_type','var') || length(sync_file_type) < file_idx || isempty(sync_file_type{file_idx}) + % Assume nmea because of legacy support which did not require this field to be set + warning('Set sync_file_type{%d} since corresponding sync_flag{%d} is set to true. Assuming sync_file_type{%d} = ''nmea''.', file_idx, file_idx, file_idx); + sync_file_type{file_idx} = 'nmea'; + end + if iscell(sync_file_type{file_idx}) + cur_file_type = sync_file_type{file_idx}{sync_fn_idx}; + else + cur_file_type = sync_file_type{file_idx}; + end + gps_fh = gps_create_fh(cur_file_type); + fprintf(' %s\n', sync_fn{sync_fn_idx}); + gps_tmp = gps_fh(sync_fn{sync_fn_idx},sync_params{file_idx}{sync_fn_idx}); + + if sync_fn_idx == 1 + sync_gps = gps_tmp; + else + sync_gps.gps_time = [sync_gps.gps_time, gps_tmp.gps_time]; + sync_gps.lat = [sync_gps.lat, gps_tmp.lat]; + sync_gps.lon = [sync_gps.lon, gps_tmp.lon]; + sync_gps.elev = [sync_gps.elev, gps_tmp.elev]; + sync_gps.roll = [sync_gps.roll, gps_tmp.roll]; + sync_gps.pitch = [sync_gps.pitch, gps_tmp.pitch]; + sync_gps.heading = [sync_gps.heading, gps_tmp.heading]; + sync_gps.comp_time = [sync_gps.comp_time, gps_tmp.comp_time]; + if isfield(sync_gps,'radar_time') + sync_gps.radar_time = [sync_gps.radar_time, gps_tmp.radar_time]; + end + if isfield(sync_gps,'profileCntr') + sync_gps.profileCntr = [sync_gps.profileCntr, gps_tmp.profileCntr]; + end + end + end + + %% Remove records with NaN + if isfield(sync_gps,'radar_time') + good_mask = ~(isnan(sync_gps.gps_time) | isnan(sync_gps.radar_time)); + sync_gps.radar_time = sync_gps.radar_time(good_mask); + else + good_mask = ~(isnan(sync_gps.gps_time) | isnan(sync_gps.comp_time)); + end + sync_gps.gps_time = sync_gps.gps_time(good_mask); + sync_gps.comp_time = sync_gps.comp_time(good_mask); + sync_gps.lat = sync_gps.lat(good_mask); + sync_gps.lon = sync_gps.lon(good_mask); + sync_gps.elev = sync_gps.elev(good_mask); + sync_gps.roll = sync_gps.roll(good_mask); + sync_gps.pitch = sync_gps.pitch(good_mask); + sync_gps.heading = sync_gps.heading(good_mask); + if isfield(sync_gps,'profileCntr') + sync_gps.profileCntr = sync_gps.profileCntr(good_mask); + end + + %% Check for day wraps + % Find jumps in the GPS time that are probably due to day interval + % 86400 seconds. + day_jumps = find(diff(sync_gps.gps_time) < -60000); + if ~isempty(day_jumps) + warning('Found a day wrap in sync gps... correcting'); + end + for jump_idx = day_jumps + sync_gps.gps_time(jump_idx+1:end) = sync_gps.gps_time(jump_idx+1:end) + 86400; + end + + %% Check/make the sync GPS data monotonic in time in case it is not + sync_gps = gps_force_monotonic(sync_gps); + end + + %% Load GPS files + if ischar(in_fn) + in_fn = {in_fn}; + end + if isstruct(params{file_idx}) + params{file_idx} = repmat({params{file_idx}},size(in_fn)); + end + clear gps; + separate_ins_data_flag = false; + fprintf('Input_files\t%s\n', datestr(now,'yyyymmdd_HHMMSS')); + for in_fn_idx = 1:length(in_fn) + if iscell(file_type{file_idx}) + cur_file_type = file_type{file_idx}{in_fn_idx}; + plus_idx = regexp(cur_file_type,'+'); + if ~isempty(plus_idx) + warning('Deprecated GPS+INS file_type format using "+". Use file_type_ins instead of combining INS type with GPS file_type. For example "awi_netcdf+awi_netcdf" should be "awi_netcdf" in file_type and "awi_netcdf" in file_type_ins.'); + separate_ins_data_flag = true; + file_type_ins{file_idx}{in_fn_idx} = cur_file_type(plus_idx+1:end); + cur_file_type = cur_file_type(1:plus_idx-1); + end + else + cur_file_type = file_type{file_idx}; + plus_idx = regexp(cur_file_type,'+'); + if ~isempty(plus_idx) + warning('Deprecated GPS+INS file_type format using "+". Use file_type_ins instead of combining INS type with GPS file_type. For example "awi_netcdf+awi_netcdf" should be "awi_netcdf" in file_type and "awi_netcdf" in file_type_ins.'); + separate_ins_data_flag = true; + file_type_ins{file_idx} = cur_file_type(plus_idx+1:end); + cur_file_type = cur_file_type(1:plus_idx-1); + end + end + gps_fh = gps_create_fh(cur_file_type); + fn = in_fn{in_fn_idx}; + [fn_dir,fn_name] = fileparts(fn); + gps_tmp = gps_fh(fn,params{file_idx}{in_fn_idx}); + if ~isempty(gps_tmp.gps_time) + % Print out filename, start/stop GPS time, and duration in seconds + fprintf('-- %s\t%s\t%s\t%s\t%g\t%s\n', fn_dir,fn_name, ... + datestr(epoch_to_datenum(gps_tmp.gps_time(1))), ... + datestr(epoch_to_datenum(gps_tmp.gps_time(end))), diff(gps_tmp.gps_time([1 end])), datestr(now,'yyyymmdd_HHMMSS')); + else + % Print out filename only because there are no entries + fprintf('-- %s\t%s\t\t\t\n', fn_dir,fn_name); + end + + if in_fn_idx == 1 + gps.gps_time = gps_tmp.gps_time; + gps.lat = gps_tmp.lat; + gps.lon = gps_tmp.lon; + gps.elev = gps_tmp.elev; + gps.roll = gps_tmp.roll; + gps.pitch = gps_tmp.pitch; + gps.heading = gps_tmp.heading; + if isfield(gps_tmp,'radar_time') + gps.radar_time = gps_tmp.radar_time; + end + if isfield(gps_tmp,'comp_time') + gps.comp_time = gps_tmp.comp_time; + end + else + gps.gps_time = [gps.gps_time, gps_tmp.gps_time]; + gps.lat = [gps.lat, gps_tmp.lat]; + gps.lon = [gps.lon, gps_tmp.lon]; + gps.elev = [gps.elev, gps_tmp.elev]; + gps.roll = [gps.roll, gps_tmp.roll]; + gps.pitch = [gps.pitch, gps_tmp.pitch]; + gps.heading = [gps.heading, gps_tmp.heading]; + if isfield(gps_tmp,'radar_time') + gps.radar_time = [gps.radar_time, gps_tmp.radar_time]; + end + if isfield(gps_tmp,'comp_time') + gps.comp_time = [gps.comp_time, gps_tmp.comp_time]; + end + end + end + + if isempty(gps.gps_time) + file_list_str = sprintf(' %s\n', in_fn{:}); + error('No GPS data loaded, isempty(gps.gps_time) == true for files:\n%s', file_list_str); + end + + %% Remove records with NaN in gps_time or trajectory + good_mask = ~(isnan(gps.gps_time) | isnan(gps.lat) ... + | isnan(gps.lon) | isnan(gps.elev)); + gps.gps_time = gps.gps_time(good_mask); + gps.lat = gps.lat(good_mask); + gps.lon = gps.lon(good_mask); + gps.elev = gps.elev(good_mask); + gps.roll = gps.roll(good_mask); + gps.pitch = gps.pitch(good_mask); + gps.heading = gps.heading(good_mask); + if isfield(gps,'radar_time') + gps.radar_time = gps.radar_time(good_mask); + end + if isfield(gps,'comp_time') + gps.comp_time = gps.comp_time(good_mask); + end + gps.gps_source = gps_source{file_idx}; + + %% Interpolate through NaN attitude data + gps.roll = interp_finite(gps.roll,0); + gps.pitch = interp_finite(gps.pitch,0); + gps.heading = interp_finite(gps.heading,0,@gps_interp1); + + %% Check for day wraps + % Find jumps in the GPS time that are probably due to day interval + % 86400 seconds. + day_jumps = find(diff(gps.gps_time) < -60000); + if ~isempty(day_jumps) + warning('Found a day wrap... correcting'); + end + for jump_idx = day_jumps + gps.gps_time(jump_idx+1:end) = gps.gps_time(jump_idx+1:end) + 86400; + end + + %% Check/make the GPS data monotonic in time in case it is not + gps = gps_force_monotonic(gps); + + %% Fabricating a heading now + [est_heading,along_track,speed] = trajectory_coord_system(gps); + + %% Load INS data for special case where it is separate from GPS + if separate_ins_data_flag ... + || (exist('in_fns_ins','var') && length(in_fns_ins) >= file_idx && ~isempty(in_fns_ins{file_idx})) + if ischar(in_fns_ins{file_idx}) + in_fns_ins{file_idx} = {in_fns_ins{file_idx}}; + end + if isstruct(params_ins{file_idx}) + params_ins{file_idx} = repmat({params_ins{file_idx}},size(in_fns_ins{file_idx})); + end + clear ins; + if isempty(in_fns_ins{file_idx}) + error('No input INS files found in in_fns{%d} variable. Usually this is because the file path is not setup.\n', file_idx); + end + for in_fn_idx = 1:length(in_fns_ins{file_idx}) + if iscell(file_type_ins{file_idx}) + cur_file_type = file_type_ins{file_idx}{in_fn_idx}; + else + cur_file_type = file_type_ins{file_idx}; + end + fprintf(' INS file %s\n', in_fns_ins{file_idx}{in_fn_idx}); + if strcmpi(cur_file_type,'applanix') + ins_tmp = read_gps_applanix(in_fns_ins{file_idx}{in_fn_idx},params_ins{file_idx}{in_fn_idx}); + elseif strcmpi(cur_file_type,'Litton') + ins_tmp = read_ins_litton(in_fns_ins{file_idx}{in_fn_idx},params_ins{file_idx}{in_fn_idx}); + elseif strcmpi(cur_file_type,'Litton_DGPS') + ins_tmp = read_gps_litton(in_fns_ins{file_idx}{in_fn_idx},params_ins{file_idx}{in_fn_idx}); + elseif strcmpi(cur_file_type,'General_ASCII') + ins_tmp = read_gps_general_ascii(in_fns_ins{file_idx}{in_fn_idx},params_ins{file_idx}{in_fn_idx}); + elseif strcmpi(cur_file_type,'awi_netcdf') + ins_tmp = read_gps_netcdf(in_fns_ins{file_idx}{in_fn_idx},params_ins{file_idx}{in_fn_idx}); + else + error('Unknown INS type %s.', cur_file_type); + end + if in_fn_idx == 1 + ins = ins_tmp; + else + ins.gps_time = [ins.gps_time, ins_tmp.gps_time]; + ins.lat = [ins.lat, ins_tmp.lat]; + ins.lon = [ins.lon, ins_tmp.lon]; + ins.elev = [ins.elev, ins_tmp.elev]; + ins.roll = [ins.roll, ins_tmp.roll]; + ins.pitch = [ins.pitch, ins_tmp.pitch]; + ins.heading = [ins.heading, ins_tmp.heading]; + end + end + + bad_mask = ins.heading == 0; + if sum(bad_mask) > 0 + fprintf('Found %d bad records out of %d records based on heading == 0\n', sum(bad_mask), length(bad_mask)); + ins.gps_time = ins.gps_time(~bad_mask); + ins.lat = ins.lat(~bad_mask); + ins.lon = ins.lon(~bad_mask); + ins.elev = ins.elev(~bad_mask); + ins.roll = ins.roll(~bad_mask); + ins.pitch = ins.pitch(~bad_mask); + ins.heading = ins.heading(~bad_mask); + end + + if isempty(ins.gps_time) + warning('INS data specified, but no good INS data found.'); + + else + + % Check for large time gaps in INS data and fill with level flight + if 0 + figure(1); clf; + bad_mask = diff(ins.gps_time) > 10; + bad_mask = grow(bad_mask,1); + time_skips = diff(ins.gps_time); + along_track = geodetic_to_along_track(gps.lat,gps.lon); + vel = diff(along_track) ./ diff(gps.gps_time); + plot(gps.gps_time(1:end-1),vel) + hold on + plot(ins.gps_time(bad_mask),time_skips(bad_mask),'r'); + hold off; + + figure(2); clf; + plot(ins.gps_time,ins.roll*180/pi,'.') + hold on + plot(ins.gps_time(bad_mask),time_skips(bad_mask),'r'); + hold off; + ylim([-20 20]) + grid on + + figure(3); clf; + plot(gps.gps_time,est_heading*180/pi,'g.'); + hold on + plot(ins.gps_time,ins.heading*180/pi,'.') + plot(ins.gps_time(bad_mask),time_skips(bad_mask),'r'); + hold off; + grid on + end + + bad_idxs = find(diff(ins.gps_time) > 20); + if ins.gps_time(1) - gps.gps_time(1) > 10 + bad_idxs = [0 bad_idxs]; + end + if gps.gps_time(end) - ins.gps_time(end) > 10 + bad_idxs = [bad_idxs length(ins.gps_time)]; + end + for bad_idxs_idx = 1:length(bad_idxs) + idx = bad_idxs(bad_idxs_idx); + + % Find gps indices in this gap + if idx == 0 + new_gps_times = gps.gps_time(1):ins.gps_time(idx+1)-10; + edge_gps_times = [gps.gps_time(1) ins.gps_time(idx+1)]; + est_heading_error = [0 ins.heading(idx+1) - gps_interp1(gps.gps_time,est_heading,ins.gps_time(idx+1))]; + elseif idx == length(ins.gps_time) + new_gps_times = ins.gps_time(idx)+10:gps.gps_time(end); + edge_gps_times = [ins.gps_time(idx) gps.gps_time(end)]; + est_heading_error = [ins.heading(idx) - gps_interp1(gps.gps_time,est_heading,ins.gps_time(idx)) 0]; + else + new_gps_times = ins.gps_time(idx)+10:ins.gps_time(idx+1)-10; + edge_gps_times = ins.gps_time(idx:idx+1); + est_heading_error = ins.heading(idx:idx+1) - gps_interp1(gps.gps_time,est_heading,ins.gps_time(idx:idx+1)); + end + bad_idxs(bad_idxs_idx+1:end) = bad_idxs(bad_idxs_idx+1:end) + length(new_gps_times); + fprintf(' Inserting %d level flight records due to missing INS data\n', length(new_gps_times)); + + ins.roll = [ins.roll(1:idx) zeros(size(new_gps_times)) ins.roll(idx+1:end)]; + ins.pitch = [ins.pitch(1:idx) zeros(size(new_gps_times)) ins.pitch(idx+1:end)]; + ins.lat = [ins.lat(1:idx) NaN*zeros(size(new_gps_times)) ins.lat(idx+1:end)]; + ins.lon = [ins.lon(1:idx) NaN*zeros(size(new_gps_times)) ins.lon(idx+1:end)]; + ins.elev = [ins.elev(1:idx) NaN*zeros(size(new_gps_times)) ins.elev(idx+1:end)]; + + inserted_heading = gps_interp1(gps.gps_time,est_heading,new_gps_times) ... + + gps_interp1(edge_gps_times,est_heading_error,new_gps_times); + ins.heading = [ins.heading(1:idx) inserted_heading ins.heading(idx+1:end)]; + + ins.gps_time = [ins.gps_time(1:idx) new_gps_times ins.gps_time(idx+1:end)]; + + end + + if 0 + figure(4); clf; + plot(gps.gps_time,est_heading*180/pi,'g.'); + hold on + plot(ins.gps_time,ins.heading*180/pi,'.') + hold off; + grid on + keyboard + end + + % Insert filler points from gps (ensures that sparse ins data does not + % create a sparse final dataset when gps data is available) + new_gps_time = union(gps.gps_time,ins.gps_time); + ins.roll = interp1(ins.gps_time,ins.roll,new_gps_time); + ins.pitch = interp1(ins.gps_time,ins.pitch,new_gps_time); + ins.heading = gps_interp1(ins.gps_time,ins.heading,new_gps_time); + interp_idxs = find(ins.gps_time >= gps.gps_time(1) & ins.gps_time <= gps.gps_time(end)); + ins.lat(interp_idxs) = interp1(gps.gps_time,gps.lat,ins.gps_time(interp_idxs)); + ins.lon(interp_idxs) = gps_interp1(gps.gps_time,gps.lon/180*pi,ins.gps_time(interp_idxs))*180/pi; + ins.elev(interp_idxs) = interp1(gps.gps_time,gps.elev,ins.gps_time(interp_idxs)); + ins.lat = interp1(ins.gps_time,ins.lat,new_gps_time); + ins.lon = gps_interp1(ins.gps_time,ins.lon/180*pi,new_gps_time)*180/pi; + ins.elev = interp1(ins.gps_time,ins.elev,new_gps_time); + ins.gps_time = new_gps_time; + gps = ins; + + %% Remove records with NaN in gps_time or trajectory + good_mask = ~(isnan(gps.gps_time) | isnan(gps.lat) ... + | isnan(gps.lon) | isnan(gps.elev)); + gps.gps_time = gps.gps_time(good_mask); + gps.lat = gps.lat(good_mask); + gps.lon = gps.lon(good_mask); + gps.elev = gps.elev(good_mask); + gps.roll = gps.roll(good_mask); + gps.pitch = gps.pitch(good_mask); + gps.heading = gps.heading(good_mask); + if isfield(gps,'radar_time') + gps.radar_time = gps.radar_time(good_mask); + end + if isfield(gps,'comp_time') + gps.comp_time = gps.comp_time(good_mask); + end + gps.gps_source = gps_source{file_idx}; + + %% Interpolate through NaN attitude data + gps.roll = interp_finite(gps.roll,0); + gps.pitch = interp_finite(gps.pitch,0); + gps.heading = interp_finite(gps.heading,0,@gps_interp1); + end + + end + + %% Now that INS data may have been added, check/make the GPS data monotonic in time in case it is not + [gps,~,monotonic_idxs] = gps_force_monotonic(gps); + + %% Using estimated heading based on the trajectory if heading is all zeros + if all(gps.heading == 0) + warning('These input files have heading(:) == 0. Using the estimated heading.'); + gps.heading = est_heading(monotonic_idxs); + end + + %% Heading and longitude modulo-2*pi + gps.heading = angle(exp(1i*gps.heading)); + gps.lon = angle(exp(1i*gps.lon/180*pi))*180/pi; + + %% Add software revision information + gps.sw_version = current_software_version; + + %% Add date_str and season_name + gps.date_str = date_str{file_idx}; + gps.season_name = season_name; + + %% Save output file + fprintf('Output_file\t%s\t%s\n', out_fn, datestr(now,'yyyymmdd_HHMMSS')); + gps.file_version = '2'; + gps.file_type = 'gps'; + if sync_flag{file_idx} + % Add the Radar Synchronization variables for mcrds, accum2, acords, + % arena + + % Synchronize computer time and radar time from NMEA file to gps time + gps.sync_gps_time = sync_gps.gps_time; + gps.sync_lat = sync_gps.lat; + gps.sync_lon = sync_gps.lon; + gps.sync_elev = sync_gps.elev; + gps.comp_time = sync_gps.comp_time; + if isfield(sync_gps,'radar_time') + gps.radar_time = sync_gps.radar_time; + end + + if isfield(gps,'radar_time') + ct_save(out_fn,'-v7.3','-STRUCT','gps','gps_time','lat','lon','elev','roll','pitch','heading','gps_source','sync_gps_time','sync_lat','sync_lon','sync_elev','comp_time','radar_time','sw_version','file_version','file_type','season_name','date_str'); + else + ct_save(out_fn,'-v7.3','-STRUCT','gps','gps_time','lat','lon','elev','roll','pitch','heading','gps_source','sync_gps_time','sync_lat','sync_lon','sync_elev','comp_time','sw_version','file_version','file_type','season_name','date_str'); + end + else + out_fields = {}; + if isfield(gps,'radar_time') + out_fields{end+1} = 'radar_time'; + end + if isfield(gps,'comp_time') + out_fields{end+1} = 'comp_time'; + end + ct_save(out_fn,'-v7.3','-STRUCT','gps','gps_time','lat','lon','elev','roll','pitch','heading','gps_source',out_fields{:},'sw_version','file_version','file_type','season_name','date_str'); + end + +end diff --git a/cresis-toolbox/gps/gps_make_fh.m b/cresis-toolbox/gps/gps_create_fh.m similarity index 80% rename from cresis-toolbox/gps/gps_make_fh.m rename to cresis-toolbox/gps/gps_create_fh.m index 12ef5459..899fb3e5 100644 --- a/cresis-toolbox/gps/gps_make_fh.m +++ b/cresis-toolbox/gps/gps_create_fh.m @@ -1,43 +1,47 @@ -function gps_fh = gps_make_fh(cur_file_type) -% gps_fh = gps_make_fh(cur_file_type) -% -% Support function for gps_make.m -% -% Author: John Paden - -%% Determine which function to use to load the GPS file -switch (lower(cur_file_type)) - case 'applanix' - gps_fh = @read_gps_applanix; - case 'arena' - gps_fh = @read_gps_arena; - case 'arena_cpu_time' - gps_fh = @read_gps_arena_cpu_time; - case 'awi_netcdf' - gps_fh = @read_gps_netcdf; - case 'cresis' - gps_fh = @read_gps_cresis; - case 'dmsraw' - gps_fh = @read_gps_dmsraw; - case 'general_ascii' - gps_fh = @read_gps_general_ascii; - case 'litton' - gps_fh = @read_ins_litton; - case 'litton_dgps' - gps_fh = @read_gps_litton; - case 'nmea' - gps_fh = @read_gps_nmea; - case 'novatel' - gps_fh = @read_gps_novatel; - case 'novatel_rpygga' - gps_fh = @read_gps_novatel_rpygga; - case 'traj' - gps_fh = @read_gps_traj; - case 'txt' - gps_fh = @read_gps_txt; - case 'csv' - gps_fh = @read_gps_csv; - otherwise - error('Unrecognized GPS file type %s', cur_file_type); -end - +function gps_fh = gps_create_fh(cur_file_type) +% gps_fh = gps_create_fh(cur_file_type) +% +% Support function for gps_create.m +% +% Author: John Paden + +%% Determine which function to use to load the GPS file +switch (lower(cur_file_type)) + case 'applanix' + gps_fh = @read_gps_applanix; + case 'arena' + gps_fh = @read_gps_arena; + case 'arena_cpu_time' + gps_fh = @read_gps_arena_cpu_time; + case 'awi_netcdf' + gps_fh = @read_gps_netcdf; + case 'cresis' + gps_fh = @read_gps_cresis; + case 'dmsraw' + gps_fh = @read_gps_dmsraw; + case 'general_ascii' + gps_fh = @read_gps_general_ascii; + case 'litton' + gps_fh = @read_ins_litton; + case 'litton_dgps' + gps_fh = @read_gps_litton; + case 'nmea' + gps_fh = @read_gps_nmea; + case 'novatel' + gps_fh = @read_gps_novatel; + case 'novatel_rpygga' + gps_fh = @read_gps_novatel_rpygga; + case 'traj' + gps_fh = @read_gps_traj; + case 'txt' + gps_fh = @read_gps_txt; + case 'csv' + gps_fh = @read_gps_csv; + case 'novatelraw' + gps_fh = @read_gps_novatelraw; + case 'utig' + gps_fh = @read_gps_utig; + otherwise + error('Unrecognized GPS file type %s', cur_file_type); +end + diff --git a/cresis-toolbox/gps/gps_make_monotonic.m b/cresis-toolbox/gps/gps_force_monotonic.m similarity index 61% rename from cresis-toolbox/gps/gps_make_monotonic.m rename to cresis-toolbox/gps/gps_force_monotonic.m index 839db678..7bc9cb24 100644 --- a/cresis-toolbox/gps/gps_make_monotonic.m +++ b/cresis-toolbox/gps/gps_force_monotonic.m @@ -1,12 +1,12 @@ -function [gps,error_flag] = gps_make_monotonic(gps) -% [gps,error_flag] = gps_make_monotonic(gps) +function [gps,error_flag,monotonic_idxs] = gps_force_monotonic(gps) +% [gps,error_flag] = gps_force_monotonic(gps) % % Check gps structure for non-monotonically increasing GPS time and return % a corrected gps structure. % % Author: John Paden % -% See also read_gps_*.m, gps_plot.m, gps_make.m, gps_make_monotonic.m +% See also read_gps_*.m, gps_plot.m, gps_create.m, gps_force_monotonic.m error_flag = false; @@ -14,16 +14,23 @@ return; end +%% Sort records according to gps.gps_time [~,sort_idxs] = sort(gps.gps_time); if any(sort_idxs ~= 1:length(sort_idxs)) - warning('GPS time is not monotonically increasing. Manual inspection is suggested. May need to run gps_make_monotonic.m in gps_make_SEASON.m for this particular file.'); + warning('GPS time is not monotonically increasing. Manual inspection is suggested. May need to run gps_force_monotonic.m in gps_create_SEASON.m for this particular file.'); gps.gps_time = gps.gps_time(sort_idxs); gps.lat = gps.lat(sort_idxs); gps.lon = gps.lon(sort_idxs); gps.elev = gps.elev(sort_idxs); - gps.roll = gps.roll(sort_idxs); - gps.pitch = gps.pitch(sort_idxs); - gps.heading = gps.heading(sort_idxs); + if isfield(gps,'roll') + gps.roll = gps.roll(sort_idxs); + end + if isfield(gps,'pitch') + gps.pitch = gps.pitch(sort_idxs); + end + if isfield(gps,'heading') + gps.heading = gps.heading(sort_idxs); + end if isfield(gps,'radar_time') gps.radar_time = gps.radar_time(sort_idxs); end @@ -36,18 +43,25 @@ error_flag = true; end +%% Remove nonmonotonically increasing records from gps.gps_time good_mask = [true, (diff(gps.gps_time) > 0)]; if ~all(good_mask) - warning('GPS time has repeated records. Manual inspection is suggested. May need to run gps_make_monotonic.m in gps_make_SEASON.m for this particular file.'); + warning('GPS time has repeated records. Manual inspection is suggested. May need to run gps_force_monotonic.m in gps_create_SEASON.m for this particular file.'); gps.gps_time = gps.gps_time(good_mask); gps.lat = gps.lat(good_mask); gps.lon = gps.lon(good_mask); gps.elev = gps.elev(good_mask); - gps.roll = gps.roll(good_mask); - gps.pitch = gps.pitch(good_mask); - gps.heading = gps.heading(good_mask); + if isfield(gps,'roll') + gps.roll = gps.roll(good_mask); + end + if isfield(gps,'pitch') + gps.pitch = gps.pitch(good_mask); + end + if isfield(gps,'heading') + gps.heading = gps.heading(good_mask); + end if isfield(gps,'radar_time') gps.radar_time = gps.radar_time(good_mask); end @@ -59,8 +73,10 @@ end error_flag = true; end +monotonic_idxs = sort_idxs(good_mask); -if isfield(gps,'radar_time') +%% Remove nonmonotonically increasing records from gps.radar_time +if isfield(gps,'radar_time') && any(~isnan(gps.radar_time)) [~,sort_idxs] = sort(gps.radar_time); if any(sort_idxs ~= 1:length(sort_idxs)) warning('Radar time is not monotonically increasing. Manual inspection is suggested.'); @@ -70,15 +86,21 @@ good_mask = [true, (diff(gps.radar_time) > 0)]; if ~all(good_mask) - warning('radar_time has repeated records. Manual inspection is suggested. May need to run gps_make_monotonic.m in gps_make_SEASON.m for this particular file.'); + warning('radar_time has repeated records. Manual inspection is suggested. May need to run gps_force_monotonic.m in gps_create_SEASON.m for this particular file.'); gps.gps_time = gps.gps_time(good_mask); gps.lat = gps.lat(good_mask); gps.lon = gps.lon(good_mask); gps.elev = gps.elev(good_mask); - gps.roll = gps.roll(good_mask); - gps.pitch = gps.pitch(good_mask); - gps.heading = gps.heading(good_mask); + if isfield(gps,'roll') + gps.roll = gps.roll(good_mask); + end + if isfield(gps,'pitch') + gps.pitch = gps.pitch(good_mask); + end + if isfield(gps,'heading') + gps.heading = gps.heading(good_mask); + end gps.radar_time = gps.radar_time(good_mask); if isfield(gps,'comp_time') gps.comp_time = gps.comp_time(good_mask); @@ -88,6 +110,8 @@ end error_flag = true; end + + monotonic_idxs = monotonic_idxs(good_mask); end end diff --git a/cresis-toolbox/gps/gps_interp1.m b/cresis-toolbox/gps/gps_interp1.m new file mode 100644 index 00000000..0e4c21ff --- /dev/null +++ b/cresis-toolbox/gps/gps_interp1.m @@ -0,0 +1,16 @@ +function yq = gps_interp1(xi,yi,xq,varargin) +% yq = gps_interp1(xi,yi,xq,varargin) +% +% Interpolates heading and longitude properly through 2*pi wraps. +% +% The data is assumed to be in radians! +% +% Examples: +% gps.lon = gps_interp1(old_gps_time, gps.lon/180*pi, new_gps_time)*180/pi; +% gps.heading = gps_interp1(old_gps_time, gps.heading, new_gps_time); + +yi_x = cos(yi); +yi_y = sin(yi); +yq_x = interp1(xi, yi_x, xq, varargin{:}); +yq_y = interp1(xi, yi_y, xq, varargin{:}); +yq = atan2(yq_y,yq_x); diff --git a/cresis-toolbox/gps/gps_load.m b/cresis-toolbox/gps/gps_load.m new file mode 100644 index 00000000..ff32176f --- /dev/null +++ b/cresis-toolbox/gps/gps_load.m @@ -0,0 +1,223 @@ +function gps = gps_load(gps_fn) +% gps = gps_load(gps_fn) +% +% gps_load('/cresis/snfs1/dataproducts/csarp_support/gps/2019_Antarctica_Ground/gps_20200107.mat'); +% +% Author: John Paden +% +% See also: gps_check, gps_load, gps_create + +gps = load(gps_fn); +old_gps = gps; + +%% Update header fields +if ~isfield(gps,'file_version') || isempty(gps.file_version) + warning('gps file missing file_version field. Adding field.'); + gps.file_version = '0'; +end + +file_version = gps.file_version(isstrprop(gps.file_version,'digit')); +if str2double(file_version) < 2 + warning('gps file is old file_version. Updating field.'); + if any(file_version == 'L') + gps.file_version = '2L'; + else + gps.file_version = '2'; + end +end + +if ~isfield(gps,'season_name') || isempty(gps.season_name) + warning('gps file missing season_name field. Adding field.'); + % Default param.season_name is the directory where the file is if the + % file is in the standard location. + [gps_fn_dir,gps_fn_name] = fileparts(gps_fn); + [~,gps.season_name] = fileparts(gps_fn_dir); + % Verify param.season_name + if 0 + season_name = input(sprintf('gps_load: Please enter the season name [%s]: ',gps.season_name),'s'); + if ~all(isstrprop(season_name,'white')) + gps.season_name = season_name; + end + end +end + +if ~isfield(gps,'date_str') || isempty(gps.date_str) + warning('gps file missing date_str field. Adding field.'); + % Default param.date_str is in the gps filename by default. + gps.date_str = gps_fn_name(5:end); + if 0 + date_str = input(sprintf('gps_load: Please enter the date string (YYYYMMDD) [%s]: ',gps.date_str),'s'); + if ~all(isstrprop(date_str,'white')) + gps.date_str = date_str; + end + end +end + +if ~isfield(gps,'file_type') || isempty(gps.file_type) + warning('gps file missing file_type field. Adding field.'); + gps.file_type = 'gps'; +end + +if ~isfield(gps,'gps_source') || isempty(gps.gps_source) + warning('gps file missing gps_source field. Adding field.'); + gps.gps_source = ''; +end + +if ~isfield(gps,'sw_version') || isempty(gps.sw_version) + warning('gps file missing sw_version field. Adding field.'); + gps.sw_version = current_software_version; +end + +%% correct shape of vectors (1xN arrays) +if size(gps.gps_time,1) > 1 + warning('gps file has column vector for gps_time. Making into row vector.'); + gps.gps_time = gps.gps_time(:).'; +end +if size(gps.lat,1) > 1 + warning('gps file has column vector for lat. Making into row vector.'); + gps.lat = gps.lat(:).'; +end +if size(gps.lon,1) > 1 + warning('gps file has column vector for lon. Making into row vector.'); + gps.lon = gps.lon(:).'; +end +if size(gps.elev,1) > 1 + warning('gps file has column vector for elev. Making into row vector.'); + gps.elev = gps.elev(:).'; +end +if size(gps.roll,1) > 1 + warning('gps file has column vector for roll. Making into row vector.'); + gps.roll = gps.roll(:).'; +end +if size(gps.pitch,1) > 1 + warning('gps file has column vector for pitch. Making into row vector.'); + gps.pitch = gps.pitch(:).'; +end +if size(gps.heading,1) > 1 + warning('gps file has column vector for heading. Making into row vector.'); + gps.heading = gps.heading(:).'; +end +if isfield(gps,'radar_time') && size(gps.radar_time,1) > 1 + warning('gps file has column vector for radar_time. Making into row vector.'); + gps.radar_time = gps.radar_time(:).'; +end +if isfield(gps,'comp_time') && size(gps.comp_time,1) > 1 + warning('gps file has column vector for comp_time. Making into row vector.'); + gps.comp_time = gps.comp_time(:).'; +end +if isfield(gps,'sync_gps_time') && size(gps.sync_gps_time,1) > 1 + warning('gps file has column vector for sync_gps_time. Making into row vector.'); + gps.sync_gps_time = gps.sync_gps_time(:).'; +end +if isfield(gps,'sync_lat') && size(gps.sync_lat,1) > 1 + warning('gps file has column vector for sync_lat. Making into row vector.'); + gps.sync_lat = gps.sync_lat(:).'; +end +if isfield(gps,'sync_lon') && size(gps.sync_lon,1) > 1 + warning('gps file has column vector for sync_lon. Making into row vector.'); + gps.sync_lon = gps.sync_lon(:).'; +end +if isfield(gps,'sync_elev') && size(gps.sync_elev,1) > 1 + warning('gps file has column vector for sync_elev. Making into row vector.'); + gps.sync_elev = gps.sync_elev(:).'; +end + +%% Remove nonmonotonically increasing records +tmp_gps = struct('gps_time',gps.gps_time); +tmp_gps.lat = gps.lat; +tmp_gps.lon = gps.lon; +tmp_gps.elev = gps.elev; +tmp_gps.roll = gps.roll; +tmp_gps.pitch = gps.pitch; +tmp_gps.heading = gps.heading; +[tmp_gps,error_flag] = gps_force_monotonic(tmp_gps); +if error_flag + warning('gps file has nonmonotonic records. Correcting.'); + gps.gps_time = tmp_gps.gps_time; + gps.lat = tmp_gps.lat; + gps.lon = tmp_gps.lon; + gps.elev = tmp_gps.elev; + gps.roll = tmp_gps.roll; + gps.pitch = tmp_gps.pitch; + gps.heading = tmp_gps.heading; +end +%% Remove records with NaN in gps_time or trajectory +good_mask = ~(isnan(gps.gps_time) | isnan(gps.lat) ... + | isnan(gps.lon) | isnan(gps.elev)); +if any(~good_mask) + warning('gps file has NaN gps_time or trajectory. Removing.'); + gps.gps_time = gps.gps_time(good_mask); + gps.lat = gps.lat(good_mask); + gps.lon = gps.lon(good_mask); + gps.elev = gps.elev(good_mask); + gps.roll = gps.roll(good_mask); + gps.pitch = gps.pitch(good_mask); + gps.heading = gps.heading(good_mask); +end + +%% Interpolate through NaN attitude data +good_mask = ~(isnan(gps.roll) | isnan(gps.pitch) | isnan(gps.heading)); +if any(~good_mask) + warning('gps file has NaN attitute. Interpolating.'); + gps.roll = interp_finite(gps.roll,0); + gps.pitch = interp_finite(gps.pitch,0); + gps.heading = interp_finite(gps.heading,0,@gps_interp1); +end + +%% sync_gps_time +if isfield(gps,'sync_gps_time') + %% sync_gps_time: Remove nonmonotonically increasing records + sync_gps = struct('gps_time',gps.sync_gps_time); + sync_gps.lat = gps.sync_lat; + sync_gps.lon = gps.sync_lon; + sync_gps.elev = gps.sync_elev; + if isfield(gps,'radar_time') + sync_gps.radar_time = gps.radar_time; + end + if isfield(gps,'comp_time') + sync_gps.comp_time = gps.comp_time; + end + [sync_gps,error_flag] = gps_force_monotonic(sync_gps); + if error_flag + warning('gps file has nonmonotonic sync_gps_time or radar_time records. Correcting.'); + end + + %% sync_gps_time: Remove records with NaN in gps_time or trajectory + if isfield(sync_gps,'radar_time') + good_mask = ~(isnan(sync_gps.gps_time) | isnan(sync_gps.radar_time)); + else + good_mask = ~(isnan(sync_gps.gps_time) | isnan(sync_gps.comp_time)); + end + if any(~good_mask) + warning('gps file has NaN sync_gps_time. Removing.'); + if isfield(sync_gps,'radar_time') + sync_gps.radar_time = sync_gps.radar_time(good_mask); + end + sync_gps.gps_time = sync_gps.gps_time(good_mask); + sync_gps.comp_time = sync_gps.comp_time(good_mask); + sync_gps.lat = sync_gps.lat(good_mask); + sync_gps.lon = sync_gps.lon(good_mask); + sync_gps.elev = sync_gps.elev(good_mask); + end + + %% sync_gps_time: Update + if error_flag || any(~good_mask) + gps.sync_gps_time = sync_gps.gps_time; + gps.sync_lat = sync_gps.lat; + gps.sync_lon = sync_gps.lon; + gps.sync_elev = sync_gps.elev; + gps.comp_time = sync_gps.comp_time; + if isfield(gps,'radar_time') + gps.radar_time = sync_gps.radar_time; + end + end +end + +%% Save updated gps record +if ~isequal(gps,old_gps) + if cluster_job_check() + error('gps file needs to be updated but may not be from cluster_job (gRadar.cluster.is_cluster_job is currently set to true). To remove this error, run gps_load on: %s', gps_fn); + end + fprintf(' Saving updated gps file %s.\n', gps_fn); + ct_save(gps_fn,'-struct','gps'); +end diff --git a/cresis-toolbox/gps/gps_merge_sync.m b/cresis-toolbox/gps/gps_merge_sync.m deleted file mode 100644 index 47249b6d..00000000 --- a/cresis-toolbox/gps/gps_merge_sync.m +++ /dev/null @@ -1,238 +0,0 @@ -function gps = gps_merge_sync(gps,merge_type) -% gps = gps_merge_sync(gps,merge_type) -% -% Merges sync NMEA and regular GPS data when sync NMEA has a longer extent than -% the regular GPS data. This function is called from gps_make.m. -% -% Author: Logan Smith - -if merge_type == 1 % sync_gps vector is longer than gps vector - warning('gps time vector is shorter than the sync gps time vector. Merging gps variables with sync gps variables...') - Ng = length(gps.gps_time); - Ns = length(gps.sync_gps_time); - gps_start_idx = find(gps.sync_gps_time >= gps.gps_time(1),1); - gps_stop_idx = find(gps.sync_gps_time <= gps.gps_time(end),1,'last'); - if gps_start_idx == 1 % gps_time starts before sync_gps_time - tmp_ext = Ns-gps_stop_idx; - tmp_len = Ng+tmp_ext; - sync_start_idx = gps_stop_idx+1; - sync_stop_idx = Ns; - elseif gps_stop_idx == Ns % gps_time ends after sync_gps_time - tmp_ext = gps_start_idx-1; - tmp_len = Ng+tmp_ext; - sync_start_idx = 1; - sync_stop_idx = gps_start_idx - 1; - else - tmp_len = Ns; % gps_time is encompassed by sync_gps_time - sync_start_idx = gps_stop_idx + 1; - sync_stop_idx = gps_start_idx - 1; - end - - tmp_gps_time = zeros(1,tmp_len); - tmp_lat = zeros(1,tmp_len); - tmp_lon = zeros(1,tmp_len); - tmp_elev = zeros(1,tmp_len); - tmp_pitch = zeros(1,tmp_len); - tmp_roll = zeros(1,tmp_len); - tmp_heading = zeros(1,tmp_len); - - if gps_start_idx == 1 % gps_time starts before sync_gps_time - tmp_gps_time(1:Ng) = gps.gps_time; - tmp_gps_time((Ng+1):end) = gps.sync_gps_time(sync_start_idx:end); - elseif tmp_len > Ns % gps_time ends after sync_gps_time - tmp_gps_time(gps_start_idx:gps_start_idx+Ng-1) = gps.gps_time; - tmp_gps_time(1:sync_stop_idx) = gps.sync_gps_time(1:sync_stop_idx); - else % gps_time is encompassed by sync_gps_time - tmp_gps_time(gps_start_idx:gps_start_idx+Ng-1) = gps.gps_time; - tmp_gps_time(1:sync_stop_idx) = gps.sync_gps_time(1:sync_stop_idx); - tmp_gps_time(sync_start_idx:end) = gps.sync_gps_time(sync_start_idx:end); - end - - if gps_start_idx == 1 % gps_time starts before sync_gps_time - tmp_lat(1:Ng) = gps.lat; - tmp_lat((Ng+1):end) = gps.sync_lat(sync_start_idx:end); - elseif tmp_len > Ns % gps_time ends after sync_gps_time - tmp_lat(gps_start_idx:gps_start_idx+Ng-1) = gps.lat; - tmp_lat(1:sync_stop_idx) = gps.sync_lat(1:sync_stop_idx); - else % gps_time is encompassed by sync_gps_time - tmp_lat(gps_start_idx:gps_start_idx+Ng-1) = gps.lat; - tmp_lat(1:sync_stop_idx) = gps.sync_lat(1:sync_stop_idx); - tmp_lat(sync_start_idx:end) = gps.sync_lat(sync_start_idx:end); - end - - if abs((gps.sync_lon(sync_stop_idx) - gps.lon(1))) > 1e-2 - gps.sync_lon = mod(gps.sync_lon,360); - end - if gps_start_idx == 1 % gps_time starts before sync_gps_time - tmp_lon(1:Ng) = gps.lon; - tmp_lon((Ng+1):end) = gps.sync_lon(sync_start_idx:end); - elseif tmp_len > Ns % gps_time ends after sync_gps_time - tmp_lon(gps_start_idx:gps_start_idx+Ng-1) = gps.lon; - tmp_lon(1:sync_stop_idx) = gps.sync_lon(1:sync_stop_idx); - else % gps_time is encompassed by sync_gps_time - tmp_lon(gps_start_idx:gps_start_idx+Ng-1) = gps.lon; - tmp_lon(1:sync_stop_idx) = gps.sync_lon(1:sync_stop_idx); - tmp_lon(sync_start_idx:end) = gps.sync_lon(sync_start_idx:end); - end - - if gps_start_idx == 1 % gps_time starts before sync_gps_time - elev_shift = gps.elev(end) - gps.sync_elev(find(gps.sync_gps_time >= gps.gps_time(end),1)) - tmp_elev(1:Ng) = gps.elev; - tmp_elev((Ng+1):end) = gps.sync_elev(sync_start_idx:end) + elev_shift; - elseif tmp_len > Ns % gps_time ends after sync_gps_time - elev_shift = gps.elev(1) - gps.sync_elev(find(gps.sync_gps_time >= gps.gps_time(1),1)) - tmp_elev(gps_start_idx:gps_start_idx+Ng-1) = gps.elev; - tmp_elev(1:sync_stop_idx) = gps.sync_elev(1:sync_stop_idx) + elev_shift; - else % gps_time is encompassed by sync_gps_time - elev_shift = gps.elev(1) - gps.sync_elev(find(gps.sync_gps_time >= gps.gps_time(1),1)) - tmp_elev(gps_start_idx:gps_start_idx+Ng-1) = gps.elev; - tmp_elev(1:sync_stop_idx) = gps.sync_elev(1:sync_stop_idx) + elev_shift; - tmp_elev(sync_start_idx:end) = gps.sync_elev(sync_start_idx:end) + elev_shift; - end - - if gps_start_idx == 1 % gps_time starts before sync_gps_time - tmp_roll(1:Ng) = gps.roll; - tmp_roll((Ng+1):end) = zeros(1,Ns-sync_start_idx+1); - elseif tmp_len > Ns % gps_time ends after sync_gps_time - tmp_roll(gps_start_idx:gps_start_idx+Ng-1) = gps.roll; - tmp_roll(1:sync_stop_idx) = zeros(1,sync_stop_idx); - else % gps_time is encompassed by sync_gps_time - tmp_roll(gps_start_idx:gps_start_idx+Ng-1) = gps.roll; - tmp_roll(1:sync_stop_idx) = zeros(1,sync_stop_idx); - tmp_roll(sync_start_idx:end) = zeros(1,Ns-sync_start_idx+1); - end - - if gps_start_idx == 1 % gps_time starts before sync_gps_time - tmp_pitch(1:Ng) = gps.pitch; - tmp_pitch((Ng+1):end) = zeros(1,Ns-sync_start_idx+1); - elseif tmp_len > Ns % gps_time ends after sync_gps_time - tmp_pitch(gps_start_idx:gps_start_idx+Ng-1) = gps.pitch; - tmp_pitch(1:sync_stop_idx) = zeros(1,sync_stop_idx); - else % gps_time is encompassed by sync_gps_time - tmp_pitch(gps_start_idx:gps_start_idx+Ng-1) = gps.pitch; - tmp_pitch(1:sync_stop_idx) = zeros(1,sync_stop_idx); - tmp_pitch(sync_start_idx:end) = zeros(1,Ns-sync_start_idx+1); - end - - if gps_start_idx == 1 % gps_time starts before sync_gps_time - tmp_heading(1:Ng) = gps.heading; - tmp_heading((Ng+1):end) = gps.sync_heading(sync_start_idx:end); - elseif tmp_len > Ns % gps_time ends after sync_gps_time - tmp_heading(gps_start_idx:gps_start_idx+Ng-1) = gps.heading; - tmp_heading(1:sync_stop_idx) = gps.sync_heading(1:sync_stop_idx); - else % gps_time is encompassed by sync_gps_time - tmp_heading(gps_start_idx:gps_start_idx+Ng-1) = gps.heading; - tmp_heading(1:sync_stop_idx) = gps.sync_heading(1:sync_stop_idx); - tmp_heading(sync_start_idx:end) = gps.sync_heading(sync_start_idx:end); - end - -elseif merge_type == 2 % sync_gps time range is longer than gps time range - Ng = length(gps.gps_time); - Ns = length(gps.sync_gps_time); - gps_start_idx = find(gps.sync_gps_time >= gps.gps_time(1),1); - gps_stop_idx = find(gps.sync_gps_time <= gps.gps_time(end),1,'last'); - if gps_start_idx == 1 % sync_gps_time extends beyond gps_time - tmp_ext = Ns-gps_stop_idx; - tmp_len = Ng+tmp_ext; - sync_start_idx = gps_stop_idx+1; - sync_stop_idx = Ns; - elseif gps_stop_idx == Ns % sync_gps_time starts before gps_time - tmp_ext = gps_start_idx-1; - tmp_len = Ng+tmp_ext; - sync_start_idx = 1; - sync_stop_idx = gps_start_idx - 1; - end - - tmp_gps_time = zeros(1,tmp_len); - tmp_lat = zeros(1,tmp_len); - tmp_lon = zeros(1,tmp_len); - tmp_elev = zeros(1,tmp_len); - tmp_pitch = zeros(1,tmp_len); - tmp_roll = zeros(1,tmp_len); - tmp_heading = zeros(1,tmp_len); - - if gps_start_idx == 1 % sync_gps_time extends beyond gps_time - tmp_gps_time(1:Ng) = gps.gps_time; - tmp_gps_time((Ng+1):end) = gps.sync_gps_time(sync_start_idx:end); - elseif gps_stop_idx == Ns % sync_gps_time starts before gps_time - tmp_gps_time(gps_start_idx:gps_start_idx+Ng-1) = gps.gps_time; - tmp_gps_time(1:sync_stop_idx) = gps.sync_gps_time(1:sync_stop_idx); - end - - if gps_start_idx == 1 % sync_gps_time extends beyond gps_time - tmp_lat(1:Ng) = gps.lat; - tmp_lat((Ng+1):end) = gps.sync_lat(sync_start_idx:end); - elseif tmp_len > Ns % sync_gps_time starts before gps_time - tmp_lat(gps_start_idx:gps_start_idx+Ng-1) = gps.lat; - tmp_lat(1:sync_stop_idx) = gps.sync_lat(1:sync_stop_idx); - end - - if abs((gps.sync_lon(sync_stop_idx) - gps.lon(1))) > 1e-2 - gps.sync_lon = mod(gps.sync_lon,360); - end - if gps_start_idx == 1 % sync_gps_time extends beyond gps_time - tmp_lon(1:Ng) = gps.lon; - tmp_lon((Ng+1):end) = gps.sync_lon(sync_start_idx:end); - elseif tmp_len > Ns % sync_gps_time starts before gps_time - tmp_lon(gps_start_idx:gps_start_idx+Ng-1) = gps.lon; - tmp_lon(1:sync_stop_idx) = gps.sync_lon(1:sync_stop_idx); - end - - if gps_start_idx == 1 % sync_gps_time extends beyond gps_time - elev_shift = gps.elev(end) - gps.sync_elev(find(gps.sync_gps_time >= gps.gps_time(end),1)) - tmp_elev(1:Ng) = gps.elev; - tmp_elev((Ng+1):end) = gps.sync_elev(sync_start_idx:end) + elev_shift; - elseif tmp_len > Ns % sync_gps_time starts before gps_time - elev_shift = gps.elev(1) - gps.sync_elev(find(gps.sync_gps_time >= gps.gps_time(1),1)) - tmp_elev(gps_start_idx:gps_start_idx+Ng-1) = gps.elev; - tmp_elev(1:sync_stop_idx) = gps.sync_elev(1:sync_stop_idx) + elev_shift; - end - - if gps_start_idx == 1 % sync_gps_time extends beyond gps_time - tmp_roll(1:Ng) = gps.roll; - tmp_roll((Ng+1):end) = zeros(1,Ns-sync_start_idx+1); - elseif tmp_len > Ns % sync_gps_time starts before gps_time - tmp_roll(gps_start_idx:gps_start_idx+Ng-1) = gps.roll; - tmp_roll(1:sync_stop_idx) = zeros(1,sync_stop_idx); - end - - if gps_start_idx == 1 % sync_gps_time extends beyond gps_time - tmp_pitch(1:Ng) = gps.pitch; - tmp_pitch((Ng+1):end) = zeros(1,Ns-sync_start_idx+1); - elseif tmp_len > Ns % sync_gps_time starts before gps_time - tmp_pitch(gps_start_idx:gps_start_idx+Ng-1) = gps.pitch; - tmp_pitch(1:sync_stop_idx) = zeros(1,sync_stop_idx); - end - - if gps_start_idx == 1 % sync_gps_time extends beyond gps_time - tmp_heading(1:Ng) = gps.heading; - tmp_heading((Ng+1):end) = gps.sync_heading(sync_start_idx:end); - elseif tmp_len > Ns % sync_gps_time starts before gps_time - tmp_heading(gps_start_idx:gps_start_idx+Ng-1) = gps.heading; - tmp_heading(1:sync_stop_idx) = gps.sync_heading(1:sync_stop_idx); - end -end - - - -fprintf('Check gps merging results now...\n') -keyboard -if 0 - plot(tmp_gps_time,'.') - plot(diff(tmp_gps_time),'.') - plot(tmp_elev,'.') - plot(diff(tmp_elev),'.') - plot(tmp_lon,tmp_lat,'.') - plot(tmp_roll,'.') - plot(tmp_pitch,'.') - plot(tmp_heading,'.') -end - -gps.gps_time = tmp_gps_time; -gps.lat = tmp_lat; -gps.lon = tmp_lon; -gps.elev = tmp_elev; -gps.roll = tmp_roll; -gps.pitch = tmp_pitch; -gps.heading = tmp_heading; -return \ No newline at end of file diff --git a/cresis-toolbox/gps/gps_plot.m b/cresis-toolbox/gps/gps_plot.m index 31af791c..e3707275 100644 --- a/cresis-toolbox/gps/gps_plot.m +++ b/cresis-toolbox/gps/gps_plot.m @@ -13,24 +13,27 @@ % % Example: % -% % Compare two GPS sources: -% fn = '/cresis/scratch1/mdce/csarp_support/gps/2009_Antarctica_DC8_GPS/20091018_ALL_pos.mat'; -% gps_plot(fn); -% fn = '/cresis/scratch1/mdce/csarp_support/gps/2009_Antarctica_DC8_DGPSwINS/20091018_ALL_pos.mat'; -% gps_plot(fn,'r'); +% % Check csarp_support/gps file +% gps_plot('/cresis/snfs1/dataproducts/csarp_support/gps/2019_Antarctica_Ground/gps_20200107.mat'); % -% % Use GPS struct -% fn = '/cresis/data2/MCoRDS/2009_Chile/GPS/IWG1_110209.log'; -% gps = read_gps_reveal(fn); -% gps_plot(gps); +% % Compare two GPS sources: +% fn = '/cresis/snfs1/dataproducts/csarp_support/gps/2009_Antarctica_DC8_GPS/20091018_ALL_pos.mat'; +% gps_plot(fn); +% fn = '/cresis/snfs1/dataproducts/csarp_support/gps/2009_Antarctica_DC8_DGPSwINS/20091018_ALL_pos.mat'; +% gps_plot(fn,'r'); % -% fn = '/cresis/scratch1/mdce/csarp_support/gps/2009_Antarctica_DC8_DGPSwINS/Javadsbet_02Nov09_PPP_Revised.out' -% gps = read_gps_applanix(fn,struct('year',2009,'month',11,'day',2)); -% gps_plot(gps); +% % Use GPS struct +% fn = '/cresis/data2/MCoRDS/2009_Chile/GPS/IWG1_110209.log'; +% gps = read_gps_reveal(fn); +% gps_plot(gps); +% +% fn = '/cresis/scratch1/mdce/csarp_support/gps/2009_Antarctica_DC8_DGPSwINS/Javadsbet_02Nov09_PPP_Revised.out' +% gps = read_gps_applanix(fn,struct('year',2009,'month',11,'day',2)); +% gps_plot(gps); % % Author: John Paden % -% See also read_gps_*.m, gps_plot.m, gps_make.m +% See also read_gps_*.m, gps_plot.m, gps_create.m if ~exist('plot_est_heading','var') || isempty(plot_est_heading) plot_est_heading = false; @@ -66,7 +69,9 @@ gps_time_datenum = epoch_to_datenum(gps.gps_time); [year month day hour minute sec] = datevec(gps_time_datenum); -GPS_sod = (day-day(1))*86400+hour*3600+minute*60+sec; % GPS seconds of day +day_jumps = day-day(1); +day_jumps(day_jumps<0) = 1; % this is the case at month jump +GPS_sod = day_jumps*86400+hour*3600+minute*60+sec; % GPS seconds of day fig = 7; if isfield(gps,'sync_gps_time') @@ -129,20 +134,24 @@ fprintf(2,'Length of heading does not match gps_time.\n', length(gps.heading), length(gps.gps_time)); end -if isfield(gps,'comp_time') && length(gps.comp_time) ~= length(gps.sync_gps_time) - fprintf(2,'Length of comp_time does not match sync_gps_time.\n', length(gps.comp_time), length(gps.sync_gps_time)); -end -if isfield(gps,'radar_time') && length(gps.radar_time) ~= length(gps.sync_gps_time) - fprintf(2,'Length of radar_time does not match sync_gps_time.\n', length(gps.radar_time), length(gps.sync_gps_time)); -end -if isfield(gps,'sync_lat') && length(gps.sync_lat) ~= length(gps.sync_gps_time) - fprintf(2,'Length of sync_lat does not match sync_gps_time.\n', length(gps.sync_lat), length(gps.sync_gps_time)); -end -if isfield(gps,'sync_lon') && length(gps.sync_lon) ~= length(gps.sync_gps_time) - fprintf(2,'Length of sync_lon does not match sync_gps_time.\n', length(gps.sync_lon), length(gps.sync_gps_time)); -end -if isfield(gps,'sync_elev') && length(gps.sync_elev) ~= length(gps.sync_gps_time) - fprintf(2,'Length of sync_elev does not match sync_gps_time.\n', length(gps.sync_elev), length(gps.sync_gps_time)); +if ~isfield(gps,'sync_gps_time'); + warning('GPS file does not have sync_gps_time field.'); +else + if isfield(gps,'comp_time') && length(gps.comp_time) ~= length(gps.sync_gps_time) + fprintf(2,'Length of comp_time does not match sync_gps_time.\n', length(gps.comp_time), length(gps.sync_gps_time)); + end + if isfield(gps,'radar_time') && length(gps.radar_time) ~= length(gps.sync_gps_time) + fprintf(2,'Length of radar_time does not match sync_gps_time.\n', length(gps.radar_time), length(gps.sync_gps_time)); + end + if isfield(gps,'sync_lat') && length(gps.sync_lat) ~= length(gps.sync_gps_time) + fprintf(2,'Length of sync_lat does not match sync_gps_time.\n', length(gps.sync_lat), length(gps.sync_gps_time)); + end + if isfield(gps,'sync_lon') && length(gps.sync_lon) ~= length(gps.sync_gps_time) + fprintf(2,'Length of sync_lon does not match sync_gps_time.\n', length(gps.sync_lon), length(gps.sync_gps_time)); + end + if isfield(gps,'sync_elev') && length(gps.sync_elev) ~= length(gps.sync_gps_time) + fprintf(2,'Length of sync_elev does not match sync_gps_time.\n', length(gps.sync_elev), length(gps.sync_gps_time)); + end end %% Elevation Plot @@ -315,7 +324,7 @@ fprintf('Approx. number of repeats (non-monotonically increasing points in sync gps time):\n %d\n', length(repeat_idxs)) end -%% Sync GPS Time Plot +%% Radar Time Plot if isfield(gps,'radar_time') fig = fig+1; figure(h_fig(fig)); diff --git a/cresis-toolbox/gps/gps_to_csv.m b/cresis-toolbox/gps/gps_to_csv.m deleted file mode 100644 index 1ccb1471..00000000 --- a/cresis-toolbox/gps/gps_to_csv.m +++ /dev/null @@ -1,683 +0,0 @@ -function gps_to_csv() -%% gps_to_csv.m -% -% Take GPS flight data and: -% (1) converts the GPS file to a structure -% (2) converts the structure to a CSV file that can be plotted or -% manipulated with MATLAB or GIS software. -% (3) plots the data on a GeoTIFF -% There are already separate functions that read the GPS files into a -% structure; this script outputs them to a convenient, usable format. -% -% This function supports the following types of GPS files: -% param.input_type = 1 --> .gps (NMEA) files -% param.input_type = 2 --> ppp.txt (Novatel) files -% param.input_type = 3 --> .out (Applanix) files -% param.input_type = 4 --> .traj (ATM) files -% param.input_type = 5 --> ln100g.asc (Litton 100) files -% -% EXAMPLE -% ------------------------------------------------------------------------- -% param.input_type = 1; -% param.region = 'Antarctica'; -% param.fn = ('P:\metadata\2012_Greenland_P3\BD960_10Apr12_PPPK_P13Jun12.out'); -% param.fn = ('/cresis/projects/metadata/2012_Greenland_P3/BD960_10Apr12_PPPK_P13Jun12.out'); -% param.dec = 11; -% param.year = 2011; -% param.month = 27; -% param.day = 10; -% param.time_reference = 'utc'; -% param.csv_merge = false; -% csv_out_path = 'Y:\sfoga\2012_Greenland_P3_GPS.csv'; -% csv_out_path = '/cresis/scratch1/sfoga/2012_Greenland_P3_GPS.csv'; -% gps_to_csv(csv_out_path,param); -% ------------------------------------------------------------------------- -% -% See 'edit gps_to_csv.m' to see the full list of param options. -% -% Date: 6 August 2012 -% Author: Steven Foga -% -% See also read_gps_applanix.m, read_gps_atm.m, read_gps_litton.m, -% read_gps_novatel.m, read_gps_traj.m - -% Specify GPS files to plot - -% fns = get_filenames('/mnt/products/csarp_support/gps/2013_Antarctica_Basler/','gps','','.mat'); -fns = get_filenames('/mnt/products/csarp_support/gps/2013_Antarctica_Ground/','gps','','.mat'); - -% Specify output directory -% out_fn_dir = '/home/cresis1/gps_csv/'; -out_fn_dir = '/home/cresis1/gps_csv_accum/'; - -% Output along track spacing -spacing = 1000; - -for fn_idx = 1:length(fns) - fn = fns{fn_idx}; - [~,fn_name] = fileparts(fn); - out_fn = fullfile(out_fn_dir,[fn_name '.csv']); - fprintf('Converting %s\n to %s\n', fn, out_fn); - - gps = load(fn); - along_track = geodetic_to_along_track(gps.lat,gps.lon,gps.elev); - decim_idxs = get_equal_alongtrack_spacing_idxs(along_track,spacing); - - if ~exist(out_fn_dir,'dir') - mkdir(out_fn_dir); - end - - [fid,msg] = fopen(out_fn,'w'); - if fid < 0 - error('Failed to open %s: %s\n', out_fn, msg); - end - - fprintf(fid, 'LAT,LON\n'); - fprintf(fid, '%f,%f\n',[gps.lat(decim_idxs); gps.lon(decim_idxs)]); - - fclose(fid); - -end - -return; - - - - - - - - - - - - - - - - - - - - - - -return; - - -%% Example User Input - -% Select type of input files that need to be converted to CSV files -% param.input_type = 1 --> .gps (NMEA) files -% param.input_type = 2 --> ppp.txt (Novatel) files -% param.input_type = 3 --> .out (Applanix) files -% param.input_type = 4 --> .traj (ATM) files -% param.input_type = 5 --> ln100g.asc (Litton 100) files -% -% param.input_type = 3; - -% Specify region for GeoTIFF plot. -% param.region = 'Greenland'; -% param.region = 'Antarctica'; -% param.region = 'Canada'; - -% param.region = 'Greenland'; - -% Find all the input files from a directory -% Example for one file: -% param.fn = '/cresis/projects/metadata/2009_Antarctica_TO/rover_ppp_antarctica_12122009_LC.txt'; -% -% Example for multiple files: -% param.fn = get_filenames('P:\metadata\2010_Greenland_P3_INS\','ln100g','','.asc'); -% param.fn = get_filenames('P:\metadata\2011_Antarctica_DC8\','','PPPK','.out'); -% For more help on finding multiple files, see 'get_filenames' Help Doc. -% param.fn = get_filenames('/cresis/projects/metadata/2012_Greenland_P3/','','','PPPK.out'); -% param.fn = ('/cresis/projects/metadata/2012_Greenland_P3/BD960_10Apr12_PPPK_P13Jun12.out'); - -% Output file (use .csv extension) -% csv_out_path = '/cresis/scratch2/sfoga/2012_Greenland_P3_GPS.csv'; -% csv_out_path = 'Z:\sfoga\test_gps_files\2010_Greenland_DC8_out.csv'; - -% Decimation - to keep every nth point -% Recommended: set .gps to 25 (Default), Novatel and Applanix (.out) to -% 1000 (Default). -% No recommended decimation on Littion or ATM; Default is 1000. -% param.dec = 1000; - -% Year/Month/Day of first day of data -- this helps to determine in what -% GPS week that data starts. -% param.year = 2012; -% param.month = 03; -% param.day = 14; - -% Time reference: should be 'utc' (Default), but try 'gps' if that fails. -% param.time_reference = 'utc'; - -% If true, there will be only one CSV output file. If false, there will -% be one CSV file for each GPS file that is converted. -% param.csv_merge = false ; - - -%% Automated section -fprintf('Start ... %s\n',datestr(now,'HH:MM:SS')); -clf; global gRadar; - -% clear gps lat lon geotiff proj RGB R; - -if strcmpi(param.region,'Greenland') | strcmpi(param.region,'Canada') - geotiff = ct_filename_gis(gRadar,'greenland/Landsat-7/Greenland_natural.tif'); - proj = geotiffinfo(geotiff); -elseif strcmpi(param.region,'Antarctica') - geotiff = ct_filename_gis(gRadar,'antarctica/Landsat-7/Antarctica_LIMA_480m.tif'); - proj = geotiffinfo(geotiff); -else - fprintf('Region not specified or incorrectly spelled. GeoTIFF cannot be displayed.\n\n'); -end -% Geotiff -figure(1); -tic; fprintf('Loading %s GeoTIFF...\n',param.region); -mapshow(geotiff); hold on; toc; - -switch param.input_type - case 1 - % if param.input_type == 1 - %% NMEA (.gps) to CSV - - % PARAMETERS - - header_lat = 'LAT'; - header_lon = 'LON'; - idx_size = size(param.fn); - - param.combine = 0; - - - if isempty(param.time_reference) - param.time_reference = 'utc'; - else - end - - if isempty(param.dec) - param.dec = 25; - else - end - - out_folder = fileparts(csv_out_path); - if ~ispc - out_folder = strcat(out_folder,'/'); - else - out_folder = strcat(out_folder,'\'); - end - - % Build structure and write file(s) - if param.csv_merge == false % One CSV file for every input - fprintf('Processing %d file(s)...\n',idx_size(1)); - for fn_idx = 1:idx_size(1) - if idx_size(1) > 1 % If more than one param.fn entry exists - tic; gps = read_gps_nmea(param.fn{fn_idx},param); toc; - else % If only one param.fn entry exists - tic; gps = read_gps_nmea(param.fn,param); toc; - end - - lat = gps.lat(1:param.dec:end); - lon = gps.lon(1:param.dec:end); - - % Write out file - if idx_size(1) > 1 % If more than one param.fn entry exists - [a filename b] = fileparts(param.fn{fn_idx}); - else % If only one param.fn entry exists - [a filename b] = fileparts(param.fn); - end - - if ~ispc - out_path = strcat('/',out_folder,filename,'.csv'); - else - out_path = strcat(out_folder,filename,'.csv'); - end - - % Open and output CSV file - output_file = strcat(filename,'.csv'); - fprintf('Writing out file %s and plotting map...\n',output_file); - fid = fopen(out_path,'w'); - - fprintf(fid,'%s,%s\n',header_lat,header_lon); - for data_idx = 1:length(lat) - fprintf(fid,'%f,%f\n',lat(data_idx),lon(data_idx)); - [x,y] = projfwd(proj,lat(data_idx),lon(data_idx)); - plot(x,y,'b-'); hold on; - end - fclose(fid); - end - else % One CSV file for any number of inputs - fprintf('Processing %d file(s)...\n',idx_size(1)); - lat = {}; - lon = {}; - gps = {}; - for fn_idx = 1:idx_size(1) - fprintf('File %d of %d...\n',fn_idx,idx_size(1)); - if idx_size(1) > 1 % If more than one param.fn entry exists - tic; gps = read_gps_nmea(param.fn{fn_idx},param); toc; - lat{fn_idx} = gps.lat(1:param.dec:end); - lon{fn_idx} = gps.lon(1:param.dec:end); - else % If only one param.fn entry exists - tic; gps = read_gps_nmea(param.fn,param); toc; - lat = gps.lat(1:param.dec:end); - lon = gps.lon(1:param.dec:end); - end - - end - [a b c] = fileparts(csv_out_path); - output_file = strcat(b,c); - fprintf('Writing out file %s and plotting map...\n',output_file); - fid = fopen(csv_out_path,'w'); - fprintf(fid,'%s,%s\n',header_lat,header_lon); - for data_idx = 1:idx_size(1) - for coord_idx = 1:length(lat{data_idx}) - fprintf(fid,'%f,%f\n',lat{data_idx}(coord_idx),lon{data_idx}(coord_idx)); - [x,y] = projfwd(proj,lat{data_idx}(coord_idx),lon{data_idx}(coord_idx)); - plot(x,y,'b-'); hold on; - end - end - fclose(fid); - - end - - case 2 - % elseif param.input_type == 2 - %% Novatel to CSV - header_lat = 'LAT'; - header_lon = 'LON'; - idx_size = size(param.fn); - - % PARAMETERS - if isempty(param.time_reference) - param.time_reference = 'utc'; - else - end - - if isempty(param.dec) - param.dec = 1000; - else - end - - out_folder = fileparts(csv_out_path); - if ~ispc - out_folder = strcat(out_folder,'/'); - else - out_folder = strcat(out_folder,'\'); - end - - % Build structure and write file(s) - if param.csv_merge == false % One CSV file for every input - fprintf('Processing %d file(s)...\n',idx_size(1)); - for fn_idx = 1:length(param.fn) - clear gps a filename b out_path; - - fprintf('File %d of %d...\n',fn_idx,idx_size(1)); - idx_size = size(param.fn); - if idx_size(1) > 1 % If more than one param.fn entry exists - tic; gps = read_gps_novatel(param.fn{fn_idx},param); toc; - else % If only one param.fn entry exists - tic; gps = read_gps_novatel(param.fn,param); toc; - end - - lat = gps.lat(1:param.dec:end); - lon = gps.lon(1:param.dec:end); - - % Write out file - if idx_size(1) > 1 % If more than one param.fn entry exists - [a filename b] = fileparts(param.fn{fn_idx}); - else % If only one param.fn entry exists - [a filename b] = fileparts(param.fn); - end - - if ~ispc - out_path = strcat('/',out_folder,filename,'.csv'); - else - out_path = strcat(out_folder,filename,'.csv'); - end - - - % Open and output CSV file - output_file = strcat(filename,'.csv'); - fprintf('Writing out file %s and plotting map...\n',output_file); - fid = fopen(csv_out_path,'w'); - fprintf(fid,'%s,%s\n',header_lat,header_lon); - for data_idx = 1:length(lat) - fprintf(fid,'%f,%f\n',lat(data_idx),lon(data_idx)); - [x,y] = projfwd(proj,lat{data_idx},lon{data_idx}); - plot(x,y,'b-'); hold on; - end - fclose(fid); - end - else % One CSV file for any number of inputs - fprintf('Processing %d file(s)...\n',idx_size(1)); - lat = {}; - lon = {}; - gps = {}; - for fn_idx = 1:length(param.fn) - fprintf('File %d of %d...\n',fn_idx,idx_size(1)); - if idx_size(1) > 1 % If more than one param.fn entry exists - tic; gps = read_gps_novatel(param.fn{fn_idx},param); toc; - else % If only one param.fn entry exists - tic; gps = read_gps_novatel(param.fn,param); toc; - end - lat{fn_idx} = gps{fn_idx}.lat(1:param.dec:end); - lon{fn_idx} = gps{fn_idx}.lon(1:param.dec:end); - end - [a b c] = fileparts(csv_out_path); - output_file = strcat(b,c); - fprintf('Writing out file %s and plotting map...\n',output_file); - fid = fopen(csv_out_path,'w'); - fprintf(fid,'%s,%s\n',header_lat,header_lon); - for data_idx = 1:length(lat) - for coord_idx = 1:length(lat{data_idx}) - fprintf(fid,'%f,%f\n',lat{data_idx}(coord_idx),lon{data_idx}(coord_idx)); - [x,y] = projfwd(proj,lat{data_idx}(coord_idx),lon{data_idx}(coord_idx)); - plot(x,y,'b-'); hold on; - end - end - fclose(fid); - end - - case 3 - % elseif param.input_type == 3 - %% Applanix (.out) to CSV - header_lat = 'LAT'; - header_lon = 'LON'; - idx_size = size(param.fn); - - % PARAMETERS - if isempty(param.time_reference) - param.time_reference = 'utc'; - else - end - - if isempty(param.dec) - param.dec = 2; - else - end - - out_folder = fileparts(csv_out_path); - if ~ispc - out_folder = strcat(out_folder,'/'); - else - out_folder = strcat(out_folder,'\'); - end - - if param.csv_merge == false % One CSV file for every input - fprintf('Processing %d file(s)...\n',idx_size(1)); - for fn_idx = 1:idx_size(1) - clear gps a filename b out_path; - - fprintf('File %d of %d...\n',fn_idx,idx_size(1)); - if idx_size(1) > 1 % If more than one param.fn entry exists - tic; gps = read_gps_applanix(param.fn{fn_idx},param); toc; - else % If only one param.fn entry exists - tic; gps = read_gps_applanix(param.fn,param); toc; - end - - lat = gps.lat(1:param.dec:end); - lon = gps.lon(1:param.dec:end); - % write out file - if idx_size(1) > 1 % If more than one param.fn entry exists - [a filename b] = fileparts(param.fn{fn_idx}); - else % If only one param.fn entry exists - [a filename b] = fileparts(param.fn); - end - - if ~ispc - out_path = strcat('/',out_folder,filename,'.csv'); - else - out_path = strcat(out_folder,filename,'.csv'); - end - - - % Open and output CSV file - output_file = strcat(filename,'.csv'); - fprintf('Writing out file %s and plotting map...\n',output_file); - fid = fopen(out_path,'w'); - fprintf(fid,'%s,%s\n',header_lat,header_lon); - for data_idx = 1:length(lat) - fprintf(fid,'%f,%f\n',lat(data_idx),lon(data_idx)); - [x,y] = projfwd(proj,lat(data_idx),lon(data_idx)); - plot(x,y,'b-'); hold on; - end - fclose(fid); - end - - else % One CSV file for any number of inputs - fprintf('Processing %d file(s)...\n',idx_size(1)); - lat = {}; - lon = {}; - gps = {}; - for fn_idx = 1:length(param.fn) - fprintf('Processing file %d of %d ...\n',fn_idx,idx_size(1)); - - idx_size = size(param.fn); - if idx_size(1) > 1 % If more than one param.fn entry exists - tic; gps = read_gps_applanix(param.fn{fn_idx},param); toc; - else % If only one param.fn entry exists - tic; gps = read_gps_applanix(param.fn,param); toc; - end - lat{fn_idx} = gps{fn_idx}.lat(1:param.dec:end); - lon{fn_idx} = gps{fn_idx}.lon(1:param.dec:end); - end - [a b c] = fileparts(csv_out_path); - output_file = strcat(b,c); - fprintf('Writing out file %s and plotting map...\n',output_file); - fid = fopen(csv_out_path,'w'); - fprintf(fid,'%s,%s\n',header_lat,header_lon); - for data_idx = 1:length(lat) - for coord_idx = 1:length(lat{data_idx}) - fprintf(fid,'%f,%f\n',lat{data_idx}(coord_idx),lon{data_idx}(coord_idx)); - [x,y] = projfwd(proj,lat{data_idx}(coord_idx),lon{data_idx}(coord_idx)); - plot(x,y,'b-'); hold on; - end - end - fclose(fid); - end - - case 4 - % elseif param.input_type == 4 - %% ATM (.traj) to CSV - header_lat = 'LAT'; - header_lon = 'LON'; - idx_size = size(param.fn); - - % PARAMETERS - % param.headerlines = 0; - % param.delimiter_type = ','; - % param.input_format = '%f%f%f%f%f%f%f%f'; - if isempty(param.time_reference) - param.time_reference = 'utc'; - else - end - - if isempty(param.dec) - param.dec = 1000; - else - end - - out_folder = fileparts(csv_out_path); - if ~ispc - out_folder = strcat(out_folder,'/'); - else - out_folder = strcat(out_folder,'\'); - end - - % Build structure and write file(s) - if param.csv_merge == false % One CSV file for every input - fprintf('Processing %d file(s)...\n',idx_size(1)); - for fn_idx = 1:length(param.fn) - clear gps a filename b out_path; - fprintf('Processing file %d of %d ...\n',fn_idx,length(param.fn)); - - idx_size = size(param.fn); - if idx_size(1) > 1 % If more than one param.fn entry exists - tic; gps = read_gps_traj(param.fn{fn_idx},param); toc; - else % If only one param.fn entry exists - tic; gps = read_gps_traj(param.fn,param); toc; - end - - lat = gps.lat(1:param.dec:end); - lon = gps.lon(1:param.dec:end); - - % write out file - if idx_size(1) > 1 % If more than one param.fn entry exists - [a filename b] = fileparts(param.fn{fn_idx}); - else % If only one param.fn entry exists - [a filename b] = fileparts(param.fn); - end - - if ~ispc - out_path = strcat('/',out_folder,filename,'.csv'); - else - out_path = strcat(out_folder,filename,'.csv'); - end - - - % Open and output CSV file - output_file = strcat(filename,'.csv'); - fprintf('Writing out file %s and plotting map...\n',output_file); - fid = fopen(csv_out_path,'w'); - fprintf(fid,'%s,%s\n',header_lat,header_lon); - for data_idx = 1:length(lat) - fprintf(fid,'%f,%f\n',lat(data_idx),lon(data_idx)); - [x,y] = projfwd(proj,lat{data_idx},lon{data_idx}); - plot(x,y,'b-'); hold on; - end - fclose(fid); - end - else % One CSV file for any number of inputs - fprintf('Processing %d file(s)...\n',idx_size(1)); - lat = {}; - lon = {}; - gps = {}; - for fn_idx = 1:length(param.fn) - fprintf('Processing file %d of %d ...\n',fn_idx,length(param.fn)); - idx_size = size(param.fn); - if idx_size(1) > 1 % If more than one param.fn entry exists - tic; gps = read_gps_traj(param.fn{fn_idx},param); toc; - else % If only one param.fn entry exists - tic; gps = read_gps_traj(param.fn,param); toc; - end - lat{fn_idx} = gps{fn_idx}.lat(1:param.dec:end); - lon{fn_idx} = gps{fn_idx}.lon(1:param.dec:end); - end - [a b c] = fileparts(csv_out_path); - output_file = strcat(b,c); - fprintf('Writing out file %s and plotting map...\n',output_file); - fid = fopen(csv_out_path,'w'); - fprintf(fid,'%s,%s\n',header_lat,header_lon); - for data_idx = 1:length(lat) - for coord_idx = 1:length(lat{data_idx}) - fprintf(fid,'%f,%f\n',lat{data_idx}(coord_idx),lon{data_idx}(coord_idx)); - [x,y] = projfwd(proj,lat{data_idx}(coord_idx),lon{data_idx}(coord_idx)); - plot(x,y,'b-'); hold on; - end - end - fclose(fid); - end - - - case 5 - % elseif param.input_type == 5 - %% Litton 100 (ln100.asc) to CSV - header_lat = 'LAT'; - header_lon = 'LON'; - idx_size = size(param.fn); - - % PARAMETERS - if isempty(param.time_reference) - param.time_reference = 'utc'; - else - end - - if isempty(param.dec) - param.dec = 250; - else - end - - out_folder = fileparts(csv_out_path); - if ~ispc - out_folder = strcat(out_folder,'/'); - else - out_folder = strcat(out_folder,'\'); - end - - if param.csv_merge == false % One CSV file for every input - fprintf('Processing %d file(s)...\n',idx_size(1)); - % Build structure and write file(s) - for fn_idx = 1:length(param.fn) - clear gps a filename b out_path; - fprintf('Processing file %d of %d ...\n',fn_idx,length(param.fn)); - idx_size = size(param.fn); - if idx_size(1) > 1 % If more than one param.fn entry exists - tic; gps = read_gps_litton(param.fn{fn_idx},param); toc; - else % If only one param.fn entry exists - tic; gps = read_gps_litton(param.fn,param); toc; - end - lat = gps.lat(1:param.dec:end); - lon = gps.lon(1:param.dec:end); - - % write out file - if idx_size(1) > 1 % If more than one param.fn entry exists - [a filename b] = fileparts(param.fn{fn_idx}); - else % If only one param.fn entry exists - [a filename b] = fileparts(param.fn); - end - - if ~ispc - out_path = strcat('/',out_folder,filename,'.csv'); - else - out_path = strcat(out_folder,filename,'.csv'); - end - - % Open and output CSV file - output_file = strcat(filename,'.csv'); - fprintf('Writing out file %s and plotting map...\n',output_file); - fid = fopen(csv_out_path,'w'); - fprintf(fid,'%s,%s\n',header_lat,header_lon); - for data_idx = 1:length(lat) - fprintf(fid,'%f,%f\n',lat(data_idx),lon(data_idx)); - [x,y] = projfwd(proj,lat(data_idx),lon(data_idx)); - plot(x,y,'b-'); hold on; - end - fclose(fid); - end - else % One CSV file for any number of inputs - fprintf('Processing %d file(s)...\n',idx_size(1)); - lat = {}; - lon = {}; - gps = {}; - for fn_idx = 1:length(param.fn) - fprintf('Processing file %d of %d ...\n',fn_idx,length(param.fn)); - idx_size = size(param.fn); - if idx_size(1) > 1 % If more than one param.fn entry exists - tic; gps = read_gps_litton(param.fn{fn_idx},param); toc; - else % If only one param.fn entry exists - tic; gps = read_gps_litton(param.fn,param); toc; - end - lat{fn_idx} = gps{fn_idx}.lat(1:param.dec:end); - lon{fn_idx} = gps{fn_idx}.lon(1:param.dec:end); - end - [a b c] = fileparts(csv_out_path); - output_file = strcat(b,c); - fprintf('Writing out file %s and plotting map...\n',output_file); - fid = fopen(csv_out_path,'w'); - fprintf(fid,'%s,%s\n',header_lat,header_lon); - for data_idx = 1:length(lat) - for coord_idx = 1:length(lat{data_idx}) - fprintf(fid,'%f,%f\n',lat{data_idx}(coord_idx),lon{data_idx}(coord_idx)); - [x,y] = projfwd(proj,lat{data_idx}(coord_idx),lon{data_idx}(coord_idx)); - plot(x,y,'b-'); hold on; - end - end - fclose(fid); - end - otherwise - % else - fprintf('Invalid param.input_type selected.\n\n'); -end -fprintf('Done ... %s\n',datestr(now,'HH:MM:SS')); \ No newline at end of file diff --git a/cresis-toolbox/gps/missions/fix_gps_2009_antarctica_TO.m b/cresis-toolbox/gps/missions/fix_gps_2009_Antarctica_TO.m similarity index 100% rename from cresis-toolbox/gps/missions/fix_gps_2009_antarctica_TO.m rename to cresis-toolbox/gps/missions/fix_gps_2009_Antarctica_TO.m diff --git a/cresis-toolbox/gps/missions/make_gps_1993_greenland_P3.m b/cresis-toolbox/gps/missions/gps_create_1993_Greenland_P3.m similarity index 95% rename from cresis-toolbox/gps/missions/make_gps_1993_greenland_P3.m rename to cresis-toolbox/gps/missions/gps_create_1993_Greenland_P3.m index 193cba5f..ac141c0c 100644 --- a/cresis-toolbox/gps/missions/make_gps_1993_greenland_P3.m +++ b/cresis-toolbox/gps/missions/gps_create_1993_Greenland_P3.m @@ -1,164 +1,164 @@ -% script make_gps_2002_greenland_P3 -% Makes the DGPSwINS????? files for 2002 Greenland P3 field season -%see icards_gps_missinNASA_csv.m to get csv files for days without -%trajectory data. (check time reference: should be gps) -tic; - -global gRadar; - -support_path = ''; -data_support_path = ''; - -if isempty(support_path) - support_path = gRadar.support_path; -end - -gps_path = fullfile(support_path,'gps','1993_Greenland_P3'); -if ~exist(gps_path,'dir') - fprintf('Making directory %s\n', gps_path); - fprintf(' Press a key to proceed\n'); - pause; - mkdir(gps_path); -end - -if isempty(data_support_path) - data_support_path = gRadar.data_support_path; -end - -% ====================================================================== -% User Settings -% ====================================================================== -debug_level = 1; - -in_base_path = fullfile(data_support_path,'1993_Greenland_P3'); - -file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; -sync_fns = {}; sync_params = {}; - -% file_idx = file_idx + 1; -% in_fns{file_idx} = fullfile(in_base_path,'19930623_nmea.csv'); -% out_fns{file_idx} = 'gps_19930623.mat'; -% file_type{file_idx} = 'csv'; -% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -% gps_source{file_idx} = 'nmea-field'; -% % -% file_idx = file_idx + 1; -% in_fns{file_idx} = fullfile(in_base_path,'19930624_nmea.csv'); -% out_fns{file_idx} = 'gps_19930624.mat'; -% file_type{file_idx} = 'csv'; -% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -% gps_source{file_idx} = 'nmea-field'; -% -% file_idx = file_idx + 1; -% in_fns{file_idx} = fullfile(in_base_path,'19930627_nmea.csv'); -% out_fns{file_idx} = 'gps_19930627.mat'; -% file_type{file_idx} = 'csv'; -% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -% gps_source{file_idx} = 'nmea-field'; -% -% file_idx = file_idx + 1; -% in_fns{file_idx} = fullfile(in_base_path,'19930628_nmea.csv'); -% out_fns{file_idx} = 'gps_19930628.mat'; -% file_type{file_idx} = 'csv'; -% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -% gps_source{file_idx} = 'nmea-field'; -% -% file_idx = file_idx + 1; -% in_fns{file_idx} = fullfile(in_base_path,'19930701_nmea.csv'); -% out_fns{file_idx} = 'gps_19930701.mat'; -% file_type{file_idx} = 'csv'; -% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -% gps_source{file_idx} = 'nmea-field'; -% -% file_idx = file_idx + 1; -% in_fns{file_idx} = fullfile(in_base_path,'19930702_nmea.csv'); -% out_fns{file_idx} = 'gps_19930702.mat'; -% file_type{file_idx} = 'csv'; -% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -% gps_source{file_idx} = 'nmea-field'; -% -% file_idx = file_idx + 1; -% in_fns{file_idx} = fullfile(in_base_path,'19930703_nmea.csv'); -% out_fns{file_idx} = 'gps_19930703.mat'; -% file_type{file_idx} = 'csv'; -% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -% gps_source{file_idx} = 'nmea-field'; -% -% file_idx = file_idx + 1; -% in_fns{file_idx} = fullfile(in_base_path,'19930707_nmea.csv'); -% out_fns{file_idx} = 'gps_19930707.mat'; -% file_type{file_idx} = 'csv'; -% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -% gps_source{file_idx} = 'nmea-field'; - -% file_idx = file_idx + 1; -% in_fns{file_idx} = fullfile(in_base_path,'19930708_nmea.csv'); -% out_fns{file_idx} = 'gps_19930708.mat'; -% file_type{file_idx} = 'csv'; -% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -% gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19930709_nmea.csv'); -out_fns{file_idx} = 'gps_19930709.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -gps_make; - -latlon_min_jump=0.005;%latitude and longitude minimum jump threshold -latlon_spike_jump=20;%latitude and longitude spike jump threshold -elev_min_jump=20;%elevation minimum jump threshold - -match_idx = strmatch('gps_19930709.mat',out_fns,'exact'); -% if ~isempty(match_idx)%----qishi -% gps_fn = fullfile(gps_path,out_fns{match_idx}); -% fprintf('Fixing GPS data for %s\n', gps_fn); -% gps = load(gps_fn); -% -% -% close all; -% gps.lat(gps.lat>median(gps.lat)+latlon_spike_jump |gps.latlatlon_min_jump))%deal with too large jumps -% gps.lat([logical(0),abs(diff(gps.lat))>latlon_min_jump])=NaN; -% gps.lat=interp_finite(gps.lat); -% end -% save(gps_fn,'-append','-struct','gps','lat'); -% -% gps.lon(gps.lon>median(gps.lon)+latlon_spike_jump |gps.lonlatlon_min_jump)%deal with too large jumps -% gps.lon([logical(0),abs(diff(gps.lon))>latlon_min_jump])=NaN; -% gps.lon=interp_finite(gps.lon); -% end -% -% jump_idxs=find(abs(diff(gps.elev))>=elev_min_jump); -% for ii=1:length(jump_idxs) -% gps.elev(jump_idxs(ii)+1:end)=gps.elev(jump_idxs(ii)+1:end)+gps.elev(jump_idxs(ii))-gps.elev(jump_idxs(ii)+1); -% end -% gps.elev=medfilt1(gps.elev);%median filter -% -% -% save(gps_fn,'-append','-struct','gps','elev'); -% end - -if ~isempty(match_idx)%---paden - gps_fn = fullfile(gps_path,out_fns{match_idx}); - fprintf('Fixing GPS data for %s\n', gps_fn); - gps = load(gps_fn); - % FIX CODE HERE - gps.elev(gps.elev > 10000) = NaN; - gps.elev = interp_finite(gps.elev); - save(gps_fn,'-append','-struct','gps','elev'); - gps.lon(gps.lon < -150) = NaN; - gps.lon = interp_finite(gps.lon); - save(gps_fn,'-append','-struct','gps','lon'); -end - - - - +% script gps_create_2002_greenland_P3 +% Makes the DGPSwINS????? files for 2002 Greenland P3 field season +%see icards_gps_missinNASA_csv.m to get csv files for days without +%trajectory data. (check time reference: should be gps) +tic; + +global gRadar; + +support_path = ''; +data_support_path = ''; + +if isempty(support_path) + support_path = gRadar.support_path; +end + +gps_path = fullfile(support_path,'gps','1993_Greenland_P3'); +if ~exist(gps_path,'dir') + fprintf('Making directory %s\n', gps_path); + fprintf(' Press a key to proceed\n'); + pause; + mkdir(gps_path); +end + +if isempty(data_support_path) + data_support_path = gRadar.data_support_path; +end + +% ====================================================================== +% User Settings +% ====================================================================== +debug_level = 1; + +in_base_path = fullfile(data_support_path,'1993_Greenland_P3'); + +file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; +sync_fns = {}; sync_params = {}; + +% file_idx = file_idx + 1; +% in_fns{file_idx} = fullfile(in_base_path,'19930623_nmea.csv'); +% out_fns{file_idx} = 'gps_19930623.mat'; +% file_type{file_idx} = 'csv'; +% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +% gps_source{file_idx} = 'nmea-field'; +% % +% file_idx = file_idx + 1; +% in_fns{file_idx} = fullfile(in_base_path,'19930624_nmea.csv'); +% out_fns{file_idx} = 'gps_19930624.mat'; +% file_type{file_idx} = 'csv'; +% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +% gps_source{file_idx} = 'nmea-field'; +% +% file_idx = file_idx + 1; +% in_fns{file_idx} = fullfile(in_base_path,'19930627_nmea.csv'); +% out_fns{file_idx} = 'gps_19930627.mat'; +% file_type{file_idx} = 'csv'; +% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +% gps_source{file_idx} = 'nmea-field'; +% +% file_idx = file_idx + 1; +% in_fns{file_idx} = fullfile(in_base_path,'19930628_nmea.csv'); +% out_fns{file_idx} = 'gps_19930628.mat'; +% file_type{file_idx} = 'csv'; +% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +% gps_source{file_idx} = 'nmea-field'; +% +% file_idx = file_idx + 1; +% in_fns{file_idx} = fullfile(in_base_path,'19930701_nmea.csv'); +% out_fns{file_idx} = 'gps_19930701.mat'; +% file_type{file_idx} = 'csv'; +% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +% gps_source{file_idx} = 'nmea-field'; +% +% file_idx = file_idx + 1; +% in_fns{file_idx} = fullfile(in_base_path,'19930702_nmea.csv'); +% out_fns{file_idx} = 'gps_19930702.mat'; +% file_type{file_idx} = 'csv'; +% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +% gps_source{file_idx} = 'nmea-field'; +% +% file_idx = file_idx + 1; +% in_fns{file_idx} = fullfile(in_base_path,'19930703_nmea.csv'); +% out_fns{file_idx} = 'gps_19930703.mat'; +% file_type{file_idx} = 'csv'; +% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +% gps_source{file_idx} = 'nmea-field'; +% +% file_idx = file_idx + 1; +% in_fns{file_idx} = fullfile(in_base_path,'19930707_nmea.csv'); +% out_fns{file_idx} = 'gps_19930707.mat'; +% file_type{file_idx} = 'csv'; +% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +% gps_source{file_idx} = 'nmea-field'; + +% file_idx = file_idx + 1; +% in_fns{file_idx} = fullfile(in_base_path,'19930708_nmea.csv'); +% out_fns{file_idx} = 'gps_19930708.mat'; +% file_type{file_idx} = 'csv'; +% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +% gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19930709_nmea.csv'); +out_fns{file_idx} = 'gps_19930709.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +gps_create; + +latlon_min_jump=0.005;%latitude and longitude minimum jump threshold +latlon_spike_jump=20;%latitude and longitude spike jump threshold +elev_min_jump=20;%elevation minimum jump threshold + +match_idx = strmatch('gps_19930709.mat',out_fns,'exact'); +% if ~isempty(match_idx)%----qishi +% gps_fn = fullfile(gps_path,out_fns{match_idx}); +% fprintf('Fixing GPS data for %s\n', gps_fn); +% gps = load(gps_fn); +% +% +% close all; +% gps.lat(gps.lat>median(gps.lat)+latlon_spike_jump |gps.latlatlon_min_jump))%deal with too large jumps +% gps.lat([logical(0),abs(diff(gps.lat))>latlon_min_jump])=NaN; +% gps.lat=interp_finite(gps.lat); +% end +% save(gps_fn,'-append','-struct','gps','lat'); +% +% gps.lon(gps.lon>median(gps.lon)+latlon_spike_jump |gps.lonlatlon_min_jump)%deal with too large jumps +% gps.lon([logical(0),abs(diff(gps.lon))>latlon_min_jump])=NaN; +% gps.lon=interp_finite(gps.lon,[],@gps_interp1); +% end +% +% jump_idxs=find(abs(diff(gps.elev))>=elev_min_jump); +% for ii=1:length(jump_idxs) +% gps.elev(jump_idxs(ii)+1:end)=gps.elev(jump_idxs(ii)+1:end)+gps.elev(jump_idxs(ii))-gps.elev(jump_idxs(ii)+1); +% end +% gps.elev=medfilt1(gps.elev);%median filter +% +% +% save(gps_fn,'-append','-struct','gps','elev'); +% end + +if ~isempty(match_idx)%---paden + gps_fn = fullfile(gps_path,out_fns{match_idx}); + fprintf('Fixing GPS data for %s\n', gps_fn); + gps = load(gps_fn); + % FIX CODE HERE + gps.elev(gps.elev > 10000) = NaN; + gps.elev = interp_finite(gps.elev); + save(gps_fn,'-append','-struct','gps','elev'); + gps.lon(gps.lon < -150) = NaN; + gps.lon = interp_finite(gps.lon,[],@gps_interp1); + save(gps_fn,'-append','-struct','gps','lon'); +end + + + + diff --git a/cresis-toolbox/gps/missions/make_gps_1995_greenland_P3.m b/cresis-toolbox/gps/missions/gps_create_1995_Greenland_P3.m similarity index 95% rename from cresis-toolbox/gps/missions/make_gps_1995_greenland_P3.m rename to cresis-toolbox/gps/missions/gps_create_1995_Greenland_P3.m index f3c59528..9f050edb 100644 --- a/cresis-toolbox/gps/missions/make_gps_1995_greenland_P3.m +++ b/cresis-toolbox/gps/missions/gps_create_1995_Greenland_P3.m @@ -1,116 +1,116 @@ -% script make_gps_1995_greenland_P3 -% -% Makes the GPS files for 1995 Greenland P3 field season - -tic; - -global gRadar; - -support_path = ''; -data_support_path = ''; - -if isempty(support_path) - support_path = gRadar.support_path; -end - -gps_path = fullfile(support_path,'gps','1995_Greenland_P3'); -if ~exist(gps_path,'dir') - fprintf('Making directory %s\n', gps_path); - fprintf(' Press a key to proceed\n'); - pause; - mkdir(gps_path); -end - -if isempty(data_support_path) - data_support_path = gRadar.data_support_path; -end - -debug_level = 1; - -in_base_path = fullfile(data_support_path,'1995_Greenland_P3'); - -file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; -sync_fns = {}; sync_params = {}; - -% file_idx = file_idx + 1; -% in_fns{file_idx} = fullfile(in_base_path,'19950518_nmea.csv'); -% out_fns{file_idx} = 'gps_19950518.mat'; -% file_type{file_idx} = 'csv'; -% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -% gps_source{file_idx} = 'nmea-field'; - -% file_idx = file_idx + 1; -% in_fns{file_idx} = fullfile(in_base_path,'19950519_nmea.csv'); -% out_fns{file_idx} = 'gps_19950519.mat'; -% file_type{file_idx} = 'csv'; -% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -% gps_source{file_idx} = 'nmea-field'; - -% file_idx = file_idx + 1; -% in_fns{file_idx} = fullfile(in_base_path,'19950520_nmea.csv'); -% out_fns{file_idx} = 'gps_19950520.mat'; -% file_type{file_idx} = 'csv'; -% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -% gps_source{file_idx} = 'nmea-field'; -% -% file_idx = file_idx + 1; -% in_fns{file_idx} = fullfile(in_base_path,'19950522_nmea.csv'); -% out_fns{file_idx} = 'gps_19950522.mat'; -% file_type{file_idx} = 'csv'; -% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -% gps_source{file_idx} = 'nmea-field'; -% -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19950523_nmea.csv'); -out_fns{file_idx} = 'gps_19950523.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; -% -% file_idx = file_idx + 1; -% in_fns{file_idx} = fullfile(in_base_path,'19950524_nmea.csv'); -% out_fns{file_idx} = 'gps_19950524.mat'; -% file_type{file_idx} = 'csv'; -% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -% gps_source{file_idx} = 'nmea-field'; -% -% file_idx = file_idx + 1; -% in_fns{file_idx} = fullfile(in_base_path,'19950526_nmea.csv'); -% out_fns{file_idx} = 'gps_19950526.mat'; -% file_type{file_idx} = 'csv'; -% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -% gps_source{file_idx} = 'nmea-field'; -% -% file_idx = file_idx + 1; -% in_fns{file_idx} = fullfile(in_base_path,'19950527_nmea.csv'); -% out_fns{file_idx} = 'gps_19950527.mat'; -% file_type{file_idx} = 'csv'; -% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -% gps_source{file_idx} = 'nmea-field'; -% -% file_idx = file_idx + 1; -% in_fns{file_idx} = fullfile(in_base_path,'19950530_nmea.csv'); -% out_fns{file_idx} = 'gps_19950530.mat'; -% file_type{file_idx} = 'csv'; -% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -% gps_source{file_idx} = 'nmea-field'; - -% ====================================================================== -% Read and translate files according to user settings -% ====================================================================== -gps_make; - -%% No GPS Data Available: Fix GPS elevation information -match_idx = strmatch('gps_19950523.mat',out_fns,'exact'); -if ~isempty(match_idx) - gps_fn = fullfile(gps_path,out_fns{match_idx}); - fprintf('Fixing GPS data for %s\n', gps_fn); - gps = load(gps_fn); - % FIX CODE HERE - gps.elev(gps.elev > 10000) = NaN; - gps.elev = interp_finite(gps.elev); - save(gps_fn,'-append','-struct','gps','elev'); - gps.lon(gps.lon < -150) = NaN; - gps.lon = interp_finite(gps.lon); - save(gps_fn,'-append','-struct','gps','lon'); +% script gps_create_1995_greenland_P3 +% +% Makes the GPS files for 1995 Greenland P3 field season + +tic; + +global gRadar; + +support_path = ''; +data_support_path = ''; + +if isempty(support_path) + support_path = gRadar.support_path; +end + +gps_path = fullfile(support_path,'gps','1995_Greenland_P3'); +if ~exist(gps_path,'dir') + fprintf('Making directory %s\n', gps_path); + fprintf(' Press a key to proceed\n'); + pause; + mkdir(gps_path); +end + +if isempty(data_support_path) + data_support_path = gRadar.data_support_path; +end + +debug_level = 1; + +in_base_path = fullfile(data_support_path,'1995_Greenland_P3'); + +file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; +sync_fns = {}; sync_params = {}; + +% file_idx = file_idx + 1; +% in_fns{file_idx} = fullfile(in_base_path,'19950518_nmea.csv'); +% out_fns{file_idx} = 'gps_19950518.mat'; +% file_type{file_idx} = 'csv'; +% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +% gps_source{file_idx} = 'nmea-field'; + +% file_idx = file_idx + 1; +% in_fns{file_idx} = fullfile(in_base_path,'19950519_nmea.csv'); +% out_fns{file_idx} = 'gps_19950519.mat'; +% file_type{file_idx} = 'csv'; +% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +% gps_source{file_idx} = 'nmea-field'; + +% file_idx = file_idx + 1; +% in_fns{file_idx} = fullfile(in_base_path,'19950520_nmea.csv'); +% out_fns{file_idx} = 'gps_19950520.mat'; +% file_type{file_idx} = 'csv'; +% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +% gps_source{file_idx} = 'nmea-field'; +% +% file_idx = file_idx + 1; +% in_fns{file_idx} = fullfile(in_base_path,'19950522_nmea.csv'); +% out_fns{file_idx} = 'gps_19950522.mat'; +% file_type{file_idx} = 'csv'; +% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +% gps_source{file_idx} = 'nmea-field'; +% +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19950523_nmea.csv'); +out_fns{file_idx} = 'gps_19950523.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; +% +% file_idx = file_idx + 1; +% in_fns{file_idx} = fullfile(in_base_path,'19950524_nmea.csv'); +% out_fns{file_idx} = 'gps_19950524.mat'; +% file_type{file_idx} = 'csv'; +% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +% gps_source{file_idx} = 'nmea-field'; +% +% file_idx = file_idx + 1; +% in_fns{file_idx} = fullfile(in_base_path,'19950526_nmea.csv'); +% out_fns{file_idx} = 'gps_19950526.mat'; +% file_type{file_idx} = 'csv'; +% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +% gps_source{file_idx} = 'nmea-field'; +% +% file_idx = file_idx + 1; +% in_fns{file_idx} = fullfile(in_base_path,'19950527_nmea.csv'); +% out_fns{file_idx} = 'gps_19950527.mat'; +% file_type{file_idx} = 'csv'; +% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +% gps_source{file_idx} = 'nmea-field'; +% +% file_idx = file_idx + 1; +% in_fns{file_idx} = fullfile(in_base_path,'19950530_nmea.csv'); +% out_fns{file_idx} = 'gps_19950530.mat'; +% file_type{file_idx} = 'csv'; +% params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +% gps_source{file_idx} = 'nmea-field'; + +% ====================================================================== +% Read and translate files according to user settings +% ====================================================================== +gps_create; + +%% No GPS Data Available: Fix GPS elevation information +match_idx = strmatch('gps_19950523.mat',out_fns,'exact'); +if ~isempty(match_idx) + gps_fn = fullfile(gps_path,out_fns{match_idx}); + fprintf('Fixing GPS data for %s\n', gps_fn); + gps = load(gps_fn); + % FIX CODE HERE + gps.elev(gps.elev > 10000) = NaN; + gps.elev = interp_finite(gps.elev); + save(gps_fn,'-append','-struct','gps','elev'); + gps.lon(gps.lon < -150) = NaN; + gps.lon = interp_finite(gps.lon,[],@gps_interp1); + save(gps_fn,'-append','-struct','gps','lon'); end \ No newline at end of file diff --git a/cresis-toolbox/gps/missions/make_gps_1996_greenland_P3.m b/cresis-toolbox/gps/missions/gps_create_1996_Greenland_P3.m similarity index 93% rename from cresis-toolbox/gps/missions/make_gps_1996_greenland_P3.m rename to cresis-toolbox/gps/missions/gps_create_1996_Greenland_P3.m index 0b1630b7..c9966151 100644 --- a/cresis-toolbox/gps/missions/make_gps_1996_greenland_P3.m +++ b/cresis-toolbox/gps/missions/gps_create_1996_Greenland_P3.m @@ -1,157 +1,157 @@ -% script make_gps_1996_greenland_P3 -% Makes the DGPSwINS????? files for 2002 Greenland P3 field season -%see icards_gps_missinNASA_csv.m to get csv files for days without -%trajectory data. (check time reference: should be gps) -tic; - -global gRadar; - -support_path = ''; -data_support_path = ''; - -if isempty(support_path) - support_path = gRadar.support_path; -end - -gps_path = fullfile(support_path,'gps','1996_Greenland_P3'); -if ~exist(gps_path,'dir') - fprintf('Making directory %s\n', gps_path); - fprintf(' Press a key to proceed\n'); - pause; - mkdir(gps_path); -end - -if isempty(data_support_path) - data_support_path = gRadar.data_support_path; -end - -% ====================================================================== -% User Settings -% ====================================================================== -debug_level = 1; - -in_base_path = fullfile(data_support_path,'1996_Greenland_P3'); - -file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; -sync_fns = {}; sync_params = {}; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19960515_nmea.csv'); -out_fns{file_idx} = 'gps_19960515.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19960520_nmea.csv'); -out_fns{file_idx} = 'gps_19960520.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19960522_nmea.csv'); -out_fns{file_idx} = 'gps_19960522.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - - - - -gps_make; -latlon_min_jump=0.005;%latitude and longitude minimum jump threshold -latlon_spike_jump=10;%latitude and longitude spike jump threshold -elev_min_jump=300;%elevation minimum jump threshold - -match_idx = strmatch('gps_19960515.mat',out_fns,'exact'); -if ~isempty(match_idx) - gps_fn = fullfile(gps_path,out_fns{match_idx}); - fprintf('Fixing GPS data for %s\n', gps_fn); - gps = load(gps_fn); - - gps.lat(gps.lat>median(gps.lat)+latlon_spike_jump |gps.latlatlon_min_jump))%deal with too large jumps - gps.lat(abs(diff(gps.lat))>latlon_min_jump)=NaN; - gps.lat=interp_finite(gps.lat); - end - save(gps_fn,'-append','-struct','gps','lat'); - - gps.lon(gps.lon>median(gps.lon)+latlon_spike_jump |gps.lonlatlon_min_jump)%deal with too large jumps - gps.lon(abs(diff(gps.lon))>latlon_min_jump)=NaN; - gps.lon=interp_finite(gps.lon); - end - save(gps_fn,'-append','-struct','gps','lon'); -end - - - -match_idx = strmatch('gps_19960520.mat',out_fns,'exact'); -if ~isempty(match_idx) - gps_fn = fullfile(gps_path,out_fns{match_idx}); - fprintf('Fixing GPS data for %s\n', gps_fn); - gps = load(gps_fn); - - gps.lat(gps.lat>median(gps.lat)+latlon_spike_jump |gps.latlatlon_min_jump))%deal with too large jumps - gps.lat(abs(diff(gps.lat))>latlon_min_jump)=NaN; - gps.lat=interp_finite(gps.lat); - end - save(gps_fn,'-append','-struct','gps','lat'); - - gps.lon(gps.lon>median(gps.lon)+latlon_spike_jump |gps.lonlatlon_min_jump)%deal with too large jumps - gps.lon(abs(diff(gps.lon))>latlon_min_jump)=NaN; - gps.lon=interp_finite(gps.lon); - end - save(gps_fn,'-append','-struct','gps','lon'); - - while any(abs(diff(gps.elev))>elev_min_jump)%deal with too large jumps - gps.elev(abs(diff(gps.elev))>elev_min_jump)=NaN; - gps.elev=interp_finite(gps.elev); - end - save(gps_fn,'-append','-struct','gps','elev'); -end - - -match_idx = strmatch('gps_19960522.mat',out_fns,'exact'); -if ~isempty(match_idx) - gps_fn = fullfile(gps_path,out_fns{match_idx}); - fprintf('Fixing GPS data for %s\n', gps_fn); - gps = load(gps_fn); - - gps.lat(gps.lat>median(gps.lat)+latlon_spike_jump |gps.latlatlon_min_jump))%deal with too large jumps - gps.lat(abs(diff(gps.lat))>latlon_min_jump)=NaN; - gps.lat=interp_finite(gps.lat); - end - save(gps_fn,'-append','-struct','gps','lat'); - - gps.lon(gps.lon>median(gps.lon)+latlon_spike_jump |gps.lonlatlon_min_jump)%deal with too large jumps - gps.lon(abs(diff(gps.lon))>latlon_min_jump)=NaN; - gps.lon=interp_finite(gps.lon); - end - save(gps_fn,'-append','-struct','gps','lon'); - - while any(abs(diff(gps.elev))>elev_min_jump)%deal with too large jumps - gps.elev(abs(diff(gps.elev))>elev_min_jump)=NaN; - gps.elev=interp_finite(gps.elev); - end - save(gps_fn,'-append','-struct','gps','elev'); -end - - +% script gps_create_1996_greenland_P3 +% Makes the DGPSwINS????? files for 2002 Greenland P3 field season +%see icards_gps_missinNASA_csv.m to get csv files for days without +%trajectory data. (check time reference: should be gps) +tic; + +global gRadar; + +support_path = ''; +data_support_path = ''; + +if isempty(support_path) + support_path = gRadar.support_path; +end + +gps_path = fullfile(support_path,'gps','1996_Greenland_P3'); +if ~exist(gps_path,'dir') + fprintf('Making directory %s\n', gps_path); + fprintf(' Press a key to proceed\n'); + pause; + mkdir(gps_path); +end + +if isempty(data_support_path) + data_support_path = gRadar.data_support_path; +end + +% ====================================================================== +% User Settings +% ====================================================================== +debug_level = 1; + +in_base_path = fullfile(data_support_path,'1996_Greenland_P3'); + +file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; +sync_fns = {}; sync_params = {}; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19960515_nmea.csv'); +out_fns{file_idx} = 'gps_19960515.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19960520_nmea.csv'); +out_fns{file_idx} = 'gps_19960520.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19960522_nmea.csv'); +out_fns{file_idx} = 'gps_19960522.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + + + + +gps_create; +latlon_min_jump=0.005;%latitude and longitude minimum jump threshold +latlon_spike_jump=10;%latitude and longitude spike jump threshold +elev_min_jump=300;%elevation minimum jump threshold + +match_idx = strmatch('gps_19960515.mat',out_fns,'exact'); +if ~isempty(match_idx) + gps_fn = fullfile(gps_path,out_fns{match_idx}); + fprintf('Fixing GPS data for %s\n', gps_fn); + gps = load(gps_fn); + + gps.lat(gps.lat>median(gps.lat)+latlon_spike_jump |gps.latlatlon_min_jump))%deal with too large jumps + gps.lat(abs(diff(gps.lat))>latlon_min_jump)=NaN; + gps.lat=interp_finite(gps.lat); + end + save(gps_fn,'-append','-struct','gps','lat'); + + gps.lon(gps.lon>median(gps.lon)+latlon_spike_jump |gps.lonlatlon_min_jump)%deal with too large jumps + gps.lon(abs(diff(gps.lon))>latlon_min_jump)=NaN; + gps.lon=interp_finite(gps.lon,[],@gps_interp1); + end + save(gps_fn,'-append','-struct','gps','lon'); +end + + + +match_idx = strmatch('gps_19960520.mat',out_fns,'exact'); +if ~isempty(match_idx) + gps_fn = fullfile(gps_path,out_fns{match_idx}); + fprintf('Fixing GPS data for %s\n', gps_fn); + gps = load(gps_fn); + + gps.lat(gps.lat>median(gps.lat)+latlon_spike_jump |gps.latlatlon_min_jump))%deal with too large jumps + gps.lat(abs(diff(gps.lat))>latlon_min_jump)=NaN; + gps.lat=interp_finite(gps.lat); + end + save(gps_fn,'-append','-struct','gps','lat'); + + gps.lon(gps.lon>median(gps.lon)+latlon_spike_jump |gps.lonlatlon_min_jump)%deal with too large jumps + gps.lon(abs(diff(gps.lon))>latlon_min_jump)=NaN; + gps.lon=interp_finite(gps.lon,[],@gps_interp1); + end + save(gps_fn,'-append','-struct','gps','lon'); + + while any(abs(diff(gps.elev))>elev_min_jump)%deal with too large jumps + gps.elev(abs(diff(gps.elev))>elev_min_jump)=NaN; + gps.elev=interp_finite(gps.elev); + end + save(gps_fn,'-append','-struct','gps','elev'); +end + + +match_idx = strmatch('gps_19960522.mat',out_fns,'exact'); +if ~isempty(match_idx) + gps_fn = fullfile(gps_path,out_fns{match_idx}); + fprintf('Fixing GPS data for %s\n', gps_fn); + gps = load(gps_fn); + + gps.lat(gps.lat>median(gps.lat)+latlon_spike_jump |gps.latlatlon_min_jump))%deal with too large jumps + gps.lat(abs(diff(gps.lat))>latlon_min_jump)=NaN; + gps.lat=interp_finite(gps.lat); + end + save(gps_fn,'-append','-struct','gps','lat'); + + gps.lon(gps.lon>median(gps.lon)+latlon_spike_jump |gps.lonlatlon_min_jump)%deal with too large jumps + gps.lon(abs(diff(gps.lon))>latlon_min_jump)=NaN; + gps.lon=interp_finite(gps.lon,[],@gps_interp1); + end + save(gps_fn,'-append','-struct','gps','lon'); + + while any(abs(diff(gps.elev))>elev_min_jump)%deal with too large jumps + gps.elev(abs(diff(gps.elev))>elev_min_jump)=NaN; + gps.elev=interp_finite(gps.elev); + end + save(gps_fn,'-append','-struct','gps','elev'); +end + + diff --git a/cresis-toolbox/gps/missions/make_gps_1997_greenland_P3.m b/cresis-toolbox/gps/missions/gps_create_1997_Greenland_P3.m similarity index 96% rename from cresis-toolbox/gps/missions/make_gps_1997_greenland_P3.m rename to cresis-toolbox/gps/missions/gps_create_1997_Greenland_P3.m index f041f9c8..77ecc8d7 100644 --- a/cresis-toolbox/gps/missions/make_gps_1997_greenland_P3.m +++ b/cresis-toolbox/gps/missions/gps_create_1997_Greenland_P3.m @@ -1,128 +1,128 @@ -% script make_gps_2002_greenland_P3 -% Makes the DGPSwINS????? files for 2002 Greenland P3 field season -%see icards_gps_missinNASA_csv.m to get csv files for days without -%trajectory data. (check time reference: should be gps) -tic; - -global gRadar; - -support_path = ''; -data_support_path = ''; - -if isempty(support_path) - support_path = gRadar.support_path; -end - -gps_path = fullfile(support_path,'gps','1997_Greenland_P3'); -if ~exist(gps_path,'dir') - fprintf('Making directory %s\n', gps_path); - fprintf(' Press a key to proceed\n'); - pause; - mkdir(gps_path); -end - -if isempty(data_support_path) - data_support_path = gRadar.data_support_path; -end - -% ====================================================================== -% User Settings -% ====================================================================== -debug_level = 1; - -in_base_path = fullfile(data_support_path,'1997_Greenland_P3'); - -file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; -sync_fns = {}; sync_params = {}; - - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19970511_nmea.csv'); -out_fns{file_idx} = 'gps_19970511.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19970513_nmea.csv'); -out_fns{file_idx} = 'gps_19970513.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19970514_nmea.csv'); -out_fns{file_idx} = 'gps_19970514.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19970515_nmea.csv'); -out_fns{file_idx} = 'gps_19970515.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19970517_nmea.csv'); -out_fns{file_idx} = 'gps_19970517.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19970519_nmea.csv'); -out_fns{file_idx} = 'gps_19970519.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19970521_nmea.csv'); -out_fns{file_idx} = 'gps_19970521.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19970523_nmea.csv'); -out_fns{file_idx} = 'gps_19970523.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19970524_nmea.csv'); -out_fns{file_idx} = 'gps_19970524.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19970527_nmea.csv'); -out_fns{file_idx} = 'gps_19970527.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19970528_nmea.csv'); -out_fns{file_idx} = 'gps_19970528.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - - -gps_make; -match_idx = strmatch('gps_19950518.mat',out_fns,'exact'); -if ~isempty(match_idx) - gps_fn = fullfile(gps_path,out_fns{match_idx}); - fprintf('Fixing GPS data for %s\n', gps_fn); - gps = load(gps_fn); - % FIX CODE HERE - gps.elev(gps.elev > 10000) = NaN; - gps.elev = interp_finite(gps.elev); - save(gps_fn,'-append','-struct','gps','elev'); -end - +% script gps_create_2002_greenland_P3 +% Makes the DGPSwINS????? files for 2002 Greenland P3 field season +%see icards_gps_missinNASA_csv.m to get csv files for days without +%trajectory data. (check time reference: should be gps) +tic; + +global gRadar; + +support_path = ''; +data_support_path = ''; + +if isempty(support_path) + support_path = gRadar.support_path; +end + +gps_path = fullfile(support_path,'gps','1997_Greenland_P3'); +if ~exist(gps_path,'dir') + fprintf('Making directory %s\n', gps_path); + fprintf(' Press a key to proceed\n'); + pause; + mkdir(gps_path); +end + +if isempty(data_support_path) + data_support_path = gRadar.data_support_path; +end + +% ====================================================================== +% User Settings +% ====================================================================== +debug_level = 1; + +in_base_path = fullfile(data_support_path,'1997_Greenland_P3'); + +file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; +sync_fns = {}; sync_params = {}; + + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19970511_nmea.csv'); +out_fns{file_idx} = 'gps_19970511.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19970513_nmea.csv'); +out_fns{file_idx} = 'gps_19970513.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19970514_nmea.csv'); +out_fns{file_idx} = 'gps_19970514.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19970515_nmea.csv'); +out_fns{file_idx} = 'gps_19970515.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19970517_nmea.csv'); +out_fns{file_idx} = 'gps_19970517.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19970519_nmea.csv'); +out_fns{file_idx} = 'gps_19970519.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19970521_nmea.csv'); +out_fns{file_idx} = 'gps_19970521.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19970523_nmea.csv'); +out_fns{file_idx} = 'gps_19970523.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19970524_nmea.csv'); +out_fns{file_idx} = 'gps_19970524.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19970527_nmea.csv'); +out_fns{file_idx} = 'gps_19970527.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19970528_nmea.csv'); +out_fns{file_idx} = 'gps_19970528.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + + +gps_create; +match_idx = strmatch('gps_19950518.mat',out_fns,'exact'); +if ~isempty(match_idx) + gps_fn = fullfile(gps_path,out_fns{match_idx}); + fprintf('Fixing GPS data for %s\n', gps_fn); + gps = load(gps_fn); + % FIX CODE HERE + gps.elev(gps.elev > 10000) = NaN; + gps.elev = interp_finite(gps.elev); + save(gps_fn,'-append','-struct','gps','elev'); +end + diff --git a/cresis-toolbox/gps/missions/make_gps_1998_greenland_P3.m b/cresis-toolbox/gps/missions/gps_create_1998_Greenland_P3.m similarity index 96% rename from cresis-toolbox/gps/missions/make_gps_1998_greenland_P3.m rename to cresis-toolbox/gps/missions/gps_create_1998_Greenland_P3.m index c5f37ac4..968eca61 100644 --- a/cresis-toolbox/gps/missions/make_gps_1998_greenland_P3.m +++ b/cresis-toolbox/gps/missions/gps_create_1998_Greenland_P3.m @@ -1,129 +1,129 @@ -% script make_gps_2002_greenland_P3 -% Makes the DGPSwINS????? files for 2002 Greenland P3 field season -%see icards_gps_missinNASA_csv.m to get csv files for days without -%trajectory data. (check time reference: should be gps) -tic; - -global gRadar; - -support_path = ''; -data_support_path = ''; - -if isempty(support_path) - support_path = gRadar.support_path; -end - -gps_path = fullfile(support_path,'gps','1998_Greenland_P3'); -if ~exist(gps_path,'dir') - fprintf('Making directory %s\n', gps_path); - fprintf(' Press a key to proceed\n'); - pause; - mkdir(gps_path); -end - -if isempty(data_support_path) - data_support_path = gRadar.data_support_path; -end - -% ====================================================================== -% User Settings -% ====================================================================== -debug_level = 1; - -in_base_path = fullfile(data_support_path,'1998_Greenland_P3'); - -file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; -sync_fns = {}; sync_params = {}; - - - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19980627_nmea.csv'); -out_fns{file_idx} = 'gps_19980627.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19980629_nmea.csv'); -out_fns{file_idx} = 'gps_19980629.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19980630_nmea.csv'); -out_fns{file_idx} = 'gps_19980630.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19980703_nmea.csv'); -out_fns{file_idx} = 'gps_19980703.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19980704_nmea.csv'); -out_fns{file_idx} = 'gps_19980704.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19980713_nmea.csv'); -out_fns{file_idx} = 'gps_19980713.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19980714_nmea.csv'); -out_fns{file_idx} = 'gps_19980714.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19980715_nmea.csv'); -out_fns{file_idx} = 'gps_19980715.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19980716_nmea.csv'); -out_fns{file_idx} = 'gps_19980716.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19980717_nmea.csv'); -out_fns{file_idx} = 'gps_19980717.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19980718_nmea.csv'); -out_fns{file_idx} = 'gps_19980718.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - - -gps_make; -match_idx = strmatch('gps_19950518.mat',out_fns,'exact'); -if ~isempty(match_idx) - gps_fn = fullfile(gps_path,out_fns{match_idx}); - fprintf('Fixing GPS data for %s\n', gps_fn); - gps = load(gps_fn); - % FIX CODE HERE - gps.elev(gps.elev > 10000) = NaN; - gps.elev = interp_finite(gps.elev); - save(gps_fn,'-append','-struct','gps','elev'); -end - +% script gps_create_2002_greenland_P3 +% Makes the DGPSwINS????? files for 2002 Greenland P3 field season +%see icards_gps_missinNASA_csv.m to get csv files for days without +%trajectory data. (check time reference: should be gps) +tic; + +global gRadar; + +support_path = ''; +data_support_path = ''; + +if isempty(support_path) + support_path = gRadar.support_path; +end + +gps_path = fullfile(support_path,'gps','1998_Greenland_P3'); +if ~exist(gps_path,'dir') + fprintf('Making directory %s\n', gps_path); + fprintf(' Press a key to proceed\n'); + pause; + mkdir(gps_path); +end + +if isempty(data_support_path) + data_support_path = gRadar.data_support_path; +end + +% ====================================================================== +% User Settings +% ====================================================================== +debug_level = 1; + +in_base_path = fullfile(data_support_path,'1998_Greenland_P3'); + +file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; +sync_fns = {}; sync_params = {}; + + + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19980627_nmea.csv'); +out_fns{file_idx} = 'gps_19980627.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19980629_nmea.csv'); +out_fns{file_idx} = 'gps_19980629.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19980630_nmea.csv'); +out_fns{file_idx} = 'gps_19980630.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19980703_nmea.csv'); +out_fns{file_idx} = 'gps_19980703.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19980704_nmea.csv'); +out_fns{file_idx} = 'gps_19980704.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19980713_nmea.csv'); +out_fns{file_idx} = 'gps_19980713.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19980714_nmea.csv'); +out_fns{file_idx} = 'gps_19980714.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19980715_nmea.csv'); +out_fns{file_idx} = 'gps_19980715.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19980716_nmea.csv'); +out_fns{file_idx} = 'gps_19980716.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19980717_nmea.csv'); +out_fns{file_idx} = 'gps_19980717.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19980718_nmea.csv'); +out_fns{file_idx} = 'gps_19980718.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + + +gps_create; +match_idx = strmatch('gps_19950518.mat',out_fns,'exact'); +if ~isempty(match_idx) + gps_fn = fullfile(gps_path,out_fns{match_idx}); + fprintf('Fixing GPS data for %s\n', gps_fn); + gps = load(gps_fn); + % FIX CODE HERE + gps.elev(gps.elev > 10000) = NaN; + gps.elev = interp_finite(gps.elev); + save(gps_fn,'-append','-struct','gps','elev'); +end + diff --git a/cresis-toolbox/gps/missions/make_gps_1999_greenland_P3.m b/cresis-toolbox/gps/missions/gps_create_1999_Greenland_P3.m similarity index 96% rename from cresis-toolbox/gps/missions/make_gps_1999_greenland_P3.m rename to cresis-toolbox/gps/missions/gps_create_1999_Greenland_P3.m index 8c86862a..9ef78ff0 100644 --- a/cresis-toolbox/gps/missions/make_gps_1999_greenland_P3.m +++ b/cresis-toolbox/gps/missions/gps_create_1999_Greenland_P3.m @@ -1,141 +1,141 @@ -% script make_gps_2002_greenland_P3 -% Makes the DGPSwINS????? files for 2002 Greenland P3 field season -%see icards_gps_missinNASA_csv.m to get csv files for days without -%trajectory data. (check time reference: should be gps) -tic; - -global gRadar; - -support_path = ''; -data_support_path = ''; - -if isempty(support_path) - support_path = gRadar.support_path; -end - -gps_path = fullfile(support_path,'gps','1999_Greenland_P3'); -if ~exist(gps_path,'dir') - fprintf('Making directory %s\n', gps_path); - fprintf(' Press a key to proceed\n'); - pause; - mkdir(gps_path); -end - -if isempty(data_support_path) - data_support_path = gRadar.data_support_path; -end - -% ====================================================================== -% User Settings -% ====================================================================== -debug_level = 1; - -in_base_path = fullfile(data_support_path,'1999_Greenland_P3'); - -file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; -sync_fns = {}; sync_params = {}; - - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19990507_nmea.csv'); -out_fns{file_idx} = 'gps_19990507.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19990510_nmea.csv'); -out_fns{file_idx} = 'gps_19990510.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19990511_nmea.csv'); -out_fns{file_idx} = 'gps_19990511.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19990512_nmea.csv'); -out_fns{file_idx} = 'gps_19990512.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19990513_nmea.csv'); -out_fns{file_idx} = 'gps_19990513.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19990514_nmea.csv'); -out_fns{file_idx} = 'gps_19990514.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19990517_nmea.csv'); -out_fns{file_idx} = 'gps_19990517.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19990518_nmea.csv'); -out_fns{file_idx} = 'gps_19990518.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19990519_nmea.csv'); -out_fns{file_idx} = 'gps_19990519.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19990521_nmea.csv'); -out_fns{file_idx} = 'gps_19990521.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19990523_nmea.csv'); -out_fns{file_idx} = 'gps_19990523.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19990524_nmea.csv'); -out_fns{file_idx} = 'gps_19990524.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'19990525_nmea.csv'); -out_fns{file_idx} = 'gps_19990525.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -gps_make; -match_idx = strmatch('gps_19950518.mat',out_fns,'exact'); -if ~isempty(match_idx) - gps_fn = fullfile(gps_path,out_fns{match_idx}); - fprintf('Fixing GPS data for %s\n', gps_fn); - gps = load(gps_fn); - % FIX CODE HERE - gps.elev(gps.elev > 10000) = NaN; - gps.elev = interp_finite(gps.elev); - save(gps_fn,'-append','-struct','gps','elev'); -end - +% script gps_create_2002_greenland_P3 +% Makes the DGPSwINS????? files for 2002 Greenland P3 field season +%see icards_gps_missinNASA_csv.m to get csv files for days without +%trajectory data. (check time reference: should be gps) +tic; + +global gRadar; + +support_path = ''; +data_support_path = ''; + +if isempty(support_path) + support_path = gRadar.support_path; +end + +gps_path = fullfile(support_path,'gps','1999_Greenland_P3'); +if ~exist(gps_path,'dir') + fprintf('Making directory %s\n', gps_path); + fprintf(' Press a key to proceed\n'); + pause; + mkdir(gps_path); +end + +if isempty(data_support_path) + data_support_path = gRadar.data_support_path; +end + +% ====================================================================== +% User Settings +% ====================================================================== +debug_level = 1; + +in_base_path = fullfile(data_support_path,'1999_Greenland_P3'); + +file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; +sync_fns = {}; sync_params = {}; + + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19990507_nmea.csv'); +out_fns{file_idx} = 'gps_19990507.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19990510_nmea.csv'); +out_fns{file_idx} = 'gps_19990510.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19990511_nmea.csv'); +out_fns{file_idx} = 'gps_19990511.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19990512_nmea.csv'); +out_fns{file_idx} = 'gps_19990512.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19990513_nmea.csv'); +out_fns{file_idx} = 'gps_19990513.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19990514_nmea.csv'); +out_fns{file_idx} = 'gps_19990514.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19990517_nmea.csv'); +out_fns{file_idx} = 'gps_19990517.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19990518_nmea.csv'); +out_fns{file_idx} = 'gps_19990518.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19990519_nmea.csv'); +out_fns{file_idx} = 'gps_19990519.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19990521_nmea.csv'); +out_fns{file_idx} = 'gps_19990521.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19990523_nmea.csv'); +out_fns{file_idx} = 'gps_19990523.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19990524_nmea.csv'); +out_fns{file_idx} = 'gps_19990524.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'19990525_nmea.csv'); +out_fns{file_idx} = 'gps_19990525.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +gps_create; +match_idx = strmatch('gps_19950518.mat',out_fns,'exact'); +if ~isempty(match_idx) + gps_fn = fullfile(gps_path,out_fns{match_idx}); + fprintf('Fixing GPS data for %s\n', gps_fn); + gps = load(gps_fn); + % FIX CODE HERE + gps.elev(gps.elev > 10000) = NaN; + gps.elev = interp_finite(gps.elev); + save(gps_fn,'-append','-struct','gps','elev'); +end + diff --git a/cresis-toolbox/gps/missions/make_gps_2001_greenland_P3.m b/cresis-toolbox/gps/missions/gps_create_2001_Greenland_P3.m similarity index 96% rename from cresis-toolbox/gps/missions/make_gps_2001_greenland_P3.m rename to cresis-toolbox/gps/missions/gps_create_2001_Greenland_P3.m index 602e03cf..a640f3a4 100644 --- a/cresis-toolbox/gps/missions/make_gps_2001_greenland_P3.m +++ b/cresis-toolbox/gps/missions/gps_create_2001_Greenland_P3.m @@ -1,93 +1,93 @@ -% script make_gps_2002_greenland_P3 -% Makes the DGPSwINS????? files for 2002 Greenland P3 field season -%see icards_gps_missinNASA_csv.m to get csv files for days without -%trajectory data. (check time reference: should be gps) -tic; - -global gRadar; - -support_path = ''; -data_support_path = ''; - -if isempty(support_path) - support_path = gRadar.support_path; -end - -gps_path = fullfile(support_path,'gps','2001_Greenland_P3'); -if ~exist(gps_path,'dir') - fprintf('Making directory %s\n', gps_path); - fprintf(' Press a key to proceed\n'); - pause; - mkdir(gps_path); -end - -if isempty(data_support_path) - data_support_path = gRadar.data_support_path; -end - -% ====================================================================== -% User Settings -% ====================================================================== -debug_level = 1; - -in_base_path = fullfile(data_support_path,'2001_Greenland_P3'); - -file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; -sync_fns = {}; sync_params = {}; - - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'20010519_nmea.csv'); -out_fns{file_idx} = 'gps_20010519.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'20010520_nmea.csv'); -out_fns{file_idx} = 'gps_20010520.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'20010521_nmea.csv'); -out_fns{file_idx} = 'gps_20010521.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'20010523_nmea.csv'); -out_fns{file_idx} = 'gps_20010523.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'20010524_nmea.csv'); -out_fns{file_idx} = 'gps_20010524.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -file_idx = file_idx + 1; -in_fns{file_idx} = fullfile(in_base_path,'20010527_nmea.csv'); -out_fns{file_idx} = 'gps_20010527.mat'; -file_type{file_idx} = 'csv'; -params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process -gps_source{file_idx} = 'nmea-field'; - -gps_make; - -match_idx = strmatch('gps_19950518.mat',out_fns,'exact'); -if ~isempty(match_idx) - gps_fn = fullfile(gps_path,out_fns{match_idx}); - fprintf('Fixing GPS data for %s\n', gps_fn); - gps = load(gps_fn); - % FIX CODE HERE - gps.elev(gps.elev > 10000) = NaN; - gps.elev = interp_finite(gps.elev); - save(gps_fn,'-append','-struct','gps','elev'); -end - +% script gps_create_2002_greenland_P3 +% Makes the DGPSwINS????? files for 2002 Greenland P3 field season +%see icards_gps_missinNASA_csv.m to get csv files for days without +%trajectory data. (check time reference: should be gps) +tic; + +global gRadar; + +support_path = ''; +data_support_path = ''; + +if isempty(support_path) + support_path = gRadar.support_path; +end + +gps_path = fullfile(support_path,'gps','2001_Greenland_P3'); +if ~exist(gps_path,'dir') + fprintf('Making directory %s\n', gps_path); + fprintf(' Press a key to proceed\n'); + pause; + mkdir(gps_path); +end + +if isempty(data_support_path) + data_support_path = gRadar.data_support_path; +end + +% ====================================================================== +% User Settings +% ====================================================================== +debug_level = 1; + +in_base_path = fullfile(data_support_path,'2001_Greenland_P3'); + +file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; +sync_fns = {}; sync_params = {}; + + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'20010519_nmea.csv'); +out_fns{file_idx} = 'gps_20010519.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'20010520_nmea.csv'); +out_fns{file_idx} = 'gps_20010520.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'20010521_nmea.csv'); +out_fns{file_idx} = 'gps_20010521.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'20010523_nmea.csv'); +out_fns{file_idx} = 'gps_20010523.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'20010524_nmea.csv'); +out_fns{file_idx} = 'gps_20010524.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +file_idx = file_idx + 1; +in_fns{file_idx} = fullfile(in_base_path,'20010527_nmea.csv'); +out_fns{file_idx} = 'gps_20010527.mat'; +file_type{file_idx} = 'csv'; +params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process +gps_source{file_idx} = 'nmea-field'; + +gps_create; + +match_idx = strmatch('gps_19950518.mat',out_fns,'exact'); +if ~isempty(match_idx) + gps_fn = fullfile(gps_path,out_fns{match_idx}); + fprintf('Fixing GPS data for %s\n', gps_fn); + gps = load(gps_fn); + % FIX CODE HERE + gps.elev(gps.elev > 10000) = NaN; + gps.elev = interp_finite(gps.elev); + save(gps_fn,'-append','-struct','gps','elev'); +end + diff --git a/cresis-toolbox/gps/missions/make_gps_2002_greenland_P3.m b/cresis-toolbox/gps/missions/gps_create_2002_Greenland_P3.m similarity index 98% rename from cresis-toolbox/gps/missions/make_gps_2002_greenland_P3.m rename to cresis-toolbox/gps/missions/gps_create_2002_Greenland_P3.m index fc761f81..ab952176 100644 --- a/cresis-toolbox/gps/missions/make_gps_2002_greenland_P3.m +++ b/cresis-toolbox/gps/missions/gps_create_2002_Greenland_P3.m @@ -1,4 +1,4 @@ -% script make_gps_2002_greenland_P3 +% script gps_create_2002_greenland_P3 % Makes the DGPSwINS????? files for 2002 Greenland P3 field season %see icards_gps_missinNASA_csv.m to get csv files for days without %trajectory data. (check time reference: should be gps) @@ -105,7 +105,7 @@ params{file_idx} = struct('input_format','%f%f%f%f%f%f%f%f%f%f%f%f%f','time_reference','utc','type',[3]);%add a new type valued 3 for "read_gps_csv" to process gps_source{file_idx} = 'nmea-field'; -gps_make; +gps_create; match_idx = strmatch('gps_19950518.mat',out_fns,'exact'); if ~isempty(match_idx) gps_fn = fullfile(gps_path,out_fns{match_idx}); diff --git a/cresis-toolbox/gps/missions/make_gps_2003_Greenland_P3.m b/cresis-toolbox/gps/missions/gps_create_2003_Greenland_P3.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2003_Greenland_P3.m rename to cresis-toolbox/gps/missions/gps_create_2003_Greenland_P3.m index cd438126..d7375b5c 100644 --- a/cresis-toolbox/gps/missions/make_gps_2003_Greenland_P3.m +++ b/cresis-toolbox/gps/missions/gps_create_2003_Greenland_P3.m @@ -1,4 +1,4 @@ -% script make_gps_2003_Greenland_P3 +% script gps_create_2003_Greenland_P3 % % Makes the GPS files for 2003 Greenland P3 field season @@ -193,5 +193,5 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; diff --git a/cresis-toolbox/gps/missions/make_gps_2004_Antarctica_P3.m b/cresis-toolbox/gps/missions/gps_create_2004_Antarctica_P3.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2004_Antarctica_P3.m rename to cresis-toolbox/gps/missions/gps_create_2004_Antarctica_P3.m index f51e3e3b..f155a4c5 100644 --- a/cresis-toolbox/gps/missions/make_gps_2004_Antarctica_P3.m +++ b/cresis-toolbox/gps/missions/gps_create_2004_Antarctica_P3.m @@ -1,4 +1,4 @@ -% script make_gps_2004_Antarctica_P3 +% script gps_create_2004_Antarctica_P3 % % Makes the GPS files for 2004 Antarctica P3 field season @@ -144,5 +144,5 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; diff --git a/cresis-toolbox/gps/missions/make_gps_2005_greenland_TO.m b/cresis-toolbox/gps/missions/gps_create_2005_Greenland_TO.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2005_greenland_TO.m rename to cresis-toolbox/gps/missions/gps_create_2005_Greenland_TO.m index a22cd4ca..3a79d02a 100644 --- a/cresis-toolbox/gps/missions/make_gps_2005_greenland_TO.m +++ b/cresis-toolbox/gps/missions/gps_create_2005_Greenland_TO.m @@ -1,4 +1,4 @@ -% script make_gps_2005_greenland_TO_GPS +% script gps_create_2005_greenland_TO_GPS % % Makes the GPS files for 2005 Greenland Twin Otter field season @@ -310,5 +310,5 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; diff --git a/cresis-toolbox/gps/missions/make_gps_2006_Greenland_TO.m b/cresis-toolbox/gps/missions/gps_create_2006_Greenland_TO.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2006_Greenland_TO.m rename to cresis-toolbox/gps/missions/gps_create_2006_Greenland_TO.m index 0e40132f..5453dfb2 100644 --- a/cresis-toolbox/gps/missions/make_gps_2006_Greenland_TO.m +++ b/cresis-toolbox/gps/missions/gps_create_2006_Greenland_TO.m @@ -1,4 +1,4 @@ -% script make_gps_2006_greenland_TO +% script gps_create_2006_greenland_TO % % Makes the GPS files for 2006 Greenland TO field season @@ -351,7 +351,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; diff --git a/cresis-toolbox/gps/missions/make_gps_2008_greenland_GPRNEEM.m b/cresis-toolbox/gps/missions/gps_create_2008_Greenland_GPRNEEM.m similarity index 98% rename from cresis-toolbox/gps/missions/make_gps_2008_greenland_GPRNEEM.m rename to cresis-toolbox/gps/missions/gps_create_2008_Greenland_GPRNEEM.m index ad7075bd..5c7d4b32 100644 --- a/cresis-toolbox/gps/missions/make_gps_2008_greenland_GPRNEEM.m +++ b/cresis-toolbox/gps/missions/gps_create_2008_Greenland_GPRNEEM.m @@ -1,4 +1,4 @@ -% script make_gps_2008_greenland_Ground_NEEM_GPS +% script gps_create_2008_greenland_Ground_NEEM_GPS % % Makes the GPS files for 2008 Greenland Ground NEEM field season @@ -95,7 +95,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; if strcmpi(in_fns{file_idx}(end-11:end-4),'20080807') gps.time_offset = 9.1617; elseif strcmpi(in_fns{file_idx}(end-11:end-4),'20080808') diff --git a/cresis-toolbox/gps/missions/make_gps_2008_greenland_TO.m b/cresis-toolbox/gps/missions/gps_create_2008_Greenland_TO.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2008_greenland_TO.m rename to cresis-toolbox/gps/missions/gps_create_2008_Greenland_TO.m index 3ec5b1dd..6c3b29b5 100644 --- a/cresis-toolbox/gps/missions/make_gps_2008_greenland_TO.m +++ b/cresis-toolbox/gps/missions/gps_create_2008_Greenland_TO.m @@ -1,4 +1,4 @@ -% script make_gps_2008_greenland_TO.m +% script gps_create_2008_greenland_TO.m % % Makes the GPS files for 2008 Greenland Twin Otter field season % @@ -536,7 +536,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; if any(strcmpi('gps_20080630.mat',out_fns)) % The DGPS data does not span the full record, so we have to use NMEA diff --git a/cresis-toolbox/gps/missions/make_gps_2009_antarctica_DC8.m b/cresis-toolbox/gps/missions/gps_create_2009_Antarctica_DC8.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2009_antarctica_DC8.m rename to cresis-toolbox/gps/missions/gps_create_2009_Antarctica_DC8.m index 3b5e0d98..91a7a76f 100644 --- a/cresis-toolbox/gps/missions/make_gps_2009_antarctica_DC8.m +++ b/cresis-toolbox/gps/missions/gps_create_2009_Antarctica_DC8.m @@ -1,4 +1,4 @@ -% script make_gps_2009_antarctica_DC8 +% script gps_create_2009_antarctica_DC8 % % Makes the GPS files for 2009 Antarctica DC-8 field season @@ -212,4 +212,4 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; diff --git a/cresis-toolbox/gps/missions/make_gps_2009_antarctica_TO.m b/cresis-toolbox/gps/missions/gps_create_2009_Antarctica_TO.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2009_antarctica_TO.m rename to cresis-toolbox/gps/missions/gps_create_2009_Antarctica_TO.m index 454f2dc2..15b5b513 100644 --- a/cresis-toolbox/gps/missions/make_gps_2009_antarctica_TO.m +++ b/cresis-toolbox/gps/missions/gps_create_2009_Antarctica_TO.m @@ -1,4 +1,4 @@ -% script make_gps_2009_antarctica_TO +% script gps_create_2009_antarctica_TO % % Makes the GPS files for 2009 Antarctica TO field season % @@ -350,7 +350,7 @@ % ====================================================================== %% Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; % ====================================================================== diff --git a/cresis-toolbox/gps/missions/make_gps_2009_antarctica_TOGambit.m b/cresis-toolbox/gps/missions/gps_create_2009_Antarctica_TOGambit.m similarity index 98% rename from cresis-toolbox/gps/missions/make_gps_2009_antarctica_TOGambit.m rename to cresis-toolbox/gps/missions/gps_create_2009_Antarctica_TOGambit.m index f35e3ca3..47692ba8 100644 --- a/cresis-toolbox/gps/missions/make_gps_2009_antarctica_TOGambit.m +++ b/cresis-toolbox/gps/missions/gps_create_2009_Antarctica_TOGambit.m @@ -1,4 +1,4 @@ -% script make_gps_2009_antarctica_TO_Gambit_DGPSwINS +% script gps_create_2009_antarctica_TO_Gambit_DGPSwINS % % Makes the DGPSwINS files for 2009 Antarctica TO field season diff --git a/cresis-toolbox/gps/missions/make_gps_2009_greenland_P3.m b/cresis-toolbox/gps/missions/gps_create_2009_Greenland_P3.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2009_greenland_P3.m rename to cresis-toolbox/gps/missions/gps_create_2009_Greenland_P3.m index f10527c9..00f4a017 100644 --- a/cresis-toolbox/gps/missions/make_gps_2009_greenland_P3.m +++ b/cresis-toolbox/gps/missions/gps_create_2009_Greenland_P3.m @@ -1,4 +1,4 @@ -% script make_gps_2009_greenland_P3_DGPSwINS +% script gps_create_2009_greenland_P3_DGPSwINS % % Makes the DGPSwINS files for 2009 Greenland P3 field season @@ -214,5 +214,5 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; diff --git a/cresis-toolbox/gps/missions/make_gps_2009_greenland_TO.m b/cresis-toolbox/gps/missions/gps_create_2009_Greenland_TO.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2009_greenland_TO.m rename to cresis-toolbox/gps/missions/gps_create_2009_Greenland_TO.m index 8b50b079..44eba07d 100644 --- a/cresis-toolbox/gps/missions/make_gps_2009_greenland_TO.m +++ b/cresis-toolbox/gps/missions/gps_create_2009_Greenland_TO.m @@ -1,4 +1,4 @@ -% script make_gps_2009_greenland_TO.m +% script gps_create_2009_greenland_TO.m % % Makes the GPS files for 2009 Greenland Twin Otter field season % @@ -223,5 +223,5 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; diff --git a/cresis-toolbox/gps/missions/make_gps_2010_antarctica_DC8.m b/cresis-toolbox/gps/missions/gps_create_2010_Antarctica_DC8.m similarity index 98% rename from cresis-toolbox/gps/missions/make_gps_2010_antarctica_DC8.m rename to cresis-toolbox/gps/missions/gps_create_2010_Antarctica_DC8.m index 2b103215..b6afdc25 100644 --- a/cresis-toolbox/gps/missions/make_gps_2010_antarctica_DC8.m +++ b/cresis-toolbox/gps/missions/gps_create_2010_Antarctica_DC8.m @@ -1,4 +1,4 @@ -% script make_gps_2010_antarctica_DC8 +% script gps_create_2010_antarctica_DC8 % % Makes the GPS files for 2010 Antarctica DC-8 field season @@ -120,5 +120,5 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; diff --git a/cresis-toolbox/gps/missions/make_gps_2010_greenland_DC8.m b/cresis-toolbox/gps/missions/gps_create_2010_Greenland_DC8.m similarity index 98% rename from cresis-toolbox/gps/missions/make_gps_2010_greenland_DC8.m rename to cresis-toolbox/gps/missions/gps_create_2010_Greenland_DC8.m index d9b021bc..fd059ac5 100644 --- a/cresis-toolbox/gps/missions/make_gps_2010_greenland_DC8.m +++ b/cresis-toolbox/gps/missions/gps_create_2010_Greenland_DC8.m @@ -1,4 +1,4 @@ -% script make_gps_2010_greenland_DC8_DGPSwINS +% script gps_create_2010_greenland_DC8_DGPSwINS % % Makes the DGPSwINS files for 2010 Greenland DC-8 field season @@ -149,5 +149,5 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; diff --git a/cresis-toolbox/gps/missions/make_gps_2010_greenland_P3.m b/cresis-toolbox/gps/missions/gps_create_2010_Greenland_P3.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2010_greenland_P3.m rename to cresis-toolbox/gps/missions/gps_create_2010_Greenland_P3.m index 9e5ae1b4..6ebb05f0 100644 --- a/cresis-toolbox/gps/missions/make_gps_2010_greenland_P3.m +++ b/cresis-toolbox/gps/missions/gps_create_2010_Greenland_P3.m @@ -1,4 +1,4 @@ -% script make_gps_2010_greenland_P3 +% script gps_create_2010_greenland_P3 % % Makes the DGPSwINS files for 2010 Greenland P3 field season @@ -151,5 +151,5 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; diff --git a/cresis-toolbox/gps/missions/make_gps_2011_antarctica_DC8.m b/cresis-toolbox/gps/missions/gps_create_2011_Antarctica_DC8.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2011_antarctica_DC8.m rename to cresis-toolbox/gps/missions/gps_create_2011_Antarctica_DC8.m index 4742fb88..e117ec19 100644 --- a/cresis-toolbox/gps/missions/make_gps_2011_antarctica_DC8.m +++ b/cresis-toolbox/gps/missions/gps_create_2011_Antarctica_DC8.m @@ -1,4 +1,4 @@ -% script make_gps_2011_antarctica_DC8_GPS +% script gps_create_2011_antarctica_DC8_GPS % % Makes the GPS files for 2011 Antarctica DC8 field season @@ -712,5 +712,5 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; diff --git a/cresis-toolbox/gps/missions/make_gps_2011_Antarctica_TO.m b/cresis-toolbox/gps/missions/gps_create_2011_Antarctica_TO.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2011_Antarctica_TO.m rename to cresis-toolbox/gps/missions/gps_create_2011_Antarctica_TO.m index d36b45bb..7b6b7671 100644 --- a/cresis-toolbox/gps/missions/make_gps_2011_Antarctica_TO.m +++ b/cresis-toolbox/gps/missions/gps_create_2011_Antarctica_TO.m @@ -1,4 +1,4 @@ -% script make_gps_2011_antarctica_TO +% script gps_create_2011_antarctica_TO % % Makes the GPS files for 2011 Antarctica TO field season @@ -283,4 +283,4 @@ % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; diff --git a/cresis-toolbox/gps/missions/make_gps_2011_Austfonna_GPR.m b/cresis-toolbox/gps/missions/gps_create_2011_Austfonna_GPR.m similarity index 98% rename from cresis-toolbox/gps/missions/make_gps_2011_Austfonna_GPR.m rename to cresis-toolbox/gps/missions/gps_create_2011_Austfonna_GPR.m index b364bbe1..90bb9e25 100644 --- a/cresis-toolbox/gps/missions/make_gps_2011_Austfonna_GPR.m +++ b/cresis-toolbox/gps/missions/gps_create_2011_Austfonna_GPR.m @@ -1,4 +1,4 @@ -% script make_gps_2011_Austfonna_GPR +% script gps_create_2011_Austfonna_GPR % % Makes the GPS files for 2011 Austfonna GPR field season @@ -106,5 +106,5 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; diff --git a/cresis-toolbox/gps/missions/make_gps_2011_greenland_P3.m b/cresis-toolbox/gps/missions/gps_create_2011_Greenland_P3.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2011_greenland_P3.m rename to cresis-toolbox/gps/missions/gps_create_2011_Greenland_P3.m index 8c8890a2..eb635a5f 100644 --- a/cresis-toolbox/gps/missions/make_gps_2011_greenland_P3.m +++ b/cresis-toolbox/gps/missions/gps_create_2011_Greenland_P3.m @@ -1,4 +1,4 @@ -% script make_gps_2011_greenland_P3_GPS +% script gps_create_2011_greenland_P3_GPS % % Makes the GPS files for 2011 Greenland P3 field season @@ -891,7 +891,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; if any(strcmpi('gps_20110328.mat',out_fns)) file_idx = find(strcmpi('gps_20110328.mat',out_fns)) diff --git a/cresis-toolbox/gps/missions/make_gps_2011_greenland_TO.m b/cresis-toolbox/gps/missions/gps_create_2011_Greenland_TO.m similarity index 98% rename from cresis-toolbox/gps/missions/make_gps_2011_greenland_TO.m rename to cresis-toolbox/gps/missions/gps_create_2011_Greenland_TO.m index 64349912..77ec34a1 100644 --- a/cresis-toolbox/gps/missions/make_gps_2011_greenland_TO.m +++ b/cresis-toolbox/gps/missions/gps_create_2011_Greenland_TO.m @@ -1,4 +1,4 @@ -% script make_gps_2011_greenland_TO +% script gps_create_2011_greenland_TO % % Makes the GPS files for 2011 Greenland Twin Otter field season @@ -130,4 +130,4 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; diff --git a/cresis-toolbox/gps/missions/make_gps_2012_antarctica_DC8.m b/cresis-toolbox/gps/missions/gps_create_2012_Antarctica_DC8.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2012_antarctica_DC8.m rename to cresis-toolbox/gps/missions/gps_create_2012_Antarctica_DC8.m index cbbfb703..9bdf063e 100644 --- a/cresis-toolbox/gps/missions/make_gps_2012_antarctica_DC8.m +++ b/cresis-toolbox/gps/missions/gps_create_2012_Antarctica_DC8.m @@ -1,4 +1,4 @@ -% script make_gps_2012_antarctica_DC8 +% script gps_create_2012_antarctica_DC8 % % Makes the GPS files for 2012 Antarctica DC8 field season @@ -482,7 +482,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; % Hand corrections to GPS files go here so that everything is automated diff --git a/cresis-toolbox/gps/missions/make_gps_2012_antarctica_GPRwais.m b/cresis-toolbox/gps/missions/gps_create_2012_Antarctica_GPRwais.m similarity index 96% rename from cresis-toolbox/gps/missions/make_gps_2012_antarctica_GPRwais.m rename to cresis-toolbox/gps/missions/gps_create_2012_Antarctica_GPRwais.m index cfa994e7..1496529c 100644 --- a/cresis-toolbox/gps/missions/make_gps_2012_antarctica_GPRwais.m +++ b/cresis-toolbox/gps/missions/gps_create_2012_Antarctica_GPRwais.m @@ -1,4 +1,4 @@ -% script make_gps_2012_antarctica_GPRwais +% script gps_create_2012_antarctica_GPRwais % % Makes the GPS files for 2012 Greenland P3 field season @@ -52,5 +52,5 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; diff --git a/cresis-toolbox/gps/missions/make_gps_2012_greenland_P3.m b/cresis-toolbox/gps/missions/gps_create_2012_Greenland_P3.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2012_greenland_P3.m rename to cresis-toolbox/gps/missions/gps_create_2012_Greenland_P3.m index 6df842a8..6f7db695 100644 --- a/cresis-toolbox/gps/missions/make_gps_2012_greenland_P3.m +++ b/cresis-toolbox/gps/missions/gps_create_2012_Greenland_P3.m @@ -1,4 +1,4 @@ -% script make_gps_2012_greenland_P3 +% script gps_create_2012_greenland_P3 % % Makes the GPS files for 2012 Greenland P3 field season @@ -1332,9 +1332,9 @@ end -%% make_gps +%% gps_create % ====================================================================== -gps_make; +gps_create; % Hand correction of gps_20120317 accum2 sync radar time information hand_idx = strmatch('gps_20120317.mat',out_fns); diff --git a/cresis-toolbox/gps/missions/make_gps_2013_antarctica_Basler.m b/cresis-toolbox/gps/missions/gps_create_2013_Antarctica_Basler.m similarity index 98% rename from cresis-toolbox/gps/missions/make_gps_2013_antarctica_Basler.m rename to cresis-toolbox/gps/missions/gps_create_2013_Antarctica_Basler.m index e39f8922..524753b1 100644 --- a/cresis-toolbox/gps/missions/make_gps_2013_antarctica_Basler.m +++ b/cresis-toolbox/gps/missions/gps_create_2013_Antarctica_Basler.m @@ -1,4 +1,4 @@ -% script make_gps_2013_antarctica_Basler +% script gps_create_2013_antarctica_Basler % % Makes the GPS files for 2013 Antarctica Basler field season @@ -241,7 +241,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; hack1_idx = cell2mat(strfind(out_fns,'gps_20130829.mat')); if ~isempty(hack1_idx) @@ -254,7 +254,7 @@ if any(strcmpi('gps_20131231.mat',out_fns)) ... || any(strcmpi('gps_20140111.mat',out_fns)) - warning('Jan 11 and Dec 31 travelled across 180 deg line and GPS file produced by Novatel may have errors! Novatel smoothing will produce points which interpolate -179.999 to +179.999 and vice versa incorrectly and you have to manually remove these points from the .txt input file before running make_gps.'); + warning('Jan 11 and Dec 31 travelled across 180 deg line and GPS file produced by Novatel may have errors! Novatel smoothing will produce points which interpolate -179.999 to +179.999 and vice versa incorrectly and you have to manually remove these points from the .txt input file before running gps_create.'); keyboard end diff --git a/cresis-toolbox/gps/missions/make_gps_2013_antarctica_Ground.m b/cresis-toolbox/gps/missions/gps_create_2013_Antarctica_Ground.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2013_antarctica_Ground.m rename to cresis-toolbox/gps/missions/gps_create_2013_Antarctica_Ground.m index 111e09b5..11e089e9 100644 --- a/cresis-toolbox/gps/missions/make_gps_2013_antarctica_Ground.m +++ b/cresis-toolbox/gps/missions/gps_create_2013_Antarctica_Ground.m @@ -1,4 +1,4 @@ -% script make_gps_2013_antarctica_Ground +% script gps_create_2013_antarctica_Ground % % Makes the GPS files for 2013 Antarctica Basler field season @@ -189,7 +189,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; % Smooth NMEA GPS data diff --git a/cresis-toolbox/gps/missions/make_gps_2013_antarctica_P3.m b/cresis-toolbox/gps/missions/gps_create_2013_Antarctica_P3.m similarity index 92% rename from cresis-toolbox/gps/missions/make_gps_2013_antarctica_P3.m rename to cresis-toolbox/gps/missions/gps_create_2013_Antarctica_P3.m index 6c7ccdcd..534362e4 100644 --- a/cresis-toolbox/gps/missions/make_gps_2013_antarctica_P3.m +++ b/cresis-toolbox/gps/missions/gps_create_2013_Antarctica_P3.m @@ -1,4 +1,4 @@ -% script make_gps_2013_antarctica_P3 +% script gps_create_2013_antarctica_P3 % % Makes the GPS files for 2013 Antarctica P3 field season @@ -238,7 +238,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; % Hand correction of gps_20131107 accum2 sync radar time information hand_idx = strmatch('gps_20131107.mat',out_fns); @@ -257,7 +257,7 @@ along_track = geodetic_to_along_track(gps.lat,gps.lon,gps.elev); rlines = get_equal_alongtrack_spacing_idxs(along_track,10); - physical_constants; + physical_constants; % Load WGS84.spheroid for rline_idx = 1:length(rlines) rline = rlines(rline_idx); if rline_idx < length(rlines) @@ -265,14 +265,14 @@ else rline_end = length(along_track); end - [origin(1),origin(2),origin(3)] = geodetic2ecef(gps.lat(rline)/180*pi,gps.lon(rline)/180*pi,gps.elev(rline),WGS84.ellipsoid); - [heading(1),heading(2),heading(3)] = geodetic2ecef(gps.lat(rline_end)/180*pi,gps.lon(rline_end)/180*pi,gps.elev(rline_end),WGS84.ellipsoid); + [origin(1),origin(2),origin(3)] = geodetic2ecef(WGS84.spheroid, gps.lat(rline),gps.lon(rline),gps.elev(rline)); + [heading(1),heading(2),heading(3)] = geodetic2ecef(WGS84.spheroid, gps.lat(rline_end),gps.lon(rline_end),gps.elev(rline_end)); heading = heading - origin; % Determine east vector - [east(1) east(2) east(3)] = lv2ecef(1,0,0,gps.lat(rline)/180*pi,gps.lon(rline)/180*pi,gps.elev(rline),WGS84.ellipsoid); + [east(1) east(2) east(3)] = enu2ecef(1,0,0,gps.lat(rline),gps.lon(rline),gps.elev(rline),WGS84.spheroid); east = east - origin; % Determine north vector - [north(1) north(2) north(3)] = lv2ecef(0,1,0,gps.lat(rline)/180*pi,gps.lon(rline)/180*pi,gps.elev(rline),WGS84.ellipsoid); + [north(1) north(2) north(3)] = enu2ecef(0,1,0,gps.lat(rline),gps.lon(rline),gps.elev(rline),WGS84.spheroid); north = north - origin; % Determine heading gps.heading(rline:rline_end) = atan2(dot(east,heading),dot(north,heading)); @@ -302,7 +302,7 @@ along_track = geodetic_to_along_track(gps.lat,gps.lon,gps.elev); rlines = get_equal_alongtrack_spacing_idxs(along_track,10); - physical_constants; + physical_constants; % Load WGS84.spheroid for rline_idx = 1:length(rlines) rline = rlines(rline_idx); if rline_idx < length(rlines) @@ -310,14 +310,14 @@ else rline_end = length(along_track); end - [origin(1),origin(2),origin(3)] = geodetic2ecef(gps.lat(rline)/180*pi,gps.lon(rline)/180*pi,gps.elev(rline),WGS84.ellipsoid); - [heading(1),heading(2),heading(3)] = geodetic2ecef(gps.lat(rline_end)/180*pi,gps.lon(rline_end)/180*pi,gps.elev(rline_end),WGS84.ellipsoid); + [origin(1),origin(2),origin(3)] = geodetic2ecef(WGS84.spheroid, gps.lat(rline),gps.lon(rline),gps.elev(rline)); + [heading(1),heading(2),heading(3)] = geodetic2ecef(WGS84.spheroid, gps.lat(rline_end),gps.lon(rline_end),gps.elev(rline_end)); heading = heading - origin; % Determine east vector - [east(1) east(2) east(3)] = lv2ecef(1,0,0,gps.lat(rline)/180*pi,gps.lon(rline)/180*pi,gps.elev(rline),WGS84.ellipsoid); + [east(1) east(2) east(3)] = enu2ecef(1,0,0,gps.lat(rline),gps.lon(rline),gps.elev(rline),WGS84.spheroid); east = east - origin; % Determine north vector - [north(1) north(2) north(3)] = lv2ecef(0,1,0,gps.lat(rline)/180*pi,gps.lon(rline)/180*pi,gps.elev(rline),WGS84.ellipsoid); + [north(1) north(2) north(3)] = enu2ecef(0,1,0,gps.lat(rline),gps.lon(rline),gps.elev(rline),WGS84.spheroid); north = north - origin; % Determine heading gps.heading(rline:rline_end) = atan2(dot(east,heading),dot(north,heading)); diff --git a/cresis-toolbox/gps/missions/make_gps_2013_antarctica_Sled.m b/cresis-toolbox/gps/missions/gps_create_2013_Antarctica_Sled.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2013_antarctica_Sled.m rename to cresis-toolbox/gps/missions/gps_create_2013_Antarctica_Sled.m index 968ae9ae..eeb0e53e 100644 --- a/cresis-toolbox/gps/missions/make_gps_2013_antarctica_Sled.m +++ b/cresis-toolbox/gps/missions/gps_create_2013_Antarctica_Sled.m @@ -1,4 +1,4 @@ -% script make_gps_2013_antarctica_Ground +% script gps_create_2013_antarctica_Ground % % Makes the GPS files for 2013 Antarctica Basler field season @@ -180,7 +180,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; % Smooth NMEA GPS data diff --git a/cresis-toolbox/gps/missions/make_gps_2013_greenland_P3.m b/cresis-toolbox/gps/missions/gps_create_2013_Greenland_P3.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2013_greenland_P3.m rename to cresis-toolbox/gps/missions/gps_create_2013_Greenland_P3.m index e259fca7..ba45e424 100644 --- a/cresis-toolbox/gps/missions/make_gps_2013_greenland_P3.m +++ b/cresis-toolbox/gps/missions/gps_create_2013_Greenland_P3.m @@ -1,4 +1,4 @@ -% script make_gps_2013_greenland_P3 +% script gps_create_2013_greenland_P3 % % Makes the GPS files for 2013 Greenland P3 field season @@ -584,7 +584,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; diff --git a/cresis-toolbox/gps/missions/make_gps_2014_alaska_TOnrl.m b/cresis-toolbox/gps/missions/gps_create_2014_Alaska_TOnrl.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2014_alaska_TOnrl.m rename to cresis-toolbox/gps/missions/gps_create_2014_Alaska_TOnrl.m index e30776f1..7e99306c 100644 --- a/cresis-toolbox/gps/missions/make_gps_2014_alaska_TOnrl.m +++ b/cresis-toolbox/gps/missions/gps_create_2014_Alaska_TOnrl.m @@ -1,4 +1,4 @@ -% script make_gps_2014_alaska_TOnrl +% script gps_create_2014_alaska_TOnrl % % Makes the GPS files for 2014 Alaska TOnrl field season @@ -141,7 +141,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; % Fabricate loopback GPS data for testing % gps.gps_time = datenum_to_epoch(datenum(2014,02,25,0,0,0:1000)); diff --git a/cresis-toolbox/gps/missions/make_gps_2014_antarctica_DC8.m b/cresis-toolbox/gps/missions/gps_create_2014_Antarctica_DC8.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2014_antarctica_DC8.m rename to cresis-toolbox/gps/missions/gps_create_2014_Antarctica_DC8.m index f27b8769..27eb5afe 100644 --- a/cresis-toolbox/gps/missions/make_gps_2014_antarctica_DC8.m +++ b/cresis-toolbox/gps/missions/gps_create_2014_Antarctica_DC8.m @@ -1,4 +1,4 @@ -% script make_gps_2014_antarctica_DC8 +% script gps_create_2014_antarctica_DC8 % % Makes the GPS files for 2014 Antarctica DC8 field season @@ -522,7 +522,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; hack_idx = cell2mat(strfind(out_fns,'gps_20140904.mat')); if ~isempty(hack_idx) diff --git a/cresis-toolbox/gps/missions/make_gps_2014_greenland_P3.m b/cresis-toolbox/gps/missions/gps_create_2014_Greenland_P3.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2014_greenland_P3.m rename to cresis-toolbox/gps/missions/gps_create_2014_Greenland_P3.m index 74fc17f1..ce5bb764 100644 --- a/cresis-toolbox/gps/missions/make_gps_2014_greenland_P3.m +++ b/cresis-toolbox/gps/missions/gps_create_2014_Greenland_P3.m @@ -1,4 +1,4 @@ -% script make_gps_2014_greenland_P3 +% script gps_create_2014_greenland_P3 % % Makes the GPS files for 2014 Greenland P3 field season @@ -1358,7 +1358,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; diff --git a/cresis-toolbox/gps/missions/make_gps_2015_antarctica_Ground.m b/cresis-toolbox/gps/missions/gps_create_2015_Antarctica_Ground.m similarity index 97% rename from cresis-toolbox/gps/missions/make_gps_2015_antarctica_Ground.m rename to cresis-toolbox/gps/missions/gps_create_2015_Antarctica_Ground.m index 935019a3..6fffe61c 100644 --- a/cresis-toolbox/gps/missions/make_gps_2015_antarctica_Ground.m +++ b/cresis-toolbox/gps/missions/gps_create_2015_Antarctica_Ground.m @@ -1,233 +1,233 @@ -% script make_gps_2015_antarctica_Ground -% -% Makes the GPS files for 2015 Antarctica Ground field season - -tic; - -global gRadar; - -support_path = ''; -data_support_path = ''; - -if isempty(support_path) - support_path = gRadar.support_path; -end - -gps_path = fullfile(support_path,'gps','2015_Antarctica_Ground'); -if ~exist(gps_path,'dir') - fprintf('Making directory %s\n', gps_path); - fprintf(' Press a key to proceed\n'); - pause; - mkdir(gps_path); -end - -if isempty(data_support_path) - data_support_path = gRadar.data_support_path; -end - -% ====================================================================== -% User Settings -% ====================================================================== -debug_level = 1; - -in_base_path = fullfile(data_support_path,'2015_Antarctica_Ground'); - -file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; -sync_fns = {}; sync_params = {}; - -gps_source_to_use = 'accum'; - -if strcmpi(gps_source_to_use,'accum') - file_idx = file_idx + 1; - year = 2015; month = 11; day = 20; - in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); - file_type{file_idx} = 'NMEA'; - params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); - gps_source{file_idx} = 'nmea-field'; - sync_flag{file_idx} = 1; - sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); - sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','format',3); - - file_idx = file_idx + 1; - year = 2015; month = 11; day = 21; - in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); - file_type{file_idx} = 'NMEA'; - params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); - gps_source{file_idx} = 'nmea-field'; - sync_flag{file_idx} = 1; - sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); - sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','format',3); - - file_idx = file_idx + 1; - year = 2015; month = 11; day = 24; - in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); - file_type{file_idx} = 'NMEA'; - params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); - gps_source{file_idx} = 'nmea-field'; - sync_flag{file_idx} = 1; - sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); - sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','format',3); - - file_idx = file_idx + 1; - year = 2015; month = 12; day = 2; - in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); - file_type{file_idx} = 'NMEA'; - params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); - gps_source{file_idx} = 'nmea-field'; - sync_flag{file_idx} = 1; - sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); - sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','format',3); - - file_idx = file_idx + 1; - year = 2015; month = 12; day = 3; - in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); - file_type{file_idx} = 'NMEA'; - params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); - gps_source{file_idx} = 'nmea-field'; - sync_flag{file_idx} = 1; - sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); - sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','format',3); - - file_idx = file_idx + 1; - year = 2015; month = 12; day = 4; - in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); - file_type{file_idx} = 'NMEA'; - params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); - gps_source{file_idx} = 'nmea-field'; - sync_flag{file_idx} = 1; - sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); - sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','format',3); - - file_idx = file_idx + 1; - year = 2015; month = 12; day = 9; - in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); - file_type{file_idx} = 'NMEA'; - params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); - gps_source{file_idx} = 'nmea-field'; - sync_flag{file_idx} = 1; - sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); - sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','format',3); - - file_idx = file_idx + 1; - year = 2015; month = 12; day = 10; - in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); - file_type{file_idx} = 'NMEA'; - params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); - gps_source{file_idx} = 'nmea-field'; - sync_flag{file_idx} = 1; - sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); - sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','format',3); - - file_idx = file_idx + 1; - year = 2015; month = 12; day = 16; - in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); - file_type{file_idx} = 'NMEA'; - params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); - gps_source{file_idx} = 'nmea-field'; - sync_flag{file_idx} = 1; - sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); - sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','format',3); - - file_idx = file_idx + 1; - year = 2015; month = 12; day = 18; - in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); - file_type{file_idx} = 'NMEA'; - params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); - gps_source{file_idx} = 'nmea-field'; - sync_flag{file_idx} = 1; - sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); - sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','format',3); - - file_idx = file_idx + 1; - year = 2015; month = 12; day = 20; - in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); - file_type{file_idx} = 'NMEA'; - params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); - gps_source{file_idx} = 'nmea-field'; - sync_flag{file_idx} = 1; - sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); - sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','format',3); - - file_idx = file_idx + 1; - year = 2015; month = 12; day = 21; - in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); - file_type{file_idx} = 'NMEA'; - params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); - gps_source{file_idx} = 'nmea-field'; - sync_flag{file_idx} = 1; - sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); - sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','format',3); - -end - -% ====================================================================== -% Read and translate files according to user settings -% ====================================================================== -gps_make; - -%% Lab Measurement Data: Fakes GPS position information -% match_idx = strmatch('gps_20150413.mat',out_fns,'exact'); -% if ~isempty(match_idx) -% gps_fn = fullfile(gps_path,out_fns{match_idx}); -% fprintf('Creating fake gps data for %s\n', gps_fn); -% gps = load(gps_fn); -% gps.lon = -45 * ones(size(gps.lon)); -% gps.lat = 70 + (1:length(gps.gps_time)) * 1e-4; -% save(gps_fn,'-append','-struct','gps','lat','lon'); -% end - -%% Conversion from GPS to UTC time since system was not allowed enough -% time to warm up before operation. -match_idx = strmatch('gps_20151202.mat',out_fns,'exact'); -if ~isempty(match_idx) - gps_fn = fullfile(gps_path,out_fns{match_idx}); - fprintf('Fixing GPS to UTC time conversion for %s\n', gps_fn); - gps = load(gps_fn); - gps.gps_time(1:340) = gps.gps_time(1:340) - utc_leap_seconds(gps.gps_time(1)); - gps.gps_time(5956:6036) = gps.gps_time(5956:6036) - utc_leap_seconds(gps.gps_time(1)); - gps.sync_gps_time(1:340) = gps.sync_gps_time(1:340) - utc_leap_seconds(gps.sync_gps_time(1)); - gps.sync_gps_time(5956:6036) = gps.sync_gps_time(5956:6036) - utc_leap_seconds(gps.gps_time(1)); - save(gps_fn,'-append','-struct','gps','gps_time','sync_gps_time'); -end -match_idx = strmatch('gps_20151221.mat',out_fns,'exact'); -if ~isempty(match_idx) - gps_fn = fullfile(gps_path,out_fns{match_idx}); - fprintf('Fixing GPS to UTC time conversion for %s\n', gps_fn); - gps = load(gps_fn); - gps.gps_time(1:107) = gps.gps_time(1:107) - utc_leap_seconds(gps.gps_time(1)); - gps.sync_gps_time(1:107) = gps.sync_gps_time(1:107) - utc_leap_seconds(gps.sync_gps_time(1)); - save(gps_fn,'-append','-struct','gps','gps_time','sync_gps_time'); -end - -%% Filter NMEA data -for idx = 1:length(out_fns) - gps_fn = fullfile(gps_path,out_fns{idx}); - fprintf('Filtering %s\n', gps_fn); - gps = load(gps_fn); - spacing = 10; - [along_track,gps.lat,gps.lon,gps.elev] = geodetic_to_along_track(gps.lat,gps.lon,gps.elev,spacing); - cur_along_track = along_track(1); cur_idx = 1; - for idx = 2:length(along_track) - if along_track(idx) <= cur_along_track - gps.lat(idx) = gps.lat(cur_idx); - gps.lon(idx) = gps.lon(cur_idx); - gps.elev(idx) = gps.elev(cur_idx); - else - cur_along_track = along_track(idx); - cur_idx = idx; - end - end - save(gps_fn,'-append','-struct','gps','lat','lon','elev'); -end +% script gps_create_2015_antarctica_Ground +% +% Makes the GPS files for 2015 Antarctica Ground field season + +tic; + +global gRadar; + +support_path = ''; +data_support_path = ''; + +if isempty(support_path) + support_path = gRadar.support_path; +end + +gps_path = fullfile(support_path,'gps','2015_Antarctica_Ground'); +if ~exist(gps_path,'dir') + fprintf('Making directory %s\n', gps_path); + fprintf(' Press a key to proceed\n'); + pause; + mkdir(gps_path); +end + +if isempty(data_support_path) + data_support_path = gRadar.data_support_path; +end + +% ====================================================================== +% User Settings +% ====================================================================== +debug_level = 1; + +in_base_path = fullfile(data_support_path,'2015_Antarctica_Ground'); + +file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; +sync_fns = {}; sync_params = {}; + +gps_source_to_use = 'accum'; + +if strcmpi(gps_source_to_use,'accum') + file_idx = file_idx + 1; + year = 2015; month = 11; day = 20; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); + file_type{file_idx} = 'NMEA'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); + gps_source{file_idx} = 'nmea-field'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); + sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','format',3); + + file_idx = file_idx + 1; + year = 2015; month = 11; day = 21; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); + file_type{file_idx} = 'NMEA'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); + gps_source{file_idx} = 'nmea-field'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); + sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','format',3); + + file_idx = file_idx + 1; + year = 2015; month = 11; day = 24; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); + file_type{file_idx} = 'NMEA'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); + gps_source{file_idx} = 'nmea-field'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); + sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','format',3); + + file_idx = file_idx + 1; + year = 2015; month = 12; day = 2; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); + file_type{file_idx} = 'NMEA'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); + gps_source{file_idx} = 'nmea-field'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); + sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','format',3); + + file_idx = file_idx + 1; + year = 2015; month = 12; day = 3; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); + file_type{file_idx} = 'NMEA'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); + gps_source{file_idx} = 'nmea-field'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); + sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','format',3); + + file_idx = file_idx + 1; + year = 2015; month = 12; day = 4; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); + file_type{file_idx} = 'NMEA'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); + gps_source{file_idx} = 'nmea-field'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); + sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','format',3); + + file_idx = file_idx + 1; + year = 2015; month = 12; day = 9; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); + file_type{file_idx} = 'NMEA'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); + gps_source{file_idx} = 'nmea-field'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); + sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','format',3); + + file_idx = file_idx + 1; + year = 2015; month = 12; day = 10; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); + file_type{file_idx} = 'NMEA'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); + gps_source{file_idx} = 'nmea-field'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); + sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','format',3); + + file_idx = file_idx + 1; + year = 2015; month = 12; day = 16; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); + file_type{file_idx} = 'NMEA'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); + gps_source{file_idx} = 'nmea-field'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); + sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','format',3); + + file_idx = file_idx + 1; + year = 2015; month = 12; day = 18; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); + file_type{file_idx} = 'NMEA'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); + gps_source{file_idx} = 'nmea-field'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); + sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','format',3); + + file_idx = file_idx + 1; + year = 2015; month = 12; day = 20; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); + file_type{file_idx} = 'NMEA'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); + gps_source{file_idx} = 'nmea-field'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); + sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','format',3); + + file_idx = file_idx + 1; + year = 2015; month = 12; day = 21; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); + file_type{file_idx} = 'NMEA'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); + gps_source{file_idx} = 'nmea-field'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','.gps'); + sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','format',3); + +end + +% ====================================================================== +% Read and translate files according to user settings +% ====================================================================== +gps_create; + +%% Lab Measurement Data: Fakes GPS position information +% match_idx = strmatch('gps_20150413.mat',out_fns,'exact'); +% if ~isempty(match_idx) +% gps_fn = fullfile(gps_path,out_fns{match_idx}); +% fprintf('Creating fake gps data for %s\n', gps_fn); +% gps = load(gps_fn); +% gps.lon = -45 * ones(size(gps.lon)); +% gps.lat = 70 + (1:length(gps.gps_time)) * 1e-4; +% save(gps_fn,'-append','-struct','gps','lat','lon'); +% end + +%% Conversion from GPS to UTC time since system was not allowed enough +% time to warm up before operation. +match_idx = strmatch('gps_20151202.mat',out_fns,'exact'); +if ~isempty(match_idx) + gps_fn = fullfile(gps_path,out_fns{match_idx}); + fprintf('Fixing GPS to UTC time conversion for %s\n', gps_fn); + gps = load(gps_fn); + gps.gps_time(1:340) = gps.gps_time(1:340) - utc_leap_seconds(gps.gps_time(1)); + gps.gps_time(5956:6036) = gps.gps_time(5956:6036) - utc_leap_seconds(gps.gps_time(1)); + gps.sync_gps_time(1:340) = gps.sync_gps_time(1:340) - utc_leap_seconds(gps.sync_gps_time(1)); + gps.sync_gps_time(5956:6036) = gps.sync_gps_time(5956:6036) - utc_leap_seconds(gps.gps_time(1)); + save(gps_fn,'-append','-struct','gps','gps_time','sync_gps_time'); +end +match_idx = strmatch('gps_20151221.mat',out_fns,'exact'); +if ~isempty(match_idx) + gps_fn = fullfile(gps_path,out_fns{match_idx}); + fprintf('Fixing GPS to UTC time conversion for %s\n', gps_fn); + gps = load(gps_fn); + gps.gps_time(1:107) = gps.gps_time(1:107) - utc_leap_seconds(gps.gps_time(1)); + gps.sync_gps_time(1:107) = gps.sync_gps_time(1:107) - utc_leap_seconds(gps.sync_gps_time(1)); + save(gps_fn,'-append','-struct','gps','gps_time','sync_gps_time'); +end + +%% Filter NMEA data +for idx = 1:length(out_fns) + gps_fn = fullfile(gps_path,out_fns{idx}); + fprintf('Filtering %s\n', gps_fn); + gps = load(gps_fn); + spacing = 10; + [along_track,gps.lat,gps.lon,gps.elev] = geodetic_to_along_track(gps.lat,gps.lon,gps.elev,spacing); + cur_along_track = along_track(1); cur_idx = 1; + for idx = 2:length(along_track) + if along_track(idx) <= cur_along_track + gps.lat(idx) = gps.lat(cur_idx); + gps.lon(idx) = gps.lon(cur_idx); + gps.elev(idx) = gps.elev(cur_idx); + else + cur_along_track = along_track(idx); + cur_idx = idx; + end + end + save(gps_fn,'-append','-struct','gps','lat','lon','elev'); +end diff --git a/cresis-toolbox/gps/missions/make_gps_2015_greenland_C130.m b/cresis-toolbox/gps/missions/gps_create_2015_Greenland_C130.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2015_greenland_C130.m rename to cresis-toolbox/gps/missions/gps_create_2015_Greenland_C130.m index ce896699..e50c50b9 100644 --- a/cresis-toolbox/gps/missions/make_gps_2015_greenland_C130.m +++ b/cresis-toolbox/gps/missions/gps_create_2015_Greenland_C130.m @@ -1,4 +1,4 @@ -% script make_gps_2015_greenland_C130 +% script gps_create_2015_greenland_C130 % % Makes the GPS files for 2015 Greenland C130 field season @@ -417,7 +417,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; diff --git a/cresis-toolbox/gps/missions/make_gps_2015_greenland_Ground.m b/cresis-toolbox/gps/missions/gps_create_2015_Greenland_Ground.m similarity index 98% rename from cresis-toolbox/gps/missions/make_gps_2015_greenland_Ground.m rename to cresis-toolbox/gps/missions/gps_create_2015_Greenland_Ground.m index cc7d54a5..37bd6b10 100644 --- a/cresis-toolbox/gps/missions/make_gps_2015_greenland_Ground.m +++ b/cresis-toolbox/gps/missions/gps_create_2015_Greenland_Ground.m @@ -1,4 +1,4 @@ -% script make_gps_2015_greenland_Ground +% script gps_create_2015_greenland_Ground % % Makes the GPS files for 2015 Greenland Ground field season @@ -87,7 +87,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; %% Lab Measurement Data: Fakes GPS position information match_idx = strmatch('gps_20150413.mat',out_fns,'exact'); diff --git a/cresis-toolbox/gps/missions/make_gps_2015_greenland_Polar6.m b/cresis-toolbox/gps/missions/gps_create_2015_Greenland_Polar6.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2015_greenland_Polar6.m rename to cresis-toolbox/gps/missions/gps_create_2015_Greenland_Polar6.m index 9e2e6c47..d28113cf 100644 --- a/cresis-toolbox/gps/missions/make_gps_2015_greenland_Polar6.m +++ b/cresis-toolbox/gps/missions/gps_create_2015_Greenland_Polar6.m @@ -1,4 +1,4 @@ -% script make_gps_2015_greenland_Polar6 +% script gps_create_2015_greenland_Polar6 % % Makes the GPS files for 2015 Greenland Polar6 field season @@ -308,7 +308,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; %% Lab Measurement Data: Fakes GPS position information match_idx = strmatch('gps_20150730.mat',out_fns,'exact'); diff --git a/cresis-toolbox/gps/missions/make_gps_2016_antarctica_DC8.m b/cresis-toolbox/gps/missions/gps_create_2016_Antarctica_DC8.m similarity index 97% rename from cresis-toolbox/gps/missions/make_gps_2016_antarctica_DC8.m rename to cresis-toolbox/gps/missions/gps_create_2016_Antarctica_DC8.m index 9f6ee83a..fc425da9 100644 --- a/cresis-toolbox/gps/missions/make_gps_2016_antarctica_DC8.m +++ b/cresis-toolbox/gps/missions/gps_create_2016_Antarctica_DC8.m @@ -1,433 +1,433 @@ -% script make_gps_2016_antarctica_DC8 -% -% Makes the GPS files for 2016 Antarctica DC8 field season - -tic; - -global gRadar; - -support_path = ''; -data_support_path = ''; - -if isempty(support_path) - support_path = gRadar.support_path; -end - -gps_path = fullfile(support_path,'gps','2016_Antarctica_DC8'); -if ~exist(gps_path,'dir') - fprintf('Making directory %s\n', gps_path); - fprintf(' Press a key to proceed\n'); - pause; - mkdir(gps_path); -end - -if isempty(data_support_path) - data_support_path = gRadar.data_support_path; -end - -% ====================================================================== -% User Settings -% ====================================================================== -debug_level = 1; - -in_base_path = fullfile(data_support_path,'2016_Antarctica_DC8'); - -file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; -sync_fns = {}; sync_params = {}; - -% gps_source_to_use = 'NMEA'; -% gps_source_to_use = 'ATM-field'; -gps_source_to_use = 'ATM'; - -if strcmpi(gps_source_to_use,'NMEA') - - % year = 2016; month = 10; day = 12; - % file_idx = file_idx + 1; - % in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); - % out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - % file_type{file_idx} = 'NMEA'; - % params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); - % gps_source{file_idx} = 'nmea-field'; - % sync_flag{file_idx} = 0; - % - % year = 2016; month = 10; day = 14; - % file_idx = file_idx + 1; - % in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); - % out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - % file_type{file_idx} = 'NMEA'; - % params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); - % gps_source{file_idx} = 'nmea-field'; - % sync_flag{file_idx} = 0; - % - % year = 2016; month = 10; day = 15; - % file_idx = file_idx + 1; - % in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); - % out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - % file_type{file_idx} = 'NMEA'; - % params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); - % gps_source{file_idx} = 'nmea-field'; - % sync_flag{file_idx} = 0; - % -% year = 2016; month = 10; day = 17; -% file_idx = file_idx + 1; -% in_fns{file_idx} = fullfile(in_base_path,'IWG1.OCT16_Science3'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'reveal'; -% params{file_idx} = struct('time_reference','utc'); -% gps_source{file_idx} = 'reveal-field'; -% sync_flag{file_idx} = 0; - -% year = 2016; month = 10; day = 20; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'NMEA'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); -% gps_source{file_idx} = 'nmea-field'; -% sync_flag{file_idx} = 0; - -% year = 2016; month = 10; day = 22; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'NMEA'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); -% gps_source{file_idx} = 'nmea-field'; -% sync_flag{file_idx} = 0; - - year = 2016; month = 11; day = 15; - file_idx = file_idx + 1; - in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - file_type{file_idx} = 'NMEA'; - params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); - gps_source{file_idx} = 'nmea-field'; - sync_flag{file_idx} = 0; - - year = 2016; month = 11; day = 17; - file_idx = file_idx + 1; - in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - file_type{file_idx} = 'NMEA'; - params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); - gps_source{file_idx} = 'nmea-field'; - sync_flag{file_idx} = 0; - - year = 2016; month = 11; day = 18; - file_idx = file_idx + 1; - in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - file_type{file_idx} = 'NMEA'; - params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); - gps_source{file_idx} = 'nmea-field'; - sync_flag{file_idx} = 0; -elseif strcmpi(gps_source_to_use,'ATM-field') - - year = 2016; month = 10; day = 4; - file_idx = file_idx + 1; - in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - file_type{file_idx} = 'applanix'; - params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); - gps_source{file_idx} = 'atm-field'; - sync_flag{file_idx} = 0; - -elseif strcmpi(gps_source_to_use,'ATM') - % Just some simple code to automate creation of the code in this section: - % - % ATM_fns = get_filenames(in_base_path,'','','.out'); - % fn_dates = []; - % for idx = 1:length(ATM_fns) - % fn = ATM_fns{idx}; - % [~,fn_name] = fileparts(fn); - % if strcmpi(fn_name(1:2),'BD') - % fn_dates(end+1) = datenum(sprintf('%s %s, 20%s', fn_name(9:11), fn_name(7:8), fn_name(12:13))); - % elseif strcmpi(fn_name(1:2),'00') - % fn_dates(end+1) = datenum(sprintf('%s %s, 20%s', fn_name(13:15), fn_name(11:12), fn_name(16:17))); - % end - % end - % fn_dates = sort(fn_dates); - % for idx = 1:length(fn_dates) - % [year,month,day] = datevec(fn_dates(idx)); - % fprintf('year = %d; month = %d; day = %d;\n', year, month, day); - % end - % !!! ALL data, including test flights, from Oct 7th to OCT29th are in - % GPS time. From Nov2nd to Nov 22nd the data is in UTC time. - -% year = 2016; month = 10; day = 14; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); -% gps_source{file_idx} = 'atm-final_20170717'; -% sync_flag{file_idx} = 0; - -% year = 2016; month = 10; day = 15; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); -% gps_source{file_idx} = 'atm-final_201710717'; -% sync_flag{file_idx} = 0; -% -% year = 2016; month = 10; day = 17; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); -% gps_source{file_idx} = 'atm-final_20171717'; -% sync_flag{file_idx} = 0; -% -% year = 2016; month = 10; day = 20; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','gps'); -% gps_source{file_idx} = 'atm-final_201710717'; -% sync_flag{file_idx} = 0; -% -% year = 2016; month = 10; day = 22; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','gps'); -% gps_source{file_idx} = 'atm-final_201710717'; -% sync_flag{file_idx} = 0; -% -% year = 2016; month = 10; day = 24; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); -% gps_source{file_idx} = 'atm-final_201710717'; -% sync_flag{file_idx} = 0; -% -% year = 2016; month = 10; day = 25; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); -% gps_source{file_idx} = 'atm-final_201710717'; -% sync_flag{file_idx} = 0; -% -% year = 2016; month = 10; day = 26; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); -% gps_source{file_idx} = 'atm-final_201710717'; -% sync_flag{file_idx} = 0; -% -% year = 2016; month = 10; day = 27; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); -% gps_source{file_idx} = 'atm-final_201710717'; -% sync_flag{file_idx} = 0; -% -% year = 2016; month = 10; day = 28; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); -% gps_source{file_idx} = 'atm-final_201710717'; -% sync_flag{file_idx} = 0; - -% year = 2016; month = 10; day = 31; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); -% gps_source{file_idx} = 'atm-final_201710717'; -% sync_flag{file_idx} = 0; -% -% year = 2016; month = 11; day = 02; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); -% gps_source{file_idx} = 'atm-final_201710717'; -% sync_flag{file_idx} = 0; -% -% year = 2016; month = 11; day = 03; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); -% gps_source{file_idx} = 'atm-final_201710222'; -% sync_flag{file_idx} = 0; - -% year = 2016; month = 11; day = 04; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); -% gps_source{file_idx} = 'atm-final_201710222'; -% sync_flag{file_idx} = 0; -% -% year = 2016; month = 11; day = 05; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); -% gps_source{file_idx} = 'atm-final_201710222'; -% sync_flag{file_idx} = 0; -% -% year = 2016; month = 11; day = 07; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); -% gps_source{file_idx} = 'atm-final_201710222'; -% sync_flag{file_idx} = 0; -% -% year = 2016; month = 11; day = 09; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); -% gps_source{file_idx} = 'atm-final_201710222'; -% sync_flag{file_idx} = 0; -% -% year = 2016; month = 11; day = 10; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); -% gps_source{file_idx} = 'atm-final_201710222'; -% sync_flag{file_idx} = 0; -% -% year = 2016; month = 11; day = 11; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); -% gps_source{file_idx} = 'atm-final_201710222'; -% sync_flag{file_idx} = 0; -% -% year = 2016; month = 11; day = 12; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); -% gps_source{file_idx} = 'atm-final_201710222'; -% sync_flag{file_idx} = 0; -% -% year = 2016; month = 11; day = 14; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); -% gps_source{file_idx} = 'atm-final_20170222'; -% sync_flag{file_idx} = 0; - -% year = 2016; month = 11; day = 15; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); -% gps_source{file_idx} = 'atm-final_20170222'; -% sync_flag{file_idx} = 0; -% -% year = 2016; month = 11; day = 17; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); -% gps_source{file_idx} = 'atm-final_20170222'; -% sync_flag{file_idx} = 0; -% - year = 2016; month = 11; day = 18; - file_idx = file_idx + 1; - in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - file_type{file_idx} = 'applanix'; - params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); - gps_source{file_idx} = 'atm-final_20170222'; - sync_flag{file_idx} = 0; - -end - - -% ====================================================================== -% Read and translate files according to user settings -% ====================================================================== -gps_make; - -% Debug code that can be used when no GPS data is available and we need to -% fake it. -hack_idx = cell2mat(strfind(out_fns,'gps_20140904.mat')); -if ~isempty(hack_idx) - out_fn = fullfile(gps_path,out_fns{hack_idx}); - - warning('Creating fake trajectory with lab data: %s', out_fn); - - gps = load(out_fn); - gps.lat(:) = gps.lat(1) + (gps.gps_time-gps.gps_time(1))*125/111e3; - gps.lon(:) = gps.lon(1); - save(out_fn,'-append','-struct','gps','lat','lon') -end - -% Reveal files are known to have GPS time errors which are corrected here. -for idx = 1:length(file_type) - out_fn = fullfile(gps_path,out_fns{idx}); - - gps = load(out_fn); - if regexp(gps.gps_source,'reveal') - - warning('Fixing non-monotonic GPS data in reveal file: %s', out_fn); - - [gps,error_flag] = make_gps_monotonic(gps); - - if error_flag - save(out_fn,'-append','-struct','gps'); - end - end -end - -% ATM and DMS files are known to have a small high frequency INS error which is -% corrected here. -for idx = 1:length(file_type) - out_fn = fullfile(gps_path,out_fns{idx}); - - gps = load(out_fn); - if regexpi(gps.gps_source,'(atm|dms)') - - warning('Smoothing INS data: %s', out_fn); - - gps.roll = sgolayfilt(gps.roll,2,101); - gps.pitch = sgolayfilt(gps.pitch,2,101); - heading_x = cos(gps.heading); - heading_y = sin(gps.heading); - heading_x = sgolayfilt(heading_x,2,101); % Adjust filter length as needed to remove high frequency noise - heading_y = sgolayfilt(heading_y,2,101); % Adjust filter length as needed to remove high frequency noise - gps.heading = atan2(heading_y,heading_x); - - save(out_fn,'-append','-struct','gps','roll','pitch','heading'); - end -end - +% script gps_create_2016_antarctica_DC8 +% +% Makes the GPS files for 2016 Antarctica DC8 field season + +tic; + +global gRadar; + +support_path = ''; +data_support_path = ''; + +if isempty(support_path) + support_path = gRadar.support_path; +end + +gps_path = fullfile(support_path,'gps','2016_Antarctica_DC8'); +if ~exist(gps_path,'dir') + fprintf('Making directory %s\n', gps_path); + fprintf(' Press a key to proceed\n'); + pause; + mkdir(gps_path); +end + +if isempty(data_support_path) + data_support_path = gRadar.data_support_path; +end + +% ====================================================================== +% User Settings +% ====================================================================== +debug_level = 1; + +in_base_path = fullfile(data_support_path,'2016_Antarctica_DC8'); + +file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; +sync_fns = {}; sync_params = {}; + +% gps_source_to_use = 'NMEA'; +% gps_source_to_use = 'ATM-field'; +gps_source_to_use = 'ATM'; + +if strcmpi(gps_source_to_use,'NMEA') + + % year = 2016; month = 10; day = 12; + % file_idx = file_idx + 1; + % in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); + % out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + % file_type{file_idx} = 'NMEA'; + % params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); + % gps_source{file_idx} = 'nmea-field'; + % sync_flag{file_idx} = 0; + % + % year = 2016; month = 10; day = 14; + % file_idx = file_idx + 1; + % in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); + % out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + % file_type{file_idx} = 'NMEA'; + % params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); + % gps_source{file_idx} = 'nmea-field'; + % sync_flag{file_idx} = 0; + % + % year = 2016; month = 10; day = 15; + % file_idx = file_idx + 1; + % in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); + % out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + % file_type{file_idx} = 'NMEA'; + % params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); + % gps_source{file_idx} = 'nmea-field'; + % sync_flag{file_idx} = 0; + % +% year = 2016; month = 10; day = 17; +% file_idx = file_idx + 1; +% in_fns{file_idx} = fullfile(in_base_path,'IWG1.OCT16_Science3'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'reveal'; +% params{file_idx} = struct('time_reference','utc'); +% gps_source{file_idx} = 'reveal-field'; +% sync_flag{file_idx} = 0; + +% year = 2016; month = 10; day = 20; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; + +% year = 2016; month = 10; day = 22; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; + + year = 2016; month = 11; day = 15; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'NMEA'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); + gps_source{file_idx} = 'nmea-field'; + sync_flag{file_idx} = 0; + + year = 2016; month = 11; day = 17; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'NMEA'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); + gps_source{file_idx} = 'nmea-field'; + sync_flag{file_idx} = 0; + + year = 2016; month = 11; day = 18; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'NMEA'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); + gps_source{file_idx} = 'nmea-field'; + sync_flag{file_idx} = 0; +elseif strcmpi(gps_source_to_use,'ATM-field') + + year = 2016; month = 10; day = 4; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'applanix'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); + gps_source{file_idx} = 'atm-field'; + sync_flag{file_idx} = 0; + +elseif strcmpi(gps_source_to_use,'ATM') + % Just some simple code to automate creation of the code in this section: + % + % ATM_fns = get_filenames(in_base_path,'','','.out'); + % fn_dates = []; + % for idx = 1:length(ATM_fns) + % fn = ATM_fns{idx}; + % [~,fn_name] = fileparts(fn); + % if strcmpi(fn_name(1:2),'BD') + % fn_dates(end+1) = datenum(sprintf('%s %s, 20%s', fn_name(9:11), fn_name(7:8), fn_name(12:13))); + % elseif strcmpi(fn_name(1:2),'00') + % fn_dates(end+1) = datenum(sprintf('%s %s, 20%s', fn_name(13:15), fn_name(11:12), fn_name(16:17))); + % end + % end + % fn_dates = sort(fn_dates); + % for idx = 1:length(fn_dates) + % [year,month,day] = datevec(fn_dates(idx)); + % fprintf('year = %d; month = %d; day = %d;\n', year, month, day); + % end + % !!! ALL data, including test flights, from Oct 7th to OCT29th are in + % GPS time. From Nov2nd to Nov 22nd the data is in UTC time. + +% year = 2016; month = 10; day = 14; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +% gps_source{file_idx} = 'atm-final_20170717'; +% sync_flag{file_idx} = 0; + +% year = 2016; month = 10; day = 15; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +% gps_source{file_idx} = 'atm-final_201710717'; +% sync_flag{file_idx} = 0; +% +% year = 2016; month = 10; day = 17; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +% gps_source{file_idx} = 'atm-final_20171717'; +% sync_flag{file_idx} = 0; +% +% year = 2016; month = 10; day = 20; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','gps'); +% gps_source{file_idx} = 'atm-final_201710717'; +% sync_flag{file_idx} = 0; +% +% year = 2016; month = 10; day = 22; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','gps'); +% gps_source{file_idx} = 'atm-final_201710717'; +% sync_flag{file_idx} = 0; +% +% year = 2016; month = 10; day = 24; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +% gps_source{file_idx} = 'atm-final_201710717'; +% sync_flag{file_idx} = 0; +% +% year = 2016; month = 10; day = 25; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +% gps_source{file_idx} = 'atm-final_201710717'; +% sync_flag{file_idx} = 0; +% +% year = 2016; month = 10; day = 26; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +% gps_source{file_idx} = 'atm-final_201710717'; +% sync_flag{file_idx} = 0; +% +% year = 2016; month = 10; day = 27; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +% gps_source{file_idx} = 'atm-final_201710717'; +% sync_flag{file_idx} = 0; +% +% year = 2016; month = 10; day = 28; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +% gps_source{file_idx} = 'atm-final_201710717'; +% sync_flag{file_idx} = 0; + +% year = 2016; month = 10; day = 31; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +% gps_source{file_idx} = 'atm-final_201710717'; +% sync_flag{file_idx} = 0; +% +% year = 2016; month = 11; day = 02; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +% gps_source{file_idx} = 'atm-final_201710717'; +% sync_flag{file_idx} = 0; +% +% year = 2016; month = 11; day = 03; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +% gps_source{file_idx} = 'atm-final_201710222'; +% sync_flag{file_idx} = 0; + +% year = 2016; month = 11; day = 04; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +% gps_source{file_idx} = 'atm-final_201710222'; +% sync_flag{file_idx} = 0; +% +% year = 2016; month = 11; day = 05; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +% gps_source{file_idx} = 'atm-final_201710222'; +% sync_flag{file_idx} = 0; +% +% year = 2016; month = 11; day = 07; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +% gps_source{file_idx} = 'atm-final_201710222'; +% sync_flag{file_idx} = 0; +% +% year = 2016; month = 11; day = 09; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +% gps_source{file_idx} = 'atm-final_201710222'; +% sync_flag{file_idx} = 0; +% +% year = 2016; month = 11; day = 10; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +% gps_source{file_idx} = 'atm-final_201710222'; +% sync_flag{file_idx} = 0; +% +% year = 2016; month = 11; day = 11; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +% gps_source{file_idx} = 'atm-final_201710222'; +% sync_flag{file_idx} = 0; +% +% year = 2016; month = 11; day = 12; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +% gps_source{file_idx} = 'atm-final_201710222'; +% sync_flag{file_idx} = 0; +% +% year = 2016; month = 11; day = 14; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +% gps_source{file_idx} = 'atm-final_20170222'; +% sync_flag{file_idx} = 0; + +% year = 2016; month = 11; day = 15; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +% gps_source{file_idx} = 'atm-final_20170222'; +% sync_flag{file_idx} = 0; +% +% year = 2016; month = 11; day = 17; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +% gps_source{file_idx} = 'atm-final_20170222'; +% sync_flag{file_idx} = 0; +% + year = 2016; month = 11; day = 18; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'applanix'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); + gps_source{file_idx} = 'atm-final_20170222'; + sync_flag{file_idx} = 0; + +end + + +% ====================================================================== +% Read and translate files according to user settings +% ====================================================================== +gps_create; + +% Debug code that can be used when no GPS data is available and we need to +% fake it. +hack_idx = cell2mat(strfind(out_fns,'gps_20140904.mat')); +if ~isempty(hack_idx) + out_fn = fullfile(gps_path,out_fns{hack_idx}); + + warning('Creating fake trajectory with lab data: %s', out_fn); + + gps = load(out_fn); + gps.lat(:) = gps.lat(1) + (gps.gps_time-gps.gps_time(1))*125/111e3; + gps.lon(:) = gps.lon(1); + save(out_fn,'-append','-struct','gps','lat','lon') +end + +% Reveal files are known to have GPS time errors which are corrected here. +for idx = 1:length(file_type) + out_fn = fullfile(gps_path,out_fns{idx}); + + gps = load(out_fn); + if regexp(gps.gps_source,'reveal') + + warning('Fixing non-monotonic GPS data in reveal file: %s', out_fn); + + [gps,error_flag] = gps_force_monotonic(gps); + + if error_flag + save(out_fn,'-append','-struct','gps'); + end + end +end + +% ATM and DMS files are known to have a small high frequency INS error which is +% corrected here. +for idx = 1:length(file_type) + out_fn = fullfile(gps_path,out_fns{idx}); + + gps = load(out_fn); + if regexpi(gps.gps_source,'(atm|dms)') + + warning('Smoothing INS data: %s', out_fn); + + gps.roll = sgolayfilt(gps.roll,2,101); + gps.pitch = sgolayfilt(gps.pitch,2,101); + heading_x = cos(gps.heading); + heading_y = sin(gps.heading); + heading_x = sgolayfilt(heading_x,2,101); % Adjust filter length as needed to remove high frequency noise + heading_y = sgolayfilt(heading_y,2,101); % Adjust filter length as needed to remove high frequency noise + gps.heading = atan2(heading_y,heading_x); + + save(out_fn,'-append','-struct','gps','roll','pitch','heading'); + end +end + diff --git a/cresis-toolbox/gps/missions/make_gps_2016_greenland_G1XB.m b/cresis-toolbox/gps/missions/gps_create_2016_Greenland_G1XB.m similarity index 98% rename from cresis-toolbox/gps/missions/make_gps_2016_greenland_G1XB.m rename to cresis-toolbox/gps/missions/gps_create_2016_Greenland_G1XB.m index 4a589422..f3045f29 100644 --- a/cresis-toolbox/gps/missions/make_gps_2016_greenland_G1XB.m +++ b/cresis-toolbox/gps/missions/gps_create_2016_Greenland_G1XB.m @@ -1,4 +1,4 @@ -% script make_gps_2016_greenland_G1XB +% script gps_create_2016_greenland_G1XB % % Makes the GPS files for 2016 Greenland G1XB field season @@ -83,4 +83,4 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; diff --git a/cresis-toolbox/gps/missions/make_gps_2016_greenland_P3.m b/cresis-toolbox/gps/missions/gps_create_2016_Greenland_P3.m similarity index 98% rename from cresis-toolbox/gps/missions/make_gps_2016_greenland_P3.m rename to cresis-toolbox/gps/missions/gps_create_2016_Greenland_P3.m index 0dad140c..6d1aff84 100644 --- a/cresis-toolbox/gps/missions/make_gps_2016_greenland_P3.m +++ b/cresis-toolbox/gps/missions/gps_create_2016_Greenland_P3.m @@ -1,4 +1,4 @@ -% script make_gps_2016_greenland_P3 +% script gps_create_2016_greenland_P3 % % Makes the GPS files for 2016 Greenland P3 field season @@ -84,7 +84,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; diff --git a/cresis-toolbox/gps/missions/make_gps_2016_greenland_Polar6.m b/cresis-toolbox/gps/missions/gps_create_2016_Greenland_Polar6.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2016_greenland_Polar6.m rename to cresis-toolbox/gps/missions/gps_create_2016_Greenland_Polar6.m index db1e3693..294f98c1 100644 --- a/cresis-toolbox/gps/missions/make_gps_2016_greenland_Polar6.m +++ b/cresis-toolbox/gps/missions/gps_create_2016_Greenland_Polar6.m @@ -1,4 +1,4 @@ -% script make_gps_2016_greenland_Polar6 +% script gps_create_2016_greenland_Polar6 % % Makes the GPS files for 2016 Greenland Polar6 field season @@ -369,7 +369,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; %% No GPS Data Available: Fakes GPS position information match_idx = strmatch('gps_20160331.mat',out_fns,'exact'); diff --git a/cresis-toolbox/gps/missions/make_gps_2016_greenland_TOdtu.m b/cresis-toolbox/gps/missions/gps_create_2016_Greenland_TOdtu.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2016_greenland_TOdtu.m rename to cresis-toolbox/gps/missions/gps_create_2016_Greenland_TOdtu.m index 84171dc1..2af0dfdd 100644 --- a/cresis-toolbox/gps/missions/make_gps_2016_greenland_TOdtu.m +++ b/cresis-toolbox/gps/missions/gps_create_2016_Greenland_TOdtu.m @@ -1,4 +1,4 @@ -% script make_gps_2016_greenland_TOdtu +% script gps_create_2016_greenland_TOdtu % % Makes the GPS files for 2016 Greenland TOdtu field season @@ -245,5 +245,5 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; diff --git a/cresis-toolbox/gps/missions/make_gps_2017_antarctica_Basler.m b/cresis-toolbox/gps/missions/gps_create_2017_Antarctica_Basler.m similarity index 98% rename from cresis-toolbox/gps/missions/make_gps_2017_antarctica_Basler.m rename to cresis-toolbox/gps/missions/gps_create_2017_Antarctica_Basler.m index b836c012..88df7724 100644 --- a/cresis-toolbox/gps/missions/make_gps_2017_antarctica_Basler.m +++ b/cresis-toolbox/gps/missions/gps_create_2017_Antarctica_Basler.m @@ -1,4 +1,4 @@ -% script make_gps_2017_antarctica_Basler +% script gps_create_2017_antarctica_Basler % % Makes the GPS files for 2017 Antarctica Basler field season @@ -177,11 +177,11 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; if any(strcmpi('gps_?.mat',out_fns)) ... || any(strcmpi('gps_?.mat',out_fns)) - warning('Jan ? and Dec ? traveled across 180 deg line and GPS file produced by Novatel may have errors! Novatel smoothing will produce points which interpolate -179.999 to +179.999 and vice versa incorrectly and you have to manually remove these points from the .txt input file before running make_gps.'); + warning('Jan ? and Dec ? traveled across 180 deg line and GPS file produced by Novatel may have errors! Novatel smoothing will produce points which interpolate -179.999 to +179.999 and vice versa incorrectly and you have to manually remove these points from the .txt input file before running gps_create.'); keyboard end diff --git a/cresis-toolbox/gps/missions/make_gps_2017_antarctica_DC8.m b/cresis-toolbox/gps/missions/gps_create_2017_Antarctica_DC8.m similarity index 95% rename from cresis-toolbox/gps/missions/make_gps_2017_antarctica_DC8.m rename to cresis-toolbox/gps/missions/gps_create_2017_Antarctica_DC8.m index b818c3a3..171263f9 100644 --- a/cresis-toolbox/gps/missions/make_gps_2017_antarctica_DC8.m +++ b/cresis-toolbox/gps/missions/gps_create_2017_Antarctica_DC8.m @@ -1,149 +1,149 @@ -% script make_gps_2017_antarctica_DC8 -% -% Makes the GPS files for 2017 Antarctica DC8 field season - -tic; - -global gRadar; - -support_path = ''; -data_support_path = ''; - -if isempty(support_path) - support_path = gRadar.support_path; -end - -gps_path = fullfile(support_path,'gps','2017_Antarctica_DC8'); -if ~exist(gps_path,'dir') - fprintf('Making directory %s\n', gps_path); - fprintf(' Press a key to proceed\n'); - pause; - mkdir(gps_path); -end - -if isempty(data_support_path) - data_support_path = gRadar.data_support_path; -end - -% ====================================================================== -% User Settings -% ====================================================================== -debug_level = 1; - -in_base_path = fullfile(data_support_path,'2017_Antarctica_DC8'); - -file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; -sync_fns = {}; sync_params = {}; - -gps_source_to_use = 'NMEA'; -% gps_source_to_use = 'ATM-field'; -% gps_source_to_use = 'ATM'; - -if strcmpi(gps_source_to_use,'NMEA') - - year = 2017; month = 10; day = 12; - file_idx = file_idx + 1; - in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - file_type{file_idx} = 'NMEA'; - params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); - gps_source{file_idx} = 'nmea-field'; - sync_flag{file_idx} = 0; - -elseif strcmpi(gps_source_to_use,'ATM-field') - - year = 2017; month = 10; day = 4; - file_idx = file_idx + 1; - in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - file_type{file_idx} = 'applanix'; - params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); - gps_source{file_idx} = 'atm-field'; - sync_flag{file_idx} = 0; - -elseif strcmpi(gps_source_to_use,'ATM') - % Just some simple code to automate creation of the code in this section: - % - % ATM_fns = get_filenames(in_base_path,'','','.out'); - % fn_dates = []; - % for idx = 1:length(ATM_fns) - % fn = ATM_fns{idx}; - % [~,fn_name] = fileparts(fn); - % if strcmpi(fn_name(1:2),'BD') - % fn_dates(end+1) = datenum(sprintf('%s %s, 20%s', fn_name(9:11), fn_name(7:8), fn_name(12:13))); - % elseif strcmpi(fn_name(1:2),'00') - % fn_dates(end+1) = datenum(sprintf('%s %s, 20%s', fn_name(13:15), fn_name(11:12), fn_name(16:17))); - % end - % end - % fn_dates = sort(fn_dates); - % for idx = 1:length(fn_dates) - % [year,month,day] = datevec(fn_dates(idx)); - % fprintf('year = %d; month = %d; day = %d;\n', year, month, day); - % end - - year = 2017; month = 10; day = 14; - file_idx = file_idx + 1; - in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - file_type{file_idx} = 'applanix'; - params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); - gps_source{file_idx} = 'atm-final_20170717'; - sync_flag{file_idx} = 0; - -end - - -% ====================================================================== -% Read and translate files according to user settings -% ====================================================================== -gps_make; - -% Debug code that can be used when no GPS data is available and we need to -% fake it. -hack_idx = cell2mat(strfind(out_fns,'gps_?.mat')); -if ~isempty(hack_idx) - out_fn = fullfile(gps_path,out_fns{hack_idx}); - - warning('Creating fake trajectory with lab data: %s', out_fn); - - gps = load(out_fn); - gps.lat(:) = gps.lat(1) + (gps.gps_time-gps.gps_time(1))*125/111e3; - gps.lon(:) = gps.lon(1); - save(out_fn,'-append','-struct','gps','lat','lon') -end - -% Reveal files are known to have GPS time errors which are corrected here. -for idx = 1:length(file_type) - out_fn = fullfile(gps_path,out_fns{idx}); - - gps = load(out_fn); - if regexp(gps.gps_source,'reveal') - - warning('Fixing non-monotonic GPS data in reveal file: %s', out_fn); - - [gps,error_flag] = make_gps_monotonic(gps); - - if error_flag - save(out_fn,'-append','-struct','gps'); - end - end -end - -% ATM files are known to have a small high frequency INS error which is -% corrected here. -for idx = 1:length(file_type) - out_fn = fullfile(gps_path,out_fns{idx}); - - gps = load(out_fn); - if regexpi(gps.gps_source,'atm') - - warning('Smoothing INS data: %s', out_fn); - - gps.roll = sgolayfilt(gps.roll,2,101); - gps.pitch = sgolayfilt(gps.pitch,2,101); - gps.heading = sgolayfilt(gps.heading,2,101); - - save(out_fn,'-append','-struct','gps','roll','pitch','heading'); - end -end - +% script gps_create_2017_antarctica_DC8 +% +% Makes the GPS files for 2017 Antarctica DC8 field season + +tic; + +global gRadar; + +support_path = ''; +data_support_path = ''; + +if isempty(support_path) + support_path = gRadar.support_path; +end + +gps_path = fullfile(support_path,'gps','2017_Antarctica_DC8'); +if ~exist(gps_path,'dir') + fprintf('Making directory %s\n', gps_path); + fprintf(' Press a key to proceed\n'); + pause; + mkdir(gps_path); +end + +if isempty(data_support_path) + data_support_path = gRadar.data_support_path; +end + +% ====================================================================== +% User Settings +% ====================================================================== +debug_level = 1; + +in_base_path = fullfile(data_support_path,'2017_Antarctica_DC8'); + +file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; +sync_fns = {}; sync_params = {}; + +gps_source_to_use = 'NMEA'; +% gps_source_to_use = 'ATM-field'; +% gps_source_to_use = 'ATM'; + +if strcmpi(gps_source_to_use,'NMEA') + + year = 2017; month = 10; day = 12; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'NMEA'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); + gps_source{file_idx} = 'nmea-field'; + sync_flag{file_idx} = 0; + +elseif strcmpi(gps_source_to_use,'ATM-field') + + year = 2017; month = 10; day = 4; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'applanix'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); + gps_source{file_idx} = 'atm-field'; + sync_flag{file_idx} = 0; + +elseif strcmpi(gps_source_to_use,'ATM') + % Just some simple code to automate creation of the code in this section: + % + % ATM_fns = get_filenames(in_base_path,'','','.out'); + % fn_dates = []; + % for idx = 1:length(ATM_fns) + % fn = ATM_fns{idx}; + % [~,fn_name] = fileparts(fn); + % if strcmpi(fn_name(1:2),'BD') + % fn_dates(end+1) = datenum(sprintf('%s %s, 20%s', fn_name(9:11), fn_name(7:8), fn_name(12:13))); + % elseif strcmpi(fn_name(1:2),'00') + % fn_dates(end+1) = datenum(sprintf('%s %s, 20%s', fn_name(13:15), fn_name(11:12), fn_name(16:17))); + % end + % end + % fn_dates = sort(fn_dates); + % for idx = 1:length(fn_dates) + % [year,month,day] = datevec(fn_dates(idx)); + % fprintf('year = %d; month = %d; day = %d;\n', year, month, day); + % end + + year = 2017; month = 10; day = 14; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'applanix'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); + gps_source{file_idx} = 'atm-final_20170717'; + sync_flag{file_idx} = 0; + +end + + +% ====================================================================== +% Read and translate files according to user settings +% ====================================================================== +gps_create; + +% Debug code that can be used when no GPS data is available and we need to +% fake it. +hack_idx = cell2mat(strfind(out_fns,'gps_?.mat')); +if ~isempty(hack_idx) + out_fn = fullfile(gps_path,out_fns{hack_idx}); + + warning('Creating fake trajectory with lab data: %s', out_fn); + + gps = load(out_fn); + gps.lat(:) = gps.lat(1) + (gps.gps_time-gps.gps_time(1))*125/111e3; + gps.lon(:) = gps.lon(1); + save(out_fn,'-append','-struct','gps','lat','lon') +end + +% Reveal files are known to have GPS time errors which are corrected here. +for idx = 1:length(file_type) + out_fn = fullfile(gps_path,out_fns{idx}); + + gps = load(out_fn); + if regexp(gps.gps_source,'reveal') + + warning('Fixing non-monotonic GPS data in reveal file: %s', out_fn); + + [gps,error_flag] = gps_force_monotonic(gps); + + if error_flag + save(out_fn,'-append','-struct','gps'); + end + end +end + +% ATM files are known to have a small high frequency INS error which is +% corrected here. +for idx = 1:length(file_type) + out_fn = fullfile(gps_path,out_fns{idx}); + + gps = load(out_fn); + if regexpi(gps.gps_source,'atm') + + warning('Smoothing INS data: %s', out_fn); + + gps.roll = sgolayfilt(gps.roll,2,101); + gps.pitch = sgolayfilt(gps.pitch,2,101); + gps.heading = sgolayfilt(gps.heading,2,101); + + save(out_fn,'-append','-struct','gps','roll','pitch','heading'); + end +end + diff --git a/cresis-toolbox/gps/missions/make_gps_2017_antarctica_P3.m b/cresis-toolbox/gps/missions/gps_create_2017_Antarctica_P3.m similarity index 98% rename from cresis-toolbox/gps/missions/make_gps_2017_antarctica_P3.m rename to cresis-toolbox/gps/missions/gps_create_2017_Antarctica_P3.m index 50b6edbe..2fbd47d6 100644 --- a/cresis-toolbox/gps/missions/make_gps_2017_antarctica_P3.m +++ b/cresis-toolbox/gps/missions/gps_create_2017_Antarctica_P3.m @@ -1,4 +1,4 @@ -% script make_gps_2017_antarctica_P3 +% script gps_create_2017_antarctica_P3 % % Makes the GPS files for 2017 Antarctica P3 field season @@ -187,7 +187,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; % Debug code that can be used when no GPS data is available and we need to % fake it. @@ -213,7 +213,7 @@ warning('Fixing non-monotonic GPS data in applanix file: %s', out_fn); - [gps,error_flag] = make_gps_monotonic(gps); + [gps,error_flag] = gps_force_monotonic(gps); if error_flag save(out_fn,'-append','-struct','gps'); diff --git a/cresis-toolbox/gps/missions/make_gps_2017_antarctica_Polar6.m b/cresis-toolbox/gps/missions/gps_create_2017_Antarctica_Polar6.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2017_antarctica_Polar6.m rename to cresis-toolbox/gps/missions/gps_create_2017_Antarctica_Polar6.m index 03a1e6bc..9b296272 100644 --- a/cresis-toolbox/gps/missions/make_gps_2017_antarctica_Polar6.m +++ b/cresis-toolbox/gps/missions/gps_create_2017_Antarctica_Polar6.m @@ -1,4 +1,4 @@ -% script make_gps_2017_antarctica_Polar6 +% script gps_create_2017_antarctica_Polar6 % % Makes the GPS files for 2017 Antarctica Polar6 field season @@ -179,7 +179,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; %% No GPS Data Available: Fakes GPS position information match_idx = strmatch('gps_20160331.mat',out_fns,'exact'); diff --git a/cresis-toolbox/gps/missions/make_gps_2017_antarctica_TObas.m b/cresis-toolbox/gps/missions/gps_create_2017_Antarctica_TObas.m similarity index 97% rename from cresis-toolbox/gps/missions/make_gps_2017_antarctica_TObas.m rename to cresis-toolbox/gps/missions/gps_create_2017_Antarctica_TObas.m index 2696e779..87f3850d 100644 --- a/cresis-toolbox/gps/missions/make_gps_2017_antarctica_TObas.m +++ b/cresis-toolbox/gps/missions/gps_create_2017_Antarctica_TObas.m @@ -1,4 +1,4 @@ -% script make_gps_2017_antarctica_TObas +% script gps_create_2017_antarctica_TObas % % Makes the GPS files for 2017 Antarctica TObas field season @@ -59,4 +59,4 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; diff --git a/cresis-toolbox/gps/missions/make_gps_2017_arctic_Polar5.m b/cresis-toolbox/gps/missions/gps_create_2017_Arctic_Polar5.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2017_arctic_Polar5.m rename to cresis-toolbox/gps/missions/gps_create_2017_Arctic_Polar5.m index ec54557c..ab7bb386 100644 --- a/cresis-toolbox/gps/missions/make_gps_2017_arctic_Polar5.m +++ b/cresis-toolbox/gps/missions/gps_create_2017_Arctic_Polar5.m @@ -1,4 +1,4 @@ -% script make_gps_2017_arctic_Polar5 +% script gps_create_2017_arctic_Polar5 % % Makes the GPS files for 2017_Arctic_Polar5 field season @@ -211,7 +211,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; %% No GPS Data Available: Fakes GPS position information match_idx = strmatch('gps_20160331.mat',out_fns,'exact'); diff --git a/cresis-toolbox/gps/missions/make_gps_2017_greenland_P3.m b/cresis-toolbox/gps/missions/gps_create_2017_Greenland_P3.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2017_greenland_P3.m rename to cresis-toolbox/gps/missions/gps_create_2017_Greenland_P3.m index 6dc87cb2..2335659b 100644 --- a/cresis-toolbox/gps/missions/make_gps_2017_greenland_P3.m +++ b/cresis-toolbox/gps/missions/gps_create_2017_Greenland_P3.m @@ -1,4 +1,4 @@ -% script make_gps_2017_greenland_P3 +% script gps_create_2017_greenland_P3 % % Makes the GPS files for 2017 Greenland P3 field season @@ -610,7 +610,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; % Debug code that sets up special processing hack_idx = cell2mat(strfind(out_fns,'gps_20170329.mat')); @@ -620,7 +620,7 @@ gps = load(out_fn); if strcmpi(gps.gps_source,'nmea-field') warning('Making monotonic gps time: %s', out_fn); - [gps,error_flag] = make_gps_monotonic(gps); + [gps,error_flag] = gps_force_monotonic(gps); save(out_fn,'-append','-struct','gps'); end end @@ -631,7 +631,7 @@ gps = load(out_fn); if strcmpi(gps.gps_source,'nmea-field') warning('Making monotonic gps time: %s', out_fn); - [gps,error_flag] = make_gps_monotonic(gps); + [gps,error_flag] = gps_force_monotonic(gps); save(out_fn,'-append','-struct','gps'); end end @@ -654,7 +654,7 @@ gps = load(out_fn); if strcmpi(gps.gps_source,'nmea-field') warning('Making monotonic gps time: %s', out_fn); - [gps,error_flag] = make_gps_monotonic(gps); + [gps,error_flag] = gps_force_monotonic(gps); save(out_fn,'-append','-struct','gps'); end end diff --git a/cresis-toolbox/gps/missions/make_gps_2018_alaska_SO.m b/cresis-toolbox/gps/missions/gps_create_2018_Alaska_SO.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2018_alaska_SO.m rename to cresis-toolbox/gps/missions/gps_create_2018_Alaska_SO.m index edcd9245..cab83d75 100644 --- a/cresis-toolbox/gps/missions/make_gps_2018_alaska_SO.m +++ b/cresis-toolbox/gps/missions/gps_create_2018_Alaska_SO.m @@ -1,4 +1,4 @@ -% script make_gps_2018_alaska_SO +% script gps_create_2018_alaska_SO % % Makes the GPS files for 2018 Alaska SO field season @@ -282,7 +282,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; % Debug code that sets up special processing @@ -292,7 +292,7 @@ gps = load(out_fn); if strcmpi(gps.gps_source,'ualidar-final') warning('Making monotonic gps time: %s', out_fn); - [gps,error_flag] = make_gps_monotonic(gps); + [gps,error_flag] = gps_force_monotonic(gps); save(out_fn,'-append','-struct','gps'); end end \ No newline at end of file diff --git a/cresis-toolbox/gps/missions/make_gps_2018_antarctica_DC8.m b/cresis-toolbox/gps/missions/gps_create_2018_Antarctica_DC8.m similarity index 97% rename from cresis-toolbox/gps/missions/make_gps_2018_antarctica_DC8.m rename to cresis-toolbox/gps/missions/gps_create_2018_Antarctica_DC8.m index 3722b945..509fb097 100644 --- a/cresis-toolbox/gps/missions/make_gps_2018_antarctica_DC8.m +++ b/cresis-toolbox/gps/missions/gps_create_2018_Antarctica_DC8.m @@ -1,593 +1,593 @@ -% script make_gps_2018_antarctica_DC8 -% -% Makes the GPS files for 2018 Antarctica DC8 field season - -tic; - -global gRadar; - -support_path = ''; -data_support_path = ''; - -if isempty(support_path) - support_path = gRadar.support_path; -end - -gps_path = fullfile(support_path,'gps','2018_Antarctica_DC8'); -if ~exist(gps_path,'dir') - fprintf('Making directory %s\n', gps_path); - fprintf(' Press a key to proceed\n'); - pause; - mkdir(gps_path); -end - -if isempty(data_support_path) - data_support_path = gRadar.data_support_path; -end - -% ====================================================================== -% User Settings -% ====================================================================== -debug_level = 1; -in_base_path = fullfile(data_support_path,'2018_Antarctica_DC8','Applanix_data'); -sync_path = fullfile(data_support_path,'2018_Antarctica_DC8','Trajectory'); - -file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; -sync_fns = {}; sync_params = {};sync_file_type = {}; - -% gps_source_to_use = 'NMEA'; -% gps_source_to_use = 'ATM-field'; -gps_source_to_use = 'ATM'; - -if strcmpi(gps_source_to_use,'NMEA') - - % year = 2018; month = 9; day = 27; - % file_idx = file_idx + 1; - % in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); - % out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - % file_type{file_idx} = 'NMEA'; - % params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); - % gps_source{file_idx} = 'nmea-field'; - % sync_flag{file_idx} = 0; - -% year = 2018; month = 10; day = 10; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'NMEA'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% gps_source{file_idx} = 'nmea-field'; -% sync_flag{file_idx} = 0; - -% year = 2018; month = 10; day = 11; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'NMEA'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% gps_source{file_idx} = 'nmea-field'; -% sync_flag{file_idx} = 0; - - % year = 2018; month = 10; day = 12; - % file_idx = file_idx + 1; - % in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); - % out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - % file_type{file_idx} = 'NMEA'; - % params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); - % gps_source{file_idx} = 'nmea-field'; - % sync_flag{file_idx} = 0; - - % year = 2018; month = 10; day = 13; - % file_idx = file_idx + 1; - % in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); - % out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - % file_type{file_idx} = 'NMEA'; - % params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); - % gps_source{file_idx} = 'nmea-field'; - % sync_flag{file_idx} = 0; - -% year = 2018; month = 10; day = 15; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'NMEA'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% gps_source{file_idx} = 'nmea-field'; -% sync_flag{file_idx} = 0; - - % year = 2018; month = 10; day = 16; - % file_idx = file_idx + 1; - % in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); - % out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - % file_type{file_idx} = 'NMEA'; - % params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); - % gps_source{file_idx} = 'nmea-field'; - % sync_flag{file_idx} = 0; - - % year = 2018; month = 10; day = 18; - % file_idx = file_idx + 1; - % in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); - % out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - % file_type{file_idx} = 'NMEA'; - % params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); - % gps_source{file_idx} = 'nmea-field'; - % sync_flag{file_idx} = 0; - - % year = 2018; month = 10; day = 19; - % file_idx = file_idx + 1; - % in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); - % out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - % file_type{file_idx} = 'NMEA'; - % params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); - % gps_source{file_idx} = 'nmea-field'; - % sync_flag{file_idx} = 0; - - % year = 2018; month = 10; day = 20; - % file_idx = file_idx + 1; - % in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); - % out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - % file_type{file_idx} = 'NMEA'; - % params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); - % gps_source{file_idx} = 'nmea-field'; - % sync_flag{file_idx} = 0; - - % year = 2018; month = 10; day = 22; - % file_idx = file_idx + 1; - % in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); - % out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - % file_type{file_idx} = 'NMEA'; - % params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); - % gps_source{file_idx} = 'nmea-field'; - % sync_flag{file_idx} = 0; - - % year = 2018; month = 10; day = 30; - % file_idx = file_idx + 1; - % in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); - % out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - % file_type{file_idx} = 'NMEA'; - % params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); - % gps_source{file_idx} = 'nmea-field'; - % sync_flag{file_idx} = 0; - - year = 2018; month = 10; day = 31; - file_idx = file_idx + 1; - in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - file_type{file_idx} = 'NMEA'; - params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); - gps_source{file_idx} = 'nmea-field'; - sync_flag{file_idx} = 0; - - - % year = 2018; month = 11; day = 3; - % file_idx = file_idx + 1; - % in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); - % out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - % file_type{file_idx} = 'NMEA'; - % params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); - % gps_source{file_idx} = 'nmea-field'; - % sync_flag{file_idx} = 0; - - year = 2018; month = 11; day = 4; - file_idx = file_idx + 1; - in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - file_type{file_idx} = 'NMEA'; - params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); - gps_source{file_idx} = 'nmea-field'; - sync_flag{file_idx} = 0; - -% year = 2018; month = 11; day = 5; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'NMEA'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% gps_source{file_idx} = 'nmea-field'; -% sync_flag{file_idx} = 0; - -% year = 2018; month = 11; day = 7; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'NMEA'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% gps_source{file_idx} = 'nmea-field'; -% sync_flag{file_idx} = 0; - -% year = 2018; month = 11; day = 9; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'NMEA'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% gps_source{file_idx} = 'nmea-field'; -% sync_flag{file_idx} = 0; - -% year = 2018; month = 11; day = 10; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'NMEA'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% gps_source{file_idx} = 'nmea-field'; -% sync_flag{file_idx} = 0; - -% year = 2018; month = 11; day = 11; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'NMEA'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% gps_source{file_idx} = 'nmea-field'; -% sync_flag{file_idx} = 0; - -% year = 2018; month = 11; day = 12; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'NMEA'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% gps_source{file_idx} = 'nmea-field'; -% sync_flag{file_idx} = 0; - -% year = 2018; month = 11; day = 14; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'NMEA'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% gps_source{file_idx} = 'nmea-field'; -% sync_flag{file_idx} = 0; - - year = 2018; month = 11; day = 15; - file_idx = file_idx + 1; - in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - file_type{file_idx} = 'NMEA'; - params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); - gps_source{file_idx} = 'nmea-field'; - sync_flag{file_idx} = 0; - - year = 2018; month = 11; day = 16; - file_idx = file_idx + 1; - in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - file_type{file_idx} = 'NMEA'; - params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); - gps_source{file_idx} = 'nmea-field'; - sync_flag{file_idx} = 0; - -elseif strcmpi(gps_source_to_use,'ATM-field') - - year = 2018; month = 09; day = 28; - file_idx = file_idx + 1; - in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - file_type{file_idx} = 'applanix'; - params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); - gps_source{file_idx} = 'atm-field'; - sync_flag{file_idx} = 0; - -elseif strcmpi(gps_source_to_use,'ATM') -% year = 2018; month = 10; day = 10; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); -% file_type{file_idx} = 'traj+applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); -% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); -% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% gps_source{file_idx} = 'atm-final_20190205'; -% sync_flag{file_idx} = 0; -% -% year = 2018; month = 10; day = 11; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); -% file_type{file_idx} = 'traj+applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); -% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); -% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% gps_source{file_idx} = 'atm-final_20190205'; -% sync_flag{file_idx} = 0; - -% year = 2018; month = 10; day = 12; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); -% file_type{file_idx} = 'traj+applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); -% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); -% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% gps_source{file_idx} = 'atm-final_20190205'; -% sync_flag{file_idx} = 0; -% -% year = 2018; month = 10; day = 13; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); -% file_type{file_idx} = 'traj+applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); -% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); -% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% gps_source{file_idx} = 'atm-final_20190205'; -% sync_flag{file_idx} = 0; -% -% year = 2018; month = 10; day = 15; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); -% file_type{file_idx} = 'traj+applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); -% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); -% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% gps_source{file_idx} = 'atm-final_20190205'; -% sync_flag{file_idx} = 0; -% -% year = 2018; month = 10; day = 16; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); -% file_type{file_idx} = 'traj+applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); -% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); -% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% gps_source{file_idx} = 'atm-final_20190205'; -% sync_flag{file_idx} = 0; -% -% year = 2018; month = 10; day = 18; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); -% file_type{file_idx} = 'traj+applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); -% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); -% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% gps_source{file_idx} = 'atm-final_20190205'; -% sync_flag{file_idx} = 0; -% -% year = 2018; month = 10; day = 19; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); -% file_type{file_idx} = 'traj+applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); -% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); -% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% gps_source{file_idx} = 'atm-final_20190205'; -% sync_flag{file_idx} = 0; -% -% year = 2018; month = 10; day = 20; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); -% file_type{file_idx} = 'traj+applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); -% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); -% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% gps_source{file_idx} = 'atm-final_20190205'; -% sync_flag{file_idx} = 0; -% -% year = 2018; month = 10; day = 22; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); -% file_type{file_idx} = 'traj+applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); -% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); -% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% gps_source{file_idx} = 'atm-final_20190205'; -% sync_flag{file_idx} = 0; -% -% year = 2018; month = 10; day = 27; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); -% file_type{file_idx} = 'traj+applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); -% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); -% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% gps_source{file_idx} = 'atm-final_20190205'; -% sync_flag{file_idx} = 0; -% -% year = 2018; month = 10; day = 30; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); -% file_type{file_idx} = 'traj+applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); -% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); -% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% gps_source{file_idx} = 'atm-final_20190205'; -% sync_flag{file_idx} = 0; - -% year = 2018; month = 10; day = 31; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); -% file_type{file_idx} = 'traj+applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); -% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); -% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% gps_source{file_idx} = 'atm-final_20190205'; -% sync_flag{file_idx} = 0; - -% year = 2018; month = 11; day = 03; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); -% file_type{file_idx} = 'traj+applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); -% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); -% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% gps_source{file_idx} = 'atm-final_20190205'; -% sync_flag{file_idx} = 0; - -% year = 2018; month = 11; day = 04; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); -% file_type{file_idx} = 'traj+applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); -% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); -% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% gps_source{file_idx} = 'atm-final_20190205'; -% sync_flag{file_idx} = 0; -% % -% year = 2018; month = 11; day = 05; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); -% file_type{file_idx} = 'traj+applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); -% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); -% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% gps_source{file_idx} = 'atm-final_20190205'; -% sync_flag{file_idx} = 0; -% -% year = 2018; month = 11; day = 07; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); -% file_type{file_idx} = 'traj+applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); -% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); -% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% gps_source{file_idx} = 'atm-final_20190205'; -% sync_flag{file_idx} = 0; -% -% year = 2018; month = 11; day = 09; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); -% file_type{file_idx} = 'traj+applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); -% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); -% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% gps_source{file_idx} = 'atm-final_20190205'; -% sync_flag{file_idx} = 0; -% -% year = 2018; month = 11; day = 10; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); -% file_type{file_idx} = 'traj+applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); -% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); -% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% gps_source{file_idx} = 'atm-final_20190205'; -% sync_flag{file_idx} = 0; -% -% year = 2018; month = 11; day = 11; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); -% file_type{file_idx} = 'traj+applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); -% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); -% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% gps_source{file_idx} = 'atm-final_20190205'; -% sync_flag{file_idx} = 0; - - year = 2018; month = 11; day = 12; - file_idx = file_idx + 1; - in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); - file_type{file_idx} = 'traj+applanix'; - params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); - in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); - params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - gps_source{file_idx} = 'atm-final_20190205'; - sync_flag{file_idx} = 0; - -% year = 2018; month = 11; day = 14; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); -% file_type{file_idx} = 'traj+applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); -% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); -% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% gps_source{file_idx} = 'atm-final_20190205'; -% sync_flag{file_idx} = 0; - -% year = 2018; month = 11; day = 15; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); -% file_type{file_idx} = 'traj+applanix'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); -% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); -% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% gps_source{file_idx} = 'atm-final_20190205'; -% sync_flag{file_idx} = 0; - - year = 2018; month = 11; day = 16; - file_idx = file_idx + 1; - in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); - file_type{file_idx} = 'traj+applanix'; - params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); - in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); - params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - gps_source{file_idx} = 'atm-final_20190205'; - sync_flag{file_idx} = 0; - -end - - -% ====================================================================== -% Read and translate files according to user settings -% ====================================================================== -gps_make; - -% Debug code that can be used when no GPS data is available and we need to -% fake it. -hack_idx = cell2mat(strfind(out_fns,'gps_?.mat')); -if ~isempty(hack_idx) - out_fn = fullfile(gps_path,out_fns{hack_idx}); - - warning('Creating fake trajectory with lab data: %s', out_fn); - - gps = load(out_fn); - gps.lat(:) = gps.lat(1) + (gps.gps_time-gps.gps_time(1))*125/111e3; - gps.lon(:) = gps.lon(1); - save(out_fn,'-append','-struct','gps','lat','lon') -end - -% Reveal files are known to have GPS time errors which are corrected here. -for idx = 1:length(file_type) - out_fn = fullfile(gps_path,out_fns{idx}); - - gps = load(out_fn); - if regexp(gps.gps_source,'reveal') - - warning('Fixing non-monotonic GPS data in reveal file: %s', out_fn); - - [gps,error_flag] = make_gps_monotonic(gps); - - if error_flag - save(out_fn,'-append','-struct','gps'); - end - end -end - -% ATM files are known to have a small high frequency INS error which is -% corrected here. -for idx = 1:length(file_type) - out_fn = fullfile(gps_path,out_fns{idx}); - - gps = load(out_fn); - if regexpi(gps.gps_source,'atm') - - warning('Smoothing INS data: %s', out_fn); - - gps.roll = sgolayfilt(gps.roll,2,101); - gps.pitch = sgolayfilt(gps.pitch,2,101); - gps.heading = sgolayfilt(gps.heading,2,101); - - save(out_fn,'-append','-struct','gps','roll','pitch','heading'); - end -end - +% script gps_create_2018_antarctica_DC8 +% +% Makes the GPS files for 2018 Antarctica DC8 field season + +tic; + +global gRadar; + +support_path = ''; +data_support_path = ''; + +if isempty(support_path) + support_path = gRadar.support_path; +end + +gps_path = fullfile(support_path,'gps','2018_Antarctica_DC8'); +if ~exist(gps_path,'dir') + fprintf('Making directory %s\n', gps_path); + fprintf(' Press a key to proceed\n'); + pause; + mkdir(gps_path); +end + +if isempty(data_support_path) + data_support_path = gRadar.data_support_path; +end + +% ====================================================================== +% User Settings +% ====================================================================== +debug_level = 1; +in_base_path = fullfile(data_support_path,'2018_Antarctica_DC8','Applanix_data'); +sync_path = fullfile(data_support_path,'2018_Antarctica_DC8','Trajectory'); + +file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; +sync_fns = {}; sync_params = {};sync_file_type = {}; + +% gps_source_to_use = 'NMEA'; +% gps_source_to_use = 'ATM-field'; +gps_source_to_use = 'ATM'; + +if strcmpi(gps_source_to_use,'NMEA') + + % year = 2018; month = 9; day = 27; + % file_idx = file_idx + 1; + % in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); + % out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + % file_type{file_idx} = 'NMEA'; + % params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); + % gps_source{file_idx} = 'nmea-field'; + % sync_flag{file_idx} = 0; + +% year = 2018; month = 10; day = 10; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; + +% year = 2018; month = 10; day = 11; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; + + % year = 2018; month = 10; day = 12; + % file_idx = file_idx + 1; + % in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); + % out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + % file_type{file_idx} = 'NMEA'; + % params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); + % gps_source{file_idx} = 'nmea-field'; + % sync_flag{file_idx} = 0; + + % year = 2018; month = 10; day = 13; + % file_idx = file_idx + 1; + % in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); + % out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + % file_type{file_idx} = 'NMEA'; + % params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); + % gps_source{file_idx} = 'nmea-field'; + % sync_flag{file_idx} = 0; + +% year = 2018; month = 10; day = 15; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; + + % year = 2018; month = 10; day = 16; + % file_idx = file_idx + 1; + % in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); + % out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + % file_type{file_idx} = 'NMEA'; + % params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); + % gps_source{file_idx} = 'nmea-field'; + % sync_flag{file_idx} = 0; + + % year = 2018; month = 10; day = 18; + % file_idx = file_idx + 1; + % in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); + % out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + % file_type{file_idx} = 'NMEA'; + % params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); + % gps_source{file_idx} = 'nmea-field'; + % sync_flag{file_idx} = 0; + + % year = 2018; month = 10; day = 19; + % file_idx = file_idx + 1; + % in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); + % out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + % file_type{file_idx} = 'NMEA'; + % params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); + % gps_source{file_idx} = 'nmea-field'; + % sync_flag{file_idx} = 0; + + % year = 2018; month = 10; day = 20; + % file_idx = file_idx + 1; + % in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); + % out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + % file_type{file_idx} = 'NMEA'; + % params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); + % gps_source{file_idx} = 'nmea-field'; + % sync_flag{file_idx} = 0; + + % year = 2018; month = 10; day = 22; + % file_idx = file_idx + 1; + % in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); + % out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + % file_type{file_idx} = 'NMEA'; + % params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); + % gps_source{file_idx} = 'nmea-field'; + % sync_flag{file_idx} = 0; + + % year = 2018; month = 10; day = 30; + % file_idx = file_idx + 1; + % in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); + % out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + % file_type{file_idx} = 'NMEA'; + % params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); + % gps_source{file_idx} = 'nmea-field'; + % sync_flag{file_idx} = 0; + + year = 2018; month = 10; day = 31; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'NMEA'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); + gps_source{file_idx} = 'nmea-field'; + sync_flag{file_idx} = 0; + + + % year = 2018; month = 11; day = 3; + % file_idx = file_idx + 1; + % in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); + % out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + % file_type{file_idx} = 'NMEA'; + % params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); + % gps_source{file_idx} = 'nmea-field'; + % sync_flag{file_idx} = 0; + + year = 2018; month = 11; day = 4; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'NMEA'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); + gps_source{file_idx} = 'nmea-field'; + sync_flag{file_idx} = 0; + +% year = 2018; month = 11; day = 5; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; + +% year = 2018; month = 11; day = 7; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; + +% year = 2018; month = 11; day = 9; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; + +% year = 2018; month = 11; day = 10; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; + +% year = 2018; month = 11; day = 11; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; + +% year = 2018; month = 11; day = 12; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; + +% year = 2018; month = 11; day = 14; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; + + year = 2018; month = 11; day = 15; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'NMEA'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); + gps_source{file_idx} = 'nmea-field'; + sync_flag{file_idx} = 0; + + year = 2018; month = 11; day = 16; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'NMEA'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); + gps_source{file_idx} = 'nmea-field'; + sync_flag{file_idx} = 0; + +elseif strcmpi(gps_source_to_use,'ATM-field') + + year = 2018; month = 09; day = 28; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'applanix'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); + gps_source{file_idx} = 'atm-field'; + sync_flag{file_idx} = 0; + +elseif strcmpi(gps_source_to_use,'ATM') +% year = 2018; month = 10; day = 10; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); +% file_type{file_idx} = 'traj+applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); +% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); +% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% gps_source{file_idx} = 'atm-final_20190205'; +% sync_flag{file_idx} = 0; +% +% year = 2018; month = 10; day = 11; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); +% file_type{file_idx} = 'traj+applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); +% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); +% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% gps_source{file_idx} = 'atm-final_20190205'; +% sync_flag{file_idx} = 0; + +% year = 2018; month = 10; day = 12; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); +% file_type{file_idx} = 'traj+applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); +% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); +% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% gps_source{file_idx} = 'atm-final_20190205'; +% sync_flag{file_idx} = 0; +% +% year = 2018; month = 10; day = 13; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); +% file_type{file_idx} = 'traj+applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); +% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); +% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% gps_source{file_idx} = 'atm-final_20190205'; +% sync_flag{file_idx} = 0; +% +% year = 2018; month = 10; day = 15; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); +% file_type{file_idx} = 'traj+applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); +% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); +% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% gps_source{file_idx} = 'atm-final_20190205'; +% sync_flag{file_idx} = 0; +% +% year = 2018; month = 10; day = 16; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); +% file_type{file_idx} = 'traj+applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); +% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); +% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% gps_source{file_idx} = 'atm-final_20190205'; +% sync_flag{file_idx} = 0; +% +% year = 2018; month = 10; day = 18; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); +% file_type{file_idx} = 'traj+applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); +% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); +% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% gps_source{file_idx} = 'atm-final_20190205'; +% sync_flag{file_idx} = 0; +% +% year = 2018; month = 10; day = 19; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); +% file_type{file_idx} = 'traj+applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); +% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); +% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% gps_source{file_idx} = 'atm-final_20190205'; +% sync_flag{file_idx} = 0; +% +% year = 2018; month = 10; day = 20; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); +% file_type{file_idx} = 'traj+applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); +% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); +% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% gps_source{file_idx} = 'atm-final_20190205'; +% sync_flag{file_idx} = 0; +% +% year = 2018; month = 10; day = 22; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); +% file_type{file_idx} = 'traj+applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); +% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); +% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% gps_source{file_idx} = 'atm-final_20190205'; +% sync_flag{file_idx} = 0; +% +% year = 2018; month = 10; day = 27; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); +% file_type{file_idx} = 'traj+applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); +% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); +% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% gps_source{file_idx} = 'atm-final_20190205'; +% sync_flag{file_idx} = 0; +% +% year = 2018; month = 10; day = 30; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); +% file_type{file_idx} = 'traj+applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); +% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); +% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% gps_source{file_idx} = 'atm-final_20190205'; +% sync_flag{file_idx} = 0; + +% year = 2018; month = 10; day = 31; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); +% file_type{file_idx} = 'traj+applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); +% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); +% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% gps_source{file_idx} = 'atm-final_20190205'; +% sync_flag{file_idx} = 0; + +% year = 2018; month = 11; day = 03; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); +% file_type{file_idx} = 'traj+applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); +% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); +% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% gps_source{file_idx} = 'atm-final_20190205'; +% sync_flag{file_idx} = 0; + +% year = 2018; month = 11; day = 04; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); +% file_type{file_idx} = 'traj+applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); +% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); +% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% gps_source{file_idx} = 'atm-final_20190205'; +% sync_flag{file_idx} = 0; +% % +% year = 2018; month = 11; day = 05; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); +% file_type{file_idx} = 'traj+applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); +% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); +% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% gps_source{file_idx} = 'atm-final_20190205'; +% sync_flag{file_idx} = 0; +% +% year = 2018; month = 11; day = 07; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); +% file_type{file_idx} = 'traj+applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); +% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); +% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% gps_source{file_idx} = 'atm-final_20190205'; +% sync_flag{file_idx} = 0; +% +% year = 2018; month = 11; day = 09; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); +% file_type{file_idx} = 'traj+applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); +% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); +% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% gps_source{file_idx} = 'atm-final_20190205'; +% sync_flag{file_idx} = 0; +% +% year = 2018; month = 11; day = 10; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); +% file_type{file_idx} = 'traj+applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); +% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); +% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% gps_source{file_idx} = 'atm-final_20190205'; +% sync_flag{file_idx} = 0; +% +% year = 2018; month = 11; day = 11; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); +% file_type{file_idx} = 'traj+applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); +% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); +% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% gps_source{file_idx} = 'atm-final_20190205'; +% sync_flag{file_idx} = 0; + + year = 2018; month = 11; day = 12; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); + file_type{file_idx} = 'traj+applanix'; + params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); + in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); + params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + gps_source{file_idx} = 'atm-final_20190205'; + sync_flag{file_idx} = 0; + +% year = 2018; month = 11; day = 14; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); +% file_type{file_idx} = 'traj+applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); +% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); +% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% gps_source{file_idx} = 'atm-final_20190205'; +% sync_flag{file_idx} = 0; + +% year = 2018; month = 11; day = 15; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); +% file_type{file_idx} = 'traj+applanix'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); +% in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); +% params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% gps_source{file_idx} = 'atm-final_20190205'; +% sync_flag{file_idx} = 0; + + year = 2018; month = 11; day = 16; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filename(sync_path,datestr(datenum(year,month,day),'yymmdd'),'amu2',''); + file_type{file_idx} = 'traj+applanix'; + params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','gps','input_format', '%f%f%f%f%f%f%f%f%f%f%f'); + in_fns_ins{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'GNSSK*.out'); + params_ins{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + gps_source{file_idx} = 'atm-final_20190205'; + sync_flag{file_idx} = 0; + +end + + +% ====================================================================== +% Read and translate files according to user settings +% ====================================================================== +gps_create; + +% Debug code that can be used when no GPS data is available and we need to +% fake it. +hack_idx = cell2mat(strfind(out_fns,'gps_?.mat')); +if ~isempty(hack_idx) + out_fn = fullfile(gps_path,out_fns{hack_idx}); + + warning('Creating fake trajectory with lab data: %s', out_fn); + + gps = load(out_fn); + gps.lat(:) = gps.lat(1) + (gps.gps_time-gps.gps_time(1))*125/111e3; + gps.lon(:) = gps.lon(1); + save(out_fn,'-append','-struct','gps','lat','lon') +end + +% Reveal files are known to have GPS time errors which are corrected here. +for idx = 1:length(file_type) + out_fn = fullfile(gps_path,out_fns{idx}); + + gps = load(out_fn); + if regexp(gps.gps_source,'reveal') + + warning('Fixing non-monotonic GPS data in reveal file: %s', out_fn); + + [gps,error_flag] = gps_force_monotonic(gps); + + if error_flag + save(out_fn,'-append','-struct','gps'); + end + end +end + +% ATM files are known to have a small high frequency INS error which is +% corrected here. +for idx = 1:length(file_type) + out_fn = fullfile(gps_path,out_fns{idx}); + + gps = load(out_fn); + if regexpi(gps.gps_source,'atm') + + warning('Smoothing INS data: %s', out_fn); + + gps.roll = sgolayfilt(gps.roll,2,101); + gps.pitch = sgolayfilt(gps.pitch,2,101); + gps.heading = sgolayfilt(gps.heading,2,101); + + save(out_fn,'-append','-struct','gps','roll','pitch','heading'); + end +end + diff --git a/cresis-toolbox/gps/missions/make_gps_2018_antarctica_Ground.m b/cresis-toolbox/gps/missions/gps_create_2018_Antarctica_Ground.m similarity index 82% rename from cresis-toolbox/gps/missions/make_gps_2018_antarctica_Ground.m rename to cresis-toolbox/gps/missions/gps_create_2018_Antarctica_Ground.m index acbdf813..e8b5b01e 100644 --- a/cresis-toolbox/gps/missions/make_gps_2018_antarctica_Ground.m +++ b/cresis-toolbox/gps/missions/gps_create_2018_Antarctica_Ground.m @@ -1,4 +1,4 @@ -% script make_gps_2018_antarctica_Ground +% script gps_create_2018_antarctica_Ground % % Makes the GPS files for 2018 Antarctica Ground field season @@ -37,8 +37,8 @@ % gps_source_to_use = 'arena'; % gps_source_to_use = 'arena_cpu_time'; -% gps_source_to_use = 'trimble_cpu_time'; -gps_source_to_use = 'trimble_cpu_time2'; +% gps_source_to_use = 'trimble_cpu_time_shun'; +gps_source_to_use = 'trimble_cpu_time_paden'; if strcmpi(gps_source_to_use,'arena') %% ARENA @@ -55,32 +55,32 @@ % sync_file_type{file_idx} = 'arena'; % sync_params{file_idx} = struct('time_reference','utc'); - % year = 2018; month = 10; day = 14; - % file_idx = file_idx + 1; - % in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); - % out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - % file_type{file_idx} = 'arena'; - % params{file_idx} = struct('year',2018,'time_reference','utc'); - % gps_source{file_idx} = 'arena-field'; - % sync_flag{file_idx} = 1; - % sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); - % sync_file_type{file_idx} = 'arena'; - % sync_params{file_idx} = struct('time_reference','utc'); + year = 2018; month = 10; day = 14; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'arena'; + params{file_idx} = struct('year',2018,'time_reference','utc'); + gps_source{file_idx} = 'arena-field'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); - year = 2018; month = 10; day = 15; - file_idx = file_idx + 1; - in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - file_type{file_idx} = 'arena'; - params{file_idx} = struct('year',2018,'time_reference','utc'); - gps_source{file_idx} = 'arena-field'; - sync_flag{file_idx} = 1; - sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); - sync_file_type{file_idx} = 'arena'; - sync_params{file_idx} = struct('time_reference','utc'); +% year = 2018; month = 10; day = 15; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'arena'; +% params{file_idx} = struct('year',2018,'time_reference','utc'); +% gps_source{file_idx} = 'arena-field'; +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); elseif strcmpi(gps_source_to_use,'arena_cpu_time') - correction = make_gps_2018_antarctica_Ground_cpu_time(in_base_path); + correction = gps_create_2018_Antarctica_Ground_cpu_time(in_base_path); % year = 2018; month = 12; day = 17; % file_idx = file_idx + 1; @@ -108,8 +108,12 @@ sync_params{file_idx} = struct('time_reference','utc', ... 'cpu_time_correction',correction); -elseif strcmpi(gps_source_to_use,'trimble_cpu_time') - correction = make_gps_2018_antarctica_Ground_cpu_time(in_base_path); +elseif strcmpi(gps_source_to_use,'trimble_cpu_time_shun') + correction = gps_create_2018_Antarctica_Ground_cpu_time(in_base_path); + + % Shun processed with ? software, but noticed unusually large errors: + % currently not using this versino of the processed data + error('Do not use this gps_source.'); % year = 2018; month = 12; % for day = [17 19 20 21 22 23 24 25 26 27 28 29] @@ -162,9 +166,13 @@ % 'cpu_time_correction',correction); % end -elseif strcmpi(gps_source_to_use,'trimble_cpu_time2') - correction = make_gps_2018_antarctica_Ground_cpu_time(in_base_path); +elseif strcmpi(gps_source_to_use,'trimble_cpu_time_paden') + correction = gps_create_2018_Antarctica_Ground_cpu_time(in_base_path); + % Paden processed with the Canadian online service. The results seem to + % be smoother than Shun's processed results and the reported errors are + % much smaller. + % % HDR GRP CANADIAN GEODETIC SURVEY, SURVEYOR GENERAL BRANCH, NATURAL RESOURCES CANADA % HDR ADR GOVERNMENT OF CANADA, 588 BOOTH STREET ROOM 334, OTTAWA ONTARIO K1A 0Y7 % HDR TEL 343-292-6617 @@ -194,7 +202,7 @@ % 'cpu_time_correction',correction); year = 2018; month = 12; - for day = 31%[19 20 21 22 23 24 25 26 27 28 29 31] + for day = 24%[19 20 21 22 23 24 25 26 27 28 29 31] file_idx = file_idx + 1; in_fns{file_idx} = get_filenames(fullfile(in_base_path,'Field_data_SM100',sprintf('%04d%02d%02d_PPP',year,month,day)),'','','0000.pos'); out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); @@ -240,7 +248,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; for idx = 1:length(file_type) out_fn = fullfile(gps_path,out_fns{idx}); @@ -266,7 +274,26 @@ end end - if regexpi(gps.gps_source,'brice') + if 0 && regexpi(gps.gps_source,'trimble_cpu_time_paden') + % Smoothing is necessary due to vibration + warning('Smoothing GPS and IMU data: %s', out_fn); + gps = load(out_fn); + + gps.elev = sgolayfilt(gps.elev,2,201); % Adjust filter length as needed to remove high frequency noise + + heading_x = cos(gps.heading); + heading_y = sin(gps.heading); + heading_x = sgolayfilt(heading_x,2,501); % Adjust filter length as needed to remove high frequency noise + heading_y = sgolayfilt(heading_y,2,501); % Adjust filter length as needed to remove high frequency noise + new_heading = atan2(heading_y,heading_x); + filter_mask = abs(new_heading-gps.heading) < 20/180*pi; + gps.heading(filter_mask) = new_heading(filter_mask); + + save(out_fn,'-append','-struct','gps','elev','heading'); + + end + + if regexpi(gps.gps_source,'trimble_cpu_time_shun') % Smoothing is necessary because data are poor quality warning('Smoothing GPS and IMU data: %s', out_fn); gps = load(out_fn); diff --git a/cresis-toolbox/gps/missions/make_gps_2018_antarctica_Ground_cpu_time.m b/cresis-toolbox/gps/missions/gps_create_2018_Antarctica_Ground_cpu_time.m similarity index 90% rename from cresis-toolbox/gps/missions/make_gps_2018_antarctica_Ground_cpu_time.m rename to cresis-toolbox/gps/missions/gps_create_2018_Antarctica_Ground_cpu_time.m index e8b82282..6af67a1d 100644 --- a/cresis-toolbox/gps/missions/make_gps_2018_antarctica_Ground_cpu_time.m +++ b/cresis-toolbox/gps/missions/gps_create_2018_Antarctica_Ground_cpu_time.m @@ -1,4 +1,4 @@ -function correction = make_gps_2018_antarctica_Ground_cpu_time(in_base_path) +function correction = gps_create_2018_antarctica_Ground_cpu_time(in_base_path) fns = get_filenames(fullfile(in_base_path,'CPU_TIME'),'cpu_time','','.csv',struct('recursive',true)); [gps_time,cpu_time] = read_cpu_time(fns); diff --git a/cresis-toolbox/gps/missions/make_gps_2018_antarctica_TObas.m b/cresis-toolbox/gps/missions/gps_create_2018_Antarctica_TObas.m similarity index 96% rename from cresis-toolbox/gps/missions/make_gps_2018_antarctica_TObas.m rename to cresis-toolbox/gps/missions/gps_create_2018_Antarctica_TObas.m index fa4da2f4..7d5682b7 100644 --- a/cresis-toolbox/gps/missions/make_gps_2018_antarctica_TObas.m +++ b/cresis-toolbox/gps/missions/gps_create_2018_Antarctica_TObas.m @@ -1,266 +1,268 @@ -% script make_gps_2018_antarctica_TObas -% -% Makes the GPS files for 2018 Antarctica TObas field season - -tic; - -global gRadar; - -support_path = ''; -data_support_path = ''; - -if isempty(support_path) - support_path = gRadar.support_path; -end - -gps_path = fullfile(support_path,'gps','2018_Antarctica_TObas'); -if ~exist(gps_path,'dir') - fprintf('Making directory %s\n', gps_path); - fprintf(' Press a key to proceed\n'); - pause; - mkdir(gps_path); -end - -if isempty(data_support_path) - data_support_path = gRadar.data_support_path; -end - -% ====================================================================== -% User Settings -% ====================================================================== -debug_level = 1; -mergegps = true; % Set to true for days that require combining Arena and BAS GPS - -if mergegps - % Merge Arena and BAS GPS - % Only one day can be enabled at once - year = 2019; month = 2; day = 4; -% year = 2019; month = 2; day = 7; - datevecs = {[year, month, day]}; - - baddatevecs = datevecs; -else - % Multiple days (BAS GPS) - % Each vec has format [YYYY, MM, DD] and it is iterated upon later[2019, 1, 29] - % Uncomment which days you want to make - - datevecs = {}; -% datevecs{end+1} = [2019, 1, 29]; -% datevecs{end+1} = [2019, 1, 30]; -% datevecs{end+1} = [2019, 1, 31]; -% datevecs{end+1} = [2019, 2, 1]; -% datevecs{end+1} = [2019, 2, 3]; -% datevecs{end+1} = [2019, 2, 5]; -% datevecs{end+1} = [2019, 2, 6]; - - baddatevecs = {}; -end - -%Find all of the subdirectories in the base path for comparison to dates -in_base_path = fullfile(data_support_path,'2018_Antarctica_TObas'); -in_base = dir(in_base_path); -in_base_dirs = {in_base(:).name}; - -file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; -sync_flag = {}; sync_fns = {}; sync_file_type = {}; sync_params = {}; - -% gps_source_to_use = 'arena'; -% gps_source_to_use = 'arena_cpu_time'; -gps_source_to_use = 'bas'; - -%Load days with "bad data" to be processed with arena settings -for date_idx = 1:length(baddatevecs) - %Get the date and make it a string - year = baddatevecs{date_idx}(1); month = baddatevecs{date_idx}(2); day = baddatevecs{date_idx}(3); - date_str = sprintf('%04d%02d%02d',year,month,day); - %Get folder names that match the date in question (looking for daya, - %dayb, etc.) - %Compare the dirs in the base path to the current date string - compcells = strfind(in_base_dirs,date_str); - %Extract the dirs that match the string - match_dirs = {in_base_dirs{cellfun(@(x) ~isempty(x),compcells)}}; - %Iterate over the directories that match the date string - for match_idx = 1:length(match_dirs) - dir_str = match_dirs{match_idx}; - file_idx = file_idx + 1; - in_fns{file_idx} = get_filenames(fullfile(in_base_path,dir_str),'','','gps.txt'); - out_fns{file_idx} = sprintf('gps_%s_arena.mat', date_str); - file_type{file_idx} = 'arena'; - params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc'); -% gps_source{file_idx} = 'arena-field'; - gps_source{file_idx} = 'arena'; - sync_flag{file_idx} = 1; - sync_fns{file_idx} = get_filenames(fullfile(in_base_path,dir_str),'','','gps.txt'); - sync_file_type{file_idx} = 'arena'; - sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc'); - end -end - -if strcmpi(gps_source_to_use,'arena') -%% ARENA -% year = 2019; month = 1; day = 26; - for date_idx = 1:length(datevecs) - %Get the date and make it a string - year = datevecs{date_idx}(1); month = datevecs{date_idx}(2); day = datevecs{date_idx}(3); - date_str = sprintf('%04d%02d%02d',year,month,day); - %Get folder names that match the date in question (looking for daya, - %dayb, etc.) - %Compare the dirs in the base path to the current date string - compcells = strfind(in_base_dirs,date_str); - %Extract the dirs that match the string - match_dirs = {in_base_dirs{cellfun(@(x) ~isempty(x),compcells)}}; - %Iterate over the directories that match the date string - for match_idx = 1:length(match_dirs) - dir_str = match_dirs{match_idx}; - file_idx = file_idx + 1; - in_fns{file_idx} = get_filenames(fullfile(in_base_path,dir_str),'','','gps.txt'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - file_type{file_idx} = 'arena'; - params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc'); - gps_source{file_idx} = 'arena-field'; - sync_flag{file_idx} = 1; - sync_fns{file_idx} = get_filenames(fullfile(in_base_path,dir_str),'','','gps.txt'); - sync_file_type{file_idx} = 'arena'; - sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc'); - end - end - -elseif strcmpi(gps_source_to_use,'arena_cpu_time') - %% Arena used computer time with no GPS inputs - -% year = 2019; month = 1; day = 27; - file_idx = file_idx + 1; - in_fns{file_idx} = fullfile(in_base_path,'GPS_FILE_20190127.gps'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - file_type{file_idx} = 'nmea'; - params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc'); - gps_source{file_idx} = 'arena-field'; - sync_flag{file_idx} = 1; - sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','awg0.txt'); - sync_file_type{file_idx} = 'arena_cpu_time'; - sync_params{file_idx} = struct('time_reference','utc', ... - 'cpu_time_fn',fullfile(in_base_path,sprintf('cpu_time_%04d%02d%02d.csv',year,month,day))); - -elseif strcmpi(gps_source_to_use,'bas') - %% BAS - for date_idx = 1:length(datevecs) - %Get the date and make it a string - year = datevecs{date_idx}(1); month = datevecs{date_idx}(2); day = datevecs{date_idx}(3); - date_str = sprintf('%04d%02d%02d',year,month,day); - - %Select the correct input files - switch date_str - case '20190129' - incase_fns = {fullfile(in_base_path,'03.txt'); fullfile(in_base_path,'04.txt')}; - case '20190130' - incase_fns = {fullfile(in_base_path,'05.txt')}; - case '20190131' - incase_fns = {fullfile(in_base_path,'06.txt')}; - case '20190201' - incase_fns = {fullfile(in_base_path,'07.txt')}; - case '20190203' - incase_fns = {fullfile(in_base_path,'08.txt')}; - case '20190204' - incase_fns = {fullfile(in_base_path,'09.txt')}; - case '20190205' - incase_fns = {fullfile(in_base_path,'11.txt')}; - case '20190206' - incase_fns = {fullfile(in_base_path,'12.txt'); fullfile(in_base_path,'13.txt')}; - case '20190207' - incase_fns = {fullfile(in_base_path,'14.txt')}; - otherwise - return - end - %Increase file id - file_idx = file_idx + 1; - - %Load the structures - in_fns{file_idx} = incase_fns; - out_fns{file_idx} = sprintf('gps_%s.mat', date_str); - file_type{file_idx} = 'General_ASCII'; - params{file_idx} = struct('time_reference','gps'); - params{file_idx}.format_str = '%s%s%f%f%f%f%f%f%f%f%f%f%f%f%f'; - params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg',... - 'elev_m','f1','f2','f3','roll_deg','pitch_deg','heading_deg','f4',... - 'f5','f6','f7'}; - params{file_idx}.textscan = {}; - params{file_idx}.headerlines = 5; - gps_source{file_idx} = 'bas-final20190313'; - sync_flag{file_idx} = 1; - sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%s',date_str)),'','','gps.txt'); - sync_file_type{file_idx} = 'arena'; -% sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc'); - sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc'); - end - - -end - -% ====================================================================== -% Read and translate files according to user settings -% ====================================================================== -gps_make; - -for idx = 1:length(file_type) - out_fn = fullfile(gps_path,out_fns{idx}); - - gps = load(out_fn,'gps_source'); - if regexpi(gps.gps_source,'arena') - % Extrapolation is necessary because GPS data starts after/stops before - % the beginning/end of the radar data. - warning('Extrapolating arena GPS data: %s', out_fn); - gps = load(out_fn); - - if length(gps.lat) >= 2 - new_gps_time = [gps.gps_time(1)-10, gps.gps_time,gps.gps_time(end)+10]; - gps.lat = interp1(gps.gps_time,gps.lat,new_gps_time,'linear','extrap'); - gps.lon = interp1(gps.gps_time,gps.lon,new_gps_time,'linear','extrap'); - gps.elev = interp1(gps.gps_time,gps.elev,new_gps_time,'linear','extrap'); - gps.roll = interp1(gps.gps_time,gps.roll,new_gps_time,'linear','extrap'); - gps.pitch = interp1(gps.gps_time,gps.pitch,new_gps_time,'linear','extrap'); - gps.heading = interp1(gps.gps_time,gps.heading,new_gps_time,'linear','extrap'); - gps.gps_time = new_gps_time; - - save(out_fn,'-append','-struct','gps','gps_time','lat','lon','elev','roll','pitch','heading'); - end - end - - if regexpi(out_fn,'20180929') - % Fake GPS for testing - warning('Faking GPS data: %s', out_fn); - gps = load(out_fn); - - velocity = 70; - gps.lat = -75.5 - (gps.gps_time-gps.gps_time(1))*velocity/111111; - gps.lon(:) = -106.75; - gps.elev(:) = 500; - gps.heading(:) = -pi; - - save(out_fn,'-append','-struct','gps','gps_time','lat','lon','elev','roll','pitch','heading'); - end - -end - -%Merge the structures for the "bad data" dates -for id_bd = 1:length(baddatevecs) - %Get the date - year = baddatevecs{id_bd}(1); month = baddatevecs{id_bd}(2); day = baddatevecs{id_bd}(3); - date_str = sprintf('%04d%02d%02d',year,month,day); - %Get the output file names - out_fn_arena = fullfile(gps_path,sprintf('gps_%s_arena',date_str)); - out_fn_bas = fullfile(gps_path,sprintf('gps_%s',date_str)); - %Load the files - gps_arena = load(out_fn_arena); - gps_bas = load(out_fn_bas); - %Append data to gps_bas - gps = gps_bas; - fappend = {'elev','heading','lat','lon','pitch','roll','gps_time'}; %Fields to append data - applogicvec = [gps_arena.gps_time>=gps_bas.gps_time(end)]; - for fid = 1:length(fappend) - basf = gps_bas.(fappend{fid}); - appf = gps_arena.(fappend{fid}); - gps.(fappend{fid}) = [basf, appf(applogicvec)]; - end - %Save the merged data - save(out_fn_bas,'-append','-struct','gps','gps_time','lat','lon','elev','roll','pitch','heading'); +% script gps_create_2018_antarctica_TObas +% +% Makes the GPS files for 2018 Antarctica TObas field season + +tic; + +global gRadar; + +support_path = ''; +data_support_path = ''; + +if isempty(support_path) + support_path = gRadar.support_path; +end + +gps_path = fullfile(support_path,'gps','2018_Antarctica_TObas'); +if ~exist(gps_path,'dir') + fprintf('Making directory %s\n', gps_path); + fprintf(' Press a key to proceed\n'); + pause; + mkdir(gps_path); +end + +if isempty(data_support_path) + data_support_path = gRadar.data_support_path; +end + +% ====================================================================== +% User Settings +% ====================================================================== +debug_level = 1; +mergegps = true; % Set to true for days that require combining Arena and BAS GPS + +if mergegps + % Merge Arena and BAS GPS + % Only one day can be enabled at once + year = 2019; month = 2; day = 4; +% year = 2019; month = 2; day = 7; + datevecs = {[year, month, day]}; + + baddatevecs = datevecs; +else + % Multiple days (BAS GPS) + % Each vec has format [YYYY, MM, DD] and it is iterated upon later[2019, 1, 29] + % Uncomment which days you want to make + + datevecs = {}; +% datevecs{end+1} = [2019, 1, 29]; +% datevecs{end+1} = [2019, 1, 30]; +% datevecs{end+1} = [2019, 1, 31]; +% datevecs{end+1} = [2019, 2, 1]; +% datevecs{end+1} = [2019, 2, 3]; +% datevecs{end+1} = [2019, 2, 5]; +% datevecs{end+1} = [2019, 2, 6]; + + baddatevecs = {}; +end + +%Find all of the subdirectories in the base path for comparison to dates +in_base_path = fullfile(data_support_path,'2018_Antarctica_TObas'); +in_base = dir(in_base_path); +in_base_dirs = {in_base(:).name}; + +file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; +sync_flag = {}; sync_fns = {}; sync_file_type = {}; sync_params = {}; + +% gps_source_to_use = 'arena'; +% gps_source_to_use = 'arena_cpu_time'; +gps_source_to_use = 'bas'; + +%Load days with "bad data" to be processed with arena settings +for date_idx = 1:length(baddatevecs) + %Get the date and make it a string + year = baddatevecs{date_idx}(1); month = baddatevecs{date_idx}(2); day = baddatevecs{date_idx}(3); + date_str = sprintf('%04d%02d%02d',year,month,day); + %Get folder names that match the date in question (looking for daya, + %dayb, etc.) + %Compare the dirs in the base path to the current date string + compcells = strfind(in_base_dirs,date_str); + %Extract the dirs that match the string + match_dirs = {in_base_dirs{cellfun(@(x) ~isempty(x),compcells)}}; + %Iterate over the directories that match the date string + for match_idx = 1:length(match_dirs) + dir_str = match_dirs{match_idx}; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,dir_str),'','','gps.txt'); + out_fns{file_idx} = sprintf('gps_%s_arena.mat', date_str); + file_type{file_idx} = 'arena'; + params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc'); +% gps_source{file_idx} = 'arena-field'; + gps_source{file_idx} = 'arena'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,dir_str),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc'); + end +end + +if strcmpi(gps_source_to_use,'arena') +%% ARENA +% year = 2019; month = 1; day = 26; + for date_idx = 1:length(datevecs) + %Get the date and make it a string + year = datevecs{date_idx}(1); month = datevecs{date_idx}(2); day = datevecs{date_idx}(3); + date_str = sprintf('%04d%02d%02d',year,month,day); + %Get folder names that match the date in question (looking for daya, + %dayb, etc.) + %Compare the dirs in the base path to the current date string + compcells = strfind(in_base_dirs,date_str); + %Extract the dirs that match the string + match_dirs = {in_base_dirs{cellfun(@(x) ~isempty(x),compcells)}}; + %Iterate over the directories that match the date string + for match_idx = 1:length(match_dirs) + dir_str = match_dirs{match_idx}; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,dir_str),'','','gps.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'arena'; + params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc'); + gps_source{file_idx} = 'arena-field'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,dir_str),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc'); + end + end + +elseif strcmpi(gps_source_to_use,'arena_cpu_time') + %% Arena used computer time with no GPS inputs + +% year = 2019; month = 1; day = 27; + file_idx = file_idx + 1; + in_fns{file_idx} = fullfile(in_base_path,'GPS_FILE_20190127.gps'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'nmea'; + params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc'); + gps_source{file_idx} = 'arena-field'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','awg0.txt'); + sync_file_type{file_idx} = 'arena_cpu_time'; + sync_params{file_idx} = struct('time_reference','utc', ... + 'cpu_time_fn',fullfile(in_base_path,sprintf('cpu_time_%04d%02d%02d.csv',year,month,day))); + +elseif strcmpi(gps_source_to_use,'bas') + %% BAS + for date_idx = 1:length(datevecs) + %Get the date and make it a string + year = datevecs{date_idx}(1); month = datevecs{date_idx}(2); day = datevecs{date_idx}(3); + date_str = sprintf('%04d%02d%02d',year,month,day); + + %Select the correct input files + switch date_str + case '20190129' + incase_fns = {fullfile(in_base_path,'03.txt'); fullfile(in_base_path,'04.txt')}; + case '20190130' + incase_fns = {fullfile(in_base_path,'05.txt')}; + case '20190131' + incase_fns = {fullfile(in_base_path,'06.txt')}; + case '20190201' + incase_fns = {fullfile(in_base_path,'07.txt')}; + case '20190203' + incase_fns = {fullfile(in_base_path,'08.txt')}; + case '20190204' + incase_fns = {fullfile(in_base_path,'09.txt')}; + case '20190205' + incase_fns = {fullfile(in_base_path,'11.txt')}; + case '20190206' + incase_fns = {fullfile(in_base_path,'12.txt'); fullfile(in_base_path,'13.txt')}; + case '20190207' + incase_fns = {fullfile(in_base_path,'14.txt')}; + otherwise + return + end + %Increase file id + file_idx = file_idx + 1; + + %Load the structures + in_fns{file_idx} = incase_fns; + out_fns{file_idx} = sprintf('gps_%s.mat', date_str); + file_type{file_idx} = 'General_ASCII'; + params{file_idx} = struct('time_reference','gps'); + params{file_idx}.format_str = '%s%s%f%f%f%f%f%f%f%f%f%f%f%f%f'; + params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg',... + 'elev_m','f1','f2','f3','roll_deg','pitch_deg','heading_deg','f4',... + 'f5','f6','f7'}; + params{file_idx}.textscan = {}; + params{file_idx}.headerlines = 5; + gps_source{file_idx} = 'bas-final20190313'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%s',date_str)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc'); + sync_params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc'); + end + + +end + +% ====================================================================== +% Read and translate files according to user settings +% ====================================================================== +gps_create; + +for idx = 1:length(file_type) + out_fn = fullfile(gps_path,out_fns{idx}); + + gps = load(out_fn,'gps_source'); + if ~isempty(regexpi(gps.gps_source,'arena')) + % Extrapolation is necessary because GPS data starts after/stops before + % the beginning/end of the radar data. + warning('Extrapolating and filtering elevation for arena GPS data: %s', out_fn); + gps = load(out_fn); + + if length(gps.lat) >= 2 + new_gps_time = [gps.gps_time(1)-10, gps.gps_time,gps.gps_time(end)+10]; + gps.lat = interp1(gps.gps_time,gps.lat,new_gps_time,'linear','extrap'); + gps.lon = interp1(gps.gps_time,gps.lon,new_gps_time,'linear','extrap'); + gps.elev = interp1(gps.gps_time,gps.elev,new_gps_time,'linear','extrap'); + gps.roll = interp1(gps.gps_time,gps.roll,new_gps_time,'linear','extrap'); + gps.pitch = interp1(gps.gps_time,gps.pitch,new_gps_time,'linear','extrap'); + gps.heading = interp1(gps.gps_time,gps.heading,new_gps_time,'linear','extrap'); + gps.gps_time = new_gps_time; + + gps.elev = fir_dec(gps.elev,ones(1,101)/101,1); + + save(out_fn,'-append','-struct','gps','gps_time','lat','lon','elev','roll','pitch','heading'); + end + end + + if regexpi(out_fn,'20180929') + % Fake GPS for testing + warning('Faking GPS data: %s', out_fn); + gps = load(out_fn); + + velocity = 70; + gps.lat = -75.5 - (gps.gps_time-gps.gps_time(1))*velocity/111111; + gps.lon(:) = -106.75; + gps.elev(:) = 500; + gps.heading(:) = -pi; + + save(out_fn,'-append','-struct','gps','gps_time','lat','lon','elev','roll','pitch','heading'); + end + +end + +%Merge the structures for the "bad data" dates +for id_bd = 1:length(baddatevecs) + %Get the date + year = baddatevecs{id_bd}(1); month = baddatevecs{id_bd}(2); day = baddatevecs{id_bd}(3); + date_str = sprintf('%04d%02d%02d',year,month,day); + %Get the output file names + out_fn_arena = fullfile(gps_path,sprintf('gps_%s_arena',date_str)); + out_fn_bas = fullfile(gps_path,sprintf('gps_%s',date_str)); + %Load the files + gps_arena = load(out_fn_arena); + gps_bas = load(out_fn_bas); + %Append data to gps_bas + gps = gps_bas; + fappend = {'elev','heading','lat','lon','pitch','roll','gps_time'}; %Fields to append data + applogicvec = [gps_arena.gps_time>=gps_bas.gps_time(end)]; + for fid = 1:length(fappend) + basf = gps_bas.(fappend{fid}); + appf = gps_arena.(fappend{fid}); + gps.(fappend{fid}) = [basf, appf(applogicvec)]; + end + %Save the merged data + save(out_fn_bas,'-append','-struct','gps','gps_time','lat','lon','elev','roll','pitch','heading'); end \ No newline at end of file diff --git a/cresis-toolbox/gps/missions/make_gps_2018_greenland_P3.m b/cresis-toolbox/gps/missions/gps_create_2018_Greenland_P3.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2018_greenland_P3.m rename to cresis-toolbox/gps/missions/gps_create_2018_Greenland_P3.m index f0238d2b..2503400f 100644 --- a/cresis-toolbox/gps/missions/make_gps_2018_greenland_P3.m +++ b/cresis-toolbox/gps/missions/gps_create_2018_Greenland_P3.m @@ -1,4 +1,4 @@ -% script make_gps_2018_greenland_P3 +% script gps_create_2018_greenland_P3 % % Makes the GPS files for 2018 Greenland P3 field season @@ -439,7 +439,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; for idx = 1:length(file_type) out_fn = fullfile(gps_path,out_fns{idx}); @@ -481,7 +481,7 @@ gps = load(out_fn); if strcmpi(gps.gps_source,'nmea-field') warning('Making monotonic gps time: %s', out_fn); - [gps,error_flag] = make_gps_monotonic(gps); + [gps,error_flag] = gps_force_monotonic(gps); save(out_fn,'-append','-struct','gps'); end end diff --git a/cresis-toolbox/gps/missions/make_gps_2018_greenland_Polar6.m b/cresis-toolbox/gps/missions/gps_create_2018_Greenland_Polar6.m similarity index 98% rename from cresis-toolbox/gps/missions/make_gps_2018_greenland_Polar6.m rename to cresis-toolbox/gps/missions/gps_create_2018_Greenland_Polar6.m index 07a44813..bcf3beb1 100644 --- a/cresis-toolbox/gps/missions/make_gps_2018_greenland_Polar6.m +++ b/cresis-toolbox/gps/missions/gps_create_2018_Greenland_Polar6.m @@ -1,4 +1,4 @@ -% script make_gps_2018_greenland_Polar6 +% script gps_create_2018_greenland_Polar6 % % Makes the GPS files for 2018 Greenland Polar6 field season @@ -104,7 +104,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; %% No GPS Data Available: Fakes GPS position information match_idx = strmatch('gps_20160331.mat',out_fns,'exact'); diff --git a/cresis-toolbox/gps/missions/make_gps_2019_alaska_SO.m b/cresis-toolbox/gps/missions/gps_create_2019_Alaska_SO.m similarity index 98% rename from cresis-toolbox/gps/missions/make_gps_2019_alaska_SO.m rename to cresis-toolbox/gps/missions/gps_create_2019_Alaska_SO.m index b7b5eb6c..67df2892 100644 --- a/cresis-toolbox/gps/missions/make_gps_2019_alaska_SO.m +++ b/cresis-toolbox/gps/missions/gps_create_2019_Alaska_SO.m @@ -1,4 +1,4 @@ -% script make_gps_2019_Alaska_SO +% script gps_create_2019_Alaska_SO % % Makes the GPS files for 2019_Alaska_SO field season @@ -55,7 +55,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; for idx = 1:length(file_type) out_fn = fullfile(gps_path,out_fns{idx}); diff --git a/cresis-toolbox/gps/missions/make_gps_2019_Antarctica_GV.m b/cresis-toolbox/gps/missions/gps_create_2019_Antarctica_GV.m similarity index 83% rename from cresis-toolbox/gps/missions/make_gps_2019_Antarctica_GV.m rename to cresis-toolbox/gps/missions/gps_create_2019_Antarctica_GV.m index 7e7c9532..f8f240e8 100644 --- a/cresis-toolbox/gps/missions/make_gps_2019_Antarctica_GV.m +++ b/cresis-toolbox/gps/missions/gps_create_2019_Antarctica_GV.m @@ -1,4 +1,4 @@ -% script make_gps_2019_Antarctica_GV +% script gps_create_2019_Antarctica_GV % % Makes the GPS files for 2019_Antarctica_GV field season @@ -35,9 +35,9 @@ file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; sync_flag = {}; sync_fns = {}; sync_file_type = {}; sync_params = {}; -gps_source_to_use = 'NMEA'; +% gps_source_to_use = 'NMEA'; % gps_source_to_use = 'ATM-field'; -% gps_source_to_use = 'ATM'; +gps_source_to_use = 'ATM'; if strcmpi(gps_source_to_use,'NMEA') %% NMEA @@ -86,14 +86,14 @@ in_fns_ins{file_idx} = get_filename(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'IWG1','17Oct2019',''); params_ins{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','headerlines',0,'format_str',... '%s%s%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f'); - params_ins{file_idx}.types = {'IWG1','date_time_sec','lat_deg','lon_deg','msl_alt_m','elev_m','press_alt_ft','radar_alt_ft',... - 'grnd_spd_mps','true_airspeed_mps','indicated_airspeed_knots','mach_number','ver_velocity_mps','heading_deg',... + params_ins{file_idx}.types = {'IWG1','date_time','lat_deg','lon_deg','elev_m','elev_m_not_used','press_alt_ft','radar_alt_ft',... + 'grnd_spd_mps','true_airspeed_mps','indicated_airspeed_knots','mach_number','vert_velocity_mps','heading_deg',... 'track_deg','drift_deg','pitch_deg','roll_deg','side_slip_deg','angle_of_attach_deg','ambient_temp_degc',... 'dew_tmp_degc','total_tmp_degc','static_press_mbar','dynamic_press_mbar','cabin_press_mbar','wind_speed_mps',... - 'wind_dir_deg','ver_wind_speed_mps','solar_zenith_deg','sun_ele_ac_deg','sun_az_grd_deg','sun_az_ac_deg'}; + 'wind_dir_deg','vert_wind_speed_mps','solar_zenith_deg','sun_ele_ac_deg','sun_az_grd_deg','sun_az_ac_deg'}; params_ins{file_idx}.textscan ={'delimiter',','}; out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - gps_source{file_idx} = 'atm-final_20190205'; + gps_source{file_idx} = 'nmea-field'; sync_flag{file_idx} = 0; % year = 2019; month = 10; day = 24; @@ -231,7 +231,7 @@ % fprintf('year = %d; month = %d; day = %d;\n', year, month, day); % end - ATM_fns = get_filenames(in_base_path,'','','PPPK*.out'); + ATM_fns = get_filenames(in_base_path,'','','BD982*.out'); fn_dates = []; for idx = 1:length(ATM_fns) fn = ATM_fns{idx}; @@ -244,12 +244,47 @@ [year,month,day] = datevec(fn_dates(idx)); fprintf('year = %d; month = %d; day = %d;\n', year, month, day); file_idx = file_idx + 1; - in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPPK*.out'); + if month == 10 & day == 17 + in_fns{file_idx} = {get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'.out'),... + get_filename(in_base_path,'IWG1','17Oct2019','')}; + else + in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'.out'); + end + % The local dates in spreadsheets are one day behind + if month == 10 & day == 31 + month = 11; + day = 1; + elseif ~(month == 10 & day == 17) + day = day + 1; + end out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - file_type{file_idx} = 'applanix'; - params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); - gps_source{file_idx} = 'atm-final_20190606'; - sync_flag{file_idx} = 0; + + % Change the day back + if month == 11 & day == 1 + month = 10; + day = 31; + elseif ~(month == 10 & day == 17) + day = day -1; + end + + if month == 10 & day == 17 + file_type{file_idx} = {'applanix','General_ASCII'}; + params{file_idx}{1} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); + params{file_idx}{2} = struct('year',year,'month',month,'day',day,'time_reference','utc','headerlines',0,'format_str',... + '%s%s%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f'); + params{file_idx}{2}.types = {'IWG1','date_time','lat_deg','lon_deg','elev_m','elev_m_not_used','press_alt_ft','radar_alt_ft',... + 'grnd_spd_mps','true_airspeed_mps','indicated_airspeed_knots','mach_number','vert_velocity_mps','heading_deg',... + 'track_deg','drift_deg','pitch_deg','roll_deg'}; + params{file_idx}{2}.textscan ={'delimiter',','}; + params{file_idx}{2}.date_time_format = 'yyyy-mm-ddTHH:MM:SS.FFF'; + gps_source{file_idx} = 'atm-final_20200110+IWG1'; + sync_flag{file_idx} = 0; + else + file_type{file_idx} = 'applanix'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); + gps_source{file_idx} = 'atm-final_20200110'; + sync_flag{file_idx} = 0; + end end end @@ -258,7 +293,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; for idx = 1:length(file_type) out_fn = fullfile(gps_path,out_fns{idx}); diff --git a/cresis-toolbox/gps/missions/make_gps_2019_Antarctica_Ground.m b/cresis-toolbox/gps/missions/gps_create_2019_Antarctica_Ground.m similarity index 90% rename from cresis-toolbox/gps/missions/make_gps_2019_Antarctica_Ground.m rename to cresis-toolbox/gps/missions/gps_create_2019_Antarctica_Ground.m index f02f6066..26ee578c 100644 --- a/cresis-toolbox/gps/missions/make_gps_2019_Antarctica_Ground.m +++ b/cresis-toolbox/gps/missions/gps_create_2019_Antarctica_Ground.m @@ -1,4 +1,4 @@ -% script make_gps_2019_Antarctica_Ground +% script gps_create_2019_Antarctica_Ground % % Makes the GPS files for 2019 Antarctica Ground field season @@ -184,24 +184,24 @@ % params{file_idx} = struct('time_reference','gps','headerlines',15,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); % params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; % params{file_idx}.textscan = {}; -% gps_source{file_idx} = 'cresis-final20200311'; +% gps_source{file_idx} = 'cresis-final20200505'; % sync_flag{file_idx} = 1; % sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); % sync_file_type{file_idx} = 'arena'; % sync_params{file_idx} = struct('time_reference','utc'); % - year = 2019; month = 12; day = 31; - file_idx = file_idx + 1; - in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'20191231','','gps.txt'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - file_type{file_idx} = 'arena'; - params{file_idx} = struct('time_reference','utc'); - gps_source{file_idx} = 'arena-field'; - sync_flag{file_idx} = 1; - sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'20191231','','gps.txt'); - sync_file_type{file_idx} = 'arena'; - sync_params{file_idx} = struct('time_reference','utc'); - +% year = 2019; month = 12; day = 31; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'20191231','','gps.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'arena'; +% params{file_idx} = struct('time_reference','utc'); +% gps_source{file_idx} = 'arena-field'; +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'20191231','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); +% % year = 2020; month = 1; day = 1; % file_idx = file_idx + 1; % in_fns{file_idx} = get_filenames(in_base_path,'',sprintf('%04d%02d%02d',year,month,day),'ver1.txt'); @@ -210,7 +210,7 @@ % params{file_idx} = struct('time_reference','gps','headerlines',15,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); % params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; % params{file_idx}.textscan = {}; -% gps_source{file_idx} = 'cresis-final20200311'; +% gps_source{file_idx} = 'cresis-final20200505'; % sync_flag{file_idx} = 1; % sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); % sync_file_type{file_idx} = 'arena'; @@ -224,7 +224,7 @@ % params{file_idx} = struct('time_reference','gps','headerlines',15,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); % params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; % params{file_idx}.textscan = {}; -% gps_source{file_idx} = 'cresis-final20200311'; +% gps_source{file_idx} = 'cresis-final20200505'; % sync_flag{file_idx} = 0; % sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); % sync_file_type{file_idx} = 'arena'; @@ -238,7 +238,7 @@ % params{file_idx} = struct('time_reference','gps','headerlines',15,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); % params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; % params{file_idx}.textscan = {}; -% gps_source{file_idx} = 'cresis-final20200311'; +% gps_source{file_idx} = 'cresis-final20200505'; % sync_flag{file_idx} = 1; % sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); % sync_file_type{file_idx} = 'arena'; @@ -252,7 +252,7 @@ % params{file_idx} = struct('time_reference','gps','headerlines',15,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); % params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; % params{file_idx}.textscan = {}; -% gps_source{file_idx} = 'cresis-final20200311'; +% gps_source{file_idx} = 'cresis-final20200505'; % sync_flag{file_idx} = 1; % sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); % sync_file_type{file_idx} = 'arena'; @@ -266,42 +266,46 @@ % params{file_idx} = struct('time_reference','gps','headerlines',15,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); % params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; % params{file_idx}.textscan = {}; -% gps_source{file_idx} = 'cresis-final20200311'; +% gps_source{file_idx} = 'cresis-final20200505'; % sync_flag{file_idx} = 1; % sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); % sync_file_type{file_idx} = 'arena'; % sync_params{file_idx} = struct('time_reference','utc'); % -% year = 2020; month = 1; day = 7; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'arena'; -% params{file_idx} = struct('time_reference','utc'); -% gps_source{file_idx} = 'arena-field'; -% sync_flag{file_idx} = 1; -% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); -% sync_file_type{file_idx} = 'arena'; -% sync_params{file_idx} = struct('time_reference','utc'); -% -% year = 2020; month = 1; day = 8; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'arena'; -% params{file_idx} = struct('time_reference','utc'); -% gps_source{file_idx} = 'arena-field'; -% sync_flag{file_idx} = 1; -% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); -% sync_file_type{file_idx} = 'arena'; -% sync_params{file_idx} = struct('time_reference','utc'); + year = 2020; month = 1; day = 7; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(in_base_path,'',sprintf('%04d%02d%02d',year,month,day),'ver1.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'General_ASCII'; + params{file_idx} = struct('time_reference','gps','headerlines',15,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); + params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; + params{file_idx}.textscan = {}; + gps_source{file_idx} = 'cresis-final20200505'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); + + year = 2020; month = 1; day = 8; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(in_base_path,'',sprintf('%04d%02d%02d',year,month,day),'ver1.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'General_ASCII'; + params{file_idx} = struct('time_reference','gps','headerlines',15,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); + params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; + params{file_idx}.textscan = {}; + gps_source{file_idx} = 'cresis-final20200505'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); end % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; for idx = 1:length(file_type) out_fn = fullfile(gps_path,out_fns{idx}); diff --git a/cresis-toolbox/gps/missions/make_gps_2019_antarctica_Polar6.m b/cresis-toolbox/gps/missions/gps_create_2019_Antarctica_Polar6.m similarity index 98% rename from cresis-toolbox/gps/missions/make_gps_2019_antarctica_Polar6.m rename to cresis-toolbox/gps/missions/gps_create_2019_Antarctica_Polar6.m index 95219a4d..75c3e066 100644 --- a/cresis-toolbox/gps/missions/make_gps_2019_antarctica_Polar6.m +++ b/cresis-toolbox/gps/missions/gps_create_2019_Antarctica_Polar6.m @@ -1,4 +1,4 @@ -% script make_gps_2019_antarctica_Polar6 +% script gps_create_2019_antarctica_Polar6 % % Makes the GPS files for 2019 Antarctic Polar6 field season @@ -103,7 +103,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; %% No GPS Data Available: Fakes GPS position information match_idx = strmatch('gps_20160331.mat',out_fns,'exact'); diff --git a/cresis-toolbox/gps/missions/gps_create_2019_Antarctica_TObas.m b/cresis-toolbox/gps/missions/gps_create_2019_Antarctica_TObas.m new file mode 100644 index 00000000..d04a1fed --- /dev/null +++ b/cresis-toolbox/gps/missions/gps_create_2019_Antarctica_TObas.m @@ -0,0 +1,537 @@ +% script gps_create_2019_Antarctica_TObas +% +% Makes the GPS files for 2019 Antarctica TObas field season + +%% Setup +% ========================================================================= +tic; + +global gRadar; + +support_path = ''; +data_support_path = ''; + +if isempty(support_path) + support_path = gRadar.support_path; +end + +gps_path = fullfile(support_path,'gps','2019_Antarctica_TObas'); +if ~exist(gps_path,'dir') + fprintf('Making directory %s\n', gps_path); + fprintf(' Press a key to proceed\n'); + pause; + mkdir(gps_path); +end + +if isempty(data_support_path) + data_support_path = gRadar.data_support_path; +end + +in_base_path = fullfile(data_support_path,'2019_Antarctica_TObas'); + +file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; +sync_flag = {}; sync_fns = {}; sync_file_type = {}; sync_params = {}; + +%% <== CHOOSE WHICH GPS SOURCE TO PROCESS +% gps_source_to_use = 'arena'; +gps_source_to_use = 'BAS'; + +if strcmpi(gps_source_to_use,'arena') + %% ARENA GPS SOURCE + % ======================================================================= + + year = 2019; month = 9; day = 23; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'arena'; + params{file_idx} = struct('year',2019,'time_reference','utc'); + gps_source{file_idx} = 'arena-field'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); + + year = 2019; month = 12; day = 13; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'arena'; + params{file_idx} = struct('year',2019,'time_reference','utc'); + gps_source{file_idx} = 'arena-field'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); + +% year = 2019; month = 12; day = 15; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'arena'; +% params{file_idx} = struct('year',2019,'time_reference','utc'); +% gps_source{file_idx} = 'arena-field'; +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); +% +% year = 2019; month = 12; day = 22; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'arena'; +% params{file_idx} = struct('year',2019,'time_reference','utc'); +% gps_source{file_idx} = 'arena-field'; +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); + +% year = 2019; month = 12; day = 25; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'arena'; +% params{file_idx} = struct('year',2019,'time_reference','utc'); +% gps_source{file_idx} = 'arena-field'; +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); +% +% year = 2019; month = 12; day = 26; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'arena'; +% params{file_idx} = struct('year',2019,'time_reference','utc'); +% gps_source{file_idx} = 'arena-field'; +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); + +% year = 2019; month = 12; day = 29; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'arena'; +% params{file_idx} = struct('year',2019,'time_reference','utc'); +% gps_source{file_idx} = 'arena-field'; +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); +% +% year = 2019; month = 12; day = 30; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'arena'; +% params{file_idx} = struct('year',2019,'time_reference','utc'); +% gps_source{file_idx} = 'arena-field'; +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); +% +% year = 2020; month = 1; day = 25; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'arena'; +% params{file_idx} = struct('year',2020,'time_reference','utc'); +% gps_source{file_idx} = 'arena-field'; +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); + +% year = 2020; month = 1; day = 26; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'arena'; +% params{file_idx} = struct('year',2020,'time_reference','utc'); +% gps_source{file_idx} = 'arena-field'; +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); +% +% year = 2020; month = 1; day = 27; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'arena'; +% params{file_idx} = struct('year',2020,'time_reference','utc'); +% gps_source{file_idx} = 'arena-field'; +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); +% +% year = 2020; month = 1; day = 28; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'arena'; +% params{file_idx} = struct('year',2020,'time_reference','utc'); +% gps_source{file_idx} = 'arena-field'; +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); + +elseif strcmpi(gps_source_to_use,'bas') + %% BAS GPS SOURCE + % ======================================================================= + + % IMU worked fine + year = 2019; month = 12; day = 15; + file_idx = file_idx + 1; + in_fns{file_idx} = {fullfile(in_base_path,'GNSS_IMU','T01.txt')}; + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'General_ASCII'; + params{file_idx} = struct('time_reference','gps'); + params{file_idx}.format_str = '%s%s%f%f%f%f%f%f%f%f%f%f%f%f%f'; + params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg',... + 'elev_m','roll_deg','pitch_deg','heading_deg','f1','f2','f3','f4',... + 'f5','f6','f7'}; + params{file_idx}.textscan = {}; + params{file_idx}.headerlines = 34; + gps_source{file_idx} = 'bas-final20200306'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); + + % IMU worked fine + year = 2019; month = 12; day = 22; + file_idx = file_idx + 1; + in_fns{file_idx} = {fullfile(in_base_path,'GNSS_IMU','T03.txt')}; + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'General_ASCII'; + params{file_idx} = struct('time_reference','gps'); + params{file_idx}.format_str = '%s%s%f%f%f%f%f%f%f%f%f%f%f%f%f'; + params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg',... + 'elev_m','roll_deg','pitch_deg','heading_deg','f1','f2','f3','f4',... + 'f5','f6','f7'}; + params{file_idx}.textscan = {}; + params{file_idx}.headerlines = 34; + gps_source{file_idx} = 'bas-final20200306'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); + + % IMU worked fine for the first part of flight, processed to IMU + year = 2019; month = 12; day = 25; + file_idx = file_idx + 1; + in_fns{file_idx} = {fullfile(in_base_path,'GNSS_IMU','T06_IMU.txt')}; + out_fns{file_idx} = sprintf('gps_%04d%02d%02d_imu.mat', year, month, day); + file_type{file_idx} = 'General_ASCII'; + params{file_idx} = struct('time_reference','gps'); + params{file_idx}.format_str = '%s%s%f%f%f%f%f%f%f%f%f%f%f%f%f'; + params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg',... + 'elev_m','roll_deg','pitch_deg','heading_deg','f1','f2','f3','f4',... + 'f5','f6','f7'}; + params{file_idx}.textscan = {}; + params{file_idx}.headerlines = 34; + gps_source{file_idx} = 'bas-final20200306'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); + + % IMU missing for latter part of flight, processed to GPS antenna + year = 2019; month = 12; day = 25; + file_idx = file_idx + 1; + in_fns{file_idx} = {fullfile(in_base_path,'GNSS_IMU','T06_GNSS.txt')}; + out_fns{file_idx} = sprintf('gps_%04d%02d%02d_gnss.mat', year, month, day); + file_type{file_idx} = 'General_ASCII'; + params{file_idx} = struct('time_reference','gps'); + params{file_idx}.format_str = '%s%s%f%f%f%f%f%f%f%f%f%f%f%f%f'; + params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg',... + 'elev_m','roll_deg','pitch_deg','heading_deg','f1','f2','f3','f4',... + 'f5','f6','f7'}; + params{file_idx}.textscan = {}; + params{file_idx}.headerlines = 33; + gps_source{file_idx} = 'bas_gnss-final20200306'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); + + % IMU worked fine + year = 2019; month = 12; day = 26; + file_idx = file_idx + 1; + in_fns{file_idx} = {fullfile(in_base_path,'GNSS_IMU','T07.txt')}; + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'General_ASCII'; + params{file_idx} = struct('time_reference','gps'); + params{file_idx}.format_str = '%s%s%f%f%f%f%f%f%f%f%f%f%f%f%f'; + params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg',... + 'elev_m','roll_deg','pitch_deg','heading_deg','f1','f2','f3','f4',... + 'f5','f6','f7'}; + params{file_idx}.textscan = {}; + params{file_idx}.headerlines = 34; + gps_source{file_idx} = 'bas-final20200306'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); + + % IMU worked fine (T14 and T15) + year = 2019; month = 12; day = 29; + file_idx = file_idx + 1; + in_fns{file_idx} = {fullfile(in_base_path,'GNSS_IMU','T14.txt')}; + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'General_ASCII'; + params{file_idx} = struct('time_reference','gps'); + params{file_idx}.format_str = '%s%s%f%f%f%f%f%f%f%f%f%f%f%f%f'; + params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg',... + 'elev_m','roll_deg','pitch_deg','heading_deg','f1','f2','f3','f4',... + 'f5','f6','f7'}; + params{file_idx}.textscan = {}; + params{file_idx}.headerlines = 34; + gps_source{file_idx} = 'bas-final20200306'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); + + % IMU worked fine (T16 and T17) + year = 2019; month = 12; day = 30; + file_idx = file_idx + 1; + in_fns{file_idx} = {fullfile(in_base_path,'GNSS_IMU','T16.txt')}; + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'General_ASCII'; + params{file_idx} = struct('time_reference','gps'); + params{file_idx}.format_str = '%s%s%f%f%f%f%f%f%f%f%f%f%f%f%f'; + params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg',... + 'elev_m','roll_deg','pitch_deg','heading_deg','f1','f2','f3','f4',... + 'f5','f6','f7'}; + params{file_idx}.textscan = {}; + params{file_idx}.headerlines = 34; + gps_source{file_idx} = 'bas-final20200306'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); + + % No IMU data, processed to GPS antenna + year = 2020; month = 1; day = 25; + file_idx = file_idx + 1; + in_fns{file_idx} = {fullfile(in_base_path,'GNSS_IMU','T19.txt')}; + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'General_ASCII'; + params{file_idx} = struct('time_reference','gps'); + params{file_idx}.format_str = '%s%s%f%f%f%f%f%f%f%f%f%f%f%f%f'; + params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg',... + 'elev_m','roll_deg','pitch_deg','heading_deg','f1','f2','f3','f4',... + 'f5','f6','f7'}; + params{file_idx}.textscan = {}; + params{file_idx}.headerlines = 33; + gps_source{file_idx} = 'bas_gnss-final20200306'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); + + % No IMU data, processed to GPS antenna + year = 2020; month = 1; day = 26; + file_idx = file_idx + 1; + in_fns{file_idx} = {fullfile(in_base_path,'GNSS_IMU','T20.txt')}; + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'General_ASCII'; + params{file_idx} = struct('time_reference','gps'); + params{file_idx}.format_str = '%s%s%f%f%f%f%f%f%f%f%f%f%f%f%f'; + params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg',... + 'elev_m','roll_deg','pitch_deg','heading_deg','f1','f2','f3','f4',... + 'f5','f6','f7'}; + params{file_idx}.textscan = {}; + params{file_idx}.headerlines = 33; + gps_source{file_idx} = 'bas_gnss-final20200306'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); + + % No IMU data, processed to GPS antenna + year = 2020; month = 1; day = 27; + file_idx = file_idx + 1; + in_fns{file_idx} = {fullfile(in_base_path,'GNSS_IMU','T21.txt')}; + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'General_ASCII'; + params{file_idx} = struct('time_reference','gps'); + params{file_idx}.format_str = '%s%s%f%f%f%f%f%f%f%f%f%f%f%f%f'; + params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg',... + 'elev_m','roll_deg','pitch_deg','heading_deg','f1','f2','f3','f4',... + 'f5','f6','f7'}; + params{file_idx}.textscan = {}; + params{file_idx}.headerlines = 33; + gps_source{file_idx} = 'bas_gnss-final20200306'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); + + % No IMU data, processed to GPS antenna + year = 2020; month = 1; day = 28; + file_idx = file_idx + 1; + in_fns{file_idx} = {fullfile(in_base_path,'GNSS_IMU','T22.txt')}; + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'General_ASCII'; + params{file_idx} = struct('time_reference','gps'); + params{file_idx}.format_str = '%s%s%f%f%f%f%f%f%f%f%f%f%f%f%f'; + params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg',... + 'elev_m','roll_deg','pitch_deg','heading_deg','f1','f2','f3','f4',... + 'f5','f6','f7'}; + params{file_idx}.textscan = {}; + params{file_idx}.headerlines = 33; + gps_source{file_idx} = 'bas_gnss-final20200306'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); + + +end + +%% gps_create +% Read and translate files according to user settings +% ========================================================================= +gps_create; + +%% custom fixes +% ========================================================================= +for idx = 1:length(file_type) + out_fn = fullfile(gps_path,out_fns{idx}); + + load(out_fn,'gps_source'); + if ~isempty(regexpi(gps_source,'arena')) + % Extrapolation is necessary because GPS data starts after/stops before + % the beginning/end of the radar data. + warning('Extrapolating and filtering elevation for arena GPS data: %s', out_fn); + gps = load(out_fn); + + if length(gps.lat) >= 2 + new_gps_time = [gps.gps_time(1)-10, gps.gps_time,gps.gps_time(end)+10]; + gps.lat = interp1(gps.gps_time,gps.lat,new_gps_time,'linear','extrap'); + gps.lon = interp1(gps.gps_time,gps.lon,new_gps_time,'linear','extrap'); + gps.elev = interp1(gps.gps_time,gps.elev,new_gps_time,'linear','extrap'); + gps.roll = interp1(gps.gps_time,gps.roll,new_gps_time,'linear','extrap'); + gps.pitch = interp1(gps.gps_time,gps.pitch,new_gps_time,'linear','extrap'); + gps.heading = interp1(gps.gps_time,gps.heading,new_gps_time,'linear','extrap'); + gps.gps_time = new_gps_time; + + gps.elev = fir_dec(gps.elev,ones(1,101)/101,1); + + save(out_fn,'-append','-struct','gps','gps_time','lat','lon','elev','roll','pitch','heading'); + end + end + + if ~isempty(regexpi(out_fn,'20191225')) && ~isempty(regexpi(gps_source,'bas-')) + % Merge of IMU and GNSS data + warning('Merging IMU and GNSS GPS data: %s', out_fn); + + gps_imu = load(out_fn); + out_fn_dir = fileparts(out_fn); + gps_gnss = load(fullfile(out_fn_dir,'gps_20191225_gnss.mat')); + + % Apply lever arm to IMU GPS data to take it to GPS antenna + trajectory_param = struct('gps_source','bas_imu_to_gps-final', ... + 'season_name','2019_Antarctica_TObas','radar_name','accum','rx_path', 0, ... + 'tx_weights', [], 'lever_arm_fh', @lever_arm); + gps_gnss_imu = trajectory_with_leverarm(gps_imu,trajectory_param); + + gnss_only_start_idx = find(gps_gnss.gps_time > gps_imu.gps_time(end),1); + gps = gps_gnss; + gps.gps_time = [gps_imu.gps_time gps_gnss.gps_time(gnss_only_start_idx:end)]; + gps.lat = [gps_gnss_imu.lat gps_gnss.lat(gnss_only_start_idx:end)]; + gps.lon = [gps_gnss_imu.lon gps_gnss.lon(gnss_only_start_idx:end)]; + gps.elev = [gps_gnss_imu.elev gps_gnss.elev(gnss_only_start_idx:end)]; + gps.roll = [gps_gnss_imu.roll gps_gnss.roll(gnss_only_start_idx:end)]; + gps.pitch = [gps_gnss_imu.pitch gps_gnss.pitch(gnss_only_start_idx:end)]; + gps.heading = [gps_gnss_imu.heading gps_gnss.heading(gnss_only_start_idx:end)]; + + if 0 + % Debug + figure(1); clf; + plot(gps_gnss.gps_time, gps_gnss.elev); % Original GNSS only + hold on; + plot(gps.gps_time, gps.elev); % Combined + plot(gps_gnss_imu.gps_time, gps_gnss_imu.elev,'--'); % IMU only + xlabel('GPS time'); + ylabel('Elevation (m)'); + legend('GNSS-only','Combined','IMU+GNSS'); + + figure(2); clf; + plot(gps_gnss.gps_time, gps_gnss.heading*180/pi); % Original GNSS only + hold on; + plot(gps.gps_time, gps.heading*180/pi); % Combined + plot(gps_gnss_imu.gps_time, gps_gnss_imu.heading*180/pi,'--'); % IMU only + xlabel('GPS time'); + ylabel('Heading (deg)'); + legend('GNSS-only','Combined','IMU+GNSS'); + + end; + + out_fn = fullfile(out_fn_dir,'gps_20191225.mat'); + ct_save(out_fn,'-struct','gps'); + end + + if ~isempty(regexpi(gps_source,'bas')) && isempty(regexpi(out_fn,'gps_20191225_gnss')) + % gps_20191225_gnss is skipped because gps_20191225 is taken care of + % when gps_20191225_imu is produced. + % + % BAS GPS data are referenced to: + % # ITRF2014(2020.07117) + % # Ellipsoid: GRS80 (a=6378137.0000, f=1/298.25722210) + % Convert to WGS84... the effect seems to be very very small (0.1 mm difference between the ellipsoids) + + warning('Converting from GRS80 ellipsoid to WGS84 ellipsoid: %s', out_fn); + + physical_constants; + gps = load(out_fn); + old_gps = gps; + [x,y,z] = geodetic2ecef(gps.lat/180*pi, gps.lon/180*pi, gps.elev, GRS80.ellipsoid); + [gps.lat,gps.lon,gps.elev] = ecef2geodetic(x, y, z, WGS84.ellipsoid); + gps.lat = gps.lat*180/pi; + gps.lon = gps.lon*180/pi; + + if 0 + figure(1); clf; + plot(gps.lon, gps.lat); + hold on; + plot(old_gps.lon, old_gps.lat,'--'); + grid on; + + figure(2); clf; + plot(gps.elev); + hold on; + plot(old_gps.elev,'--'); + grid on; + end + + save(out_fn,'-append','-struct','gps','lat','lon','elev'); + + end + + if ~isempty(regexpi(out_fn,'201910XX')) + % Fake GPS for testing + warning('Faking GPS data: %s', out_fn); + gps = load(out_fn); + + velocity = 4; + gps.lat = -75.5 - (gps.gps_time-gps.gps_time(1))*velocity/111111; + gps.lon(:) = -106.75; + gps.elev(:) = 500; + gps.heading(:) = -pi; + + save(out_fn,'-append','-struct','gps','gps_time','lat','lon','elev','roll','pitch','heading'); + end + +end diff --git a/cresis-toolbox/gps/missions/make_gps_2019_arctic_GV.m b/cresis-toolbox/gps/missions/gps_create_2019_Arctic_GV.m similarity index 98% rename from cresis-toolbox/gps/missions/make_gps_2019_arctic_GV.m rename to cresis-toolbox/gps/missions/gps_create_2019_Arctic_GV.m index 7b2b0372..db646960 100644 --- a/cresis-toolbox/gps/missions/make_gps_2019_arctic_GV.m +++ b/cresis-toolbox/gps/missions/gps_create_2019_Arctic_GV.m @@ -1,4 +1,4 @@ -% script make_gps_2019_Arctic_GV +% script gps_create_2019_Arctic_GV % % Makes the GPS files for 2019_Arctic_GV field season @@ -171,11 +171,11 @@ [year,month,day] = datevec(fn_dates(idx)); fprintf('year = %d; month = %d; day = %d;\n', year, month, day); file_idx = file_idx + 1; - in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPRTX*.out'); + in_fns{file_idx} = get_filename(in_base_path,'BD982_',datestr(datenum(year,month,day),'ddmmmyy'),'PPRTX*eth.out'); out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); file_type{file_idx} = 'applanix'; params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); - gps_source{file_idx} = 'atm-final_20190606'; + gps_source{file_idx} = 'atm-final_20200103'; sync_flag{file_idx} = 0; end @@ -185,7 +185,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; for idx = 1:length(file_type) out_fn = fullfile(gps_path,out_fns{idx}); diff --git a/cresis-toolbox/gps/missions/make_gps_2019_greenland_P3.m b/cresis-toolbox/gps/missions/gps_create_2019_Greenland_P3.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2019_greenland_P3.m rename to cresis-toolbox/gps/missions/gps_create_2019_Greenland_P3.m index a621255b..c5fb24aa 100644 --- a/cresis-toolbox/gps/missions/make_gps_2019_greenland_P3.m +++ b/cresis-toolbox/gps/missions/gps_create_2019_Greenland_P3.m @@ -1,4 +1,4 @@ -% script make_gps_2019_Greenland_P3 +% script gps_create_2019_Greenland_P3 % % Makes the GPS files for 2019_Greenland_P3 field season @@ -486,7 +486,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; for idx = 1:length(file_type) out_fn = fullfile(gps_path,out_fns{idx}); diff --git a/cresis-toolbox/gps/missions/make_gps_2019_greenland_TO.m b/cresis-toolbox/gps/missions/gps_create_2019_Greenland_TO.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2019_greenland_TO.m rename to cresis-toolbox/gps/missions/gps_create_2019_Greenland_TO.m index 9176300a..bcd9befd 100644 --- a/cresis-toolbox/gps/missions/make_gps_2019_greenland_TO.m +++ b/cresis-toolbox/gps/missions/gps_create_2019_Greenland_TO.m @@ -1,4 +1,4 @@ -% script make_gps_2019_greenland_TO +% script gps_create_2019_greenland_TO % % Makes the GPS files for 2019_Greenland_TO field season @@ -147,7 +147,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; for idx = 1:length(file_type) out_fn = fullfile(gps_path,out_fns{idx}); diff --git a/cresis-toolbox/gps/missions/gps_create_2019_SouthDakota_N1KU.m b/cresis-toolbox/gps/missions/gps_create_2019_SouthDakota_N1KU.m new file mode 100644 index 00000000..18810425 --- /dev/null +++ b/cresis-toolbox/gps/missions/gps_create_2019_SouthDakota_N1KU.m @@ -0,0 +1,217 @@ +% script gps_create_2019_SouthDakota_N1KU +% +% Makes the GPS files for 2019_South_Dakota_N1KU field season + +tic; + +global gRadar; + +support_path = ''; +data_support_path = ''; + +if isempty(support_path) + support_path = gRadar.support_path; +end + +gps_path = fullfile(support_path,'gps','2019_SouthDakota_N1KU'); +if ~exist(gps_path,'dir') + fprintf('Making directory %s\n', gps_path); + fprintf(' Press a key to proceed\n'); + pause; + mkdir(gps_path); +end + +if isempty(data_support_path) + data_support_path = gRadar.data_support_path; +end + +% ====================================================================== +% User Settings +% ====================================================================== +debug_level = 1; + +in_base_path = fullfile(data_support_path,'2019_SouthDakota_N1KU'); + +file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; +sync_fns = {}; sync_params = {}; + +% gps_source_to_use = 'NMEA'; +gps_source_to_use = 'novatel'; + +if strcmpi(gps_source_to_use,'NMEA') + + year = 2020; month = 01; day = 29; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'nmea','','.gps'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'NMEA'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); + gps_source{file_idx} = 'nmea-field'; + sync_flag{file_idx} = 0; + + year = 2020; month = 01; day = 28; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'nmea','','.gps'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'NMEA'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); + gps_source{file_idx} = 'nmea-field'; + sync_flag{file_idx} = 0; + +% year = 2020; month = 01; day = 16; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'nmea','','.gps'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; + +% year = 2019; month = 12; day = 11 ; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'nmea','','.gps'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; + +elseif strcmpi(gps_source_to_use,'novatel') + %% NOVATEL + + year = 2020; month = 1; day = 28; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(in_base_path,'',sprintf('%04d%02d%02d',year,month,day),'ver2.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'General_ASCII'; + params{file_idx} = struct('time_reference','gps','headerlines',21,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); + params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; + params{file_idx}.textscan = {}; + gps_source{file_idx} = 'cresis-final20200601'; + sync_flag{file_idx} = 0; + + year = 2020; month = 1; day = 29; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(in_base_path,'',sprintf('%04d%02d%02d',year,month,day),'ver2.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'General_ASCII'; + params{file_idx} = struct('time_reference','gps','headerlines',21,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); + params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; + params{file_idx}.textscan = {}; + gps_source{file_idx} = 'cresis-final20200601'; + sync_flag{file_idx} = 0; + + year = 2020; month = 1; day = 31; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(in_base_path,'',sprintf('%04d%02d%02d',year,month,day),'ver2.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'General_ASCII'; + params{file_idx} = struct('time_reference','gps','headerlines',21,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); + params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; + params{file_idx}.textscan = {}; + gps_source{file_idx} = 'cresis-final20200601'; + sync_flag{file_idx} = 0; + + year = 2020; month = 2; day = 1; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(in_base_path,'',sprintf('%04d%02d%02d',year,month,day),'ver2.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'General_ASCII'; + params{file_idx} = struct('time_reference','gps','headerlines',21,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); + params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; + params{file_idx}.textscan = {}; + gps_source{file_idx} = 'cresis-final20200601'; + sync_flag{file_idx} = 0; + + year = 2020; month = 2; day = 2; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(in_base_path,'',sprintf('%04d%02d%02d',year,month,day),'ver2.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'General_ASCII'; + params{file_idx} = struct('time_reference','gps','headerlines',21,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); + params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; + params{file_idx}.textscan = {}; + gps_source{file_idx} = 'cresis-final20200601'; + sync_flag{file_idx} = 0; + + year = 2020; month = 2; day = 4; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(in_base_path,'',sprintf('%04d%02d%02d',year,month,day),'ver2.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'General_ASCII'; + params{file_idx} = struct('time_reference','gps','headerlines',21,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); + params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; + params{file_idx}.textscan = {}; + gps_source{file_idx} = 'cresis-final20200601'; + sync_flag{file_idx} = 0; + + year = 2020; month = 2; day = 5; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(in_base_path,'',sprintf('%04d%02d%02d',year,month,day),'ver2.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'General_ASCII'; + params{file_idx} = struct('time_reference','gps','headerlines',21,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); + params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; + params{file_idx}.textscan = {}; + gps_source{file_idx} = 'cresis-final20200601'; + sync_flag{file_idx} = 0; + + year = 2020; month = 2; day = 8; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(in_base_path,'',sprintf('%04d%02d%02d',year,month,day),'ver2.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'General_ASCII'; + params{file_idx} = struct('time_reference','gps','headerlines',21,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); + params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; + params{file_idx}.textscan = {}; + gps_source{file_idx} = 'cresis-final20200601'; + sync_flag{file_idx} = 0; + + year = 2020; month = 2; day = 9; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(in_base_path,'',sprintf('%04d%02d%02d',year,month,day),'ver2.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'General_ASCII'; + params{file_idx} = struct('time_reference','gps','headerlines',21,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); + params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; + params{file_idx}.textscan = {}; + gps_source{file_idx} = 'cresis-final20200601'; + sync_flag{file_idx} = 0; + + year = 2020; month = 2; day = 10; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(in_base_path,'',sprintf('%04d%02d%02d',year,month,day),'ver2.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'General_ASCII'; + params{file_idx} = struct('time_reference','gps','headerlines',21,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); + params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; + params{file_idx}.textscan = {}; + gps_source{file_idx} = 'cresis-final20200601'; + sync_flag{file_idx} = 0; + +end + +% ====================================================================== +% Read and translate files according to user settings +% ====================================================================== +gps_create; + +for idx = 1:length(file_type) + out_fn = fullfile(gps_path,out_fns{idx}); + + gps = load(out_fn); + if regexpi(gps.gps_source,'nmea') + warning('Making monotonic gps time: %s', out_fn); + [gps,error_flag] = gps_force_monotonic(gps); + + warning('Smoothing elevation and heading data: %s', out_fn); + gps.elev = sgolayfilt(gps.elev,2,101); % Adjust filter length as needed to remove high frequency noise + heading_x = cos(gps.heading); + heading_y = sin(gps.heading); + heading_x = sgolayfilt(heading_x,2,101); % Adjust filter length as needed to remove high frequency noise + heading_y = sgolayfilt(heading_y,2,101); % Adjust filter length as needed to remove high frequency noise + gps.heading = atan2(heading_y,heading_x); + + save(out_fn,'-append','-struct','gps','elev','heading'); + end +end diff --git a/cresis-toolbox/gps/missions/make_gps_2019_arctic_Polar6.m b/cresis-toolbox/gps/missions/gps_create_2019_arctic_Polar6.m similarity index 99% rename from cresis-toolbox/gps/missions/make_gps_2019_arctic_Polar6.m rename to cresis-toolbox/gps/missions/gps_create_2019_arctic_Polar6.m index 3cfbfae0..b460a398 100644 --- a/cresis-toolbox/gps/missions/make_gps_2019_arctic_Polar6.m +++ b/cresis-toolbox/gps/missions/gps_create_2019_arctic_Polar6.m @@ -1,4 +1,4 @@ -% script make_gps_2019_arctic_Polar6 +% script gps_create_2019_arctic_Polar6 % % Makes the GPS files for 2019_Arctic_Polar6 field season @@ -212,7 +212,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; %% No GPS Data Available: Fakes GPS position information match_idx = strmatch('gps_20160331.mat',out_fns,'exact'); diff --git a/cresis-toolbox/gps/missions/make_gps_2020_arctic_Polar6.m b/cresis-toolbox/gps/missions/gps_create_2020_Arctic_Polar6.m similarity index 97% rename from cresis-toolbox/gps/missions/make_gps_2020_arctic_Polar6.m rename to cresis-toolbox/gps/missions/gps_create_2020_Arctic_Polar6.m index 965eb495..157bdd6c 100644 --- a/cresis-toolbox/gps/missions/make_gps_2020_arctic_Polar6.m +++ b/cresis-toolbox/gps/missions/gps_create_2020_Arctic_Polar6.m @@ -1,4 +1,4 @@ -% script make_gps_2020_arctic_Polar6 +% script gps_create_2020_arctic_Polar6 % % Makes the GPS files for 2020_Arctic_Polar6 field season @@ -56,7 +56,7 @@ % ====================================================================== % Read and translate files according to user settings % ====================================================================== -gps_make; +gps_create; %% No GPS Data Available: Fakes GPS position information match_idx = strmatch('gps_20160331.mat',out_fns,'exact'); diff --git a/cresis-toolbox/gps/missions/gps_create_2020_SouthDakota_N1KU.m b/cresis-toolbox/gps/missions/gps_create_2020_SouthDakota_N1KU.m new file mode 100644 index 00000000..4bb37408 --- /dev/null +++ b/cresis-toolbox/gps/missions/gps_create_2020_SouthDakota_N1KU.m @@ -0,0 +1,363 @@ +% script gps_create_2020_SouthDakota_N1KU +% +% Makes the GPS files for 2020_South_Dakota_N1KU field season + +%% Set Season Information +% ====================================================================== +season_name = '2020_SouthDakota_N1KU'; + +%% Construct default paths/make output directory +% ====================================================================== + +global gRadar; + +support_path = ''; +data_support_path = ''; + +if isempty(support_path) + support_path = gRadar.support_path; +end + +gps_path = fullfile(support_path,'gps',season_name); +if ~exist(gps_path,'dir') + fprintf('Making directory %s\n', gps_path); + fprintf(' Press a key to proceed\n'); + pause; + mkdir(gps_path); +end + +if isempty(data_support_path) + data_support_path = gRadar.data_support_path; +end + +%% Set GPS File Information +% ====================================================================== +debug_level = 1; + +in_base_path = fullfile(data_support_path,season_name); + +file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; +sync_fns = {}; sync_params = {}; + +gps_source_to_use = 'NMEA'; +% gps_source_to_use = 'novatel'; + +if strcmpi(gps_source_to_use,'NMEA') + %% NMEA + % ====================================================================== + +% year = 2021; month = 02; day = 04; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); +% +% year = 2021; month = 02; day = 05; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + +% year = 2021; month = 02; day = 09; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + +% year = 2021; month = 02; day = 16; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + +% year = 2021; month = 02; day = 18; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + +% year = 2021; month = 02; day = 19; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + +% year = 2021; month = 02; day = 20; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + +% year = 2021; month = 02; day = 24; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + +% year = 2021; month = 02; day = 25; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + +% year = 2021; month = 02; day = 26; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + +% year = 2021; month = 02; day = 28; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); +% +% year = 2021; month = 03; day = 01; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); +% +% year = 2021; month = 03; day = 02; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + + year = 2021; month = 03; day = 28; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'GPS','','.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'NMEA'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); + gps_source{file_idx} = 'nmea-field'; + sync_flag{file_idx} = 0; + date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + +elseif strcmpi(gps_source_to_use,'novatel') + %% NOVATEL + % ====================================================================== + +% year = 2021; month = 02; day = 05; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(in_base_path,'',sprintf('%04d%02d%02d',year,month,day),'ver1.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'General_ASCII'; +% params{file_idx} = struct('time_reference','gps','headerlines',21,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); +% params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; +% params{file_idx}.textscan = {}; +% gps_source{file_idx} = 'cresis-final20210323'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); +% +% year = 2021; month = 02; day = 09; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(in_base_path,'',sprintf('%04d%02d%02d',year,month,day),'ver1.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'General_ASCII'; +% params{file_idx} = struct('time_reference','gps','headerlines',21,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); +% params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; +% params{file_idx}.textscan = {}; +% gps_source{file_idx} = 'cresis-final20210323'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); +% +% year = 2021; month = 02; day = 16; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(in_base_path,'',sprintf('%04d%02d%02d',year,month,day),'ver1.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'General_ASCII'; +% params{file_idx} = struct('time_reference','gps','headerlines',21,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); +% params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; +% params{file_idx}.textscan = {}; +% gps_source{file_idx} = 'cresis-final20210323'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); +% +% year = 2021; month = 02; day = 18; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(in_base_path,'',sprintf('%04d%02d%02d',year,month,day),'ver1.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'General_ASCII'; +% params{file_idx} = struct('time_reference','gps','headerlines',21,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); +% params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; +% params{file_idx}.textscan = {}; +% gps_source{file_idx} = 'cresis-final20210323'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + +% year = 2021; month = 02; day = 19; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(in_base_path,'',sprintf('%04d%02d%02d',year,month,day),'ver1.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'General_ASCII'; +% params{file_idx} = struct('time_reference','gps','headerlines',21,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); +% params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; +% params{file_idx}.textscan = {}; +% gps_source{file_idx} = 'cresis-final20210323'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + +% year = 2021; month = 02; day = 20; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(in_base_path,'',sprintf('%04d%02d%02d',year,month,day),'ver1.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'General_ASCII'; +% params{file_idx} = struct('time_reference','gps','headerlines',21,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); +% params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; +% params{file_idx}.textscan = {}; +% gps_source{file_idx} = 'cresis-final20210323'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); +% +% year = 2021; month = 02; day = 24; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(in_base_path,'',sprintf('%04d%02d%02d',year,month,day),'ver1.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'General_ASCII'; +% params{file_idx} = struct('time_reference','gps','headerlines',21,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); +% params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; +% params{file_idx}.textscan = {}; +% gps_source{file_idx} = 'cresis-final20210323'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + + year = 2021; month = 02; day = 25; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(in_base_path,'',sprintf('%04d%02d%02d',year,month,day),'ver1.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'General_ASCII'; + params{file_idx} = struct('time_reference','gps','headerlines',21,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); + params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; + params{file_idx}.textscan = {}; + gps_source{file_idx} = 'cresis-final20210323'; + sync_flag{file_idx} = 0; + date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + +% year = 2021; month = 02; day = 26; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(in_base_path,'',sprintf('%04d%02d%02d',year,month,day),'ver1.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'General_ASCII'; +% params{file_idx} = struct('time_reference','gps','headerlines',21,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); +% params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; +% params{file_idx}.textscan = {}; +% gps_source{file_idx} = 'cresis-final20210323'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); +% +% year = 2021; month = 02; day = 28; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(in_base_path,'',sprintf('%04d%02d%02d',year,month,day),'ver1.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'General_ASCII'; +% params{file_idx} = struct('time_reference','gps','headerlines',21,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); +% params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; +% params{file_idx}.textscan = {}; +% gps_source{file_idx} = 'cresis-final20210323'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); +% +% year = 2021; month = 03; day = 1; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(in_base_path,'',sprintf('%04d%02d%02d',year,month,day),'ver1.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'General_ASCII'; +% params{file_idx} = struct('time_reference','gps','headerlines',21,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); +% params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; +% params{file_idx}.textscan = {}; +% gps_source{file_idx} = 'cresis-final20210323'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); +% +% year = 2021; month = 03; day = 2; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(in_base_path,'',sprintf('%04d%02d%02d',year,month,day),'ver1.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'General_ASCII'; +% params{file_idx} = struct('time_reference','gps','headerlines',21,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); +% params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; +% params{file_idx}.textscan = {}; +% gps_source{file_idx} = 'cresis-final20210323'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + +end + +%% Make GPS files +% ====================================================================== +gps_create; + +%% Manual fixes to GPS files +% ====================================================================== +for idx = 1:length(file_type) + out_fn = fullfile(gps_path,out_fns{idx}); + + gps = load(out_fn); + if regexpi(gps.gps_source,'nmea') + warning('Making monotonic gps time: %s', out_fn); + [gps,error_flag] = gps_force_monotonic(gps); + + warning('Smoothing elevation and heading data: %s', out_fn); + gps.elev = ct_sgolayfilt(gps.elev,2,101); % Adjust filter length as needed to remove high frequency noise + heading_x = cos(gps.heading); + heading_y = sin(gps.heading); + heading_x = ct_sgolayfilt(heading_x,2,101); % Adjust filter length as needed to remove high frequency noise + heading_y = ct_sgolayfilt(heading_y,2,101); % Adjust filter length as needed to remove high frequency noise + gps.heading = atan2(heading_y,heading_x); + + save(out_fn,'-append','-struct','gps','elev','heading'); + end +end diff --git a/cresis-toolbox/gps/missions/gps_create_2021_Alaska_SO.m b/cresis-toolbox/gps/missions/gps_create_2021_Alaska_SO.m new file mode 100644 index 00000000..f4706a57 --- /dev/null +++ b/cresis-toolbox/gps/missions/gps_create_2021_Alaska_SO.m @@ -0,0 +1,291 @@ + +% script gps_create_2021_alaska_SO +% +% Makes the GPS files for 2021 Alaska SO field season + +tic; + +global gRadar; + +support_path = ''; +data_support_path = ''; + +if isempty(support_path) + support_path = gRadar.support_path; +end + +gps_path = fullfile(support_path,'gps','2021_Alaska_SO'); +if ~exist(gps_path,'dir') + fprintf('Making directory %s\n', gps_path); + fprintf(' Press a key to proceed\n'); + pause; + mkdir(gps_path); +end + +if isempty(data_support_path) + data_support_path = gRadar.data_support_path; +end + +% ====================================================================== +% User Settings +% ====================================================================== +debug_level = 1; + +in_base_path = fullfile(data_support_path,'2021_Alaska_SO'); + +file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; +sync_fns = {}; sync_params = {}; + +season_name = '2021_Alaska_SO'; +% gps_source_to_use = 'NMEA'; +gps_source_to_use = 'Lidar-traj'; + +if strcmpi(gps_source_to_use,'NMEA') + +% year = 2021; month = 4; day = 20; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'nmea','','.gps'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = '20210420'; + +% year = 2021; month = 5; day = 2; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'nmea','','.gps'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = '20210502'; + +% year = 2021; month = 5; day = 3; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'nmea','','.gps'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = '20210503'; + +% year = 2021; month = 5; day = 5; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'nmea','','.gps'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = '20210505'; + +% year = 2021; month = 5; day = 6; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'nmea','','.gps'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = '20210506'; + +% year = 2021; month = 5; day = 9; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'nmea','','.gps'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = '20210509'; + +% year = 2021; month = 5; day = 10; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'nmea','','.gps'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = '20210510'; + +% year = 2021; month = 5; day = 12; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'nmea','','.gps'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = '20210512'; + +year = 2021; month = 5; day = 13; +file_idx = file_idx + 1; +in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'nmea','','.gps'); +out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +file_type{file_idx} = 'NMEA'; +params{file_idx} = struct('year',year,'month',month,'day',day,'format',3,'time_reference','utc'); +gps_source{file_idx} = 'nmea-field'; +sync_flag{file_idx} = 0; +date_str{file_idx} = '20210513'; + + +elseif strcmpi(gps_source_to_use,'Lidar-traj') + +% year = 2021; month = 5; day = 2; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(in_base_path,datestr(datenum(year,month,day),'yymmdd'),'','.pos'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'General_ASCII'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','headerlines',1,'format_str','%f%f%f%f%f%f%f'); +% params{file_idx}.types = {'sec','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg'}; +% params{file_idx}.textscan = {}; +% gps_source{file_idx} = 'lidar-final'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = '20210502'; + +% year = 2021; month = 5; day = 3; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(in_base_path,datestr(datenum(year,month,day),'yymmdd'),'','.pos'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'General_ASCII'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','headerlines',1,'format_str','%f%f%f%f%f%f%f'); +% params{file_idx}.types = {'sec','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg'}; +% params{file_idx}.textscan = {}; +% gps_source{file_idx} = 'lidar-final'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = '20210503'; + +% year = 2021; month = 5; day = 5; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(in_base_path,datestr(datenum(year,month,day),'yymmdd'),'','.pos'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'General_ASCII'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','headerlines',1,'format_str','%f%f%f%f%f%f%f'); +% params{file_idx}.types = {'sec','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg'}; +% params{file_idx}.textscan = {}; +% gps_source{file_idx} = 'lidar-final'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = '20210505'; + +% year = 2021; month = 5; day = 6; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(in_base_path,datestr(datenum(year,month,day),'yymmdd'),'','.pos'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'General_ASCII'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','headerlines',1,'format_str','%f%f%f%f%f%f%f'); +% params{file_idx}.types = {'sec','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg'}; +% params{file_idx}.textscan = {}; +% gps_source{file_idx} = 'lidar-final'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = '20210506'; +% +% year = 2021; month = 5; day = 9; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(in_base_path,datestr(datenum(year,month,day),'yymmdd'),'','.pos'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'General_ASCII'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','headerlines',1,'format_str','%f%f%f%f%f%f%f'); +% params{file_idx}.types = {'sec','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg'}; +% params{file_idx}.textscan = {}; +% gps_source{file_idx} = 'lidar-final'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = '20210509'; +% +% year = 2021; month = 5; day = 10; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(in_base_path,datestr(datenum(year,month,day),'yymmdd'),'','.pos'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'General_ASCII'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','headerlines',1,'format_str','%f%f%f%f%f%f%f'); +% params{file_idx}.types = {'sec','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg'}; +% params{file_idx}.textscan = {}; +% gps_source{file_idx} = 'lidar-final'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = '20210510'; +% + year = 2021; month = 5; day = 12; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(in_base_path,datestr(datenum(year,month,day),'yymmdd'),'','.pos'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'General_ASCII'; + params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','headerlines',1,'format_str','%f%f%f%f%f%f%f'); + params{file_idx}.types = {'sec','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg'}; + params{file_idx}.textscan = {}; + gps_source{file_idx} = 'lidar-final'; + sync_flag{file_idx} = 0; + date_str{file_idx} = '20210512'; +% +% year = 2021; month = 5; day = 13; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(in_base_path,datestr(datenum(year,month,day),'yymmdd'),'','.pos'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'General_ASCII'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'time_reference','utc','headerlines',1,'format_str','%f%f%f%f%f%f%f'); +% params{file_idx}.types = {'sec','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg'}; +% params{file_idx}.textscan = {}; +% gps_source{file_idx} = 'lidar-final'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = '20210513'; +end + +% ====================================================================== +% Read and translate files according to user settings +% ====================================================================== +gps_create; + +% Debug code that sets up special processing + +hack_idx = cell2mat(strfind(out_fns,'gps_20210502.mat')); % snow radar data collected before LiDAR data, so need to combine radar NMEA and LiDAR gps times. +if ~isempty(hack_idx) + out_fn = fullfile(gps_path,out_fns{hack_idx}); + gps = load(out_fn); + gps_nmea = load(fullfile(gps_path,'NMEA_gps',out_fns{hack_idx})); + append_idxs = find(gps_nmea.gps_time < gps.gps_time(1)); + gps.gps_time = [gps_nmea.gps_time(append_idxs),gps.gps_time]; + gps.lat = [gps_nmea.lat(append_idxs),gps.lat]; + gps.lon = [gps_nmea.lon(append_idxs),gps.lon]; + gps.elev = [gps_nmea.elev(append_idxs),gps.elev]; + gps.pitch = [gps_nmea.pitch(append_idxs),gps.pitch]; + gps.roll = [gps_nmea.roll(append_idxs),gps.roll]; + gps.heading = [gps_nmea.heading(append_idxs),gps.heading]; + gps.gps_source = 'lidar-final+NMEA'; + save(out_fn,'-append','-struct','gps'); +end + +hack_idx = cell2mat(strfind(out_fns,'gps_20210505.mat')); % snow radar data collected after LiDAR data, so need to combine radar NMEA and LiDAR gps times. +if ~isempty(hack_idx) + out_fn = fullfile(gps_path,out_fns{hack_idx}); + gps = load(out_fn); + gps_nmea = load(fullfile(gps_path,'NMEA_gps',out_fns{hack_idx})); + append_idxs = find(gps_nmea.gps_time > gps.gps_time(end)); + gps.gps_time = [gps.gps_time,gps_nmea.gps_time(append_idxs)]; + gps.lat = [gps.lat,gps_nmea.lat(append_idxs)]; + gps.lon = [gps.lon,gps_nmea.lon(append_idxs)]; + gps.elev = [gps.elev,gps_nmea.elev(append_idxs)]; + gps.pitch = [gps.pitch,gps_nmea.pitch(append_idxs)]; + gps.roll = [gps.roll,gps_nmea.roll(append_idxs)]; + gps.heading = [gps.heading,gps_nmea.heading(append_idxs)]; + gps.gps_source = 'lidar-final+NMEA'; + save(out_fn,'-append','-struct','gps'); +end + +hack_idx = cell2mat(strfind(out_fns,'gps_20210512.mat')); % snow radar data collected after LiDAR data, so need to combine radar NMEA and LiDAR gps times. +if ~isempty(hack_idx) + out_fn = fullfile(gps_path,out_fns{hack_idx}); + gps = load(out_fn); + gps_nmea = load(fullfile(gps_path,'NMEA_gps',out_fns{hack_idx})); + append_idxs = find(gps_nmea.gps_time > gps.gps_time(end)); + gps.gps_time = [gps.gps_time,gps_nmea.gps_time(append_idxs)]; + gps.lat = [gps.lat,gps_nmea.lat(append_idxs)]; + gps.lon = [gps.lon,gps_nmea.lon(append_idxs)]; + gps.elev = [gps.elev,gps_nmea.elev(append_idxs)]; + gps.pitch = [gps.pitch,gps_nmea.pitch(append_idxs)]; + gps.roll = [gps.roll,gps_nmea.roll(append_idxs)]; + gps.heading = [gps.heading,gps_nmea.heading(append_idxs)]; + gps.gps_source = 'lidar-final+NMEA'; + save(out_fn,'-append','-struct','gps'); +end \ No newline at end of file diff --git a/cresis-toolbox/gps/missions/gps_create_2021_Arctic_Vanilla.m b/cresis-toolbox/gps/missions/gps_create_2021_Arctic_Vanilla.m new file mode 100644 index 00000000..6e87c633 --- /dev/null +++ b/cresis-toolbox/gps/missions/gps_create_2021_Arctic_Vanilla.m @@ -0,0 +1,107 @@ +% script gps_create_2021_Arctic_Vanilla +% +% Makes the GPS files for 2021 Aarctic Vanilla field season + +%% Setup +% ========================================================================= +tic; + +global gRadar; + +support_path = ''; +data_support_path = ''; + +if isempty(support_path) + support_path = gRadar.support_path; +end + +gps_path = fullfile(support_path,'gps','2021_Arctic_Vanilla'); +if ~exist(gps_path,'dir') + fprintf('Making directory %s\n', gps_path); + fprintf(' Press a key to proceed\n'); + pause; + mkdir(gps_path); +end + +if isempty(data_support_path) + data_support_path = gRadar.data_support_path; +end + +% ====================================================================== +% User Settings +% ====================================================================== +debug_level = 1; +in_base_path = fullfile(data_support_path,'2021_Arctic_Vanilla'); + +file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; +sync_flag = {}; sync_fns = {}; sync_file_type = {}; sync_params = {}; + +season_name = '2021_Alaska_SO'; +gps_source_to_use = 'arena'; + +if strcmpi(gps_source_to_use,'arena') + %% ARENA GPS SOURCE + % ======================================================================= + +% year = 2021; month = 4; day = 14; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day),'config_logs'),'','','gps.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'arena'; +% params{file_idx} = struct('year',2021,'month',4,'day',14,'time_reference','utc'); +% gps_source{file_idx} = 'arena-field'; +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day),'config_logs'),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('year',2021,'month',4,'day',14,'time_reference','utc'); +% date_str{file_idx} = '20210414'; + +year = 2021; month = 8; day = 19; +file_idx = file_idx + 1; +in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day),'logs'),'','','gps.txt'); +out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +file_type{file_idx} = 'arena'; +params{file_idx} = struct('year',2021,'month',8,'day',19,'time_reference','utc'); +gps_source{file_idx} = 'arena-field'; +sync_flag{file_idx} = 1; +sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day),'logs'),'','','gps.txt'); +sync_file_type{file_idx} = 'arena'; +sync_params{file_idx} = struct('year',2021,'month',8,'day',19,'time_reference','utc'); +date_str{file_idx} = '20210819'; +end + +%% gps_create +% Read and translate files according to user settings +% ========================================================================= +gps_create; + +%% custom fixes +% ========================================================================= +for idx = 1:length(file_type) + if strcmpi(date_str{idx},'20210414') || strcmpi(date_str{idx},'20210819') % Radar time is 2 seconds beyond GPS time + out_fn = fullfile(gps_path,out_fns{idx}); + + load(out_fn,'gps_source'); + if ~isempty(regexpi(gps_source,'arena')) + % Extrapolation is necessary because GPS data starts after/stops before + % the beginning/end of the radar data. + warning('Extrapolating and filtering elevation for arena GPS data: %s', out_fn); + gps = load(out_fn); + + if length(gps.lat) >= 2 + new_gps_time = [gps.gps_time,gps.gps_time(end)+5]; + gps.lat = interp1(gps.gps_time,gps.lat,new_gps_time,'linear','extrap'); + gps.lon = interp1(gps.gps_time,gps.lon,new_gps_time,'linear','extrap'); + gps.elev = interp1(gps.gps_time,gps.elev,new_gps_time,'linear','extrap'); + gps.roll = interp1(gps.gps_time,gps.roll,new_gps_time,'linear','extrap'); + gps.pitch = interp1(gps.gps_time,gps.pitch,new_gps_time,'linear','extrap'); + gps.heading = interp1(gps.gps_time,gps.heading,new_gps_time,'linear','extrap'); + gps.gps_time = new_gps_time; + +% gps.elev = fir_dec(gps.elev,ones(1,101)/101,1); + + save(out_fn,'-append','-struct','gps','gps_time','lat','lon','elev','roll','pitch','heading'); + end + end + end +end diff --git a/cresis-toolbox/gps/missions/gps_create_2021_Greenland_Polar5.m b/cresis-toolbox/gps/missions/gps_create_2021_Greenland_Polar5.m new file mode 100644 index 00000000..37a39748 --- /dev/null +++ b/cresis-toolbox/gps/missions/gps_create_2021_Greenland_Polar5.m @@ -0,0 +1,139 @@ +% script gps_create_2021_Greenland_Polar5 +% +% Makes the GPS files for 2021 Greenland Polar5 field season + +tic; + +global gRadar; + +support_path = ''; +data_support_path = ''; +season_name = '2021_Greenland_Polar5' + +% use NMEA directly after the flight in the field +gps_source_to_use = 'NMEA'; + +% use AWI as soon the post processed GPS_R_L1_......nc files are available +%gps_source_to_use = 'AWI'; + +%================================================================% +%Please edit this line for each new campaign flight +year = 2021; month = 7; day = 28; + + +%================================================================% + + +date_str = join([compose('%04d',year) compose('%02d',month) compose('%02d',day)],''); +date_str = date_str{1}; +gps.date_str = date_str; + +if isempty(support_path) + support_path = gRadar.support_path; +end + +gps_path = fullfile(support_path,'gps',season_name); +if ~exist(gps_path,'dir') + fprintf('Making directory %s\n', gps_path); + fprintf(' Press a key to proceed\n'); + pause; + mkdir(gps_path); +end + +if isempty(data_support_path) + data_support_path = gRadar.data_support_path; +end + +debug_level = 1; + +in_base_path = fullfile(data_support_path,season_name); + +file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; +sync_fns = {}; sync_params = {}; + + +if strcmpi(gps_source_to_use,'NMEA') + % ======================================================================= + % NMEA + % ======================================================================= + + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(in_base_path,sprintf('GPS_%04d%02d%02d',year,month,day),'','.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); + file_type{file_idx} = 'NMEA'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); + gps_source{file_idx} = 'nmea-field'; + sync_flag{file_idx} = 0; + +elseif strcmpi(gps_source_to_use,'AWI') + % ======================================================================= + % AWI + % ======================================================================= + field_gps_in_base_path = fullfile(in_base_path,''); + + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(field_gps_in_base_path,sprintf('GPS_R_L1_%04d%02d%02d',year,month,day),'','.nc'); + in_fns_ins{file_idx} = get_filenames(in_base_path,sprintf('INS_L1_%04d%02d%02d',year,month,day),'','.nc'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); + file_type{file_idx} = 'awi_netcdf+awi_netcdf'; + gps_source{file_idx} = 'awi-field'; + sync_flag{file_idx} = 0; + params{file_idx} = struct('time_reference','utc'); + params{file_idx}.nc_field = {'TIME','LATITUDE','LONGITUDE','ALTITUDE','YEAR','MONTH','DAY'}; + params{file_idx}.nc_type = {'v','v','v','v','a','a','a'}; + params{file_idx}.types = {'sec','lat_deg','lon_deg','elev_m','year','month','day'}; + params{file_idx}.scale = [1e-3 1 1 1 1 1 1]; + params_ins{file_idx} = struct('time_reference','utc'); + params_ins{file_idx}.nc_field = {'TIME','ROLL','PITCH','THDG','YEAR','MONTH','DAY'}; + params_ins{file_idx}.nc_type = {'v','v','v','v','a','a','a'}; + params_ins{file_idx}.types = {'sec','roll_deg','pitch_deg','heading_deg','year','month','day'}; + params_ins{file_idx}.scale = [1e-3 1 1 1 1 1 1]; + +elseif strcmpi(gps_source_to_use,'AWI_final') + + % ======================================================================= + % AWI_final + % ======================================================================= + + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(in_base_path,sprintf('GPS_R_L1_%04d%02d%02d',year,month,day),'','.nc'); + in_fns_ins{file_idx} = get_filenames(in_base_path,sprintf('INS_L1_%04d%02d%02d',year,month,day),'','.nc'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); + file_type{file_idx} = 'awi_netcdf+awi_netcdf'; + gps_source{file_idx} = 'awi-final_20161109'; + sync_flag{file_idx} = 0; + params{file_idx} = struct('time_reference','utc'); + params{file_idx}.nc_field = {'TIME','LATITUDE','LONGITUDE','ALTITUDE','YEAR','MONTH','DAY'}; + params{file_idx}.nc_type = {'v','v','v','v','a','a','a'}; + params{file_idx}.types = {'sec','lat_deg','lon_deg','elev_m','year','month','day'}; + params{file_idx}.scale = [1e-3 1 1 1 1 1 1]; + params_ins{file_idx} = struct('time_reference','utc'); + params_ins{file_idx}.nc_field = {'TIME','ROLL','PITCH','THDG','YEAR','MONTH','DAY'}; + params_ins{file_idx}.nc_type = {'v','v','v','v','a','a','a'}; + params_ins{file_idx}.types = {'sec','roll_deg','pitch_deg','heading_deg','year','month','day'}; + params_ins{file_idx}.scale = [1e-3 1 1 1 1 1 1]; + +end + +% ====================================================================== +% Read and translate files according to user settings +% ====================================================================== + gps_create; + +%% No GPS Data Available: Fakes GPS position information +match_idx = strmatch('gps_20160331.mat',out_fns,'exact'); +if ~isempty(match_idx) + gps_fn = fullfile(gps_path,out_fns{match_idx}); + fprintf('Creating fake gps data for %s\n', gps_fn); + gps = load(gps_fn); + gps.gps_time = datenum_to_epoch(datenum(2016,3,31) + (50000:65000)/86400); + gps.lon = -45 * ones(size(gps.gps_time)); + gps.lat = 70 + (1:length(gps.gps_time)) * 6e-4; + gps.elev = 500 * ones(size(gps.gps_time)); + gps.roll = zeros(size(gps.gps_time)); + gps.pitch = zeros(size(gps.gps_time)); + gps.heading = zeros(size(gps.gps_time)); + save(gps_fn,'-append','-struct','gps','gps_time','lat','lon','elev','roll','pitch','heading'); +end + + diff --git a/cresis-toolbox/gps/missions/gps_create_2022_Antarctica_BaslerMKB.m b/cresis-toolbox/gps/missions/gps_create_2022_Antarctica_BaslerMKB.m new file mode 100644 index 00000000..eab44caf --- /dev/null +++ b/cresis-toolbox/gps/missions/gps_create_2022_Antarctica_BaslerMKB.m @@ -0,0 +1,183 @@ +% script gps_create_2022_Antarctica_BaslerMKB +% +% Makes the GPS files for 2022_Antarctica_BaslerMKB field season + +%% Setup +% ========================================================================= +tic; + +global gRadar; + +support_path = ''; +data_support_path = ''; + +if isempty(support_path) + support_path = gRadar.support_path; +end + +season_name = '2022_Antarctica_BaslerMKB'; + +gps_path = fullfile(support_path,'gps',season_name); +if ~exist(gps_path,'dir') + fprintf('Making directory %s\n', gps_path); + fprintf(' Press a key to proceed\n'); + pause; + mkdir(gps_path); +end + +if isempty(data_support_path) + data_support_path = gRadar.data_support_path; +end + +in_base_path = fullfile(data_support_path,season_name); + +file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; +sync_flag = {}; sync_fns = {}; sync_file_type = {}; sync_params = {}; + +%% <== CHOOSE WHICH GPS SOURCE TO PROCESS +gps_source_to_use = 'novatelraw'; +% gps_source_to_use = 'cresis'; + +if strcmpi(gps_source_to_use,'novatelraw') + %% sonntag_nav GPS SOURCE + % ======================================================================= + +% year = 2023; month = 1; day = 10; +% datestr_year = 2023; datestr_month = 1; datestr_day = 9; % <--- UPDATE TO MATCH WHAT PREPROCESS PRINTS OUT +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', datestr_year, datestr_month, datestr_day); +% date_str{file_idx} = sprintf('%04d%02d%02d', datestr_year, datestr_month, datestr_day); +% file_type{file_idx} = 'novatelraw'; +% params{file_idx} = struct('time_reference','utc'); +% gps_source{file_idx} = 'novatelraw-field'; +% sync_flag{file_idx} = 0; + +% year = 2023; month = 1; day = 11; +% datestr_year = 2023; datestr_month = 1; datestr_day = 10; % <--- UPDATE TO MATCH WHAT PREPROCESS PRINTS OUT +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', datestr_year, datestr_month, datestr_day); +% date_str{file_idx} = sprintf('%04d%02d%02d', datestr_year, datestr_month, datestr_day); +% file_type{file_idx} = 'novatelraw'; +% params{file_idx} = struct('time_reference','utc'); +% gps_source{file_idx} = 'novatelraw-field'; +% sync_flag{file_idx} = 0; + +% year = 2023; month = 1; day = 14; +% datestr_year = 2023; datestr_month = 1; datestr_day = 13; % <--- UPDATE TO MATCH WHAT PREPROCESS PRINTS OUT +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', datestr_year, datestr_month, datestr_day); +% date_str{file_idx} = sprintf('%04d%02d%02d', datestr_year, datestr_month, datestr_day); +% file_type{file_idx} = 'novatelraw'; +% params{file_idx} = struct('time_reference','utc'); +% gps_source{file_idx} = 'novatelraw-field'; +% sync_flag{file_idx} = 0; + +% year = 2023; month = 1; day = 16; +% datestr_year = 2023; datestr_month = 1; datestr_day = 16; % <--- UPDATE TO MATCH WHAT PREPROCESS PRINTS OUT +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', datestr_year, datestr_month, datestr_day); +% date_str{file_idx} = sprintf('%04d%02d%02d', datestr_year, datestr_month, datestr_day); +% file_type{file_idx} = 'novatelraw'; +% params{file_idx} = struct('time_reference','utc'); +% gps_source{file_idx} = 'novatelraw-field'; +% sync_flag{file_idx} = 0; + +% year = 2023; month = 1; day = 20; +% datestr_year = 2023; datestr_month = 1; datestr_day = 20; % <--- UPDATE TO MATCH WHAT PREPROCESS PRINTS OUT +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','aq-field22','gps'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', datestr_year, datestr_month, datestr_day); +% date_str{file_idx} = sprintf('%04d%02d%02d', datestr_year, datestr_month, datestr_day); +% file_type{file_idx} = 'novatelraw'; +% params{file_idx} = struct('time_reference','utc'); +% gps_source{file_idx} = 'novatelraw-field'; +% sync_flag{file_idx} = 0; + +% year = 2023; month = 1; day = 25; +% datestr_year = 2023; datestr_month = 1; datestr_day = 24; % <--- UPDATE TO MATCH WHAT PREPROCESS PRINTS OUT +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','aq-field22','gps'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', datestr_year, datestr_month, datestr_day); +% date_str{file_idx} = sprintf('%04d%02d%02d', datestr_year, datestr_month, datestr_day); +% file_type{file_idx} = 'novatelraw'; +% params{file_idx} = struct('time_reference','utc'); +% gps_source{file_idx} = 'novatelraw-field'; +% sync_flag{file_idx} = 0; + +% year = 2023; month = 1; day = 26; +% datestr_year = 2023; datestr_month = 1; datestr_day = 25; % <--- UPDATE TO MATCH WHAT PREPROCESS PRINTS OUT +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','aq-field22','gps'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', datestr_year, datestr_month, datestr_day); +% date_str{file_idx} = sprintf('%04d%02d%02d', datestr_year, datestr_month, datestr_day); +% file_type{file_idx} = 'novatelraw'; +% params{file_idx} = struct('time_reference','utc'); +% gps_source{file_idx} = 'novatelraw-field'; +% sync_flag{file_idx} = 0; +% +% year = 2023; month = 1; day = 27; +% datestr_year = 2023; datestr_month = 1; datestr_day = 26; % <--- UPDATE TO MATCH WHAT PREPROCESS PRINTS OUT +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','aq-field22','gps'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', datestr_year, datestr_month, datestr_day); +% date_str{file_idx} = sprintf('%04d%02d%02d', datestr_year, datestr_month, datestr_day); +% file_type{file_idx} = 'novatelraw'; +% params{file_idx} = struct('time_reference','utc'); +% gps_source{file_idx} = 'novatelraw-field'; +% sync_flag{file_idx} = 0; + + year = 2023; month = 1; day = 28; + datestr_year = 2023; datestr_month = 1; datestr_day = 27; % <--- UPDATE TO MATCH WHAT PREPROCESS PRINTS OUT + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','aq-field22','gps'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', datestr_year, datestr_month, datestr_day); + date_str{file_idx} = sprintf('%04d%02d%02d', datestr_year, datestr_month, datestr_day); + file_type{file_idx} = 'novatelraw'; + params{file_idx} = struct('time_reference','utc'); + gps_source{file_idx} = 'novatelraw-field'; + sync_flag{file_idx} = 0; + + +elseif strcmpi(gps_source_to_use,'cresis') + %% CReSIS GPS SOURCE + % ======================================================================= + + % DO NOT USE IN FIELD +end + +%% gps_create +% Read and translate files according to user settings +% ========================================================================= +gps_create; + +%% custom fixes +% ========================================================================= +for idx = 1:length(file_type) + out_fn = fullfile(gps_path,out_fns{idx}); + + gps = load(out_fn); + + if ~isempty(regexpi(gps_source,'novatelraw')) && ~isempty(regexpi(out_fn,'20230113')) + gps.roll(:) = 0; + gps.pitch(:) = 0; + [est_heading,along_track,speed] = trajectory_coord_system(gps); + gps.heading = est_heading; + + save(out_fn,'-append','-struct','gps','roll','pitch','heading'); + end + + % Arena 500's radar_time is UTC time, fill in sync fields with this + % information. + gps.radar_time = gps.gps_time - utc_leap_seconds(gps.gps_time(1)); + gps.sync_gps_time = gps.gps_time; + gps.sync_lat = gps.lat; + gps.sync_lon = gps.lon; + gps.sync_elev = gps.elev; + + fprintf('Saving sync fields into %s\n', out_fn); + save(out_fn,'-append','-struct','gps','radar_time','sync_gps_time','sync_lat','sync_lon','sync_elev'); +end diff --git a/cresis-toolbox/gps/missions/gps_create_2022_Antarctica_BaslerMKB_UTIG.m b/cresis-toolbox/gps/missions/gps_create_2022_Antarctica_BaslerMKB_UTIG.m new file mode 100644 index 00000000..00380117 --- /dev/null +++ b/cresis-toolbox/gps/missions/gps_create_2022_Antarctica_BaslerMKB_UTIG.m @@ -0,0 +1,97 @@ +% script gps_create_2022_Antarctica_BaslerMKB +% +% Makes the GPS files for 2022_Antarctica_BaslerMKB field season + +%% Setup +% ========================================================================= +tic; + +global gRadar; + +support_path = ''; +data_support_path = ''; + +if isempty(support_path) + support_path = gRadar.support_path; +end + +season_name = '2022_Antarctica_BaslerMKB'; + +gps_path = fullfile(support_path,'gps',season_name); +if ~exist(gps_path,'dir') + fprintf('Making directory %s\n', gps_path); + fprintf(' Press a key to proceed\n'); + pause; + mkdir(gps_path); +end + +if isempty(data_support_path) + data_support_path = gRadar.data_support_path; +end + +in_base_path = fullfile(data_support_path,season_name); + +file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; +sync_flag = {}; sync_fns = {}; sync_file_type = {}; sync_params = {}; + +%% <== CHOOSE WHICH GPS SOURCE TO PROCESS +% gps_source_to_use = 'novatelraw'; +gps_source_to_use = 'utig'; +% gps_source_to_use = 'cresis'; + +if strcmpi(gps_source_to_use,'utig') + %% "utig ELSA" GPS SOURCE + % ======================================================================= + +% year = 2023; month = 1; day = 16; +% datestr_year = 2023; datestr_month = 1; datestr_day = 16; % <--- UPDATE TO MATCH WHAT PREPROCESS PRINTS OUT +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day),'ELSA'),'serial','',''); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d_utig.mat', datestr_year, datestr_month, datestr_day); +% date_str{file_idx} = sprintf('%04d%02d%02d', datestr_year, datestr_month, datestr_day); +% file_type{file_idx} = 'utig'; +% params{file_idx} = struct('time_reference','utc'); +% gps_source{file_idx} = 'utig-field'; +% sync_flag{file_idx} = 0; + +% year = 2023; month = 1; day = 20; +% datestr_year = 2023; datestr_month = 1; datestr_day = 20; % <--- UPDATE TO MATCH WHAT PREPROCESS PRINTS OUT +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day),'ELSA'),'serial','',''); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d_utig.mat', datestr_year, datestr_month, datestr_day); +% date_str{file_idx} = sprintf('%04d%02d%02d', datestr_year, datestr_month, datestr_day); +% file_type{file_idx} = 'utig'; +% params{file_idx} = struct('time_reference','utc'); +% gps_source{file_idx} = 'utig-field'; +% sync_flag{file_idx} = 0; + +% year = 2023; month = 1; day = 25; +% datestr_year = 2023; datestr_month = 1; datestr_day = 25; % <--- UPDATE TO MATCH WHAT PREPROCESS PRINTS OUT +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day),'ELSA'),'serial','',''); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d_utig.mat', datestr_year, datestr_month, datestr_day); +% date_str{file_idx} = sprintf('%04d%02d%02d', datestr_year, datestr_month, datestr_day); +% file_type{file_idx} = 'utig'; +% params{file_idx} = struct('time_reference','utc'); +% gps_source{file_idx} = 'utig-field'; +% sync_flag{file_idx} = 0; + +elseif strcmpi(gps_source_to_use,'cresis') + %% CReSIS GPS SOURCE + % ======================================================================= + + % DO NOT USE IN FIELD +end + +%% gps_create +% Read and translate files according to user settings +% ========================================================================= +gps_create; + +%% custom fixes +% ========================================================================= +for idx = 1:length(file_type) + out_fn = fullfile(gps_path,out_fns{idx}); + +% gps = load(out_fn); +end diff --git a/cresis-toolbox/gps/missions/gps_create_2022_Antarctica_Ground.m b/cresis-toolbox/gps/missions/gps_create_2022_Antarctica_Ground.m new file mode 100644 index 00000000..06545811 --- /dev/null +++ b/cresis-toolbox/gps/missions/gps_create_2022_Antarctica_Ground.m @@ -0,0 +1,258 @@ +% script gps_create_2022_Antarctica_Ground +% +% Makes the GPS files for 2022_Antarctica_Ground field season + +%% Setup +% ========================================================================= +tic; + +global gRadar; + +support_path = ''; +data_support_path = ''; + +if isempty(support_path) + support_path = gRadar.support_path; +end + +season_name = '2022_Antarctica_Ground'; + +gps_path = fullfile(support_path,'gps',season_name); +if ~exist(gps_path,'dir') + fprintf('Making directory %s\n', gps_path); + fprintf(' Press a key to proceed\n'); + pause; + mkdir(gps_path); +end + +if isempty(data_support_path) + data_support_path = gRadar.data_support_path; +end + +in_base_path = fullfile(data_support_path,season_name); + +file_idx = 0; in_fns = {}; out_fns = {}; date_str = {}; file_type = {}; params = {}; gps_source = {}; +sync_flag = {}; sync_fns = {}; sync_file_type = {}; sync_params = {}; + +%% <== CHOOSE WHICH GPS SOURCE TO PROCESS +gps_source_to_use = 'arena'; +% gps_source_to_use = 'cresis'; + +if strcmpi(gps_source_to_use,'arena') + %% ARENA GPS SOURCE + % ======================================================================= + +% year = 2022; month = 12; day = 1; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% date_str{file_idx} = sprintf('%04d%02d%02d', year, month, day); +% file_type{file_idx} = 'arena'; +% params{file_idx} = struct('time_reference','utc'); +% gps_source{file_idx} = 'arena-field'; +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); + +% year = 2022; month = 12; day = 9; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% date_str{file_idx} = sprintf('%04d%02d%02d', year, month, day); +% file_type{file_idx} = 'arena'; +% params{file_idx} = struct('time_reference','utc'); +% gps_source{file_idx} = 'arena-field'; +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); + +% year = 2022; month = 12; day = 21; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% date_str{file_idx} = sprintf('%04d%02d%02d', year, month, day); +% file_type{file_idx} = 'arena'; +% params{file_idx} = struct('time_reference','utc'); +% gps_source{file_idx} = 'arena-field'; +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); + +% year = 2023; month = 1; day = 13; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% date_str{file_idx} = sprintf('%04d%02d%02d', year, month, day); +% file_type{file_idx} = 'arena'; +% params{file_idx} = struct('time_reference','utc'); +% gps_source{file_idx} = 'arena-field'; +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); +% +% year = 2023; month = 1; day = 14; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% date_str{file_idx} = sprintf('%04d%02d%02d', year, month, day); +% file_type{file_idx} = 'arena'; +% params{file_idx} = struct('time_reference','utc'); +% gps_source{file_idx} = 'arena-field'; +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); +% +% year = 2023; month = 1; day = 15; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% date_str{file_idx} = sprintf('%04d%02d%02d', year, month, day); +% file_type{file_idx} = 'arena'; +% params{file_idx} = struct('time_reference','utc'); +% gps_source{file_idx} = 'arena-field'; +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); +% % +% year = 2023; month = 1; day = 18; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% date_str{file_idx} = sprintf('%04d%02d%02d', year, month, day); +% file_type{file_idx} = 'arena'; +% params{file_idx} = struct('time_reference','utc'); +% gps_source{file_idx} = 'arena-field'; +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); + + year = 2023; month = 1; day = 19; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + date_str{file_idx} = sprintf('%04d%02d%02d', year, month, day); + file_type{file_idx} = 'arena'; + params{file_idx} = struct('time_reference','utc'); + gps_source{file_idx} = 'arena-field'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); + +% year = 2023; month = 1; day = 21; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% date_str{file_idx} = sprintf('%04d%02d%02d', year, month, day); +% file_type{file_idx} = 'arena'; +% params{file_idx} = struct('time_reference','utc'); +% gps_source{file_idx} = 'arena-field'; +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); + +elseif strcmpi(gps_source_to_use,'cresis') + %% CReSIS GPS SOURCE + % ======================================================================= + +% % IMU worked fine + year = 2022; month = 12; day = 2; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('ie_%04d%02d%02d',year,month,day)),sprintf('ie_%04d%02d%02d',year,month,day),'','.OUT'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'applanix'; + params{file_idx} = struct('time_reference','gps','year',year,'month',month,'day',day); + gps_source{file_idx} = 'cresis-field20220421'; + sync_flag{file_idx} = 0; + date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); + +end + +%% gps_create +% Read and translate files according to user settings +% ========================================================================= +gps_create; + +%% custom fixes +% ========================================================================= +for idx = 1:length(file_type) + out_fn = fullfile(gps_path,out_fns{idx}); + + load(out_fn,'gps_source'); + if ~isempty(regexpi(gps_source,'arena')) + % Extrapolation is necessary because GPS data starts after/stops before + % the beginning/end of the radar data. + warning('Extrapolating and filtering elevation for arena GPS data: %s', out_fn); + gps = load(out_fn); + + if length(gps.lat) >= 2 + new_gps_time = [gps.gps_time(1)-20, gps.gps_time,gps.gps_time(end)+10]; + gps.lat = interp1(gps.gps_time,gps.lat,new_gps_time,'linear','extrap'); + gps.lon = interp1(gps.gps_time,gps.lon,new_gps_time,'linear','extrap'); + gps.elev = interp1(gps.gps_time,gps.elev,new_gps_time,'linear','extrap'); + gps.roll = interp1(gps.gps_time,gps.roll,new_gps_time,'linear','extrap'); + gps.pitch = interp1(gps.gps_time,gps.pitch,new_gps_time,'linear','extrap'); + gps.heading = interp1(gps.gps_time,gps.heading,new_gps_time,'linear','extrap'); + gps.gps_time = new_gps_time; + + gps.elev = fir_dec(gps.elev,ones(1,101)/101,1); + + save(out_fn,'-append','-struct','gps','gps_time','lat','lon','elev','roll','pitch','heading'); + end + end + + if ~isempty(regexpi(gps_source,'arena')) && ~isempty(regexpi(out_fn,'20221209')) + [out_fn_dir,out_fn_name,out_fn_ext] = fileparts(out_fn); + new_out_fn = fullfile(out_fn_dir,[out_fn_name(1:end-2),'10',out_fn_ext]); + warning('Copying 20221209 to 20221210:\n %s\n %s\n', out_fn, new_out_fn); + copyfile(out_fn,new_out_fn); + new_out_fn = fullfile(out_fn_dir,[out_fn_name(1:end-2),'11',out_fn_ext]); + warning('Copying 20221209 to 20221211:\n %s\n %s\n', out_fn, new_out_fn); + copyfile(out_fn,new_out_fn); + new_out_fn = fullfile(out_fn_dir,[out_fn_name(1:end-2),'12',out_fn_ext]); + warning('Copying 20221209 to 20221212:\n %s\n %s\n', out_fn, new_out_fn); + copyfile(out_fn,new_out_fn); + end + + if ~isempty(regexpi(out_fn,'20220419')) + % Fix bad radar time + warning('Fixing bad radar_time: %s', out_fn); + gps = load(out_fn); + + if gps.radar_time(1) == 0 + gps.radar_time = gps.radar_time(2:end); + gps.sync_elev = gps.sync_elev(2:end); + gps.sync_gps_time = gps.sync_gps_time(2:end); + gps.sync_lat = gps.sync_lat(2:end); + gps.sync_lon = gps.sync_lon(2:end); + end + save(out_fn,'-append','-struct','gps','radar_time','sync_elev','sync_gps_time','sync_lat','sync_lon'); + end + + if ~isempty(regexpi(out_fn,'201910XX')) + % Fake GPS for testing + warning('Faking GPS data: %s', out_fn); + gps = load(out_fn); + + velocity = 4; + gps.lat = -75.5 - (gps.gps_time-gps.gps_time(1))*velocity/111111; + gps.lon(:) = -106.75; + gps.elev(:) = 500; + gps.heading(:) = -pi; + + save(out_fn,'-append','-struct','gps','gps_time','lat','lon','elev','roll','pitch','heading'); + end + +end diff --git a/cresis-toolbox/gps/missions/gps_create_2022_Antarctica_GroundGHOST.m b/cresis-toolbox/gps/missions/gps_create_2022_Antarctica_GroundGHOST.m new file mode 100644 index 00000000..bc079df0 --- /dev/null +++ b/cresis-toolbox/gps/missions/gps_create_2022_Antarctica_GroundGHOST.m @@ -0,0 +1,210 @@ +% script gps_create_2022_Antarctica_GroundGHOST +% +% Makes the GPS files for 2022_Antarctica_GroundGHOST field season + +%% Setup +% ========================================================================= +tic; + +global gRadar; + +support_path = ''; +data_support_path = ''; + +if isempty(support_path) + support_path = gRadar.support_path; +end + +season_name = '2022_Antarctica_GroundGHOST'; + +gps_path = fullfile(support_path,'gps',season_name); +if ~exist(gps_path,'dir') + fprintf('Making directory %s\n', gps_path); + fprintf(' Press a key to proceed\n'); + pause; + mkdir(gps_path); +end + +if isempty(data_support_path) + data_support_path = gRadar.data_support_path; +end + +in_base_path = fullfile(data_support_path,season_name); + +file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; +sync_flag = {}; sync_fns = {}; sync_file_type = {}; sync_params = {}; + +%% <== CHOOSE WHICH GPS SOURCE TO PROCESS +gps_source_to_use = 'arena'; +% gps_source_to_use = 'cresis'; + +if strcmpi(gps_source_to_use,'arena') + %% ARENA GPS SOURCE + % ======================================================================= + +% year = 2022; month = 12; day = 25; +% datestr_year = 2022; datestr_month = 12; datestr_day = 24; % <--- UPDATE TO MATCH WHAT PREPROCESS PRINTS OUT +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', datestr_year, datestr_month, datestr_day); +% %date_str{file_idx} = sprintf('%04d%02d%02d', year, month, day); +% date_str{file_idx} = sprintf('%04d%02d%02d', datestr_year, datestr_month, datestr_day); +% file_type{file_idx} = 'arena'; +% params{file_idx} = struct('time_reference','utc'); +% gps_source{file_idx} = 'arena-field'; +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); + +% year = 2023; month = 1; day = 29; +% datestr_year = 2023; datestr_month = 1; datestr_day = 29; % <--- UPDATE TO MATCH WHAT PREPROCESS PRINTS OUT +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', datestr_year, datestr_month, datestr_day); +% %date_str{file_idx} = sprintf('%04d%02d%02d', year, month, day); +% date_str{file_idx} = sprintf('%04d%02d%02d', datestr_year, datestr_month, datestr_day); +% file_type{file_idx} = 'arena'; +% params{file_idx} = struct('time_reference','utc'); +% gps_source{file_idx} = 'arena-field'; +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); + +% year = 2023; month = 2; day = 1; +% datestr_year = 2023; datestr_month = 2; datestr_day = 1; % <--- UPDATE TO MATCH WHAT PREPROCESS PRINTS OUT +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', datestr_year, datestr_month, datestr_day); +% %date_str{file_idx} = sprintf('%04d%02d%02d', year, month, day); +% date_str{file_idx} = sprintf('%04d%02d%02d', datestr_year, datestr_month, datestr_day); +% file_type{file_idx} = 'arena'; +% params{file_idx} = struct('time_reference','utc'); +% gps_source{file_idx} = 'arena-field'; +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); + +% year = 2023; month = 2; day = 2; +% datestr_year = 2023; datestr_month = 2; datestr_day = 1; % <--- UPDATE TO MATCH WHAT PREPROCESS PRINTS OUT +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', datestr_year, datestr_month, datestr_day); +% %date_str{file_idx} = sprintf('%04d%02d%02d', year, month, day); +% date_str{file_idx} = sprintf('%04d%02d%02d', datestr_year, datestr_month, datestr_day); +% file_type{file_idx} = 'arena'; +% params{file_idx} = struct('time_reference','utc'); +% gps_source{file_idx} = 'arena-field'; +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); + + year = 2023; month = 2; day = 3; + datestr_year = 2023; datestr_month = 2; datestr_day = 2; % <--- UPDATE TO MATCH WHAT PREPROCESS PRINTS OUT + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', datestr_year, datestr_month, datestr_day); + %date_str{file_idx} = sprintf('%04d%02d%02d', year, month, day); + date_str{file_idx} = sprintf('%04d%02d%02d', datestr_year, datestr_month, datestr_day); + file_type{file_idx} = 'arena'; + params{file_idx} = struct('time_reference','utc'); + gps_source{file_idx} = 'arena-field'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); + +elseif strcmpi(gps_source_to_use,'cresis') + %% CReSIS GPS SOURCE + % ======================================================================= + + % DO NOT USE IN FIELD +end + +%% gps_create +% Read and translate files according to user settings +% ========================================================================= +gps_create; + +%% custom fixes +% ========================================================================= +for idx = 1:length(file_type) + out_fn = fullfile(gps_path,out_fns{idx}); + + load(out_fn,'gps_source'); + if ~isempty(regexpi(gps_source,'arena')) + % Extrapolation is necessary because GPS data starts after/stops before + % the beginning/end of the radar data. + warning('Extrapolating and filtering elevation for arena GPS data: %s', out_fn); + gps = load(out_fn); + + if length(gps.lat) >= 2 + new_gps_time = [gps.gps_time(1)-10, gps.gps_time,gps.gps_time(end)+10]; + gps.lat = interp1(gps.gps_time,gps.lat,new_gps_time,'linear','extrap'); + gps.lon = interp1(gps.gps_time,gps.lon,new_gps_time,'linear','extrap'); + gps.elev = interp1(gps.gps_time,gps.elev,new_gps_time,'linear','extrap'); + gps.roll = interp1(gps.gps_time,gps.roll,new_gps_time,'linear','extrap'); + gps.pitch = interp1(gps.gps_time,gps.pitch,new_gps_time,'linear','extrap'); + gps.heading = interp1(gps.gps_time,gps.heading,new_gps_time,'linear','extrap'); + gps.gps_time = new_gps_time; + + gps.elev = fir_dec(gps.elev,ones(1,101)/101,1); + + good_idxs = gps.radar_time~=0; + gps.radar_time = gps.radar_time(good_idxs); + gps.comp_time = gps.comp_time(good_idxs); + gps.sync_elev = gps.sync_elev(good_idxs); + gps.sync_gps_time = gps.sync_gps_time(good_idxs); + gps.sync_lat = gps.sync_lat(good_idxs); + gps.sync_lon = gps.sync_lon(good_idxs); + + save(out_fn,'-struct','gps'); + end + end + + if ~isempty(regexpi(gps_source,'arena')) && ~isempty(regexpi(out_fn,'20221209')) + [out_fn_dir,out_fn_name,out_fn_ext] = fileparts(out_fn); + new_out_fn = fullfile(out_fn_dir,[out_fn_name(1:end-2),'10',out_fn_ext]); + warning('Copying 20221209 to 20221210:\n %s\n %s\n', out_fn, new_out_fn); + copyfile(out_fn,new_out_fn); + new_out_fn = fullfile(out_fn_dir,[out_fn_name(1:end-2),'11',out_fn_ext]); + warning('Copying 20221209 to 20221211:\n %s\n %s\n', out_fn, new_out_fn); + copyfile(out_fn,new_out_fn); + new_out_fn = fullfile(out_fn_dir,[out_fn_name(1:end-2),'12',out_fn_ext]); + warning('Copying 20221209 to 20221212:\n %s\n %s\n', out_fn, new_out_fn); + copyfile(out_fn,new_out_fn); + end + + if ~isempty(regexpi(out_fn,'20220419')) + % Fix bad radar time + warning('Fixing bad radar_time: %s', out_fn); + gps = load(out_fn); + + if gps.radar_time(1) == 0 + gps.radar_time = gps.radar_time(2:end); + gps.sync_elev = gps.sync_elev(2:end); + gps.sync_gps_time = gps.sync_gps_time(2:end); + gps.sync_lat = gps.sync_lat(2:end); + gps.sync_lon = gps.sync_lon(2:end); + end + save(out_fn,'-append','-struct','gps','radar_time','sync_elev','sync_gps_time','sync_lat','sync_lon'); + end + + if ~isempty(regexpi(out_fn,'201910XX')) + % Fake GPS for testing + warning('Faking GPS data: %s', out_fn); + gps = load(out_fn); + + velocity = 4; + gps.lat = -75.5 - (gps.gps_time-gps.gps_time(1))*velocity/111111; + gps.lon(:) = -106.75; + gps.elev(:) = 500; + gps.heading(:) = -pi; + + save(out_fn,'-append','-struct','gps','gps_time','lat','lon','elev','roll','pitch','heading'); + end + +end diff --git a/cresis-toolbox/gps/missions/gps_create_2022_Antarctica_Polar5.m b/cresis-toolbox/gps/missions/gps_create_2022_Antarctica_Polar5.m new file mode 100644 index 00000000..46d7bff4 --- /dev/null +++ b/cresis-toolbox/gps/missions/gps_create_2022_Antarctica_Polar5.m @@ -0,0 +1,223 @@ +% script gps_create_2022_Antarctica_Polar5 +% +% Makes the GPS files for 2022 Antarctica Polar5 field season + +tic; + +global gRadar; + +support_path = ''; +data_support_path = ''; +season_name = '2022_Antarctica_Polar5' + +% use NMEA directly after the flight in the field +% gps_source_to_use = 'NMEA'; + +% use AWI as soon the post processed GPS_R_L1_......nc files are available +gps_source_to_use = 'AWI_final'; + +%================================================================% +%Please edit this line for each new campaign flight +year = 2022; month = 12; day = 3; + + +%================================================================% + + +date_str = join([compose('%04d',year) compose('%02d',month) compose('%02d',day)],''); +date_str = date_str{1}; +gps.date_str = date_str; + +if isempty(support_path) + support_path = gRadar.support_path; +end + +gps_path = fullfile(support_path,'gps',season_name); +if ~exist(gps_path,'dir') + fprintf('Making directory %s\n', gps_path); + fprintf(' Press a key to proceed\n'); + pause; + mkdir(gps_path); +end + +if isempty(data_support_path) + data_support_path = gRadar.data_support_path; +end + +debug_level = 1; + +in_base_path = fullfile(data_support_path,season_name); + +file_idx = 0; in_fns = {}; out_fns = {}; date_str = {}; file_type = {}; params = {}; gps_source = {}; +sync_flag = {}; sync_fns = {}; sync_file_type = {}; sync_params = {}; + + +if strcmpi(gps_source_to_use,'NMEA') + % ======================================================================= + % NMEA + % ======================================================================= + + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(in_base_path,sprintf('GPS_%04d%02d%02d',year,month,day),'','.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); + date_str{file_idx} = sprintf('%04d%02d%02d', year, month, day); + file_type{file_idx} = 'NMEA'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); + gps_source{file_idx} = 'nmea-field'; + sync_flag{file_idx} = 0; + +elseif strcmpi(gps_source_to_use,'AWI') + % ======================================================================= + % AWI + % ======================================================================= + field_gps_in_base_path = fullfile(in_base_path,''); + + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(field_gps_in_base_path,sprintf('GPS_R_L1_%04d%02d%02d',year,month,day),'','.nc'); + in_fns_ins{file_idx} = get_filenames(in_base_path,sprintf('INS_L1_%04d%02d%02d',year,month,day),'','.nc'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); + date_str{file_idx} = sprintf('%04d%02d%02d', year, month, day); + file_type{file_idx} = 'awi_netcdf+awi_netcdf'; + gps_source{file_idx} = 'awi-field'; + sync_flag{file_idx} = 0; + params{file_idx} = struct('time_reference','utc'); + params{file_idx}.nc_field = {'TIME','LATITUDE','LONGITUDE','ALTITUDE','YEAR','MONTH','DAY'}; + params{file_idx}.nc_type = {'v','v','v','v','a','a','a'}; + params{file_idx}.types = {'sec','lat_deg','lon_deg','elev_m','year','month','day'}; + params{file_idx}.scale = [1e-3 1 1 1 1 1 1]; + params_ins{file_idx} = struct('time_reference','utc'); + params_ins{file_idx}.nc_field = {'TIME','ROLL','PITCH','THDG','YEAR','MONTH','DAY'}; + params_ins{file_idx}.nc_type = {'v','v','v','v','a','a','a'}; + params_ins{file_idx}.types = {'sec','roll_deg','pitch_deg','heading_deg','year','month','day'}; + params_ins{file_idx}.scale = [1e-3 1 1 1 1 1 1]; + +elseif strcmpi(gps_source_to_use,'AWI_final') + + % ======================================================================= + % AWI_final + % ======================================================================= + + file_idx = file_idx + 1; + year = 2022; month = 9; day = 14; + in_fns{file_idx} = get_filenames(in_base_path,sprintf('GPS_R_L1_%04d%02d%02d',year,month,day),'','.nc'); + in_fns_ins{file_idx} = get_filenames(in_base_path,sprintf('INS_L1_%04d%02d%02d',year,month,day),'','.nc'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); + date_str{file_idx} = sprintf('%04d%02d%02d', year, month, day); + file_type{file_idx} = 'awi_netcdf+awi_netcdf'; + gps_source{file_idx} = 'awi-final_20230127'; + sync_flag{file_idx} = 0; + params{file_idx} = struct('time_reference','utc'); + params{file_idx}.nc_field = {'TIME','LATITUDE','LONGITUDE','ALTITUDE','YEAR','MONTH','DAY'}; + params{file_idx}.nc_type = {'v','v','v','v','a','a','a'}; + params{file_idx}.types = {'sec','lat_deg','lon_deg','elev_m','year','month','day'}; + params{file_idx}.scale = [1e-3 1 1 1 1 1 1]; + params_ins{file_idx} = struct('time_reference','utc'); + params_ins{file_idx}.nc_field = {'TIME','ROLL','PITCH','THDG','YEAR','MONTH','DAY'}; + params_ins{file_idx}.nc_type = {'v','v','v','v','a','a','a'}; + params_ins{file_idx}.types = {'sec','roll_deg','pitch_deg','heading_deg','year','month','day'}; + params_ins{file_idx}.scale = [1e-3 1 1 1 1 1 1]; + + file_idx = file_idx + 1; + year = 2022; month = 12; day = 3; + in_fns{file_idx} = get_filenames(in_base_path,sprintf('GPS_R_L1_%04d%02d%02d',year,month,day),'','.nc'); + in_fns_ins{file_idx} = get_filenames(in_base_path,sprintf('INS_L1_%04d%02d%02d',year,month,day),'','.nc'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); + date_str{file_idx} = sprintf('%04d%02d%02d', year, month, day); + file_type{file_idx} = 'awi_netcdf+awi_netcdf'; + gps_source{file_idx} = 'awi-final_20230127'; + sync_flag{file_idx} = 0; + params{file_idx} = struct('time_reference','utc'); + params{file_idx}.nc_field = {'TIME','LATITUDE','LONGITUDE','ALTITUDE','YEAR','MONTH','DAY'}; + params{file_idx}.nc_type = {'v','v','v','v','a','a','a'}; + params{file_idx}.types = {'sec','lat_deg','lon_deg','elev_m','year','month','day'}; + params{file_idx}.scale = [1e-3 1 1 1 1 1 1]; + params_ins{file_idx} = struct('time_reference','utc'); + params_ins{file_idx}.nc_field = {'TIME','ROLL','PITCH','THDG','YEAR','MONTH','DAY'}; + params_ins{file_idx}.nc_type = {'v','v','v','v','a','a','a'}; + params_ins{file_idx}.types = {'sec','roll_deg','pitch_deg','heading_deg','year','month','day'}; + params_ins{file_idx}.scale = [1e-3 1 1 1 1 1 1]; + + file_idx = file_idx + 1; + year = 2022; month = 12; day = 4; + in_fns{file_idx} = get_filenames(in_base_path,sprintf('GPS_R_L1_%04d%02d%02d',year,month,day),'','.nc'); + in_fns_ins{file_idx} = get_filenames(in_base_path,sprintf('INS_L1_%04d%02d%02d',year,month,day),'','.nc'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); + date_str{file_idx} = sprintf('%04d%02d%02d', year, month, day); + file_type{file_idx} = 'awi_netcdf+awi_netcdf'; + gps_source{file_idx} = 'awi-final_20230127'; + sync_flag{file_idx} = 0; + params{file_idx} = struct('time_reference','utc'); + params{file_idx}.nc_field = {'TIME','LATITUDE','LONGITUDE','ALTITUDE','YEAR','MONTH','DAY'}; + params{file_idx}.nc_type = {'v','v','v','v','a','a','a'}; + params{file_idx}.types = {'sec','lat_deg','lon_deg','elev_m','year','month','day'}; + params{file_idx}.scale = [1e-3 1 1 1 1 1 1]; + params_ins{file_idx} = struct('time_reference','utc'); + params_ins{file_idx}.nc_field = {'TIME','ROLL','PITCH','THDG','YEAR','MONTH','DAY'}; + params_ins{file_idx}.nc_type = {'v','v','v','v','a','a','a'}; + params_ins{file_idx}.types = {'sec','roll_deg','pitch_deg','heading_deg','year','month','day'}; + params_ins{file_idx}.scale = [1e-3 1 1 1 1 1 1]; + + file_idx = file_idx + 1; + year = 2022; month = 12; day = 5; + in_fns{file_idx} = get_filenames(in_base_path,sprintf('GPS_R_L1_%04d%02d%02d',year,month,day),'','.nc'); + in_fns_ins{file_idx} = get_filenames(in_base_path,sprintf('INS_L1_%04d%02d%02d',year,month,day),'','.nc'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); + date_str{file_idx} = sprintf('%04d%02d%02d', year, month, day); + file_type{file_idx} = 'awi_netcdf+awi_netcdf'; + gps_source{file_idx} = 'awi-final_20230127'; + sync_flag{file_idx} = 0; + params{file_idx} = struct('time_reference','utc'); + params{file_idx}.nc_field = {'TIME','LATITUDE','LONGITUDE','ALTITUDE','YEAR','MONTH','DAY'}; + params{file_idx}.nc_type = {'v','v','v','v','a','a','a'}; + params{file_idx}.types = {'sec','lat_deg','lon_deg','elev_m','year','month','day'}; + params{file_idx}.scale = [1e-3 1 1 1 1 1 1]; + params_ins{file_idx} = struct('time_reference','utc'); + params_ins{file_idx}.nc_field = {'TIME','ROLL','PITCH','THDG','YEAR','MONTH','DAY'}; + params_ins{file_idx}.nc_type = {'v','v','v','v','a','a','a'}; + params_ins{file_idx}.types = {'sec','roll_deg','pitch_deg','heading_deg','year','month','day'}; + params_ins{file_idx}.scale = [1e-3 1 1 1 1 1 1]; + + file_idx = file_idx + 1; + year = 2022; month = 12; day = 9; + in_fns{file_idx} = get_filenames(in_base_path,sprintf('GPS_R_L1_%04d%02d%02d',year,month,day),'','.nc'); + in_fns_ins{file_idx} = get_filenames(in_base_path,sprintf('INS_L1_%04d%02d%02d',year,month,day),'','.nc'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); + date_str{file_idx} = sprintf('%04d%02d%02d', year, month, day); + file_type{file_idx} = 'awi_netcdf+awi_netcdf'; + gps_source{file_idx} = 'awi-final_20230127'; + sync_flag{file_idx} = 0; + params{file_idx} = struct('time_reference','utc'); + params{file_idx}.nc_field = {'TIME','LATITUDE','LONGITUDE','ALTITUDE','YEAR','MONTH','DAY'}; + params{file_idx}.nc_type = {'v','v','v','v','a','a','a'}; + params{file_idx}.types = {'sec','lat_deg','lon_deg','elev_m','year','month','day'}; + params{file_idx}.scale = [1e-3 1 1 1 1 1 1]; + params_ins{file_idx} = struct('time_reference','utc'); + params_ins{file_idx}.nc_field = {'TIME','ROLL','PITCH','THDG','YEAR','MONTH','DAY'}; + params_ins{file_idx}.nc_type = {'v','v','v','v','a','a','a'}; + params_ins{file_idx}.types = {'sec','roll_deg','pitch_deg','heading_deg','year','month','day'}; + params_ins{file_idx}.scale = [1e-3 1 1 1 1 1 1]; + +end + +% ====================================================================== +% Read and translate files according to user settings +% ====================================================================== + gps_create; + +%% No GPS Data Available: Fakes GPS position information +match_idx = strmatch('gps_20160331.mat',out_fns,'exact'); +if ~isempty(match_idx) + gps_fn = fullfile(gps_path,out_fns{match_idx}); + fprintf('Creating fake gps data for %s\n', gps_fn); + gps = load(gps_fn); + gps.gps_time = datenum_to_epoch(datenum(2016,3,31) + (50000:65000)/86400); + gps.lon = -45 * ones(size(gps.gps_time)); + gps.lat = 70 + (1:length(gps.gps_time)) * 6e-4; + gps.elev = 500 * ones(size(gps.gps_time)); + gps.roll = zeros(size(gps.gps_time)); + gps.pitch = zeros(size(gps.gps_time)); + gps.heading = zeros(size(gps.gps_time)); + save(gps_fn,'-append','-struct','gps','gps_time','lat','lon','elev','roll','pitch','heading'); +end + + diff --git a/cresis-toolbox/gps/missions/gps_create_2022_Greenland_Ground.m b/cresis-toolbox/gps/missions/gps_create_2022_Greenland_Ground.m new file mode 100644 index 00000000..fc99945e --- /dev/null +++ b/cresis-toolbox/gps/missions/gps_create_2022_Greenland_Ground.m @@ -0,0 +1,239 @@ +% script make_gps_2022_Greenland_Ground +% +% Makes the GPS files for 2022_Greenland_Ground field season + +%% Setup +% ========================================================================= +tic; + +global gRadar; + +support_path = ''; +data_support_path = ''; + +if isempty(support_path) + support_path = gRadar.support_path; +end + +season_name = '2022_Greenland_Ground'; + +gps_path = fullfile(support_path,'gps',season_name); +if ~exist(gps_path,'dir') + fprintf('Making directory %s\n', gps_path); + fprintf(' Press a key to proceed\n'); + pause; + mkdir(gps_path); +end + +if isempty(data_support_path) + data_support_path = gRadar.data_support_path; +end + +in_base_path = fullfile(data_support_path,season_name); + +file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; +sync_flag = {}; sync_fns = {}; sync_file_type = {}; sync_params = {}; + +%% <== CHOOSE WHICH GPS SOURCE TO PROCESS +% gps_source_to_use = 'arena'; +gps_source_to_use = 'cresis'; + +if strcmpi(gps_source_to_use,'arena') + %% ARENA GPS SOURCE + % ======================================================================= +% + year = 2022; month = 5; day = 24; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + file_type{file_idx} = 'arena'; + params{file_idx} = struct('year',year,'time_reference','utc'); + gps_source{file_idx} = 'arena-field'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); + + year = 2022; month = 06; day = 02; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + file_type{file_idx} = 'arena'; + params{file_idx} = struct('year',year,'time_reference','utc'); + gps_source{file_idx} = 'arena-field'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); + + year = 2022; month = 06; day = 07; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + file_type{file_idx} = 'arena'; + params{file_idx} = struct('year',year,'time_reference','utc'); + gps_source{file_idx} = 'arena-field'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); + + year = 2022; month = 06; day = 13; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + file_type{file_idx} = 'arena'; + params{file_idx} = struct('year',year,'time_reference','utc'); + gps_source{file_idx} = 'arena-field'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); + + year = 2022; month = 06; day = 27; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + file_type{file_idx} = 'arena'; + params{file_idx} = struct('year',year,'time_reference','utc'); + gps_source{file_idx} = 'arena-field'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); + +elseif strcmpi(gps_source_to_use,'cresis') + %% CRESIS GPS SOURCE + % ======================================================================= + +% year = 2022; month = 5; day = 24; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('ie_%04d%02d%02d',year,month,day)),sprintf('ie_%04d%02d%02d',year,month,day),'','*.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'General_ASCII'; +% params{file_idx} = struct('time_reference','gps','headerlines',16,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); +% params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; +% params{file_idx}.textscan = {}; +% gps_source{file_idx} = 'cresis-final_20230304'; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); +% +% year = 2022; month = 6; day = 2; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('ie_%04d%02d%02d',year,month,day)),sprintf('ie_%04d%02d%02d',year,month,day),'','*.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'General_ASCII'; +% params{file_idx} = struct('time_reference','gps','headerlines',16,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); +% params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; +% params{file_idx}.textscan = {}; +% gps_source{file_idx} = 'cresis-final_20230304'; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); + + year = 2022; month = 6; day = 7; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('ie_%04d%02d%02d',year,month,day)),sprintf('ie_%04d%02d%02d',year,month,day),'','*.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'General_ASCII'; + params{file_idx} = struct('time_reference','gps','headerlines',16,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); + params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; + params{file_idx}.textscan = {}; + gps_source{file_idx} = 'cresis-final_20230304'; + date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); + +% year = 2022; month = 6; day = 13; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('ie_%04d%02d%02d',year,month,day)),sprintf('ie_%04d%02d%02d',year,month,day),'','*.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'General_ASCII'; +% params{file_idx} = struct('time_reference','gps','headerlines',16,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); +% params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; +% params{file_idx}.textscan = {}; +% gps_source{file_idx} = 'cresis-final_20230304'; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); +% +% year = 2022; month = 6; day = 27; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('ie_%04d%02d%02d',year,month,day)),sprintf('ie_%04d%02d%02d',year,month,day),'','*.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'General_ASCII'; +% params{file_idx} = struct('time_reference','gps','headerlines',16,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); +% params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; +% params{file_idx}.textscan = {}; +% gps_source{file_idx} = 'cresis-final_20230304'; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); + + +end + +%% gps_create +% Read and translate files according to user settings +% ========================================================================= +gps_create; + +%% custom fixes +% ========================================================================= +for idx = 1:length(file_type) + out_fn = fullfile(gps_path,out_fns{idx}); + + load(out_fn,'gps_source'); + if ~isempty(regexpi(gps_source,'arena')) + % Extrapolation is necessary because GPS data starts after/stops before + % the beginning/end of the radar data. + warning('Extrapolating and filtering elevation for arena GPS data: %s', out_fn); + gps = load(out_fn); + + if length(gps.lat) >= 2 + new_gps_time = [gps.gps_time(1)-10, gps.gps_time,gps.gps_time(end)+10]; + gps.lat = interp1(gps.gps_time,gps.lat,new_gps_time,'linear','extrap'); + gps.lon = interp1(gps.gps_time,gps.lon,new_gps_time,'linear','extrap'); + gps.elev = interp1(gps.gps_time,gps.elev,new_gps_time,'linear','extrap'); + gps.roll = interp1(gps.gps_time,gps.roll,new_gps_time,'linear','extrap'); + gps.pitch = interp1(gps.gps_time,gps.pitch,new_gps_time,'linear','extrap'); + gps.heading = interp1(gps.gps_time,gps.heading,new_gps_time,'linear','extrap'); + gps.gps_time = new_gps_time; + + gps.elev = fir_dec(gps.elev,ones(1,101)/101,1); + + save(out_fn,'-append','-struct','gps','gps_time','lat','lon','elev','roll','pitch','heading'); + end + end + + if ~isempty(regexpi(out_fn,'201910XX')) + % Fake GPS for testing + warning('Faking GPS data: %s', out_fn); + gps = load(out_fn); + + velocity = 4; + gps.lat = -75.5 - (gps.gps_time-gps.gps_time(1))*velocity/111111; + gps.lon(:) = -106.75; + gps.elev(:) = 500; + gps.heading(:) = -pi; + + save(out_fn,'-append','-struct','gps','gps_time','lat','lon','elev','roll','pitch','heading'); + end + +end diff --git a/cresis-toolbox/gps/missions/make_gps_2019_antarctica_TObas.m b/cresis-toolbox/gps/missions/gps_create_2022_Greenland_P3.m similarity index 53% rename from cresis-toolbox/gps/missions/make_gps_2019_antarctica_TObas.m rename to cresis-toolbox/gps/missions/gps_create_2022_Greenland_P3.m index 92c1199e..990e4598 100644 --- a/cresis-toolbox/gps/missions/make_gps_2019_antarctica_TObas.m +++ b/cresis-toolbox/gps/missions/gps_create_2022_Greenland_P3.m @@ -1,185 +1,225 @@ -% script make_gps_2019_antarctica_TObas -% -% Makes the GPS files for 2019 Antarctica TObas field season - -tic; - -global gRadar; - -support_path = ''; -data_support_path = ''; - -if isempty(support_path) - support_path = gRadar.support_path; -end - -gps_path = fullfile(support_path,'gps','2019_Antarctica_TObas'); -if ~exist(gps_path,'dir') - fprintf('Making directory %s\n', gps_path); - fprintf(' Press a key to proceed\n'); - pause; - mkdir(gps_path); -end - -if isempty(data_support_path) - data_support_path = gRadar.data_support_path; -end - -% ====================================================================== -% User Settings -% ====================================================================== -debug_level = 1; - -in_base_path = fullfile(data_support_path,'2019_Antarctica_TObas'); - -file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; -sync_flag = {}; sync_fns = {}; sync_file_type = {}; sync_params = {}; - -gps_source_to_use = 'arena'; - -if strcmpi(gps_source_to_use,'arena') - %% ARENA - -% year = 2019; month = 9; day = 23; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'arena'; -% params{file_idx} = struct('year',2019,'time_reference','utc'); -% gps_source{file_idx} = 'arena-field'; -% sync_flag{file_idx} = 1; -% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); -% sync_file_type{file_idx} = 'arena'; -% sync_params{file_idx} = struct('time_reference','utc'); - -% year = 2019; month = 12; day = 13; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'arena'; -% params{file_idx} = struct('year',2019,'time_reference','utc'); -% gps_source{file_idx} = 'arena-field'; -% sync_flag{file_idx} = 1; -% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); -% sync_file_type{file_idx} = 'arena'; -% sync_params{file_idx} = struct('time_reference','utc'); -% -% year = 2019; month = 12; day = 15; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'arena'; -% params{file_idx} = struct('year',2019,'time_reference','utc'); -% gps_source{file_idx} = 'arena-field'; -% sync_flag{file_idx} = 1; -% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); -% sync_file_type{file_idx} = 'arena'; -% sync_params{file_idx} = struct('time_reference','utc'); -% -% year = 2019; month = 12; day = 22; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'arena'; -% params{file_idx} = struct('year',2019,'time_reference','utc'); -% gps_source{file_idx} = 'arena-field'; -% sync_flag{file_idx} = 1; -% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); -% sync_file_type{file_idx} = 'arena'; -% sync_params{file_idx} = struct('time_reference','utc'); - - year = 2019; month = 12; day = 25; - file_idx = file_idx + 1; - in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - file_type{file_idx} = 'arena'; - params{file_idx} = struct('year',2019,'time_reference','utc'); - gps_source{file_idx} = 'arena-field'; - sync_flag{file_idx} = 1; - sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); - sync_file_type{file_idx} = 'arena'; - sync_params{file_idx} = struct('time_reference','utc'); - - year = 2019; month = 12; day = 26; - file_idx = file_idx + 1; - in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - file_type{file_idx} = 'arena'; - params{file_idx} = struct('year',2019,'time_reference','utc'); - gps_source{file_idx} = 'arena-field'; - sync_flag{file_idx} = 1; - sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); - sync_file_type{file_idx} = 'arena'; - sync_params{file_idx} = struct('time_reference','utc'); - - year = 2019; month = 12; day = 29; - file_idx = file_idx + 1; - in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - file_type{file_idx} = 'arena'; - params{file_idx} = struct('year',2019,'time_reference','utc'); - gps_source{file_idx} = 'arena-field'; - sync_flag{file_idx} = 1; - sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); - sync_file_type{file_idx} = 'arena'; - sync_params{file_idx} = struct('time_reference','utc'); - - year = 2019; month = 12; day = 30; - file_idx = file_idx + 1; - in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - file_type{file_idx} = 'arena'; - params{file_idx} = struct('year',2019,'time_reference','utc'); - gps_source{file_idx} = 'arena-field'; - sync_flag{file_idx} = 1; - sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); - sync_file_type{file_idx} = 'arena'; - sync_params{file_idx} = struct('time_reference','utc'); - - -end - -% ====================================================================== -% Read and translate files according to user settings -% ====================================================================== -gps_make; - -for idx = 1:length(file_type) - out_fn = fullfile(gps_path,out_fns{idx}); - - gps = load(out_fn,'gps_source'); - if regexpi(gps.gps_source,'arena') - % Extrapolation is necessary because GPS data starts after/stops before - % the beginning/end of the radar data. - warning('Extrapolating arena GPS data: %s', out_fn); - gps = load(out_fn); - - if length(gps.lat) >= 2 - new_gps_time = [gps.gps_time(1)-10, gps.gps_time,gps.gps_time(end)+10]; - gps.lat = interp1(gps.gps_time,gps.lat,new_gps_time,'linear','extrap'); - gps.lon = interp1(gps.gps_time,gps.lon,new_gps_time,'linear','extrap'); - gps.elev = interp1(gps.gps_time,gps.elev,new_gps_time,'linear','extrap'); - gps.roll = interp1(gps.gps_time,gps.roll,new_gps_time,'linear','extrap'); - gps.pitch = interp1(gps.gps_time,gps.pitch,new_gps_time,'linear','extrap'); - gps.heading = interp1(gps.gps_time,gps.heading,new_gps_time,'linear','extrap'); - gps.gps_time = new_gps_time; - - save(out_fn,'-append','-struct','gps','gps_time','lat','lon','elev','roll','pitch','heading'); - end - end - - if regexpi(out_fn,'201910XX') - % Fake GPS for testing - warning('Faking GPS data: %s', out_fn); - gps = load(out_fn); - - velocity = 4; - gps.lat = -75.5 - (gps.gps_time-gps.gps_time(1))*velocity/111111; - gps.lon(:) = -106.75; - gps.elev(:) = 500; - gps.heading(:) = -pi; - - save(out_fn,'-append','-struct','gps','gps_time','lat','lon','elev','roll','pitch','heading'); - end - -end +% script gps_create_2022_Greenland_P3 +% +% Makes the GPS files for 2022_Greenland_P3 field season + +%% Setup +% ========================================================================= +tic; + +global gRadar; + +support_path = ''; +data_support_path = ''; + +if isempty(support_path) + support_path = gRadar.support_path; +end + +season_name = '2022_Greenland_P3'; + +gps_path = fullfile(support_path,'gps',season_name); +if ~exist(gps_path,'dir') + fprintf('Making directory %s\n', gps_path); + fprintf(' Press a key to proceed\n'); + pause; + mkdir(gps_path); +end + +if isempty(data_support_path) + data_support_path = gRadar.data_support_path; +end + +in_base_path = fullfile(data_support_path,season_name); + +file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; +sync_flag = {}; sync_fns = {}; sync_file_type = {}; sync_params = {}; + +%% <== CHOOSE WHICH GPS SOURCE TO PROCESS +gps_source_to_use = 'arena'; +% gps_source_to_use = 'cresis'; + +if strcmpi(gps_source_to_use,'arena') + %% ARENA GPS SOURCE + % ======================================================================= + +% year = 2022; month = 4; day = 12; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% date_str{file_idx} = '20220412'; +% file_type{file_idx} = 'arena'; +% params{file_idx} = struct('year',year,'time_reference','utc'); +% gps_source{file_idx} = 'arena-field'; +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); + +% year = 2022; month = 4; day = 19; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% date_str{file_idx} = '20220419'; +% file_type{file_idx} = 'arena'; +% params{file_idx} = struct('year',year,'time_reference','utc'); +% gps_source{file_idx} = 'arena-field'; +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); + +% year = 2022; month = 4; day = 22; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% date_str{file_idx} = '20220422'; +% file_type{file_idx} = 'arena'; +% params{file_idx} = struct('year',year,'time_reference','utc'); +% gps_source{file_idx} = 'arena-field'; +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); + + year = 2022; month = 4; day = 27; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + date_str{file_idx} = '20220427'; + file_type{file_idx} = 'arena'; + params{file_idx} = struct('year',year,'time_reference','utc'); + gps_source{file_idx} = 'arena-field'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); + +elseif strcmpi(gps_source_to_use,'cresis') + %% CReSIS GPS SOURCE + % ======================================================================= + +% % IMU worked fine +% year = 2022; month = 4; day = 12; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('ie_%04d%02d%02d',year,month,day)),sprintf('ie_%04d%02d%02d',year,month,day),'','.OUT'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'applanix'; +% params{file_idx} = struct('time_reference','gps','year',year,'month',month,'day',day); +% gps_source{file_idx} = 'cresis-field20220421'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); + +% % IMU worked fine +% year = 2022; month = 4; day = 19; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('ie_%04d%02d%02d',year,month,day)),sprintf('ie_%04d%02d%02d',year,month,day),'','.OUT'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'applanix'; +% params{file_idx} = struct('time_reference','gps','year',year,'month',month,'day',day); +% gps_source{file_idx} = 'cresis-field20220421'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); + +% % IMU worked fine +% year = 2022; month = 4; day = 22; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('ie_%04d%02d%02d',year,month,day)),sprintf('ie_%04d%02d%02d',year,month,day),'','.OUT'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'applanix'; +% params{file_idx} = struct('time_reference','gps','year',year,'month',month,'day',day); +% gps_source{file_idx} = 'cresis-field20220421'; +% sync_flag{file_idx} = 0; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); + +% % IMU worked fine + year = 2022; month = 4; day = 27; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('ie_%04d%02d%02d',year,month,day)),sprintf('ie_%04d%02d%02d',year,month,day),'','.OUT'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'applanix'; + params{file_idx} = struct('time_reference','gps','year',year,'month',month,'day',day); + gps_source{file_idx} = 'cresis-field20220421'; + sync_flag{file_idx} = 0; + date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); + +end + +%% gps_create +% Read and translate files according to user settings +% ========================================================================= +gps_create; + +%% custom fixes +% ========================================================================= +for idx = 1:length(file_type) + out_fn = fullfile(gps_path,out_fns{idx}); + + load(out_fn,'gps_source'); + if ~isempty(regexpi(gps_source,'arena')) + % Extrapolation is necessary because GPS data starts after/stops before + % the beginning/end of the radar data. + warning('Extrapolating and filtering elevation for arena GPS data: %s', out_fn); + gps = load(out_fn); + + if length(gps.lat) >= 2 + new_gps_time = [gps.gps_time(1)-10, gps.gps_time,gps.gps_time(end)+10]; + gps.lat = interp1(gps.gps_time,gps.lat,new_gps_time,'linear','extrap'); + gps.lon = interp1(gps.gps_time,gps.lon,new_gps_time,'linear','extrap'); + gps.elev = interp1(gps.gps_time,gps.elev,new_gps_time,'linear','extrap'); + gps.roll = interp1(gps.gps_time,gps.roll,new_gps_time,'linear','extrap'); + gps.pitch = interp1(gps.gps_time,gps.pitch,new_gps_time,'linear','extrap'); + gps.heading = interp1(gps.gps_time,gps.heading,new_gps_time,'linear','extrap'); + gps.gps_time = new_gps_time; + + gps.elev = fir_dec(gps.elev,ones(1,101)/101,1); + + save(out_fn,'-append','-struct','gps','gps_time','lat','lon','elev','roll','pitch','heading'); + end + end + + if ~isempty(regexpi(out_fn,'20220419')) + % Fix bad radar time + warning('Fixing bad radar_time: %s', out_fn); + gps = load(out_fn); + + if gps.radar_time(1) == 0 + gps.radar_time = gps.radar_time(2:end); + gps.sync_elev = gps.sync_elev(2:end); + gps.sync_gps_time = gps.sync_gps_time(2:end); + gps.sync_lat = gps.sync_lat(2:end); + gps.sync_lon = gps.sync_lon(2:end); + end + save(out_fn,'-append','-struct','gps','radar_time','sync_elev','sync_gps_time','sync_lat','sync_lon'); + end + + if ~isempty(regexpi(out_fn,'201910XX')) + % Fake GPS for testing + warning('Faking GPS data: %s', out_fn); + gps = load(out_fn); + + velocity = 4; + gps.lat = -75.5 - (gps.gps_time-gps.gps_time(1))*velocity/111111; + gps.lon(:) = -106.75; + gps.elev(:) = 500; + gps.heading(:) = -pi; + + save(out_fn,'-append','-struct','gps','gps_time','lat','lon','elev','roll','pitch','heading'); + end + +end diff --git a/cresis-toolbox/gps/missions/gps_create_2022_Greenland_Polar5.m b/cresis-toolbox/gps/missions/gps_create_2022_Greenland_Polar5.m new file mode 100644 index 00000000..846a667a --- /dev/null +++ b/cresis-toolbox/gps/missions/gps_create_2022_Greenland_Polar5.m @@ -0,0 +1,232 @@ +% script gps_create_2018_Greenland_Polar5 +% +% Makes the GPS files for 2022_Greenland_Polar5 field season + +%% Setup +% ========================================================================= +tic; + +global gRadar; + +support_path = ''; +data_support_path = ''; + +if isempty(support_path) + support_path = gRadar.support_path; +end + +season_name = '2022_Greenland_Polar5'; + +gps_path = fullfile(support_path,'gps',season_name); +if ~exist(gps_path,'dir') + fprintf('Making directory %s\n', gps_path); + fprintf(' Press a key to proceed\n'); + pause; + mkdir(gps_path); +end + +if isempty(data_support_path) + data_support_path = gRadar.data_support_path; +end + +in_base_path = fullfile(data_support_path,season_name); + +file_idx = 0; in_fns = {}; in_fns_ins = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; +sync_flag = {}; sync_fns = {}; sync_file_type = {}; sync_params = {}; + +%% <== CHOOSE WHICH GPS SOURCE TO PROCESS +gps_source_to_use = 'NMEA'; +% gps_source_to_use = 'AWI'; + +if strcmpi(gps_source_to_use,'NMEA') + %% NMEA + % ======================================================================= + +% file_idx = file_idx + 1; +% year = 2022; month = 5; day = 12; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path, ... +% sprintf('%04d%02d%02d',year,month,day)),'GPS_','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; + +% file_idx = file_idx + 1; +% year = 2022; month = 6; day = 3; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path, ... +% sprintf('%04d%02d%02d',year,month,day)),'GPS_','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; + +% file_idx = file_idx + 1; +% year = 2022; month = 6; day = 7; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path, ... +% sprintf('%04d%02d%02d',year,month,day)),'GPS_','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; +% +% file_idx = file_idx + 1; +% year = 2022; month = 6; day = 8; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path, ... +% sprintf('%04d%02d%02d',year,month,day)),'GPS_','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; +% +% file_idx = file_idx + 1; +% year = 2022; month = 6; day = 9; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path, ... +% sprintf('%04d%02d%02d',year,month,day)),'GPS_','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; + +% file_idx = file_idx + 1; +% year = 2022; month = 6; day = 10; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path, ... +% sprintf('%04d%02d%02d',year,month,day)),'GPS_','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; + +% file_idx = file_idx + 1; +% year = 2022; month = 6; day = 11; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path, ... +% sprintf('%04d%02d%02d',year,month,day)),'GPS_','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; + +% file_idx = file_idx + 1; +% year = 2022; month = 6; day = 12; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path, ... +% sprintf('%04d%02d%02d',year,month,day)),'GPS_','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; + +% file_idx = file_idx + 1; +% year = 2022; month = 6; day = 13; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path, ... +% sprintf('%04d%02d%02d',year,month,day)),'GPS_','','.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); +% file_type{file_idx} = 'NMEA'; +% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); +% gps_source{file_idx} = 'nmea-field'; +% sync_flag{file_idx} = 0; + + file_idx = file_idx + 1; + year = 2022; month = 6; day = 16; + in_fns{file_idx} = get_filenames(fullfile(in_base_path, ... + sprintf('%04d%02d%02d',year,month,day)),'GPS_','','.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); + date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + file_type{file_idx} = 'NMEA'; + params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); + gps_source{file_idx} = 'nmea-field'; + sync_flag{file_idx} = 0; + + +elseif strcmpi(gps_source_to_use,'AWI') + % ======================================================================= + % AWI + % ======================================================================= + field_gps_in_base_path = fullfile(in_base_path,''); + + file_idx = file_idx + 1; + year = 2017; month = 1; day = 21; + in_fns{file_idx} = get_filenames(field_gps_in_base_path,sprintf('GPS_R_L1_%04d%02d%02d',year,month,day),'','.nc'); + in_fns_ins{file_idx} = get_filenames(in_base_path,sprintf('INS_L1_%04d%02d%02d',year,month,day),'','.nc'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); + date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + file_type{file_idx} = 'awi_netcdf+awi_netcdf'; + gps_source{file_idx} = 'awi-field'; + sync_flag{file_idx} = 0; + params{file_idx} = struct('time_reference','utc'); + params{file_idx}.nc_field = {'TIME','LATITUDE','LONGITUDE','ALTITUDE','YEAR','MONTH','DAY'}; + params{file_idx}.nc_type = {'v','v','v','v','a','a','a'}; + params{file_idx}.types = {'sec','lat_deg','lon_deg','elev_m','year','month','day'}; + params{file_idx}.scale = [1e-3 1 1 1 1 1 1]; + params_ins{file_idx} = struct('time_reference','utc'); + params_ins{file_idx}.nc_field = {'TIME','ROLL','PITCH','THDG','YEAR','MONTH','DAY'}; + params_ins{file_idx}.nc_type = {'v','v','v','v','a','a','a'}; + params_ins{file_idx}.types = {'sec','roll_deg','pitch_deg','heading_deg','year','month','day'}; + params_ins{file_idx}.scale = [1e-3 1 1 1 1 1 1]; + +elseif strcmpi(gps_source_to_use,'AWI_final') + + % ======================================================================= + % AWI_final + % ======================================================================= + + file_idx = file_idx + 1; + year = 2018; month = 1; day = 1; + in_fns{file_idx} = get_filenames(in_base_path,sprintf('GPS_R_L1_%04d%02d%02d',year,month,day),'','.nc'); + in_fns_ins{file_idx} = get_filenames(in_base_path,sprintf('INS_L1_%04d%02d%02d',year,month,day),'','.nc'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat',year,month,day); + date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + file_type{file_idx} = 'awi_netcdf+awi_netcdf'; + gps_source{file_idx} = 'awi-final_20161109'; + sync_flag{file_idx} = 0; + params{file_idx} = struct('time_reference','utc'); + params{file_idx}.nc_field = {'TIME','LATITUDE','LONGITUDE','ALTITUDE','YEAR','MONTH','DAY'}; + params{file_idx}.nc_type = {'v','v','v','v','a','a','a'}; + params{file_idx}.types = {'sec','lat_deg','lon_deg','elev_m','year','month','day'}; + params{file_idx}.scale = [1e-3 1 1 1 1 1 1]; + params_ins{file_idx} = struct('time_reference','utc'); + params_ins{file_idx}.nc_field = {'TIME','ROLL','PITCH','THDG','YEAR','MONTH','DAY'}; + params_ins{file_idx}.nc_type = {'v','v','v','v','a','a','a'}; + params_ins{file_idx}.types = {'sec','roll_deg','pitch_deg','heading_deg','year','month','day'}; + params_ins{file_idx}.scale = [1e-3 1 1 1 1 1 1]; + +end + +% ====================================================================== +% Read and translate files according to user settings +% ====================================================================== +gps_create; + +%% No GPS Data Available: Fakes GPS position information +match_idx = strmatch('gps_20220613.mat',out_fns,'exact'); +if ~isempty(match_idx) + gps_fn = fullfile(gps_path,out_fns{match_idx}); + fprintf('Creating fake gps data for %s\n', gps_fn); + gps = load(gps_fn); + gps.gps_time = datenum_to_epoch(datenum(2022,6,13) + (0:86400)/86400); + gps.lon = -45 * ones(size(gps.gps_time)); + gps.lat = 70 + (1:length(gps.gps_time)) * 6e-4; + gps.elev = 500 * ones(size(gps.gps_time)); + gps.roll = zeros(size(gps.gps_time)); + gps.pitch = zeros(size(gps.gps_time)); + gps.heading = zeros(size(gps.gps_time)); + save(gps_fn,'-append','-struct','gps','gps_time','lat','lon','elev','roll','pitch','heading'); +end + + + diff --git a/cresis-toolbox/gps/missions/gps_create_2022_Greenland_Vapor.m b/cresis-toolbox/gps/missions/gps_create_2022_Greenland_Vapor.m new file mode 100644 index 00000000..5c9609f1 --- /dev/null +++ b/cresis-toolbox/gps/missions/gps_create_2022_Greenland_Vapor.m @@ -0,0 +1,241 @@ +% script make_gps_2022_Greenland_Vapor +% +% Makes the GPS files for 2022_Greenland_Vapor field season + +%% Setup +% ========================================================================= +tic; + +global gRadar; + +support_path = ''; +data_support_path = ''; + +if isempty(support_path) + support_path = gRadar.support_path; +end + +season_name = '2022_Greenland_Vapor'; + +gps_path = fullfile(support_path,'gps',season_name); +if ~exist(gps_path,'dir') + fprintf('Making directory %s\n', gps_path); + fprintf(' Press a key to proceed\n'); + pause; + mkdir(gps_path); +end + +if isempty(data_support_path) + data_support_path = gRadar.data_support_path; +end + +in_base_path = fullfile(data_support_path,season_name); + +file_idx = 0; in_fns = {}; out_fns = {}; date_str={}; file_type = {}; params = {}; gps_source = {}; +sync_flag = {}; sync_fns = {}; sync_file_type = {}; sync_params = {}; + +%% <== CHOOSE WHICH GPS SOURCE TO PROCESS +% gps_source_to_use = 'arena'; +gps_source_to_use = 'cresis'; + +if strcmpi(gps_source_to_use,'arena') + %% ARENA GPS SOURCE + % ======================================================================= +% + year = 2022; month = 5; day = 24; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + file_type{file_idx} = 'arena'; + params{file_idx} = struct('year',year,'time_reference','utc'); + gps_source{file_idx} = 'arena-field'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); + + year = 2022; month = 06; day = 02; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + file_type{file_idx} = 'arena'; + params{file_idx} = struct('year',year,'time_reference','utc'); + gps_source{file_idx} = 'arena-field'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); + + year = 2022; month = 06; day = 07; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + file_type{file_idx} = 'arena'; + params{file_idx} = struct('year',year,'time_reference','utc'); + gps_source{file_idx} = 'arena-field'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); + + year = 2022; month = 06; day = 13; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + file_type{file_idx} = 'arena'; + params{file_idx} = struct('year',year,'time_reference','utc'); + gps_source{file_idx} = 'arena-field'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); + + year = 2022; month = 06; day = 27; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + file_type{file_idx} = 'arena'; + params{file_idx} = struct('year',year,'time_reference','utc'); + gps_source{file_idx} = 'arena-field'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); + +elseif strcmpi(gps_source_to_use,'cresis') + %% CRESIS GPS SOURCE + % ======================================================================= + +% year = 2022; month = 5; day = 24; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('ie_%04d%02d%02d',year,month,day)),sprintf('ie_%04d%02d%02d',year,month,day),'','*.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'General_ASCII'; +% params{file_idx} = struct('time_reference','gps','headerlines',16,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); +% params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; +% params{file_idx}.textscan = {}; +% gps_source{file_idx} = 'cresis-final_20230304'; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); +% +% year = 2022; month = 6; day = 2; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('ie_%04d%02d%02d',year,month,day)),sprintf('ie_%04d%02d%02d',year,month,day),'','*.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'General_ASCII'; +% params{file_idx} = struct('time_reference','gps','headerlines',16,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); +% params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; +% params{file_idx}.textscan = {}; +% gps_source{file_idx} = 'cresis-final_20230304'; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); + + year = 2022; month = 8; day = 01; + file_idx = file_idx + 1; + %in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('ie_%04d%02d%02d',year,month,day)),sprintf('ie_%04d%02d%02d',year,month,day),'','*.txt'); + fn_start='ie_log000'; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('ie_%04d%02d%02d',year,month,day)),fn_start,'','*.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + file_type{file_idx} = 'General_ASCII'; + params{file_idx} = struct('time_reference','gps','headerlines',16,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); + params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; + params{file_idx}.textscan = {}; + gps_source{file_idx} = 'cresis-final_20221005'; + date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); + sync_flag{file_idx} = 0; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); + +% year = 2022; month = 6; day = 13; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('ie_%04d%02d%02d',year,month,day)),sprintf('ie_%04d%02d%02d',year,month,day),'','*.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'General_ASCII'; +% params{file_idx} = struct('time_reference','gps','headerlines',16,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); +% params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; +% params{file_idx}.textscan = {}; +% gps_source{file_idx} = 'cresis-final_20230304'; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); +% +% year = 2022; month = 6; day = 27; +% file_idx = file_idx + 1; +% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('ie_%04d%02d%02d',year,month,day)),sprintf('ie_%04d%02d%02d',year,month,day),'','*.txt'); +% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); +% file_type{file_idx} = 'General_ASCII'; +% params{file_idx} = struct('time_reference','gps','headerlines',16,'format_str','%s%s%f%f%f%f%f%f%f%f%f'); +% params{file_idx}.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; +% params{file_idx}.textscan = {}; +% gps_source{file_idx} = 'cresis-final_20230304'; +% date_str{file_idx} = sprintf('%04d%02d%02d',year,month,day); +% sync_flag{file_idx} = 1; +% sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); +% sync_file_type{file_idx} = 'arena'; +% sync_params{file_idx} = struct('time_reference','utc'); + + +end + +%% gps_create +% Read and translate files according to user settings +% ========================================================================= +gps_create; + +%% custom fixes +% ========================================================================= +for idx = 1:length(file_type) + out_fn = fullfile(gps_path,out_fns{idx}); + + load(out_fn,'gps_source'); + if ~isempty(regexpi(gps_source,'arena')) + % Extrapolation is necessary because GPS data starts after/stops before + % the beginning/end of the radar data. + warning('Extrapolating and filtering elevation for arena GPS data: %s', out_fn); + gps = load(out_fn); + + if length(gps.lat) >= 2 + new_gps_time = [gps.gps_time(1)-10, gps.gps_time,gps.gps_time(end)+10]; + gps.lat = interp1(gps.gps_time,gps.lat,new_gps_time,'linear','extrap'); + gps.lon = interp1(gps.gps_time,gps.lon,new_gps_time,'linear','extrap'); + gps.elev = interp1(gps.gps_time,gps.elev,new_gps_time,'linear','extrap'); + gps.roll = interp1(gps.gps_time,gps.roll,new_gps_time,'linear','extrap'); + gps.pitch = interp1(gps.gps_time,gps.pitch,new_gps_time,'linear','extrap'); + gps.heading = interp1(gps.gps_time,gps.heading,new_gps_time,'linear','extrap'); + gps.gps_time = new_gps_time; + + gps.elev = fir_dec(gps.elev,ones(1,101)/101,1); + + save(out_fn,'-append','-struct','gps','gps_time','lat','lon','elev','roll','pitch','heading'); + end + end + + if ~isempty(regexpi(out_fn,'201910XX')) + % Fake GPS for testing + warning('Faking GPS data: %s', out_fn); + gps = load(out_fn); + + velocity = 4; + gps.lat = -75.5 - (gps.gps_time-gps.gps_time(1))*velocity/111111; + gps.lon(:) = -106.75; + gps.elev(:) = 500; + gps.heading(:) = -pi; + + save(out_fn,'-append','-struct','gps','gps_time','lat','lon','elev','roll','pitch','heading'); + end + +end diff --git a/cresis-toolbox/gps/missions/gps_create_2023_Greenland_Ground.m b/cresis-toolbox/gps/missions/gps_create_2023_Greenland_Ground.m new file mode 100644 index 00000000..4d015d83 --- /dev/null +++ b/cresis-toolbox/gps/missions/gps_create_2023_Greenland_Ground.m @@ -0,0 +1,140 @@ +% script gps_create_2023_Greenland_Ground +% +% Makes the GPS files for 2023_Greenland_Ground field season + +%% Setup +% ========================================================================= +tic; + +global gRadar; + +support_path = ''; +data_support_path = ''; + +if isempty(support_path) + support_path = gRadar.support_path; +end + +season_name = '2023_Greenland_Ground'; + +gps_path = fullfile(support_path,'gps',season_name); +if ~exist(gps_path,'dir') + fprintf('Making directory %s\n', gps_path); + fprintf(' Press a key to proceed\n'); + pause; + mkdir(gps_path); +end + +if isempty(data_support_path) + data_support_path = gRadar.data_support_path; +end + +in_base_path = fullfile(data_support_path,season_name); + +file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; +sync_flag = {}; sync_fns = {}; sync_file_type = {}; sync_params = {}; + +%% <== CHOOSE WHICH GPS SOURCE TO PROCESS +gps_source_to_use = 'arena'; +% gps_source_to_use = 'cresis'; + +if strcmpi(gps_source_to_use,'arena') + %% ARENA GPS SOURCE + % ======================================================================= + + year = 2023; month = 4; day = 7; + file_idx = file_idx + 1; + in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); + date_str{file_idx} = sprintf('%04d%02d%02d', year, month, day); + file_type{file_idx} = 'arena'; + params{file_idx} = struct('time_reference','utc'); + gps_source{file_idx} = 'arena-field'; + sync_flag{file_idx} = 1; + sync_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'','','gps.txt'); + sync_file_type{file_idx} = 'arena'; + sync_params{file_idx} = struct('time_reference','utc'); + +elseif strcmpi(gps_source_to_use,'cresis') + %% CReSIS GPS SOURCE + % ======================================================================= + + +end + +%% gps_create +% Read and translate files according to user settings +% ========================================================================= +gps_create; + +%% custom fixes +% ========================================================================= +for idx = 1:length(file_type) + out_fn = fullfile(gps_path,out_fns{idx}); + + load(out_fn,'gps_source'); + if ~isempty(regexpi(gps_source,'arena')) + % Extrapolation is necessary because GPS data starts after/stops before + % the beginning/end of the radar data. + warning('Extrapolating and filtering elevation for arena GPS data: %s', out_fn); + gps = load(out_fn); + + if length(gps.lat) >= 2 + new_gps_time = [gps.gps_time(1)-10, gps.gps_time,gps.gps_time(end)+10]; + gps.lat = interp1(gps.gps_time,gps.lat,new_gps_time,'linear','extrap'); + gps.lon = interp1(gps.gps_time,gps.lon,new_gps_time,'linear','extrap'); + gps.elev = interp1(gps.gps_time,gps.elev,new_gps_time,'linear','extrap'); + gps.roll = interp1(gps.gps_time,gps.roll,new_gps_time,'linear','extrap'); + gps.pitch = interp1(gps.gps_time,gps.pitch,new_gps_time,'linear','extrap'); + gps.heading = interp1(gps.gps_time,gps.heading,new_gps_time,'linear','extrap'); + gps.gps_time = new_gps_time; + + gps.elev = fir_dec(gps.elev,ones(1,101)/101,1); + + save(out_fn,'-append','-struct','gps','gps_time','lat','lon','elev','roll','pitch','heading'); + end + end + + if ~isempty(regexpi(gps_source,'arena')) && ~isempty(regexpi(out_fn,'20221209')) + [out_fn_dir,out_fn_name,out_fn_ext] = fileparts(out_fn); + new_out_fn = fullfile(out_fn_dir,[out_fn_name(1:end-2),'10',out_fn_ext]); + warning('Copying 20221209 to 20221210:\n %s\n %s\n', out_fn, new_out_fn); + copyfile(out_fn,new_out_fn); + new_out_fn = fullfile(out_fn_dir,[out_fn_name(1:end-2),'11',out_fn_ext]); + warning('Copying 20221209 to 20221211:\n %s\n %s\n', out_fn, new_out_fn); + copyfile(out_fn,new_out_fn); + new_out_fn = fullfile(out_fn_dir,[out_fn_name(1:end-2),'12',out_fn_ext]); + warning('Copying 20221209 to 20221212:\n %s\n %s\n', out_fn, new_out_fn); + copyfile(out_fn,new_out_fn); + end + + if ~isempty(regexpi(out_fn,'20220419')) + % Fix bad radar time + warning('Fixing bad radar_time: %s', out_fn); + gps = load(out_fn); + + if gps.radar_time(1) == 0 + gps.radar_time = gps.radar_time(2:end); + gps.sync_elev = gps.sync_elev(2:end); + gps.sync_gps_time = gps.sync_gps_time(2:end); + gps.sync_lat = gps.sync_lat(2:end); + gps.sync_lon = gps.sync_lon(2:end); + end + save(out_fn,'-append','-struct','gps','radar_time','sync_elev','sync_gps_time','sync_lat','sync_lon'); + end + + if ~isempty(regexpi(out_fn,'201910XX')) + % Fake GPS for testing + warning('Faking GPS data: %s', out_fn); + gps = load(out_fn); + + velocity = 4; + gps.lat = -75.5 - (gps.gps_time-gps.gps_time(1))*velocity/111111; + gps.lon(:) = -106.75; + gps.elev(:) = 500; + gps.heading(:) = -pi; + + save(out_fn,'-append','-struct','gps','gps_time','lat','lon','elev','roll','pitch','heading'); + end + +end diff --git a/cresis-toolbox/gps/missions/make_gps_2019_southdakota_CESSNA.m b/cresis-toolbox/gps/missions/make_gps_2019_southdakota_CESSNA.m deleted file mode 100644 index e9dfe646..00000000 --- a/cresis-toolbox/gps/missions/make_gps_2019_southdakota_CESSNA.m +++ /dev/null @@ -1,103 +0,0 @@ -% script make_gps_2019_southdakota_CESSNA -% -% Makes the GPS files for 2018 South Dakota CESSNA field season - -tic; - -global gRadar; - -support_path = ''; -data_support_path = ''; - -if isempty(support_path) - support_path = gRadar.support_path; -end - -gps_path = fullfile(support_path,'gps','2019_SouthDakota_CESSNA'); -if ~exist(gps_path,'dir') - fprintf('Making directory %s\n', gps_path); - fprintf(' Press a key to proceed\n'); - pause;plo - mkdir(gps_path); -end - -if isempty(data_support_path) - data_support_path = gRadar.data_support_path; -end - -% ====================================================================== -% User Settings -% ====================================================================== -debug_level = 1; - -in_base_path = fullfile(data_support_path,'2020_SouthDakota_CESSNA'); - -file_idx = 0; in_fns = {}; out_fns = {}; file_type = {}; params = {}; gps_source = {}; -sync_fns = {}; sync_params = {}; - -gps_source_to_use = 'NMEA'; -if strcmpi(gps_source_to_use,'NMEA') - - year = 2020; month = 01; day = 29; - file_idx = file_idx + 1; - in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'nmea','','.gps'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - file_type{file_idx} = 'NMEA'; - params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); - gps_source{file_idx} = 'nmea-field'; - sync_flag{file_idx} = 0; - - year = 2020; month = 01; day = 28; - file_idx = file_idx + 1; - in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'nmea','','.gps'); - out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); - file_type{file_idx} = 'NMEA'; - params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); - gps_source{file_idx} = 'nmea-field'; - sync_flag{file_idx} = 0; - -% year = 2020; month = 01; day = 16; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'nmea','','.gps'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'NMEA'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% gps_source{file_idx} = 'nmea-field'; -% sync_flag{file_idx} = 0; - -% year = 2019; month = 12; day = 11 ; -% file_idx = file_idx + 1; -% in_fns{file_idx} = get_filenames(fullfile(in_base_path,sprintf('%04d%02d%02d',year,month,day)),'nmea','','.gps'); -% out_fns{file_idx} = sprintf('gps_%04d%02d%02d.mat', year, month, day); -% file_type{file_idx} = 'NMEA'; -% params{file_idx} = struct('year',year,'month',month,'day',day,'format',1,'time_reference','utc'); -% gps_source{file_idx} = 'nmea-field'; -% sync_flag{file_idx} = 0; - - -end - -% ====================================================================== -% Read and translate files according to user settings -% ====================================================================== -gps_make; - -for idx = 1:length(file_type) - out_fn = fullfile(gps_path,out_fns{idx}); - - gps = load(out_fn); - if regexpi(gps.gps_source,'nmea') - warning('Making monotonic gps time: %s', out_fn); - [gps,error_flag] = make_gps_monotonic(gps); - - warning('Smoothing elevation and heading data: %s', out_fn); - gps.elev = sgolayfilt(gps.elev,2,101); % Adjust filter length as needed to remove high frequency noise - heading_x = cos(gps.heading); - heading_y = sin(gps.heading); - heading_x = sgolayfilt(heading_x,2,101); % Adjust filter length as needed to remove high frequency noise - heading_y = sgolayfilt(heading_y,2,101); % Adjust filter length as needed to remove high frequency noise - gps.heading = atan2(heading_y,heading_x); - - save(out_fn,'-append','-struct','gps','elev','heading'); - end -end diff --git a/cresis-toolbox/gps/read_gps_applanix.m b/cresis-toolbox/gps/read_gps_applanix.m index b22669fe..94bc375f 100644 --- a/cresis-toolbox/gps/read_gps_applanix.m +++ b/cresis-toolbox/gps/read_gps_applanix.m @@ -74,7 +74,7 @@ % % Author: John Paden % -% See also read_gps_*.m, gps_plot.m, gps_make.m +% See also read_gps_*.m, gps_plot.m, gps_create.m if ~isfield(param,'roll_byte_offset') || isempty(param.roll_byte_offset) param.roll_byte_offset = 56; diff --git a/cresis-toolbox/gps/read_gps_arena.m b/cresis-toolbox/gps/read_gps_arena.m index e2344756..6a62694d 100644 --- a/cresis-toolbox/gps/read_gps_arena.m +++ b/cresis-toolbox/gps/read_gps_arena.m @@ -51,8 +51,116 @@ % % Author: John Paden % -% See also read_gps_*.m, gps_plot.m, gps_make.m +% See also read_gps_*.m, gps_plot.m, gps_create.m +%% NMEA String Format +% ============================================================================= +% NMEA-0183 message: GGA +% Related Topics +% NMEA-0183 messages: Overview +% Time, position, and fix related data +% An example of the GGA message string is: +% +% $GPGGA,172814.0,3723.46587704,N,12202.26957864,W,2,6,1.2,18.893,M,-25.669,M,2.0,0031*4F +% +% Note: The data string exceeds the NMEA standard length. +% +% GGA message fields +% Field Meaning +% 0 Message ID $GPGGA +% 1 UTC of position fix +% 2 Latitude +% 3 Direction of latitude: +% N: North +% S: South +% 4 Longitude +% 5 Direction of longitude: +% E: East +% W: West +% 6 GPS Quality indicator: +% 0: Fix not valid +% 1: GPS fix +% 2: Differential GPS fix, OmniSTAR VBS +% 4: Real-Time Kinematic, fixed integers +% 5: Real-Time Kinematic, float integers, OmniSTAR XP/HP or Location RTK +% 7 Number of SVs in use, range from 00 through to 24+ +% 8 HDOP +% 9 Orthometric height (MSL reference) +% 10 M: unit of measure for orthometric height is meters +% 11 Geoid separation +% 12 M: geoid separation measured in meters +% 13 Age of differential GPS data record, Type 1 or Type 9. Null field when DGPS is not used. +% 14 Reference station ID, range 0000-4095. A null field when any reference station ID is selected and no corrections are received1. +% 15 +% The checksum data, always begins with * +% +% Note: If a user-defined geoid model, or an inclined plane is loaded into the receiver, then the height output in the NMEA GGA string is always the orthometric height (height above a geoid). The orthometric height is output even if no user-defined geoid is loaded (there is a simplified default geoid in the receiver), or if a user-defined geoid is loaded, or if an inclined plane is used. +% +% ============================================================================= +% NMEA-0183 message: RMC +% Related Topics +% NMEA-0183 messages: Overview +% Position, velocity, and time +% The RMC string is: +% +% $GPRMC,123519,A,4807.038,N,01131.000,E,022.4,084.4,230394,003.1,W*6A +% +% GPRMC message fields +% Field Meaning +% 0 Message ID $GPRMC +% 1 UTC of position fix +% 2 Status A=active or V=void +% 3 Latitude +% 4 Longitude +% 5 Speed over the ground in knots +% 6 Track angle in degrees (True) +% 7 Date +% 8 Magnetic variation in degrees +% 9 The checksum data, always begins with * +% +% ============================================================================= +% NMEA-0183 message: ZDA +% Related Topics +% NMEA-0183 messages: Overview +% Date and time +% The ZDA string is: +% +% $GPZDA,034558.00,09,12,2022,,*61 +% +% GPZDA message fields +% Field Meaning +% 0 Message ID $GPZDA +% 1 UTC of position fix +% 2 Day +% 3 Month +% 4 Year +% 5 +% 6 The checksum data, always begins with * +% +% ============================================================================= +% NMEA-0183 message: VTG +% Related Topics +% NMEA-0183 messages: Overview +% Track made good and speed over ground +% An example of the VTG message string is: +% +% $GPVTG,,T,,M,0.00,N,0.00,K*4E +% +% VTG message fields +% Field Meaning +% 0 Message ID $GPVTG +% 1 Track made good (degrees true) +% 2 T: track made good is relative to true north +% 3 Track made good (degrees magnetic) +% 4 M: track made good is relative to magnetic north +% 5 Speed, in knots +% 6 N: speed is measured in knots +% 7 Speed over ground in kilometers/hour (kph) +% 8 K: speed over ground is measured in kph +% 9 The checksum data, always begins with * +% ============================================================================= + +%% Input checks if ~exist('param','var') || isempty(param) error('Year, month, day must be specified in param struct'); end @@ -60,11 +168,13 @@ param.clk = 10e6; end +%% Open file [fid,msg] = fopen(fn,'r'); if fid < 0 error('Error opening %s: %s', fn, msg); end +%% Process file line by line UTC_time_file = []; latitude = []; N_S = []; @@ -77,13 +187,19 @@ gps_date = []; heading = []; +% $GPZDA,034558.00,09,12,2022,,*61 +format_str_GPZDA = '%s%f%s%s%s%s%s'; +% $GPRMC,123519,A,4807.038,N,01131.000,E,022.4,084.4,230394,003.1,W*6A format_str_GPRMC = '%s%f%s%f%c%f%c%f%f%f%f%s%s'; +% $GPGGA,194325.010,3857.135037,N,09515.861757,W,1,9,0.88,311.412,M,-29.504,M,,*65 format_str_GPGGA = '%s%f%f%c%f%c%u%u%f%f%c%f%c%s%s'; nmea_idx = 1; relTimeCntrTmp = NaN; profileCntrTmp = NaN; ppsCntrTmp = NaN; heading_tmp = NaN; +gps_date_tmp = NaN; +gps_date_time_tmp = NaN; line_num = 0; while ~feof(fid) str = fgets(fid); @@ -94,7 +210,7 @@ GPGGA_str = remain(2:end); C = textscan(GPGGA_str,format_str_GPGGA,'delimiter',', ','emptyvalue',NaN); [tag,UTC_time_file_tmp,latitude_tmp,N_S_tmp,longitude_tmp,E_W_tmp,fix,NoSatelite,dilution,... - altitude_tmp,alt_unit,geode_ref,geode_unit,dgps,checksum] = deal(C{:}); + altitude_tmp,alt_unit,geoid_ref,geoid_unit,dgps,checksum] = deal(C{:}); if ~isnan(relTimeCntrTmp) && ~isnan(profileCntrTmp) && ~isnan(ppsCntrTmp) if nmea_idx > 1 @@ -105,7 +221,7 @@ fprintf(2, ' RADAR REPEAT LINE %d: %.0f\n', line_num, relTimeCntrTmp); end if relTimeCntrTmp < relTimeCntr(nmea_idx-1) - warning(2, 'Radar time is not monotonic on line %d: %.0f <= %.0f\n', line_num, relTimeCntrTmp, relTimeCntr(nmea_idx-1)); + fprintf(2, 'Radar time is not monotonic on line %d: %.0f <= %.0f\n', line_num, relTimeCntrTmp, relTimeCntr(nmea_idx-1)); end end @@ -121,11 +237,18 @@ longitude(nmea_idx) = longitude_tmp; E_W(nmea_idx) = E_W_tmp; altitude(nmea_idx) = altitude_tmp; + if ~isempty(geoid_ref) && isfinite(geoid_ref) + altitude(nmea_idx) = altitude(nmea_idx) + geoid_ref; + end relTimeCntr(nmea_idx) = relTimeCntrTmp; profileCntr(nmea_idx) = profileCntrTmp; ppsCntr(nmea_idx) = ppsCntrTmp; heading(nmea_idx) = heading_tmp; - gps_date(nmea_idx) = NaN; + if UTC_time_file(nmea_idx) == gps_date_time_tmp + gps_date(nmea_idx) = gps_date_tmp; + else + gps_date(nmea_idx) = NaN; + end nmea_idx = nmea_idx + 1; relTimeCntrTmp = NaN; profileCntrTmp = NaN; @@ -135,21 +258,56 @@ fprintf(2, ' BAD LINE %d: %s\n', line_num, GPGGA_str(GPGGA_str ~= 10)); end end + elseif numel(str)>=11 && strcmp(str(7:11),'GPZDA') + C = textscan(remain(2:end),format_str_GPZDA,'delimiter',', ','emptyvalue',NaN); + [tag,gps_date_time_tmp,day_tmp,month_tmp,year_tmp,unknown,checksum] = deal(C{:}); + heading_tmp = NaN; % Do not use (low quality) + if nmea_idx > 1 + % The GPZDA string may come before or after the corresponding GPZDA + % string with the same time stamp. + % + % If it comes afterwards, this string is just updating fields that + % the GPGGA string already provided. The most important fields are + % the day, month, and year, which the GPGGA does not provide. We + % __think__ that all the other fields should be the same as what + % were in the GPGGA string. + % + % If GPZDA comes before the GPGGA string, we do nothing now, but + % use the gps_date_tmp and gps_date_time_tmp fields to update the + % next GPGGA string when it comes. + if ~isempty(day_tmp{1}) && ~isempty(month_tmp{1}) && ~isempty(year_tmp{1}) + gps_date_tmp = str2double([day_tmp{1},month_tmp{1},year_tmp{1}(3:4)]); + if UTC_time_file(nmea_idx-1) == gps_date_time_tmp + gps_date(nmea_idx-1) = gps_date_tmp; + gps_date_tmp = NaN; + gps_date_time_tmp = NaN; + % fprintf(2, ' GPZDA WITH DIFFERENT TIME THAN LAST GPGGA LINE %d: %.14g ~= %.14g\n', line_num, UTC_time_file_tmp, UTC_time_file(nmea_idx-1)); + end + end + end elseif numel(str)>=11 && strcmp(str(7:11),'GPRMC') C = textscan(remain(2:end),format_str_GPRMC,'delimiter',', ','emptyvalue',NaN); - [tag,UTC_time_file_tmp,nav_rx_warning,latitude_tmp,N_S_tmp,longitude_tmp,E_W_tmp,speed,heading_tmp,... + [tag,gps_date_time_tmp,nav_rx_warning,latitude_tmp,N_S_tmp,longitude_tmp,E_W_tmp,speed,heading_tmp,... gps_date_tmp,mag_Var,mag_var_E_W,checksum] = deal(C{:}); + heading_tmp = NaN; % Do not use (low quality) if nmea_idx > 1 - % We __think__ that the GPRMC string always comes after the - % corresponding GPGGA string with the same time stamp. Therefore, - % this string is just updating fields that the GPGGA string already - % provided. The most important field is the gps_date which the - % GPGGA does not provide. We __think__ that all the other fields - % should be the same as what were in the GPGGA string. - if UTC_time_file(nmea_idx-1) == UTC_time_file_tmp + % The GPRMC string may come before or after the corresponding GPGGA + % string with the same time stamp. + % + % If it comes afterwards, this string is just updating fields that + % the GPGGA string already provided. The most important field is + % the gps_date which the GPGGA does not provide. We __think__ that + % all the other fields should be the same as what were in the GPGGA + % string. + % + % If GPRMC comes before the GPGGA string, we do nothing now, but + % use the gps_date_tmp and gps_date_time_tmp fields to update the + % next GPGGA string when it comes. + if UTC_time_file(nmea_idx-1) == gps_date_time_tmp gps_date(nmea_idx-1) = gps_date_tmp; - else - fprintf(2, ' GPRMC WITH DIFFERENT TIME THAN LAST GPGGA LINE %d: %.14g ~= %.14g\n', line_num, UTC_time_file_tmp, UTC_time_file(nmea_idx-1)); + gps_date_tmp = NaN; + gps_date_time_tmp = NaN; + % fprintf(2, ' GPRMC WITH DIFFERENT TIME THAN LAST GPGGA LINE %d: %.14g ~= %.14g\n', line_num, UTC_time_file_tmp, UTC_time_file(nmea_idx-1)); end end end @@ -163,6 +321,8 @@ end fclose(fid); +%% Parse file contents + if nmea_idx < 2 gps.gps_time = []; gps.lat = []; @@ -246,8 +406,7 @@ end end -% =================================================================== -% Store outputs in structure +%% Store outputs in structure % =================================================================== if strcmpi(param.time_reference,'utc') % UTC time stored in file, so need to add leap seconds back in diff --git a/cresis-toolbox/gps/read_gps_arena_cpu_time.m b/cresis-toolbox/gps/read_gps_arena_cpu_time.m index 6e02eb83..557bb355 100644 --- a/cresis-toolbox/gps/read_gps_arena_cpu_time.m +++ b/cresis-toolbox/gps/read_gps_arena_cpu_time.m @@ -46,7 +46,7 @@ % % Author: John Paden % -% See also read_gps_*.m, gps_plot.m, gps_make.m +% See also read_gps_*.m, gps_plot.m, gps_create.m if ~isfield(param,'clk') || isempty(param.clk) param.clk = 10e6; diff --git a/cresis-toolbox/gps/read_gps_cresis.m b/cresis-toolbox/gps/read_gps_cresis.m index f4d6d6ac..4133ebc6 100644 --- a/cresis-toolbox/gps/read_gps_cresis.m +++ b/cresis-toolbox/gps/read_gps_cresis.m @@ -34,7 +34,7 @@ % % Author: John Paden % -% See also read_gps_*.m, gps_plot.m, gps_make.m +% See also read_gps_*.m, gps_plot.m, gps_create.m [fid,msg] = fopen(fn,'r'); diff --git a/cresis-toolbox/gps/read_gps_csv.m b/cresis-toolbox/gps/read_gps_csv.m index 5da022a9..bf54b759 100644 --- a/cresis-toolbox/gps/read_gps_csv.m +++ b/cresis-toolbox/gps/read_gps_csv.m @@ -43,7 +43,7 @@ % % Author: John Paden % -% See also read_gps_*.m, gps_plot.m, gps_make.m +% See also read_gps_*.m, gps_plot.m, gps_create.m debug_level = 1; diff --git a/cresis-toolbox/gps/read_gps_dmsraw.m b/cresis-toolbox/gps/read_gps_dmsraw.m index c3f385b9..b09b26d7 100644 --- a/cresis-toolbox/gps/read_gps_dmsraw.m +++ b/cresis-toolbox/gps/read_gps_dmsraw.m @@ -17,7 +17,7 @@ % % Author: Huan Zhao, John Paden % -% See also read_gps_*.m, gps_plot.m, gps_make.m +% See also read_gps_*.m, gps_plot.m, gps_create.m [fid,msg] = fopen(fn,'r'); if fid < 1 diff --git a/cresis-toolbox/gps/read_gps_general_ascii.m b/cresis-toolbox/gps/read_gps_general_ascii.m index ecfc990a..96e65bd1 100644 --- a/cresis-toolbox/gps/read_gps_general_ascii.m +++ b/cresis-toolbox/gps/read_gps_general_ascii.m @@ -124,6 +124,57 @@ % param.day = 10; % gps = read_gps_general_ascii(fn,param); % +% % Example 8: CReSIS/University of Kansas Novatel Inertial Explorer Custom Output without IMU (2019_Antarctica_Ground) +% +% Date GPSTime Latitude Longitude H-Ell Roll Pitch Heading SDNorth SDEast SDHeight +% (YMD) (HMS) (Deg) (Deg) (m) (Deg) (Deg) (Deg) (m) (m) (m) +% 2020/01/06 21:04:35.03 -86.1831485554 -107.5646197725 2581.715 -0.7619420000 -2.9489170000 32.0370050000 0.003 0.003 0.006 +% +% fn = 'ie_20200106_01_ver1.txt'; +% param = []; +% param.format_str = '%s%s%f%f%f%f%f%f%f%f%f'; +% param.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; +% param.textscan = {}; +% param.headerlines = 15; +% param.time_reference = 'gps'; +% gps = read_gps_general_ascii(fn,param); +% +% % Example 9: CReSIS/University of Kansas Novatel Inertial Explorer Custom Output with IMU (2019_Antarctica_Ground) +% +% Date GPSTime Latitude Longitude H-Ell Roll Pitch Heading SDNorth SDEast SDHeight +% (YMD) (HMS) (Deg) (Deg) (m) (Deg) (Deg) (Deg) (m) (m) (m) +% 2020/01/06 21:04:35.03 -86.1831485554 -107.5646197725 2581.715 -0.7619420000 -2.9489170000 32.0370050000 0.003 0.003 0.006 +% +% fn = 'ie_20200106_01_ver2.txt'; +% param = []; +% param.format_str = '%s%s%f%f%f%f%f%f%f%f%f'; +% param.types = {'date_MDY','time_HMS','lat_deg','lon_deg','elev_m','roll_deg','pitch_deg','heading_deg','tmp_1','tmp_2','tmp_3'}; +% param.textscan = {}; +% param.headerlines = 21; +% param.time_reference = 'gps'; +% gps = read_gps_general_ascii(fn,param); +% +% % Example 10: Gulfstream V (GV) IWG1 file with IMU (2019_Antarctica_GV) +% +% IWG1 string,Date/Time(utc),Latitude(deg),Longitude(deg),GPS_MSL_Alt(m),WGS_84_Alt(m),Press_Alt(ft),Radar_Alt(ft)',... +% Grnd_Spd(m/s),True_Airspeed(m/s),Indicated_Airspeed(knots),Mach_Number,Ver_Velocity(m/s),Heading(deg),... +% Track(deg),Drift(deg),Pitch(deg),Roll(deg),Side_slip(deg),'Angle_of_Attach(deg),Ambient_Temp(degc),... +% Dew_Tmp(degc),Total_Tmp(degc),Static_Press(mbar),Dynamic_Press(mbar),Cabin_press(mbar),Wind_Speed(m/s),... +% Wind_Dir(deg),Vert_Wind_Speed(m/s),Solar_Zenith(deg),Sun_Ele_AC(deg),Sun_Az_Grd(deg),Sun_Az_AC(deg) +% IWG1,2019-10-17T14:10:33.003,,,,,-89.0,,,0.0,30.0,0.031,,,,,,,,,14.0,,14.2,30.0,1.8,1019.5,,,,,,1571320712.8, +% +% fn = '/cresis/snfs1/dataproducts/metadata/2019_Antarctica_GV/IWG1_17Oct2019-2224.txt'; +% param = []; +% param.format_str = '%s%s%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f%f'; +% param.types = {'IWG1','date_time','lat_deg','lon_deg','gps_msl_alt','WGS_84_alt','press_alt',... +% 'radar_alt','grnd_spd','true_airspeed','indicated_airspeed','Mach_number','vert_velocity','heading_deg',... +% 'track','drift','pitch_deg','roll_deg','pitch_deg'}; +% param.date_time_format = 'yyyy-mm-ddTHH:MM:SS.FFF'; +% param.textscan = {'delimiter',','}; +% param.headerlines = 0; +% param.time_reference = 'utc'; +% gps = read_gps_general_ascii(fn,param); + % Author: John Paden [fid,msg] = fopen(fn,'r'); @@ -177,6 +228,24 @@ minute = []; sec = []; +if isfield(tmp_gps,'date_time') + % Handles all the standard date-time formats including: + % yyyy/mm/dd HH:MM:SS, mm/dd/yyyy HH:MM:SS + datenums = zeros(size(tmp_gps.date_time)); + for row=1:length(tmp_gps.date_time) + try + if isfield(param,'date_time_format') + datenums(row) = datenum(tmp_gps.date_time{row},param.date_time_format); + else + datenums(row) = datenum(tmp_gps.date_time{row}); + end + catch ME + warning('Row %d failed: %s\n', row, ME.getReport); + datenums(row) = NaN; + end + end + [year,month,day,hour,minute,sec] = datevec(datenums); +end if isfield(tmp_gps,'date_MDY') % Handles all the standard date formats including: % yyyy/mm/dd, mm/dd/yyyy @@ -209,24 +278,6 @@ end [~,~,~,hour,minute,sec] = datevec(datenums); end -if isfield(tmp_gps,'IWG1') - % Handles date-time in aircraft IWG1 string (2019_Antarctica_GV test flight on 20191017) - tmp_gps.year = size(tmp_gps.IWG1); - tmp_gps.month = size(tmp_gps.IWG1); - tmp_gps.day = size(tmp_gps.IWG1); - tmp_gps.hour = size(tmp_gps.IWG1); - tmp_gps.minute = size(tmp_gps.IWG1); - tmp_gps.sec = size(tmp_gps.IWG1); - for row=1:length(tmp_gps.IWG1) - tmp_gps.year(row) = str2num(tmp_gps.date_time_sec{row}(1:4)); - tmp_gps.month(row) = str2num(tmp_gps.date_time_sec{row}(6:7)); - tmp_gps.day(row) = str2num(tmp_gps.date_time_sec{row}(9:10)); - tmp_gps.hour(row) = str2num(tmp_gps.date_time_sec{row}(12:13)); - tmp_gps.minute(row) = str2num(tmp_gps.date_time_sec{row}(15:16)); - tmp_gps.sec(row) = str2num(tmp_gps.date_time_sec{row}(18:23)); - tmp_gps.sec(row) = str2num(tmp_gps.date_time_sec{row}(18:23)); - end -end if isfield(tmp_gps,'year') year = tmp_gps.year; elseif isfield(param,'year') @@ -471,5 +522,3 @@ if ~isfield(gps,'range') gps.range = zeros(1,num_rows); end - -return; diff --git a/cresis-toolbox/gps/read_gps_litton.m b/cresis-toolbox/gps/read_gps_litton.m index 0749d566..90f0f967 100644 --- a/cresis-toolbox/gps/read_gps_litton.m +++ b/cresis-toolbox/gps/read_gps_litton.m @@ -36,7 +36,7 @@ % % Author: John Paden % -% See also read_gps_*.m, gps_plot.m, gps_make.m +% See also read_gps_*.m, gps_plot.m, gps_create.m if ischar(litton_fn) tmp = litton_fn; diff --git a/cresis-toolbox/gps/read_gps_netcdf.m b/cresis-toolbox/gps/read_gps_netcdf.m index 36ee6e68..359dfa98 100644 --- a/cresis-toolbox/gps/read_gps_netcdf.m +++ b/cresis-toolbox/gps/read_gps_netcdf.m @@ -73,7 +73,7 @@ % % Author: John Paden % -% See also read_gps_*.m, gps_plot.m, gps_make.m +% See also read_gps_*.m, gps_plot.m, gps_create.m %% Check Inputs if ~isfield(param,'scale') || isempty(param.scale) diff --git a/cresis-toolbox/gps/read_gps_nmea.m b/cresis-toolbox/gps/read_gps_nmea.m index d8ab78fe..83f14b51 100644 --- a/cresis-toolbox/gps/read_gps_nmea.m +++ b/cresis-toolbox/gps/read_gps_nmea.m @@ -1,312 +1,266 @@ -function gps = read_gps_nmea(in_fn, param) -% gps = read_gps_nmea(in_fn, param) -% -% Reads in NMEA files and modified NMEA files that include the MCRDS -% radar time stamps. GPS file must contain only GPGGA strings!!! If -% non-compatible strings are present you must remove them. In Linux: -% grep GPGGA OLD_FILENAME >NEW_FILENAME -% -% $GPGGA,120448.00,6952.4649165,N,03256.5105286,W,1,10,0.80,3135.1254,M,55.3383,M,,*7B -% $GPGGA,221343.00,7928.1873712,S,11203.3163302,W,0,10, ,1769.932,M, , , ,*5e -% -% Input Args: -% in_fn = string containing input NMEA filename -% OR cell array of strings containging input NMEA filenames -% If the length of the cell array is one, then it operates on that one -% file as if just a string was passed in. If the length is more than -% one, then param.combine is set to true. -% param = tells the file GPS type and the year, month, day to determine -% absolute time (GGA NMEA files just give the time of day) -% .format = scalar integer from 1 to 4, default is 1 -% 1. Standard NMEA -% 2. Standard NMEA comp_time_sec comp_time_usec -% comp_time_sec + comp_time_usec/1e6 -% = seconds since the epoch Jan 1, 1970 00:00:00 -% 3. Standard NMEA comp_time_sec comp_time_usec radar_32MSB radar_32LSB -% radar_32MSB*2^32 + radar_32LSB (64 bit MCRDS radar time, 10 MHz clock) -% 4. From reveal system -% .year -% .month -% .day -% .time_reference = 'gps' or 'utc' (should always be 'utc') -% .combine = logical (default is false) -% When enabled, all input files will be concatenated together -% in the temp directory and this file will be used. -% If in_fn is a string, then a search is done for all files in the -% directory specified by in_fn which have the string YYYYMMDD in -% their filename. -% If in_fn is a cell array, then these specific files will be used. -% .nmea_tag = NEMA string to identify good lines (e.g. '$GPGGA') -% -% Output Args: -% gps = output structure with fields -% .time = GPS time in seconds since Jan 1, 1970 epoch (sec) -% .lat = latitude (deg) -% .lon = longitude (deg) -% .elev = elevation (m) -% .roll = roll (rad) -% .pitch = pitch (rad) -% .heading = true heading (rad) -% .comp_time = computer time in seconds since Jan 1, 1970 epoch (sec) -% .radar_time = radar time, 64 bit counter free-running at 10 MHz -% -% Example: -% fn = '/cresis/data2/MCoRDS/2010_Antarctica/GPS_new/GGA/RevealGPS_20100103A'; -% gps = read_gps_nmea(fn, struct('year',2010,'month',1,'day',3)); -% plot(gps.lon,gps.lat); -% datestr(epoch_to_datenum(gps.gps_time(1))); -% gps.utc_time = gps.gps_time - utc_leap_seconds(gps.gps_time(1)) -% gps_plot(gps) -% -% Author: William Blake, John Paden, Anthony Hoch, Logan Smith -% -% See also read_gps_*.m, gps_plot.m, gps_make.m - -global gRadar - -debug_level = 1; - -if ~exist('param','var') || isempty(param) - error('Year, month, day must be specified in param struct'); -end -if ~isfield(param,'format') - param.format = 1; -end -if ~isfield(param,'combine') - param.combine = 0; -end -if iscell(in_fn) - % If in_fn is a cell array of more than one file name, then we need - % to combined all of these files - if length(in_fn) == 1 - in_fn = in_fn{1}; - else - param.combine = 1; - end -end - -if param.combine - date_str = sprintf('%04d%02d%02d',param.year,param.month,param.day); - if iscell(in_fn) - % in_fn is already of cell array of filenames - in_fns = in_fn; - else - in_fns = get_filenames(in_fn,'',date_str,''); - end - tmp_nmea_path = fullfile(gRadar.tmp_path,sprintf('tmp_nmea_%s.txt',date_str)); - syscmd = sprintf('cat %s > %s',in_fns{1},tmp_nmea_path); - system(syscmd); - for idx=2:length(in_fns) - syscmd = sprintf('cat %s >> %s',in_fns{idx},tmp_nmea_path); - system(syscmd); - end - in_fn = tmp_nmea_path; -end - -if ~exist(in_fn,'file') - error('File does not exist %s\n', in_fn); -end - -% LOAD NMEA FILE - -switch param.format - case 1 - format_str = '%s%f%f%c%f%c%u%u%f%f%c%f%c%s%s'; - case 2 - format_str = '%s%f%f%c%f%c%u%u%f%f%c%f%c%s%s%f%f'; - case 3 - format_str = '%s%f%f%c%f%c%u%u%f%f%c%f%c%s%s%f%f%f%f'; - case 4 - format_str = '%s%f%f%c%f%c%u%u%f%f%f%f%s'; - case 5 - format_str = '%s%f%f%c%f%c%u%u%f%f%c%f%c%s%s%f%f%f%f'; -end - -[fid,msg] = fopen(in_fn,'r'); -if fid < 0 - error('Error opening %s: %s', in_fn, msg); -end -finfo = dir(in_fn); -C_final = {}; -num_lines = 0; -while ftell(fid) < finfo.bytes - % This reader uses textscan to read in the file. If textscan finds a bad - % line, it stops reading before the end of the file - C = textscan(fid,format_str,'delimiter',', ','emptyvalue',NaN); - good_lines = length(C{end}); - num_lines = num_lines + good_lines+1; - if ftell(fid) < finfo.bytes - readchar = fread(fid,1,'uint8'); - while readchar ~= 10 && ftell(fid) > 1 - fseek(fid,-2,0); - readchar = fread(fid,1,'uint8'); - end - readline = fgets(fid); - if readline(end) ~= 10 - readline(end+1) = 10; - end - fprintf('Bad line %d: %s', num_lines, readline); - end - Clength = cellfun(@length,C); - if good_lines > 0 - for field_idx = 1:length(C) - if length(C_final) < field_idx - C_final{field_idx} = C{field_idx}(1:good_lines,1); - else - C_final{field_idx} = cat(1,C_final{field_idx}, C{field_idx}(1:good_lines)); - end - end - end -end -fclose(fid); - -if isempty(C_final) - C_final = cell(1,sum(format_str=='%')); -end - -switch param.format - case 1 - [tag,UTC_time_file,latitude,N_S,longitude,E_W,fix,NoSatelite,dilution,... - altitude,alt_unit,geode_ref,geode_unit,dgps,checksum] = deal(C_final{:}); - case 2 - [tag,UTC_time_file,latitude,N_S,longitude,E_W,fix,NoSatelite,dilution,... - altitude,alt_unit,geode_ref,geode_unit,dgps,checksum,time1,time2] = deal(C_final{:}); - case 3 - [tag,UTC_time_file,latitude,N_S,longitude,E_W,fix,NoSatelite,dilution,... - altitude,alt_unit,geode_ref,geode_unit,dgps,checksum,time1,time2,time3,time4] = deal(C_final{:}); - case 4 - [tag,UTC_time_file,latitude,N_S,longitude,E_W,fix,NoSatelite,dilution,... - altitude,alt_unit,geode_ref,checksum] = deal(C_final{:}); - case 5 - [tag,UTC_time_file,latitude,N_S,longitude,E_W,fix,NoSatelite,dilution,... - altitude,alt_unit,geode_ref,geode_unit,dgps,checksum,time1,time2,time3,time4] = deal(C_final{:}); -end - -if param.format == 2 || param.format == 3 - comp_time = time1 + time2/1e6; -end -if param.format == 3 - radar_time = (time3*2^32 + time4)/10e6; -end - -if isfield(param,'nmea_tag') - good_idxs = strmatch(param.nmea_tag,tag); - tag = tag(good_idxs); - UTC_time_file = UTC_time_file(good_idxs); - latitude = latitude(good_idxs); - N_S = N_S(good_idxs); - longitude = longitude(good_idxs); - E_W = E_W(good_idxs); - altitude = altitude(good_idxs); - if param.format == 2 || param.format == 3 - comp_time = comp_time(good_idxs); - end - if param.format == 3 - radar_time = radar_time(good_idxs); - end -end - -% CONVERT LATITUDE AND LONGITUDE TO [DD.DDD] FROM [DDDMM.MMM] -lat_MM = mod(latitude,100); -lat_DD = (latitude - lat_MM)./100; -lat = lat_DD + lat_MM./60; -lon_MM = mod(longitude,100); -lon_DD = (longitude - lon_MM)./100; -lon = lon_DD + lon_MM./60; - -% IMPLY NEGATIVE LATITUDE AND LONGITUDE TO SOUTH AND WEST COORDINATES -lat(N_S == 'S') = -1 * lat(N_S == 'S'); -lon(E_W == 'W') = -1 * lon(E_W == 'W'); - -% CREATE NEW ELEVATION VARIABLE -elev = altitude; - -% Convert HHMMSS format to ANSI-C standard, seconds since Jan 1 1970 -sec = mod(UTC_time_file,100); -min = mod([UTC_time_file-sec]./100,100); -hour = [[UTC_time_file-sec]./100 - min]./100; -UTC_time = datenum_to_epoch(datenum(param.year,param.month,param.day,hour,min,sec)); - -% ENSURE ALL VECTORS IN 1xN FORMAT -UTC_time = reshape(UTC_time,[1 length(UTC_time)]); -lat = reshape(lat,[1 length(lat)]); -lon = reshape(lon,[1 length(lon)]); -elev = reshape(elev,[1 length(elev)]); -if param.format == 2 || param.format == 3 - comp_time = reshape(comp_time,[1 length(comp_time)]); -end -if param.format == 3 - radar_time = reshape(radar_time,[1 length(radar_time)]); -end - - -goodIdxs = find(~isnan(lat)); -UTC_time = UTC_time(goodIdxs); -lat = lat(goodIdxs); -lon = lon(goodIdxs); -elev = elev(goodIdxs); -if param.format == 2 || param.format == 3 - comp_time = comp_time(goodIdxs); -end -if param.format == 3 - radar_time = radar_time(goodIdxs); -end - -if param.format == 4 - % Reveal file tends to have a lot of problems... this catches some - bad_idxs = find(abs(diff(lat)) > 0.005); - if ~isempty(bad_idxs) - plot(diff(lat(:))); - fprintf('Some likely bad indices found. Set bad_idxs in the code and run.\n') - keyboard - % For example, two bad ranges: bad_idxs = [12690:12768, 16608:16643]; - bad_idxs = []; % SET THIS TO APPROPRIATE VALUE EACH TIME - good_idxs = setdiff(1:length(lat), bad_idxs); - plot(diff(lat(good_idxs)),'.'); - keyboard - UTC_time = UTC_time(good_idxs); - lat = lat(good_idxs); - lon = lon(good_idxs); - elev = elev(good_idxs); - end -end - -% =================================================================== -% Find jumps in the GPS time that are probably due to day interval -% 86400 seconds. -day_jumps = find(diff(UTC_time) < -60000); -for jump_idx = day_jumps - UTC_time(jump_idx+1:end) = UTC_time(jump_idx+1:end) + 86400; -end - -% =================================================================== -% Store outputs in structure -% =================================================================== -if strcmpi(param.time_reference,'utc') - % UTC time stored in file, so need to add leap seconds back in - if ~isempty(UTC_time) - gps.gps_time = UTC_time + utc_leap_seconds(UTC_time(1)); - else - gps.gps_time = []; - end -else - warning('NMEA files are usually always UTC time, but GPS time has been specified.'); - gps.gps_time = UTC_time; -end - -gps.lat = lat; -gps.lon = lon; -gps.elev = elev; - -gps.roll = zeros(size(gps.lat)); -gps.pitch = zeros(size(gps.lat)); -gps.heading = zeros(size(gps.lat)); - -if param.format == 2 || param.format == 3 - gps.comp_time = comp_time; -end -if param.format == 3 - gps.radar_time = radar_time; -end - -return; +function gps = read_gps_nmea(in_fns, param) +% gps = read_gps_nmea(in_fns, param) +% +% Reads in NMEA files and modified NMEA files that include the MCRDS +% radar time stamps. +% +% Example strings: +% $GPGGA,120448.00,6952.4649165,N,03256.5105286,W,1,10,0.80,3135.1254,M,55.3383,M,,*7B +% $GPGGA,221343.00,7928.1873712,S,11203.3163302,W,0,10, ,1769.932,M, , , ,*5e +% $GPGGA,151922.00,4402.1108,N,10330.6311,W,1,08,0.0,2030.41,M,-14.10,M,,*69 +% $GPZDA,151923.00,02,03,2021,,*6B +% $GPRMC,194325.010,A,3857.135037,N,09515.861757,W,0.000,0.00,121018,,,A*46 +% +% Special MCRDS GPS string with radar timing at the end: +% $GPGGA,163202.50,6909.94391,N,05045.73509,W,1,10,01.1,+00351,M,,M,,*6E,1216243966,190018,0,1204901900 +% +% Input Args: +% in_fns: string containing input NMEA filename +% OR cell array of strings containing input NMEA filenames +% param: controls how the NMEA file is read +% .format = scalar integer from 1 to 4, default is 1 +% 1. Standard NMEA +% 2. Standard NMEA comp_time_sec comp_time_usec +% comp_time_sec + comp_time_usec/1e6 +% = seconds since the epoch Jan 1, 1970 00:00:00 +% 3. Standard NMEA comp_time_sec comp_time_usec radar_32MSB radar_32LSB +% radar_32MSB*2^32 + radar_32LSB (64 bit MCRDS radar time, 10 MHz clock) +% 4. From reveal system +% .year: Optionally override the year (this field must be set if the +% NMEA strings do not include the date) +% .month: Optionally override the year (this field must be set if the +% NMEA strings do not include the date) +% .day: Optionally override the year (this field must be set if the +% NMEA strings do not include the date) +% .time_reference = 'gps' or 'utc' (should always be 'utc') +% +% Output Args: +% gps: output structure with fields +% .gps_time: GPS time in seconds since Jan 1, 1970 epoch (sec) +% .lat: latitude (deg) +% .lon: longitude (deg) +% .elev: elevation (m) +% .roll: roll (rad) +% .pitch: pitch (rad) +% .heading: true heading (rad) +% .comp_time: computer time in seconds since Jan 1, 1970 epoch (sec) +% .radar_time: radar time, 64 bit counter free-running at 10 MHz +% +% Example: +% fn = '/cresis/data2/MCoRDS/2010_Antarctica/GPS_new/GGA/RevealGPS_20100103A'; +% gps = read_gps_nmea(fn, struct('year',2010,'month',1,'day',3)); +% plot(gps.lon,gps.lat); +% datestr(epoch_to_datenum(gps.gps_time(1))); +% gps.utc_time = gps.gps_time - utc_leap_seconds(gps.gps_time(1)) +% gps_plot(gps) +% +% fn = '/cresis/snfs1/dataproducts/metadata/2020_SouthDakota_N1KU/20210302/GPS_20210302_151952.txt'; +% gps = read_gps_nmea(fn); +% gps_plot(gps); +% +% fn = '/cresis/snfs1/dataproducts/metadata/2008_Greenland_TO/nmea.20080716163242.gps'; +% gps = read_gps_nmea(fn,struct('year',2008,'month',7,'day',16)); +% gps_plot(gps); +% +% Author: William Blake, John Paden, Anthony Hoch, Logan Smith +% +% See also read_gps_*.m, gps_plot.m, gps_create.m + +if ~exist('param','var') + param = []; +end + +if ~isfield(param,'year') || isempty(param.year) + param.year = NaN; +end + +if ~isfield(param,'month') || isempty(param.month) + param.month = NaN; +end + +if ~isfield(param,'day') || isempty(param.day) + param.day = NaN; +end + +if ~isfield(param,'time_reference') || isempty(param.time_reference) + param.time_reference = 'utc'; +end + +if ischar(in_fns) + in_fns = {in_fns}; +end + +for in_fns_idx = 1:length(in_fns) + in_fn = in_fns{in_fns_idx}; + fprintf('Opening_file\t%s\n', in_fn); + if ~exist(in_fn,'file') + error('File does not exist %s\n', in_fn); + end + + [fid,msg] = fopen(in_fn,'rb'); + if fid < 0 + error('Error opening %s: %s', in_fn, msg); + end + + file_contents = fread(fid,inf,'char=>char').'; + fclose(fid); + + line_start_idxs = [find(file_contents=='$') numel(file_contents)+1]; + gps_idx = 0; + gps.gps_time = nan(1,length(line_start_idxs)); + gps.lat = nan(1,length(line_start_idxs)); + gps.lon = nan(1,length(line_start_idxs)); + gps.elev= nan(1,length(line_start_idxs)); + gps.comp_time = nan(1,length(line_start_idxs)); + gps.radar_time = nan(1,length(line_start_idxs)); + cur_hour = NaN; + cur_min = NaN; + cur_sec = NaN; + start_time = -inf; + for line = 1:length(line_start_idxs)-1 + if now > start_time+1/86400 + fprintf('File_line\t%d\tof\t%d\t%s\n', line, length(line_start_idxs), datestr(now,'yyyymmdd_HHMMSS')); + start_time = now; + end + line_str = file_contents(line_start_idxs(line) : line_start_idxs(line+1)-1); + + comma_idxs = find(line_str == ','); + star_idx = find(line_str == '*'); + + if length(comma_idxs) > 1 && ~isempty(star_idx) && length(line_str) >= star_idx(end)+2 + line_uint8 = uint8(line_str); + + % This is a simple calculator to compute the checksum field for the + % NMEA protocol. The checksum is simple, just an XOR of all the bytes + % between the $ and the * (not including the delimiters themselves), + % and written in hexadecimal. + check_sum = line_uint8(2); + for idx = 3:star_idx-1 + check_sum = bitxor(check_sum,line_uint8(idx)); + end + + if check_sum == sscanf(line_str(star_idx(end)+(1:2)),'%x') + % Checksum is correct + + if strcmp('GPGGA',line_str(2:comma_idxs(1)-1)) && length(comma_idxs) >= 10 ... + && comma_idxs(2) - comma_idxs(1) >= 2 ... + && comma_idxs(3) - comma_idxs(2) >= 2 ... + && comma_idxs(4) - comma_idxs(3) >= 2 ... + && comma_idxs(5) - comma_idxs(4) >= 2 ... + && comma_idxs(6) - comma_idxs(5) >= 2 ... + && comma_idxs(10) - comma_idxs(9) >= 2 + + try + [~,~,~,cur_hour,cur_min,cur_sec] = datevec(datenum(line_str(comma_idxs(1)+1 : comma_idxs(2)-1), 'HHMMSS.FFF')); + end + + lat = 10*(line_str(comma_idxs(2)+1)-48) + (line_str(comma_idxs(2)+2)-48) ... + + sscanf(line_str(comma_idxs(2)+3:comma_idxs(3)-1),'%f')/60; + if line_str(comma_idxs(3)+1) == 'S' + lat = -lat; + end + lon = 100*(line_str(comma_idxs(4)+1)-48) + 10*(line_str(comma_idxs(4)+2)-48) + (line_str(comma_idxs(4)+3)-48) ... + + sscanf(line_str(comma_idxs(4)+4:comma_idxs(5)-1),'%f')/60; + if line_str(comma_idxs(5)+1) == 'W' + lon = -lon; + end + elev = sscanf(line_str(comma_idxs(9)+1:comma_idxs(10)-1),'%f'); + + gps_idx = gps_idx + 1; + gps.gps_time(gps_idx) = datenum(param.year,param.month,param.day,cur_hour,cur_min,cur_sec); + if isnan(gps.gps_time(gps_idx)) + warning('NaN gps_time.'); + end + gps.lat(gps_idx) = lat; + gps.lon(gps_idx) = lon; + gps.elev(gps_idx) = elev; + + if length(comma_idxs) == 16 ... + && comma_idxs(16) - comma_idxs(15) >= 2 ... + && length(line_str) - comma_idxs(16) >= 2 + % NMEA file with computer time stamps at end + + [time_fields,count] = sscanf(line_str(comma_idxs(15)+1:end),'%f,%f'); + if count == 2 + gps.comp_time(gps_idx) = time_fields(1) + time_fields(2)/10e6; + end + + elseif length(comma_idxs) == 18 ... + && comma_idxs(16) - comma_idxs(15) >= 2 ... + && comma_idxs(17) - comma_idxs(16) >= 2 ... + && comma_idxs(18) - comma_idxs(17) >= 2 ... + && length(line_str) - comma_idxs(18) >= 2 + % MCRDS NMEA file (computer and radar time stamps at the end) + + [time_fields,count] = sscanf(line_str(comma_idxs(15)+1:end),'%f,%f,%f,%f'); + if count == 4 + gps.comp_time(gps_idx) = time_fields(1) + time_fields(2)/1e6; + gps.radar_time(gps_idx) = time_fields(3) + time_fields(4)/10e6; + end + end + + elseif strcmp('GPZDA',line_str(2:comma_idxs(1)-1)) && length(comma_idxs) >= 5 ... + && comma_idxs(2) - comma_idxs(1) >= 2 ... + && comma_idxs(3) - comma_idxs(2) >= 2 ... + && comma_idxs(4) - comma_idxs(3) >= 2 ... + && comma_idxs(5) - comma_idxs(4) >= 2 + + try + gps_time = datenum(line_str(comma_idxs(1)+1 : comma_idxs(5)-1), 'HHMMSS.FFF,dd,mm,yyyy'); + [param.year,param.month,param.day,test_hour,test_min,test_sec] = datevec(gps_time); + end + + if test_hour == cur_hour && test_min == cur_min && test_sec == cur_sec + gps.gps_time(gps_idx) = gps_time; + end + + elseif strcmp('GPRMC',line_str(2:comma_idxs(1)-1)) && length(comma_idxs) >= 10 ... + && comma_idxs(2) - comma_idxs(1) >= 2 ... + && comma_idxs(10) - comma_idxs(9) >= 2 + + try + gps_time = datenum(line_str([comma_idxs(1)+1 : comma_idxs(2)-1,comma_idxs(9) : comma_idxs(10)-1]), 'HHMMSS.FFF,ddmmyy'); + [param.year,param.month,param.day,test_hour,test_min,test_sec] = datevec(gps_time); + end + + if test_hour == cur_hour && test_min == cur_min && test_sec == cur_sec + gps.gps_time(gps_idx) = gps_time; + end + end + end + end + + end +end + +good_mask = ~isnan(gps.gps_time); +gps.gps_time = gps.gps_time(good_mask); +gps.lat = gps.lat(good_mask); +gps.lon = gps.lon(good_mask); +gps.elev = gps.elev(good_mask); +gps.comp_time = gps.comp_time(good_mask); +gps.radar_time = gps.radar_time(good_mask); + +gps.roll = zeros(size(gps.lat)); +gps.pitch = zeros(size(gps.lat)); +gps.heading = zeros(size(gps.lat)); + +%% Day Jumps +% ========================================================================= +% Find negative jumps in the GPS time of more than 75% of a day that are +% probably due to day wraps of 1. +day_jumps = find(diff(gps.gps_time) < -0.75); +for jump_idx = day_jumps + gps.gps_time(jump_idx+1:end) = gps.gps_time(jump_idx+1:end) + 1; +end + +%% UTC or GPS time reference +% ========================================================================= +% Convert gps_time into ANSI-C standard (seconds since 1970) +gps.gps_time = datenum_to_epoch(gps.gps_time); +if strcmpi(param.time_reference,'utc') + % UTC time stored in file, so need to add leap seconds back in + gps.gps_time = gps.gps_time + utc_leap_seconds(gps.gps_time(1)); +else + warning('NMEA files are usually always UTC time, but GPS time has been specified.'); +end diff --git a/cresis-toolbox/gps/read_gps_novatel.m b/cresis-toolbox/gps/read_gps_novatel.m index 7b7d9976..9d30bb6e 100644 --- a/cresis-toolbox/gps/read_gps_novatel.m +++ b/cresis-toolbox/gps/read_gps_novatel.m @@ -20,7 +20,7 @@ % % Author: Huan Zhao, John Paden % -% See also read_gps_*.m, gps_plot.m, gps_make.m +% See also read_gps_*.m, gps_plot.m, gps_create.m % 2009 Antarctica TO FORMAT: % ========================================================================= diff --git a/cresis-toolbox/gps/read_gps_novatel_rpygga.m b/cresis-toolbox/gps/read_gps_novatel_rpygga.m index 88b20be5..0146ef47 100644 --- a/cresis-toolbox/gps/read_gps_novatel_rpygga.m +++ b/cresis-toolbox/gps/read_gps_novatel_rpygga.m @@ -40,7 +40,7 @@ % % Author: William Blake, John Paden, Anthony Hoch, Logan Smith % -% See also read_gps_*.m, gps_plot.m, gps_make.m +% See also read_gps_*.m, gps_plot.m, gps_create.m global gRadar diff --git a/cresis-toolbox/gps/read_gps_novatelraw.m b/cresis-toolbox/gps/read_gps_novatelraw.m new file mode 100644 index 00000000..01d70cca --- /dev/null +++ b/cresis-toolbox/gps/read_gps_novatelraw.m @@ -0,0 +1,198 @@ +function gps = read_gps_novatelraw(fn, param) +% gps = read_gps_novatelraw(fn, param) +% +% Parses Novatel binary commands BESTPOSB,TIMEB,INSATTB +% See Novatel OEM6 or OEM7 command manuals. +% +% GPS+INS: +% LOG USB1 BESTPOSB ONTIME 0.2 +% LOG USB1 TIMEB ONTIME 0.2 +% LOG USB1 INSATTB ONTIME 0.2 +% +% GPS ONLY: +% LOG USB1 BESTPOSB ONTIME 0.05 +% LOG USB1 TIMEB ONTIME 0.05 +% +% Example: +% +% fn = '/scratch/metadata/2022_Antarctica_BaslerMKB/20230110/GPS_Novatel_raw_aq-field22_20230110_010949.gps' +% gps = read_gps_novatelraw(fn); +% gps_plot(gps); +% datestr(epoch_to_datenum(gps.gps_time(1))); +% gps.utc_time = gps.gps_time - utc_leap_seconds(gps.gps_time(1)) +% +% Author: John Paden +% +% See also read_gps_*.m, gps_plot.m, gps_create.m + +if ~exist('param','var') + param = []; +end + +if ~isfield(param,'first_byte') + param.first_byte = 0; +end + +%% Read in file + +% Open file +[fid,msg] = fopen(fn,'rb'); +if fid < 1 + fprintf('Could not open file %s\n', fn); + error(msg); +end +fseek(fid,param.first_byte,-1); + +% Read entire file (may require at lot of RAM!) +A = fread(fid,inf,'uint8=>uint8'); + +% Close file +fclose(fid); + +%% Find Frame Syncs in Novatel Data + +frame_sync_byte1 = 170; % '0xAA' +frame_sync_byte2 = 68; % '0x44' +frame_sync_byte3 = 18; % '0x12' +frame_sync_byte3_or = 19; % '0x13' + +frame_start_mask = A(1:end-2)==frame_sync_byte1 & A(2:end-1)==frame_sync_byte2 & (A(3:end)==frame_sync_byte3 | A(3:end)==frame_sync_byte3_or); + +frame_start_idxs = find(frame_start_mask); + +%% Parse each frame + +% Preallocate output arrays: Assume the longest possible length for outputs +% (only BESTPOS and TIME messages present) +gps.gps_time = zeros(1,floor(length(frame_start_idxs)/2)); +gps.lat = zeros(1,floor(length(frame_start_idxs)/2)); +gps.lon = zeros(1,floor(length(frame_start_idxs)/2)); +gps.elev = zeros(1,floor(length(frame_start_idxs)/2)); +gps.roll = zeros(1,floor(length(frame_start_idxs)/2)); +gps.pitch = zeros(1,floor(length(frame_start_idxs)/2)); +gps.heading = zeros(1,floor(length(frame_start_idxs)/2)); + +if isempty(frame_start_idxs) + return; +end + +next_start_idx = frame_start_idxs(1); +out_idx = 0; +have_time = false; +have_ins = false; +roll = 0; +pitch = 0; +heading = 0; +message_ID_list = []; +try + for frame = 1:length(frame_start_idxs) + %fprintf('FRAME %10d of %10d\n', frame, length(frame_start_idxs)); + + % Check to see if this 3-byte frame sync occurs where we think it should + % based on the last frame sync and the length of that record. If it does + % not, then ignore this 3-byte frame sync since it may have randomly + % occurred. + if frame_start_idxs(frame) < next_start_idx + fprintf('Unexpected frame position at byte offset %d. Skipping 3-byte frame sync.\n', frame_start_idxs(frame)); + if frame == length(frame_start_idxs) + break; + end + next_start_idx = frame_start_idxs(frame+1); + continue; + end + if A(frame_start_idxs(frame)+2) == 18 + message_type = 'GPS'; + header_length = double(A(frame_start_idxs(frame)+3)); + message_ID = typecast(swapbytes(A(frame_start_idxs(frame)+(4:5))),'uint16'); + gps_week = typecast(swapbytes(A(frame_start_idxs(frame)+(14:15))),'uint16'); + gps_msec = typecast(swapbytes(A(frame_start_idxs(frame)+(16:19))),'uint32'); + message_length = double(typecast(swapbytes(A(frame_start_idxs(frame)+(8:9))),'uint16')); + else % if A(frame_start_idxs(frame)+2) == 19 + message_type = 'INS'; + header_length = 12; + message_length = double(A(frame_start_idxs(frame)+3)); + message_ID = typecast(swapbytes(A(frame_start_idxs(frame)+(4:5))),'uint16'); + gps_week = typecast(swapbytes(A(frame_start_idxs(frame)+(6:7))),'uint16'); + gps_msec = typecast(swapbytes(A(frame_start_idxs(frame)+(8:11))),'uint32'); + end + + % Debug print out + %fprintf('%s ID: %4d Length: %4d %5d Time: %4d %6.1f\n', message_type, message_ID, header_length, message_length, gps_week, gps_msec/1000); + % Debug message_ID_list: + %message_ID_list(end+1) = message_ID; + + next_start_idx = frame_start_idxs(frame)+header_length+message_length+4; + %fprintf('%g %g %g\n', next_start_idx, frame_start_idxs(frame+1),frame_start_idxs(frame+1)-next_start_idx); + + if message_ID==101 + % TIME MESSAGE + % Contain UTC time + %fprintf('%4d %4d %5d %6.1f\n', header_length, message_ID, gps_week, gps_msec/1000); + year = typecast(swapbytes(A(frame_start_idxs(frame)+header_length+(28:31))),'uint32'); + month = A(frame_start_idxs(frame)+header_length+32); + day = A(frame_start_idxs(frame)+header_length+33); + hour = A(frame_start_idxs(frame)+header_length+34); + minute = A(frame_start_idxs(frame)+header_length+35); + sec = double(typecast(swapbytes(A(frame_start_idxs(frame)+header_length+(36:39))),'uint32'))/1000; + %fprintf('%04d_%02d_%02d %02d:%02d:%04.1f\n', year, month, day, hour, minute, sec); + have_time = true; + + elseif message_ID==42 + % BESTPOS MESSAGE + %fprintf('%4d %4d %5d %6.1f\n', header_length, message_ID, gps_week, gps_msec/1000); + if ~have_time + fprintf('BESTPOS without preceding TIME. Skipping frame.\n', frame_start_idxs(frame)); + continue; + end + lat = typecast(swapbytes(A(frame_start_idxs(frame)+header_length+(8:15))),'double'); + lon = typecast(swapbytes(A(frame_start_idxs(frame)+header_length+(16:23))),'double'); + elev = typecast(swapbytes(A(frame_start_idxs(frame)+header_length+(24:31))),'double'); + %fprintf('%8.4f %8.4f %6.1f\n', lat, lon, elev); + out_idx = out_idx + 1; + % NOTE: gps.gps_time is actually UTC time from the TIME field. Will add + % leap seconds in to convert UTC to GPS time once finished loading. + gps.gps_time(out_idx) = datenum([double([year,month,day,hour,minute]),sec]); + gps.lat(out_idx) = lat; + gps.lon(out_idx) = lon; + gps.elev(out_idx) = elev; + gps.roll(out_idx) = roll; + gps.pitch(out_idx) = pitch; + gps.heading(out_idx) = heading; + have_time = false; + have_ins = false; + + elseif message_ID==263 + % INSATT MESSAGE + %fprintf('%4d %4d %5d %6.1f\n', header_length, message_ID, gps_week, gps_msec/1000); + roll = typecast(swapbytes(A(frame_start_idxs(frame)+header_length+(12:19))),'double') / 180*pi; + pitch = typecast(swapbytes(A(frame_start_idxs(frame)+header_length+(20:27))),'double') / 180*pi; + heading = typecast(swapbytes(A(frame_start_idxs(frame)+header_length+(28:35))),'double') / 180*pi; + %fprintf('%8.4f %8.4f %8.4f\n', roll, pitch, heading); + have_ins = true; + + % elseif message_ID==1465 + % lat = typecast(swapbytes(A(frame_start_idxs(frame)+header_length+(8:15))),'double'); + % lon = typecast(swapbytes(A(frame_start_idxs(frame)+header_length+(16:23))),'double'); + % elev = typecast(swapbytes(A(frame_start_idxs(frame)+header_length+(24:31))),'double'); + % ext_sol_stat = typecast(swapbytes(A(frame_start_idxs(frame)+header_length+(120:123))),'uint32'); + % time_since_update = typecast(swapbytes(A(frame_start_idxs(frame)+header_length+(124:125))),'uint16'); + % CRC = typecast(swapbytes(A(frame_start_idxs(frame)+header_length+(126:129))),'uint32'); + end + end +catch ME + warning(ME.getReport); +end + +%% Prepare Output + +% Only keep the records that we filled: +gps.gps_time = datenum_to_epoch(gps.gps_time(1:out_idx)); % Convert from Matlab datenum to ANSI-C standard time of seconds since Jan 1, 1970 +if ~isempty(gps.gps_time) + gps.gps_time = gps.gps_time + utc_leap_seconds(gps.gps_time(1)); % Convert gps.gps_time from UTC to GPS time +end +gps.lat = gps.lat(1:out_idx); +gps.lon = gps.lon(1:out_idx); +gps.elev = gps.elev(1:out_idx); +gps.roll = gps.roll(1:out_idx); +gps.pitch = gps.pitch(1:out_idx); +gps.heading = gps.heading(1:out_idx); diff --git a/cresis-toolbox/gps/read_gps_reveal.m b/cresis-toolbox/gps/read_gps_reveal.m index a8ff431b..1576369d 100644 --- a/cresis-toolbox/gps/read_gps_reveal.m +++ b/cresis-toolbox/gps/read_gps_reveal.m @@ -27,7 +27,7 @@ % % Author: William Blake, John Paden % -% See also read_gps_*.m, gps_plot.m, gps_make.m +% See also read_gps_*.m, gps_plot.m, gps_create.m debug_level = 1; diff --git a/cresis-toolbox/gps/read_gps_sonntagsequence2.m b/cresis-toolbox/gps/read_gps_sonntagsequence2.m new file mode 100644 index 00000000..2de3917c --- /dev/null +++ b/cresis-toolbox/gps/read_gps_sonntagsequence2.m @@ -0,0 +1,36 @@ +function wp = read_gps_sonntagsequence2(fn) +% wp = read_gps_sonntagsequence2(fn) +% +% Sonntag navigation software waypoint reader. +% +% John Sonntag's navigation software takes a waypoints file where each line +% contains one waypoint, space delimited fields, with fields: +% 1. waypoint string with no spaces +% 2. lat (deg,N) +% 3. lon (deg,E) +% 4. elev (meters, WGS-84) +% +% Example: +% NPX -89.9990 90.0000 11200 +% cas_340_6000 -89.3300 -20.0000 14950 +% CLXr84p90 -88.4822 38.0367 15500 +% CLXr84p88 -88.3292 41.7100 11650 +% CLXr84p46 -84.6462 64.1158 13450 +% CLXr84p42 -84.2835 64.7233 13650 +% CLXr76p42 -84.3068 66.8772 13650 +% CLXr76p46 -84.6733 66.6340 13450 +% CLXr76p88 -88.5053 56.8550 12000 +% CLXr76p90 -88.6835 54.9785 15950 +% NPX -89.9990 90.0000 15500 +% NPX -89.9990 90.0000 11200 + + +fid = fopen(fn); +A = textscan(fid,'%s %f %f %f'); +fclose(fid); + +wp.name = A{1}; +wp.lat = A{2}; +wp.lon = A{3}; +% Convert from feet to meters +wp.elev = A{4} * 12*2.54/100; diff --git a/cresis-toolbox/gps/read_gps_traj.m b/cresis-toolbox/gps/read_gps_traj.m index 28a70c44..c1e129d3 100644 --- a/cresis-toolbox/gps/read_gps_traj.m +++ b/cresis-toolbox/gps/read_gps_traj.m @@ -55,7 +55,7 @@ % % Author: John Paden, Kyle Purdon % -% See also read_gps_*.m, gps_plot.m, gps_make.m +% See also read_gps_*.m, gps_plot.m, gps_create.m % Set Defaults (If var does not exist) if ~isfield(param,'headerlines') diff --git a/cresis-toolbox/gps/read_gps_txt.m b/cresis-toolbox/gps/read_gps_txt.m index 431cefba..5fb99b65 100644 --- a/cresis-toolbox/gps/read_gps_txt.m +++ b/cresis-toolbox/gps/read_gps_txt.m @@ -32,7 +32,7 @@ % % Author: John Paden % -% See also read_gps_*.m, gps_plot.m, gps_make.m +% See also read_gps_*.m, gps_plot.m, gps_create.m debug_level = 1; diff --git a/cresis-toolbox/gps/read_gps_utig.m b/cresis-toolbox/gps/read_gps_utig.m new file mode 100644 index 00000000..d29698c9 --- /dev/null +++ b/cresis-toolbox/gps/read_gps_utig.m @@ -0,0 +1,174 @@ +function gps = read_gps_utig(fn, param) +% gps = read_gps_utig(fn, param) +% +% Read in UTIG ELSA raw file and extract GPS data from GPSnc1 packets. +% Ignores all other packets. Uses CX headers for radar_time (ct_time) and +% comp_time (ct_clk). ct_clk is not precise enough, but is absolute time +% and therefore monotonically increasing. ct_time is more precise than +% ct_clk, but it resets when the radar is reset so timing is ambiguous. +% +% Example: +% +% fn = '/data/UTIG/orig/xped/CXA1/acqn/ELSA/F13/serial0_20230116-200145-0001.dat'; +% gps = read_gps_utig(fn); +% gps_plot(gps); +% datestr(epoch_to_datenum(gps.gps_time(1))); +% gps.utc_time = gps.gps_time - utc_leap_seconds(gps.gps_time(1)) +% +% Author: John Paden +% +% See also read_gps_*.m, gps_plot.m, gps_create.m + +% =============================================================== +%% Open file little-endian for reading +% =============================================================== +[fid,msg] = fopen(fn,'r','ieee-le'); +if fid < 1 + fprintf('Could not open file %s\n', fn); + error(msg); +end +fseek(fid,0,1); +file_length = ftell(fid); +Nx_max = floor(file_length/179); +fseek(fid,0,-1); + +gps = []; +gps.gps_time = zeros(1,Nx_max); +gps.lat = zeros(1,Nx_max); +gps.lon = zeros(1,Nx_max); +gps.elev = zeros(1,Nx_max); +gps.roll = zeros(1,Nx_max); +gps.pitch = zeros(1,Nx_max); +gps.heading = zeros(1,Nx_max); +gps.radar_time = zeros(1,Nx_max); +gps.comp_time = zeros(1,Nx_max); + +state = 0; +locked = false; +gps_rec = 0; +while ~feof(fid) + test_byte = fread(fid,1,'char'); + switch (state) + case 0 + if test_byte == 'P' + state = state+1; + elseif locked + warning('Lost "PACKET" lock at byte %d.', ftell(fid)); + locked = false; + end + case 1 + if test_byte == 'A' + state = state+1; + else + state = 0; + end + case 2 + if test_byte == 'C' + state = state+1; + else + state = 0; + end + case 3 + if test_byte == 'K' + state = state+1; + else + state = 0; + end + case 4 + if test_byte == 'E' + state = state+1; + else + state = 0; + end + case 5 + if test_byte == 'T' + % keyboard + locked = true; + + fseek(fid,2,0); + %ftell(fid) + project = char(fread(fid,8,'char').'); + set = char(fread(fid,8,'char').'); + transect = char(fread(fid,8,'char').'); + stream_name = char(fread(fid,8,'char').'); + sequence_number = fread(fid,1,'uint32',0,'ieee-le'); + rec_length = fread(fid,1,'uint32',0,'ieee-le')+48; % Record length in bytes including the 68 byte CX header + % CT[48:67] + ct_clk_packed = fread(fid,1,'uint32',0,'ieee-le'); + year = 1000*mod(floor(ct_clk_packed/2^4),2^4) + 100*mod(ct_clk_packed,2^4) ... + + 10*mod(floor(ct_clk_packed/2^12),2^4) + mod(floor(ct_clk_packed/2^8),2^4); + month = 10*mod(floor(ct_clk_packed/2^20),2^4) + mod(floor(ct_clk_packed/2^16),2^4); + day= 10*mod(floor(ct_clk_packed/2^28),2^4) + mod(floor(ct_clk_packed/2^24),2^4); + ct_clk_packed = fread(fid,1,'uint32',0,'ieee-le'); + hour = 10*mod(floor(ct_clk_packed/2^4),2^4) + mod(ct_clk_packed,2^4); + min = 10*mod(floor(ct_clk_packed/2^12),2^4) + mod(floor(ct_clk_packed/2^8),2^4); + sec = 10*mod(floor(ct_clk_packed/2^20),2^4) + mod(floor(ct_clk_packed/2^16),2^4); + frac = 10*mod(floor(ct_clk_packed/2^28),2^4) + mod(floor(ct_clk_packed/2^24),2^4); + ct_clk = datenum([year,month,day,hour,min,sec]); + ct_time = fread(fid,1,'uint32',0,'ieee-le'); + fseek(fid,8,0); + + if strcmp(stream_name(1:6),'GPSnc1') + gps_rec = gps_rec + 1; + + gps_time = fread(fid,1,'int64',0,'ieee-be'); + gps_subsecs = fread(fid,1,'uint64',0,'ieee-be'); + + % Convert from NI time to ANSI C time + gps.gps_time(gps_rec) = datenum([1904 0 0 0 0 gps_time+gps_subsecs/2^64]); + + pps_time = fread(fid,1,'int64',0,'ieee-be'); + pps_subsecs = fread(fid,1,'uint64',0,'ieee-be'); + query_time = fread(fid,1,'int64',0,'ieee-be'); + query_subsecs = fread(fid,1,'uint64',0,'ieee-be'); + + time_source = fread(fid,1,'uint8',0,'ieee-be'); + + pps_ct = fread(fid,1,'uint32',0,'ieee-be'); + + query_ct = fread(fid,1,'uint32',0,'ieee-be'); + + ct_flags = fread(fid,1,'uint8',0,'ieee-be'); + + EW_vel = fread(fid,1,'float',0,'ieee-be'); + NS_vel = fread(fid,1,'float',0,'ieee-be'); + vert_vel = fread(fid,1,'float',0,'ieee-be'); + + gps.lat(gps_rec) = fread(fid,1,'float',0,'ieee-be'); % lat_ang + gps.lon(gps_rec) = fread(fid,1,'float',0,'ieee-be'); % lon_ang + gps.elev(gps_rec) = fread(fid,1,'float',0,'ieee-be'); % vert_cor + + gps.radar_time(gps_rec) = ct_time; + gps.comp_time(gps_rec) = ct_clk; + + gps_state = fread(fid,1,'uint8',0,'ieee-be'); + state = fread(fid,1,'uint8',0,'ieee-be'); + self_survey = fread(fid,1,'uint8',0,'ieee-be'); + + time_offset = fread(fid,1,'double',0,'ieee-be'); + time_corr = fread(fid,1,'double',0,'ieee-be'); + + utc_offset = fread(fid,1,'uint32',0,'ieee-be'); + nsv = fread(fid,1,'uint8',0,'ieee-be'); + sv_time = fread(fid,1,'uint32',0,'ieee-be'); + sw_state = fread(fid,1,'uint8',0,'ieee-be'); + else + fseek(fid,rec_length-68,0); + end + end + state = 0; + end +end + +fclose(fid); + +gps.gps_time = datenum_to_epoch(gps.gps_time(1:gps_rec)); +gps.lat = gps.lat(1:gps_rec); +gps.lon = gps.lon(1:gps_rec); +gps.elev = gps.elev(1:gps_rec); +gps.roll = gps.roll(1:gps_rec); +gps.pitch = gps.pitch(1:gps_rec); +gps.heading = gps.heading(1:gps_rec); + +gps.radar_time = gps.radar_time(1:gps_rec); +gps.comp_time = datenum_to_epoch(gps.comp_time(1:gps_rec)); diff --git a/cresis-toolbox/gps/read_gps_wepilot.m b/cresis-toolbox/gps/read_gps_wepilot.m index e58029ef..c2d861f1 100644 --- a/cresis-toolbox/gps/read_gps_wepilot.m +++ b/cresis-toolbox/gps/read_gps_wepilot.m @@ -1,102 +1,102 @@ -function gps = read_gps_mat(in_fn, param) -% gps = read_gps_mat(in_fn, param) -% -% Reads in Matlab mat files. A generic Variable and Attribute mapping is -% possible with the param structure. Includes wePilot. -% -% Input Args: -% in_fn (string) input .mat filename -% param = control parameter structure -% -% Output Args: -% gps = output structure with fields -% .time = GPS time in seconds since Jan 1, 1970 epoch (sec) -% .lat = latitude (deg) -% .lon = longitude (deg) -% .elev = elevation (m) -% .roll = roll (rad) -% .pitch = pitch (rad) -% .heading = true heading (rad) -% -% Example: -% See make_gps_2016_Greenland_G1XB.m -% -% fn = 'E:\tmp\uav_gps_imu\weData_20160413_1851AUX.mat'; -% gps = read_gps_wepilot(fn); -% -% Author: John Paden -% -% See also read_gps_*.m, gps_plot.m, gps_make.m - -load(in_fn); - -gps = []; - -if any(strcmp(OnBoard.signals.labels,'UTCYEAR')) - utc_year = OnBoard.signals.values(:,find(strcmp(OnBoard.signals.labels,'UTCYEAR'))); - utc_month = OnBoard.signals.values(:,find(strcmp(OnBoard.signals.labels,'UTCMONTH'))); - utc_day = OnBoard.signals.values(:,find(strcmp(OnBoard.signals.labels,'UTCDAY'))); - utc_hours = OnBoard.signals.values(:,find(strcmp(OnBoard.signals.labels,'UTCHOURS'))); - utc_min = OnBoard.signals.values(:,find(strcmp(OnBoard.signals.labels,'UTCMIN'))); - utc_sec = OnBoard.signals.values(:,find(strcmp(OnBoard.signals.labels,'UTCSEC'))); - utc_time = datenum(utc_year,utc_month,utc_day,utc_hours,utc_min,utc_sec); - datestr(utc_time(1)) - gps.gps_time = datenum_to_epoch(utc_time); - gps.gps_time = gps.gps_time + utc_leap_seconds(gps.gps_time(1)); - - relative_time = OnBoard.signals.values(:,find(strcmp(OnBoard.signals.labels,'TIME'))); - relative_time = relative_time-relative_time(1); - - old_gps_time = gps.gps_time(1); - for gps_idx = 2:length(gps.gps_time) - if (gps.gps_time(gps_idx) - old_gps_time) < 1e-3 - gps.gps_time(gps_idx) = gps.gps_time(gps_idx-1) + (relative_time(gps_idx) - relative_time(gps_idx-1)); - else - old_gps_time = gps.gps_time(gps_idx); - end - end - - gps_relative_time = gps.gps_time-gps.gps_time(1); - relative_time = relative_time + median(gps_relative_time-relative_time); - - plot(gps_relative_time-relative_time,'.') - pause; - ref_idx = find(abs(gps_relative_time-relative_time)<1e-3,1); - gps.gps_time = gps.gps_time(ref_idx) + relative_time - relative_time(ref_idx); - -else - gps_date = OnBoard.signals.values(:,find(strcmp(OnBoard.signals.labels,'GPSDATE'))); - gps_time = OnBoard.signals.values(:,find(strcmp(OnBoard.signals.labels,'GPSTIME'))); - error('Unsupported format'); - utc_time = datenum(utc_year,utc_month,utc_day,utc_hours,utc_min,utc_sec); - datestr(utc_time(1)) - gps.gps_time = datenum_to_epoch(utc_time); - gps.gps_time = gps.gps_time + utc_leap_seconds(gps.gps_time(1)); -end - -gps.roll = OnBoard.signals.values(:,find(strcmp(OnBoard.signals.labels,'ROLL'))); -gps.pitch = OnBoard.signals.values(:,find(strcmp(OnBoard.signals.labels,'PITCH'))); -gps.heading = OnBoard.signals.values(:,find(strcmp(OnBoard.signals.labels,'YAW'))); -gps.lat = OnBoard.signals.values(:,find(strcmp(OnBoard.signals.labels,'LAT')))*180/pi; -gps.lon = OnBoard.signals.values(:,find(strcmp(OnBoard.signals.labels,'LON')))*180/pi; -gps.elev = OnBoard.signals.values(:,find(strcmp(OnBoard.signals.labels,'ALT'))); - -return; - -fns = get_filenames('E:\tmp\uav_gps_imu','','','AUX.mat'); -for fn_idx=1:length(fns)-1 - fns{fn_idx} - gps = read_gps_wepilot(fns{fn_idx}) - gps_plot(gps); - pause -end - -% fns = get_filenames('E:\tmp\uav_gps_imu','','','AUX.mat'); -% for fn_idx=1:length(fns) -% gps = load(fns{fn_idx}) -% end - -% tmp = load('weData_010714_35MHz_flight3.mat'); -% tmp.OnBoard = tmp.data; -% tmp = rmfield(tmp,'data'); -% save('weData_201604XX_XXXAUX.mat','-struct','tmp'); +function gps = read_gps_mat(in_fn, param) +% gps = read_gps_mat(in_fn, param) +% +% Reads in Matlab mat files. A generic Variable and Attribute mapping is +% possible with the param structure. Includes wePilot. +% +% Input Args: +% in_fn (string) input .mat filename +% param = control parameter structure +% +% Output Args: +% gps = output structure with fields +% .time = GPS time in seconds since Jan 1, 1970 epoch (sec) +% .lat = latitude (deg) +% .lon = longitude (deg) +% .elev = elevation (m) +% .roll = roll (rad) +% .pitch = pitch (rad) +% .heading = true heading (rad) +% +% Example: +% See make_gps_2016_Greenland_G1XB.m +% +% fn = 'E:\tmp\uav_gps_imu\weData_20160413_1851AUX.mat'; +% gps = read_gps_wepilot(fn); +% +% Author: John Paden +% +% See also read_gps_*.m, gps_plot.m, gps_create.m + +load(in_fn); + +gps = []; + +if any(strcmp(OnBoard.signals.labels,'UTCYEAR')) + utc_year = OnBoard.signals.values(:,find(strcmp(OnBoard.signals.labels,'UTCYEAR'))); + utc_month = OnBoard.signals.values(:,find(strcmp(OnBoard.signals.labels,'UTCMONTH'))); + utc_day = OnBoard.signals.values(:,find(strcmp(OnBoard.signals.labels,'UTCDAY'))); + utc_hours = OnBoard.signals.values(:,find(strcmp(OnBoard.signals.labels,'UTCHOURS'))); + utc_min = OnBoard.signals.values(:,find(strcmp(OnBoard.signals.labels,'UTCMIN'))); + utc_sec = OnBoard.signals.values(:,find(strcmp(OnBoard.signals.labels,'UTCSEC'))); + utc_time = datenum(utc_year,utc_month,utc_day,utc_hours,utc_min,utc_sec); + datestr(utc_time(1)) + gps.gps_time = datenum_to_epoch(utc_time); + gps.gps_time = gps.gps_time + utc_leap_seconds(gps.gps_time(1)); + + relative_time = OnBoard.signals.values(:,find(strcmp(OnBoard.signals.labels,'TIME'))); + relative_time = relative_time-relative_time(1); + + old_gps_time = gps.gps_time(1); + for gps_idx = 2:length(gps.gps_time) + if (gps.gps_time(gps_idx) - old_gps_time) < 1e-3 + gps.gps_time(gps_idx) = gps.gps_time(gps_idx-1) + (relative_time(gps_idx) - relative_time(gps_idx-1)); + else + old_gps_time = gps.gps_time(gps_idx); + end + end + + gps_relative_time = gps.gps_time-gps.gps_time(1); + relative_time = relative_time + median(gps_relative_time-relative_time); + + plot(gps_relative_time-relative_time,'.') + pause; + ref_idx = find(abs(gps_relative_time-relative_time)<1e-3,1); + gps.gps_time = gps.gps_time(ref_idx) + relative_time - relative_time(ref_idx); + +else + gps_date = OnBoard.signals.values(:,find(strcmp(OnBoard.signals.labels,'GPSDATE'))); + gps_time = OnBoard.signals.values(:,find(strcmp(OnBoard.signals.labels,'GPSTIME'))); + error('Unsupported format'); + utc_time = datenum(utc_year,utc_month,utc_day,utc_hours,utc_min,utc_sec); + datestr(utc_time(1)) + gps.gps_time = datenum_to_epoch(utc_time); + gps.gps_time = gps.gps_time + utc_leap_seconds(gps.gps_time(1)); +end + +gps.roll = OnBoard.signals.values(:,find(strcmp(OnBoard.signals.labels,'ROLL'))); +gps.pitch = OnBoard.signals.values(:,find(strcmp(OnBoard.signals.labels,'PITCH'))); +gps.heading = OnBoard.signals.values(:,find(strcmp(OnBoard.signals.labels,'YAW'))); +gps.lat = OnBoard.signals.values(:,find(strcmp(OnBoard.signals.labels,'LAT')))*180/pi; +gps.lon = OnBoard.signals.values(:,find(strcmp(OnBoard.signals.labels,'LON')))*180/pi; +gps.elev = OnBoard.signals.values(:,find(strcmp(OnBoard.signals.labels,'ALT'))); + +return; + +fns = get_filenames('E:\tmp\uav_gps_imu','','','AUX.mat'); +for fn_idx=1:length(fns)-1 + fns{fn_idx} + gps = read_gps_wepilot(fns{fn_idx}) + gps_plot(gps); + pause +end + +% fns = get_filenames('E:\tmp\uav_gps_imu','','','AUX.mat'); +% for fn_idx=1:length(fns) +% gps = load(fns{fn_idx}) +% end + +% tmp = load('weData_010714_35MHz_flight3.mat'); +% tmp.OnBoard = tmp.data; +% tmp = rmfield(tmp,'data'); +% save('weData_201604XX_XXXAUX.mat','-struct','tmp'); diff --git a/cresis-toolbox/gps/read_ins_applanix.m b/cresis-toolbox/gps/read_ins_applanix.m index ede6813c..814102e2 100644 --- a/cresis-toolbox/gps/read_ins_applanix.m +++ b/cresis-toolbox/gps/read_ins_applanix.m @@ -1,174 +1,174 @@ -% script read_ins_applanix -% -% Very basic script for loading and plotting the display plot exported -% results of acceleration from POSPac. -% -% Author: John Paden - -% fn: Exported file from Applanix POSPac software -fn = 'C:\tmp\Forward Processed Trajectory, Reference Frame.txt'; -save_dir = 'C:\tmp'; - -%% Read in Acceleration file -fid = fopen(fn); - -A = textscan(fid, '%f %f %f %f %f','HeaderLines',1); -feof(fid); - -fclose(fid); - -%% All events -time = A{1} - A{1}(1); -x = A{2}; -y = A{3}; -z = A{4}; -total = A{5}; -dt = time(2)-time(1); -Nt = length(time); -df = 1/(Nt*dt); -freq = df*(0:Nt-1); - -if 0 - [B_filt,A_filt] = butter(2,0.13); - [B_filt_lat,A_filt_lat] = butter(2,0.09); - freqz(B_filt,A_filt); - z_filt = filter(B_filt,A_filt,z); - x_filt = filter(B_filt_lat,A_filt_lat,x); - y_filt = filter(B_filt_lat,A_filt_lat,y); - - figure(1); clf; - clf - plot(freq, lp(fft(z))) - hold on - plot(freq, lp(fft(z_filt))) - xlabel('Frequency (Hz)'); - ylabel('Relative acceleration (dB)'); - legend('z','z isolated'); - - figure(4); clf; - clf - plot(freq, lp(fft(x))) - hold on - plot(freq, lp(fft(y))) - xlabel('Frequency (Hz)'); - ylabel('Relative acceleration (dB)'); - legend('x','y'); - - figure(5); clf; - total_filt = sqrt(x_filt.^2 + y_filt.^2 + z_filt.^2); - clf - plot(time, total) - hold on - plot(time, total_filt) - xlabel('Time (sec)'); - ylabel('Acceleration (g)'); - legend('total','total isolated'); - - figure(2); clf; - plot(time,z); - hold on - plot(time,z_filt); - xlabel('Time (sec)'); - ylabel('Acceleration (g)'); - legend('z','z isolated'); - - figure(3); clf; - plot(time,x); - hold on; - plot(time,y); - xlabel('Time (sec)'); - ylabel('Acceleration (g)'); - legend('x','y'); - - return -end - -figure(1); clf; -plot(time,A{5},'b') -xlabel('Time (sec)'); -ylabel('Acceleration (g)'); -hold on; -ylims = ylim; -plot([1805 1831.5 1831.5 1805 1805], ylims([1 1 2 2 1]),'r'); -plot([3280 3283 3283 3280 3280], ylims([1 1 2 2 1]),'g'); -plot(time,A{5},'b') -legend('Total XYZ', 'Event 1', 'Event 2'); -saveas(1,fullfile(save_dir,'Total.jpg')); - -%% Event 1 - -good_mask = time>2800 & time<2980; - -% figure(1); clf; -% plot(time(good_mask),x(good_mask)) -% hold on; -% plot(time(good_mask),y(good_mask)) -% plot(time(good_mask),z(good_mask)) - -dt = time(2)-time(1) -fs = 1/dt - -figure(2); clf; -periodogram(x(good_mask),[],'onesided',512,fs) -title('X Vibration PSD'); -figure(3); clf; -periodogram(y(good_mask),[],'onesided',512,fs) -title('Y Vibration PSD'); -figure(4); clf; -periodogram(z(good_mask),[],'onesided',512,fs) -title('Z Vibration PSD'); -saveas(2,fullfile(save_dir,'Background1_X.jpg')); -saveas(3,fullfile(save_dir,'Background1_Y.jpg')); -saveas(4,fullfile(save_dir,'Background1_Z.jpg')); - -%% Event 1 - -good_mask = time>1805 & time<1831.5; - -% figure(1); clf; -% plot(time(good_mask),x(good_mask)) -% hold on; -% plot(time(good_mask),y(good_mask)) -% plot(time(good_mask),z(good_mask)) - -dt = time(2)-time(1) -fs = 1/dt - -figure(2); clf; -periodogram(x(good_mask),[],'onesided',512,fs) -title('X Vibration PSD'); -figure(3); clf; -periodogram(y(good_mask),[],'onesided',512,fs) -title('Y Vibration PSD'); -figure(4); clf; -periodogram(z(good_mask),[],'onesided',512,fs) -title('Z Vibration PSD'); -saveas(2,fullfile(save_dir,'Event1_X.jpg')); -saveas(3,fullfile(save_dir,'Event1_Y.jpg')); -saveas(4,fullfile(save_dir,'Event1_Z.jpg')); - -%% Event 2 - -good_mask = time>3280 & time<3283; - -figure(1); clf; -plot(time(good_mask),x(good_mask)) -hold on; -plot(time(good_mask),y(good_mask)) -plot(time(good_mask),z(good_mask)) - -dt = time(2)-time(1) -fs = 1/dt - -figure(2); clf; -periodogram(x(good_mask),[],'onesided',512,fs) -title('X Vibration PSD'); -figure(3); clf; -periodogram(y(good_mask),[],'onesided',512,fs) -title('Y Vibration PSD'); -figure(4); clf; -periodogram(z(good_mask),[],'onesided',512,fs) -title('Z Vibration PSD'); -saveas(2,fullfile(save_dir,'Event2_X.jpg')); -saveas(3,fullfile(save_dir,'Event2_Y.jpg')); -saveas(4,fullfile(save_dir,'Event2_Z.jpg')); +% script read_ins_applanix +% +% Very basic script for loading and plotting the display plot exported +% results of acceleration from POSPac. +% +% Author: John Paden + +% fn: Exported file from Applanix POSPac software +fn = 'C:\tmp\Forward Processed Trajectory, Reference Frame.txt'; +save_dir = 'C:\tmp'; + +%% Read in Acceleration file +fid = fopen(fn); + +A = textscan(fid, '%f %f %f %f %f','HeaderLines',1); +feof(fid); + +fclose(fid); + +%% All events +time = A{1} - A{1}(1); +x = A{2}; +y = A{3}; +z = A{4}; +total = A{5}; +dt = time(2)-time(1); +Nt = length(time); +df = 1/(Nt*dt); +freq = df*(0:Nt-1); + +if 0 + [B_filt,A_filt] = butter(2,0.13); + [B_filt_lat,A_filt_lat] = butter(2,0.09); + freqz(B_filt,A_filt); + z_filt = filter(B_filt,A_filt,z); + x_filt = filter(B_filt_lat,A_filt_lat,x); + y_filt = filter(B_filt_lat,A_filt_lat,y); + + figure(1); clf; + clf + plot(freq, lp(fft(z))) + hold on + plot(freq, lp(fft(z_filt))) + xlabel('Frequency (Hz)'); + ylabel('Relative acceleration (dB)'); + legend('z','z isolated'); + + figure(4); clf; + clf + plot(freq, lp(fft(x))) + hold on + plot(freq, lp(fft(y))) + xlabel('Frequency (Hz)'); + ylabel('Relative acceleration (dB)'); + legend('x','y'); + + figure(5); clf; + total_filt = sqrt(x_filt.^2 + y_filt.^2 + z_filt.^2); + clf + plot(time, total) + hold on + plot(time, total_filt) + xlabel('Time (sec)'); + ylabel('Acceleration (g)'); + legend('total','total isolated'); + + figure(2); clf; + plot(time,z); + hold on + plot(time,z_filt); + xlabel('Time (sec)'); + ylabel('Acceleration (g)'); + legend('z','z isolated'); + + figure(3); clf; + plot(time,x); + hold on; + plot(time,y); + xlabel('Time (sec)'); + ylabel('Acceleration (g)'); + legend('x','y'); + + return +end + +figure(1); clf; +plot(time,A{5},'b') +xlabel('Time (sec)'); +ylabel('Acceleration (g)'); +hold on; +ylims = ylim; +plot([1805 1831.5 1831.5 1805 1805], ylims([1 1 2 2 1]),'r'); +plot([3280 3283 3283 3280 3280], ylims([1 1 2 2 1]),'g'); +plot(time,A{5},'b') +legend('Total XYZ', 'Event 1', 'Event 2'); +saveas(1,fullfile(save_dir,'Total.jpg')); + +%% Event 1 + +good_mask = time>2800 & time<2980; + +% figure(1); clf; +% plot(time(good_mask),x(good_mask)) +% hold on; +% plot(time(good_mask),y(good_mask)) +% plot(time(good_mask),z(good_mask)) + +dt = time(2)-time(1) +fs = 1/dt + +figure(2); clf; +periodogram(x(good_mask),[],'onesided',512,fs) +title('X Vibration PSD'); +figure(3); clf; +periodogram(y(good_mask),[],'onesided',512,fs) +title('Y Vibration PSD'); +figure(4); clf; +periodogram(z(good_mask),[],'onesided',512,fs) +title('Z Vibration PSD'); +saveas(2,fullfile(save_dir,'Background1_X.jpg')); +saveas(3,fullfile(save_dir,'Background1_Y.jpg')); +saveas(4,fullfile(save_dir,'Background1_Z.jpg')); + +%% Event 1 + +good_mask = time>1805 & time<1831.5; + +% figure(1); clf; +% plot(time(good_mask),x(good_mask)) +% hold on; +% plot(time(good_mask),y(good_mask)) +% plot(time(good_mask),z(good_mask)) + +dt = time(2)-time(1) +fs = 1/dt + +figure(2); clf; +periodogram(x(good_mask),[],'onesided',512,fs) +title('X Vibration PSD'); +figure(3); clf; +periodogram(y(good_mask),[],'onesided',512,fs) +title('Y Vibration PSD'); +figure(4); clf; +periodogram(z(good_mask),[],'onesided',512,fs) +title('Z Vibration PSD'); +saveas(2,fullfile(save_dir,'Event1_X.jpg')); +saveas(3,fullfile(save_dir,'Event1_Y.jpg')); +saveas(4,fullfile(save_dir,'Event1_Z.jpg')); + +%% Event 2 + +good_mask = time>3280 & time<3283; + +figure(1); clf; +plot(time(good_mask),x(good_mask)) +hold on; +plot(time(good_mask),y(good_mask)) +plot(time(good_mask),z(good_mask)) + +dt = time(2)-time(1) +fs = 1/dt + +figure(2); clf; +periodogram(x(good_mask),[],'onesided',512,fs) +title('X Vibration PSD'); +figure(3); clf; +periodogram(y(good_mask),[],'onesided',512,fs) +title('Y Vibration PSD'); +figure(4); clf; +periodogram(z(good_mask),[],'onesided',512,fs) +title('Z Vibration PSD'); +saveas(2,fullfile(save_dir,'Event2_X.jpg')); +saveas(3,fullfile(save_dir,'Event2_Y.jpg')); +saveas(4,fullfile(save_dir,'Event2_Z.jpg')); diff --git a/cresis-toolbox/gps/read_ins_litton.m b/cresis-toolbox/gps/read_ins_litton.m index 02991867..a18f0b9f 100644 --- a/cresis-toolbox/gps/read_ins_litton.m +++ b/cresis-toolbox/gps/read_ins_litton.m @@ -37,7 +37,7 @@ % % Author: John Paden % -% See also read_gps_*.m, gps_plot.m, gps_make.m +% See also read_gps_*.m, gps_plot.m, gps_create.m if ischar(litton_fn) tmp = litton_fn; diff --git a/cresis-toolbox/gps/records_create_sync_gps.m b/cresis-toolbox/gps/records_create_sync_gps.m index 55a39de7..ac498d5a 100644 --- a/cresis-toolbox/gps/records_create_sync_gps.m +++ b/cresis-toolbox/gps/records_create_sync_gps.m @@ -35,7 +35,9 @@ end %% Load the GPS data -gps = load(ct_filename_support(param,param.records.gps.fn,'gps',true)); +gps_fn = ct_filename_support(param,param.records.gps.fn,'gps',true); +fprintf('Loading GPS data: %s (%s)\n', gps_fn, datestr(now)); +gps = gps_load(gps_fn); %% Check for non-monotonically increasing gps time if any(diff(gps.gps_time) <= 0) @@ -132,11 +134,19 @@ % UTUA RDS based systems radar_gps_time = radar_time + max(param.records.gps.time_offset); +elseif any(param.records.file.version == [415]) + % UTIG RDS based systems + good_mask = gps.comp_time >= comp_time(1) & gps.comp_time <= comp_time(end); + radar_gps_time = interp1(gps.radar_time(good_mask), gps.gps_time(good_mask), ... + radar_time + max(param.records.gps.time_offset),'linear','extrap'); + elseif any(param.records.file.version == [9 10 103 412]) % Arena based systems % Interpolate gps.sync_gps_time to radar gps_time using gps.radar_time % and radar_time + % Note: radar_time is relative on older Arena systems + % radar_time is UTC time from NMEA GPRMC message on newer Arena systems radar_gps_time = interp1(gps.radar_time, gps.sync_gps_time, ... radar_time + max(param.records.gps.time_offset),'linear','extrap'); @@ -162,12 +172,17 @@ end %% Synchronize times to get positions and absolute time -my_struct.lat = double(interp1(gps.gps_time,gps.lat,radar_gps_time)); -my_struct.lon = double(mod(interp1(gps.gps_time,unwrap(gps.lon/180*pi),radar_gps_time)*180/pi+180, 360)-180); -my_struct.elev = double(interp1(gps.gps_time,gps.elev,radar_gps_time)); -my_struct.roll = double(interp1(gps.gps_time,gps.roll,radar_gps_time)); -my_struct.pitch = double(interp1(gps.gps_time,gps.pitch,radar_gps_time)); -my_struct.heading = double(mod(interp1(gps.gps_time,unwrap(gps.heading),radar_gps_time)+pi,2*pi)-pi); +% NOTE: Extrapolation outside the time frame that the GPS data were +% collected is not allowed as it results in bad data, so the linear +% interpolation is set to NaN for extrapolation. Rather than rely on +% extrapolation, modify the make_gps_SEASON_NAME.m function to extend the +% GPS records as needed. +my_struct.lat = double(interp1(gps.gps_time,gps.lat,radar_gps_time,'spline',NaN)); +my_struct.lon = double(gps_interp1(gps.gps_time,gps.lon/180*pi,radar_gps_time,'spline',NaN))*180/pi; +my_struct.elev = double(interp1(gps.gps_time,gps.elev,radar_gps_time,'spline',NaN)); +my_struct.roll = double(interp1(gps.gps_time,gps.roll,radar_gps_time,'spline',NaN)); +my_struct.pitch = double(interp1(gps.gps_time,gps.pitch,radar_gps_time,'spline',NaN)); +my_struct.heading = double(gps_interp1(gps.gps_time,gps.heading,radar_gps_time,'spline',NaN)); my_struct.gps_time = radar_gps_time; my_struct.gps_source = gps.gps_source; diff --git a/cresis-toolbox/gps/run_all_gps_check.m b/cresis-toolbox/gps/run_all_gps_check.m new file mode 100644 index 00000000..d38373da --- /dev/null +++ b/cresis-toolbox/gps/run_all_gps_check.m @@ -0,0 +1,17 @@ +% script run_all_frames_check +% +% Script for running records check on many seasons at once. +% +% Author: John Paden +% +% See also: check_data_products, frames_check, gps_check, records_check +% gps_load, gps_create +% run_all_frames_check, run_all_gps_check, run_all_records_check + +run_all; + +for param_fns_idx = 1:length(param_fns) + param_fn = param_fns{param_fns_idx}; + param = regexp(param_fn,'(?\w+)_param_(?\w+)','names'); + gps_check(param); +end diff --git a/cresis-toolbox/gps/run_google_map.m b/cresis-toolbox/gps/run_google_map.m index 64eab84f..8c25099c 100644 --- a/cresis-toolbox/gps/run_google_map.m +++ b/cresis-toolbox/gps/run_google_map.m @@ -3,7 +3,8 @@ delete(gGoogle); end -gGoogle = google_map(); +gGoogle = google_map('AIzaSyCNexiP6WcIda8ZEa2MnwznWrGotDoLu0w'); +% global gRadar; gGoogle = google_map(gRadar.ops.google_map_api_key); % [A,x_axis,y_axis] = gGoogle.request_google_map(10,23,10,23); diff --git a/cresis-toolbox/gps/trajectory_coord_system.m b/cresis-toolbox/gps/trajectory_coord_system.m new file mode 100644 index 00000000..68ebf1e9 --- /dev/null +++ b/cresis-toolbox/gps/trajectory_coord_system.m @@ -0,0 +1,69 @@ +function [est_heading,along_track,speed,origin,x,y,z,pos] = trajectory_coord_system(gps) +% [est_heading,along_track,speed,origin,x,y,z,pos] = trajectory_coord_system(gps) + +physical_constants; % Load WGS84.spheroid + +%% Along-track vector +along_track = geodetic_to_along_track(gps.lat,gps.lon,gps.elev); +rlines = get_equal_alongtrack_spacing_idxs(along_track,40); + +%% Loop to calculate trajectory every 40 meters +est_heading = zeros(1,length(gps.lat)); +x = zeros(3,length(gps.lat)); +y = zeros(3,length(gps.lat)); +z = zeros(3,length(gps.lat)); +clear origin heading east north up; +for rline_idx = 1:length(rlines)-1 + rline = rlines(rline_idx); + if rline_idx < length(rlines) + rline_end = rlines(rline_idx+1); + else + rline_end = length(along_track); + end + + %% Estimate heading + [origin(1),origin(2),origin(3)] = geodetic2ecef(WGS84.spheroid,gps.lat(rline),gps.lon(rline),gps.elev(rline)); + [heading(1),heading(2),heading(3)] = geodetic2ecef(WGS84.spheroid,gps.lat(rline_end),gps.lon(rline_end),gps.elev(rline_end)); + heading = heading - origin; + % Determine east vector + [east(1) east(2) east(3)] = enu2ecef(1,0,0,gps.lat(rline),gps.lon(rline),gps.elev(rline),WGS84.spheroid); + east = east - origin; + % Determine north vector + [north(1) north(2) north(3)] = enu2ecef(0,1,0,gps.lat(rline),gps.lon(rline),gps.elev(rline),WGS84.spheroid); + north = north - origin; + % Determine up vector + [up(1) up(2) up(3)] = enu2ecef(0,0,1,gps.lat(rline),gps.lon(rline),gps.elev(rline),WGS84.spheroid); + up = up - origin; + % Determine heading (North is zero, positive towards east) + est_heading(rline:rline_end) = pi/2-atan2(dot(north,heading),dot(east,heading)); + + %% Create local trajectory coordinate system: x,y,z + % Matches standard SAR coordinate system: + % x along-track + % y pointing left + % z pointing up + x_vec = heading(:)/norm(heading); + z_vec = up(:) - x_vec*dot(up(:),x_vec); + z_vec = z_vec/norm(z_vec); + y_vec = cross(z_vec,x_vec); + x(:,rline:rline_end) = repmat(x_vec,[1 rline_end-rline+1]); + y(:,rline:rline_end) = repmat(y_vec,[1 rline_end-rline+1]); + z(:,rline:rline_end) = repmat(z_vec,[1 rline_end-rline+1]); +end + +%% Complete local coordinate system: origin, pos +origin = zeros(3,length(gps.lat)); +[origin(1,:),origin(2,:),origin(3,:)] = geodetic2ecef(WGS84.spheroid,gps.lat,gps.lon,gps.elev); +pos = zeros(3,length(gps.lat)); + +%% Calculate speed and interpolate heading through low-speed sections +if isfield(gps,'gps_time') + speed = diff(along_track)./diff(gps.gps_time); + speed(end+1) = speed(end); +else + speed = nan(size(along_track)); +end + +% Don't trust heading when speed is too low +est_heading(speed < 0.5) = NaN; +est_heading = interp_finite(est_heading,0,@gps_interp1); diff --git a/cresis-toolbox/gps/write_garmin_fpl.m b/cresis-toolbox/gps/write_garmin_fpl.m index 0feda861..721a775b 100644 --- a/cresis-toolbox/gps/write_garmin_fpl.m +++ b/cresis-toolbox/gps/write_garmin_fpl.m @@ -1,123 +1,123 @@ -function status = write_garmin_fpl(fn, fline) -% status = write_garmin_fpl(fn, fline) -% -% fn = filename to write out to. If there are too many waypoints -% to fit into a single file (MAX_WAYPOINTS variable), then -% multiple files are written with "_M" appended to fn where M -% is the file number. If this happens, there is one overlapping -% waypoint added to each file. For example, with MAX_WAYPOINTS -% equal to 30 and an input with 35 waypoints, you will get two -% files. The first file will have waypoints 1-30 and the next -% file will have waypoints 30-35. -% fline = struct array with flight line information -% .lat = N x 1 double vector of latitudes (degrees, positive is north) -% .lon = N x 1 double vector of longitudes (degrees, positive is east) -% .wpnt_names = N x 1 cell vector of way point char arrays -% -% status = return 1 on success and 0 on failure -% -% Author: John Paden - -MAX_WAYPOINTS = 30; - -status = 0; - -%% Remove bad characters (not sure what these are, so being conservative) -% 1. Alphanumeric (and remove '_' characters) -% 2. Not longer than 5 characters -% 3. All upper case% -for wpnt_idx = 1:length(fline.lat) - fline.wpnt_names{wpnt_idx} ... - = fline.wpnt_names{wpnt_idx}(... - isstrprop(fline.wpnt_names{wpnt_idx},'alphanum') ... - & fline.wpnt_names{wpnt_idx} ~= '_'); - - fline.wpnt_names{wpnt_idx} ... - = upper(fline.wpnt_names{wpnt_idx}(1:min(5,length(fline.wpnt_names{wpnt_idx})))); -end - -%% Check for any duplicates that don't have matching lat/lon -for wpnt_idx = 1:length(fline.lat) - match_idxs = find(strcmp(fline.wpnt_names{wpnt_idx}, fline.wpnt_names)); - for match_idx = match_idxs(:).' - if fline.lat(match_idx) ~= fline.lat(wpnt_idx) ... - || fline.lon(match_idx) ~= fline.lon(wpnt_idx) - warning('No Garmin flight plan written because duplicate way points exist with nonmatching coordinates: %s', fline.wpnt_names{wpnt_idx}); - return; - end - end -end - -%% Determine how many files we have to use to get all the -% waypoints written. -num_files = ceil( (length(fline.lat) - 1)/(MAX_WAYPOINTS-1) ); - -%% Write the files out -[fn_dir fn_name fn_ext] = fileparts(fn); -orig_fline = fline; -for file_num = 1:num_files - if num_files > 1 - fn_out = fullfile(fn_dir,sprintf('%s_%d%s', fn_name, file_num, fn_ext)); - else - fn_out = fn; - end - - if file_num < num_files - fline.lat = orig_fline.lat((file_num-1)*(MAX_WAYPOINTS-1) + (1:MAX_WAYPOINTS)); - fline.lon = orig_fline.lon((file_num-1)*(MAX_WAYPOINTS-1) + (1:MAX_WAYPOINTS)); - fline.wpnt_names = orig_fline.wpnt_names((file_num-1)*(MAX_WAYPOINTS-1) + (1:MAX_WAYPOINTS)); - else - fline.lat = orig_fline.lat((file_num-1)*(MAX_WAYPOINTS-1) + 1 : end); - fline.lon = orig_fline.lon((file_num-1)*(MAX_WAYPOINTS-1) + 1 : end); - fline.wpnt_names = orig_fline.wpnt_names((file_num-1)*(MAX_WAYPOINTS-1) + 1 : end); - end - - [fid,msg] = fopen(fn_out,'w'); - if fid<0 - warning('Failed to open %s: %s', fn_out, msg); - return; - end - - %% Write header to file - fprintf(fid, '\r\n'); - fprintf(fid, '\r\n'); - fprintf(fid, ' %s\r\n', datestr(now, 'YYYY-mm-DDTHH:MM:SSZ')); - - %% Write way points to file - [~,unique_idxs] = unique(fline.wpnt_names); - fprintf(fid, ' \r\n'); - for wpnt_idx = unique_idxs' - fprintf(fid, ' \r\n'); - fprintf(fid, ' %s\r\n', fline.wpnt_names{wpnt_idx}); - fprintf(fid, ' USER WAYPOINT\r\n'); - fprintf(fid, ' \r\n'); - fprintf(fid, ' %.6f\r\n', fline.lat(wpnt_idx)); - fprintf(fid, ' %.7f\r\n', fline.lon(wpnt_idx)); - fprintf(fid, ' \r\n'); - fprintf(fid, ' \r\n'); - end - fprintf(fid, ' \r\n\r\n'); - - %% Write route to file - fprintf(fid, ' \r\n'); - fprintf(fid, ' %s/%s\r\n', fline.wpnt_names{1}, fline.wpnt_names{end}); - fprintf(fid, ' Generated with vector_editor.m\r\n'); - fprintf(fid, ' 1\r\n'); - - for wpnt_idx = 1:length(fline.lat) - fprintf(fid, ' \r\n'); - fprintf(fid, ' %s\r\n', fline.wpnt_names{wpnt_idx}); - fprintf(fid, ' USER WAYPOINT\r\n'); - fprintf(fid, ' \r\n'); - fprintf(fid, ' \r\n'); - end - fprintf(fid, ' \r\n'); - - %% Write footer and close file - fprintf(fid, '\r\n'); - fclose(fid); -end - -status = 1; - -return; +function status = write_garmin_fpl(fn, fline) +% status = write_garmin_fpl(fn, fline) +% +% fn = filename to write out to. If there are too many waypoints +% to fit into a single file (MAX_WAYPOINTS variable), then +% multiple files are written with "_M" appended to fn where M +% is the file number. If this happens, there is one overlapping +% waypoint added to each file. For example, with MAX_WAYPOINTS +% equal to 30 and an input with 35 waypoints, you will get two +% files. The first file will have waypoints 1-30 and the next +% file will have waypoints 30-35. +% fline = struct array with flight line information +% .lat = N x 1 double vector of latitudes (degrees, positive is north) +% .lon = N x 1 double vector of longitudes (degrees, positive is east) +% .wpnt_names = N x 1 cell vector of way point char arrays +% +% status = return 1 on success and 0 on failure +% +% Author: John Paden + +MAX_WAYPOINTS = 30; + +status = 0; + +%% Remove bad characters (not sure what these are, so being conservative) +% 1. Alphanumeric (and remove '_' characters) +% 2. Not longer than 5 characters +% 3. All upper case% +for wpnt_idx = 1:length(fline.lat) + fline.wpnt_names{wpnt_idx} ... + = fline.wpnt_names{wpnt_idx}(... + isstrprop(fline.wpnt_names{wpnt_idx},'alphanum') ... + & fline.wpnt_names{wpnt_idx} ~= '_'); + + fline.wpnt_names{wpnt_idx} ... + = upper(fline.wpnt_names{wpnt_idx}(1:min(5,length(fline.wpnt_names{wpnt_idx})))); +end + +%% Check for any duplicates that don't have matching lat/lon +for wpnt_idx = 1:length(fline.lat) + match_idxs = find(strcmp(fline.wpnt_names{wpnt_idx}, fline.wpnt_names)); + for match_idx = match_idxs(:).' + if fline.lat(match_idx) ~= fline.lat(wpnt_idx) ... + || fline.lon(match_idx) ~= fline.lon(wpnt_idx) + warning('No Garmin flight plan written because duplicate way points exist with nonmatching coordinates: %s', fline.wpnt_names{wpnt_idx}); + return; + end + end +end + +%% Determine how many files we have to use to get all the +% waypoints written. +num_files = ceil( (length(fline.lat) - 1)/(MAX_WAYPOINTS-1) ); + +%% Write the files out +[fn_dir fn_name fn_ext] = fileparts(fn); +orig_fline = fline; +for file_num = 1:num_files + if num_files > 1 + fn_out = fullfile(fn_dir,sprintf('%s_%d%s', fn_name, file_num, fn_ext)); + else + fn_out = fn; + end + + if file_num < num_files + fline.lat = orig_fline.lat((file_num-1)*(MAX_WAYPOINTS-1) + (1:MAX_WAYPOINTS)); + fline.lon = orig_fline.lon((file_num-1)*(MAX_WAYPOINTS-1) + (1:MAX_WAYPOINTS)); + fline.wpnt_names = orig_fline.wpnt_names((file_num-1)*(MAX_WAYPOINTS-1) + (1:MAX_WAYPOINTS)); + else + fline.lat = orig_fline.lat((file_num-1)*(MAX_WAYPOINTS-1) + 1 : end); + fline.lon = orig_fline.lon((file_num-1)*(MAX_WAYPOINTS-1) + 1 : end); + fline.wpnt_names = orig_fline.wpnt_names((file_num-1)*(MAX_WAYPOINTS-1) + 1 : end); + end + + [fid,msg] = fopen(fn_out,'w'); + if fid<0 + warning('Failed to open %s: %s', fn_out, msg); + return; + end + + %% Write header to file + fprintf(fid, '\r\n'); + fprintf(fid, '\r\n'); + fprintf(fid, ' %s\r\n', datestr(now, 'YYYY-mm-DDTHH:MM:SSZ')); + + %% Write way points to file + [~,unique_idxs] = unique(fline.wpnt_names); + fprintf(fid, ' \r\n'); + for wpnt_idx = unique_idxs' + fprintf(fid, ' \r\n'); + fprintf(fid, ' %s\r\n', fline.wpnt_names{wpnt_idx}); + fprintf(fid, ' USER WAYPOINT\r\n'); + fprintf(fid, ' \r\n'); + fprintf(fid, ' %.6f\r\n', fline.lat(wpnt_idx)); + fprintf(fid, ' %.7f\r\n', fline.lon(wpnt_idx)); + fprintf(fid, ' \r\n'); + fprintf(fid, ' \r\n'); + end + fprintf(fid, ' \r\n\r\n'); + + %% Write route to file + fprintf(fid, ' \r\n'); + fprintf(fid, ' %s/%s\r\n', fline.wpnt_names{1}, fline.wpnt_names{end}); + fprintf(fid, ' Generated with vector_editor.m\r\n'); + fprintf(fid, ' 1\r\n'); + + for wpnt_idx = 1:length(fline.lat) + fprintf(fid, ' \r\n'); + fprintf(fid, ' %s\r\n', fline.wpnt_names{wpnt_idx}); + fprintf(fid, ' USER WAYPOINT\r\n'); + fprintf(fid, ' \r\n'); + fprintf(fid, ' \r\n'); + end + fprintf(fid, ' \r\n'); + + %% Write footer and close file + fprintf(fid, '\r\n'); + fclose(fid); +end + +status = 1; + +return; diff --git a/cresis-toolbox/gps/write_gps_sonntagsequence2.m b/cresis-toolbox/gps/write_gps_sonntagsequence2.m new file mode 100644 index 00000000..5dd9c7bf --- /dev/null +++ b/cresis-toolbox/gps/write_gps_sonntagsequence2.m @@ -0,0 +1,91 @@ +function wp = write_gps_sonntagsequence2(fn, wp, along_track_spacing_minimum, interp_method,dem_fn,constant_AGL_ft) +% wp = write_gps_sonntagsequence2(fn, wp, along_track_spacing_minimum, interp_method) +% +% Sonntag navigation software waypoint writer. This includes the option to +% oversample the flight track using the Mathworks Mapping toolbox interpm +% function. +% +% See read_gps_sonntagsequence2 for details on the file format. +% +% Inputs: +% wp.name = A{1}; +% wp.lat = A{2}; +% wp.lon = A{3}; +% wp.elev = A{4}; +% +% Example: +% wp = read_gps_sonntagsequence2('SAD_backup1.sequence2'); +% wp_over_sampled = write_gps_sonntagsequence2('SAD_backup1_OVER_SAMPLED.sequence2', wp); +% +% Author: John Paden +% +% See also: read_gps_sonntagsequence2.m, read_gps*.m + +%% Input checks +if ~exist('along_track_spacing_minimum','var') + along_track_spacing_minimum = 100000; +end + +if ~exist('interp_method','var') + interp_method = 'gc'; +end + +if ~exist('dem_fn','var') + global gRadar; + if wp.lat(1) < 0 + dem_fn = fullfile(gRadar.gis_path,'antarctica','DEM','REMA','REMA_1km_dem_filled.tif'); + else + dem_fn = fullfile(gRadar.gis_path,'arctic','ArcticDEM','arcticdem_mosaic_500m_v3.0.tif'); + end +end + +if ~exist('constant_AGL_ft','var') + constant_AGL_ft = 2000; +end + +% Earth Radius (m) from WGS-84 ellipsoid (larger: equatorial radius, +% smaller: polar radius) +% earthRadius = 0.5*(6378137+6356752.31424518); +earthRadius = 6.367444657122590e+06; + +%% Interpolation +orig_lat = wp.lat; +orig_lon = wp.lon; +orig_name = wp.name; +% Interpolate +[wp.lat,wp.lon] = interpm(wp.lat,wp.lon,along_track_spacing_minimum/earthRadius*180/pi,'gc'); +% Find original points +last_match_idx = 0; +for idx = 1:length(wp.lat) + match_idx = find(wp.lat(idx) == orig_lat & wp.lon(idx) == orig_lon); + if ~isempty(match_idx) + % This is one of the original waypoints, so copy its name + wp.name{idx} = orig_name{match_idx}; + sub_waypoint_idx = 0; + last_match_idx = match_idx; + elseif last_match_idx ~= 0 + sub_waypoint_idx = sub_waypoint_idx + 1; + %wp.name{idx} = sprintf('%s_%03d', orig_name{last_match_idx}, sub_waypoint_idx); + wp.name{idx} = '-'; + end +end + + +%% Load DEM +if ~isempty(dem_fn) + [dem_RGB, dem_R, ~] = geotiffread(dem_fn); + dem_proj = geotiffinfo(dem_fn); + dem_x_axis = dem_R(3,1) + dem_R(2,1)*(1:size(dem_RGB,2)); + dem_y_axis = dem_R(3,2) + dem_R(1,2)*(1:size(dem_RGB,1)); + + [dem_x,dem_y] = projfwd(dem_proj,wp.lat,wp.lon); + elev_ground = double(interp2(dem_x_axis,dem_y_axis,dem_RGB,dem_x,dem_y)); + wp.elev = elev_ground + constant_AGL_ft*12*2.54/100; +end + +%% Write file +fid = fopen(fn,'w'); +for idx = 1:length(wp.name) + fprintf(fid,'%s %f %f %.1f\n', wp.name{idx}, wp.lat(idx), wp.lon(idx), wp.elev(idx) * 100/2.54/12); +end +fclose(fid); diff --git a/cresis-toolbox/gui/@geotiff/geotiff.m b/cresis-toolbox/gui/@geotiff/geotiff.m index 981c281f..f927a140 100644 --- a/cresis-toolbox/gui/@geotiff/geotiff.m +++ b/cresis-toolbox/gui/@geotiff/geotiff.m @@ -102,7 +102,7 @@ if isempty(geotiff_fn) error('No geotiff file specified, using lat/lon.'); end - [obj.proj] = plot_geotiff(geotiff_fn,[],[],[obj.h_fig obj.h_axes]); + [obj.proj] = geotiff_plot(geotiff_fn,[],[],[obj.h_fig obj.h_axes]); xlabel(obj.h_axes,'X (km)'); ylabel(obj.h_axes,'Y (km)'); obj.xlims = xlim(obj.h_axes); diff --git a/cresis-toolbox/gui/@imagewin/create_ui.m b/cresis-toolbox/gui/@imagewin/create_ui.m index 74e16386..30ff1ec5 100644 --- a/cresis-toolbox/gui/@imagewin/create_ui.m +++ b/cresis-toolbox/gui/@imagewin/create_ui.m @@ -63,9 +63,11 @@ function create_ui(obj) addlistener(obj.h_gui.hist_slider,'slider_changed',@obj.update_caxis); signal_filters = {'10*log10(X)', ... - '10*log10(filter2(ones(3)/9,X))', ... - '10*log10(filter2(ones(round(param1*2+1),round(param2*2+1))/round(param1*2+1)/round(param2*2+1),X))', ... - '10*log10(wiener2(X,[3 3]))'}; + '10*log10(echo_filt(X,[param1 param2]))', ... + 'echo_xcorr(10*log10(echo_filt(X,[param1 param2])),echo_xcorr_profile('''',''h_filt_offset'',-4,''h_filt_len'',6))', ... + 'echo_xcorr(10*log10(echo_filt(X,[param1 param2])),echo_xcorr_profile('''',''h_filt_offset'',-16,''h_filt_len'',20))', ... + 'echo_xcorr(10*log10(echo_filt(X,[param1 param2])),echo_xcorr_profile('''',''h_filt_offset'',-40,''h_filt_len'',50))', ... + }; obj.h_gui.signal_listbox = listbox(obj.h_fig,'Signal','listbox',signal_filters,1); diff --git a/cresis-toolbox/gui/@imagewin/update_caxis.m b/cresis-toolbox/gui/@imagewin/update_caxis.m index d8adde46..81b537e4 100644 --- a/cresis-toolbox/gui/@imagewin/update_caxis.m +++ b/cresis-toolbox/gui/@imagewin/update_caxis.m @@ -48,4 +48,5 @@ function update_caxis(obj,varargin) obj.h_gui.min_slider.set_value(clims(1)); obj.h_gui.max_slider.set_value(clims(end)); end + end diff --git a/cresis-toolbox/gui/@listbox/listbox.m b/cresis-toolbox/gui/@listbox/listbox.m index 229a112f..8587f8cd 100644 --- a/cresis-toolbox/gui/@listbox/listbox.m +++ b/cresis-toolbox/gui/@listbox/listbox.m @@ -96,6 +96,9 @@ function CM_callback(obj, h_obj, event) function list_callback(obj, h_obj, event) list_string = get(obj.h_list,'String'); cur_entry = get(obj.h_list,'Value'); + if isempty(cur_entry) + cur_entry = length(list_string); + end if h_obj == obj.h_list % User selected a value in the list if strcmpi(get(gcf,'SelectionType'),'Open') @@ -108,17 +111,16 @@ function list_callback(obj, h_obj, event) if strcmpi(obj.cur_entry_mode,'add') % Add entry to the list new_entry = get(obj.h_LE,'String'); - [list_string,sort_idxs] = sort({list_string{:}, new_entry}); + list_string = {list_string{1:cur_entry}, new_entry, list_string{cur_entry+1:end}}; set(obj.h_list,'String',list_string); - set(obj.h_list,'Value',find(sort_idxs==length(list_string))); + set(obj.h_list,'Value',cur_entry+1); obj.cur_entry_mode = 'edit'; - elseif strcmpi(obj.cur_entry_mode,'edit') + elseif strcmpi(obj.cur_entry_mode,'edit') && cur_entry > 0 % Replace an entry in the list new_entry = get(obj.h_LE,'String'); list_string{cur_entry} = new_entry; - [list_string,sort_idxs] = sort(list_string); set(obj.h_list,'String',list_string); - set(obj.h_list,'Value',find(sort_idxs==cur_entry)); + set(obj.h_list,'Value',cur_entry); end end end diff --git a/cresis-toolbox/gui/@selectionbox/selectionbox.m b/cresis-toolbox/gui/@selectionbox/selectionbox.m index 56f0b9d1..9d827fb7 100644 --- a/cresis-toolbox/gui/@selectionbox/selectionbox.m +++ b/cresis-toolbox/gui/@selectionbox/selectionbox.m @@ -16,6 +16,12 @@ fh_available fh_selected enable_mode + + % Data in listboxes + strings + ids + orders + selected_mask end properties (SetAccess = private, GetAccess = private) @@ -26,10 +32,24 @@ end methods - function obj = selectionbox(parent,name,list_values,init_value) + %% constructor + function obj = selectionbox(parent,name,list_values,init_value,list_ids,list_orders) % List boxes start enabled by default obj.enable_mode = true; + obj.strings = list_values; + if exist('list_ids','var') && ~isempty(list_ids) + obj.ids = list_ids; + else + obj.ids = 1:length(list_values); + end + if exist('list_orders','var') && ~isempty(list_orders) + obj.orders = list_orders; + else + obj.orders = 1:length(list_values); + end + obj.selected_mask = false(size(obj.strings)); + obj.h_text = uicontrol('Parent',parent); set(obj.h_text,'Style','Text'); set(obj.h_text,'String',name); @@ -50,16 +70,25 @@ obj.h_list_availableCM = uicontextmenu('Parent',parent); % Define the context menu items and install their callbacks - uimenu(obj.h_list_availableCM, 'Label', 'Add', 'Callback', @obj.add_callback); + uimenu(obj.h_list_availableCM, 'Label', 'Add', 'Callback', @obj.available_callback); + uimenu(obj.h_list_availableCM, 'Label', 'Up', 'Callback', @obj.available_callback); + uimenu(obj.h_list_availableCM, 'Label', 'Down', 'Callback', @obj.available_callback); + uimenu(obj.h_list_availableCM, 'Label', 'Top', 'Callback', @obj.available_callback); + uimenu(obj.h_list_availableCM, 'Label', 'Bottom', 'Callback', @obj.available_callback); set(obj.h_list_available,'uicontextmenu',obj.h_list_availableCM) obj.h_list_selectedCM = uicontextmenu('Parent',parent); % Define the context menu items and install their callbacks - uimenu(obj.h_list_selectedCM, 'Label', 'Remove', 'Callback', @obj.remove_callback); + uimenu(obj.h_list_selectedCM, 'Label', 'Remove', 'Callback', @obj.selected_callback); + uimenu(obj.h_list_selectedCM, 'Label', 'Up', 'Callback', @obj.selected_callback); + uimenu(obj.h_list_selectedCM, 'Label', 'Down', 'Callback', @obj.selected_callback); + uimenu(obj.h_list_selectedCM, 'Label', 'Top', 'Callback', @obj.selected_callback); + uimenu(obj.h_list_selectedCM, 'Label', 'Bottom', 'Callback', @obj.selected_callback); set(obj.h_list_selected,'uicontextmenu',obj.h_list_selectedCM) end + %% destructor function delete(obj) try delete(obj.h_list_available); @@ -69,94 +98,549 @@ function delete(obj) end end - function add_callback(obj,status,event) + %% available_callback + function available_callback(obj,status,event) h_status = get(status); - if isfield(h_status,'Label') && strcmp(get(status,'Label'),'Add') + if isfield(h_status,'Label') + context_menu = get(status,'Label'); + elseif isfield(h_status,'text') + context_menu = get(status,'text'); + else + context_menu = ''; + end + + if strcmpi(context_menu,'Add') + %% available_callback: add + % Get the highlighted entries in the available box, return if + % nothing is selected since there will be nothing to do match_idxs = get(obj.h_list_available,'Value'); - available_list = get(obj.h_list_available,'string'); - if isempty(match_idxs) || isempty(available_list); return; end; - selected_list = get(obj.h_list_selected,'string'); - selected_list = cat(1,selected_list,available_list{match_idxs}); - selected_list = sort(selected_list); - moved_mask = zeros(size(available_list)); - moved_mask(match_idxs) = 1; - available_list = available_list(~moved_mask); - set(obj.h_list_available,'value',1); + if isempty(match_idxs) + return + end + + % Create sorted selected_mask + [~,sort_idxs] = sort(obj.orders); + [~,unsort_idxs] = sort(sort_idxs); + sort_selected_mask = obj.selected_mask(sort_idxs); + + % Get indexes into the obj.selected_mask that are selected and then + % update these entries to be moved to the selected listbox + change_idxs = find(~sort_selected_mask); + sort_selected_mask(change_idxs(match_idxs)) = true; + obj.selected_mask = sort_selected_mask(unsort_idxs); + + % Create updated sorted selected_mask + sort_selected_mask = obj.selected_mask(sort_idxs); + + % Reset selected entry in the available list + set(obj.h_list_available,'value',min(sum(~obj.selected_mask),min(match_idxs))); + + % Set selected entries in the selected list to the new items + new_value = zeros(size(match_idxs)); + for idx = 1:length(match_idxs) + new_value(idx) = sum(sort_selected_mask(1:change_idxs(match_idxs(idx)))); + end + set(obj.h_list_selected,'value',new_value); + + % Update the available and selected listboxes + available_list = obj.strings(sort_idxs(~sort_selected_mask)); set(obj.h_list_available,'string',available_list); + selected_list = obj.strings(sort_idxs(sort_selected_mask)); set(obj.h_list_selected,'string',selected_list); + + % Notify event that selection has changed notify(obj,'selection_changed'); + + elseif strcmpi(context_menu,'Up') + %% available_callback: up + % Check to see if there is an element above the top selected + % element, if not quit and do nothing + match_idxs = get(obj.h_list_available,'Value'); + if isempty(match_idxs) || match_idxs(1) == 1 + return + end + + for match_idx = match_idxs(:).'; + % Create sorted selected_mask + [sort_orders,sort_idxs] = sort(obj.orders); + sort_selected_mask = obj.selected_mask(sort_idxs); + + % Create list of sorted orders with just the available items in + % it + sort_orders_masked = sort_orders(~sort_selected_mask); + + % Find order values in the range + idxs = find(obj.orders >= sort_orders_masked(match_idx-1) & obj.orders <= sort_orders_masked(match_idx)); + + % Shift the item before the selected items so that it is at the + % end. + obj.orders(idxs) = obj.orders(idxs([2:end 1])); + end + + % Create sorted selected_mask + [~,sort_idxs] = sort(obj.orders); + sort_selected_mask = obj.selected_mask(sort_idxs); + + % Update the listbox + available_list = obj.strings(sort_idxs(~sort_selected_mask)); + set(obj.h_list_available,'string',available_list); + + % Update selected entries + set(obj.h_list_available,'value',match_idxs-1); + + elseif strcmpi(context_menu,'Down') + %% available_callback: down + % Check to see if there is an element below the bottom selected + % element, if not quit and do nothing + match_idxs = get(obj.h_list_available,'Value'); + if isempty(match_idxs) || match_idxs(end) == sum(~obj.selected_mask) + return + end + + for match_idx = fliplr(match_idxs(:).'); + % Create sorted selected_mask + [sort_orders,sort_idxs] = sort(obj.orders); + sort_selected_mask = obj.selected_mask(sort_idxs); + + sort_orders_masked = sort_orders(~sort_selected_mask); + + idxs = find(obj.orders >= sort_orders_masked(match_idx) & obj.orders <= sort_orders_masked(match_idx+1)); + + obj.orders(idxs) = obj.orders(idxs([end 1:end-1])); + end + + % Create sorted selected_mask + [~,sort_idxs] = sort(obj.orders); + sort_selected_mask = obj.selected_mask(sort_idxs); + + % Update the listbox + available_list = obj.strings(sort_idxs(~sort_selected_mask)); + set(obj.h_list_available,'string',available_list); + + % Update selected entries + set(obj.h_list_available,'value',match_idxs+1); + + elseif strcmpi(context_menu,'Top') + %% available_callback: top + % Check to see if there is an element above the top selected + % element, if not quit and do nothing + match_idxs = get(obj.h_list_available,'Value'); + if isempty(match_idxs) || match_idxs(1) == 1 + return + end + + % Create sorted selected_mask + [sort_orders,sort_idxs] = sort(obj.orders); + [~,unsort_idxs] = sort(sort_idxs); + sort_selected_mask = obj.selected_mask(sort_idxs); + + % Get indexes into the obj.selected_mask that are selected and then + % update these entries to be moved to the selected listbox + sort_orders_masked = sort_orders(~sort_selected_mask); + + % Find range of orders to move + orders_mask = obj.orders >= sort_orders_masked(match_idxs(1)) & obj.orders <= sort_orders_masked(match_idxs(end)); + + % Determine where in the sorted list these (need to be moved) orders are + sort_orders_mask = orders_mask(sort_idxs); + + % Move these orders around in the sorted domain + new_sort_orders(sort_orders_mask) = sort_orders(1:sum(sort_orders_mask)); + new_sort_orders(~sort_orders_mask) = sort_orders(sum(sort_orders_mask)+1:end); + + % Convert the new ordering back to the unsorted domain + obj.orders = new_sort_orders(unsort_idxs); + + % Update sorted selected_mask + [~,sort_idxs] = sort(obj.orders); + sort_selected_mask = obj.selected_mask(sort_idxs); + + % Update the listbox + available_list = obj.strings(sort_idxs(~sort_selected_mask)); + set(obj.h_list_available,'string',available_list); + + % Update selected entries + set(obj.h_list_available,'value',1:sum(sort_orders_mask)); + + elseif strcmpi(context_menu,'Bottom') + %% available_callback: bottom + % Check to see if there is an element above the top selected + % element, if not quit and do nothing + match_idxs = get(obj.h_list_available,'Value'); + if isempty(match_idxs) || match_idxs(end) == sum(~obj.selected_mask) + return + end + + % Create sorted selected_mask + [sort_orders,sort_idxs] = sort(obj.orders); + [~,unsort_idxs] = sort(sort_idxs); + sort_selected_mask = obj.selected_mask(sort_idxs); + + % Get indexes into the obj.selected_mask that are selected and then + % update these entries to be moved to the selected listbox + sort_orders_masked = sort_orders(~sort_selected_mask); + + % Find range of orders to move + orders_mask = obj.orders >= sort_orders_masked(match_idxs(1)) & obj.orders <= sort_orders_masked(match_idxs(end)); + + % Determine where in the sorted list these (need to be moved) orders are + sort_orders_mask = orders_mask(sort_idxs); + + % Move these orders around in the sorted domain + new_sort_orders(sort_orders_mask) = sort_orders(end+(-sum(sort_orders_mask)+1:0)); + new_sort_orders(~sort_orders_mask) = sort_orders(1:end-sum(sort_orders_mask)); + + % Convert the new ordering back to the unsorted domain + obj.orders = new_sort_orders(unsort_idxs); + + % Update sorted selected_mask + [~,sort_idxs] = sort(obj.orders); + sort_selected_mask = obj.selected_mask(sort_idxs); + + % Update the listbox + available_list = obj.strings(sort_idxs(~sort_selected_mask)); + set(obj.h_list_available,'string',available_list); + + % Update selected entries + set(obj.h_list_available,'value',length(available_list)+(-sum(sort_orders_mask)+1:0)); + else obj.fh_available(status,event); end end - function remove_callback(obj,status,event) + %% selected_callback + function selected_callback(obj,status,event) + % =================================================================== + % =================================================================== + % COPIED FROM available_callback WITH CHANGES INDICATED WITH "-->" + % =================================================================== + % =================================================================== h_status = get(status); - if isfield(h_status,'Label') && strcmp(get(status,'Label'),'Remove') + if isfield(h_status,'Label') + context_menu = get(status,'Label'); + elseif isfield(h_status,'text') + context_menu = get(status,'text'); + else + context_menu = ''; + end + + if strcmpi(context_menu,'Remove') + %% selected_callback: remove + % Get the highlighted entries in the selected box, return if + % nothing is selected since there will be nothing to do match_idxs = get(obj.h_list_selected,'Value'); - available_list = get(obj.h_list_available,'string'); - selected_list = get(obj.h_list_selected,'string'); - if isempty(match_idxs) || isempty(selected_list); return; end; - available_list = cat(1,available_list,selected_list{match_idxs}); - available_list = sort(available_list); - moved_mask = zeros(size(selected_list)); - moved_mask(match_idxs) = 1; - selected_list = selected_list(~moved_mask); - set(obj.h_list_selected,'value',1); + if isempty(match_idxs) + return + end + + % Create sorted selected_mask + [~,sort_idxs] = sort(obj.orders); + [~,unsort_idxs] = sort(sort_idxs); + sort_selected_mask = obj.selected_mask(sort_idxs); + + % Get indexes into the obj.selected_mask that are selected and then + % update these entries to be moved to the selected listbox + change_idxs = find(sort_selected_mask); % ~sort_selected_mask --> sort_selected_mask + sort_selected_mask(change_idxs(match_idxs)) = false; % true --> false + obj.selected_mask = sort_selected_mask(unsort_idxs); + + % Create updated sorted selected_mask + sort_selected_mask = obj.selected_mask(sort_idxs); + + % Reset selected entry in the selected list + set(obj.h_list_selected,'value',min(sum(obj.selected_mask),min(match_idxs))); % h_list_available --> h_list_selected + + % Set selected entries in the available list to the new items + new_value = zeros(size(match_idxs)); + for idx = 1:length(match_idxs) + new_value(idx) = sum(~sort_selected_mask(1:change_idxs(match_idxs(idx)))); % ~sort_selected_mask --> sort_selected_mask + end + set(obj.h_list_available,'value',new_value); % h_list_selected --> h_list_available + + % Update the available and selected listboxes + available_list = obj.strings(sort_idxs(~sort_selected_mask)); set(obj.h_list_available,'string',available_list); + selected_list = obj.strings(sort_idxs(sort_selected_mask)); set(obj.h_list_selected,'string',selected_list); + + % Notify event that selection has changed notify(obj,'selection_changed'); + + elseif strcmpi(context_menu,'Up') + %% selected_callback: up + % Check to see if there is an element above the top selected + % element, if not quit and do nothing + match_idxs = get(obj.h_list_selected,'Value'); % h_list_available --> h_list_selected + if isempty(match_idxs) || match_idxs(1) == 1 + return + end + + for match_idx = match_idxs(:).'; + % Create sorted selected_mask + [sort_orders,sort_idxs] = sort(obj.orders); + sort_selected_mask = obj.selected_mask(sort_idxs); + + % Create list of sorted orders with just the available items in + % it + sort_orders_masked = sort_orders(sort_selected_mask); % ~sort_selected_mask --> sort_selected_mask + + % Find order values in the range + idxs = find(obj.orders >= sort_orders_masked(match_idx-1) & obj.orders <= sort_orders_masked(match_idx)); + + % Shift the item before the selected items so that it is at the + % end. + obj.orders(idxs) = obj.orders(idxs([2:end 1])); + end + + % Create sorted selected_mask + [~,sort_idxs] = sort(obj.orders); + sort_selected_mask = obj.selected_mask(sort_idxs); + + % Update the listbox + selected_list = obj.strings(sort_idxs(sort_selected_mask)); % ~sort_selected_mask --> sort_selected_mask + set(obj.h_list_selected,'string',selected_list); % h_list_available --> h_list_selected + + % Update selected entries + set(obj.h_list_selected,'value',match_idxs-1); % h_list_available --> h_list_selected + + elseif strcmpi(context_menu,'Down') + %% selected_callback: down + % Check to see if there is an element below the bottom selected + % element, if not quit and do nothing + match_idxs = get(obj.h_list_selected,'Value'); % h_list_available --> h_list_selected + if isempty(match_idxs) || match_idxs(end) == sum(obj.selected_mask) % ~obj.selected_mask --> obj.selected_mask + return + end + + for match_idx = fliplr(match_idxs(:).'); + % Create sorted selected_mask + [sort_orders,sort_idxs] = sort(obj.orders); + sort_selected_mask = obj.selected_mask(sort_idxs); + + sort_orders_masked = sort_orders(sort_selected_mask); % ~sort_selected_mask --> sort_selected_mask + + idxs = find(obj.orders >= sort_orders_masked(match_idx) & obj.orders <= sort_orders_masked(match_idx+1)); + + obj.orders(idxs) = obj.orders(idxs([end 1:end-1])); + end + + % Create sorted selected_mask + [~,sort_idxs] = sort(obj.orders); + sort_selected_mask = obj.selected_mask(sort_idxs); + + % Update the listbox + selected_list = obj.strings(sort_idxs(sort_selected_mask)); % ~sort_selected_mask --> sort_selected_mask + set(obj.h_list_selected,'string',selected_list); % h_list_available --> h_list_selected + + % Update selected entries + set(obj.h_list_selected,'value',match_idxs+1); % h_list_available --> h_list_selected + + elseif strcmpi(context_menu,'Top') + %% selected_callback: top + % Check to see if there is an element above the top selected + % element, if not quit and do nothing + match_idxs = get(obj.h_list_selected,'Value'); % h_list_available --> h_list_selected + if isempty(match_idxs) || match_idxs(1) == 1 + return + end + + % Create sorted selected_mask + [sort_orders,sort_idxs] = sort(obj.orders); + [~,unsort_idxs] = sort(sort_idxs); + sort_selected_mask = obj.selected_mask(sort_idxs); + + % Get indexes into the obj.selected_mask that are selected and then + % update these entries to be moved to the selected listbox + sort_orders_masked = sort_orders(sort_selected_mask); % ~sort_selected_mask --> sort_selected_mask + + % Find range of orders to move + orders_mask = obj.orders >= sort_orders_masked(match_idxs(1)) & obj.orders <= sort_orders_masked(match_idxs(end)); + + % Determine where in the sorted list these (need to be moved) orders are + sort_orders_mask = orders_mask(sort_idxs); + + % Move these orders around in the sorted domain + new_sort_orders(sort_orders_mask) = sort_orders(1:sum(sort_orders_mask)); + new_sort_orders(~sort_orders_mask) = sort_orders(sum(sort_orders_mask)+1:end); + + % Convert the new ordering back to the unsorted domain + obj.orders = new_sort_orders(unsort_idxs); + + % Update sorted selected_mask + [~,sort_idxs] = sort(obj.orders); + sort_selected_mask = obj.selected_mask(sort_idxs); + + % Update the listbox + selected_list = obj.strings(sort_idxs(sort_selected_mask)); % ~sort_selected_mask --> sort_selected_mask + set(obj.h_list_selected,'string',selected_list); % h_list_available --> h_list_selected + + % Update selected entries + set(obj.h_list_selected,'value',1:sum(sort_orders_mask)); % h_list_available --> h_list_selected + + elseif strcmpi(context_menu,'Bottom') + %% selected_callback: bottom + % Check to see if there is an element above the top selected + % element, if not quit and do nothing + match_idxs = get(obj.h_list_selected,'Value'); % h_list_available --> h_list_selected + if isempty(match_idxs) || match_idxs(end) == sum(obj.selected_mask) % ~obj.selected_mask --> obj.selected_mask + return + end + + % Create sorted selected_mask + [sort_orders,sort_idxs] = sort(obj.orders); + [~,unsort_idxs] = sort(sort_idxs); + sort_selected_mask = obj.selected_mask(sort_idxs); + + % Get indexes into the obj.selected_mask that are selected and then + % update these entries to be moved to the selected listbox + sort_orders_masked = sort_orders(sort_selected_mask); % ~sort_selected_mask --> sort_selected_mask + + % Find range of orders to move + orders_mask = obj.orders >= sort_orders_masked(match_idxs(1)) & obj.orders <= sort_orders_masked(match_idxs(end)); + + % Determine where in the sorted list these (need to be moved) orders are + sort_orders_mask = orders_mask(sort_idxs); + + % Move these orders around in the sorted domain + new_sort_orders(sort_orders_mask) = sort_orders(end+(-sum(sort_orders_mask)+1:0)); + new_sort_orders(~sort_orders_mask) = sort_orders(1:end-sum(sort_orders_mask)); + + % Convert the new ordering back to the unsorted domain + obj.orders = new_sort_orders(unsort_idxs); + + % Update sorted selected_mask + [~,sort_idxs] = sort(obj.orders); + sort_selected_mask = obj.selected_mask(sort_idxs); + + % Update the listbox + selected_list = obj.strings(sort_idxs(sort_selected_mask)); % ~sort_selected_mask --> sort_selected_mask + set(obj.h_list_selected,'string',selected_list); % h_list_available --> h_list_selected + + % Update selected entries + set(obj.h_list_selected,'value',length(selected_list)+(-sum(sort_orders_mask)+1:0)); % h_list_available --> h_list_selected + else obj.fh_available(status,event); end end + %% list_callback function list_callback(obj, h_obj, event) if strcmpi(get(gcf,'SelectionType'),'Open') % Double click causes the currently selected item to switch lists - match_idx = get(h_obj,'Value'); - available_list = get(obj.h_list_available,'string'); - selected_list = get(obj.h_list_selected,'string'); + + % Get the highlighted entries in the available box, return if + % nothing is selected since there will be nothing to do + match_idxs = get(h_obj,'Value'); + if isempty(match_idxs) + return; + end + + % Create sorted selected_mask + [~,sort_idxs] = sort(obj.orders); + [~,unsort_idxs] = sort(sort_idxs); + sort_selected_mask = obj.selected_mask(sort_idxs); + + % Determine which list was double clicked in if h_obj == obj.h_list_available - selected_list{end+1} = available_list{match_idx}; - selected_list = sort(selected_list); - available_list = available_list([1:match_idx-1 match_idx+1:end]); + % Move from available to selected + % --------------------------------------------------------------- + + % Get indexes into the obj.selected_mask that are selected and then + % update these entries to be moved to the selected listbox + change_idxs = find(~sort_selected_mask); + sort_selected_mask(change_idxs(match_idxs)) = true; + obj.selected_mask = sort_selected_mask(unsort_idxs); + + % Create updated sorted selected_mask + sort_selected_mask = obj.selected_mask(sort_idxs); + + % Reset selected entry in the available list + set(obj.h_list_available,'value',min(sum(~obj.selected_mask),min(match_idxs))); + + % Set selected entries in the selected list to the new items + new_value = zeros(size(match_idxs)); + for idx = 1:length(match_idxs) + new_value(idx) = sum(sort_selected_mask(1:change_idxs(match_idxs(idx)))); + end + set(obj.h_list_selected,'value',new_value); + else - available_list{end+1} = selected_list{match_idx}; - available_list = sort(available_list); - selected_list = selected_list([1:match_idx-1 match_idx+1:end]); - end - avail_val = get(obj.h_list_available,'value'); - if avail_val > length(available_list) - set(obj.h_list_available,'value',1); - end - select_val = get(obj.h_list_selected,'value'); - if select_val > length(selected_list) - set(obj.h_list_selected,'value',1); + % Move from selected to available + % --------------------------------------------------------------- + + % Get indexes into the obj.selected_mask that are selected and then + % update these entries to be moved to the selected listbox + change_idxs = find(sort_selected_mask); + sort_selected_mask(change_idxs(match_idxs)) = false; + obj.selected_mask = sort_selected_mask(unsort_idxs); + + % Create updated sorted selected_mask + sort_selected_mask = obj.selected_mask(sort_idxs); + + % Reset selected entry in the selected list + set(obj.h_list_selected,'value',min(sum(obj.selected_mask),min(match_idxs))); + + % Set selected entries in the available list to the new items + new_value = zeros(size(match_idxs)); + for idx = 1:length(match_idxs) + new_value(idx) = sum(~sort_selected_mask(1:change_idxs(match_idxs(idx)))); + end + set(obj.h_list_available,'value',new_value); end + + % Update the available and selected listboxes + available_list = obj.strings(sort_idxs(~sort_selected_mask)); set(obj.h_list_available,'string',available_list); + selected_list = obj.strings(sort_idxs(sort_selected_mask)); set(obj.h_list_selected,'string',selected_list); + + % Notify event that selection has changed notify(obj,'selection_changed'); + end end + %% get_selected_values function vals = get_selected_values(obj) vals = get(obj.h_list_selected,'Value'); end - function vals = get_selected_strings(obj) - vals = get(obj.h_list_selected,'string'); + %% get_selected_strings + function strings = get_selected_strings(obj) + strings = get(obj.h_list_selected,'string'); end - function set_list(obj, vals) - % val = cell vector of character strings + %% set_list + function set_list(obj, strings, ids, orders) + % strings: cell vector of character strings that will be displayed + % + % ids: corresponding id for each string (optional). Default is + % integers from 1 to length(strings) + % + % orders: corresponding order that determines in what order the + % strings will be listed (optional). Default is the order that the + % values are passed in. % % If items in val already exist in either available or selected then % they will stay in that listbox, items not in val are removed, new % items from val are placed in available. - vals = sort(vals); + obj.strings = strings; + if exist('ids','var') && ~isempty(ids) + obj.ids = ids; + else + obj.ids = 1:length(obj.strings); + end + if exist('orders','var') && ~isempty(orders) + obj.orders = orders; + else + obj.orders = 1:length(obj.strings); + end + % Keep track of which values are in the "available" box and which ones + % of those are selected. available_values = get(obj.h_list_available,'value'); available_list = get(obj.h_list_available,'string'); if ~isempty(available_list) @@ -165,6 +649,8 @@ function set_list(obj, vals) available_highlighted = {}; end + % Keep track of which values are in the "selected" box and which ones + % of those are selected. selected_values = get(obj.h_list_selected,'value'); selected_list = get(obj.h_list_selected,'string'); if ~isempty(selected_list) @@ -173,14 +659,23 @@ function set_list(obj, vals) selected_highlighted = {}; end - selected_list = intersect(vals,selected_list); - available_list = intersect(vals,available_list); - new_available_list = setdiff(vals,[available_list(:); selected_list(:)]); - available_list = [available_list(:); new_available_list(:)]; - available_list = sort(available_list); + % Any items that were in the "selected" box, will stay in the + % selected box. All other items will be placed in the available box. + obj.selected_mask = false(size(obj.strings)); + for idx = 1:length(obj.strings) + obj.selected_mask(idx) = any(strcmpi(obj.strings{idx},selected_list)); + end + [~,sort_idxs] = sort(obj.orders); + sort_selected_mask = obj.selected_mask(sort_idxs); + + available_list = obj.strings(sort_idxs(~sort_selected_mask)); set(obj.h_list_available,'string',available_list); + selected_list = obj.strings(sort_idxs(sort_selected_mask)); set(obj.h_list_selected,'string',selected_list); + % Check which of the new items needs to be selected and also check to + % see if the selection has changed because an item is no longer + % available. available_values = []; available_highlighted_mask = false(size(available_highlighted)); for idx = 1:length(available_highlighted) @@ -201,85 +696,88 @@ function set_list(obj, vals) end end + % Reselect the same items that were selected before set(obj.h_list_available,'value',available_values); set(obj.h_list_selected,'value',selected_values); + % If the selection changed in either listbox, then notify event if any(~available_highlighted_mask) || any(~selected_highlighted_mask) notify(obj,'selection_changed'); end end - function set_available(obj, vals) - % val = cell vector of character strings (resets selectionbox) - if ~isempty(vals) + %% set_available + function set_available(obj, strings, ids, orders) + % Same as set_list, but resets the listbox so that everything shows + % up in the available box. + + obj.strings = strings; + if exist('ids','var') && ~isempty(ids) + obj.ids = ids; + else + obj.ids = 1:length(obj.strings); + end + if exist('orders','var') && ~isempty(orders) + obj.orders = orders; + else + obj.orders = 1:length(obj.strings); + end + + % Any items that were in the "selected" box, will stay in the + % selected box. All other items will be placed in the available box. + obj.selected_mask = false(size(obj.strings)); + + available_list = obj.strings; + set(obj.h_list_available,'string',available_list); + selected_list = {}; + set(obj.h_list_selected,'string',selected_list); + + % Reselect the same items that were selected before + if ~isempty(obj.strings) set(obj.h_list_available,'value',1); + else + set(obj.h_list_available,'value',[]); end - set(obj.h_list_available,'string',sort(vals)); - set(obj.h_list_selected,'string',[]); + set(obj.h_list_selected,'value',[]); + + % If the selection changed in either listbox, then notify event notify(obj,'selection_changed'); end - function set_selected(obj, vals, selected) - % val = cell vector of character strings or array of indices to strings - % selected = logical (true to select, false to deselect) + %% set_selected + function set_selected(obj, strings, selected_state) + % strings: cell vector of character strings or array of indices to + % strings + % + % selected_state: logical (true to select and move to the selected + % list, false to deselect and move to the available list) % % Strings or indices that match will be (de)selected selection_changed = false; - for val_idx = 1:length(vals) - if iscell(vals) - val = vals{val_idx}; - else - val = vals(val_idx); - end - - available_list = get(obj.h_list_available,'string'); - selected_list = get(obj.h_list_selected,'string'); - - if selected == true - % User wishes to select val - string_list = available_list; - else - % User wishes to deselect val - string_list = selected_list; - end - if ischar(val) - % Search through string list for a match - match_idx = find(strcmp(string_list,val),1); - else - % Assume val is index into string list - match_idx = val; - end - if ~isempty(match_idx) && match_idx >= 1 && match_idx <= length(string_list) - % Move selection from one list to the other depending on selected - if selected == true - selected_list{end+1} = available_list{match_idx}; - selected_list = sort(selected_list); - available_list = available_list([1:match_idx-1 match_idx+1:end]); - else - available_list{end+1} = selected_list{match_idx}; - available_list = sort(available_list); - selected_list = selected_list([1:match_idx-1 match_idx+1:end]); - end - avail_val = get(obj.h_list_available,'value'); - if avail_val > length(available_list) - set(obj.h_list_available,'value',1); - end - select_val = get(obj.h_list_selected,'value'); - if select_val > length(selected_list) - set(obj.h_list_selected,'value',1); - end - set(obj.h_list_available,'string',available_list); - set(obj.h_list_selected,'string',selected_list); + for str_idx = 1:length(strings) + match_idx = find(strcmp(strings{str_idx},obj.strings)); + if ~isempty(match_idx) && obj.selected_mask(match_idx) ~= selected_state selection_changed = true; + + obj.selected_mask(match_idx) = selected_state; end end if selection_changed + [~,sort_idxs] = sort(obj.orders); + sort_selected_mask = obj.selected_mask(sort_idxs); + + available_list = obj.strings(sort_idxs(~sort_selected_mask)); + set(obj.h_list_available,'string',available_list); + selected_list = obj.strings(sort_idxs(sort_selected_mask)); + set(obj.h_list_selected,'string',selected_list); + notify(obj,'selection_changed'); end end + %% set_enable function set_enable(obj,enable_mode) if obj.enable_mode ~= enable_mode obj.enable_mode = enable_mode; diff --git a/cresis-toolbox/gui/delete_figures.m b/cresis-toolbox/gui/delete_figures.m new file mode 100644 index 00000000..12ff46eb --- /dev/null +++ b/cresis-toolbox/gui/delete_figures.m @@ -0,0 +1,15 @@ +function delete_figures(max_fig_num) +% delete_figures(max_fig_num) +% +% Convenience function to delete all figures. This is useful when figures +% have on close callbacks or have visible set of "off". + +if ~exist('max_fig_num','var') + max_fig_num = 10000; +end + +for delete_figures_idx = 1:max_fig_num + try + delete(delete_figures_idx); + end +end diff --git a/cresis-toolbox/gui/get_figures.m b/cresis-toolbox/gui/get_figures.m index faa46623..53149ed0 100644 --- a/cresis-toolbox/gui/get_figures.m +++ b/cresis-toolbox/gui/get_figures.m @@ -32,7 +32,12 @@ else fig_visible = 'off'; end -h_figs = get(0,'Children'); +try + h_figs = get(0,'Children'); +catch ME + warning(ME.getReport); + h_figs = []; +end if num_figs <= 0 h_fig = []; diff --git a/cresis-toolbox/display/gmean.m b/cresis-toolbox/gui/gui_stats.m similarity index 100% rename from cresis-toolbox/display/gmean.m rename to cresis-toolbox/gui/gui_stats.m diff --git a/cresis-toolbox/display/fiddle_plot.m b/cresis-toolbox/gui/gui_zoom.m similarity index 100% rename from cresis-toolbox/display/fiddle_plot.m rename to cresis-toolbox/gui/gui_zoom.m diff --git a/cresis-toolbox/gui/link_axes.m b/cresis-toolbox/gui/link_axes.m deleted file mode 100644 index 91d46241..00000000 --- a/cresis-toolbox/gui/link_axes.m +++ /dev/null @@ -1,34 +0,0 @@ -function link_axes(fig_list,option) -% link_axes(fig_list,option) -% -% Simple function to link axes more quickly. -% -% fig_list: list of figure handles or numbers, default is all figures -% option: options to linkaxes, default is 'xy' -% -% Author: John Paden -% -% See also: - -h_axes = []; - -if ~exist('fig_list','var') || isempty(fig_list) - fig_list = get(0,'Children'); -end -if ~exist('option','var') || isempty(option) - option = 'xy'; -end - -for f_idx = 1:length(fig_list) - h_children = get(fig_list(f_idx),'Children'); - - for h_idx = 1:length(h_children) - if isa(h_children(h_idx),'matlab.graphics.axis.Axes') - h_axes(end+1) = h_children(h_idx); - end - end -end - -linkaxes(h_axes,option); - -end diff --git a/cresis-toolbox/gui/link_caxis.m b/cresis-toolbox/gui/link_caxis.m new file mode 100644 index 00000000..d755a2a3 --- /dev/null +++ b/cresis-toolbox/gui/link_caxis.m @@ -0,0 +1,42 @@ +function [h_axes,clims] = link_caxis(h_fig,clims) +% [h_axes] = link_caxis(h_fig,clims) +% +% Function for conveniently linking all the axes of a set of figures. The +% figures can be referenced by their double value (e.g. 1,2,3) or by their +% Matlab object handle. +% +% h_fig: list of figures (either double array or Matlab figure objects +% option: optional argument passed to linkaxes, default is 'xy' +% +% Author: John Paden +% +% See also: linkaxes.m + +if ~exist('h_fig','var') || isempty(h_fig) + h_fig = get(0,'Children'); +end +if ~exist('clims','var') || isempty(clims) + clims = []; +end + +% Get all the axes children of each figure that is passed in +h_axes = []; +for fig_idx = 1:length(h_fig) + h_children = get(h_fig(fig_idx),'children'); + for child_idx = 1:length(h_children) + if isa(h_children(child_idx),'matlab.graphics.axis.Axes') + h_axes(end+1) = h_children(child_idx); + end + end +end + +if ~isempty(h_axes) + % Set all caxis equal + if isempty(clims) + clims = caxis(h_axes(1)); + end + clims = sort(clims); + for axis_idx = 1:length(h_axes) + caxis(h_axes(axis_idx),clims); + end +end diff --git a/cresis-toolbox/gui/link_figures.m b/cresis-toolbox/gui/link_figures.m index f06b0897..cde4849d 100644 --- a/cresis-toolbox/gui/link_figures.m +++ b/cresis-toolbox/gui/link_figures.m @@ -12,6 +12,9 @@ % % See also: linkaxes.m +if ~exist('h_fig','var') || isempty(h_fig) + h_fig = get(0,'Children'); +end if ~exist('option','var') || isempty(option) option = 'xy'; end @@ -19,10 +22,12 @@ % Get all the axes children of each figure that is passed in h_axes = []; for fig_idx = 1:length(h_fig) - h_children = get(h_fig(fig_idx),'children'); - for child_idx = 1:length(h_children) - if isa(h_children(child_idx),'matlab.graphics.axis.Axes') - h_axes(end+1) = h_children(child_idx); + try + h_children = get(h_fig(fig_idx),'children'); + for child_idx = 1:length(h_children) + if isa(h_children(child_idx),'matlab.graphics.axis.Axes') + h_axes(end+1) = h_children(child_idx); + end end end end diff --git a/cresis-toolbox/gui/table/table_draw.m b/cresis-toolbox/gui/table/table_draw.m index a1ff5b5a..95170b88 100644 --- a/cresis-toolbox/gui/table/table_draw.m +++ b/cresis-toolbox/gui/table/table_draw.m @@ -1,8 +1,47 @@ function table_draw(table) % table_draw(table) % -% Initializes table ui handles. -% Also called during resizing +% Part of table_* container functions. table_draw is the user interface to +% the container functions. All other functions should not be called +% directly. +% +% Initializes table ui handles and sizes all ui elements in table. Also +% called for resizing. +% +% INPUTS: +% table: structure controlling table object +% +% .width_margin: Nrow by Ncol matrix indicating margin from left, leave +% undefined or set to NaN to use default value 0 +% +% .height_margin: Nrow by Ncol matrix indicating margin from top, leave +% undefined or set to NaN to use default value 0 +% +% .false_width: Nrow by Ncol matrix indicating width adjustment for +% contained object, leave undefined or set to NaN to use default value 0 +% +% .false_height: Nrow by Ncol matrix indicating height adjustment for +% contained object, leave undefined or set to NaN to use default value 0 +% +% .width: Nrow by Ncol matrix indicating width of contained object, set to +% inf to use the remaining space available, multiple inf entries in a row +% will dividing the remaining space equally +% +% .height: Nrow by Ncol matrix indicating height of contained object, set +% to inf to use the remaining space available, multiple inf entries in a +% row will dividing the remaining space equally +% +% .handles: Nrow by Ncol matrix of ui handles +% +% .ui: ui object that the table is associated with +% +% .origin: [0 0] by default +% +% .offset: [0 0] by default +% +% Author: John Paden +% +% See also: table_draw, table_pos, table_resize, table_size if ~isempty(table.ui) table.size = table_size(table); @@ -25,17 +64,17 @@ function table_draw(table) end end end -if ~isfield(table,'false_height') +if ~isfield(table,'false_width') for row = 1:table.rows for col = 1:table.cols - table.false_height(row,col) = NaN; + table.false_width(row,col) = NaN; end end end -if ~isfield(table,'false_width') +if ~isfield(table,'false_height') for row = 1:table.rows for col = 1:table.cols - table.false_width(row,col) = NaN; + table.false_height(row,col) = NaN; end end end @@ -122,9 +161,12 @@ function table_draw(table) if row == table.rows % Last row is always on the bottom newPos = [table_pos(table,row,col,curPos,height) width-2*table.width_margin(row,col)-table.false_width(row,col) height-2*table.height_margin(row,col)-table.false_height(row,col)]; + newPos(1) = newPos(1)-table.width_margin(row,col); newPos(2) = table.height_margin(row,col); else newPos = [table_pos(table,row,col,curPos,height) width-2*table.width_margin(row,col)-table.false_width(row,col) height-2*table.height_margin(row,col)-table.false_height(row,col)]; + newPos(1) = newPos(1)-table.width_margin(row,col); + newPos(2) = newPos(2)-table.height_margin(row,col); end newPos(1:2) = newPos(1:2) + table.origin; if ~isempty(table.handles{row,col}) diff --git a/cresis-toolbox/gui/table/table_pos.m b/cresis-toolbox/gui/table/table_pos.m index 975e88ca..31ed475f 100644 --- a/cresis-toolbox/gui/table/table_pos.m +++ b/cresis-toolbox/gui/table/table_pos.m @@ -1,9 +1,15 @@ function pos = table_pos(table, row, col, in_pos, height) % pos = table_pos(table, row, col, in_pos) % -% Convert from natural units to Matlab units +% Part of table_* container functions. table_draw is the user interface to +% the container functions. All other functions should not be called +% directly. +% +% Convert from natural units to Matlab units. Support function for table_draw.m. +% +% Author: John Paden +% +% See also: table_draw, table_pos, table_resize, table_size pos(1) = table.offset(1)+table.width_margin(row,col)+in_pos(1); pos(2) = -table.offset(2)+table.height_margin(row,col)+table.size(2)-in_pos(2)-height; - -return; diff --git a/cresis-toolbox/gui/table/table_resize.m b/cresis-toolbox/gui/table/table_resize.m index 6ccdad8a..ff4bce6c 100644 --- a/cresis-toolbox/gui/table/table_resize.m +++ b/cresis-toolbox/gui/table/table_resize.m @@ -1,10 +1,18 @@ function table_resize(hObject, eventdata, handles) % table_resize(hObject, eventdata, handles) +% +% Part of table_* container functions. table_draw is the user interface to +% the container functions. All other functions should not be called +% directly. +% +% Resize event handler for table container. +% +% Author: John Paden +% +% See also: table_draw, table_pos, table_resize, table_size [hObject,figure] = gcbo; table = get(hObject,'UserData'); table_draw(table); - -return; diff --git a/cresis-toolbox/gui/table/table_size.m b/cresis-toolbox/gui/table/table_size.m index eabbb8f8..959dd0d9 100644 --- a/cresis-toolbox/gui/table/table_size.m +++ b/cresis-toolbox/gui/table/table_size.m @@ -1,7 +1,15 @@ function Tsize = table_size(table) % Tsize = table_size(table) % -% Size of table's user interface control. Called by table_draw.m. +% Part of table_* container functions. table_draw is the user interface to +% the container functions. All other functions should not be called +% directly. +% +% Size of table's user interface control. Support function for table_draw.m. +% +% Author: John Paden +% +% See also: table_draw, table_pos, table_resize, table_size set(table.ui,'ResizeFcn',[]); set(table.ui,'Units','Points'); @@ -13,6 +21,3 @@ set(table.ui,'Units','Normalized'); set(table.ui,'ResizeFcn',@table_resize); - - -return; \ No newline at end of file diff --git a/cresis-toolbox/hardware/NAread.m b/cresis-toolbox/hardware/NAread.m index a5329f49..e4a33e4e 100644 --- a/cresis-toolbox/hardware/NAread.m +++ b/cresis-toolbox/hardware/NAread.m @@ -37,9 +37,9 @@ % data(2,1,:,1) represents first S21 measurement taken % % Examples: -% [hdr, data] = NAread([1 2],struct('ip_addr','10.117.4.117')); +% [hdr, data] = NAread([2 3],struct('ip_addr','172.16.0.201')); % [hdr, data] = NAread([1 2],struct('usb_addr','USB0::0x0957::0x0118::MY51421306::0::INSTR')) -% [hdr, data] = NAread([1 3 4],struct('ip_addr','10.117.4.117')); +% [hdr, data] = NAread([1 3 4],struct('ip_addr','172.16.0.201')); % % Author: Hara Talasila and John Paden % diff --git a/cresis-toolbox/fmcw/keysight_waveform_32GS_two_waveforms.m b/cresis-toolbox/hardware/keysight_waveform_32GS_two_waveforms.m similarity index 100% rename from cresis-toolbox/fmcw/keysight_waveform_32GS_two_waveforms.m rename to cresis-toolbox/hardware/keysight_waveform_32GS_two_waveforms.m diff --git a/cresis-toolbox/hardware/keysight_waveform_32GS_two_waveforms_taper.m b/cresis-toolbox/hardware/keysight_waveform_32GS_two_waveforms_taper.m new file mode 100644 index 00000000..2f289c33 --- /dev/null +++ b/cresis-toolbox/hardware/keysight_waveform_32GS_two_waveforms_taper.m @@ -0,0 +1,48 @@ +close all; clear all; + +fs = 32e9; %256 samples at 125MHz +dt = 1/fs; +tau = 200e-6; +max_del = 50e-6; +del = 1.756e-6; +fif = 70e6; +nt_tau = round(tau/dt); +nt_del = round(del/dt); +nt = round((tau+max_del)/dt); +tt = (0:dt:(nt-1)*dt)'; + +f0 = 2e9; +bw = 12e9; +k = bw/tau; + +chirp1 = [tukeywin(nt_tau,.1);zeros(nt-nt_tau,1)].*sin(2*pi*(f0*tt + 0.5*k*(tt.*tt))); +chirp2 = [zeros(nt_del,1);tukeywin(nt_tau,.1);zeros(nt-nt_tau-nt_del,1)].*sin(2*pi*((f0+fif)*(tt-del) + 0.5*k*((tt-del).*(tt-del)))); + +m_clk = zeros(4*nt,1); +wave_id = uint64(hex2dec('464D4357')); +for index = 1:32, + if mod(bitshift(wave_id,1-index),2), + m_clk((index-1)*2048+4*1024+1:(index-1)*2048+4*1024+2048) = ones(2048,1); + end; +end; +wave_id = uint64(hex2dec('4F49425F')); +for index = 33:64, + if mod(bitshift(wave_id,33-index),2), + m_clk((index-1)*2048+4*1024+1:(index-1)*2048+4*1024+2048) = ones(2048,1); + end; +end; + +data = zeros(4*nt,4); +data(:,1) = [ chirp1;-chirp1;-chirp1; chirp1]; +data(:,2) = [ chirp2; chirp2;-chirp2;-chirp2]; +data(0*nt+4+(4097:8192),3) = ones(4096,1); %PRI Trigger +data(1*nt+4+(4097:8192),3) = ones(4096,1); %PRI Trigger +data(2*nt+4+(4097:8192),3) = ones(4096,1); %PRI Trigger +data(3*nt+4+(4097:8192),3) = ones(4096,1); %PRI Trigger +data(:,4) = m_clk(1:4*nt); +fid1 = fopen('fmcw_2ch_32GSPS_2to14GHz_200us_4khz_01756ns_70MHz.csv','w'); +fprintf(fid1,'SampleRate=32000000000\r\n'); +fprintf(fid1,'SetConfig=true\r\n'); +fprintf(fid1,'Y1,Y2,SampleMarker1,SampleMarker2\r\n'); +fprintf(fid1,'%12.10f,%12.10f,%i,%i\r\n',data'); +fclose(fid1); \ No newline at end of file diff --git a/cresis-toolbox/hardware/keysight_waveform_32GS_two_waveforms_taper_single.m b/cresis-toolbox/hardware/keysight_waveform_32GS_two_waveforms_taper_single.m new file mode 100644 index 00000000..d11bd613 --- /dev/null +++ b/cresis-toolbox/hardware/keysight_waveform_32GS_two_waveforms_taper_single.m @@ -0,0 +1,95 @@ +fs = 32e9; %256 samples at 125MHz +dt = 1/fs; +f0 = 2.5e9; +bw = 4e9; +tau = 240e-6; +range_ft = 16500; +del = range_ft*12*2.54/100 / (3e8/2); +PRI = 300e-6; +fif = 200e6; + +% Calculate the maximum delay, PRI should be larger than this +max_del = tau + del + 10e-6 + +% Adjust delay of LO to create desired IF frequency +del = del - tau / bw * fif + +sequence = [0 300e-6 600e-6 900e-6]; +sequence1_scale = [1 1 -1 -1]; +sequence2_scale = [1 -1 1 -1]; +total_duration = 1200e-6-10e-6; + +nt = round((total_duration)/dt); +tt = (0:dt:(nt-1)*dt)'; + +taper_ratio=0.85; %linear + +k = bw/tau; +TUKEY_WIN = 2e-6/tau; + +chirp1_sequence = zeros(nt,1); +chirp2_sequence = zeros(nt,1); +for idx = 1:length(sequence) + tt_delay = (tt - sequence(idx)); + % y-intercept is 0.85 at time sequence(idx) + % y-intercept is 1 at time sequence(idx)+tau + slope = (1-taper_ratio) / tau; + y_intercept = taper_ratio; + taper = slope*(tt-sequence(idx)) + y_intercept; + chirp1_sequence = chirp1_sequence + taper.*sequence1_scale(idx).*tukeywin_cont((tt_delay-tau/2)/tau,TUKEY_WIN) .* sin(2*pi*(f0*tt_delay + 0.5*k*(tt_delay.*tt_delay))); + + tt_delay = (tt - sequence(idx) - del); + taper = slope*(tt-sequence(idx) - del) + y_intercept; + chirp2_sequence = chirp2_sequence + taper.*sequence2_scale(idx).*tukeywin_cont((tt_delay-tau/2)/tau,TUKEY_WIN) .* sin(2*pi*(f0*tt_delay + 0.5*k*(tt_delay.*tt_delay))); + figure(1); clf; + plot(tt,chirp1_sequence); + hold on; + plot(tt,chirp2_sequence); + figure(2); clf; + df = 1/(dt*nt); + freq = ifftshift(df*(-floor(nt/2):floor((nt-1)/2))); + plot(freq/1e9,db(fft(chirp1_sequence))) + hold on + plot(freq/1e9,db(fft(chirp2_sequence))) +end + +% MARKERS ARE PROBABLY NOT CORRECT... NEED TO BE FIXED +m_clk = zeros(nt,1); +wave_id = uint64(hex2dec('464D4357')); +for index = 1:32, + if mod(bitshift(wave_id,1-index),2), + m_clk((index-1)*2048+4*1024+1:(index-1)*2048+4*1024+2048) = ones(2048,1); + end; +end; +wave_id = uint64(hex2dec('4F49425F')); +for index = 33:64, + if mod(bitshift(wave_id,33-index),2), + m_clk((index-1)*2048+4*1024+1:(index-1)*2048+4*1024+2048) = ones(2048,1); + end; +end; + +data = zeros(nt,4); +data(:,1) = [chirp1_sequence]; +data(:,2) = [chirp2_sequence]; +data(0*nt+4+(4097:8192),3) = ones(4096,1); %PRI Trigger +% data(1*nt+4+(4097:8192),3) = ones(4096,1); %PRI Trigger +% data(2*nt+4+(4097:8192),3) = ones(4096,1); %PRI Trigger +% data(3*nt+4+(4097:8192),3) = ones(4096,1); %PRI Trigger +data(:,4) = m_clk(1:nt); + +fid1 = fopen('AITT_2ch_32GSPS_2G5to6G5GHz_300us_18000ft_200MHz.csv','w'); +fprintf(fid1,'SampleRate=32000000000\r\n'); +fprintf(fid1,'SetConfig=true\r\n'); +fprintf(fid1,'Y1,Y2,SampleMarker1,SampleMarker2\r\n'); +fprintf(fid1,'%12.10f,%12.10f,%i,%i\r\n',data'); +fclose(fid1); + +% Single transmitter +% Keep along-track sampling the same with 16 presums, but double range gate +Tgate = 150e6*240e-6/4e9 +Rgate = Tgate * 3e8/2 +Rgate_ft = Rgate * 100/12/2.54 +% Use two modes with zero pi mod +% Minimum 4: [1 -1 1 -1] +% 300 us PRI +% 1190 us total lengths diff --git a/cresis-toolbox/fmcw/keysight_waveform_64GS_one_waveform.m b/cresis-toolbox/hardware/keysight_waveform_64GS_one_waveform.m similarity index 100% rename from cresis-toolbox/fmcw/keysight_waveform_64GS_one_waveform.m rename to cresis-toolbox/hardware/keysight_waveform_64GS_one_waveform.m diff --git a/cresis-toolbox/hardware/keysight_waveform_64GS_one_waveform_taper.m b/cresis-toolbox/hardware/keysight_waveform_64GS_one_waveform_taper.m new file mode 100644 index 00000000..3f2eaa71 --- /dev/null +++ b/cresis-toolbox/hardware/keysight_waveform_64GS_one_waveform_taper.m @@ -0,0 +1,97 @@ +close all; clear all; + +low_alt = true; + +fs = 64e9; %512 samples corresponds to 1 cycle at 125MHz +dt = 1/fs; +tau = 240e-6; +if low_alt + max_del = 0e-6; +else + max_del = 50e-6; +end +zeros_at_end = false; +nt_tau = round(tau/dt); +nt = round((tau+max_del)/dt); +taper_ratio_dB=3; %dB +taper_ratio=1-10^(-taper_ratio_dB/20); +taper_ratio=0.85; %linear +f0 = 2.5e9; +bw = 15e9; +% bw = 2.5e9; % Wide altitude range +k = bw/tau; + +TUKEY_WIN=2e-6/tau; + +if zeros_at_end + % Zeros at end + % tt = (0:dt:(nt-1)*dt)'; + % chirp1 = [tukeywin(nt_tau,TUKEY_WIN);zeros(nt-nt_tau,1)].*sin(2*pi*(f0*tt + 0.5*k*(tt.*tt))); +else + % Zeros at start (to create delay when using this signal as LO) + tt = (0:dt:(nt-1)*dt)' - max_del; + chirp1 = [zeros(nt-nt_tau,1); tukeywin(nt_tau,TUKEY_WIN)].*sin(2*pi*(f0*tt + 0.5*k*(tt.*tt))); +end + +FreqWeight = 1 + [0.0 : [taper_ratio/nt_tau] : taper_ratio]; +if zeros_at_end + % Zeros at end + % taper=[FreqWeight(1:nt_tau)';zeros(nt-nt_tau,1)]; +else + % Zeros at start + taper=[zeros(nt-nt_tau,1);FreqWeight(1:nt_tau)']; +end +chirp1=(taper.*chirp1); +chirp1 = chirp1./max(chirp1); +figure(1); +plot(chirp1); +hold on; +plot(taper, 'k'); +m_clk = zeros(nt,1); + +%wave_id is used to uniquely identify the current waveform +%it is written to the header of the data file every epri through marker 2 +%64e9/4096 is 125/8 (using 3rd bit (number 2) of the counter +wave_id = uint64(hex2dec('464D4357')); +for index = 1:32, + if mod(bitshift(wave_id,1-index),2), + m_clk((index-1)*4096+2*2048:(index-1)*4096+2*2048+4095) = ones(4096,1); + end; +end; +wave_id = uint64(hex2dec('4F49425F')); +for index = 33:64, + if mod(bitshift(wave_id,33-index),2), + m_clk((index-1)*4096+2*2048:(index-1)*4096+2*2048+4095) = ones(4096,1); + end; +end; + +data = zeros(nt,4); +data(:,1) = chirp1; + +%marker 1 is used as the pri trigger which is sent from the AWG to the DAQ +%the location of the pri trigger can be adjusted to account for setup/hold +%instabilities due to cable lengths +%64e9/4096 is 125/8 +data(0+(4097:8192),3) = ones(4096,1); %PRI Trigger +data(:,4) = m_clk(1:nt); + +%write the file +fid1 = fopen('AITT_fmcw_1ch_64GSPS_2G5to17G5_240us_4khz_taper0.85_0us_delay.csv','w'); +% fid1 = fopen('AITT_fmcw_1ch_64GSPS_2G5to17G5_240us_4khz_taper0.85_50us_delay.csv','w'); +fprintf(fid1,'SampleRate=64000000000\r\n'); +fprintf(fid1,'SetConfig=true\r\n'); +fprintf(fid1,'Y1,Y2,SampleMarker1,SampleMarker2\r\n'); +fprintf(fid1,'%12.10f,%12.10f,%i,%i\r\n',data'); +fclose(fid1); + +if 0 + + df = 1/(dt*nt); +freq = ifftshift(df*(-floor(nt/2):floor((nt-1)/2))); + + clf; +plot(freq/1e9,db(fft(chirp1))) + + +end + diff --git a/cresis-toolbox/fmcw/keysight_waveform_load.m b/cresis-toolbox/hardware/keysight_waveform_load.m similarity index 100% rename from cresis-toolbox/fmcw/keysight_waveform_load.m rename to cresis-toolbox/hardware/keysight_waveform_load.m diff --git a/cresis-toolbox/lidar/lasdata.m b/cresis-toolbox/lidar/lasdata.m new file mode 100644 index 00000000..1a87add9 --- /dev/null +++ b/cresis-toolbox/lidar/lasdata.m @@ -0,0 +1,2075 @@ +classdef lasdata < handle + %lasdata LAS data reader / writer / converter + % Reads LAS files in format 1.0,1.1,1.2,1.3 and 1.4, supports point + % formats 0-10. This tool is intended for quick manual LAS file open, + % edit, repair and saving. + % + % Load LAS file into class with c = lasdata('file.las'); + % Default load only loads xyz coordinates. Additional variables can + % be loaded with the accessor functions get_(variable name). + % + % Not much checks are made for the modified data. Feel free to modify + % class member variables, but don't break your data. ;) + % + % Data is written in the format set in the class header, except + % that the target format can be given with the write_las function. + % + % Copyright (C) Teemu Kumpumki / Tampere University of Technology 2014 + % Licence: see the included BSD licence. + % + % See lasdata_license.txt for license information. + % + % DOWNLOADED FROM: https://www.mathworks.com/matlabcentral/fileexchange/48073-lasdata + properties + x; %X coordinate + y; %Y coordinate + z; %Z coordinate + intensity; %Return intensity + bits; %Data in bitfields + bits2;%Data in bitfields from 1.4+ + classification; %Classification + user_data; %User data + scan_angle; %Scan angle + point_source_id; %Point source id + gps_time; %GPS timestamp + red; %Red color channel + green; %Green color channel + blue; %Blue color channel + nir; %Near infrared color channel + extradata; %extra data found at the end of the point record + + Xt; %Waveform line x(t) parameter + Yt; %Waveform line y(t) parameter + Zt; %Waveform line z(t) parameter + wave_return_point; %Time in picoseconds from wave recording start + + wave_packet_descriptor; %Waveform packet descriptor index + wave_byte_offset; %Byte offset to waveform data (in/ext file) + wave_packet_size; %Size of single waveform packet + + header; %Contain LAS header + variablerecords; %Contain LAS variable records + extendedvariables; %Contains LAS extended variable records + + selection; %Filtering logical index + + wavedescriptors; %Waveform descriptor structures + filename; %Name of the datafile loaded + end + + properties (Access=private) + originalname; + waveformfile; %name of the waveform file + waveformfilefid; + isLAZ; %laz compression check (to delete temporary file in destructor) + end + +methods (Access=public) + + function obj = lasdata(filename,command) + obj.waveformfilefid = -1; + obj.filename = filename; + obj.originalname = filename; + + if exist('command','var') && ischar(command) + if strcmp(command,'createemptyobj') + obj.createemptyheader(); + disp('Empty object created, please set header & data values and then write las file. You need to read the LAS specification for header values in each format.') + elseif strcmp(command,'loadall') + obj.las_read(filename); + %try reading all variables + obj.read_intensity(); + obj.read_classification(); + obj.read_user_data(); + obj.read_scan_angle(); + obj.read_point_source_id(); + obj.read_point_wave_info(); + obj.read_gps_time(); + obj.read_bits(); + obj.read_extradata(); + obj.read_color(); + else + error(['Invalid command: ' command]); + end + else + obj.las_read(filename); + end + end + + function delete(obj) + if obj.waveformfilefid > 2 + fclose(obj.waveformfilefid); + end + if obj.isLAZ + delete([obj.filename]); + end + end + + function obj = normalize_xyz( obj ) + if isempty(obj.x) + warning('No data loaded.') + return; + end + obj.x = double(obj.x) * obj.header.scale_factor_x + obj.header.x_offset; + obj.y = double(obj.y) * obj.header.scale_factor_y + obj.header.y_offset; + obj.z = double(obj.z) * obj.header.scale_factor_z + obj.header.z_offset; + end + + function data = toint32_xyz( obj ) + if isempty(obj.x) + warning('No data loaded.') + return; + end + if isa(obj.x,'int32') + return; + end + data(:,1) = obj.x - obj.header.x_offset; + data(:,2) = obj.y - obj.header.y_offset; + data(:,3) = obj.z - obj.header.z_offset; + data(:,1) = round(data(:,1) / obj.header.scale_factor_x); + data(:,2) = round(data(:,2) / obj.header.scale_factor_y); + data(:,3) = round(data(:,3) / obj.header.scale_factor_z); + data = int32(data); + end + + function x = get_x(obj) + x = obj.x; + end + + function y = get_y(obj) + y = obj.y; + end + + function z = get_z(obj) + z = obj.z; + end + + function xy = get_xy(obj) + xy = [obj.x obj.y]; + end + + function xyz = get_xyz(obj) + xyz = [obj.x obj.y obj.z]; + end + + + function intensity = get_intensity(obj) + if isempty(obj.intensity) + obj.read_intensity(); + end + intensity = obj.intensity; + end + + function classification = get_classification(obj) + if isempty(obj.classification) + obj.read_classification(); + end + classification = obj.classification; + end + + function user_data = get_user_data(obj) + if isempty(obj.user_data) + obj.read_user_data(); + end + user_data = obj.user_data; + end + + function scan_angle = get_scan_angle(obj) + if isempty(obj.scan_angle) + obj.read_scan_angle(); + end + scan_angle = obj.scan_angle; + end + + function point_source_id = get_point_source_id(obj) + if isempty(obj.point_source_id) + obj.read_point_source_id(); + end + point_source_id = obj.point_source_id; + end + + function gps_time = get_gps_time(obj) + if isempty(obj.gps_time) + obj.read_gps_time(); + end + gps_time = obj.gps_time; + end + + function color = get_color(obj) + if isempty(obj.red) + obj.read_color(); + end + if any(obj.header.point_data_format == [8 10]) + color = [obj.red obj.green obj.blue obj.nir]; + else + color = [obj.red obj.green obj.blue]; + end + end + + function waveXYZ = get_waveXYZ(obj) + if isempty(obj.Xt) + obj.read_point_wave_info(); + end + waveXYZ = [obj.Xt obj.Yt obj.Zt]; + end + + function return_point = get_wave_return_point(obj) + if isempty(obj.Xt) + obj.read_point_wave_info(); + end + return_point = obj.wave_return_point; + end + + function wavedesc = get_wave_descriptor(obj) + if isempty(obj.Xt) + obj.read_point_wave_info(); + end + wavedesc = obj.wavedescriptors; + end + + function returns = get_return_number(obj) + if isempty(obj.bits) + obj.read_bits(); + end + if obj.header.point_data_format < 6 + returns = bitand(obj.bits,7); + else + returns = bitand(obj.bits,15); + end + end + + function returns = get_number_of_returns(obj) + if isempty(obj.bits) + obj.read_bits(); + end + if obj.header.point_data_format < 6 + returns = bitshift(bitand(obj.bits,56),-3); + else + returns = bitshift(bitand(obj.bits,240),-4); + end + end + + function returns = get_classification_flags(obj) + if isempty(obj.bits) + obj.read_bits(); + end + if obj.header.point_data_format < 6 + returns = []; + else + returns = bitand(obj.bits2,15); + end + end + + function returns = get_scan_direction_flag(obj) + if isempty(obj.bits) + obj.read_bits(); + end + if obj.header.point_data_format < 6 + returns = bitand(obj.bits,64); + else + returns = bitand(obj.bits2,64); + end + end + + function returns = get_edge_of_flight_line(obj) + if isempty(obj.bits) + obj.read_bits(); + end + if obj.header.point_data_format < 6 + returns = bitand(obj.bits,128); + else + returns = bitand(obj.bits2,128); + end + end + + function returns = get_scanner_channel(obj) + if isempty(obj.bits) + obj.read_bits(); + end + if obj.header.point_data_format < 6 + returns = []; + else + returns = bitshift(bitand(obj.bits2,48),-4); + end + end + + function waves = getwaveforms(obj,points) + if isempty(obj.wave_packet_descriptor) + obj = read_point_wave_info(obj); + end + + bitstrs = cell(length(obj.wavedescriptors),1); + for k=1:length(obj.wavedescriptors) + bitstrs{k} = ['*ubit' num2str(obj.wavedescriptors(obj.wave_packet_descriptor(k)).bits)]; + end + + fid = fopen(obj.waveformfile,'r'); %multithreading fix + + DATASTART = obj.header.start_of_waveform_data; + waves = cell(length(points),1); + for k=1:length(points) + p = points(k); + if fseek(fid,double(DATASTART+obj.wave_byte_offset(p)),-1)<0 + error('fseek failed'); + end + if obj.wave_packet_descriptor(p) %if wavepacket id not zero then waveform exists + DATAPOINTS = floor(double(obj.wave_packet_size(p))*8 / ... + double(obj.wavedescriptors(obj.wave_packet_descriptor(p)).bits)); + waves{k} = fread(fid,DATAPOINTS,... + bitstrs{obj.wave_packet_descriptor(p)}); + else + waves{k} = []; + end + end + fclose(fid); + end + + function obj = setfilter(obj,varargin) + % setfilter adds given filter command to the filter stack. + % To reset filtering, reload the class from the original data. + % Specify filter with setfilter('filter',value,'filter',value...) + % or give filters sequentially obj.setfilter(...); obj.setfilter(...), + % however this method is slower as it causes reloading of the data. + + if nargin < 3 %+1 comes from the class obj + error('Incorrect parameter count.'); + end + + %clear filter + if isempty(obj.selection) + obj.selection = true(obj.header.number_of_point_records,1); + end + + k=1; + while k<=length(varargin) + switch( varargin{k} ) + case 'area' + area = varargin{k+1}; + if numel(area) ~=4 + error('Area definition requires rectangle [x,y,width,height].') + end + + %reload if data is not in the original condition + if isempty(obj.x) || ( length(obj.x) ~= length(obj.selection) ) + obj.read_xyz(1); + end + + obj.selection = obj.selection & ... + (obj.x >= area(1) & obj.x<= area(1)+area(3) & ... + obj.y >= area(2) & obj.y<= area(2)+area(4)); + k = k+2; + case 'classification' + val = varargin{k+1}; + if ~isnumeric(val) + error('Classification comparison requires a vector input of classification codes.') + end + %reload if data is not in the original condition + if isempty(obj.classification) || ( length(obj.classification) ~= length(obj.selection) ) + obj.read_classification(1); + end + + obj.selection = obj.selection & ( ismember(obj.classification, val) ); + k = k+2; + case 'user_data' + val = varargin{k+1}; + if numel(val) ~=1 || ~isnumeric(val) + error('User data comparison requires a single number.') + end + %reload if data is not in the original condition + if isempty(obj.user_data) || ( length(obj.user_data) ~= length(obj.selection) ) + obj.read_user_data(1); + end + + obj.selection = obj.selection & (obj.user_data == val); + k = k+2; + case 'scan_angle' + operator = varargin{k+1}; + if ~ischar(operator) && any(strncmp(operator,{'==','~=','<','>','<=','>='},2)) + error('Comparison operator must be one of [==,~=,<,>,<=,>=].') + end + + val = varargin{k+2}; + if numel(val) ~=1 || ~isnumeric(val) + error('Scan angle comparison requires a single number.') + end + %reload if data is not in the original condition + if isempty(obj.scan_angle) || ( length(obj.scan_angle) ~= length(obj.selection) ) + obj.read_scan_angle(1); + end + + eval(['obj.selection = obj.selection & (obj.scan_angle ' operator ' val);']); + k = k+3; + case 'custom' + fil = varargin{k+1}; + + if length(fil) == length(obj.x) + idx = find(obj.selection); + idx = fil(:).*idx; + fil = false(size(obj.selection)); + idx(idx==0) = []; + fil(idx) = true; + end + if numel(fil) ~=length(obj.selection) || ~islogical(fil) + error('Custom filter must be as long as is the length of the original or current data') + end + + obj.selection = obj.selection & fil; + k = k+2; + otherwise + error('Unknown filtering command'); + end + end + + %reload existing datas and set filter + if length(obj.x) ~= length(obj.selection) + obj.read_xyz(1); + end + obj.x = obj.x(obj.selection); + obj.y = obj.y(obj.selection); + obj.z = obj.z(obj.selection); + + if ~isempty(obj.intensity) && ( length(obj.intensity) ~= length(obj.selection) ) + obj.read_intensity(1); + end + if ~isempty(obj.intensity) + obj.intensity = obj.intensity(obj.selection); + end + + if ~isempty(obj.classification) && ( length(obj.classification) ~= length(obj.selection) ) + obj.read_classification(1); + end + if ~isempty(obj.classification) + obj.classification = obj.classification(obj.selection); + end + + if ~isempty(obj.bits) && ( length(obj.bits) ~= length(obj.selection) ) + obj.read_bits(1); + end + if ~isempty(obj.bits) + obj.bits = obj.bits(obj.selection); + if ~isempty(obj.bits2) + obj.bits2 = obj.bits2(obj.selection); + end + end + + if ~isempty(obj.user_data) && ( length(obj.user_data) ~= length(obj.selection) ) + obj.read_user_data(1); + end + if ~isempty(obj.user_data) + obj.user_data = obj.user_data(obj.selection); + end + + if ~isempty(obj.scan_angle) && ( length(obj.scan_angle) ~= length(obj.selection) ) + obj.read_scan_angle(1); + end + if ~isempty(obj.scan_angle) + obj.scan_angle = obj.scan_angle(obj.selection); + end + + if ~isempty(obj.point_source_id) && ( length(obj.point_source_id) ~= length(obj.selection) ) + obj.read_point_source_id(1); + end + if ~isempty(obj.point_source_id) + obj.point_source_id = obj.point_source_id(obj.selection); + end + + if ~isempty(obj.gps_time) && ( length(obj.gps_time) ~= length(obj.selection) ) + obj.read_gps_time(1); + end + if ~isempty(obj.gps_time) + obj.gps_time = obj.gps_time(obj.selection); + end + + if ~isempty(obj.red) && ( length(obj.red) ~= length(obj.selection) ) + obj.read_color(1); + end + if ~isempty(obj.red) + obj.red = obj.red(obj.selection); + obj.green = obj.green(obj.selection); + obj.blue = obj.blue(obj.selection); + if ~isempty(obj.nir) + obj.nir = obj.nir(obj.selection); + end + end + + if ~isempty(obj.extradata) && ( length(obj.extradata) ~= length(obj.selection) ) + obj.read_extradata(1); + end + if ~isempty(obj.extradata) + obj.extradata = obj.extradata(obj.selection,:); + end + + if ~isempty(obj.Xt) && ( length(obj.Xt) ~= length(obj.selection) ) + obj.read_point_wave_info(1); + end + if ~isempty(obj.Xt) + obj.Xt = obj.Xt(obj.selection); + obj.Yt = obj.Yt(obj.selection); + obj.Zt = obj.Zt(obj.selection); + obj.wave_return_point = obj.wave_return_point(obj.selection); + + obj.wave_packet_descriptor = obj.wave_packet_descriptor(obj.selection); + obj.wave_byte_offset = obj.wave_byte_offset(obj.selection); + obj.wave_packet_size = obj.wave_packet_size(obj.selection); + end + end + + function obj = read_xyz(obj,donotfilter) + fid = fopen(obj.filename); + fseek(fid,double(obj.header.offset_to_point_data),-1); + + LEN = obj.header.point_data_record_length; + POINTS = obj.header.number_of_point_records; + OFFSET = obj.header.offset_to_point_data; + fseek(fid,OFFSET,-1); + obj.x = fread(fid,POINTS,'*int32',LEN-4); %x + fseek(fid,OFFSET+4,-1); + obj.y = fread(fid,POINTS,'*int32',LEN-4); %y + fseek(fid,OFFSET+8,-1); + obj.z = fread(fid,POINTS,'*int32',LEN-4); %z + + fclose(fid); + + %apply filter + if ~exist('donotfilter','var') + obj.x = obj.x(obj.selection); + obj.y = obj.y(obj.selection); + obj.z = obj.z(obj.selection); + end + obj.normalize_xyz(); + end + + function obj = write_xyz(obj) + fid = fopen(obj.header.filename,'r+'); + fseek(fid,double(obj.header.offset_to_point_data),-1); + + LEN = obj.header.point_data_record_length; + OFFSET = 0; + + fseek(fid,double(obj.header.offset_to_point_data),-1); + obj.columndatafwrite(fid,obj.toint32_xyz(),OFFSET,LEN); + + fclose(fid); + end + + function obj = read_intensity(obj,donotfilter) + fid = fopen(obj.filename); + + LEN = obj.header.point_data_record_length; + POINTS = obj.header.number_of_point_records; + OFFSET = obj.header.offset_to_point_data + 12; + fseek(fid,OFFSET,-1); + obj.intensity = fread(fid,POINTS,'*uint16',LEN-2);%intensity + + fclose(fid); + + %apply filter + if ~exist('donotfilter','var') + obj.intensity = obj.intensity(obj.selection); + end + end + + function obj = write_intensity(obj) + fid = fopen(obj.header.filename,'r+'); + LEN = obj.header.point_data_record_length; + OFFSET = 12; + fseek(fid,double(obj.header.offset_to_point_data),-1); + obj.columndatafwrite(fid,obj.intensity,OFFSET,LEN); + fclose(fid); + end + + function obj = read_bits(obj,donotfilter) + fid = fopen(obj.filename); + + LEN = obj.header.point_data_record_length; + POINTS = obj.header.number_of_point_records; + OFFSET = obj.header.offset_to_point_data + 14; + + fseek(fid,OFFSET,-1); + obj.bits = fread(fid,POINTS,'*uint8',LEN-1); %bits + + if obj.header.version_minor > 3 && obj.header.point_data_format > 5 %1.4 && pointformat >=6 + fseek(fid,OFFSET+1,-1); + obj.bits2 = fread(fid,POINTS,'*uint8',LEN-1); %bits2 + end + + fclose(fid); + + %apply filter + if ~exist('donotfilter','var') + obj.bits = obj.bits(obj.selection); + if ~isempty(obj.bits2) + obj.bits2 = obj.bits2(obj.selection); + end + end + end + + function obj = write_bits(obj) + fid = fopen(obj.header.filename,'r+'); + LEN = obj.header.point_data_record_length; + OFFSET = 14; + + fseek(fid,double(obj.header.offset_to_point_data),-1); + obj.columndatafwrite(fid,obj.bits,OFFSET,LEN); + + if obj.header.version_minor > 3 && obj.header.point_data_format > 5 %1.4 & pointformat >=6 + fseek(fid,double(obj.header.offset_to_point_data),-1); + obj.columndatafwrite(fid,obj.bits2,OFFSET+1,LEN); + end + fclose(fid); + end + + function obj = read_classification(obj,donotfilter) + fid = fopen(obj.filename); + fseek(fid,double(obj.header.offset_to_point_data),-1); + + LEN = obj.header.point_data_record_length; + POINTS = obj.header.number_of_point_records; + + offsettable = [15 15 15 15 15 15 16 16 16 16 16]; + + OFFSET = offsettable(obj.header.point_data_format+1)+obj.header.offset_to_point_data; + + fseek(fid,OFFSET,-1); + obj.classification= fread(fid,POINTS,'*uint8',LEN-1); + + fclose(fid); + + %apply filter + if ~exist('donotfilter','var') + obj.classification = obj.classification(obj.selection); + end + end + + function obj = write_classification(obj) + fid = fopen(obj.header.filename,'r+'); + + LEN = obj.header.point_data_record_length; + offsettable = [15 15 15 15 15 15 16 16 16 16 16]; + OFFSET = offsettable(obj.header.point_data_format+1); + + fseek(fid,double(obj.header.offset_to_point_data),-1); + obj.columndatafwrite(fid,obj.classification,OFFSET,LEN); + fclose(fid); + end + + function obj = read_scan_angle(obj,donotfilter) + fid = fopen(obj.filename); + + LEN = obj.header.point_data_record_length; + POINTS = obj.header.number_of_point_records; + + offsettable = [16 16 16 16 16 16 18 18 18 18 18]; + datatypetable = {'*int8', '*int8', '*int8', '*int8', '*int8', ... + '*int8', '*int16', '*int16', '*int16', '*int16', '*int16'}; + datasizetable = [1 1 1 1 1 1 2 2 2 2 2]; + + OFFSET = offsettable(obj.header.point_data_format+1)+obj.header.offset_to_point_data; + DATATYPE = datatypetable{obj.header.point_data_format+1}; + DATASIZE = datasizetable(obj.header.point_data_format+1); + + fseek(fid,OFFSET,-1); + obj.scan_angle= fread(fid,POINTS,DATATYPE,LEN-DATASIZE); + + fclose(fid); + + %apply filter + if ~exist('donotfilter','var') + obj.scan_angle = obj.scan_angle(obj.selection); + end + end + + function obj = write_scan_angle(obj) + fid = fopen(obj.header.filename,'r+'); + LEN = obj.header.point_data_record_length; + + offsettable = [16 16 16 16 16 16 18 18 18 18 18]; + OFFSET = offsettable(obj.header.point_data_format+1); + datatypetable = {'int8', 'int8', 'int8', 'int8', 'int8', ... + 'int8', 'int16', 'int16', 'int16', 'int16', 'int16'}; + + DATATYPE = datatypetable{obj.header.point_data_format+1}; + + if ~isa(obj.scan_angle,DATATYPE) + error(['Scan angle datatype is not: ' DATATYPE]) + end + + fseek(fid,double(obj.header.offset_to_point_data),-1); + obj.columndatafwrite(fid,obj.scan_angle,OFFSET,LEN); + fclose(fid); + end + + function obj = read_user_data(obj,donotfilter) + fid = fopen(obj.filename); + + LEN = obj.header.point_data_record_length; + POINTS = obj.header.number_of_point_records; + + offsettable = [17 17 17 17 17 17 17 17 17 17 17]; + OFFSET = offsettable(obj.header.point_data_format+1); + DATATYPE ='*uint8'; + DATASIZE = 1; + + fseek(fid,OFFSET,-1); + obj.user_data= fread(fid,POINTS,DATATYPE,LEN-DATASIZE); + + fclose(fid); + + %apply filter + if ~exist('donotfilter','var') + obj.user_data = obj.user_data(obj.selection); + end + end + + function obj = write_user_data(obj) + fid = fopen(obj.header.filename,'r+'); + LEN = obj.header.point_data_record_length; + DATATYPE ='uint8'; + offsettable = [17 17 17 17 17 17 17 17 17 17 17]; + OFFSET = offsettable(obj.header.point_data_format+1); + + if ~isa(obj.user_data,DATATYPE) + error(['User data datatype is not: ' DATATYPE]) + end + + fseek(fid,double(obj.header.offset_to_point_data),-1); + + obj.columndatafwrite(fid,obj.user_data,OFFSET,LEN); + fclose(fid); + end + + function obj = read_point_source_id(obj,donotfilter) + fid = fopen(obj.filename); + + LEN = obj.header.point_data_record_length; + POINTS = obj.header.number_of_point_records; + + offsettable = [18 18 18 18 18 18 20 20 20 20 20]; + + OFFSET = offsettable(obj.header.point_data_format+1)+obj.header.offset_to_point_data; + DATATYPE = '*uint16'; + DATASIZE = 2; + + fseek(fid,OFFSET,-1); + obj.point_source_id= fread(fid,POINTS,DATATYPE,LEN-DATASIZE); + + fclose(fid); + + %apply filter + if ~exist('donotfilter','var') + obj.point_source_id = obj.point_source_id(obj.selection); + end + end + + function obj = write_point_source_id(obj) + fid = fopen(obj.header.filename,'r+'); + LEN = obj.header.point_data_record_length; + offsettable = [18 18 18 18 18 18 20 20 20 20 20]; + OFFSET = offsettable(obj.header.point_data_format+1); + DATATYPE = 'uint16'; + + if ~isa(obj.point_source_id,DATATYPE) + error(['Point source id datatype is not: ' DATATYPE]) + end + + fseek(fid,double(obj.header.offset_to_point_data),-1); + obj.columndatafwrite(fid,obj.point_source_id,OFFSET,LEN); + fclose(fid); + end + + function obj = read_gps_time(obj,donotfilter) + %check if not in this point format + if any(obj.header.point_data_format == [0 2]) + return; + end + + fid = fopen(obj.filename); + + LEN = obj.header.point_data_record_length; + POINTS = obj.header.number_of_point_records; + + offsettable = [20 20 20 20 20 20 22 22 22 22 22]; + + OFFSET = offsettable(obj.header.point_data_format+1)+obj.header.offset_to_point_data; + DATATYPE = '*double'; + DATASIZE = 8; + + fseek(fid,OFFSET,-1); + obj.gps_time= fread(fid,POINTS,DATATYPE,LEN-DATASIZE); + + fclose(fid); + + %apply filter + if ~exist('donotfilter','var') + obj.gps_time = obj.gps_time(obj.selection); + end + end + + + function obj = write_gps_time(obj) + fid = fopen(obj.header.filename,'r+'); + %check if not in this point format + if any(obj.header.point_data_format == [0 2]) + return; + end + + LEN = obj.header.point_data_record_length; + offsettable = [20 20 20 20 20 20 22 22 22 22 22]; + OFFSET = offsettable(obj.header.point_data_format+1); + DATATYPE = 'double'; + + if ~isa(obj.gps_time,DATATYPE) + error(['GPS time datatype is not: ' DATATYPE]) + end + + fseek(fid,double(obj.header.offset_to_point_data),-1); + obj.columndatafwrite(fid,obj.gps_time,OFFSET,LEN); + fclose(fid); + end + + function obj = read_color(obj,donotfilter) + %check if not in this point format + if any(obj.header.point_data_format == [0 1 2 4 6 9]) + return; + end + + fid = fopen(obj.filename); + + LEN = obj.header.point_data_record_length; + POINTS = obj.header.number_of_point_records; + + offsettable = [20 20 20 28 28 28 30 30 30 30 30]; + + OFFSET = offsettable(obj.header.point_data_format+1)+obj.header.offset_to_point_data; + DATATYPE = '*uint16'; + DATASIZE = 2; + + fseek(fid,OFFSET,-1); + obj.red= fread(fid,POINTS,DATATYPE,LEN-DATASIZE); + fseek(fid,OFFSET+2,-1); + obj.green= fread(fid,POINTS,DATATYPE,LEN-DATASIZE); + fseek(fid,OFFSET+4,-1); + obj.blue= fread(fid,POINTS,DATATYPE,LEN-DATASIZE); + if any(obj.header.point_data_format == [8 10]) + fseek(fid,OFFSET+6,-1); + obj.nir= fread(fid,POINTS,DATATYPE,LEN-DATASIZE); + end + + fclose(fid); + + %apply filter + if ~exist('donotfilter','var') + obj.red = obj.red(obj.selection); + obj.green = obj.green(obj.selection); + obj.blue = obj.blue(obj.selection); + if ~isempty(obj.nir) + obj.nir = obj.nir(obj.selection); + end + end + end + + function obj = write_color(obj) + fid = fopen(obj.header.filename,'r+'); + %check if not in this point format + if any(obj.header.point_data_format == [0 1 2 4 6 9]) + return; + end + + LEN = obj.header.point_data_record_length; + offsettable = [20 20 20 28 28 28 30 30 30 30 30]; + OFFSET = offsettable(obj.header.point_data_format+1); + DATATYPE = 'uint16'; + + if ~isa(obj.red,DATATYPE) + error(['Color datatype is not: ' DATATYPE]) + end + + fseek(fid,double(obj.header.offset_to_point_data),-1); + obj.columndatafwrite(fid,[obj.red obj.green obj.blue],OFFSET,LEN); + + if any(obj.header.point_data_format == [8 10]) + fseek(fid,double(obj.header.offset_to_point_data),-1); + obj.columndatafwrite(fid,obj.nir,OFFSET+6,LEN); + end + + fclose(fid); + end + + function obj = read_point_wave_info(obj,donotfilter) + %check if not in this point format + if any(obj.header.point_data_format == [0 1 2 3 6 7 8]) + return; + end + + fid = fopen(obj.filename); + + LEN = obj.header.point_data_record_length; + POINTS = obj.header.number_of_point_records; + + offsettable = [28 28 28 28 28 28 28 28 28 30 38]; + + OFFSET = offsettable(obj.header.point_data_format+1)+obj.header.offset_to_point_data; + DATATYPE = '*uint8'; + DATASIZE = 1; + + fseek(fid,OFFSET,-1); + obj.wave_packet_descriptor= fread(fid,POINTS,DATATYPE,LEN-DATASIZE); + + DATATYPE = '*uint64'; + DATASIZE = 8; + fseek(fid,OFFSET+1,-1); + obj.wave_byte_offset= fread(fid,POINTS,DATATYPE,LEN-DATASIZE); + + DATATYPE = '*uint32'; + DATASIZE = 4; + fseek(fid,OFFSET+9,-1); + obj.wave_packet_size= fread(fid,POINTS,DATATYPE,LEN-DATASIZE); + + DATATYPE = '*single'; + DATASIZE = 4; + fseek(fid,OFFSET+13,-1); + obj.wave_return_point= fread(fid,POINTS,DATATYPE,LEN-DATASIZE); + + DATATYPE = '*single'; + DATASIZE = 4; + fseek(fid,OFFSET+17,-1); + obj.Xt= fread(fid,POINTS,DATATYPE,LEN-DATASIZE); + + DATATYPE = '*single'; + DATASIZE = 4; + fseek(fid,OFFSET+21,-1); + obj.Yt= fread(fid,POINTS,DATATYPE,LEN-DATASIZE); + + DATATYPE = '*single'; + DATASIZE = 4; + fseek(fid,OFFSET+25,-1); + obj.Zt= fread(fid,POINTS,DATATYPE,LEN-DATASIZE); + + fclose(fid); + + %apply filter + if ~exist('donotfilter','var') + obj.wave_packet_descriptor = obj.wave_packet_descriptor(obj.selection); + obj.wave_byte_offset = obj.wave_byte_offset(obj.selection); + obj.wave_packet_size = obj.wave_packet_size(obj.selection); + obj.wave_return_point = obj.wave_return_point(obj.selection); + obj.Xt = obj.Xt(obj.selection); + obj.Yt = obj.Yt(obj.selection); + obj.Zt = obj.Zt(obj.selection); + end + end + + function obj = write_point_wave_info(obj) + fid = fopen(obj.header.filename,'r+'); + %check if not in this point format + if any(obj.header.point_data_format == [0 1 2 3 6 7 8]) + return; + end + + LEN = obj.header.point_data_record_length; + + offsettable = [28 28 28 28 28 28 28 28 28 30 38]; + + DATATYPE = 'uint8'; + if ~isa(obj.wave_packet_descriptor,DATATYPE) + error(['Wave packet descriptor datatype is not: ' DATATYPE]) + end + + fseek(fid,double(obj.header.offset_to_point_data),-1); + OFFSET = offsettable(obj.header.point_data_format+1); + obj.columndatafwrite(fid,obj.wave_packet_descriptor,OFFSET,LEN); + + DATATYPE = 'uint64'; + if ~isa(obj.wave_byte_offset,DATATYPE) + error(['Wave byte offset datatype is not: ' DATATYPE]) + end + + fseek(fid,double(obj.header.offset_to_point_data),-1); + OFFSET = offsettable(obj.header.point_data_format+1)+1; + obj.columndatafwrite(fid,obj.wave_byte_offset,OFFSET,LEN); + + DATATYPE = 'uint32'; + if ~isa(obj.wave_packet_size,DATATYPE) + error(['Wave packet size datatype is not: ' DATATYPE]) + end + fseek(fid,double(obj.header.offset_to_point_data),-1); + OFFSET = offsettable(obj.header.point_data_format+1)+9; + obj.columndatafwrite(fid,obj.wave_packet_size,OFFSET,LEN); + + DATATYPE = 'single'; + if ~isa(obj.wave_return_point,DATATYPE) + error(['Wave return point datatype is not: ' DATATYPE]) + end + fseek(fid,double(obj.header.offset_to_point_data),-1); + OFFSET = offsettable(obj.header.point_data_format+1)+13; + obj.columndatafwrite(fid,obj.wave_return_point,OFFSET,LEN); + + DATATYPE = 'single'; + if ~isa(obj.Xt,DATATYPE) + error(['Xt datatype is not: ' DATATYPE]) + end + fseek(fid,double(obj.header.offset_to_point_data),-1); + OFFSET = offsettable(obj.header.point_data_format+1)+17; + obj.columndatafwrite(fid,obj.Xt,OFFSET,LEN); + + DATATYPE = 'single'; + if ~isa(obj.Yt,DATATYPE) + error(['Yt datatype is not: ' DATATYPE]) + end + fseek(fid,double(obj.header.offset_to_point_data),-1); + OFFSET = offsettable(obj.header.point_data_format+1)+21; + obj.columndatafwrite(fid,obj.Yt,OFFSET,LEN); + + DATATYPE = 'single'; + if ~isa(obj.Zt,DATATYPE) + error(['Zt datatype is not: ' DATATYPE]) + end + fseek(fid,double(obj.header.offset_to_point_data),-1); + OFFSET = offsettable(obj.header.point_data_format+1)+25; + obj.columndatafwrite(fid,obj.Zt,OFFSET,LEN); + fclose(fid); + end + + function obj = read_extradata(obj) + LEN = obj.header.point_data_record_length; + POINTS = obj.header.number_of_point_records; + + offsettable = [20 28 26 34 57 63 30 36 38 59 67]; %magic numbers from point record byte lengths + OFFSET = offsettable(obj.header.point_data_format+1)+obj.header.offset_to_point_data; + + extralen = obj.header.point_data_record_length - offsettable(obj.header.point_data_format+1); + if extralen>0 %unknown extra data exists + fid = fopen(obj.filename); + + fseek(fid,OFFSET,-1); + obj.extradata = zeros(POINTS,extralen,'uint8'); + for k=1:POINTS + obj.extradata(k,:) = fread(fid,extralen,'*uint8'); + fseek(fid,LEN-extralen,0); + end + + fclose(fid); + end + + %apply filter + if ~exist('donotfilter','var') && ~isempty(obj.extradata) + obj.extradata = obj.extradata(obj.selection,:); + end + end + + + function obj = write_extradata(obj) + fid = fopen(obj.header.filename,'r+'); + LEN = obj.header.point_data_record_length; + offsettable = [20 28 26 34 57 63 30 36 38 59 67]; %magic numbers from point record byte lengths + OFFSET = offsettable(obj.header.point_data_format+1); + + extralen = size(obj.extradata,2); + if extralen %unknown extra data exists + if ~isa(obj.extradata,'uint8') + error(['Row extra data datatype is not: uint8']) + end + + fseek(fid,double(obj.header.offset_to_point_data),-1); + obj.columndatafwrite(fid,obj.extradata,OFFSET,LEN); + end + fclose(fid); + end + + function plot_xyz(obj,pointlimit) + if ~exist('pointlimit','var') + pointlimit = 10000; + end + if pointlimit == -1 + pointlimit = length(obj.x); + end + if isempty(obj.x) + warning('No points to plot, check filtering!') + return; + end + sel = randi(length(obj.x),pointlimit,1); + set(gcf,'renderer','opengl') + obj.get_intensity(); + scatter3(obj.x(sel),obj.y(sel),obj.z(sel),100,obj.intensity(sel),'r.') + end + + function plot_intensity(obj,pointlimit) + if ~exist('pointlimit','var') + pointlimit = 10000; + end + if pointlimit == -1 + pointlimit = length(obj.x); + end + if isempty(obj.x) + warning('No points to plot, check filtering!') + return; + end + + sel = randi(length(obj.x),pointlimit,1); + set(gcf,'renderer','opengl') + obj.get_intensity(); + scatter3(obj.x(sel),obj.y(sel),obj.z(sel),100,obj.intensity(sel),'.') + end + + function plot_classification(obj,pointlimit,no_z) + if ~exist('pointlimit','var') + pointlimit = 10000; + end + if pointlimit == -1 + pointlimit = length(obj.x); + end + if isempty(obj.x) + warning('No points to plot, check filtering!') + return; + end + + sel = randi(length(obj.x),pointlimit,1); + set(gcf,'renderer','opengl') + obj.get_classification(); + if ~exist('no_z','var') + scatter(obj.x(sel),obj.y(sel),100,obj.classification(sel),'.') + else + scatter3(obj.x(sel),obj.y(sel),obj.z(sel),100,obj.classification(sel),'.') + end + end + + function plot_waveforms(obj,pointlimit,alphaon) + if ~exist('pointlimit','var') + pointlimit = 1000; + end + if pointlimit == -1 + pointlimit = length(obj.x); + end + if isempty(obj.x) + warning('No points to plot, check filtering!') + return; + end + + + %select last returns + subset = find(obj.get_return_number() == obj.get_number_of_returns()); + %and random amount of these points or all + subset = subset(randi(size(subset,1),pointlimit,1)); + xyz = obj.get_xyz(); + + %remove points with no waves + waves = obj.getwaveforms(subset); + xyz = xyz(subset,:); + + amplitudemax = double(max(cellfun(@max,waves))); + firstpeaks = zeros(length(waves),1); + for k=1:length(waves) + [~,firstpeaks(k)] = findpeaks(double(waves{k}),'npeaks',1,'minpeakheight',mean(waves{k})); + end + + %temporal spacing from descriptor (*1000000 scaling) + step = double([obj.wavedescriptors(obj.wave_packet_descriptor(subset)).temporal_sample_spacing])*1000000; + samples = double([obj.wavedescriptors(obj.wave_packet_descriptor(subset)).number_of_samples]); + %calculate height of the waveform + waveheight = (298925574./step(:)).*samples(:)/2; + wavegroundoffset = (298925574./step(:)).*firstpeaks(:)/2; + + + set(gcf,'renderer','opengl') + hold on + colors = jet(amplitudemax+1); + + %generate ground + tri = delaunay(xyz(:,1),xyz(:,2)); + trisurf(tri,xyz(:,1),xyz(:,2),xyz(:,3),'edgecolor','none') + + faces = []; + vertices = []; + col = []; + valpha = []; + len = 0; + for k=1:length(subset) + tmpx = linspace(xyz(k,1),xyz(k,1),samples(k))'+double(waves{k})/amplitudemax/2; + tmpy = linspace(xyz(k,2),xyz(k,2),samples(k))'; + tmpz = linspace(xyz(k,3)-wavegroundoffset(k),... + xyz(k,3)+waveheight(k)-wavegroundoffset(k),samples(k))'; + tmpc = colors(double(waves{k})+1,:); + + p = [tmpx tmpy tmpz]; + + fc = [(1:size(p,1)-1)' (1:size(p,1)-1)' (2:size(p,1))']+ len; + + len = len + size(p,1); + + faces = [faces; fc]; + vertices = [vertices; p]; + col = [col; tmpc]; + valpha = [valpha; double(waves{k})/amplitudemax]; + end + + if exist('alphaon','var') + valpha = log(valpha); + valpha = valpha -(min(valpha)); + valpha = valpha / max(valpha); + patch( 'Vertices', vertices, 'Faces', faces,'FaceColor','none',... + 'EdgeColor',[0 0 0],... + 'FaceVertexAlphaData', valpha,... + 'EdgeAlpha','interp','AlphaDataMapping','none'); + else + patch( 'Vertices', vertices, 'Faces', faces,'FaceColor','none',... + 'FaceVertexCData',col,'EdgeColor','interp'); + end + end + + + function plot_waveform_layers(obj) + + if isempty(obj.x) + warning('No points to plot, check filtering!') + return; + end + + %select last returns + subset = find(obj.get_return_number() == obj.get_number_of_returns()); + xyz = obj.get_xyz(); + xyz = xyz(subset,:); + + xlimorg = [min(xyz(:,1)) max(xyz(:,1))]; + ylimorg = [min(xyz(:,2)) max(xyz(:,2))]; + ZADD = 30; + zlimorg = [min(xyz(:,3)) max(xyz(:,3))+ZADD]; + + xyz(:,1) = xyz(:,1)-min(xyz(:,1)); + xyz(:,2) = xyz(:,2)-min(xyz(:,2)); + xyz(:,3) = xyz(:,3)-min(xyz(:,3)); + + waves = obj.getwaveforms(subset); + removeempty = cellfun(@isempty,waves); + subset = subset(~removeempty); + xyz = xyz(~removeempty,:); + waves = waves(~removeempty); + + %create voxel structure + NX = 256; NY = 256; NZ = 256; + layers = zeros(NX,NY,NZ,'uint8'); + xlim = [min(xyz(:,1)) max(xyz(:,1))]; + ylim = [min(xyz(:,2)) max(xyz(:,2))]; + zlim = [min(xyz(:,3)) max(xyz(:,3))+ZADD]; + + amplitudemax = double(max(cellfun(@max,waves))); + firstpeaks = zeros(1,length(waves)); + for k=1:length(waves) + [~,firstpeaks(k)] = findpeaks(double(waves{k}),'npeaks',1,'minpeakheight',mean(waves{k})); + end + + %temporal spacing from descriptor (*1000000 scaling) + step = double([obj.wavedescriptors(obj.wave_packet_descriptor(subset)).temporal_sample_spacing])*1000000; + samples = double([obj.wavedescriptors(obj.wave_packet_descriptor(subset)).number_of_samples]); + %calculate height of the waveform + waveheight = (298925574./step).*samples/2; + wavegroundoffset = (298925574./step).*firstpeaks/2; + + set(gcf,'renderer','opengl') + hold on + + for k=1:length(subset) + tmpx = linspace(xyz(k,1),xyz(k,1),samples(k))'; + tmpy = linspace(xyz(k,2),xyz(k,2),samples(k))'; + tmpz = linspace(xyz(k,3)-wavegroundoffset(k),... + xyz(k,3)+waveheight(k)-wavegroundoffset(k),samples(k))'; + amp = (double(waves{k})/amplitudemax)*255; + + px = floor((tmpx)/(xlim(2))*(NX-1))+1; + py = floor((tmpy)/(ylim(2))*(NY-1))+1; + pz = floor((tmpz)/(zlim(2))*(NZ-1))+1; + + for r=1:length(px) + if pz(r)<=NZ && pz(r)>0 && layers(px(r),py(r),pz(r)) < amp(r) + layers(px(r),py(r),pz(r)) = amp(r); + end + end + end + + alim([0 255]); + for k = 1:NZ + zk = zlimorg(1)+(zlimorg(2)-zlimorg(1))/NZ*(k-1); + surf([xlimorg(1) xlimorg(2); xlimorg(1) xlimorg(2)],... + [ylimorg(1) ylimorg(1); ylimorg(2) ylimorg(2)],... + [zk zk; zk zk],'CData',layers(:,:,k),'FaceColor','texturemap',... + 'EdgeColor','none','FaceAlpha', 'texturemap', ... + 'AlphaDataMapping', 'scaled', 'AlphaData',layers(:,:,k)); + end + end + + function obj = write_las(obj, filename, majorversion, minorversion, pointformat) + + %prevent overwriting + %if you like to overwrite files, then read all variables to memory + %at this point and disable check + [pathtmp,filetmp,ext]=fileparts(obj.filename); + if isempty(pathtmp); pathtmp = pwd; end + orgfile = [pathtmp '/' filetmp ext]; + [pathtmp,filetmp]=fileparts(filename); + if isempty(pathtmp); pathtmp = pwd; end + newfile = [pathtmp '/' filetmp ext]; + + if strcmpi(orgfile,newfile) + error('Overwriting is not allowed.') + end + + newheader = obj.header; + oldheader = obj.header; + if ~exist('filename','var') + error('Please input target filename.') + end + if exist('majorvarsion','var') + newheader.version_major = majorversion; + end + if exist('minorversion','var') + newheader.version_minor = minorversion; + end + if exist('pointformat','var') + newheader.point_data_format = pointformat; + end + newheader.number_of_point_records = length(obj.x); + newheader.max_x = max(obj.x); + newheader.min_x = min(obj.x); + newheader.max_y = max(obj.y); + newheader.min_y = min(obj.y); + newheader.max_z = max(obj.z); + newheader.min_z = min(obj.z); + newheader.filename = filename; + + fid = fopen(filename,'w'); + try + obj.header = newheader; + obj.writeheader(fid); + catch err + obj.header = oldheader; + error(['Error writing las header: ' err.getReport]); + end + obj.header = oldheader; + + LEN = length(obj.x); + if isempty(obj.intensity) + obj.read_intensity(); + if isempty(obj.intensity) + warning('Adding zeros to intensity') + obj.intensity = zeros(LEN,1,'uint16'); + end + end + if isempty(obj.bits) + obj.read_bits(); + if isempty(obj.bits) + warning('Adding zeros to bit values (return nr, scan dir. flag, edge of flight line)') + obj.bits = zeros(LEN,1,'uint8'); + end + end + + if any(newheader.point_data_format == [6 7 8 9 10]) + if isempty(obj.bits2) + obj.bits2 = zeros(LEN,1,'uint8'); + end + %convert to new point formats + if oldheader.point_data_format < 6 + obj.bits = bitor(obj.get_return_number, bitshift(obj.get_number_of_returns,4)); + obj.bits2 = bitor(bitshift(obj.get_scan_direction_flag,6), bitshift(obj.get_edge_of_flight_line,7)); + end + end + if any(oldheader.point_data_format > 5) && newheader.point_data_format < 6 + %convert to old point formats + if oldheader.point_data_format > 5 + obj.bits = bitor(bitand(obj.get_return_number,7), bitand(bitshift(obj.get_number_of_returns,3),7)); + obj.bits = bitor(obj.bits, bitor(bitshift(obj.get_scan_direction_flag,6), bitshift(obj.get_edge_of_flight_line,7))); + end + end + + if isempty(obj.classification) + obj.read_classification(); + if isempty(obj.classification) + warning('Adding zeros to classification') + obj.classification = zeros(LEN,1,'uint8'); + end + end + + if isempty(obj.scan_angle) + obj.read_scan_angle(); + if isempty(obj.scan_angle) + warning('Adding zeros to scan angle') + obj.scan_angle = zeros(LEN,1,'uint8'); + end + end + + if isempty(obj.user_data) + obj.read_user_data(); + if isempty(obj.user_data) + warning('Adding zeros to user data') + obj.user_data = zeros(LEN,1,'uint8'); + end + end + + if isempty(obj.point_source_id) + obj.read_point_source_id(); + if isempty(obj.point_source_id) + warning('Adding zeros to point source id') + obj.point_source_id = zeros(LEN,1,'uint16'); + end + end + + if ~any(newheader.point_data_format == [0 2]) + if isempty(obj.gps_time) + obj.read_gps_time(); + if isempty(obj.gps_time) + warning('Adding zeros to gps time') + obj.gps_time = zeros(LEN,1,'double'); + end + end + end + + if any(newheader.point_data_format == [3 5 7 8 10]) + if isempty(obj.red) + obj.read_color(); + if isempty(obj.red) + warning('Adding zeros to RGB color') + obj.red = zeros(LEN,1,'uint16'); + obj.green = zeros(LEN,1,'uint16'); + obj.blue = zeros(LEN,1,'uint16'); + end + if any(newheader.point_data_format == [8 10]) + if isempty(obj.nir) + warning('Adding zeros to nir color') + obj.nir = zeros(LEN,1,'uint16'); + end + end + end + end + + if any(newheader.point_data_format == [4 5 9 10]) + if isempty(obj.wave_return_point) + obj.read_point_wave_info(); + if isempty(obj.wave_return_point) + warning('Adding zeros to wave packet info') + obj.wave_packet_descriptor = zeros(LEN,1,'uint8'); + obj.wave_byte_offset = zeros(LEN,1,'uint64'); + obj.wave_packet_size = zeros(LEN,1,'uint32'); + obj.wave_return_point = zeros(LEN,1,'single'); + obj.Xt = zeros(LEN,1,'single'); + obj.Yt = zeros(LEN,1,'single'); + obj.Zt = zeros(LEN,1,'single'); + end + end + end + + obj.header = newheader; + + %%% variable length records + try + obj.write_variable_records(fid); + catch err + error(['Error writing variable length records: ' err.getReport]); + end + + if obj.header.version_major==1 && obj.header.version_minor == 0 + tmp = char([hex2dec('DD') hex2dec('CC')]); %write las 1.0 variable record start + fwrite(fid,tmp,'uint8'); + end + + %find offset to point data and write it to header in file + tmppos = ftell(fid); + obj.header.offset_to_point_data = tmppos; + fseek(fid,96,-1); + fwrite(fid,uint32(tmppos),'uint32'); + fseek(fid,tmppos,-1); + + %calculate point record length and write it to header in file + record_lengths = [20 28 26 34 57 63 30 36 38 59 67 ]; + obj.header.point_data_record_length = ... + record_lengths(obj.header.point_data_format+1) + size(obj.extradata,2); + fseek(fid,105,-1); + fwrite(fid,obj.header.point_data_record_length,'uint16'); + + %update extendedvariable offset + if newheader.version_minor > 3 %1.4 + obj.header.start_of_extended_variable_length_record = ... + obj.header.offset_to_point_data + length(obj.x)*obj.header.point_data_record_length; + fseek(fid,235,-1); + fwrite(fid,obj.header.start_of_extended_variable_length_record,'uint64'); + end + + fclose(fid); + + try + obj.write_xyz(); + obj.write_intensity(); + obj.write_bits(); + obj.write_classification(); + obj.write_scan_angle(); + obj.write_user_data(); + obj.write_point_source_id(); + obj.write_gps_time(); + obj.write_color(); + obj.write_point_wave_info(); + obj.write_extradata(); + catch err + error(['Error writing point data: ' err.getReport]); + end + + fid = fopen(filename,'r+'); + fseek(fid,double(obj.header.offset_to_point_data+length(obj.x)*obj.header.point_data_record_length),-1); + + try + obj.write_extended_variables(fid); + catch err + error(['Error writing extended variable data: ' err.getReport]); + end + + fclose(fid); + obj.header = oldheader; + + end + + + function add_waveform_packet_desc(obj,bits,compression,numberofsamples,samplespacing,gain,offset) + obj.header.number_of_variable_records = obj.header.number_of_variable_records + 1; + idx = obj.header.number_of_variable_records; + obj.variablerecords(idx).reserved = uint16(0); + tmp = 'LASF_Spec'; + obj.variablerecords(idx).user_id = [tmp zeros(1,16-length(tmp),'uint8')]; + %calculate new descriptor number + if ~isempty(obj.wavedescriptors) + tmp = [obj.variablerecords.record_id]; + tmp(tmp<100 | tmp >355) = []; %remove non wavedescriptors + val = max(tmp); + obj.variablerecords(idx).record_id = uint16(val+1); + else + obj.variablerecords(idx).record_id = uint16(100); %first id + end + obj.variablerecords(idx).record_length = uint16(26); + tmp = 'Waveform Packet Descriptor'; + obj.variablerecords(idx).description = [tmp zeros(1,32-length(tmp),'uint8')]; + + data(1) = uint8(bits); + data(2) = uint8(compression); + data(3:6) = typecast(uint32(numberofsamples),'uint8'); + data(7:10) = typecast(uint32(samplespacing),'uint8'); + data(11:18) = typecast(double(gain),'uint8'); + data(19:26) = typecast(double(offset),'uint8'); + + obj.variablerecords(idx).data = data; + obj.wavedescriptors = obj.decode_waveform_packet_desc(obj.variablerecords); + end + + function add_waveforms_to_external_file(obj,pointids,descriptorids,waves,xts,yts,zts,returnpoints) + + if size(waves,2)==1 + error('Input wavedatas as rows.') + end + + if obj.waveformfilefid < 3 + [pathtmp,filetmp]=fileparts(obj.filename); + if isempty(pathtmp) + pathtmp = pwd; + end + obj.waveformfile = [pathtmp '\\' filetmp '.wdp']; + obj.waveformfilefid = fopen(obj.waveformfile,'a+'); + end + fseek(obj.waveformfilefid,0,'eof'); + + %set external waveform file to global encoding + obj.header.global_encoding = bitor(bitand(uint16(obj.header.global_encoding),uint16(65533)) , bitshift(uint16(1),2)); + + %load existing data if not loaded, or data is empty + if isempty(obj.Xt) || isempty(obj.Yt) || isempty(obj.Zt) || ... + isempty(obj.wave_return_point) || isempty(obj.wave_packet_descriptor)... + || isempty(obj.wave_byte_offset) || isempty(obj.wave_packet_size) + obj.read_point_wave_info(); + end + if size(obj.Xt) ~= size(obj.x); obj.Xt = zeros(size(obj.x),'single'); end + if size(obj.Yt) ~= size(obj.x); obj.Yt = zeros(size(obj.x),'single'); end + if size(obj.Zt) ~= size(obj.x); obj.Zt = zeros(size(obj.x),'single'); end + if size(obj.wave_return_point) ~= size(obj.x); obj.wave_return_point = zeros(size(obj.x),'single'); end + if size(obj.wave_packet_descriptor) ~= size(obj.x); obj.wave_packet_descriptor = zeros(size(obj.x),'uint8'); end + if size(obj.wave_byte_offset) ~= size(obj.x); obj.wave_byte_offset = zeros(size(obj.x),'uint64'); end + if size(obj.wave_packet_size) ~= size(obj.x); obj.wave_packet_size = zeros(size(obj.x),'uint32'); end + + obj.Xt(pointids) = xts; + obj.Yt(pointids) = yts; + obj.Zt(pointids) = zts; + obj.wave_return_point(pointids) = returnpoints; + obj.wave_packet_descriptor(pointids) = descriptorids; + + %multiple points with same wave + if size(waves,1)~= length(pointids) + pos = ftell(obj.waveformfilefid); + len = size(waves,2); + obj.wave_byte_offset(pointids) = pos; + obj.wave_packet_size(pointids) = len; + fwrite(obj.waveformfilefid,waves,'uint8'); + else + %insert multiple points with different waves + for k=1:length(pointids) + pos = ftell(obj.waveformfilefid); + len = size(waves,2); + obj.wave_byte_offset(pointids(k)) = pos; + obj.wave_packet_size(pointids(k)) = len; + fwrite(obj.waveformfilefid,waves,'uint8'); + end + end + end + +end + +methods (Access=private) + + function obj = las_read( obj, file ) + if ~exist(file,'file') + error('File not found.') + end + fid = fopen(file,'r'); + + obj = readheader(obj,fid); + + obj.isLAZ = 0; + + %check for LAZ compressed file + if any(obj.header.point_data_format >=127) %127 laz shift on pointformat? + obj.isLAZ = 1; + fclose(fid); + if ~exist('laszip.exe','file') + error('Cannot decompress LAZ file without laszip.exe') + end + obj.filename = [file '_tmp.las']; + system(['laszip -i ' file ' -o ' obj.filename]); + + fid = fopen(obj.filename,'r'); + + obj = readheader(obj,fid); + end + + %set filter off + obj.selection = true(obj.header.number_of_point_records,1); + + obj = read_variable_records(obj,fid); + + if obj.header.version_major == 1 && obj.header.version_minor == 0 + check = fread(fid,2,'uint8'); + if check(1) ~= hex2dec('DD') && check(2) ~= hex2dec('CC') + warning('File position do not match offset to the point data, continuing reading using header position.') + fseek(fid,double(obj.header.offset_to_point_data),-1); + end + end + if ftell(fid) ~= obj.header.offset_to_point_data + [ftell(fid) obj.header.offset_to_point_data] + warning('File position do not match offset to the point data, continuing reading using header position.') + fseek(fid,double(obj.header.offset_to_point_data),-1); + end + + obj.read_xyz(); + + obj.extendedvariables = []; + if isfield(obj.header,'number_of_extended_variable_length_record') ... + && obj.header.number_of_extended_variable_length_record > 0 ... + && ~feof(fid) + fseek(fid,obj.header.start_of_extended_variable_length_record,-1); + read_extended_variables(obj,fid); + end + + fclose(fid); + + if any(obj.header.point_data_format == [4 5]) + obj.wavedescriptors = obj.decode_waveform_packet_desc(obj.variablerecords); + obj.setwaveformsource(); + end + end + + function obj = createemptyheader(obj) + obj.header.source_id = []; + obj.header.global_encoding = []; + obj.header.project_id_guid1 = []; + obj.header.project_id_guid2 = []; + obj.header.project_id_guid3 = []; + obj.header.project_id_guid4 = []; + obj.header.version_major = []; + obj.header.version_minor = []; + obj.header.system_identifier = []; + obj.header.generating_software = []; + obj.header.file_creation_daobj.y = []; + obj.header.file_creation_year = []; + obj.header.header_size = []; + obj.header.offset_to_point_data = []; + obj.header.number_of_variable_records = []; + obj.header.point_data_format = []; + obj.header.point_data_record_length = []; + obj.header.number_of_point_records = []; + obj.header.number_of_points_by_return = []; + obj.header.scale_factor_x = []; + obj.header.scale_factor_y = []; + obj.header.scale_factor_z = []; + obj.header.x_offset = []; + obj.header.y_offset = []; + obj.header.z_offset = []; + obj.header.max_x = []; + obj.header.min_x = []; + obj.header.max_y = []; + obj.header.min_y = []; + obj.header.max_z = []; + obj.header.min_z = []; + obj.header.start_of_extended_variable_length_record = []; + obj.header.number_of_extended_variable_length_record = []; + obj.header.start_of_waveform_data = []; + end + + function obj = readheader(obj,fid) + str = fscanf(fid,'%c',4); + if strcmp('LASF',str)==0 + fclose(fid); + error([file ' is not a LAS file.']) + end + + try + obj.header.source_id = fread(fid,1,'uint16'); + obj.header.global_encoding = fread(fid,1,'uint16'); + obj.header.project_id_guid1 = fread(fid,1,'uint32'); + obj.header.project_id_guid2 = fread(fid,1,'uint16'); + obj.header.project_id_guid3 = fread(fid,1,'uint16'); + obj.header.project_id_guid4 = fread(fid,8,'int8'); + obj.header.version_major = fread(fid,1,'uint8'); + obj.header.version_minor = fread(fid,1,'uint8'); + obj.header.system_identifier = fscanf(fid,'%c',32); + obj.header.generating_software = fscanf(fid,'%c',32); + obj.header.file_creation_daobj.y = fread(fid,1,'uint16'); + obj.header.file_creation_year = fread(fid,1,'uint16'); + obj.header.header_size = fread(fid,1,'uint16'); + obj.header.offset_to_point_data = fread(fid,1,'uint32'); + obj.header.number_of_variable_records = fread(fid,1,'uint32'); + obj.header.point_data_format = fread(fid,1,'uint8'); + obj.header.point_data_record_length = fread(fid,1,'uint16'); + obj.header.number_of_point_records = fread(fid,1,'uint32'); + obj.header.number_of_points_by_return = fread(fid,5,'uint32'); + obj.header.scale_factor_x = fread(fid,1,'double'); + obj.header.scale_factor_y = fread(fid,1,'double'); + obj.header.scale_factor_z = fread(fid,1,'double'); + obj.header.x_offset = fread(fid,1,'double'); + obj.header.y_offset = fread(fid,1,'double'); + obj.header.z_offset = fread(fid,1,'double'); + obj.header.max_x = fread(fid,1,'double'); + obj.header.min_x = fread(fid,1,'double'); + obj.header.max_y = fread(fid,1,'double'); + obj.header.min_y = fread(fid,1,'double'); + obj.header.max_z = fread(fid,1,'double'); + obj.header.min_z = fread(fid,1,'double'); + if obj.header.version_minor > 2 %1.3 + obj.header.start_of_waveform_data = fread(fid,1,'uint64'); + end + % add one EVLR for 1.3, if point format 4 or 5 and internal waveforms + if obj.header.version_major == 1 && obj.header.version_minor == 3 ... + && bitand(obj.header.global_encoding,2) && ~bitand(obj.header.global_encoding,4) + obj.header.number_of_extended_variable_length_record = 1; + end + if obj.header.version_minor > 3 %1.4 + obj.header.start_of_extended_variable_length_record = fread(fid,1,'uint64'); + obj.header.number_of_extended_variable_length_record = fread(fid,1,'uint32'); + %copy legacy values to show + obj.header.legacy_number_of_point_records_READ_ONLY = obj.header.number_of_point_records; + obj.header.number_of_point_records = fread(fid,1,'uint64'); + obj.header.legacy_number_of_points_by_return_READ_ONLY = obj.header.number_of_points_by_return; + obj.header.number_of_points_by_return = fread(fid,15,'uint64'); + end + catch ex + fclose(fid); + disp('Error while processing header information.') + throw(ex) + end + + if obj.header.version_major > 1 || obj.header.version_minor > 4 + warning('Trying to parse unsupported LAS version.'); + end + end + + function obj = writeheader(obj,fid) + fseek(fid,0,-1); + fprintf(fid,'LASF'); + + fwrite(fid, obj.header.source_id,'uint16'); + fwrite(fid, obj.header.global_encoding,'uint16'); + fwrite(fid, obj.header.project_id_guid1,'uint32'); + fwrite(fid, obj.header.project_id_guid2,'uint16'); + fwrite(fid, obj.header.project_id_guid3,'uint16'); + fwrite(fid, obj.header.project_id_guid4,'int8'); + fwrite(fid, obj.header.version_major,'uint8'); + fwrite(fid, obj.header.version_minor,'uint8'); + tmp = obj.header.system_identifier; + tmp = [tmp zeros(1,32-length(tmp),'uint8')]; + fprintf(fid, '%c',tmp); + tmp = obj.header.generating_software; + tmp = [tmp zeros(1,32-length(tmp),'uint8')]; + fprintf(fid, '%c', tmp); + fwrite(fid, obj.header.file_creation_daobj.y,'uint16'); + fwrite(fid, obj.header.file_creation_year,'uint16'); + fwrite(fid, obj.header.header_size,'uint16'); + fwrite(fid, obj.header.offset_to_point_data,'uint32'); + fwrite(fid, obj.header.number_of_variable_records,'uint32'); + fwrite(fid, obj.header.point_data_format,'uint8'); + fwrite(fid, obj.header.point_data_record_length,'uint16'); + if obj.header.number_of_point_records < 2^32 %if legacy compatible + fwrite(fid, obj.header.number_of_point_records,'uint32'); + else + fwrite(fid, 0,'uint32'); + end + + %add lecagy only if possible by pointcount limited by uint32 + if obj.header.number_of_point_records < 2^32 && ... + (length(obj.header.number_of_points_by_return) == 15 && ... + all(obj.header.number_of_points_by_return(6:15)==0)) + tmpp = obj.header.number_of_points_by_return; + if length(obj.header.number_of_points_by_return)==15 + tmpp = tmpp(1:5); + end + + fwrite(fid, tmpp,'uint32'); + else + fwrite(fid, zeros(5,1,'uint32'),'uint32'); + end + fwrite(fid, obj.header.scale_factor_x,'double'); + fwrite(fid, obj.header.scale_factor_y,'double'); + fwrite(fid, obj.header.scale_factor_z,'double'); + fwrite(fid, obj.header.x_offset,'double'); + fwrite(fid, obj.header.y_offset,'double'); + fwrite(fid, obj.header.z_offset,'double'); + fwrite(fid, obj.header.max_x,'double'); + fwrite(fid, obj.header.min_x,'double'); + fwrite(fid, obj.header.max_y,'double'); + fwrite(fid, obj.header.min_y,'double'); + fwrite(fid, obj.header.max_z,'double'); + fwrite(fid, obj.header.min_z,'double'); + if obj.header.version_minor > 2 %1.3 + if ~isfield(obj.header,'start_of_waveform_data') + fwrite(fid, 0,'uint64'); + else + fwrite(fid, obj.header.start_of_waveform_data,'uint64'); + end + end + if obj.header.version_minor > 3 %1.4 + if ~isfield(obj.header,'start_of_extended_variable_length_record') + fwrite(fid, 0,'uint64'); + else + fwrite(fid, obj.header.start_of_extended_variable_length_record,'uint64'); + end + + if ~isfield(obj.header,'number_of_extended_variable_length_record') + fwrite(fid, 0,'uint32'); + else + fwrite(fid, obj.header.number_of_extended_variable_length_record,'uint32'); + end + + if ~isfield(obj.header,'number_of_point_records') + fwrite(fid, 0,'uint64'); + else + fwrite(fid, obj.header.number_of_point_records,'uint64'); + end + + if ~isfield(obj.header,'number_of_points_by_return') + fwrite(fid, 15,'uint64'); + else + fwrite(fid, obj.header.number_of_points_by_return,'uint64'); + end + end + %write header length + pos = ftell(fid); + fseek(fid,94,-1); + fwrite(fid,pos,'uint16'); + fseek(fid,pos,-1); + end + + function obj = read_variable_records(obj,fid) + for k=1:obj.header.number_of_variable_records + obj.variablerecords(k).reserved = fread(fid,1,'*uint16'); + obj.variablerecords(k).user_id = fscanf(fid,'%c',16); + obj.variablerecords(k).record_id = fread(fid,1,'*uint16'); + obj.variablerecords(k).record_length = fread(fid,1,'*uint16'); + obj.variablerecords(k).description = fscanf(fid,'%c',32); + obj.variablerecords(k).data = fread(fid,obj.variablerecords(k).record_length,'*uint8'); + obj.variablerecords(k).data_as_text = char(obj.variablerecords(k).data(:))'; + end + end + + function obj = write_variable_records(obj,fid) + for k=1:obj.header.number_of_variable_records + if obj.header.version_major==1 && obj.header.version_minor == 0 + tmp = char([hex2dec('BB') hex2dec('AA')]); %write las 1.0 variable record start + fwrite(fid,tmp,'uint8'); + end + + fwrite(fid,obj.variablerecords(k).reserved,'uint16'); + tmp = obj.variablerecords(k).user_id; + tmp = [tmp zeros(1,16-length(tmp))]; + fprintf(fid,'%c',tmp); + fwrite(fid,obj.variablerecords(k).record_id,'uint16'); + fwrite(fid,length(obj.variablerecords(k).data),'uint16'); + tmp = obj.variablerecords(k).description; + tmp = [tmp zeros(1,32-length(tmp))]; + fprintf(fid,'%c',tmp); + fwrite(fid,obj.variablerecords(k).data,'uint8'); + end + end + + function obj = read_extended_variables(obj,fid) + for k=1:obj.header.number_of_extended_variable_length_record + obj.extendedvariables(k).reserved = fread(fid,1,'*uint16'); + obj.extendedvariables(k).user_id = fread(fid,16,'*int8'); + obj.extendedvariables(k).record_id = fread(fid,1,'*uint16'); + obj.extendedvariables(k).record_length_after_obj_header = fread(fid,1,'*uint64'); + obj.extendedvariables(k).description = fread(fid,32,'*int8'); + if isfield(obj.header,'start_of_waveform_data') && ftell(fid) == obj.header.start_of_waveform_data + warning('Skipping waveform data while reading extended variables. Access waveforms with waveform reading function.'); + obj.extendedvariables(k).data = []; + else + obj.extendedvariables(k).data = fread(fid,obj.extendedvariables.record_length_after_obj_header,'*uint8'); + end + end + end + + function obj = write_extended_variables(obj,fid) + if ~isfield(obj.header,'number_of_extended_variable_length_record') + obj.header.number_of_extended_variable_length_record = 0; + end + + for k=1:obj.header.number_of_extended_variable_length_record + fwrite(fid,obj.extendedvariables(k).reserved,'uint16'); + fwrite(fid,obj.extendedvariables(k).user_id,'int8'); + fwrite(fid,obj.extendedvariables(k).record_id,'uint16'); + fwrite(fid,length(obj.extendedvariables(k).data),'uint64'); + fwrite(fid,obj.extendedvariables(k).description,'int8'); + fwrite(fid,obj.extendedvariables(k).data,'uint8'); + end + end + + function obj = setwaveformsource(obj) + %check if external waveform file + if bitand(obj.header.global_encoding,4) + [pathtmp,filetmp]=fileparts(obj.originalname); + if isempty(pathtmp) + pathtmp = pwd; + end + obj.waveformfile = [pathtmp '\\' filetmp '.wdp']; + if ~exist(obj.waveformfile,'file') + warning(['External waveform file ' obj.waveformfile ' not found!']) + return; + end + else + obj.waveformfile = obj.filename; + end + end + +end + +methods(Static,Access=public) + + %line to triangle converter for plotting + function arr = FlattenTriangleArrayComponent(arr) + arr = padarray(arr',2,'post'); + d = arr(:); + arr = d; + arr = arr + [0; d(1:end-1)]; + arr = arr(1:end-3) + [0; 0; d(4:end-2)]; + end + + function [codes] = list_classification_codes(format,noprint) + + if format==1.0 || format==1.1 || format==1.2 || format==1.3 + + codes={ 'Created, never classified',... + 'Unclassified','Ground', 'Low Vegetation',... + 'Medium vegetation', 'High vegetation', 'Building',... + 'Low point (noise)','Model key-point (mass point)',... + 'Water','Reserved', 'Reserved', 'Overlap Points'}; + + if ~exist('noprint','var') + fprintf('\nPoint Record Types 0-5\n'); + for k=1:length(codes) + fprintf('%d %s\n',k-1,codes{k}) + end + fprintf('13-31 Reserved\n\n') + end + for k=13:31; codes{k} = 'Reserved'; end + else + codes = {'Created, never classified',... + 'Unclassified','Ground', 'Low Vegetation',... + 'Medium vegetation', 'High vegetation', 'Building',... + 'Low point (noise)','Reserved','Water',... + 'Rail', 'Road surface', 'Reserved',... + 'Wire - guard (Shield)','Wire - conductor (Phase)',... + 'Transmission tower','Wire-structure connector (insulator)',... + 'Bridge deck','High noise'}; + + if ~exist('noprint','var') + fprintf('\nPoint record types 6-10\n'); + for k=1:length(codes) + fprintf('%d %s\n',k-1,codes{k}) + end + fprintf('19-63 Reserved\n') + fprintf('64-255 User definable\n\n') + end + for k=13:31; codes{k} = 'Reserved'; end + for k=64:255; codes{k} = 'User definable'; end + end + end + + %helper class not operating on class data + function desc = decode_waveform_packet_desc(data) + r=1; + for k=1:length(data) + if data(k).record_id >=100 && data(k).record_id < 356 %waveform record types + desc(r).bits = typecast(data(k).data(1),'uint8'); + desc(r).compression = typecast(data(k).data(2),'uint8'); + desc(r).number_of_samples = typecast(data(k).data(3:6),'uint32'); + desc(r).temporal_sample_spacing = typecast(data(k).data(7:10),'uint32'); + desc(r).digitizer_gain = typecast(data(k).data(11:18),'double'); + desc(r).digitizer_offset = typecast(data(k).data(19:26),'double'); + r=r+1; + end + end + if ~exist('desc','var') + warning('Expecting waveform descriptors, but found none.') + desc = []; + end + end + + function columndatafwrite(fid,data,columnpos,rowlength) + %fwrite with block read/write to have faster column writes + BLOCKROWS = 20000; + + insertdatalen = length(typecast(data(1,:),'uint8')); + for k=1:BLOCKROWS:size(data,1) + pos = ftell(fid); + bend = k+BLOCKROWS-1; + if bend > size(data,1) + bend = size(data,1); + end + + %find file size + fseek(fid,0,1); + filelen = ftell(fid); + fseek(fid,pos,-1); + %create empty space, because reading will fail otherwise + %next column write will be faster + if filelen < ftell(fid)+(bend-k+1) + need_to_allocate = (bend-k+1)*rowlength - (filelen-pos); + fwrite(fid,zeros(need_to_allocate,1,'uint8')); + end + fseek(fid,pos,-1); + + %read block to memory + block = fread(fid,(bend-k+1)*rowlength,'*uint8'); + block = reshape(block,rowlength,[])'; + + %add data in memory + tmp = data(k:bend,:)'; + tmp = typecast(tmp(:),'uint8'); + tmp = reshape(tmp,insertdatalen,[])'; + block(:,columnpos+1:columnpos+insertdatalen) = tmp; + block = block'; + %write block back to file + fseek(fid,pos,-1); + fwrite(fid,block(:)); + end + end + +end +end + diff --git a/cresis-toolbox/lidar/lasdata_license.txt b/cresis-toolbox/lidar/lasdata_license.txt new file mode 100644 index 00000000..9d429cfc --- /dev/null +++ b/cresis-toolbox/lidar/lasdata_license.txt @@ -0,0 +1,27 @@ +Copyright (c) 2016, Teemu Kumpumäki +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution + * Neither the name of the Tampere University of Technology nor the names + of its contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/cresis-toolbox/lidar/read_lidar_atm.m b/cresis-toolbox/lidar/read_lidar_atm.m index fa97a24e..a1a46bbf 100644 --- a/cresis-toolbox/lidar/read_lidar_atm.m +++ b/cresis-toolbox/lidar/read_lidar_atm.m @@ -252,8 +252,15 @@ elseif strcmpi(file_type,'csv') %% Use the "CSV" Read Method % ===================================================================== + % Skip header lines first, and then read data fid = fopen(atm_fn); - A = textscan(fid,'%f%f%f%f%f%f%f%f%f%f%f', 'Delimiter',',', 'HeaderLines', 10); + while ~feof(fid) + hdline = fgetl(fid); + if strfind(hdline,'UTC_Seconds_Of_Day') + break + end + end + A = textscan(fid,'%f%f%f%f%f%f%f%f%f%f%f', 'Delimiter',','); fclose(fid); % Some files are empty so skip these diff --git a/cresis-toolbox/lidar/read_lidar_dtu.m b/cresis-toolbox/lidar/read_lidar_dtu.m index a2c3af16..422686ba 100644 --- a/cresis-toolbox/lidar/read_lidar_dtu.m +++ b/cresis-toolbox/lidar/read_lidar_dtu.m @@ -7,7 +7,7 @@ % % lidar = struct of position and LIDAR data, each N x 1 vectors % where N is the number of records in the file(s). The fields are: -% .gps_time = GPS time in dec.hour(UTC) +% .gps_time = GPS time in decimal hours (UTC) % .lat = latitude (deg) % .lon = longitude (deg) % .surface = WGS-84 surface elevation (m) diff --git a/cresis-toolbox/lidar/read_lidar_las.m b/cresis-toolbox/lidar/read_lidar_las.m new file mode 100644 index 00000000..82c71145 --- /dev/null +++ b/cresis-toolbox/lidar/read_lidar_las.m @@ -0,0 +1,83 @@ +function [lidar] = read_lidar_las(fns, param) +% [lidar] = read_lidar_las(fns, param) +% +% Reads LIDAR data in the LAS format. UAF single otter rds and snow radar +% data and BAS Twin Otter accum and rds data may come in this format. +% +% This process should work with any LAZ/LAS LIDAR files as long as the x, +% y, and z fields correspond to longitude, latitude, and WGS-84 elevation. +% The filenames and file locations need to be modified to match the +% conventions required to work with read_lidar_las and get_filenames_lidar. +% +% ------------------------------------------------------------------------- +% IMPORTANT NOTE: +% The x,y,z fields in the LAS file must be lat, lon, WGS-84 elevation +% ------------------------------------------------------------------------- +% +% 1. Obtain LAZ (compressed LAS) files. If LAS files already skip the first +% two steps. Technically lasdata.m can read LAZ files too if laszip +% executable is in the system path, but this has not been tested. +% 2. Use laszip to decompress the laz files into las files. LAZ files use a +% lidar-specific compression scheme. +% Open Source Solution for decompression: https://laszip.org/ +% They have: +% 1. Linux C++ source files that can be compiled into executable +% 2. Windows based GUI binary: laszip.exe +% 3. Windows based CLI binary: laszip-cli.exe +% 3. Rename the LAS files to follow this convention: +% YYYYMMDD_N.las such as 20191215_1.las, 20191224_1.las, 20191224_2.las +% +% YYYYMMDD is the date of the data collection and needs to match the +% date segment convention in the radar parameter segments. +% +% N is used to distinguish files when more than one file was collected +% on a particular day. Numbering should be done in chronological order. +% 4. Place LAS files in the directory: +% metadata/LIDAR_LAS/SEASON_NAME/ +% +% param: struct that controls reading of file(s) +% .season_name: String containing the season name such as +% '2019_Antarctica_TObas', should match parameter spreadsheet season_name +% field. This field is not required by read_lidar_las, but is required by +% get_filenames_lidar. +% +% .date_string: String containing the date in this format: 'YYYYMMDD'. +% This field is not required by read_lidar_las, but is required by +% get_filenames_lidar. +% To convert gps time in seconds since Jan 1, 1970: +% param.date_str = datestr(epoch_to_datenum(gps_time),'yyyymmdd')) +% To convert day_seg field: +% param.date_str = param.day_seg(1:8); +% +% lidar: struct of position and LIDAR data, each 1xNx vectors +% where 1xNx is the number of records in the file(s). The fields are: +% .gps_time: filled with NaN since this field is not available +% .lat: latitude (deg) +% .lon: longitude (deg) +% .surface: WGS-84 surface elevation (m) +% +% Example: +% +% See also: read_lidar_atm, read_lidar_awi, read_lidar_dtu, read_lidar_las, +% get_filenames_lidar + +%% Read Lidar Data from the LAS data files +lidar = []; +lidar.lat = []; +lidar.lon = []; +lidar.surface = []; +for fn_idx = 1:length(fns) + fn = fns{fn_idx}; + + % Create lasdata class with contents from file "fn" + lidar_tmp = lasdata(fn); + + % Concatenate contents of this file to the lidar output structure fields: + lidar.lat(1,end+(1:length(lidar_tmp.x))) = lidar_tmp.x; + lidar.lon(1,end+(1:length(lidar_tmp.x))) = lidar_tmp.y; + lidar.surface(1,end+(1:length(lidar_tmp.x))) = lidar_tmp.z; + + % Delete handle class or it will stay in memory + delete(lidar_tmp); +end +lidar.gps_time = nan(size(lidar.lon)); diff --git a/cresis-toolbox/matlab_speed_test.m b/cresis-toolbox/matlab_speed_test.m new file mode 100644 index 00000000..6f60a885 --- /dev/null +++ b/cresis-toolbox/matlab_speed_test.m @@ -0,0 +1,93 @@ +% script matlab_speed_test.m + +%% CPU Test +if 1 + %% CPU: Date and Time, Version, OS + fprintf('date_time\t%s\n', datestr(now,'yyyymmdd_HHMMSS')); + A = ver('Matlab'); + fprintf('version\t%s\n', A.Release); + if ~isempty(which('detect_os')) + [OS, OSVersion] = detect_os; + fprintf('OS\t%s\t%s\n', OS, mat2str_generic(OSVersion)); + else + if ispc + fprintf('OS\t%s\t%s\n', 'Windows', ''); + elseif isunix + fprintf('OS\t%s\t%s\n', 'Linux', ''); + elseif ismac + fprintf('OS\t%s\t%s\n', 'Mac', ''); + end + end + + %% CPU: FFT speed test + start_time = tic; + fprintf('fft_start\t%g\n', toc(start_time)); + A = randn(10e3,5e3) + 1i*randn(10e3,5e3); + fprintf('fft_data_creation\t%g\n', toc(start_time)); + for run = 1:500 + B = fft(A); + end + fprintf('fft_done\t%g\n', toc(start_time)); + + %% CPU: Matrix Inversion speed test + start_time = tic; + fprintf('mat_inv_start\t%g\n', toc(start_time)); + A = randn(15,15,1e5) + 1i*randn(15,15,1e5); + fprintf('mat_inv_creation\t%g\n', toc(start_time)); + B = zeros(size(A)); + for run = 1:40 + for rline = 1:size(A,3) + B(:,:,rline) = inv(A(:,:,rline)); + end + end + fprintf('mat_inv_done\t%g\n', toc(start_time)); + + %% CPU: Component wise matrix multiplies + start_time = tic; + fprintf('mat_mult_start\t%g\n', toc(start_time)); + A = randn(10e3,5e3) + 1i*randn(10e3,5e3); + B = randn(10e3,5e3) + 1i*randn(10e3,5e3); + fprintf('mat_mult_creation\t%g\n', toc(start_time)); + for run = 1:300 + C = A .* B; + end + fprintf('mat_mult_done\t%g\n', toc(start_time)); +end + +%% GPU Section (verify parallel toolbox gpuDeviceCount is in the path) +if 1 && ~isempty(which('gpuDeviceCount')) + % gpuDeviceTable + for gpu_idx = 1:gpuDeviceCount + gpu_dev = gpuDevice(gpu_idx); + + %% GPU: Determine size of matrix to operate on based on GPU memory + % 8 bytes per sample, 3 copies in memory, 10e3 rows, 80% utilization + cols = min(10e3,floor(gpu_dev.TotalMemory / 10e3 / 8 / 3 * 0.8)); + if cols == 10e3 + fprintf('GPU\t%s\n', gpu_dev.Name); + else + fprintf('GPU\t%s: only using %d instead of 10000 columns due to VRAM limitation\n', gpu_dev.Name, cols); + end + + %% GPU: FFT speed test + start_time = tic; + fprintf('fft_start\t%g\n', toc(start_time)); + A = randn(10e3,10e3,'single') + 1i*randn(10e3,10e3,'single'); + % gpurng(0, 'Philox'); + % B = randn(10e3,5e3,'single','gpuArray') + 1i*randn(10e3,5e3,'single','gpuArray'); + B = gpuArray(A); + % underlyingType(B) + fprintf('fft_data_creation\t%g\n', toc(start_time)); + for run = 1:500 + C = fft(B); + end + + %% GPU: Copy result to CPU RAM + % gather ensures all GPU operations are completed and then copies the + % results back into CPU RAM + D = gather(C); + fprintf('fft_done\t%g\n', toc(start_time)); + E = fft(A); + fprintf('fft_error\t%g\n', sum(abs(E(:)-D(:)))); + end +end diff --git a/cresis-toolbox/mcords5/basic_load_mcords5.m b/cresis-toolbox/mcords5/basic_load_mcords5.m index f82e1477..162142cc 100644 --- a/cresis-toolbox/mcords5/basic_load_mcords5.m +++ b/cresis-toolbox/mcords5/basic_load_mcords5.m @@ -10,24 +10,38 @@ % NOTE: 64-bit computer may be essential to load a 256 MB file since it will % consume 512 MB of memory after loading. % -% fn = filename of MCoRDS-5 data -% param = struct controlling loading of data -% .clk = clock (Hz), default 1600e6, used to interpret -% counts in the header fields. -% .recs = 2 element vector for records to load [start_rec num_rec] -% start_rec uses zero-indexing, default is [0 inf] -% .debug_level = 1 is default, 2 generates plots/print-outs -% .first_byte = first byte to start reading at (default is zero0) -% .start_index_time_offset = time offset between transmit waveform start -% and the start index. The default is -1.0665e-05 implying that the first -% sample collected is 10.665 us before the transmit event starts. This -% value can be very different for each radar system (e.g. AWI system -% is 0.6 us). -% .presum_bug_fixed: default is false. Set to true if presum number in -% file matches actual presums. +% Inputs: +% ========================================================================= % -% hdr = file header for each record -% data = cell vector of single matrices of radar data where each entry +% fn: filename of MCoRDS-5 data +% +% param: struct controlling loading of data +% +% .clk = clock (Hz), default 200e6 (1600 MHz sampling frequency divided by +% 8), used to interpret counts in the header fields. +% +% .recs = 2 element vector for records to load [start_rec num_rec] +% start_rec uses zero-indexing, default is [0 inf] +% +% .first_byte = first byte to start reading at (default is zero0) +% +% .start_index_time_offset = time offset between transmit waveform start +% and the start index. The default is -1.0665e-05 implying that the first +% sample collected is 10.665 us before the transmit event starts. This +% value can be very different for each radar system (e.g. AWI system is +% 0.6 us). +% +% .presum_mode: default is 1 which means there is an extra unused waveform +% and the presum header values are 1 higher than what they should be +% (8-channel DDS Ledford/Leuschen has this bug). Set to 0 if the presum +% header values are correct (i.e. header values match the actual number of +% presums. +% +% Outputs: +% ========================================================================= +% +% hdr: file header for each record +% data: cell vector of single matrices of radar data where each entry % in the cell vector is a 3-D array for that waveform. Dimensions % 1: fast-time/range-bin % 2: slow-time/range-line/records @@ -45,33 +59,26 @@ if ~exist('param','var') param = struct(); end -if ~isfield(param,'clk'); - param.clk = 1600e6; +if ~isfield(param,'clk') + param.clk = 200e6; end -if ~isfield(param,'recs'); - param.recs = [0 inf]; +if param.clk == 1600e6 + error('param.clk is now defined to be the actual header counter clock which is 200e6 instead of 1600e6.'); end -if ~isfield(param,'debug_level'); - param.debug_level = 1; +if ~isfield(param,'recs') || isempty(param.recs) + param.recs = [0 inf]; end -if ~isfield(param,'first_byte'); +if ~isfield(param,'first_byte') || isempty(param.first_byte) param.first_byte = 0; end -if ~isfield(param,'start_index_time_offset') +if ~isfield(param,'start_index_time_offset') || isempty(param.start_index_time_offset) param.start_index_time_offset = -1.0665e-05; end -if ~isfield(param,'header_only') - if nargout <= 1 - param.header_only = 1; - else - param.header_only = 0; - end -end -if ~isfield(param,'record_mode') - param.record_mode = 1; +if ~isfield(param,'presum_mode') || isempty(param.presum_mode) + param.presum_mode = 1; end -if ~isfield(param,'presum_bug_fixed') - param.presum_bug_fixed = 0; +if isfield(param,'presum_bug_fixed') + error('presum_bug_fixed no longer used. Change it to presum_mode. Change param.presum_bug_fixed = 1 to param.presum_mode = 0 and presum_bug_fixed = 0 to presum_mode = 1.'); end % Reset/clear hdr struct @@ -146,14 +153,19 @@ for wf = 1:num_waveforms % Read in waveform header fseek(fid,2,0); - if param.presum_bug_fixed - hdr.wfs(wf).presums = fread(fid, 1, 'uint8') + 1; - else - hdr.wfs(wf).presums = fread(fid, 1, 'uint8'); + % hdr.wfs(wf).presums: field in file contains the number of hardware + % presums/stacking/averages minus one + hdr.wfs(wf).presums = fread(fid, 1, 'uint8') + 1; + if param.presum_mode == 1 + % For 8-channel Ledford/Leuschen DDS waveform generator, an extra bad + % waveform is transmitted which is not used in the presum. The header + % shows the number of transmitted waveforms including the bad waveform + % so we need to subtract one from the presums. + hdr.wfs(wf).presums = hdr.wfs(wf).presums - 1; end hdr.wfs(wf).bit_shifts = -fread(fid, 1, 'int8'); hdr.wfs(wf).start_idx = fread(fid, 1, 'uint16'); - hdr.wfs(wf).t0 = hdr.wfs(wf).start_idx / param.clk*8 + param.start_index_time_offset; + hdr.wfs(wf).t0 = hdr.wfs(wf).start_idx / param.clk + param.start_index_time_offset; if DDC == 1 hdr.wfs(wf).num_sam = 8*fread(fid, 1, 'uint16'); @@ -200,7 +212,7 @@ hdr.file_size = ftell(fid); if hdr.rec_size ~= median(diff(hdr.sync_offsets))/2 - error('Estimated header size is wrong'); + error('Estimated header size (%d) does not appear to match the header size found in the file (%d)', hdr.rec_size, median(diff(hdr.sync_offsets))/2); % keyboard; % For badly recorded files (e.g. DDS settings not matching ADC settings) % you can try uncommenting the following line: @@ -230,7 +242,7 @@ fseek(fid,8,-1); end hdr.counter = fread(fid,1,'uint64'); - hdr.utc_time_sod = hdr.seconds + hdr.fractions / param.clk*8; + hdr.utc_time_sod = hdr.seconds + hdr.fractions / param.clk; hdr.comp_time_sod = double(fread(fid,1,'uint64')); fclose(fid); @@ -321,7 +333,7 @@ + 2^16*hdr_data(15+3*HACK_OFFSET,:) ... + hdr_data(16+3*HACK_OFFSET,:); -hdr.utc_time_sod = hdr.seconds + hdr.fractions/param.clk*8; +hdr.utc_time_sod = hdr.seconds + hdr.fractions/param.clk; hdr.comp_time_sod= (2^48*hdr_data(17+4*HACK_OFFSET,:) ... + 2^32*hdr_data(18+4*HACK_OFFSET,:) ... diff --git a/cresis-toolbox/mcords5/mcords5_fill_missing_headers.m b/cresis-toolbox/mcords5/mcords5_fill_missing_headers.m index d9f59e60..5993e0b2 100644 --- a/cresis-toolbox/mcords5/mcords5_fill_missing_headers.m +++ b/cresis-toolbox/mcords5/mcords5_fill_missing_headers.m @@ -96,7 +96,6 @@ reuse_tmp_files = true; % Set to false if you want to overwrite current results file_prefix_override = ''; % most of the time counter_correction_en = true; - presum_bug_fixed = true; % Seasons from 2015 Greenland Polar6 onward should be set to true union_time_epri_gaps = true; % Parameters below this point OFTEN NEEDS TO BE CHANGED diff --git a/cresis-toolbox/missions/create_configs_2019_Antarctica_Ground_rds.m b/cresis-toolbox/missions/create_configs_2019_Antarctica_Ground_rds.m index be204428..45088fc0 100644 --- a/cresis-toolbox/missions/create_configs_2019_Antarctica_Ground_rds.m +++ b/cresis-toolbox/missions/create_configs_2019_Antarctica_Ground_rds.m @@ -137,3 +137,73 @@ write_radar_config(config); config = old_config; end + +%% Ping Pong Mode +% <4500 m thick ice +ice_thickness = [4500]; +for freq_idx = [1] + param = default_radar_params_2019_Antarctica_Ground_rds; + config = param.config; + + config.flight_hours = 10; + config.prf = prf; + + config.zeropimods = [0 180]; % 180 causes a special mode in some ADC/DAC + + config.tg.staged_recording = {[1 2 3 3]}; % HERE + config.tg.altitude_guard = 100*12*2.54/100; + config.tg.Haltitude = 100*12*2.54/100; + config.tg.Hice_thick = ice_thickness(freq_idx); + + config.create_IQ = false; + + config.f0 = f0_list(freq_idx); + config.f1 = f1_list(freq_idx); + config.DDC_freq = 160e6; + config.zeropimods = [0 180]; + + config.wfs(1).presums = 128; % HERE + config.wfs(2).presums = 128; % HERE + config.wfs(3).presums = 256; + config.wfs(4).presums = 256; % HERE + config.tukey = 0.08; + config.wfs(1).Tpd = 0.5e-6; + config.wfs(2).Tpd = 3e-6; + config.wfs(3).Tpd = 10e-6; + config.wfs(4).Tpd = 10e-6; % HERE + config.wfs(1).tukey = 0.1; + config.wfs(2).tukey = 0.1; + config.wfs(3).tukey = 0.1; + config.wfs(4).tukey = 0.1; % HERE + + config.wfs(1).name = '0p5us_high'; + config.wfs(2).name = '3us_high'; + config.wfs(3).name = '10us_lefttx'; % HERE + config.wfs(4).name = '10us_righttx'; % HERE + + config.wfs(1).tx_weights = final_DDS_amp{cal_settings_list(freq_idx)}; % HERE + config.wfs(1).tx_enable = [1 1 1 1]; % HERE + config.wfs(2).tx_weights = final_DDS_amp{cal_settings_list(freq_idx)}; % HERE + config.wfs(2).tx_enable = [1 1 1 1]; % HERE + config.wfs(3).tx_weights = final_DDS_amp{cal_settings_list(freq_idx)}; % HERE + config.wfs(3).tx_weights([2 4]) = 0; % HERE + config.wfs(3).tx_enable = [1 0 1 0]; % HERE + config.wfs(4).tx_weights = final_DDS_amp{cal_settings_list(freq_idx)}; % HERE + config.wfs(4).tx_weights([1 3]) = 0; % HERE + config.wfs(4).tx_enable = [0 1 0 1]; % HERE + + config.wfs(1).phase = final_DDS_phase{cal_settings_list(freq_idx)}; + config.wfs(2).phase = final_DDS_phase{cal_settings_list(freq_idx)}; + config.wfs(3).phase = final_DDS_phase{cal_settings_list(freq_idx)}; + config.wfs(4).phase = final_DDS_phase{cal_settings_list(freq_idx)}; + + config.wfs(1).delay = final_DDS_time{cal_settings_list(freq_idx)}; + config.wfs(2).delay = final_DDS_time{cal_settings_list(freq_idx)}; + config.wfs(3).delay = final_DDS_time{cal_settings_list(freq_idx)}; + config.wfs(4).delay = final_DDS_time{cal_settings_list(freq_idx)}; + + config.arena.psc_name = sprintf('pingpong_%.0f-%.0fMHz_%.0fft_%.0fus_%.0fmthick', ... + config.f0/1e6,config.f1/1e6,config.tg.Haltitude*100/12/2.54,config.wfs(end).Tpd*1e6,config.tg.Hice_thick); % HERE + config.arena.fn = fullfile(arena_base_dir,sprintf('%s.xml',config.arena.psc_name)); + write_radar_config(config); +end diff --git a/cresis-toolbox/missions/create_configs_2021_Greenland_Polar5.m b/cresis-toolbox/missions/create_configs_2021_Greenland_Polar5.m new file mode 100644 index 00000000..9b78da42 --- /dev/null +++ b/cresis-toolbox/missions/create_configs_2021_Greenland_Polar5.m @@ -0,0 +1,719 @@ +% script create_configs_2021_Greenland_Polar6 +% +% Creates NI radar depth sounder settings +% +% See link_budgets.xls + +physical_constants; % c = speed of light + +% Define waveforms +if ispc + base_dir = 'C:\waveforms\'; + rss_base_dir = 'C:\Users\Administrator\Desktop\Arena_Shared\configs\'; +else + base_dir = '~/waveforms/'; + rss_base_dir = '~/rss_waveforms/'; +end + +f0_list = [150e6 180e6 320e6]; +f1_list = [520e6 210e6 350e6]; +DDC_select_list = [1 2 1]; % Which DDC mode to use +cal_settings = [1 2 1]; +prf = 10000; + +% presums: Ensure lambda/4 sampling (fudge factor allows difference) and an +% that presums are an even number. +velocity = [100 100 100]; % m/s +presums = round(c./abs(f1_list+f0_list)/2 ./ velocity * prf / 2)*2 + +final_DDS_phase = []; +final_DDS_phase_no_time = []; +final_DDS_amp = []; +final_DDS_time = []; +if 0 + % Initial conditions (usually all zeros phase/time with max amplitude) + for idx = 1:length(f0_list) + final_DDS_phase{idx} = [0 0 0 0 0 0 0 0]; + final_DDS_phase_no_time = [0 0 0 0 0 0 0 0]; % not used usually + final_DDS_amp{idx} = [4000 4000 4000 4000 4000 4000 4000 4000]; + final_DDS_time{idx} = [0 0 2.5 2.5 3.125 3.125 0 0]; + end +else + % COPY AND PASTE RESULTS FROM basic_tx_chan_equalization_SEASON_NAME.m + % HERE: + + % 150-520 MHz + final_DDS_phase{end+1} = [127.0 151.7 -4.3 0.0 -13.2 21.7 161.8 149.6]; + final_DDS_phase_no_time{end+1} = [0 0 0 0 0 0 0 0]; % not used usually + final_DDS_amp{end+1} = [1312 2849 2657 3572 4000 2618 2574 1386]; + final_DDS_time{end+1} = [0.9209 1.0190 2.5389 2.5000 2.3667 2.6164 0.7276 0.4423]; + + % 180-210 MHz + final_DDS_phase{end+1} = [11.3 18.9 7.1 0.0 -161.3 151.7 2.4 34.9]; + final_DDS_phase_no_time{end+1} = [0 0 0 0 0 0 0 0]; % not used usually + final_DDS_amp{end+1} = [1031 2982 2741 4000 3496 3103 2035 1151]; + final_DDS_time{end+1} = [-0.5180 -0.3989 3.1638 2.5000 0.2090 4.4217 -0.8502 -0.9472]; + + % 320-350 MHz + final_DDS_phase{end+1} = final_DDS_phase{1}; + final_DDS_phase_no_time{end+1} = final_DDS_phase_no_time{1}; + final_DDS_amp{end+1} = final_DDS_amp{1}; + final_DDS_time{end+1} = final_DDS_time{1}; +end + +% Hwindow_orig: Desired window created during transmit calibration +% This is used any time a window that is different from that used +% during calibration is to be used. +Hwindow_orig = chebwin(8,30).'; + +%% SETUP +% ========================================================================= + +param = []; +param.season_name = '2021_Greenland_Polar6'; +param.radar_name = 'rds'; +param.gps_source = 'awi-final'; +clear phase_centers; +for tx_chan = 1:8 + % Just enable the current antenna to determine its phase center + tx_weights = zeros(1,8); + tx_weights(tx_chan) = 1; + rxchan = 4; % Fix the receiver (it should not matter which one you choose) + % Determine phase center for the antenna + phase_centers(:,tx_chan) = lever_arm(param, tx_weights, rxchan); +end +% Adjust phase centers to the mean phase center position +phase_centers = bsxfun(@minus,phase_centers,mean(phase_centers,2)); + +% Create waveforms directories if they do not exist +if ~exist(base_dir,'dir') + mkdir(base_dir); +end +calval_dir = fullfile(base_dir, 'calval'); +if ~exist(calval_dir,'dir') + mkdir(calval_dir); +end + +%% AWI MCoRDS Arena Parameters +[~,defaults] = default_radar_params_2021_Greenland_Polar6_mcords; +arena = defaults{1}.arena; + +%% Survey Mode + loopback, noise, and deconv modes +% <4000 m thick ice, 1200 +/- 700 ft AGL +ice_thickness = [4000 4000]; +for freq_idx = [1 2] + param = struct('radar_name','mcords5','num_chan',8,'aux_dac',[255 255 255 255 255 255 255 255],'version','14.0f1','TTL_prog_delay',650,'xml_version',2.0,'fs',1600e6,'fs_sync',90.0e6,'fs_dds',1440e6,'TTL_clock',1440e6/16,'TTL_mode',[2.5e-6 260e-9 -1100e-9],'rss_base_dir',rss_base_dir); + param.max_tx = [4000 4000 4000 4000 4000 4000 4000 4000]; param.max_data_rate = 755; param.flight_hours = 3.5; param.sys_delay = 0.75e-6; param.use_mcords4_names = true; param.arena = arena; + param.DDC_select = DDC_select_list(freq_idx); + param.max_duty_cycle = 0.12; + param.create_IQ = false; + param.tg.staged_recording = [1 2 3]; + param.tg.altitude_guard = 700*12*2.54/100; + param.tg.Haltitude = 1200*12*2.54/100; + param.tg.Hice_thick = ice_thickness(freq_idx); + param.prf = prf; + param.presums = [2 4 presums(freq_idx)-6]; + param.wfs(1).atten = 37; + param.wfs(2).atten = 0; + param.wfs(3).atten = 0; + DDS_amp = final_DDS_amp{cal_settings(freq_idx)}; + param.tx_weights = DDS_amp; + param.tukey = 0.08; + param.wfs(1).Tpd = 1e-6; + param.wfs(2).Tpd = 3e-6; + param.wfs(3).Tpd = 10e-6; + param.wfs(1).phase = final_DDS_phase{cal_settings(freq_idx)}; + param.wfs(2).phase = final_DDS_phase{cal_settings(freq_idx)}; + param.wfs(3).phase = final_DDS_phase{cal_settings(freq_idx)}; + param.delay = final_DDS_time{cal_settings(freq_idx)}; + param.f0 = f0_list(freq_idx); + param.f1 = f1_list(freq_idx); + param.DDC_freq = (param.f0+param.f1)/2; + [param.wfs(1:3).tx_mask] = deal([0 0 0 0 0 0 0 0]); + param.fn = fullfile(base_dir,sprintf('survey_%.0f-%.0fMHz_%.0fft_%.0fus_%.0fmthick.xml',param.f0/1e6,param.f1/1e6,param.tg.Haltitude*100/12/2.54,param.wfs(end).Tpd*1e6,param.tg.Hice_thick)); + write_cresis_xml(param); + if freq_idx == 1 + % Default Mode + param.fn = fullfile(base_dir,'default.xml'); + write_cresis_xml(param); + end + % Loopback Mode without delay line + param.tg.staged_recording = false; + param.tg.altitude_guard = 1000*12*2.54/100; + param.tg.Haltitude = 0e-6 * c/2; + param.tg.Hice_thick = 0; % Long enough for 10 us delay line + param.fn = fullfile(calval_dir,sprintf('survey_%.0f-%.0fMHz_%.0fus_LOOPBACK_NO_DELAY.xml',param.f0/1e6,param.f1/1e6,param.wfs(end).Tpd*1e6)); + write_cresis_xml(param); + % Loopback Mode (10e-6 delay line) + param.tg.staged_recording = false; + param.tg.altitude_guard = 1000*12*2.54/100; + param.tg.Haltitude = 10e-6 * c/2; + param.tg.Hice_thick = 0; % Long enough for 10 us delay line + param.fn = fullfile(calval_dir,sprintf('survey_%.0f-%.0fMHz_%.0fus_LOOPBACK.xml',param.f0/1e6,param.f1/1e6,param.wfs(end).Tpd*1e6)); + write_cresis_xml(param); + % Deconvolution Mode (for over calm lake or sea ice lead) + param.wfs(1).atten = 43; + param.wfs(2).atten = 43; + param.wfs(3).atten = 43; + param.tg.staged_recording = false; + param.tg.altitude_guard = 3000*12*2.54/100; + param.tg.Haltitude = 4000*12*2.54/100; + param.tg.Hice_thick = 0 * 12*2.54/100/sqrt(er_ice); + param.fn = fullfile(calval_dir,sprintf('survey_%.0f-%.0fMHz_%.0fft_%.0fus_DECONV.xml',param.f0/1e6,param.f1/1e6,param.tg.Haltitude*100/12/2.54,param.wfs(end).Tpd*1e6)); + write_cresis_xml(param); + if freq_idx == 1 + % Noise Mode + param.tx_weights = [0 0 0 0 0 0 0 0]; + [param.wfs(1:3).tx_mask] = deal([1 1 1 1 1 1 1 1]); + param.wfs(1).atten = 37; + param.wfs(2).atten = 0; + param.wfs(3).atten = 0; + param.tg.staged_recording = [1 2 3]; + param.tg.altitude_guard = 500*12*2.54/100; + param.tg.Haltitude = 1400*12*2.54/100; + param.tg.Hice_thick = 3250; + param.fn = fullfile(calval_dir,sprintf('survey_%.0f-%.0fMHz_%.0fus_NOISE.xml',param.f0/1e6,param.f1/1e6,param.wfs(end).Tpd*1e6)); + write_cresis_xml(param); + end +end + +%% Survey Mode +% <3250 m thick ice, 1200 +/- 700 ft AGL +ice_thickness = [3250 3250]; +for freq_idx = [1 2] + param = struct('radar_name','mcords5','num_chan',8,'aux_dac',[255 255 255 255 255 255 255 255],'version','14.0f1','TTL_prog_delay',650,'xml_version',2.0,'fs',1600e6,'fs_sync',90.0e6,'fs_dds',1440e6,'TTL_clock',1440e6/16,'TTL_mode',[2.5e-6 260e-9 -1100e-9],'rss_base_dir',rss_base_dir); + param.max_tx = [4000 4000 4000 4000 4000 4000 4000 4000]; param.max_data_rate = 750; param.flight_hours = 3.5; param.sys_delay = 0.75e-6; param.use_mcords4_names = true; param.arena = arena; + param.DDC_select = DDC_select_list(freq_idx); + param.max_duty_cycle = 0.12; + param.create_IQ = false; + param.tg.staged_recording = [1 2 3]; + param.tg.altitude_guard = 700*12*2.54/100; + param.tg.Haltitude = 1200*12*2.54/100; + param.tg.Hice_thick = ice_thickness(freq_idx); + param.prf = prf; + param.presums = [2 4 presums(freq_idx)-6]; + param.wfs(1).atten = 37; + param.wfs(2).atten = 0; + param.wfs(3).atten = 0; + DDS_amp = final_DDS_amp{cal_settings(freq_idx)}; + param.tx_weights = DDS_amp; + param.tukey = 0.08; + param.wfs(1).Tpd = 1e-6; + param.wfs(2).Tpd = 3e-6; + param.wfs(3).Tpd = 10e-6; + param.wfs(1).phase = final_DDS_phase{cal_settings(freq_idx)}; + param.wfs(2).phase = final_DDS_phase{cal_settings(freq_idx)}; + param.wfs(3).phase = final_DDS_phase{cal_settings(freq_idx)}; + param.delay = final_DDS_time{cal_settings(freq_idx)}; + param.f0 = f0_list(freq_idx); + param.f1 = f1_list(freq_idx); + param.DDC_freq = (param.f0+param.f1)/2; + [param.wfs(1:3).tx_mask] = deal([0 0 0 0 0 0 0 0]); + param.fn = fullfile(base_dir,sprintf('survey_%.0f-%.0fMHz_%.0fft_%.0fus_%.0fmthick.xml',param.f0/1e6,param.f1/1e6,param.tg.Haltitude*100/12/2.54,param.wfs(end).Tpd*1e6,param.tg.Hice_thick)); + write_cresis_xml(param); +end + +%% Survey Mode for thin ice +% <2500 m thick ice, 1200 +/- 700 ft AGL +ice_thickness = [2500 2500]; +for freq_idx = [1 2] + param = struct('radar_name','mcords5','num_chan',8,'aux_dac',[255 255 255 255 255 255 255 255],'version','14.0f1','TTL_prog_delay',650,'xml_version',2.0,'fs',1600e6,'fs_sync',90.0e6,'fs_dds',1440e6,'TTL_clock',1440e6/16,'TTL_mode',[2.5e-6 260e-9 -1100e-9],'rss_base_dir',rss_base_dir); + param.max_tx = [4000 4000 4000 4000 4000 4000 4000 4000]; param.max_data_rate = 750; param.flight_hours = 3.5; param.sys_delay = 0.75e-6; param.use_mcords4_names = true; param.arena = arena; + param.DDC_select = DDC_select_list(freq_idx); + param.max_duty_cycle = 0.12; + param.create_IQ = false; + param.tg.staged_recording = [1 2 3]; + param.tg.rg_stop_offset = [500*sqrt(3.15) 0 0]; % Keep waveform 1 on for 500 m of ice + param.tg.altitude_guard = 700*12*2.54/100; + param.tg.Haltitude = 1200*12*2.54/100; + param.tg.Hice_thick = ice_thickness(freq_idx); + param.prf = prf; + param.presums = [8 2 presums(freq_idx)-10]; + param.wfs(1).atten = 35; + param.wfs(2).atten = 0; + param.wfs(3).atten = 0; + DDS_amp = final_DDS_amp{cal_settings(freq_idx)}; + param.tx_weights = DDS_amp; + param.tukey = 0.08; + param.wfs(1).Tpd = 1e-6; + param.wfs(2).Tpd = 1e-6; + param.wfs(3).Tpd = 3e-6; + param.wfs(1).phase = final_DDS_phase{cal_settings(freq_idx)}; + param.wfs(2).phase = final_DDS_phase{cal_settings(freq_idx)}; + param.wfs(3).phase = final_DDS_phase{cal_settings(freq_idx)}; + param.delay = final_DDS_time{cal_settings(freq_idx)}; + param.f0 = f0_list(freq_idx); + param.f1 = f1_list(freq_idx); + param.DDC_freq = (param.f0+param.f1)/2; + [param.wfs(1:3).tx_mask] = deal([0 0 0 0 0 0 0 0]); + param.fn = fullfile(base_dir,sprintf('thinice_%.0f-%.0fMHz_%.0fft_%.0fus_%.0fmthick.xml',param.f0/1e6,param.f1/1e6,param.tg.Haltitude*100/12/2.54,param.wfs(end).Tpd*1e6,param.tg.Hice_thick)); + write_cresis_xml(param); +end + +%% Sea Ice +% 1200 +/- 1200 ft AGL +for freq_idx = [1] + param = struct('radar_name','mcords5','num_chan',8,'aux_dac',[255 255 255 255 255 255 255 255],'version','14.0f1','TTL_prog_delay',650,'xml_version',2.0,'fs',1600e6,'fs_sync',90.0e6,'fs_dds',1440e6,'TTL_clock',1440e6/16,'TTL_mode',[2.5e-6 260e-9 -1100e-9],'rss_base_dir',rss_base_dir); + param.max_tx = [4000 4000 4000 4000 4000 4000 4000 4000]; param.max_data_rate = 750; param.flight_hours = 3.5; param.sys_delay = 0.75e-6; param.use_mcords4_names = true; param.arena = arena; + param.DDC_select = DDC_select_list(freq_idx); + param.max_duty_cycle = 0.12; + param.create_IQ = false; + param.tg.staged_recording = [0 0]; + param.tg.rg_stop_offset = [0 0]; % Keep waveform 1 on for 500 m of ice + param.tg.altitude_guard = 1200*12*2.54/100; + param.tg.Haltitude = 1200*12*2.54/100; + param.tg.Hice_thick = 0; + param.prf = prf; + param.presums = [round(presums(freq_idx)/2/2)*2 round(presums(freq_idx)/2/2)*2]; + param.wfs(1).atten = 13; + param.wfs(2).atten = 13; + DDS_amp = final_DDS_amp{cal_settings(freq_idx)}; + param.tx_weights = DDS_amp; + param.tukey = 0.08; + param.wfs(1).Tpd = 1e-6; + param.wfs(2).Tpd = 1e-6; + param.wfs(1).phase = final_DDS_phase{cal_settings(freq_idx)}; + param.wfs(2).phase = final_DDS_phase{cal_settings(freq_idx)}; + param.delay = final_DDS_time{cal_settings(freq_idx)}; + param.f0 = f0_list(freq_idx); + param.f1 = f1_list(freq_idx); + param.DDC_freq = (param.f0+param.f1)/2; + param.wfs(1).tx_mask = deal([0 1 1 1 1 1 1 1]); + param.wfs(2).tx_mask = deal([1 1 1 1 1 1 1 0]); + param.fn = fullfile(base_dir,sprintf('seaice_%.0f-%.0fMHz_%.0fft_%.0fus_%.0fmthick.xml',param.f0/1e6,param.f1/1e6,param.tg.Haltitude*100/12/2.54,param.wfs(end).Tpd*1e6,param.tg.Hice_thick)); + write_cresis_xml(param); +end + +%% Image Mode (EGRIP Low Altitude, Thick Ice) +% Ice thickness "param.tg.Hice_thick_min" m to "param.tg.Hice_thick" m, "param.tg.Haltitude" +/- "param.tg.altitude_guard" ft AGL +for freq_idx = 2 + param = struct('radar_name','mcords5','num_chan',8,'aux_dac',[255 255 255 255 255 255 255 255],'version','14.0f1','TTL_prog_delay',650,'xml_version',2.0,'fs',1600e6,'fs_sync',90.0e6,'fs_dds',1440e6,'TTL_clock',1440e6/16,'TTL_mode',[2.5e-6 260e-9 -1100e-9],'rss_base_dir',rss_base_dir); + param.max_tx = [4000 4000 4000 4000 4000 4000 4000 4000]; param.max_data_rate = 700; param.flight_hours = 4; param.sys_delay = 0.75e-6; param.use_mcords4_names = true; param.arena = arena; + param.max_data_rate = 755; + param.DDC_select = DDC_select_list(freq_idx); + param.max_duty_cycle = 0.12; + param.create_IQ = false; + param.tg.staged_recording = [1 2 3 3]; + param.tg.altitude_guard = 700 * 12*2.54/100; + param.tg.Haltitude = 1200 * 12*2.54/100; + param.tg.Hice_thick = 3250; + param.tg.look_angle_deg = [0 0 40 40]; + param.prf = prf; + param.presums = [2 4 ceil((presums(freq_idx)-6)/4)*2 ceil((presums(freq_idx)-6)/4)*2]; + % Switch from tx calibration window to hanning window to broaden beam + DDS_amp = final_DDS_amp{freq_idx} .* hanning(8).' ./ Hwindow_orig; + % Renormalize the amplitudes + [~,relative_max_idx] = max(DDS_amp./param.max_tx); + DDS_amp = round(DDS_amp .* param.max_tx(relative_max_idx) / DDS_amp(relative_max_idx)); + param.tx_weights = DDS_amp; + param.tukey = 0.08; + param.wfs(1).Tpd = 1e-6; + param.wfs(2).Tpd = 3e-6; + param.wfs(3).Tpd = 10e-6; + param.wfs(4).Tpd = 10e-6; + param.wfs(1).phase = final_DDS_phase{freq_idx}; + param.wfs(2).phase = final_DDS_phase{freq_idx}; + param.wfs(3).phase = final_DDS_phase{freq_idx}; + param.wfs(4).phase = final_DDS_phase{freq_idx}; + param.wfs(1).name = 'nadir1'; + param.wfs(2).name = 'nadir3'; + param.wfs(3).name = 'left'; + param.wfs(4).name = 'right'; + % Add in time delays to each position, subtract out the nadir time delays since tx_equalization already took care of those + beam_angle_deg = 0; % Nadir + param.wfs(1).delay = final_DDS_time{freq_idx} ... + - (phase_centers(2,:) / (c/2) * sind(beam_angle_deg))*1e9 ... + - (phase_centers(3,:) / (c/2) * cosd(beam_angle_deg))*1e9 ... + + (phase_centers(3,:) / (c/2) * cosd(0))*1e9; + beam_angle_deg = 0; % Nadir + param.wfs(2).delay = final_DDS_time{freq_idx} ... + - (phase_centers(2,:) / (c/2) * sind(beam_angle_deg))*1e9 ... + - (phase_centers(3,:) / (c/2) * cosd(beam_angle_deg))*1e9 ... + + (phase_centers(3,:) / (c/2) * cosd(0))*1e9; + beam_angle_deg = 20; % Positive to the left + param.wfs(3).delay = final_DDS_time{freq_idx} ... + - (phase_centers(2,:) / (c/2) * sind(beam_angle_deg))*1e9 ... + - (phase_centers(3,:) / (c/2) * cosd(beam_angle_deg))*1e9 ... + + (phase_centers(3,:) / (c/2) * cosd(0))*1e9; + beam_angle_deg = -20; % Negative to the right + param.wfs(4).delay = final_DDS_time{freq_idx} ... + - (phase_centers(2,:) / (c/2) * sind(beam_angle_deg))*1e9 ... + - (phase_centers(3,:) / (c/2) * cosd(beam_angle_deg))*1e9 ... + + (phase_centers(3,:) / (c/2) * cosd(0))*1e9; + param.f0 = f0_list(freq_idx); + param.f1 = f1_list(freq_idx); + param.DDC_freq = (param.f0+param.f1)/2; + [param.wfs(1:4).tx_mask] = deal([0 0 0 0 0 0 0 0]); + param.wfs(1).atten = 37; + param.wfs(2).atten = 0; + param.wfs(3).atten = 0; + param.wfs(4).atten = 0; + param.fn = fullfile(base_dir,sprintf('egrip_image_%.0f-%.0fMHz_%.0fft_%.0fus_%.0fmthick.xml', ... + param.f0/1e6,param.f1/1e6,param.tg.Haltitude*100/2.54/12,param.wfs(end).Tpd*1e6,param.tg.Hice_thick)); + write_cresis_xml(param); +end + +%% Image Mode (Low Altitude, Thick Ice) +% Ice thickness "param.tg.Hice_thick_min" m to "param.tg.Hice_thick" m, "param.tg.Haltitude" +/- "param.tg.altitude_guard" ft AGL +for freq_idx = [1 2] + param = struct('radar_name','mcords5','num_chan',8,'aux_dac',[255 255 255 255 255 255 255 255],'version','14.0f1','TTL_prog_delay',650,'xml_version',2.0,'fs',1600e6,'fs_sync',90.0e6,'fs_dds',1440e6,'TTL_clock',1440e6/16,'TTL_mode',[2.5e-6 260e-9 -1100e-9],'rss_base_dir',rss_base_dir); + param.max_tx = [4000 4000 4000 4000 4000 4000 4000 4000]; param.max_data_rate = 700; param.flight_hours = 3.5; param.sys_delay = 0.75e-6; param.use_mcords4_names = true; param.arena = arena; + param.max_data_rate = 755; + param.DDC_select = DDC_select_list(freq_idx); + param.max_duty_cycle = 0.12; + param.create_IQ = false; + param.tg.staged_recording = [0 0 0]; + param.tg.start_ref = {'bottom','surface','bottom'}; + param.tg.stop_ref = {'bottom','surface','bottom'}; + param.tg.altitude_guard = 700 * 12*2.54/100; + param.tg.Haltitude = 1200 * 12*2.54/100; + param.tg.Hice_thick_min = 1500; + param.tg.Hice_thick = 3000; + param.tg.look_angle_deg = [40 0 40]; + param.prf = prf; + param.presums = [ceil(presums(freq_idx)/4)*2 2 ceil(presums(freq_idx)/4)*2]; + % Switch from tx calibration window to hanning window to broaden beam + DDS_amp = final_DDS_amp{freq_idx} .* hanning(8).' ./ Hwindow_orig; + % Renormalize the amplitudes + [~,relative_max_idx] = max(DDS_amp./param.max_tx); + DDS_amp = round(DDS_amp .* param.max_tx(relative_max_idx) / DDS_amp(relative_max_idx)); + param.tx_weights = DDS_amp; + param.tukey = 0.08; + param.wfs(1).Tpd = 10e-6; + param.wfs(2).Tpd = 1e-6; + param.wfs(3).Tpd = 10e-6; + param.wfs(1).phase = final_DDS_phase{freq_idx}; + param.wfs(2).phase = final_DDS_phase{freq_idx}; + param.wfs(3).phase = final_DDS_phase{freq_idx}; + param.wfs(1).name = 'left'; + param.wfs(2).name = 'nadir'; + param.wfs(3).name = 'right'; + % Add in time delays to each position, subtract out the nadir time delays since tx_equalization already took care of those + beam_angle_deg = 20; % Positive to the left + param.wfs(1).delay = final_DDS_time{freq_idx} ... + - (phase_centers(2,:) / (c/2) * sind(beam_angle_deg))*1e9 ... + - (phase_centers(3,:) / (c/2) * cosd(beam_angle_deg))*1e9 ... + + (phase_centers(3,:) / (c/2) * cosd(0))*1e9; + beam_angle_deg = 0; % Nadir + param.wfs(2).delay = final_DDS_time{freq_idx} ... + - (phase_centers(2,:) / (c/2) * sind(beam_angle_deg))*1e9 ... + - (phase_centers(3,:) / (c/2) * cosd(beam_angle_deg))*1e9 ... + + (phase_centers(3,:) / (c/2) * cosd(0))*1e9; + beam_angle_deg = -20; % Negative to the right + param.wfs(3).delay = final_DDS_time{freq_idx} ... + - (phase_centers(2,:) / (c/2) * sind(beam_angle_deg))*1e9 ... + - (phase_centers(3,:) / (c/2) * cosd(beam_angle_deg))*1e9 ... + + (phase_centers(3,:) / (c/2) * cosd(0))*1e9; + param.f0 = f0_list(freq_idx); + param.f1 = f1_list(freq_idx); + param.DDC_freq = (param.f0+param.f1)/2; + [param.wfs(1:3).tx_mask] = deal([0 0 0 0 0 0 0 0]); + param.wfs(1).atten = 0; + param.wfs(2).atten = 37; + param.wfs(3).atten = 0; + param.fn = fullfile(base_dir,sprintf('image_%.0f-%.0fMHz_%.0fft_%.0fus_%.0fmthick.xml', ... + param.f0/1e6,param.f1/1e6,param.tg.Haltitude*100/2.54/12,param.wfs(end).Tpd*1e6,param.tg.Hice_thick)); + write_cresis_xml(param); +end + +%% Image Mode Pattern Measurements +% 3500 ft +/- 1000 ft AGL +for freq_idx = [1 2] + param = struct('radar_name','mcords5','num_chan',8,'aux_dac',[255 255 255 255 255 255 255 255],'version','14.0f1','TTL_prog_delay',650,'xml_version',2.0,'fs',1600e6,'fs_sync',90.0e6,'fs_dds',1440e6,'TTL_clock',1440e6/16,'TTL_mode',[2.5e-6 260e-9 -1100e-9],'rss_base_dir',rss_base_dir); + param.max_tx = [4000 4000 4000 4000 4000 4000 4000 4000]; param.max_data_rate = 700; param.flight_hours = 3.5; param.sys_delay = 0.75e-6; param.use_mcords4_names = true; param.arena = arena; + param.max_data_rate = 755; + param.DDC_select = DDC_select_list(freq_idx); + param.max_duty_cycle = 0.12; + param.create_IQ = false; + param.tg.staged_recording = [0 0]; + param.tg.start_ref = {'surface','surface'}; + param.tg.altitude_guard = 1000 * 12*2.54/100; + param.tg.Haltitude = 3500 * 12*2.54/100; + param.tg.Hice_thick_min = 0; + param.tg.Hice_thick = 0; + param.tg.look_angle_deg = [0 0 0]; + param.prf = prf; + param.presums = [ceil(presums(freq_idx)/4)*2 ceil(presums(freq_idx)/4)*2]; + % Switch from tx calibration window to hanning window to broaden beam + DDS_amp = final_DDS_amp{freq_idx} .* hanning(8).' ./ Hwindow_orig; + % Renormalize the amplitudes + [~,relative_max_idx] = max(DDS_amp./param.max_tx); + DDS_amp = round(DDS_amp .* param.max_tx(relative_max_idx) / DDS_amp(relative_max_idx)); + param.tx_weights = DDS_amp; + param.tukey = 0.08; + param.wfs(1).Tpd = 3e-6; + param.wfs(2).Tpd = 3e-6; + param.wfs(1).phase = final_DDS_phase{freq_idx}; + param.wfs(2).phase = final_DDS_phase{freq_idx}; + param.wfs(1).name = 'left'; + param.wfs(2).name = 'right'; + % Add in time delays to each position, subtract out the nadir time delays since tx_equalization already took care of those + beam_angle_deg = 20; % Positive to the left + param.wfs(1).delay = final_DDS_time{freq_idx} ... + - (phase_centers(2,:) / (c/2) * sind(beam_angle_deg))*1e9 ... + - (phase_centers(3,:) / (c/2) * cosd(beam_angle_deg))*1e9 ... + + (phase_centers(3,:) / (c/2) * cosd(0))*1e9; + beam_angle_deg = -20; % Negative to the right + param.wfs(2).delay = final_DDS_time{freq_idx} ... + - (phase_centers(2,:) / (c/2) * sind(beam_angle_deg))*1e9 ... + - (phase_centers(3,:) / (c/2) * cosd(beam_angle_deg))*1e9 ... + + (phase_centers(3,:) / (c/2) * cosd(0))*1e9; + param.f0 = f0_list(freq_idx); + param.f1 = f1_list(freq_idx); + param.DDC_freq = (param.f0+param.f1)/2; + [param.wfs(1:2).tx_mask] = deal([0 0 0 0 0 0 0 0]); + [param.wfs(1:2).atten] = deal(43); + param.fn = fullfile(calval_dir,sprintf('image_%.0f-%.0fMHz_%.0fft_%.0fus_PATTERN.xml', ... + param.f0/1e6,param.f1/1e6,param.tg.Haltitude*100/2.54/12,param.wfs(end).Tpd*1e6)); + write_cresis_xml(param); +end + +%% Image Mode (High Altitude, Thin Ice) +% Ice thickness "param.tg.Hice_thick_min" m to "param.tg.Hice_thick" m, "param.tg.Haltitude" +/- "param.tg.altitude_guard" ft AGL +freq_idx_WB = 1; +freq_idx_NB = 3; +param = struct('radar_name','mcords5','num_chan',8,'aux_dac',[255 255 255 255 255 255 255 255],'version','14.0f1','TTL_prog_delay',650,'xml_version',2.0,'fs',1600e6,'fs_sync',90.0e6,'fs_dds',1440e6,'TTL_clock',1440e6/16,'TTL_mode',[2.5e-6 260e-9 -1100e-9],'rss_base_dir',rss_base_dir); +param.max_tx = [4000 4000 4000 4000 4000 4000 4000 4000]; param.max_data_rate = 700; param.flight_hours = 3.5; param.sys_delay = 0.75e-6; param.use_mcords4_names = true; param.arena = arena; +param.max_data_rate = 755; +param.DDC_select = DDC_select_list(freq_idx_WB); +param.max_duty_cycle = 0.12; +param.create_IQ = false; +param.tg.staged_recording = [0 0 0]; +param.tg.start_ref = {'bottom','surface','bottom'}; +param.tg.stop_ref = {'bottom','bottom','bottom'}; +param.tg.altitude_guard = 1000 * 12*2.54/100; +param.tg.Haltitude = 6000 * 12*2.54/100; +param.tg.Hice_thick_min = 0; +param.tg.Hice_thick = 1000; +param.tg.look_angle_deg = [40 0 40]; +param.prf = prf; +param.presums = [ceil(presums(freq_idx_WB)/4)*2 4 ceil(presums(freq_idx_WB)/4)*2]; +% Switch from tx calibration window to hanning window to broaden beam +DDS_amp = final_DDS_amp{freq_idx_WB} .* hanning(8).' ./ Hwindow_orig; +% Renormalize the amplitudes +[~,relative_max_idx] = max(DDS_amp./param.max_tx); +DDS_amp = round(DDS_amp .* param.max_tx(relative_max_idx) / DDS_amp(relative_max_idx)); +param.tx_weights = DDS_amp; +param.tukey = 0.08; +param.wfs(1).Tpd = 3e-6; +param.wfs(2).Tpd = 1e-6; +param.wfs(3).Tpd = 3e-6; +param.wfs(1).phase = final_DDS_phase{freq_idx_NB}; +param.wfs(2).phase = final_DDS_phase{freq_idx_WB}; +param.wfs(3).phase = final_DDS_phase{freq_idx_NB}; +param.wfs(1).name = 'left'; +param.wfs(2).name = 'nadir'; +param.wfs(3).name = 'right'; +% Add in time delays to each position, subtract out the nadir time delays since tx_equalization already took care of those +beam_angle_deg = 20; % Positive to the left +param.wfs(1).delay = final_DDS_time{freq_idx_NB} ... + - (phase_centers(2,:) / (c/2) * sind(beam_angle_deg))*1e9 ... + - (phase_centers(3,:) / (c/2) * cosd(beam_angle_deg))*1e9 ... + + (phase_centers(3,:) / (c/2) * cosd(0))*1e9; +beam_angle_deg = 0; % Nadir +param.wfs(2).delay = final_DDS_time{freq_idx_WB} ... + - (phase_centers(2,:) / (c/2) * sind(beam_angle_deg))*1e9 ... + - (phase_centers(3,:) / (c/2) * cosd(beam_angle_deg))*1e9 ... + + (phase_centers(3,:) / (c/2) * cosd(0))*1e9; +beam_angle_deg = -20; % Negative to the right +param.wfs(3).delay = final_DDS_time{freq_idx_NB} ... + - (phase_centers(2,:) / (c/2) * sind(beam_angle_deg))*1e9 ... + - (phase_centers(3,:) / (c/2) * cosd(beam_angle_deg))*1e9 ... + + (phase_centers(3,:) / (c/2) * cosd(0))*1e9; +param.wfs(1).f0 = f0_list(freq_idx_NB); +param.wfs(2).f0 = f0_list(freq_idx_WB); +param.wfs(3).f0 = f0_list(freq_idx_NB); +param.wfs(1).f1 = f1_list(freq_idx_NB); +param.wfs(2).f1 = f1_list(freq_idx_WB); +param.wfs(3).f1 = f1_list(freq_idx_NB); +param.DDC_freq = (param.wfs(2).f0+param.wfs(2).f1)/2; +[param.wfs(1:3).tx_mask] = deal([0 0 0 0 0 0 0 0]); +param.wfs(1).atten = 0; +param.wfs(2).atten = 23; +param.wfs(3).atten = 0; +param.fn = fullfile(base_dir,sprintf('imagehighthin_%.0f-%.0fMHz_%.0fft_%.0fus_%.0fmthick.xml', ... + param.wfs(2).f0/1e6,param.wfs(2).f1/1e6,param.tg.Haltitude*100/2.54/12,param.wfs(end).Tpd*1e6,param.tg.Hice_thick)); +write_cresis_xml(param); + +%% Image Mode (Low Altitude, Ice <2000 m thick) +% Ice thickness "param.tg.Hice_thick_min" m to "param.tg.Hice_thick" m, "param.tg.Haltitude" +/- "param.tg.altitude_guard" ft AGL +freq_idx_WB = 1; +freq_idx_NB = 2; +param = struct('radar_name','mcords5','num_chan',8,'aux_dac',[255 255 255 255 255 255 255 255],'version','14.0f1','TTL_prog_delay',650,'xml_version',2.0,'fs',1600e6,'fs_sync',90.0e6,'fs_dds',1440e6,'TTL_clock',1440e6/16,'TTL_mode',[2.5e-6 260e-9 -1100e-9],'rss_base_dir',rss_base_dir); +param.max_tx = [4000 4000 4000 4000 4000 4000 4000 4000]; param.max_data_rate = 700; param.flight_hours = 3.5; param.sys_delay = 0.75e-6; param.use_mcords4_names = true; param.arena = arena; +param.max_data_rate = 755; +param.DDC_select = DDC_select_list(freq_idx_NB); +param.max_duty_cycle = 0.12; +param.create_IQ = false; +param.tg.staged_recording = [0 0 0]; +param.tg.start_ref = {'bottom','surface','bottom'}; +param.tg.stop_ref = {'bottom','bottom','bottom'}; +param.tg.altitude_guard = 500 * 12*2.54/100; +param.tg.Haltitude = 1200 * 12*2.54/100; +param.tg.Hice_thick_min = 0; +param.tg.Hice_thick = 2000; +param.tg.look_angle_deg = [40 0 40]; +param.prf = prf; +param.presums = [8 8 8]; %[ceil(presums(freq_idx_WB)/4)*2 4 ceil(presums(freq_idx_WB)/4)*2]; +% Switch from tx calibration window to hanning window to broaden beam +DDS_amp = final_DDS_amp{freq_idx_NB} .* hanning(8).' ./ Hwindow_orig; +% Renormalize the amplitudes +[~,relative_max_idx] = max(DDS_amp./param.max_tx); +DDS_amp = round(DDS_amp .* param.max_tx(relative_max_idx) / DDS_amp(relative_max_idx)); +param.tx_weights = DDS_amp; +param.tukey = 0.08; +param.wfs(1).Tpd = 3e-6; +param.wfs(2).Tpd = 3e-6; +param.wfs(3).Tpd = 3e-6; +param.wfs(1).phase = final_DDS_phase{freq_idx_NB}; +param.wfs(2).phase = final_DDS_phase{freq_idx_NB}; +param.wfs(3).phase = final_DDS_phase{freq_idx_NB}; +param.wfs(1).name = 'left'; +param.wfs(2).name = 'nadir'; +param.wfs(3).name = 'right'; +% Add in time delays to each position, subtract out the nadir time delays since tx_equalization already took care of those +beam_angle_deg = 20; % Positive to the left +param.wfs(1).delay = final_DDS_time{freq_idx_NB} ... + - (phase_centers(2,:) / (c/2) * sind(beam_angle_deg))*1e9 ... + - (phase_centers(3,:) / (c/2) * cosd(beam_angle_deg))*1e9 ... + + (phase_centers(3,:) / (c/2) * cosd(0))*1e9; +beam_angle_deg = 0; % Nadir +param.wfs(2).delay = final_DDS_time{freq_idx_NB} ... + - (phase_centers(2,:) / (c/2) * sind(beam_angle_deg))*1e9 ... + - (phase_centers(3,:) / (c/2) * cosd(beam_angle_deg))*1e9 ... + + (phase_centers(3,:) / (c/2) * cosd(0))*1e9; +beam_angle_deg = -20; % Negative to the right +param.wfs(3).delay = final_DDS_time{freq_idx_NB} ... + - (phase_centers(2,:) / (c/2) * sind(beam_angle_deg))*1e9 ... + - (phase_centers(3,:) / (c/2) * cosd(beam_angle_deg))*1e9 ... + + (phase_centers(3,:) / (c/2) * cosd(0))*1e9; +param.wfs(1).f0 = f0_list(freq_idx_NB); +param.wfs(2).f0 = f0_list(freq_idx_NB); +param.wfs(3).f0 = f0_list(freq_idx_NB); +param.wfs(1).f1 = f1_list(freq_idx_NB); +param.wfs(2).f1 = f1_list(freq_idx_NB); +param.wfs(3).f1 = f1_list(freq_idx_NB); +param.DDC_freq = (param.wfs(2).f0+param.wfs(2).f1)/2; +[param.wfs(1:3).tx_mask] = deal([0 0 0 0 0 0 0 0]); +param.wfs(1).atten = 0; +param.wfs(2).atten = 0; +param.wfs(3).atten = 0; +param.fn = fullfile(base_dir,sprintf('image_angelika_%.0f-%.0fMHz_%.0fft_%.0fus_%.0fmthick.xml', ... + param.wfs(2).f0/1e6,param.wfs(2).f1/1e6,param.tg.Haltitude*100/2.54/12,param.wfs(end).Tpd*1e6,param.tg.Hice_thick)); +write_cresis_xml(param); + +%% Image Mode (Low Altitude, Ice <1400 m thick) +% Ice thickness "param.tg.Hice_thick_min" m to "param.tg.Hice_thick" m, "param.tg.Haltitude" +/- "param.tg.altitude_guard" ft AGL +freq_idx_WB = 1; +freq_idx_NB = 2; +param = struct('radar_name','mcords5','num_chan',8,'aux_dac',[255 255 255 255 255 255 255 255],'version','14.0f1','TTL_prog_delay',650,'xml_version',2.0,'fs',1600e6,'fs_sync',90.0e6,'fs_dds',1440e6,'TTL_clock',1440e6/16,'TTL_mode',[2.5e-6 260e-9 -1100e-9],'rss_base_dir',rss_base_dir); +param.max_tx = [4000 4000 4000 4000 4000 4000 4000 4000]; param.max_data_rate = 700; param.flight_hours = 3.5; param.sys_delay = 0.75e-6; param.use_mcords4_names = true; param.arena = arena; +param.max_data_rate = 755; +param.DDC_select = DDC_select_list(freq_idx_NB); +param.max_duty_cycle = 0.12; +param.create_IQ = false; +param.tg.staged_recording = [0 0 0]; +param.tg.start_ref = {'bottom','surface','bottom'}; +param.tg.stop_ref = {'bottom','bottom','bottom'}; +param.tg.altitude_guard = 500 * 12*2.54/100; +param.tg.Haltitude = 1200 * 12*2.54/100; +param.tg.Hice_thick_min = 0; +param.tg.Hice_thick = 1400; +param.tg.look_angle_deg = [40 0 40]; +param.prf = prf; +param.presums = [8 8 8]; %[ceil(presums(freq_idx_WB)/4)*2 4 ceil(presums(freq_idx_WB)/4)*2]; +% Switch from tx calibration window to hanning window to broaden beam +DDS_amp = final_DDS_amp{freq_idx_NB} .* hanning(8).' ./ Hwindow_orig; +% Renormalize the amplitudes +[~,relative_max_idx] = max(DDS_amp./param.max_tx); +DDS_amp = round(DDS_amp .* param.max_tx(relative_max_idx) / DDS_amp(relative_max_idx)); +param.tx_weights = DDS_amp; +param.tukey = 0.08; +param.wfs(1).Tpd = 3e-6; +param.wfs(2).Tpd = 3e-6; +param.wfs(3).Tpd = 3e-6; +param.wfs(1).phase = final_DDS_phase{freq_idx_NB}; +param.wfs(2).phase = final_DDS_phase{freq_idx_NB}; +param.wfs(3).phase = final_DDS_phase{freq_idx_NB}; +param.wfs(1).name = 'left'; +param.wfs(2).name = 'nadir'; +param.wfs(3).name = 'right'; +% Add in time delays to each position, subtract out the nadir time delays since tx_equalization already took care of those +beam_angle_deg = 20; % Positive to the left +param.wfs(1).delay = final_DDS_time{freq_idx_NB} ... + - (phase_centers(2,:) / (c/2) * sind(beam_angle_deg))*1e9 ... + - (phase_centers(3,:) / (c/2) * cosd(beam_angle_deg))*1e9 ... + + (phase_centers(3,:) / (c/2) * cosd(0))*1e9; +beam_angle_deg = 0; % Nadir +param.wfs(2).delay = final_DDS_time{freq_idx_NB} ... + - (phase_centers(2,:) / (c/2) * sind(beam_angle_deg))*1e9 ... + - (phase_centers(3,:) / (c/2) * cosd(beam_angle_deg))*1e9 ... + + (phase_centers(3,:) / (c/2) * cosd(0))*1e9; +beam_angle_deg = -20; % Negative to the right +param.wfs(3).delay = final_DDS_time{freq_idx_NB} ... + - (phase_centers(2,:) / (c/2) * sind(beam_angle_deg))*1e9 ... + - (phase_centers(3,:) / (c/2) * cosd(beam_angle_deg))*1e9 ... + + (phase_centers(3,:) / (c/2) * cosd(0))*1e9; +param.wfs(1).f0 = f0_list(freq_idx_NB); +param.wfs(2).f0 = f0_list(freq_idx_NB); +param.wfs(3).f0 = f0_list(freq_idx_NB); +param.wfs(1).f1 = f1_list(freq_idx_NB); +param.wfs(2).f1 = f1_list(freq_idx_NB); +param.wfs(3).f1 = f1_list(freq_idx_NB); +param.DDC_freq = (param.wfs(2).f0+param.wfs(2).f1)/2; +[param.wfs(1:3).tx_mask] = deal([0 0 0 0 0 0 0 0]); +param.wfs(1).atten = 0; +param.wfs(2).atten = 0; +param.wfs(3).atten = 0; +param.fn = fullfile(base_dir,sprintf('image_angelika_%.0f-%.0fMHz_%.0fft_%.0fus_%.0fmthick.xml', ... + param.wfs(2).f0/1e6,param.wfs(2).f1/1e6,param.tg.Haltitude*100/2.54/12,param.wfs(end).Tpd*1e6,param.tg.Hice_thick)); +write_cresis_xml(param); + +%% Equalization (Using Ocean) +% Haltitude +/- 1000 ft +% For lower altitude, increase attenuation +% Use these settings over ocean or sea ice for fast-time equalization, +% transmit equalization, and receiver equalization. +% Creates one waveform for each of N DDS-transmitters plus a combined +% waveform with all transmitters going. +Haltitude = [1500 1500 0 3000 6000]; +Tpd_list = [1e-6 1e-6 3e-6 3e-6 3e-6]; +attenuation = [43 39 43 43 43]; +fn_hint = {'WATER','ICE','NO_DELAY','WATER','WATER'}; +for Tpd_idx = 1:length(Tpd_list) + Tpd = Tpd_list(Tpd_idx); + for freq_idx = [1 2] + param = struct('radar_name','mcords5','num_chan',8,'aux_dac',[255 255 255 255 255 255 255 255],'version','14.0f1','TTL_prog_delay',650,'xml_version',2.0,'fs',1600e6,'fs_sync',90.0e6,'fs_dds',1440e6,'TTL_clock',1440e6/16,'TTL_mode',[2.5e-6 260e-9 -1100e-9],'rss_base_dir',rss_base_dir); + param.max_tx = [4000 4000 4000 4000 4000 4000 4000 4000]; param.max_data_rate = 700; param.flight_hours = 3.5; param.sys_delay = 0.75e-6; param.use_mcords4_names = true; param.arena = arena; + param.DDC_select = DDC_select_list(freq_idx); + param.max_duty_cycle = 0.12; + param.create_IQ = false; + param.tg.staged_recording = false; + param.tg.altitude_guard = 1000*12*2.54/100; + param.tg.Haltitude = Haltitude(Tpd_idx)*12*2.54/100; + param.tg.Hice_thick = 0; + param.prf = prf; + param.presums = [10 10 10 10 10 10 10 10 10]; + [param.wfs(1:8).atten] = deal(attenuation(Tpd_idx)-12); + [param.wfs(9:9).atten] = deal(attenuation(Tpd_idx)); + param.tx_weights = final_DDS_amp{cal_settings(freq_idx)}; + param.tukey = 0.08; + param.Tpd = Tpd; + for wf=1:9 + param.wfs(wf).phase = final_DDS_phase{cal_settings(freq_idx)}; + end + param.delay = final_DDS_time{cal_settings(freq_idx)}; + param.f0 = f0_list(freq_idx); + param.f1 = f1_list(freq_idx); + param.DDC_freq = (param.f0+param.f1)/2; + for wf=1:8 + param.wfs(wf).tx_mask = ones(1,8); + param.wfs(wf).tx_mask(9-wf) = 0; + end + for wf=9:9 + param.wfs(wf).tx_mask = [0 0 0 0 0 0 0 0]; + end + param.fn = fullfile(calval_dir,sprintf('txequal_%.0f-%.0fMHz_%.0fft_%.0fus_%s.xml',param.f0/1e6,param.f1/1e6,param.tg.Haltitude*100/12/2.54,param.Tpd*1e6,fn_hint{Tpd_idx})); + write_cresis_xml(param); + end +end diff --git a/cresis-toolbox/missions/create_configs_2022_Greenland_Polar5.m b/cresis-toolbox/missions/create_configs_2022_Greenland_Polar5.m new file mode 100644 index 00000000..c90d832b --- /dev/null +++ b/cresis-toolbox/missions/create_configs_2022_Greenland_Polar5.m @@ -0,0 +1,604 @@ +% script create_configs_2022_Greenland_Polar5 +% +% Creates NI and Arena radar depth sounder settings + +%% User Settings +% ========================================================================= + +% File locations +% ------------------------------------------------------------------------- +% NI_base_dir: Location where National Instruments configuration files will +% be written. +% arena_base_dir: Location where Arena configuration files will be written. +if ispc + NI_base_dir = 'C:\waveforms\'; + % Air 1 Server: C:\Users\Administrator\Desktop\Arena_Shared\configs\ + arena_base_dir = 'C:\Users\Administrator\Desktop\Arena_Shared\configs\'; + %arena_base_dir = 'C:\arena_waveforms\'; % Comment out on Air 1 Server +else + NI_base_dir = '~/waveforms/'; + arena_base_dir = '~/rss_waveforms/'; +end + +% NI_calval_dir: Location where National Instruments configuration files +% for calibration and validation settings will be written (usually a +% subdirectory "calval") in the NI_base_dir +NI_calval_dir = fullfile(NI_base_dir, 'calval'); +% ------------------------------------------------------------------------- +% End file locations + +% Specify a list of start/stop frequencies that you want to generate +% settings for. THIS OFTEN NEEDS TO BE CHANGED FOR A CAMPAIGN. +if 0 + f0_list = [180e6]; + f1_list = [210e6]; +else + f0_list = [180e6 150e6]; + f1_list = [210e6 520e6]; +end + +% Cal Settings +final_cal_fc = []; +final_DDS_phase = []; +final_DDS_phase_no_time = []; +final_DDS_amp = []; +final_DDS_time = []; +if 0 + % Initial conditions (usually all zeros phase/time with max amplitude) + for idx = 1:length(f0_list) + final_DDS_phase{idx} = [0 0 0 0 0 0 0 0]; + final_DDS_phase_no_time = [0 0 0 0 0 0 0 0]; % not used usually + final_DDS_amp{idx} = [4000 4000 4000 4000 4000 4000 4000 4000]; + final_DDS_time{idx} = [0 0 2.5 2.5 3.125 3.125 0 0]; + end +else + % COPY AND PASTE RESULTS FROM basic_tx_chan_equalization_SEASON_NAME.m + % HERE: + + % 150-520 MHz + final_cal_fc(end+1) = 335e6; + final_DDS_phase{end+1} = [127.0 151.7 -4.3 0.0 -13.2 21.7 161.8 149.6]; + final_DDS_phase_no_time{end+1} = [0 0 0 0 0 0 0 0]; % not used usually + final_DDS_amp{end+1} = [1312 2849 2657 3572 4000 2618 2574 1386]; + final_DDS_time{end+1} = [0.9209 1.0190 2.5389 2.5000 2.3667 2.6164 0.7276 0.4423]; + + % 180-210 MHz + final_cal_fc(end+1) = 195e6; + final_DDS_phase_no_time{end+1} = [0 0 0 0 0 0 0 0]; % not used usually + final_DDS_time{end+1} = [-0.13 -0.24 4.93 2.50 0.97 4.47 -0.09 -1.78]; + final_DDS_amp{end+1} = [942 2585 2958 4000 3480 3183 2196 1053]; + final_DDS_phase{end+1} = [65.0 62.2 164.2 0.0 -117.9 155.4 59.3 -65.0]; +end + +% prf: Scaler double with units of Hertz. Desired pulse repetition +% frequency. Should be 10000 Hz. +prf = 10000; + +% Hwindow_orig: Desired window created during transmit calibration. This is +% used any time a window that is different from that used during +% calibration is to be used. Should be chebwin(8,30).'. +Hwindow_orig = chebwin(8,30).'; + +% speed: scaler double in meters/second. Typical cruise speed of Basler is +% 90 m/s. Typical survey speed is 80 m/s. Set this speed to the maximum +% expected ground speed considering wind conditions. Using a setting of 100 +% m/s is generally sufficient for all surveys. +speed = [100]; +% presums: Positive scaler integer containing the number of hardware +% presums. Presums are also known as "stacking" and "coherent averaging". +% The radar only has one antenna arrayed in the along-track direction. This +% antenna has almost no directivity so the sample spacing (after +% presumming) should be one fourth of a wavelength to meet Nyquist criteria +% in all situations. We calculate the wavelength at the center frequency. +% Typically, this equation should not be modified: +% presums = round(c./abs(f1_list+f0_list)/2 ./ speed * prf / 2)*2 +physical_constants; % Loads "c" as speed of light +presums = round(c./abs(f1_list+f0_list)/2 ./ speed * prf / 2)*2 + +%% Load Antenna Positions/Lever Arms (do not change) +% ========================================================================= +param = []; +param.season_name = '2022_Greenland_Polar5'; % Any polar5/6 season works +param.radar_name = 'rds'; +param.gps_source = 'awi-final'; +clear phase_centers; +for tx_chan = 1:8 + % Just enable the current antenna to determine its phase center + tx_weights = zeros(1,8); + tx_weights(tx_chan) = 1; + rxchan = 4; % Fix the receiver (it should not matter which one you choose) + % Determine phase center for the antenna + phase_centers(:,tx_chan) = lever_arm(param, tx_weights, rxchan); +end +% Adjust phase centers to the mean phase center position +phase_centers = bsxfun(@minus,phase_centers,mean(phase_centers,2)); + +%% Load Arena Parameters (do not change) +% ========================================================================= +[param_defaults,defaults] = default_radar_params_2022_Greenland_Polar5_rds; +arena = defaults{1}.arena; + +%% Survey Mode + loopback, noise, and deconv modes +% thick ice, 1200 +/- 700 ft AGL +% ========================================================================= +ice_thickness = 3250; +for freq_idx = [] + param = struct('radar_name','mcords5','num_chan',8,'aux_dac',[255 255 255 255 255 255 255 255],'version','14.0f1','TTL_prog_delay',650,'xml_version',2.0,'fs',1600e6,'fs_sync',90.0e6,'fs_dds',1440e6,'TTL_clock',1440e6/16,'TTL_mode',[2.5e-6 260e-9 -1100e-9],'arena_base_dir',arena_base_dir); + param = merge_structs(param,param_defaults); + param.arena = arena; + param.max_tx = [4000 4000 4000 4000 4000 4000 4000 4000]; param.max_data_rate = 755; param.flight_hours = 3.5; param.sys_delay = 0.75e-6; param.use_mcords4_names = true; param.arena = arena; + BW = abs(f1_list(freq_idx)-f0_list(freq_idx)); + if BW <= 170e6 + param.DDC_select = 2; % 200 MHz sampling mode + elseif BW <= 370e6 + param.DDC_select = 1; % 400 MHz sampling mode + else + error('Bandwidth (%g MHz) is too large. Must be less than or equal to 370 MHz.', BW/1e6) + end + cal_used_idx = -1; + fc = abs(f0_list+f1_list)/2; + for cal_idx = 1:length(final_cal_fc) + if abs(final_cal_fc(cal_idx) - fc(freq_idx)) < 1e6 + cal_used_idx = cal_idx; + break; + end + end + if cal_used_idx == -1 + error('The center frequency %g MHz does not match any of the cal setting center frequencies specified in final_cal_fc. Either change the f0_list and f1_list or add a calibration setting with this center frequency.', fc/1e6); + end + param.max_duty_cycle = 0.12; + param.create_IQ = false; + param.tg.staged_recording = [1 2 3]; + param.tg.altitude_guard = 700*12*2.54/100; + param.tg.Haltitude = 1200*12*2.54/100; + param.tg.Hice_thick = ice_thickness; + param.tg.rg_stop_offset = [300 300 0]; + param.prf = prf; + param.presums = [4 4 presums(freq_idx)-8]; + param.wfs(1).atten = 37; + param.wfs(2).atten = 0; + param.wfs(3).atten = 0; + DDS_amp = final_DDS_amp{cal_used_idx}; + param.tx_weights = DDS_amp; + param.tukey = 0.08; + param.wfs(1).Tpd = 1e-6; + param.wfs(2).Tpd = 3e-6; + param.wfs(3).Tpd = 10e-6; + param.wfs(1).phase = final_DDS_phase{cal_used_idx}; + param.wfs(2).phase = final_DDS_phase{cal_used_idx}; + param.wfs(3).phase = final_DDS_phase{cal_used_idx}; + param.delay = final_DDS_time{cal_used_idx}; + param.f0 = f0_list(freq_idx); + param.f1 = f1_list(freq_idx); + param.DDC_freq = (param.f0+param.f1)/2; + [param.wfs(1:3).tx_mask] = deal([0 0 0 0 0 0 0 0]); + param.fn = fullfile(NI_base_dir,sprintf('survey_%.0f-%.0fMHz_%.0fft_%.0fus_%.0fmthick.xml',param.f0/1e6,param.f1/1e6,param.tg.Haltitude*100/12/2.54,param.wfs(end).Tpd*1e6,param.tg.Hice_thick)); + write_cresis_xml(param); + if freq_idx == 1 + % Default Mode + param.fn = fullfile(NI_base_dir,'default.xml'); + write_cresis_xml(param); + end + % Loopback Mode without delay line + param.tg.staged_recording = false; + param.tg.altitude_guard = 1000*12*2.54/100; + param.tg.Haltitude = 0e-6 * c/2; + param.tg.Hice_thick = 0; % Long enough for 10 us delay line + param.fn = fullfile(NI_calval_dir,sprintf('survey_%.0f-%.0fMHz_%.0fus_LOOPBACK_NO_DELAY.xml',param.f0/1e6,param.f1/1e6,param.wfs(end).Tpd*1e6)); + write_cresis_xml(param); + % Loopback Mode (10e-6 delay line) + param.tg.staged_recording = false; + param.tg.altitude_guard = 1000*12*2.54/100; + param.tg.Haltitude = 10e-6 * c/2; + param.tg.Hice_thick = 0; % Long enough for 10 us delay line + param.fn = fullfile(NI_calval_dir,sprintf('survey_%.0f-%.0fMHz_%.0fus_LOOPBACK.xml',param.f0/1e6,param.f1/1e6,param.wfs(end).Tpd*1e6)); + write_cresis_xml(param); + % Deconvolution Mode (for over calm lake or sea ice lead) + param.wfs(1).atten = 43; + param.wfs(2).atten = 43; + param.wfs(3).atten = 43; + param.tg.staged_recording = false; + param.tg.altitude_guard = 3000*12*2.54/100; + param.tg.Haltitude = 4000*12*2.54/100; + param.tg.Hice_thick = 0 * 12*2.54/100/sqrt(er_ice); + param.fn = fullfile(NI_calval_dir,sprintf('survey_%.0f-%.0fMHz_%.0fft_%.0fus_DECONV.xml',param.f0/1e6,param.f1/1e6,param.tg.Haltitude*100/12/2.54,param.wfs(end).Tpd*1e6)); + write_cresis_xml(param); + if freq_idx == 1 + % Noise Mode + param.tx_weights = [0 0 0 0 0 0 0 0]; + [param.wfs(1:3).tx_mask] = deal([1 1 1 1 1 1 1 1]); + param.wfs(1).atten = 37; + param.wfs(2).atten = 0; + param.wfs(3).atten = 0; + param.tg.staged_recording = [1 2 3]; + param.tg.altitude_guard = 500*12*2.54/100; + param.tg.Haltitude = 1400*12*2.54/100; + param.tg.Hice_thick = 3250; + param.fn = fullfile(NI_calval_dir,sprintf('survey_%.0f-%.0fMHz_%.0fus_NOISE.xml',param.f0/1e6,param.f1/1e6,param.wfs(end).Tpd*1e6)); + write_cresis_xml(param); + end +end + +%% Survey Mode + loopback, noise, and deconv modes +% thin ice, 1200 +/- 700 ft AGL +% ========================================================================= +ice_thickness = 1800; +for freq_idx = [2] + param = struct('radar_name','mcords5','num_chan',8,'aux_dac',[255 255 255 255 255 255 255 255],'version','14.0f1','TTL_prog_delay',650,'xml_version',2.0,'fs',1600e6,'fs_sync',90.0e6,'fs_dds',1440e6,'TTL_clock',1440e6/16,'TTL_mode',[2.5e-6 260e-9 -1100e-9],'arena_base_dir',arena_base_dir); + param = merge_structs(param,param_defaults); + param.arena = arena; + param.max_tx = [4000 4000 4000 4000 4000 4000 4000 4000]; param.max_data_rate = 755; param.flight_hours = 3.5; param.sys_delay = 0.75e-6; param.use_mcords4_names = true; param.arena = arena; + BW = abs(f1_list(freq_idx)-f0_list(freq_idx)); + if BW <= 170e6 + param.DDC_select = 2; % 200 MHz sampling mode + elseif BW <= 370e6 + param.DDC_select = 1; % 400 MHz sampling mode + else + error('Bandwidth (%g MHz) is too large. Must be less than or equal to 370 MHz.', BW/1e6) + end + cal_used_idx = -1; + fc = abs(f0_list+f1_list)/2; + for cal_idx = 1:length(final_cal_fc) + if abs(final_cal_fc(cal_idx) - fc(freq_idx)) < 1e6 + cal_used_idx = cal_idx; + break; + end + end + if cal_used_idx == -1 + error('The center frequency %g MHz does not match any of the cal setting center frequencies specified in final_cal_fc. Either change the f0_list and f1_list or add a calibration setting with this center frequency.', fc/1e6); + end + param.max_duty_cycle = 0.12; + param.create_IQ = false; + param.tg.staged_recording = [1 2]; + param.tg.altitude_guard = 700*12*2.54/100; + param.tg.Haltitude = 1200*12*2.54/100; + param.tg.Hice_thick = ice_thickness; + param.tg.rg_stop_offset = [300 300]; + param.prf = prf; + param.presums = [4 presums(freq_idx)-4]; + param.wfs(1).atten = 37; + param.wfs(2).atten = 0; + DDS_amp = final_DDS_amp{cal_used_idx}; + param.tx_weights = DDS_amp; + param.tukey = 0.08; + param.wfs(1).Tpd = 1e-6; + param.wfs(2).Tpd = 3e-6; + param.wfs(1).phase = final_DDS_phase{cal_used_idx}; + param.wfs(2).phase = final_DDS_phase{cal_used_idx}; + param.delay = final_DDS_time{cal_used_idx}; + param.f0 = f0_list(freq_idx); + param.f1 = f1_list(freq_idx); + param.DDC_freq = (param.f0+param.f1)/2; + [param.wfs(1:2).tx_mask] = deal([0 0 0 0 0 0 0 0]); + param.fn = fullfile(NI_base_dir,sprintf('thinsurvey_%.0f-%.0fMHz_%.0fft_%.0fus_%.0fmthick.xml',param.f0/1e6,param.f1/1e6,param.tg.Haltitude*100/12/2.54,param.wfs(end).Tpd*1e6,param.tg.Hice_thick)); + write_cresis_xml(param); + if freq_idx == 2 + % Default Mode + param.fn = fullfile(NI_base_dir,'default.xml'); + write_cresis_xml(param); + end + % Loopback Mode without delay line + param.tg.staged_recording = false; + param.tg.altitude_guard = 1000*12*2.54/100; + param.tg.Haltitude = 0e-6 * c/2; + param.tg.Hice_thick = 0; % Long enough for 10 us delay line + param.fn = fullfile(NI_calval_dir,sprintf('survey_%.0f-%.0fMHz_%.0fus_LOOPBACK_NO_DELAY.xml',param.f0/1e6,param.f1/1e6,param.wfs(end).Tpd*1e6)); + write_cresis_xml(param); + % Loopback Mode (10e-6 delay line) + param.tg.staged_recording = false; + param.tg.altitude_guard = 1000*12*2.54/100; + param.tg.Haltitude = 10e-6 * c/2; + param.tg.Hice_thick = 0; % Long enough for 10 us delay line + param.fn = fullfile(NI_calval_dir,sprintf('survey_%.0f-%.0fMHz_%.0fus_LOOPBACK.xml',param.f0/1e6,param.f1/1e6,param.wfs(end).Tpd*1e6)); + write_cresis_xml(param); + % Deconvolution Mode (for over calm lake or sea ice lead) + param.wfs(1).atten = 43; + param.wfs(2).atten = 43; + param.tg.staged_recording = false; + param.tg.altitude_guard = 3000*12*2.54/100; + param.tg.Haltitude = 4000*12*2.54/100; + param.tg.Hice_thick = 0 * 12*2.54/100/sqrt(er_ice); + param.fn = fullfile(NI_calval_dir,sprintf('survey_%.0f-%.0fMHz_%.0fft_%.0fus_DECONV.xml',param.f0/1e6,param.f1/1e6,param.tg.Haltitude*100/12/2.54,param.wfs(end).Tpd*1e6)); + write_cresis_xml(param); + if 1 + % Noise Mode + param.tx_weights = [0 0 0 0 0 0 0 0]; + [param.wfs(1:2).tx_mask] = deal([1 1 1 1 1 1 1 1]); + param.wfs(1).atten = 37; + param.wfs(2).atten = 0; + param.tg.staged_recording = [1 2]; + param.tg.altitude_guard = 500*12*2.54/100; + param.tg.Haltitude = 1400*12*2.54/100; + param.tg.Hice_thick = 3250; + param.fn = fullfile(NI_calval_dir,sprintf('survey_%.0f-%.0fMHz_%.0fus_NOISE.xml',param.f0/1e6,param.f1/1e6,param.wfs(end).Tpd*1e6)); + write_cresis_xml(param); + end +end + +%% Equalization (Using Ocean) +% Haltitude +/- 2000 ft +% For lower altitude, increase attenuation +% Use these settings over ocean or sea ice for fast-time equalization, +% transmit equalization, and receiver equalization. +% Creates one waveform for each of N DDS-transmitters plus a combined +% waveform with all transmitters going. +% ========================================================================= +Haltitude = [2500 2500 0 3500 6000]; +Tpd_list = [1e-6 1e-6 3e-6 3e-6 3e-6]; +attenuation = [62 45 43 61 55]; +fn_hint = {'WATER','ICE','NO_DELAY','WATER','WATER'}; +for Tpd_idx = 1:length(Tpd_list) + Tpd = Tpd_list(Tpd_idx); + for freq_idx = [2] + param = struct('radar_name','mcords5','num_chan',8,'aux_dac',[255 255 255 255 255 255 255 255],'version','14.0f1','TTL_prog_delay',650,'xml_version',2.0,'fs',1600e6,'fs_sync',90.0e6,'fs_dds',1440e6,'TTL_clock',1440e6/16,'TTL_mode',[2.5e-6 260e-9 -1100e-9],'arena_base_dir',arena_base_dir); + param = merge_structs(param,param_defaults); + param.arena = arena; + param.max_tx = [4000 4000 4000 4000 4000 4000 4000 4000]; param.max_data_rate = 700; param.flight_hours = 3.5; param.sys_delay = 0.75e-6; param.use_mcords4_names = true; param.arena = arena; + BW = abs(f1_list(freq_idx)-f0_list(freq_idx)); + if BW <= 170e6 + param.DDC_select = 2; % 200 MHz sampling mode + elseif BW <= 370e6 + param.DDC_select = 1; % 400 MHz sampling mode + else + error('Bandwidth (%g MHz) is too large. Must be less than or equal to 370 MHz.', BW/1e6) + end + cal_used_idx = -1; + fc = abs(f0_list+f1_list)/2; + for cal_idx = 1:length(final_cal_fc) + if abs(final_cal_fc(cal_idx) - fc(freq_idx)) < 1e6 + cal_used_idx = cal_idx; + break; + end + end + if cal_used_idx == -1 + error('The center frequency %g MHz does not match any of the cal setting center frequencies specified in final_cal_fc. Either change the f0_list and f1_list or add a calibration setting with this center frequency.', fc/1e6); + end + param.max_duty_cycle = 0.12; + param.create_IQ = false; + param.tg.staged_recording = false; + param.tg.altitude_guard = 2000*12*2.54/100; + param.tg.Haltitude = Haltitude(Tpd_idx)*12*2.54/100; + param.tg.Hice_thick = 0; + param.prf = prf; + param.presums = [10 10 10 10 10 10 10 10 10]; + [param.wfs(1:8).atten] = deal(attenuation(Tpd_idx)-12); + [param.wfs(9:9).atten] = deal(attenuation(Tpd_idx)); + param.tx_weights = final_DDS_amp{cal_used_idx}; + param.tukey = 0.08; + param.Tpd = Tpd; + for wf=1:9 + param.wfs(wf).phase = final_DDS_phase{cal_used_idx}; + end + param.delay = final_DDS_time{cal_used_idx}; + param.f0 = f0_list(freq_idx); + param.f1 = f1_list(freq_idx); + param.DDC_freq = (param.f0+param.f1)/2; + for wf=1:8 + param.wfs(wf).tx_mask = ones(1,8); + param.wfs(wf).tx_mask(9-wf) = 0; + end + for wf=9:9 + param.wfs(wf).tx_mask = [0 0 0 0 0 0 0 0]; + end + param.fn = fullfile(NI_calval_dir,sprintf('txequal_%.0f-%.0fMHz_%.0fft_%.0fft_%.0fus_%s.xml', ... + param.f0/1e6, param.f1/1e6, (param.tg.Haltitude+[-param.tg.altitude_guard param.tg.altitude_guard])*100/12/2.54, ... + param.Tpd*1e6,fn_hint{Tpd_idx})); + write_cresis_xml(param); + end +end + +%% Polarimetric Mode 3250m +% <3250 m thick ice, 1200 +/- 700 ft AGL +% ========================================================================= +ice_thickness = 3250; +polarimetric_f0_list = f0_list; +polarimetric_f1_list = f1_list; +for freq_idx = [] + param = struct('radar_name','mcords5','num_chan',8,'aux_dac',[255 255 255 255 255 255 255 255],'version','14.0f1','TTL_prog_delay',650,'xml_version',2.0,'fs',1600e6,'fs_sync',90.0e6,'fs_dds',1440e6,'TTL_clock',1440e6/16,'TTL_mode',[2.5e-6 260e-9 -1100e-9],'arena_base_dir',arena_base_dir); + param = merge_structs(param,param_defaults); + param.arena = arena; + param.max_tx = [4000 4000 4000 4000 4000 4000 4000 4000]; param.max_data_rate = 700; param.flight_hours = 3.5; param.sys_delay = 0.75e-6; param.use_mcords4_names = true; param.arena = arena; + BW = abs(polarimetric_f1_list(freq_idx)-polarimetric_f0_list(freq_idx)); + if BW <= 170e6 + param.DDC_select = 2; % 200 MHz sampling mode + elseif BW <= 370e6 + param.DDC_select = 1; % 400 MHz sampling mode + else + error('Bandwidth (%g MHz) is too large. Must be less than or equal to 370 MHz.', BW/1e6) + end + cal_used_idx = -1; + fc = abs(polarimetric_f0_list+polarimetric_f1_list)/2; + for cal_idx = 1:length(final_cal_fc) + if abs(final_cal_fc(cal_idx) - fc(freq_idx)) < 1e6 + cal_used_idx = cal_idx; + break; + end + end + if cal_used_idx == -1 + error('The center frequency %g MHz does not match any of the cal setting center frequencies specified in final_cal_fc. Either change the f0_list and f1_list or add a calibration setting with this center frequency.', fc/1e6); + end + param.max_duty_cycle = 0.12; + param.create_IQ = false; + param.tg.staged_recording = [1 1 2 2 3 3]; + param.tg.altitude_guard = 700*12*2.54/100; + param.tg.Haltitude = 1200*12*2.54/100; + param.tg.Hice_thick = ice_thickness; + param.prf = prf; + param.presums = [2 4 2 4 16 16]; + param.wfs(1).atten = 37; + param.wfs(2).atten = 37; + param.wfs(3).atten = 0; + param.wfs(4).atten = 0; + param.wfs(5).atten = 0; + param.wfs(6).atten = 0; + param.tx_weights = [4000 4000 4000 4000 4000 4000 4000 4000]; + param.tukey = 0.08; + param.wfs(1).Tpd = 1e-6; + param.wfs(2).Tpd = 1e-6; + param.wfs(3).Tpd = 3e-6; + param.wfs(4).Tpd = 3e-6; + param.wfs(5).Tpd = 10e-6; + param.wfs(6).Tpd = 10e-6; + param.wfs(1).phase = final_DDS_phase{cal_used_idx}; + param.wfs(2).phase = final_DDS_phase{cal_used_idx}; + param.wfs(3).phase = final_DDS_phase{cal_used_idx}; + param.wfs(4).phase = final_DDS_phase{cal_used_idx}; + param.wfs(5).phase = final_DDS_phase{cal_used_idx}; + param.wfs(6).phase = final_DDS_phase{cal_used_idx}; + param.delay = final_DDS_time{cal_used_idx}; + param.f0 = polarimetric_f0_list(freq_idx); + param.f1 = polarimetric_f1_list(freq_idx); + param.DDC_freq = (param.f0+param.f1)/2; + [param.wfs([1 3 5]).tx_mask] = deal([1 1 0 0 0 0 1 1]); + [param.wfs([2 4 6]).tx_mask] = deal([0 0 1 1 1 1 0 0]); + param.tg.rg_stop_offset = [300 300 300 300 0 0]; + param.fn = fullfile(NI_base_dir,sprintf('polarimetric_%.0f-%.0fMHz_%.0fft_%.0fus_%.0fmthick.xml',param.f0/1e6,param.f1/1e6,param.tg.Haltitude*100/12/2.54,param.wfs(end).Tpd*1e6,param.tg.Hice_thick)); + write_cresis_xml(param); +end + +%% Survey Mode 3250m with Polarimetric Setup +% <3250 m thick ice, 1200 +/- 700 ft AGL +% ========================================================================= +ice_thickness = 3250; +for freq_idx = [] + param = struct('radar_name','mcords5','num_chan',8,'aux_dac',[255 255 255 255 255 255 255 255],'version','14.0f1','TTL_prog_delay',650,'xml_version',2.0,'fs',1600e6,'fs_sync',90.0e6,'fs_dds',1440e6,'TTL_clock',1440e6/16,'TTL_mode',[2.5e-6 260e-9 -1100e-9],'arena_base_dir',arena_base_dir); + param = merge_structs(param,param_defaults); + param.arena = arena; + param.max_tx = [4000 4000 4000 4000 4000 4000 4000 4000]; param.max_data_rate = 700; param.flight_hours = 3.5; param.sys_delay = 0.75e-6; param.use_mcords4_names = true; param.arena = arena; + BW = abs(f1_list(freq_idx)-f0_list(freq_idx)); + if BW <= 170e6 + param.DDC_select = 2; % 200 MHz sampling mode + elseif BW <= 370e6 + param.DDC_select = 1; % 400 MHz sampling mode + else + error('Bandwidth (%g MHz) is too large. Must be less than or equal to 370 MHz.', BW/1e6) + end + cal_used_idx = -1; + fc = abs(f0_list+f1_list)/2; + for cal_idx = 1:length(final_cal_fc) + if abs(final_cal_fc(cal_idx) - fc(freq_idx)) < 1e6 + cal_used_idx = cal_idx; + break; + end + end + if cal_used_idx == -1 + error('The center frequency %g MHz does not match any of the cal setting center frequencies specified in final_cal_fc. Either change the f0_list and f1_list or add a calibration setting with this center frequency.', fc/1e6); + end + param.max_duty_cycle = 0.12; + param.create_IQ = false; + param.tg.staged_recording = [1 2 3]; + param.tg.altitude_guard = 700*12*2.54/100; + param.tg.Haltitude = 1200*12*2.54/100; + param.tg.Hice_thick = ice_thickness; + param.prf = prf; + param.presums = [4 4 34]; + param.wfs(1).atten = 37; + param.wfs(2).atten = 0; + param.wfs(3).atten = 0; + param.tx_weights = [4000 4000 4000 4000 4000 4000 4000 4000]; + param.tukey = 0.08; + param.wfs(1).Tpd = 1e-6; + param.wfs(2).Tpd = 3e-6; + param.wfs(3).Tpd = 10e-6; + param.wfs(1).phase = final_DDS_phase{cal_used_idx}; + param.wfs(2).phase = final_DDS_phase{cal_used_idx}; + param.wfs(3).phase = final_DDS_phase{cal_used_idx}; + param.delay = final_DDS_time{cal_used_idx}; + param.f0 = f0_list(freq_idx); + param.f1 = f1_list(freq_idx); + param.DDC_freq = (param.f0+param.f1)/2; + [param.wfs([1 2 3]).tx_mask] = deal([1 1 0 0 0 0 1 1]); + param.tg.rg_stop_offset = [300 300 0]; + param.fn = fullfile(NI_base_dir,sprintf('sur_pol_%.0f-%.0fMHz_%.0fft_%.0fus_%.0fmthick.xml',param.f0/1e6,param.f1/1e6,param.tg.Haltitude*100/12/2.54,param.wfs(end).Tpd*1e6,param.tg.Hice_thick)); + write_cresis_xml(param); +end + +%% Image Mode (EGRIP Low Altitude, Thick Ice) +% Ice thickness "param.tg.Hice_thick_min" m to "param.tg.Hice_thick" m, "param.tg.Haltitude" +/- "param.tg.altitude_guard" ft AGL +% ========================================================================= +for freq_idx = [] + param = struct('radar_name','mcords5','num_chan',8,'aux_dac',[255 255 255 255 255 255 255 255],'version','14.0f1','TTL_prog_delay',650,'xml_version',2.0,'fs',1600e6,'fs_sync',90.0e6,'fs_dds',1440e6,'TTL_clock',1440e6/16,'TTL_mode',[2.5e-6 260e-9 -1100e-9],'arena_base_dir',arena_base_dir); + param = merge_structs(param,param_defaults); + param.arena = arena; + param.max_tx = [4000 4000 4000 4000 4000 4000 4000 4000]; param.max_data_rate = 700; param.flight_hours = 3.5; param.sys_delay = 0.75e-6; param.use_mcords4_names = true; param.arena = arena; + BW = abs(f1_list(freq_idx)-f0_list(freq_idx)); + if BW <= 170e6 + param.DDC_select = 2; % 200 MHz sampling mode + elseif BW <= 370e6 + param.DDC_select = 1; % 400 MHz sampling mode + else + error('Bandwidth (%g MHz) is too large. Must be less than or equal to 370 MHz.', BW/1e6) + end + cal_used_idx = -1; + fc = abs(f0_list+f1_list)/2; + for cal_idx = 1:length(final_cal_fc) + if abs(final_cal_fc(cal_idx) - fc(freq_idx)) < 1e6 + cal_used_idx = cal_idx; + break; + end + end + if cal_used_idx == -1 + error('The center frequency %g MHz does not match any of the cal setting center frequencies specified in final_cal_fc. Either change the f0_list and f1_list or add a calibration setting with this center frequency.', fc/1e6); + end + param.max_duty_cycle = 0.12; + param.create_IQ = false; + param.tg.staged_recording = [1 2 3 3]; + param.tg.altitude_guard = 700 * 12*2.54/100; + param.tg.Haltitude = 1200 * 12*2.54/100; + param.tg.rg_stop_offset = [300 300 0 0]; + param.tg.Hice_thick = 3250; + param.tg.look_angle_deg = [0 0 40 40]; + param.prf = prf; + param.presums = [4 4 ceil((presums(freq_idx)-6)/4)*2 ceil((presums(freq_idx)-6)/4)*2]; + % Switch from tx calibration window to hanning window to broaden beam + DDS_amp = final_DDS_amp{cal_used_idx} .* hanning(8).' ./ Hwindow_orig; + % Renormalize the amplitudes + [~,relative_max_idx] = max(DDS_amp./param.max_tx); + DDS_amp = round(DDS_amp .* param.max_tx(relative_max_idx) / DDS_amp(relative_max_idx)); + param.tx_weights = DDS_amp; + param.tukey = 0.08; + param.wfs(1).Tpd = 1e-6; + param.wfs(2).Tpd = 3e-6; + param.wfs(3).Tpd = 10e-6; + param.wfs(4).Tpd = 10e-6; + param.wfs(1).phase = final_DDS_phase{cal_used_idx}; + param.wfs(2).phase = final_DDS_phase{cal_used_idx}; + param.wfs(3).phase = final_DDS_phase{cal_used_idx}; + param.wfs(4).phase = final_DDS_phase{cal_used_idx}; + param.wfs(1).name = 'nadir1'; + param.wfs(2).name = 'nadir3'; + param.wfs(3).name = 'left'; + param.wfs(4).name = 'right'; + % Loop through each waveform and adjust time delays to match the + % desired beam angle for that waveform. Note that the delays that need to + % be added in are the delay RELATIVE to a nadir beam since final_DDS_time + % contains the equalized delays for a nadir beam. + % param.tg.look_angle (negative is to the left, positive is to the right) + param.tg.look_angle = [0 0 -20 20]; + for wf = 1:length(param.tg.look_angle) + nadir_vec = [0 0 1]; + beam_vec = [0 sind(param.tg.look_angle(wf)) cosd(param.tg.look_angle(wf))]; + nadir_delay = -nadir_vec * phase_centers; + beam_delay = -beam_vec * phase_centers; + % Only need relative phases, so adjust by a constant + nadir_delay = nadir_delay - nadir_delay(4); + beam_delay = beam_delay - beam_delay(4); + % Convert from range to one-way delay + nadir_delay = nadir_delay / c; + beam_delay = beam_delay / c - nadir_delay; % Only include delay relative to nadir + beam_delay = beam_delay - mean(beam_delay); + % param.wfs(wf).delay and final_DDS_time{cal_used_idx} are in units of ns + param.wfs(wf).delay = (final_DDS_time{cal_used_idx} - beam_delay*1e9); + end + param.f0 = f0_list(freq_idx); + param.f1 = f1_list(freq_idx); + param.DDC_freq = (param.f0+param.f1)/2; + [param.wfs(1:4).tx_mask] = deal([0 0 0 0 0 0 0 0]); + param.wfs(1).atten = 37; + param.wfs(2).atten = 0; + param.wfs(3).atten = 0; + param.wfs(4).atten = 0; + param.fn = fullfile(NI_base_dir,sprintf('egrip_image_%.0f-%.0fMHz_%.0fft_%.0fus_%.0fmthick.xml', ... + param.f0/1e6,param.f1/1e6,param.tg.Haltitude*100/2.54/12,param.wfs(end).Tpd*1e6,param.tg.Hice_thick)); + write_cresis_xml(param); +end diff --git a/cresis-toolbox/missions/create_configs_2023_Canada_Polar5.m b/cresis-toolbox/missions/create_configs_2023_Canada_Polar5.m new file mode 100644 index 00000000..f0ad4326 --- /dev/null +++ b/cresis-toolbox/missions/create_configs_2023_Canada_Polar5.m @@ -0,0 +1,614 @@ +% script create_configs_2022_Greenland_Polar5 +% +% Creates NI and Arena radar depth sounder settings + +%% User Settings +% ========================================================================= +% freq_idx [1] narrow band +% freq_idx [2] broadband +% freq_idx [] skip +% File locations +% ------------------------------------------------------------------------- +% NI_base_dir: Location where National Instruments configuration files will +% be written. +% arena_base_dir: Location where Arena configuration files will be written. +if ispc + NI_base_dir = 'C:\waveforms\'; +% NI_base_dir = 'C:\Users\uwb\Desktop\waveforms\2023\'; % for new AIR1 server + % Air 1 Server: C:\Users\Administrator\Desktop\Arena_Shared\configs\ + %arena_base_dir = 'C:\Users\Administrator\Desktop\Arena_Shared\configs\'; +% arena_base_dir = 'C:\Users\uwb\Desktop\Arena_Shared\configs\2023\'; % for new AIR1 server + arena_base_dir = 'C:\arena_waveforms\'; % Comment out on Air 1 Server +else + NI_base_dir = '~/waveforms/'; + arena_base_dir = '~/rss_waveforms/'; +end + +% NI_calval_dir: Location where National Instruments configuration files +% for calibration and validation settings will be written (usually a +% subdirectory "calval") in the NI_base_dir +NI_calval_dir = fullfile(NI_base_dir, 'calval'); +% ------------------------------------------------------------------------- +% End file locations + +% Specify a list of start/stop frequencies that you want to generate +% settings for. THIS OFTEN NEEDS TO BE CHANGED FOR A CAMPAIGN. +if 0 + f0_list = [180e6]; + f1_list = [210e6]; +else + f0_list = [180e6 150e6]; + f1_list = [210e6 520e6]; +end + +% Cal Settings +final_cal_fc = []; +final_DDS_phase = []; +final_DDS_phase_no_time = []; +final_DDS_amp = []; +final_DDS_time = []; +if 0 + % Initial conditions (usually all zeros phase/time with max amplitude) + for idx = 1:length(f0_list) + final_DDS_phase{idx} = [0 0 0 0 0 0 0 0]; + final_DDS_phase_no_time = [0 0 0 0 0 0 0 0]; % not used usually + final_DDS_amp{idx} = [4000 4000 4000 4000 4000 4000 4000 4000]; + final_DDS_time{idx} = [0 0 2.5 2.5 3.125 3.125 0 0]; + end +else + % COPY AND PASTE RESULTS FROM basic_tx_chan_equalization_SEASON_NAME.m + % HERE: + + % 150-520 MHz + final_cal_fc(end+1) = 335e6; + final_DDS_phase{end+1} = [-138.4 145.2 -33.0 0.0 -163.3 -82.7 111.3 -85.6]; + final_DDS_phase_no_time{end+1} = [0 0 0 0 0 0 0 0]; % not used usually + final_DDS_amp{end+1} = [1426 3350 2957 3952 4000 3548 2615 273]; + final_DDS_time{end+1} = [-4.19 -5.07 -3.85 -3.43 -5.39 -4.53 -5.31 -4.64]; + + % 180-210 MHz + final_cal_fc(end+1) = 195e6; + final_DDS_phase_no_time{end+1} = [0 0 0 0 0 0 0 0]; % not used usually + final_DDS_time{end+1} = [-7.35 -5.57 -2.39 -3.43 0.97 -0.75 -5.91 -4.64]; + final_DDS_amp{end+1} = [942 2585 2958 4000 3480 3183 2196 1053]; + final_DDS_phase{end+1} = [65.0 62.2 164.2 0.0 -117.9 155.4 59.3 -65.0]; +end + +% prf: Scaler double with units of Hertz. Desired pulse repetition +% frequency. Should be 10000 Hz. +prf = 10000; + +% Hwindow_orig: Desired window created during transmit calibration. This is +% used any time a window that is different from that used during +% calibration is to be used. Should be chebwin(8,30).'. +Hwindow_orig = chebwin(8,30).'; + +% speed: scaler double in meters/second. Typical cruise speed of Basler is +% 90 m/s. Typical survey speed is 80 m/s. Set this speed to the maximum +% expected ground speed considering wind conditions. Using a setting of 100 +% m/s is generally sufficient for all surveys. +speed = [100]; +% presums: Positive scaler integer containing the number of hardware +% presums. Presums are also known as "stacking" and "coherent averaging". +% The radar only has one antenna arrayed in the along-track direction. This +% antenna has almost no directivity so the sample spacing (after +% presumming) should be one fourth of a wavelength to meet Nyquist criteria +% in all situations. We calculate the wavelength at the center frequency. +% Typically, this equation should not be modified: +% presums = round(c./abs(f1_list+f0_list)/2 ./ speed * prf / 2)*2 +physical_constants; % Loads "c" as speed of light +presums = round(c./abs(f1_list+f0_list)/2 ./ speed * prf / 2)*2 + +%% Load Antenna Positions/Lever Arms (do not change) +% ========================================================================= +param = []; +param.season_name = '2022_Greenland_Polar5'; % Any polar5/6 season works +param.radar_name = 'rds'; +param.gps_source = 'awi-final'; +clear phase_centers; +for tx_chan = 1:8 + % Just enable the current antenna to determine its phase center + tx_weights = zeros(1,8); + tx_weights(tx_chan) = 1; + rxchan = 4; % Fix the receiver (it should not matter which one you choose) + % Determine phase center for the antenna + phase_centers(:,tx_chan) = lever_arm(param, tx_weights, rxchan); +end +% Adjust phase centers to the mean phase center position +phase_centers = bsxfun(@minus,phase_centers,mean(phase_centers,2)); + +%% Load Arena Parameters (do not change) +% ========================================================================= +[param_defaults,defaults] = default_radar_params_2022_Greenland_Polar5_rds; +arena = defaults{1}.arena; + +%% Survey Mode + loopback, noise, and deconv modes +% thick ice, 1200 +/- 700 ft AGL +% ========================================================================= +ice_thickness = 3250; +for freq_idx = [1] + param = struct('radar_name','mcords5','num_chan',8,'aux_dac',[255 255 255 255 255 255 255 255],'version','14.0f1','TTL_prog_delay',650,'xml_version',2.0,'fs',1600e6,'fs_sync',90.0e6,'fs_dds',1440e6,'TTL_clock',1440e6/16,'TTL_mode',[2.5e-6 260e-9 -1100e-9],'arena_base_dir',arena_base_dir); + param = merge_structs(param,param_defaults); + param.arena = arena; + param.max_tx = [4000 4000 4000 4000 4000 4000 4000 4000]; param.max_data_rate = 755; param.flight_hours = 3.5; param.sys_delay = 0.75e-6; param.use_mcords4_names = true; param.arena = arena; + BW = abs(f1_list(freq_idx)-f0_list(freq_idx)); + if BW <= 170e6 + param.DDC_select = 2; % 200 MHz sampling mode + elseif BW <= 370e6 + param.DDC_select = 1; % 400 MHz sampling mode + else + error('Bandwidth (%g MHz) is too large. Must be less than or equal to 370 MHz.', BW/1e6) + end + cal_used_idx = -1; + fc = abs(f0_list+f1_list)/2; + for cal_idx = 1:length(final_cal_fc) + if abs(final_cal_fc(cal_idx) - fc(freq_idx)) < 1e6 + cal_used_idx = cal_idx; + break; + end + end + if cal_used_idx == -1 + error('The center frequency %g MHz does not match any of the cal setting center frequencies specified in final_cal_fc. Either change the f0_list and f1_list or add a calibration setting with this center frequency.', fc/1e6); + end + param.max_duty_cycle = 0.12; + param.create_IQ = false; + param.tg.staged_recording = [1 2 3]; + param.tg.altitude_guard = 700*12*2.54/100; + param.tg.Haltitude = 1200*12*2.54/100; + param.tg.Hice_thick = ice_thickness; + param.tg.rg_stop_offset = [300 300 0]; + param.prf = prf; + param.presums = [4 4 presums(freq_idx)-8]; + param.wfs(1).atten = 37; + param.wfs(2).atten = 0; + param.wfs(3).atten = 0; + DDS_amp = final_DDS_amp{cal_used_idx}; + param.tx_weights = DDS_amp; + param.tukey = 0.08; + param.wfs(1).Tpd = 1e-6; + param.wfs(2).Tpd = 3e-6; + param.wfs(3).Tpd = 10e-6; + param.wfs(1).phase = final_DDS_phase{cal_used_idx}; + param.wfs(2).phase = final_DDS_phase{cal_used_idx}; + param.wfs(3).phase = final_DDS_phase{cal_used_idx}; + param.delay = final_DDS_time{cal_used_idx}; + param.f0 = f0_list(freq_idx); + param.f1 = f1_list(freq_idx); + param.DDC_freq = (param.f0+param.f1)/2; + [param.wfs(1:3).tx_mask] = deal([0 0 0 0 0 0 0 0]); + param.fn = fullfile(NI_base_dir,sprintf('survey_%.0f-%.0fMHz_%.0fft_%.0fus_%.0fmthick.xml',param.f0/1e6,param.f1/1e6,param.tg.Haltitude*100/12/2.54,param.wfs(end).Tpd*1e6,param.tg.Hice_thick)); + write_cresis_xml(param); + if freq_idx == 1 + % Default Mode + param.fn = fullfile(NI_base_dir,'default.xml'); + write_cresis_xml(param); + end + % Loopback Mode without delay line + param.tg.staged_recording = false; + param.tg.altitude_guard = 1000*12*2.54/100; + param.tg.Haltitude = 0e-6 * c/2; + param.tg.Hice_thick = 0; % Long enough for 10 us delay line + param.fn = fullfile(NI_calval_dir,sprintf('survey_%.0f-%.0fMHz_%.0fus_LOOPBACK_NO_DELAY.xml',param.f0/1e6,param.f1/1e6,param.wfs(end).Tpd*1e6)); + write_cresis_xml(param); + % Loopback Mode (10e-6 delay line) + param.tg.staged_recording = false; + param.tg.altitude_guard = 1000*12*2.54/100; + param.tg.Haltitude = 10e-6 * c/2; + param.tg.Hice_thick = 0; % Long enough for 10 us delay line + param.fn = fullfile(NI_calval_dir,sprintf('survey_%.0f-%.0fMHz_%.0fus_LOOPBACK.xml',param.f0/1e6,param.f1/1e6,param.wfs(end).Tpd*1e6)); + write_cresis_xml(param); + % Deconvolution Mode (for over calm lake or sea ice lead) + param.wfs(1).atten = 43; + param.wfs(2).atten = 43; + param.wfs(3).atten = 43; + param.tg.staged_recording = false; + param.tg.altitude_guard = 3000*12*2.54/100; + param.tg.Haltitude = 4000*12*2.54/100; + param.tg.Hice_thick = 0 * 12*2.54/100/sqrt(er_ice); + param.fn = fullfile(NI_calval_dir,sprintf('survey_%.0f-%.0fMHz_%.0fft_%.0fus_DECONV.xml',param.f0/1e6,param.f1/1e6,param.tg.Haltitude*100/12/2.54,param.wfs(end).Tpd*1e6)); + write_cresis_xml(param); + if freq_idx == 1 + % Noise Mode + param.tx_weights = [0 0 0 0 0 0 0 0]; + [param.wfs(1:3).tx_mask] = deal([1 1 1 1 1 1 1 1]); + param.wfs(1).atten = 37; + param.wfs(2).atten = 0; + param.wfs(3).atten = 0; + param.tg.staged_recording = [1 2 3]; + param.tg.altitude_guard = 500*12*2.54/100; + param.tg.Haltitude = 1400*12*2.54/100; + param.tg.Hice_thick = 3250; + param.fn = fullfile(NI_calval_dir,sprintf('survey_%.0f-%.0fMHz_%.0fus_NOISE.xml',param.f0/1e6,param.f1/1e6,param.wfs(end).Tpd*1e6)); + write_cresis_xml(param); + end +end + +%% Survey Mode + loopback, noise, and deconv modes +% thin ice, 1200 +/- 700 ft AGL +% ========================================================================= +ice_thickness = 1800; +for freq_idx = [2] + param = struct('radar_name','mcords5','num_chan',8,'aux_dac',[255 255 255 255 255 255 255 255],'version','14.0f1','TTL_prog_delay',650,'xml_version',2.0,'fs',1600e6,'fs_sync',90.0e6,'fs_dds',1440e6,'TTL_clock',1440e6/16,'TTL_mode',[2.5e-6 260e-9 -1100e-9],'arena_base_dir',arena_base_dir); + param = merge_structs(param,param_defaults); + param.arena = arena; + param.max_tx = [4000 4000 4000 4000 4000 4000 4000 4000]; param.max_data_rate = 755; param.flight_hours = 3.5; param.sys_delay = 0.75e-6; param.use_mcords4_names = true; param.arena = arena; + BW = abs(f1_list(freq_idx)-f0_list(freq_idx)); + if BW <= 170e6 + param.DDC_select = 2; % 200 MHz sampling mode + elseif BW <= 370e6 + param.DDC_select = 1; % 400 MHz sampling mode + else + error('Bandwidth (%g MHz) is too large. Must be less than or equal to 370 MHz.', BW/1e6) + end + cal_used_idx = -1; + fc = abs(f0_list+f1_list)/2; + for cal_idx = 1:length(final_cal_fc) + if abs(final_cal_fc(cal_idx) - fc(freq_idx)) < 1e6 + cal_used_idx = cal_idx; + break; + end + end + if cal_used_idx == -1 + error('The center frequency %g MHz does not match any of the cal setting center frequencies specified in final_cal_fc. Either change the f0_list and f1_list or add a calibration setting with this center frequency.', fc/1e6); + end + param.max_duty_cycle = 0.12; + param.create_IQ = false; + param.tg.staged_recording = [1 2 3]; + param.tg.altitude_guard = 700*12*2.54/100; + param.tg.Haltitude = 1200*12*2.54/100; + param.tg.Hice_thick = ice_thickness; + param.tg.rg_start_offset = [-100 -100 -100]; % Ensure good overlap between waveforms + param.tg.rg_stop_offset = [300 300 300]; % Ensure good overlap between waveforms + param.prf = prf; + param.presums = [8 2 presums(freq_idx)-10]; + param.wfs(1).atten = 35; + param.wfs(2).atten = 0; + param.wfs(3).atten = 0; + DDS_amp = final_DDS_amp{cal_used_idx}; + param.tx_weights = DDS_amp; + param.tukey = 0.08; + param.wfs(1).Tpd = 1e-6; + param.wfs(2).Tpd = 1e-6; + param.wfs(3).Tpd = 3e-6; + param.wfs(1).phase = final_DDS_phase{cal_used_idx}; + param.wfs(2).phase = final_DDS_phase{cal_used_idx}; + param.wfs(3).phase = final_DDS_phase{cal_used_idx}; + param.delay = final_DDS_time{cal_used_idx}; + param.f0 = f0_list(freq_idx); + param.f1 = f1_list(freq_idx); + param.DDC_freq = (param.f0+param.f1)/2; + [param.wfs(1:3).tx_mask] = deal([0 0 0 0 0 0 0 0]); + param.fn = fullfile(NI_base_dir,sprintf('thinsurvey_%.0f-%.0fMHz_%.0fft_%.0fus_%.0fmthick.xml',param.f0/1e6,param.f1/1e6,param.tg.Haltitude*100/12/2.54,param.wfs(end).Tpd*1e6,param.tg.Hice_thick)); + write_cresis_xml(param); + if freq_idx == 2 + % Default Mode + param.fn = fullfile(NI_base_dir,'default.xml'); + write_cresis_xml(param); + end + % Loopback Mode without delay line + param.tg.staged_recording = false; + param.tg.altitude_guard = 1000*12*2.54/100; + param.tg.Haltitude = 0e-6 * c/2; + param.tg.Hice_thick = 0; % Long enough for 10 us delay line + param.fn = fullfile(NI_calval_dir,sprintf('survey_%.0f-%.0fMHz_%.0fus_LOOPBACK_NO_DELAY.xml',param.f0/1e6,param.f1/1e6,param.wfs(end).Tpd*1e6)); + write_cresis_xml(param); + % Loopback Mode (10e-6 delay line) + param.tg.staged_recording = false; + param.tg.altitude_guard = 1000*12*2.54/100; + param.tg.Haltitude = 10e-6 * c/2; + param.tg.Hice_thick = 0; % Long enough for 10 us delay line + param.fn = fullfile(NI_calval_dir,sprintf('survey_%.0f-%.0fMHz_%.0fus_LOOPBACK.xml',param.f0/1e6,param.f1/1e6,param.wfs(end).Tpd*1e6)); + write_cresis_xml(param); + % Deconvolution Mode (for over calm lake or sea ice lead) + param.wfs(1).atten = 43; + param.wfs(2).atten = 43; + param.wfs(3).atten = 43; + param.tg.staged_recording = false; + param.tg.altitude_guard = 3000*12*2.54/100; + param.tg.Haltitude = 4000*12*2.54/100; + param.tg.Hice_thick = 0 * 12*2.54/100/sqrt(er_ice); + param.fn = fullfile(NI_calval_dir,sprintf('survey_%.0f-%.0fMHz_%.0fft_%.0fus_DECONV.xml',param.f0/1e6,param.f1/1e6,param.tg.Haltitude*100/12/2.54,param.wfs(end).Tpd*1e6)); + write_cresis_xml(param); + if 1 + % Noise Mode + param.tx_weights = [0 0 0 0 0 0 0 0]; + [param.wfs(1:3).tx_mask] = deal([1 1 1 1 1 1 1 1]); + param.wfs(1).atten = 35; + param.wfs(2).atten = 0; + param.wfs(3).atten = 0; + param.tg.staged_recording = [1 2 3]; + param.tg.altitude_guard = 500*12*2.54/100; + param.tg.Haltitude = 1400*12*2.54/100; + param.tg.Hice_thick = 3250; + param.fn = fullfile(NI_calval_dir,sprintf('survey_%.0f-%.0fMHz_%.0fus_NOISE.xml',param.f0/1e6,param.f1/1e6,param.wfs(end).Tpd*1e6)); + write_cresis_xml(param); + end +end + +%% Equalization (Using Ocean) +% Haltitude +/- 2000 ft +% For lower altitude, increase attenuation +% Use these settings over ocean or sea ice for fast-time equalization, +% transmit equalization, and receiver equalization. +% Creates one waveform for each of N DDS-transmitters plus a combined +% waveform with all transmitters going. +% ========================================================================= +Haltitude = [2500 2500 0 3500 6000]; +Tpd_list = [1e-6 1e-6 3e-6 3e-6 3e-6]; +attenuation = [62 45 43 61 55]; +fn_hint = {'WATER','ICE','NO_DELAY','WATER','WATER'}; +for Tpd_idx = 1:length(Tpd_list) + Tpd = Tpd_list(Tpd_idx); + for freq_idx = [2] + param = struct('radar_name','mcords5','num_chan',8,'aux_dac',[255 255 255 255 255 255 255 255],'version','14.0f1','TTL_prog_delay',650,'xml_version',2.0,'fs',1600e6,'fs_sync',90.0e6,'fs_dds',1440e6,'TTL_clock',1440e6/16,'TTL_mode',[2.5e-6 260e-9 -1100e-9],'arena_base_dir',arena_base_dir); + param = merge_structs(param,param_defaults); + param.arena = arena; + param.max_tx = [4000 4000 4000 4000 4000 4000 4000 4000]; param.max_data_rate = 700; param.flight_hours = 3.5; param.sys_delay = 0.75e-6; param.use_mcords4_names = true; param.arena = arena; + BW = abs(f1_list(freq_idx)-f0_list(freq_idx)); + if BW <= 170e6 + param.DDC_select = 2; % 200 MHz sampling mode + elseif BW <= 370e6 + param.DDC_select = 1; % 400 MHz sampling mode + else + error('Bandwidth (%g MHz) is too large. Must be less than or equal to 370 MHz.', BW/1e6) + end + cal_used_idx = -1; + fc = abs(f0_list+f1_list)/2; + for cal_idx = 1:length(final_cal_fc) + if abs(final_cal_fc(cal_idx) - fc(freq_idx)) < 1e6 + cal_used_idx = cal_idx; + break; + end + end + if cal_used_idx == -1 + error('The center frequency %g MHz does not match any of the cal setting center frequencies specified in final_cal_fc. Either change the f0_list and f1_list or add a calibration setting with this center frequency.', fc/1e6); + end + param.max_duty_cycle = 0.12; + param.create_IQ = false; + param.tg.staged_recording = false; + param.tg.altitude_guard = 2000*12*2.54/100; + param.tg.Haltitude = Haltitude(Tpd_idx)*12*2.54/100; + param.tg.Hice_thick = 0; + param.prf = prf; + param.presums = [10 10 10 10 10 10 10 10 10]; + [param.wfs(1:8).atten] = deal(attenuation(Tpd_idx)-12); + [param.wfs(9:9).atten] = deal(attenuation(Tpd_idx)); + param.tx_weights = final_DDS_amp{cal_used_idx}; + param.tukey = 0.08; + param.Tpd = Tpd; + for wf=1:9 + param.wfs(wf).phase = final_DDS_phase{cal_used_idx}; + end + param.delay = final_DDS_time{cal_used_idx}; + param.f0 = f0_list(freq_idx); + param.f1 = f1_list(freq_idx); + param.DDC_freq = (param.f0+param.f1)/2; + for wf=1:8 + param.wfs(wf).tx_mask = ones(1,8); + param.wfs(wf).tx_mask(9-wf) = 0; + end + for wf=9:9 + param.wfs(wf).tx_mask = [0 0 0 0 0 0 0 0]; + end + param.fn = fullfile(NI_calval_dir,sprintf('txequal_%.0f-%.0fMHz_%.0fft_%.0fft_%.0fus_%s.xml', ... + param.f0/1e6, param.f1/1e6, (param.tg.Haltitude+[-param.tg.altitude_guard param.tg.altitude_guard])*100/12/2.54, ... + param.Tpd*1e6,fn_hint{Tpd_idx})); + write_cresis_xml(param); + end +end + +%% Polarimetric Mode 3250m +% <3250 m thick ice, 1200 +/- 700 ft AGL +% ========================================================================= +ice_thickness = 3250; +polarimetric_f0_list = f0_list; +polarimetric_f1_list = f1_list; +for freq_idx = [] + param = struct('radar_name','mcords5','num_chan',8,'aux_dac',[255 255 255 255 255 255 255 255],'version','14.0f1','TTL_prog_delay',650,'xml_version',2.0,'fs',1600e6,'fs_sync',90.0e6,'fs_dds',1440e6,'TTL_clock',1440e6/16,'TTL_mode',[2.5e-6 260e-9 -1100e-9],'arena_base_dir',arena_base_dir); + param = merge_structs(param,param_defaults); + param.arena = arena; + param.max_tx = [4000 4000 4000 4000 4000 4000 4000 4000]; param.max_data_rate = 700; param.flight_hours = 3.5; param.sys_delay = 0.75e-6; param.use_mcords4_names = true; param.arena = arena; + BW = abs(polarimetric_f1_list(freq_idx)-polarimetric_f0_list(freq_idx)); + if BW <= 170e6 + param.DDC_select = 2; % 200 MHz sampling mode + elseif BW <= 370e6 + param.DDC_select = 1; % 400 MHz sampling mode + else + error('Bandwidth (%g MHz) is too large. Must be less than or equal to 370 MHz.', BW/1e6) + end + cal_used_idx = -1; + fc = abs(polarimetric_f0_list+polarimetric_f1_list)/2; + for cal_idx = 1:length(final_cal_fc) + if abs(final_cal_fc(cal_idx) - fc(freq_idx)) < 1e6 + cal_used_idx = cal_idx; + break; + end + end + if cal_used_idx == -1 + error('The center frequency %g MHz does not match any of the cal setting center frequencies specified in final_cal_fc. Either change the f0_list and f1_list or add a calibration setting with this center frequency.', fc/1e6); + end + param.max_duty_cycle = 0.12; + param.create_IQ = false; + param.tg.staged_recording = [1 1 2 2 3 3]; + param.tg.altitude_guard = 700*12*2.54/100; + param.tg.Haltitude = 1200*12*2.54/100; + param.tg.Hice_thick = ice_thickness; + param.prf = prf; + param.presums = [2 4 2 4 16 16]; + param.wfs(1).atten = 37; + param.wfs(2).atten = 37; + param.wfs(3).atten = 0; + param.wfs(4).atten = 0; + param.wfs(5).atten = 0; + param.wfs(6).atten = 0; + param.tx_weights = [4000 4000 4000 4000 4000 4000 4000 4000]; + param.tukey = 0.08; + param.wfs(1).Tpd = 1e-6; + param.wfs(2).Tpd = 1e-6; + param.wfs(3).Tpd = 3e-6; + param.wfs(4).Tpd = 3e-6; + param.wfs(5).Tpd = 10e-6; + param.wfs(6).Tpd = 10e-6; + param.wfs(1).phase = final_DDS_phase{cal_used_idx}; + param.wfs(2).phase = final_DDS_phase{cal_used_idx}; + param.wfs(3).phase = final_DDS_phase{cal_used_idx}; + param.wfs(4).phase = final_DDS_phase{cal_used_idx}; + param.wfs(5).phase = final_DDS_phase{cal_used_idx}; + param.wfs(6).phase = final_DDS_phase{cal_used_idx}; + param.delay = final_DDS_time{cal_used_idx}; + param.f0 = polarimetric_f0_list(freq_idx); + param.f1 = polarimetric_f1_list(freq_idx); + param.DDC_freq = (param.f0+param.f1)/2; + [param.wfs([1 3 5]).tx_mask] = deal([1 1 0 0 0 0 1 1]); + [param.wfs([2 4 6]).tx_mask] = deal([0 0 1 1 1 1 0 0]); + param.tg.rg_stop_offset = [300 300 300 300 0 0]; + param.fn = fullfile(NI_base_dir,sprintf('polarimetric_%.0f-%.0fMHz_%.0fft_%.0fus_%.0fmthick.xml',param.f0/1e6,param.f1/1e6,param.tg.Haltitude*100/12/2.54,param.wfs(end).Tpd*1e6,param.tg.Hice_thick)); + write_cresis_xml(param); +end + +%% Survey Mode 3250m with Polarimetric Setup +% <3250 m thick ice, 1200 +/- 700 ft AGL +% ========================================================================= +ice_thickness = 3250; +for freq_idx = [] + param = struct('radar_name','mcords5','num_chan',8,'aux_dac',[255 255 255 255 255 255 255 255],'version','14.0f1','TTL_prog_delay',650,'xml_version',2.0,'fs',1600e6,'fs_sync',90.0e6,'fs_dds',1440e6,'TTL_clock',1440e6/16,'TTL_mode',[2.5e-6 260e-9 -1100e-9],'arena_base_dir',arena_base_dir); + param = merge_structs(param,param_defaults); + param.arena = arena; + param.max_tx = [4000 4000 4000 4000 4000 4000 4000 4000]; param.max_data_rate = 700; param.flight_hours = 3.5; param.sys_delay = 0.75e-6; param.use_mcords4_names = true; param.arena = arena; + BW = abs(f1_list(freq_idx)-f0_list(freq_idx)); + if BW <= 170e6 + param.DDC_select = 2; % 200 MHz sampling mode + elseif BW <= 370e6 + param.DDC_select = 1; % 400 MHz sampling mode + else + error('Bandwidth (%g MHz) is too large. Must be less than or equal to 370 MHz.', BW/1e6) + end + cal_used_idx = -1; + fc = abs(f0_list+f1_list)/2; + for cal_idx = 1:length(final_cal_fc) + if abs(final_cal_fc(cal_idx) - fc(freq_idx)) < 1e6 + cal_used_idx = cal_idx; + break; + end + end + if cal_used_idx == -1 + error('The center frequency %g MHz does not match any of the cal setting center frequencies specified in final_cal_fc. Either change the f0_list and f1_list or add a calibration setting with this center frequency.', fc/1e6); + end + param.max_duty_cycle = 0.12; + param.create_IQ = false; + param.tg.staged_recording = [1 2 3]; + param.tg.altitude_guard = 700*12*2.54/100; + param.tg.Haltitude = 1200*12*2.54/100; + param.tg.Hice_thick = ice_thickness; + param.prf = prf; + param.presums = [4 4 34]; + param.wfs(1).atten = 37; + param.wfs(2).atten = 0; + param.wfs(3).atten = 0; + param.tx_weights = [4000 4000 4000 4000 4000 4000 4000 4000]; + param.tukey = 0.08; + param.wfs(1).Tpd = 1e-6; + param.wfs(2).Tpd = 3e-6; + param.wfs(3).Tpd = 10e-6; + param.wfs(1).phase = final_DDS_phase{cal_used_idx}; + param.wfs(2).phase = final_DDS_phase{cal_used_idx}; + param.wfs(3).phase = final_DDS_phase{cal_used_idx}; + param.delay = final_DDS_time{cal_used_idx}; + param.f0 = f0_list(freq_idx); + param.f1 = f1_list(freq_idx); + param.DDC_freq = (param.f0+param.f1)/2; + [param.wfs([1 2 3]).tx_mask] = deal([1 1 0 0 0 0 1 1]); + param.tg.rg_stop_offset = [300 300 0]; + param.fn = fullfile(NI_base_dir,sprintf('sur_pol_%.0f-%.0fMHz_%.0fft_%.0fus_%.0fmthick.xml',param.f0/1e6,param.f1/1e6,param.tg.Haltitude*100/12/2.54,param.wfs(end).Tpd*1e6,param.tg.Hice_thick)); + write_cresis_xml(param); +end + +%% Image Mode (EGRIP Low Altitude, Thick Ice) +% Ice thickness "param.tg.Hice_thick_min" m to "param.tg.Hice_thick" m, "param.tg.Haltitude" +/- "param.tg.altitude_guard" ft AGL +% ========================================================================= +for freq_idx = [] + param = struct('radar_name','mcords5','num_chan',8,'aux_dac',[255 255 255 255 255 255 255 255],'version','14.0f1','TTL_prog_delay',650,'xml_version',2.0,'fs',1600e6,'fs_sync',90.0e6,'fs_dds',1440e6,'TTL_clock',1440e6/16,'TTL_mode',[2.5e-6 260e-9 -1100e-9],'arena_base_dir',arena_base_dir); + param = merge_structs(param,param_defaults); + param.arena = arena; + param.max_tx = [4000 4000 4000 4000 4000 4000 4000 4000]; param.max_data_rate = 700; param.flight_hours = 3.5; param.sys_delay = 0.75e-6; param.use_mcords4_names = true; param.arena = arena; + BW = abs(f1_list(freq_idx)-f0_list(freq_idx)); + if BW <= 170e6 + param.DDC_select = 2; % 200 MHz sampling mode + elseif BW <= 370e6 + param.DDC_select = 1; % 400 MHz sampling mode + else + error('Bandwidth (%g MHz) is too large. Must be less than or equal to 370 MHz.', BW/1e6) + end + cal_used_idx = -1; + fc = abs(f0_list+f1_list)/2; + for cal_idx = 1:length(final_cal_fc) + if abs(final_cal_fc(cal_idx) - fc(freq_idx)) < 1e6 + cal_used_idx = cal_idx; + break; + end + end + if cal_used_idx == -1 + error('The center frequency %g MHz does not match any of the cal setting center frequencies specified in final_cal_fc. Either change the f0_list and f1_list or add a calibration setting with this center frequency.', fc/1e6); + end + param.max_duty_cycle = 0.12; + param.create_IQ = false; + param.tg.staged_recording = [1 2 3 3]; + param.tg.altitude_guard = 700 * 12*2.54/100; + param.tg.Haltitude = 1200 * 12*2.54/100; + param.tg.rg_stop_offset = [300 300 0 0]; + param.tg.Hice_thick = 3250; + param.tg.look_angle_deg = [0 0 40 40]; + param.prf = prf; + param.presums = [4 4 ceil((presums(freq_idx)-6)/4)*2 ceil((presums(freq_idx)-6)/4)*2]; + % Switch from tx calibration window to hanning window to broaden beam + DDS_amp = final_DDS_amp{cal_used_idx} .* hanning(8).' ./ Hwindow_orig; + % Renormalize the amplitudes + [~,relative_max_idx] = max(DDS_amp./param.max_tx); + DDS_amp = round(DDS_amp .* param.max_tx(relative_max_idx) / DDS_amp(relative_max_idx)); + param.tx_weights = DDS_amp; + param.tukey = 0.08; + param.wfs(1).Tpd = 1e-6; + param.wfs(2).Tpd = 3e-6; + param.wfs(3).Tpd = 10e-6; + param.wfs(4).Tpd = 10e-6; + param.wfs(1).phase = final_DDS_phase{cal_used_idx}; + param.wfs(2).phase = final_DDS_phase{cal_used_idx}; + param.wfs(3).phase = final_DDS_phase{cal_used_idx}; + param.wfs(4).phase = final_DDS_phase{cal_used_idx}; + param.wfs(1).name = 'nadir1'; + param.wfs(2).name = 'nadir3'; + param.wfs(3).name = 'left'; + param.wfs(4).name = 'right'; + % Loop through each waveform and adjust time delays to match the + % desired beam angle for that waveform. Note that the delays that need to + % be added in are the delay RELATIVE to a nadir beam since final_DDS_time + % contains the equalized delays for a nadir beam. + % param.tg.look_angle (negative is to the left, positive is to the right) + param.tg.look_angle = [0 0 -20 20]; + for wf = 1:length(param.tg.look_angle) + nadir_vec = [0 0 1]; + beam_vec = [0 sind(param.tg.look_angle(wf)) cosd(param.tg.look_angle(wf))]; + nadir_delay = -nadir_vec * phase_centers; + beam_delay = -beam_vec * phase_centers; + % Only need relative phases, so adjust by a constant + nadir_delay = nadir_delay - nadir_delay(4); + beam_delay = beam_delay - beam_delay(4); + % Convert from range to one-way delay + nadir_delay = nadir_delay / c; + beam_delay = beam_delay / c - nadir_delay; % Only include delay relative to nadir + beam_delay = beam_delay - mean(beam_delay); + % param.wfs(wf).delay and final_DDS_time{cal_used_idx} are in units of ns + param.wfs(wf).delay = (final_DDS_time{cal_used_idx} - beam_delay*1e9); + end + param.f0 = f0_list(freq_idx); + param.f1 = f1_list(freq_idx); + param.DDC_freq = (param.f0+param.f1)/2; + [param.wfs(1:4).tx_mask] = deal([0 0 0 0 0 0 0 0]); + param.wfs(1).atten = 37; + param.wfs(2).atten = 0; + param.wfs(3).atten = 0; + param.wfs(4).atten = 0; + param.fn = fullfile(NI_base_dir,sprintf('egrip_image_%.0f-%.0fMHz_%.0fft_%.0fus_%.0fmthick.xml', ... + param.f0/1e6,param.f1/1e6,param.tg.Haltitude*100/2.54/12,param.wfs(end).Tpd*1e6,param.tg.Hice_thick)); + write_cresis_xml(param); +end diff --git a/cresis-toolbox/missions/default_radar_params_2011_Greenland_P3_rds.m b/cresis-toolbox/missions/default_radar_params_2011_Greenland_P3_rds.m index be89e209..34d0ad3a 100644 --- a/cresis-toolbox/missions/default_radar_params_2011_Greenland_P3_rds.m +++ b/cresis-toolbox/missions/default_radar_params_2011_Greenland_P3_rds.m @@ -50,7 +50,7 @@ default.records.file.adcs = [2:16]; default.records.frames.mode = 1; default.records.frames.geotiff_fn = 'greenland\Landsat-7\mzl7geo_90m_lzw.tif'; -default.records.presum_bug_fixed = 0; +default.records.presum_mode = 1; %% Qlook worksheet default.qlook.out_path = ''; @@ -108,8 +108,6 @@ default.array.dbin = 1; default.array.dline = 6; default.array.DCM = []; -default.array.three_dim.en = 0; -default.array.three_dim.layer_fn = ''; default.array.Nsv = 1; default.array.theta_rng = [0 0]; default.array.sv_fh = @array_proc_sv; diff --git a/cresis-toolbox/missions/default_radar_params_2013_Antarctica_Basler_rds.m b/cresis-toolbox/missions/default_radar_params_2013_Antarctica_Basler_rds.m index 665b123f..b9b6b4d3 100644 --- a/cresis-toolbox/missions/default_radar_params_2013_Antarctica_Basler_rds.m +++ b/cresis-toolbox/missions/default_radar_params_2013_Antarctica_Basler_rds.m @@ -50,7 +50,7 @@ default.records.file.adcs = [2:16]; default.records.frames.mode = 1; default.records.frames.geotiff_fn = 'greenland\Landsat-7\mzl7geo_90m_lzw.tif'; -default.records.presum_bug_fixed = 0; +default.records.presum_mode = 1; %% Qlook worksheet default.qlook.out_path = ''; @@ -108,8 +108,6 @@ default.array.dbin = 1; default.array.dline = 6; default.array.DCM = []; -default.array.three_dim.en = 0; -default.array.three_dim.layer_fn = ''; default.array.Nsv = 1; default.array.theta_rng = [0 0]; default.array.sv_fh = @array_proc_sv; diff --git a/cresis-toolbox/missions/default_radar_params_2013_Greenland_P3_snow.m b/cresis-toolbox/missions/default_radar_params_2013_Greenland_P3_snow.m index e74c5aa6..15a4c972 100644 --- a/cresis-toolbox/missions/default_radar_params_2013_Greenland_P3_snow.m +++ b/cresis-toolbox/missions/default_radar_params_2013_Greenland_P3_snow.m @@ -109,8 +109,6 @@ default.array.dbin = 1; default.array.dline = 6; default.array.DCM = []; -default.array.three_dim.en = 0; -default.array.three_dim.layer_fn = ''; default.array.Nsv = 1; default.array.theta_rng = [0 0]; default.array.sv_fh = @array_proc_sv; diff --git a/cresis-toolbox/missions/default_radar_params_2014_Greenland_P3_mcords.m b/cresis-toolbox/missions/default_radar_params_2014_Greenland_P3_mcords.m deleted file mode 100644 index 16587e0d..00000000 --- a/cresis-toolbox/missions/default_radar_params_2014_Greenland_P3_mcords.m +++ /dev/null @@ -1,233 +0,0 @@ -function [param,defaults] = default_radar_params_2014_Greenland_P3_mcords -% [param,defaults] = default_radar_params_2014_Greenland_P3_mcords -% -% MCORDS 3: 2014 Greenland P3 -% -% Creates base "param" struct -% Creates defaults cell array for each type of radar setting -% -% Author: John Paden - -param.season_name = '2014_Greenland_P3'; -param.radar_name = 'mcords3'; - -%% Control parameters (not used in the parameter spreadsheet directly) -default.xml_file_prefix = 'mcords3'; -default.data_file_prefix = 'mcords3'; -default.header_load_func = @basic_load_mcords3; -default.header_load_params = struct('clk',1e9/9,'presum_bug_fixed',false); -default.xml_version = 1.4; - -default.noise_50ohm = [-45.5 -44.3 -44.8 -44.9 -40.2 -40.8 -40.5 -41.4 -39.9 -40.2 -42.0 -43.6 -43.0 -44.1 -44.6]; - -default.Pt = 175 * sum([1 1 1 1 1 1 1]); -default.Gt = 7*4; -default.Ae = 2*0.468 * 0.468; - -default.system_loss_dB = 10.^(-5.88/10); -default.max_DDS_RAM = 40000; -default.tx_voltage = sqrt(1000*50)*10^(-2/20); - -default.iq_mode = 0; -default.tx_DDS_mask = [1 1 1 1 1 1 1 0]; - -default.radar_worksheet_headers = {'Tpd','Tadc','Tadc_adjust','f0','f1','ref_fn','tukey','tx_weights','rx_paths','adc_gains','chan_equal_dB','chan_equal_deg','Tsys','DC_adjust','DDC_mode','DDC_freq'}; -default.radar_worksheet_headers_type = {'r','r','r','r','r','r','r','r','r','r','r','r','r','r','r','r'}; - -default.basic_surf_track_min_time = 2e-6; % Normally -inf for lab test, 2e-6 for flight test -default.basic_surf_track_Tpd_factor = 1.1; % Normally -inf for lab test, 1.1 for flight test -default.adc_folder_name = 'board%b'; - -if 1 - % Example 1: Normal configuration: - % Connect antenna N to WFG N for all N = 1 to 7 - ref_adc = 14; - default.txequal.img = [(1:7).', ref_adc*ones(7,1)]; - default.txequal.ref_wf_adc = 3; - default.txequal.wf_mapping = [1 2 3 4 5 6 7 0]; - default.txequal.Hwindow_desired = [1 1 1 1 1 1 1 0]; - default.txequal.max_DDS_amp = [40000 40000 40000 40000 40000 40000 40000 0]; - default.txequal.time_delay_desired = [0 0 0 0 0 0 0 0]; - default.txequal.phase_desired = [0 0 0 0 0 0 0 0]; - default.txequal.time_validation = [3 3 3 3 3 3 3 3]*1e-9; - default.txequal.amp_validation = [3 3 3 3 3 3 3 3]; - default.txequal.phase_validation = [35 35 35 35 35 35 35 35]; - default.txequal.remove_linear_phase_en = true; -end - -%% Vectors worksheet in parameter spreadsheet -default.vectors.gps.time_offset = 1; - -%% Records worksheet in parameter spreadsheet -default.records.geotiff_fn = 'greenland\Landsat-7\mzl7geo_90m_lzw.tif'; -default.records.file.adcs = [2:16]; -default.records.file.adc_headers = [2:16]; -default.records.gps.en = 1; -default.records.frame_mode = 0; -default.records.presum_bug_fixed = 0; -default.records.tmp_fn_uses_adc_folder_name = 1; - -%% Get heights (quick-look) worksheet in parameter spreadsheet -default.get_heights.qlook.out_path = ''; -default.get_heights.qlook.en = 1; -default.get_heights.block_size = 10000; -default.get_heights.frm_types = {0,[0 1],0,0,-1}; -default.get_heights.coh_noise_method = []; -default.get_heights.coh_noise_arg = []; -default.get_heights.ft_wind = @hanning; -default.get_heights.ft_wind_time = false; -default.get_heights.ft_dec = true; -default.get_heights.pulse_comp = []; -default.get_heights.pulse_rfi.en = []; -default.get_heights.pulse_rfi.inc_ave= []; -default.get_heights.pulse_rfi.thresh_scale = []; -default.get_heights.roll_correction = 0; -default.get_heights.lever_arm_fh = @lever_arm; -default.get_heights.elev_correction = 0; -default.get_heights.B_filter = ones(1,20)/20; -default.get_heights.decimate_factor = 20; -default.get_heights.inc_ave = 10; -default.get_heights.surf.en = 1; -default.get_heights.surf.method = 'threshold'; -default.get_heights.surf.noise_rng = [0 -50 10]; -default.get_heights.surf.min_bin = 2e-6; -default.get_heights.surf.max_bin = []; -default.get_heights.surf.threshold = 9; -default.get_heights.surf.sidelobe = 15; -default.get_heights.surf.medfilt = 3; -default.get_heights.surf.search_rng = [0:2]; - -%% CSARP worksheet in parameter spreadsheet -default.csarp.out_path = ''; -default.csarp.imgs = {[1*ones(4,1),(2:5).'],[2*ones(4,1),(2:5).'],[3*ones(4,1),(2:5).']}; -default.csarp.frm_types = {0,[0 1],0,0,-1}; -default.csarp.chunk_len = 5000; -default.csarp.chunk_overlap = 10; -default.csarp.frm_overlap = 0; -default.csarp.coh_noise_removal = 0; -default.csarp.combine_rx = 0; -default.csarp.time_of_full_support = 3.5e-5; -default.csarp.pulse_rfi.en = []; -default.csarp.pulse_rfi.inc_ave= []; -default.csarp.pulse_rfi.thresh_scale = []; -default.csarp.trim_vals = []; -default.csarp.pulse_comp = 1; -default.csarp.ft_dec = 1; -default.csarp.ft_wind = @hanning; -default.csarp.ft_wind_time = 0; -default.csarp.lever_arm_fh = @lever_arm; -default.csarp.mocomp.en = 1; -default.csarp.mocomp.type = 2; -default.csarp.mocomp.filter = {@butter [2] [0.1000]}; -default.csarp.mocomp.uniform_en = 1; -default.csarp.sar_type = 'f-k'; -default.csarp.sigma_x = 2.5; -default.csarp.sub_aperture_steering = 0; -default.csarp.st_wind = @hanning; -default.csarp.start_eps = 3.15; - -%% Combine worksheet in parameter spreadsheet -default.combine.in_path = ''; -default.combine.array_path = ''; -default.combine.out_path = ''; -default.combine.method = 'standard'; -default.combine.window = @hanning; -default.combine.bin_rng = 0; -default.combine.rline_rng = -5:5; -default.combine.dbin = 1; -default.combine.dline = 6; -default.combine.DCM = []; -default.combine.three_dim.en = 0; -default.combine.three_dim.layer_fn = ''; -default.combine.Nsv = 1; -default.combine.theta_rng = [0 0]; -default.combine.sv_fh = @array_proc_sv; -default.combine.diag_load = 0; -default.combine.Nsig = 2; - -%% Radar worksheet in parameter spreadsheet -default.radar.fs = 1e9/9; -default.radar.Tadc = []; % normally leave empty to use value in file header -default.radar.adc_bits = 14; -default.radar.adc_full_scale = 2; -default.radar.rx_paths = [1 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15]; -default.radar.noise_figure = 2; -default.radar.rx_gain = 51.5; -default.radar.adc_SNR_dB = 70; -default.radar.Tadc_adjust = -1.4455e-06; % System time delay: leave this empty or set it to zero at first, determine this value later using data over surface with known height or from surface multiple - -defaults = {}; - -%% Settings -default.radar.wfs(1).chan_equal_Tsys = [2.8 2.3 0 6.1 1.7 1 1.7 -19 -24 -24.5 -29.5 -29.5 -25.7 -24.6 -19.3]/1e9; -default.radar.wfs(1).chan_equal_dB = [2.2 1.1 0 -3.1 -0.3 0.9 1.8 1.9 -1.2 0.5 1.6 2.2 0.4 -0.8 2.1]; -default.radar.wfs(1).chan_equal_deg = [-168.3 101.4 0 -87.8 84.1 52.9 119.3 -159.6 82 -70 -142.2 -104.5 -129.3 112 -104.2]; - -% survey mode -default.get_heights.qlook.img_comb = [3e-06 -inf 1e-06 1e-05 -inf 3e-06]; -default.get_heights.imgs = {[1*ones(4,1),(2:5).'],[2*ones(4,1),(2:5).'],[3*ones(4,1),(2:5).']}; -default.csarp.imgs = default.get_heights.imgs; -default.combine.imgs = default.get_heights.imgs; -default.combine.img_comb = default.get_heights.qlook.img_comb; -default.radar.DC_adjust = {'','',''}; -default.radar.ref_fn = ''; -default.xml_regexp = 'survey_.*thick.xml'; -default.name = 'Survey Mode'; -defaults{end+1} = default; - -% survey mode -default.get_heights.qlook.img_comb = [3e-06 -inf 1e-06]; -default.get_heights.imgs = {[1*ones(4,1),(2:5).'],[2*ones(4,1),(2:5).']}; -default.csarp.imgs = default.get_heights.imgs; -default.combine.imgs = default.get_heights.imgs; -default.combine.img_comb = default.get_heights.qlook.img_comb; -default.radar.DC_adjust = {'','',''}; -default.radar.ref_fn = ''; -default.xml_regexp = 'survey_.*thin_ice.xml'; -default.name = 'Thin Ice Mode'; -defaults{end+1} = default; - -% high altitude mode -default.get_heights.qlook.img_comb = [1e-05 -inf 3e-06]; -default.get_heights.imgs = {[1*ones(4,1),(2:5).'],[2*ones(4,1),(2:5).']}; -default.csarp.imgs = default.get_heights.imgs; -default.combine.imgs = default.get_heights.imgs; -default.combine.img_comb = default.get_heights.qlook.img_comb; -default.radar.DC_adjust = {'','',''}; -default.radar.ref_fn = ''; -default.xml_regexp = 'survey_.*high_altitude.xml'; -default.name = 'High Altitude Mode'; -defaults{end+1} = default; - -% deconvolution mode -default.get_heights.qlook.img_comb = []; -default.get_heights.imgs = {[1*ones(4,1),(2:5).'],[2*ones(4,1),(2:5).'],[3*ones(4,1),(2:5).']}; -default.csarp.imgs = default.get_heights.imgs; -default.combine.imgs = default.get_heights.imgs; -default.combine.img_comb = default.get_heights.qlook.img_comb; -default.radar.DC_adjust = {'','',''}; -default.radar.ref_fn = ''; -default.xml_regexp = 'survey_.*DECONVOLUTION.xml'; -default.name = 'Deconvolution Mode'; -defaults{end+1} = default; - -%% Other settings - -default.get_heights.qlook.img_comb = []; -default.get_heights.imgs = []; -default.combine.imgs = default.get_heights.imgs; -default.combine.img_comb = default.get_heights.qlook.img_comb; - -default.radar.DC_adjust = {'DC_20160413_04_wf1.mat','DC_20160413_04_wf2.mat','DC_20160413_04_wf2.mat'}; -default.radar.ref_fn = 'deconv_wf_%w_adc_%a_20160413_06'; -default.xml_regexp = 'survey_180-210MHz_.*DECONV.xml'; -default.name = 'Deconv 180-210 MHz'; -defaults{end+1} = default; - -default.radar.DC_adjust = []; -default.radar.ref_fn = ''; -default.xml_regexp = '.*'; -default.name = 'Other Settings'; -defaults{end+1} = default; - -return; diff --git a/cresis-toolbox/missions/default_radar_params_2014_Greenland_P3_rds.m b/cresis-toolbox/missions/default_radar_params_2014_Greenland_P3_rds.m new file mode 100644 index 00000000..92b1833b --- /dev/null +++ b/cresis-toolbox/missions/default_radar_params_2014_Greenland_P3_rds.m @@ -0,0 +1,259 @@ +function param = default_radar_params_2014_Greenland_P3_rds +% param = default_radar_params_2014_Greenland_P3_rds +% +% MCORDS 3: 2014 Greenland P3 +% +% Creates base "param" struct +% Creates defaults cell array for each type of radar setting +% +% Author: John Paden + +% param.season_name = '2014_Greenland_P3'; +% param.radar_name = 'mcords3'; +% +% %% Control parameters (not used in the parameter spreadsheet directly) +% default.xml_file_prefix = 'mcords3'; +% default.data_file_prefix = 'mcords3'; +% default.header_load_func = @basic_load_mcords3; +% default.header_load_params = struct('clk',1e9/9,'presum_mode',1); +% default.xml_version = 1.4; +% +% default.noise_50ohm = [-45.5 -44.3 -44.8 -44.9 -40.2 -40.8 -40.5 -41.4 -39.9 -40.2 -42.0 -43.6 -43.0 -44.1 -44.6]; +% +% default.Pt = 175 * sum([1 1 1 1 1 1 1]); +% default.Gt = 7*4; +% default.Ae = 2*0.468 * 0.468; +% +% default.system_loss_dB = 10.^(-5.88/10); +% default.max_DDS_RAM = 40000; +% default.tx_voltage = sqrt(1000*50)*10^(-2/20); +% +% default.iq_mode = 0; +% default.tx_DDS_mask = [1 1 1 1 1 1 1 0]; +% +% default.radar_worksheet_headers = {'Tpd','Tadc','Tadc_adjust','f0','f1','ref_fn','tukey','tx_weights','rx_paths','adc_gains','chan_equal_dB','chan_equal_deg','Tsys','DC_adjust','DDC_mode','DDC_freq'}; +% default.radar_worksheet_headers_type = {'r','r','r','r','r','r','r','r','r','r','r','r','r','r','r','r'}; +% +% default.basic_surf_track_min_time = 2e-6; % Normally -inf for lab test, 2e-6 for flight test +% default.basic_surf_track_Tpd_factor = 1.1; % Normally -inf for lab test, 1.1 for flight test +% default.adc_folder_name = 'board%b'; +% +% if 1 +% % Example 1: Normal configuration: +% % Connect antenna N to WFG N for all N = 1 to 7 +% ref_adc = 14; +% default.txequal.img = [(1:7).', ref_adc*ones(7,1)]; +% default.txequal.ref_wf_adc = 3; +% default.txequal.wf_mapping = [1 2 3 4 5 6 7 0]; +% default.txequal.Hwindow_desired = [1 1 1 1 1 1 1 0]; +% default.txequal.max_DDS_amp = [40000 40000 40000 40000 40000 40000 40000 0]; +% default.txequal.time_delay_desired = [0 0 0 0 0 0 0 0]; +% default.txequal.phase_desired = [0 0 0 0 0 0 0 0]; +% default.txequal.time_validation = [3 3 3 3 3 3 3 3]*1e-9; +% default.txequal.amp_validation = [3 3 3 3 3 3 3 3]; +% default.txequal.phase_validation = [35 35 35 35 35 35 35 35]; +% default.txequal.remove_linear_phase_en = true; +% end + +%% Preprocess parameters +param.season_name = '2014_Greenland_P3'; +param.radar_name = 'mcords3'; + +param.config.file.version = 403; +param.config.file.prefix = param.radar_name; +param.config.file.suffix = '.bin'; +param.config.max_time_gap = 10; +param.config.min_seg_size = 2; + +param.config.daq_type = 'cresis'; +param.config.wg_type = 'cresis'; +param.config.header_load_func = @basic_load_mcords3; +param.config.board_map = {'board0','board1','board2','board3'}; +param.config.tx_map = {'','','','','','','',''}; + +param.config.daq.xml_version = 1.4; + +param.config.max_data_rate = 100; +param.config.max_duty_cycle = 0.12; +param.config.prf_multiple = []; % Power supply sync signal that PRF must be a factor of these numbers +param.config.PRI_guard = 10e-6; +param.config.PRI_guard_percentage = 1; +param.config.tx_enable = [1 1 1 1 1 1 1 0]; +param.config.max_tx = 40000; +param.config.max_tx_voltage = sqrt(250*50)*10^(-2/20); % voltage at max_tx + +%% CReSIS parameters +param.config.cresis.clk = 1e9/9; +param.config.cresis.rx_gain_dB = 51.5; +param.config.cresis.gps_file_mask = 'GPS*'; + + +%% Command worksheet +default.cmd.records = 1; +default.cmd.qlook = 1; +default.cmd.generic = 1; + +%% Records worksheet +default.records.gps.time_offset = 1; +default.records.file.adcs = [2:16]; +default.records.frames.mode = 1; +default.records.frames.geotiff_fn = 'greenland\Landsat-7\mzl7geo_90m_lzw.tif'; +default.records.presum_mode = 1; + +%% Qlook worksheet +default.qlook.out_path = ''; +default.qlook.block_size = 10000; +default.qlook.dec = 50; +default.qlook.inc_dec = 10; +default.qlook.surf.en = 1; +default.qlook.surf.method = 'threshold'; +default.qlook.surf.noise_rng = [0 -50 10]; +default.qlook.surf.min_bin = 1.8e-6; +default.qlook.surf.max_bin = []; +default.qlook.surf.threshold = 15; +default.qlook.surf.sidelobe = 15; +default.qlook.surf.medfilt = 3; +default.qlook.surf.search_rng = [0:2]; + +%% SAR worksheet +default.sar.out_path = ''; +default.sar.imgs = {[1*ones(4,1),(9:12).'],[2*ones(4,1),(9:12).'],[3*ones(4,1),(9:12).']}; +default.sar.frm_types = {0,[0 1],0,0,-1}; +default.sar.chunk_len = 5000; +default.sar.array_rx = 0; +default.sar.time_of_full_support = 3.5e-5; +default.sar.mocomp.en = 1; +default.sar.mocomp.type = 2; +default.sar.mocomp.filter = {@butter [2] [0.1000]}; +default.sar.mocomp.uniform_en = 1; +default.sar.sar_type = 'fk'; +default.sar.sigma_x = 2.5; +default.sar.sub_aperture_steering = 0; +default.sar.st_wind = @hanning; +default.sar.start_eps = 3.15; + +%% Array worksheet +default.array.in_path = ''; +default.array.array_path = ''; +default.array.out_path = ''; +default.array.method = 'standard'; +default.array.window = @hanning; +default.array.bin_rng = 0; +default.array.line_rng = -5:5; +default.array.dbin = 1; +default.array.dline = 6; +default.array.DCM = []; +default.array.tomo_en = 0; +default.array.Nsv = 1; +default.array.theta_rng = [0 0]; +default.array.sv_fh = @array_proc_sv; +default.array.diag_load = 0; +default.array.Nsrc = 2; + +%% Radar worksheet +default.radar.fs = 1e9/9; +default.radar.Tadc = []; % normally leave empty to use value in file header +default.radar.adc_bits = 14; +default.radar.Vpp_scale = 2; + +default.radar.rx_paths = [1 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15]; +default.radar.wfs.noise_figure = 2; +default.radar.wfs.Tadc_adjust = -1.4455e-06; % System time delay: leave this empty or set it to zero at first, determine this value later using data over surface with known height or from surface multiple + +%% Post worksheet +default.post.data_dirs = {'qlook'}; +default.post.layer_dir = 'layerData'; +default.post.maps_en = 1; +default.post.echo_en = 1; +default.post.layers_en = 0; +default.post.data_en = 0; +default.post.csv_en = 1; +default.post.concat_en = 1; +default.post.pdf_en = 1; +default.post.map.location = 'Greenland'; +default.post.map.type = 'combined'; +default.post.echo.elev_comp = 3; +default.post.echo.depth = '[publish_echogram_switch(Bbad,0.25,Surface_Elev,-3500,DBottom,-100),max(Surface_Elev+100)]'; +default.post.echo.er_ice = 3.15; +default.post.ops.en = 0; +default.post.ops.location = 'arctic'; +default.post.ops.layers = {'bottom','surface'}; +default.post.ops.gaps_dist = [300 60]; + +defaults = {}; + +%% Settings +default.radar.wfs(1).chan_equal_Tsys = [2.8 2.3 0 6.1 1.7 1 1.7 -19 -24 -24.5 -29.5 -29.5 -25.7 -24.6 -19.3]/1e9; +default.radar.wfs(1).chan_equal_dB = [2.2 1.1 0 -3.1 -0.3 0.9 1.8 1.9 -1.2 0.5 1.6 2.2 0.4 -0.8 2.1]; +default.radar.wfs(1).chan_equal_deg = [-168.3 101.4 0 -87.8 84.1 52.9 119.3 -159.6 82 -70 -142.2 -104.5 -129.3 112 -104.2]; + +% survey mode +default.qlook.qlook.img_comb = [3e-06 -inf 1e-06 1e-05 -inf 3e-06]; +default.qlook.imgs = {[1*ones(4,1),(2:5).'],[2*ones(4,1),(2:5).'],[3*ones(4,1),(2:5).']}; +default.sar.imgs = default.qlook.imgs; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.qlook.img_comb; +default.radar.DC_adjust = {'','',''}; +default.radar.ref_fn = ''; +default.xml_regexp = 'survey_.*thick.xml'; +default.name = 'Survey Mode'; +defaults{end+1} = default; + +% survey mode +default.qlook.qlook.img_comb = [3e-06 -inf 1e-06]; +default.qlook.imgs = {[1*ones(4,1),(2:5).'],[2*ones(4,1),(2:5).']}; +default.sar.imgs = default.qlook.imgs; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.qlook.img_comb; +default.radar.DC_adjust = {'','',''}; +default.radar.ref_fn = ''; +default.xml_regexp = 'survey_.*thin_ice.xml'; +default.name = 'Thin Ice Mode'; +defaults{end+1} = default; + +% high altitude mode +default.qlook.qlook.img_comb = [1e-05 -inf 3e-06]; +default.qlook.imgs = {[1*ones(4,1),(2:5).'],[2*ones(4,1),(2:5).']}; +default.sar.imgs = default.qlook.imgs; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.qlook.img_comb; +default.radar.DC_adjust = {'','',''}; +default.radar.ref_fn = ''; +default.xml_regexp = 'survey_.*high_altitude.xml'; +default.name = 'High Altitude Mode'; +defaults{end+1} = default; + +% deconvolution mode +default.qlook.qlook.img_comb = []; +default.qlook.imgs = {[1*ones(4,1),(2:5).'],[2*ones(4,1),(2:5).'],[3*ones(4,1),(2:5).']}; +default.sar.imgs = default.qlook.imgs; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.qlook.img_comb; +default.radar.DC_adjust = {'','',''}; +default.radar.ref_fn = ''; +default.xml_regexp = 'survey_.*DECONVOLUTION.xml'; +default.name = 'Deconvolution Mode'; +defaults{end+1} = default; + +%% Other settings + +default.qlook.qlook.img_comb = []; +default.qlook.imgs = []; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.qlook.img_comb; + +default.radar.DC_adjust = {'DC_20160413_04_wf1.mat','DC_20160413_04_wf2.mat','DC_20160413_04_wf2.mat'}; +default.radar.ref_fn = 'deconv_wf_%w_adc_%a_20160413_06'; +default.xml_regexp = 'survey_180-210MHz_.*DECONV.xml'; +default.name = 'Deconv 180-210 MHz'; +defaults{end+1} = default; + +default.radar.DC_adjust = []; +default.radar.ref_fn = ''; +default.xml_regexp = '.*'; +default.name = 'Other Settings'; +defaults{end+1} = default; + +%% Add default settings + +param.config.defaults = defaults; diff --git a/cresis-toolbox/missions/default_radar_params_2014_Greenland_P3_snow.m b/cresis-toolbox/missions/default_radar_params_2014_Greenland_P3_snow.m index 957f8917..e41a4e24 100644 --- a/cresis-toolbox/missions/default_radar_params_2014_Greenland_P3_snow.m +++ b/cresis-toolbox/missions/default_radar_params_2014_Greenland_P3_snow.m @@ -109,8 +109,6 @@ default.array.dbin = 1; default.array.dline = 6; default.array.DCM = []; -default.array.three_dim.en = 0; -default.array.three_dim.layer_fn = ''; default.array.Nsv = 1; default.array.theta_rng = [0 0]; default.array.sv_fh = @array_proc_sv; diff --git a/cresis-toolbox/missions/default_radar_params_2016_Antarctica_DC8_mcords.m b/cresis-toolbox/missions/default_radar_params_2016_Antarctica_DC8_mcords.m index 17227553..9db57c60 100644 --- a/cresis-toolbox/missions/default_radar_params_2016_Antarctica_DC8_mcords.m +++ b/cresis-toolbox/missions/default_radar_params_2016_Antarctica_DC8_mcords.m @@ -15,7 +15,7 @@ default.xml_file_prefix = 'mcords3'; default.data_file_prefix = 'mcords3'; default.header_load_func = @basic_load_mcords3; -default.header_load_params = struct('clk',150e6,'presum_bug_fixed',false); +default.header_load_params = struct('clk',150e6,'presum_mode',1); default.xml_version = 2.0; % default.noise_50ohm = [-41.6 -42.2 -42.4 -41.9 -42.5 -42.9 -41.7 -43.0 -44.1 -44.7 -43.1 -44.1 -41.8 -42.6 -41.4 -42.6 -41.8 -43.1 -42.0 -42.7 -41.1 -43.4 -42.1 -41.9]; @@ -65,7 +65,7 @@ default.records.file.adc_headers = [1:6]; default.records.gps.en = 1; default.records.frame_mode = 0; -default.records.presum_bug_fixed = 0; +default.records.presum_mode = 1; default.records.tmp_fn_uses_adc_folder_name = 1; %% Get heights (quick-look) worksheet in parameter spreadsheet @@ -138,8 +138,6 @@ default.combine.dbin = 1; default.combine.dline = 6; default.combine.DCM = []; -default.combine.three_dim.en = 0; -default.combine.three_dim.layer_fn = ''; default.combine.Nsv = 1; default.combine.theta_rng = [0 0]; default.combine.sv_fh = @array_proc_sv; diff --git a/cresis-toolbox/missions/default_radar_params_2016_Greenland_Polar6_mcords.m b/cresis-toolbox/missions/default_radar_params_2016_Greenland_Polar6_mcords.m index 635a8677..fb9462c8 100644 --- a/cresis-toolbox/missions/default_radar_params_2016_Greenland_Polar6_mcords.m +++ b/cresis-toolbox/missions/default_radar_params_2016_Greenland_Polar6_mcords.m @@ -15,7 +15,7 @@ default.xml_file_prefix = 'mcords5'; default.data_file_prefix = 'mcords5'; default.header_load_func = @basic_load_mcords5; -default.header_load_params = struct('clk',1600e6,'presum_bug_fixed',true); +default.header_load_params = struct('clk',200e6,'presum_mode',0); default.xml_version = 2.0; % default.noise_50ohm = [-41.6 -42.2 -42.4 -41.9 -42.5 -42.9 -41.7 -43.0 -44.1 -44.7 -43.1 -44.1 -41.8 -42.6 -41.4 -42.6 -41.8 -43.1 -42.0 -42.7 -41.1 -43.4 -42.1 -41.9]; @@ -110,7 +110,7 @@ default.records.file.adc_headers = [1:24]; default.records.gps.en = 1; default.records.frame_mode = 0; -default.records.presum_bug_fixed = 1; +default.records.presum_mode = 0; default.records.tmp_fn_uses_adc_folder_name = 1; %% Get heights (quick-look) worksheet in parameter spreadsheet @@ -183,8 +183,6 @@ default.combine.dbin = 1; default.combine.dline = 6; default.combine.DCM = []; -default.combine.three_dim.en = 0; -default.combine.three_dim.layer_fn = ''; default.combine.Nsv = 1; default.combine.theta_rng = [0 0]; default.combine.sv_fh = @array_proc_sv; diff --git a/cresis-toolbox/missions/default_radar_params_2016_Greenland_Polar6_snow.m b/cresis-toolbox/missions/default_radar_params_2016_Greenland_Polar6_snow.m index 185fd86e..b44965e0 100644 --- a/cresis-toolbox/missions/default_radar_params_2016_Greenland_Polar6_snow.m +++ b/cresis-toolbox/missions/default_radar_params_2016_Greenland_Polar6_snow.m @@ -116,8 +116,6 @@ default.combine.dbin = 1; default.combine.dline = 6; default.combine.DCM = []; -default.combine.three_dim.en = 0; -default.combine.three_dim.layer_fn = ''; default.combine.Nsv = 1; default.combine.theta_rng = [0 0]; default.combine.sv_fh = @array_proc_sv; diff --git a/cresis-toolbox/missions/default_radar_params_2016_Greenland_TOdtu.m b/cresis-toolbox/missions/default_radar_params_2016_Greenland_TOdtu.m index 2c491755..0a799a25 100644 --- a/cresis-toolbox/missions/default_radar_params_2016_Greenland_TOdtu.m +++ b/cresis-toolbox/missions/default_radar_params_2016_Greenland_TOdtu.m @@ -15,7 +15,7 @@ default.xml_file_prefix = 'mcords5'; default.data_file_prefix = 'mcords5'; default.header_load_func = @basic_load_mcords5; -default.header_load_params = struct('clk',1600e6,'presum_bug_fixed',true); +default.header_load_params = struct('clk',200e6,'presum_mode',0); default.xml_version = 100.0; default.noise_50ohm = [0 0 0 0]; @@ -64,7 +64,7 @@ default.records.file.adc_headers = [1:4]; default.records.gps.en = 1; default.records.frame_mode = 0; -default.records.presum_bug_fixed = 1; +default.records.presum_mode = 0; default.records.tmp_fn_uses_adc_folder_name = 1; %% Get heights (quick-look) worksheet in parameter spreadsheet @@ -137,8 +137,6 @@ default.combine.dbin = 1; default.combine.dline = 6; default.combine.DCM = []; -default.combine.three_dim.en = 0; -default.combine.three_dim.layer_fn = ''; default.combine.Nsv = 1; default.combine.theta_rng = [0 0]; default.combine.sv_fh = @array_proc_sv; diff --git a/cresis-toolbox/missions/default_radar_params_2017_Antarctica_Basler.m b/cresis-toolbox/missions/default_radar_params_2017_Antarctica_Basler.m index 52339582..46347929 100644 --- a/cresis-toolbox/missions/default_radar_params_2017_Antarctica_Basler.m +++ b/cresis-toolbox/missions/default_radar_params_2017_Antarctica_Basler.m @@ -15,7 +15,7 @@ default.xml_file_prefix = 'mcords5'; default.data_file_prefix = 'mcords5'; default.header_load_func = @basic_load_mcords5; -default.header_load_params = struct('clk',1600e6,'presum_bug_fixed',true); +default.header_load_params = struct('clk',200e6,'presum_mode',0); default.xml_version = 2.0; default.noise_50ohm = [-45.4 -45.7 -45.5 -45.6 -46.2 -46.7 -44.8 -46.1]; @@ -175,7 +175,7 @@ default.records.file.adc_headers = [1:8]; default.records.gps.en = 1; default.records.frame_mode = 0; -default.records.presum_bug_fixed = 1; +default.records.presum_mode = 0; default.records.tmp_fn_uses_adc_folder_name = 1; %% Get heights (quick-look) worksheet in parameter spreadsheet @@ -248,8 +248,6 @@ default.combine.dbin = 1; default.combine.dline = 6; default.combine.DCM = []; -default.combine.three_dim.en = 0; -default.combine.three_dim.layer_fn = ''; default.combine.Nsv = 1; default.combine.theta_rng = [0 0]; default.combine.sv_fh = @array_proc_sv; diff --git a/cresis-toolbox/missions/default_radar_params_2017_Antarctica_P3_accum.m b/cresis-toolbox/missions/default_radar_params_2017_Antarctica_P3_accum.m index 2b2f25b5..dc6c3974 100644 --- a/cresis-toolbox/missions/default_radar_params_2017_Antarctica_P3_accum.m +++ b/cresis-toolbox/missions/default_radar_params_2017_Antarctica_P3_accum.m @@ -15,7 +15,7 @@ default.xml_file_prefix = 'mcords5'; default.data_file_prefix = 'mcords5'; default.header_load_func = @basic_load_mcords5; -default.header_load_params = struct('clk',1600e6,'presum_bug_fixed',true); +default.header_load_params = struct('clk',200e6,'presum_mode',0); default.xml_version = 2.0; default.noise_50ohm = [0 0 0 0]; @@ -64,7 +64,7 @@ default.records.file.adc_headers = [1:4]; default.records.gps.en = 1; default.records.frame_mode = 0; -default.records.presum_bug_fixed = 1; +default.records.presum_mode = 0; default.records.tmp_fn_uses_adc_folder_name = 1; %% Get heights (quick-look) worksheet in parameter spreadsheet @@ -137,8 +137,6 @@ default.combine.dbin = 1; default.combine.dline = 6; default.combine.DCM = []; -default.combine.three_dim.en = 0; -default.combine.three_dim.layer_fn = ''; default.combine.Nsv = 1; default.combine.theta_rng = [0 0]; default.combine.sv_fh = @array_proc_sv; diff --git a/cresis-toolbox/missions/default_radar_params_2017_Antarctica_P3_mcords.m b/cresis-toolbox/missions/default_radar_params_2017_Antarctica_P3_mcords.m index 40f4c993..92758776 100644 --- a/cresis-toolbox/missions/default_radar_params_2017_Antarctica_P3_mcords.m +++ b/cresis-toolbox/missions/default_radar_params_2017_Antarctica_P3_mcords.m @@ -15,7 +15,7 @@ default.xml_file_prefix = 'mcords3'; default.data_file_prefix = 'mcords3'; default.header_load_func = @basic_load_mcords3; -default.header_load_params = struct('clk',1e9/9,'presum_bug_fixed',false); +default.header_load_params = struct('clk',1e9/9,'presum_mode',1); default.xml_version = 2.0; default.noise_50ohm = [-32.2 -26.9 -34.1 -44.4 -43.1 -43.0 -42.3 -42.7 -42.3 -41.8 -40.2 -64.4 -40.3 -39.2 -41.8]; @@ -64,7 +64,7 @@ default.records.file.adc_headers = [2:16]; default.records.gps.en = 1; default.records.frame_mode = 0; -default.records.presum_bug_fixed = 0; +default.records.presum_mode = 1; default.records.tmp_fn_uses_adc_folder_name = 1; %% Get heights (quick-look) worksheet in parameter spreadsheet @@ -137,8 +137,6 @@ default.combine.dbin = 1; default.combine.dline = 6; default.combine.DCM = []; -default.combine.three_dim.en = 0; -default.combine.three_dim.layer_fn = ''; default.combine.Nsv = 1; default.combine.theta_rng = [0 0]; default.combine.sv_fh = @array_proc_sv; diff --git a/cresis-toolbox/missions/default_radar_params_2017_Antarctica_Polar6_mcords.m b/cresis-toolbox/missions/default_radar_params_2017_Antarctica_Polar6_mcords.m index 41eca78c..b3194ba7 100644 --- a/cresis-toolbox/missions/default_radar_params_2017_Antarctica_Polar6_mcords.m +++ b/cresis-toolbox/missions/default_radar_params_2017_Antarctica_Polar6_mcords.m @@ -15,7 +15,7 @@ default.xml_file_prefix = 'mcords5'; default.data_file_prefix = 'mcords5'; default.header_load_func = @basic_load_mcords5; -default.header_load_params = struct('clk',1600e6,'presum_bug_fixed',true); +default.header_load_params = struct('clk',200e6,'presum_mode',0); default.xml_version = 2.0; % default.noise_50ohm = [-41.6 -42.2 -42.4 -41.9 -42.5 -42.9 -41.7 -43.0 -44.1 -44.7 -43.1 -44.1 -41.8 -42.6 -41.4 -42.6 -41.8 -43.1 -42.0 -42.7 -41.1 -43.4 -42.1 -41.9]; @@ -111,7 +111,7 @@ default.records.file.adc_headers = [1:24]; default.records.gps.en = 1; default.records.frame_mode = 0; -default.records.presum_bug_fixed = 1; +default.records.presum_mode = 0; default.records.tmp_fn_uses_adc_folder_name = 1; %% Get heights (quick-look) worksheet in parameter spreadsheet @@ -184,8 +184,6 @@ default.combine.dbin = 1; default.combine.dline = 6; default.combine.DCM = []; -default.combine.three_dim.en = 0; -default.combine.three_dim.layer_fn = ''; default.combine.Nsv = 1; default.combine.theta_rng = [0 0]; default.combine.sv_fh = @array_proc_sv; diff --git a/cresis-toolbox/missions/default_radar_params_2017_Antarctica_TObas_rds.m b/cresis-toolbox/missions/default_radar_params_2017_Antarctica_TObas_rds.m index ccb895c4..ef8361bf 100644 --- a/cresis-toolbox/missions/default_radar_params_2017_Antarctica_TObas_rds.m +++ b/cresis-toolbox/missions/default_radar_params_2017_Antarctica_TObas_rds.m @@ -65,8 +65,6 @@ default.array.dbin = 1; default.array.dline = 6; default.array.DCM = []; -default.array.three_dim.en = 0; -default.array.three_dim.layer_fn = ''; default.array.Nsv = 1; default.array.theta_rng = [0 0]; default.array.sv_fh = @array_proc_sv; diff --git a/cresis-toolbox/missions/default_radar_params_2017_Arctic_Polar5_snow.m b/cresis-toolbox/missions/default_radar_params_2017_Arctic_Polar5_snow.m index 6c14616d..b8d1df6e 100644 --- a/cresis-toolbox/missions/default_radar_params_2017_Arctic_Polar5_snow.m +++ b/cresis-toolbox/missions/default_radar_params_2017_Arctic_Polar5_snow.m @@ -102,8 +102,6 @@ default.array.dbin = 1; default.array.dline = 5; default.array.DCM = []; -default.array.three_dim.en = 0; -default.array.three_dim.layer_fn = ''; default.array.Nsv = 1; default.array.theta_rng = [0 0]; default.array.sv_fh = @array_proc_sv; diff --git a/cresis-toolbox/missions/default_radar_params_2017_Greenland_P3_accum.m b/cresis-toolbox/missions/default_radar_params_2017_Greenland_P3_accum.m index e240ab73..5dfa4cf9 100644 --- a/cresis-toolbox/missions/default_radar_params_2017_Greenland_P3_accum.m +++ b/cresis-toolbox/missions/default_radar_params_2017_Greenland_P3_accum.m @@ -15,7 +15,7 @@ default.xml_file_prefix = 'mcords5'; default.data_file_prefix = 'mcords5'; default.header_load_func = @basic_load_mcords5; -default.header_load_params = struct('clk',1600e6,'presum_bug_fixed',true); +default.header_load_params = struct('clk',200e6,'presum_mode',0); default.xml_version = 2.0; default.noise_50ohm = [0 0 0 0]; @@ -64,7 +64,7 @@ default.records.file.adc_headers = [1:4]; default.records.gps.en = 1; default.records.frame_mode = 0; -default.records.presum_bug_fixed = 1; +default.records.presum_mode = 0; default.records.tmp_fn_uses_adc_folder_name = 1; %% Get heights (quick-look) worksheet in parameter spreadsheet @@ -137,8 +137,6 @@ default.combine.dbin = 1; default.combine.dline = 6; default.combine.DCM = []; -default.combine.three_dim.en = 0; -default.combine.three_dim.layer_fn = ''; default.combine.Nsv = 1; default.combine.theta_rng = [0 0]; default.combine.sv_fh = @array_proc_sv; diff --git a/cresis-toolbox/missions/default_radar_params_2017_Greenland_P3_mcords.m b/cresis-toolbox/missions/default_radar_params_2017_Greenland_P3_mcords.m index 8a6dcc16..3b037a67 100644 --- a/cresis-toolbox/missions/default_radar_params_2017_Greenland_P3_mcords.m +++ b/cresis-toolbox/missions/default_radar_params_2017_Greenland_P3_mcords.m @@ -8,154 +8,178 @@ % % Author: John Paden +% param.season_name = '2017_Greenland_P3'; +% param.radar_name = 'mcords3'; +% +% %% Control parameters (not used in the parameter spreadsheet directly) +% default.xml_file_prefix = 'mcords3'; +% default.data_file_prefix = 'mcords3'; +% default.header_load_func = @basic_load_mcords3; +% default.header_load_params = struct('clk',1e9/9,'presum_mode',1); +% default.xml_version = 2.0; +% +% default.noise_50ohm = [-45.5 -44.3 -44.8 -44.9 -40.2 -40.8 -40.5 -41.4 -39.9 -40.2 -42.0 -43.6 -43.0 -44.1 -44.6]; +% +% default.Pt = 1000 * [1 1 1 1 1 1 1]; +% default.Gt = 7*4; +% default.Ae = 2*0.468 * 0.468; +% +% default.system_loss_dB = 10.^(-5.88/10); +% default.max_DDS_RAM = 40000; +% default.tx_voltage = sqrt(1000*50)*10^(-2/20); +% +% default.iq_mode = 0; +% default.tx_DDS_mask = [1 1 1 1 1 1 1 0]; +% +% default.radar_worksheet_headers = {'Tpd','Tadc','Tadc_adjust','f0','f1','ref_fn','tukey','tx_weights','rx_paths','adc_gains','chan_equal_dB','chan_equal_deg','Tsys','DC_adjust','DDC_mode','DDC_freq'}; +% default.radar_worksheet_headers_type = {'r','r','r','r','r','r','r','r','r','r','r','r','r','r','r','r'}; +% +% default.basic_surf_track_min_time = 2e-6; % Normally -inf for lab test, 2e-6 for flight test +% default.basic_surf_track_Tpd_factor = 1.1; % Normally -inf for lab test, 1.1 for flight test +% default.adc_folder_name = 'board%b'; +% +% if 1 +% % Example 1: Normal configuration: +% % Connect antenna N to WFG N for all N = 1 to 7 +% ref_adc = 14; +% default.txequal.img = [(1:7).', ref_adc*ones(7,1)]; +% default.txequal.ref_wf_adc = 3; +% default.txequal.wf_mapping = [1 2 3 4 5 6 7 0]; +% default.txequal.Hwindow_desired = [1 1 1 1 1 1 1 0]; +% default.txequal.max_DDS_amp = [40000 40000 40000 40000 40000 40000 40000 0]; +% default.txequal.time_delay_desired = [0 0 0 0 0 0 0 0]; +% default.txequal.phase_desired = [0 0 0 0 0 0 0 0]; +% default.txequal.time_validation = [3 3 3 3 3 3 3 3]*1e-9; +% default.txequal.amp_validation = [3 3 3 3 3 3 3 3]; +% default.txequal.phase_validation = [35 35 35 35 35 35 35 35]; +% default.txequal.remove_linear_phase_en = true; +% end + +%% Preprocess parameters param.season_name = '2017_Greenland_P3'; param.radar_name = 'mcords3'; -%% Control parameters (not used in the parameter spreadsheet directly) -default.xml_file_prefix = 'mcords3'; -default.data_file_prefix = 'mcords3'; -default.header_load_func = @basic_load_mcords3; -default.header_load_params = struct('clk',1e9/9,'presum_bug_fixed',false); -default.xml_version = 2.0; - -default.noise_50ohm = [-45.5 -44.3 -44.8 -44.9 -40.2 -40.8 -40.5 -41.4 -39.9 -40.2 -42.0 -43.6 -43.0 -44.1 -44.6]; - -default.Pt = 1000 * [1 1 1 1 1 1 1]; -default.Gt = 7*4; -default.Ae = 2*0.468 * 0.468; - -default.system_loss_dB = 10.^(-5.88/10); -default.max_DDS_RAM = 40000; -default.tx_voltage = sqrt(1000*50)*10^(-2/20); - -default.iq_mode = 0; -default.tx_DDS_mask = [1 1 1 1 1 1 1 0]; - -default.radar_worksheet_headers = {'Tpd','Tadc','Tadc_adjust','f0','f1','ref_fn','tukey','tx_weights','rx_paths','adc_gains','chan_equal_dB','chan_equal_deg','Tsys','DC_adjust','DDC_mode','DDC_freq'}; -default.radar_worksheet_headers_type = {'r','r','r','r','r','r','r','r','r','r','r','r','r','r','r','r'}; - -default.basic_surf_track_min_time = 2e-6; % Normally -inf for lab test, 2e-6 for flight test -default.basic_surf_track_Tpd_factor = 1.1; % Normally -inf for lab test, 1.1 for flight test -default.adc_folder_name = 'board%b'; - -if 1 - % Example 1: Normal configuration: - % Connect antenna N to WFG N for all N = 1 to 7 - ref_adc = 14; - default.txequal.img = [(1:7).', ref_adc*ones(7,1)]; - default.txequal.ref_wf_adc = 3; - default.txequal.wf_mapping = [1 2 3 4 5 6 7 0]; - default.txequal.Hwindow_desired = [1 1 1 1 1 1 1 0]; - default.txequal.max_DDS_amp = [40000 40000 40000 40000 40000 40000 40000 0]; - default.txequal.time_delay_desired = [0 0 0 0 0 0 0 0]; - default.txequal.phase_desired = [0 0 0 0 0 0 0 0]; - default.txequal.time_validation = [3 3 3 3 3 3 3 3]*1e-9; - default.txequal.amp_validation = [3 3 3 3 3 3 3 3]; - default.txequal.phase_validation = [35 35 35 35 35 35 35 35]; - default.txequal.remove_linear_phase_en = true; -end - -%% Vectors worksheet in parameter spreadsheet -default.vectors.gps.time_offset = 1; - -%% Records worksheet in parameter spreadsheet -default.records.geotiff_fn = 'greenland\Landsat-7\mzl7geo_90m_lzw.tif'; +param.config.file.version = 403; +param.config.file.prefix = param.radar_name; +param.config.file.suffix = '.bin'; +param.config.max_time_gap = 10; +param.config.min_seg_size = 2; + +param.config.daq_type = 'cresis'; +param.config.wg_type = 'cresis'; +param.config.header_load_func = @basic_load_mcords3; +param.config.board_map = {'board0','board1','board2','board3'}; +param.config.tx_map = {'','','','','','','',''}; + +param.config.daq.xml_version = 2.0; + +param.config.max_data_rate = 100; +param.config.max_duty_cycle = 0.12; +param.config.prf_multiple = []; % Power supply sync signal that PRF must be a factor of these numbers +param.config.PRI_guard = 10e-6; +param.config.PRI_guard_percentage = 1; +param.config.tx_enable = [1 1 1 1 1 1 1 0]; +param.config.max_tx = 40000; +param.config.max_tx_voltage = sqrt(250*50)*10^(-2/20); % voltage at max_tx + +%% CReSIS parameters +param.config.cresis.clk = 1e9/9; +param.config.cresis.rx_gain_dB = 51.5; +param.config.cresis.gps_file_mask = 'GPS*'; + + +%% Command worksheet +default.cmd.records = 1; +default.cmd.qlook = 1; +default.cmd.generic = 1; + +%% Records worksheet +default.records.gps.time_offset = 1; default.records.file.adcs = [2:16]; -default.records.file.adc_headers = [2:16]; -default.records.gps.en = 1; -default.records.frame_mode = 0; -default.records.presum_bug_fixed = 0; -default.records.tmp_fn_uses_adc_folder_name = 1; - -%% Get heights (quick-look) worksheet in parameter spreadsheet -default.get_heights.qlook.out_path = ''; -default.get_heights.qlook.en = 1; -default.get_heights.block_size = 10000; -default.get_heights.frm_types = {0,[0 1],0,0,-1}; -default.get_heights.coh_noise_method = []; -default.get_heights.coh_noise_arg = []; -default.get_heights.ft_wind = @hanning; -default.get_heights.ft_wind_time = false; -default.get_heights.ft_dec = true; -default.get_heights.pulse_comp = []; -default.get_heights.pulse_rfi.en = []; -default.get_heights.pulse_rfi.inc_ave= []; -default.get_heights.pulse_rfi.thresh_scale = []; -default.get_heights.roll_correction = 0; -default.get_heights.lever_arm_fh = @lever_arm; -default.get_heights.elev_correction = 0; -default.get_heights.B_filter = ones(1,20)/20; -default.get_heights.decimate_factor = 20; -default.get_heights.inc_ave = 10; -default.get_heights.surf.en = 1; -default.get_heights.surf.method = 'threshold'; -default.get_heights.surf.noise_rng = [0 -50 10]; -default.get_heights.surf.min_bin = 2e-6; -default.get_heights.surf.max_bin = []; -default.get_heights.surf.threshold = 9; -default.get_heights.surf.sidelobe = 15; -default.get_heights.surf.medfilt = 3; -default.get_heights.surf.search_rng = [0:2]; - -%% CSARP worksheet in parameter spreadsheet -default.csarp.out_path = ''; -default.csarp.imgs = {[1*ones(4,1),(2:5).'],[2*ones(4,1),(2:5).'],[3*ones(4,1),(2:5).']}; -default.csarp.frm_types = {0,[0 1],0,0,-1}; -default.csarp.chunk_len = 5000; -default.csarp.chunk_overlap = 10; -default.csarp.frm_overlap = 0; -default.csarp.coh_noise_removal = 0; -default.csarp.combine_rx = 0; -default.csarp.time_of_full_support = 3.5e-5; -default.csarp.pulse_rfi.en = []; -default.csarp.pulse_rfi.inc_ave= []; -default.csarp.pulse_rfi.thresh_scale = []; -default.csarp.trim_vals = []; -default.csarp.pulse_comp = 1; -default.csarp.ft_dec = 1; -default.csarp.ft_wind = @hanning; -default.csarp.ft_wind_time = 0; -default.csarp.lever_arm_fh = @lever_arm; -default.csarp.mocomp.en = 1; -default.csarp.mocomp.type = 2; -default.csarp.mocomp.filter = {@butter [2] [0.1000]}; -default.csarp.mocomp.uniform_en = 1; -default.csarp.sar_type = 'f-k'; -default.csarp.sigma_x = 2.5; -default.csarp.sub_aperture_steering = 0; -default.csarp.st_wind = @hanning; -default.csarp.start_eps = 3.15; - -%% Combine worksheet in parameter spreadsheet -default.combine.in_path = ''; -default.combine.array_path = ''; -default.combine.out_path = ''; -default.combine.method = 'standard'; -default.combine.window = @hanning; -default.combine.bin_rng = 0; -default.combine.rline_rng = -5:5; -default.combine.dbin = 1; -default.combine.dline = 6; -default.combine.DCM = []; -default.combine.three_dim.en = 0; -default.combine.three_dim.layer_fn = ''; -default.combine.Nsv = 1; -default.combine.theta_rng = [0 0]; -default.combine.sv_fh = @array_proc_sv; -default.combine.diag_load = 0; -default.combine.Nsig = 2; - -%% Radar worksheet in parameter spreadsheet +default.records.frames.mode = 1; +default.records.frames.geotiff_fn = 'greenland\Landsat-7\mzl7geo_90m_lzw.tif'; +default.records.presum_mode = 1; + +%% Qlook worksheet +default.qlook.out_path = ''; +default.qlook.block_size = 10000; +default.qlook.dec = 50; +default.qlook.inc_dec = 10; +default.qlook.surf.en = 1; +default.qlook.surf.method = 'threshold'; +default.qlook.surf.noise_rng = [0 -50 10]; +default.qlook.surf.min_bin = 1.8e-6; +default.qlook.surf.max_bin = []; +default.qlook.surf.threshold = 15; +default.qlook.surf.sidelobe = 15; +default.qlook.surf.medfilt = 3; +default.qlook.surf.search_rng = [0:2]; + +%% SAR worksheet +default.sar.out_path = ''; +default.sar.imgs = {[1*ones(4,1),(9:12).'],[2*ones(4,1),(9:12).'],[3*ones(4,1),(9:12).']}; +default.sar.frm_types = {0,[0 1],0,0,-1}; +default.sar.chunk_len = 5000; +default.sar.array_rx = 0; +default.sar.time_of_full_support = 3.5e-5; +default.sar.mocomp.en = 1; +default.sar.mocomp.type = 2; +default.sar.mocomp.filter = {@butter [2] [0.1000]}; +default.sar.mocomp.uniform_en = 1; +default.sar.sar_type = 'fk'; +default.sar.sigma_x = 2.5; +default.sar.sub_aperture_steering = 0; +default.sar.st_wind = @hanning; +default.sar.start_eps = 3.15; + +%% Array worksheet +default.array.in_path = ''; +default.array.array_path = ''; +default.array.out_path = ''; +default.array.method = 'standard'; +default.array.window = @hanning; +default.array.bin_rng = 0; +default.array.line_rng = -5:5; +default.array.dbin = 1; +default.array.dline = 6; +default.array.DCM = []; +default.array.tomo_en = 0; +default.array.Nsv = 1; +default.array.theta_rng = [0 0]; +default.array.sv_fh = @array_proc_sv; +default.array.diag_load = 0; +default.array.Nsrc = 2; + +%% Radar worksheet default.radar.fs = 1e9/9; default.radar.Tadc = []; % normally leave empty to use value in file header default.radar.adc_bits = 14; -default.radar.adc_full_scale = 2; -default.radar.rx_paths = [1,1:15]; +default.radar.Vpp_scale = 2; + +%default.radar.rx_paths = [1,1:15]; default.radar.rx_paths = [1 8 9 10 11 1 2 3 4 5 6 7 12 13 14 15]; -default.radar.noise_figure = 2; -default.radar.rx_gain = 51.5; -default.radar.adc_SNR_dB = 70; -default.radar.Tadc_adjust = -1.4455e-06; % System time delay: leave this empty or set it to zero at first, determine this value later using data over surface with known height or from surface multiple +default.radar.wfs.noise_figure = 2; +default.radar.wfs.Tadc_adjust = -1.4455e-06; % System time delay: leave this empty or set it to zero at first, determine this value later using data over surface with known height or from surface multiple + +%% Post worksheet +default.post.data_dirs = {'qlook'}; +default.post.layer_dir = 'layerData'; +default.post.maps_en = 1; +default.post.echo_en = 1; +default.post.layers_en = 0; +default.post.data_en = 0; +default.post.csv_en = 1; +default.post.concat_en = 1; +default.post.pdf_en = 1; +default.post.map.location = 'Greenland'; +default.post.map.type = 'combined'; +default.post.echo.elev_comp = 3; +default.post.echo.depth = '[publish_echogram_switch(Bbad,0.25,Surface_Elev,-3500,DBottom,-100),max(Surface_Elev+100)]'; +default.post.echo.er_ice = 3.15; +default.post.ops.en = 0; +default.post.ops.location = 'arctic'; +default.post.ops.layers = {'bottom','surface'}; +default.post.ops.gaps_dist = [300 60]; defaults = {}; @@ -165,11 +189,11 @@ default.radar.wfs(1).chan_equal_deg = [97.1 25.2 -60.1 -126.1 -38.7 6.4 -0.0 -131.0 0.6 -25.1 -47.5 -86.3 -33.5 165.4 -106.6]; % survey mode -default.get_heights.qlook.img_comb = [3e-06 -inf 1e-06 1e-05 -inf 3e-06]; -default.get_heights.imgs = {[1*ones(4,1),(2:5).'],[2*ones(4,1),(2:5).'],[3*ones(4,1),(2:5).']}; -default.csarp.imgs = default.get_heights.imgs; -default.combine.imgs = default.get_heights.imgs; -default.combine.img_comb = default.get_heights.qlook.img_comb; +default.qlook.qlook.img_comb = [3e-06 -inf 1e-06 1e-05 -inf 3e-06]; +default.qlook.imgs = {[1*ones(4,1),(2:5).'],[2*ones(4,1),(2:5).'],[3*ones(4,1),(2:5).']}; +default.sar.imgs = default.qlook.imgs; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.qlook.img_comb; default.radar.DC_adjust = {'','',''}; default.radar.ref_fn = ''; default.xml_regexp = 'survey_.*thick.xml'; @@ -177,11 +201,11 @@ defaults{end+1} = default; % survey mode -default.get_heights.qlook.img_comb = [3e-06 -inf 1e-06]; -default.get_heights.imgs = {[1*ones(4,1),(2:5).'],[2*ones(4,1),(2:5).']}; -default.csarp.imgs = default.get_heights.imgs; -default.combine.imgs = default.get_heights.imgs; -default.combine.img_comb = default.get_heights.qlook.img_comb; +default.qlook.qlook.img_comb = [3e-06 -inf 1e-06]; +default.qlook.imgs = {[1*ones(4,1),(2:5).'],[2*ones(4,1),(2:5).']}; +default.sar.imgs = default.qlook.imgs; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.qlook.img_comb; default.radar.DC_adjust = {'','',''}; default.radar.ref_fn = ''; default.xml_regexp = 'survey_.*thin_ice.xml'; @@ -189,11 +213,11 @@ defaults{end+1} = default; % high altitude mode -default.get_heights.qlook.img_comb = [1e-05 -inf 3e-06]; -default.get_heights.imgs = {[1*ones(4,1),(2:5).'],[2*ones(4,1),(2:5).']}; -default.csarp.imgs = default.get_heights.imgs; -default.combine.imgs = default.get_heights.imgs; -default.combine.img_comb = default.get_heights.qlook.img_comb; +default.qlook.qlook.img_comb = [1e-05 -inf 3e-06]; +default.qlook.imgs = {[1*ones(4,1),(2:5).'],[2*ones(4,1),(2:5).']}; +default.sar.imgs = default.qlook.imgs; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.qlook.img_comb; default.radar.DC_adjust = {'','',''}; default.radar.ref_fn = ''; default.xml_regexp = 'survey_.*high_altitude.xml'; @@ -201,11 +225,11 @@ defaults{end+1} = default; % deconvolution mode -default.get_heights.qlook.img_comb = []; -default.get_heights.imgs = {[1*ones(4,1),(2:5).'],[2*ones(4,1),(2:5).'],[3*ones(4,1),(2:5).']}; -default.csarp.imgs = default.get_heights.imgs; -default.combine.imgs = default.get_heights.imgs; -default.combine.img_comb = default.get_heights.qlook.img_comb; +default.qlook.qlook.img_comb = []; +default.qlook.imgs = {[1*ones(4,1),(2:5).'],[2*ones(4,1),(2:5).'],[3*ones(4,1),(2:5).']}; +default.sar.imgs = default.qlook.imgs; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.qlook.img_comb; default.radar.DC_adjust = {'','',''}; default.radar.ref_fn = ''; default.xml_regexp = 'survey_.*DECONVOLUTION.xml'; @@ -214,10 +238,10 @@ %% Other settings -default.get_heights.qlook.img_comb = []; -default.get_heights.imgs = []; -default.combine.imgs = default.get_heights.imgs; -default.combine.img_comb = default.get_heights.qlook.img_comb; +default.qlook.qlook.img_comb = []; +default.qlook.imgs = []; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.qlook.img_comb; default.radar.DC_adjust = {'DC_20160413_04_wf1.mat','DC_20160413_04_wf2.mat','DC_20160413_04_wf2.mat'}; default.radar.ref_fn = 'deconv_wf_%w_adc_%a_20160413_06'; diff --git a/cresis-toolbox/missions/default_radar_params_2017_Greenland_P3_snow.m b/cresis-toolbox/missions/default_radar_params_2017_Greenland_P3_snow.m index 95154f5a..e88634f8 100644 --- a/cresis-toolbox/missions/default_radar_params_2017_Greenland_P3_snow.m +++ b/cresis-toolbox/missions/default_radar_params_2017_Greenland_P3_snow.m @@ -31,7 +31,8 @@ param.config.max_time_gap = 10; param.config.min_seg_size = 2; -param.config.ni.clk = 125e6; +% param.config.ni.clk = 125e6; +param.config.cresis.clk = 125e6; %% Command worksheet default.cmd.records = 1; @@ -104,8 +105,6 @@ default.array.dbin = 1; default.array.dline = 6; default.array.DCM = []; -default.array.three_dim.en = 0; -default.array.three_dim.layer_fn = ''; default.array.Nsv = 1; default.array.theta_rng = [0 0]; default.array.sv_fh = @array_proc_sv; diff --git a/cresis-toolbox/missions/default_radar_params_2018_Alaska_SO_snow.m b/cresis-toolbox/missions/default_radar_params_2018_Alaska_SO_snow.m index 5be6742d..eb3ec868 100644 --- a/cresis-toolbox/missions/default_radar_params_2018_Alaska_SO_snow.m +++ b/cresis-toolbox/missions/default_radar_params_2018_Alaska_SO_snow.m @@ -135,8 +135,8 @@ % Survey Mode 2-18 GHz default.radar.wfs(1).f0 = 2e9; default.radar.wfs(1).f1 = 8e9; -default.radar.wfs(1).Tpd = 240e-6; -default.radar.wfs(1).BW_window = [2.1e9 7.75e9]; +default.radar.wfs(1).Tpd = 250e-6; +default.radar.wfs(1).BW_window = [2.100096e9 7.749888e9]; default.radar.wfs(1).t_ref = 0; default.config_regexp = '.*'; diff --git a/cresis-toolbox/missions/default_radar_params_2018_Antarctica_DC8_rds.m b/cresis-toolbox/missions/default_radar_params_2018_Antarctica_DC8_rds.m index 92d7c504..bc7cca7c 100644 --- a/cresis-toolbox/missions/default_radar_params_2018_Antarctica_DC8_rds.m +++ b/cresis-toolbox/missions/default_radar_params_2018_Antarctica_DC8_rds.m @@ -49,7 +49,7 @@ default.records.gps.time_offset = 1; default.records.frames.mode = 1; default.records.frames.geotiff_fn = 'antarctica/Landsat-7/Antarctica_LIMA_480m'; -default.records.presum_bug_fixed = 0; +default.records.presum_mode = 1; %% Qlook worksheet default.qlook.out_path = ''; diff --git a/cresis-toolbox/missions/default_radar_params_2018_Antarctica_DC8_snow.m b/cresis-toolbox/missions/default_radar_params_2018_Antarctica_DC8_snow.m index 261501f0..69b385ba 100644 --- a/cresis-toolbox/missions/default_radar_params_2018_Antarctica_DC8_snow.m +++ b/cresis-toolbox/missions/default_radar_params_2018_Antarctica_DC8_snow.m @@ -102,8 +102,6 @@ default.array.dbin = 1; default.array.dline = 6; default.array.DCM = []; -default.array.three_dim.en = 0; -default.array.three_dim.layer_fn = ''; default.array.Nsv = 1; default.array.theta_rng = [0 0]; default.array.sv_fh = @array_proc_sv; diff --git a/cresis-toolbox/missions/default_radar_params_2018_Antarctica_GroundLab_rds.m b/cresis-toolbox/missions/default_radar_params_2018_Antarctica_GroundLab_rds.m index 5e4794a2..986daebe 100644 --- a/cresis-toolbox/missions/default_radar_params_2018_Antarctica_GroundLab_rds.m +++ b/cresis-toolbox/missions/default_radar_params_2018_Antarctica_GroundLab_rds.m @@ -207,8 +207,6 @@ default.array.dbin = 1; default.array.dline = 6; default.array.DCM = []; -default.array.three_dim.en = 0; -default.array.three_dim.layer_fn = ''; default.array.Nsv = 1; default.array.theta_rng = [0 0]; default.array.sv_fh = @array_proc_sv; diff --git a/cresis-toolbox/missions/default_radar_params_2018_Antarctica_Ground_rds.m b/cresis-toolbox/missions/default_radar_params_2018_Antarctica_Ground_rds.m index e7652c29..c2abaeab 100644 --- a/cresis-toolbox/missions/default_radar_params_2018_Antarctica_Ground_rds.m +++ b/cresis-toolbox/missions/default_radar_params_2018_Antarctica_Ground_rds.m @@ -249,8 +249,6 @@ default.array.dbin = 1; default.array.dline = 6; default.array.DCM = []; -default.array.three_dim.en = 0; -default.array.three_dim.layer_fn = ''; default.array.Nsv = 1; default.array.theta_rng = [0 0]; default.array.sv_fh = @array_proc_sv; diff --git a/cresis-toolbox/missions/default_radar_params_2018_Antarctica_TObas_accum.m b/cresis-toolbox/missions/default_radar_params_2018_Antarctica_TObas_accum.m index 3ea11182..d51f6d3a 100644 --- a/cresis-toolbox/missions/default_radar_params_2018_Antarctica_TObas_accum.m +++ b/cresis-toolbox/missions/default_radar_params_2018_Antarctica_TObas_accum.m @@ -1,11 +1,13 @@ function param = default_radar_params_2018_Antarctica_TObas_accum % param = default_radar_params_2018_Antarctica_TObas_accum % -% Accum: 2018_Antarctica_TObas +% Accum: 2018_Antarctica_TObas and 2019_Antarctica_TObas % % Creates base "param" struct % Creates defaults cell array for each type of radar setting % +% Set the param.season_name to the correct season before running. +% % Author: John Paden %% Preprocess parameters @@ -216,8 +218,6 @@ default.array.dbin = 1; default.array.dline = 6; default.array.DCM = []; -default.array.three_dim.en = 0; -default.array.three_dim.layer_fn = ''; default.array.Nsv = 1; default.array.theta_rng = [0 0]; default.array.sv_fh = @array_proc_sv; @@ -227,7 +227,7 @@ %% Radar worksheet default.radar.adc_bits = 14; default.radar.Vpp_scale = 1.5; % Digital receiver gain is 5, full scale Vpp is 2 -default.radar.Tadc_adjust = 8.3042e-06; % System time delay: leave this empty or set it to zero at first, determine this value later using data over surface with known height or from surface multiple +default.radar.Tadc_adjust = -189e-9; % System time delay: leave this empty or set it to zero at first, determine this value later using data over surface with known height or from surface multiple default.radar.lever_arm_fh = @lever_arm; % default.radar.wfs(1).adc_gains_dB = 27; % Gain from the first LNA to the ADC % default.radar.wfs(2).adc_gains_dB = 45; % Gain from the first LNA to the ADC diff --git a/cresis-toolbox/missions/default_radar_params_2018_Greenland_P3_accum.m b/cresis-toolbox/missions/default_radar_params_2018_Greenland_P3_accum.m index e5afd75c..33553275 100644 --- a/cresis-toolbox/missions/default_radar_params_2018_Greenland_P3_accum.m +++ b/cresis-toolbox/missions/default_radar_params_2018_Greenland_P3_accum.m @@ -38,7 +38,6 @@ %% CReSIS Parameters param.config.cresis.clk = 1.6e9/8; param.config.cresis.rx_gain_dB = 45; -param.config.cresis.presum_bug_fixed = true; %% BAS ACCUM Arena Parameters arena = []; @@ -138,7 +137,7 @@ default.records.frames.mode = 2; default.records.gps.en = 1; default.records.gps.time_offset = 1; -default.records.presum_bug_fixed = 1; +default.records.presum_mode = 0; %% Qlook worksheet default.qlook.out_path = ''; @@ -210,8 +209,6 @@ default.array.dbin = 1; default.array.dline = 6; default.array.DCM = []; -default.array.three_dim.en = 0; -default.array.three_dim.layer_fn = ''; default.array.Nsv = 1; default.array.theta_rng = [0 0]; default.array.sv_fh = @array_proc_sv; diff --git a/cresis-toolbox/missions/default_radar_params_2018_Greenland_P3_rds.m b/cresis-toolbox/missions/default_radar_params_2018_Greenland_P3_rds.m index 29474a5f..79d7155b 100644 --- a/cresis-toolbox/missions/default_radar_params_2018_Greenland_P3_rds.m +++ b/cresis-toolbox/missions/default_radar_params_2018_Greenland_P3_rds.m @@ -50,7 +50,7 @@ default.records.file.adcs = [2:16]; default.records.frames.mode = 1; default.records.frames.geotiff_fn = 'greenland\Landsat-7\mzl7geo_90m_lzw.tif'; -default.records.presum_bug_fixed = 0; +default.records.presum_mode = 1; %% Qlook worksheet default.qlook.out_path = ''; diff --git a/cresis-toolbox/missions/default_radar_params_2018_Greenland_P3_snow.m b/cresis-toolbox/missions/default_radar_params_2018_Greenland_P3_snow.m index ff9c65d2..a1d3e974 100644 --- a/cresis-toolbox/missions/default_radar_params_2018_Greenland_P3_snow.m +++ b/cresis-toolbox/missions/default_radar_params_2018_Greenland_P3_snow.m @@ -1,5 +1,5 @@ -function param = default_radar_params_2018_Greenland_P3_snow -% param = default_radar_params_2018_Greenland_P3_snow +function [param,defaults] = default_radar_params_2018_Greenland_P3_snow +% [param,defaults] = default_radar_params_2018_Greenland_P3_snow % % Snow: 2018_Greenland_P3 % @@ -12,9 +12,6 @@ param.season_name = '2018_Greenland_P3'; param.radar_name = 'snow8'; -param.config.file.version = 8; -param.config.file.prefix = param.radar_name; -param.config.file.suffix = '.bin'; param.config.max_time_gap = 10; param.config.min_seg_size = 2; @@ -32,130 +29,134 @@ param.config.cresis.clk = 125e6; %% Command worksheet -default.cmd.records = 1; -default.cmd.qlook = 1; -default.cmd.generic = 1; +param.cmd.records = 1; +param.cmd.qlook = 1; +param.cmd.generic = 1; %% Records worksheet -default.records.file.boards = [1]; -default.records.frames.geotiff_fn = 'greenland/Landsat-7/Greenland_natural_150m.tif'; -default.records.frames.mode = 2; -default.records.gps.en = 1; -default.records.gps.time_offset = 1; +param.records.file.boards = {''}; +param.records.file.version = 8; +param.records.file.prefix = param.radar_name; +param.records.file.suffix = '.bin'; +param.records.file.clk = 125000000; +param.records.frames.geotiff_fn = 'greenland/Landsat-7/Greenland_natural_150m.tif'; +param.records.frames.mode = 2; +param.records.gps.en = 1; +param.records.gps.time_offset = 1; %% Qlook worksheet -default.qlook.img_comb = []; -default.qlook.imgs = {[1 1]}; -default.qlook.out_path = ''; -default.qlook.block_size = 2000; -default.qlook.motion_comp = 0; -default.qlook.dec = 4; -default.qlook.inc_dec = 5; -default.qlook.surf.en = 1; -default.qlook.surf.min_bin = 1e-6; -default.qlook.surf.method = 'threshold'; -default.qlook.surf.threshold = 17; -default.qlook.surf.filter_len = 7; -default.qlook.surf.sidelobe = 19; -default.qlook.surf.max_diff = 1.2e-7; -default.qlook.surf.noise_rng = [100 -700 -300]; -default.qlook.surf.search_rng = [0:9]; +param.qlook.img_comb = []; +param.qlook.imgs = {[1 1]}; +param.qlook.out_path = ''; +param.qlook.block_size = 2000; +param.qlook.motion_comp = 0; +param.qlook.dec = 4; +param.qlook.inc_dec = 5; +param.qlook.surf.en = 1; +param.qlook.surf.min_bin = 1e-6; +param.qlook.surf.method = 'threshold'; +param.qlook.surf.threshold = 17; +param.qlook.surf.filter_len = 7; +param.qlook.surf.sidelobe = 19; +param.qlook.surf.max_diff = 1.2e-7; +param.qlook.surf.noise_rng = [100 -700 -300]; +param.qlook.surf.search_rng = [0:9]; %% SAR worksheet -default.sar.out_path = ''; -default.sar.imgs = default.qlook.imgs; -default.sar.frm_types = {0,[0 1],0,0,-1}; -default.sar.chunk_len = 2000; -default.sar.frm_overlap = 0; -default.sar.coh_noise_removal = 0; -default.sar.combine_rx = 0; -default.sar.time_of_full_support = inf; -default.sar.pulse_rfi.en = []; -default.sar.pulse_rfi.inc_ave= []; -default.sar.pulse_rfi.thresh_scale = []; -default.sar.trim_vals = []; -default.sar.pulse_comp = 1; -default.sar.ft_dec = 1; -default.sar.ft_wind = @hanning; -default.sar.ft_wind_time = 0; -default.sar.lever_arm_fh = @lever_arm; -default.sar.mocomp.en = 1; -default.sar.mocomp.type = 2; -default.sar.mocomp.filter = {@butter [2] [0.1000]}; -default.sar.mocomp.uniform_en = 1; -default.sar.sar_type = 'fk'; -default.sar.sigma_x = 1; -default.sar.sub_aperture_steering = 0; -default.sar.st_wind = @hanning; -default.sar.start_eps = 3.15; +param.sar.out_path = ''; +param.sar.imgs = param.qlook.imgs; +param.sar.frm_types = {0,[0 1],0,0,-1}; +param.sar.chunk_len = 2000; +param.sar.frm_overlap = 0; +param.sar.coh_noise_removal = 0; +param.sar.combine_rx = 0; +param.sar.time_of_full_support = inf; +param.sar.pulse_rfi.en = []; +param.sar.pulse_rfi.inc_ave= []; +param.sar.pulse_rfi.thresh_scale = []; +param.sar.trim_vals = []; +param.sar.pulse_comp = 1; +param.sar.ft_dec = 1; +param.sar.ft_wind = @hanning; +param.sar.ft_wind_time = 0; +param.sar.lever_arm_fh = @lever_arm; +param.sar.mocomp.en = 1; +param.sar.mocomp.type = 2; +param.sar.mocomp.filter = {@butter [2] [0.1000]}; +param.sar.mocomp.uniform_en = 1; +param.sar.sar_type = 'fk'; +param.sar.sigma_x = 1; +param.sar.sub_aperture_steering = 0; +param.sar.st_wind = @hanning; +param.sar.start_eps = 3.15; %% Array worksheet -default.array.in_path = ''; -default.array.array_path = ''; -default.array.out_path = ''; -default.array.imgs = default.qlook.imgs; -default.array.img_comb = default.qlook.img_comb; -default.array.method = 'standard'; -default.array.window = @hanning; -default.array.bin_rng = 0; -default.array.rline_rng = -5:5; -default.array.dbin = 1; -default.array.dline = 6; -default.array.DCM = []; -default.array.three_dim.en = 0; -default.array.three_dim.layer_fn = ''; -default.array.Nsv = 1; -default.array.theta_rng = [0 0]; -default.array.sv_fh = @array_proc_sv; -default.array.diag_load = 0; -default.array.Nsig = 2; +param.array.in_path = ''; +param.array.array_path = ''; +param.array.out_path = ''; +param.array.imgs = param.qlook.imgs; +param.array.img_comb = param.qlook.img_comb; +param.array.method = 'standard'; +param.array.window = @hanning; +param.array.bin_rng = 0; +param.array.rline_rng = -5:5; +param.array.dbin = 1; +param.array.dline = 6; +param.array.DCM = []; +param.array.Nsv = 1; +param.array.theta_rng = [0 0]; +param.array.sv_fh = @array_proc_sv; +param.array.diag_load = 0; +param.array.Nsig = 2; %% Radar worksheet -default.radar.prf = 1/256e-6; -default.radar.fs = 250e6; -default.radar.adc_bits = 14; -default.radar.Vpp_scale = 2; % Digital receiver gain is 5, full scale Vpp is 2 -default.radar.Tadc_adjust = []; % System time delay: leave this empty or set it to zero at first, determine this value later using data over surface with known height or from surface multiple -default.radar.lever_arm_fh = @lever_arm; +param.radar.prf = 1/256e-6; +param.radar.fs = 250e6; +param.radar.adc_bits = 14; +param.radar.Vpp_scale = 2; % Digital receiver gain is 5, full scale Vpp is 2 +param.radar.Tadc_adjust = []; % System time delay: leave this empty or set it to zero at first, determine this value later using data over surface with known height or from surface multiple +param.radar.lever_arm_fh = @lever_arm; chan_equal_Tsys = [0]/1e9; chan_equal_dB = [0]; chan_equal_deg = [0]; for wf = 1:1 - default.radar.wfs(wf).tx_weights = 1; % Watts - default.radar.wfs(wf).adc_gains_dB = 95.8; % Radiometric calibration to 1/R^2 - default.radar.wfs(wf).rx_paths = [1]; % ADC to rx path mapping - default.radar.wfs(wf).ref_fn = ''; - default.radar.wfs(wf).chan_equal_Tsys = chan_equal_Tsys; - default.radar.wfs(wf).chan_equal_dB = chan_equal_dB; - default.radar.wfs(wf).chan_equal_deg = chan_equal_deg; - default.radar.wfs(wf).adcs = [1]; - default.radar.wfs(wf).nz_trim = {[0 0],[0 2],[0 0],[0 0]}; - default.radar.wfs(wf).nz_valid = [0 1 2 3]; + param.radar.wfs(wf).tx_weights = 1; % Watts + param.radar.wfs(wf).adc_gains_dB = 95.8; % Radiometric calibration to 1/R^2 + param.radar.wfs(wf).rx_paths = [1]; % ADC to rx path mapping + param.radar.wfs(wf).ref_fn = ''; + param.radar.wfs(wf).chan_equal_Tsys = chan_equal_Tsys; + param.radar.wfs(wf).chan_equal_dB = chan_equal_dB; + param.radar.wfs(wf).chan_equal_deg = chan_equal_deg; + param.radar.wfs(wf).adcs = [1]; + param.radar.wfs(wf).nz_trim = {[0 0],[0 2],[0 0],[0 0]}; + param.radar.wfs(wf).nz_valid = [0 1 2 3]; end %% Post worksheet -default.post.data_dirs = {'qlook'}; -default.post.layer_dir = 'layerData'; -default.post.maps_en = 1; -default.post.echo_en = 1; -default.post.layers_en = 0; -default.post.data_en = 0; -default.post.csv_en = 1; -default.post.concat_en = 1; -default.post.pdf_en = 1; -default.post.map.location = 'Greenland'; -default.post.map.type = 'combined'; -default.post.echo.elev_comp = 2; -default.post.echo.depth = '[min(Surface_Depth)-2 max(Surface_Depth)+25]'; -% default.post.echo.elev_comp = 3; -% default.post.echo.depth = '[min(Surface_Elev)-25 max(Surface_Elev)+2]'; -default.post.echo.er_ice = round((1+0.51*0.3)^3 * 100)/100; -default.post.ops.location = 'arctic'; +param.post.data_dirs = {'qlook'}; +param.post.layer_dir = 'layerData'; +param.post.maps_en = 1; +param.post.echo_en = 1; +param.post.layers_en = 0; +param.post.data_en = 0; +param.post.csv_en = 1; +param.post.concat_en = 1; +param.post.pdf_en = 1; +param.post.map.location = 'Greenland'; +param.post.map.type = 'combined'; +param.post.echo.elev_comp = 2; +param.post.echo.depth = '[min(Surface_Depth)-2 max(Surface_Depth)+25]'; +% param.post.echo.elev_comp = 3; +% param.post.echo.depth = '[min(Surface_Elev)-25 max(Surface_Elev)+2]'; +param.post.echo.er_ice = round((1+0.51*0.3)^3 * 100)/100; +param.post.ops.location = 'arctic'; %% Radar Settings defaults = {}; +default = param; + % Survey Mode 2-18 GHz default.radar.wfs(1).f0 = 2e9; default.radar.wfs(1).f1 = 18e9; @@ -188,7 +189,3 @@ default.config_regexp = '.*'; default.name = 'Survey Mode 2-14 GHz Dual Waveform'; defaults{end+1} = default; - -%% Add default settings - -param.config.defaults = defaults; diff --git a/cresis-toolbox/missions/default_radar_params_2018_Greenland_Polar6_mcords.m b/cresis-toolbox/missions/default_radar_params_2018_Greenland_Polar6_mcords.m index 4bafa6ab..cd8fd748 100644 --- a/cresis-toolbox/missions/default_radar_params_2018_Greenland_Polar6_mcords.m +++ b/cresis-toolbox/missions/default_radar_params_2018_Greenland_Polar6_mcords.m @@ -15,7 +15,7 @@ default.xml_file_prefix = 'mcords5'; default.data_file_prefix = 'mcords5'; default.header_load_func = @basic_load_mcords5; -default.header_load_params = struct('clk',1600e6,'presum_bug_fixed',true); +default.header_load_params = struct('clk',200e6,'presum_mode',0); default.xml_version = 2.0; default.noise_50ohm = [-39.8 -41.0 -40.1 -39.6 -38.4 -39.1 -38.3 -39.6 ]; @@ -191,7 +191,7 @@ default.records.file.adc_headers = [1:8]; default.records.gps.en = 1; default.records.frame_mode = 0; -default.records.presum_bug_fixed = 1; +default.records.presum_mode = 0; default.records.tmp_fn_uses_adc_folder_name = 1; %% Get heights (quick-look) worksheet in parameter spreadsheet @@ -264,8 +264,6 @@ default.combine.dbin = 1; default.combine.dline = 6; default.combine.DCM = []; -default.combine.three_dim.en = 0; -default.combine.three_dim.layer_fn = ''; default.combine.Nsv = 1; default.combine.theta_rng = [0 0]; default.combine.sv_fh = @array_proc_sv; diff --git a/cresis-toolbox/missions/default_radar_params_2018_Greenland_Polar6_snow.m b/cresis-toolbox/missions/default_radar_params_2018_Greenland_Polar6_snow.m index b2209676..b7ab99fd 100644 --- a/cresis-toolbox/missions/default_radar_params_2018_Greenland_Polar6_snow.m +++ b/cresis-toolbox/missions/default_radar_params_2018_Greenland_Polar6_snow.m @@ -104,8 +104,6 @@ default.array.dbin = 1; default.array.dline = 5; default.array.DCM = []; -default.array.three_dim.en = 0; -default.array.three_dim.layer_fn = ''; default.array.Nsv = 1; default.array.theta_rng = [0 0]; default.array.sv_fh = @array_proc_sv; diff --git a/cresis-toolbox/missions/default_radar_params_2019_Antarctica_GV_rds.m b/cresis-toolbox/missions/default_radar_params_2019_Antarctica_GV_rds.m index e04f0e2c..1d98fbfd 100644 --- a/cresis-toolbox/missions/default_radar_params_2019_Antarctica_GV_rds.m +++ b/cresis-toolbox/missions/default_radar_params_2019_Antarctica_GV_rds.m @@ -49,7 +49,7 @@ default.records.gps.time_offset = 1; default.records.frames.mode = 2; default.records.frames.geotiff_fn = fullfile('antarctica','Landsat-7','Antarctica_LIMA_480m.tif'); -default.records.presum_bug_fixed = 0; +default.records.presum_mode = 1; %% Qlook worksheet default.qlook.out_path = ''; diff --git a/cresis-toolbox/missions/default_radar_params_2019_Antarctica_Ground_rds.m b/cresis-toolbox/missions/default_radar_params_2019_Antarctica_Ground_rds.m index 926fb070..48408306 100644 --- a/cresis-toolbox/missions/default_radar_params_2019_Antarctica_Ground_rds.m +++ b/cresis-toolbox/missions/default_radar_params_2019_Antarctica_Ground_rds.m @@ -1,5 +1,5 @@ -function [param,defaults] = default_radar_params_2019_Antarctica_Ground_mcords6 -% [param,defaults] = default_radar_params_2019_Antarctica_Ground_mcords6 +function [param,defaults] = default_radar_params_2019_Antarctica_Ground_rds +% [param,defaults] = default_radar_params_2019_Antarctica_Ground_rds % % MCoRDS6: 2019_Antarctica_Ground % @@ -263,8 +263,6 @@ default.array.dbin = 1; default.array.dline = 6; default.array.DCM = []; -default.array.three_dim.en = 0; -default.array.three_dim.layer_fn = ''; default.array.Nsv = 1; default.array.theta_rng = [0 0]; default.array.sv_fh = @array_proc_sv; diff --git a/cresis-toolbox/missions/default_radar_params_2019_Antarctica_LC130_rds.m b/cresis-toolbox/missions/default_radar_params_2019_Antarctica_LC130_rds.m new file mode 100644 index 00000000..7db755c0 --- /dev/null +++ b/cresis-toolbox/missions/default_radar_params_2019_Antarctica_LC130_rds.m @@ -0,0 +1,406 @@ +function [param,defaults] = default_radar_params_2019_Antarctica_Ground_rds +% [param,defaults] = default_radar_params_2019_Antarctica_Ground_rds +% +% RDS: 2019_Antarctica_LC130 +% +% Creates base "param" struct +% Creates defaults cell array for each type of radar setting +% +% Author: John Paden + +%% Preprocess parameters +param.season_name = '2019_Antarctica_LC130'; +param.radar_name = 'mcords6'; + +param.config.daq_type = 'arena'; +param.config.wg_type = 'arena'; +param.config.header_load_func = @basic_load_arena; +param.config.board_map = {'digrx0','digrx1','digrx2','digrx3'}; +param.config.tx_map = {'awg0','awg1','awg2','awg3'}; + +param.config.file.version = 103; +param.config.file.prefix = param.radar_name; +param.config.file.suffix = '.bin'; +param.config.max_time_gap = 10; +param.config.min_seg_size = 1; + +param.config.max_data_rate = 60; +param.config.max_duty_cycle = 0.1; +param.config.prf_multiple = [10e6 10e6/20]; % Power supply sync signal that PRF must be a factor of these numbers +param.config.PRI_guard = 1e-6; +param.config.PRI_guard_percentage = 1; +param.config.tx_enable = [1 1 1 1]; +param.config.max_tx = 1.0; +param.config.max_tx_voltage = sqrt(1000*50)*10^(-2/20); % voltage at max_tx + +%% MCoRDS6 Arena Parameters +arena = []; +fs = 640e6; +fs_dac = 1280e6; +subsystem_idx = 0; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'ARENA-CTU'; +arena.subsystem(subsystem_idx).subSystem{1} = 'ctu'; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'ARENA0'; +arena.subsystem(subsystem_idx).subSystem{1} = 'awg0'; +arena.subsystem(subsystem_idx).subSystem{2} = 'digrx0'; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'ARENA1'; +arena.subsystem(subsystem_idx).subSystem{1} = 'awg1'; +arena.subsystem(subsystem_idx).subSystem{2} = 'digrx1'; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'ARENA2'; +arena.subsystem(subsystem_idx).subSystem{1} = 'awg2'; +arena.subsystem(subsystem_idx).subSystem{2} = 'digrx2'; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'ARENA3'; +arena.subsystem(subsystem_idx).subSystem{1} = 'awg3'; +arena.subsystem(subsystem_idx).subSystem{2} = 'digrx3'; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'Data Server'; + +dac_idx = 0; +dac_idx = dac_idx + 1; +arena.dac(dac_idx).name = 'awg0'; +arena.dac(dac_idx).type = 'dac-ad9129_0012'; +arena.dac(dac_idx).dacClk = fs_dac; +arena.dac(dac_idx).desiredAlignMin = -10; +arena.dac(dac_idx).desiredAlignMax = 4; +arena.dac(dac_idx).dcoPhase = 80; +dac_idx = dac_idx + 1; +arena.dac(dac_idx).name = 'awg1'; +arena.dac(dac_idx).type = 'dac-ad9129_0012'; +arena.dac(dac_idx).dacClk = fs_dac; +arena.dac(dac_idx).desiredAlignMin = -14; +arena.dac(dac_idx).desiredAlignMax = 0; +arena.dac(dac_idx).dcoPhase = 80; +dac_idx = dac_idx + 1; +arena.dac(dac_idx).name = 'awg2'; +arena.dac(dac_idx).type = 'dac-ad9129_0012'; +arena.dac(dac_idx).dacClk = fs_dac; +arena.dac(dac_idx).desiredAlignMin = 4; +arena.dac(dac_idx).desiredAlignMax = 18; +arena.dac(dac_idx).dcoPhase = 80; +dac_idx = dac_idx + 1; +arena.dac(dac_idx).name = 'awg3'; +arena.dac(dac_idx).type = 'dac-ad9129_0012'; +arena.dac(dac_idx).dacClk = fs_dac; +arena.dac(dac_idx).desiredAlignMin = 0; +arena.dac(dac_idx).desiredAlignMax = 14; +arena.dac(dac_idx).dcoPhase = 80; + +adc_idx = 0; +adc_idx = adc_idx + 1; +arena.adc(adc_idx).name = 'digrx0'; +arena.adc(adc_idx).type = 'adc-ad9680_0017'; +arena.adc(adc_idx).sampFreq = fs; +arena.adc(adc_idx).adcMode = 2; +arena.adc(adc_idx).desiredAlignMin = -12; +arena.adc(adc_idx).desiredAlignMax = 8; +arena.adc(adc_idx).stream = 'tcp'; +arena.adc(adc_idx).ip = '10.0.0.100'; +arena.adc(adc_idx).outputSelect = 0; +arena.adc(adc_idx).gain_dB = [2.4 2.4]; +adc_idx = adc_idx + 1; +arena.adc(adc_idx).name = 'digrx1'; +arena.adc(adc_idx).type = 'adc-ad9680_0017'; +arena.adc(adc_idx).sampFreq = fs; +arena.adc(adc_idx).adcMode = 2; +arena.adc(adc_idx).desiredAlignMin = -17; +arena.adc(adc_idx).desiredAlignMax = 3; +arena.adc(adc_idx).stream = 'tcp'; +arena.adc(adc_idx).ip = '10.0.0.100'; +arena.adc(adc_idx).outputSelect = 0; +arena.adc(adc_idx).gain_dB = [2.4 2.4]; +adc_idx = adc_idx + 1; +arena.adc(adc_idx).name = 'digrx2'; +arena.adc(adc_idx).type = 'adc-ad9680_0017'; +arena.adc(adc_idx).sampFreq = fs; +arena.adc(adc_idx).adcMode = 2; +arena.adc(adc_idx).desiredAlignMin = -21; +arena.adc(adc_idx).desiredAlignMax = -1; +arena.adc(adc_idx).stream = 'tcp'; +arena.adc(adc_idx).ip = '10.0.0.100'; +arena.adc(adc_idx).outputSelect = 0; +arena.adc(adc_idx).gain_dB = [2.4 2.4]; +adc_idx = adc_idx + 1; +arena.adc(adc_idx).name = 'digrx3'; +arena.adc(adc_idx).type = 'adc-ad9680_0017'; +arena.adc(adc_idx).sampFreq = fs; +arena.adc(adc_idx).adcMode = 2; +arena.adc(adc_idx).desiredAlignMin = -20; +arena.adc(adc_idx).desiredAlignMax = 0; +arena.adc(adc_idx).stream = 'tcp'; +arena.adc(adc_idx).ip = '10.0.0.100'; +arena.adc(adc_idx).outputSelect = 0; +arena.adc(adc_idx).gain_dB = [2.4 2.4]; + +daq_idx = 0; +daq_idx = daq_idx + 1; +arena.daq(daq_idx).name = 'daq0'; +arena.daq(daq_idx).type = 'daq_0001'; +arena.daq(daq_idx).auxDir = '/data/'; +arena.daq(daq_idx).fileStripe = '/data/%b/'; +arena.daq(daq_idx).fileName = 'mcords'; + +arena.system.name = 'ku0001'; + +arena.param.tx_max = [1 1 1 1]; +arena.param.PA_setup_time = 2e-6; % Time required to enable PA before transmit +arena.param.TTL_time_delay = 0.0; % TTL time delay relative to transmit start +arena.param.ADC_time_delay = 3.0720e-6; % ADC time delay relative to transmit start + +% mode 0, subchannel 0, board_idx 1 is wf-adc 1-1 +% mode 0, subchannel 0, board_idx 2 is wf-adc 2-1 + +arena.psc.type = 'psc_0003'; + +arena.daq.type = 'daq_0001'; + +arena.ctu.name = 'ctu'; +arena.ctu.type = 'ctu_001D'; +if 0 + % External GPS + arena.ctu.nmea = 31; + arena.ctu.nmea_baud = 9600; + arena.ctu.pps = 10; + arena.ctu.pps_polarity = 1; +else + % Internal GPS + arena.ctu.nmea = 60; + arena.ctu.nmea_baud = 115200; + arena.ctu.pps = 63; + arena.ctu.pps_polarity = 1; +end +idx = 0; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'EPRI'; +arena.ctu.out.bit_group(idx).bits = 0; +arena.ctu.out.bit_group(idx).epri = [1 0 0]; +arena.ctu.out.bit_group(idx).pri = [0 0 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'PRI'; +arena.ctu.out.bit_group(idx).bits = 1; +arena.ctu.out.bit_group(idx).epri = [1 0 0]; +arena.ctu.out.bit_group(idx).pri = [1 0 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'TR'; +arena.ctu.out.bit_group(idx).bits = 2; +arena.ctu.out.bit_group(idx).epri = [1 0 0]; +arena.ctu.out.bit_group(idx).pri = [1 0 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'Isolation'; +arena.ctu.out.bit_group(idx).bits = 3; +arena.ctu.out.bit_group(idx).epri = [1 1 0]; +arena.ctu.out.bit_group(idx).pri = [1 1 0]; +% idx = idx + 1; +% arena.ctu.out.bit_group(idx).name = 'Atten'; +% arena.ctu.out.bit_group(idx).bits = 4:8; +% arena.ctu.out.bit_group(idx).epri = [0 0]; +% arena.ctu.out.bit_group(idx).pri = [0 0]; + +arena.ctu.out.time_cmd = {'2e-6+param.wfs(wf).Tpd+0.5e-6' '2e-6+param.wfs(wf).Tpd+2.5e-6' '2/param.prf'}; + +param.config.arena = arena; + +%% Command worksheet +default.cmd.records = 1; +default.cmd.qlook = 1; +default.cmd.generic = 1; + +%% Records worksheet +default.records.gps.time_offset = 0; +default.records.frames.geotiff_fn = 'antarctica/Landsat-7/Antarctica_LIMA_480m.tif'; +default.records.frames.mode = 1; + +%% Quick Look worksheet +default.qlook.out_path = ''; +default.qlook.block_size = 5000; +default.qlook.motion_comp = 0; +default.qlook.dec = 20; +default.qlook.inc_dec = 10; +default.qlook.surf.en = 1; +default.qlook.surf.method = 'fixed'; +default.qlook.surf.fixed_value = 0; + +%% SAR worksheet +default.sar.out_path = ''; +default.sar.frm_types = {0,[0 1],0,0,-1}; +default.sar.chunk_len = 5000; +default.sar.chunk_overlap = 10; +default.sar.frm_overlap = 0; +default.sar.coh_noise_removal = 0; +default.sar.combine_rx = 0; +default.sar.time_of_full_support = inf; +default.sar.pulse_rfi.en = []; +default.sar.pulse_rfi.inc_ave= []; +default.sar.pulse_rfi.thresh_scale = []; +default.sar.trim_vals = []; +default.sar.pulse_comp = 1; +default.sar.ft_dec = 1; +default.sar.ft_wind = @hanning; +default.sar.ft_wind_time = 0; +default.sar.lever_arm_fh = @lever_arm; +default.sar.mocomp.en = 1; +default.sar.mocomp.type = 2; +default.sar.mocomp.filter = {@butter [2] [0.1000]}; +default.sar.mocomp.uniform_en = 1; +default.sar.sar_type = 'fk'; +default.sar.sigma_x = 2.5; +default.sar.sub_aperture_steering = 0; +default.sar.st_wind = @hanning; +default.sar.start_eps = 3.15; + +%% Array worksheet +default.array.in_path = ''; +default.array.array_path = ''; +default.array.out_path = ''; +default.array.method = 'standard'; +default.array.window = @hanning; +default.array.bin_rng = 0; +default.array.rline_rng = -5:5; +default.array.dbin = 1; +default.array.dline = 6; +default.array.DCM = []; +default.array.three_dim.en = 0; +default.array.three_dim.layer_fn = ''; +default.array.Nsv = 1; +default.array.theta_rng = [0 0]; +default.array.sv_fh = @array_proc_sv; +default.array.diag_load = 0; +default.array.Nsig = 2; + +%% Radar worksheet +default.radar.adc_bits = 14; +default.radar.Vpp_scale = 1.37; +default.radar.Tadc_adjust = 8.3042e-06; % System time delay: leave this empty or set it to zero at first, determine this value later using data over surface with known height or from surface multiple +default.radar.lever_arm_fh = @lever_arm; +Tsys = [0 0 0 0 0 0 0 0]/1e9; +chan_equal_dB = [0 0 0 0 0 0 0 0]; +chan_equal_deg = [0 0 0 0 0 0 0 0]; + +%% Post worksheet +default.post.data_dirs = {'qlook'}; +default.post.layer_dir = 'layerData'; +default.post.maps_en = 1; +default.post.echo_en = 1; +default.post.layers_en = 0; +default.post.data_en = 0; +default.post.csv_en = 1; +default.post.concat_en = 1; +default.post.pdf_en = 1; +default.post.map.location = 'Antarctica'; +default.post.map.type = 'combined'; +default.post.echo.elev_comp = 2; +default.post.echo.depth = '[min(Surface_Depth)-10 max(Surface_Depth)+4500]'; +% default.post.echo.elev_comp = 3; +% default.post.echo.depth = '[min(Surface_Elev)-4500 max(Surface_Elev)+10]'; +default.post.echo.er_ice = 3.15; +default.post.ops.location = 'antarctic'; + +%% Analysis worksheet +default.analysis.block_size = 5000; +default.analysis.imgs = {[1*ones(8,1),(1:8).'],[2*ones(8,1),(1:8).'],[3*ones(8,1),(1:8).']}; +cmd_idx = 0; +cmd_idx = cmd_idx + 1; +default.analysis.cmd{cmd_idx}.method = 'statistics'; +default.analysis.cmd{cmd_idx}.out_path = 'analysis_mean'; +default.analysis.cmd{cmd_idx}.block_ave = 1000; +default.analysis.cmd{cmd_idx}.stats = {@(x)nanmean(x.*conj(x),2)}; +cmd_idx = cmd_idx + 1; +default.analysis.cmd{cmd_idx}.method = 'statistics'; +default.analysis.cmd{cmd_idx}.out_path = 'analysis_freq'; +default.analysis.cmd{cmd_idx}.block_ave = 1000; +default.analysis.cmd{cmd_idx}.start_time = 's=es.Tend-es.Tpd-3e-6;'; +default.analysis.cmd{cmd_idx}.stop_time = 's=es.Tend-es.Tpd;'; +default.analysis.cmd{cmd_idx}.stats = {@(x)mean(abs(fft(x)).^2,2) @(x)mean(abs(fft(fir_dec(x,10))).^2,2) @(x)mean(abs(fft(fir_dec(x,100))).^2,2)}; +cmd_idx = cmd_idx + 1; +default.analysis.cmd{cmd_idx}.method = 'statistics'; +default.analysis.cmd{cmd_idx}.out_path = 'analysis_max'; +default.analysis.cmd{cmd_idx}.block_ave = 1; +default.analysis.cmd{cmd_idx}.pulse_comp = 1; +default.analysis.cmd{cmd_idx}.start_time = 's=min(es.Tend-2*es.Tpd,es.Tpd+6e-6);'; +default.analysis.cmd{cmd_idx}.stats = {'analysis_task_stats_max'}; +cmd_idx = cmd_idx + 1; +default.analysis.cmd{cmd_idx}.method = 'statistics'; +default.analysis.cmd{cmd_idx}.en = 0; +default.analysis.cmd{cmd_idx}.out_path = 'analysis_kx'; +default.analysis.cmd{cmd_idx}.block_ave = 5000; +default.analysis.cmd{cmd_idx}.stats = {'analysis_task_stats_kx'}; +default.analysis.cmd{cmd_idx}.kx = 1000; + + +%% Radar Settings + +defaults = {}; + +% Noise Mode +default.records.data_map = {[2 0 1 1;2 1 1 2;5 0 2 1;5 1 2 2;8 0 3 1;8 1 3 2],[2 0 1 3;2 1 1 4;5 0 2 3;5 1 2 4;8 0 3 3;8 1 3 4],[2 0 1 5;2 1 1 6;5 0 2 5;5 1 2 6;8 0 3 5;8 1 3 6],[2 0 1 7;2 1 1 8;5 0 2 7;5 1 2 8;8 0 3 7;8 1 3 8]}; +default.qlook.img_comb = []; +default.qlook.imgs = {[1*ones(8,1),(1:8).'],[2*ones(8,1),(1:8).'],[3*ones(8,1),(1:8).']}; +default.sar.imgs = default.qlook.imgs; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.radar.ref_fn = ''; +for wf = 1:3 + default.radar.wfs(wf).Tsys = Tsys; + default.radar.wfs(wf).chan_equal_dB = chan_equal_dB; + default.radar.wfs(wf).chan_equal_deg = chan_equal_deg; + default.radar.wfs(wf).adcs = [1 2 3 4 5 6 7 8]; + default.radar.wfs(wf).rx_paths = [2 4 1 3 5 6 7 8]; + default.radar.wfs(wf).tx_paths = [1 2 3 4]; + default.radar.wfs(wf).adc_gains_dB = [45 45 45 45 45 45 45 45]; % Gain from the first LNA to the ADC +end + +default.config_regexp = '.*NOISE.*'; +default.name = 'Noise Mode 180-210 MHz'; +defaults{end+1} = default; + +% Survey Mode +default.records.data_map = {[2 0 1 1;2 1 1 2;5 0 2 1;5 1 2 2;8 0 3 1;8 1 3 2],[2 0 1 3;2 1 1 4;5 0 2 3;5 1 2 4;8 0 3 3;8 1 3 4],[2 0 1 5;2 1 1 6;5 0 2 5;5 1 2 6;8 0 3 5;8 1 3 6],[2 0 1 7;2 1 1 8;5 0 2 7;5 1 2 8;8 0 3 7;8 1 3 8]}; +default.qlook.img_comb = [3e-6 -inf 0.5e-6 10e-06 -inf 3e-06]; +default.qlook.imgs = {[1*ones(8,1),(1:8).'],[2*ones(8,1),(1:8).'],[3*ones(8,1),(1:8).']}; +default.sar.imgs = default.qlook.imgs; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.radar.ref_fn = ''; +for wf = 1:3 + default.radar.wfs(wf).Tsys = Tsys; + default.radar.wfs(wf).chan_equal_dB = chan_equal_dB; + default.radar.wfs(wf).chan_equal_deg = chan_equal_deg; + default.radar.wfs(wf).adcs = [1 2 3 4 5 6 7 8]; + default.radar.wfs(wf).rx_paths = [2 4 1 3 5 6 7 8]; + default.radar.wfs(wf).tx_paths = [1 2 3 4]; + default.radar.wfs(wf).adc_gains_dB = [46 46 46 46 46 46 46 46]; % Gain from the first LNA to the ADC +end + +default.config_regexp = '.*survey.*'; +default.name = 'Survey Mode 180-210 MHz'; +defaults{end+1} = default; + +% Other settings +default.records.data_map = {[2 0 1 1;2 1 1 2;5 0 2 1;5 1 2 2;8 0 3 1;8 1 3 2],[2 0 1 3;2 1 1 4;5 0 2 3;5 1 2 4;8 0 3 3;8 1 3 4],[2 0 1 5;2 1 1 6;5 0 2 5;5 1 2 6;8 0 3 5;8 1 3 6],[2 0 1 7;2 1 1 8;5 0 2 7;5 1 2 8;8 0 3 7;8 1 3 8]}; +default.qlook.img_comb = []; +default.qlook.imgs = {[1*ones(8,1),(1:8).'],[2*ones(8,1),(1:8).'],[3*ones(8,1),(1:8).']}; +default.sar.imgs = default.qlook.imgs; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.radar.ref_fn = ''; +for wf = 1:3 + default.radar.wfs(wf).Tsys = Tsys; + default.radar.wfs(wf).chan_equal_dB = chan_equal_dB; + default.radar.wfs(wf).chan_equal_deg = chan_equal_deg; + default.radar.wfs(wf).adcs = [1 2 3 4 5 6 7 8]; + default.radar.wfs(wf).rx_paths = [2 4 1 3 5 6 7 8]; + default.radar.wfs(wf).tx_paths = [1 2 3 4]; + default.radar.wfs(wf).adc_gains_dB = [46 46 46 46 46 46 46 46]; % Gain from the first LNA to the ADC +end + +default.config_regexp = '.*'; +default.name = 'Default 180-210 MHz'; +defaults{end+1} = default; + +%% Add default settings + +param.config.defaults = defaults; diff --git a/cresis-toolbox/missions/default_radar_params_2019_Antarctica_Polar6_mcords.m b/cresis-toolbox/missions/default_radar_params_2019_Antarctica_Polar6_mcords.m index 628a8f2c..50f00090 100644 --- a/cresis-toolbox/missions/default_radar_params_2019_Antarctica_Polar6_mcords.m +++ b/cresis-toolbox/missions/default_radar_params_2019_Antarctica_Polar6_mcords.m @@ -15,7 +15,7 @@ default.xml_file_prefix = 'mcords5'; default.data_file_prefix = 'mcords5'; default.header_load_func = @basic_load_mcords5; -default.header_load_params = struct('clk',1600e6,'presum_bug_fixed',true); +default.header_load_params = struct('clk',200e6,'presum_mode',0); default.xml_version = 2.0; default.noise_50ohm = [-39.8 -41.0 -40.1 -39.6 -38.4 -39.1 -38.3 -39.6 ]; @@ -191,7 +191,7 @@ default.records.file.adc_headers = [1:8]; default.records.gps.en = 1; default.records.frame_mode = 0; -default.records.presum_bug_fixed = 1; +default.records.presum_mode = 0; default.records.tmp_fn_uses_adc_folder_name = 1; %% Get heights (quick-look) worksheet in parameter spreadsheet @@ -264,8 +264,6 @@ default.combine.dbin = 1; default.combine.dline = 6; default.combine.DCM = []; -default.combine.three_dim.en = 0; -default.combine.three_dim.layer_fn = ''; default.combine.Nsv = 1; default.combine.theta_rng = [0 0]; default.combine.sv_fh = @array_proc_sv; diff --git a/cresis-toolbox/missions/default_radar_params_2019_Arctic_GV_snow.m b/cresis-toolbox/missions/default_radar_params_2019_Arctic_GV_snow.m index 2b6c4af6..19f3f6dc 100644 --- a/cresis-toolbox/missions/default_radar_params_2019_Arctic_GV_snow.m +++ b/cresis-toolbox/missions/default_radar_params_2019_Arctic_GV_snow.m @@ -15,7 +15,7 @@ param.config.file.version = 8; param.config.file.prefix = param.radar_name; param.config.file.suffix = '.bin'; -param.config.max_time_gap = 10; +param.config.max_time_gap = 5; param.config.min_seg_size = 2; param.config.daq_type = 'cresis'; diff --git a/cresis-toolbox/missions/default_radar_params_2019_Arctic_Polar6_snow.m b/cresis-toolbox/missions/default_radar_params_2019_Arctic_Polar6_snow.m index 51b349ce..5a90e692 100644 --- a/cresis-toolbox/missions/default_radar_params_2019_Arctic_Polar6_snow.m +++ b/cresis-toolbox/missions/default_radar_params_2019_Arctic_Polar6_snow.m @@ -98,8 +98,6 @@ default.array.dbin = 1; default.array.dline = 5; default.array.DCM = []; -default.array.three_dim.en = 0; -default.array.three_dim.layer_fn = ''; default.array.Nsv = 1; default.array.theta_rng = [0 0]; default.array.sv_fh = @array_proc_sv; diff --git a/cresis-toolbox/missions/default_radar_params_2019_Greenland_P3_rds.m b/cresis-toolbox/missions/default_radar_params_2019_Greenland_P3_rds.m index b5ff7281..1c297360 100644 --- a/cresis-toolbox/missions/default_radar_params_2019_Greenland_P3_rds.m +++ b/cresis-toolbox/missions/default_radar_params_2019_Greenland_P3_rds.m @@ -41,92 +41,90 @@ param.config.cresis.gps_file_mask = 'GPS*'; %% Command worksheet -default.cmd.records = 1; -default.cmd.qlook = 1; -default.cmd.generic = 1; +param.cmd.records = 1; +param.cmd.qlook = 1; +param.cmd.generic = 1; %% Records worksheet -default.records.gps.time_offset = 1; -default.records.frames.mode = 2; -default.records.frames.geotiff_fn = fullfile('greenland','Landsat-7','Greenland_natural_150m.tif'); -default.records.presum_bug_fixed = 0; +param.records.gps.time_offset = 1; +param.records.frames.mode = 1; +param.records.frames.geotiff_fn = fullfile('greenland','Landsat-7','Greenland_natural_150m.tif'); +param.records.presum_mode = 1; %% Qlook worksheet -default.qlook.out_path = ''; -default.qlook.en = 1; -default.qlook.block_size = 10000; -default.qlook.dec = 20; -default.qlook.inc_dec = 5; -default.qlook.surf.en = 1; -default.qlook.surf.profile = 'RDS_OIB'; +param.qlook.out_path = ''; +param.qlook.en = 1; +param.qlook.block_size = 10000; +param.qlook.dec = 20; +param.qlook.inc_dec = 5; +param.qlook.surf.en = 1; +param.qlook.surf.profile = 'RDS_OIB'; %% SAR worksheet -default.sar.out_path = ''; -default.sar.chunk_len = 5000; -default.sar.combine_rx = 0; -default.sar.mocomp.en = 1; -default.sar.mocomp.type = 2; -default.sar.mocomp.filter = {@butter [2] [0.1000]}; -default.sar.mocomp.uniform_en = 1; -default.sar.sar_type = 'fk'; -default.sar.sigma_x = 2.5; -default.sar.sub_aperture_steering = 0; -default.sar.st_wind = @hanning; -default.sar.start_eps = 3.15; +param.sar.out_path = ''; +param.sar.chunk_len = 5000; +param.sar.combine_rx = 0; +param.sar.mocomp.en = 1; +param.sar.mocomp.type = 2; +param.sar.mocomp.filter = {@butter [2] [0.1000]}; +param.sar.mocomp.uniform_en = 1; +param.sar.sar_type = 'fk'; +param.sar.sigma_x = 2.5; +param.sar.sub_aperture_steering = 0; +param.sar.st_wind = @hanning; +param.sar.start_eps = 3.15; %% Array worksheet -default.array.in_path = ''; -default.array.array_path = ''; -default.array.out_path = ''; -default.array.method = 'standard'; -default.array.window = @hanning; -default.array.bin_rng = 0; -default.array.line_rng = -5:5; -default.array.dbin = 1; -default.array.dline = 6; -default.array.DCM = []; -default.array.tomo_en = 0; -default.array.tomo.layer_fn = ''; -default.array.Nsv = 1; -default.array.theta_rng = [0 0]; -default.array.sv_fh = @array_proc_sv; -default.array.diag_load = 0; -default.array.Nsrc = 2; +param.array.in_path = ''; +param.array.array_path = ''; +param.array.out_path = ''; +param.array.method = 'standard'; +param.array.window = @hanning; +param.array.bin_rng = 0; +param.array.line_rng = -5:5; +param.array.dbin = 1; +param.array.dline = 6; +param.array.DCM = []; +param.array.tomo_en = 0; +param.array.Nsv = 1; +param.array.theta_rng = [0 0]; +param.array.sv_fh = @array_proc_sv; +param.array.diag_load = 0; +param.array.Nsrc = 2; %% Radar worksheet -default.radar.fs = 1e9/9; -default.radar.Tadc = []; % normally leave empty to use value in file header -default.radar.adc_bits = 14; -default.radar.Vpp_scale = 2; -default.radar.lever_arm_fh = @lever_arm; +param.radar.fs = 1e9/9; +param.radar.Tadc = []; % normally leave empty to use value in file header +param.radar.adc_bits = 14; +param.radar.Vpp_scale = 2; +param.radar.lever_arm_fh = @lever_arm; -default.radar.wfs.rx_paths = [1 2 3 4 5 6 7 NaN]; -default.radar.wfs.noise_figure = 2; -default.radar.wfs.Tadc_adjust = -1.4455e-06; % System time delay: leave this empty or set it to zero at first, determine this value later using data over surface with known height or from surface multiple +param.radar.wfs.rx_paths = [1 2 3 4 5 6 7 NaN]; +param.radar.wfs.Tadc_adjust = -1.4455e-06; % System time delay: leave this empty or set it to zero at first, determine this value later using data over surface with known height or from surface multiple -default.radar.wfs(1).Tsys = [0 0 0 0 0 0 0]/1e9; -default.radar.wfs(1).chan_equal_dB = [0 0 0 0 0 0 0]; -default.radar.wfs(1).chan_equal_deg = [0 0 0 0 0 0 0]; +param.radar.wfs(1).Tsys = [0 0 0 0 0 0 0]/1e9; +param.radar.wfs(1).chan_equal_dB = [0 0 0 0 0 0 0]; +param.radar.wfs(1).chan_equal_deg = [0 0 0 0 0 0 0]; %% Post worksheet -default.post.data_dirs = {'qlook'}; -default.post.layer_dir = 'layerData'; -default.post.maps_en = 1; -default.post.echo_en = 1; -default.post.layers_en = 0; -default.post.data_en = 0; -default.post.csv_en = 1; -default.post.concat_en = 1; -default.post.pdf_en = 1; -default.post.map.location = 'Greenland'; -default.post.map.type = 'combined'; -default.post.echo.elev_comp = 3; -default.post.echo.depth = '[publish_echogram_switch(Bbad,0.25,Surface_Elev,-3500,DBottom,-100),max(Surface_Elev+100)]'; -default.post.echo.er_ice = 3.15; -default.post.ops.en = 0; -default.post.ops.location = 'arctic'; -default.post.ops.layers = {'bottom','surface'}; -default.post.ops.gaps_dist = [300 60]; +param.post.data_dirs = {'qlook'}; +param.post.layer_dir = 'layerData'; +param.post.maps_en = 1; +param.post.echo_en = 1; +param.post.layers_en = 0; +param.post.data_en = 0; +param.post.csv_en = 1; +param.post.concat_en = 1; +param.post.pdf_en = 1; +param.post.map.location = 'Greenland'; +param.post.map.type = 'combined'; +param.post.echo.elev_comp = 3; +param.post.echo.depth = '[publish_echogram_switch(Bbad,0.25,Surface_Elev,-3500,DBottom,-100),max(Surface_Elev+100)]'; +param.post.echo.er_ice = 3.15; +param.post.ops.en = 0; +param.post.ops.location = 'arctic'; +param.post.ops.layers = {'bottom','surface'}; +param.post.ops.gaps_dist = [300 60]; %% Radar Settings diff --git a/cresis-toolbox/missions/default_radar_params_2019_Greenland_TO_kaband.m b/cresis-toolbox/missions/default_radar_params_2019_Greenland_TO_kaband.m index 6dfe8010..29049b1c 100644 --- a/cresis-toolbox/missions/default_radar_params_2019_Greenland_TO_kaband.m +++ b/cresis-toolbox/missions/default_radar_params_2019_Greenland_TO_kaband.m @@ -137,8 +137,8 @@ % Survey Mode 32-38 GHz default.radar.wfs(1).f0 = 38e9; default.radar.wfs(1).f1 = 32e9; -default.radar.wfs(1).Tpd = 240e-6; -default.radar.wfs(1).BW_window = [32.1e9 37.75e9]; +default.radar.wfs(1).Tpd = 250e-6; +default.radar.wfs(1).BW_window = [32.100096e9 37.749888e9]; default.radar.wfs(1).t_ref = 0; default.config_regexp = '.*'; diff --git a/cresis-toolbox/missions/default_radar_params_2019_Greenland_TO_kuband.m b/cresis-toolbox/missions/default_radar_params_2019_Greenland_TO_kuband.m index 88a749ad..de07a9db 100644 --- a/cresis-toolbox/missions/default_radar_params_2019_Greenland_TO_kuband.m +++ b/cresis-toolbox/missions/default_radar_params_2019_Greenland_TO_kuband.m @@ -137,8 +137,8 @@ % Survey Mode 12-18 GHz default.radar.wfs(1).f0 = 18e9; default.radar.wfs(1).f1 = 12e9; -default.radar.wfs(1).Tpd = 240e-6; -default.radar.wfs(1).BW_window = [12.1e9 17.75e9]; +default.radar.wfs(1).Tpd = 250e-6; +default.radar.wfs(1).BW_window = [12.100096e9 17.749888e9]; default.radar.wfs(1).t_ref = 0; default.config_regexp = '.*'; diff --git a/cresis-toolbox/missions/default_radar_params_2019_SouthDakota_CESSNA_snow.m b/cresis-toolbox/missions/default_radar_params_2019_SouthDakota_N1KU_snow.m similarity index 91% rename from cresis-toolbox/missions/default_radar_params_2019_SouthDakota_CESSNA_snow.m rename to cresis-toolbox/missions/default_radar_params_2019_SouthDakota_N1KU_snow.m index e51eea8c..f3fbbd61 100644 --- a/cresis-toolbox/missions/default_radar_params_2019_SouthDakota_CESSNA_snow.m +++ b/cresis-toolbox/missions/default_radar_params_2019_SouthDakota_N1KU_snow.m @@ -1,7 +1,7 @@ -function param = default_radar_params_2019_SouthDakota_CESSNA_snow -% param = default_radar_params_2019_SouthDakota_CESSNA_snow +function param = default_radar_params_2019_SouthDakota_N1KU_snow +% param = default_radar_params_2019_SouthDakota_N1KU_snow % -% Snow: 2019_SouthDakota_CESSNA +% Snow: 2019_SouthDakota_N1KU % % Creates base "param" struct % Creates defaults cell array for each type of radar setting @@ -9,7 +9,7 @@ % Author: John Paden %% Preprocess parameters -param.season_name = '2019_SouthDakota_CESSNA'; +param.season_name = '2019_SouthDakota_N1KU'; param.radar_name = 'snow'; param.config.file.version = 4; @@ -54,7 +54,7 @@ default.qlook.dec = 8; default.qlook.inc_dec = 5; default.qlook.surf.en = 1; -default.qlook.surf.profile = 'SNOW_AWI'; +default.qlook.surf.profile = 'SNOW'; %% SAR worksheet @@ -98,7 +98,7 @@ chan_equal_dB = [0]; chan_equal_deg = [0]; for wf = 1 - default.radar.wfs(wf).tx_weights = 1; % Watts + default.radar.wfs(wf).tx_weights = 1; default.radar.wfs(wf).adc_gains_dB = 95.8; % Radiometric calibration to 1/R^2 default.radar.wfs(wf).rx_paths = [1]; % ADC to rx path mapping default.radar.wfs(wf).ref_fn = ''; @@ -141,6 +141,8 @@ default.radar.wfs(1).Tpd = 250e-6; default.radar.wfs(1).BW_window = [2.5e9 7.72016e9]; default.radar.wfs(1).t_ref = 0; +fc = (default.radar.wfs(1).f0+default.radar.wfs(1).f1)/2; +default.radar.wfs(1).system_dB = 10*log10(0.1)+10.5+10.5+20*log10(300000000/(8*pi*fc))+10*log10(50); % Survey Mode 2-18 GHz % default.radar.wfs(1).f0 = 2e9; diff --git a/cresis-toolbox/missions/default_radar_params_2020_SouthDakota_N1KU_snow.m b/cresis-toolbox/missions/default_radar_params_2020_SouthDakota_N1KU_snow.m new file mode 100644 index 00000000..7c91d704 --- /dev/null +++ b/cresis-toolbox/missions/default_radar_params_2020_SouthDakota_N1KU_snow.m @@ -0,0 +1,163 @@ +function param = default_radar_params_2020_SouthDakota_N1KU_snow +% param = default_radar_params_2020_SouthDakota_N1KU_snow +% +% Snow: 2020_SouthDakota_N1KU +% +% Creates base "param" struct +% Creates defaults cell array for each type of radar setting +% +% Author: John Paden + +%% Preprocess parameters +param.season_name = '2020_SouthDakota_N1KU'; +param.radar_name = 'snow'; + +param.config.file.version = 11; +param.config.file.prefix = 'data'; +param.config.file.suffix = '.bin'; +param.config.max_time_gap = 10; +param.config.min_seg_size = 2; + +param.config.daq_type = 'cresis'; +param.config.wg_type = 'cresis'; +param.config.header_load_func = @basic_load; +param.config.board_map = {''}; +param.config.tx_map = {''}; + +param.config.daq.xml_version = -1; % No XML file available + +param.config.tx_enable = [1]; + +%% CReSIS parameters +param.config.cresis.clk = 125e6; +param.config.cresis.expected_rec_sizes = [250448]; +param.config.cresis.gps_file_mask = 'GPS*'; + +%% Command worksheet +default.cmd.records = 1; +default.cmd.qlook = 1; +default.cmd.generic = 1; + +%% Records worksheet +default.records.data_map = {[1 1 1 1; 1 2 1 2; 1 3 1 3; 1 4 1 4]}; +default.records.file.boards = [1]; +default.records.frames.geotiff_fn = fullfile('southdakota','','south_dakota.tif'); +default.records.frames.mode = 2; +default.records.frames.length = 2000; +default.records.gps.en = 1; +default.records.gps.time_offset = 1; + +%% Qlook worksheet +default.qlook.img_comb = []; +default.qlook.imgs = {[1 1]}; +default.qlook.out_path = ''; +default.qlook.block_size = 2000; +default.qlook.motion_comp = 0; +default.qlook.dec = 8; +default.qlook.inc_dec = 5; +default.qlook.surf.en = 1; +default.qlook.surf.profile = 'SNOW'; + + +%% SAR worksheet +default.sar.out_path = ''; +default.sar.imgs = default.qlook.imgs; +default.sar.frm_types = {0,[0 1],0,0,-1}; +default.sar.chunk_len = 2000; +default.sar.combine_rx = 0; +default.sar.time_of_full_support = 4e-6; +default.sar.mocomp.en = 1; +default.sar.mocomp.type = 2; +default.sar.mocomp.filter = {@butter [2] [0.1000]}; +default.sar.mocomp.uniform_en = 1; +default.sar.sar_type = 'fk'; +default.sar.sigma_x = 1; +default.sar.sub_aperture_steering = 0; +default.sar.st_wind = @hanning; +default.sar.start_eps = 1; + +%% Array worksheet +default.array.in_path = ''; +default.array.array_path = ''; +default.array.out_path = ''; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.array.method = 'standard'; +default.array.window = @hanning; +default.array.bin_rng = 0; +default.array.line_rng = -2:2; +default.array.dbin = 1; +default.array.dline = 5; + +%% Radar worksheet +default.radar.prf = 3125; +default.radar.fs = 125e6; +default.radar.adc_bits = 14; +default.radar.Vpp_scale = 2; % Digital receiver gain is 5, full scale Vpp is 2 +default.radar.Tadc_adjust = []; % System time delay: leave this empty or set it to zero at first, determine this value later using data over surface with known height or from surface multiple +default.radar.lever_arm_fh = @lever_arm; +chan_equal_Tsys = [0]/1e9; +chan_equal_dB = [0]; +chan_equal_deg = [0]; +for wf = 1 + default.radar.wfs(wf).tx_weights = 1; + default.radar.wfs(wf).adc_gains_dB = [95.8 95.8 95.8 95.8]; % Radiometric calibration to 1/R^2 + default.radar.wfs(wf).rx_paths = [1 2 3 4]; % ADC to rx path mapping + default.radar.wfs(wf).ref_fn = ''; + default.radar.wfs(wf).chan_equal_Tsys = chan_equal_Tsys; + default.radar.wfs(wf).chan_equal_dB = chan_equal_dB; + default.radar.wfs(wf).chan_equal_deg = chan_equal_deg; + default.radar.wfs(wf).adcs = [1]; + default.radar.wfs(wf).nz_trim = {[0 0],[0 0],[0 0],[0 0]}; + default.radar.wfs(wf).nz_valid = [0 1 2 3]; +end + +%% Post worksheet +default.post.data_dirs = {'qlook'}; +default.post.layer_dir = 'layerData'; +default.post.maps_en = 1; +default.post.echo_en = 1; +default.post.layers_en = 0; +default.post.data_en = 0; +default.post.csv_en = 1; +default.post.concat_en = 1; +default.post.pdf_en = 1; +default.post.map.location = 'geotiff'; +default.post.map.type = 'geotiff'; +default.post.img_dpi = 600; +default.post.map.geotiff = {'southdakota/Model_Landsat_UTM.tif','Black Hills, South Dakota'}; +% default.post.echo.elev_comp = 2; +% default.post.echo.depth = '[min(Surface_Depth)-2 max(Surface_Depth)+25]'; +default.post.echo.elev_comp = 3; +default.post.echo.depth = '[min(Surface_Elev)-10 max(Surface_Elev)+5]'; +default.post.echo.er_ice = 1; +default.post.echo.plot_params = {'PaperPosition',[0.25 2.5 3 6]}; +default.post.ops.location = 'arctic'; + +%% Radar Settings + +defaults = {}; + +% Survey Mode 2.5-8 GHz +default.radar.wfs(1).f0 = 2.5e9; +default.radar.wfs(1).f1 = 6.5e9; +default.radar.wfs(1).Tpd = 250e-6; +default.radar.wfs(1).BW_window = [2560000000 6208000000]; +default.radar.wfs(1).t_ref = 0; +fc = (default.radar.wfs(1).f0+default.radar.wfs(1).f1)/2; +default.radar.wfs(1).system_dB = 10*log10(0.1)+10.5+10.5+20*log10(300000000/(8*pi*fc))+10*log10(50); + +% Survey Mode 2-18 GHz +% default.radar.wfs(1).f0 = 2e9; +% default.radar.wfs(1).f1 = 8e9; +% default.radar.wfs(1).Tpd = 240e-6; +% default.radar.wfs(1).BW_window = [2.1e9 7.75e9]; +% default.radar.wfs(1).t_ref = 0; + +default.config_regexp = '.*'; +default.name = 'Survey Mode'; +defaults{end+1} = default; + +%% Add default settings + +param.config.defaults = defaults; diff --git a/cresis-toolbox/missions/default_radar_params_2021_Alaska_SO_snow.m b/cresis-toolbox/missions/default_radar_params_2021_Alaska_SO_snow.m new file mode 100644 index 00000000..fe34338f --- /dev/null +++ b/cresis-toolbox/missions/default_radar_params_2021_Alaska_SO_snow.m @@ -0,0 +1,148 @@ +function param = default_radar_params_2021_Alaska_SO_snow +% param = default_radar_params_2021_Alaska_SO_snow +% +% Snow: 2021_Alaska_SO +% +% Creates base "param" struct +% Creates defaults cell array for each type of radar setting +% +% Author: John Paden + +%% Preprocess parameters +param.season_name = '2021_Alaska_SO'; +param.radar_name = 'snow'; + +param.config.file.version = 4; +param.config.file.prefix = 'snow'; +param.config.file.suffix = '.bin'; +param.config.max_time_gap = 10; +param.config.min_seg_size = 2; + +param.config.daq_type = 'cresis'; +param.config.wg_type = 'cresis'; +param.config.header_load_func = @basic_load_fmcw2; +param.config.board_map = {''}; +param.config.tx_map = {''}; + +param.config.daq.xml_version = -1; % No XML file available + +param.config.tx_enable = [1]; + +%% CReSIS parameters +param.config.cresis.clk = 125e6; +param.config.cresis.expected_rec_sizes = [125240]; + +%% Command worksheet +default.cmd.records = 1; +default.cmd.qlook = 1; +default.cmd.generic = 1; + +%% Records worksheet +default.records.file.boards = [1]; +default.records.frames.geotiff_fn = fullfile('alaska','Landsat-7','Alaska_90m.tif'); +default.records.frames.mode = 2; +default.records.gps.en = 1; +default.records.gps.time_offset = 1; + +%% Qlook worksheet +default.qlook.img_comb = []; +default.qlook.imgs = {[1 1]}; +default.qlook.out_path = ''; +default.qlook.block_size = 2000; +default.qlook.motion_comp = 0; +default.qlook.dec = 4; +default.qlook.inc_dec = 5; +default.qlook.surf.en = 1; +default.qlook.surf.profile = 'SNOW_AWI'; + + +%% SAR worksheet +default.sar.out_path = ''; +default.sar.imgs = default.qlook.imgs; +default.sar.frm_types = {0,[0 1],0,0,-1}; +default.sar.chunk_len = 2000; +default.sar.combine_rx = 0; +default.sar.time_of_full_support = 4e-6; +default.sar.mocomp.en = 1; +default.sar.mocomp.type = 2; +default.sar.mocomp.filter = {@butter [2] [0.1000]}; +default.sar.mocomp.uniform_en = 1; +default.sar.sar_type = 'fk'; +default.sar.sigma_x = 1; +default.sar.sub_aperture_steering = 0; +default.sar.st_wind = @hanning; +default.sar.start_eps = 3.15; + +%% Array worksheet +default.array.in_path = ''; +default.array.array_path = ''; +default.array.out_path = ''; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.array.method = 'standard'; +default.array.window = @hanning; +default.array.bin_rng = 0; +default.array.line_rng = -2:2; +default.array.dbin = 1; +default.array.dline = 5; + +%% Radar worksheet +default.radar.prf = 4000; +default.radar.fs = 125e6; +default.radar.adc_bits = 14; +default.radar.Vpp_scale = 2; % Digital receiver gain is 5, full scale Vpp is 2 +default.radar.Tadc_adjust = []; % System time delay: leave this empty or set it to zero at first, determine this value later using data over surface with known height or from surface multiple +default.radar.lever_arm_fh = @lever_arm; +chan_equal_Tsys = [0]/1e9; +chan_equal_dB = [0]; +chan_equal_deg = [0]; +for wf = 1 + default.radar.wfs(wf).tx_weights = 1; % Watts + default.radar.wfs(wf).adc_gains_dB = 95.8; % Radiometric calibration to 1/R^2 + default.radar.wfs(wf).rx_paths = [1]; % ADC to rx path mapping + default.radar.wfs(wf).ref_fn = ''; + default.radar.wfs(wf).chan_equal_Tsys = chan_equal_Tsys; + default.radar.wfs(wf).chan_equal_dB = chan_equal_dB; + default.radar.wfs(wf).chan_equal_deg = chan_equal_deg; + default.radar.wfs(wf).adcs = [1]; + default.radar.wfs(wf).nz_trim = {[0 0],[0 0],[0 0],[0 0]}; + default.radar.wfs(wf).nz_valid = [0 1 2 3]; +end + +%% Post worksheet +default.post.data_dirs = {'qlook'}; +default.post.layer_dir = 'layerData'; +default.post.maps_en = 1; +default.post.echo_en = 1; +default.post.layers_en = 0; +default.post.data_en = 0; +default.post.csv_en = 1; +default.post.concat_en = 1; +default.post.pdf_en = 1; +default.post.map.location = 'Alaska'; +default.post.map.type = 'combined'; +default.post.echo.elev_comp = 2; +default.post.echo.depth = '[min(Surface_Depth)-2 max(Surface_Depth)+25]'; +% default.post.echo.elev_comp = 3; +% default.post.echo.depth = '[min(Surface_Elev)-25 max(Surface_Elev)+2]'; +default.post.echo.er_ice = round((1+0.51*0.3)^3 * 100)/100; +default.post.ops.location = 'alaska'; + +%% Radar Settings + +defaults = {}; + +% Survey Mode 2-18 GHz +default.radar.wfs(1).f0 = 2e9; +default.radar.wfs(1).f1 = 6e9; +default.radar.wfs(1).Tpd = 250e-6; +default.radar.wfs(1).BW_window = [2e9 6e9]; +default.radar.wfs(1).t_ref = -6.358e-6; + +default.config_regexp = '.*'; +default.name = 'Survey Mode'; +defaults{end+1} = default; + +%% Add default settings + +param.config.defaults = defaults; diff --git a/cresis-toolbox/missions/default_radar_params_2021_Arctic_Vanilla_snow.m b/cresis-toolbox/missions/default_radar_params_2021_Arctic_Vanilla_snow.m new file mode 100644 index 00000000..558532ed --- /dev/null +++ b/cresis-toolbox/missions/default_radar_params_2021_Arctic_Vanilla_snow.m @@ -0,0 +1,265 @@ +function param = default_radar_params_2021_Arctic_Vanilla_snow +% param = default_radar_params_2021_Arctic_Vanilla_snow +% +% Snow: 2021_Arctic_Vanilla +% +% Creates base "param" struct +% Creates defaults cell array for each type of radar setting +% +% Author: John Paden + +%% Preprocess parameters +param.season_name = '2021_Arctic_Vanilla'; +param.radar_name = 'snow9'; + +param.config.file.version = 9; +param.config.file.prefix = ''; +param.config.file.suffix = '.dat'; +param.config.max_time_gap = 10; +param.config.min_seg_size = 2; + +param.config.daq_type = 'arena'; +param.config.wg_type = 'arena'; +param.config.header_load_func = @basic_load_arena; +param.config.board_map = {'digrx0'}; +param.config.tx_map = {'awg0'}; + +param.config.daq.xml_version = -1; % No XML file available + +param.config.tx_enable = [1]; + +%% VANILLA SNOW Arena Parameters +arena = []; +arena.clk = 10e6; +fs = 1000e6; +fs_dac = 2000e6; +subsystem_idx = 0; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'ARENA0'; +arena.subsystem(subsystem_idx).subSystem{1} = 'ctu'; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'ARENA0'; +arena.subsystem(subsystem_idx).subSystem{1} = 'awg0'; +arena.subsystem(subsystem_idx).subSystem{2} = 'digrx0'; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'Data Server'; + +dac_idx = 0; +dac_idx = dac_idx + 1; +arena.dac(dac_idx).name = 'awg0'; +arena.dac(dac_idx).type = 'dac-ad9129_0012'; +arena.dac(dac_idx).dacClk = fs_dac; +arena.dac(dac_idx).desiredAlignMin = -5; +arena.dac(dac_idx).desiredAlignMax = 5; +arena.dac(dac_idx).dcoPhase = 0; + +adc_idx = 0; +adc_idx = adc_idx + 1; +arena.adc(adc_idx).name = 'digrx0'; +arena.adc(adc_idx).type = 'adc-ad9680_0017'; +arena.adc(adc_idx).sampFreq = fs; +arena.adc(adc_idx).adcMode = 1; +arena.adc(adc_idx).desiredAlignMin = -6; +arena.adc(adc_idx).desiredAlignMax = 10; +arena.adc(adc_idx).stream = 'file'; +arena.adc(adc_idx).ip = '10.0.0.51'; +arena.adc(adc_idx).outputSelect = 1; +arena.adc(adc_idx).wf_set = 1; +arena.adc(adc_idx).gain_dB = [-6 -6]; +% +% 0 +% dataStream0 +% f0 +% NewItem +% + + +daq_idx = 0; +daq_idx = daq_idx + 1; +arena.daq(daq_idx).name = 'daq0'; +arena.daq(daq_idx).type = 'daq_0001'; +arena.daq(daq_idx).auxDir = '/data/'; +arena.daq(daq_idx).fileStripe = '/data/%b/'; +arena.daq(daq_idx).fileName = 'accum3'; + +arena.system.name = 'ku0002'; + +arena.param.tx_max = [1 1]; +arena.param.PA_setup_time = 2e-6; % Time required to enable PA before transmit +arena.param.TTL_time_delay = 0.0; % TTL time delay relative to transmit start +arena.param.ADC_time_delay = 3.0720e-6; % ADC time delay relative to transmit start + +arena.psc.type = 'psc_0003'; + +arena.daq.type = 'daq_0001'; + +arena.ctu.name = 'ctu00'; +arena.ctu.type = 'ctu_0032'; +if 1 + % External GPS + arena.ctu.nmea = 31; + arena.ctu.nmea_baud = 9600; + arena.ctu.pps = 10; + arena.ctu.pps_polarity = 1; +else + % Internal GPS + arena.ctu.nmea = 60; + arena.ctu.nmea_baud = 115200; + arena.ctu.pps = 63; + arena.ctu.pps_polarity = 1; +end +idx = 0; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'EPRI'; +arena.ctu.out.bit_group(idx).bits = 0; +arena.ctu.out.bit_group(idx).epri = [1 0]; +arena.ctu.out.bit_group(idx).pri = [0 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'PRI'; +arena.ctu.out.bit_group(idx).bits = 1; +arena.ctu.out.bit_group(idx).epri = [1 0]; +arena.ctu.out.bit_group(idx).pri = [1 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'TR'; +arena.ctu.out.bit_group(idx).bits = 2; +arena.ctu.out.bit_group(idx).epri = [1 0]; +arena.ctu.out.bit_group(idx).pri = [1 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'CalSwitch'; +arena.ctu.out.bit_group(idx).bits = 3; +arena.ctu.out.bit_group(idx).epri = [1 0]; +arena.ctu.out.bit_group(idx).pri = [1 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'AttenFirst18dB'; % 1 is low gain/disables attenuator +arena.ctu.out.bit_group(idx).bits = 4; +arena.ctu.out.bit_group(idx).epri = [0 0]; +arena.ctu.out.bit_group(idx).pri = [0 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'AttenSecond7dB'; % 1 is high gain/disables attenuator +arena.ctu.out.bit_group(idx).bits = 5; +arena.ctu.out.bit_group(idx).epri = [1 1]; +arena.ctu.out.bit_group(idx).pri = [1 1]; + +arena.ctu.out.time_cmd = {'2e-6+param.wfs(wf).Tpd+0.1e-6' '2/param.prf'}; + +param.config.arena = arena; + +%% Command worksheet +default.cmd.records = 1; +default.cmd.qlook = 1; +default.cmd.generic = 1; + +%% Records worksheet +default.records.file.boards = [1]; +default.records.frames.geotiff_fn = fullfile('greenland','Landsat-7','Greenland_natural_150m.tif'); +default.records.frames.mode = 2; +default.records.gps.en = 1; +default.records.gps.time_offset = 1; + +%% Qlook worksheet +default.qlook.img_comb = []; +default.qlook.imgs = {[1 1]}; +default.qlook.out_path = ''; +default.qlook.block_size = 2000; +default.qlook.motion_comp = 0; +default.qlook.dec = 4; +default.qlook.inc_dec = 5; +default.qlook.surf.en = 1; +default.qlook.surf.profile = 'SNOW_AWI'; + +%% SAR worksheet +default.sar.out_path = ''; +default.sar.imgs = default.qlook.imgs; +default.sar.frm_types = {0,[0 1],0,0,-1}; +default.sar.chunk_len = 2000; +default.sar.combine_rx = 0; +default.sar.time_of_full_support = 4e-6; +default.sar.mocomp.en = 1; +default.sar.mocomp.type = 2; +default.sar.mocomp.filter = {@butter [2] [0.1000]}; +default.sar.mocomp.uniform_en = 1; +default.sar.sar_type = 'fk'; +default.sar.sigma_x = 1; +default.sar.sub_aperture_steering = 0; +default.sar.st_wind = @hanning; +default.sar.start_eps = 3.15; + +%% Array worksheet +default.array.in_path = ''; +default.array.array_path = ''; +default.array.out_path = ''; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.array.method = 'standard'; +default.array.window = @hanning; +default.array.bin_rng = 0; +default.array.line_rng = -2:2; +default.array.dbin = 1; +default.array.dline = 5; +default.array.DCM = []; + +%% Radar worksheet +default.radar.prf = 1/256e-6; +default.radar.fs = 1000e6; +default.radar.adc_bits = 14; +default.radar.Vpp_scale = 2; % Digital receiver gain is 5, full scale Vpp is 2 +default.radar.lever_arm_fh = @lever_arm; +chan_equal_Tsys = [0]/1e9; +chan_equal_dB = [0]; +chan_equal_deg = [0]; +for wf = 1:1 + default.radar.wfs(wf).tx_weights = 1; % Watts + default.radar.wfs(wf).Tadc_adjust = []; % System time delay: leave this empty or set it to zero at first, determine this value later using data over surface with known height or from surface multiple + default.radar.wfs(wf).adc_gains_dB = 105.8; % Radiometric calibration to 1/R^2 + default.radar.wfs(wf).rx_paths = [1]; % ADC to rx path mapping + default.radar.wfs(wf).ref_fn = ''; + default.radar.wfs(wf).chan_equal_Tsys = chan_equal_Tsys; + default.radar.wfs(wf).chan_equal_dB = chan_equal_dB; + default.radar.wfs(wf).chan_equal_deg = chan_equal_deg; + default.radar.wfs(wf).adcs = [1]; + default.radar.wfs(wf).coh_noise_method = 'estimated'; + default.radar.wfs(wf).nz_trim = {[0 0],[0 0],[0 0],[0 0]}; + default.radar.wfs(wf).nz_valid = [0 1 2 3]; +end + +%% Post worksheet +default.post.data_dirs = {'qlook'}; +default.post.layer_dir = 'layerData'; +default.post.maps_en = 1; +default.post.echo_en = 1; +default.post.layers_en = 0; +default.post.data_en = 0; +default.post.csv_en = 0; +default.post.concat_en = 0; +default.post.pdf_en = 0; +default.post.map.location = 'Greenland'; +default.post.map.type = 'combined'; +default.post.echo.elev_comp = 2; +default.post.echo.depth = '[min(Surface_Depth)-2 max(Surface_Depth)+25]'; +% default.post.echo.elev_comp = 3; +% default.post.echo.depth = '[min(Surface_Elev)-25 max(Surface_Elev)+2]'; +default.post.echo.er_ice = round((1+0.51*0.3)^3 * 100)/100; +default.post.ops.location = 'arctic'; + +%% Radar Settings + +defaults = {}; + +% Survey Mode 2-8 GHz +default.records.data_map = {[0 0 1 1]}; +default.radar.wfs(1).f0 = 500e6; +default.radar.wfs(1).f1 = 750e6; +default.radar.wfs(1).fmult = 24; +default.radar.wfs(1).fLO = -10e9 +default.radar.wfs(1).Tpd = 180e-6; +default.radar.wfs(1).BW_window = [2e9 8e9]; +default.radar.wfs(1).t_ref = 0; +default.radar.wfs(1).tx_paths = [1]; + +default.config_regexp = '.*'; +default.name = 'Survey Mode 2-8 GHz'; +defaults{end+1} = default; + +%% Add default settings + +param.config.defaults = defaults; diff --git a/cresis-toolbox/missions/default_radar_params_2021_Greenland_Polar5_mcords.m b/cresis-toolbox/missions/default_radar_params_2021_Greenland_Polar5_mcords.m new file mode 100644 index 00000000..5cbe3551 --- /dev/null +++ b/cresis-toolbox/missions/default_radar_params_2021_Greenland_Polar5_mcords.m @@ -0,0 +1,421 @@ +function [param] = default_radar_params_2021_Greenland_Polar_mcords +% [param] = default_radar_params_2021_Greenland_Polar_mcords +% +% MCORDS 5: 2021 Greenland Polar5 +% +% Creates base "param" struct +% Creates defaults cell array for each type of radar setting +% +% Author: John Paden + +param.season_name = '2021_Greenland_Polar5'; +param.radar_name = 'mcords5'; + +%% CReSIS parameters +param.config.file.version = 407; +param.config.file.prefix = param.radar_name; +param.config.file.suffix = '.bin'; +param.config.max_time_gap = 10; +param.config.min_seg_size = 2; + +param.config.daq_type = 'cresis'; +param.config.wg_type = 'cresis'; +param.config.header_load_func = @basic_load_mcords5; +param.config.board_map = {'chan1','chan2','chan3','chan4','chan5','chan6','chan7','chan8'}; +param.config.tx_map = {'','','','','','','',''}; + +param.config.daq.xml_version = 2.0; + +param.config.max_data_rate = 100; +param.config.max_duty_cycle = 0.12; +param.config.prf_multiple = []; % Power supply sync signal that PRF must be a factor of these numbers +param.config.PRI_guard = 10e-6; +param.config.PRI_guard_percentage = 1; +param.config.tx_enable = [1 1 1 1 1 1 1 1]; +param.config.max_tx = [4000 4000 4000 4000 4000 4000 4000 4000]; +param.config.max_tx_voltage = sqrt([500 500 1000 1000 1000 1000 500 500]*50)*10^(-2/20); % voltage at max_tx + +%% CReSIS parameters +param.config.cresis.clk = 200e6; +param.config.cresis.rx_gain_dB = 48; +param.config.cresis.gps_file_mask = 'GPS*'; + + +%% Control parameters (not used in the parameter spreadsheet directly) +% default.xml_file_prefix = 'mcords5'; +% default.data_file_prefix = 'mcords5'; +% default.header_load_func = @basic_load_mcords5; +% default.header_load_params = struct('clk',200e6,'presum_mode',0); +% default.xml_version = 2.0; +% +% default.noise_50ohm = [-39.8 -41.0 -40.1 -39.6 -38.4 -39.1 -38.3 -39.6 ]; +% +% default.Pt = (4*1000 + 4*500) * sum(chebwin(8,30).^2)/8; +% default.Gt = 8*4; +% default.Ae = 2*0.468 * 0.468; +% +% default.system_loss_dB = 10.^(-5.88/10); +% default.max_DDS_RAM = 4000; +% default.tx_voltage = sqrt(1000*50)*10^(-2/20); +% +% default.iq_mode = 0; +% default.tx_DDS_mask = [1 1 1 1 1 1 1 1]; +% + default.radar_worksheet_headers = {'Tpd','Tadc','Tadc_adjust','f0','f1','ft_dec','ref_fn','tukey','tx_weights','rx_paths','adc_gains','chan_equal_dB','chan_equal_deg','Tsys','DC_adjust','DDC_mode','DDC_freq','bit_shifts'}; + default.radar_worksheet_headers_type = {'r','r','r','r','r','r','r','r','r','r','r','r','r','r','r','r','r','r'}; +% + default.basic_surf_track_min_time = 2e-6; % Normally 0e-6 for lab test, 2e-6 for flight test + default.basic_surf_track_Tpd_factor = 1.1; % Normally -inf for lab test, 1.1 for flight test +% default.adc_folder_name = 'chan%d'; +% +% if 1 +% % Example 1: Normal configuration: +% % Connect antenna N to WFG N for all N = 1 to 8 +% ref_adc = 1; +% default.txequal.img = [(1:8).', ref_adc*ones(8,1)]; +% default.txequal.ref_wf_adc = 4; +% default.txequal.wf_mapping = [1 2 3 4 5 6 7 8]; +% default.txequal.Hwindow_desired = chebwin(8,30).'; +% default.txequal.max_DDS_amp = [4000 4000 4000 4000 4000 4000 4000 4000]; +% default.txequal.time_delay_desired = [0 0 0 0 0 0 0 0]; +% default.txequal.phase_desired = [0 0 0 0 0 0 0 0]; +% default.txequal.time_validation = [0.4 0.4 0.4 0.4 0.4 0.4 0.4 0.4]*1e-9; +% default.txequal.amp_validation = [3 3 3 3 3 3 3 3]; +% default.txequal.phase_validation = [35 35 35 35 35 35 35 35]; +% default.txequal.remove_linear_phase_en = true; +% elseif 0 +% % Channel 4 ADC is bad: +% ref_adc = 5; +% default.txequal.img = [(1:8).', ref_adc*ones(8,1)]; +% default.txequal.ref_wf_adc = 4; +% default.txequal.wf_mapping = [1 2 3 4 5 6 7 8]; +% default.txequal.Hwindow_desired = chebwin(8,30).'; +% default.txequal.max_DDS_amp = [4000 4000 4000 4000 4000 4000 4000 4000]; +% default.txequal.time_delay_desired = [0 0 0 0 0 0 0 0]; +% default.txequal.phase_desired = [0 0 0 0 0 0 0 0]; +% default.txequal.time_validation = [0.4 0.4 0.4 0.4 0.4 0.4 0.4 0.4]*1e-9; +% default.txequal.amp_validation = [3 3 3 3 3 3 3 3]; +% default.txequal.phase_validation = [35 35 35 35 35 35 35 35]; +% default.txequal.remove_linear_phase_en = true; +% elseif 0 +% % DDS 3 is not used, but create settings not changed: +% % Connect antenna 1 to a 50 oxhm load +% % Connect antenna 2 to WFG 1 +% % Connect antenna 3 to WFG 2 +% ref_adc = 4; +% default.txequal.img = [(1:8).', ref_adc*ones(8,1)]; +% default.txequal.wf_mapping = [1 2 0 4 5 6 7 8]; +% default.txequal.ref_wf_adc = 4; +% default.txequal.Hwindow_desired = chebwin(7,30).'; +% default.txequal.max_DDS_amp = [4000 4000 0 4000 4000 4000 4000 4000]; +% default.txequal.time_delay_desired = [0 0 0 0 0 0 0 0]; +% default.txequal.phase_desired = [0 0 0 0 0 0 0 0]; +% default.txequal.time_validation = [0.4 0.4 0.4 0.4 0.4 0.4 0.4 0.4]*1e-9; +% default.txequal.amp_validation = [3 3 3 3 3 3 3 3]; +% default.txequal.phase_validation = [35 35 35 35 35 35 35 35]; +% default.txequal.remove_linear_phase_en = true; +% elseif 0 +% % DDS 3 to 8 are not used and ADC's 3 to 24 are not used, create settings +% % also only uses first two DDS +% ref_adc = 1; +% default.txequal.img = [(1:2).', ref_adc*ones(2,1)]; +% default.wf_mapping = [1 2 0 0 0 0 0 0]; +% default.txequal.ref_wf_adc = 1; +% default.txequal.Hwindow_desired = [1 1]; +% default.txequal.max_DDS_amp = [4000 4000 0 0 0 0 0 0]; +% default.txequal.time_delay_desired = [0 0 0 0 0 0 0 0]; +% default.txequal.phase_desired = [0 0 0 0 0 0 0 0]; +% default.txequal.time_validation = [0.4 0.4 0.4 0.4 0.4 0.4 0.4 0.4]*1e-9; +% default.txequal.amp_validation = [3 3 3 3 3 3 3 3]; +% default.txequal.phase_validation = [35 35 35 35 35 35 35 35]; +% default.txequal.remove_linear_phase_en = false; +% end + +%% AWI MCoRDS Arena Parameters +arena.awg = []; +arena.awg(end+1).awg = 0; +arena.awg(end).dacs = [0 1]; +arena.awg(end).dacClk = [1600e6 1600e6]; +arena.awg(end).desiredAlignMin = [3 10]; +arena.awg(end).desiredAlignMax = [17 24]; +arena.awg(end+1).awg = 1; +arena.awg(end).dacs = [2 3]; +arena.awg(end).dacClk = [1600e6 1600e6]; +arena.awg(end).desiredAlignMin = [-3 11]; +arena.awg(end).desiredAlignMax = [11 25]; +arena.awg(end+1).awg = 2; +arena.awg(end).dacs = [4 5]; +arena.awg(end).dacClk = [1600e6 1600e6]; +arena.awg(end).desiredAlignMin = [4 10]; +arena.awg(end).desiredAlignMax = [18 24]; +arena.awg(end+1).awg = 3; +arena.awg(end).dacs = [6 7]; +arena.awg(end).dacClk = [1600e6 1600e6]; +arena.awg(end).desiredAlignMin = [3 13]; +arena.awg(end).desiredAlignMax = [17 27]; +arena.dacs = [0 1 2 3 4 5 6 7]; +arena.dacs_sampFreq = [1600e6 1600e6 1600e6 1600e6 1600e6 1600e6 1600e6 1600e6]; +arena.max_tx = [0.63 0.63 0.63 0.63 0.63 0.63 0.63 0.63]; +arena.zeropimods = [0 180]; +arena.TTL_time = [0.1 0.2 2.2]; +arena.dacs_internal_delay = 0.0; +arena.dacs_start_delay = 0.0; + +arena.TTL_names = {}; +for PA = 1:8 + arena.TTL_names{end+1} = sprintf('PA ENA %d',PA); +end +arena.TTL_names{end+1} = 'T/R'; +arena.TTL_names{end+1} = 'ISO'; +arena.TTL_names{end+1} = 'EPRI'; +arena.TTL_names{end+1} = 'PRI'; +arena.TTL_names{end+1} = 'EPRI'; +arena.TTL_names{end+1} = 'PRI'; +arena.TTL_names{end+1} = 'EPRI'; +arena.TTL_names{end+1} = 'PRI'; +arena.TTL_states{1} = [ + 0 1 1 0 % PA ENA 1 + 0 1 1 0 % PA ENA 2 + 0 1 1 0 % PA ENA 3 + 0 1 1 0 % PA ENA 4 + 0 1 1 0 % PA ENA 5 + 0 1 1 0 % PA ENA 6 + 0 1 1 0 % PA ENA 7 + 0 1 1 0 % PA ENA 8 + 0 1 1 0 % T/R + 0 1 1 0 % ISO + 1 0 0 0 % EPRI + 0 1 0 0 % PRI + 1 0 0 0 % EPRI + 0 1 0 0 % PRI + 1 0 0 0 % EPRI + 0 1 0 0 % PRI + ]; +arena.TTL_states{2} = [ + 0 1 1 0 % PA ENA 1 + 0 1 1 0 % PA ENA 2 + 0 1 1 0 % PA ENA 3 + 0 1 1 0 % PA ENA 4 + 0 1 1 0 % PA ENA 5 + 0 1 1 0 % PA ENA 6 + 0 1 1 0 % PA ENA 7 + 0 1 1 0 % PA ENA 8 + 0 1 1 0 % T/R + 0 1 1 0 % ISO + 0 0 0 0 % EPRI + 0 1 0 0 % PRI + 0 0 0 0 % EPRI + 0 1 0 0 % PRI + 0 0 0 0 % EPRI + 0 1 0 0 % PRI + ]; + +default.arena = arena; + +%% Command worksheet +default.cmd.records = 1; +default.cmd.qlook = 1; +default.cmd.generic = 1; + +%% Records worksheet +default.records.gps.time_offset = 1; +default.records.frames.mode = 2; +%default.records.frames.geotiff_fn = fullfile('antarctica','Landsat-7','Antarctica_LIMA_480m.tif'); +default.records.frames.geotiff_fn = fullfile('greenland','Landsat-7','Greenland_natural_150m.tif'); +default.records.presum_mode = 0; + +%% Qlook worksheet +default.qlook.block_size = 5000; +default.qlook.dec = 20; +default.qlook.inc_dec = 5; +default.qlook.surf.en = 1; +default.qlook.surf.profile = 'RDS_OIB'; + +%% SAR worksheet +default.sar.out_path = ''; +default.sar.imgs = {[1*ones(8,1),(1:8).'],[2*ones(8,1),(1:8).'],[3*ones(8,1),(1:8).']}; +default.sar.chunk_len = 3500; +default.sar.mocomp.en = 1; +default.sar.mocomp.type = 2; +default.sar.mocomp.filter = {@butter [2] [0.1000]}; +default.sar.mocomp.uniform_en = 1; +default.sar.sar_type = 'fk'; +default.sar.sigma_x = 2.5; +default.sar.sub_aperture_steering = 0; +default.sar.st_wind = @hanning; +default.sar.start_eps = 3.15; + +%% Combine worksheet in parameter spreadsheet +default.array.in_path = ''; +default.array.array_path = ''; +default.array.out_path = ''; +default.array.method = 'standard'; +default.array.window = @hanning; +default.array.bin_rng = 0; +default.array.rline_rng = -5:5; +default.array.dbin = 1; +default.array.dline = 6; +default.array.DCM = []; +default.array.Nsv = 1; +default.array.theta_rng = [0 0]; +default.array.sv_fh = @array_proc_sv; +default.array.diag_load = 0; +default.array.Nsig = 2; + +%% Radar worksheet in parameter spreadsheet +default.radar.fs = 1600e6; +default.radar.Tadc = []; % normally leave empty to use value in file header +default.radar.fs = 1600e6; +default.radar.adc_bits = 12; +default.radar.adc_full_scale = 2; +default.radar.rx_paths = [1:8]; +default.radar.noise_figure = 2; +default.radar.rx_gain = 48; +default.radar.adc_SNR_dB = 59; +default.radar.Tadc_adjust = 0.00000851003077; % System time delay: leave this empty or set it to zero at first, determine this value later using data over surface with known height or from surface multiple +default.radar.lever_arm_fh = @lever_arm; + +defaults = {}; + +%% Wideband settings +default.radar.wfs(1).Tsys = [-31.8 -32.1 -31.7 -31.6 -31.2 -30.8 -31.5 -31.1]/1e9; +default.radar.wfs(1).chan_equal_dB = [-4 -4.3 -2.9 -4.6 -1.2 -1.5 -0.9 -2.2]; +default.radar.wfs(1).chan_equal_deg = [113.1 64.8 124.3 133.7 108 138.1 71.6 102.9]; +default.radar.wfs(1).rx_paths = [1:8]; +default.radar.wfs(1).Tadc_adjust = 0.00000851003077; + + % survey mode +default.qlook.img_comb = [3e-06 -inf 1e-06 1e-05 -inf 3e-06]; +default.qlook.imgs = {[1*ones(8,1),(1:8).'],[2*ones(8,1),(1:8).'],[3*ones(8,1),(1:8).']}; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.config_regexp = 'survey_150-520MHz_.*thick.xml'; +default.name = 'Survey Mode 150-520 MHz'; +defaults{end+1} = default; + + % thin ice mode +default.qlook.img_comb = [1e-06 -inf 1e-06 3e-06 -inf 1e-06]; +default.qlook.imgs = {[1*ones(8,1),(1:8).'],[2*ones(8,1),(1:8).'],[3*ones(8,1),(1:8).']}; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.config_regexp = 'thinice_150-520MHz_.*thick.xml'; +default.name = 'Thin Ice Mode 150-520 MHz'; +defaults{end+1} = default; + + % 2 beam imaging mode +default.qlook.img_comb = []; +default.qlook.imgs = {[1*ones(8,1),(1:8).']}; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.config_regexp = 'image_150-520MHz_.*thick.xml'; +default.name = '2 Beam Image Mode 150-520 MHz'; +defaults{end+1} = default; + + % 3 beam imaging mode +default.qlook.img_comb = []; +default.qlook.imgs = {[2*ones(8,1),(1:8).']}; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.config_regexp = 'image3_150-520MHz_.*thick.xml'; +default.name = '3 Beam Image Mode 150-520 MHz'; +defaults{end+1} = default; + + % sea ice mode +default.qlook.img_comb = []; +default.qlook.imgs = {[1*ones(8,1),(1:8).']}; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.config_regexp = 'seaice_150-520MHz_.*.xml'; +default.name = 'Sea Ice 150-520 MHz'; +defaults{end+1} = default; + + % image high thin with narrowband +default.qlook.img_comb = []; +default.qlook.imgs = {[2*ones(8,1),(1:8).']}; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.config_regexp = 'imagehighthin_150-520MHz_.*.xml'; +default.name = 'High Alt Thin Ice Image Mode 150-520 MHz'; +defaults{end+1} = default; + +%% Narrowband settings +default.radar.wfs(1).Tsys = [-31.8 -32.1 -31.7 -31.6 -31.2 -30.8 -31.5 -31.1]/1e9; +default.radar.wfs(1).chan_equal_dB = [0 0.9 0.2 0 3.2 1.7 3.9 2.4]; +default.radar.wfs(1).chan_equal_deg = [-14.6 -69.7 -6.8 0 -13.9 -1.8 -48.8 -15.8]; + +% survey mode +default.qlook.img_comb = [3e-06 -inf 1e-06 1e-05 -inf 3e-06]; +default.qlook.imgs = {[1*ones(8,1),(1:8).'],[2*ones(8,1),(1:8).'],[3*ones(8,1),(1:8).']}; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.config_regexp = 'survey_180-210MHz_.*thick.xml'; +default.name = 'Survey Mode 180-210 MHz'; +defaults{end+1} = default; + +% thin ice mode +default.qlook.img_comb = [1e-06 -inf 1e-06 3e-06 -inf 1e-06]; +default.qlook.imgs = {[1*ones(8,1),(1:8).'],[2*ones(8,1),(1:8).'],[3*ones(8,1),(1:8).']}; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.config_regexp = 'thinice_180-210MHz_.*thick.xml'; +default.name = 'Thin Ice Mode 180-210 MHz'; +defaults{end+1} = default; + + % 2 beam imaging mode +default.qlook.img_comb = []; +default.qlook.imgs = {[1*ones(8,1),(1:8).']}; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.config_regexp = '^image_180-210MHz_.*thick.xml'; +default.name = '2 Beam Image Mode 180-210 MHz'; +defaults{end+1} = default; + +% 3 beam imaging mode +default.qlook.img_comb = []; +default.qlook.imgs = {[2*ones(8,1),(1:8).']}; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.config_regexp = 'image3_180-210MHz_.*thick.xml'; +default.name = '3 Beam Image Mode 180-210 MHz'; +defaults{end+1} = default; + +% egrip imaging mode +default.qlook.img_comb = [1e-06 -inf 1e-06 3e-06 -inf 1e-06]; +default.qlook.imgs = {[1*ones(8,1),(1:8).'],[2*ones(8,1),(1:8).'],[3*ones(8,1),(1:8).'; 4*ones(8,1),(1:8).']}; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.config_regexp = 'egrip_image.*.xml'; +default.name = 'EGRIP Image 180-210 MHz'; +defaults{end+1} = default; + +%% Other settings + +default.qlook.img_comb = []; +default.qlook.imgs = []; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; + +default.config_regexp = 'survey_180-210MHz_.*DECONV.xml'; +default.name = 'Deconv 180-210 MHz'; +defaults{end+1} = default; + +default.config_regexp = '.*180-210MHz.*'; +default.name = 'Other Settings 180-210 MHz'; +defaults{end+1} = default; + +default.radar.wfs(1).Tsys = [-31.8 -32.1 -31.7 -31.6 -31.2 -30.8 -31.5 -31.1]/1e9; +default.radar.wfs(1).chan_equal_dB = [-4 -4.3 -2.9 -4.6 -1.2 -1.5 -0.9 -2.2]; +default.radar.wfs(1).chan_equal_deg = [113.1 64.8 124.3 133.7 108 138.1 71.6 102.9]; + +default.config_regexp = 'survey_150-520MHz_.*DECONV.xml'; +default.name = 'Deconv 150-520 MHz'; +defaults{end+1} = default; + +default.config_regexp = '.*150-520MHz.*'; +default.name = 'Other Settings 150-520 MHz'; +defaults{end+1} = default; + +%% Add default settings + +param.config.defaults = defaults; + diff --git a/cresis-toolbox/missions/default_radar_params_2022_Antarctica_BaslerMKB_accum.m b/cresis-toolbox/missions/default_radar_params_2022_Antarctica_BaslerMKB_accum.m new file mode 100644 index 00000000..a2a059df --- /dev/null +++ b/cresis-toolbox/missions/default_radar_params_2022_Antarctica_BaslerMKB_accum.m @@ -0,0 +1,311 @@ +function [param,defaults] = default_radar_params_2022_Antarctica_BaslerMKB_accum +% param = default_radar_params_2022_Antarctica_BaslerMKB_accum +% +% accum: 2022_Antarctica_BaslerMKB +% +% Creates base "param" struct +% Creates defaults cell array for each type of radar setting +% +% Set the param.season_name to the correct season before running. +% +% Author: John Paden +% +% See also: default_radar_params_2022_Antarctica_BaslerMKB_accum_data_map + +%% Preprocess parameters +param.season_name = '2022_Antarctica_BaslerMKB'; +param.radar_name = 'accum'; + +% Reading in files +param.config.daq_type = 'arena'; +param.config.wg_type = 'arena'; +param.config.header_load_func = @basic_load_arena; +param.config.board_map = {'digrx0','digrx1'}; +param.config.tx_map = {'awg0','awg1'}; + +% Creating segments +param.config.max_time_gap = 10; +param.config.min_seg_size = 1; + +% Creating settings files +param.config.max_data_rate = 60; +param.config.max_duty_cycle = 0.1; +param.config.prf_multiple = [10e6 10e6/20]; % Power supply sync signal that PRF must be a factor of these numbers +param.config.PRI_guard = 1e-6; +param.config.PRI_guard_percentage = 450e6/500e6; +param.config.tx_enable = [1 1]; +param.config.max_tx = 1.0; +param.config.max_tx_voltage = sqrt(400*50)*10^(-2/20); % voltage at max_tx + +%% COLDEX accum Arena Parameters +arena = []; +arena.clk = 10e6; +fs = 280e6*16; +fs_dac = 280e6*32; +subsystem_idx = 0; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'ARENA5xxRF'; +arena.subsystem(subsystem_idx).subSystem{1} = 'dac1'; +arena.subsystem(subsystem_idx).subSystem{2} = 'adc'; +arena.subsystem(subsystem_idx).subSystem{3} = 'ctu1'; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'ARENA5xxRF_2'; +arena.subsystem(subsystem_idx).subSystem{1} = 'dac2'; +arena.subsystem(subsystem_idx).subSystem{2} = 'adc'; +arena.subsystem(subsystem_idx).subSystem{3} = 'ctu1'; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'Data Server'; + +dac_idx = 0; +dac_idx = dac_idx + 1; +arena.dac(dac_idx).name = 'dac1'; +arena.dac(dac_idx).type = 'rfsoc_dac_1002'; +arena.dac(dac_idx).dacClk = fs_dac; +arena.dac(dac_idx).desiredAlignMin = -10; +arena.dac(dac_idx).desiredAlignMax = 0; +arena.dac(dac_idx).dcoPhase = 80; +dac_idx = dac_idx + 1; +arena.dac(dac_idx).name = 'dac2'; +arena.dac(dac_idx).type = 'rfsoc_dac_1002'; +arena.dac(dac_idx).dacClk = fs_dac; +arena.dac(dac_idx).desiredAlignMin = -4; +arena.dac(dac_idx).desiredAlignMax = 6; +arena.dac(dac_idx).dcoPhase = 80; + +adc_idx = 0; +adc_idx = adc_idx + 1; +arena.adc(adc_idx).name = 'digrx0'; +arena.adc(adc_idx).type = 'rfsoc_adc_2002'; +arena.adc(adc_idx).sampFreq = fs; +arena.adc(adc_idx).adcMode = 1; +arena.adc(adc_idx).desiredAlignMin = -8; +arena.adc(adc_idx).desiredAlignMax = 2; +arena.adc(adc_idx).stream = 'socket'; +arena.adc(adc_idx).ip = '172.16.0.122'; +arena.adc(adc_idx).outputSelect = 1; +arena.adc(adc_idx).wf_set = 1; +arena.adc(adc_idx).gain_dB = [0 0]; +adc_idx = adc_idx + 1; +arena.adc(adc_idx).name = 'digrx1'; +arena.adc(adc_idx).type = 'rfsoc_adc_2002'; +arena.adc(adc_idx).sampFreq = fs; +arena.adc(adc_idx).adcMode = 1; +arena.adc(adc_idx).desiredAlignMin = NaN; +arena.adc(adc_idx).desiredAlignMax = NaN; +arena.adc(adc_idx).stream = 'socket'; +arena.adc(adc_idx).ip = '172.16.0.123'; +arena.adc(adc_idx).outputSelect = 1; +arena.adc(adc_idx).wf_set = 2; +arena.adc(adc_idx).gain_dB = [0 0]; + +daq_idx = 0; +daq_idx = daq_idx + 1; +arena.daq(daq_idx).name = 'daq0'; +arena.daq(daq_idx).type = 'daq_0001'; +arena.daq(daq_idx).auxDir = '/data/'; +arena.daq(daq_idx).fileStripe = '/data/%b/'; +arena.daq(daq_idx).fileName = 'accum_%b'; +arena.daq.udp_packet_headers = false; +% arena.daq.udp_packet_headers = true; % HACK FOR TEST DATASET + +arena.system.name = 'extsyncarena5xx'; + +arena.param.tx_max = [1 1]; +arena.param.PA_setup_time = 2e-6; % Time required to enable PA before transmit +arena.param.TTL_time_delay = 0.0; % TTL time delay relative to transmit start +arena.param.ADC_time_delay = -2e-6; % ADC time delay relative to transmit start + +arena.psc.type = 'psc_0008'; + +arena.daq.type = 'daq_0001'; + +arena.ctu.name = 'ctu'; +arena.ctu.type = 'ctu_001D'; +if 1 + % External GPS + arena.ctu.nmea = 31; + arena.ctu.nmea_baud = 115200; + arena.ctu.pps = 10; + arena.ctu.pps_polarity = 1; +else + % Internal GPS + arena.ctu.nmea = 60; + arena.ctu.nmea_baud = 115200; + arena.ctu.pps = 63; + arena.ctu.pps_polarity = 1; +end +idx = 0; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'EPRI'; +arena.ctu.out.bit_group(idx).bits = 0; +arena.ctu.out.bit_group(idx).epri = [1 0]; +arena.ctu.out.bit_group(idx).pri = [0 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'PRI'; +arena.ctu.out.bit_group(idx).bits = 1; +arena.ctu.out.bit_group(idx).epri = [1 0]; +arena.ctu.out.bit_group(idx).pri = [1 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'PA'; % 1 enables the transmitter +arena.ctu.out.bit_group(idx).bits = 2; +arena.ctu.out.bit_group(idx).epri = [1 0]; +arena.ctu.out.bit_group(idx).pri = [1 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'TR'; % 1 enables tx-ant path +arena.ctu.out.bit_group(idx).bits = 3; +arena.ctu.out.bit_group(idx).epri = [1 0]; +arena.ctu.out.bit_group(idx).pri = [1 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'Blank'; % 1 enables rx blank mode +arena.ctu.out.bit_group(idx).bits = 4; +arena.ctu.out.bit_group(idx).epri = [1 0]; +arena.ctu.out.bit_group(idx).pri = [1 0]; + +arena.ctu.out.time_cmd = {'2e-6+param.wfs(wf).Tpd+0.1e-6' '2e-6+param.wfs(wf).Tpd+0.1e-6' '2e-6+param.wfs(wf).Tpd+0.1e-6' '2/param.prf'}; + +default.arena = arena; + +%% Command worksheet +param.cmd.records = 1; +param.cmd.qlook = 1; +param.cmd.generic = 1; + +%% Records worksheet +param.records.gps.time_offset = 0; +param.records.frames.geotiff_fn = 'antarctica\Landsat-7\Antarctica_LIMA.tif'; +param.records.frames.mode = 1; +param.records.file.version = 103; +param.records.file.prefix = param.radar_name; +param.records.file.suffix = '.bin'; +param.records.file.boards = {'digrx0','digrx1'}; +param.records.file.board_folder_name = '%b'; +param.records.file.clk = 10e6; + +%% Qlook worksheet +param.qlook.out_path = ''; +param.qlook.block_size = 5000; +param.qlook.motion_comp = 0; +param.qlook.dec = 20; +param.qlook.inc_dec = 10; +param.qlook.surf.en = 1; +param.qlook.surf.method = 'accum'; +param.qlook.resample = [2 1]; + +%% SAR worksheet +param.sar.out_path = ''; +param.sar.frm_types = {0,[0 1],0,0,-1}; +param.sar.chunk_len = 5000; +param.sar.chunk_overlap = 10; +param.sar.frm_overlap = 0; +param.sar.coh_noise_removal = 0; +param.sar.combine_rx = 0; +param.sar.time_of_full_support = inf; +param.sar.pulse_rfi.en = []; +param.sar.pulse_rfi.inc_ave= []; +param.sar.pulse_rfi.thresh_scale = []; +param.sar.trim_vals = []; +param.sar.pulse_comp = 1; +param.sar.ft_dec = 1; +param.sar.ft_wind = @hanning; +param.sar.ft_wind_time = 0; +param.sar.lever_arm_fh = @lever_arm; +param.sar.mocomp.en = 1; +param.sar.mocomp.type = 2; +param.sar.mocomp.filter = {@butter [2] [0.1000]}; +param.sar.mocomp.uniform_en = 1; +param.sar.sar_type = 'fk'; +param.sar.sigma_x = 1; +param.sar.sub_aperture_steering = 0; +param.sar.st_wind = @hanning; +param.sar.start_eps = 3.15; + +%% Array worksheet +param.array.in_path = ''; +param.array.array_path = ''; +param.array.out_path = ''; +param.array.method = 'standard'; +param.array.window = @hanning; +param.array.bin_rng = 0; +param.array.line_rng = -51:51; +param.array.dbin = 1; +param.array.dline = 26; +param.array.DCM = []; +param.array.Nsv = 1; +param.array.theta_rng = [0 0]; +param.array.sv_fh = @array_proc_sv; +param.array.diag_load = 0; +param.array.Nsig = 2; + +%% Radar worksheet +param.radar.adc_bits = 14; +param.radar.Vpp_scale = 1.5; +param.radar.lever_arm_fh = @lever_arm; +for wf = 1:4 + param.radar.wfs(wf).adc_gains_dB = [38 38 38 38 38 38 38 38 38 38 38 38 38 38 38 38]; % ADC gain + param.radar.wfs(wf).adcs = [1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16]; % ADCs + param.radar.wfs(wf).rx_paths = [1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16]; % ADC to rx path mapping + param.radar.wfs(wf).gain_en = [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]; % Disable fast-time gain correction + param.radar.wfs(wf).coh_noise_method = ''; % No coherent noise removal + param.radar.wfs(wf).Tadc_adjust = -1.5e-6; + param.radar.wfs(wf).bit_shifts = [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]; +end +Tsys = [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]/1e9; +chan_equal_dB = [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]; +chan_equal_deg = [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]; + +%% Post worksheet +param.post.data_dirs = {'qlook'}; +param.post.img = 0; +param.post.layer_dir = 'layerData'; +param.post.maps_en = 1; +param.post.echo_en = 1; +param.post.layers_en = 0; +param.post.data_en = 0; +param.post.csv_en = 1; +param.post.concat_en = 1; +param.post.pdf_en = 1; +param.post.map.location = 'Antarctica'; +param.post.map.type = 'combined'; +param.post.echo.elev_comp = 2; +param.post.echo.depth = '[min(Surface_Depth)-100 max(Surface_Depth)+3500]'; +% param.post.echo.elev_comp = 3; +% param.post.echo.depth = '[min(Surface_Elev)-1500 max(Surface_Elev)+100]'; +param.post.echo.er_ice = 3.15; +param.post.ops.location = 'antarctic'; + +%% Analysis worksheet +param.analysis_noise.block_size = 10000; +cmd_idx = 0; +cmd_idx = cmd_idx + 1; +param.analysis_noise.cmd{cmd_idx}.method = 'coh_noise'; +param.analysis_noise.cmd{cmd_idx}.distance_weight = 1; % Enable distance weighting of the average + +%% Radar Settings + +defaults = {}; + +% Survey Mode Thick Ice Single Polarization +default.records.arena.total_presums = 80; +default.records.data_map = default_radar_params_2022_Antarctica_BaslerMKB_accum_data_map(); +default.qlook.img_comb = [9e-06 -inf 1e-06]; +default.qlook.imgs = {[ones(16,1) (1:16).'],[3*ones(16,1) (1:16).']}; +default.sar.imgs = default.qlook.imgs; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.analysis_noise.imgs = default.qlook.imgs; +default.radar.ref_fn = ''; +default.radar.wfs = param.radar.wfs(1:4); +for wf = 1:4 + default.radar.wfs(wf).Tsys = Tsys; + default.radar.wfs(wf).chan_equal_dB = chan_equal_dB; + default.radar.wfs(wf).chan_equal_deg = chan_equal_deg; + default.radar.wfs(wf).tx_paths = {[4 3 2 1 inf inf inf inf],[16 15 14 13 inf inf inf inf]}; % dac1-ch1->Ant4, dac1-ch2->Ant3, dac1-ch3->Ant2, dac1-ch4->Ant1, dac2-ch1->Ant16, dac2-ch2->Ant15, dac2-ch3->Ant14, dac2-ch4->Ant13 + default.radar.wfs(wf).DDC_dec = 8; % 280 MHz / 8 = 35 MHz + default.radar.wfs(wf).bit_shifts = zeros(1,16); +end +default.post.echo.depth = '[min(Surface_Depth)-5 max(Surface_Depth)+4200]'; +% Note psc config name was incorrectly set, but it is for shallow ice: +default.config_regexp = 'psc_survey'; +default.name = 'Survey Mode 675-725 MHz Thick Ice'; +defaults{end+1} = default; + diff --git a/cresis-toolbox/missions/default_radar_params_2022_Antarctica_BaslerMKB_accum_data_map.m b/cresis-toolbox/missions/default_radar_params_2022_Antarctica_BaslerMKB_accum_data_map.m new file mode 100644 index 00000000..e58edbe9 --- /dev/null +++ b/cresis-toolbox/missions/default_radar_params_2022_Antarctica_BaslerMKB_accum_data_map.m @@ -0,0 +1,47 @@ +function data_map = default_radar_params_2022_Antarctica_BaslerMKB_accum_data_map(day_seg) +% data_map = default_radar_params_2022_Antarctica_BaslerMKB_accum_data_map(day_seg) +% +% Used to create param.records.data_map. +% +% Support function for default_radar_params_2022_Antarctica_BaslerMKB_accum +% and in general for the dataset corresponding to +% season: 2022_Antarctica_BaslerMKB +% radar: accum +% +% Author: John Paden +% +% See also: default_radar_params_2022_Antarctica_BaslerMKB_accum + +if ~exist('day_seg','var') || isempty(day_seg) + data_map = {}; + for board_idx = 1:2 + data_map{board_idx} = []; + profile = 0; + wf_list = [1 2 3 4]; + mode = 0; + for wf = wf_list + for subchannel=0:7 + data_map{board_idx}(end+1,1:5) = [profile mode subchannel wf subchannel+1+(board_idx-1)*8]; + data_map{board_idx}(end+1,1:5) = [profile mode+1 subchannel wf subchannel+1+(board_idx-1)*8]; + profile = profile+1; + end + mode = mode+2; + end + end +elseif strcmp(day_seg,'20230115_03') + data_map = {}; + for board_idx = 1 + data_map{board_idx} = []; + profile = 0; + wf_list = [1 2 3 4]; + mode = 0; + for wf = wf_list + for subchannel=0:7 + data_map{board_idx}(end+1,1:5) = [profile mode subchannel wf subchannel+1+(board_idx)*8]; + data_map{board_idx}(end+1,1:5) = [profile mode+1 subchannel wf subchannel+1+(board_idx)*8]; + profile = profile+1; + end + mode = mode+2; + end + end +end \ No newline at end of file diff --git a/cresis-toolbox/missions/default_radar_params_2022_Antarctica_BaslerMKB_rds.m b/cresis-toolbox/missions/default_radar_params_2022_Antarctica_BaslerMKB_rds.m new file mode 100644 index 00000000..96572cc2 --- /dev/null +++ b/cresis-toolbox/missions/default_radar_params_2022_Antarctica_BaslerMKB_rds.m @@ -0,0 +1,179 @@ +function [param,defaults] = default_radar_params_2022_Antarctica_BaslerMKB_rds +% param = default_radar_params_2022_Antarctica_BaslerMKB_rds +% +% rds: 2022_Antarctica_BaslerMKB +% +% Creates base "param" struct +% Creates defaults cell array for each type of radar setting +% +% Set the param.season_name to the correct season before running. +% +% Author: John Paden +% +% See also: default_radar_params_*.m + +%% Preprocess parameters +param.season_name = '2022_Antarctica_BaslerMKB'; +param.radar_name = 'rds'; + +% Reading in files +param.config.daq_type = 'utig'; +param.config.header_load_func = @basic_load_utig; +param.config.file.prefix = 'radar'; +param.config.board_map = {''}; + +% Creating segments +param.config.max_time_gap = 10; +param.config.min_seg_size = 1; + +default = []; + +%% Command worksheet +param.cmd.records = 1; +param.cmd.qlook = 1; +param.cmd.generic = 1; + +%% Records worksheet +param.records.gps.time_offset = 0; +param.records.gps.en = 1; +param.records.frames.geotiff_fn = 'antarctica\Landsat-7\Antarctica_LIMA.tif'; +param.records.frames.mode = 1; +param.records.file.version = 415; +param.records.file.prefix = 'radar0'; +param.records.file.suffix = '.dat'; +param.records.file.boards = {''}; +param.records.file.board_folder_name = ''; +param.records.file.clk = 10e6; + +%% Qlook worksheet +param.qlook.out_path = ''; +param.qlook.block_size = 20000; +param.qlook.motion_comp = 0; +param.qlook.dec = 10; +param.qlook.inc_dec = 5; +param.qlook.surf.en = 1; +param.qlook.surf.method = 'rds'; +param.qlook.resample = [2 1]; + +%% SAR worksheet +param.sar.out_path = ''; +param.sar.frm_types = {0,[0 1],0,0,-1}; +param.sar.chunk_len = 5000; +param.sar.chunk_overlap = 10; +param.sar.frm_overlap = 0; +param.sar.coh_noise_removal = 0; +param.sar.combine_rx = 0; +param.sar.time_of_full_support = inf; +param.sar.pulse_rfi.en = []; +param.sar.pulse_rfi.inc_ave= []; +param.sar.pulse_rfi.thresh_scale = []; +param.sar.trim_vals = []; +param.sar.pulse_comp = 1; +param.sar.ft_dec = 1; +param.sar.ft_wind = @hanning; +param.sar.ft_wind_time = 0; +param.sar.lever_arm_fh = @lever_arm; +param.sar.mocomp.en = 1; +param.sar.mocomp.type = 2; +param.sar.mocomp.filter = {@butter [2] [0.1000]}; +param.sar.mocomp.uniform_en = 1; +param.sar.sar_type = 'fk'; +param.sar.sigma_x = 2.5; +param.sar.sub_aperture_steering = 0; +param.sar.st_wind = @hanning; +param.sar.start_eps = 3.15; + +%% Array worksheet +param.array.in_path = ''; +param.array.array_path = ''; +param.array.out_path = ''; +param.array.method = 'standard'; +param.array.window = @hanning; +param.array.bin_rng = 0; +param.array.line_rng = -10:10; +param.array.dbin = 1; +param.array.dline = 11; +param.array.DCM = []; +param.array.Nsv = 1; +param.array.theta_rng = [0 0]; +param.array.sv_fh = @array_proc_sv; +param.array.diag_load = 0; +param.array.Nsig = 2; + +%% Radar worksheet +param.radar.fs = 50e6; +param.radar.prf = 6250; +param.radar.adc_bits = 14; +param.radar.Vpp_scale = 2; +param.radar.lever_arm_fh = @lever_arm; +param.radar.wfs = []; +for wf = 1:1 + param.radar.wfs(wf).adcs = [1 2 3 4]; % ADCs + param.radar.wfs(wf).f0 = 52.5e6; + param.radar.wfs(wf).f1 = 67.5e6; + param.radar.wfs(wf).DDC_dec = 1; + param.radar.wfs(wf).DDC_freq = 0; + param.radar.wfs(wf).BW_window = [param.radar.wfs(wf).f0 param.radar.wfs(wf).f1]; + param.radar.wfs(wf).ft_dec = [1 1]; + param.radar.wfs(wf).Tpd = 1e-6; + param.radar.wfs(wf).tukey = 0.1; + param.radar.wfs(wf).system_dB = 0; + param.radar.wfs(wf).rx_paths = [1 2 1 2]; + param.radar.wfs(wf).adc_gains_dB = [4 50 4 50]; % ADC gain + param.radar.wfs(wf).chan_equal_dB = [0 0 0 0]; + param.radar.wfs(wf).chan_equal_deg = [0 0 0 0]; + param.radar.wfs(wf).Tsys = [0 0 0 0]; + param.radar.wfs(wf).presums = 32; + param.radar.wfs(wf).bit_shifts = [0 0 0 0]; + param.radar.wfs(wf).Tadc_adjust = -1.2e-6; + param.radar.wfs(wf).Tadc = 0; + param.radar.wfs(wf).gain_en = [0 0 0 0]; % Disable fast-time gain correction + param.radar.wfs(wf).coh_noise_method = ''; % No coherent noise removal +end + +%% Post worksheet +param.post.data_dirs = {'qlook'}; +param.post.img = 0; +param.post.layer_dir = 'layerData'; +param.post.maps_en = 1; +param.post.echo_en = 1; +param.post.layers_en = 0; +param.post.data_en = 0; +param.post.csv_en = 1; +param.post.concat_en = 1; +param.post.pdf_en = 1; +param.post.map.location = 'Antarctica'; +param.post.map.type = 'combined'; +param.post.echo.elev_comp = 2; +param.post.echo.depth = '[min(Surface_Depth)-100 max(Surface_Depth)+3500]'; +% param.post.echo.elev_comp = 3; +% param.post.echo.depth = '[min(Surface_Elev)-1500 max(Surface_Elev)+100]'; +param.post.echo.er_ice = 3.15; +param.post.ops.location = 'antarctic'; + +%% Analysis worksheet +param.analysis_noise.block_size = 10000; +cmd_idx = 0; +cmd_idx = cmd_idx + 1; +param.analysis_noise.cmd{cmd_idx}.method = 'coh_noise'; +param.analysis_noise.cmd{cmd_idx}.distance_weight = 1; % Enable distance weighting of the average + +%% Radar Settings + +defaults = {}; + +% Survey Mode Thick Ice +default.records.arena.total_presums = 32; +default.qlook.img_comb = [4e-06 -inf 1e-06]; +default.qlook.imgs = {[ones(2,1) [1 3].'],[ones(2,1) [2 4].']}; +default.sar.imgs = default.qlook.imgs; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.analysis_noise.imgs = default.qlook.imgs; +default.radar.ref_fn = ''; +default.radar.wfs = param.radar.wfs(1:1); +default.post.echo.depth = '[min(Surface_Depth)-5 max(Surface_Depth)+4200]'; +default.config_regexp = 'survey'; +default.name = 'Survey Mode 52.5-67.5 MHz Thick Ice'; +defaults{end+1} = default; + diff --git a/cresis-toolbox/missions/default_radar_params_2022_Antarctica_GroundGHOST_rds.m b/cresis-toolbox/missions/default_radar_params_2022_Antarctica_GroundGHOST_rds.m new file mode 100644 index 00000000..fb26dd47 --- /dev/null +++ b/cresis-toolbox/missions/default_radar_params_2022_Antarctica_GroundGHOST_rds.m @@ -0,0 +1,362 @@ +function [param,defaults] = default_radar_params_2022_Antarctica_GroundGHOST_rds +% param = default_radar_params_2022_Antarctica_GroundGHOST_rds +% +% rds: 2022_Antarctica_GroundGHOST +% +% Creates base "param" struct +% Creates defaults cell array for each type of radar setting +% +% Set the param.season_name to the correct season before running. +% +% Author: John Paden + +%% Preprocess parameters +param.season_name = '2022_Antarctica_GroundGHOST'; +param.radar_name = 'rds'; + +% Reading in files +param.config.daq_type = 'arena'; +param.config.wg_type = 'arena'; +param.config.header_load_func = @basic_load_arena; +param.config.board_map = {'digrx1','digrx2'}; +param.config.tx_map = {'awg1'}; + +% Creating segments +param.config.max_time_gap = 10; +param.config.min_seg_size = 1; + +% Creating settings files +param.config.max_data_rate = 60; +param.config.max_duty_cycle = 0.1; +param.config.prf_multiple = [10e6 10e6/20]; % Power supply sync signal that PRF must be a factor of these numbers +param.config.PRI_guard = 1e-6; +param.config.PRI_guard_percentage = 450e6/500e6; +param.config.tx_enable = [1 1]; +param.config.max_tx = 1.0; +param.config.max_tx_voltage = sqrt(400*50)*10^(-2/20); % voltage at max_tx + +%% GHOST 1 rds Arena Parameters +arena = []; +arena.clk = 10e6; +fs = 480e6; +fs_snow = 1000e6; +fs_dac = 480e6; +subsystem_idx = 0; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'CTU'; +arena.subsystem(subsystem_idx).subSystem{1} = 'ctu'; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'SNOW'; +arena.subsystem(subsystem_idx).subSystem{1} = 'awg0'; +arena.subsystem(subsystem_idx).subSystem{2} = 'digrx0'; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'RDS_RX1'; +arena.subsystem(subsystem_idx).subSystem{1} = 'digrx1'; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'RDS_AWG_RX2'; +arena.subsystem(subsystem_idx).subSystem{1} = 'awg1'; +arena.subsystem(subsystem_idx).subSystem{2} = 'digrx2'; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'Data Server'; + +dac_idx = 0; +dac_idx = dac_idx + 1; +arena.dac(dac_idx).name = 'awg0'; +arena.dac(dac_idx).type = 'dac-ad9129_0012'; +arena.dac(dac_idx).dacClk = fs_dac; +arena.dac(dac_idx).desiredAlignMin = -10; +arena.dac(dac_idx).desiredAlignMax = 0; +arena.dac(dac_idx).dcoPhase = 80; +dac_idx = dac_idx + 1; +arena.dac(dac_idx).name = 'awg1'; +arena.dac(dac_idx).type = 'dac-ad9783_002C'; +arena.dac(dac_idx).dacClk = fs_dac; +arena.dac(dac_idx).desiredAlignMin = -4; +arena.dac(dac_idx).desiredAlignMax = 6; +arena.dac(dac_idx).dcoPhase = 80; + +adc_idx = 0; +adc_idx = adc_idx + 1; +arena.adc(adc_idx).name = 'digrx0'; +arena.adc(adc_idx).type = 'adc-ad9680_0017'; +arena.adc(adc_idx).sampFreq = fs_snow; +arena.adc(adc_idx).adcMode = 1; +arena.adc(adc_idx).desiredAlignMin = -8; +arena.adc(adc_idx).desiredAlignMax = 2; +arena.adc(adc_idx).stream = 'socket'; +arena.adc(adc_idx).ip = '172.16.0.121'; +arena.adc(adc_idx).outputSelect = 1; +arena.adc(adc_idx).wf_set = 1; +arena.adc(adc_idx).gain_dB = [0 0]; +adc_idx = adc_idx + 1; +arena.adc(adc_idx).name = 'digrx1'; +arena.adc(adc_idx).type = 'adc-ad9684_002D'; +arena.adc(adc_idx).sampFreq = fs; +arena.adc(adc_idx).adcMode = 1; +arena.adc(adc_idx).desiredAlignMin = NaN; +arena.adc(adc_idx).desiredAlignMax = NaN; +arena.adc(adc_idx).stream = 'socket'; +arena.adc(adc_idx).ip = '172.16.0.18'; +arena.adc(adc_idx).outputSelect = 1; +arena.adc(adc_idx).wf_set = 2; +arena.adc(adc_idx).gain_dB = [0 0]; +adc_idx = adc_idx + 1; +arena.adc(adc_idx).name = 'digrx2'; +arena.adc(adc_idx).type = 'adc-ad9684_002D'; +arena.adc(adc_idx).sampFreq = fs; +arena.adc(adc_idx).adcMode = 1; +arena.adc(adc_idx).desiredAlignMin = NaN; +arena.adc(adc_idx).desiredAlignMax = NaN; +arena.adc(adc_idx).stream = 'socket'; +arena.adc(adc_idx).ip = '172.16.0.121'; +arena.adc(adc_idx).outputSelect = 1; +arena.adc(adc_idx).wf_set = 2; +arena.adc(adc_idx).gain_dB = [0 0]; + +daq_idx = 0; +daq_idx = daq_idx + 1; +arena.daq(daq_idx).name = 'daq0'; +arena.daq(daq_idx).type = 'daq_0001'; +arena.daq(daq_idx).auxDir = '/data/'; +arena.daq(daq_idx).fileStripe = '/data/%b/'; +arena.daq(daq_idx).fileName = 'rds_%b'; +arena.daq.udp_packet_headers = false; + +arena.system.name = 'ku0001'; + +arena.param.tx_max = [1 1]; +arena.param.PA_setup_time = 2e-6; % Time required to enable PA before transmit +arena.param.TTL_time_delay = 0.0; % TTL time delay relative to transmit start +arena.param.ADC_time_delay = -2e-6; % ADC time delay relative to transmit start + +arena.psc.type = 'psc_0003'; + +arena.daq.type = 'daq_0001'; + +arena.ctu.name = 'ctu'; +arena.ctu.type = 'ctu_001D'; +if 1 + % External GPS + arena.ctu.nmea = 31; + arena.ctu.nmea_baud = 115200; + arena.ctu.pps = 10; + arena.ctu.pps_polarity = 1; +else + % Internal GPS + arena.ctu.nmea = 60; + arena.ctu.nmea_baud = 115200; + arena.ctu.pps = 63; + arena.ctu.pps_polarity = 1; +end +idx = 0; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'EPRI'; +arena.ctu.out.bit_group(idx).bits = 0; +arena.ctu.out.bit_group(idx).epri = [1 0]; +arena.ctu.out.bit_group(idx).pri = [0 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'PRI'; +arena.ctu.out.bit_group(idx).bits = 1; +arena.ctu.out.bit_group(idx).epri = [1 0]; +arena.ctu.out.bit_group(idx).pri = [1 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'PA'; % 1 enables the transmitter +arena.ctu.out.bit_group(idx).bits = 2; +arena.ctu.out.bit_group(idx).epri = [1 0]; +arena.ctu.out.bit_group(idx).pri = [1 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'TR'; % 1 enables tx-ant path +arena.ctu.out.bit_group(idx).bits = 3; +arena.ctu.out.bit_group(idx).epri = [1 0]; +arena.ctu.out.bit_group(idx).pri = [1 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'Blank'; % 1 enables rx blank mode +arena.ctu.out.bit_group(idx).bits = 4; +arena.ctu.out.bit_group(idx).epri = [1 0]; +arena.ctu.out.bit_group(idx).pri = [1 0]; + +arena.ctu.out.time_cmd = {'2e-6+param.wfs(wf).Tpd+0.1e-6' '2e-6+param.wfs(wf).Tpd+0.1e-6' '2e-6+param.wfs(wf).Tpd+0.1e-6' '2/param.prf'}; + +default.arena = arena; + +%% Command worksheet +param.cmd.records = 1; +param.cmd.qlook = 1; +param.cmd.generic = 1; + +%% Records worksheet +param.records.gps.time_offset = 0; +param.records.frames.geotiff_fn = 'antarctica\Landsat-7\Antarctica_LIMA.tif'; +param.records.frames.mode = 1; +param.records.file.version = 103; +param.records.file.prefix = param.radar_name; +param.records.file.suffix = '.bin'; +param.records.file.boards = {'digrx1','digrx2'}; +param.records.file.board_folder_name = '%b'; +param.records.file.clk = 10e6; + +%% Qlook worksheet +param.qlook.out_path = ''; +param.qlook.block_size = 5000; +param.qlook.motion_comp = 0; +param.qlook.dec = 20; +param.qlook.inc_dec = 10; +param.qlook.surf.en = 1; +param.qlook.surf.method = 'fixed'; +param.qlook.surf.fixed_value = 0; +param.qlook.resample = [2 1]; + +%% SAR worksheet +param.sar.out_path = ''; +param.sar.frm_types = {0,[0 1],0,0,-1}; +param.sar.chunk_len = 5000; +param.sar.chunk_overlap = 10; +param.sar.frm_overlap = 0; +param.sar.coh_noise_removal = 0; +param.sar.combine_rx = 0; +param.sar.time_of_full_support = inf; +param.sar.pulse_rfi.en = []; +param.sar.pulse_rfi.inc_ave= []; +param.sar.pulse_rfi.thresh_scale = []; +param.sar.trim_vals = []; +param.sar.pulse_comp = 1; +param.sar.ft_dec = 1; +param.sar.ft_wind = @hanning; +param.sar.ft_wind_time = 0; +param.sar.lever_arm_fh = @lever_arm; +param.sar.mocomp.en = 1; +param.sar.mocomp.type = 2; +param.sar.mocomp.filter = {@butter [2] [0.1000]}; +param.sar.mocomp.uniform_en = 1; +param.sar.sar_type = 'fk'; +param.sar.sigma_x = 2.5; +param.sar.sub_aperture_steering = 0; +param.sar.st_wind = @hanning; +param.sar.start_eps = 3.15; + +%% Array worksheet +param.array.in_path = ''; +param.array.array_path = ''; +param.array.out_path = ''; +param.array.method = 'standard'; +param.array.window = @hanning; +param.array.bin_rng = 0; +param.array.line_rng = -5:5; +param.array.dbin = 1; +param.array.dline = 6; +param.array.DCM = []; +param.array.Nsv = 1; +param.array.theta_rng = [0 0]; +param.array.sv_fh = @array_proc_sv; +param.array.diag_load = 0; +param.array.Nsig = 2; + +%% Radar worksheet +param.radar.adc_bits = 14; +param.radar.Vpp_scale = 1.5; +param.radar.lever_arm_fh = @lever_arm; +for wf = 1:4 + param.radar.wfs(wf).adc_gains_dB = [38 38 38 38 38 38 38 38]; % ADC gain + param.radar.wfs(wf).adcs = [1 2 3 4 5 6 7 8]; % ADCs + param.radar.wfs(wf).rx_paths = [1 2 3 4 1 2 3 4]; % ADC to rx path mapping + param.radar.wfs(wf).gain_en = [0 0 0 0 0 0 0 0]; % Disable fast-time gain correction + param.radar.wfs(wf).coh_noise_method = ''; % No coherent noise removal + param.radar.wfs(wf).Tadc_adjust = 0; + param.radar.wfs(wf).bit_shifts = [0 0 0 0 0 0 0 0]; +end +Tsys = [0 0 0 0 0 0 0 0]/1e9; +chan_equal_dB = [0 0 0 0 0 0 0 0]; +chan_equal_deg = [0 0 0 0 0 0 0 0]; + +%% Post worksheet +param.post.data_dirs = {'qlook'}; +param.post.img = 0; +param.post.layer_dir = 'layerData'; +param.post.maps_en = 1; +param.post.echo_en = 1; +param.post.layers_en = 0; +param.post.data_en = 0; +param.post.csv_en = 1; +param.post.concat_en = 1; +param.post.pdf_en = 1; +param.post.map.location = 'Antarctica'; +param.post.map.type = 'combined'; +param.post.echo.elev_comp = 2; +param.post.echo.depth = '[min(Surface_Depth)-100 max(Surface_Depth)+1200]'; +% param.post.echo.elev_comp = 3; +% param.post.echo.depth = '[min(Surface_Elev)-1500 max(Surface_Elev)+100]'; +param.post.echo.er_ice = 3.15; +param.post.ops.location = 'antarctic'; + +%% Analysis worksheet +param.analysis_noise.block_size = 10000; +cmd_idx = 0; +cmd_idx = cmd_idx + 1; +param.analysis_noise.cmd{cmd_idx}.method = 'coh_noise'; +param.analysis_noise.cmd{cmd_idx}.distance_weight = 1; % Enable distance weighting of the average + +%% Radar Settings + +defaults = {}; + +% Survey Mode Thick Ice Single Polarization +% Note data_map has unusual ordering with profile 1 first instead of +% profile 0. Profile 1 corresponds to mode 0 which was transmitted first. +default.records.arena.total_presums = 400; +% default.records.data_map = {[0 0 1 1;1 0 1 1;2 0 2 1;3 0 2 1]}; +% Each row of param.records.data_map{board_idx} = [profile mode_latch subchannel wf adc] +default.records.data_map = {... + [0 0 0 1 1 + 1 2 0 2 1 + 2 4 0 3 1 + 3 6 0 4 1 + 4 0 1 1 2 + 5 2 1 2 2 + 6 4 1 3 2 + 7 6 1 4 2 + 8 0 2 1 3 + 9 2 2 2 3 + 10 4 2 3 3 + 11 6 2 4 3 + 12 0 3 1 4 + 13 2 3 2 4 + 14 4 3 3 4 + 15 6 3 4 4], ... + [0 0 0 1 5 + 1 2 0 2 5 + 2 4 0 3 5 + 3 6 0 4 5 + 4 0 1 1 6 + 5 2 1 2 6 + 6 4 1 3 6 + 7 6 1 4 6 + 8 0 2 1 7 + 9 2 2 2 7 + 10 4 2 3 7 + 11 6 2 4 7 + 12 0 3 1 8 + 13 2 3 2 8 + 14 4 3 3 8 + 15 6 3 4 8]}; +default.qlook.img_comb = [3e-06 -inf 1e-06]; +default.qlook.imgs = {[1 1; 1 2; 1 3; 1 4;],[3 1; 3 2; 3 3; 3 4;]}; +default.sar.imgs = default.qlook.imgs; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.analysis_noise.imgs = default.qlook.imgs; +default.radar.ref_fn = ''; +default.radar.wfs = param.radar.wfs(1:4); +for wf = 1:4 + default.radar.wfs(wf).Tsys = Tsys; + default.radar.wfs(wf).chan_equal_dB = chan_equal_dB; + default.radar.wfs(wf).chan_equal_deg = chan_equal_deg; + default.radar.wfs(wf).bit_shifts = [0 0 0 0 0 0 0 0]; + default.radar.wfs(wf).tx_paths = {[1 2 3 4 5 6]}; % awg1-CH0-->ANT1, awg1-CH1-->ANT2, awg1-CH2-->ANT5, awg1-CH3-->ANT6 + default.radar.wfs(wf).DDC_dec = 4; +end +default.post.echo.depth = '[min(Surface_Depth)-5 max(Surface_Depth)+1200]'; +% Note psc config name was incorrectly set, but it is for shallow ice: +default.config_regexp = 'ghost_config'; +default.name = 'Survey Mode 180-210 MHz Thick Ice'; +defaults{end+1} = default; + diff --git a/cresis-toolbox/missions/default_radar_params_2022_Antarctica_GroundGHOST_rds_data_map.m b/cresis-toolbox/missions/default_radar_params_2022_Antarctica_GroundGHOST_rds_data_map.m new file mode 100644 index 00000000..8f77bf06 --- /dev/null +++ b/cresis-toolbox/missions/default_radar_params_2022_Antarctica_GroundGHOST_rds_data_map.m @@ -0,0 +1,47 @@ +function data_map = default_radar_params_2022_Antarctica_GroundGHOST_rds_data_map(day_seg) +% data_map = default_radar_params_2022_Antarctica_GroundGHOST_rds_data_map(day_seg) +% +% Used to create param.records.data_map. +% +% Support function for default_radar_params_2022_Antarctica_GroundGHOST_rds +% and in general for the dataset corresponding to +% season: 2022_Antarctica_BaslerMKB +% radar: rds +% +% Author: John Paden +% +% See also: default_radar_params_2022_Antarctica_GroundGHOST_rds + +data_map = {... + [0 0 0 1 1 + 1 2 0 2 1 + 2 4 0 3 1 + 3 6 0 4 1 + 4 0 1 1 2 + 5 2 1 2 2 + 6 4 1 3 2 + 7 6 1 4 2 + 8 0 2 1 3 + 9 2 2 2 3 + 10 4 2 3 3 + 11 6 2 4 3 + 12 0 3 1 4 + 13 2 3 2 4 + 14 4 3 3 4 + 15 6 3 4 4], ... + [0 0 0 1 5 + 1 2 0 2 5 + 2 4 0 3 5 + 3 6 0 4 5 + 4 0 1 1 6 + 5 2 1 2 6 + 6 4 1 3 6 + 7 6 1 4 6 + 8 0 2 1 7 + 9 2 2 2 7 + 10 4 2 3 7 + 11 6 2 4 7 + 12 0 3 1 8 + 13 2 3 2 8 + 14 4 3 3 8 + 15 6 3 4 8]}; diff --git a/cresis-toolbox/missions/default_radar_params_2022_Antarctica_Ground_accum.m b/cresis-toolbox/missions/default_radar_params_2022_Antarctica_Ground_accum.m new file mode 100644 index 00000000..a1a44aa1 --- /dev/null +++ b/cresis-toolbox/missions/default_radar_params_2022_Antarctica_Ground_accum.m @@ -0,0 +1,377 @@ +function [param,defaults] = default_radar_params_2022_Antarctica_Ground_accum +% [param,defaults] = default_radar_params_2022_Antarctica_Ground_accum +% +% Accum: 2022_Antarctica_Ground +% +% Creates base "param" struct +% Creates defaults cell array for each type of radar setting +% +% Set the param.season_name to the correct season before running. +% +% Author: John Paden + +%% Preprocess parameters +param.season_name = '2022_Antarctica_Ground'; +param.radar_name = 'accum3'; + +% Reading in files +param.config.daq_type = 'arena'; +param.config.wg_type = 'arena'; +param.config.header_load_func = @basic_load_arena; +%param.config.board_map = {'digrx0'}; % H only polarization +param.config.board_map = {'digrx0','digrx1'}; % HV polarization +param.config.tx_map = {'awg0','awg1'}; + +% Creating segments +param.config.max_time_gap = 10; +param.config.min_seg_size = 1; + +% Creating settings files +param.config.max_data_rate = 60; +param.config.max_duty_cycle = 0.1; +param.config.prf_multiple = [10e6 10e6/20]; % Power supply sync signal that PRF must be a factor of these numbers +param.config.PRI_guard = 1e-6; +param.config.PRI_guard_percentage = 450e6/500e6; +param.config.tx_enable = [1 1]; +param.config.max_tx = 1.0; +param.config.max_tx_voltage = sqrt(400*50)*10^(-2/20); % voltage at max_tx + +%% EAGER 1 ACCUM Arena Parameters +arena = []; +arena.clk = 10e6; +fs = 1000e6; +fs_dac = 2000e6; +subsystem_idx = 0; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'CTU51'; +arena.subsystem(subsystem_idx).subSystem{1} = 'ctu'; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'ARENA60'; +arena.subsystem(subsystem_idx).subSystem{1} = 'awg0'; +arena.subsystem(subsystem_idx).subSystem{2} = 'digrx0'; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'ARENA61'; +arena.subsystem(subsystem_idx).subSystem{1} = 'awg1'; +arena.subsystem(subsystem_idx).subSystem{1} = 'digrx1'; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'Data Server'; + +dac_idx = 0; +dac_idx = dac_idx + 1; +arena.dac(dac_idx).name = 'awg0'; +arena.dac(dac_idx).type = 'dac-ad9129_0012'; +arena.dac(dac_idx).dacClk = fs_dac; +arena.dac(dac_idx).desiredAlignMin = -10; +arena.dac(dac_idx).desiredAlignMax = 0; +arena.dac(dac_idx).dcoPhase = 80; +dac_idx = dac_idx + 1; +arena.dac(dac_idx).name = 'awg1'; +arena.dac(dac_idx).type = 'dac-ad9129_0012'; +arena.dac(dac_idx).dacClk = fs_dac; +arena.dac(dac_idx).desiredAlignMin = -4; +arena.dac(dac_idx).desiredAlignMax = 6; +arena.dac(dac_idx).dcoPhase = 80; + +adc_idx = 0; +adc_idx = adc_idx + 1; +arena.adc(adc_idx).name = 'digrx0'; +arena.adc(adc_idx).type = 'adc-ad9680_0017'; +arena.adc(adc_idx).sampFreq = fs; +arena.adc(adc_idx).adcMode = 1; +arena.adc(adc_idx).desiredAlignMin = -8; +arena.adc(adc_idx).desiredAlignMax = 2; +arena.adc(adc_idx).stream = 'socket'; +arena.adc(adc_idx).ip = '172.16.0.118'; +arena.adc(adc_idx).outputSelect = 1; +arena.adc(adc_idx).wf_set = 1; +arena.adc(adc_idx).gain_dB = [0 0]; +adc_idx = adc_idx + 1; +arena.adc(adc_idx).name = 'digrx1'; +arena.adc(adc_idx).type = 'adc-ad9680_0017'; +arena.adc(adc_idx).sampFreq = fs; +arena.adc(adc_idx).adcMode = 1; +arena.adc(adc_idx).desiredAlignMin = -2; +arena.adc(adc_idx).desiredAlignMax = 8; +arena.adc(adc_idx).stream = 'socket'; +arena.adc(adc_idx).ip = '172.16.0.118'; +arena.adc(adc_idx).outputSelect = 1; +arena.adc(adc_idx).wf_set = 2; +arena.adc(adc_idx).gain_dB = [0 0]; + +daq_idx = 0; +daq_idx = daq_idx + 1; +arena.daq(daq_idx).name = 'daq0'; +arena.daq(daq_idx).type = 'daq_0001'; +arena.daq(daq_idx).auxDir = '/data/'; +arena.daq(daq_idx).fileStripe = '/data/%b/'; +arena.daq(daq_idx).fileName = 'accum'; +arena.daq.udp_packet_headers = false; + +arena.system.name = 'ku0001'; + +arena.param.tx_max = [1 1]; +arena.param.PA_setup_time = 2e-6; % Time required to enable PA before transmit +arena.param.TTL_time_delay = 0.0; % TTL time delay relative to transmit start +arena.param.ADC_time_delay = 3.0720e-6; % ADC time delay relative to transmit start + +arena.psc.type = 'psc_0003'; + +arena.daq.type = 'daq_0001'; + +arena.ctu.name = 'ctu'; +arena.ctu.type = 'ctu_001D'; +if 1 + % External GPS + arena.ctu.nmea = 31; + arena.ctu.nmea_baud = 115200; + arena.ctu.pps = 10; + arena.ctu.pps_polarity = 1; +else + % Internal GPS + arena.ctu.nmea = 60; + arena.ctu.nmea_baud = 115200; + arena.ctu.pps = 63; + arena.ctu.pps_polarity = 1; +end +idx = 0; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'EPRI'; +arena.ctu.out.bit_group(idx).bits = 0; +arena.ctu.out.bit_group(idx).epri = [1 0]; +arena.ctu.out.bit_group(idx).pri = [0 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'PRI'; +arena.ctu.out.bit_group(idx).bits = 1; +arena.ctu.out.bit_group(idx).epri = [1 0]; +arena.ctu.out.bit_group(idx).pri = [1 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'TX_EN'; % 1 enables the transmitter, sets the T/R to transmit +arena.ctu.out.bit_group(idx).bits = 2; +arena.ctu.out.bit_group(idx).epri = [1 0]; +arena.ctu.out.bit_group(idx).pri = [1 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'CAL_EN'; % 1 enables rx calibration switch +arena.ctu.out.bit_group(idx).bits = 3; +arena.ctu.out.bit_group(idx).epri = [1 0]; +arena.ctu.out.bit_group(idx).pri = [1 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'Atten1_TRISO_NOT_USED'; % unused +arena.ctu.out.bit_group(idx).bits = 4; +arena.ctu.out.bit_group(idx).epri = [0 0]; +arena.ctu.out.bit_group(idx).pri = [0 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'LOW_GAIN_EN'; % 1 is low gain +arena.ctu.out.bit_group(idx).bits = 5; +arena.ctu.out.bit_group(idx).epri = [1 1]; +arena.ctu.out.bit_group(idx).pri = [1 1]; + +arena.ctu.out.time_cmd = {'2e-6+param.wfs(wf).Tpd+0.1e-6' '2e-6+param.wfs(wf).Tpd+0.1e-6' '2e-6+param.wfs(wf).Tpd+0.1e-6' '2/param.prf'}; + +%% Command worksheet +param.cmd.records = 1; +param.cmd.qlook = 1; +param.cmd.generic = 1; + +%% Records worksheet +param.records.gps.time_offset = 0; +param.records.frames.geotiff_fn = 'antarctica\Landsat-7\Antarctica_LIMA.tif'; +param.records.frames.mode = 1; +param.records.file.version = 103; +param.records.file.prefix = param.radar_name; +param.records.file.suffix = '.bin'; +%param.records.file.boards = {'digrx0'}; % H only polarization +param.records.file.boards = {'digrx0','digrx1'}; % HV Polarization +param.records.file.board_folder_name = '%b'; +param.records.file.clk = 10e6; + +%% Qlook worksheet +param.qlook.out_path = ''; +param.qlook.block_size = 5000; +param.qlook.motion_comp = 0; +param.qlook.dec = 20; +param.qlook.inc_dec = 10; +param.qlook.surf.en = 1; +param.qlook.surf.method = 'fixed'; +param.qlook.surf.fixed_value = 0; +param.qlook.resample = [2 1]; + +%% SAR worksheet +param.sar.out_path = ''; +param.sar.frm_types = {0,[0 1],0,0,-1}; +param.sar.chunk_len = 5000; +param.sar.chunk_overlap = 10; +param.sar.frm_overlap = 0; +param.sar.coh_noise_removal = 0; +param.sar.combine_rx = 0; +param.sar.time_of_full_support = inf; +param.sar.pulse_rfi.en = []; +param.sar.pulse_rfi.inc_ave= []; +param.sar.pulse_rfi.thresh_scale = []; +param.sar.trim_vals = []; +param.sar.pulse_comp = 1; +param.sar.ft_dec = 1; +param.sar.ft_wind = @hanning; +param.sar.ft_wind_time = 0; +param.sar.lever_arm_fh = @lever_arm; +param.sar.mocomp.en = 1; +param.sar.mocomp.type = 2; +param.sar.mocomp.filter = {@butter [2] [0.1000]}; +param.sar.mocomp.uniform_en = 1; +param.sar.sar_type = 'fk'; +param.sar.sigma_x = 2.5; +param.sar.sub_aperture_steering = 0; +param.sar.st_wind = @hanning; +param.sar.start_eps = 3.15; + +%% Array worksheet +param.array.in_path = ''; +param.array.array_path = ''; +param.array.out_path = ''; +param.array.method = 'standard'; +param.array.window = @hanning; +param.array.bin_rng = 0; +param.array.rline_rng = -5:5; +param.array.dbin = 1; +param.array.dline = 6; +param.array.DCM = []; +param.array.Nsv = 1; +param.array.theta_rng = [0 0]; +param.array.sv_fh = @array_proc_sv; +param.array.diag_load = 0; +param.array.Nsig = 2; + +%% Radar worksheet +param.radar.adc_bits = 14; +param.radar.Vpp_scale = 1.5; +param.radar.lever_arm_fh = @lever_arm; +for wf = 1:4 + param.radar.wfs(wf).adc_gains_dB = [38 38]; % ADC gain + param.radar.wfs(wf).adcs = [1 2]; % ADC to rx path mapping + param.radar.wfs(wf).rx_paths = [1 2]; % ADC to rx path mapping + param.radar.wfs(wf).gain_en = [0 0]; % Disable fast-time gain correction + param.radar.wfs(wf).coh_noise_method = ''; % No coherent noise removal + param.radar.wfs(wf).Tadc_adjust = 0; + param.radar.wfs(wf).bit_shifts = [6 8]; +end +Tsys = [0 0]/1e9; +chan_equal_dB = [0 0]; +chan_equal_deg = [0 0]; + +%% Post worksheet +param.post.data_dirs = {'qlook'}; +param.post.img = 1; +param.post.layer_dir = 'layerData'; +param.post.maps_en = 1; +param.post.echo_en = 1; +param.post.layers_en = 0; +param.post.data_en = 0; +param.post.csv_en = 1; +param.post.concat_en = 1; +param.post.pdf_en = 1; +param.post.map.location = 'Antarctica'; +param.post.map.type = 'combined'; +param.post.echo.elev_comp = 2; +param.post.echo.depth = '[min(Surface_Depth)-100 max(Surface_Depth)+2900]'; +% param.post.echo.elev_comp = 3; +% param.post.echo.depth = '[min(Surface_Elev)-1500 max(Surface_Elev)+100]'; +param.post.echo.er_ice = 3.15; +param.post.ops.location = 'antarctic'; + +%% Analysis worksheet +param.analysis_noise.block_size = 10000; +cmd_idx = 0; +cmd_idx = cmd_idx + 1; +param.analysis_noise.cmd{cmd_idx}.method = 'coh_noise'; +param.analysis_noise.cmd{cmd_idx}.distance_weight = 1; % Enable distance weighting of the average + +%% Radar Settings + +defaults = {}; + +default = param; +default.arena = arena; + + +% % Survey Mode Shallow Ice +% % Note data_map has unusual ordering with mode 4 first instead of mode 0. This is due to +% % an error in the settings where mode 0 and mode 1 both acquired data that should have only +% % gone to mode 0. +% default.records.data_map = {[4 0 2 1;0 0 1 1;1 0 1 1],[4 0 2 2;0 0 1 2;1 0 1 2;]}; +% default.qlook.img_comb = []; +% default.qlook.imgs = {[1 1],[2 1],[1 2],[2 2]}; +% default.sar.imgs = default.qlook.imgs; +% default.array.imgs = default.qlook.imgs; +% default.array.img_comb = default.qlook.img_comb; +% default.analysis_noise.imgs = default.qlook.imgs; +% default.radar.ref_fn = ''; +% default.radar.wfs = param.radar.wfs(1:2); +% for wf = 1:2 +% default.radar.wfs(wf).Tsys = Tsys; +% default.radar.wfs(wf).chan_equal_dB = chan_equal_dB; +% default.radar.wfs(wf).chan_equal_deg = chan_equal_deg; +% param.radar.wfs(wf).bit_shifts = [7 7]; +% param.radar.wfs(wf).tx_paths = [1 2]; +% end +% default.post.echo.depth = '[min(Surface_Depth)-5 max(Surface_Depth)+300]'; +% % Note psc config name was incorrectly set, but it is for shallow ice: +% default.config_regexp = 'psc_survey_600-900MHz_0usDelay_2us_LOOPBACK'; +% default.name = 'Survey Mode 600-900 MHz Shallow Ice'; +% defaults{end+1} = default; + +% % Survey Mode Thick Ice Single Polarization +% % Note data_map has unusual ordering with mode 4 first instead of mode 0. This is due to +% % an error in the settings where mode 0 and mode 1 both acquired data that should have only +% % gone to mode 0. +% default.records.arena.total_presums = 300; +% default.records.data_map = {[0 0 1 1;1 0 1 1;2 0 2 1;3 0 2 1]}; +% default.qlook.img_comb = []; +% default.qlook.imgs = {[1 1],[2 1]}; +% default.sar.imgs = default.qlook.imgs; +% default.array.imgs = default.qlook.imgs; +% default.array.img_comb = default.qlook.img_comb; +% default.analysis_noise.imgs = default.qlook.imgs; +% default.radar.ref_fn = ''; +% default.radar.wfs = param.radar.wfs(1:2); +% for wf = 1:2 +% default.radar.wfs(wf).Tsys = Tsys; +% default.radar.wfs(wf).chan_equal_dB = chan_equal_dB; +% default.radar.wfs(wf).chan_equal_deg = chan_equal_deg; +% default.radar.wfs(wf).bit_shifts = [6 8]; +% default.radar.wfs(wf).tx_paths = [1 inf]; +% end +% default.post.echo.depth = '[min(Surface_Depth)-5 max(Surface_Depth)+4300]'; +% % Note psc config name was incorrectly set, but it is for shallow ice: +% default.config_regexp = 'psc_eager_config'; +% default.name = 'Survey Mode 600-900 MHz Thick Ice'; +% defaults{end+1} = default; + +% Survey Mode Thick Ice Polarimetric +% Note data_map has unusual ordering with mode 4 first instead of mode 0. This is due to +% an error in the settings where mode 0 and mode 1 both acquired data that should have only +% gone to mode 0. +default.records.arena.total_presums = 600; +default.records.data_map = {[0 0 1 1;1 0 1 1;2 0 2 1;3 0 2 1;4 0 3 1;5 0 3 1;6 0 4 1;7 0 4 1], ... + [0 0 1 2;1 0 1 2;2 0 2 2;3 0 2 2;4 0 3 2;5 0 3 2;6 0 4 2;7 0 4 2]}; +default.qlook.img_comb = []; +default.qlook.imgs = {[1 1],[2 1],[3 1],[4 1],[1 2],[2 2],[3 2],[4 2]}; +default.sar.imgs = default.qlook.imgs; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.analysis_noise.imgs = default.qlook.imgs; +default.radar.ref_fn = ''; +default.radar.wfs = param.radar.wfs(1:4); +for wf = 1:4 + default.radar.wfs(wf).Tsys = Tsys; + default.radar.wfs(wf).chan_equal_dB = chan_equal_dB; + default.radar.wfs(wf).chan_equal_deg = chan_equal_deg; + default.radar.wfs(wf).bit_shifts = [6 8]; + default.radar.wfs(wf).tx_paths = {[1],[2]}; +end +default.post.echo.depth = '[min(Surface_Depth)-5 max(Surface_Depth)+4300]'; +% Note psc config name was incorrectly set, but it is for shallow ice: +%default.config_regexp = 'psc_eager_configHV'; +default.config_regexp = '.*'; +default.name = 'Polarimetric Mode 600-900 MHz Thick Ice'; +defaults{end+1} = default; + + diff --git a/cresis-toolbox/missions/default_radar_params_2022_Antarctica_Polar5_snow.m b/cresis-toolbox/missions/default_radar_params_2022_Antarctica_Polar5_snow.m new file mode 100644 index 00000000..4552044a --- /dev/null +++ b/cresis-toolbox/missions/default_radar_params_2022_Antarctica_Polar5_snow.m @@ -0,0 +1,180 @@ +function [param,defaults] = default_radar_params_2022_Antarctica_Polar5_snow +% [param,defaults] = default_radar_params_2022_Antarctica_Polar5_snow +% +% Snow: 2022_Antarctica_Polar5 +% +% Creates base "param" struct +% Creates defaults cell array for each type of radar setting +% +% Author: John Paden + +%% Preprocess parameters +param.season_name = '2022_Antarctica_Polar5'; +param.radar_name = 'snow5'; + +param.config.max_time_gap = 10; +param.config.min_seg_size = 2; + +param.config.daq_type = 'cresis'; +param.config.wg_type = 'cresis'; +param.config.header_load_func = @basic_load; +% param.config.board_map = {'chan1'}; +param.config.tx_map = {'',''}; + +param.config.daq.xml_version = -1; % No XML file available + +param.config.tx_enable = [1]; + +%% CReSIS parameters +param.config.cresis.clk = 125e6; +param.config.cresis.expected_rec_sizes = [30288 60480 120864 181296]; + +%% Command worksheet +param.cmd.records = 1; +param.cmd.qlook = 1; +param.cmd.generic = 1; + +%% Records worksheet +param.records.file.boards = {'chan1','chan2'}; +param.records.file.clk = 125e6; +param.records.file.prefix = param.radar_name; +param.records.file.suffix = '.bin'; +param.records.file.version = 7; +%param.records.frames.geotiff_fn = 'arctic/NaturalEarth_Data/Arctic_NaturalEarth.tif'; +param.records.frames.geotiff_fn = 'antarctica\NaturalEarth_Data\Antarctica_NaturalEarth.tif'; +param.records.frames.mode = 2; +param.records.gps.en = 1; +param.records.gps.time_offset = 1; + +%% Qlook worksheet +param.qlook.img_comb = []; +param.qlook.imgs = {[2 1],[1 1],[1 2],[2 2]}; +% param.qlook.imgs = {[2 1],[1 1]}; +param.qlook.out_path = ''; +param.qlook.block_size = 2000; +param.qlook.motion_comp = 0; +param.qlook.dec = 4; +param.qlook.inc_dec = 5; +param.qlook.surf.en = 1; +param.qlook.surf.profile = 'snow_AWI'; + +%% SAR worksheet +param.sar.out_path = ''; +param.sar.imgs = param.qlook.imgs; +param.sar.frm_types = {0,[0 1],0,0,-1}; +param.sar.chunk_len = 500; +param.sar.frm_overlap = 0; +param.sar.coh_noise_removal = 0; +param.sar.combine_rx = 0; +param.sar.time_of_full_support = inf; +param.sar.pulse_rfi.en = []; +param.sar.pulse_rfi.inc_ave= []; +param.sar.pulse_rfi.thresh_scale = []; +param.sar.trim_vals = []; +param.sar.pulse_comp = 1; +param.sar.ft_dec = 1; +param.sar.ft_wind = @hanning; +param.sar.ft_wind_time = 0; +param.sar.lever_arm_fh = @lever_arm; +param.sar.mocomp.en = 1; +param.sar.mocomp.type = 2; +param.sar.mocomp.filter = {@butter [2] [0.1000]}; +param.sar.mocomp.uniform_en = 1; +param.sar.sar_type = 'fk'; +param.sar.sigma_x = 1; +param.sar.sub_aperture_steering = 0; +param.sar.st_wind = @hanning; +param.sar.start_eps = 1.53; + +%% Array worksheet +param.array.in_path = ''; +param.array.array_path = ''; +param.array.out_path = ''; +param.array.imgs = param.qlook.imgs; +param.array.img_comb = param.qlook.img_comb; +param.array.method = 'standard'; +param.array.window = @hanning; +param.array.bin_rng = 0; +param.array.line_rng = -2:2; +param.array.dbin = 1; +param.array.dline = 5; +param.array.DCM = []; +param.array.Nsv = 1; +param.array.theta_rng = [0 0]; +param.array.sv_fh = @array_proc_sv; +param.array.diag_load = 0; +param.array.Nsig = 2; + +%% Radar worksheet +param.radar.prf = 1/256e-6; +param.radar.fs = 125e6; +param.radar.adc_bits = 14; +param.radar.Vpp_scale = 2; % Digital receiver gain is 5, full scale Vpp is 2 +param.radar.Tadc_adjust = []; % System time delay: leave this empty or set it to zero at first, determine this value later using data over surface with known height or from surface multiple +param.radar.lever_arm_fh = @lever_arm; +chan_equal_Tsys = [0]/1e9; +chan_equal_dB = [0]; +chan_equal_deg = [0]; +param.radar.wfs(1).tx_weights = [0.1 0]; % Watts +param.radar.wfs(2).tx_weights = [0 0.1]; % Watts +for wf = 1:2 + param.radar.wfs(wf).fmult = 16; + param.radar.wfs(wf).prepulse_H.type = 'NI_DDC_2019'; + %param.radar.wfs(wf).coh_noise_method = 'analysis'; % Post-processing + param.radar.wfs(wf).coh_noise_method = 'estimated'; % Field processing + %param.radar.wfs(wf).coh_noise_method = ''; % Lab data + param.radar.wfs(wf).fLO = -20e9; + param.radar.wfs(wf).adc_gains_dB = [95.8 95.8]; % Radiometric calibration to 1/R^2 + param.radar.wfs(wf).rx_paths = [1 2]; % ADC to rx path mapping + param.radar.wfs(wf).ref_fn = ''; + param.radar.wfs(wf).chan_equal_Tsys = chan_equal_Tsys; + param.radar.wfs(wf).chan_equal_dB = chan_equal_dB; + param.radar.wfs(wf).chan_equal_deg = chan_equal_deg; + param.radar.wfs(wf).nz_trim = {[100 100],[0 0],[0 0],[0 0]}; +end + +%% Post worksheet +param.post.data_dirs = {'qlook'}; +param.post.layer_dir = 'layerData'; +param.post.maps_en = 0; +param.post.echo_en = 1; +param.post.layers_en = 0; +param.post.data_en = 0; +param.post.csv_en = 0; +param.post.concat_en = 0; +param.post.pdf_en = 0; +param.post.map.location = 'Antarctic'; +param.post.map.type = 'contour'; +param.post.echo.elev_comp = 3; +param.post.echo.depth = '[min(Surface_Elev)-2.5 max(Surface_Elev)+4]'; +% param.post.echo.elev_comp = 3; +% param.post.echo.depth = '[min(Surface_Elev)-25 max(Surface_Elev)+2]'; +param.post.echo.er_ice = round((1+0.51*0.3)^3 * 100)/100; +param.post.ops.location = 'arctic'; + +%% Add default settings + +% Initialize the list of default settings +defaults = {}; + +% Create a default setting +default = param; + +%% Radar Settings + +% Survey Mode 2-18 GHz +for wf = 1:2 + default.radar.wfs(wf).f1 = 2.375e9; + default.radar.wfs(wf).f0 = 1.375e9; + default.radar.wfs(wf).Tpd = 240e-6; + %default.radar.wfs(wf).BW_window = [2.5e9 17.493e9]; + default.radar.wfs(wf).BW_window = [2798933333.33 16947200000]; + default.radar.wfs(wf).t_ref = -0.000000040063; +end + +default.config_regexp = '.*'; +default.name = 'Survey Mode 2-18 GHz'; + +% Add the default setting to the list of default settings +defaults{end+1} = default; + diff --git a/cresis-toolbox/missions/default_radar_params_2022_Greenland_Ground_accum.m b/cresis-toolbox/missions/default_radar_params_2022_Greenland_Ground_accum.m new file mode 100644 index 00000000..2105fdd9 --- /dev/null +++ b/cresis-toolbox/missions/default_radar_params_2022_Greenland_Ground_accum.m @@ -0,0 +1,396 @@ +function [param,defaults] = default_radar_params_2022_Greenland_Ground_accum +% [param,defaults] = default_radar_params_2022_Greenland_Ground_accum +% +% Accum: 2022_Greenland_Ground +% +% * Creates "param" struct for preprocess.m +% * Creates "defaults" cell array for each type of radar setting. Each cell +% array element contains the default "param" struct for a particular radar +% setting. +% +% Set the param.season_name to the correct season before running. +% +% Author: John Paden + +%% Preprocess parameters +param.season_name = '2022_Greenland_Ground'; +param.radar_name = 'accum3'; + +% Reading in files +param.config.daq_type = 'arena'; +param.config.wg_type = 'arena'; +param.config.header_load_func = @basic_load_arena; +param.config.board_map = {'digrx0','digrx1'}; +param.config.tx_map = {'awg0'}; + +% Creating segments +param.config.max_time_gap = 10; +param.config.min_seg_size = 1; + +% Creating settings files +param.config.max_data_rate = 60; +param.config.max_duty_cycle = 0.1; +param.config.prf_multiple = [10e6 10e6/20]; % Power supply sync signal that PRF must be a factor of these numbers +param.config.PRI_guard = 1e-6; +param.config.PRI_guard_percentage = 450e6/500e6; +param.config.tx_enable = [1]; +param.config.max_tx = 1.0; +param.config.max_tx_voltage = sqrt(400*50)*10^(-2/20); % voltage at max_tx + +%% BAS ACCUM Arena Parameters +arena = []; +arena.clk = 10e6; +fs = 1000e6; +fs_dac = 2000e6; +subsystem_idx = 0; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'ARENA-CTU'; +arena.subsystem(subsystem_idx).subSystem{1} = 'ctu'; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'ARENA0'; +arena.subsystem(subsystem_idx).subSystem{1} = 'awg0'; +arena.subsystem(subsystem_idx).subSystem{2} = 'digrx0'; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'ARENA1'; +arena.subsystem(subsystem_idx).subSystem{1} = 'digrx1'; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'Data Server'; + +dac_idx = 0; +dac_idx = dac_idx + 1; +arena.dac(dac_idx).name = 'awg0'; +arena.dac(dac_idx).type = 'dac-ad9129_0012'; +arena.dac(dac_idx).dacClk = fs_dac; +arena.dac(dac_idx).desiredAlignMin = -4; +arena.dac(dac_idx).desiredAlignMax = 10; +arena.dac(dac_idx).dcoPhase = 80; + +adc_idx = 0; +adc_idx = adc_idx + 1; +arena.adc(adc_idx).name = 'digrx0'; +arena.adc(adc_idx).type = 'adc-ad9680_0017'; +arena.adc(adc_idx).sampFreq = fs; +arena.adc(adc_idx).adcMode = 1; +arena.adc(adc_idx).desiredAlignMin = -15; +arena.adc(adc_idx).desiredAlignMax = 0; +arena.adc(adc_idx).stream = 'socket'; +arena.adc(adc_idx).ip = '10.0.0.100'; +arena.adc(adc_idx).outputSelect = 1; +arena.adc(adc_idx).wf_set = 1; +arena.adc(adc_idx).gain_dB = [0 0]; +adc_idx = adc_idx + 1; +arena.adc(adc_idx).name = 'digrx1'; +arena.adc(adc_idx).type = 'adc-ad9680_0017'; +arena.adc(adc_idx).sampFreq = fs; +arena.adc(adc_idx).adcMode = 1; +arena.adc(adc_idx).desiredAlignMin = -14; +arena.adc(adc_idx).desiredAlignMax = -0; +arena.adc(adc_idx).stream = 'socket'; +arena.adc(adc_idx).ip = '10.0.0.100'; +arena.adc(adc_idx).outputSelect = 1; +arena.adc(adc_idx).wf_set = 2; +arena.adc(adc_idx).gain_dB = [0 0]; + +daq_idx = 0; +daq_idx = daq_idx + 1; +arena.daq(daq_idx).name = 'daq0'; +arena.daq(daq_idx).type = 'daq_0001'; +arena.daq(daq_idx).auxDir = '/data/'; +arena.daq(daq_idx).fileStripe = '/data/%b/'; +arena.daq(daq_idx).fileName = 'accum3'; +arena.daq.udp_packet_headers = false; + +arena.system.name = 'ku0001'; + +arena.param.tx_max = [1 1]; +arena.param.PA_setup_time = 2e-6; % Time required to enable PA before transmit +arena.param.TTL_time_delay = 0.0; % TTL time delay relative to transmit start +arena.param.ADC_time_delay = 3.0720e-6; % ADC time delay relative to transmit start + +arena.psc.type = 'psc_0003'; + +arena.daq.type = 'daq_0001'; + +arena.ctu.name = 'ctu'; +arena.ctu.type = 'ctu_001D'; +if 0 + % External GPS + arena.ctu.nmea = 31; + arena.ctu.nmea_baud = 9600; + arena.ctu.pps = 10; + arena.ctu.pps_polarity = 1; +else + % Internal GPS + arena.ctu.nmea = 60; + arena.ctu.nmea_baud = 115200; + arena.ctu.pps = 63; + arena.ctu.pps_polarity = 1; +end +idx = 0; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'EPRI'; +arena.ctu.out.bit_group(idx).bits = 0; +arena.ctu.out.bit_group(idx).epri = [1 0]; +arena.ctu.out.bit_group(idx).pri = [0 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'PRI'; +arena.ctu.out.bit_group(idx).bits = 1; +arena.ctu.out.bit_group(idx).epri = [1 0]; +arena.ctu.out.bit_group(idx).pri = [1 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'TR'; +arena.ctu.out.bit_group(idx).bits = 2; +arena.ctu.out.bit_group(idx).epri = [1 0]; +arena.ctu.out.bit_group(idx).pri = [1 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'CalSwitch'; +arena.ctu.out.bit_group(idx).bits = 3; +arena.ctu.out.bit_group(idx).epri = [1 0]; +arena.ctu.out.bit_group(idx).pri = [1 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'AttenFirst18dB'; % 1 is low gain/disables attenuator +arena.ctu.out.bit_group(idx).bits = 4; +arena.ctu.out.bit_group(idx).epri = [0 0]; +arena.ctu.out.bit_group(idx).pri = [0 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'AttenSecond7dB'; % 1 is high gain/disables attenuator +arena.ctu.out.bit_group(idx).bits = 5; +arena.ctu.out.bit_group(idx).epri = [1 1]; +arena.ctu.out.bit_group(idx).pri = [1 1]; + +arena.ctu.out.time_cmd = {'2e-6+param.wfs(wf).Tpd+0.1e-6' '2/param.prf'}; + +%% Command worksheet +param.cmd.records = 1; +param.cmd.qlook = 1; +param.cmd.generic = 1; + +%% Records worksheet +param.records.gps.time_offset = 0; +param.records.frames.geotiff_fn = 'greenland\Landsat-7\mzl7geo_90m_lzw.tif'; +param.records.frames.mode = 1; +param.records.file.version = 103; +param.records.file.prefix = param.radar_name; +param.records.file.suffix = '.bin'; +param.records.file.boards = {'digrx0','digrx1'}; +param.records.file.board_folder_name = '%b'; +param.records.file.clk = 10e6; + +%% Qlook worksheet +param.qlook.out_path = ''; +param.qlook.block_size = 5000; +param.qlook.motion_comp = 0; +param.qlook.dec = 20; +param.qlook.inc_dec = 10; +param.qlook.surf.en = 1; +param.qlook.surf.method = 'fixed'; +param.qlook.surf.fixed_value = 0; +param.qlook.resample = [2 1]; + +%% SAR worksheet +param.sar.out_path = ''; +param.sar.imgs = {[1*ones(4,1),(1:4).'],[2*ones(4,1),(1:4).']}; +param.sar.frm_types = {0,[0 1],0,0,-1}; +param.sar.chunk_len = 5000; +param.sar.chunk_overlap = 10; +param.sar.frm_overlap = 0; +param.sar.coh_noise_removal = 0; +param.sar.combine_rx = 0; +param.sar.time_of_full_support = inf; +param.sar.pulse_rfi.en = []; +param.sar.pulse_rfi.inc_ave= []; +param.sar.pulse_rfi.thresh_scale = []; +param.sar.trim_vals = []; +param.sar.pulse_comp = 1; +param.sar.ft_dec = 1; +param.sar.ft_wind = @hanning; +param.sar.ft_wind_time = 0; +param.sar.lever_arm_fh = @lever_arm; +param.sar.mocomp.en = 1; +param.sar.mocomp.type = 2; +param.sar.mocomp.filter = {@butter [2] [0.1000]}; +param.sar.mocomp.uniform_en = 1; +param.sar.sar_type = 'fk'; +param.sar.sigma_x = 2.5; +param.sar.sub_aperture_steering = 0; +param.sar.st_wind = @hanning; +param.sar.start_eps = 3.15; + +%% Array worksheet +param.array.in_path = ''; +param.array.array_path = ''; +param.array.out_path = ''; +param.array.method = 'standard'; +param.array.window = @hanning; +param.array.bin_rng = 0; +param.array.rline_rng = -5:5; +param.array.dbin = 1; +param.array.dline = 6; +param.array.DCM = []; +param.array.Nsv = 1; +param.array.theta_rng = [0 0]; +param.array.sv_fh = @array_proc_sv; +param.array.diag_load = 0; +param.array.Nsig = 2; + +%% Radar worksheet +param.radar.adc_bits = 14; +param.radar.Vpp_scale = 1.5; % Digital receiver gain is 5, full scale Vpp is 2 +param.radar.Tadc_adjust = -189e-9; % System time delay: leave this empty or set it to zero at first, determine this value later using data over surface with known height or from surface multiple +param.radar.lever_arm_fh = @lever_arm; +% param.radar.wfs(1).adc_gains_dB = 27; % Gain from the first LNA to the ADC +% param.radar.wfs(2).adc_gains_dB = 45; % Gain from the first LNA to the ADC +param.radar.wfs(1).adc_gains_dB = 32.7; % After radiometric calibration +param.radar.wfs(2).adc_gains_dB = 50.7; % After radiometric calibration +param.radar.wfs(3).adc_gains_dB = 32.7; % After radiometric calibration +param.radar.wfs(4).adc_gains_dB = 50.7; % After radiometric calibration +for wf = 1:4 + param.radar.wfs(wf).rx_paths = [1]; % ADC to rx path mapping + param.radar.wfs(wf).gain_en = 1; % Enable fast-time gain correction + param.radar.wfs(wf).coh_noise_method = 'analysis'; % Coherent noise removal + param.radar.wfs(wf).Tadc_adjust = -0.0000003065; +end +Tsys = [0]/1e9; +chan_equal_dB = [0]; +chan_equal_deg = [0]; + +%% Post worksheet +param.post.data_dirs = {'qlook'}; +param.post.img = 1; +param.post.layer_dir = 'layerData'; +param.post.maps_en = 1; +param.post.echo_en = 1; +param.post.layers_en = 0; +param.post.data_en = 0; +param.post.csv_en = 1; +param.post.concat_en = 1; +param.post.pdf_en = 1; +param.post.map.location = 'Greenland'; +param.post.map.type = 'combined'; +param.post.echo.elev_comp = 2; +param.post.echo.depth = '[min(Surface_Depth)-100 max(Surface_Depth)+2900]'; +% param.post.echo.elev_comp = 3; +% param.post.echo.depth = '[min(Surface_Elev)-1500 max(Surface_Elev)+100]'; +param.post.echo.er_ice = 3.15; +param.post.ops.location = 'arctic'; + +%% Analysis worksheet +param.analysis_noise.block_size = 10000; +param.analysis_noise.imgs = {[1 1],[2 1],[3 1],[4 1]}; +cmd_idx = 0; +cmd_idx = cmd_idx + 1; +param.analysis_noise.cmd{cmd_idx}.method = 'coh_noise'; +param.analysis_noise.cmd{cmd_idx}.distance_weight = 1; % Enable distance weighting of the average + +%% Radar Settings + +defaults = {}; + +default = param; +default.arena = arena; + +% Deconvolution Mode +default.records.data_map = {[2 0 1 1; 4 0 3 1],[2 0 2 1; 4 0 4 1]}; +default.qlook.img_comb = []; +default.qlook.imgs = {[1 1],[2 1],[3 1],[4 1]}; +default.sar.imgs = default.qlook.imgs; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.radar.ref_fn = ''; +for wf = 1:4 + default.radar.wfs(wf).Tsys = Tsys; + default.radar.wfs(wf).chan_equal_dB = chan_equal_dB; + default.radar.wfs(wf).chan_equal_deg = chan_equal_deg; + default.radar.wfs(wf).adcs = [1]; + default.radar.wfs(wf).bit_shifts = [8]; + default.radar.wfs(wf).tx_paths = {[1]}; +end + +default.config_regexp = '.*deconv.*'; +default.name = 'Deconv Mode 600-900 MHz'; +defaults{end+1} = default; + +% Noise Mode +default.records.data_map = {[2 0 1 1; 4 0 3 1],[2 0 2 1; 4 0 4 1]}; +default.qlook.img_comb = []; +default.qlook.imgs = {[1 1],[2 1],[3 1],[4 1]}; +default.sar.imgs = default.qlook.imgs; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.radar.ref_fn = ''; +for wf = 1:4 + default.radar.wfs(wf).Tsys = Tsys; + default.radar.wfs(wf).chan_equal_dB = chan_equal_dB; + default.radar.wfs(wf).chan_equal_deg = chan_equal_deg; + default.radar.wfs(wf).adcs = [1]; + default.radar.wfs(wf).bit_shifts = [8]; + default.radar.wfs(wf).tx_paths = {[1]}; +end + +default.config_regexp = '.*noise.*'; +default.name = 'Noise Mode 600-900 MHz'; +defaults{end+1} = default; + +% Loopback Mode +default.records.data_map = {[2 0 1 1; 4 0 3 1],[2 0 2 1; 4 0 4 1]}; +default.qlook.img_comb = []; +default.qlook.imgs = {[1 1],[2 1],[3 1],[4 1]}; +default.sar.imgs = default.qlook.imgs; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.radar.ref_fn = ''; +for wf = 1:4 + default.radar.wfs(wf).Tsys = Tsys; + default.radar.wfs(wf).chan_equal_dB = chan_equal_dB; + default.radar.wfs(wf).chan_equal_deg = chan_equal_deg; + default.radar.wfs(wf).adcs = [1]; + default.radar.wfs(wf).bit_shifts = [8]; + default.radar.wfs(wf).tx_paths = {[1]}; +end + +default.config_regexp = '.*loopback.*'; +default.name = 'Loopback Mode 600-900 MHz'; +defaults{end+1} = default; + +% Survey Mode +default.records.data_map = {[2 0 1 1; 4 0 3 1],[2 0 2 1; 4 0 4 1]}; +default.qlook.img_comb = []; +default.qlook.imgs = {[1 1],[2 1],[3 1],[4 1]}; +default.sar.imgs = default.qlook.imgs; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.radar.ref_fn = ''; +for wf = 1:4 + default.radar.wfs(wf).Tsys = Tsys; + default.radar.wfs(wf).chan_equal_dB = chan_equal_dB; + default.radar.wfs(wf).chan_equal_deg = chan_equal_deg; + default.radar.wfs(wf).adcs = [1]; + default.radar.wfs(wf).bit_shifts = [8]; + default.radar.wfs(wf).tx_paths = {[1]}; +end + +default.config_regexp = '.*survey.*'; +default.name = 'Survey Mode 600-900 MHz'; +defaults{end+1} = default; + +% Other settings + +default.records.data_map = {[2 0 1 1; 4 0 3 1],[2 0 2 1; 4 0 4 1]}; +default.qlook.img_comb = []; +default.qlook.imgs = {[1 1],[2 1],[3 1],[4 1]}; +default.sar.imgs = default.qlook.imgs; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.radar.ref_fn = ''; +for wf = 1:4 + default.radar.wfs(wf).Tsys = Tsys; + default.radar.wfs(wf).chan_equal_dB = chan_equal_dB; + default.radar.wfs(wf).chan_equal_deg = chan_equal_deg; + default.radar.wfs(wf).adcs = [1]; + default.radar.wfs(wf).bit_shifts = [8]; + default.radar.wfs(wf).tx_paths = {[1]}; +end + +default.config_regexp = '.*'; +default.name = 'Default 600-900 MHz'; +defaults{end+1} = default; diff --git a/cresis-toolbox/missions/default_radar_params_2022_Greenland_P3_snow.m b/cresis-toolbox/missions/default_radar_params_2022_Greenland_P3_snow.m new file mode 100644 index 00000000..238d338e --- /dev/null +++ b/cresis-toolbox/missions/default_radar_params_2022_Greenland_P3_snow.m @@ -0,0 +1,437 @@ +function [param,defaults] = default_radar_params_2022_Greenland_P3_snow +% [param,defaults] = default_radar_params_2022_Greenland_P3_snow +% +% Snow: 2022_Greenland_P3 +% +% Creates base "param" struct +% Creates defaults cell array for each type of radar setting +% +% Author: John Paden + +%% Preprocess parameters +param.season_name = '2022_Greenland_P3'; +param.radar_name = 'snow'; + +param.config.daq_type = 'arena'; +param.config.wg_type = 'arena'; +param.config.header_load_func = @basic_load_arena; +param.config.board_map = {'digrx0','digrx1','digrx2','digrx3'}; +param.config.tx_map = {'awg0'}; + +param.config.file.version = 103; +param.config.file.prefix = param.radar_name; +param.config.file.suffix = '.bin'; +param.config.max_time_gap = 10; +param.config.min_seg_size = 1; + +param.config.max_data_rate = 60; +param.config.max_duty_cycle = 0.1; +param.config.prf_multiple = [10e6 10e6/20]; % Power supply sync signal that PRF must be a factor of these numbers +param.config.PRI_guard = 1e-6; +param.config.PRI_guard_percentage = 1; +param.config.tx_enable = [1 1 1 1]; +param.config.max_tx = 1.0; +param.config.max_tx_voltage = sqrt(1000*50)*10^(-2/20); % voltage at max_tx + +%% snow Arena Parameters +arena = []; +fs = 1200e6; +fs_dac = 2400e6; +subsystem_idx = 0; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'ARENA-CTU'; +arena.subsystem(subsystem_idx).subSystem{1} = 'ctu'; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'ARENA0'; +arena.subsystem(subsystem_idx).subSystem{1} = 'awg0'; +arena.subsystem(subsystem_idx).subSystem{2} = 'digrx0'; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'ARENA1'; +arena.subsystem(subsystem_idx).subSystem{1} = 'awg1'; +arena.subsystem(subsystem_idx).subSystem{2} = 'digrx1'; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'ARENA2'; +arena.subsystem(subsystem_idx).subSystem{1} = 'awg2'; +arena.subsystem(subsystem_idx).subSystem{2} = 'digrx2'; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'ARENA3'; +arena.subsystem(subsystem_idx).subSystem{1} = 'awg3'; +arena.subsystem(subsystem_idx).subSystem{2} = 'digrx3'; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'Data Server'; + +dac_idx = 0; +dac_idx = dac_idx + 1; +arena.dac(dac_idx).name = 'awg0'; +arena.dac(dac_idx).type = 'dac-ad9129_0012'; +arena.dac(dac_idx).dacClk = fs_dac; +arena.dac(dac_idx).desiredAlignMin = -10; +arena.dac(dac_idx).desiredAlignMax = 4; +arena.dac(dac_idx).dcoPhase = 80; +dac_idx = dac_idx + 1; +arena.dac(dac_idx).name = 'awg1'; +arena.dac(dac_idx).type = 'dac-ad9129_0012'; +arena.dac(dac_idx).dacClk = fs_dac; +arena.dac(dac_idx).desiredAlignMin = -14; +arena.dac(dac_idx).desiredAlignMax = 0; +arena.dac(dac_idx).dcoPhase = 80; +dac_idx = dac_idx + 1; +arena.dac(dac_idx).name = 'awg2'; +arena.dac(dac_idx).type = 'dac-ad9129_0012'; +arena.dac(dac_idx).dacClk = fs_dac; +arena.dac(dac_idx).desiredAlignMin = 4; +arena.dac(dac_idx).desiredAlignMax = 18; +arena.dac(dac_idx).dcoPhase = 80; +dac_idx = dac_idx + 1; +arena.dac(dac_idx).name = 'awg3'; +arena.dac(dac_idx).type = 'dac-ad9129_0012'; +arena.dac(dac_idx).dacClk = fs_dac; +arena.dac(dac_idx).desiredAlignMin = 0; +arena.dac(dac_idx).desiredAlignMax = 14; +arena.dac(dac_idx).dcoPhase = 80; + +adc_idx = 0; +adc_idx = adc_idx + 1; +arena.adc(adc_idx).name = 'digrx0'; +arena.adc(adc_idx).type = 'adc-ad9680_0017'; +arena.adc(adc_idx).sampFreq = fs; +arena.adc(adc_idx).adcMode = 2; +arena.adc(adc_idx).desiredAlignMin = -12; +arena.adc(adc_idx).desiredAlignMax = 8; +arena.adc(adc_idx).stream = 'tcp'; +arena.adc(adc_idx).ip = '10.0.0.100'; +arena.adc(adc_idx).outputSelect = 0; +arena.adc(adc_idx).gain_dB = [2.4 2.4]; +adc_idx = adc_idx + 1; +arena.adc(adc_idx).name = 'digrx1'; +arena.adc(adc_idx).type = 'adc-ad9680_0017'; +arena.adc(adc_idx).sampFreq = fs; +arena.adc(adc_idx).adcMode = 2; +arena.adc(adc_idx).desiredAlignMin = -17; +arena.adc(adc_idx).desiredAlignMax = 3; +arena.adc(adc_idx).stream = 'tcp'; +arena.adc(adc_idx).ip = '10.0.0.100'; +arena.adc(adc_idx).outputSelect = 0; +arena.adc(adc_idx).gain_dB = [2.4 2.4]; +adc_idx = adc_idx + 1; +arena.adc(adc_idx).name = 'digrx2'; +arena.adc(adc_idx).type = 'adc-ad9680_0017'; +arena.adc(adc_idx).sampFreq = fs; +arena.adc(adc_idx).adcMode = 2; +arena.adc(adc_idx).desiredAlignMin = -21; +arena.adc(adc_idx).desiredAlignMax = -1; +arena.adc(adc_idx).stream = 'tcp'; +arena.adc(adc_idx).ip = '10.0.0.100'; +arena.adc(adc_idx).outputSelect = 0; +arena.adc(adc_idx).gain_dB = [2.4 2.4]; +adc_idx = adc_idx + 1; +arena.adc(adc_idx).name = 'digrx3'; +arena.adc(adc_idx).type = 'adc-ad9680_0017'; +arena.adc(adc_idx).sampFreq = fs; +arena.adc(adc_idx).adcMode = 2; +arena.adc(adc_idx).desiredAlignMin = -20; +arena.adc(adc_idx).desiredAlignMax = 0; +arena.adc(adc_idx).stream = 'tcp'; +arena.adc(adc_idx).ip = '10.0.0.100'; +arena.adc(adc_idx).outputSelect = 0; +arena.adc(adc_idx).gain_dB = [2.4 2.4]; + +daq_idx = 0; +daq_idx = daq_idx + 1; +arena.daq(daq_idx).name = 'daq0'; +arena.daq(daq_idx).type = 'daq_0001'; +arena.daq(daq_idx).auxDir = '/data/'; +arena.daq(daq_idx).fileStripe = '/data/%b/'; +arena.daq(daq_idx).fileName = 'mcords'; + +arena.system.name = 'ku0001'; + +arena.param.tx_max = [1 1 1 1]; +arena.param.PA_setup_time = 2e-6; % Time required to enable PA before transmit +arena.param.TTL_time_delay = 0.0; % TTL time delay relative to transmit start +arena.param.ADC_time_delay = 3.0720e-6; % ADC time delay relative to transmit start + +% mode 0, subchannel 0, board_idx 1 is wf-adc 1-1 +% mode 0, subchannel 0, board_idx 2 is wf-adc 2-1 + +arena.psc.type = 'psc_0003'; + +arena.daq.type = 'daq_0001'; + +arena.ctu.name = 'ctu'; +arena.ctu.type = 'ctu_001D'; +if 0 + % External GPS + arena.ctu.nmea = 31; + arena.ctu.nmea_baud = 9600; + arena.ctu.pps = 10; + arena.ctu.pps_polarity = 1; +else + % Internal GPS + arena.ctu.nmea = 60; + arena.ctu.nmea_baud = 115200; + arena.ctu.pps = 63; + arena.ctu.pps_polarity = 1; +end +idx = 0; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'EPRI'; +arena.ctu.out.bit_group(idx).bits = 0; +arena.ctu.out.bit_group(idx).epri = [1 0 0]; +arena.ctu.out.bit_group(idx).pri = [0 0 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'PRI'; +arena.ctu.out.bit_group(idx).bits = 1; +arena.ctu.out.bit_group(idx).epri = [1 0 0]; +arena.ctu.out.bit_group(idx).pri = [1 0 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'TR'; +arena.ctu.out.bit_group(idx).bits = 2; +arena.ctu.out.bit_group(idx).epri = [1 0 0]; +arena.ctu.out.bit_group(idx).pri = [1 0 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'Isolation'; +arena.ctu.out.bit_group(idx).bits = 3; +arena.ctu.out.bit_group(idx).epri = [1 1 0]; +arena.ctu.out.bit_group(idx).pri = [1 1 0]; +% idx = idx + 1; +% arena.ctu.out.bit_group(idx).name = 'Atten'; +% arena.ctu.out.bit_group(idx).bits = 4:8; +% arena.ctu.out.bit_group(idx).epri = [0 0]; +% arena.ctu.out.bit_group(idx).pri = [0 0]; + +arena.ctu.out.time_cmd = {'2e-6+param.wfs(wf).Tpd+0.5e-6' '2e-6+param.wfs(wf).Tpd+2.5e-6' '2/param.prf'}; + +arena.daq.udp_packet_headers = false; % False except for 20220412 + +param.config.arena = arena; + +%% Command worksheet +default.cmd.records = 1; +default.cmd.qlook = 1; +default.cmd.generic = 1; + +%% Records worksheet +default.records.gps.time_offset = 0; +default.records.frames.geotiff_fn = 'greenland\Landsat-7\mzl7geo_90m_lzw.tif'; +default.records.frames.mode = 1; + +%% Quick Look worksheet +default.qlook.imgs = {[1 1],[1 2],[1 3],[1 4],[1 5],[1 6],[2 1],[2 2],[2 3],[2 4],[2 5],[2 6]}; +% default.qlook.imgs = {[1*ones(6,1),(1:6).'; 2*ones(6,1),(1:6).']}; +default.qlook.out_path = ''; +default.qlook.block_size = 2000; +default.qlook.motion_comp = 0; +default.qlook.dec = 2; +default.qlook.inc_dec = 5; +default.qlook.surf.en = 1; +default.qlook.surf.profile = 'SNOW_AWI'; + +%% SAR worksheet +default.sar.imgs = default.qlook.imgs; +default.sar.out_path = ''; +default.sar.frm_types = {0,[0 1],0,0,-1}; +default.sar.chunk_len = 2000; +default.sar.chunk_overlap = 10; +default.sar.frm_overlap = 0; +default.sar.coh_noise_removal = 0; +default.sar.combine_rx = 0; +default.sar.time_of_full_support = inf; +default.sar.pulse_rfi.en = []; +default.sar.pulse_rfi.inc_ave= []; +default.sar.pulse_rfi.thresh_scale = []; +default.sar.trim_vals = []; +default.sar.pulse_comp = 1; +default.sar.ft_dec = 1; +default.sar.ft_wind = @hanning; +default.sar.ft_wind_time = 0; +default.sar.lever_arm_fh = @lever_arm; +default.sar.mocomp.en = 1; +default.sar.mocomp.type = 2; +default.sar.mocomp.filter = {@butter [2] [0.1000]}; +default.sar.mocomp.uniform_en = 1; +default.sar.sar_type = 'fk'; +default.sar.sigma_x = 0.5; +default.sar.sub_aperture_steering = 0; +default.sar.st_wind = @hanning; +default.sar.start_eps = 1.53; + +%% Array worksheet +default.sar.imgs = default.qlook.imgs; +default.array.in_path = ''; +default.array.array_path = ''; +default.array.out_path = ''; +default.array.method = 'standard'; +default.array.window = @hanning; +default.array.bin_rng = 0; +default.array.rline_rng = -5:5; +default.array.dbin = 1; +default.array.dline = 6; +default.array.DCM = []; +default.array.Nsv = 1; +default.array.theta_rng = [0 0]; +default.array.sv_fh = @array_proc_sv; +default.array.diag_load = 0; +default.array.Nsig = 2; + +%% Radar worksheet +default.radar.adc_bits = 14; +default.radar.Vpp_scale = 1; +default.radar.Tadc_adjust = 8.3042e-06; % System time delay: leave this empty or set it to zero at first, determine this value later using data over surface with known height or from surface multiple +default.radar.lever_arm_fh = @lever_arm; +Tsys = [0 0 0 0 0 0 0 0]/1e9; +chan_equal_dB = [0 0 0 0 0 0 0 0]; +chan_equal_deg = [0 0 0 0 0 0 0 0]; + +%% Post worksheet +default.post.data_dirs = {'qlook'}; +default.post.layer_dir = 'layerData'; +default.post.maps_en = 1; +default.post.echo_en = 1; +default.post.layers_en = 0; +default.post.data_en = 0; +default.post.csv_en = 1; +default.post.concat_en = 1; +default.post.pdf_en = 1; +default.post.map.location = 'Greenland'; +default.post.map.type = 'combined'; +default.post.echo.elev_comp = 2; +default.post.echo.depth = '[min(Surface_Depth)-5 max(Surface_Depth)+25]'; +% default.post.echo.elev_comp = 3; +% default.post.echo.depth = '[min(Surface_Elev)-4500 max(Surface_Elev)+10]'; +default.post.echo.er_ice = 3.15; +default.post.ops.location = 'arctic'; + +%% Analysis worksheet +default.analysis.block_size = 5000; +default.analysis.imgs = {[1*ones(6,1),(1:6).'; 2*ones(6,1),(1:6).']}; +cmd_idx = 0; +cmd_idx = cmd_idx + 1; +default.analysis.cmd{cmd_idx}.method = 'statistics'; +default.analysis.cmd{cmd_idx}.out_path = 'analysis_mean'; +default.analysis.cmd{cmd_idx}.block_ave = 1000; +default.analysis.cmd{cmd_idx}.stats = {@(x)nanmean(x.*conj(x),2)}; +cmd_idx = cmd_idx + 1; +default.analysis.cmd{cmd_idx}.method = 'statistics'; +default.analysis.cmd{cmd_idx}.out_path = 'analysis_freq'; +default.analysis.cmd{cmd_idx}.block_ave = 1000; +default.analysis.cmd{cmd_idx}.start_time = 's=es.Tend-es.Tpd-3e-6;'; +default.analysis.cmd{cmd_idx}.stop_time = 's=es.Tend-es.Tpd;'; +default.analysis.cmd{cmd_idx}.stats = {@(x)mean(abs(fft(x)).^2,2) @(x)mean(abs(fft(fir_dec(x,10))).^2,2) @(x)mean(abs(fft(fir_dec(x,100))).^2,2)}; +cmd_idx = cmd_idx + 1; +default.analysis.cmd{cmd_idx}.method = 'statistics'; +default.analysis.cmd{cmd_idx}.out_path = 'analysis_max'; +default.analysis.cmd{cmd_idx}.block_ave = 1; +default.analysis.cmd{cmd_idx}.pulse_comp = 1; +default.analysis.cmd{cmd_idx}.start_time = 's=min(es.Tend-2*es.Tpd,es.Tpd+6e-6);'; +default.analysis.cmd{cmd_idx}.stats = {'analysis_task_stats_max'}; +cmd_idx = cmd_idx + 1; +default.analysis.cmd{cmd_idx}.method = 'statistics'; +default.analysis.cmd{cmd_idx}.en = 0; +default.analysis.cmd{cmd_idx}.out_path = 'analysis_kx'; +default.analysis.cmd{cmd_idx}.block_ave = 5000; +default.analysis.cmd{cmd_idx}.stats = {'analysis_task_stats_kx'}; +default.analysis.cmd{cmd_idx}.kx = 1000; + + +%% Radar Settings + +defaults = {}; + +% Noise Mode +default.records.data_map = {[2 0 1 1;2 1 1 2;5 0 2 1;5 1 2 2;8 0 3 1;8 1 3 2],[2 0 1 3;2 1 1 4;5 0 2 3;5 1 2 4;8 0 3 3;8 1 3 4],[2 0 1 5;2 1 1 6;5 0 2 5;5 1 2 6;8 0 3 5;8 1 3 6],[2 0 1 7;2 1 1 8;5 0 2 7;5 1 2 8;8 0 3 7;8 1 3 8]}; +default.qlook.img_comb = []; +default.qlook.imgs = {[1*ones(8,1),(1:8).'],[2*ones(8,1),(1:8).'],[3*ones(8,1),(1:8).']}; +default.sar.imgs = default.qlook.imgs; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.radar.ref_fn = ''; +for wf = 1:3 + default.radar.wfs(wf).Tsys = Tsys; + default.radar.wfs(wf).chan_equal_dB = chan_equal_dB; + default.radar.wfs(wf).chan_equal_deg = chan_equal_deg; + default.radar.wfs(wf).adcs = [1 2 3 4 5 6 7 8]; + default.radar.wfs(wf).rx_paths = [2 4 1 3 5 6 7 8]; + default.radar.wfs(wf).tx_paths = [1 2 3 4]; + default.radar.wfs(wf).adc_gains_dB = [45 45 45 45 45 45 45 45]; % Gain from the first LNA to the ADC +end + +default.config_regexp = '.*NOISE.*'; +default.name = 'Noise Mode 180-210 MHz'; +defaults{end+1} = default; + +% Survey Mode +default.records.data_map = {[2 0 1 1;2 1 1 2;5 0 2 1;5 1 2 2;8 0 3 1;8 1 3 2],[2 0 1 3;2 1 1 4;5 0 2 3;5 1 2 4;8 0 3 3;8 1 3 4],[2 0 1 5;2 1 1 6;5 0 2 5;5 1 2 6;8 0 3 5;8 1 3 6],[2 0 1 7;2 1 1 8;5 0 2 7;5 1 2 8;8 0 3 7;8 1 3 8]}; +default.qlook.img_comb = [3e-6 -inf 0.5e-6 10e-06 -inf 3e-06]; +default.qlook.imgs = {[1*ones(8,1),(1:8).'],[2*ones(8,1),(1:8).'],[3*ones(8,1),(1:8).']}; +default.sar.imgs = default.qlook.imgs; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +for wf = 1:3 + default.radar.wfs(wf).Tsys = Tsys; + default.radar.wfs(wf).chan_equal_dB = chan_equal_dB; + default.radar.wfs(wf).chan_equal_deg = chan_equal_deg; + default.radar.wfs(wf).adcs = [1 2 3 4 5 6 7 8]; + default.radar.wfs(wf).rx_paths = [7 8 1 2 3 4 5 6]; + default.radar.wfs(wf).tx_paths = [1 2 3 4]; + default.radar.wfs(wf).adc_gains_dB = [46 46 46 46 46 46 46 46]; % Gain from the first LNA to the ADC +end + +default.config_regexp = '.*survey.*'; +default.name = 'Survey Mode 180-210 MHz'; +defaults{end+1} = default; + +%%%%%%%%%%%%%%%%%%%%%%%%%%% +% PingPong Mode + +default.records.data_map = {[0 0 1 1; 0 1 1 2; 1 0 2 1; 1 1 2 2],[0 0 1 3; 0 1 1 4; 1 0 2 3; 1 1 2 4],[0 0 1 5; 0 1 1 6; 1 0 2 5; 1 1 2 6]}; +default.qlook.img_comb = []; +default.qlook.imgs = {[1*ones(8,1),(1:8).'; 2*ones(8,1),(1:8).']}; +default.sar.imgs = default.qlook.imgs; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +wf = 1; +default.radar.wfs(wf).Tsys = Tsys; +default.radar.wfs(wf).chan_equal_dB = chan_equal_dB; +default.radar.wfs(wf).chan_equal_deg = chan_equal_deg; +default.radar.wfs(wf).adcs = [1 2 3 4 5 6]; +default.radar.wfs(wf).rx_paths = [1 2 3 4 5 6]; +default.radar.wfs(wf).tx_paths = [1]; +default.radar.wfs(wf).adc_gains_dB = [46 46 46 46 46 46 46 46]; % Gain from the first LNA to the ADC +wf = 2; +default.radar.wfs(wf).Tsys = Tsys; +default.radar.wfs(wf).chan_equal_dB = chan_equal_dB; +default.radar.wfs(wf).chan_equal_deg = chan_equal_deg; +default.radar.wfs(wf).adcs = [1 2 3 4 5 6]; +default.radar.wfs(wf).rx_paths = [1 2 3 4 5 6]; +default.radar.wfs(wf).tx_paths = [2]; +default.radar.wfs(wf).adc_gains_dB = [46 46 46 46 46 46 46 46]; % Gain from the first LNA to the ADC + +default.config_regexp = '.*2Tx.*'; +default.name = 'Ping Pong 2-18 GHz'; +defaults{end+1} = default; + +% Other settings +default.records.data_map = {[0 0 1 1; 0 1 1 2],[0 0 1 3; 0 1 1 4],[0 0 1 5; 0 1 1 6],[0 0 1 7; 0 1 1 8]}; +default.qlook.img_comb = []; +default.qlook.imgs = {[1*ones(8,1),(1:8).']}; +default.sar.imgs = default.qlook.imgs; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.radar.ref_fn = ''; +wf = 1; +default.radar.wfs(wf).Tsys = Tsys; +default.radar.wfs(wf).chan_equal_dB = chan_equal_dB; +default.radar.wfs(wf).chan_equal_deg = chan_equal_deg; +default.radar.wfs(wf).adcs = [1 2 3 4 5 6]; +default.radar.wfs(wf).rx_paths = [1 2 3 4 5 6]; +default.radar.wfs(wf).tx_paths = [1]; +default.radar.wfs(wf).adc_gains_dB = [46 46 46 46 46 46 46 46]; % Gain from the first LNA to the ADC + +default.config_regexp = '.*'; +default.name = 'Single 2-18 GHz'; +defaults{end+1} = default; + +%% Add default settings + +param.config.defaults = defaults; diff --git a/cresis-toolbox/missions/default_radar_params_2022_Greenland_Polar5_rds.m b/cresis-toolbox/missions/default_radar_params_2022_Greenland_Polar5_rds.m new file mode 100644 index 00000000..6d815d7b --- /dev/null +++ b/cresis-toolbox/missions/default_radar_params_2022_Greenland_Polar5_rds.m @@ -0,0 +1,623 @@ +function [param,defaults] = default_radar_params_2022_Greenland_Polar5_rds +% [param,defaults] = default_radar_params_2022_Greenland_Polar5_rds +% +% MCORDS 5: 2022_Greenland_Polar5 +% +% Creates base "param" struct +% Creates defaults cell array for each type of radar setting +% +% Author: John Paden + +%% Preprocess parameters +param.season_name = '2022_Greenland_Polar5'; +param.radar_name = 'mcords5'; + +% Reading in files +param.config.daq_type = 'cresis'; +param.config.wg_type = 'arena'; +param.config.header_load_func = @basic_load_mcords5; +param.config.header_load_param = struct('presum_mode',true,'clk',200e6); +param.config.tx_map = {'awg0','awg1','awg2','awg3','awg4','awg5','awg6','awg7'}; + +% Creating segments +param.config.max_time_gap = 10; +param.config.segment_end_file_trim = 2; + +% Creating settings files +param.config.max_data_rate = 240; +param.config.max_duty_cycle = 0.1; +param.config.prf_multiple = [10e6]; % Power supply sync signal that PRF must be a factor of these numbers +param.config.PRI_guard = 1e-6; +param.config.PRI_guard_percentage = 1; +param.config.tx_enable = [1 1 1 1 1 1 1 1]; +param.config.max_tx = 0.63; +param.config.max_tx_voltage = sqrt(1000*50)*10^(-2/20); % voltage at max_tx + +% Interpretting settings +param.config.cresis.config_version = 2.0; +param.config.cresis.rx_gain_dB = 48; +param.config.gps_file_mask = 'GPS*'; + +%% Control parameters (not used in the parameter spreadsheet directly) +% default.xml_file_prefix = 'mcords5'; +% default.data_file_prefix = 'mcords5'; + +param.config.adc_SNR_dB = 59; +param.config.noise_figure = 2; +param.config.max_tx = 0.63; + +param.config.noise_50ohm = [-39.8 -41.0 -40.1 -39.6 -38.4 -39.1 -38.3 -39.6 ]; + +param.config.Pt = (4*1000 + 4*500) * sum(chebwin(8,30).^2)/8; +param.config.Gt = 8*4; +param.config.Ae = 2*0.468 * 0.468; + +param.config.system_loss_dB = 10.^(-5.88/10); +param.config.max_DDS_RAM = 4000; +param.config.tx_voltage = sqrt(1000*50)*10^(-2/20); + +param.config.iq_mode = 0; +param.config.tx_DDS_mask = [1 1 1 1 1 1 1 1]; % Used by basic_rx_chan_equalization + +% For airborne test: +% param.config.basic_surf_track_min_time = 2e-6; % Normally 0e-6 for lab test, 2e-6 for flight test +% param.config.basic_surf_track_Tpd_factor = 1.1; % Normally -inf for lab test, 1.1 for flight test +% For ground test: +param.config.basic_surf_track_min_time = 0e-6; % Normally 0e-6 for lab test, 2e-6 for flight test +param.config.basic_surf_track_Tpd_factor = -inf; % Normally -inf for lab test, 1.1 for flight test + +% default.board_folder_name = 'chan%d'; + +if 1 + % Example 1: Normal configuration: + % Connect antenna N to WFG N for all N = 1 to 8 + ref_adc = 1; + param.config.txequal.img = [(1:8).', ref_adc*ones(8,1)]; + param.config.txequal.ref_wf_adc = 4; + param.config.txequal.wf_mapping = [1 2 3 4 5 6 7 8]; + param.config.txequal.Hwindow_desired = chebwin(8,30).'; + param.config.txequal.max_DDS_amp = [4000 4000 4000 4000 4000 4000 4000 4000]; + param.config.txequal.time_delay_desired = [0 0 0 0 0 0 0 0]; + param.config.txequal.phase_desired = [0 0 0 0 0 0 0 0]; + param.config.txequal.time_validation = [0.4 0.4 0.4 0.4 0.4 0.4 0.4 0.4]*1e-9; + param.config.txequal.amp_validation = [3 3 3 3 3 3 3 3]; + param.config.txequal.phase_validation = [35 35 35 35 35 35 35 35]; + param.config.txequal.remove_linear_phase_en = true; +elseif 0 + % Channel 4 ADC is bad: + ref_adc = 5; + param.config.txequal.img = [(1:8).', ref_adc*ones(8,1)]; + param.config.txequal.ref_wf_adc = 4; + param.config.txequal.wf_mapping = [1 2 3 4 5 6 7 8]; + param.config.txequal.Hwindow_desired = chebwin(8,30).'; + param.config.txequal.max_DDS_amp = [4000 4000 4000 4000 4000 4000 4000 4000]; + param.config.txequal.time_delay_desired = [0 0 0 0 0 0 0 0]; + param.config.txequal.phase_desired = [0 0 0 0 0 0 0 0]; + param.config.txequal.time_validation = [0.4 0.4 0.4 0.4 0.4 0.4 0.4 0.4]*1e-9; + param.config.txequal.amp_validation = [3 3 3 3 3 3 3 3]; + param.config.txequal.phase_validation = [35 35 35 35 35 35 35 35]; + param.config.txequal.remove_linear_phase_en = true; +elseif 0 + % DDS 3 is not used, but create settings not changed: + % Connect antenna 1 to a 50 oxhm load + % Connect antenna 2 to WFG 1 + % Connect antenna 3 to WFG 2 + ref_adc = 4; + param.config.txequal.img = [(1:8).', ref_adc*ones(8,1)]; + param.config.txequal.wf_mapping = [1 2 0 4 5 6 7 8]; + param.config.txequal.ref_wf_adc = 4; + param.config.txequal.Hwindow_desired = chebwin(7,30).'; + param.config.txequal.max_DDS_amp = [4000 4000 0 4000 4000 4000 4000 4000]; + param.config.txequal.time_delay_desired = [0 0 0 0 0 0 0 0]; + param.config.txequal.phase_desired = [0 0 0 0 0 0 0 0]; + param.config.txequal.time_validation = [0.4 0.4 0.4 0.4 0.4 0.4 0.4 0.4]*1e-9; + param.config.txequal.amp_validation = [3 3 3 3 3 3 3 3]; + param.config.txequal.phase_validation = [35 35 35 35 35 35 35 35]; + param.config.txequal.remove_linear_phase_en = true; +elseif 0 + % DDS 3 to 8 are not used and ADC's 3 to 24 are not used, create settings + % also only uses first two DDS + ref_adc = 1; + param.config.txequal.img = [(1:2).', ref_adc*ones(2,1)]; + default.wf_mapping = [1 2 0 0 0 0 0 0]; + param.config.txequal.ref_wf_adc = 1; + param.config.txequal.Hwindow_desired = [1 1]; + param.config.txequal.max_DDS_amp = [4000 4000 0 0 0 0 0 0]; + param.config.txequal.time_delay_desired = [0 0 0 0 0 0 0 0]; + param.config.txequal.phase_desired = [0 0 0 0 0 0 0 0]; + param.config.txequal.time_validation = [0.4 0.4 0.4 0.4 0.4 0.4 0.4 0.4]*1e-9; + param.config.txequal.amp_validation = [3 3 3 3 3 3 3 3]; + param.config.txequal.phase_validation = [35 35 35 35 35 35 35 35]; + param.config.txequal.remove_linear_phase_en = false; +end + +%% AWI MCoRDS Arena Parameters +arena = []; +arena.clk = 10e6; +% fs = 1600e6; +fs_dac = 1600e6; +subsystem_idx = 0; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'arena-awg0'; +arena.subsystem(subsystem_idx).subSystem{1} = 'awg0'; +arena.subsystem(subsystem_idx).subSystem{2} = 'awg1'; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'arena-awg1'; +arena.subsystem(subsystem_idx).subSystem{1} = 'awg2'; +arena.subsystem(subsystem_idx).subSystem{2} = 'awg3'; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'arena-awg2'; +arena.subsystem(subsystem_idx).subSystem{1} = 'awg4'; +arena.subsystem(subsystem_idx).subSystem{2} = 'awg5'; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'arena-awg3'; +arena.subsystem(subsystem_idx).subSystem{1} = 'awg6'; +arena.subsystem(subsystem_idx).subSystem{2} = 'awg7'; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'arena-ctu0'; +arena.subsystem(subsystem_idx).subSystem{1} = 'ctu0'; + +dac_idx = 0; +dac_idx = dac_idx + 1; +arena.dac(dac_idx).name = 'awg0'; +arena.dac(dac_idx).type = 'dac-ad9129_0014'; +arena.dac(dac_idx).dacClk = fs_dac; +arena.dac(dac_idx).desiredAlignMin = 3; +arena.dac(dac_idx).desiredAlignMax = 17; +arena.dac(dac_idx).dcoPhase = 80; +dac_idx = dac_idx + 1; +arena.dac(dac_idx).name = 'awg1'; +arena.dac(dac_idx).type = 'dac-ad9129_0014'; +arena.dac(dac_idx).dacClk = fs_dac; +arena.dac(dac_idx).desiredAlignMin = 10; +arena.dac(dac_idx).desiredAlignMax = 24; +arena.dac(dac_idx).dcoPhase = 80; +dac_idx = dac_idx + 1; +arena.dac(dac_idx).name = 'awg2'; +arena.dac(dac_idx).type = 'dac-ad9129_0014'; +arena.dac(dac_idx).dacClk = fs_dac; +arena.dac(dac_idx).desiredAlignMin = -3; +arena.dac(dac_idx).desiredAlignMax = 11; +arena.dac(dac_idx).dcoPhase = 80; +dac_idx = dac_idx + 1; +arena.dac(dac_idx).name = 'awg3'; +arena.dac(dac_idx).type = 'dac-ad9129_0014'; +arena.dac(dac_idx).dacClk = fs_dac; +arena.dac(dac_idx).desiredAlignMin = 11; +arena.dac(dac_idx).desiredAlignMax = 25; +arena.dac(dac_idx).dcoPhase = 80; +dac_idx = dac_idx + 1; +arena.dac(dac_idx).name = 'awg4'; +arena.dac(dac_idx).type = 'dac-ad9129_0014'; +arena.dac(dac_idx).dacClk = fs_dac; +arena.dac(dac_idx).desiredAlignMin = 4; +arena.dac(dac_idx).desiredAlignMax = 18; +arena.dac(dac_idx).dcoPhase = 80; +dac_idx = dac_idx + 1; +arena.dac(dac_idx).name = 'awg5'; +arena.dac(dac_idx).type = 'dac-ad9129_0014'; +arena.dac(dac_idx).dacClk = fs_dac; +arena.dac(dac_idx).desiredAlignMin = 10; +arena.dac(dac_idx).desiredAlignMax = 24; +arena.dac(dac_idx).dcoPhase = 80; +dac_idx = dac_idx + 1; +arena.dac(dac_idx).name = 'awg6'; +arena.dac(dac_idx).type = 'dac-ad9129_0014'; +arena.dac(dac_idx).dacClk = fs_dac; +arena.dac(dac_idx).desiredAlignMin = 3; +arena.dac(dac_idx).desiredAlignMax = 17; +arena.dac(dac_idx).dcoPhase = 80; +dac_idx = dac_idx + 1; +arena.dac(dac_idx).name = 'awg7'; +arena.dac(dac_idx).type = 'dac-ad9129_0014'; +arena.dac(dac_idx).dacClk = fs_dac; +arena.dac(dac_idx).desiredAlignMin = 13; +arena.dac(dac_idx).desiredAlignMax = 27; +arena.dac(dac_idx).dcoPhase = 80; + +arena.zeropimods = [0 180]; + +% arena.awg = []; +% arena.awg(end+1).awg = 0; +% arena.awg(end).dacs = [0 1]; +% arena.awg(end).dacClk = [1600e6 1600e6]; +% arena.awg(end).desiredAlignMin = [3 10]; +% arena.awg(end).desiredAlignMax = [17 24]; +% arena.awg(end+1).awg = 1; +% arena.awg(end).dacs = [2 3]; +% arena.awg(end).dacClk = [1600e6 1600e6]; +% arena.awg(end).desiredAlignMin = [-3 11]; +% arena.awg(end).desiredAlignMax = [11 25]; +% arena.awg(end+1).awg = 2; +% arena.awg(end).dacs = [4 5]; +% arena.awg(end).dacClk = [1600e6 1600e6]; +% arena.awg(end).desiredAlignMin = [4 10]; +% arena.awg(end).desiredAlignMax = [18 24]; +% arena.awg(end+1).awg = 3; +% arena.awg(end).dacs = [6 7]; +% arena.awg(end).dacClk = [1600e6 1600e6]; +% arena.awg(end).desiredAlignMin = [3 13]; +% arena.awg(end).desiredAlignMax = [17 27]; +% arena.dacs = [0 1 2 3 4 5 6 7]; +% arena.dacs_sampFreq = [1600e6 1600e6 1600e6 1600e6 1600e6 1600e6 1600e6 1600e6]; +arena.max_tx = [0.63 0.63 0.63 0.63 0.63 0.63 0.63 0.63]; +arena.TTL_time = [0.1 0.2 2.2]; +arena.dacs_internal_delay = 0.0; +arena.dacs_start_delay = 0.0; + +arena.TTL_names = {}; +for PA = 1:8 + arena.TTL_names{end+1} = sprintf('PA ENA %d',PA); +end +arena.TTL_names{end+1} = 'T/R'; +arena.TTL_names{end+1} = 'ISO'; +arena.TTL_names{end+1} = 'EPRI'; +arena.TTL_names{end+1} = 'PRI'; +arena.TTL_names{end+1} = 'EPRI'; +arena.TTL_names{end+1} = 'PRI'; +arena.TTL_names{end+1} = 'EPRI'; +arena.TTL_names{end+1} = 'PRI'; +arena.TTL_states{1} = [ + 0 1 1 0 % PA ENA 1 + 0 1 1 0 % PA ENA 2 + 0 1 1 0 % PA ENA 3 + 0 1 1 0 % PA ENA 4 + 0 1 1 0 % PA ENA 5 + 0 1 1 0 % PA ENA 6 + 0 1 1 0 % PA ENA 7 + 0 1 1 0 % PA ENA 8 + 0 1 1 0 % T/R + 0 1 1 0 % ISO + 1 0 0 0 % EPRI + 0 1 0 0 % PRI + 1 0 0 0 % EPRI + 0 1 0 0 % PRI + 1 0 0 0 % EPRI + 0 1 0 0 % PRI + ]; +arena.TTL_states{2} = [ + 0 1 1 0 % PA ENA 1 + 0 1 1 0 % PA ENA 2 + 0 1 1 0 % PA ENA 3 + 0 1 1 0 % PA ENA 4 + 0 1 1 0 % PA ENA 5 + 0 1 1 0 % PA ENA 6 + 0 1 1 0 % PA ENA 7 + 0 1 1 0 % PA ENA 8 + 0 1 1 0 % T/R + 0 1 1 0 % ISO + 0 0 0 0 % EPRI + 0 1 0 0 % PRI + 0 0 0 0 % EPRI + 0 1 0 0 % PRI + 0 0 0 0 % EPRI + 0 1 0 0 % PRI + ]; + +arena.system.name = 'ku0001'; + +arena.param.tx_max = [1 1]; +arena.param.PA_setup_time = 2e-6; % Time required to enable PA before transmit +arena.param.TTL_time_delay = 0.0; % TTL time delay relative to transmit start +arena.param.ADC_time_delay = 3.0720e-6; % ADC time delay relative to transmit start + +arena.psc.type = 'psc_0001'; + +arena.ctu.type = 'ctu_0013'; +arena.ctu.name = 'ctu0'; + +if 0 + % External GPS + arena.ctu.nmea = 31; + arena.ctu.nmea_baud = 9600; + arena.ctu.pps = 10; + arena.ctu.pps_polarity = 1; +else + % Internal GPS + arena.ctu.nmea = 60; + arena.ctu.nmea_baud = 115200; + arena.ctu.pps = 63; + arena.ctu.pps_polarity = 1; +end +for idx = 1:8 + arena.ctu.out.bit_group(idx).name = sprintf('PA ENA %d',idx); + arena.ctu.out.bit_group(idx).bits = 0; + arena.ctu.out.bit_group(idx).epri = [0 1 1 0]; + arena.ctu.out.bit_group(idx).pri = [0 1 1 0]; +end +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'T/R'; +arena.ctu.out.bit_group(idx).bits = 1; +arena.ctu.out.bit_group(idx).epri = [0 1 1 0]; +arena.ctu.out.bit_group(idx).pri = [0 1 1 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'ISO'; +arena.ctu.out.bit_group(idx).bits = 2; +arena.ctu.out.bit_group(idx).epri = [0 1 1 0]; +arena.ctu.out.bit_group(idx).pri = [0 1 1 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'EPRI'; +arena.ctu.out.bit_group(idx).bits = 3; +arena.ctu.out.bit_group(idx).epri = [1 0 0 0]; +arena.ctu.out.bit_group(idx).pri = [0 0 0 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'PRI'; +arena.ctu.out.bit_group(idx).bits = 4; +arena.ctu.out.bit_group(idx).epri = [0 1 0 0]; +arena.ctu.out.bit_group(idx).pri = [0 1 0 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'EPRI'; +arena.ctu.out.bit_group(idx).bits = 3; +arena.ctu.out.bit_group(idx).epri = [1 0 0 0]; +arena.ctu.out.bit_group(idx).pri = [0 0 0 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'PRI'; +arena.ctu.out.bit_group(idx).bits = 4; +arena.ctu.out.bit_group(idx).epri = [0 1 0 0]; +arena.ctu.out.bit_group(idx).pri = [0 1 0 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'EPRI'; +arena.ctu.out.bit_group(idx).bits = 3; +arena.ctu.out.bit_group(idx).epri = [1 0 0 0]; +arena.ctu.out.bit_group(idx).pri = [0 0 0 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'PRI'; +arena.ctu.out.bit_group(idx).bits = 4; +arena.ctu.out.bit_group(idx).epri = [0 1 0 0]; +arena.ctu.out.bit_group(idx).pri = [0 1 0 0]; + +arena.ctu.out.time_cmd = {'2e-6+param.wfs(wf).Tpd+0.1e-6' '2e-6+param.wfs(wf).Tpd+0.1e-6' '2e-6+param.wfs(wf).Tpd+0.1e-6' '2/param.prf'}; + +default.arena = arena; + +%% Records worksheet in parameter spreadsheet +param.records.file.version = 407; +param.records.file.boards = {'chan1','chan2','chan3','chan4','chan5','chan6','chan7','chan8'}; +param.records.file.board_folder_name = '%b'; +param.records.file.prefix = param.radar_name; +param.records.file.suffix = '.bin'; +param.records.file.clk = 200e6; +param.records.gps.time_offset = 1; +param.records.presum_mode = 0; +param.records.frames.mode = 1; +param.records.frames.geotiff_fn = fullfile('greenland','Landsat-7','Greenland_natural_150m.tif'); + +%% Qlook worksheet in parameter spreadsheet +param.qlook.out_path = ''; +param.qlook.imgs = {[1*ones(8,1),(1:8).'],[2*ones(8,1),(1:8).'],[3*ones(8,1),(1:8).']}; +param.qlook.en = 1; +param.qlook.block_size = 10000; +param.qlook.dec = 20; +param.qlook.inc_dec = 5; +param.qlook.surf.en = 1; +param.qlook.surf.profile = 'RDS'; + +%% SAR worksheet in parameter spreadsheet +param.sar.out_path = ''; +param.sar.imgs = {[1*ones(8,1),(1:8).'],[2*ones(8,1),(1:8).'],[3*ones(8,1),(1:8).']}; +param.sar.chunk_len = 5000; +param.sar.combine_rx = 0; +param.sar.mocomp.en = 1; +param.sar.mocomp.type = 2; +param.sar.mocomp.filter = {@butter [2] [0.1000]}; +param.sar.mocomp.uniform_en = 1; +param.sar.sar_type = 'fk'; +param.sar.sigma_x = 2.5; +param.sar.sub_aperture_steering = 0; +param.sar.st_wind = @hanning; +param.sar.start_eps = 3.15; + +%% Array worksheet in parameter spreadsheet +param.array.in_path = ''; +param.array.array_path = ''; +param.array.out_path = ''; +param.array.method = 'standard'; +param.array.window = @hanning; +param.array.bin_rng = 0; +param.array.line_rng = -5:5; +param.array.dbin = 1; +param.array.dline = 6; +param.array.DCM = []; +param.array.tomo_en = 0; +param.array.Nsv = 1; +param.array.theta_rng = [0 0]; +param.array.sv_fh = @array_proc_sv; +param.array.diag_load = 0; +param.array.Nsrc = 2; + +%% Radar worksheet in parameter spreadsheet +param.radar.fs = 1600e6; +param.radar.Tadc = []; % normally leave empty to use value in file header +param.radar.adc_bits = 12; +param.radar.Vpp_scale = 2; +param.radar.lever_arm_fh = @lever_arm; + +param.radar.wfs.rx_paths = [1:8]; +param.radar.wfs.Tadc_adjust = 0.000010179163-1.936020e-6; % System time delay: leave this empty or set it to zero at first, determine this value later using data over surface with known height or from surface multiple + +param.radar.wfs(1).Tsys = [0 0 0 0 0 0 0 0]/1e9; +param.radar.wfs(1).chan_equal_dB = [0 0 0 0 0 0 0 0]; +param.radar.wfs(1).chan_equal_deg = [0 0 0 0 0 0 0 0]; + +%% Post worksheet +param.post.data_dirs = {'qlook'}; +param.post.layer_dir = 'layerData'; +param.post.maps_en = 1; +param.post.echo_en = 1; +param.post.layers_en = 0; +param.post.data_en = 0; +param.post.csv_en = 1; +param.post.concat_en = 1; +param.post.pdf_en = 1; +param.post.map.location = 'Greenland'; +param.post.map.type = 'combined'; +% param.post.echo.elev_comp = 2; +% param.post.echo.depth = '[publish_echogram_switch(Bbad,0.25,Surface_Depth,-2800,DBottom,-100),max(Surface_Depth+100)]'; +param.post.echo.elev_comp = 3; +param.post.echo.depth = '[publish_echogram_switch(Bbad,0.25,Surface_Elev,-3500,DBottom,-100),max(Surface_Elev+100)]'; +param.post.echo.er_ice = 3.15; +param.post.ops.location = 'arctic'; + +defaults = {}; + +%% Wideband settings +default.radar.wfs(1).Tsys = [-31.8 -32.1 -31.7 -31.6 -31.2 -30.8 -31.5 -31.1]/1e9; +default.radar.wfs(1).chan_equal_dB = [-4 -4.3 -2.9 -4.6 -1.2 -1.5 -0.9 -2.2]; +default.radar.wfs(1).chan_equal_deg = [113.1 64.8 124.3 133.7 108 138.1 71.6 102.9]; +default.radar.ft_dec = [37 40]; + + % survey mode +default.qlook.img_comb = [3e-06 -inf 1e-06 1e-05 -inf 3e-06]; +default.qlook.imgs = {[1*ones(8,1),(1:8).'],[2*ones(8,1),(1:8).'],[3*ones(8,1),(1:8).']}; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.radar.ref_fn = ''; +default.config_regexp = 'survey_150-520MHz_.*thick.xml'; +default.name = 'Survey Mode 150-520 MHz'; +defaults{end+1} = default; + + % thin ice mode +default.qlook.img_comb = [1e-06 -inf 1e-06 3e-06 -inf 1e-06]; +default.qlook.imgs = {[1*ones(8,1),(1:8).'],[2*ones(8,1),(1:8).'],[3*ones(8,1),(1:8).']}; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.radar.ref_fn = ''; +default.config_regexp = 'thinice_150-520MHz_.*thick.xml'; +default.name = 'Thin Ice Mode 150-520 MHz'; +defaults{end+1} = default; + + % 2 beam imaging mode +default.qlook.img_comb = []; +default.qlook.imgs = {[1*ones(8,1),(1:8).']}; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.radar.ref_fn = ''; +default.config_regexp = 'image_150-520MHz_.*thick.xml'; +default.name = '2 Beam Image Mode 150-520 MHz'; +defaults{end+1} = default; + + % 3 beam imaging mode +default.qlook.img_comb = []; +default.qlook.imgs = {[2*ones(8,1),(1:8).']}; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.radar.ref_fn = ''; +default.config_regexp = 'image3_150-520MHz_.*thick.xml'; +default.name = '3 Beam Image Mode 150-520 MHz'; +defaults{end+1} = default; + + % sea ice mode +default.qlook.img_comb = []; +default.qlook.imgs = {[1*ones(8,1),(1:8).']}; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.radar.ref_fn = ''; +default.config_regexp = 'seaice_150-520MHz_.*.xml'; +default.name = 'Sea Ice 150-520 MHz'; +defaults{end+1} = default; + + % image high thin with narrowband +default.qlook.img_comb = []; +default.qlook.imgs = {[2*ones(8,1),(1:8).']}; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.radar.ref_fn = ''; +default.config_regexp = 'imagehighthin_150-520MHz_.*.xml'; +default.name = 'High Alt Thin Ice Image Mode 150-520 MHz'; +defaults{end+1} = default; + +%% Narrowband settings +default.radar.wfs(1).Tsys = [-31.8 -32.1 -31.7 -31.6 -31.2 -30.8 -31.5 -31.1]/1e9; +default.radar.wfs(1).chan_equal_dB = [0 0.9 0.2 0 3.2 1.7 3.9 2.4]; +default.radar.wfs(1).chan_equal_deg = [-14.6 -69.7 -6.8 0 -13.9 -1.8 -48.8 -15.8]; +default.radar.ft_dec = [3 20]; + +% survey mode +default.qlook.img_comb = [3e-06 -inf 1e-06 1e-05 -inf 3e-06]; +default.qlook.imgs = {[1*ones(8,1),(1:8).'],[2*ones(8,1),(1:8).'],[3*ones(8,1),(1:8).']}; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.config_regexp = 'survey_180-210MHz_.*thick.xml'; +default.name = 'Survey Mode 180-210 MHz'; +defaults{end+1} = default; + +% polarimetric mode +default.qlook.img_comb = [3e-06 -inf 1e-06 1e-05 -inf 3e-06]; +default.qlook.imgs = {[1*ones(4,1),(3:6).'],[3*ones(4,1),(3:6).'],[5*ones(4,1),(3:6).']}; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.config_regexp = 'polarimetric_180-210MHz_.*thick.xml'; +default.name = 'Polarimetric Mode 180-210 MHz'; +defaults{end+1} = default; + +% survey polarimetric mode +default.qlook.img_comb = [3e-06 -inf 1e-06 1e-05 -inf 3e-06]; +default.qlook.imgs = {[1*ones(4,1),(3:6).'],[2*ones(4,1),(3:6).'],[3*ones(4,1),(3:6).']}; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.config_regexp = 'sur_pol_180-210MHz_.*thick.xml'; +default.name = 'Survey Polarimetric Mode 180-210 MHz'; +defaults{end+1} = default; + +% thin ice mode +default.qlook.img_comb = [1e-06 -inf 1e-06 3e-06 -inf 1e-06]; +default.qlook.imgs = {[1*ones(8,1),(1:8).'],[2*ones(8,1),(1:8).'],[3*ones(8,1),(1:8).']}; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.config_regexp = 'thinice_180-210MHz_.*thick.xml'; +default.name = 'Thin Ice Mode 180-210 MHz'; +defaults{end+1} = default; + + % 2 beam imaging mode +default.qlook.img_comb = []; +default.qlook.imgs = {[1*ones(8,1),(1:8).']}; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.config_regexp = '^image_180-210MHz_.*thick.xml'; +default.name = '2 Beam Image Mode 180-210 MHz'; +defaults{end+1} = default; + +% 3 beam imaging mode +default.qlook.img_comb = []; +default.qlook.imgs = {[2*ones(8,1),(1:8).']}; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.config_regexp = 'image3_180-210MHz_.*thick.xml'; +default.name = '3 Beam Image Mode 180-210 MHz'; +defaults{end+1} = default; + +% egrip imaging mode +default.qlook.img_comb = [1e-06 -inf 1e-06 3e-06 -inf 1e-06]; +default.qlook.imgs = {[1*ones(8,1),(1:8).'],[2*ones(8,1),(1:8).'],[3*ones(8,1),(1:8).'; 4*ones(8,1),(1:8).']}; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.config_regexp = 'egrip_image.*.xml'; +default.name = 'EGRIP Image 180-210 MHz'; +defaults{end+1} = default; + +%% Other settings + +default.qlook.img_comb = []; +default.qlook.imgs = []; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; + +default.config_regexp = 'survey_180-210MHz_.*DECONV.xml'; +default.name = 'Deconv 180-210 MHz'; +defaults{end+1} = default; + +default.config_regexp = '.*180-210MHz.*'; +default.name = 'Other Settings 180-210 MHz'; +defaults{end+1} = default; + +default.radar.wfs(1).Tsys = [-31.8 -32.1 -31.7 -31.6 -31.2 -30.8 -31.5 -31.1]/1e9; +default.radar.wfs(1).chan_equal_dB = [-4 -4.3 -2.9 -4.6 -1.2 -1.5 -0.9 -2.2]; +default.radar.wfs(1).chan_equal_deg = [113.1 64.8 124.3 133.7 108 138.1 71.6 102.9]; + +default.config_regexp = 'survey_150-520MHz_.*DECONV.xml'; +default.name = 'Deconv 150-520 MHz'; +defaults{end+1} = default; + +default.config_regexp = '.*150-520MHz.*'; +default.name = 'Other Settings 150-520 MHz'; +defaults{end+1} = default; diff --git a/cresis-toolbox/missions/default_radar_params_2023_Greenland_Ground_accum.m b/cresis-toolbox/missions/default_radar_params_2023_Greenland_Ground_accum.m new file mode 100644 index 00000000..dd40c944 --- /dev/null +++ b/cresis-toolbox/missions/default_radar_params_2023_Greenland_Ground_accum.m @@ -0,0 +1,378 @@ +function [param,defaults] = default_radar_params_2023_Greenland_Ground_accum +% [param,defaults] = default_radar_params_2023_Greenland_Ground_accum +% +% Accum: 2023_Greenland_Ground +% +% Creates base "param" struct +% Creates defaults cell array for each type of radar setting +% +% Set the param.season_name to the correct season before running. +% +% Author: John Paden + +physical_constants; + +%% Preprocess parameters +param.season_name = '2023_Greenland_Ground'; +param.radar_name = 'accum3'; + +% Reading in files +param.config.daq_type = 'arena'; +param.config.wg_type = 'arena'; +param.config.header_load_func = @basic_load_arena; +%param.config.board_map = {'digrx0'}; % H only polarization +param.config.board_map = {'digrx0','digrx1'}; % HV polarization +param.config.tx_map = {'awg0','awg1'}; + +% Creating segments +param.config.max_time_gap = 10; +param.config.min_seg_size = 1; + +% Creating settings files +param.config.max_data_rate = 60; +param.config.max_duty_cycle = 0.1; +param.config.prf_multiple = [10e6 10e6/20]; % Power supply sync signal that PRF must be a factor of these numbers +param.config.PRI_guard = 1e-6; +param.config.PRI_guard_percentage = 450e6/500e6; +param.config.tx_enable = [1 1]; +param.config.max_tx = 1.0; +param.config.max_tx_voltage = sqrt(400*50)*10^(-2/20); % voltage at max_tx + +%% EAGER 1 ACCUM Arena Parameters +arena = []; +arena.clk = 10e6; +fs = 1000e6; +fs_dac = 2000e6; +subsystem_idx = 0; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'CTU51'; +arena.subsystem(subsystem_idx).subSystem{1} = 'ctu'; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'ARENA60'; +arena.subsystem(subsystem_idx).subSystem{1} = 'awg0'; +arena.subsystem(subsystem_idx).subSystem{2} = 'digrx0'; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'ARENA61'; +arena.subsystem(subsystem_idx).subSystem{1} = 'awg1'; +arena.subsystem(subsystem_idx).subSystem{1} = 'digrx1'; +subsystem_idx = subsystem_idx + 1; +arena.subsystem(subsystem_idx).name = 'Data Server'; + +dac_idx = 0; +dac_idx = dac_idx + 1; +arena.dac(dac_idx).name = 'awg0'; +arena.dac(dac_idx).type = 'dac-ad9129_0012'; +arena.dac(dac_idx).dacClk = fs_dac; +arena.dac(dac_idx).desiredAlignMin = -10; +arena.dac(dac_idx).desiredAlignMax = 0; +arena.dac(dac_idx).dcoPhase = 80; +dac_idx = dac_idx + 1; +arena.dac(dac_idx).name = 'awg1'; +arena.dac(dac_idx).type = 'dac-ad9129_0012'; +arena.dac(dac_idx).dacClk = fs_dac; +arena.dac(dac_idx).desiredAlignMin = -4; +arena.dac(dac_idx).desiredAlignMax = 6; +arena.dac(dac_idx).dcoPhase = 80; + +adc_idx = 0; +adc_idx = adc_idx + 1; +arena.adc(adc_idx).name = 'digrx0'; +arena.adc(adc_idx).type = 'adc-ad9680_0017'; +arena.adc(adc_idx).sampFreq = fs; +arena.adc(adc_idx).adcMode = 1; +arena.adc(adc_idx).desiredAlignMin = -8; +arena.adc(adc_idx).desiredAlignMax = 2; +arena.adc(adc_idx).stream = 'socket'; +arena.adc(adc_idx).ip = '172.16.0.118'; +arena.adc(adc_idx).outputSelect = 1; +arena.adc(adc_idx).wf_set = 1; +arena.adc(adc_idx).gain_dB = [0 0]; +adc_idx = adc_idx + 1; +arena.adc(adc_idx).name = 'digrx1'; +arena.adc(adc_idx).type = 'adc-ad9680_0017'; +arena.adc(adc_idx).sampFreq = fs; +arena.adc(adc_idx).adcMode = 1; +arena.adc(adc_idx).desiredAlignMin = -2; +arena.adc(adc_idx).desiredAlignMax = 8; +arena.adc(adc_idx).stream = 'socket'; +arena.adc(adc_idx).ip = '172.16.0.118'; +arena.adc(adc_idx).outputSelect = 1; +arena.adc(adc_idx).wf_set = 2; +arena.adc(adc_idx).gain_dB = [0 0]; + +daq_idx = 0; +daq_idx = daq_idx + 1; +arena.daq(daq_idx).name = 'daq0'; +arena.daq(daq_idx).type = 'daq_0001'; +arena.daq(daq_idx).auxDir = '/data/'; +arena.daq(daq_idx).fileStripe = '/data/%b/'; +arena.daq(daq_idx).fileName = 'accum'; +arena.daq.udp_packet_headers = false; + +arena.system.name = 'ku0001'; + +arena.param.tx_max = [1 1]; +arena.param.PA_setup_time = 2e-6; % Time required to enable PA before transmit +arena.param.TTL_time_delay = 0.0; % TTL time delay relative to transmit start +arena.param.ADC_time_delay = 3.0720e-6; % ADC time delay relative to transmit start + +arena.psc.type = 'psc_0003'; + +arena.daq.type = 'daq_0001'; + +arena.ctu.name = 'ctu'; +arena.ctu.type = 'ctu_001D'; +if 1 + % External GPS + arena.ctu.nmea = 31; + arena.ctu.nmea_baud = 115200; + arena.ctu.pps = 10; + arena.ctu.pps_polarity = 1; +else + % Internal GPS + arena.ctu.nmea = 60; + arena.ctu.nmea_baud = 115200; + arena.ctu.pps = 63; + arena.ctu.pps_polarity = 1; +end +idx = 0; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'EPRI'; +arena.ctu.out.bit_group(idx).bits = 0; +arena.ctu.out.bit_group(idx).epri = [1 0]; +arena.ctu.out.bit_group(idx).pri = [0 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'PRI'; +arena.ctu.out.bit_group(idx).bits = 1; +arena.ctu.out.bit_group(idx).epri = [1 0]; +arena.ctu.out.bit_group(idx).pri = [1 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'TX_EN'; % 1 enables the transmitter, sets the T/R to transmit +arena.ctu.out.bit_group(idx).bits = 2; +arena.ctu.out.bit_group(idx).epri = [1 0]; +arena.ctu.out.bit_group(idx).pri = [1 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'CAL_EN'; % 1 enables rx calibration switch +arena.ctu.out.bit_group(idx).bits = 3; +arena.ctu.out.bit_group(idx).epri = [1 0]; +arena.ctu.out.bit_group(idx).pri = [1 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'Atten1_TRISO_NOT_USED'; % unused +arena.ctu.out.bit_group(idx).bits = 4; +arena.ctu.out.bit_group(idx).epri = [0 0]; +arena.ctu.out.bit_group(idx).pri = [0 0]; +idx = idx + 1; +arena.ctu.out.bit_group(idx).name = 'LOW_GAIN_EN'; % 1 is low gain +arena.ctu.out.bit_group(idx).bits = 5; +arena.ctu.out.bit_group(idx).epri = [1 1]; +arena.ctu.out.bit_group(idx).pri = [1 1]; + +arena.ctu.out.time_cmd = {'2e-6+param.wfs(wf).Tpd+0.1e-6' '2e-6+param.wfs(wf).Tpd+0.1e-6' '2e-6+param.wfs(wf).Tpd+0.1e-6' '2/param.prf'}; + +%% Command worksheet +param.cmd.records = 1; +param.cmd.qlook = 1; +param.cmd.generic = 1; + +%% Records worksheet +param.records.gps.time_offset = 0; +param.records.frames.geotiff_fn = 'greenland\Landsat-7\mzl7geo_90m_lzw.tif'; +param.records.frames.mode = 1; +param.records.file.version = 103; +param.records.file.prefix = param.radar_name; +param.records.file.suffix = '.bin'; +%param.records.file.boards = {'digrx0'}; % H only polarization +param.records.file.boards = {'digrx0','digrx1'}; % HV Polarization +param.records.file.board_folder_name = '%b'; +param.records.file.clk = 10e6; + +%% Qlook worksheet +param.qlook.out_path = ''; +param.qlook.block_size = 5000; +param.qlook.motion_comp = 0; +param.qlook.dec = 20; +param.qlook.inc_dec = 10; +param.qlook.surf.en = 1; +param.qlook.surf.method = 'fixed'; +param.qlook.surf.fixed_value = 0; +param.qlook.resample = [2 1]; + +%% SAR worksheet +param.sar.out_path = ''; +param.sar.frm_types = {0,[0 1],0,0,-1}; +param.sar.chunk_len = 5000; +param.sar.chunk_overlap = 10; +param.sar.frm_overlap = 0; +param.sar.coh_noise_removal = 0; +param.sar.combine_rx = 0; +param.sar.time_of_full_support = inf; +param.sar.pulse_rfi.en = []; +param.sar.pulse_rfi.inc_ave= []; +param.sar.pulse_rfi.thresh_scale = []; +param.sar.trim_vals = []; +param.sar.pulse_comp = 1; +param.sar.ft_dec = 1; +param.sar.ft_wind = @hanning; +param.sar.ft_wind_time = 0; +param.sar.lever_arm_fh = @lever_arm; +param.sar.mocomp.en = 1; +param.sar.mocomp.type = 2; +param.sar.mocomp.filter = {@butter [2] [0.1000]}; +param.sar.mocomp.uniform_en = 1; +param.sar.sar_type = 'fk'; +param.sar.sigma_x = 2.5; +param.sar.sub_aperture_steering = 0; +param.sar.st_wind = @hanning; +param.sar.start_eps = 3.15; + +%% Array worksheet +param.array.in_path = ''; +param.array.array_path = ''; +param.array.out_path = ''; +param.array.method = 'standard'; +param.array.window = @hanning; +param.array.bin_rng = 0; +param.array.rline_rng = -5:5; +param.array.dbin = 1; +param.array.dline = 6; +param.array.DCM = []; +param.array.Nsv = 1; +param.array.theta_rng = [0 0]; +param.array.sv_fh = @array_proc_sv; +param.array.diag_load = 0; +param.array.Nsig = 2; + +%% Radar worksheet +param.radar.adc_bits = 14; +param.radar.Vpp_scale = 1.5; +param.radar.lever_arm_fh = @lever_arm; +for wf = 1:4 + param.radar.wfs(wf).adc_gains_dB = [38 38]; % ADC gain + param.radar.wfs(wf).adcs = [1 2]; % ADC to rx path mapping + param.radar.wfs(wf).rx_paths = [1 2]; % ADC to rx path mapping + param.radar.wfs(wf).gain_en = [0 0]; % Disable fast-time gain correction + param.radar.wfs(wf).coh_noise_method = ''; % No coherent noise removal + param.radar.wfs(wf).Tadc_adjust = -150e-9 - 2*((2+2+1.5+0.5)/0.695+2/3+41/0.85)*12*2.54/100 / c; % Remove system delay plus cabling to antennas + param.radar.wfs(wf).bit_shifts = [6 8]; +end +Tsys = [0 0]/1e9; +chan_equal_dB = [0 0]; +chan_equal_deg = [0 0]; + +%% Post worksheet +param.post.data_dirs = {'qlook'}; +param.post.img = 1; +param.post.layer_dir = 'layerData'; +param.post.maps_en = 1; +param.post.echo_en = 1; +param.post.layers_en = 0; +param.post.data_en = 0; +param.post.csv_en = 1; +param.post.concat_en = 1; +param.post.pdf_en = 1; +param.post.map.location = 'Greenland'; +param.post.map.type = 'combined'; +param.post.echo.elev_comp = 2; +param.post.echo.depth = '[min(Surface_Depth)-100 max(Surface_Depth)+2900]'; +% param.post.echo.elev_comp = 3; +% param.post.echo.depth = '[min(Surface_Elev)-1500 max(Surface_Elev)+100]'; +param.post.echo.er_ice = 3.15; +param.post.ops.location = 'arctic'; + +%% Analysis worksheet +param.analysis_noise.block_size = 10000; +cmd_idx = 0; +cmd_idx = cmd_idx + 1; +param.analysis_noise.cmd{cmd_idx}.method = 'coh_noise'; +param.analysis_noise.cmd{cmd_idx}.distance_weight = 1; % Enable distance weighting of the average + +%% Radar Settings + +defaults = {}; + +default = param; +default.arena = arena; + + +% % Survey Mode Shallow Ice +% % Note data_map has unusual ordering with mode 4 first instead of mode 0. This is due to +% % an error in the settings where mode 0 and mode 1 both acquired data that should have only +% % gone to mode 0. +% default.records.data_map = {[4 0 2 1;0 0 1 1;1 0 1 1],[4 0 2 2;0 0 1 2;1 0 1 2;]}; +% default.qlook.img_comb = []; +% default.qlook.imgs = {[1 1],[2 1],[1 2],[2 2]}; +% default.sar.imgs = default.qlook.imgs; +% default.array.imgs = default.qlook.imgs; +% default.array.img_comb = default.qlook.img_comb; +% default.analysis_noise.imgs = default.qlook.imgs; +% default.radar.ref_fn = ''; +% default.radar.wfs = param.radar.wfs(1:2); +% for wf = 1:2 +% default.radar.wfs(wf).Tsys = Tsys; +% default.radar.wfs(wf).chan_equal_dB = chan_equal_dB; +% default.radar.wfs(wf).chan_equal_deg = chan_equal_deg; +% param.radar.wfs(wf).bit_shifts = [7 7]; +% param.radar.wfs(wf).tx_paths = [1 2]; +% end +% default.post.echo.depth = '[min(Surface_Depth)-5 max(Surface_Depth)+300]'; +% % Note psc config name was incorrectly set, but it is for shallow ice: +% default.config_regexp = 'psc_survey_600-900MHz_0usDelay_2us_LOOPBACK'; +% default.name = 'Survey Mode 600-900 MHz Shallow Ice'; +% defaults{end+1} = default; + +% % Survey Mode Thick Ice Single Polarization +% % Note data_map has unusual ordering with mode 4 first instead of mode 0. This is due to +% % an error in the settings where mode 0 and mode 1 both acquired data that should have only +% % gone to mode 0. +% default.records.arena.total_presums = 300; +% default.records.data_map = {[0 0 1 1;1 0 1 1;2 0 2 1;3 0 2 1]}; +% default.qlook.img_comb = []; +% default.qlook.imgs = {[1 1],[2 1]}; +% default.sar.imgs = default.qlook.imgs; +% default.array.imgs = default.qlook.imgs; +% default.array.img_comb = default.qlook.img_comb; +% default.analysis_noise.imgs = default.qlook.imgs; +% default.radar.ref_fn = ''; +% default.radar.wfs = param.radar.wfs(1:2); +% for wf = 1:2 +% default.radar.wfs(wf).Tsys = Tsys; +% default.radar.wfs(wf).chan_equal_dB = chan_equal_dB; +% default.radar.wfs(wf).chan_equal_deg = chan_equal_deg; +% default.radar.wfs(wf).bit_shifts = [6 8]; +% default.radar.wfs(wf).tx_paths = [1 inf]; +% end +% default.post.echo.depth = '[min(Surface_Depth)-5 max(Surface_Depth)+4300]'; +% % Note psc config name was incorrectly set, but it is for shallow ice: +% default.config_regexp = 'psc_eager_config'; +% default.name = 'Survey Mode 600-900 MHz Thick Ice'; +% defaults{end+1} = default; + +% Survey Mode Thick Ice Polarimetric +% Note data_map has unusual ordering with mode 4 first instead of mode 0. This is due to +% an error in the settings where mode 0 and mode 1 both acquired data that should have only +% gone to mode 0. +default.records.arena.total_presums = 600; +default.records.data_map = {[0 0 1 1;1 0 1 1;2 0 2 1;3 0 2 1;4 0 3 1;5 0 3 1;6 0 4 1;7 0 4 1], ... + [0 0 1 2;1 0 1 2;2 0 2 2;3 0 2 2;4 0 3 2;5 0 3 2;6 0 4 2;7 0 4 2]}; +default.qlook.img_comb = []; +default.qlook.imgs = {[1 1],[2 1],[3 1],[4 1],[1 2],[2 2],[3 2],[4 2]}; +default.sar.imgs = default.qlook.imgs; +default.array.imgs = default.qlook.imgs; +default.array.img_comb = default.qlook.img_comb; +default.analysis_noise.imgs = default.qlook.imgs; +default.radar.ref_fn = ''; +default.radar.wfs = param.radar.wfs(1:4); +for wf = 1:4 + default.radar.wfs(wf).Tsys = Tsys; + default.radar.wfs(wf).chan_equal_dB = chan_equal_dB; + default.radar.wfs(wf).chan_equal_deg = chan_equal_deg; + default.radar.wfs(wf).bit_shifts = [6 8]; + default.radar.wfs(wf).tx_paths = {[1],[2]}; +end +default.post.echo.depth = '[min(Surface_Depth)-5 max(Surface_Depth)+4300]'; +% Note psc config name was incorrectly set, but it is for shallow ice: +default.config_regexp = 'psc_eager_configHV'; +default.name = 'Polarimetric Mode 600-900 MHz Thick Ice'; +defaults{end+1} = default; + + diff --git a/cresis-toolbox/missions/fast_time_gain_meas_2022_Greenland_Ground.m b/cresis-toolbox/missions/fast_time_gain_meas_2022_Greenland_Ground.m new file mode 100644 index 00000000..144809e9 --- /dev/null +++ b/cresis-toolbox/missions/fast_time_gain_meas_2022_Greenland_Ground.m @@ -0,0 +1,208 @@ +% Create fast-time gain compensation files for 2022_Greenland_Ground +% accum/2022_Greenland_Ground/CSARP_analysis/gain_wf_[1-4]_adc_1.mat +% +% Created manually since there were no measurements of fast-time gain. + +param = read_param_xls(ct_filename_param('accum_param_2022_Greenland_Ground.xls'),'20220607_04',{'analysis_noise','analysis'}); +global gRadar; +param = merge_structs(gRadar,param); +fn_dir = ct_filename_out(param,'analysis',[],true); + +for wf = 1:4 + fn = fullfile(fn_dir, sprintf('/gain_wf_%d_adc_1.mat',wf)); + ftg = struct(); + ftg.param_collate_gain.radar.wfs(wf).Tadc_adjust = -3.065000000000000e-07; + ftg.param_collate_gain.radar.wfs(wf).time_correction = 0; + if wf == 1 + t0 = -1.266500000000000e-06; + Nt = 5008; + dt = 1.999999999999938e-09; + ftg.param_collate_gain.radar.wfs(wf).time = t0 + (0:Nt-1)*dt; + + ftg.Gain_raw = zeros(Nt,1); + ftg.Gain_raw(1:1370) = 10.^((42-9.6+22/2+4/2)/20); + ftg.Gain_raw(1460:1590) = 10.^((32-9.6+22/2)/20); + ftg.Gain_raw(1620:end) = 10.^(0/20); + elseif wf == 2 + t0 = -1.266500000000000e-06; + Nt = 5008; + dt = 1.999999999999938e-09; + ftg.param_collate_gain.radar.wfs(wf).time = t0 + (0:Nt-1)*dt; + + ftg.Gain_raw = zeros(Nt,1); + ftg.Gain_raw(1:1370) = 10.^((0+57)/20); + ftg.Gain_raw(1460:1590) = 10.^((0+42)/20); + ftg.Gain_raw(1650:end) = 10.^(0/20); + elseif wf == 3 + t0 = -1.266500000000000e-06; + Nt = 19504; + dt = 1.999999999999938e-09; + ftg.param_collate_gain.radar.wfs(wf).time = t0 + (0:Nt-1)*dt; + + ftg.Gain_raw = zeros(Nt,1); + ftg.Gain_raw(1:3380) = 10.^((32+13)/20); + ftg.Gain_raw(3460:3590) = 10.^((29+13)/20); + ftg.Gain_raw(3630:end) = 10.^(0/20); + elseif wf == 4 + t0 = -1.266500000000000e-06; + Nt = 19504; + dt = 1.999999999999938e-09; + ftg.param_collate_gain.radar.wfs(wf).time = t0 + (0:Nt-1)*dt; + + ftg.Gain_raw = zeros(Nt,1); + ftg.Gain_raw(1:3380) = 10.^((39+7/2)/20); + ftg.Gain_raw(3460:3585) = 10.^((32+4/2)/20); + ftg.Gain_raw(3820:end) = 10.^(0/20); + end + + fprintf('Saving: %s\n', fn); + ftg.file_version = '1'; + ftg.file_type = 'gain'; + ftg.param_collate_gain.sw_version = current_software_version; + save(fn,'-v7.3','-struct','ftg'); +end + + +% wf = +% 1 +% dd = +% struct with fields: +% +% Tadc_adjust: -3.065000000000000e-07 +% time_correction: 2.266500000000071e-06 +% t0: -1.266500000000000e-06 +% Nt: 5008 +% dt: 1.999999999999938e-09 +% 933 if wf == 1 +% Applying fast time gain compensation 2-1 +% wf = +% 2 +% dd = +% struct with fields: +% +% Tadc_adjust: -3.065000000000000e-07 +% time_correction: 2.266500000000071e-06 +% t0: -1.266500000000000e-06 +% Nt: 5008 +% dt: 1.999999999999938e-09 +% Applying fast time gain compensation 3-1 +% wf = +% 3 +% dd = +% struct with fields: +% +% Tadc_adjust: -3.065000000000000e-07 +% time_correction: 6.266500000000168e-06 +% t0: -1.266500000000000e-06 +% Nt: 19504 +% dt: 1.999999999999938e-09 +% Applying fast time gain compensation 4-1 +% wf = +% 4 +% dd = +% struct with fields: +% +% Tadc_adjust: -3.065000000000000e-07 +% time_correction: 6.266500000000168e-06 +% t0: -1.266500000000000e-06 +% Nt: 19504 +% dt: 1.999999999999938e-09 + + + + + + + + + + + + + +% 2022_Greenland_Ground + +% corr_Time = ftg.param_analysis.radar.wfs(wf).time... % Actual Time axis +% + (ftg.param_analysis.radar.wfs(wf).Tadc_adjust - 1*param.radar.wfs(wf).Tadc_adjust)... % Difference in Tadc_adjust +% -1*ftg.param_analysis.radar.wfs(wf).time_correction; +% % +-ftg.param_analysis.radar.wfs(wf).time(1); +% % plot(wfs(wf).time_raw(1:hdr.Nt{img}(1))/1e-6,data{img}(:,1,wf_adc)); +% % temp_data = bsxfun(@times,data{img}(:,:,wf_adc),interp1(corr_Time, 1./ftg.Gain_raw, wfs(wf).time_raw(1:hdr.Nt{img}(1)), 'linear','extrap')); +% % hold on; plot(wfs(wf).time_raw(1:hdr.Nt{img}(1))/1e-6,temp_data(:,1),'--'); +% if wf == 1 +% corr_Time = wfs(wf).time_raw(1:hdr.Nt{img}(1)); +% ftg.Gain_raw = zeros(hdr.Nt{img}(1),1); +% ftg.Gain_raw(1:1370) = 10.^((42-9.6)/20); +% ftg.Gain_raw(1460:1590) = 10.^((32-9.6)/20); +% ftg.Gain_raw(1620:end) = 10.^(0/20); +% elseif wf == 2 +% corr_Time = wfs(wf).time_raw(1:hdr.Nt{img}(1)); +% ftg.Gain_raw = zeros(hdr.Nt{img}(1),1); +% ftg.Gain_raw(1:1370) = 10.^(57/20); +% ftg.Gain_raw(1460:1590) = 10.^(42/20); +% ftg.Gain_raw(1650:end) = 10.^(0/20); +% elseif wf == 3 +% corr_Time = wfs(wf).time_raw(1:hdr.Nt{img}(1)); +% ftg.Gain_raw = zeros(hdr.Nt{img}(1),1); +% ftg.Gain_raw(1:3380) = 10.^(32/20); +% ftg.Gain_raw(3460:3590) = 10.^(29/20); +% ftg.Gain_raw(3630:end) = 10.^(0/20); +% elseif wf == 4 +% corr_Time = wfs(wf).time_raw(1:hdr.Nt{img}(1)); +% ftg.Gain_raw = zeros(hdr.Nt{img}(1),1); +% ftg.Gain_raw(1:3380) = 10.^(39/20); +% ftg.Gain_raw(3460:3585) = 10.^(32/20); +% ftg.Gain_raw(3820:end) = 10.^(0/20); +% end +% if 0 +% figure(1); clf; +% plot(lp(mean(abs(data{img}(:,:,wf_adc)).^2,2))); +% grid on; +% keyboard +% end +% data{img}(:,:,wf_adc) = bsxfun(@times,data{img}(:,:,wf_adc),interp1(corr_Time, ftg.Gain_raw, wfs(wf).time_raw(1:hdr.Nt{img}(1)), 'linear','extrap')); + + + + + + + + % +-ftg.param_analysis.radar.wfs(wf).time(1); + % plot(wfs(wf).time_raw(1:hdr.Nt{img}(1))/1e-6,data{img}(:,1,wf_adc)); + % temp_data = bsxfun(@times,data{img}(:,:,wf_adc),interp1(corr_Time, 1./ftg.Gain_raw, wfs(wf).time_raw(1:hdr.Nt{img}(1)), 'linear','extrap')); + % hold on; plot(wfs(wf).time_raw(1:hdr.Nt{img}(1))/1e-6,temp_data(:,1),'--'); + +% wf +% dd.Tadc_adjust = param.radar.wfs(wf).Tadc_adjust; +% dd.time_correction = ftg.param_analysis.radar.wfs(wf).time_correction; +% dd.t0 = wfs(wf).time_raw(1); +% dd.Nt = hdr.Nt{img}(1); +% dd.dt = wfs(wf).time_raw(2)-wfs(wf).time_raw(1); +% dd + +% if wf == 1 +% corr_Time = wfs(wf).time_raw(1:hdr.Nt{img}(1)); +% ftg.Gain_raw = zeros(hdr.Nt{img}(1),1); +% ftg.Gain_raw(1:1370) = 10.^((42-9.6)/20); +% ftg.Gain_raw(1460:1590) = 10.^((32-9.6)/20); +% ftg.Gain_raw(1620:end) = 10.^(0/20); +% elseif wf == 2 +% corr_Time = wfs(wf).time_raw(1:hdr.Nt{img}(1)); +% ftg.Gain_raw = zeros(hdr.Nt{img}(1),1); +% ftg.Gain_raw(1:1370) = 10.^(57/20); +% ftg.Gain_raw(1460:1590) = 10.^(42/20); +% ftg.Gain_raw(1650:end) = 10.^(0/20); +% elseif wf == 3 +% corr_Time = wfs(wf).time_raw(1:hdr.Nt{img}(1)); +% ftg.Gain_raw = zeros(hdr.Nt{img}(1),1); +% ftg.Gain_raw(1:3380) = 10.^(32/20); +% ftg.Gain_raw(3460:3590) = 10.^(29/20); +% ftg.Gain_raw(3630:end) = 10.^(0/20); +% elseif wf == 4 +% corr_Time = wfs(wf).time_raw(1:hdr.Nt{img}(1)); +% ftg.Gain_raw = zeros(hdr.Nt{img}(1),1); +% ftg.Gain_raw(1:3380) = 10.^(39/20); +% ftg.Gain_raw(3460:3585) = 10.^(32/20); +% ftg.Gain_raw(3820:end) = 10.^(0/20); +% end diff --git a/cresis-toolbox/missions/negis_ice_core_2014_Greenland_P3.m b/cresis-toolbox/missions/negis_ice_core_2014_Greenland_P3.m index 050a7732..09405ed7 100644 --- a/cresis-toolbox/missions/negis_ice_core_2014_Greenland_P3.m +++ b/cresis-toolbox/missions/negis_ice_core_2014_Greenland_P3.m @@ -330,7 +330,7 @@ if length(param.er_depth) > 1 TWtime = genPropProfileFromPerm(param.er_depth,param.er_ice, param.er_freq); profile_idxs = depth_axis > 0 & depth_axis < param.er_depth(end); - depth_time(profile_idxs) = interp1(param.er_depth, [0; TWtime], depth_axis(profile_idxs)); + depth_time(profile_idxs) = interp1(param.er_depth, TWtime, depth_axis(profile_idxs)); end depth_time = depth_time'; depth_time = [0;depth_time]; %ensure vectors are the same length diff --git a/cresis-toolbox/missions/rds_settings.m b/cresis-toolbox/missions/rds_settings.m index 2bb4430b..eb915d81 100644 --- a/cresis-toolbox/missions/rds_settings.m +++ b/cresis-toolbox/missions/rds_settings.m @@ -3,8 +3,41 @@ % Script that should be called from run scripts to do standard processing of RDS data. % Has many seasons and different projects setup. +%% cmd % Example to run a specific segment and frame by overriding parameter spreadsheet values params = ct_set_params(params,['cmd.' cmd_method],0); + + +%% cmd: Accumulation Radar +% ========================================================================= +% ------------------------------------------------------------------------- +% 2018 Antarctica TObas +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190130_01'); +% params = ct_set_params(params,'cmd.frms',[1]); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190131_03'); + +% ------------------------------------------------------------------------- +% 2019 Antarctica TObas +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20191215_02'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20191215_03'); % DO NOT SAR PROCESS +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20191222_01'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20191225_01'); +% params = ct_set_params(params,'cmd.frms',[20:22],'day_seg','20191225_01'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20191226_01'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20191229_01'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20191230_01'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20191230_02'); % DECONV +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20200125_03'); % DECONV +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20200125_05'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20200125_06'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20200126_01'); % DECONV +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20200127_01'); +% params = ct_set_params(params,'cmd.frms',[33:34],'day_seg','20200127_01'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20200128_01'); % DECONV +% params = ct_set_params(params,'cmd.frms',[],'day_seg','20200128_01'); % DECONV + +%% cmd: Multipass +% ========================================================================= % ------------------------------------------------------------------------- % Eqip Line 1 % params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20140414_02'); @@ -57,26 +90,87 @@ % params = ct_set_params(params,'cmd.frms',[1:3],'day_seg','20180418_05'); % 1 2 % ------------------------------------------------------------------------- % CAA +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20140325_01'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20140325_02'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20140325_04'); % params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20140325_05'); % params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20140325_06'); % params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20140325_07'); % params = ct_set_params(params,'cmd.frms',[4 5],'day_seg','20140325_07'); -% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20140401_03'); -% params = ct_set_params(params,'cmd.frms',[1 2],'day_seg','20140401_03'); +params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20140401_03'); +params = ct_set_params(params,'cmd.frms',[1:5],'day_seg','20140401_03'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20140506_01'); +% params = ct_set_params(params,'cmd.frms',[3 4],'day_seg','20140506_01'); % params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20140506_01'); -% params = ct_set_params(params,'cmd.frms',[3:4],'day_seg','20140506_01'); +% params = ct_set_params(params,'cmd.frms',[3 4],'day_seg','20140506_01'); % ------------------------------------------------------------------------- % Multipass Camp Century % params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20140429_01'); % params = ct_set_params(params,'cmd.frms',[67]); +% Multipass EGIG +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20110426_11'); +% params = ct_set_params(params,'cmd.frms',[5 6 7]); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20120411_02'); +% params = ct_set_params(params,'cmd.frms',[7 8 9 10]); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20140410_01'); +% params = ct_set_params(params,'cmd.frms',[57 58 59]); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20170506_01'); +% params = ct_set_params(params,'cmd.frms',[57 58 59]); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20180501_01'); +% params = ct_set_params(params,'cmd.frms',[51 52 53 54 55]); % Multipass Summit % params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20120330_03'); % params = ct_set_params(params,'cmd.frms',[8 9]); % params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20140502_01'); % params = ct_set_params(params,'cmd.frms',[41 42]); +%% cmd: Radar Depth Sounder +% ========================================================================= + % ------------------------------------------------------------------------- +% 2013 Antarctica Basler +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20140109_03'); + +% ------------------------------------------------------------------------- +% 2014 Greenland P3 +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20140325_04'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20140502_01'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20140506_01'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20140512_01'); + +% ------------------------------------------------------------------------- +% 2018 Greenland P3 +% params = ct_set_params(params,'cmd.generic',0,'day_seg','20180315_10'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20180322_03'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20180322_04'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20180404_02'); % 4 wfs +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20180405'); % no digital errors +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20180406'); % 2 wfs, no digital errors +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20180418_04'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20180418_05'); % 4 wfs, no digital errors +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20180418_06'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20180419_01'); % 4 wfs, 12 frames, frames 9 and 11 +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20180419_02'); % Remember to look at end of segment for false alarms and implement valid_gps_times +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20180421'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20180422'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20180423'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20180425'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20180426_01'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20180426_02'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20180426_03'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20180426_04'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20180427_01'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20180427_03'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20180429'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20180430'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20180501'); + % 2018 Antarctica Ground +% ------------------------------------------------------------------------- +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20181224_03'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20181224_02'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20181224_03'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20181224_04'); % params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20181217'); % params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20181219'); % params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20181220'); @@ -98,10 +192,51 @@ % params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190107'); % params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190108'); % params = ct_set_params(params,['cmd.' cmd_method],0,'cmd.notes','do not process'); + +% ------------------------------------------------------------------------- +% 2019_Greenland_P3 +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190403_02'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190403_03'); % Deconvolution/equalization/array-calibration segment +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190405_01'); % Frame 7 last block is a good coh noise check +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190405_02'); % Test dataset +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190405_03'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190405_04'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190406_01'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190406_02'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190406_03'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190409_01'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190409_02'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190409_03'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190410_01'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190410_02'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190415_01'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190416_01'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190417_01'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190417_02'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190418_01'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190420_01'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190420_02'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190423_01'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190423_02'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190423_03'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190505_01'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190505_02'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190506_01'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190506_02'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190507_01'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190508_01'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190512_01'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190512_02'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190513_01'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190514_01'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190515_01'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190516_01'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190516_02'); + % ------------------------------------------------------------------------- % 2019 Antarctica Ground % params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20190925_04'); -params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20200107_01'); +% params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20200107_01'); % params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20191230'); % params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20191231'); % params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20200101'); @@ -111,10 +246,7 @@ % params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20200107'); % params = ct_set_params(params,['cmd.' cmd_method],1,'day_seg','20200108'); - - -% params = ct_set_params(params,'radar.wfs(1).deconv.en',true); -% params = ct_set_params(params,'radar.wfs(2).deconv.en',true); +output_dir = ct_output_dir(params(1).radar_name); for param_idx = 1:length(params) param = params(param_idx); @@ -128,25 +260,122 @@ %% qlook params = ct_set_params(params,'qlook.out_path','qlook'); + params = ct_set_params(params,'qlook.surf_layer',struct('name','surface','source','layerdata','layerdata_source','layer')); + params = ct_set_params(params,'qlook.resample',[2 1; 1 1]); + if strcmpi(params(param_idx).season_name,'2018_Antarctica_TObas') + params(param_idx).qlook.surf.en = false; + elseif strcmpi(params(param_idx).season_name,'2019_Antarctica_TObas') + params(param_idx).qlook.surf.en = false; + elseif strcmpi(params(param_idx).season_name,'2018_Greenland_P3') + params(param_idx).qlook.surf.en = false; + %params(param_idx).qlook.nan_dec = true; + params(param_idx).qlook.nan_dec = false; + params(param_idx).qlook.out_path = 'qlook'; + params(param_idx).qlook.motion_comp = false; + adcs = [13:16]; Nchan = length(adcs); + if length(params(param_idx).radar.wfs) == 6 + params(param_idx).qlook.imgs = {[ones(1,Nchan) 2*ones(1,Nchan); adcs adcs].', [3*ones(1,Nchan) 4*ones(1,Nchan); adcs adcs].', [5*ones(1,Nchan) 6*ones(1,Nchan); adcs adcs].'}; + elseif length(params(param_idx).radar.wfs) == 4 + params(param_idx).qlook.imgs = {[ones(1,Nchan) 2*ones(1,Nchan); adcs adcs].', [3*ones(1,Nchan) 4*ones(1,Nchan); adcs adcs].'}; + elseif length(params(param_idx).radar.wfs) == 2 + params(param_idx).qlook.imgs = {[ones(1,Nchan) 2*ones(1,Nchan); adcs adcs].'}; + end + elseif strcmpi(params(param_idx).season_name,'2018_Antarctica_Ground') + %params(param_idx).qlook.out_path = 'qlook_test'; + params(param_idx).qlook.out_path = 'qlook_test_adcs5678'; + %params(param_idx).qlook.out_path = 'qlook'; + params(param_idx).qlook.motion_comp = false; + if 0 + adcs = [1:4]; Nchan = length(adcs); + params(param_idx).qlook.imgs = {[ones(1,Nchan); adcs].', [2*ones(1,Nchan); adcs].'}; + else + adcs = [5:8]; Nchan = length(adcs); + params(param_idx).qlook.imgs = {[ones(1,Nchan); adcs].', [2*ones(1,Nchan); adcs].'}; + end + elseif strcmpi(params(param_idx).season_name,'2019_Greenland_P3') + params(param_idx).qlook.surf.en = true; + params(param_idx).qlook.nan_dec = false; + params(param_idx).qlook.out_path = 'qlook'; + params(param_idx).qlook.motion_comp = true; + adcs = [1:7]; Nchan = length(adcs); + if length(params(param_idx).radar.wfs) == 6 + params(param_idx).qlook.imgs = {[ones(1,Nchan) 2*ones(1,Nchan); adcs adcs].', [3*ones(1,Nchan) 4*ones(1,Nchan); adcs adcs].', [5*ones(1,Nchan) 6*ones(1,Nchan); adcs adcs].'}; + params(param_idx).qlook.imgs{3} = params(param_idx).qlook.imgs{3}(1:end-1,:); + params(param_idx).qlook.imgs{3} = params(param_idx).qlook.imgs{3}([1 3:end],:); + params(param_idx).qlook.imgs{2} = params(param_idx).qlook.imgs{2}(1:end-1,:); + params(param_idx).qlook.imgs{1} = params(param_idx).qlook.imgs{1}(1:end-1,:); + elseif length(params(param_idx).radar.wfs) == 4 + params(param_idx).qlook.imgs = {[ones(1,Nchan) 2*ones(1,Nchan); adcs adcs].', [3*ones(1,Nchan) 4*ones(1,Nchan); adcs adcs].'}; + params(param_idx).qlook.imgs{2} = params(param_idx).qlook.imgs{2}(1:end-1,:); + params(param_idx).qlook.imgs{1} = params(param_idx).qlook.imgs{1}(1:end-1,:); + elseif length(params(param_idx).radar.wfs) == 2 + if isempty(params(param_idx).qlook.img_comb) + params(param_idx).qlook.imgs = {[ones(1,Nchan) 2*ones(1,Nchan); adcs adcs].'}; + params(param_idx).qlook.imgs{1} = params(param_idx).qlook.imgs{1}(1:end-1,:); + else + params(param_idx).qlook.imgs = {[ones(1,Nchan); adcs].',[2*ones(1,Nchan); adcs].'}; + params(param_idx).qlook.imgs{2} = params(param_idx).qlook.imgs{2}(1:end-1,:); + end + end + elseif strcmpi(params(param_idx).season_name,'2019_Antarctica_Ground') + %params(param_idx).qlook.out_path = 'qlook_test'; + params(param_idx).qlook.out_path = 'qlook'; + records = records_load(params(param_idx),'gps_source'); + if ~isempty(regexpi(records.gps_source,'cresis')) + params(param_idx).qlook.motion_comp = true; + else + params(param_idx).qlook.motion_comp = false; + end + params(param_idx).qlook.motion_comp = false; + adcs = [1:6]; Nchan = length(adcs); + if length(params(param_idx).radar.wfs) == 3 + params(param_idx).qlook.imgs = {[ones(1,Nchan); adcs].', [2*ones(1,Nchan); adcs].', [3*ones(1,Nchan); adcs].'}; + else + params(param_idx).qlook.imgs = {[ones(1,Nchan); adcs].', [2*ones(1,Nchan); adcs].', [3*ones(1,Nchan) 4*ones(1,Nchan); adcs adcs].'}; + end + end + + %% sar if strcmpi(params(param_idx).season_name,'2018_Antarctica_Ground') - params = ct_set_params(params,'qlook.out_path','qlook'); + params = ct_set_params(params,'sar.out_path','sar'); + params = ct_set_params(params,'sar.sigma_x',1); + params(param_idx).sar.mocomp.en = true; elseif strcmpi(params(param_idx).season_name,'2019_Antarctica_Ground') + params = ct_set_params(params,'sar.out_path','sar'); + %params = ct_set_params(params,'sar.out_path','sar_tukey'); + params = ct_set_params(params,'sar.sigma_x',1); + records = records_load(params(param_idx),'gps_source'); + if ~isempty(regexpi(records.gps_source,'cresis')) + params(param_idx).sar.mocomp.en = true; + else + params(param_idx).sar.mocomp.en = false; + end + params(param_idx).sar.mocomp.en = false; end %% general if strcmpi(params(param_idx).season_name,'2018_Antarctica_Ground') elseif strcmpi(params(param_idx).season_name,'2019_Antarctica_Ground') -% params = ct_set_params(params,'records.data_map',{[2 0 1 1;2 1 1 2;5 0 2 1;5 1 2 2;8 0 3 1;8 1 3 2],[2 0 1 3;2 1 1 4;5 0 2 3;5 1 2 4;8 0 3 3;8 1 3 4],[2 0 1 5;2 1 1 6;5 0 2 5;5 1 2 6;8 0 3 5;8 1 3 6],[2 0 1 7;2 1 1 8;5 0 2 7;5 1 2 8;8 0 3 7;8 1 3 8]}); -% params = ct_set_params(params,'records.data_map',{[2 0 1 1; 2 1 1 2; 5 0 2 1; 5 1 2 2; 8 0 3 1; 8 1 3 2; 11 0 4 1; 11 1 4 2],[2 0 ... -% 1 3; 2 1 1 4; 5 0 2 3; 5 1 2 4; 8 0 3 3; 8 1 3 4; 11 0 4 3; 11 1 4 4],[2 0 1 5; 2 1 1 6; 5 0 2 5; 5 1 2 6; 8 0 3 ... -% 5; 8 1 3 6; 11 0 4 5; 11 1 4 6],[2 0 1 7; 2 1 1 8; 5 0 2 7; 5 1 2 8; 8 0 3 7; 8 1 3 8; 11 0 4 7; 11 1 4 8]}, ... -% 'day_seg','20200101_02|20200101_03|20200101_04|20200104_01|20200104_02|20200104_03|20200104_04|20200104_05|20200105_01|20200105_02|20200105_03|20200105_04|20200106_01|20200106_02|20200107_01|20200107_02|20200107_03|20200108_01|20200108_02'); + % params = ct_set_params(params,'records.data_map',{[2 0 1 1;2 1 1 2;5 0 2 1;5 1 2 2;8 0 3 1;8 1 3 2],[2 0 1 3;2 1 1 4;5 0 2 3;5 1 2 4;8 0 3 3;8 1 3 4],[2 0 1 5;2 1 1 6;5 0 2 5;5 1 2 6;8 0 3 5;8 1 3 6],[2 0 1 7;2 1 1 8;5 0 2 7;5 1 2 8;8 0 3 7;8 1 3 8]}); + % params = ct_set_params(params,'records.data_map',{[2 0 1 1; 2 1 1 2; 5 0 2 1; 5 1 2 2; 8 0 3 1; 8 1 3 2; 11 0 4 1; 11 1 4 2],[2 0 ... + % 1 3; 2 1 1 4; 5 0 2 3; 5 1 2 4; 8 0 3 3; 8 1 3 4; 11 0 4 3; 11 1 4 4],[2 0 1 5; 2 1 1 6; 5 0 2 5; 5 1 2 6; 8 0 3 ... + % 5; 8 1 3 6; 11 0 4 5; 11 1 4 6],[2 0 1 7; 2 1 1 8; 5 0 2 7; 5 1 2 8; 8 0 3 7; 8 1 3 8; 11 0 4 7; 11 1 4 8]}, ... + % 'day_seg','20200101_02|20200101_03|20200101_04|20200104_01|20200104_02|20200104_03|20200104_04|20200104_05|20200105_01|20200105_02|20200105_03|20200105_04|20200106_01|20200106_02|20200107_01|20200107_02|20200107_03|20200108_01|20200108_02'); end - + %% radar.wfs for wf = 1:length(params(param_idx).radar.wfs) + params(param_idx).radar.wfs(wf).bad_value = NaN; params(param_idx).radar.wfs(wf).deconv.en = 0; - if strcmpi(params(param_idx).season_name,'2010_Greenland_P3') + if strcmpi(params(param_idx).season_name,'2018_Antarctica_TObas') + params(param_idx).radar.wfs(wf).deconv.en = 1; + params(param_idx).radar.wfs(wf).coh_noise_method = 'analysis'; + params(param_idx).radar.wfs(wf).coh_noise_arg.fn = 'analysis_threshold'; + elseif strcmpi(params(param_idx).season_name,'2019_Antarctica_TObas') + params(param_idx).radar.wfs(wf).deconv.en = 1; + params(param_idx).radar.wfs(wf).coh_noise_method = 'analysis'; + params(param_idx).radar.wfs(wf).coh_noise_arg.fn = 'analysis_threshold'; + elseif strcmpi(params(param_idx).season_name,'2010_Greenland_P3') elseif strcmpi(params(param_idx).season_name,'2010_Greenland_DC8') elseif strcmpi(params(param_idx).season_name,'2011_Greenland_P3') params(param_idx).radar.wfs(wf).Tadc_adjust = -7.8534e-07; % 854.656e-9 shift @@ -160,7 +389,7 @@ elseif strcmpi(params(param_idx).season_name,'2012_Greenland_P3') elseif strcmpi(params(param_idx).season_name,'2013_Greenland_P3') elseif strcmpi(params(param_idx).season_name,'2014_Greenland_P3') - if any(strcmp(param.day_seg,{'20140429_01','20140502_01'})) + if any(strcmp(param.day_seg,{'20140325_04','20140429_01','20140502_01'})) else params(param_idx).radar.wfs(wf).coh_noise_method = 'analysis'; params(param_idx).radar.wfs(wf).coh_noise_arg.fn = 'analysis_threshold'; @@ -171,159 +400,890 @@ params(param_idx).radar.wfs(wf).Tsys = [3.73 3.09 0 5.59 3.05 0.92 2.28 -19.24 -22.03 -26.23 -29.8 -28.74 -25.71 -22.8 -19.42]/1e9; elseif strcmpi(params(param_idx).season_name,'2016_Greenland_P3') elseif strcmpi(params(param_idx).season_name,'2018_Greenland_P3') + if wf < 3 || length(params(param_idx).radar.wfs) < 4 + params(param_idx).radar.wfs(wf).burst.en = false; + else + %params(param_idx).radar.wfs(wf).burst.en = true; + params(param_idx).radar.wfs(wf).burst.en = false; + params(param_idx).radar.wfs(wf).burst.fn = 'analysis_burst'; + end + params(param_idx).radar.wfs(wf).deconv.en = false; params(param_idx).radar.wfs(wf).Tadc_adjust = -0.00000164; params(param_idx).radar.wfs(wf).Tsys = [0.46 -4.66 0.14 -1.77 0 -2.63 -3.38 -69.66 -75.57 -75.45 -80.42 -80.49 -75.71 -77.69 -70.53]/1e9; params(param_idx).radar.wfs(wf).chan_equal_dB = [6.8 -0.6 3 0.1 0 3.5 3.9 7 3.3 4.8 6.1 6.2 4.6 3.1 6.2]; params(param_idx).radar.wfs(wf).chan_equal_deg = [-166.2 -142.7 177 -95.9 0 -25.9 -86.5 -27.4 128.1 41.6 -46.8 43 90.7 121.3 31.6]; + if ~isempty(regexpi(params(param_idx).cmd.notes,'DECONVOLUTION.xml')) + % Deconvolution segment + params(param_idx).radar.wfs(wf).coh_noise_method = []; + else + params(param_idx).radar.wfs(wf).coh_noise_method = 'analysis'; + params(param_idx).radar.wfs(wf).coh_noise_arg.fn = 'analysis_threshold'; + end elseif strcmpi(params(param_idx).season_name,'2018_Antarctica_Ground') - params(param_idx).radar.wfs(wf).Tadc_adjust = -0.25e-6; + params(param_idx).radar.wfs(wf).Tadc_adjust = -0.259e-6; + if wf == 1 + params(param_idx).radar.wfs(wf).chan_equal_dB = [-0.9 0.7 0 0.5 1.6 -0.7 -0.8 0.4]; + params(param_idx).radar.wfs(wf).chan_equal_deg = [-172.7 -173.5 -174.8 -169.3 -51.1 0 -26 -34.5]; + elseif wf == 2 + params(param_idx).radar.wfs(wf).chan_equal_dB = [-0.9 0.7 0 0.5 1.6 -0.7 -0.8 0.4]; + params(param_idx).radar.wfs(wf).chan_equal_deg = [-172.7 -173.5 -174.8 -169.3 -51.1 0 -26 -34.5]; + end + elseif strcmpi(params(param_idx).season_name,'2019_Greenland_P3') + if wf < 3 || length(params(param_idx).radar.wfs) < 4 + params(param_idx).radar.wfs(wf).burst.en = false; + else + %params(param_idx).radar.wfs(wf).burst.en = true; + params(param_idx).radar.wfs(wf).burst.en = false; + %params(param_idx).radar.wfs(wf).burst.fn = 'analysis_burst'; + end + params(param_idx).radar.wfs(wf).deconv.en = false; + % params(param_idx).radar.wfs(wf).Tadc_adjust = -0.0000014455; OLD/WRONG + params(param_idx).radar.wfs(wf).Tadc_adjust = -1627.7e-9; % NEW/GOOD + + if ~isempty(regexpi(params(param_idx).cmd.notes,'Image Thick Ice Mode')) || ~isempty(regexpi(params(param_idx).cmd.notes,'Image Thin Ice Mode')) + params(param_idx).radar.wfs(1).chan_equal_dB = [1.7 -4 0 -2.6 -2.6 -1.2 0.7]; + params(param_idx).radar.wfs(1).chan_equal_deg = [39.1 45.2 0 -128.5 -135.5 -55.4 89.8]; + params(param_idx).radar.wfs(1).Tsys = [0.49 -4.63 0 -5.56 4.24 -3.19 -3.45]/1e9; + params(param_idx).radar.wfs(2).chan_equal_dB = [2.9 -2.7 1.4 -1 -1.2 0.4 2.4]; + params(param_idx).radar.wfs(2).chan_equal_deg = [-29 -15.4 -58.5 174.4 172.8 -105.5 18.3]; + params(param_idx).radar.wfs(2).Tsys = [-0.55 -5.6 -1.04 -6.63 3.35 -4.07 -4.7]/1e9; + params(param_idx).radar.wfs(3).chan_equal_dB = [1.8 -4 0.1 -2.5 -2.5 -1.2 0.8]; + params(param_idx).radar.wfs(3).chan_equal_deg = [-39 -32.8 -85 160.4 139.6 -133.5 18.7]; + params(param_idx).radar.wfs(3).Tsys = [6.77 1.71 6.23 0.78 10.42 3.11 2.88]/1e9; + params(param_idx).radar.wfs(4).chan_equal_dB = [3 -2.7 1.5 -1 -1.1 0.4 2.5]; + params(param_idx).radar.wfs(4).chan_equal_deg = [-78.2 -57.7 -107.8 110.9 102.5 -161.9 -38.1]; + params(param_idx).radar.wfs(4).Tsys = [6.11 1.17 5.66 -0.08 9.67 2.48 1.95]/1e9; + end + + if ~isempty(regexpi(params(param_idx).cmd.notes,'Image Thick Ice Mode')) + params(param_idx).radar.wfs(5).chan_equal_dB = [2.1 -3.6 0.5 -2.1 -2.2 -0.8 1.1]; + params(param_idx).radar.wfs(5).chan_equal_deg = [54.5 60.8 15.6 -99 -126.9 -32.9 112.3]; + params(param_idx).radar.wfs(5).Tsys = [-4.96 -10.06 -5.52 -10.93 -1.36 -8.62 -8.87]/1e9; + params(param_idx).radar.wfs(6).chan_equal_dB = [3.1 -2.6 1.6 -0.9 -1 0.5 2.5]; + params(param_idx).radar.wfs(6).chan_equal_deg = [0.3 13.8 -36.4 -170.5 -179 -76.4 68.5]; + params(param_idx).radar.wfs(6).Tsys = [-5.92 -10.87 -6.36 -12.08 -2.29 -9.4 -9.71]/1e9; + end + + if ~isempty(regexpi(params(param_idx).cmd.notes,'DECONVOLUTION')) + % Disable coherent noise removal for deconvolution segments (20190403_03) + params(param_idx).radar.wfs(wf).coh_noise_method = []; + else + params(param_idx).radar.wfs(wf).coh_noise_method = []; +% params(param_idx).radar.wfs(wf).coh_noise_method = 'analysis'; +% params(param_idx).radar.wfs(wf).coh_noise_arg.fn = 'analysis_threshold'; + end elseif strcmpi(params(param_idx).season_name,'2019_Antarctica_Ground') + if strcmpi(cmd_method,'sar') + if strcmp(params(param_idx).sar.out_path,'sar') + params(param_idx).radar.wfs(wf).ft_wind = @hanning; + params(param_idx).radar.wfs(wf).coh_noise_method = 'analysis'; + params(param_idx).radar.wfs(wf).coh_noise_arg.fn = 'analysis_threshold'; + elseif strcmp(params(param_idx).sar.out_path,'sar_tukey') + params(param_idx).radar.wfs(wf).ft_wind = @(N) tukeywin_trim(N,0.5); + params(param_idx).radar.wfs(wf).coh_noise_method = 'analysis'; + params(param_idx).radar.wfs(wf).coh_noise_arg.fn = 'analysis_threshold_tukey'; + end + elseif strcmpi(cmd_method,'qlook') + if strcmp(params(param_idx).qlook.out_path,'qlook') + params(param_idx).radar.wfs(wf).ft_wind = @(N) tukeywin_trim(N,0.5); + params(param_idx).radar.wfs(wf).coh_noise_method = 'analysis'; + params(param_idx).radar.wfs(wf).coh_noise_arg.fn = 'analysis_threshold_tukey'; + elseif strcmp(params(param_idx).qlook.out_path,'qlook_test') + params(param_idx).radar.wfs(wf).ft_wind = @(N) tukeywin_trim(N,0.5); + end + end params(param_idx).radar.wfs(wf).Tadc_adjust = -0.25e-6; + if wf < 3 + params(param_idx).radar.wfs(wf).chan_equal_dB = [10 6 10 9.6 -11.4 -0.8 0 9]; + params(param_idx).radar.wfs(wf).chan_equal_deg = [-53.4 -60.5 116.8 134 118.4 64.7 0 -57]; + elseif wf == 3 + params(param_idx).radar.wfs(wf).chan_equal_dB = [10.2 6.4 17.8 12.6 15.3 11.7 0 9.2]; + params(param_idx).radar.wfs(wf).chan_equal_deg = [-52.7 -59 117.5 134.2 92.6 73.2 0 -57.6]; + elseif wf == 4 + params(param_idx).radar.wfs(wf).chan_equal_dB = [10.6 6.8 20.2 16.3 20.2 6.1 1.4 10.6]; + params(param_idx).radar.wfs(wf).chan_equal_deg = [-47.1 -53.7 127.2 147.1 111.9 122.6 11.2 -35.2]; + end + elseif strcmpi(params(param_idx).season_name,'2018_Greenland_P3') + params(param_idx).radar.wfs(wf).system_dB = 10*log10(337*sum([1,1,1,1,1,0,0])^2)+6+6+20*log10(300000000/(8*pi*(180e6+210e6)/2))+10*log10(50); + params(param_idx).radar.wfs(wf).Tsys = [0.46 -4.66 0.14 -1.77 0 -2.63 -3.38 -69.66 -75.57 -75.45 -80.42 -80.49 -75.71 -77.69 -70.53]/1e9; + params(param_idx).radar.wfs(wf).chan_equal_dB = [6.8 -0.6 3 0.1 0 3.5 3.9 7 3.3 4.8 6.1 6.2 4.6 3.1 6.2]; + params(param_idx).radar.wfs(wf).chan_equal_deg = [-166.2 -142.7 177 -95.9 0 -25.9 -86.5 -27.4 128.1 41.6 -46.8 43 90.7 121.3 31.6]; + + elseif strcmpi(params(param_idx).season_name,'2019_Greenland_P3') + params(param_idx).radar.wfs(wf).system_dB = 10*log10(337*sum([1,1,1,0,0,0,0])^2)+6+6+20*log10(300000000/(8*pi*(180e6+210e6)/2))+10*log10(50); + else params(param_idx).radar.wfs(wf).coh_noise_method = 'analysis'; params(param_idx).radar.wfs(wf).coh_noise_arg.fn = 'analysis_threshold'; end end -end -if 1 - % Standard - params = ct_set_params(params,'array.tomo_en',false); - params = ct_set_params(params,'array.out_path','standard'); - params = ct_set_params(params,'array.in_path','sar'); - if strcmpi(params(param_idx).season_name,'2018_Greenland_P3') - params = ct_set_params(params,'array.imgs',{[ones(1,7); 6:12].', [2*ones(1,7); 6:12].', [3*ones(1,7); 6:12].'}); - elseif strcmpi(params(param_idx).season_name,'2010_Greenland_DC8') - elseif strcmpi(params(param_idx).season_name,'2010_Greenland_P3') - elseif strcmpi(params(param_idx).season_name,'2011_Greenland_P3') - params = ct_set_params(params,'array.imgs',{[ones(1,7); 2:8].', [2*ones(1,7); 2:8].'}); - elseif strcmpi(params(param_idx).season_name,'2012_Greenland_P3') - params = ct_set_params(params,'array.imgs',{[ones(1,7); 2:8].', [2*ones(1,7); 2:8].'}); - elseif strcmpi(params(param_idx).season_name,'2013_Greenland_P3') - elseif strcmpi(params(param_idx).season_name,'2016_Greenland_P3') - elseif any(strcmpi(params(param_idx).season_name,{'2014_Greenland_P3'})) - params = ct_set_params(params,'array.imgs',{[ones(1,7); 2:8].', [2*ones(1,7); 2:8].', [3*ones(1,7); 2:8].'}); - elseif strcmpi(params(param_idx).season_name,'2018_Antarctica_Ground') - elseif strcmpi(params(param_idx).season_name,'2019_Antarctica_Ground') - else - keyboard + %% analysis coh_noise cmd + if isfield(params(param_idx),'analysis') && ~isempty(params(param_idx).analysis.cmd) ... + && strcmp(params(param_idx).analysis.cmd{1}.method,'coh_noise') + + if any(strcmp(params(param_idx).day_seg,{'20200127_01'})) + params(param_idx).analysis.cmd{1}.threshold_coh_ave = 101; + end + + % radar.wfs + if isfield(param_override,'analysis') && isfield(param_override.analysis,'out_path') + for wf = 1:length(params(param_idx).radar.wfs) + if ~isempty(regexp(param_override.analysis.out_path,'tukey')) + params(param_idx).radar.wfs(wf).ft_wind = @(N) tukeywin_trim(N,0.5); + if ~isempty(regexp(param_override.analysis.out_path,'threshold')) + params = ct_set_params(params,'analysis.cmd{1}.threshold','analysis_tukey'); + end + else + if ~isempty(regexp(param_override.analysis.out_path,'threshold')) + params = ct_set_params(params,'analysis.cmd{1}.threshold',''); + end + end + end + end + + if isfield(param_override,'collate_coh_noise') + param_override.collate_coh_noise.debug_out_dir = regexprep(param_override.collate_coh_noise.in_path,'analysis','collate_coh_noise'); + for img = 1:length(params(param_idx).analysis.imgs) + for wf_adc = 1:size(params(param_idx).analysis.imgs{img},1) + wf = params(param_idx).analysis.imgs{img}(wf_adc,1); + adc = params(param_idx).analysis.imgs{img}(wf_adc,2); + Tpd = params(param_idx).radar.wfs(wf).Tpd; + if strcmpi(params(param_idx).season_name,'2019_Antarctica_Ground') + params(param_idx).collate_coh_noise.method{img} = 'firdec'; + params(param_idx).collate_coh_noise.firdec_fs{img} = 1/7.5; + params(param_idx).collate_coh_noise.firdec_fcutoff{img} = @(t)(t=Tpd+1.2e-6)*1/120; + if wf == 1 + params(param_idx).collate_coh_noise.threshold_eval{img}{wf_adc} ... + = 'threshold = max_filt1(threshold+6,11);'; + elseif wf == 2 + params(param_idx).collate_coh_noise.threshold_eval{img}{wf_adc} ... + = 'threshold = max_filt1(threshold+6,11);'; + elseif any(wf == [3 4]) + if any(adc == [1 2 3 4 5 6 7 8]) + params(param_idx).collate_coh_noise.threshold_eval{img}{wf_adc} ... + = 'threshold = max(nanmin(threshold(time>Tpd+1.2e-6 & time=Tpd+0e-6 & time<=Tpd+13e-6)-1e6*(time>(Tpd+13e-6)),21));'; + end + else + params(param_idx).collate_coh_noise.threshold_eval{img}{wf_adc} = 'threshold = -inf(size(threshold));'; + end + + elseif strcmpi(params(param_idx).season_name,'2019_Antarctica_TObas') + params(param_idx).collate_coh_noise.method{img} = 'firdec'; + params(param_idx).collate_coh_noise.firdec_fs{img} = 1/7.5; + params(param_idx).collate_coh_noise.firdec_fcutoff{img} = @(t) 1/30; + + %params(param_idx).collate_coh_noise.threshold_eval{wf} = 'threshold = max(min(-100,threshold + 20),10*log10(abs(noise.dft(:,1)).^2)+6);'; + if any(strcmp(params(param_idx).day_seg,{'20200127_01'})) + if wf == 1 + params(param_idx).collate_coh_noise.threshold_eval{img} = 'threshold(:) = -150;'; + else + params(param_idx).collate_coh_noise.threshold_eval{img} = 'threshold = max(nanmin(orig_threshold(time>Tpd+1.2e-6 & time=1.834e-6); + params(param_idx).collate_coh_noise.dft_corr_time(img) = inf; + + else + % Second pass + % Runs each of the modes one at a time + mode_2018_Greenland_P3 = 1; + + if mode_2018_Greenland_P3 == 1 + % (except adc 9-10 for Apr 21 and later) + if img < 3 + % Waveforms 1 and 2 + params(param_idx).collate_coh_noise.method{img} = 'firdec'; + else + % Waveforms 3, 4, 5, and 6 + params(param_idx).collate_coh_noise.method{img} = 'dft'; + end + params(param_idx).collate_coh_noise.firdec_fs{img} = 1/30; + params(param_idx).collate_coh_noise.firdec_fcutoff{img} = @(t) 1/120*(t<1.834e-6) + -1*(t>=1.834e-6); + params(param_idx).collate_coh_noise.dft_corr_time(img) = inf; + params(param_idx).collate_coh_noise.wf_adcs{img} = 1:15; + + elseif mode_2018_Greenland_P3 == 2 + if datenum(param.day_seg,'yyyymmdd') < datenum('20180421','yyyymmdd') + error('Do not run mode 2 on segments before April 21.'); + end + params(param_idx).collate_coh_noise.imgs = 3:length(params(param_idx).radar.wfs); + params(param_idx).collate_coh_noise.wf_adcs{img} = [9 10]; + params(param_idx).collate_coh_noise.method{img} = 'firdec'; + params(param_idx).collate_coh_noise.firdec_fs{img} = 1/7.5; + params(param_idx).collate_coh_noise.firdec_fcutoff{img} = @(t) 1/30*(t<30e-6); + end + + end + end + + elseif strcmpi(params(param_idx).season_name,'2019_Greenland_P3') + + params(param_idx).collate_coh_noise.min_samples = 0.5; % 50% of samples must be good to use data + params(param_idx).collate_coh_noise.threshold_en = true; + + if any(wf == [1 2]) + params(param_idx).collate_coh_noise.threshold_eval{wf} = 'threshold(time>Tpd+0.85e-6 & threshold>-110) = -110; threshold(time<=Tpd+0.85e-6) = inf; threshold = threshold+20;'; + elseif any(wf == [3 4]) + params(param_idx).collate_coh_noise.threshold_eval{wf} = 'threshold(time>Tpd+2.3e-6 & threshold>-130) = -130; threshold = threshold+20;'; + elseif any(wf == [5 6]) + params(param_idx).collate_coh_noise.threshold_eval{wf} = 'threshold(time>Tpd+3e-6 & threshold>-142) = -122; threshold(time<=Tpd+3e-6) = threshold(time<=Tpd+3e-6)+20;'; + params(param_idx).collate_coh_noise.threshold_eval{wf} = 'threshold(time>Tpd+3e-6 & threshold>-142) = -142; threshold = threshold+20;'; + else + keyboard + end + + if length(params(param_idx).radar.wfs) == 2 ... + || length(params(param_idx).radar.wfs) == 3 + % Only a single pass required + params(param_idx).collate_coh_noise.method{img} = 'dft'; + params(param_idx).collate_coh_noise.dft_corr_time(img) = inf; + params(param_idx).collate_coh_noise.in_path = 'analysis'; + params(param_idx).collate_coh_noise.out_path = 'analysis_threshold'; + + elseif length(params(param_idx).radar.wfs) == 4 ... + || length(params(param_idx).radar.wfs) == 6 + + if isempty(regexp(param_override.collate_coh_noise.in_path,'threshold')) + % First pass + if img < 3 + % Waveforms 1 and 2 + params(param_idx).collate_coh_noise.method{img} = 'firdec'; + else + % Waveforms 3, 4, 5, and 6 + params(param_idx).collate_coh_noise.method{img} = 'dft'; + end + params(param_idx).collate_coh_noise.firdec_fs{img} = 1/30; + params(param_idx).collate_coh_noise.firdec_fcutoff{img} = @(t) 1/120*(t<1.834e-6) + -1*(t>=1.834e-6); + params(param_idx).collate_coh_noise.dft_corr_time(img) = inf; + + else + % Second pass + % Runs each of the modes one at a time + mode_2018_Greenland_P3 = 1; + + if mode_2018_Greenland_P3 == 1 + % (except adc 9-10 for Apr 21 and later) + if img < 3 + % Waveforms 1 and 2 + params(param_idx).collate_coh_noise.method{img} = 'firdec'; + else + % Waveforms 3, 4, 5, and 6 + params(param_idx).collate_coh_noise.method{img} = 'dft'; + end + params(param_idx).collate_coh_noise.firdec_fs{img} = 1/30; + params(param_idx).collate_coh_noise.firdec_fcutoff{img} = @(t) 1/120*(t<1.834e-6) + -1*(t>=1.834e-6); + params(param_idx).collate_coh_noise.dft_corr_time(img) = inf; + params(param_idx).collate_coh_noise.wf_adcs{img} = 1:15; + + elseif mode_2018_Greenland_P3 == 2 + params(param_idx).collate_coh_noise.imgs = 3:length(params(param_idx).radar.wfs); + params(param_idx).collate_coh_noise.wf_adcs{img} = [9 10]; + params(param_idx).collate_coh_noise.method{img} = 'firdec'; + params(param_idx).collate_coh_noise.firdec_fs{img} = 1/7.5; + params(param_idx).collate_coh_noise.firdec_fcutoff{img} = @(t) 1/30*(t<30e-6); + end + + end + end + + + else + % Coherent noise estimate by finding DC of the entire segment + % (as opposed to using a firdec low pass filter): + params(param_idx).collate_coh_noise.method{img} = 'dft'; + params(param_idx).collate_coh_noise.dft_corr_time{img} = inf; + + % For time >= Tpd+1.2e-6, the minimum threshold used that is + % estimated by the collate process is used in the region of time >= + % Tpd+1.2e-6 and time(end)-Tpd. The first part of the time gate is + % ignored because the transmission is occuring and the last part is + % ignored due to pulse compression roll off. A guard of 6 dB is + % added to this threshold to ensure random noise fluctuation is + % accounted for. There is some risk that this data dependent minimum + % threshold value is too high and a fixed threshold should be used + % instead of "nanmin(threshold(time>Tpd+1.2e-6 & time 20; + end + for img = 3:length(params(param_idx).analysis.imgs) + % Total power detection: good for detecting cable disconnects + params(param_idx).analysis.cmd{1}.signal_fh{img} = @(raw_data,wfs) []; + params(param_idx).analysis.cmd{1}.noise_fh{img} = @(raw_data,wfs) -lp(mean(abs(raw_data).^2,1)); + params(param_idx).analysis.cmd{1}.test_fh{img} = @(data_signal,data_noise,wfs) data_noise; + params(param_idx).analysis.cmd{1}.threshold_fh{img} = @(data_signal,data_noise,test_metric,wfs) data_noise > 80; + end + else + error('Burst noise settings not determined yet.'); + end else - keyboard + for img = 1:length(params(param_idx).analysis.imgs) + % 2D filter for CFAR noise, 1D filter for signal: good for detecting short bursts. + params(param_idx).analysis.cmd{1}.signal_fh{img} = @(raw_data,wfs) lp(fir_dec(abs(raw_data.').^2,ones(1,11)/11,1).'); + params(param_idx).analysis.cmd{1}.noise_fh{img} = @(raw_data,wfs) lp(fir_dec(fir_dec(abs(raw_data.').^2,ones(1,11)/11,1).',ones(1,101)/101,1)); + params(param_idx).analysis.cmd{1}.test_fh{img} = @(data_signal,data_noise,wfs) max(data_signal-data_noise,[],1); + params(param_idx).analysis.cmd{1}.threshold_fh{img} = @(data_signal,data_noise,test_metric,wfs) data_signal-data_noise > 20; + end end - else - % 7 Elements - params = ct_set_params(params,'array.out_path','mvdr'); - if any(strcmpi(params(param_idx).season_name,{'2011_Greenland_P3','2012_Greenland_P3','2014_Greenland_P3'})) - params = ct_set_params(params,'array.imgs',{[ones(1,7); 2:8].', [2*ones(1,7); 2:8].', [3*ones(1,7); 2:8].'}); - elseif strcmpi(params(param_idx).season_name,'2018_Antarctica_Ground') - elseif strcmpi(params(param_idx).season_name,'2019_Antarctica_Ground') - else - keyboard + + if isfield(param_override,'collate_burst_noise') + if strcmpi(params(param_idx).season_name,'2018_Greenland_P3') + params(param_idx).collate_burst_noise.bit_mask = 8; + params(param_idx).collate_burst_noise.debug_max_plot_size = 0; + params(param_idx).collate_burst_noise.filt_length = 101; + params(param_idx).collate_burst_noise.filt_threshold = 0.15; + params(param_idx).collate_burst_noise.imgs = 3:length(params(param_idx).analysis.imgs); + %params(param_idx).collate_burst_noise.imgs = 3:2:length(params(param_idx).analysis.imgs); + for img = 1:length(params(param_idx).analysis.imgs) + param_override.collate_burst_noise.wf_adcs{img} = [1:4,12:15]; + %param_override.collate_burst_noise.wf_adcs{img} = [1]; + for wf_adc = 1:4 + wf = params(param_idx).analysis.imgs{img}(wf_adc,1); + adc = params(param_idx).analysis.imgs{img}(wf_adc,2); + param_override.collate_burst_noise.test_wf_adcs{img}{wf_adc} = [wf 13; wf 6]; + params(param_idx).collate_burst_noise.threshold_fh{img}{wf_adc} = @(noise,wfs) 10*log10(fir_dec(10.^(interp_finite(noise{1}.test_metric)/10), ones(1,101)/101,1)) - 10*log10(fir_dec(10.^(interp_finite(noise{2}.test_metric)/10), ones(1,101)/101,1)) > 4; + end + for wf_adc = 12:15 + wf = params(param_idx).analysis.imgs{img}(wf_adc,1); + adc = params(param_idx).analysis.imgs{img}(wf_adc,2); + param_override.collate_burst_noise.test_wf_adcs{img}{wf_adc} = [wf 13; wf 6]; + params(param_idx).collate_burst_noise.threshold_fh{img}{wf_adc} = @(noise,wfs) 10*log10(fir_dec(10.^(interp_finite(noise{1}.test_metric)/10), ones(1,101)/101,1)) - 10*log10(fir_dec(10.^(interp_finite(noise{2}.test_metric)/10), ones(1,101)/101,1)) > 10; + end + end + end end end - params = ct_set_params(params,'array.Nsv',1); - params = ct_set_params(params,'array.DCM.bin_rng',[-3:3]); - params = ct_set_params(params,'array.DCM.line_rng',[-30:30]); - params = ct_set_params(params,'array.bin_rng',[0]); - params = ct_set_params(params,'array.line_rng',[-5:5]); - params = ct_set_params(params,'array.Nsrc',1); -elseif 0 - % MUSIC - params = ct_set_params(params,'array.tomo_en',true); - params = ct_set_params(params,'array.out_path','music3D'); - params = ct_set_params(params,'array.method','music'); - if strcmpi(params(param_idx).season_name,'2018_Greenland_P3') - params = ct_set_params(params,'array.imgs',{[ones(1,15); [1:4,6:16]].', [2*ones(1,15); [1:4,6:16]].', [3*ones(1,15); [1:4,6:16]].'}); - elseif any(strcmpi(params(param_idx).season_name,{'2011_Greenland_P3','2012_Greenland_P3'})) - params = ct_set_params(params,'array.imgs',{[ones(1,15); 2:16].', [2*ones(1,15); 2:16].'}); - elseif any(strcmpi(params(param_idx).season_name,{'2014_Greenland_P3'})) - params = ct_set_params(params,'array.imgs',{[ones(1,15); 2:16].', [2*ones(1,15); 2:16].', [3*ones(1,15); 2:16].'}); - elseif strcmpi(params(param_idx).season_name,'2018_Antarctica_Ground') - elseif strcmpi(params(param_idx).season_name,'2019_Antarctica_Ground') - else - keyboard - end - params = ct_set_params(params,'array.Nsv',128); - params = ct_set_params(params,'array.bin_rng',[-1:1]); - params = ct_set_params(params,'array.line_rng',[-10:10]); - params = ct_set_params(params,'array.Nsrc',3); -elseif 1 - % GEONULL - params = ct_set_params(params,'array.tomo_en',true); - params = ct_set_params(params,'array.method','geonull_cal'); - params = ct_set_params(params,'array.surf_layer.source','surfData'); - params = ct_set_params(params,'array.surf_layer.name','top twtt'); - if any(strcmpi(params(param_idx).season_name,{'2011_Greenland_P3','2012_Greenland_P3'})) - params = ct_set_params(params,'array.imgs',{[ones(1,7); 2:8].', [2*ones(1,7); 2:8].'}); - elseif any(strcmpi(params(param_idx).season_name,{'2014_Greenland_P3'})) - params = ct_set_params(params,'array.imgs',{[ones(1,7); 2:8].', [2*ones(1,7); 2:8].', [3*ones(1,7); 2:8].'}); - elseif strcmpi(params(param_idx).season_name,'2018_Antarctica_Ground') - elseif strcmpi(params(param_idx).season_name,'2019_Antarctica_Ground') - else - keyboard + + %% analysis specular cmd (deconvolution) + if isfield(params(param_idx),'analysis') && ~isempty(params(param_idx).analysis.cmd) ... + && strcmp(params(param_idx).analysis.cmd{1}.method,'specular') + + if isfield(param_override,'collate_deconv') + param_override.collate_deconv.debug_out_dir = regexprep(param_override.collate_deconv.in_path,'analysis','collate_deconv'); + for img = 1:length(params(param_idx).analysis.imgs) + for wf_adc = 1:size(params(param_idx).analysis.imgs{img},1) + wf = params(param_idx).analysis.imgs{img}(wf_adc,1); + adc = params(param_idx).analysis.imgs{img}(wf_adc,2); + Tpd = params(param_idx).radar.wfs(wf).Tpd; + BW = abs(params(param_idx).radar.wfs(wf).f1-params(param_idx).radar.wfs(wf).f0); + if strcmpi(params(param_idx).season_name,'2019_Antarctica_TObas') + param_override.collate_deconv.rbins{1} = [-(50+104) 520+104]; + params = ct_set_params(params,'collate_deconv.f0',615e6); + params = ct_set_params(params,'collate_deconv.f1',885e6); + params = ct_set_params(params,'collate_deconv.abs_metric',[90 3.8 -34 -34 -30 -30]); + params = ct_set_params(params,'collate_deconv.SL_guard_bins',6); + elseif strcmpi(params(param_idx).season_name,'2018_Greenland_P3') + if wf == 1 + param_override.collate_deconv.rbins{img} = round([-Tpd*BW*1.1 Tpd*BW*1.1]); + params = ct_set_params(params,'collate_deconv.abs_metric',[60 3.3 -34 -45 -24 -35]); + elseif wf == 2 + param_override.collate_deconv.rbins{img} = round([-Tpd*BW*1.1 Tpd*BW*1.1]); + params = ct_set_params(params,'collate_deconv.abs_metric',[60 3.4 -34 -45 -24 -35]); + elseif wf == 3 + %param_override.collate_deconv.rbins{img} = round([-Tpd*BW*1.1 Tpd*BW*1.1]); + param_override.collate_deconv.rbins{img} = [-25 30]; + params = ct_set_params(params,'collate_deconv.abs_metric',[60 3.4 -34 -45 -24 -35]); + end + params = ct_set_params(params,'collate_deconv.f0',181e6); + params = ct_set_params(params,'collate_deconv.f1',209e6); + params = ct_set_params(params,'collate_deconv.SL_guard_bins',6); + elseif strcmpi(params(param_idx).season_name,'2019_Greenland_P3') + if wf == 1 || wf == 2 + param_override.collate_deconv.rbins{img} = [-35 30]; + %params = ct_set_params(params,'collate_deconv.abs_metric',[60 3.3 -34 -45 -24 -35]); + elseif wf == 3 || wf == 4 + param_override.collate_deconv.rbins{img} = [-35 30]; + %params = ct_set_params(params,'collate_deconv.abs_metric',[60 3.4 -34 -45 -24 -35]); + elseif wf == 5 || wf == 6 + %param_override.collate_deconv.rbins{img} = round([-Tpd*BW*1.1 Tpd*BW*1.1]); + param_override.collate_deconv.rbins{img} = [-35 30]; + %params = ct_set_params(params,'collate_deconv.abs_metric',[60 3.4 -34 -45 -24 -35]); + end + params = ct_set_params(params,'collate_deconv.f0',181e6); + params = ct_set_params(params,'collate_deconv.f1',209e6); + params = ct_set_params(params,'collate_deconv.SL_guard_bins',6); + end + end + end + end + end - params = ct_set_params(params,'array.Nsv',1); - params = ct_set_params(params,'array.bin_rng',[0]); - params = ct_set_params(params,'array.line_rng',[-5:5]); - params = ct_set_params(params,'array.Nsrc',1); -elseif 0 - % GSLC - params = ct_set_params(params,'array.tomo_en',true); - params = ct_set_params(params,'array.out_path','gslc'); - params = ct_set_params(params,'array.method','music'); - if any(strcmpi(params(param_idx).season_name,{'2011_Greenland_P3','2012_Greenland_P3'})) - params = ct_set_params(params,'array.imgs',{[ones(1,7); 2:8].', [2*ones(1,7); 2:8].'}); - elseif any(strcmpi(params(param_idx).season_name,{'2014_Greenland_P3'})) - params = ct_set_params(params,'array.imgs',{[ones(1,7); 2:8].', [2*ones(1,7); 2:8].', [3*ones(1,7); 2:8].'}); - elseif strcmpi(params(param_idx).season_name,'2018_Antarctica_Ground') - elseif strcmpi(params(param_idx).season_name,'2019_Antarctica_Ground') - else - keyboard + + %% analysis waveform cmd (equalization) + if isfield(params(param_idx),'analysis') && ~isempty(params(param_idx).analysis.cmd) ... + && strcmp(params(param_idx).analysis.cmd{1}.method,'waveform') && isfield(param_override,'analysis') + if strcmpi(params(param_idx).season_name,'2018_Antarctica_Ground') + if strcmp(param_override.analysis.out_path,'analysis_equal_001') + params(param_idx).analysis.imgs = {[1*ones([8 1]),(1:8).']}; + params(param_idx).analysis.cmd{1}.start_time = struct('name','equal_001','eval',struct('cmd','s=s-0.75e-6;')); + elseif strcmp(param_override.analysis.out_path,'analysis_equal_002') + params(param_idx).analysis.imgs = {[2*ones([8 1]),(1:8).']}; + params(param_idx).analysis.cmd{1}.start_time = struct('name','equal_002','eval',struct('cmd','s=s-0.75e-6;')); + end + end + if strcmpi(params(param_idx).season_name,'2019_Antarctica_Ground') + if strcmp(param_override.analysis.out_path,'analysis_equal_002') + params(param_idx).analysis.imgs = {[2*ones([8 1]),(1:8).']}; + params(param_idx).analysis.cmd{1}.start_time = struct('name','equal_002','eval',struct('cmd','s=s-1.5e-6;')); + elseif strcmp(param_override.analysis.out_path,'analysis_equal_003') + params(param_idx).analysis.imgs = {[[3*ones([8 1]),(1:8).']; [4*ones([8 1]),(1:8).']]}; + params(param_idx).analysis.cmd{1}.start_time = struct('name','equal_003','eval',struct('cmd','s=s-1.5e-6;')); + end + end end - params = ct_set_params(params,'array.Nsv',1); - params = ct_set_params(params,'array.DCM.bin_rng',[-3:3]); - params = ct_set_params(params,'array.DCM.line_rng',[-30:30]); - params = ct_set_params(params,'array.bin_rng',[0]); - params = ct_set_params(params,'array.line_rng',[-5:5]); - params = ct_set_params(params,'array.Nsrc',1); -elseif 1 - % SNAPSHOT - params = ct_set_params(params,'array.tomo_en',true); - params = ct_set_params(params,'array.in_path','sar_air'); - params = ct_set_params(params,'array.out_path','snapshot'); - params = ct_set_params(params,'array.method','snapshot'); - params = ct_set_params(params,'array.surf_layer.source','surfData'); - params = ct_set_params(params,'array.surf_layer.name','top twtt'); - if strcmpi(params(param_idx).season_name,'2018_Greenland_P3') - params = ct_set_params(params,'array.imgs',{[ones(1,15); [1:4,6:16]].', [2*ones(1,15); [1:4,6:16]].', [3*ones(1,15); [1:4,6:16]].'}); - elseif any(strcmpi(params(param_idx).season_name,{'2011_Greenland_P3','2012_Greenland_P3'})) - params = ct_set_params(params,'array.imgs',{[ones(1,15); 2:16].', [2*ones(1,15); 2:16].'}); - elseif any(strcmpi(params(param_idx).season_name,{'2014_Greenland_P3'})) - params = ct_set_params(params,'array.imgs',{[ones(1,15); 2:16].', [2*ones(1,15); 2:16].', [3*ones(1,15); 2:16].'}); - else - keyboard + + if isfield(param_override,'collate_equal') + if strcmpi(params(param_idx).season_name,'2014_Greenland_P3') && strcmpi(params(param_idx).day_seg,'20140325_07') + param_override.collate_equal.rlines = [14000:18500]; + param_override.collate_equal.ref = 3; + param_override.collate_equal.debug_plots = {'before_comp','after_comp','surf','final','comp_image'}; + param_override.collate_equal.retrack_en = true; + + elseif strcmpi(params(param_idx).season_name,'2011_Greenland_P3') && strcmpi(params(param_idx).day_seg,'20110506_02') + if 1 + param_override.collate_equal.img_lists = {[2]}; + param_override.collate_equal.rlines = [170000:180000]; + else + param_override.collate_equal.img_lists = {[1]}; + param_override.collate_equal.rlines = [80000:117500]; + end + param_override.collate_equal.wf_adcs = {{[2:16]}}; + param_override.collate_equal.ref = 3; + param_override.collate_equal.debug_plots = {'before_comp','after_comp','surf','final','comp_image'}; + + param_override.collate_equal.retrack_en = true; + + elseif strcmpi(params(param_idx).season_name,'2018_Antarctica_Ground') && strcmpi(params(param_idx).day_seg,'20181224_03') + if strcmp(param_override.collate_equal.in_path,'analysis_equal_001') + params(param_idx).analysis.imgs = {[1*ones([8 1]),(1:8).']}; + param_override.collate_equal.rlines = []; % wf == 1, equal_001 layer + param_override.collate_equal.debug_out_dir = 'collate_equal_001'; + param_override.collate_equal.ref = 6; + elseif strcmp(param_override.collate_equal.in_path,'analysis_equal_002') + params(param_idx).analysis.imgs = {[2*ones([8 1]),(1:8).']}; + param_override.collate_equal.rlines = []; % wf == 2 equal_002 layer + param_override.collate_equal.debug_out_dir = 'collate_equal_002'; + param_override.collate_equal.ref = 6; + end + %param_override.collate_equal.debug_plots = {'visible','before_comp','after_comp','surf','final','comp_image'}; + param_override.collate_equal.debug_plots = {'before_comp','after_comp','surf','final','comp_image'}; + param_override.collate_equal.retrack_en = false; + + elseif strcmpi(params(param_idx).season_name,'2019_Greenland_P3') + param_override.collate_equal.rlines = [1:660]; + param_override.collate_equal.ref = 3; + param_override.collate_equal.debug_plots = {'visible','before_comp','after_comp','surf','final','comp_image'}; + %param_override.collate_equal.debug_plots = {'before_comp','after_comp','surf','final','comp_image'}; + param_override.collate_equal.retrack_en = false; + + elseif strcmpi(params(param_idx).season_name,'2019_Antarctica_Ground') && strcmpi(params(param_idx).day_seg,'20200107_01') + if strcmp(param_override.collate_equal.in_path,'analysis_equal_002') + params(param_idx).analysis.imgs = {[2*ones([8 1]),(1:8).']}; + param_override.collate_equal.rlines = []; % wf == 2 equal_002 layer + param_override.collate_equal.debug_out_dir = 'collate_equal_002'; + elseif strcmp(param_override.collate_equal.in_path,'analysis_equal_003') + params(param_idx).analysis.imgs = {[[3*ones([8 1]),(1:8).']; [4*ones([8 1]),(1:8).']]}; + param_override.collate_equal.rlines = []; % wf == 3,4, equal_003 layer + param_override.collate_equal.debug_out_dir = 'collate_equal_003'; + end + param_override.collate_equal.debug_plots = {'visible','before_comp','after_comp','surf','final','comp_image'}; + % param_override.collate_equal.debug_plots = {'before_comp','after_comp','surf','final','comp_image'}; + param_override.collate_equal.retrack_en = false; + + end + end -% params = ct_set_params(params,'array.imgs',{[ones(1,7); 2:8].', [2*ones(1,7); 2:8].', [3*ones(1,7); 2:8].'}); - params = ct_set_params(params,'array.imgs',{[ones(1,7); 2:8].', [2*ones(1,7); 2:8].', [3*ones(1,7); 2:8].'}); - params = ct_set_params(params,'array.dline',1); + end -params = ct_set_params(params,'array.dline',6); +%% array +if isfield(param_override,'array') && isfield(param_override.array,'out_path') + for param_idx = 1:length(params) + param = params(param_idx); + if strcmpi(cmd_method,'generic') + if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic + continue; + end + elseif ~param.cmd.(cmd_method) + continue; + end + + if strcmpi(params(param_idx).season_name,'2018_Antarctica_Ground') + params(param_idx).array.in_path = 'sar'; + elseif strcmpi(params(param_idx).season_name,'2019_Antarctica_Ground') + if strcmpi(param_override.array.out_path,'standard') + params(param_idx).array.in_path = 'sar'; + elseif strcmpi(param_override.array.out_path,'standard_tukey') + params(param_idx).array.in_path = 'sar_tukey'; + elseif strcmpi(param_override.array.out_path,'mvdr') + params(param_idx).array.in_path = 'sar'; + elseif strcmpi(param_override.array.out_path,'music3D') + params(param_idx).array.in_path = 'sar'; + end + end + params(param_idx).array.ft_over_sample = 2; + + params(param_idx).array.dline = 6; + if ~isempty(regexp(param_override.array.out_path,'standard')) + % Standard + params(param_idx).array.tomo_en = false; + params(param_idx).array.method = 'standard'; + params(param_idx).array.Nsv = 1; + params(param_idx).array.bin_rng = [0]; + params(param_idx).array.line_rng = [-5:5]; + params(param_idx).array.Nsrc = 1; + if strcmpi(output_dir,'accum') && strcmpi(params(param_idx).season_name,'2018_Antarctica_TObas') + params(param_idx).array.line_rng = [-10:10]; + params(param_idx).array.dline = 11; + elseif strcmpi(output_dir,'accum') && strcmpi(params(param_idx).season_name,'2019_Antarctica_TObas') + params(param_idx).array.line_rng = [-10:10]; + params(param_idx).array.dline = 11; + elseif strcmpi(output_dir,'rds') && strcmpi(params(param_idx).season_name,'2010_Greenland_DC8') + elseif strcmpi(output_dir,'rds') && strcmpi(params(param_idx).season_name,'2010_Greenland_P3') + elseif strcmpi(output_dir,'rds') && strcmpi(params(param_idx).season_name,'2011_Greenland_P3') + params = ct_set_params(params,'array.imgs',{[ones(1,7); 2:8].', [2*ones(1,7); 2:8].'}); + elseif strcmpi(output_dir,'rds') && strcmpi(params(param_idx).season_name,'2012_Greenland_P3') + params = ct_set_params(params,'array.imgs',{[ones(1,7); 2:8].', [2*ones(1,7); 2:8].'}); + elseif strcmpi(output_dir,'rds') && strcmpi(params(param_idx).season_name,'2013_Greenland_P3') + elseif strcmpi(output_dir,'rds') && strcmpi(params(param_idx).season_name,'2014_Greenland_P3') + params = ct_set_params(params,'array.imgs',{[ones(1,7); 2:8].', [2*ones(1,7); 2:8].', [3*ones(1,7); 2:8].'}); + elseif strcmpi(output_dir,'rds') && strcmpi(params(param_idx).season_name,'2016_Greenland_P3') + elseif strcmpi(output_dir,'rds') && strcmpi(params(param_idx).season_name,'2018_Greenland_P3') + %adcs = [13:16]; Nchan = length(adcs); % right wing + adcs = [6:12]; Nchan = length(adcs); % fuselage + if length(params(param_idx).radar.wfs) == 6 + params(param_idx).array.imgs = {[ones(1,Nchan) 2*ones(1,Nchan); adcs adcs].', [3*ones(1,Nchan) 4*ones(1,Nchan); adcs adcs].', [5*ones(1,Nchan) 6*ones(1,Nchan); adcs adcs].'}; + elseif length(params(param_idx).radar.wfs) == 4 + params(param_idx).array.imgs = {[ones(1,Nchan) 2*ones(1,Nchan); adcs adcs].', [3*ones(1,Nchan) 4*ones(1,Nchan); adcs adcs].'}; + elseif length(params(param_idx).radar.wfs) == 3 + params(param_idx).array.imgs = {[ones(1,Nchan); adcs].', [2*ones(1,Nchan) adcs].', [3*ones(1,Nchan) adcs].'}; + elseif length(params(param_idx).radar.wfs) == 2 + params(param_idx).array.imgs = {[ones(1,Nchan) 2*ones(1,Nchan); adcs adcs].'}; + end + elseif strcmpi(params(param_idx).season_name,'2018_Antarctica_Ground') + params(param_idx).array.imgs = {{[ones(1,4); 1:4].',[ones(1,4); 5:8].'},{[2*ones(1,4); 1:4].',[2*ones(1,4); 5:8].'}}; + elseif strcmpi(output_dir,'rds') && strcmpi(params(param_idx).season_name,'2019_Greenland_P3') + adcs = [1:7]; Nchan = length(adcs); % fuselage + if length(params(param_idx).radar.wfs) == 6 + params(param_idx).array.imgs = {[ones(1,Nchan) 2*ones(1,Nchan); adcs adcs].', [3*ones(1,Nchan) 4*ones(1,Nchan); adcs adcs].', [5*ones(1,Nchan) 6*ones(1,Nchan); adcs adcs].'}; + params(param_idx).array.imgs{3} = params(param_idx).array.imgs{3}(1:end-1,:); + params(param_idx).array.imgs{3} = params(param_idx).array.imgs{3}([1 3:end],:); + params(param_idx).array.imgs{2} = params(param_idx).array.imgs{2}(1:end-1,:); + params(param_idx).array.imgs{1} = params(param_idx).array.imgs{1}(1:end-1,:); + elseif length(params(param_idx).radar.wfs) == 4 + params(param_idx).array.imgs = {[ones(1,Nchan) 2*ones(1,Nchan); adcs adcs].', [3*ones(1,Nchan) 4*ones(1,Nchan); adcs adcs].'}; + params(param_idx).array.imgs{2} = params(param_idx).array.imgs{2}(1:end-1,:); + params(param_idx).array.imgs{1} = params(param_idx).array.imgs{1}(1:end-1,:); + elseif length(params(param_idx).radar.wfs) == 2 + if isempty(params(param_idx).array.img_comb) + params(param_idx).array.imgs = {[ones(1,Nchan) 2*ones(1,Nchan); adcs adcs].'}; + params(param_idx).array.imgs{1} = params(param_idx).array.imgs{1}(1:end-1,:); + else + params(param_idx).array.imgs = {[ones(1,Nchan); adcs].',[2*ones(1,Nchan); adcs].'}; + params(param_idx).array.imgs{2} = params(param_idx).array.imgs{2}(1:end-1,:); + end + end + + elseif strcmpi(params(param_idx).season_name,'2019_Antarctica_Ground') + adcs = [1:6]; Nchan = length(adcs); + if length(params(param_idx).radar.wfs) == 3 + params(param_idx).array.imgs = {[ones(1,Nchan); adcs].', [2*ones(1,Nchan); adcs].', [3*ones(1,Nchan); adcs].'}; + else + params(param_idx).array.imgs = {[ones(1,Nchan); adcs].', [2*ones(1,Nchan); adcs].', [3*ones(1,Nchan) 4*ones(1,Nchan); adcs adcs].'}; + end + else + keyboard + end + + elseif ~isempty(regexp(param_override.array.out_path,'mvdr')) + % MVDR + params(param_idx).array.tomo_en = false; + params(param_idx).array.method = 'mvdr'; + if strcmpi(params(param_idx).season_name,'2019_Antarctica_Ground') + adcs = [1:6]; Nchan = length(adcs); + if length(params(param_idx).radar.wfs) == 3 + params(param_idx).array.imgs = {[ones(1,Nchan); adcs].', [2*ones(1,Nchan); adcs].', [3*ones(1,Nchan); adcs].'}; + else + params(param_idx).array.imgs = {[ones(1,Nchan); adcs].', [2*ones(1,Nchan); adcs].', [3*ones(1,Nchan) 4*ones(1,Nchan); adcs adcs].'}; + end + else + if 0 + % 15 Elements + params = ct_set_params(params,'array.out_path','mvdr'); + if any(strcmpi(params(param_idx).season_name,{'2011_Greenland_P3','2012_Greenland_P3'})) + params = ct_set_params(params,'array.imgs',{[ones(1,15); 2:16].', [2*ones(1,15); 2:16].'}); + elseif any(strcmpi(params(param_idx).season_name,{'2014_Greenland_P3'})) + params = ct_set_params(params,'array.imgs',{[ones(1,15); 2:16].', [2*ones(1,15); 2:16].', [3*ones(1,15); 2:16].'}); + elseif strcmpi(params(param_idx).season_name,'2018_Greenland_P3') + adcs = [1:4,6:16]; Nchan = length(adcs); + if length(params(param_idx).radar.wfs) == 6 + params(param_idx).array.imgs = {[ones(1,Nchan) 2*ones(1,Nchan); adcs adcs].', [3*ones(1,Nchan) 4*ones(1,Nchan); adcs adcs].', [5*ones(1,Nchan) 6*ones(1,Nchan); adcs adcs].'}; + elseif length(params(param_idx).radar.wfs) == 4 + params(param_idx).array.imgs = {[ones(1,Nchan) 2*ones(1,Nchan); adcs adcs].', [3*ones(1,Nchan) 4*ones(1,Nchan); adcs adcs].'}; + end + + else + keyboard + end + else + % 7 Elements + params = ct_set_params(params,'array.out_path','mvdr'); + if any(strcmpi(params(param_idx).season_name,{'2011_Greenland_P3','2012_Greenland_P3','2014_Greenland_P3'})) + params = ct_set_params(params,'array.imgs',{[ones(1,7); 2:8].', [2*ones(1,7); 2:8].', [3*ones(1,7); 2:8].'}); + elseif strcmpi(params(param_idx).season_name,'2018_Greenland_P3') + %adcs = [13:16]; Nchan = length(adcs); % right wing + adcs = [6:12]; Nchan = length(adcs); % fuselage + if length(params(param_idx).radar.wfs) == 6 + params(param_idx).array.imgs = {[ones(1,Nchan) 2*ones(1,Nchan); adcs adcs].', [3*ones(1,Nchan) 4*ones(1,Nchan); adcs adcs].', [5*ones(1,Nchan) 6*ones(1,Nchan); adcs adcs].'}; + elseif length(params(param_idx).radar.wfs) == 4 + params(param_idx).array.imgs = {[ones(1,Nchan) 2*ones(1,Nchan); adcs adcs].', [3*ones(1,Nchan) 4*ones(1,Nchan); adcs adcs].'}; + elseif length(params(param_idx).radar.wfs) == 3 + params(param_idx).array.imgs = {[ones(1,Nchan); adcs].', [2*ones(1,Nchan) adcs].', [3*ones(1,Nchan) adcs].'}; + elseif length(params(param_idx).radar.wfs) == 2 + params(param_idx).array.imgs = {[ones(1,Nchan) 2*ones(1,Nchan); adcs adcs].'}; + end + elseif strcmpi(params(param_idx).season_name,'2018_Antarctica_Ground') + elseif strcmpi(params(param_idx).season_name,'2019_Greenland_P3') + adcs = [1:7]; Nchan = length(adcs); % fuselage + if length(params(param_idx).radar.wfs) == 6 + params(param_idx).array.imgs = {[ones(1,Nchan) 2*ones(1,Nchan); adcs adcs].', [3*ones(1,Nchan) 4*ones(1,Nchan); adcs adcs].', [5*ones(1,Nchan) 6*ones(1,Nchan); adcs adcs].'}; + elseif length(params(param_idx).radar.wfs) == 4 + params(param_idx).array.imgs = {[ones(1,Nchan) 2*ones(1,Nchan); adcs adcs].', [3*ones(1,Nchan) 4*ones(1,Nchan); adcs adcs].'}; + elseif length(params(param_idx).radar.wfs) == 3 + params(param_idx).array.imgs = {[ones(1,Nchan); adcs].', [2*ones(1,Nchan) adcs].', [3*ones(1,Nchan) adcs].'}; + elseif length(params(param_idx).radar.wfs) == 2 + params(param_idx).array.imgs = {[ones(1,Nchan) 2*ones(1,Nchan); adcs adcs].'}; + end + elseif strcmpi(params(param_idx).season_name,'2019_Antarctica_Ground') + else + keyboard + end + end + end + params(param_idx).array.Nsv = 1; + params(param_idx).array.bin_rng = [0]; + params(param_idx).array.line_rng = [-5:5]; + params(param_idx).array.Nsrc = 1; + params(param_idx).array.DCM.bin_rng = [-3:3]; + params(param_idx).array.DCM.line_rng = [-30:30]; + elseif ~isempty(regexp(param_override.array.out_path,'music3D')) + % MUSIC + params(param_idx).array.tomo_en = true; + params(param_idx).array.method = 'music'; + params(param_idx).array.out_path = 'music3D'; + if any(strcmpi(params(param_idx).season_name,{'2011_Greenland_P3','2012_Greenland_P3'})) + params = ct_set_params(params,'array.imgs',{[ones(1,15); 2:16].', [2*ones(1,15); 2:16].'}); + params = ct_set_params(params,'array.Nsv',128); + params = ct_set_params(params,'array.bin_rng',[-1:1]); + params = ct_set_params(params,'array.line_rng',[-10:10]); + params = ct_set_params(params,'array.Nsrc',3); + elseif any(strcmpi(params(param_idx).season_name,{'2014_Greenland_P3'})) + params = ct_set_params(params,'array.imgs',{[ones(1,15); 2:16].', [2*ones(1,15); 2:16].', [3*ones(1,15); 2:16].'}); + params = ct_set_params(params,'array.Nsv',128); + params = ct_set_params(params,'array.bin_rng',[-1:1]); + params = ct_set_params(params,'array.line_rng',[-10:10]); + params = ct_set_params(params,'array.Nsrc',3); + elseif strcmpi(params(param_idx).season_name,'2018_Greenland_P3') + adcs = [1:4,6:16]; Nchan = length(adcs); + if length(params(param_idx).radar.wfs) == 6 + params(param_idx).array.imgs = {[ones(1,Nchan); adcs].', [2*ones(1,Nchan); adcs].', [3*ones(1,Nchan); adcs].', [4*ones(1,Nchan); adcs].', [5*ones(1,Nchan); adcs].' [6*ones(1,Nchan); adcs].'}; + params = ct_set_params(params,'array.Nsv',128); + params = ct_set_params(params,'array.bin_rng',[-1:1]); + params = ct_set_params(params,'array.line_rng',[-10:10]); + params = ct_set_params(params,'array.Nsrc',3); + elseif length(params(param_idx).radar.wfs) == 4 + params(param_idx).array.imgs = {[ones(1,Nchan) 2*ones(1,Nchan); adcs adcs].', [3*ones(1,Nchan) 4*ones(1,Nchan); adcs adcs].'}; + elseif length(params(param_idx).radar.wfs) == 3 + params(param_idx).array.imgs = {[ones(1,Nchan); adcs].', [2*ones(1,Nchan) adcs].', [3*ones(1,Nchan) adcs].'}; + elseif length(params(param_idx).radar.wfs) == 2 + params(param_idx).array.imgs = {[ones(1,Nchan) 2*ones(1,Nchan); adcs adcs].'}; + params = ct_set_params(params,'array.Nsv',128); + params = ct_set_params(params,'array.bin_rng',[-1:1]); + params = ct_set_params(params,'array.line_rng',[-10:10]); + params = ct_set_params(params,'array.Nsrc',3); + end + elseif strcmpi(params(param_idx).season_name,'2018_Antarctica_Ground') + params(param_idx).array.Nsv = 64; + params(param_idx).array.bin_rng = [-1:1]; + params(param_idx).array.line_rng = [-10:10]; + params(param_idx).array.Nsrc = 2; + elseif strcmpi(params(param_idx).season_name,'2019_Greenland_P3') + adcs = [1:7]; Nchan = length(adcs); + if length(params(param_idx).radar.wfs) == 6 + params(param_idx).array.imgs = {[ones(1,Nchan); adcs].', [2*ones(1,Nchan); adcs].', [3*ones(1,Nchan); adcs].', [4*ones(1,Nchan); adcs].', [5*ones(1,Nchan); adcs].' [6*ones(1,Nchan); adcs].'}; + params = ct_set_params(params,'array.Nsv',128); + params = ct_set_params(params,'array.bin_rng',[-1:1]); + params = ct_set_params(params,'array.line_rng',[-10:10]); + params = ct_set_params(params,'array.Nsrc',3); + elseif length(params(param_idx).radar.wfs) == 4 + params(param_idx).array.imgs = {[ones(1,Nchan) 2*ones(1,Nchan); adcs adcs].', [3*ones(1,Nchan) 4*ones(1,Nchan); adcs adcs].'}; + elseif length(params(param_idx).radar.wfs) == 3 + params(param_idx).array.imgs = {[ones(1,Nchan); adcs].', [2*ones(1,Nchan) adcs].', [3*ones(1,Nchan) adcs].'}; + elseif length(params(param_idx).radar.wfs) == 2 + params(param_idx).array.imgs = {[ones(1,Nchan) 2*ones(1,Nchan); adcs adcs].'}; + params = ct_set_params(params,'array.Nsv',128); + params = ct_set_params(params,'array.bin_rng',[-1:1]); + params = ct_set_params(params,'array.line_rng',[-10:10]); + params = ct_set_params(params,'array.Nsrc',3); + end + elseif strcmpi(params(param_idx).season_name,'2019_Antarctica_Ground') + adcs = [1:6]; Nchan = length(adcs); + if length(params(param_idx).radar.wfs) == 3 + params(param_idx).array.imgs = {[ones(1,Nchan); adcs].', [2*ones(1,Nchan); adcs].', [3*ones(1,Nchan); adcs].'}; + else + params(param_idx).array.imgs = {[ones(1,Nchan); adcs].', [2*ones(1,Nchan); adcs].', [3*ones(1,Nchan) 4*ones(1,Nchan); adcs adcs].'}; + end + params(param_idx).array.Nsv = 64; + params(param_idx).array.bin_rng = [-1:1]; + params(param_idx).array.line_rng = [-10:10]; + params(param_idx).array.Nsrc = 2; + else + keyboard + end + elseif 0 + % GEONULL + params = ct_set_params(params,'array.tomo_en',true); + params = ct_set_params(params,'array.method','geonull_cal'); + params = ct_set_params(params,'array.surf_layer.source','surfData'); + params = ct_set_params(params,'array.surf_layer.name','top twtt'); + if any(strcmpi(params(param_idx).season_name,{'2011_Greenland_P3','2012_Greenland_P3'})) + params = ct_set_params(params,'array.imgs',{[ones(1,7); 2:8].', [2*ones(1,7); 2:8].'}); + elseif any(strcmpi(params(param_idx).season_name,{'2014_Greenland_P3'})) + params = ct_set_params(params,'array.imgs',{[ones(1,7); 2:8].', [2*ones(1,7); 2:8].', [3*ones(1,7); 2:8].'}); + elseif strcmpi(params(param_idx).season_name,'2018_Antarctica_Ground') + elseif strcmpi(params(param_idx).season_name,'2019_Antarctica_Ground') + else + keyboard + end + params = ct_set_params(params,'array.Nsv',1); + params = ct_set_params(params,'array.bin_rng',[0]); + params = ct_set_params(params,'array.line_rng',[-5:5]); + params = ct_set_params(params,'array.Nsrc',1); + elseif 0 + % GSLC + params = ct_set_params(params,'array.tomo_en',true); + params = ct_set_params(params,'array.out_path','gslc'); + params = ct_set_params(params,'array.method','music'); + if any(strcmpi(params(param_idx).season_name,{'2011_Greenland_P3','2012_Greenland_P3'})) + params = ct_set_params(params,'array.imgs',{[ones(1,7); 2:8].', [2*ones(1,7); 2:8].'}); + elseif any(strcmpi(params(param_idx).season_name,{'2014_Greenland_P3'})) + params = ct_set_params(params,'array.imgs',{[ones(1,7); 2:8].', [2*ones(1,7); 2:8].', [3*ones(1,7); 2:8].'}); + elseif strcmpi(params(param_idx).season_name,'2018_Antarctica_Ground') + elseif strcmpi(params(param_idx).season_name,'2019_Antarctica_Ground') + else + keyboard + end + params = ct_set_params(params,'array.Nsv',1); + params = ct_set_params(params,'array.DCM.bin_rng',[-3:3]); + params = ct_set_params(params,'array.DCM.line_rng',[-30:30]); + params = ct_set_params(params,'array.bin_rng',[0]); + params = ct_set_params(params,'array.line_rng',[-5:5]); + params = ct_set_params(params,'array.Nsrc',1); + elseif ~isempty(regexp(param_override.array.out_path,'snapshot')) + % SNAPSHOT + params = ct_set_params(params,'array.tomo_en',true); + params = ct_set_params(params,'array.in_path','sar_air'); +% params = ct_set_params(params,'array.out_path','snapshot'); + params = ct_set_params(params,'array.method','snapshot'); + params = ct_set_params(params,'array.surf_layer.source','surf_sar'); + params = ct_set_params(params,'array.surf_layer.name','top twtt'); + if strcmpi(params(param_idx).season_name,'2018_Greenland_P3') + params = ct_set_params(params,'array.imgs',{[ones(1,15); [1:4,6:16]].', [2*ones(1,15); [1:4,6:16]].', [3*ones(1,15); [1:4,6:16]].'}); + elseif any(strcmpi(params(param_idx).season_name,{'2011_Greenland_P3','2012_Greenland_P3'})) + params = ct_set_params(params,'array.imgs',{[ones(1,15); 2:16].', [2*ones(1,15); 2:16].'}); + elseif any(strcmpi(params(param_idx).season_name,{'2014_Greenland_P3'})) + params = ct_set_params(params,'array.imgs',{[ones(1,15); 2:16].', [2*ones(1,15); 2:16].', [3*ones(1,15); 2:16].'}); + else + keyboard + end + % params = ct_set_params(params,'array.imgs',{[ones(1,7); 2:8].', [2*ones(1,7); 2:8].', [3*ones(1,7); 2:8].'}); + params = ct_set_params(params,'array.imgs',{[ones(1,7); 2:8].', [2*ones(1,7); 2:8].', [3*ones(1,7); 2:8].'}); + params(param_idx).array.dline = 1; + end + end +end +end diff --git a/cresis-toolbox/missions/run_basic_noise_analysis.m b/cresis-toolbox/missions/run_basic_noise_analysis.m index b78d36a5..3dc00403 100644 --- a/cresis-toolbox/missions/run_basic_noise_analysis.m +++ b/cresis-toolbox/missions/run_basic_noise_analysis.m @@ -13,31 +13,31 @@ %% RDS: MCORDS5 if strcmpi(radar_setup,'MCORDS5') - [param,defaults] = default_radar_params_2018_Greenland_Polar6_mcords; + [param,defaults] = default_radar_params_2022_Greenland_Polar5_rds; % .file_search_mode: Specify how to search for a file: 'last_file', % 'specific', 'default', or empty to be asked - param.file_search_mode = 'last_file'; + param.config.file_search_mode = 'last_file'; % .base_dir_search: cell vector of paths to search for data files - param.base_dir_search = {'D:\awi\','/mnt/AWI_SSD0/1604261101/UWB/','/mnt/AWI_SSD0/1604261202/UWB/'}; + param.config.base_dir_search = {'G:\20220603\','C:\rds\2022_Greenland_Polar5\20220512\','D:\awi\','\\192.168.1.100\D\AWI\'}; % .pdf_en: Enable time domain, probability density function, and quantization plots - param.pdf_en = false; + param.basic_noise_analysis.pdf_en = false; % .psd_en: Enable PSD plot - param.psd_en = true; + param.basic_noise_analysis.psd_en = true; % .img: wf-adc pair list which specifies which waveform-adc pairs to % analyze wf = 3; adcs = 1:8; - param.img = cat(2, wf*ones(length(adcs),1), adcs.'); + param.config.img = cat(2, wf*ones(length(adcs),1), adcs.'); % .recs: two element vector specifying which records/range-lines to load % [start_record num_records] - param.recs = [0 250]; + param.config.recs = [0 250]; % .presums: Number of additional software presums (coherent averaging) to do - param.presums = 1; + param.config.presums = 1; end %% RDS: MCORDS5 @@ -133,5 +133,3 @@ %% Automated Section basic_noise_analysis(param,defaults); - -return; diff --git a/cresis-toolbox/missions/run_preprocess_2022_Antarctica_Polar5.m b/cresis-toolbox/missions/run_preprocess_2022_Antarctica_Polar5.m new file mode 100644 index 00000000..ec0cd06b --- /dev/null +++ b/cresis-toolbox/missions/run_preprocess_2022_Antarctica_Polar5.m @@ -0,0 +1,57 @@ +% script run_preprocess.m +% +% Runs script preprocess.m +% +% Instructions: +% 1. Set your default radar parameters file +% 2. Set the input directory of the data files +% 3. Set the input directory of the config files if applicable +% 4. Run the script +% +% Author: John Paden + +%% User Setup +% ========================================================================= +param_override = []; param = []; + +run_preprocess_settings_2022_Antarctica_Polar5; % REPLACE THIS LINE WITH CORRECT SETTINGS SCRIPT + +dbstop if error; +param_override.config.reuse_tmp_files = false; +% param_override.cluster.type = 'torque'; +% param_override.cluster.type = 'matlab'; +param_override.cluster.type = 'debug'; +% param_override.cluster.rerun_only = true; +param_override.cluster.desired_time_per_job = 0*60; +% param_override.cluster.cpu_time_mult = 2; +% param_override.cluster.mem_mult = 2; + +%% Automated Section +% ===================================================================== + +% Input checking +global gRadar; +if exist('param_override','var') + param_override = merge_structs(gRadar,param_override); +else + param_override = gRadar; +end + +ctrl_chain = preprocess(param,param_override); + +cluster_print_chain(ctrl_chain); + +[chain_fn,chain_id] = cluster_save_chain(ctrl_chain); + +% Potentially stop and inspect cluster_print_chain output to adjust +% cluster control parameters before running or to run the next lines on a +% different computer (the save/load functions are for this purpose). + +return +%ctrl_chain = cluster_set_chain(ctrl_chain,'cluster.desired_time_per_job',5*60); +%ctrl_chain = cluster_set_chain(ctrl_chain,'cluster.cpu_time_mult',2); +%ctrl_chain = cluster_set_chain(ctrl_chain,'cluster.mem_mult',2); + +[ctrl_chain,chain_fn] = cluster_load_chain([],chain_id); +ctrl_chain = cluster_run(ctrl_chain); + diff --git a/cresis-toolbox/missions/run_preprocess_AWI.m b/cresis-toolbox/missions/run_preprocess_AWI.m deleted file mode 100644 index cf676d54..00000000 --- a/cresis-toolbox/missions/run_preprocess_AWI.m +++ /dev/null @@ -1,64 +0,0 @@ -% script run_preprocess_AWI.m -% -% Support script for run_preprocess.m - -param.config.default = []; - - -%% MCORDS3 SINGLE DAY -% cur_idx = length(param.config.default)+1; -% param.config.default{cur_idx} = default_radar_params_2018_Greenland_P3_rds(); -% param.config.base_dir{cur_idx} = '/N/dcwan/projects/cresis/2018_Greenland_P3/'; -% param.config.config_folder_names{cur_idx} = '20180405/mcords'; -% param.config.board_folder_names{cur_idx} = '20180405/mcords/%b'; -% param.config.date_strs{cur_idx} = '20180405'; - -%% SNOW8 SINGLE DAY -cur_idx = length(param.config.default)+1; -param.config.default{cur_idx} = default_radar_params_2019_Arctic_Polar6_snow(); -param.config.base_dir{cur_idx} = '/work/ollie/ajutila/Data/mnt/HDD5/'; -param.config.config_folder_names{cur_idx} = '1903080101/UWBM/'; -param.config.board_folder_names{cur_idx} = '1903080101/UWBM/%b'; -param.config.date_strs{cur_idx} = '20190308'; - -%% MCORDS5-ACCUM SINGLE DAY -% cur_idx = length(param.config.default)+1; -% param.config.default{cur_idx} = default_radar_params_2018_Greenland_P3_accum(); -% param.config.base_dir{cur_idx} = '/N/dcwan/projects/cresis/2018_Greenland_P3/'; -% param.config.config_folder_names{cur_idx} = '20180405/accum'; -% param.config.board_folder_names{cur_idx} = '20180405/accum/%b'; -% param.config.date_strs{cur_idx} = '20180405'; - -%return; - -%% MCORDS3 MULTIPLE DAYS -% date_strs = {'20180315','20180322','20180404','20180405','20180406','20180418','20180419','20180420','20180421','20180422','20180423','20180425','20180426','20180427','20180429','20180430','20180501'}; -% config_format_str = '%s/mcords/'; -% board_format_str = '%s/mcords/%%b'; -% defaults_fh = @default_radar_params_2018_Greenland_P3_rds; -% base_dir = '/N/dcwan/projects/cresis/2018_Greenland_P3/'; - -% for idx = 1:length(date_strs) - % cur_idx = length(param.config.default)+1; - % param.config.default{cur_idx} = defaults_fh(); - % param.config.base_dir{cur_idx} = base_dir; - % param.config.config_folder_names{cur_idx} = sprintf(config_format_str,date_strs{idx}); - % param.config.board_folder_names{cur_idx} = sprintf(board_format_str,date_strs{idx}); - % param.config.date_strs{cur_idx} = date_strs{idx}; -% end - -% %% SNOW8 MULTIPLE DAYS -% date_strs = {'1703301101','1704021401','1704041501','1704061601','1704082001','1704102101'}; -% config_format_str = '%s/UWBM/'; -% board_format_str = '%s/UWBM/%%b'; -% defaults_fh = @default_radar_params_2017_Arctic_Polar5_snow; -% base_dir = '/work/ollie/ajutila/Data/mnt/HDD5/'; -% -% for idx = 1:length(date_strs) -% cur_idx = length(param.config.default)+1; -% param.config.default{cur_idx} = defaults_fh(); -% param.config.base_dir{cur_idx} = base_dir; -% param.config.config_folder_names{cur_idx} = sprintf(config_format_str,date_strs{idx}); -% param.config.board_folder_names{cur_idx} = sprintf(board_format_str,date_strs{idx}); -% param.config.date_strs{cur_idx} = ['20' date_strs{idx}(1:6)]; -% end diff --git a/cresis-toolbox/missions/run_preprocess_BAS.m b/cresis-toolbox/missions/run_preprocess_BAS.m deleted file mode 100644 index d73882e2..00000000 --- a/cresis-toolbox/missions/run_preprocess_BAS.m +++ /dev/null @@ -1,13 +0,0 @@ -% script run_preprocess_BAS.m -% -% Support script for run_preprocess.m - -param.config.default = []; - -%% ACCUM3 TObas -cur_idx = length(param.config.default)+1; -param.config.default{cur_idx} = default_radar_params_2018_Antarctica_TObas_accum(); -param.config.base_dir{cur_idx} = '/data/'; -param.config.config_folder_names{cur_idx} = '20181004'; -param.config.board_folder_names{cur_idx} = '20181004/%b'; -param.config.date_strs{cur_idx} = '20181004'; diff --git a/cresis-toolbox/missions/run_preprocess_2013_Antarctica_Basler.m b/cresis-toolbox/missions/run_preprocess_settings_2013_Antarctica_Basler.m similarity index 100% rename from cresis-toolbox/missions/run_preprocess_2013_Antarctica_Basler.m rename to cresis-toolbox/missions/run_preprocess_settings_2013_Antarctica_Basler.m diff --git a/cresis-toolbox/missions/run_preprocess_settings_2018_Greenland_P3.m b/cresis-toolbox/missions/run_preprocess_settings_2018_Greenland_P3.m new file mode 100644 index 00000000..81df8c87 --- /dev/null +++ b/cresis-toolbox/missions/run_preprocess_settings_2018_Greenland_P3.m @@ -0,0 +1,80 @@ +% script run_preprocess_settings_2018_Greenland_P3.m +% +% Support script for run_preprocess.m + +param.config.default = []; + + +%% MCORDS3 SINGLE DAY +% cur_idx = length(param.config.default)+1; +% param.config.default{cur_idx} = @default_radar_params_2019_Antarctica_GV_rds; +% param.config.base_dir{cur_idx} = '/cresis/snfs1/data/MCoRDS/2019_Antarctica_GV/'; +% param.config.config_folder_names{cur_idx} = '20191009/'; +% param.config.board_folder_names{cur_idx} = '20191009/%b'; +% param.config.date_str{cur_idx} = '20191009'; + +%% SNOW8 SINGLE DAY +cur_idx = length(param.config.default)+1; +param.config.default{cur_idx} = @default_radar_params_2018_Greenland_P3_snow; +param.config.base_dir{cur_idx} = '/mnt/raid_ssdorange/2018_Greenland_P3/'; +param.config.config_folder_names{cur_idx} = '20180320'; +param.config.board_folder_names{cur_idx} = '20180320'; +param.config.date_str{cur_idx} = '20180320'; + +%% MCORDS5-ACCUM SINGLE DAY +% cur_idx = length(param.config.default)+1; +% param.config.default{cur_idx} = @default_radar_params_2018_Greenland_P3_accum; +% param.config.base_dir{cur_idx} = '/N/dcwan/projects/cresis/2018_Greenland_P3/'; +% param.config.config_folder_names{cur_idx} = '20180405/accum'; +% param.config.board_folder_names{cur_idx} = '20180405/accum/%b'; +% param.config.date_str{cur_idx} = '20180405'; + +% return; + +%% MCORDS3 MULTIPLE DAYS +% date_str = {'20180315','20180322','20180404','20180405','20180406','20180418','20180419','20180420','20180421','20180422','20180423','20180425','20180426','20180427','20180429','20180430','20180501'}; +% config_format_str = '%s/mcords/'; +% board_format_str = '%s/mcords/%%b'; +% defaults_fh = @default_radar_params_2018_Greenland_P3_rds; +% base_dir = '/N/dcwan/projects/cresis/2018_Greenland_P3/'; +% +% for idx = 1:length(date_str) +% cur_idx = length(param.config.default)+1; +% param.config.default{cur_idx} = defaults_fh(); +% param.config.base_dir{cur_idx} = base_dir; +% param.config.config_folder_names{cur_idx} = sprintf(config_format_str,date_str{idx}); +% param.config.board_folder_names{cur_idx} = sprintf(board_format_str,date_str{idx}); +% param.config.date_str{cur_idx} = date_str{idx}; +% end + +%% SNOW8 MULTIPLE DAYS +% date_str = {'20180322','20180403','20180404','20180405','20180406','20180407','20180408','20180414','20180416','20180418','20180419','20180421','20180422','20180423','20180425','20180426','20180427','20180429','20180430','20180501'}; +% config_format_str = '%s/fmcw/snow/'; +% board_format_str = '%s/fmcw/snow/'; +% defaults_fh = @default_radar_params_2018_Greenland_P3_snow; +% base_dir = '/N/dcwan/projects/cresis/2018_Greenland_P3/'; +% +% for idx = 1:length(date_str) +% cur_idx = length(param.config.default)+1; +% param.config.default{cur_idx} = defaults_fh(); +% param.config.base_dir{cur_idx} = base_dir; +% param.config.config_folder_names{cur_idx} = sprintf(config_format_str,date_str{idx}); +% param.config.board_folder_names{cur_idx} = sprintf(board_format_str,date_str{idx}); +% param.config.date_str{cur_idx} = date_str{idx}; +% end + +%% MCORDS5-ACCUM MULTIPLE DAYS +% date_str = {'20180315','20180404','20180405','20180406','20180418','20180419','20180420','20180421','20180422','20180423','20180425','20180426','20180427','20180429','20180430','20180501'}; +% config_format_str = '%s/accum/'; +% board_format_str = '%s/accum/%%b'; +% defaults_fh = @default_radar_params_2018_Greenland_P3_accum; +% base_dir = '/N/dcwan/projects/cresis/2018_Greenland_P3/'; +% +% for idx = 1:length(date_str) +% cur_idx = length(param.config.default)+1; +% param.config.default{cur_idx} = defaults_fh(); +% param.config.base_dir{cur_idx} = base_dir; +% param.config.config_folder_names{cur_idx} = sprintf(config_format_str,date_str{idx}); +% param.config.board_folder_names{cur_idx} = sprintf(board_format_str,date_str{idx}); +% param.config.date_str{cur_idx} = date_str{idx}; +% end diff --git a/cresis-toolbox/missions/run_preprocess_settings_2022_Antarctica_BaslerMKB.m b/cresis-toolbox/missions/run_preprocess_settings_2022_Antarctica_BaslerMKB.m new file mode 100644 index 00000000..78e0ec67 --- /dev/null +++ b/cresis-toolbox/missions/run_preprocess_settings_2022_Antarctica_BaslerMKB.m @@ -0,0 +1,42 @@ +% script run_preprocess_settings_2022_Antarctica_GroundGHOST.m +% +% Support script for run_preprocess.m +% +% Preprocess setup script for 2022_Antarctica_GroundGHOST. + +param.config.default = []; + +%% RDS SINGLE DAY +cur_idx = length(param.config.default)+1; +param.config.default{cur_idx} = @default_radar_params_2022_Antarctica_BaslerMKB_accum; +param.config.base_dir{cur_idx} = '/data/'; +% param.config.config_folder_names{cur_idx} = '20221225a'; +% param.config.board_folder_names{cur_idx} = '20221225a/%b'; +% param.config.date_str{cur_idx} = '20221224'; +% param.config.config_folder_names{cur_idx} = '20230110'; +% param.config.board_folder_names{cur_idx} = '20230110/%b'; +% param.config.date_str{cur_idx} = '20230110'; +% param.config.config_folder_names{cur_idx} = '20230114'; +% param.config.board_folder_names{cur_idx} = '20230114/%b'; +% param.config.date_str{cur_idx} = '20230114'; +% param.config.config_folder_names{cur_idx} = '20230115'; +% param.config.board_folder_names{cur_idx} = '20230115/%b'; +% param.config.date_str{cur_idx} = '20230115'; +% param.config.config_folder_names{cur_idx} = '20230116'; +% param.config.board_folder_names{cur_idx} = '20230116/%b'; +% param.config.date_str{cur_idx} = '20230116'; +% param.config.config_folder_names{cur_idx} = '20230125'; +% param.config.board_folder_names{cur_idx} = '20230125/%b'; +% param.config.date_str{cur_idx} = '20230125'; +% param.config.config_folder_names{cur_idx} = '20230126'; +% param.config.board_folder_names{cur_idx} = '20230126/%b'; +% param.config.date_str{cur_idx} = '20230126'; +% param.config.config_folder_names{cur_idx} = '20230127'; +% param.config.board_folder_names{cur_idx} = '20230127/%b'; +% param.config.date_str{cur_idx} = '20230127'; +param.config.config_folder_names{cur_idx} = '20230128'; +param.config.board_folder_names{cur_idx} = '20230128/%b'; +param.config.date_str{cur_idx} = '20230128'; + +% 20230109_191654_accum_digrx0_0113.dat + diff --git a/cresis-toolbox/missions/run_preprocess_settings_2022_Antarctica_BaslerMKB_UTIG.m b/cresis-toolbox/missions/run_preprocess_settings_2022_Antarctica_BaslerMKB_UTIG.m new file mode 100644 index 00000000..d92deaa2 --- /dev/null +++ b/cresis-toolbox/missions/run_preprocess_settings_2022_Antarctica_BaslerMKB_UTIG.m @@ -0,0 +1,110 @@ +% script run_preprocess_settings_2022_Antarctica_GroundGHOST.m +% +% Support script for run_preprocess.m +% +% Preprocess setup script for 2022_Antarctica_GroundGHOST. + +param.config.default = []; + +%% RDS SINGLE DAY +% cur_idx = length(param.config.default)+1; +% param.config.default{cur_idx} = @default_radar_params_2022_Antarctica_BaslerMKB_rds; +% param.config.base_dir{cur_idx} = '/data/UTIG_all/orig/xped/CXA1/acqn/MARFA/'; +% param.config.config_folder_names{cur_idx} = '../../ELSA/F14/'; +% param.config.board_folder_names{cur_idx} = 'F14'; +% param.config.date_str{cur_idx} = '20230123'; + +%% RDS EVERY DAY + +%good_flights = [3 4 11 12 14 15 16 19 20]; +%missing_flights = [17 18] +for flight = [1 2 5 6 7 8 9 10 13] + cur_idx = length(param.config.default)+1; + param.config.default{cur_idx} = @default_radar_params_2022_Antarctica_BaslerMKB_rds; + param.config.base_dir{cur_idx} = '/data/UTIG_all/orig/xped/CXA1/acqn/MARFA/'; + param.config.config_folder_names{cur_idx} = sprintf('../../ELSA/F%02d/', flight); + param.config.board_folder_names{cur_idx} = sprintf('F%02d', flight); + if flight == 1 + param.config.date_str{cur_idx} = '20221210'; + % utig_pkt_strip 7/35 /data/UTIG_all/orig/xped/CXA1/acqn/MARFA/F01/radar0_20221210-005833-0006.dat (01-May-2023 22:00:54) + % /scratch/ct_tmp/headers/rds/2022_Antarctica_BaslerMKB/F01/radar0_20221210-005833-0006.dat + % Error using basic_load_utig (line 216) + % Bad record + % Error in preprocess_task_utig (line 119) + % hdr = basic_load_utig(fn); + elseif flight == 2 + param.config.date_str{cur_idx} = '20221212'; + % utig_pkt_strip 14/49 /data/UTIG_all/orig/xped/CXA1/acqn/MARFA/F02/radar0_20221212-000310-0013.dat (01-May-2023 22:11:37) + % /scratch/ct_tmp/headers/rds/2022_Antarctica_BaslerMKB/F02/radar0_20221212-000310-0013.dat + % Error using basic_load_utig (line 216) + % Bad record + elseif flight == 3 + param.config.date_str{cur_idx} = '20221213'; + elseif flight == 4 + param.config.date_str{cur_idx} = '20221228'; + elseif flight == 5 + param.config.date_str{cur_idx} = '20221229'; + % utig_pkt_strip 15/44 /data/UTIG_all/orig/xped/CXA1/acqn/MARFA/F05/radar0_20221229-192430-0016.dat (04-May-2023 03:56:03) + % /scratch/ct_tmp/headers/rds/2022_Antarctica_BaslerMKB/F05/radar0_20221229-192430-0016.dat + % Error using basic_load_utig (line 216) + % Bad record + elseif flight == 6 + param.config.date_str{cur_idx} = '20230106'; + % line 129: if hdr{1}.rseq(idx) ~= hdr{3}.rseq(idx) + % preprocess_task_utig.m + elseif flight == 7 + param.config.date_str{cur_idx} = '20230107'; +% utig_pkt_strip 39/40 /data/UTIG_all/orig/xped/CXA1/acqn/MARFA/F07/radar0_20230107-191130-0044.dat (04-May-2023 04:36:51) +% /scratch/ct_tmp/headers/rds/2022_Antarctica_BaslerMKB/F07/radar0_20230107-191130-0044.dat +% Error using basic_load_utig (line 216) +% Bad record + elseif flight == 8 + param.config.date_str{cur_idx} = '20230108'; +% utig_pkt_strip 4/44 /data/UTIG_all/orig/xped/CXA1/acqn/MARFA/F08/radar0_20230108-215526-0004.dat (04-May-2023 11:02:01) +% /scratch/ct_tmp/headers/rds/2022_Antarctica_BaslerMKB/F08/radar0_20230108-215526-0004.dat +% Error using basic_load_utig (line 216) +% Bad record + elseif flight == 9 + param.config.date_str{cur_idx} = '20230109'; +% utig_pkt_strip 42/45 /data/UTIG_all/orig/xped/CXA1/acqn/MARFA/F09/radar0_20230109-205352-0041.dat (04-May-2023 11:41:28) +% /scratch/ct_tmp/headers/rds/2022_Antarctica_BaslerMKB/F09/radar0_20230109-205352-0041.dat +% Error using basic_load_utig (line 216) +% Bad record + elseif flight == 10 + param.config.date_str{cur_idx} = '20230110'; +% utig_pkt_strip 1/44 /data/UTIG_all/orig/xped/CXA1/acqn/MARFA/F10/radar0_20230110-203509-0001.dat (04-May-2023 13:34:28) +% /scratch/ct_tmp/headers/rds/2022_Antarctica_BaslerMKB/F10/radar0_20230110-203509-0001.dat +% Copy /data/UTIG_all/orig/xped/CXA1/acqn/MARFA/F10/../../ELSA/F10/serial* +% /scratch/metadata/2022_Antarctica_BaslerMKB/20230110/ELSA +% Error using basic_load_utig (line 216) +% Bad record + elseif flight == 11 + param.config.date_str{cur_idx} = '20230112'; + elseif flight == 12 + param.config.date_str{cur_idx} = '20230113'; + elseif flight == 13 + param.config.date_str{cur_idx} = '20230116'; + % utig_pkt_strip 2/45 /data/UTIG_all/orig/xped/CXA1/acqn/MARFA/F13/radar0_20230116-200145-0001.dat (04-May-2023 14:58:08) + % /scratch/ct_tmp/headers/rds/2022_Antarctica_BaslerMKB/F13/radar0_20230116-200145-0001.dat + % Warning: Variable 'comp_time' not found. + % > In preprocess_task_utig (line 111) + % In preprocess_task (line 29) + % In cluster_exec_task (line 122) + % In cluster_submit_job (line 194) + % In cluster_run (line 202) + % In cluster_run (line 99) + % Unrecognized function or variable 'comp_time'. + % Error in preprocess_task_utig (line 113) + % board_hdrs{1}.comp_time(end+(1:length(radar_time))) = comp_time; + elseif flight == 14 % CANCELLED FLIGHT? + param.config.date_str{cur_idx} = '20230117'; + elseif flight == 15 + param.config.date_str{cur_idx} = '20230120'; + elseif flight == 16 + param.config.date_str{cur_idx} = '20230124'; + elseif flight == 19 + param.config.date_str{cur_idx} = '20230127'; + elseif flight == 20 + param.config.date_str{cur_idx} = '20230129'; + end +end diff --git a/cresis-toolbox/missions/run_preprocess_settings_2022_Antarctica_Ground.m b/cresis-toolbox/missions/run_preprocess_settings_2022_Antarctica_Ground.m new file mode 100644 index 00000000..5881d3bd --- /dev/null +++ b/cresis-toolbox/missions/run_preprocess_settings_2022_Antarctica_Ground.m @@ -0,0 +1,29 @@ +% script run_preprocess_settings_2022_Antarctica_Ground.m +% +% Support script for run_preprocess.m +% +% Preprocess setup script for 2022_Antarctica_Ground. + +param.config.default = []; + +%% ACCUM3 SINGLE DAY +% cur_idx = length(param.config.default)+1; +% param.config.default{cur_idx} = @default_radar_params_2022_Antarctica_Ground_accum; +% param.config.base_dir{cur_idx} = '/data/2022_Antarctica_Ground/'; +% param.config.config_folder_names{cur_idx} = '20221207'; +% param.config.board_folder_names{cur_idx} = '20221207/%b'; +% param.config.date_str{cur_idx} = '20221207'; + + +%% ACCUM3 MULTIPLE DAYS +date_str = {'20221206', '20221207', '20221209', '20221221', '20230108', '20230109', '20230110', '20230111', '20230112', '20230113', '20230114', '20230115', '20230118', '20230119', '20230121'}; +% date_str = {'20230119'}; +for idx = 1:length(date_str) + cur_idx = length(param.config.default)+1; + param.config.default{cur_idx} = @default_radar_params_2022_Antarctica_Ground_accum; + param.config.base_dir{cur_idx} = '/data/2022_Antarctica_Ground/'; + param.config.config_folder_names{cur_idx} = date_str{idx}; + param.config.board_folder_names{cur_idx} = [date_str{idx} '/%b']; + param.config.date_str{cur_idx} = date_str{idx}; +end + diff --git a/cresis-toolbox/missions/run_preprocess_settings_2022_Antarctica_GroundGHOST.m b/cresis-toolbox/missions/run_preprocess_settings_2022_Antarctica_GroundGHOST.m new file mode 100644 index 00000000..f0b2fa3d --- /dev/null +++ b/cresis-toolbox/missions/run_preprocess_settings_2022_Antarctica_GroundGHOST.m @@ -0,0 +1,27 @@ +% script run_preprocess_settings_2022_Antarctica_GroundGHOST.m +% +% Support script for run_preprocess.m +% +% Preprocess setup script for 2022_Antarctica_GroundGHOST. + +param.config.default = []; + +%% RDS SINGLE DAY +% cur_idx = length(param.config.default)+1; +% param.config.default{cur_idx} = @default_radar_params_2022_Antarctica_GroundGHOST_rds; +% param.config.base_dir{cur_idx} = '/data/'; +% param.config.config_folder_names{cur_idx} = '20230202d'; +% param.config.board_folder_names{cur_idx} = '20230202d/%b'; +% param.config.date_str{cur_idx} = '20230202d'; + +%% RDS MULTIPLE DAYS +date_str = {'20230202', '20230203', '20230206', '20230207'}; +% date_str = {'20230119'}; +for idx = 1:length(date_str) + cur_idx = length(param.config.default)+1; + param.config.default{cur_idx} = @default_radar_params_2022_Antarctica_GroundGHOST_rds; + param.config.base_dir{cur_idx} = '/cresis/snfs1/data/MCoRDS/2022_Antarctica_GroundGHOST/'; + param.config.config_folder_names{cur_idx} = date_str{idx}; + param.config.board_folder_names{cur_idx} = [date_str{idx} '/%b']; + param.config.date_str{cur_idx} = date_str{idx}; +end \ No newline at end of file diff --git a/cresis-toolbox/missions/run_preprocess_settings_2022_Antarctica_Polar5.m b/cresis-toolbox/missions/run_preprocess_settings_2022_Antarctica_Polar5.m new file mode 100644 index 00000000..40e4d7ef --- /dev/null +++ b/cresis-toolbox/missions/run_preprocess_settings_2022_Antarctica_Polar5.m @@ -0,0 +1,22 @@ +% script run_preprocess_settings_AWI.m +% +% Support script for run_preprocess.m + +param.config.default = []; + +%% MCORDS5 AWI UWB - SINGLE DAY +% cur_idx = length(param.config.default)+1; +% param.config.default{cur_idx} = @default_radar_params_2022_Greenland_Polar5_rds; +% param.config.base_dir{cur_idx} = 'G:\'; +% param.config.config_folder_names{cur_idx} = '20220603'; +% param.config.board_folder_names{cur_idx} = fullfile('20220603','%b'); +% param.config.date_str{cur_idx} = '20220603'; + +%% SNOW8 SINGLE DAY +cur_idx = length(param.config.default)+1; +param.config.default{cur_idx} = @default_radar_params_2022_Antarctica_Polar5_snow; +param.config.base_dir{cur_idx} = '/albedo/work/user/ajutila/Data/'; +param.config.config_folder_names{cur_idx} = fullfile('2212030901'); +param.config.board_folder_names{cur_idx} = fullfile('2212030901','%b'); +param.config.date_str{cur_idx} = '20221203'; + diff --git a/cresis-toolbox/missions/run_preprocess_settings_2022_Greenland_Ground.m b/cresis-toolbox/missions/run_preprocess_settings_2022_Greenland_Ground.m new file mode 100644 index 00000000..947a6707 --- /dev/null +++ b/cresis-toolbox/missions/run_preprocess_settings_2022_Greenland_Ground.m @@ -0,0 +1,43 @@ +% script run_preprocess_settings_2022_Greenland_Ground.m +% +% Support script for run_preprocess.m +% +% Preprocess setup script for 2022_Greenland_Ground. + +param.config.default = []; + +%% ACCUM3 SINGLE DAY +cur_idx = length(param.config.default)+1; +param.config.default{cur_idx} = @default_radar_params_2022_Greenland_Ground_accum; +param.config.base_dir{cur_idx} = '/data/2022_Greenland_Ground/'; +param.config.config_folder_names{cur_idx} = '20220524'; +param.config.board_folder_names{cur_idx} = '20220524/%b'; +param.config.date_str{cur_idx} = '20220524'; + +% cur_idx = length(param.config.default)+1; +% param.config.default{cur_idx} = @default_radar_params_2022_Greenland_Ground_accum; +% param.config.base_dir{cur_idx} = '/data/2022_Greenland_Ground/'; +% param.config.config_folder_names{cur_idx} = '20220602'; +% param.config.board_folder_names{cur_idx} = '20220602/%b'; +% param.config.date_str{cur_idx} = '20220602'; +% +% cur_idx = length(param.config.default)+1; +% param.config.default{cur_idx} = @default_radar_params_2022_Greenland_Ground_accum; +% param.config.base_dir{cur_idx} = '/data/2022_Greenland_Ground/'; +% param.config.config_folder_names{cur_idx} = '20220607'; +% param.config.board_folder_names{cur_idx} = '20220607/%b'; +% param.config.date_str{cur_idx} = '20220607'; +% +% cur_idx = length(param.config.default)+1; +% param.config.default{cur_idx} = @default_radar_params_2022_Greenland_Ground_accum; +% param.config.base_dir{cur_idx} = '/data/2022_Greenland_Ground/'; +% param.config.config_folder_names{cur_idx} = '20220613'; +% param.config.board_folder_names{cur_idx} = '20220613/%b'; +% param.config.date_str{cur_idx} = '20220613'; +% +% cur_idx = length(param.config.default)+1; +% param.config.default{cur_idx} = @default_radar_params_2022_Greenland_Ground_accum; +% param.config.base_dir{cur_idx} = '/data/2022_Greenland_Ground/'; +% param.config.config_folder_names{cur_idx} = '20220627'; +% param.config.board_folder_names{cur_idx} = '20220627/%b'; +% param.config.date_str{cur_idx} = '20220627'; diff --git a/cresis-toolbox/missions/run_preprocess_settings_2022_Greenland_P3.m b/cresis-toolbox/missions/run_preprocess_settings_2022_Greenland_P3.m new file mode 100644 index 00000000..10b9b896 --- /dev/null +++ b/cresis-toolbox/missions/run_preprocess_settings_2022_Greenland_P3.m @@ -0,0 +1,13 @@ +% script run_preprocess_2022_Greenland_P3.m +% +% Support script for run_preprocess.m + +param.config.default = []; + +%% HERC MCORDS6 GROUND +cur_idx = length(param.config.default)+1; +param.config.default{cur_idx} = default_radar_params_2022_Greenland_P3_snow(); +param.config.base_dir{cur_idx} = '/mnt/data/'; +param.config.config_folder_names{cur_idx} = '20220419/'; +param.config.board_folder_names{cur_idx} = '20220419/%b'; +param.config.date_strs{cur_idx} = '20220419'; diff --git a/cresis-toolbox/missions/run_preprocess_settings_2023_Greenland_Ground.m b/cresis-toolbox/missions/run_preprocess_settings_2023_Greenland_Ground.m new file mode 100644 index 00000000..1ff69f56 --- /dev/null +++ b/cresis-toolbox/missions/run_preprocess_settings_2023_Greenland_Ground.m @@ -0,0 +1,15 @@ +% script run_preprocess_settings_2023_Greenland_Ground.m +% +% Support script for run_preprocess.m +% +% Preprocess setup script for 2023_Greenland_Ground. + +param.config.default = []; + +%% ACCUM3 SINGLE DAY +cur_idx = length(param.config.default)+1; +param.config.default{cur_idx} = @default_radar_params_2023_Greenland_Ground_accum; +param.config.base_dir{cur_idx} = '/data/'; +param.config.config_folder_names{cur_idx} = '20230407'; +param.config.board_folder_names{cur_idx} = '20230407/%b'; +param.config.date_str{cur_idx} = '20230407'; diff --git a/cresis-toolbox/missions/run_preprocess_settings_AWI.m b/cresis-toolbox/missions/run_preprocess_settings_AWI.m new file mode 100644 index 00000000..b66e3a80 --- /dev/null +++ b/cresis-toolbox/missions/run_preprocess_settings_AWI.m @@ -0,0 +1,21 @@ +% script run_preprocess_settings_AWI.m +% +% Support script for run_preprocess.m + +param.config.default = []; + +%% MCORDS5 AWI UWB - SINGLE DAY +cur_idx = length(param.config.default)+1; +param.config.default{cur_idx} = @default_radar_params_2022_Greenland_Polar5_rds; +param.config.base_dir{cur_idx} = 'G:\'; +param.config.config_folder_names{cur_idx} = '20220603'; +param.config.board_folder_names{cur_idx} = fullfile('20220603','%b'); +param.config.date_str{cur_idx} = '20220603'; + +%% SNOW8 SINGLE DAY +% cur_idx = length(param.config.default)+1; +% param.config.default{cur_idx} = default_radar_params_2019_Arctic_Polar6_snow(); +% param.config.base_dir{cur_idx} = '/work/ollie/ajutila/Data/mnt/HDD5/'; +% param.config.config_folder_names{cur_idx} = fullfile('1903080101','UWBM'); +% param.config.board_folder_names{cur_idx} = fullfile('1903080101','UWBM','%b'); +% param.config.date_str{cur_idx} = '20190308'; diff --git a/cresis-toolbox/missions/run_preprocess_settings_BAS.m b/cresis-toolbox/missions/run_preprocess_settings_BAS.m new file mode 100644 index 00000000..fd47ff98 --- /dev/null +++ b/cresis-toolbox/missions/run_preprocess_settings_BAS.m @@ -0,0 +1,34 @@ +% script run_preprocess_BAS.m +% +% Support script for run_preprocess.m +% +% Preprocess setup script for 2018_Antarctica_TObas and +% 2019_Antarctica_TObas. + +param.config.default = []; + +%% ACCUM3 SINGLE DAY +% cur_idx = length(param.config.default)+1; +% param.config.default{cur_idx} = default_radar_params_2018_Antarctica_TObas_accum(); +% param.config.base_dir{cur_idx} = '/cresis/snfs1/data/Accum_Data/2019_Antarctica_TObas/'; +% param.config.config_folder_names{cur_idx} = '20191225'; +% param.config.board_folder_names{cur_idx} = '20191225/%b'; +% param.config.date_strs{cur_idx} = '20191225'; + + +%% ACCUM3 MULTIPLE DAYS +% date_strs = {'20191215','20191222','20191225','20191226','20191229','20191230'}; +date_strs = {'20200125','20200126','20200127','20200128'}; +config_format_str = '%s/'; +board_format_str = '%s/%%b'; +defaults_fh = @default_radar_params_2018_Antarctica_TObas_accum; +base_dir = '/cresis/snfs1/data/Accum_Data/2019_Antarctica_TObas/'; + +for idx = 1:length(date_strs) + cur_idx = length(param.config.default)+1; + param.config.default{cur_idx} = defaults_fh(); + param.config.base_dir{cur_idx} = base_dir; + param.config.config_folder_names{cur_idx} = sprintf(config_format_str,date_strs{idx}); + param.config.board_folder_names{cur_idx} = sprintf(board_format_str,date_strs{idx}); + param.config.date_strs{cur_idx} = date_strs{idx}; +end diff --git a/cresis-toolbox/missions/run_preprocess_BAS_RDS.m b/cresis-toolbox/missions/run_preprocess_settings_BAS_RDS.m similarity index 100% rename from cresis-toolbox/missions/run_preprocess_BAS_RDS.m rename to cresis-toolbox/missions/run_preprocess_settings_BAS_RDS.m diff --git a/cresis-toolbox/missions/run_preprocess_DOME.m b/cresis-toolbox/missions/run_preprocess_settings_DOME.m similarity index 100% rename from cresis-toolbox/missions/run_preprocess_DOME.m rename to cresis-toolbox/missions/run_preprocess_settings_DOME.m diff --git a/cresis-toolbox/missions/run_preprocess_HERC.m b/cresis-toolbox/missions/run_preprocess_settings_HERC.m similarity index 100% rename from cresis-toolbox/missions/run_preprocess_HERC.m rename to cresis-toolbox/missions/run_preprocess_settings_HERC.m diff --git a/cresis-toolbox/missions/run_preprocess_MINISNOW.m b/cresis-toolbox/missions/run_preprocess_settings_MINISNOW.m similarity index 100% rename from cresis-toolbox/missions/run_preprocess_MINISNOW.m rename to cresis-toolbox/missions/run_preprocess_settings_MINISNOW.m diff --git a/cresis-toolbox/missions/run_preprocess_settings_N1KU.m b/cresis-toolbox/missions/run_preprocess_settings_N1KU.m new file mode 100644 index 00000000..a39095da --- /dev/null +++ b/cresis-toolbox/missions/run_preprocess_settings_N1KU.m @@ -0,0 +1,34 @@ +% script run_preprocess_N1KU.m +% +% Support script for run_preprocess.m + +param.config.default = []; + +%% SNOW SINGLE DAY +cur_idx = length(param.config.default)+1; +param.config.default{cur_idx} = default_radar_params_2020_SouthDakota_N1KU_snow(); +param.config.base_dir{cur_idx} = '/cresis/snfs1/data/SnowRadar/2020_SouthDakota_CESSNA/'; +param.config.base_dir{cur_idx} = '/run/media/cresis1/KUSNOW/'; +param.config.config_folder_names{cur_idx} = '20210209/gps'; +param.config.board_folder_names{cur_idx} = '20210209/snow'; +param.config.date_strs{cur_idx} = '20210209'; + +% return; + +%% SNOW MULTIPLE DAYS +% date_strs = {'20210204','20210205','20210209'}; +% date_strs = {'20210209'}; +% config_format_str = '%s/gps'; +% board_format_str = '%s/snow'; +% defaults_fh = @default_radar_params_2020_SouthDakota_N1KU_snow; +% %base_dir = '/cresis/snfs1/data/SnowRadar/2020_SouthDakota_N1KU/'; +% base_dir = '/run/media/cresis1/KUSNOW/'; +% +% for idx = 1:length(date_strs) +% cur_idx = length(param.config.default)+1; +% param.config.default{cur_idx} = defaults_fh(); +% param.config.base_dir{cur_idx} = base_dir; +% param.config.config_folder_names{cur_idx} = sprintf(config_format_str,date_strs{idx}); +% param.config.board_folder_names{cur_idx} = sprintf(board_format_str,date_strs{idx}); +% param.config.date_strs{cur_idx} = date_strs{idx}; +% end diff --git a/cresis-toolbox/missions/run_preprocess_OIB.m b/cresis-toolbox/missions/run_preprocess_settings_OIB.m similarity index 100% rename from cresis-toolbox/missions/run_preprocess_OIB.m rename to cresis-toolbox/missions/run_preprocess_settings_OIB.m diff --git a/cresis-toolbox/missions/run_preprocess_UTUA.m b/cresis-toolbox/missions/run_preprocess_settings_UTUA.m similarity index 100% rename from cresis-toolbox/missions/run_preprocess_UTUA.m rename to cresis-toolbox/missions/run_preprocess_settings_UTUA.m diff --git a/cresis-toolbox/missions/run_preprocess_settings_VANILLA.m b/cresis-toolbox/missions/run_preprocess_settings_VANILLA.m new file mode 100644 index 00000000..50e40f10 --- /dev/null +++ b/cresis-toolbox/missions/run_preprocess_settings_VANILLA.m @@ -0,0 +1,22 @@ +% script run_preprocess_VANILLA.m +% +% Support script for run_preprocess.m +% +% Preprocess setup script for 2021_Arctic_Vanilla. + +param.config.default = []; + +% Snow9 SINGLE DAY +% cur_idx = length(param.config.default)+1; +% param.config.default{cur_idx} = default_radar_params_2021_Arctic_Vanilla_snow(); +% param.config.base_dir{cur_idx} = '/cresis/snfs1/data/SnowRadar/2021_Arctic_Vanilla/'; +% param.config.config_folder_names{cur_idx} = '/20210414'; +% param.config.board_folder_names{cur_idx} = '/20210414'; +% param.config.date_strs{cur_idx} = '20210414'; + +cur_idx = length(param.config.default)+1; +param.config.default{cur_idx} = default_radar_params_2021_Arctic_Vanilla_snow(); +param.config.base_dir{cur_idx} = '/cresis/snfs1/data/SnowRadar/2021_Arctic_Vanilla/'; +param.config.config_folder_names{cur_idx} = '/20210819/sysconfig'; +param.config.board_folder_names{cur_idx} = '/20210819/data'; +param.config.date_strs{cur_idx} = '20210819'; \ No newline at end of file diff --git a/cresis-toolbox/posting/check_data_products.m b/cresis-toolbox/posting/check_data_products.m index 98f1c1f3..2a33d33e 100644 --- a/cresis-toolbox/posting/check_data_products.m +++ b/cresis-toolbox/posting/check_data_products.m @@ -177,7 +177,7 @@ fprintf(' Records %s\n', records_fn); if exist(records_fn,'file') try - records = load(records_fn); + records = records_load(param); if isfield(records,'records') fprintf(' Exists: %s\n', records.records.gps_source); else @@ -211,8 +211,7 @@ %% Check echogram outputs for output_idx = 1:length(outputs) - frames_fn = ct_filename_support(param,'','frames'); - load(frames_fn); + frames = frames_load(param); out_dir = fullfile(ct_filename_out(param,'','',1),['CSARP_' outputs_post_dir], ... ['CSARP_' outputs{output_idx}],param.day_seg); fprintf(' Output %s\n', out_dir); @@ -252,7 +251,7 @@ delete(fn); end else - if exist('gps_sources','var') && ~isempty(gps_sources) && ~strcmp(outputs{output_idx},'layerData') + if exist('gps_sources','var') && ~isempty(gps_sources) && ~strcmp(outputs{output_idx},'layer') if strcmp(outputs{output_idx},'CSARP_out') fns2 = get_filenames(fn,'','',''); fn = fns2{1}; @@ -315,7 +314,7 @@ %% Check for expected image files frames_fn = ct_filename_support(param,'','frames'); - load(frames_fn); + frames = load(frames_fn); for image_idx = 1:length(images) image_dir = fullfile(ct_filename_out(param, ... outputs_post_dir,'', true),'images',param.day_seg); diff --git a/cresis-toolbox/posting/check_nsidc_files.m b/cresis-toolbox/posting/check_nsidc_files.m new file mode 100644 index 00000000..de41108a --- /dev/null +++ b/cresis-toolbox/posting/check_nsidc_files.m @@ -0,0 +1,98 @@ +function [total_num,nc_num,supplement_num,SP_num,map_num,img_num,csv_num] = check_nsidc_files(param, param_override) +% [total_num,nc_num,supplement_num,SP_num,map_num,img_num] = check_nsidc_files(param, param_override) +% +% This function count and check the numbers of nsidc files generated by run_nsidc_delivery_script.m +% param = struct with processing parameters +% param_override = parameters in this struct will override parameters +% in param. This struct must also contain the gRadar fields. +% Typically global gRadar; param_override = gRadar; +% +% Example: +% See run_check_nsidc_files.m for how to run this function directly. +% Authors: Jilu Li + +param = merge_structs(param, param_override); + +if ~isfield(param,'nsidc_file_type') || isempty(param.nsidc_file_type) + param.nsidc_file_type = 'L1B'; +end + +fprintf('=====================================================================\n'); +fprintf('%s: %s (%s)\n', mfilename, param.day_seg, datestr(now)); +fprintf('=====================================================================\n'); + +frames = frames_load(param); +bad_frame_idxs = find(frames.proc_mode == 20); +echo_file_num = length(frames.frame_idxs); +if ~isempty(bad_frame_idxs) + echo_file_num = length(frames.frame_idxs) - length(bad_frame_idxs); +end +if strcmpi(param.nsidc_file_type,'L1B') + nsidc_SP_file_num_expected = 2*echo_file_num; % each frame has one premet and one spatial file + map_num = echo_file_num; + csv_num = 0; +elseif strcmpi(param.nsidc_file_type,'L2') + map_num = 0; + csv_num = 1; % each segment has one csv file + nsidc_SP_file_num_expected = 2*csv_num; +end +SP_num = nsidc_SP_file_num_expected; +nc_num = 0; +supplement_num = 0; +img_num = 0; +if strcmpi(param.radar_name,'snow8') % 2-18 GHz, with uwb,snow and kuband data products + nsidc_sensor_str = 'IRSNO1B'; + if regexpi(param.cmd.notes, '2-8 GHz') % 2-8 GHz settings + nc_num = 2*echo_file_num; % snow qlook and deconv only + img_num = echo_file_num; % one image (snow decon) + nsidc_file_num_expected = map_num + nc_num + img_num; + if regexpi(param.cmd.mission_names,'sea ice') + supplement_num = echo_file_num; % one supplement file + nsidc_file_num_expected = nsidc_file_num_expected + supplement_num; + end + else + nc_num = 4*echo_file_num; % uwb qlook and deconv, snow deconv and kuband deconv + img_num = 3*echo_file_num; % three echogram images (snow, uwb and kuband, all decon) + nsidc_file_num_expected = map_num + nc_num + img_num; + if regexpi(param.cmd.mission_names,'sea ice') + supplement_num = 3*echo_file_num; % three supplement files (snow,uwb and kuband) + nsidc_file_num_expected = nsidc_file_num_expected + supplement_num; + end + end +elseif strcmpi(param.radar_name,'mcords3') + if strcmpi(param.nsidc_file_type,'L1B') + nsidc_sensor_str = 'IRMCR1B'; + nc_num = echo_file_num; % standard or mvdr + img_num = 2*echo_file_num; % two images with/without picks + nsidc_file_num_expected = map_num + nc_num + img_num; + elseif strcmpi(param.nsidc_file_type,'L2') + nsidc_sensor_str = 'IRMCR2'; + nsidc_file_num_expected = csv_num; + end +else % other systems to be added +end +total_num = nsidc_file_num_expected; + +%% Check nsidc files +% ========================================================================= +nsidc_fns = get_filenames(param.nsidc_file_folder,nsidc_sensor_str,param.day_seg,''); +if length(nsidc_fns) == nsidc_file_num_expected + display('all files found') +elseif length(nsidc_fns) > nsidc_file_num_expected + display('got extra files!!!!!!') +else + display('missing files!!!!!!') +end +nsidc_SP_fns = get_filenames(param.nsidc_SP_file_folder,nsidc_sensor_str,param.day_seg,''); +if length(nsidc_SP_fns) == nsidc_SP_file_num_expected + display('all spatial and premet files found') +elseif length(nsidc_SP_fns) > nsidc_SP_file_num_expected + display('got extra spatial and premet files!!!!!!') +else + display('missing spatial and premet files!!!!!!') +end + + + + + diff --git a/cresis-toolbox/fmcw/check_images.m b/cresis-toolbox/posting/check_posted_images.m similarity index 67% rename from cresis-toolbox/fmcw/check_images.m rename to cresis-toolbox/posting/check_posted_images.m index 105c3ac8..fd13c619 100644 --- a/cresis-toolbox/fmcw/check_images.m +++ b/cresis-toolbox/posting/check_posted_images.m @@ -1,8 +1,27 @@ +% script check_posted_images +% +% Checks posted images (map and echogram). This is a script for quickly and +% conveniently checking a few CSARP_post images in each segment to ensure +% proper operation. Often it is better to view the images in the Windows +% file explorer since all images can be viewed quickly this way by setting +% the view setting to "view large icons" or similar, but this script is +% available when that is not an option. +% +% Author: John Paden + base_dir = '/N/dc/projects/cresis/output/snow/2012_Antarctica_DC8/CSARP_post/images/'; % base_dir = '/N/dc/projects/cresis/output/kuband/2012_Antarctica_DC8/CSARP_post/images/'; +% Set the search string here to select specific segments or all segments in +% the base_dir input_dirs = get_filenames(base_dir, '2012','','',struct('type','d')); +% Just look at a sample of num_images images per segment +num_images = 4; + +% ========================================================================= +% Automated Section + figure(1); clf; h_axes = axes; set(h_axes,'Position',[0 0 1 1]); @@ -19,8 +38,6 @@ fprintf('dir_idx = %d\n', dir_idx); input_dir = input_dirs{dir_idx}; fns = get_filenames(input_dir,'','','echo.jpg'); - % Just look at a sample of num_images images - num_images = 4; fn_idxs = unique(round(linspace(2,length(fns)-1,num_images))); % Look at every image: %fn_idxs = 1:length(fns); diff --git a/cresis-toolbox/utility/create_picking_spreadsheet.m b/cresis-toolbox/posting/create_picking_spreadsheet.m similarity index 91% rename from cresis-toolbox/utility/create_picking_spreadsheet.m rename to cresis-toolbox/posting/create_picking_spreadsheet.m index a9a4845b..d79cad44 100644 --- a/cresis-toolbox/utility/create_picking_spreadsheet.m +++ b/cresis-toolbox/posting/create_picking_spreadsheet.m @@ -20,9 +20,15 @@ function create_picking_spreadsheet(param_fn,xls_fn_dir,pick_param) % % EXAMPLE: % param_fn = ct_filename_param('rds_param_2013_Antarctica_P3.xls'); -% xls_fn_dir = 'C:\tmp\'; +% xls_fn_dir = 'C:\'; % create_picking_spreadsheet(param_fn,xls_fn_dir); % +% params = read_param_xls(ct_filename_param('accum_param_2019_Antarctica_TObas.xls')); +% params = ct_set_params(params,'cmd.generic',1); +% params = ct_set_params(params,'cmd.generic',0,'cmd.notes','do not process'); +% xls_fn_dir = '~/'; +% create_picking_spreadsheet(params,xls_fn_dir); +% % Author: John King % % See also: read_param_xls.m @@ -39,7 +45,11 @@ function create_picking_spreadsheet(param_fn,xls_fn_dir,pick_param) end %% Read in the param spreadsheet -params = read_param_xls(param_fn); +if ischar(param_fn) + params = read_param_xls(param_fn); +else + params = param_fn; +end %% Create Excel spreadsheet output_dir = ct_output_dir(params(1).radar_name); @@ -80,8 +90,7 @@ function create_picking_spreadsheet(param_fn,xls_fn_dir,pick_param) continue; end - frames_fn = ct_filename_support(param,'','frames'); - load(frames_fn); % Load frames into variable "frames" + frames = frames_load(param); % Load frames into variable "frames" if strcmpi(pick_param.mode,'ascii') for frm = 1:length(frames.frame_idxs) diff --git a/cresis-toolbox/posting/echo_plot.m b/cresis-toolbox/posting/echo_plot.m new file mode 100644 index 00000000..072afb62 --- /dev/null +++ b/cresis-toolbox/posting/echo_plot.m @@ -0,0 +1,155 @@ +function [mdata,x_axis,y_axis,surf_comp,layers_comp,h] = echo_plot(echo_fn,echo_plot_param,layer_params) +% [mdata,x_axis,y_axis,surf_comp,layers_comp,h] = echo_plot(echo_fn,echo_plot_param,layer_params) +% +% Function for loading and plotting data with layers using +% elevation_compensation.m +% +% Plots data based on echo_plot_param. Loads and plots layers based on +% layer_params. +% +% Inputs: +% ========================================================================= +% +% echo_fn: string containing an echogram filename or an echogram structure +% (e.g. loaded with echo_load.m) +% +% echo_plot(echo_fn,echo_plot_param,layer_params) +% +% echo_plot_param: See echo_plot_profile.m. Can be either a profile string +% or a parameter structure. +% +% layer_params: See layerdata.m profile() static function. Can be either a +% profile string or a opsLoadLayers.m parameter structure. +% +% Outputs: +% ========================================================================= +% +% data,x_axis,y_axis,surf_comp,layers_comp: From elevation_compensation.m +% +% h: figure, axes, image, and plot handles created or used by this function +% +% Examples: See run_echo_plot.m +% +% Author: John Paden, Dhagash Kapadia +% +% See also: echo_plot.m, echo_plot_profile.m, elevation_compensation.m, +% load_L1B.m, run_echo_plot.m + + +%% Input Checks +% ========================================================================= + +if ~exist('echo_plot_param', 'var') || isempty(echo_plot_param) + echo_plot_param = []; +end + +echo_plot_param = echo_plot_profile(echo_plot_param); + +if ~exist('layer_params', 'var') + layer_params = []; +end + +layer_params = layerdata.profile(layer_params); + +%% Load Data +% ========================================================================= +if isstruct(echo_fn) + mdata = echo_fn; +else + mdata = echo_load(echo_fn); +end + +%% Load Layers +% ========================================================================= +layers_twtt = {}; +if ~isempty(layer_params) + param = echo_param(mdata); + % Determine the range of frames that this echogram covers + [~,frm_id] = get_frame_id(param,mdata.GPS_time([1 end])); + % Get the frame before/after so that layers will be interpolated + % correctly. + param.cmd.frms = floor(frm_id(1))-1 : floor(frm_id(end))+1; + % Load the layers + [layers] = opsLoadLayers(param,layer_params); + % Format each layer + for lay_idx = 1:length(layers) + ops_layer = []; + ops_layer{1}.gps_time = layers(lay_idx).gps_time; + ops_layer{1}.type = layers(lay_idx).type; + ops_layer{1}.quality = layers(lay_idx).quality; + ops_layer{1}.twtt = layers(lay_idx).twtt; + ops_layer{1}.type(isnan(ops_layer{1}.type)) = 2; + ops_layer{1}.quality(isnan(ops_layer{1}.quality)) = 1; + lay = opsInterpLayersToMasterGPSTime(mdata,ops_layer,[300 60]); + layers_twtt{lay_idx} = lay.layerData{1}.value{2}.data; + layers_twtt{lay_idx} = interp_finite(layers_twtt{lay_idx}, 0); + end +end + +%% Elevation compensation +% ========================================================================= +[mdata,x_axis,y_axis,surf_comp,layers_comp] = elevation_compensation(mdata,echo_plot_param,layers_twtt); + +%% Plot results +% ========================================================================= +% h: output GUI handles +h = []; +if ~echo_plot_param.plot_en + return; +end +if isempty(echo_plot_param.h_fig) + h.fig = get_figures(1,true); +else + h.fig = echo_plot_param.h_fig; + if ~ishandle(h.fig) + figure(h.fig); + end +end +clf(h.fig); +h.axes = axes('parent',h.fig); +if ~isreal(mdata.Data) + % Complex (voltage data are always represented in complex baseband format) + h.image = imagesc(x_axis, y_axis, db(mdata.Data), 'parent', h.axes); +else + % Linear power + h.image = imagesc(x_axis, y_axis, db(mdata.Data,'power'), 'parent', h.axes); +end +colormap(h.axes,1-gray(256)); + +%% Plot x-axis +% ========================================================================= +if strcmpi(echo_plot_param.mode_x_axis, 'range_line') + xlabel(h.axes,'Range line'); +elseif strcmpi(echo_plot_param.mode_x_axis, 'gps_time') + xlabel(h.axes,'GPS time (sec)'); +elseif strcmpi(echo_plot_param.mode_x_axis, 'along_track') + xlabel(h.axes,'Along track (km)'); +end + +%% Plot y-axis +% ========================================================================= +if strcmpi(echo_plot_param.mode_y_axis, 'TWTT') + ylabel(h.axes,'Two way travel time (sec)') + set(h.axes,'YDir','reverse') + +elseif strcmpi(echo_plot_param.mode_y_axis,'RANGE') + ylabel(h.axes,'Range (m)') + set(h.axes,'YDir','reverse') + +elseif strcmpi(echo_plot_param.mode_y_axis, 'WGS84') + ylabel(h.axes,'WGS-84 (m)') + set(h.axes,'YDir','normal') + +elseif strcmpi(echo_plot_param.mode_y_axis, 'DEPTH') + ylabel(h.axes,'Depth (m)'); + set(h.axes,'YDir','reverse') +end + +%% Plot layers +% ========================================================================= +hold(h.axes,'on'); +h.plot = []; +for layer_idx = 1:length(layers_comp) + h.plot(layer_idx) = plot(h.axes,x_axis,layers_comp{layer_idx}); +end +hold(h.axes,'off') diff --git a/cresis-toolbox/posting/echo_plot_profile.m b/cresis-toolbox/posting/echo_plot_profile.m new file mode 100644 index 00000000..6da52f49 --- /dev/null +++ b/cresis-toolbox/posting/echo_plot_profile.m @@ -0,0 +1,93 @@ +function echo_plot_param = echo_plot_profile(profile_str) +% echo_plot_param = echo_plot_profile(profile_str) +% +% Converts echo_plot profile_str string into corresponding echo_plot_param +% structure. If profile_str is a struct, then this struct is used. Either +% way, input checks are done on the structure. +% +% Inputs: +% ========================================================================= +% +% echo_plot_param: A structure or string. If a string, it should contain +% the profile name (e.g. 'TWTT', 'WGS84', 'DEPTH', or 'RANGE'). If a +% struct, it should be an echo_plot echo_plot_param structure. +% +% Outputs: +% ========================================================================= +% +% echo_plot_param: structure containing parameters for echo_plot.m. If +% profile_str is a string, then this struct contains the default values for +% the profile selected. See "Input Checks" section for details. Structure also +% contains elevation_compensation "param" input fields. See that function +% for details on those fields. +% +% Examples: +% echo_plot_param = echo_plot_profile('DEPTH'); +% +% Author: John Paden, Dhagash Kapadia +% +% See also: echo_plot.m, echo_plot_profile.m, elevation_compensation.m, +% load_L1B.m, run_echo_plot.m + +%% echo_plot_param profile fill +% ========================================================================= +if isempty(profile_str) + profile_str = 'TWTT'; +end +if ischar(profile_str) + + echo_plot_param = []; + + if strcmpi(profile_str, 'DEPTH') + echo_plot_param.mode_y_axis = 'DEPTH'; + + elseif strcmpi(profile_str, 'RANGE') + echo_plot_param.mode_y_axis = 'range'; + + elseif strcmpi(profile_str, 'NONE') + echo_plot_param.mode_y_axis = 'twtt'; + echo_plot_param.plot_en = false; + + elseif strcmpi(profile_str, 'TWTT') + echo_plot_param.mode_y_axis = 'twtt'; + + elseif strcmpi(profile_str, 'WGS84') + echo_plot_param.mode_y_axis = 'wgs84'; + end + +elseif ~isstruct(profile_str) + error('echo_plot_param input must be a string or structure.'); + +else + % Must be a struct which we copy into echo_plot_param + echo_plot_param = profile_str; +end + +%% Input checks: echo_plot specific +% ========================================================================= +% elevation_compensation.m specific parameters (input "param") are stored +% in this strcuture, but that function will do input checks on those +% parameters + +% h_fig: figure handle to use in echo_plot +if ~isfield(echo_plot_param, 'h_fig') || isempty(echo_plot_param.h_fig) + echo_plot_param.h_fig = []; +end + +% .mode_x_axis: String specifying the x-axis type. The options are +% 'ALONG_TRACK', 'GPS_TIME', or 'RANGE_LINE'. +if ~isfield(echo_plot_param, 'mode_x_axis') || isempty(echo_plot_param.mode_x_axis) + echo_plot_param.mode_x_axis = 'RANGE_LINE'; +end + +% .mode_y_axis: String specifying the y-axis type. The options are 'DEPTH', +% 'RANGE', 'TWTT', or 'WGS84'. +if ~isfield(echo_plot_param, 'mode_y_axis') || isempty(echo_plot_param.mode_y_axis) + echo_plot_param.mode_y_axis = 'TWTT'; +end + +% plot_en: logical to enable plotting or not, default is true. If false, +% echo_plot just loads the variables and prepares them for plotting. +if ~isfield(echo_plot_param, 'plot_en') || isempty(echo_plot_param.plot_en) + echo_plot_param.plot_en = true; +end diff --git a/cresis-toolbox/posting/echogram_to_jpeg.m b/cresis-toolbox/posting/echo_to_jpeg.m similarity index 99% rename from cresis-toolbox/posting/echogram_to_jpeg.m rename to cresis-toolbox/posting/echo_to_jpeg.m index cb1d8214..99068358 100644 --- a/cresis-toolbox/posting/echogram_to_jpeg.m +++ b/cresis-toolbox/posting/echo_to_jpeg.m @@ -92,7 +92,7 @@ end % Remove frames that do not exist from param.cmd.frms list -load(ct_filename_support(param,'','frames')); +frames = frames_load(param); if isempty(param.cmd.frms) param.cmd.frms = 1:length(frames.frame_idxs); end diff --git a/cresis-toolbox/posting/elevation_compensation.m b/cresis-toolbox/posting/elevation_compensation.m index d2d10b9c..3368d39d 100644 --- a/cresis-toolbox/posting/elevation_compensation.m +++ b/cresis-toolbox/posting/elevation_compensation.m @@ -1,237 +1,584 @@ -function [mdata,depth_axis] = elevation_compensation(mdata,param) -% function [mdata,depth_axis] = elevation_compensation(mdata,param) -% -% mdata = L1B data structure from load_L1B.m. Requires -% .Data is Nt x Nx matrix -% .Time is Nt x 1 vector -% .Elevation is 1 x Nx vector -% .Surface is 1 x Nx vector -% param = structure controlling compensation -% .update_surf = logical, default false, if true the function uses -% threshold and sidelobe fields to track the surface again -% This will cause "mdata.Surface" to be updated. -% .filter_surf = logical, default false, applies an along-track median -% filter to the surface. -% This will cause "mdata.Surface" to be updated. -% .threshold = used with update_surf (specified in power magnitude -% assuming mdata.Data is in power magnitude units), this threshold -% will be used for tracking the surface. Default is estimated from -% the data. -% .sidelobe = special case which helps the tracker avoid tracking side -% lobes at the relative power specified or below. Actual threshold -% used is: -% max of param.threshold and max_value_of_current_rangeline*param.sidelobe -% Default is "10^(-13/10) -% .elev_comp = elevation compensation mode (same as publish_echogram.m) -% .er_ice = default is 3.15, vector of dielectrics -% .er_depth = default is 0, depth vector to align with er_ice, first element -% must always be zero -% .depth = string to be evaluated in the calculation of depth_axis. -% This specifies which WGS-84 elevation bins to include in case a -% subset of the data is desired. A vector "Surface_Elev" is available -% which contains the ice surface elevation at each range line. -% DEFAULT: '[-inf inf]'; -% EXAMPLE: '[min(Surface_Elev)-600 max(Surface_Elev)+20]'; -% EXAMPLE: '[1200 1800]'; - -% mdata = updated data structure -% .Data = updated to be gridded on constant WGS-84 axes -% .Elevation, .Surface_Elev = updated to account for compensation -% of .Data on constant grid (only for mode 3) -% depth_axis = Depends on elevation compensation mode -% mode 2: depth axis -% mode 3: indices specified by param.depth -% -% Examples: See plot_L1B.m -% -% Author: John Paden -% -% See also: load_L1B.m, plot_L1B.m +function [mdata,x_axis,y_axis,surf_comp,layers_comp] = elevation_compensation(mdata,param,layers_twtt) +% function [data,x_axis,y_axis,surf_comp,layers_comp] = elevation_compensation(mdata,param,layers_twtt) +% +% Inputs radar echogram (mdata.Data) and optional layers (layers_twtt) and +% creates a reinterpolated output image and layers onto a "WGS-84" +% elevation axis or a "depth" axis. +% +% Inputs +% ========================================================================= +% +% mdata: L1B data structure from load_L1B.m. Requires +% +% .Data: Nt x Nx numeric matrix +% +% .Time: is Nt x 1 numeric vector +% +% .Elevation: 1 x Nx numeric vector +% +% .Surface: 1 x Nx numeric vector (an air-ice surface is required) +% +% param: structure controlling how elevation compensation is done. See +% "Input Checks" section for details. +% +% layers_twtt: cell array of numeric vectors. Each numeric vector contains +% the two way travel time to a layer. +% +% Outputs +% ========================================================================= +% +% mdata: The input L1B data structure with resampled variables: +% +% .Data will be Nt_out by Nx_out output matrix sampled at the specified +% x-axis and y-axis units +% +% .GPS_time, .Lat, .Lon, .Elev, .Roll, .Pitch, .Heading, .Surface, +% .Bottom: all Nx_out by 1 +% +% .Time: Remains unchanged, but no longer will match the data matrix if +% y-axis resampling is done. Use y_axis instead. +% +% x_axis: 1 by Nx_out numeric vector holding the x-axis corresponding to +% the row dimension of data output +% +% y_axis: Nt_out by 1 numeric vector holding the y-axis corresponding to +% the column dimension of data output +% +% surf_comp: 1 by Nx_out numeric vector holding the surface in y-axis units +% +% layers_comp: cell array of layers in y-axis units; corresponds with +% layers_twtt. Each cell array contains a 1 by Nx_out numeric vector. +% +% Examples: +% +% param = read_param_xls(ct_filename_param('snow_param_2012_Greenland_P3.xls'),'20120330_04'); +% [mdata,echo_fn] = echo_load(param,'CSARP_post/qlook',3); +% [data,x_axis,y_axis] = elevation_compensation(mdata,echo_plot_profile('DEPTH')); +% imagesc(x_axis,y_axis,10*log10(data)); +% +% % Also see echo_plot.m for examples +% +% Author: John Paden, Dhagash Kapadia +% +% See also: echo_plot.m, echo_plot_profile.m, elevation_compensation.m, +% load_L1B.m, run_echo_plot.m + +%% Input checks + +physical_constants; if ~exist('param','var') param = []; end -if ~isfield(param,'update_surf') - param.update_surf = false; +% .er_depth: Numeric vector of same length as er_ice field and aligns with +% the same. Default value is 0. +if ~isfield(param, 'er_depth') || isempty(param.er_depth) + param.er_depth = 0; end +% Force param.er_depth to be a column vector +param.er_depth = param.er_depth(:); -if ~isfield(param,'filter_surf') - param.filter_surf = false; +% .er_freq: Scalar numeric. Frequency to evaluate the permittivity at to +% determine loss. Only needed when the imaginary part of er_ice is +% non-zero. +if ~isfield(param,'er_freq') + param.er_freq = 150e6; end -if ~isfield(param,'threshold') - param.threshold = median(max(mdata.Data)) / 10; +% .er_ice: Nonempty numeric vector. Default value is a scalar pulled from +% physical_constants "er_ice" variable. Represents the relative +% permittivity of the medium at each depth specified in er_depth. A value +% of 1 represents a vacuum. +if ~isfield(param, 'er_ice') || isempty(param.er_ice) + param.er_ice = er_ice; end +% Force param.er_ice to be a column vector +param.er_ice = param.er_ice(:); -if ~isfield(param,'sidelobe') - param.sidelobe = 10^(-13/10); +% .mode_x_axis: String specifying the x-axis type. The options are +% 'ALONG_TRACK', 'GPS_TIME', or 'RANGE_LINE'. +% +% 'ALONG_TRACK': interpolates onto uniform along-track spatial sampling. +% +% 'GPS_TIME': interpolates onto uniform gps time sampling. +% +% 'RANGE_LINE': Default mode. Data are assumed to be uniformily sampled +% in range lines and no interpolation occurs. +if ~isfield(param, 'mode_x_axis') || isempty(param.mode_x_axis) + param.mode_x_axis = 'RANGE_LINE'; end -if ~isfield(param,'er_depth') - param.er_depth = 0; +% .mode_y_axis: String specifying the y-axis type. The options are 'DEPTH', +% 'RANGE', 'TWTT', or 'WGS84'. +% +% 'DEPTH': flattens the surface and interpolates onto a constant depth +% grid where the surface is zero depth. +% +% 'RANGE': interpolates onto range grid (each row is a constant range +% from the radar). +% +% 'TWTT': Default mode. Data are assumed to be uniformily sampled in time +% and no interpolation occurs. +% +% 'WGS84': interpolates onto WGS-84 elevation grid (each row is a +% constant WGS-84 value). +if ~isfield(param, 'mode_y_axis') || isempty(param.mode_y_axis) + param.mode_y_axis = 'TWTT'; end -param.er_depth = reshape(param.er_depth,[length(param.er_depth) 1]); -if ~isfield(param,'er_ice') - param.er_ice = 3.15; +% .surf: fields dealing with surface +if ~isfield(param,'surf') || isempty(param.surf) + param.surf = []; end -param.er_ice = reshape(param.er_ice,[length(param.er_ice) 1]); -if ~isfield(param,'er_freq') - param.er_freq = 150e6; +% .filter_en: enables low pass filtering of the surface to reduce high +% frequency content distorting the interpolation result. Useful when +% surface has a lot of high frequency content. +if ~isfield(param.surf,'filter_en') || isempty(param.surf.filter_en) + param.surf.filter_en = false; end -if ~isfield(param,'depth') - param.depth = '[-inf inf]'; +% .filter_percent: scalar numeric, sets the filtering length, default is +% 0.05 or 5% of the whole dataset/range-line axis +if ~isfield(param.surf,'filter_percent') || isempty(param.surf.filter_percent) + param.surf.filter_percent = 0.05; end -physical_constants; +% surf.source: source of the surface (0 to use mdata.Surface) +if ~isfield(param.surf,'source') || isempty(param.surf.source) + param.surf.source = 0; +end + +% surf.update_en: logical to update the surface using layer_tracker_task.m, +% useful when the surface is missing or poorly tracked +if ~isfield(param.surf,'update_en') || isempty(param.surf.update_en) + param.surf.update_en = false; +end + +% surf.update_params: layer_tracker parameters to be used with +% layer_tracker_task.m +% NOT IMPLEMENTED + +% .trim_nan_en: logical scalar, default true, if true, it removes rows that +% are all NaN from the top and bottom of the image. +if ~exist('param.trim_nan_en','var') || isempty(param.trim_nan_en) + param.trim_nan_en = true; +end + +% .ylims_cmd: Matlab command string to be evaluated in the calculation of +% y_axis. This specifies which bins to include in case a subset of the +% data is desired. Several variables are made available to use with the +% ylims_cmd command: +% +% .elev: elevation 1 by Nx vector +% +% .layers_comp: Cell array that corresponds to layers_twtt in y-axis units +% +% .surf_comp: Numeric vector of surface location in y-axis units +% +% Default value: '[-inf inf]' +% +% Example for 'depth' mode_y_axis: '[-2 20]'; +% +% Example for 'wgs84' mode_y_axis: '[min(surf_comp)-600 max(surf_comp)+20]'; +if ~isfield(param,'ylims_cmd') || isempty(param.ylims_cmd) + param.ylims_cmd = '[-inf inf]'; +end + +if ~exist('layers_twtt','var') + layers_twtt = {}; +end + +%% Setup -if param.update_surf +Nt = size(mdata.Data,1); +Nx = size(mdata.Data,2); + +%% Get surface +if param.surf.source == 0 + if isfield(mdata,'Surface') + surf = interp_finite(mdata.Surface,0); + else + surf = zeros(1,Nx); + end +else + surf = interp_finite(layers_twtt{param.surf.source},0); +end + +if param.surf.update_en %% Update surface: + + surf_param = param; + surf_param.layer_tracker.frms = frm; + surf_param.layer_tracker.echogram_source = struct('Data',Data_Surface,'Time',Time_Surface,'GPS_time',GPS_time,'Latitude',Latitude,'Longitude',Longitude,'Elevation',Elevation,'Roll',Roll); + if length(Time_Surface) < 2 + % This frame has all bad records, so surface tracking cannot be completed. + Surface = nan(size(GPS_time)); + else + Surface = layer_tracker_task(surf_param); + end + + + surf_bin = zeros(1,size(mdata.Data,2)); for rline = 1:size(mdata.Data,2) new_surf = find(mdata.Data(:,rline) ... > max(param.threshold,max(mdata.Data(:,rline))*param.sidelobe),1); if isempty(new_surf) - mdata.Surface_Bin(rline) = NaN; + surf_bin(rline) = NaN; else - mdata.Surface_Bin(rline) = new_surf; + surf_bin(rline) = new_surf; end end - mdata.Surface = interp1(1:length(mdata.Time),mdata.Time,mdata.Surface_Bin); - nan_mask = isnan(mdata.Surface); - mdata.Surface = interp1(find(~nan_mask),mdata.Surface(~nan_mask),1:length(mdata.Surface)); + layers_twtt{1} = interp1(1:length(mdata.Time),mdata.Time, surf_bin); + nan_mask = isnan(layers_twtt{1}); + layers_twtt{1} = interp1(find(~nan_mask),layers_twtt{1}(~nan_mask),1:length(layers_twtt{1})); end -if param.filter_surf + +if param.surf.filter_en %% Filter surface - if length(mdata.Surface) >= 100 - [Bfilt,Afilt] = butter(2,0.02); - surf_filt = filtfilt(Bfilt,Afilt,mdata.Surface); - tmp = polyval(polyfit(1:51,reshape(mdata.Surface(1:51),[1 51]),2),1:51); + if length(layers_twtt{1}) >= 100 + [Bfilt,Afilt] = butter(2,length(layers_twtt{1}) * param.surf.filter_percent); + surf_filt = filtfilt(Bfilt,Afilt,layers_twtt{1}); + tmp = polyval(polyfit(1:51,reshape(layers_twtt{1}(1:51),[1 51]),2),1:51); surf_filt(1:50) = tmp(1:50) - tmp(51) + surf_filt(51); - tmp = polyval(polyfit(1:51,reshape(mdata.Surface(end-50:end),[1 51]),2),1:51); + tmp = polyval(polyfit(1:51,reshape(layers_twtt{1}(end-50:end),[1 51]),2),1:51); surf_filt(end-49:end) = tmp(2:51) - tmp(1) + surf_filt(end-50); - mdata.Surface = surf_filt; + layers_twtt{1} = surf_filt; else - surf_filt = medfilt1(mdata.Surface,round(length(mdata.Surface)/20)*2+1); - mdata.Surface = surf_filt; + surf_filt = medfilt1(layers_twtt{1},round(length(layers_twtt{1})/20)*2+1); + layers_twtt{1} = surf_filt; + end +else + surf_filt = surf; +end + +%% Resample data in x-dimension as required +if strcmpi(param.mode_x_axis,'along_track') + % Resample data to be uniformily sampled in along_track + along_track = geodetic_to_along_track(mdata); + dx = median(diff(along_track)); + x_axis = along_track(1) : dx : along_track(end); + mdata.Data = interp1(along_track,mdata.Data.',x_axis).'; + surf_filt = interp1(along_track,surf_filt,x_axis); + Nx = length(x_axis); + elev = interp1(along_track,mdata.Elevation,x_axis); + for layer_idx = 1:length(layers_twtt) + layers_twtt{layer_idx} = interp1(along_track,layers_twtt{layer_idx},x_axis); + end + if isfield(mdata,'Latitude') + mdata.Latitude = interp1(along_track,mdata.Latitude,x_axis); + end + if isfield(mdata,'Longitude') + mdata.Longitude = interp1(along_track,mdata.Longitude,x_axis); end + if isfield(mdata,'Elevation') + mdata.Elevation = interp1(along_track,mdata.Elevation,x_axis); + end + if isfield(mdata,'Roll') + mdata.Roll = interp1(along_track,mdata.Roll,x_axis); + end + if isfield(mdata,'Pitch') + mdata.Pitch = interp1(along_track,mdata.Pitch,x_axis); + end + if isfield(mdata,'Heading') + mdata.Heading = interp1(along_track,mdata.Heading,x_axis); + end + if isfield(mdata,'Surface') + mdata.Surface = interp1(along_track,mdata.Surface,x_axis); + end + if isfield(mdata,'Bottom') + mdata.Bottom = interp1(along_track,mdata.Bottom,x_axis); + end + if isfield(mdata,'GPS_time') + mdata.GPS_time = interp1(along_track,mdata.GPS_time,x_axis); + end +elseif strcmpi(param.mode_x_axis,'gps_time') + % Resample data to be uniformily sampled in mdata.GPS_time + dx = median(diff(mdata.GPS_time)); + x_axis = mdata.GPS_time(1) : dx : mdata.GPS_time(end); + mdata.Data = interp1(mdata.GPS_time,mdata.Data.',x_axis).'; + surf_filt = interp1(mdata.GPS_time,surf_filt,x_axis); + Nx = length(x_axis); + elev = interp1(mdata.GPS_time,mdata.Elevation,x_axis); + for layer_idx = 1:length(layers_twtt) + layers_twtt{layer_idx} = interp1(mdata.GPS_time,layers_twtt{layer_idx},x_axis); + end + if isfield(mdata,'Latitude') + mdata.Latitude = interp1(mdata.GPS_time,mdata.Latitude,x_axis); + end + if isfield(mdata,'Longitude') + mdata.Longitude = interp1(mdata.GPS_time,mdata.Longitude,x_axis); + end + if isfield(mdata,'Elevation') + mdata.Elevation = interp1(mdata.GPS_time,mdata.Elevation,x_axis); + end + if isfield(mdata,'Roll') + mdata.Roll = interp1(mdata.GPS_time,mdata.Roll,x_axis); + end + if isfield(mdata,'Pitch') + mdata.Pitch = interp1(mdata.GPS_time,mdata.Pitch,x_axis); + end + if isfield(mdata,'Heading') + mdata.Heading = interp1(mdata.GPS_time,mdata.Heading,x_axis); + end + if isfield(mdata,'Surface') + mdata.Surface = interp1(mdata.GPS_time,mdata.Surface,x_axis); + end + if isfield(mdata,'Bottom') + mdata.Bottom = interp1(mdata.GPS_time,mdata.Bottom,x_axis); + end + if isfield(mdata,'GPS_time') + mdata.GPS_time = x_axis; + end +elseif strcmpi(param.mode_x_axis,'range_line') + % Nothing needs to be done, the echogram is already uniformily in + % range_line units. + x_axis = 1:Nx; + elev = mdata.Elevation; else - surf_filt = mdata.Surface; + error('param.mode_x_axis %s not supported', param.mode_x_axis); end -%% Remove data before zero time -negative_bins = mdata.Time < 0; -mdata.Time = mdata.Time(~negative_bins); -mdata.Data = mdata.Data(~negative_bins,:); +% Create two way travel time axis for below the surface based on the permittivity model +[TWtime,~,param.er_depth,param.er_ice] = genPropProfileFromPerm(param.er_depth,param.er_ice,param.er_freq); +% Add an extra point for "air/vacuum" above the ice/media surface +TWtime = [-1e-6 TWtime]; +param.er_depth = [-1e-6*c/2 param.er_depth]; -if param.elev_comp == 1 - %% Relative elevation compensation - error('Not supported'); -elseif param.elev_comp == 2 - %% Compensate for surface variations (i.e. flatten to low pass filtered - % version of surface) +%% Resample data in y-dimension as required +if strcmpi(param.mode_y_axis,'depth') + %% DEPTH elevation compensation ============== + + %% Depth: Find the min/max depths represented in the data - % Determine start/stop depths + % Find the maximum elevation above ground level represented in the + % echogram (i.e. what is the maximum AGL of the first pixel over all of + % the range lines). This code tries to handle all cases, including if the first + % pixel in the echogram is below the surface. + min_y = interp1(TWtime, param.er_depth, min(mdata.Time(1)-surf_filt),'linear','extrap'); - %% Limit plotted depths according to input param.depth - % Example: param.depth = '[-2 20]'; - depth_range = eval(param.depth); + % Find the minimum elevation above ground level represented in the + % echogram. Normally this will be negative since we are generally looking + % at targets below the surface (i.e. what is the minimum AGL of the last + % pixel over all of the range lines). This code tries to handle all + % cases, including if the last pixel in the echogram is above the + % surface. + max_y = interp1(TWtime, param.er_depth, max(mdata.Time(end)-surf_filt),'linear','extrap'); + + %% Depth: Convert surface and layers to new y-axis + % Surface is at "zero" depth (surf_comp can be used in param.ylims_cmd) + surf_comp = zeros(1,Nx); + % layers_twtt-->layers_comp (layers_comp can be used in param.ylims_cmd) + layers_comp = cell(size(layers_twtt)); + for layer_idx = 1:length(layers_twtt) + layers_comp{layer_idx} = interp1(TWtime, param.er_depth, layers_twtt{layer_idx} - surf_filt,'linear','extrap'); + end + + %% Depth: Limit plotted y-axis according to input param.ylims_cmd + % Example: param.ylims_cmd = '[-2 20]'; + ylims = eval(param.ylims_cmd); + + % Check for inf limits (which get clipped to the ends of the time gate) + if ~isfinite(ylims(1)) + ylims(1) = min_y; + end + if ~isfinite(ylims(2)) + ylims(end) = max_y; + end + + %% Depth: Create depth axis % Determine sampling max_er = max(param.er_ice); dt = mdata.Time(2) - mdata.Time(1); dz = dt * c/2/sqrt(max_er); - % Depth axis - depth = (round(depth_range(1)/dz)*dz : dz : depth_range(end)).'; - - % Depth time axis - depth_time = size(depth); - % Above surface - depth_time(depth <= 0) = depth(depth <= 0) / (c/2); - % Below surface and within dielectric profile defined depth - if length(param.er_depth) > 1 - TWtime = genPropProfileFromPerm(param.er_depth,param.er_ice,param.er_freq); - profile_idxs = depth > 0 & depth < param.er_depth(end); - depth_time(profile_idxs) = interp1(param.er_depth, [0; TWtime], depth(profile_idxs)); - end - % Below surface and below dielectric profile defined depth - const_idxs = depth >= param.er_depth(end); - depth_time(const_idxs) = TWtime(end) + (depth(const_idxs) - param.er_depth(end)) / (c/2/sqrt(param.er_ice(end))); - - % Re-interpolate data to constant depth axis - newData = zeros(length(depth),size(mdata.Data,2)); - for rline = 1:size(mdata.Data,2) - newData(:,rline) = interp1(mdata.Time, mdata.Data(:,rline), ... - mdata.Surface(rline) + depth_time); + % y_axis: y-axis for output elevation compensated echogram + y_axis = (round(ylims(1)/dz)*dz : dz : round(ylims(end)/dz)*dz).'; + + % Convert Y-axis into time relative to the surface return (above the + % surface is negative and below the surface is positive time) + y_axis_time = interp1(param.er_depth,TWtime,y_axis,'linear','extrap'); + + %% Depth: Interpolate data onto new y-axis + data = zeros(length(y_axis),Nx); + for rline = 1:Nx + data(:,rline) = interp1(mdata.Time-surf_filt(rline),mdata.Data(:,rline),y_axis_time); + end + +elseif strcmpi(param.mode_y_axis,'range') + %% RANGE elevation compensation ============== + + %% Range: Find the min/max ranges represented in the data + + % Find the minimum range represented in the echogram. This code tries to + % handle all cases, including if the first pixel in the echogram is below + % the surface. + min_surf_filt = min(surf_filt); + min_y = min_surf_filt*c/2 + interp1(TWtime, param.er_depth, mdata.Time(1)-min_surf_filt,'linear','extrap'); + + % Find the maximum range represented in the echogram. This code tries to + % handle all cases, including if the last pixel in the echogram is above + % or below the surface. + max_surf_filt = max(surf_filt); + max_y = max_surf_filt*c/2 + interp1(TWtime, param.er_depth, mdata.Time(end) - max_surf_filt,'linear','extrap'); + + %% Range: Convert surface and layers to new y-axis + % Surface range (surf_comp can be used in param.ylims_cmd) + surf_comp = surf_filt*c/2; + % layers_twtt-->layers_comp (layers_comp can be used in param.ylims_cmd) + layers_comp = cell(size(layers_twtt)); + for layer_idx = 1:length(layers_twtt) + layers_comp{layer_idx} = surf_filt*c/2 ... + + interp1(TWtime, param.er_depth, layers_twtt{layer_idx}-surf_filt,'linear','extrap'); + end + + %% Range: Limit plotted y-axis according to input param.ylims_cmd + % Example: param.ylims_cmd = 'surf_comp + [-2 20]'; + ylims = eval(param.ylims_cmd); + + % Check for inf limits (which get clipped to the ends of the time gate) + if ~isfinite(ylims(1)) + ylims(1) = min_y; + end + if ~isfinite(ylims(2)) + ylims(end) = max_y; + end + + %% Range: Create range axis + + % Determine sampling + max_er = max(param.er_ice); + dt = mdata.Time(2) - mdata.Time(1); + dz = dt * c/2/sqrt(max_er); + + % y_axis: y-axis for output elevation compensated echogram + y_axis = (round(ylims(1)/dz)*dz : dz : round(ylims(end)/dz)*dz).'; + + %% Range: Interpolate data onto new y-axis + data = zeros(length(y_axis),Nx); + for rline = 1:Nx + % Create range axis for this dataset + range = surf_filt(rline)*c/2 ... + + interp1(TWtime,param.er_depth,mdata.Time-surf_filt(rline),'linear','extrap'); + data(:,rline) = interp1(range,mdata.Data(:,rline),y_axis); end - mdata.Data = newData; - depth_axis = depth; +elseif strcmpi(param.mode_y_axis,'twtt') + %% TWTT elevation compensation ============== -elseif param.elev_comp == 3 - %% Elevation compensate to WGS-84 y-axis + %% twtt: Find the min/max ranges represented in the data + min_y = mdata.Time(1); + max_y = mdata.Time(end); + + %% twtt: Convert surface and layers to new y-axis + % Surface/layers are already in this format, so no work is needed. + surf_comp = surf_filt; + layers_comp = layers_twtt; + + %% twtt: Limit plotted y-axis according to input param.ylims_cmd + % Example: param.ylims_cmd = 'surf_comp + [-2e-9 20e-9]'; + ylims = eval(param.ylims_cmd); - if length(param.er_ice) > 1 - error('Vector form of er_ice not supported'); + % Check for inf limits (which get clipped to the ends of the time gate) + if ~isfinite(ylims(1)) + ylims(1) = min_y; + end + if ~isfinite(ylims(2)) + ylims(end) = max_y; end + + %% twtt: Interpolate data onto new y-axis + % Data is already in this format, so just need to truncate to ylims + y_axis = mdata.Time(mdata.Time>=ylims(1) & mdata.Time<=ylims(end)); + data = mdata.Data(mdata.Time>=ylims(1) & mdata.Time<=ylims(end),:); - %% Create elevation axis to interpolate to - max_elev = max(mdata.Elevation); - min_elev = min(mdata.Elevation - surf_filt*c/2 - (mdata.Time(end)-surf_filt)*c/2/sqrt(param.er_ice)); - dt = mdata.Time(2)-mdata.Time(1); - dr = dt * c/2 / sqrt(param.er_ice); - elev_axis = max_elev:-dr:min_elev; - mdata.Elevation_Fasttime = elev_axis; +elseif strcmpi(param.mode_y_axis,'wgs84') + %% WGS84 elevation compensation ============== - % Zero pad data to create space for interpolated data - zero_pad_len = length(elev_axis) - length(mdata.Time); - mdata.Data = cat(1,mdata.Data,zeros(zero_pad_len,size(mdata.Data,2))); + %% wgs84: Find the min/max ranges represented in the data - % Determine the corrections to apply to elevation and layers - dRange = max_elev - mdata.Elevation; - dBins = round(dRange / (c/2) / dt); - dtime = dRange/(c/2); + % Find the minimum wgs-84 elevation represented in the echogram. This + % code tries to handle all cases, including if the last pixel in the + % echogram is below the surface. + min_y = min(elev ... + - (surf_filt*c/2 + ... + interp1(TWtime, param.er_depth, mdata.Time(end)-surf_filt,'linear','extrap'))); + + % Find the maximum wgs-84 elevation represented in the echogram. This + % code tries to handle all cases, including if the first pixel in the + % echogram is above or below the surface. + max_y = max(elev ... + - (surf_filt*c/2 + ... + interp1(TWtime, param.er_depth, mdata.Time(1)-surf_filt,'linear','extrap'))); + + %% wgs84: Convert surface and layers to new y-axis + % Surface range (surf_comp can be used in param.ylims_cmd) + surf_comp = elev - surf_filt*c/2; + % layers_twtt-->layers_comp (layers_comp can be used in param.ylims_cmd) + layers_comp = cell(size(layers_twtt)); + for layer_idx = 1:length(layers_twtt) + layers_comp{layer_idx} = elev ... + - (surf_filt*c/2 ... + + interp1(TWtime, param.er_depth, layers_twtt{layer_idx}-surf_filt,'linear','extrap')); + end + + %% wgs84: Limit plotted y-axis according to input param.ylims_cmd + ylims = eval(param.ylims_cmd); + + % Check for inf limits (which get clipped to the ends of the time gate) + if ~isfinite(ylims(1)) + ylims(1) = min_y; + end + if ~isfinite(ylims(2)) + ylims(end) = max_y; + end + + %% wgs84: Create range axis + + % Determine sampling + max_er = max(param.er_ice); + dt = mdata.Time(2) - mdata.Time(1); + dz = dt * c/2/sqrt(max_er); + + % y_axis: y-axis for output elevation compensated echogram + y_axis = (round(ylims(1)/dz)*dz : dz : round(ylims(end)/dz)*dz).'; + + %% wgs84: Interpolate data onto new y-axis + data = zeros(length(y_axis),Nx); + for rline = 1:Nx + % Create range axis for this dataset + wgs84 = elev(rline) ... + - (surf_filt(rline)*c/2 ... + + interp1(TWtime,param.er_depth,mdata.Time-surf_filt(rline),'linear','extrap')); + data(:,rline) = interp1(wgs84,mdata.Data(:,rline),y_axis); + end - warning off - for rline = 1:size(mdata.Data,2) - % Determine elevation bins before surface - surf_elev = mdata.Elevation(rline) - surf_filt(rline) * c/2; - dt_air = dr/(c/2); - time0 = -(max_elev - mdata.Elevation(rline))/(c/2); - last_air_idx = find(elev_axis > surf_elev,1,'last'); - new_time = (time0 + dt_air*(0:last_air_idx-1)).'; - if isempty(last_air_idx) - % Radar is on the surface of the ice - last_air_idx = 0; - end - if last_air_idx < length(elev_axis) - % Determine elevation bins after surface - dt_ice = dr/(c/2/sqrt(param.er_ice)); - first_ice_idx = last_air_idx + 1; - time0 = surf_filt(rline) + (surf_elev - elev_axis(first_ice_idx))/(c/2/sqrt(param.er_ice)); - new_time = cat(1,new_time, (time0 + dt_ice*(0:length(elev_axis)-length(new_time)-1)).'); - end - mdata.Data(:,rline) = interp1(mdata.Time, mdata.Data(1:length(mdata.Time),rline), new_time, 'linear',0); - mdata.Elevation(rline) = mdata.Elevation(rline) + dRange(rline); - mdata.Surface(rline) = mdata.Surface(rline) + dtime(rline); - %mdata.Bottom(rline) = mdata.Bottom(rline) + dtime(rline); - end - warning on - - %% Limit plotted depths according to input param.depth - DSurface = mdata.Elevation - mdata.Surface*c/2; - Surface_Elev = DSurface; - mdata.Surface_Elev = DSurface; - % Example: param.depth = '[min(Surface_Elev) - 15 max(Surface_Elev)+3]'; - % Example: param.depth = '[100 120]'; - depth_range = eval(param.depth); - depth_axis = find(elev_axis >= depth_range(1) & elev_axis <= depth_range(end)); else - error('Not supported'); + error('param.mode_y_axis %s not supported', param.mode_y_axis); end -return; +mdata.Data = data; + +%% Trim NaN from start/end of record +trim_nan_en = true; +if trim_nan_en + start_bin = nan(1,Nx); + stop_bin = nan(1,Nx); + for rline = 1:Nx + start_bin_tmp = find(~isnan(mdata.Data(:,rline)),1); + if ~isempty(start_bin_tmp) + start_bin(rline) = start_bin_tmp; + end + stop_bin_tmp = find(~isnan(mdata.Data(:,rline)),1,'last'); + if ~isempty(stop_bin_tmp) + stop_bin(rline) = stop_bin_tmp; + end + end + start_bin = min(start_bin); + stop_bin = max(stop_bin); + mdata.Data = mdata.Data(start_bin:stop_bin,:); + y_axis = y_axis(start_bin:stop_bin); +end diff --git a/cresis-toolbox/posting/flowline_post.m b/cresis-toolbox/posting/flowline_post.m index 9494a318..bc1af75b 100644 --- a/cresis-toolbox/posting/flowline_post.m +++ b/cresis-toolbox/posting/flowline_post.m @@ -8,7 +8,7 @@ % 3. Outputs a CSV file with mass conservation, radar parameters, and % best-fit radar thickness [NOT DONE YET] % -% Authors: John Paden, Levi Sedlock +% Authors: John Paden, Levi Sedlock, Leigh Stearns %% User Settings diff --git a/cresis-toolbox/posting/load_L1B.m b/cresis-toolbox/posting/load_L1B.m index b2790fc4..0a91c8d5 100644 --- a/cresis-toolbox/posting/load_L1B.m +++ b/cresis-toolbox/posting/load_L1B.m @@ -3,7 +3,7 @@ % % fn = filename string and must contain correct extension ('.nc' or '.mat') % -% Loads L1B cresis echogram files. +% Loads L1B cresis echogram (radar image) files. % L1B files can be in netcdf or mat format. % They can be "compressed" (compression is used for Snow and Kuband radar data % to make them smaller... main difference is that the surface is tracked @@ -20,9 +20,11 @@ % % See also: plot_L1B.m, uncompress_echogram.m +%% Load File [fn_dir,fn_name,fn_ext] = fileparts(fn); if strcmpi(fn_ext,'.nc') + % NETCDF mdata = netcdf_to_mat(fn); mdata.Latitude = mdata.lat; @@ -52,6 +54,7 @@ mdata = rmfield(mdata,'heading'); elseif strcmpi(fn_ext,'.mat') + % MAT mdata = load(fn); else @@ -60,21 +63,280 @@ mdata = uncompress_echogram(mdata); +%% param_get_heights if isfield(mdata,'param_get_heights') - mdata.param_qlook.qlook = mdata.param_get_heights; + if isequal({'get_heights'},fieldnames(mdata.param_get_heights)) + % param_get_heights is incomplete structure, supplement with param_records + mdata.param_qlook = mdata.param_records; + mdata.param_qlook = rmfield(mdata.param_qlook,'get_heights'); + mdata.param_qlook.qlook = mdata.param_get_heights; + mdata.file_version = '0'; + mdata.file_type = 'echo'; + else + % param_get_heights is complete structure, just rename to qlook + mdata.param_qlook = mdata.param_get_heights; + mdata.param_qlook.qlook = mdata.param_qlook.get_heights; + mdata.param_qlook = rmfield(mdata.param_qlook,'get_heights'); + % Ensure param_records has standard 3 fields (some old files do not + % have) + mdata.param_records.day_seg = mdata.param_qlook.day_seg; + mdata.param_records.season_name = mdata.param_qlook.season_name; + mdata.param_records.radar_name = mdata.param_qlook.radar_name; + if isfield(mdata.param_records,'gps') + mdata.param_records.records.gps = mdata.param_records.gps; + mdata.param_qlook.records.gps = mdata.param_records.gps; + mdata.param_records = rmfield(mdata.param_records,'gps'); + end + if isfield(mdata.param_records,'file') + mdata.param_records.records.file = mdata.param_records.file; + mdata.param_records = rmfield(mdata.param_records,'file'); + end + if isfield(mdata.param_records,'records_fn') + mdata.param_records = rmfield(mdata.param_records,'records_fn'); + end + % Ensure both parameter structs have sw_version + if ~isfield(mdata.param_records,'sw_version') + mdata.param_records.sw_version.ver = ''; + mdata.param_records.sw_version.date_time = ''; + mdata.param_records.sw_version.cur_date_time = ''; + mdata.param_records.sw_version.rev = ''; + mdata.param_records.sw_version.URL = ''; + end + if ~isfield(mdata.param_qlook,'sw_version') + mdata.param_qlook.sw_version = mdata.param_records.sw_version; + end + % Ensure both parameter structs have radar.lever_arm_fh + if ~isfield(mdata.param_records,'radar') + mdata.param_records.radar = []; + end + if ~isfield(mdata.param_qlook,'radar') + mdata.param_qlook.radar = []; + end + if ~isfield(mdata.param_records.radar,'lever_arm_fh') + mdata.param_records.radar.lever_arm_fh = []; + end + if ~isfield(mdata.param_qlook.radar,'lever_arm_fh') + mdata.param_qlook.radar.lever_arm_fh = []; + end + % Add file_version + mdata.file_version = '0'; + mdata.file_type = 'echo'; + end mdata = rmfield(mdata,'param_get_heights'); + if isfield(mdata.param_qlook,'proc') + mdata.param_qlook.load.frm = mdata.param_qlook.proc.frm; + end end +%% param_csarp if isfield(mdata,'param_csarp') - mdata.param_sar.sar = mdata.param_csarp; - mdata = rmfield(mdata,'param_csarp'); + if ~isfield(mdata.param_csarp,'csarp') + % Very old file format + mdata.param_sar.sar = mdata.param_csarp; + mdata.param_sar.sw_version= mdata.param_csarp.sw_version; + mdata = rmfield(mdata,'param_csarp'); + else + mdata.param_sar = mdata.param_csarp; + mdata = rmfield(mdata,'param_csarp'); + mdata.param_sar.sar = mdata.param_sar.csarp; + mdata.param_sar = rmfield(mdata.param_sar,'csarp'); + end + if isfield(mdata.param_records,'vectors') && isfield(mdata.param_records.vectors,'gps') && isfield(mdata.param_records.vectors.gps,'time_offset') + mdata.param_records.records.gps.time_offset = mdata.param_records.vectors.gps.time_offset; + mdata.param_sar.records.gps.time_offset = mdata.param_records.vectors.gps.time_offset; + else + if ~isfield(mdata.param_records,'records') || ~isfield(mdata.param_records.records,'gps') || ~isfield(mdata.param_records.records.gps,'time_offset') + mdata.param_records.records.gps.time_offset = 0; + end + if ~isfield(mdata.param_sar,'records') || ~isfield(mdata.param_sar.records,'gps') || ~isfield(mdata.param_sar.records.gps,'time_offset') + mdata.param_sar.records.gps.time_offset = 0; + end + end + mdata.file_version = '0'; + mdata.file_type = 'echo'; end +%% param_combine +if isfield(mdata,'param_combine') + mdata.param_array = mdata.param_combine; + mdata = rmfield(mdata,'param_combine'); + if isfield(mdata.param_sar.sar,'lever_arm_fh') + lever_arm_fh = mdata.param_sar.sar.lever_arm_fh; + else + lever_arm_fh = []; + end + if ~isfield(mdata.param_records.radar,'lever_arm_fh') + mdata.param_records.radar.lever_arm_fh = lever_arm_fh; + end + if ~isfield(mdata.param_array.radar,'lever_arm_fh') + mdata.param_array.radar.lever_arm_fh = lever_arm_fh; + end + if ~isfield(mdata.param_sar.radar,'lever_arm_fh') + mdata.param_sar.radar.lever_arm_fh = lever_arm_fh; + end + if isfield(mdata.param_records,'vectors') && isfield(mdata.param_records.vectors,'gps') && isfield(mdata.param_records.vectors.gps,'time_offset') + mdata.param_array.records.gps.time_offset = mdata.param_records.vectors.gps.time_offset; + elseif ~isfield(mdata.param_array,'records') || ~isfield(mdata.param_array.records,'gps') || ~isfield(mdata.param_array.records.gps,'time_offset') + mdata.param_array.records.gps.time_offset = 0; + end + mdata.file_version = '0'; + mdata.file_type = 'echo'; +end + +%% param_combine_wf_chan if isfield(mdata,'param_combine_wf_chan') mdata.param_array.array = mdata.param_combine_wf_chan; mdata = rmfield(mdata,'param_combine_wf_chan'); + + if isfield(mdata.param_array.array,'array') + warning('Very old file format. Fixing array field.'); + tmp_array = mdata.param_array.array.array; + mdata.param_array.array = rmfield(mdata.param_array.array,'array'); + mdata.param_array.array = merge_structs(mdata.param_array.array,tmp_array); + end + % Ensure param_records has standard 3 fields (some old files do not + % have) + if ~isfield(mdata.param_sar,'day_seg') + warning('Very old file format. No day_seg field present for this L1B data.'); + mdata.param_sar.day_seg = ''; + end + if ~isfield(mdata.param_sar,'season_name') + warning('Very old file format. No season_name field present for this L1B data.'); + mdata.param_sar.season_name = ''; + end + if ~isfield(mdata.param_sar,'radar_name') + warning('Very old file format. No radar_name field present for this L1B data.'); + mdata.param_sar.radar_name = ''; + end + mdata.param_records.day_seg = mdata.param_sar.day_seg; + mdata.param_records.season_name = mdata.param_sar.season_name; + mdata.param_records.radar_name = mdata.param_sar.radar_name; + mdata.param_array.day_seg = mdata.param_sar.day_seg; + mdata.param_array.season_name = mdata.param_sar.season_name; + mdata.param_array.radar_name = mdata.param_sar.radar_name; + if isfield(mdata.param_records,'gps') + mdata.param_records.records.gps = mdata.param_records.gps; + mdata.param_array.records.gps = mdata.param_records.gps; + mdata.param_records = rmfield(mdata.param_records,'gps'); + end + if isfield(mdata.param_records,'file') + mdata.param_records.records.file = mdata.param_records.file; + mdata.param_records = rmfield(mdata.param_records,'file'); + end + if isfield(mdata.param_records,'records_fn') + mdata.param_records = rmfield(mdata.param_records,'records_fn'); + end + % Ensure both parameter structs have sw_version + if ~isfield(mdata.param_records,'sw_version') + mdata.param_records.sw_version.ver = ''; + mdata.param_records.sw_version.date_time = ''; + mdata.param_records.sw_version.cur_date_time = ''; + mdata.param_records.sw_version.rev = ''; + mdata.param_records.sw_version.URL = ''; + end + if ~isfield(mdata.param_array,'sw_version') + mdata.param_array.sw_version = mdata.param_records.sw_version; + end + if ~isfield(mdata.param_array,'sw_version') + mdata.param_sar.sw_version = mdata.param_records.sw_version; + end + % Ensure both parameter structs have radar.lever_arm_fh + if ~isfield(mdata.param_records,'radar') + mdata.param_records.radar = []; + end + if ~isfield(mdata.param_array,'radar') + mdata.param_array.radar = []; + end + if ~isfield(mdata.param_sar,'radar') + mdata.param_sar.radar = []; + end + if ~isfield(mdata.param_records.radar,'lever_arm_fh') + mdata.param_records.radar.lever_arm_fh = []; + end + if ~isfield(mdata.param_array.radar,'lever_arm_fh') + mdata.param_array.radar.lever_arm_fh = []; + end + if ~isfield(mdata.param_sar.radar,'lever_arm_fh') + mdata.param_sar.radar.lever_arm_fh = []; + end + + mdata.file_version = '0'; + mdata.file_type = 'echo'; +end + +%% records.gps.time_offset, radar.lever_arm_fh +if isfield(mdata,'param_qlook') + if ~isfield(mdata.param_qlook.records,'gps') + mdata.param_qlook.records.gps = []; + end + if ~isfield(mdata.param_qlook.records.gps,'time_offset') + mdata.param_qlook.records.gps.time_offset = NaN; + end + if ~isfield(mdata.param_qlook.radar,'lever_arm_fh') + if isfield(mdata.param_qlook.qlook.lever_arm_fh) + mdata.param_qlook.radar.lever_arm_fh = mdata.param_qlook.qlook.lever_arm_fh; + mdata.param_qlook.qlook = rmfield(mdata.param_qlook.qlook,'lever_arm_fh'); + else + mdata.param_qlook.radar.lever_arm_fh = []; + end + end +end + +if isfield(mdata,'param_array') + if ~isfield(mdata.param_array.records,'gps') + mdata.param_array.records.gps = []; + end + if ~isfield(mdata.param_array.records.gps,'time_offset') + mdata.param_array.records.gps.time_offset = NaN; + end + if ~isfield(mdata.param_array.radar,'lever_arm_fh') + if isfield(mdata.param_array.sar.lever_arm_fh) + mdata.param_array.radar.lever_arm_fh = mdata.param_array.sar.lever_arm_fh; + mdata.param_array.sar = rmfield(mdata.param_array.sar,'lever_arm_fh'); + else + mdata.param_array.radar.lever_arm_fh = []; + end + end end +%% Detect LDEO SIR or DICE file +if isfield(mdata,'Campaign') && isfield(mdata,'FlightNo') && isfield(mdata,'prodLevel') && isfield(mdata,'UTC_time') + mdata.GPS_time = mdata.UTC_time + utc_leap_seconds(mdata.UTC_time(1)); + mdata.param_qlook.season_name = mdata.Campaign; + mdata.param_qlook.radar_name = 'LDEO_SIR'; + mdata.param_qlook.cmd.mission_names = mdata.FlightNo; + mdata.param_qlook.qlook.dec = mdata.numAvg; + mdata.param_qlook.qlook.B_filter = ones(1,mdata.numAvg)/mdata.numAvg; + mdata.param_qlook.qlook.inc_dec = 0; + mdata.file_type = 'qlook'; + mdata.file_version = '1'; + + mdata = rmfield(mdata,'Campaign'); + mdata = rmfield(mdata,'FlightNo'); + mdata = rmfield(mdata,'layerInd'); + mdata = rmfield(mdata,'prodLevel'); + mdata = rmfield(mdata,'UTC_time'); + mdata = rmfield(mdata,'Surf_Elev'); + mdata = rmfield(mdata,'numAvg'); + + % Shallow Ice Radar (SIR) <-- RS02_L870_20161129_031707_level1a_SIR_177.mat + % 1 minute files + % Pulse compressed (FFT, deramp on receive) + % Motion compensation + % GPS synchronization + % Surface tracked + + % 5 minute files + % Pulse compressed + % Motion compensation + % GPS synchronization + % Surface tracked + % Unfocused SAR (averaging and decimation) + + % Deep Ice Radar (DICE) +end + +%% Roll, Pitch, Heading if ~isfield(mdata,'Roll') && isfield(mdata,'GPS_time') mdata.Roll = zeros(size(mdata.GPS_time)); end @@ -84,6 +346,15 @@ end if ~isfield(mdata,'Heading') && isfield(mdata,'GPS_time') - mdata.Heading = zeros(size(mdata.GPS_time)); + mdata.Heading = nan(size(mdata.GPS_time)); end +%% Time, Depth field correction for old files +if size(mdata.Time,1) == 1 && size(mdata.Time,2) > 1 + warning('mdata.Time is a row vector and should always be a column vector. Fixing.'); + mdata.Time = mdata.Time.'; +end +if isfield(mdata,'Depth') + warning('Old file format. mdata.Depth is deprecated. Removing field.'); + mdata = rmfield(mdata,'Depth'); +end diff --git a/cresis-toolbox/posting/load_data_by_gps_time.m b/cresis-toolbox/posting/load_data_by_gps_time.m index 0d7b205b..224066b4 100644 --- a/cresis-toolbox/posting/load_data_by_gps_time.m +++ b/cresis-toolbox/posting/load_data_by_gps_time.m @@ -196,40 +196,32 @@ % ==================================================================== % ==================================================================== - % Get list of records and frames files - % records_fn = '/cresis/scratch1/mdce/csarp_support/records/mcords/2009_Antarctica_DC8/records_20091016_seg1.mat'; - % records = records_aux_files_read(records_fn,[1 4000]); - % check = load(records_fn); - % Get list of records files - records_dir = ct_filename_support(param, '', 'records'); - fns = get_filenames(records_dir,'records','','.nc'); + frames_dir = ct_filename_support(param, '', 'frames'); + fns = get_filenames(frames_dir,'frames','','.mat'); if isempty(fns) - error('No records file found in %s', records_dir); + error('No frames file found in %s', frames_dir); end % Load in first and last record from each records file first_gps_time = []; + last_gps_time = []; for file_idx = 1:length(fns) - records_fn = fns{file_idx}; - [path name] = fileparts(records_fn); - cdf_fn = fullfile(path, sprintf('%s.nc', name)); + frames_fn = fns{file_idx}; try - ncid = netcdf.open(cdf_fn,'NOWRITE'); + % Continue to run even if a frames file fails to load + frames = frames_load(frames_fn); + first_gps_time(file_idx) = frames.gps_time(1); + last_gps_time(file_idx) = frames.gps_time(end); catch ME - warning('Exception during file opening'); - ME - keyboard + warning(ME.getReport) end - var_idx = netcdf.inqVarID(ncid,'gps_time'); - first_gps_time(file_idx) = netcdf.getVar(ncid,var_idx,[0 0],[1 1]); - netcdf.close(ncid); end start_file_idx = find(start.gps_time >= first_gps_time,1,'last'); - stop_file_idx = find(stop.gps_time >= first_gps_time,1,'last'); + stop_file_idx = find(stop.gps_time <= last_gps_time,1); if isempty(start_file_idx) error('Start time (%s) is before any radar data exists (%s)', ... @@ -247,31 +239,13 @@ datestr(epoch_to_datenum(stop.gps_time))); end - records_fn = [fns{start_file_idx}(1:end-2) 'mat']; - [tmp records_fn_name] = fileparts(records_fn); - param.day_seg = records_fn_name(9:end); - - frames_fn = ct_filename_support(param, '', 'frames'); - load(frames_fn); - - records_ver = load(records_fn,'ver','file_version'); - if isfield(records_ver,'ver') || isfield(records_ver,'file_version') - records = load(records_fn); - else - load(records_fn, 'records'); - end + frames = frames_load(fns{start_file_idx}); + param.day_seg = frames.param.day_seg; + records = records_load(param); - % Determine which records must be loaded - start_record = find(records.gps_time >= start.gps_time,1); - stop_record = find(records.gps_time <= stop.gps_time,1,'last'); - - if isempty(start_record) - error('No data found!'); - end - - % Determine which frames contain these records - start_frame = find(frames.frame_idxs <= start_record,1,'last'); - stop_frame = find(frames.frame_idxs <= stop_record,1,'last'); + % Determine which frames need to be loaded + start_frame = find(start.gps_time >= frames.gps_time,1,'last'); + stop_frame = find(stop.gps_time >= frames.gps_time,1,'last'); start_frm_id = sprintf('%s_%03.0f', param.day_seg, start_frame); stop_frm_id = sprintf('%s_%03.0f', param.day_seg, stop_frame); @@ -294,12 +268,8 @@ end % Determine this frames valid GPS time range - frm_gps_time_start = records.gps_time(frames.frame_idxs(frm)); - if frm < length(frames.frame_idxs) - frm_gps_time_stop = records.gps_time(frames.frame_idxs(frm+1)); - else - frm_gps_time_stop = records.gps_time(end); - end + frm_gps_time_start = frames.gps_time(start_frame); + frm_gps_time_stop = frames.gps_time(stop_frame+1); fprintf(' %s\n', out_fn); if frm_idx == 1 diff --git a/cresis-toolbox/posting/nsidc_delivery_script_task.m b/cresis-toolbox/posting/nsidc_delivery_script_task.m index 282136b2..45380b8b 100644 --- a/cresis-toolbox/posting/nsidc_delivery_script_task.m +++ b/cresis-toolbox/posting/nsidc_delivery_script_task.m @@ -26,7 +26,7 @@ %% Load the frames information frames_fn = ct_filename_support(param,'','frames'); -load(frames_fn); +frames = frames_load(frames_fn); if isempty(param.cmd.frms) param.cmd.frms = 1:length(frames.frame_idxs); end @@ -79,6 +79,9 @@ elseif strcmpi(platform,'SO') premet_param.nsidc_platform_short_name = 'DHC-3'; premet_param.nsidc_aircraft_id = 'N226UT'; +elseif strcmpi(platform,'GV') + premet_param.nsidc_platform_short_name = 'G-V'; + premet_param.nsidc_aircraft_id = 'N95NA'; else error('Unsupported platform %s\n', platform); end @@ -188,8 +191,7 @@ data_type = '1B'; % Find the records information for updating global attributes - records_fn = ct_filename_support(param,'','records'); - records = load(records_fn); + records = records_load(param); % Remove frames that do not exist from param.cmd.frms list [valid_frms,keep_idxs] = intersect(param.cmd.frms, 1:length(frames.frame_idxs)); @@ -302,7 +304,7 @@ end % create extra image filenames - for extra_idx = 1:size(image_extra,1) + for extra_idx = 1:size(image_extra,2) echogram_fn = fullfile(ct_filename_out(param,'post','',1),'images', ... param.day_seg, sprintf('%s_%03d_1echo_%s.jpg',param.day_seg,frm,image_extra{extra_idx})); new_echogram_fn = fullfile(out_data_dir, ... @@ -410,7 +412,7 @@ netcdf_from_mat(out_fn_netcdf,supplement,supplement_netcdf_param); if exist('L1B_supplement_name_extra','var') || ~isempty(L1B_supplement_name_extra) - for extra_idx = 1:size(L1B_supplement_name_extra,1) + for extra_idx = 1:size(L1B_supplement_name_extra,2) out_fn_netcdf = fullfile(ct_filename_out(param,USER_SPECIFIED_DIRECTORY,'',1), ... sprintf('%s%s_Files', radar_type, data_type),data_files_dir, ... sprintf('%s%s_%s_%03d_%s_%s.nc',radar_type, data_type, param.day_seg, frm,L1B_supplement_name_extra{extra_idx}, L1B_supplement_name)); diff --git a/cresis-toolbox/posting/nsidc_netcdf_attributes.m b/cresis-toolbox/posting/nsidc_netcdf_attributes.m index ea213fbb..e8aaa88e 100644 --- a/cresis-toolbox/posting/nsidc_netcdf_attributes.m +++ b/cresis-toolbox/posting/nsidc_netcdf_attributes.m @@ -34,7 +34,7 @@ if isfield(data,'param_qlook') proc_param = data.param_qlook; else - proc_param = data.param_csarp; + proc_param = data.param_sar; if ~isfield(proc_param,'sw_version') %% Legacy file format fix proc_param.sw_version = proc_param.csarp.sw_version; @@ -72,6 +72,8 @@ netcdf.putAtt(ncid,varid,'location','NASABasler'); elseif strcmpi(parse_season_name{3},'SO') netcdf.putAtt(ncid,varid,'location','UltraThuleSO'); +elseif strcmpi(parse_season_name{3},'GV') + netcdf.putAtt(ncid,varid,'location','NASAGV'); else error('Platform not supported'); end @@ -96,8 +98,11 @@ '2011_Greenland_P3','2011_Antarctica_DC8','2012_Greenland_P3', ... '2012_Antarctica_DC8'})) netcdf.putAtt(ncid,varid,'investigators','C.Leuschen, C. Allen, R. Hale, J. Paden, F. Rodriguez'); -elseif any(strcmpi(param.season_name,{'2013_Greenland_P3', ... - '2013_Antarctica_P3','2014_Greenland_P3','2014_Antarctica_DC8','2015_Greenland_C130','2016_Greenland_P3','2016_Antarctica_DC8','2017_Greenland_P3','2017_Antarctica_P3','2017_Antarctica_Basler','2018_Greenland_P3','2018_Alaska_SO','2018_Antarctica_DC8','2019_Greenland_P3'})) +elseif any(strcmpi(param.season_name,{'2013_Greenland_P3','2013_Antarctica_P3',... + '2014_Greenland_P3','2014_Antarctica_DC8','2015_Greenland_C130',... + '2016_Greenland_P3','2016_Antarctica_DC8','2017_Greenland_P3','2017_Antarctica_P3','2017_Antarctica_Basler',... + '2018_Greenland_P3','2018_Alaska_SO','2018_Antarctica_DC8',... + '2019_Greenland_P3','2019_Arctic_GV','2019_Antarctica_GV','2021_Alaska_SO'})) netcdf.putAtt(ncid,varid,'investigators','C.Leuschen, R. Hale, J. Li, J. Paden, F. Rodriguez'); else error('Unsupported season\n'); @@ -169,7 +174,7 @@ '2014_Greenland_P3'})) rfparams_val = cat(2,rfparams_val, ... sprintf('Transmitter: 1 GSPS DDS, 5 W\n')); - elseif any(strcmpi(param.season_name,{'2017_Greenland_P3','2017_Antarctica_P3'})) + elseif any(strcmpi(param.season_name,{'2017_Greenland_P3','2017_Antarctica_P3','2018_Greenland_P3'})) rfparams_val = cat(2,rfparams_val, ... sprintf('Transmitter: 2.4 GSPS AWG, 400 W\n')); else @@ -223,7 +228,7 @@ elseif any(strcmpi(param.season_name,{'2014_Greenland_P3'})) rfparams_val = cat(2,rfparams_val, ... sprintf('Transmitters: 7-channel DDS with phase/amplitude control, 300W SSPA per channel\n')); - elseif any(strcmpi(param.season_name,{'2014_Antarctica_DC8','2016_Antarctica_DC8'})) + elseif any(strcmpi(param.season_name,{'2014_Antarctica_DC8','2016_Antarctica_DC8','2018_Antarctica_DC8'})) rfparams_val = cat(2,rfparams_val, ... sprintf('Transmitters: 6-channel DDS with phase/amplitude control, 1000W SSPA per channel\n')); elseif any(strcmpi(param.season_name,{'2015_Greenland_C130'})) @@ -232,12 +237,15 @@ elseif any(strcmpi(param.season_name,{'2016_Greenland_P3'})) rfparams_val = cat(2,rfparams_val, ... sprintf('Transmitters: 2-channel DDS with phase/amplitude control, 250W SSPA per channel\n')); - elseif any(strcmpi(param.season_name,{'2017_Greenland_P3','2017_Antarctica_P3'})) + elseif any(strcmpi(param.season_name,{'2017_Greenland_P3','2017_Antarctica_P3','2018_Greenland_P3'})) rfparams_val = cat(2,rfparams_val, ... sprintf('Transmitters: 7-channel DDS with phase/amplitude control, 500W SSPA per channel\n')); elseif any(strcmpi(param.season_name,{'2017_Antarctica_Basler'})) rfparams_val = cat(2,rfparams_val, ... sprintf('Transmitters: 8-channel DDS with phase/amplitude control, 225W SSPA per channel\n')); + elseif any(strcmpi(param.season_name,{'2019_Antarctica_GV'})) + rfparams_val = cat(2,rfparams_val, ... + sprintf('Transmitters: 4-channel DDS with phase/amplitude control, 500W SSPA per channel\n')); else error('Unsupported season\n'); end @@ -258,7 +266,7 @@ elseif any(strcmpi(param.season_name,{'2018_Greenland_P3','2018_Antarctica_DC8','2019_Greenland_P3','2019_Arctic_GV','2019_Antarctica_GV'})) rfparams_val = cat(2,rfparams_val, ... sprintf('Transmitter: Keysight M8195A AWG 65 GSPS, 1 W, 2-18 GHz\n')); - elseif any(strcmpi(param.season_name,{'2018_Alaska_SO'})) + elseif any(strcmpi(param.season_name,{'2018_Alaska_SO','2021_Alaska_SO'})) rfparams_val = cat(2,rfparams_val, ... sprintf('Transmitter: DDS with 20x frequency multiplier, 1 W\n')); else @@ -287,6 +295,9 @@ elseif any(strcmpi(param.season_name,{'2017_Greenland_P3','2017_Antarctica_P3'})) rfparams_val = cat(2,rfparams_val, ... sprintf('Multichannel: Forward 4 elements tx with power splitter, aft 4 elements used for rx with individual channels digitized\n')); + elseif any(strcmpi(param.season_name,{'2018_Greenland_P3'})) + rfparams_val = cat(2,rfparams_val, ... + sprintf('Single channel mode: all 8 elements tx combined with a power combiner, all 8 elements used for rx with individual channels combined to be digitized\n')); else error('Unsupported season\n'); end @@ -312,6 +323,9 @@ elseif any(strcmpi(param.season_name,{'2017_Antarctica_Basler'})) rfparams_val = cat(2,rfparams_val, ... sprintf('TX/RX mode: Channels 1-8 are Tx/Rx (installed under fuselage)\n')); + elseif any(strcmpi(param.season_name,{'2019_Antarctica_GV'})) + rfparams_val = cat(2,rfparams_val, ... + sprintf('TX/RX mode: Channels 1-4 are Tx/Rx (installed under fuselage)\n')); else error('Unsupported season\n'); end @@ -320,13 +334,14 @@ '2010_Greenland_DC8','2010_Antarctica_DC8', ... '2011_Antarctica_DC8', ... '2012_Antarctica_DC8', ... - '2014_Antarctica_DC8','2015_Greenland_C130','2016_Antarctica_DC8','2018_Antarctica_DC8'})) + '2014_Antarctica_DC8','2015_Greenland_C130','2016_Antarctica_DC8','2018_Antarctica_DC8','2019_Arctic_GV','2019_Antarctica_GV'})) rfparams_val = cat(2,rfparams_val, ... sprintf('Transmit out of left antenna, receive from right antenna\n')); elseif any(strcmpi(param.season_name,{'2009_Greenland_P3','2010_Greenland_P3',... '2011_Greenland_P3','2012_Greenland_P3', ... '2013_Greenland_P3', '2013_Antarctica_P3', ... - '2014_Greenland_P3','2017_Greenland_P3','2017_Antarctica_P3','2018_Greenland_P3','2018_Alaska_SO','2019_Greenland_P3'})) + '2014_Greenland_P3','2017_Greenland_P3','2017_Antarctica_P3','2018_Greenland_P3','2018_Alaska_SO',... + '2019_Greenland_P3','2021_Alaska_SO'})) rfparams_val = cat(2,rfparams_val, ... sprintf('Transmit out of forward antenna, receive from aft antenna\n')); elseif any(strcmpi(param.season_name,{'2016_Greenland_P3'})) @@ -398,7 +413,7 @@ sprintf('Digitizer platform: NI PXIe Windows PC running LabView\n')); digparams_val = cat(2,digparams_val, ... sprintf('Digitizer manufacturer: National Instruments/Sundance')); - elseif any(strcmpi(param.season_name,{'2017_Greenland_P3','2017_Antarctica_P3'})) + elseif any(strcmpi(param.season_name,{'2017_Greenland_P3','2017_Antarctica_P3','2018_Greenland_P3'})) digparams_val = cat(2,digparams_val, ... sprintf('Digitizer maximum range: 2 volt peak to peak; 10 dBm max power')); digparams_val = cat(2,digparams_val, ... @@ -443,12 +458,13 @@ sprintf('Digitizer manufacturer: Analog Devices and Custom')); elseif any(strcmpi(param.season_name,{'2011_Greenland_P3','2011_Antarctica_DC8','2012_Greenland_P3', ... '2012_Antarctica_DC8','2013_Greenland_P3', '2013_Antarctica_P3', ... - '2014_Greenland_P3','2014_Antarctica_DC8','2016_Antarctica_DC8','2017_Greenland_P3','2017_Antarctica_P3','2018_Greenland_P3','2019_Greenland_P3'})) + '2014_Greenland_P3','2014_Antarctica_DC8','2016_Antarctica_DC8','2017_Greenland_P3','2017_Antarctica_P3',... + '2018_Greenland_P3','2018_Antarctica_DC8','2019_Greenland_P3','2019_Antarctica_GV'})) digparams_val = cat(2,digparams_val, ... sprintf('Digitizer maximum range: 2 volt peak to peak; 10 dBm max power')); digparams_val = cat(2,digparams_val, ... sprintf('Record rate: variable')); - if any(strcmpi(param.season_name,{'2013_Greenland_P3','2019_Greenland_P3'})) + if any(strcmpi(param.season_name,{'2013_Greenland_P3','2019_Greenland_P3','2019_Antarctica_GV'})) digparams_val = cat(2,digparams_val, ... sprintf('Channels: 7\n')); else @@ -530,8 +546,10 @@ digparams_val = cat(2,digparams_val, ... sprintf('Digitizer manufacturer: Analog Devices and Custom')); elseif any(strcmpi(param.season_name,{'2012_Greenland_P3', ... - '2012_Antarctica_DC8','2013_Greenland_P3', '2013_Antarctica_P3', ... - '2014_Greenland_P3','2014_Antarctica_DC8','2015_Greenland_C130','2016_Greenland_P3','2016_Antarctica_DC8','2017_Greenland_P3','2017_Antarctica_P3','2018_Greenland_P3','2018_Alaska_SO','2018_Antarctica_DC8','2019_Greenland_P3'})) + '2012_Antarctica_DC8','2013_Greenland_P3', '2013_Antarctica_P3','2014_Greenland_P3','2014_Antarctica_DC8', ... + '2015_Greenland_C130','2016_Greenland_P3','2016_Antarctica_DC8','2017_Greenland_P3','2017_Antarctica_P3', ... + '2018_Greenland_P3','2018_Alaska_SO','2018_Antarctica_DC8','2019_Greenland_P3','2019_Arctic_GV','2019_Antarctica_GV' ... + '2021_Alaska_SO'})) digparams_val = cat(2,digparams_val, ... sprintf('Digitizer maximum range: 2 volt peak to peak; 10 dBm max power')); digparams_val = cat(2,digparams_val, ... @@ -575,6 +593,9 @@ elseif any(strcmpi(param.season_name,{'2017_Greenland_P3','2017_Antarctica_P3'})) antparams_val = cat(2,antparams_val, ... sprintf('Antennas: 2 along-track x 4 cross-track elliptical dipole array (forward four for Tx, aft four for Rx)\n')); + elseif any(strcmpi(param.season_name,{'2018_Greenland_P3'})) + antparams_val = cat(2,antparams_val, ... + sprintf('Antennas: 2 along-track x 4 cross-track elliptical dipole array (all combined for Tx and Rx)\n')); else error('Unsupported season\n'); end @@ -601,13 +622,12 @@ sprintf('Antennas: Byers et al., IEEE Tran. Instruments and Methods, May 2012\n')); elseif any(strcmpi(param.season_name,{'2009_Antarctica_DC8', ... '2010_Greenland_DC8','2010_Antarctica_DC8', ... - '2011_Antarctica_DC8',... - '2012_Antarctica_DC8','2014_Antarctica_DC8','2016_Antarctica_DC8'})) + '2011_Antarctica_DC8','2012_Antarctica_DC8'})); antparams_val = cat(2,antparams_val, ... sprintf('Antennas: Loaded dipole array with 5 elements\n')); antparams_val = cat(2,antparams_val, ... sprintf('Antennas: Allen et al., IEEE Aerospace and Electronic Systems Magazine, March 2012\n')); - elseif any(strcmpi(param.season_name,{'2014_Antarctica_DC8','2016_Antarctica_DC8'})) + elseif any(strcmpi(param.season_name,{'2014_Antarctica_DC8','2016_Antarctica_DC8','2018_Antarctica_DC8'})) antparams_val = cat(2,antparams_val, ... sprintf('Antennas: Loaded dipole array with 6 elements\n')); antparams_val = cat(2,antparams_val, ... @@ -622,6 +642,11 @@ sprintf('Antennas: Eight custom CReSIS dipole antennas\n')); antparams_val = cat(2,antparams_val, ... sprintf('Antennas: Custom built 150-600 MHz antenna with matching network\n')); + elseif any(strcmpi(param.season_name,{'2019_Antarctica_GV'})) + antparams_val = cat(2,antparams_val, ... + sprintf('Antennas: Four custom CReSIS U-slot patch antennas\n')); + antparams_val = cat(2,antparams_val, ... + sprintf('Antennas: Custom built 236-254 MHz antenna with impedence matching network\n')); else error('Unsupported season\n'); end @@ -630,9 +655,14 @@ '2010_Greenland_P3','2010_Antarctica_DC8', ... '2011_Greenland_P3','2011_Antarctica_DC8','2012_Greenland_P3', ... '2012_Antarctica_DC8','2013_Greenland_P3', '2013_Antarctica_P3', ... - '2014_Greenland_P3','2014_Greenland_DC8','2014_Antarctica_DC8','2016_Greenland_P3','2016_Antarctica_DC8','2017_Greenland_P3','2017_Antarctica_P3','2018_Greenland_P3','2019_Greenland_P3'})) + '2014_Greenland_P3','2014_Greenland_DC8','2014_Antarctica_DC8',... + '2016_Greenland_P3','2016_Antarctica_DC8','2017_Greenland_P3',... + '2017_Antarctica_P3','2018_Greenland_P3','2018_Antarctica_DC8','2019_Greenland_P3'})) antparams_val = cat(2,antparams_val, ... sprintf('Antennas: ETS Lindgren TEM horn antenna model ETS 3115\n')); + elseif any(strcmpi(param.season_name,{'2019_Arctic_GV','2019_Antarctica_GV'})) + antparams_val = cat(2,antparams_val, ... + sprintf('Antennas: A-info LB-20180-N\n')); elseif any(strcmpi(param.season_name,{'2009_Antarctica_DC8'})) antparams_val = cat(2,antparams_val, ... sprintf('Antennas: Pasternack PE9861-15\n')); @@ -642,7 +672,7 @@ elseif any(strcmpi(param.season_name,{'2015_Greenland_C130'})) antparams_val = cat(2,antparams_val, ... sprintf('Antennas: Custom CReSIS Vivaldi Antenna fed with 12:1 power divider, only inner 8 elements used\n')); - elseif any(strcmpi(param.season_name,{'2018_Alaska_SO'})) + elseif any(strcmpi(param.season_name,{'2018_Alaska_SO','2021_Alaska_SO'})) antparams_val = cat(2,antparams_val, ... sprintf('Antennas: Q-Par Angus WBH2-18 (TX) and Steatite Q-Par QWH-SL--2-18-S-HG-R (RX)\n')); else @@ -698,7 +728,7 @@ sprintf('Coherent stacking: SAR Processing\n')); procparams_val = cat(2,procparams_val, ... sprintf('Incoherent averaging: [%d %d]\n', ... - length(data.param_array.array.rline_rng), ... + length(data.param_array.array.line_rng), ... length(data.param_array.array.bin_rng))); end procparams_val = cat(2,procparams_val, ... diff --git a/cresis-toolbox/posting/plot_L1B.m b/cresis-toolbox/posting/plot_L1B.m index 68a5689b..5e5363eb 100644 --- a/cresis-toolbox/posting/plot_L1B.m +++ b/cresis-toolbox/posting/plot_L1B.m @@ -1,4 +1,5 @@ -% script plot_L1B +function [mdata] = plot_L1B(echo_fn,layerdata_source) +% [mdata] = plot_L1B(echo_fn,layerdata_source) % % Example of loading data with load_L1B.m and using elevation_compensation.m % @@ -7,20 +8,22 @@ % Figure 2: range on y-axis % Figure 3: time-delay on y-axis % +% Examples: +% +% fn = 'IRMCR1B_V01_20130408_01_020.nc'; +% mdata = plot_L1B(fn); +% +% fn = '/cresis/snfs1/dataproducts/ct_data/rds/2012_Greenland_P3/CSARP_post/CSARP_qlook/20120412_01/Data_img_01_20120412_01_001.mat'; +% [mdata,lay] = plot_L1B(fn,'CSARP_post/layerData'); +% % Author: John Paden % % See also: load_L1B.m, elevation_compensation.m -% fn = 'IRMCR1B_V01_20130408_01_020.nc'; -% mdata = load_L1B(fn); - -fn = '/cresis/snfs1/dataproducts/ct_data/snow/2014_Greenland_P3/CSARP_post/CSARP_qlook/20140324_01/Data_20140324_01_010.mat'; -mdata = load_L1B(fn); -if 0 - % Replace L1B echogram surface with L2 surface from layer data file - fn = '/cresis/snfs1/dataproducts/ct_data/snow/2014_Greenland_P3/CSARP_post/CSARP_layerData/20140324_01/Data_20140324_01_010.mat'; - lay = load(fn); - mdata.Surface = interp1(lay.GPS_time,lay.layerData{1}.value{2}.data,mdata.GPS_time); +mdata = load_L1B(echo_fn); +if exist('layerdata_source','var') + % Replace L1B echogram surface and bottom with L2 surface and bottom from layer data file + [mdata.Surface,mdata.Bottom] = layerdata.load_layers(mdata,layerdata_source,'surface','bottom'); end %% Set which bins to plot @@ -34,6 +37,9 @@ colormap(1-gray(256)) hold on plot(mdata.Surface*1e6); +if isfield(mdata,'Bottom') + plot(mdata.Bottom*1e6); +end hold off; %% Elevation Correction Example @@ -54,6 +60,9 @@ colormap(1-gray(256)) hold on plot(mdata_WGS84.Elevation(1) - mdata_WGS84.Surface_Elev); + if isfield(mdata,'Bottom') + plot(mdata_WGS84.Elevation(1) - mdata_WGS84.Bottom_Elev); + end hold off; %% Plot versus WGS-84 elevation @@ -65,6 +74,9 @@ colormap(1-gray(256)) hold on plot(mdata_WGS84.Surface_Elev); + if isfield(mdata,'Bottom') + plot(mdata_WGS84.Bottom_Elev); + end hold off; else @@ -81,5 +93,3 @@ ylabel('Depth (m)') colormap(1-gray(256)) end - -return; diff --git a/cresis-toolbox/posting/post.m b/cresis-toolbox/posting/post.m index 80b458d9..496227ab 100644 --- a/cresis-toolbox/posting/post.m +++ b/cresis-toolbox/posting/post.m @@ -10,7 +10,9 @@ function post(param, param_override) %% General Setup % ===================================================================== -param = merge_structs(param, param_override); +if exist('param_override','var') + param = merge_structs(param, param_override); +end fprintf('=====================================================================\n'); fprintf('%s: %s (%s)\n', mfilename, param.day_seg, datestr(now)); @@ -28,19 +30,7 @@ function post(param, param_override) % Load the frames file frames = frames_load(param); - -if isempty(param.cmd.frms) - param.cmd.frms = 1:length(frames.frame_idxs); -end -% Remove frames that do not exist from param.cmd.frms list -[valid_frms,keep_idxs] = intersect(param.cmd.frms, 1:length(frames.frame_idxs)); -if length(valid_frms) ~= length(param.cmd.frms) - bad_mask = ones(size(param.cmd.frms)); - bad_mask(keep_idxs) = 0; - warning('Nonexistent frames specified in param.cmd.frms (e.g. frame "%g" is invalid), removing these', ... - param.cmd.frms(find(bad_mask,1))); - param.cmd.frms = valid_frms; -end +param.cmd.frms = frames_param_cmd_frms(param,frames); if isempty(param.cmd.frms) % No valid frames specified to post @@ -57,6 +47,10 @@ function post(param, param_override) if ~isfield(param.post,'csv_en') || isempty(param.post.csv_en) param.post.csv_en = 0; end +if param.post.concat_en && ~param.post.csv_en + warning('post.csv_en must be true when post.concat_en is true since concatenation uses the CSV files. Setting post.csv_en to true.'); + param.post.csv_en = 1; +end if ~isfield(param.post,'data_dirs') || isempty(param.post.data_dirs) % Cell array of data files to post. The first is the master and will be @@ -81,6 +75,12 @@ function post(param, param_override) param.post.echo_en = false; end +% echo_with_no_layer_en: only activated if echo_en is true. When true, an +% echogram image file will be created with no layers plotted on it. +if ~isfield(param.post,'echo_with_no_layer_en') || isempty(param.post.echo_with_no_layer_en) + param.post.echo_with_no_layer_en = true; +end + if ~isfield(param.post,'frm_types') || isempty(param.post.frm_types) param.post.frm_types = {-1,0,-1,-1,-1}; end @@ -161,7 +161,10 @@ function post(param, param_override) tmp_param = param; tmp_param.cmd.frms = []; % Get layer information for all frames layers = []; if param.post.layers_en - layers = opsLoadLayers(tmp_param, param.post.layers); + % Load layers to output + [layers,param.post.layers] = opsLoadLayers(tmp_param, param.post.layers); + % Setup output layerdata class + out_layers = layerdata(param, fullfile(post_path,'CSARP_layer')); end surface_layer = {opsLoadLayers(tmp_param, param.post.surface_source)}; @@ -369,7 +372,7 @@ function post(param, param_override) echo_param.fig_hand = []; end - echo_param.frm_id = sprintf('%s_03d', param.day_seg, frm); + echo_param.frm_id = sprintf('%s_%03d', param.day_seg, frm); echo_info = publish_echogram(echo_param,mdata,lay, surface_lay.layerData{1}); for handle_idx = 1:length(echo_info.h_layers) @@ -411,8 +414,10 @@ function post(param, param_override) set(echo_info.fig_hand(1),'PaperUnits','inches'); set(echo_info.fig_hand(1),param.post.echo.plot_params{1},param.post.echo.plot_params{2}); set(echo_info.fig_hand(1),'PaperOrientation','Portrait'); - fprintf(' Saving output %s\n', echo_fn); - print(echo_info.fig_hand(1),print_device,print_dpi,echo_fn); + if param.post.echo_with_no_layer_en + fprintf(' Saving output %s\n', echo_fn); + print(echo_info.fig_hand(1),print_device,print_dpi,echo_fn); + end if param.post.pdf_en % Remove current file and save new file @@ -550,14 +555,33 @@ function post(param, param_override) %% Post Loop: Create layer files if param.post.layers_en - % Copy layer file - layer_out_fn_dir = fullfile(post_path,sprintf('CSARP_layerData'),param.day_seg); - if ~exist(layer_out_fn_dir,'dir') - mkdir(layer_out_fn_dir) + % NEW FILE FORMAT + for layer_idx = 1:length(layers_to_post) + id = out_layers.get_id(layers_to_post{layer_idx}.name); + if isempty(id) + layer_organizer = []; + layer_organizer.age = layers_to_post{layer_idx}.age; + layer_organizer.age_source = {layers_to_post{layer_idx}.age_source}; + layer_organizer.lyr_desc = {layers_to_post{layer_idx}.desc}; + layer_organizer.lyr_group_name = {layers_to_post{layer_idx}.group_name}; + layer_organizer.lyr_name = {layers_to_post{layer_idx}.name}; + id = out_layers.insert_layers(layer_organizer); + end + out_layers.update_layer(frm, id, layers_to_post{layer_idx}.gps_time, ... + layers_to_post{layer_idx}.twtt,layers_to_post{layer_idx}.quality,layers_to_post{layer_idx}.type); + end + if 1 + % OLD FILE FORMAT + + % Copy layer file + layer_out_fn_dir = fullfile(post_path,sprintf('CSARP_layerData'),param.day_seg); + if ~exist(layer_out_fn_dir,'dir') + mkdir(layer_out_fn_dir) + end + layer_out_fn = fullfile(layer_out_fn_dir,sprintf('Data_%s.mat',frm_id)); + + save(layer_out_fn,'-struct','lay','GPS_time','Latitude','Longitude','Elevation','layerData'); end - layer_out_fn = fullfile(layer_out_fn_dir,sprintf('Data_%s.mat',frm_id)); - - save(layer_out_fn,'-struct','lay','GPS_time','Latitude','Longitude','Elevation','layerData'); end %% Post Loop: Copy data files @@ -611,6 +635,11 @@ function post(param, param_override) end end +% Save layer files +% ======================================================================= +if param.post.layers_en + out_layers.save(); +end % ======================================================================= %% Create segment CSV/KML files @@ -620,35 +649,25 @@ function post(param, param_override) fprintf(' Creating csv and kml files (%s)\n', datestr(now)); csv_dir = fullfile(post_path,'csv',param.day_seg); - kml_base_dir = fullfile(post_path,'kml'); - [csv_dir_path csv_dir_name] = fileparts(csv_dir); - out_fn = fullfile(csv_dir_path,sprintf('Data_%s.csv',csv_dir_name)); - concatenate_thickness_files(csv_dir,'*.csv',out_fn,','); - - % Create KML browse files for each segment - % Extract day_seg from filename - in_fn = out_fn; - [in_fn_dir in_fn_name] = fileparts(in_fn); - kml_out_fn = fullfile(kml_base_dir, ['Browse_' in_fn_name '.kml']); - day_seg = in_fn_name(6:end); - kml_write_cresis(in_fn, kml_out_fn, day_seg,'segment',[inf 40]); + if ~exist(csv_dir,'dir') + warning('No csv files to concatenate.'); + else + [csv_dir_path csv_dir_name] = fileparts(csv_dir); + out_fn = fullfile(csv_dir_path,sprintf('Data_%s.csv',csv_dir_name)); + concatenate_thickness_files(csv_dir,'*.csv',out_fn,','); + end % Repeat for csv_good and kml_good csv_dir = fullfile(post_path,'csv_good',param.day_seg); - kml_base_dir = fullfile(post_path,'kml_good'); - [csv_dir_path csv_dir_name] = fileparts(csv_dir); - out_fn = fullfile(csv_dir_path,sprintf('Data_%s.csv',csv_dir_name)); - concatenate_thickness_files(csv_dir,'*.csv',out_fn,','); - - % Create KML browse files for each segment - % Extract day_seg from filename - in_fn = out_fn; - [in_fn_dir in_fn_name] = fileparts(in_fn); - kml_out_fn = fullfile(kml_base_dir, ['Browse_' in_fn_name '.kml']); - day_seg = in_fn_name(6:end); - kml_write_cresis(in_fn, kml_out_fn, day_seg,'segment',[inf 40]); + if ~exist(csv_dir,'dir') + warning('No csv files to concatenate.'); + else + [csv_dir_path csv_dir_name] = fileparts(csv_dir); + out_fn = fullfile(csv_dir_path,sprintf('Data_%s.csv',csv_dir_name)); + concatenate_thickness_files(csv_dir,'*.csv',out_fn,','); + end end % ======================================================================= diff --git a/cresis-toolbox/posting/post_csv_kml.m b/cresis-toolbox/posting/post_csv_kml.m index dd497bd1..505ca438 100644 --- a/cresis-toolbox/posting/post_csv_kml.m +++ b/cresis-toolbox/posting/post_csv_kml.m @@ -11,7 +11,7 @@ %% User Settings -params = read_param_xls(ct_filename_param('snow_param_2017_Greenland_P3.xls')); +params = read_param_xls(ct_filename_param('snow_param_2019_SouthDakota_CESSNA.xls')); params = ct_set_params(params,'cmd.generic',1); params = ct_set_params(params,'cmd.generic',0,'cmd.notes','Do not process'); @@ -60,14 +60,15 @@ mkdir(kml_dir) end - records_fn = ct_filename_support(param, '', 'records'); - fprintf('Loading %s\n', records_fn); - records = load(records_fn); - UTC_sod = epoch_to_sod(records.gps_time,param.day_seg(1:8)); - - frames_fn = ct_filename_support(param, '', 'frames'); - fprintf('Loading %s\n', frames_fn); - load(frames_fn); + try + records = records_load(param); + UTC_sod = epoch_to_sod(records.gps_time,param.day_seg(1:8)); + + frames = frames_load(param); + catch ME + warning(ME.getReport); + continue; + end % Create new filename csv_fn = sprintf('Browse_Data_%s.csv',param.day_seg); diff --git a/cresis-toolbox/posting/publish_echogram.m b/cresis-toolbox/posting/publish_echogram.m index 13931e53..7ae65312 100644 --- a/cresis-toolbox/posting/publish_echogram.m +++ b/cresis-toolbox/posting/publish_echogram.m @@ -194,7 +194,7 @@ if length(param.er_depth) > 1 TWtime = genPropProfileFromPerm(param.er_depth,param.er_ice,1); profile_idxs = depth > 0 & depth < param.er_depth(end); - depth_time(profile_idxs) = interp1(param.er_depth, [0; TWtime], depth(profile_idxs)); + depth_time(profile_idxs) = interp1(param.er_depth, TWtime, depth(profile_idxs)); else TWtime = 0; end @@ -641,8 +641,8 @@ end end else - xtl = create_standard_x_labels(mdata.Latitude,mdata.Longitude,mdata.Elevation,param.num_x_tics); - add_x_labels(ah_echo,xtl,{'dist','lat','lon'}); + xtl = xlabel_create(mdata.Latitude,mdata.Longitude,mdata.Elevation,param.num_x_tics); + xlabel_add(ah_echo,xtl,{'dist','lat','lon'}); set(ah_echo_time,'Units','normalized'); set(ah_echo,'Units','normalized'); set(ah_echo_time,'Position',get(ah_echo,'Position')); diff --git a/cresis-toolbox/posting/run_check_nsidc_files.m b/cresis-toolbox/posting/run_check_nsidc_files.m new file mode 100644 index 00000000..0966a3ef --- /dev/null +++ b/cresis-toolbox/posting/run_check_nsidc_files.m @@ -0,0 +1,54 @@ +% script run_check_nsidc_files +% +% Script for checking if the number of nsidc files generated matches with expected +% Authors: Jilu Li +% +% See also: check_nsidc_files +%% User Setup +% ===================================================================== +param_override = []; +params = read_param_xls(ct_filename_param('rds_param_2018_Greenland_P3.xls')); + +% Example to run a specific segment and frame by overriding parameter spreadsheet values +% params = ct_set_params(params,'cmd.generic',0); +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20181027_02'); +param_override.nsidc_file_folder = '/cresis/snfs1/scratch/jliwestc/nsidc/2018_Greenland_P3/rds/IRMCR1B_Files/Data_2018_GR'; +param_override.nsidc_SP_file_folder = '/cresis/snfs1/scratch/jliwestc/nsidc/2018_Greenland_P3/rds/IRMCR1B_Files/Spatial_Premet_Files'; +param_override.nsidc_file_type = 'L1B'; % L1B or L2 + +%% Automated Section +% ===================================================================== +% Input checking +global gRadar; +if exist('param_override','var') + param_override = merge_structs(gRadar,param_override); +else + param_override = gRadar; +end + +%% Process each of the segments +% ===================================================================== +all_file_num = 0; % (not include Spatial and Premet files) +all_nc_file_num = 0; +all_supplement_file_num = 0; +all_SP_file_num = 0; +all_map_file_num = 0; +all_img_file_num = 0; +all_csv_file_num = 0; +for param_idx = 1:length(params) + param = params(param_idx); + if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic + continue; + end + [total_num,nc_num,supplement_num,SP_num,map_num,img_num,csv_num] = check_nsidc_files(param,param_override); + all_file_num = all_file_num + total_num; + all_nc_file_num = all_nc_file_num + nc_num; + all_supplement_file_num = all_supplement_file_num + supplement_num; + all_SP_file_num = all_SP_file_num + SP_num; + all_map_file_num = all_map_file_num + map_num; + all_img_file_num = all_img_file_num + img_num; + all_csv_file_num = all_csv_file_num + csv_num; +end +sprintf('number of files to deliver = %d\nnumber of nc files = %d\nnumber of supplement files = %d\nnumber of Spatial and Premet files = %d\nnumber of frames or map files = %d\nnumber of echogram images = %d\nnumber of csv files = %d\n',... + all_file_num, all_nc_file_num,all_supplement_file_num,all_SP_file_num,all_map_file_num, all_img_file_num,all_csv_file_num) + diff --git a/cresis-toolbox/posting/run_echo_plot.m b/cresis-toolbox/posting/run_echo_plot.m new file mode 100644 index 00000000..3b4aaf0a --- /dev/null +++ b/cresis-toolbox/posting/run_echo_plot.m @@ -0,0 +1,91 @@ +% script run_echo_plot.m +% +% Example and test script for echo_plot.m +% +% Author: John Paden, Dhagash Kapadia +% +% See also: echo_plot.m, echo_plot_profile.m, elevation_compensation.m, +% load_L1B.m, run_echo_plot.m + +%% Choose example (enable just one) +% test_case_str = 'no_layers'; +% test_case_str = 'no_layers_fn'; +% test_case_str = 'no_layers_twtt_fn'; +% test_case_str = 'default_layers'; +% test_case_str = 'nonstandard_surface'; +% test_case_str = 'no_plot'; +% test_case_str = 'posted_layers'; +% test_case_str = 'arbitrary_layers'; +test_case_str = 'uniform_sampling'; +% test_case_str = 'layer_params'; +% test_case_str = 'wildcard_layers'; +% test_case_str = 'surface_nan'; + +%% Examples + +param = read_param_xls(ct_filename_param('snow_param_2012_Greenland_P3.xls'),'20120330_04'); +[mdata,echo_fn] = echo_load(param,'CSARP_post/qlook',3); + +if strcmpi(test_case_str,'no_layers') + [mdata,x_axis,y_axis,surf_comp,layers_comp,h] = echo_plot(mdata); +end + +if strcmpi(test_case_str,'no_layers_fn') + [mdata,x_axis,y_axis,surf_comp,layers_comp,h] = echo_plot(echo_fn); +end + +if strcmpi(test_case_str,'no_layers_twtt_fn') + [mdata,x_axis,y_axis,surf_comp,layers_comp,h] = echo_plot(echo_fn,'TWTT'); +end + +if strcmpi(test_case_str,'default_layers') + [mdata,x_axis,y_axis,surf_comp,layers_comp,h] = echo_plot(echo_fn, [], 'rds_layers'); +end + +if strcmpi(test_case_str,'nonstandard_surface') + echo_plot_param = echo_plot_profile('DEPTH'); + echo_plot_param.surf.source = 0; + echo_plot_param.h_fig = 1; + [mdata,x_axis,y_axis,surf_comp,layers_comp,h] = echo_plot(echo_fn, echo_plot_param, 'rds_layers'); + echo_plot_param.surf.source = 1; + echo_plot_param.h_fig = 2; + [mdata,x_axis,y_axis,surf_comp,layers_comp,h] = echo_plot(echo_fn, echo_plot_param, 'rds_layers'); +end + +if strcmpi(test_case_str,'no_plot') + % Just loads the data (twtt) and provides all the variables to plot + [mdata,x_axis,y_axis,surf_comp,layers_comp] = echo_plot(mdata,'NONE','rds_layers'); +end + +if strcmpi(test_case_str,'posted_layers') + layer_params = struct('name',{'surface','bottom'},'layerdata_source','layer_old','existence_check',false); + [data, layers_comp, h] = echo_plot(echo_fn, [], layer_params); +end + +if strcmpi(test_case_str,'arbitrary_layers') + layer_names = {'surface','bottom','layer_01'}; + [mdata,x_axis,y_axis,surf_comp,layers_comp,h] = echo_plot(echo_fn, setfield(echo_plot_profile('DEPTH'),'h_fig',1), struct('name',layer_names, 'source', 'layerdata','existence_check',false)); + [mdata,x_axis,y_axis,surf_comp,layers_comp,h] = echo_plot(echo_fn, setfield(echo_plot_profile('RANGE'),'h_fig',2), struct('name',layer_names, 'source', 'layerdata','existence_check',false)); + [mdata,x_axis,y_axis,surf_comp,layers_comp,h] = echo_plot(echo_fn, setfield(echo_plot_profile('TWTT'),'h_fig',3), struct('name',layer_names, 'source', 'layerdata','existence_check',false)); + [mdata,x_axis,y_axis,surf_comp,layers_comp,h] = echo_plot(echo_fn, setfield(echo_plot_profile('WGS84'),'h_fig',4), struct('name',layer_names, 'source', 'layerdata','existence_check',false)); +end + +if strcmpi(test_case_str,'uniform_sampling') + [mdata,x_axis,y_axis,surf_comp,layers_comp,h] = echo_plot(echo_fn, setfield(setfield(echo_plot_profile('WGS84'),'h_fig',4),'mode_x_axis','ALONG_TRACK')); +end + +if strcmpi(test_case_str,'layer_params') + layer_params = struct('source','layerdata','name','surface'); + [mdata,x_axis,y_axis,surf_comp,layers_comp,h] = echo_plot(echo_fn, [], layer_params); +end + +if strcmpi(test_case_str,'wildcard_layers') + layer_params = struct('source','layerdata','name',{'surface',''},'regexp',{[],'layer.*'},'existence_check',false); + [mdata,x_axis,y_axis,surf_comp,layers_comp,h] = echo_plot(echo_fn, 'TWTT', layer_params); +end + +if strcmpi(test_case_str,'surface_nan') + [mdata,x_axis,y_axis,surf_comp,layers_comp,h] = echo_plot(mdata, setfield(echo_plot_profile('DEPTH'),'h_fig',1), struct('name',{'surface', 'bottom'}, 'source', 'layerdata')); + mdata.Surface(:) = NaN; + [mdata,x_axis,y_axis,surf_comp,layers_comp,h] = echo_plot(mdata, setfield(echo_plot_profile('DEPTH'),'h_fig',2), struct('name',{'surface', 'bottom'}, 'source', 'layerdata')); +end diff --git a/cresis-toolbox/posting/run_echogram_to_jpeg.m b/cresis-toolbox/posting/run_echo_to_jpeg.m similarity index 100% rename from cresis-toolbox/posting/run_echogram_to_jpeg.m rename to cresis-toolbox/posting/run_echo_to_jpeg.m diff --git a/cresis-toolbox/posting/run_echo_to_jpeg_full_resolution.m b/cresis-toolbox/posting/run_echo_to_jpeg_full_resolution.m new file mode 100644 index 00000000..78d6c68d --- /dev/null +++ b/cresis-toolbox/posting/run_echo_to_jpeg_full_resolution.m @@ -0,0 +1,87 @@ +% script echogram_to_jpeg +% +% Converts specified echogram image .mat files into small 8-bit jpegs after +% truncation and decimation. +% +% Example to load datafiles: +% +% mdata_orig = load('/cresis/snfs1/dataproducts/public/data/accum/2018_Antarctica_TObas/CSARP_standard/20190205_01/Data_20190205_01_010.mat'); +% mdata = load('/cresis/snfs1/dataproducts/public/data/accum/2018_Antarctica_TObas/CSARP_small_mat/20190205_01/Data_20190205_01_010.mat'); +% mdata.Data = imread('/cresis/snfs1/dataproducts/public/data/accum/2018_Antarctica_TObas/CSARP_small_jpg/20190205_01/Data_20190205_01_010.jpg'); +% mdata.Data = single(mdata.Data)/255*mdata.Data_Scale + mdata.Data_Offset; +% +% figure(1); clf; +% imagesc(mdata.Data); +% colormap(1-gray(256)); +% hold on; +% plot(mdata.Surface); +% plot(mdata.Bottom); +% +% figure(2); clf; +% plot(mdata_orig.Time, 10*log10(mdata_orig.Data(:,1))); +% hold on; +% plot(mdata.Time, mdata.Data(:,1)); +% +% Author: John Paden +% +% See also: run_post.m, post.m, run_echogram_to_jpeg.m, +% echogram_to_jpeg.m + +%% User Setup +% ===================================================================== + +param_override = []; + +params = read_param_xls(ct_filename_param('snow_param_2020_SouthDakota_N1KU.xls')); + +% params = ct_set_params(params,'cmd.generic',1); +% params = ct_set_params(params,'cmd.generic',0,'cmd.notes','do not process'); + +params = ct_set_params(params,'cmd.generic',0); +params = ct_set_params(params,'cmd.generic',1,'day_seg','20210205_01'); +params = ct_set_params(params,'cmd.frms',30); + +% Input images +param_override.echogram_to_jpeg.data_type = 'qlook_noise_deconv'; + +% Input top/bottom layers +param_override.echogram_to_jpeg.layers = [struct('name', 'surface', 'source', 'layerdata', 'layerdata_source', 'layer', 'existence_check', false) ... + struct('name', 'bottom', 'source', 'layerdata', 'layerdata_source', 'layer', 'existence_check', false)]; + +% Output path +param_override.echogram_to_jpeg.mat_out_path = 'CSARP_post/small_mat'; +param_override.echogram_to_jpeg.jpeg_out_path = 'CSARP_post/small_jpg'; + +% Truncation parameters +param_override.echogram_to_jpeg.N_before_surface = 500; +param_override.echogram_to_jpeg.N_after_surface = 1500; +param_override.echogram_to_jpeg.N_after_bottom = 100; + +% Decimation parameters +param_override.echogram_to_jpeg.decimate_fasttime = 1; +param_override.echogram_to_jpeg.decimate_slowtime = 1; + +% Combined image (0) or img_II (II) +param_override.echogram_to_jpeg.data_img = 1; + +%% Automated Section +% ===================================================================== + +% Input checking +global gRadar; +if exist('param_override','var') + param_override = merge_structs(gRadar,param_override); +else + param_override = gRadar; +end + +%% Process each of the segments +% ===================================================================== +for param_idx = 1:length(params) + param = params(param_idx); + if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic + continue; + end + %echo_to_jpeg(param,param_override); + echo_to_jpeg +end diff --git a/cresis-toolbox/posting/run_load_data_by_gps_time.m b/cresis-toolbox/posting/run_load_data_by_gps_time.m index ae3214b8..1d878bc8 100644 --- a/cresis-toolbox/posting/run_load_data_by_gps_time.m +++ b/cresis-toolbox/posting/run_load_data_by_gps_time.m @@ -81,10 +81,14 @@ gaps_dist = [100 30]; -surface_source = struct('name','surface','source','layerData', 'layerdata_source','layerData_koenig'); % <== CHANGE HERE +% surface_source: location of surface information (used in elevation +% compensation). For example: +% struct('name','surface','source','layerdata', 'layerdata_source','layer') +% struct('name','surface','source','ops') +surface_source = struct('name','surface','source','layerdata', 'layerdata_source','layer'); % <== CHANGE HERE % param.img_name: output data product image. For example: -% '': combined product, 'img_01_', , 'img_02_' +% '': combined product, 'img_01_': image 1, , 'img_02_': image 2, etc. param.img_name = ''; if echo_param.elev_comp == 3 @@ -130,10 +134,10 @@ % param.layer_params: set to plot layers on echograms layer_params = []; idx = 0; -if 1 % Enable to plot layers on echograms - layer_params = struct('name','surface','source','layerData','layerdata_source','layerData_koenig'); +if 0 % Enable to plot layers on echograms + layer_params = struct('name','surface','source','layerdata','layerdata_source','layerData_koenig'); for idx = 2:30 - layer_params(end+1) = struct('name',sprintf('Koenig_%d',idx),'source','layerData','layerdata_source','layerData_koenig'); + layer_params(end+1) = struct('name',sprintf('Koenig_%d',idx),'source','layerdata','layerdata_source','layerData_koenig'); end end param.layer_params = layer_params; @@ -256,10 +260,10 @@ % param.layer_params: set to plot layers on echograms layer_params = []; idx = 0; - if 1 % Enable to plot layers on echograms - layer_params = struct('name','surface','source','layerData','layerdata_source','layerData_koenig'); + if 0 % Enable to plot layers on echograms + layer_params = struct('name','surface','source','layerdata','layerdata_source','layerData_koenig'); for idx = 2:30 - layer_params(end+1) = struct('name',sprintf('Koenig_%d',idx),'source','layerData','layerdata_source','layerData_koenig'); + layer_params(end+1) = struct('name',sprintf('Koenig_%d',idx),'source','layerdata','layerdata_source','layerData_koenig'); end end param.layer_params = layer_params; @@ -508,16 +512,14 @@ master.Elevation = ds.Elevation; if ds.Latitude<0 - ds.param_records.post.location = 'antarctic'; + ds.param_records.post.ops.location = 'antarctic'; else - ds.param_records.post.location = 'arctic'; + ds.param_records.post.ops.location = 'arctic'; end global gRadar; + param = merge_structs(param,ds.param_records); param = merge_structs(param,gRadar); - param.day_seg = ds.frm_id(1:11); - param.cmd.frms = ds.start_frame:ds.stop_frame; - param.post.ops.location = ds.param_records.post.location; surface_layer = {opsLoadLayers(param, surface_source)}; diff --git a/cresis-toolbox/posting/run_post.m b/cresis-toolbox/posting/run_post.m index 9d6606aa..f8cc80dd 100644 --- a/cresis-toolbox/posting/run_post.m +++ b/cresis-toolbox/posting/run_post.m @@ -44,7 +44,7 @@ for param_idx = 1:length(params) param = params(param_idx); cmd = param.cmd; - if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic + if ct_generic_en(param) continue; end if param.post.concat_en diff --git a/cresis-toolbox/processing/analysis.m b/cresis-toolbox/processing/analysis.m index 13ee94e3..e557837e 100644 --- a/cresis-toolbox/processing/analysis.m +++ b/cresis-toolbox/processing/analysis.m @@ -26,7 +26,7 @@ fprintf('%s: %s (%s)\n', mfilename, param.day_seg, datestr(now)); fprintf('=====================================================================\n'); -%% Input Checks +%% Input Checks: cmd % ===================================================================== if ~isempty(param.cmd.frms) @@ -34,15 +34,13 @@ param.cmd.frms = []; % All frames end +%% Input Checks: analysis +% ===================================================================== + if ~isfield(param,'analysis') || isempty(param.analysis) error('The analysis field (worksheet) is missing.'); end -if ~isfield(param.analysis,'bit_mask') || isempty(param.analysis.bit_mask) - % Set to 3 to mask out stationary and bad records (useful for coherent noise estimation on ground based data that may have stationary records) - param.analysis.bit_mask = 1; -end - if ~isfield(param.analysis,'block_size') || isempty(param.analysis.block_size) param.analysis.block_size = 6000; end @@ -64,15 +62,24 @@ param.analysis.presums = 1; end +if ~isfield(param.analysis,'resample') || isempty(param.analysis.resample) + param.analysis.resample = [1 1; 1 1]; +end +if ~isequal(param.analysis.resample(2,1:2),[1 1]) + error('The bottom row of param.analysis.resample must be [1 1] because resampling in the along-track is not supported. Use line_rng and dline instead.'); +end + if ~isfield(param.analysis,'surf_layer') || isempty(param.analysis.surf_layer) param.analysis.surf_layer.name = 'surface'; - param.analysis.surf_layer.source = 'layerData'; + param.analysis.surf_layer.source = 'layerdata'; end % Never check for the existence of files param.analysis.surf_layer.existence_check = false; +%% Input Checks: analysis.cmd +% ===================================================================== % For each command in the list, set its default settings -enabled_cmds = 0; +enabled_cmds = {}; for cmd_idx = 1:length(param.analysis.cmd) cmd = param.analysis.cmd{cmd_idx}; @@ -82,7 +89,6 @@ if ~cmd.en continue; end - enabled_cmds = enabled_cmds + 1; if ~isfield(cmd,'out_path') || isempty(cmd.out_path) cmd.out_path = param.analysis.out_path; @@ -161,7 +167,89 @@ cmd.method = lower(cmd.method); switch cmd.method case {'burst_noise'} + % Set defaults for burst noise analysis method + + % max_bad_waveforms: wf-adc recorded waveforms/range-lines with burst + % noise detected are saved, but only up to this many. Set to inf to + % save all waveforms with burst noise detected. + if ~isfield(cmd,'max_bad_waveforms') || isempty(cmd.max_bad_waveforms) + cmd.max_bad_waveforms = 100; + end + + % noise_fh: This function is used to create the temporary variable + % data_noise in analysis burst. This output is available to test_fh + % and threshold_fh. % + % Examples: + % cmd.noise_fh{img} = @(raw_data,wfs) lp(fir_dec(fir_dec(abs(raw_data.').^2,ones(1,11)/11,1).',ones(1,101)/101,1)); + % cmd.noise_fh{img} = @(raw_data,wfs) lp(medfilt2(abs(raw_data).^2,[11 101])); + % cmd.noise_fh{img} = @(raw_data,wfs) []; + if ~isfield(cmd,'noise_fh') || isempty(cmd.noise_fh) + for img = 1:length(param.analysis.imgs) + cmd.noise_fh{img} = @(raw_data,wfs) lp(fir_dec(fir_dec(abs(raw_data.').^2,ones(1,11)/11,1).',ones(1,101)/101,1)); + end + end + + % signal_fh: This function is used to create the temporary variable + % data_signal in analysis burst. This output is available to test_fh + % and threshold_fh. + % + % Examples: + % cmd.signal_fh{img} = @(raw_data,wfs) lp(fir_dec(abs(raw_data.').^2,ones(1,11)/11,1).'); + % cmd.signal_fh{img} = @(raw_data,wfs) lp(abs(raw_data).^2,1)); + % cmd.signal_fh{img} = @(raw_data,wfs) lp(abs(fft(raw_data(300:end,:))).^2); + % cmd.signal_fh{img} = @(raw_data,wfs) []; + if ~isfield(cmd,'signal_fh') || isempty(cmd.signal_fh) + for img = 1:length(param.analysis.imgs) + cmd.signal_fh{img} = @(raw_data,wfs) lp(fir_dec(abs(raw_data.').^2,ones(1,11)/11,1).'); + end + end + + % test_fh: This function is used to create the output variable + % test_metric in analysis burst. This output is available to + % threshold_fh and is also stored in the output file. + % + % Examples: + % cmd.test_fh{img} = @(data_signal,data_noise,wfs) max(data_signal-data_noise,[],1); + % cmd.test_fh{img} = @(data_signal,data_noise,wfs) data_signal-data_noise; + % cmd.test_fh{img} = @(data_signal,data_noise,wfs)% max(data_signal(10:end-9,:))-median(data_signal(10:end-9,:)); + % cmd.test_fh{img} = @(raw_data,wfs) []; + if ~isfield(cmd,'test_fh') || isempty(cmd.test_fh) + for img = 1:length(param.analysis.imgs) + cmd.test_fh{img} = @(data_signal,data_noise,wfs) max(data_signal-data_noise,[],1); + end + end + + % test_fh: This function is used to create the temporary variable + % bad_samples in analysis burst. If the output is a row vector, then + % it is assumed to be a mask of the bad records (interpretation is + % that the entire record is bad). If the output is a matrix, then it + % is assumed to be a mask of the bad samples (interpretation is that + % just those samples are bad and not the whole record). + % + % Examples: + % cmd.threshold_fh{img} = @(data_signal,data_noise,test_metric,wfs) data_noise > 80; + % cmd.threshold_fh{img} = @(data_signal,data_noise,test_metric,wfs) test_metric > 20; + % cmd.threshold_fh{img} = @(data_signal,data_noise,test_metric,wfs) repmat(data_signal-data_noise > 20,[wfs.Nt_raw 1]); + % cmd.threshold_fh{img} = @(data_signal,data_noise,test_metric,wfs) max(data_signal(10:end-9,:))-median(data_signal(10:end-9,:)); + if ~isfield(cmd,'threshold_fh') || isempty(cmd.threshold_fh) + for img = 1:length(param.analysis.imgs) + cmd.threshold_fh{img} = @(data_signal,data_noise,test_metric,wfs) test_metric > 20; + end + end + + % valid_bins: Restricts the range of bins where burst detections are + % allowed to occur. The default is [1 inf] which allows burst + % detections anywhere in the range line. Typical usage would be to + % exclude the feedthrough or nadir surface signal that may be large + % and variable and lead to false alarms if it is nonstationary. + if ~isfield(cmd,'valid_bins') || isempty(cmd.valid_bins) + cmd.valid_bins = {}; + for img = 1:length(param.analysis.imgs) + cmd.valid_bins{img} = [1 inf]; + end + end + case {'coh_noise'} % Set defaults for coherent noise analysis method @@ -173,6 +261,15 @@ param.analysis.block_size, cmd.block_ave); end + % distance_weight: default is false; if true, then the coherent + % averaging is weighted by the distance travelled. This is primarily + % useful for ground based traverses where long stops can bias the + % coherent noise estimate. Setting this to true will reduce the + % affect the long stops have on the estimated coherent noise mean. + if ~isfield(cmd,'distance_weight') || isempty(cmd.distance_weight) + cmd.distance_weight = false; + end + if ~isfield(cmd,'mag_en') || isempty(cmd.mag_en) % Default is to collect magnitude sums (coh_ave_mag) in addition to % phase-coherent sums (coh_ave) @@ -196,6 +293,12 @@ end end + if ~isfield(cmd,'threshold_coh_ave') + cmd.threshold_coh_ave = 1; + elseif mod(cmd.threshold_coh_ave-1,2) + error('param.analysis.cmd{%d}.threshold_coh_ave must be odd; currently %g.', cmd_idx, cmd.threshold_coh_ave); + end + if ~isfield(cmd,'threshold_removeDC') || isempty(cmd.threshold_removeDC) % Default is to not remove slow-time DC before determining good % samples to use in coh_ave and coh_ave_mag @@ -209,30 +312,64 @@ case {'specular'} % Set defaults for specular analysis method + % cmd.gps_times: vector of GPS times in ANSI C format (seconds since + % Jan 1, 1970) for which the STFT block will automatically have its + % waveform extracted even if the cmd.threshold is not exceeded. if ~isfield(cmd,'gps_times') || isempty(cmd.gps_times) cmd.gps_times = []; end + % cmd.max_rlines: maximum number of STFT waveforms to extract (so + % even if there are more than cmd.max_rlines blocks which detect a + % coherent/specular scatterer only the first cmd.max_rlines will be + % extracted for deconvolution) if ~isfield(cmd,'max_rlines') || isempty(cmd.max_rlines) cmd.max_rlines = 10; end + % cmd.rlines: STFT block size, the data are broken into 50% + % overlapping blocks of this length to detect coherent/specular + % scatterers using a short time Fourier transform if ~isfield(cmd,'rlines') || isempty(cmd.rlines) cmd.rlines = 128; end + % cmd.noise_doppler_bins: bins of STFT to use for clutter power + % detection + guard = round(cmd.rlines/32); if ~isfield(cmd,'noise_doppler_bins') || isempty(cmd.noise_doppler_bins) - cmd.noise_doppler_bins = [12:cmd.rlines-11]; + cmd.noise_doppler_bins = [1+3*guard:cmd.rlines-3*guard]; end + % cmd.signal_doppler_bins: bins of STFT to use for signal power + % estimation if ~isfield(cmd,'signal_doppler_bins') || isempty(cmd.signal_doppler_bins) - cmd.signal_doppler_bins = [1:4 cmd.rlines+(-3:0)]; + cmd.signal_doppler_bins = [1:guard cmd.rlines+(-guard+1:0)]; end + % cmd.threshold: peakiness threshold to decide whether or not to + % extract the waveform from a particular STFT block. This is in dB + % and is the ratio (peak signal to mean noise/clutter power): + % peakiness = lp(max(abs(H(cmd.signal_doppler_bins,:)).^2) ./ mean(abs(H(cmd.noise_doppler_bins,:)).^2)); if ~isfield(cmd,'threshold') || isempty(cmd.threshold) cmd.threshold = 40; end + % cmd.peak_sgolay_filt: cell array with 2 elements containing the + % 2nd and 3rd input arguments to sgolayfilt that is used to filter + % the peak values in the along-track dimension. This ensures that the + % frame size to sgolayfilt.m is odd and is approximately 40% of the + % length of the cmd.rlines. + if ~isfield(cmd,'peak_sgolay_filt') || isempty(cmd.peak_sgolay_filt) + cmd.peak_sgolay_filt = {3,round(0.2*cmd.rlines)*2+1}; + end + if length(cmd.peak_sgolay_filt) < 2 + cmd.peak_sgolay_filt{2} = round(0.2*cmd.rlines)*2+1; + end + if ~mod(cmd.peak_sgolay_filt{2},2) + cmd.peak_sgolay_filt{2} = cmd.peak_sgolay_filt{2}+1; + end + case {'statistics'} % Set defaults for statistical analysis method @@ -290,13 +427,31 @@ % Update the command structure param.analysis.cmd{cmd_idx} = cmd; + enabled_cmds{end+1} = cmd.method; end -if enabled_cmds == 0 +if isempty(enabled_cmds) ctrl_chain = {}; return; end +if ~isfield(param.analysis,'bit_mask') || isempty(param.analysis.bit_mask) + % Set to 3 to mask out stationary and bad records (useful for coherent noise estimation on ground based data that may have stationary records) + if all(strcmp('coh_noise',enabled_cmds)) + % Remove bad records (bit_mask==1), remove stationary records + % (bit_mask==2), and remove bad records (bit_mask==4) + param.analysis.bit_mask = 1 + 2 + 4; + elseif all(strcmp('burst_noise',enabled_cmds)) + % Remove bad records (bit_mask==1), leave stationary records + % (bit_mask==2), and leave bad records (bit_mask==4) + param.analysis.bit_mask = 1; + else + % Remove bad records (bit_mask==1), leave stationary records + % (bit_mask==2), and remove bad records (bit_mask==4) + param.analysis.bit_mask = 1 + 4; + end +end + %% Setup processing % ===================================================================== @@ -304,8 +459,7 @@ [~,~,radar_name] = ct_output_dir(param.radar_name); % Load records file -records_fn = ct_filename_support(param,'','records'); -records = load(records_fn); +records = records_load(param); % Apply presumming if param.analysis.presums > 1 records.lat = fir_dec(records.lat,param.analysis.presums); @@ -315,7 +469,6 @@ records.pitch = fir_dec(records.pitch,param.analysis.presums); records.heading = fir_dec(records.heading,param.analysis.presums); records.gps_time = fir_dec(records.gps_time,param.analysis.presums); - records.surface = fir_dec(records.surface,param.analysis.presums); end % Compute all estimates with pulse compressed numbers even though raw data @@ -342,6 +495,11 @@ cpu_time_mult = 4e-8; mem_mult = 17; +elseif strcmpi(radar_name,'snow9') + total_num_sam = 45000 * ones(size(param.analysis.imgs)); + cpu_time_mult = 4e-8; + mem_mult = 17; + else error('radar_name %s not supported yet.', radar_name); @@ -364,7 +522,15 @@ switch cmd.method case {'burst_noise'} - % + for wf_adc = param.analysis.cmd{cmd_idx}.wf_adcs{img}(:).' + wf = param.analysis.imgs{img}(wf_adc,1); + adc = param.analysis.imgs{img}(wf_adc,2); + out_fn = fullfile(out_segment_fn_dir,sprintf('burst_noise_%s_wf_%d_adc_%d.mat',param.day_seg,wf,adc)); + combine_file_success{end+1} = out_fn; + if ~ctrl.cluster.rerun_only && exist(out_fn,'file') + ct_file_lock_check(out_fn,3); + end + end case {'coh_noise'} for wf_adc = param.analysis.cmd{cmd_idx}.wf_adcs{img}(:).' @@ -474,7 +640,7 @@ dparam.file_success = {}; success_error = 64; % Loading in the data: cpu_time and mem - dparam.mem = 250e6; + dparam.mem = 500e6; for img = 1:length(param.analysis.imgs) dparam.cpu_time = dparam.cpu_time + 10 + size(param.analysis.imgs{img},1)*Nx*total_num_sam(img)*log2(Nx)*cpu_time_mult; dparam.mem = dparam.mem + size(param.analysis.imgs{img},1)*Nx*total_num_sam(img)*mem_mult; @@ -498,7 +664,20 @@ % Process commands switch cmd.method case {'burst_noise'} - % + for wf_adc = param.analysis.cmd{cmd_idx}.wf_adcs{img}(:).' + wf = param.analysis.imgs{img}(wf_adc,1); + adc = param.analysis.imgs{img}(wf_adc,2); + out_fn = fullfile(tmp_out_fn_dir,sprintf('burst_noise_wf_%d_adc_%d_%d_%d.mat',wf,adc,actual_cur_recs)); + dparam.file_success{end+1} = out_fn; + if ~ctrl.cluster.rerun_only && exist(out_fn,'file') + delete(out_fn); + end + dparam.cpu_time = dparam.cpu_time + 10 + Nx*total_num_sam(img)*log2(Nx)*cpu_time_mult; + dparam.mem = max(dparam.mem,data_load_memory + Nx*total_num_sam(img)*mem_mult); + if isempty(cmd_method_str) + cmd_method_str = '_burst_noise'; + end + end case {'coh_noise'} for wf_adc = param.analysis.cmd{cmd_idx}.wf_adcs{img}(:).' @@ -590,8 +769,8 @@ % Rerun only mode: Test to see if we need to run this task % ================================================================= - dparam.notes = sprintf('%s%s:%s:%s %s %d of %d recs %d-%d', ... - mfilename, cmd_method_str, param.radar_name, param.season_name, param.day_seg, ... + dparam.notes = sprintf('%s%s %s:%s:%s %s %d of %d recs %d-%d', ... + mfilename, cmd_method_str, param.analysis.cmd{1}.out_path, param.radar_name, param.season_name, param.day_seg, ... break_idx, length(breaks), actual_cur_recs); if ctrl.cluster.rerun_only % If we are in rerun only mode AND the analysis task file success @@ -658,13 +837,17 @@ switch cmd.method case {'burst_noise'} - % + for wf_adc = param.analysis.cmd{cmd_idx}.wf_adcs{img}(:).' + num_sam_hint = 1; + sparam.cpu_time = sparam.cpu_time + Nx*num_sam_hint*log2(Nx)*cpu_time_mult; + sparam.mem = max(sparam.mem,500e6 + records_var.bytes + Nx*num_sam_hint*mem_mult); + end case {'coh_noise'} Nx_cmd = Nx / cmd.block_ave; for wf_adc = param.analysis.cmd{cmd_idx}.wf_adcs{img}(:).' sparam.cpu_time = sparam.cpu_time + Nx_cmd*num_sam_hint*log2(Nx_cmd)*cpu_time_mult; - sparam.mem = max(sparam.mem,350e6 + records_var.bytes + Nx_cmd*num_sam_hint*mem_mult); + sparam.mem = max(sparam.mem,500e6 + records_var.bytes + Nx_cmd*num_sam_hint*mem_mult); end case {'qlook'} @@ -677,14 +860,14 @@ Nx_cmd = Nx / param.analysis.block_size * cmd.max_rlines; for wf_adc = param.analysis.cmd{cmd_idx}.wf_adcs{img}(:).' sparam.cpu_time = sparam.cpu_time + Nx_cmd*num_sam_hint*log2(Nx_cmd)*cpu_time_mult; - sparam.mem = max(sparam.mem,350e6 + records_var.bytes + Nx_cmd*num_sam_hint*mem_mult*1.5); + sparam.mem = max(sparam.mem,500e6 + records_var.bytes + Nx_cmd*num_sam_hint*mem_mult*1.5); end case {'statistics'} Nx_cmd = Nx / cmd.block_ave; for wf_adc = param.analysis.cmd{cmd_idx}.wf_adcs{img}(:).' sparam.cpu_time = sparam.cpu_time + Nx_cmd*num_sam_hint*log2(Nx_cmd)*cpu_time_mult; - sparam.mem = max(sparam.mem,350e6 + records_var.bytes + Nx_cmd*num_sam_hint*mem_mult); + sparam.mem = max(sparam.mem,500e6 + records_var.bytes + Nx_cmd*num_sam_hint*mem_mult); end case {'waveform'} @@ -696,20 +879,18 @@ end for wf_adc = param.analysis.cmd{cmd_idx}.wf_adcs{img}(:).' sparam.cpu_time = sparam.cpu_time + Nx_cmd*Nt*log2(Nx_cmd)*cpu_time_mult; - sparam.mem = max(sparam.mem,350e6 + records_var.bytes + Nx_cmd*Nt*mem_mult); + sparam.mem = max(sparam.mem,500e6 + records_var.bytes + Nx_cmd*Nt*mem_mult); end end end end sparam.file_success = combine_file_success; -sparam.notes = sprintf('%s:%s:%s %s combine', ... - mfilename, param.radar_name, param.season_name, param.day_seg); +sparam.notes = sprintf('%s %s:%s:%s %s combine', ... + mfilename, param.analysis.cmd{1}.out_path, param.radar_name, param.season_name, param.day_seg); ctrl = cluster_new_task(ctrl,sparam,[]); ctrl_chain{end+1} = ctrl; fprintf('Done %s\n', datestr(now)); - -return diff --git a/cresis-toolbox/processing/analysis_combine_task.m b/cresis-toolbox/processing/analysis_combine_task.m index 8e7c81d1..971560ce 100644 --- a/cresis-toolbox/processing/analysis_combine_task.m +++ b/cresis-toolbox/processing/analysis_combine_task.m @@ -16,8 +16,7 @@ %% Setup % Load records file -records_fn = ct_filename_support(param,'','records'); -records = load(records_fn); +records = records_load(param); % Apply presumming if param.analysis.presums > 1 records.lat = fir_dec(records.lat,param.analysis.presums); @@ -127,7 +126,7 @@ spec = []; spec.deconv_fc = []; spec.deconv_t0 = []; - spec.dt = []; + spec.dt = NaN; spec.deconv_gps_time = []; spec.deconv_mean = {}; spec.deconv_std = {}; @@ -164,6 +163,9 @@ fprintf(' Load %s (%s)\n', out_fn, datestr(now)); spec_in = load(out_fn); + if isfinite(spec_in.dt) + spec.dt = spec_in.dt; + end % Concatenate spec.deconv_gps_time(end+(1:numel(spec_in.deconv_gps_time))) = spec_in.deconv_gps_time; spec.deconv_fc(end+(1:numel(spec_in.deconv_fc))) = spec_in.deconv_fc; @@ -188,7 +190,6 @@ %% Specular: Store concatenated output % ================================================================= - spec.dt = spec_in.dt; spec.param_analysis = spec_in.param_analysis; spec.param_records = spec_in.param_records; if param.ct_file_lock @@ -205,6 +206,76 @@ end + elseif strcmpi(cmd.method,{'burst_noise'}) + %% Burst Noise + % =================================================================== + % =================================================================== + for img = 1:length(param.analysis.imgs) + + for wf_adc = cmd.wf_adcs{img}(:).' + wf = param.analysis.imgs{img}(wf_adc,1); + adc = param.analysis.imgs{img}(wf_adc,2); + + %% Coh Noise: Loop through all the coherent noise tracker files and combine + % ===================================================================== + bad_bins = []; + bad_recs = []; + bad_waveforms_recs = {}; + bad_waveforms = {}; + test_metric = []; + + for block_idx = 1:length(blocks) + rec_load_start = blocks(block_idx); + + if block_idx == length(blocks) + rec_load_stop = length(records.gps_time); + else + rec_load_stop = rec_load_start+param.analysis.block_size-1; + end + + % Load each block and concatenate + % ===================================================================== + cur_recs = [rec_load_start rec_load_stop]; + actual_cur_recs = [(cur_recs(1)-1)*param.analysis.presums+1, ... + cur_recs(end)*param.analysis.presums]; + + out_fn = fullfile(tmp_out_fn_dir, ... + sprintf('burst_noise_wf_%d_adc_%d_%d_%d.mat',wf,adc,actual_cur_recs)); + + noise = load(out_fn); + + bad_bins(end+(1:length(noise.bad_bins))) = noise.bad_bins; + bad_recs(end+(1:length(noise.bad_recs))) = rec_load_start-1 + noise.bad_recs; + bad_recs_unique = unique(noise.bad_recs); + bad_waveforms_recs{block_idx} = rec_load_start + bad_recs_unique(1:size(noise.bad_waveforms,2)) - 1; + bad_waveforms{block_idx} = noise.bad_waveforms; + test_metric(end+(1:length(noise.test_metric))) = noise.test_metric(:).'; + end + + % Constant noise fields carried over from last file loaded: + % param_analysis, param_records + + % Overwrite concatenated dynamic fields for the whole segment: + noise.bad_bins = bad_bins; + noise.bad_recs = bad_recs; + noise.bad_waveforms_recs = bad_waveforms_recs; + noise.bad_waveforms = bad_waveforms; + noise.test_metric = test_metric; + + if param.ct_file_lock + noise.file_version = '1L'; + else + noise.file_version = '1'; + end + noise.file_type = 'analysis_burst_noise'; + + out_segment_fn = fullfile(out_segment_fn_dir,sprintf('burst_noise_%s_wf_%d_adc_%d.mat', param.day_seg, wf, adc)); + fprintf('Saving output %s (%s)\n', out_segment_fn, datestr(now)); + ct_save(out_segment_fn,'-struct','noise'); % Use HDF because of the large file size + end + end + + elseif strcmpi(cmd.method,{'coh_noise'}) %% Coh Noise % =================================================================== @@ -219,6 +290,8 @@ % ===================================================================== Nt = []; t0 = []; + fc = NaN; + dt = NaN; gps_time = []; lat = []; lon = []; @@ -226,6 +299,7 @@ roll = []; pitch = []; heading = []; + along_track = []; surface = []; nyquist_zone = []; bad_rec = []; @@ -260,6 +334,12 @@ Nt(block_idx) = noise.Nt; t0(block_idx) = noise.t0; + if ~isnan(noise.fc) + fc = noise.fc; + end + if ~isnan(noise.dt) + dt = noise.dt; + end gps_time(end+(1:length(noise.gps_time))) = noise.gps_time; lat(end+(1:length(noise.lat))) = noise.lat; @@ -268,6 +348,7 @@ roll(end+(1:length(noise.roll))) = noise.roll; pitch(end+(1:length(noise.pitch))) = noise.pitch; heading(end+(1:length(noise.heading))) = noise.heading; + along_track(end+(1:length(noise.along_track))) = noise.along_track; surface(end+(1:length(noise.surface))) = noise.surface; nyquist_zone(end+(1:length(noise.nyquist_zone))) = noise.nyquist_zone; bad_rec(end+(1:length(noise.bad_rec))) = noise.bad_rec; @@ -298,6 +379,10 @@ % Constant noise fields carried over from last file loaded: % dt, fc, param_analysis, param_records + % Handle special case where a block had all bad records and fc and + % dt were NaN. + noise.fc = fc; + noise.dt = dt; % Overwrite concatenated dynamic fields for the whole segment: noise.Nt = Nt; @@ -310,6 +395,7 @@ noise.roll = roll; noise.pitch = pitch; noise.heading = heading; + noise.along_track = along_track; noise.surface = surface; noise.nyquist_zone = nyquist_zone; noise.bad_rec = bad_rec; @@ -358,6 +444,7 @@ heading = []; wf_data = []; time_rng = []; + layer_nan_mask = []; for block_idx = 1:length(blocks) rec_load_start = blocks(block_idx); @@ -385,6 +472,7 @@ heading = cat(2,heading,waveform.heading); wf_data = cat(2,wf_data,waveform.wf_data); time_rng = cat(2,time_rng,waveform.time_rng); + layer_nan_mask = cat(2,layer_nan_mask,waveform.layer_nan_mask); end % Constant waveform fields carried over from last file loaded: @@ -400,6 +488,7 @@ waveform.heading = heading; waveform.wf_data = wf_data; waveform.time_rng = time_rng; + waveform.layer_nan_mask = layer_nan_mask; if param.ct_file_lock waveform.file_version = '1L'; diff --git a/cresis-toolbox/processing/analysis_task.m b/cresis-toolbox/processing/analysis_task.m index 0bea3bf5..c1833452 100644 --- a/cresis-toolbox/processing/analysis_task.m +++ b/cresis-toolbox/processing/analysis_task.m @@ -13,7 +13,6 @@ %% Load records file % ========================================================================= -records_fn = ct_filename_support(param,'','records'); % Adjust the load records to account for filtering and decimation. Care is % taken to ensure that when blocks and frames are concatenated together, @@ -32,7 +31,7 @@ load_recs_ps(1) = floor((param.load.recs(1)-1)/param.analysis.presums)+1; load_recs_ps(2) = floor(param.load.recs(2)/param.analysis.presums); -records = records_aux_files_read(records_fn,param.load.recs); +records = records_load(param,param.load.recs); % Store the parameters that were used to create the records file param_records = records.param_records; @@ -51,11 +50,12 @@ % Determine which frames have the records that are needed frms = find(task_recs(1) >= frames.frame_idxs,1,'last') : find(task_recs(2) >= frames.frame_idxs,1,'last'); tmp_param.cmd.frms = max(1,min(frms)-1) : min(length(frames.frame_idxs),max(frms)+1); +% Load the surface layer surf_layer = opsLoadLayers(tmp_param,param.analysis.surf_layer); -if isempty(surf_layer.gps_time) - records.surface(:) = 0; +if isempty(surf_layer.gps_time) || any(~isfinite(surf_layer.gps_time)) + records.surface = zeros(size(records.gps_time)); elseif length(surf_layer.gps_time) == 1; - records.surface(:) = surf_layer.twtt; + records.surface = surf_layer.twtt*ones(size(records.gps_time)); else records.surface = interp_finite(interp1(surf_layer.gps_time,surf_layer.twtt,records.gps_time),0); end @@ -184,171 +184,190 @@ pitch = []; heading = []; - % Pulse compression - tmp_param.radar.wfs(wf).deconv.en = false; - [tmp_hdr,data,tmp_param] = data_pulse_compress(tmp_param,tmp_hdr,{raw_data{1}(:,:,wf_adc)}); - - [tmp_hdr,data] = data_merge_combine(tmp_param,tmp_hdr,data); - - [tmp_hdr,data] = data_trim(tmp_hdr,data,tmp_param.load.trim); - - data = data{1}; - - % Correct all the data to a constant elevation (no zero padding is - % applied so wrap around could be an issue for DDC data) - data(isnan(data)) = 0; % NaN values will create problems in ifft(fft(data)) - for rline = 1:size(data,2) - elev_dt = (tmp_hdr.records{1,1}.elev(rline) - tmp_hdr.records{1,1}.elev(1)) / (c/2); - data(:,rline,1) = ifft(fft(data(:,rline,1)) .* exp(1i*2*pi*tmp_hdr.freq{1,1}*elev_dt)); - end - - %% Specular: Coherence (STFT) Estimation - - % Grab the peak values - if ~isfield(cmd,'min_bin') || isempty(cmd.min_bin) - if strcmpi(radar_type,'deramp') - cmd.min_bin = 0; - else - cmd.min_bin = wfs(wf).Tpd; + if isempty(raw_data{1}) + % There are no good records in the image + % =============================================================== + deconv_gps_time = []; + deconv_mean = []; + deconv_std = []; + deconv_sample = []; + deconv_twtt = []; + deconv_forced = []; + peakiness = nan(size(gps_time)); + deconv_fc = []; + deconv_t0 = []; + dt = NaN; + + else + % There is at least one good record in the image + % =============================================================== + + % Pulse compression + tmp_param.radar.wfs(wf).deconv.en = false; + [tmp_hdr,data,tmp_param] = data_pulse_compress(tmp_param,tmp_hdr,{raw_data{1}(:,:,wf_adc)}); + + [tmp_hdr,data] = data_merge_combine(tmp_param,tmp_hdr,data); + + [tmp_hdr,data] = data_trim(tmp_hdr,data,tmp_param.load.trim); + + data = data{1}; + + % Correct all the data to a constant elevation (no zero padding is + % applied so wrap around could be an issue for DDC data) + data(isnan(data)) = 0; % NaN values will create problems in ifft(fft(data)) + for rline = 1:size(data,2) + elev_dt = (tmp_hdr.records{1,1}.elev(rline) - tmp_hdr.records{1,1}.elev(1)) / (c/2); + data(:,rline,1) = ifft(fft(data(:,rline,1)) .* exp(1i*2*pi*tmp_hdr.freq{1,1}*elev_dt)); end - end - min_bin_idxs = find(tmp_hdr.time{1,1} >= cmd.min_bin,1); - [max_value,max_idx_unfilt] = max(data(min_bin_idxs:end,:,1)); - max_idx_unfilt = max_idx_unfilt + min_bin_idxs(1) - 1; - - % Perform STFT (short time Fourier transform) (i.e. overlapping short FFTs in slow-time) - H = spectrogram(double(max_value),hanning(cmd.rlines),cmd.rlines/2,cmd.rlines); - - % Since there may be a little slope in the ice, we sum the powers from - % the lower frequency doppler bins rather than just taking DC. It seems to help - % a lot to normalize by the sum of the middle/high-frequency Doppler bins. A coherent/specular - % surface will have high power in the low bins and low power in the high bins - % so this ratio makes sense. - peakiness = lp(max(abs(H(cmd.signal_doppler_bins,:)).^2) ./ mean(abs(H(cmd.noise_doppler_bins,:)).^2)); - - if 0 - figure(1); clf; - imagesc(lp(data(:,:,1))) - figure(2); clf; - plot(peakiness) - keyboard - end - - % Threshold to find high peakiness range lines. (Note these are not - % actual range line numbers, but rather indices into the STFT groups - % of range lines.) - good_rlines = find(peakiness > cmd.threshold); - - % Force there to be two good STFT groups in a row before storing - % it to the specular file for deconvolution. - good_rlines_idxs = diff(good_rlines) == 1; - final_good_rlines = good_rlines(good_rlines_idxs); - - if ~isempty(cmd.max_rlines) - [~,sort_idxs] = sort( peakiness(final_good_rlines)+peakiness(final_good_rlines+1) , 'descend'); - final_good_rlines = final_good_rlines(sort_idxs); - final_good_rlines = final_good_rlines(1 : min(end,cmd.max_rlines)); - end - - % Prepare outputs for file - peakiness_rlines = round((1:length(peakiness)+0.5)*cmd.rlines/2); - gps_time = tmp_hdr.gps_time(peakiness_rlines); - lat = tmp_hdr.records{1,1}.lat(peakiness_rlines); - lon = tmp_hdr.records{1,1}.lon(peakiness_rlines); - elev = tmp_hdr.records{1,1}.elev(peakiness_rlines); - roll = tmp_hdr.records{1,1}.roll(peakiness_rlines); - pitch = tmp_hdr.records{1,1}.pitch(peakiness_rlines); - heading = tmp_hdr.records{1,1}.heading(peakiness_rlines); - surface = tmp_hdr.surface(peakiness_rlines); - - %% Specular: Forced GPS Check - deconv_forced = zeros(size(final_good_rlines)); - if isfield(cmd,'gps_times') && ~isempty(cmd.gps_times) - for idx = 1:length(cmd.gps_times) - force_gps_time = cmd.gps_times(idx); - if records.gps_time(1) <= force_gps_time && records.gps_time(end) >= force_gps_time - % This forced GPS time is in the block, find the peakiness block - % closest to this time and force it to be included in final_good_rlines - % if it is not already. - [~,force_final_good_rline] = min(abs(gps_time - force_gps_time)); - match_idx = find(final_good_rlines == force_final_good_rline); - if isempty(match_idx) - final_good_rlines = [final_good_rlines force_final_good_rline]; - [final_good_rlines,new_idxs] = sort(final_good_rlines); - deconv_forced(new_idxs(end)) = 1; - else - deconv_forced(match_idx) = 1; - end + + %% Specular: Coherence (STFT) Estimation + + % Grab the peak values + if ~isfield(cmd,'min_bin') || isempty(cmd.min_bin) + if strcmpi(radar_type,'deramp') + cmd.min_bin = 0; + else + cmd.min_bin = wfs(wf).Tpd; end end - end - - %% Specular: Extract specular waveforms - deconv_gps_time = []; - deconv_mean = {}; - deconv_std = {}; - deconv_sample = {}; - deconv_twtt = []; - for good_rline_idx = 1:length(final_good_rlines) - % Get the specific STFT group we will be extracting an answer from - final_good_rline = final_good_rlines(good_rline_idx); + min_bin_idxs = find(tmp_hdr.time{1,1} >= cmd.min_bin,1); + [max_value,max_idx_unfilt] = max(data(min_bin_idxs:end,:,1)); + max_idx_unfilt = max_idx_unfilt + min_bin_idxs(1) - 1; - % Determine the center range line that this STFT group corresponds to - center_rline = (final_good_rline+0.5)*cmd.rlines/2; + % Perform STFT (short time Fourier transform) (i.e. overlapping short FFTs in slow-time) + H = spectrogram(double(max_value),hanning(cmd.rlines),cmd.rlines/2,cmd.rlines); - fprintf(' SPECULAR %d %s (%s)!\n', center_rline, ... - datestr(epoch_to_datenum(tmp_hdr.gps_time(center_rline)),'YYYYmmDD HH:MM:SS.FFF'), ... - datestr(now)); + % Since there may be a little slope in the ice, we sum the powers from + % the lower frequency doppler bins rather than just taking DC. It seems to help + % a lot to normalize by the sum of the middle/high-frequency Doppler bins. A coherent/specular + % surface will have high power in the low bins and low power in the high bins + % so this ratio makes sense. + peakiness = lp(max(abs(H(cmd.signal_doppler_bins,:)).^2) ./ mean(abs(H(cmd.noise_doppler_bins,:)).^2)); - % Find the max values and correponding indices for all the range lines - % in this group. Since we over-interpolate by Mt and the memory - % requirements may be prohibitive, we do this in a loop - % Enforce the same DDC filter in this group. Skip groups that have DDC filter swiches. - STFT_rlines = -cmd.rlines/4 : cmd.rlines/4-1; - % if any(strcmpi(radar_name,{'kuband','kuband2','kuband3','kaband3','snow','snow2','snow3','snow5','snow8'})) - % if any(diff(img_Mt(center_rline + STFT_rlines))) - % fprintf(' Including different DDC filters, skipped.\n'); - % continue - % end - % end - Mt = 100; - max_value = zeros(size(STFT_rlines)); - max_idx_unfilt = zeros(size(STFT_rlines)); - for offset_idx = 1:length(STFT_rlines) - offset = STFT_rlines(offset_idx); - oversampled_rline = interpft(data(:,center_rline+offset),size(data,1)*Mt); - [max_value(offset_idx),max_idx_unfilt(offset_idx)] ... - = max(oversampled_rline(min_bin_idxs(1)*Mt:end)); - max_idx_unfilt(offset_idx) = max_idx_unfilt(offset_idx) + min_bin_idxs(1)*Mt - 1; + if 0 + figure(1); clf; + imagesc(lp(data(:,:,1))) + figure(2); clf; + plot(peakiness) + keyboard end - % Filter the max and phase vectors - max_idx = sgolayfilt(max_idx_unfilt/100,3,51); - phase_corr = sgolayfilt(double(unwrap(angle(max_value))),3,51); + % Threshold to find high peakiness range lines. (Note these are not + % actual range line numbers, but rather indices into the STFT groups + % of range lines.) + good_rlines = find(peakiness > cmd.threshold); - % Compensate range lines for amplitude, phase, and delay variance - % in the peak value + % Force there to be two good STFT groups in a row before storing + % it to the specular file for deconvolution. + good_rlines_idxs = diff(good_rlines) == 1; + final_good_rlines = good_rlines(good_rlines_idxs); + + if ~isempty(cmd.max_rlines) + [~,sort_idxs] = sort( peakiness(final_good_rlines)+peakiness(final_good_rlines+1) , 'descend'); + final_good_rlines = final_good_rlines(sort_idxs); + final_good_rlines = final_good_rlines(1 : min(end,cmd.max_rlines)); + end - % Apply true time delay shift to flatten surface - dt = diff(tmp_hdr.time{1,1}(1:2)); - Nt = size(data,1); - comp_data = ifft(fft(data(:,center_rline+STFT_rlines,1)) .* exp(1i*2*pi*tmp_hdr.freq{1,1}*max_idx*dt) ); - % Apply amplitude correction - %comp_data = max(abs(max_value)) * comp_data .* repmat(1./abs(max_value), [Nt 1]); - % Apply phase correction (compensating for phase from time delay shift) - comp_data = comp_data .* repmat(exp(-1i*(phase_corr + 2*pi*tmp_hdr.freq{1,1}(1)*max_idx*dt)), [Nt 1]); + % Prepare outputs for file + peakiness_rlines = round((1:length(peakiness)+0.5)*cmd.rlines/2); + gps_time = tmp_hdr.gps_time(peakiness_rlines); + lat = tmp_hdr.records{1,1}.lat(peakiness_rlines); + lon = tmp_hdr.records{1,1}.lon(peakiness_rlines); + elev = tmp_hdr.records{1,1}.elev(peakiness_rlines); + roll = tmp_hdr.records{1,1}.roll(peakiness_rlines); + pitch = tmp_hdr.records{1,1}.pitch(peakiness_rlines); + heading = tmp_hdr.records{1,1}.heading(peakiness_rlines); + surface = tmp_hdr.surface(peakiness_rlines); - deconv_gps_time(end+1) = tmp_hdr.gps_time(center_rline); - deconv_mean{end+1} = mean(comp_data,2); - deconv_std{end+1} = std(comp_data,[],2); - deconv_sample{end+1} = data(:,center_rline+1+cmd.rlines/4,1); - deconv_twtt(:,end+1) = tmp_hdr.time{1,1}(round(mean(max_idx))); + %% Specular: Forced GPS Check + deconv_forced = zeros(size(final_good_rlines)); + if isfield(cmd,'gps_times') && ~isempty(cmd.gps_times) + for idx = 1:length(cmd.gps_times) + force_gps_time = cmd.gps_times(idx); + if records.gps_time(1) <= force_gps_time && records.gps_time(end) >= force_gps_time + % This forced GPS time is in the block, find the peakiness block + % closest to this time and force it to be included in final_good_rlines + % if it is not already. + [~,force_final_good_rline] = min(abs(gps_time - force_gps_time)); + match_idx = find(final_good_rlines == force_final_good_rline); + if isempty(match_idx) + final_good_rlines = [final_good_rlines force_final_good_rline]; + [final_good_rlines,new_idxs] = sort(final_good_rlines); + deconv_forced(new_idxs(end)) = 1; + else + deconv_forced(match_idx) = 1; + end + end + end + end + + %% Specular: Extract specular waveforms + deconv_gps_time = []; + deconv_mean = {}; + deconv_std = {}; + deconv_sample = {}; + deconv_twtt = []; + for good_rline_idx = 1:length(final_good_rlines) + % Get the specific STFT group we will be extracting an answer from + final_good_rline = final_good_rlines(good_rline_idx); + + % Determine the center range line that this STFT group corresponds to + center_rline = round((final_good_rline+0.5)*cmd.rlines/2); + + fprintf(' SPECULAR %d %s (%s)!\n', center_rline, ... + datestr(epoch_to_datenum(tmp_hdr.gps_time(center_rline)),'YYYYmmDD HH:MM:SS.FFF'), ... + datestr(now)); + + % Find the max values and correponding indices for all the range lines + % in this group. Since we over-interpolate by Mt and the memory + % requirements may be prohibitive, we do this in a loop + % Enforce the same DDC filter in this group. Skip groups that have DDC filter swiches. + STFT_rlines = -round(cmd.rlines/4) : round(cmd.rlines/4)-1; + % if any(strcmpi(radar_name,{'kuband','kuband2','kuband3','kaband3','snow','snow2','snow3','snow5','snow8'})) + % if any(diff(img_Mt(center_rline + STFT_rlines))) + % fprintf(' Including different DDC filters, skipped.\n'); + % continue + % end + % end + Mt = 100; + max_value = zeros(size(STFT_rlines)); + max_idx_unfilt = zeros(size(STFT_rlines)); + for offset_idx = 1:length(STFT_rlines) + offset = STFT_rlines(offset_idx); + oversampled_rline = interpft(data(:,center_rline+offset),size(data,1)*Mt); + [max_value(offset_idx),max_idx_unfilt(offset_idx)] ... + = max(oversampled_rline(min_bin_idxs(1)*Mt:end)); + max_idx_unfilt(offset_idx) = max_idx_unfilt(offset_idx) + min_bin_idxs(1)*Mt - 1; + end + + % Filter the max and phase vectors + max_idx = sgolayfilt(max_idx_unfilt/100,cmd.peak_sgolay_filt{1},cmd.peak_sgolay_filt{2}); + phase_corr = sgolayfilt(double(unwrap(angle(max_value))),cmd.peak_sgolay_filt{1},cmd.peak_sgolay_filt{2}); + + % Compensate range lines for amplitude, phase, and delay variance + % in the peak value + + % Apply true time delay shift to flatten surface + dt = diff(tmp_hdr.time{1,1}(1:2)); + Nt = size(data,1); + comp_data = ifft(fft(data(:,center_rline+STFT_rlines,1)) .* exp(1i*2*pi*tmp_hdr.freq{1,1}*max_idx*dt) ); + % Apply amplitude correction + %comp_data = max(abs(max_value)) * comp_data .* repmat(1./abs(max_value), [Nt 1]); + % Apply phase correction (compensating for phase from time delay shift) + comp_data = comp_data .* repmat(exp(-1i*(phase_corr + 2*pi*tmp_hdr.freq{1,1}(1)*max_idx*dt)), [Nt 1]); + + deconv_gps_time(end+1) = tmp_hdr.gps_time(center_rline); + deconv_mean{end+1} = mean(comp_data,2); + deconv_std{end+1} = std(comp_data,[],2); + deconv_sample{end+1} = data(:,center_rline+1+cmd.rlines/4,1); + deconv_twtt(:,end+1) = tmp_hdr.time{1,1}(round(mean(max_idx))); + end + + deconv_fc = tmp_hdr.freq{1,1}(1) * ones(size(deconv_gps_time)); + deconv_t0 = tmp_hdr.time{1,1}(1) * ones(size(deconv_gps_time)); + dt = tmp_hdr.time{1,1}(2)-tmp_hdr.time{1,1}(1); end - deconv_fc = tmp_hdr.freq{1,1}(1) * ones(size(deconv_gps_time)); - deconv_t0 = tmp_hdr.time{1,1}(1) * ones(size(deconv_gps_time)); - dt = tmp_hdr.time{1,1}(2)-tmp_hdr.time{1,1}(1); - %% Specular: Save Results out_fn = fullfile(tmp_out_fn_dir, ... sprintf('specular_wf_%d_adc_%d_%d_%d.mat',wf,adc,task_recs)); @@ -366,6 +385,122 @@ end + elseif strcmpi(cmd.method,{'burst_noise'}) + %% Burst Noise + % =================================================================== + % =================================================================== + + for wf_adc = cmd.wf_adcs{cmd_img}(:).' + wf = param.load.imgs{img}(wf_adc,1); + adc = param.load.imgs{img}(wf_adc,2); + + % data_signal: create the temporary "signal" variable Format is + % user defined and depends on how user plans to use data_signal in + % test_fh and threshold_fh. Set the function to return [] if not + % used. + data_signal = cmd.signal_fh{cmd_img}(raw_data{1}(:,:,wf_adc),wfs(wf)); + % data_noise: create the temporary "noise" variable Format is user + % defined and depends on how user plans to use data_signal in + % test_fh and threshold_fh. Set the function to return [] if not + % used. + data_noise = cmd.noise_fh{cmd_img}(raw_data{1}(:,:,wf_adc),wfs(wf),data_signal); + % test_metric: create the output "test_metric" variable Format is + % user defined and depends on how user plans to use data_signal in + % threshold_fh. Format is restricted by the concatenation operation + % in analysis_combine_task (i.e. some data types may not + % concatenate properly but regular matrices should have no + % problems). Set the function to return [] if not used. + test_metric = cmd.test_fh{cmd_img}(data_signal,data_noise,wfs(wf)); % optional + % bad_samples: create the output "bad_samples" variable The format + % of this must be a row vector or matrix equal to the size of the + % data. If a row vector, then bad_samples must be a mask + % representing bad records. If a matrix, then bad_samples must be a + % mask representing bad samples. Set the function to return [] if + % not used. This will cause bad bins, bad_recs, and bad_waveforms + % to all be []. + bad_samples = cmd.threshold_fh{cmd_img}(data_signal,data_noise,test_metric,wfs(wf)); + + if size(bad_samples,1) < 2 + % bad_samples (row vector or empty) represents bad records + % --------------------------------------------------------------- + bad_bins = []; + bad_recs = find(bad_samples); + else + % bad_samples represents bad samples + % --------------------------------------------------------------- + + % Convert peaks to range-bin/records + bad_idxs = find(bad_samples); + Nt = size(raw_data{1},1); + bad_bins = mod(bad_idxs-1,Nt)+1; + bad_recs = floor((bad_idxs-1)/Nt)+1; + % Remove detections that fall outside the valid bin range + valid_mask = bad_bins >= cmd.valid_bins{img}(1) & bad_bins <= cmd.valid_bins{img}(2); + bad_bins = bad_bins(valid_mask); + bad_recs = bad_recs(valid_mask); + end + + % Extract waveforms + bad_recs_unique = unique(bad_recs); + bad_waveforms = raw_data{1}(:,bad_recs_unique(1:min(end,cmd.max_bad_waveforms)),wf_adc); + + if 0 && ~isdeployed + % Debug code (enable for debugging, does not run when compiled) + figure(1); clf; + subplot(3,1,1); + imagesc(data_signal); + axis tight + subplot(3,1,2); + if size(data_noise,1) < 10 + plot(test_metric.'); + else + imagesc(test_metric); + end + grid on; + axis tight + subplot(3,1,3); + if size(data_noise,1) < 2 + plot(bad_samples.'); + else + imagesc(bad_samples); + end + axis tight + if size(data_noise,1) < 2 || size(data_noise,1) < 10 + link_figures(1,'x'); + else + link_figures(1,'xy'); + end + keyboard; + end + + if 0 && ~isdeployed + % Debug plots + figure(1); clf; + plot(bad_recs, bad_bins, 'x') + figure(2); clf; + imagesc(data_signal) + set(gca,'YDir','normal') + link_figures([2 1],'xy'); + end + + %% Burst Noise: Save Results + out_fn = fullfile(tmp_out_fn_dir, ... + sprintf('burst_noise_wf_%d_adc_%d_%d_%d.mat',wf,adc,task_recs)); + param_analysis = tmp_param; + fprintf(' Saving outputs %s (%s)\n', out_fn, datestr(now)); + if param.ct_file_lock + file_version = '1L'; + else + file_version = '1'; + end + file_type = 'analysis_noise_tmp'; + if size(bad_samples,1) >= 2 + test_metric = test_metric(find(bad_samples)); + end + ct_save(out_fn, 'bad_recs', 'bad_bins', 'bad_waveforms', 'test_metric', 'param_analysis', 'param_records','file_type','file_version'); + end + + elseif strcmpi(cmd.method,{'coh_noise'}) %% Coh Noise % =================================================================== @@ -395,13 +530,19 @@ roll = []; pitch = []; heading = []; + along_track = []; bad_rec = []; DDC_dec = []; raw_or_DDC = []; DDC_freq_min = []; DDC_freq_max = []; - % Pulse compression + % Pulse compression (special settings for coherent noise) + if strcmp(radar_type,'deramp') + tmp_param.radar.wfs(wf).chan_equal_dB(:) = 0; + tmp_param.radar.wfs(wf).chan_equal_deg(:) = 0; + tmp_param.radar.wfs(wf).Tsys(:) = 0; + end tmp_param.radar.wfs(wf).coh_noise_method = ''; tmp_param.radar.wfs(wf).deconv.en = false; tmp_param.radar.wfs(wf).nz_trim = {}; @@ -446,7 +587,12 @@ noise_fn_dir = fileparts(ct_filename_out(param,cmd.threshold, '')); noise_fn = fullfile(noise_fn_dir,sprintf('coh_noise_simp_%s_wf_%d_adc_%d.mat', param.day_seg, wf, adc)); fprintf(' Loading coh_noise threshold: %s\n', noise_fn); - load(noise_fn,'threshold'); + if strcmp(radar_type,'deramp') + load(noise_fn,'threshold','threshold_time'); + threshold = interp1(threshold_time, threshold, tmp_hdr.time{1}, 'nearest', 'extrap'); + else + load(noise_fn,'threshold'); + end else % threshold is a scalar constant threshold = cmd.threshold; @@ -470,33 +616,72 @@ if cmd.threshold_removeDC good_samples = bsxfun(@lt, lp(bsxfun(@minus,data(:,rlines),mu)), threshold); else - good_samples = bsxfun(@lt, lp(data(:,rlines)), threshold); + if cmd.threshold_coh_ave == 1 + good_samples = bsxfun(@lt, lp(data(:,rlines)), threshold); + else + good_samples = bsxfun(@lt, lp(nan_fir_dec(data(:,rlines),ones(1,cmd.threshold_coh_ave)/cmd.threshold_coh_ave,1, [], [], [], [], 2)), threshold); + end end end %% Coh Noise: Debug coh_ave.threshold if 0 + num_coh_ave = cmd.threshold_coh_ave; figure(1); clf; - imagesc(lp(data(:,rlines))); + imagesc(lp(nan_fir_dec(data(:,rlines),ones(1,num_coh_ave)/num_coh_ave,1, [], [], [], [], 2))); + title(sprintf('Raw data wf %d adc %d blk %d of %d', wf, adc, rline0_idx, length(rline0_list))) + cc = caxis; a1 = gca; figure(2); clf; + title(sprintf('Good sample mask wf %d adc %d blk %d of %d', wf, adc, rline0_idx, length(rline0_list))) imagesc(good_samples); colormap(gray); caxis([0 1]); title('Good sample mask (black is thresholded)'); a2 = gca; figure(3); clf; - imagesc( lp(bsxfun(@minus,data(:,rlines),mu)) ); + if 0 + % Subtract mu (mean of data with high variance signals removed) + imagesc( lp(bsxfun(@minus,nan_fir_dec(data(:,rlines),ones(1,num_coh_ave)/num_coh_ave,1, [], [], [], [], 2), mu )) ); + caxis(cc); + else + % Subtract the mean + tmp = nan_fir_dec(data(:,rlines),ones(1,num_coh_ave)/num_coh_ave,1, [], [], [], [], 2); + tmp(~good_samples) = NaN; + tmp_mean = nanmean(tmp,2); + imagesc( lp(bsxfun(@minus, tmp, tmp_mean )) ); + caxis(cc); + end + title(sprintf('Raw data coh noise removed wf %d adc %d blk %d of %d', wf, adc, rline0_idx, length(rline0_list))) a3 = gca; - linkaxes([a1 a2 a3], 'xy'); + figure(4); clf; + imagesc(angle(fir_dec(data(:,rlines),ones(1,num_coh_ave)/num_coh_ave,1))); + title(sprintf('Raw data phase wf %d adc %d blk %d of %d', wf, adc, rline0_idx, length(rline0_list))) + a4 = gca; + linkaxes([a1 a2 a3 a4], 'xy'); keyboard end %% Coh Noise: Concatenate Info + tmp_along_track = geodetic_to_along_track(hdr.records{img,wf_adc}.lat(rlines),hdr.records{img,wf_adc}.lon(rlines)); coh_ave_samples(:,rline0_idx) = sum(good_samples,2); - coh_ave(:,rline0_idx) = nansum(data(:,rlines) .* good_samples,2) ./ coh_ave_samples(:,rline0_idx); + if ~cmd.distance_weight || tmp_along_track(end) < 10 + % Too short, all good_sample measurements equal weight + sum_weight = good_samples; + sum_weight = bsxfun(@times,sum_weight,1./sum(sum_weight,2)); + else + sum_weight = diff(tmp_along_track); + sum_weight = [sum_weight(1) sum_weight]; + sum_weight = bsxfun(@times,good_samples,sum_weight); + sum_weight = bsxfun(@times,sum_weight,1./sum(sum_weight,2)); + end + if cmd.threshold_coh_ave == 1 + coh_ave(:,rline0_idx) = nansum(data(:,rlines) .* sum_weight,2); + else + coh_ave(:,rline0_idx) = nansum(nan_fir_dec(data(:,rlines),ones(1,cmd.threshold_coh_ave)/cmd.threshold_coh_ave,1, [], [], [], [], 2) .* sum_weight,2); + end if cmd.mag_en - coh_ave_mag(:,rline0_idx) = nansum(abs(data(:,rlines)) .* good_samples,2) ./ coh_ave_samples(:,rline0_idx); + coh_ave_mag(:,rline0_idx) = nansum(abs(data(:,rlines)) .* sum_weight,2); else coh_ave_mag = []; end @@ -509,7 +694,7 @@ % nyquist zone will be 3 which is 0011 in binary and positions 0 % and 1 are set to 1. nz_mask = char('0'*ones(1,32)); - nz_mask(32-unique(hdr.nyquist_zone_hw{img,wf_adc}(rlines))) = '1'; + nz_mask(32-unique(hdr.nyquist_zone_hw{img}(rlines))) = '1'; nyquist_zone(1,rline0_idx) = bin2dec(nz_mask); else nyquist_zone(1,rline0_idx) = 1; @@ -523,6 +708,7 @@ roll(rline0_idx) = mean(hdr.records{img,wf_adc}.roll(rlines)); pitch(rline0_idx) = mean(hdr.records{img,wf_adc}.pitch(rlines)); heading(rline0_idx) = mean(hdr.records{img,wf_adc}.heading(rlines)); + along_track(rline0_idx) = tmp_along_track(end); bad_rec(rline0_idx) = mean(hdr.bad_rec{1}(rlines)); tmp_DDC_dec = unique(hdr.DDC_dec{1}(rlines)); @@ -539,9 +725,16 @@ %% Coh Noise: Save results Nt = length(tmp_hdr.time{1}); - fc = tmp_hdr.freq{1}(1); - t0 = tmp_hdr.time{1}(1); - dt = tmp_hdr.time{1}(2)-tmp_hdr.time{1}(1); + if isempty(tmp_hdr.freq{1}) + % All records were bad + fc = NaN; + t0 = NaN; + dt = NaN; + else + fc = tmp_hdr.freq{1}(1); + t0 = tmp_hdr.time{1}(1); + dt = tmp_hdr.time{1}(2)-tmp_hdr.time{1}(1); + end out_fn = fullfile(tmp_out_fn_dir, ... sprintf('coh_noise_wf_%d_adc_%d_%d_%d.mat',wf,adc,task_recs)); @@ -554,7 +747,7 @@ end file_type = 'analysis_noise_tmp'; ct_save(out_fn, 'coh_ave', 'coh_ave_samples', 'coh_ave_mag', 'doppler', 'Nt', 'fc', 't0', 'dt', 'gps_time', 'surface', 'lat', ... - 'lon', 'elev', 'roll', 'pitch', 'heading', 'param_analysis', 'param_records','nyquist_zone','file_type','file_version', 'bad_rec', 'DDC_dec', 'DDC_freq_min', 'DDC_freq_max', 'raw_or_DDC'); + 'lon', 'elev', 'roll', 'pitch', 'heading', 'along_track', 'param_analysis', 'param_records','nyquist_zone','file_type','file_version', 'bad_rec', 'DDC_dec', 'DDC_freq_min', 'DDC_freq_max', 'raw_or_DDC'); end elseif strcmpi(cmd.method,{'waveform'}) @@ -602,6 +795,7 @@ t0 = time(1); fc = freq(1); Tpd = tmp_param.radar.wfs(wf).Tpd; + layer_nan_mask = []; if isnumeric(cmd.start_time) start_bin = find(time>=cmd.start_time,1)*ones(1,size(data,2)); if isempty(start_bin) @@ -613,8 +807,9 @@ cmd.start_time.eval.Tstart = time(1); cmd.start_time.eval.Tend = time(end); layers = opsLoadLayers(param,cmd.start_time); - layers.twtt = interp1(layers.gps_time, layers.twtt, gps_time); + layer_nan_mask = isnan(interp1(layers.gps_time, layers.twtt, gps_time)); layers.twtt = interp_finite(layers.twtt,0); + layers.twtt = interp1(layers.gps_time, layers.twtt, gps_time); start_bin = round(interp1(time, 1:length(time), layers.twtt,'linear','extrap')); start_bin = min(max(1,start_bin),size(data,1)); elseif ischar(cmd.start_time) @@ -662,7 +857,7 @@ end file_type = 'analysis_waveform_tmp'; ct_save(out_fn, 'wf_data','time_rng', 'gps_time', 'lat', ... - 'lon', 'elev', 'roll', 'pitch', 'heading', 'dt', 'fc', 'param_analysis', 'param_records','file_type','file_version'); + 'lon', 'elev', 'roll', 'pitch', 'heading', 'dt', 'fc', 'layer_nan_mask', 'param_analysis', 'param_records','file_type','file_version'); end @@ -706,8 +901,8 @@ cmd.start_time.eval.Tstart = time(1); cmd.start_time.eval.Tend = time(end); layers = opsLoadLayers(param,cmd.start_time); - layers.twtt = interp1(layers.gps_time, layers.twtt, hdr.gps_time(1,:)); layers.twtt = interp_finite(layers.twtt,0); + layers.twtt = interp1(layers.gps_time, layers.twtt, hdr.gps_time(1,:)); start_bin = round(interp1(time, 1:length(time), layers.twtt,'linear','extrap')); start_bin = min(max(1,start_bin),size(data,1)); elseif ischar(cmd.start_time) diff --git a/cresis-toolbox/processing/arbitrary_resample.m b/cresis-toolbox/processing/arbitrary_resample.m index 6b2e1b16..1ed7ce4a 100644 --- a/cresis-toolbox/processing/arbitrary_resample.m +++ b/cresis-toolbox/processing/arbitrary_resample.m @@ -45,9 +45,13 @@ elseif length(Hwin) == 1 data(:,rline) = data_in(:,idxs); else - Hwin = Hwin / param.dx * param.filt_len/length(idxs); - Hwin = Hwin .* [x_off(2)-x_off(1); (x_off(3:end)-x_off(1:end-2))/2; x_off(end)-x_off(end-1)]; - Hwin = Hwin / sum(Hwin); + % Normalize Hwin based on the distance travelled (this is mostly to + % deal with changes in speed which happen for ground based data). We + % also prevent discontinuities in the along-track from being weighted + % to much by capping the normalization at param.dx (i.e. a single + % input cannot be weighted more than one output sample). + norm_Hwin = min(param.dx,abs([x_off(2)-x_off(1); (x_off(3:end)-x_off(1:end-2))/2; x_off(end)-x_off(end-1)])) / param.dx; + Hwin = Hwin .* norm_Hwin; data(:,rline) = data_in(:,idxs)*Hwin; end end diff --git a/cresis-toolbox/processing/array.m b/cresis-toolbox/processing/array.m index 7b820a6e..bbd28e90 100644 --- a/cresis-toolbox/processing/array.m +++ b/cresis-toolbox/processing/array.m @@ -39,7 +39,9 @@ %% General Setup % ===================================================================== -param = merge_structs(param, param_override); +if exist('param_override','var') + param = merge_structs(param, param_override); +end fprintf('=====================================================================\n'); fprintf('%s: %s (%s)\n', mfilename, param.day_seg, datestr(now)); @@ -73,7 +75,10 @@ % {{[1 1],[1 2],[1 3],[1 4],[1 5]},{[2 1],[2 2],[2 3],[2 4],[2 5]}} % If the image is a cell array it describes multilooking across apertures if ~iscell(param.array.imgs{1}) - % No special multilooking, reformat old syntax to new multilooking syntax + % param.array.imgs is not using multilooking syntax; reformat + % param.array.imgs non-multilooking syntax to multilooking syntax to + % ensure param.array.imgs is always in the multilooking syntax. This + % makes coding easier since the format is always the same. for img = 1:length(param.array.imgs) param.array.imgs{img} = {param.array.imgs{img}}; end @@ -102,14 +107,29 @@ end end +if ~isfield(param.array,'fcs_pos_averaged') || isempty(param.array.fcs_pos_averaged) + param.array.fcs_pos_averaged = true; +end + if ~isfield(param.array,'frm_types') || isempty(param.array.frm_types) param.array.frm_types = {-1,-1,-1,-1,-1}; end +if ~isfield(param.array,'ft_over_sample') || isempty(param.array.ft_over_sample) + param.array.ft_over_sample = 1; +end + if ~isfield(param.array,'img_comb') || isempty(param.array.img_comb) param.array.img_comb = []; end +% Check img_comb length +if ~isempty(param.array.img_comb) && length(param.array.img_comb) ~= 3*(length(param.array.imgs)-1) + error('param.array.img_comb not the right length. Since it is not empty, there should be 3 entries for each image combination interface ([Tpd second image for surface saturation, -inf for second image blank, Tpd first image to avoid roll off] is typical).'); +end + +param = img_combine_input_check(param,'array'); + if ~isfield(param.array,'in_path') || isempty(param.array.in_path) param.array.in_path = 'sar'; end @@ -118,10 +138,6 @@ param.array.out_path = param.array.method; end -if ~isfield(param.array,'fcs_pos_averaged') || isempty(param.array.fcs_pos_averaged) - param.array.fcs_pos_averaged = true; -end - if ~isfield(param.array,'presums') || isempty(param.array.presums) if ~isfield(param.sar,'presums') || isempty(param.sar.presums) param.sar.presums = 1; @@ -133,6 +149,10 @@ end end +if ~isfield(param.array,'radiometric_corr_dB') || isempty(param.array.radiometric_corr_dB) + param.array.radiometric_corr_dB = NaN; +end + if ~isfield(param.array,'sar_type') || isempty(param.array.sar_type) if ~isfield(param.sar,'sar_type') || isempty(param.sar.sar_type) param.array.sar_type = 'fk'; @@ -187,6 +207,14 @@ end end +if ~isfield(param.array,'sv_model') || isempty(param.array.sv_model) + param.array.sv_model = 'ideal'; +end + +if ~isfield(param.array,'sv_lut_path') || isempty(param.array.sv_lut_path) + param.array.sv_lut_path = 'analysis'; +end + %% Setup processing % ===================================================================== @@ -197,8 +225,7 @@ array_out_dir = ct_filename_out(param, param.array.out_path); % Load records file -records_fn = ct_filename_support(param,'','records'); -records = load(records_fn); +records = records_load(param); % Apply presumming if param.sar.presums > 1 records.lat = fir_dec(records.lat,param.sar.presums); @@ -227,8 +254,10 @@ total_num_sam_input = zeros(size(param.array.imgs)); total_num_sam_output = zeros(size(param.array.imgs)); if param.array.tomo_en - Nsv = param.array.Nsv; + % Accounts for "Data" and "Tomo.img" + Nsv = 1 + param.array.Nsv; else + % Accounts for "Data" Nsv = 1; end if any(strcmpi(radar_name,{'acords','hfrds','hfrds2','mcords','mcords2','mcords3','mcords4','mcords5','mcords6','mcrds','rds','seaice','accum2','accum3'})) @@ -240,9 +269,9 @@ for ml_idx = 1:length(param.array.imgs{img}) num_chan = num_chan + size(param.array.imgs{img}{ml_idx},1); end - total_num_sam_output(img) = total_num_sam_output(img) + wfs(wf).Nt/param.array.dbin*num_chan; + total_num_sam_output(img) = total_num_sam_output(img) + wfs(wf).Nt/param.array.dbin*num_chan*param.array.ft_over_sample; else - total_num_sam_output(img) = total_num_sam_output(img) + wfs(wf).Nt/param.array.dbin*Nsv; + total_num_sam_output(img) = total_num_sam_output(img) + wfs(wf).Nt/param.array.dbin*Nsv*param.array.ft_over_sample; end for ml_idx = 1:length(param.array.imgs{img}) @@ -252,22 +281,22 @@ * size(param.array.imgs{img}{ml_idx},1) * numel(param.array.subaps{img}{ml_idx}); end end - cpu_time_mult = 6e-9; - mem_mult = 32; + cpu_time_mult = 1e-9; + mem_mult = 40; elseif any(strcmpi(radar_name,{'snow','kuband','snow2','kuband2','snow3','kuband3','kaband','kaband3','snow5','snow8'})) estimated_num_sam = 32000; for img = 1:length(param.array.imgs) wf = param.array.imgs{img}{1}(1,1); - total_num_sam_output(img) = total_num_sam_output(img) + estimated_num_sam/param.array.dbin*Nsv; + total_num_sam_output(img) = total_num_sam_output(img) + estimated_num_sam/param.array.dbin*Nsv*param.array.ft_over_sample; for ml_idx = 1:length(param.array.imgs{img}) wf = param.array.imgs{img}{ml_idx}(1,1); total_num_sam_input(img) = total_num_sam_input(img) + estimated_num_sam ... - * size(param.array.imgs{img}{ml_idx},1) * numel(param.array.subaps{img}{ml_idx}); + * size(param.array.imgs{img}{ml_idx},1) * numel(param.array.subaps{img}{ml_idx}) * param.array.ft_over_sample; end end cpu_time_mult = 4e-9; - mem_mult = 32; + mem_mult = 40; else error('radar_name %s not supported yet.', radar_name); @@ -275,24 +304,33 @@ end cpu_time_method_mult = 0; +Nsv_mult = 0; for method_idx = 1:length(param.array.method) switch (param.array.method(method_idx)) case STANDARD_METHOD cpu_time_method_mult = cpu_time_method_mult + 1; + Nsv_mult = Nsv_mult + 0.2; case MVDR_METHOD - cpu_time_method_mult = cpu_time_method_mult + 4; + cpu_time_method_mult = cpu_time_method_mult + 1.5; + Nsv_mult = Nsv_mult + 0.2; case MUSIC_METHOD - cpu_time_method_mult = cpu_time_method_mult + 6; + cpu_time_method_mult = cpu_time_method_mult + 1.5; + Nsv_mult = Nsv_mult + 0.2; case GEONULL_METHOD cpu_time_method_mult = cpu_time_method_mult + 8; + Nsv_mult = Nsv_mult + 0.2; case GSLC_METHOD cpu_time_method_mult = cpu_time_method_mult + 8; + Nsv_mult = Nsv_mult + 0.2; case SNAPSHOT_METHOD cpu_time_method_mult = cpu_time_method_mult + 32; + Nsv_mult = Nsv_mult + 0.2; case MLE_METHOD cpu_time_method_mult = cpu_time_method_mult + 480; + Nsv_mult = Nsv_mult + 1; otherwise cpu_time_method_mult = cpu_time_method_mult + 1; + Nsv_mult = Nsv_mult + 0.2; end end cpu_time_mult = cpu_time_mult * cpu_time_method_mult; @@ -301,7 +339,6 @@ sparam.argsin{1} = param; % Static parameters sparam.task_function = 'array_task'; sparam.num_args_out = 1; -prev_frm_num_chunks = []; for frm_idx = 1:length(param.cmd.frms); frm = param.cmd.frms(frm_idx); @@ -355,6 +392,28 @@ warning('Frame %d length (%g m) is smaller than the param.array.chunk_len (%g m), there could be problems. Consider making the chunk length smaller for this frame. Possibly the frame is too small and should be combined with a neighboring frame.', frm_dist, param.array.chunk_len); num_chunks = 1; end + + % Determine number of chunks in the previous frame + if frm == 1 + prev_frm_num_chunks = 0; + else + % Current frame goes from the start record specified in the frames file + % to the record just before the start record of the next frame. For + % the last frame, the stop record is just the last record in the segment. + prev_frm_start_rec = ceil(frames.frame_idxs(frm-1)/param.sar.presums); + if frm-1 < length(frames.frame_idxs) + prev_frm_stop_rec = ceil((frames.frame_idxs(frm)-1)/param.sar.presums); + else + prev_frm_stop_rec = length(records.gps_time); + end + + prev_frm_frm_dist = along_track_approx(prev_frm_stop_rec) - along_track_approx(prev_frm_start_rec); + prev_frm_num_chunks = round(prev_frm_frm_dist / param.array.chunk_len); + if prev_frm_num_chunks == 0 + warning('Previous frame %d length (%g m) is smaller than the param.array.chunk_len (%g m), there could be problems. Consider making the chunk length smaller for this frame. Possibly the frame is too small and should be combined with a neighboring frame.', frm_dist, param.array.chunk_len); + prev_frm_num_chunks = 1; + end + end %% Process each chunk (unless it is a skip frame) for chunk_idx = 1:num_chunks*~skip_frame @@ -381,7 +440,7 @@ % Rerun only mode: Test to see if we need to run this task % ================================================================= dparam.notes = sprintf('%s %s:%s:%s %s_%03d (%d of %d)/%d of %d', ... - sparam.task_function, array_proc_method_str(param.array.method(1)), param.radar_name, param.season_name, param.day_seg, frm, frm_idx, length(param.cmd.frms), ... + sparam.task_function, param.array.out_path, param.radar_name, param.season_name, param.day_seg, frm, frm_idx, length(param.cmd.frms), ... chunk_idx, num_chunks); if ctrl.cluster.rerun_only % If we are in rerun only mode AND the array task file success @@ -402,17 +461,27 @@ Nx = round(frm_dist / num_chunks / param.sar.sigma_x); dparam.cpu_time = 0; dparam.mem = 0; + [~,max_img] = max(total_num_sam_output); for img = 1:length(param.array.imgs) - dparam.cpu_time = dparam.cpu_time + 10 + Nx*total_num_sam_input(img)*total_num_sam_output(img)*cpu_time_mult; + dparam.cpu_time = dparam.cpu_time + 10 + Nx*total_num_sam_input(img)*total_num_sam_output(img)*cpu_time_mult/Nsv*(1 + (Nsv-1)*Nsv_mult); % Take the max of the input data size and the output data size - dparam.mem = max(dparam.mem,250e6 + Nx*total_num_sam_input(img)*mem_mult ... + dparam.mem = max(dparam.mem,500e6 + Nx*total_num_sam_input(img)*mem_mult ... + Nx/param.array.dline*total_num_sam_output(img)*mem_mult ); + if img == max_img + % Account for the fact that any command operating on the whole + % image usually requires twice the image memory to complete the + % operation. + dparam.mem = max(dparam.mem,500e6 + 2*Nx*total_num_sam_input(img)*mem_mult ... + + Nx/param.array.dline*total_num_sam_output(img)*mem_mult ); + else + dparam.mem = max(dparam.mem,500e6 + Nx*total_num_sam_input(img)*mem_mult ... + + Nx/param.array.dline*total_num_sam_output(img)*mem_mult ); + end end ctrl = cluster_new_task(ctrl,sparam,dparam,'dparam_save',0); end - prev_frm_num_chunks = num_chunks; end ctrl = cluster_save_dparam(ctrl); @@ -424,8 +493,8 @@ ctrl = cluster_new_batch(param); if any(strcmpi(radar_name,{'acords','hfrds','hfrds2','mcords','mcords2','mcords3','mcords4','mcords5','mcords6','mcrds','rds','seaice','accum2','accum3'})) - cpu_time_mult = 1e-6; - mem_mult = 16; + cpu_time_mult = 2e-6; + mem_mult = 8; elseif any(strcmpi(radar_name,{'snow','kuband','snow2','kuband2','snow3','kuband3','kaband','kaband3','snow5','snow8'})) cpu_time_mult = 1e-6; @@ -459,16 +528,17 @@ records_var = whos('records'); for img = 1:length(param.array.imgs) sparam.cpu_time = sparam.cpu_time + (Nx*total_num_sam_output(img)/Nsv*cpu_time_mult) * (1 + (Nsv-1)*0.2); + if isempty(param.array.img_comb) % Individual images, so need enough memory to hold the largest image - sparam.mem = max(sparam.mem,350e6 + records_var.bytes + Nx_max*total_num_sam_output(img)*mem_mult); + sparam.mem = max(sparam.mem,500e6 + records_var.bytes + Nx_max*total_num_sam_output(img)*mem_mult); else % Images combined into one so need enough memory to hold all images - sparam.mem = max(sparam.mem,350e6 + records_var.bytes + Nx_max*sum(total_num_sam_output)*mem_mult); + sparam.mem = max(sparam.mem,500e6 + records_var.bytes + Nx_max*sum(total_num_sam_output)*mem_mult); end end sparam.notes = sprintf('%s %s:%s:%s %s combine frames', ... - sparam.task_function, array_proc_method_str(param.array.method(1)), param.radar_name, param.season_name, param.day_seg); + sparam.task_function, param.array.out_path, param.radar_name, param.season_name, param.day_seg); % Create success condition sparam.file_success = {}; @@ -503,4 +573,3 @@ fprintf('Done %s\n', datestr(now)); -return; diff --git a/cresis-toolbox/processing/array_combine_task.m b/cresis-toolbox/processing/array_combine_task.m index 9e56e6ed..91c60b70 100644 --- a/cresis-toolbox/processing/array_combine_task.m +++ b/cresis-toolbox/processing/array_combine_task.m @@ -36,8 +36,7 @@ frames = frames_load(param); % Load records file -records_fn = ct_filename_support(param,'','records'); -records = load(records_fn); +records = records_load(param); % Apply presumming if param.sar.presums > 1 records.lat = fir_dec(records.lat,param.sar.presums); @@ -54,11 +53,41 @@ % Array_proc_methods array_proc_methods; +% Radiometric correction (dB) +radiometric_corr_dB = param.array.radiometric_corr_dB; + %% Combine chunks into each frame % ===================================================================== +[output_dir,radar_type] = ct_output_dir(param.radar_name); for frm_idx = 1:length(param.cmd.frms); frm = param.cmd.frms(frm_idx); + outputs_done = true; + for img = 1:length(param.array.imgs) + if length(param.array.imgs) == 1 + out_fn = fullfile(array_out_dir, sprintf('Data_%s_%03d.mat', ... + param.day_seg, frm)); + else + out_fn = fullfile(array_out_dir, sprintf('Data_img_%02d_%s_%03d.mat', ... + img, param.day_seg, frm)); + end + if ~ct_file_lock_check(out_fn,4) + outputs_done = false; + end + end + if ~param.array.tomo_en && (~isempty(param.array.img_comb) || (length(param.array.imgs) == 1 && ~strcmpi(radar_type,'deramp'))) + out_fn = fullfile(array_out_dir, sprintf('Data_%s_%03d.mat', ... + param.day_seg, frm)); + + if ~ct_file_lock_check(out_fn,4) + outputs_done = false; + end + end + if outputs_done + fprintf('Done, skipping %s\n', out_fn); + continue; + end + if ct_proc_frame(frames.proc_mode(frm),param.array.frm_types) fprintf('array_combine %s_%03i (%i of %i) %s\n', param.day_seg, frm, frm_idx, length(param.cmd.frms), datestr(now)); else @@ -173,7 +202,7 @@ param_array.array_proc.fcs.z = tmp.param_array.array_proc.fcs{1}{1}.z(:,tmp.param_array.array_proc.lines); param_array.array_proc.fcs.origin = tmp.param_array.array_proc.fcs{1}{1}.origin(:,tmp.param_array.array_proc.lines); if param.array.fcs_pos_averaged - % Average the fcs position + % Average the fcs position (default) pos = zeros(3,length(tmp.param_array.array_proc.lines)); Nc = 0; for ml_idx = 1:length(tmp.param_array.array_proc.fcs) @@ -184,7 +213,8 @@ end param_array.array_proc.fcs.pos = pos/Nc; else - % Store all the fcs positions without averaging + % Store all the fcs positions without averaging (just used for + % estimating steering vectors) pos = zeros(3,length(tmp.param_array.array_proc.lines),0); Nc = 0; for ml_idx = 1:length(tmp.param_array.array_proc.fcs) @@ -259,7 +289,12 @@ Tomo.(fields{field_idx}) = cat(2,Tomo.(fields{field_idx}), nan(Nt,Nnans,Nx_old)); end end - Tomo.(fields{field_idx}) = cat(max_dim,Tomo.(fields{field_idx}),tmp.Tomo.(fields{field_idx})); + % Memory expensive and slow concatenation: + % Tomo.(fields{field_idx}) = cat(max_dim,Tomo.(fields{field_idx}),tmp.Tomo.(fields{field_idx})); + % Memory efficient concatenation: + cat_idx = numel(Tomo.(fields{field_idx})); + Tomo.(fields{field_idx})(1,1,end+size(tmp.Tomo.(fields{field_idx}),max_dim)) = 0; + Tomo.(fields{field_idx})(cat_idx+(1:numel(tmp.Tomo.(fields{field_idx})))) = tmp.Tomo.(fields{field_idx}); end end @@ -281,19 +316,24 @@ else file_version = '1'; end - Data = single(Data); + file_type = 'array'; + if isnan(radiometric_corr_dB) + Data = single(Data); + else + Data = single(Data * 10^(radiometric_corr_dB/10)); + end if ~param.array.tomo_en % Do not save 3D surface - save('-v7.3',out_fn,'Time','Latitude','Longitude', ... + ct_save('-v7.3',out_fn,'Time','Latitude','Longitude', 'radiometric_corr_dB', ... 'Elevation','GPS_time','Data','Surface','Bottom', ... 'param_array','param_records','param_sar', ... - 'Roll', 'Pitch', 'Heading','file_version'); + 'Roll', 'Pitch', 'Heading', 'file_type', 'file_version'); else % Save 3D surface - save('-v7.3',out_fn,'Tomo','Time','Latitude', ... + ct_save('-v7.3',out_fn,'Tomo','Time','Latitude', 'radiometric_corr_dB', ... 'Longitude','Elevation','GPS_time','Data','Surface','Bottom', ... 'param_array','param_records','param_sar', ... - 'Roll', 'Pitch', 'Heading','file_version'); + 'Roll', 'Pitch', 'Heading', 'file_type', 'file_version'); end end @@ -314,7 +354,6 @@ end %% Combine images - [output_dir,radar_type] = ct_output_dir(param.radar_name); if ~param.array.tomo_en && (~isempty(param.array.img_comb) || (length(param.array.imgs) == 1 && ~strcmpi(radar_type,'deramp'))) % Combine images into a single image and/or trim invalid times with % img_comb_trim @@ -331,10 +370,10 @@ fprintf(' Writing output to %s\n', out_fn); % Note that image combining here never includes "Tomo" variable. Use % tomo.run_collate.m to create the combined image with the "Tomo" variable. - save('-v7.3',out_fn,'Time','Latitude','Longitude', ... + ct_save('-v7.3',out_fn,'Time','Latitude','Longitude', 'radiometric_corr_dB', ... 'Elevation','GPS_time','Data','Surface','Bottom', ... 'param_array','param_records','param_sar', ... - 'Roll', 'Pitch', 'Heading','file_version'); + 'Roll', 'Pitch', 'Heading', 'file_type', 'file_version'); end end diff --git a/cresis-toolbox/processing/array_proc.m b/cresis-toolbox/processing/array_proc.m index 6e816499..8741bd51 100644 --- a/cresis-toolbox/processing/array_proc.m +++ b/cresis-toolbox/processing/array_proc.m @@ -309,6 +309,7 @@ param.array.bin_rng = 0; param.array.line_rng = 0; param.array.Nsrc = 1; + param.array.array_proc.lut = []; end param.array.method = method_integer; end @@ -397,6 +398,13 @@ % grid/alternating projection initialization methods and in the % alternating projection optimization method. +% .sv_dielectric +% Scalar numeric containing relative dielectric (default is 1 for +% vacuum). +if ~isfield(param.array,'sv_dielectric') || isempty(param.array.sv_dielectric) + param.array.sv_dielectric = 1; +end + % .sv_fh % Steering vector function handle. Defaults to array_proc_sv.m and should % generally not be changed. @@ -412,7 +420,7 @@ if ~isfield(param.array,'Nsv') || isempty(param.array.Nsv) param.array.Nsv = 1; end - theta = fftshift(param.array.sv_fh(param.array.Nsv, 1)); + theta = fftshift(param.array.sv_fh(1,[],[],param.array.Nsv, [],[])); end % .Nsubband: @@ -476,6 +484,113 @@ param.array.layer_name = ''; end +if ~isfield(param.array,'sv_model') || isempty(param.array.sv_model) + param.array.sv_model = 'ideal'; % 'ideal' or 'lookup_table' +end + +if ~isfield(param.array,'process_training_override_flag') || isempty(param.array.process_training_override_flag) + param.array.process_training_override_flag = false; +end + + +if strcmpi(param.array.sv_model,'lookup_table') + if ~isfield(param.array,'lut_dayseg')|| isempty(param.array.lut_dayseg); + param.array.lut_dayseg = param.day_seg; + param.array.lut_type = 'process'; + end +end + +if strcmpi(param.array.sv_model,'lookup_table') + if ~isfield(param.array,'lut_type') || isempty(param.array.lut_type) + if ~isempty(param.array.lut_dayseg) + if strcmpi(param.array.lut_dayseg,param.day_seg) + param.array.lut_type = 'process'; + elseif ~strcmpi(param.array.lut_dayseg,param.day_seg) + param.array.lut_type = 'training'; + end + end + end +end + +if strcmpi(param.array.sv_model,'lookup_table') + if strcmpi(param.array.lut_type,'training') && strcmpi(param.array.lut_dayseg, param.day_seg) ... + && ~param.array.process_training_override_flag + param.array.sv_model = 'ideal'; + warning('Cannot use training data on processing from same day_seg. Converting manifold model to ideal'); + end +end + +if strcmpi(param.array.sv_model,'lookup_table') + if ~isfield(param.array,'lut_method') || isempty(param.array.lut_method) + param.array.lut_method = 'evd'; + end + if~isfield(param.array,'lut_path') || isempty(param.array.lut_path) + param.array.lut_path = 'array_manifold'; + end + if ~isfield(param.array,'lut_fn') || isempty(param.array.lut_fn) + param.array.lut_fn = []; + end + +end + +% +% if ~strcmpi(param.array.sv_model,'ideal') && isempty(param.array.lut_path) || ~isfield(param.array,'lut_path') +% param.array.lut_path = 'array_manifold'; +% end + +% if ~strcmpi(param.array.sv_model,'ideal') && isempty(param.array.sv_lut_path) +% param.array.sv_lut_path = []; +% end + +% if ~isfield(param.array,'lut_path') +% param.array.sv_lut_path = []; +% end +% +% if strcmpi(param.array.sv_model,'lookup_table') +% if~isfield(param.array,'lut_path') || isempty(param.array.lut_path) +% param.array.lut_path = 'array_manifold'; +% end +% end + + +% User can specify lookup table full filename directly +if strcmpi(param.array.sv_model,'lookup_table') && ~isempty(param.array.lut_fn) + if ~exist(param.array.lut_fn)==2 + warning('No valid lookup table found. Converting to ideal manifold'); + param.array.sv_model = 'ideal'; + param.array.lut_dayseg = []; + param.array.lut_type = []; + param.array.lut_path = []; + end +end + +% Check for the existence of the lut +if strcmpi(param.array.sv_model,'lookup_table') && isempty(param.array.lut_fn) + if strcmpi(param.array.lut_type,'process') + lut_path = ct_fileparts(ct_filename_out(param, param.array.lut_path,'')); + lut_fn = fullfile(lut_path,sprintf('lut_process_%s.mat',param.day_seg)); + elseif strcmpi(param.array.lut_type,'training') + lut_path = fullfile(ct_fileparts(ct_filename_out(param, param.array.lut_path,'')),sprintf('%s',param.array.lut_dayseg)); + lut_fn = fullfile(lut_path,sprintf('lut_training_%s.mat',param.array.lut_dayseg)); + end + + if ~isdir(lut_path) + warning('Lookup table path does not exist. Converting to ideal manifold'); + param.array.sv_model = 'ideal'; + param.array.lut_dayseg = []; + param.array.lut_type = []; + param.array.lut_path = []; + elseif ~exist(lut_fn)==2 + warning('No valid lookup table found. Converting to ideal manifold'); + param.array.sv_model = 'ideal'; + param.array.lut_dayseg = []; + param.array.lut_type = []; + param.array.lut_path = []; + else + param.array.lut_fn = lut_fn; + end +end + if nargin == 1 % No input data provided so just checking input arguments return; @@ -673,7 +788,7 @@ physical_constants; % c: speed of light % Change er_ice to 1 for ice top -if ~isempty(param.array.layer_name) && (strcmpi(param.array.layer_name,'top') || strcmpi(param.array.layer_name,'surface')) +if ~isempty(cfg.layer_name) && (strcmpi(cfg.layer_name,'top') || strcmpi(cfg.layer_name,'surface')) er_ice = 1; end @@ -721,6 +836,7 @@ doa_param.theta = theta; % This changes inside S-MAP loop theta_copy = theta; doa_param.seq = cfg.doa_seq; + doa_param.lut = cfg.lut; % Setup cfgeterization for DCM if cfg.method == DCM_METHOD doa_param.h = cfg.imp_resp.vals(:); @@ -749,22 +865,25 @@ % sv_fh_arg1 % ------------------------------------------------------------------------- % First argument to sv_fh -sv_fh_arg1 = {'theta'}; -sv_fh_arg1{2} = theta; +% sv_fh_arg1 = {'theta'}; +% sv_fh_arg1{2} = theta; +sv_fh_arg1.theta = theta; +sv_arg_opt.theta = theta; + % Set the output of any ignored image to NaN % ------------------------------------------------------------------------- -if ~isempty(cfg.ignored_img_idx) && (cfg.ignored_img_idx == param.array_proc.imgs{1}(1,1)) +if ~isempty(cfg.ignored_img_idx) && (cfg.ignored_img_idx == cfg.imgs{1}(1,1)) % Copy temporary outputs for first method to dout dout = tout.(array_proc_method_str(cfg.method(1))); return; end %% Load layerData: used to define the first range-bin and to skip bad-data range-line (those that have NAN/Inf in their layerData entry) -if ~isempty(param.array.layerData_folder) && ~isempty(param.array.layer_name) - layerData_fn = fullfile(ct_filename_out(param, param.array.layerData_folder), ... +if ~isempty(cfg.layerData_folder) && ~isempty(cfg.layer_name) + layerData_fn = fullfile(ct_filename_out(param, cfg.layerData_folder), ... sprintf('Data_%s_%03d', param.day_seg, param.load.frm)); - layer_name = param.array.layer_name; + layer_name = cfg.layer_name; if strcmpi(layer_name,'surface') layer_name = 'top'; end @@ -803,19 +922,37 @@ last_fprintf_time = -inf; last_fprintf_time_bin = -inf; -if any(cfg.method == GEONULL_METHOD) - % HACK - LUT = load('/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/sv_LUT.mat'); - LUT.bins = LUT.bins.'/180*pi; - LUT.sv = (sqrt(LUT.power_SVmean) .* exp(1i*LUT.angle_SVmean)).'; - LUT.sv_real = real(LUT.sv); - LUT.sv_imag = imag(LUT.sv); -end +% If the steering vector model is specified as a lookup table, then load +% if strcmpi(cfg.sv_model,'lookup_table') +% +% % Future paths after updated estimate_sv_lut (leaves off dayseg folder) +% % lut_folder = ct_filename_out(param,cfg.sv_lut_path,[],1); +% % lut_fn = fullfile(lut_folder,'sv_lut.mat'); +% +% lut_folder = ct_filename_out(param,cfg.sv_lut_path); +% lut_fn = fullfile(lut_folder, sprintf('sv_lut_%s.mat',param.day_seg)); +% load(lut_fn,'sv_rel_mag','sv_rel_phase','doa_bins'); +% cfg.lut.bins = doa_bins.'/180*pi; +% cfg.lut.sv = ((sv_rel_mag) .* exp(1i*sv_rel_phase)).'; +% cfg.lut.sv_real = real(cfg.lut.sv); +% cfg.lut.sv_imag = imag(cfg.lut.sv); +% else +% cfg.lut = []; +% end + +% if any(cfg.method == GEONULL_METHOD) +% % HACK +% LUT = load('/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/sv_LUT.mat'); +% LUT.bins = LUT.bins.'/180*pi; +% LUT.sv = (sqrt(LUT.power_SVmean) .* exp(1i*LUT.angle_SVmean)).'; +% LUT.sv_real = real(LUT.sv); +% LUT.sv_imag = imag(LUT.sv); +% end for line_idx = 1:1:Nx_out %% Array: Setup rline = cfg.lines(line_idx); - if now > last_fprintf_time+param.array.last_fprintf_time_delay/86400 + if now > last_fprintf_time+cfg.last_fprintf_time_delay/86400 fprintf(' Record %.0f (%.0f of %.0f) bin start (%s)\n', rline, line_idx, ... Nx_out, datestr(now)); last_fprintf_time = now; @@ -865,8 +1002,18 @@ y_pos{ml_idx}(wf_adc_idx,1) = cfg.fcs{ml_idx}{wf_adc_idx}.pos(2,rline); z_pos{ml_idx}(wf_adc_idx,1) = cfg.fcs{ml_idx}{wf_adc_idx}.pos(3,rline); end + + sv_arg_opt = []; + sv_arg_opt.theta = theta; + + % Arguments for array_proc_sv + sv_arg = []; + sv_arg = {cfg.wfs.fc.*sqrt(cfg.sv_dielectric), y_pos{ml_idx}, z_pos{ml_idx},sv_arg_opt, cfg.lut, cfg.fcs{1}{1}.roll(rline)}; + % Determine Steering Vector - [~,sv{ml_idx}] = cfg.sv_fh(sv_fh_arg1,cfg.wfs.fc,y_pos{ml_idx},z_pos{ml_idx}); + [~,sv{ml_idx}] = cfg.sv_fh(sv_arg{:}); + +% [~,sv{ml_idx}] = cfg.sv_fh(sv_fh_arg1,cfg.wfs.fc*sqrt(cfg.sv_dielectric),y_pos{ml_idx},z_pos{ml_idx}); end if 0 @@ -891,7 +1038,13 @@ if 0 % Debug: Plot steering vector correlation matrix ml_idx = 1; - [theta_plot,sv{ml_idx}] = cfg.sv_fh(cfg.Nsv,cfg.fc,y_pos{ml_idx},z_pos{ml_idx}); + sv_arg_opt = []; + sv_arg_opt = cfg.Nsv; + sv_arg = []; + sv_arg = {cfg.wfs.fc.*sqrt(cfg.sv_dielectric), y_pos{ml_idx}, z_pos{ml_idx}, sv_arg_opt, cfg.lut, cfg.fcs{1}{1}.roll(rline)}; + [theta_plot,sv{ml_idx}] = cfg.sv_fh(sv_arg{:}); + +% [theta_plot,sv{ml_idx}] = cfg.sv_fh(cfg.Nsv,cfg.fc*sqrt(cfg.sv_dielectric),y_pos{ml_idx},z_pos{ml_idx}); sv_table = fftshift(sv{ml_idx}.',1); theta_plot = fftshift(theta_plot); @@ -924,6 +1077,7 @@ doa_param.y_pc = y_pos{1}; doa_param.z_pc = z_pos{1}; doa_param.SV = fftshift(sv{1},2); + doa_param.lut_roll = cfg.fcs{1}{1}.roll(rline); end % .range: Range vector. It may have negative values. So, better to ignore @@ -933,7 +1087,7 @@ negative_range_idxs = find(cfg.range<0); if ~isempty(negative_range_idxs) && cfg.bin_restriction.start_bin(rline)= cfg.bin_restriction.start_bin(rline) & cfg.bins <= cfg.bin_restriction.stop_bin(rline)); early_stop = false; - if any(param.array.method < DOA_METHOD_THRESHOLD) + if any(cfg.method < DOA_METHOD_THRESHOLD) % Beam Forming Method - m = array_proc_method_str(cfg.method(param.array.method < DOA_METHOD_THRESHOLD)); + m = array_proc_method_str(cfg.method(cfg.method < DOA_METHOD_THRESHOLD)); Sarray.(m) = zeros(cfg.Nsv,Nt_out); end clear prev_active_rbin first_rbin_idx first_active_doa prev_rbin sources_number% For S-MAP tracker only % This is used with S-MAP, where the number of snapshots changes when the % change of DOA over range crosses some threshold. - cfg.bin_rng = param.array.bin_rng; + cfg.bin_rng = cfg.bin_rng; std_doa_accum = []; % For S-MAP to track stdev using Kalman filter std_doa_accum_new = []; % Interpolate twtt-dependant DOAs from the surfData onto the radar twtt % ----------------------------------------------------------------------- - if isfield(cfg,'surf_layer') && strcmpi(cfg.surf_layer.source, 'surfData') && strcmpi(cfg.surf_layer.name,'top twtt') + if isfield(cfg,'surf_layer') && strcmpi(cfg.surf_layer.source, 'surf_sar') && strcmpi(cfg.surf_layer.name,'top twtt') % TWTT of radar radar_twtt = cfg.wfs.time; @@ -1058,7 +1212,7 @@ surf_ice_mask(rbin_idx,1:length(bin_ice_mask)) = bin_ice_mask; if cfg.debug_plots - plot(bin_theta,y0,'mp','MarkerSize',4,'MarkerFaceColor','g','LineWidth',2) + plot(bin_theta,y0,'mo','MarkerSize',4,'MarkerFaceColor','g','LineWidth',2) end end end @@ -1068,7 +1222,7 @@ for bin_idx = bin_idxs(:).' %% Array: Array Process Each Bin bin = cfg.bins(bin_idx); - if now > last_fprintf_time_bin+2*param.array.last_fprintf_time_delay/86400 + if now > last_fprintf_time_bin+2*cfg.last_fprintf_time_delay/86400 fprintf(' Record %.0f (%.0f of %.0f) bin %d of %d (%s)\n', rline, line_idx, ... Nx_out, bin_idx, length(bin_idxs), datestr(now)); last_fprintf_time_bin = now; @@ -1213,7 +1367,7 @@ dataSample = reshape(dataSample,[length(cfg.bin_rng)*length(line_rng)*Na*Nb Nc]); if isempty(sv) - Sarray.music(:,bin_idx) = pmusic(dataSample,cfg.Nsrc,param.array.Nsv); + Sarray.music(:,bin_idx) = pmusic(dataSample,cfg.Nsrc,cfg.Nsv); else Rxx = 1/size(dataSample,1) * (dataSample' * dataSample); [V,D] = eig(Rxx); @@ -1304,6 +1458,7 @@ elseif any(cfg.method == GEONULL_METHOD) + warning off %% Array: GEONULL % The geonull method is a non-adaptive beamformer that steers nulls % towards predicted clutter angles from the air-ice interfaces @@ -1333,8 +1488,8 @@ g = vertcat(1,zeros(size(keep_surf_doas))); surf_doas_rad = keep_surf_doas*pi/180; - sv_fh_arg_geonull = {'theta'}; - sv_fh_arg_geonull{2} = [theta_desired(des_idx), surf_doas_rad(:)']; % array_proc_sv breaks if this is a column vector -- fix this! +% sv_fh_arg_geonull = {'theta'}; +% sv_fh_arg_geonull{2} = [theta_desired(des_idx), surf_doas_rad(:)']; % array_proc_sv breaks if this is a column vector -- fix this! for ml_idx = 1:length(cfg.fcs) % Make column vectors of y and z-positions @@ -1342,18 +1497,25 @@ y_pos{ml_idx}(wf_adc_idx,1) = cfg.fcs{ml_idx}{wf_adc_idx}.pos(2,rline); z_pos{ml_idx}(wf_adc_idx,1) = cfg.fcs{ml_idx}{wf_adc_idx}.pos(3,rline); end - % Determine Steering Vectors for target and interference - %[~,A] = cfg.sv_fh(sv_fh_arg_geonull,cfg.wfs.fc,y_pos{ml_idx},z_pos{ml_idx}); - - roll = param.array_proc.fcs{1}{1}.roll(rline); - - [~,A] = cfg.sv_fh(sv_fh_arg_geonull, cfg.wfs.fc, y_pos{ml_idx}, z_pos{ml_idx}, roll, LUT, []); + % Determine Steering Vectors for target and interference + sv_opt_arg = []; + sv_opt_arg.theta = [theta_desired(des_idx), surf_doas_rad(:)']; + sv_arg = []; + sv_arg = {cfg.wfs.fc*sqrt(cfg.sv_dielectric), y_pos{ml_idx}, z_pos{ml_idx}, sv_opt_arg, cfg.lut, cfg.fcs{1}{1}.roll(rline)}; + [~,A] = cfg.sv_fh(sv_arg{:}); +% [~,A] = cfg.sv_fh(cfg.wfs.fc*sqrt(cfg.sv_dielectric), y_pos{ml_idx}, z_pos{ml_idx}, sv_opt_arg, cfg.lut, cfg.fcs{1}{1}.roll(rline)); +% [~,A] = cfg.sv_fh(sv_fh_arg_geonull,cfg.wfs.fc,y_pos{ml_idx},z_pos{ml_idx}); +% roll = cfg.fcs{1}{1}.roll(rline); + % DEBUG ONLY bin 501-502, line 1308 if 0 - sv_fh_arg_geonull = {'theta'}; - sv_fh_arg_geonull{2} = [theta_desired(des_idx)]; % array_proc_sv breaks if this is a column vector -- fix this! - [~,Atarget] = cfg.sv_fh(sv_fh_arg_geonull,cfg.wfs.fc,y_pos{ml_idx},z_pos{ml_idx}); + sv_opt_arg = []; + sv_opt_arg.theta = [theta_desired(des_idx)]; + sv_arg = []; + sv_arg = {cfg.wfs.fc*sqrt(cfg.sv_dielectric), y_pos{ml_idx}, z_pos{ml_idx}, sv_opt_arg, cfg.lut, cfg.fcs{1}{1}.roll(rline)}; + [~,Atarget] = cfg.sv_fh(sv_arg{:}); + Atarget = Atarget ./ sqrt(Atarget'*Atarget); Aint = [0.3075 + 0.0000i, -0.3686 + 0.0469i, 0.4972 - 0.1680i, -0.4738 - 0.1331i, 0.2880 + 0.2123i, -0.2115 - 0.1632i, 0.0480 + 0.2192i]; % Aint = [0.2090 + 0.0000i, -0.2008 + 0.2626i, -0.2076 - 0.3079i, 0.4031 - 0.1203i, -0.0308 + 0.4104i, -0.3837 - 0.1338i, 0.3912 - 0.2112i]; @@ -1398,28 +1560,87 @@ % end if cfg.debug_plots + % Plot Geonull beamformer, normalized periodogram and normalize + % Music pseudospectrum + + % Matrix of steering vectors theta_vec = linspace(-pi/2,pi/2,256); - sv_patt_arg = sv_fh_arg_geonull; - sv_patt_arg{2} = [theta_vec]; - [~,Amanifold] = cfg.sv_fh(sv_patt_arg,cfg.wfs.fc,y_pos{ml_idx},z_pos{ml_idx}); - Pattmle = w'*Amanifold; + sv_opt_arg = []; + sv_opt_arg.theta = [theta_vec]; + sv_arg = []; + sv_arg = {cfg.wfs.fc*sqrt(cfg.sv_dielectric), y_pos{1}, z_pos{1}, sv_opt_arg, cfg.lut, cfg.fcs{1}{1}.roll(rline)}; + [~,SV_lut] = cfg.sv_fh(sv_arg{:}); + + sv_arg = []; + sv_arg = {cfg.wfs.fc*sqrt(cfg.sv_dielectric), y_pos{1}, z_pos{1}, sv_opt_arg, [], []}; + [~,SV_nom] = cfg.sv_fh(sv_arg{:}); + + % Geonull beamformer + Wgeo_lut = wgeo'*SV_lut; + Wgeo_nom = wgeo'*SV_nom; + + % Normalized periodogram + Sp_lut = (1/size(dataSample,1))*abs(sum(conj(dataSample)*SV_lut,1)).^2; + Sp_lut = Sp_lut./max(abs(Sp_lut)); + + Sp_nom = (1/size(dataSample,1))*abs(sum(conj(dataSample)*SV_nom,1)).^2; + Sp_nom = Sp_nom./max(abs(Sp_nom)); + + % Music pseudospectrum + Rxx = 1/size(dataSample,1) * (dataSample' * dataSample); + [V,D] = eig(Rxx); + eigenVals = diag(D); + [eigenVals noiseIdxs] = sort(eigenVals); + Nsrc = length(g); + noiseIdxs = noiseIdxs(1:end-Nsrc); - figure(97);clf;plot(theta_vec*180/pi,20*log10(abs(Pattmle)),'LineWidth',2) + Sm_lut = mean(abs(SV_lut(:,:).'*V(:,noiseIdxs)).^2,2); + Sm_lut = Sm_lut ./ max(abs(Sm_lut)); + + Sm_nom = mean(abs(SV_nom(:,:).'*V(:,noiseIdxs)).^2,2); + Sm_nom = Sm_nom ./ max(abs(Sm_nom)); + + colorvec = hsv(7); + figure(97);clf; + plot(theta_vec*180/pi,20*log10(abs(Wgeo_lut)),'LineWidth',2,'Color',colorvec(1,:)) + hold on + grid on + grid minor + axis tight + plot(theta_vec*180/pi,10*log10(abs(Sp_lut)),'LineWidth',2,'Color',colorvec(2,:)) + plot(theta_vec*180/pi,10*log10(abs(Sm_lut)),'LineWidth',2,'Color',colorvec(3,:)) + [day, rem] = strtok(param.day_seg, '_'); + seg = rem(2:end); + tgps = datetime(1970,01,01,00,00,00,00) + seconds(cfg.fcs{1}{1}.gps_time(rline)); + titlestr = sprintf('%s %s Measured Manifold, %s',day,seg,datestr(tgps)); + title(titlestr) + ylims = ylim; + if ~isempty(surf_doas) + DOAclutter = surf_doas; + line([DOAclutter(1) DOAclutter(1)],[min(ylims) max(ylims)],'LineStyle','--','Color',[0.3 0.3 0.3]) + line([DOAclutter(2) DOAclutter(2)],[min(ylims) max(ylims)],'LineStyle','--','Color',[0.3 0.3 0.3]) + legend('Geonull Beamformer','Normalized Periodogram','Normalized MUSIC','Clutter','Clutter','Location','SouthWest') + end + + figure(98);clf; + plot(theta_vec*180/pi,20*log10(abs(Wgeo_nom)),'LineWidth',2,'Color',colorvec(1,:)) hold on grid on grid minor axis tight + plot(theta_vec*180/pi,10*log10(abs(Sp_nom)),'LineWidth',2,'Color',colorvec(2,:)) + plot(theta_vec*180/pi,10*log10(abs(Sm_nom)),'LineWidth',2,'Color',colorvec(3,:)) [day, rem] = strtok(param.day_seg, '_'); seg = rem(2:end); tgps = datetime(1970,01,01,00,00,00,00) + seconds(cfg.fcs{1}{1}.gps_time(rline)); - titlestr = sprintf('%s %s GEONULL PATTERN, %s',day,seg,datestr(tgps)); + titlestr = sprintf('%s %s Nominal Manifold, %s',day,seg,datestr(tgps)); title(titlestr) ylims = ylim; if ~isempty(surf_doas) DOAclutter = surf_doas; line([DOAclutter(1) DOAclutter(1)],[min(ylims) max(ylims)],'LineStyle','--','Color',[0.3 0.3 0.3]) line([DOAclutter(2) DOAclutter(2)],[min(ylims) max(ylims)],'LineStyle','--','Color',[0.3 0.3 0.3]) - legend('Geonull','Clutter','Clutter') + legend('Geonull Beamformer','Normalized Periodogram','Normalized MUSIC','Clutter','Clutter','Location','SouthWest') end keyboard end @@ -1456,8 +1677,10 @@ dim_Aperp = Nc - Nsrc; surf_doas_rad = keep_surf_doas*pi/180; - sv_fh_arg_gslc = {'theta'}; - sv_fh_arg_gslc{2} = [theta_desired(des_idx), surf_doas_rad(:)']; % array_proc_sv breaks if this is a column vector -- fix this! +% sv_fh_arg_gslc = {'theta'}; +% sv_fh_arg_gslc{2} = [theta_desired(des_idx), surf_doas_rad(:)']; % array_proc_sv breaks if this is a column vector -- fix this! + sv_opt_arg = []; + sv_opt_arg.theta = [theta_desired(des_idx), surf_doas_rad(:)']; for ml_idx = 1:length(cfg.fcs) % Make column vectors of y and z-positions @@ -1465,8 +1688,11 @@ y_pos{ml_idx}(wf_adc_idx,1) = cfg.fcs{ml_idx}{wf_adc_idx}.pos(2,rline); z_pos{ml_idx}(wf_adc_idx,1) = cfg.fcs{ml_idx}{wf_adc_idx}.pos(3,rline); end + % Determine Steering Vectors for target and interference - [~,A] = cfg.sv_fh(sv_fh_arg_gslc,cfg.wfs.fc,y_pos{ml_idx},z_pos{ml_idx}); + sv_arg = []; + sv_arg = {cfg.wfs.fc*sqrt(cfg.sv_dielectric), y_pos{ml_idx}, z_pos{ml_idx}, sv_opt_arg, cfg.lut, cfg.fcs{1}{1}.roll(rline)}; + [~,A] = cfg.sv_fh(sv_arg{:}); % Find orthonormal basis for the orthogonal complement of C(A) % (nullspace of A transpose) Ca = zeros(Nc,dim_Aperp); @@ -1510,17 +1736,27 @@ g_geo = vertcat(1,zeros(Nsrc-1,1)); % Unity gain towards nadir, nulls in clutter directions sv_fh_arg_geonull = {'theta'}; sv_fh_arg_geonull{2} = [theta, surf_doas_rad(:)']; % array_proc_sv breaks if this is a column vector -- fix this! + sv_opt_arg = []; + sv_opt_arg.theta = [theta, surf_doas_rad(:)']; % array_proc_sv breaks if this is a column vector -- fix this! % Determine Steering Vectors for target and interference - [~,Aint] = cfg.sv_fh(sv_fh_arg_geonull,cfg.wfs.fc,y_pos{ml_idx},z_pos{ml_idx}); + sv_arg = []; + sv_arg = {cfg.wfs.fc*sqrt(cfg.sv_dielectric), y_pos{ml_idx}, z_pos{ml_idx}, sv_opt_arg, cfg.lut, cfg.fcs{1}{1}.roll(rline)}; + [~,Aint] = cfg.sv_fh(sv_arg{:}); +% [~,Aint] = cfg.sv_fh(sv_fh_arg_geonull,cfg.wfs.fc*sqrt(cfg.sv_dielectric),y_pos{ml_idx},z_pos{ml_idx}); % Apply pseudoinverse to g (project orthogonally to interference) wgeo = Aint * inv(Aint'*Aint) *g_geo; theta_vec = linspace(-pi/2,pi/2,256); - sv_patt_arg = sv_fh_arg_gslc; - sv_patt_arg{2} = [theta_vec]; - [~,Amanifold] = cfg.sv_fh(sv_patt_arg,cfg.wfs.fc,y_pos{ml_idx},z_pos{ml_idx}); +% sv_patt_arg = sv_fh_arg_gslc; +% sv_patt_arg{2} = [theta_vec]; + sv_opt_arg = []; + sv_opt_arg.theta = [theta_vec]; + sv_arg = []; + sv_arg = {cfg.wfs.fc*sqrt(cfg.sv_dielectric), y_pos{ml_idx}, z_pos{ml_idx}, sv_opt_arg, cfg.lut, cfg.fcs{1}{1}.roll(rline)}; + [~,Amanifold] = cfg.sv_fh(sv_arg{:}); +% [~,Amanifold] = cfg.sv_fh(sv_patt_arg,cfg.wfs.fc*sqrt(cfg.sv_dielectric),y_pos{ml_idx},z_pos{ml_idx}); % Constituent patterns of the sidelobe canceller Pattgslc = w_gslc'*Amanifold; @@ -1666,6 +1902,7 @@ doa_param.theta_guard = cfg.doa_theta_guard/180*pi; doa_param.search_type = cfg.doa_init; doa_param.options = optimoptions(@fmincon,'Display','off','Algorithm','sqp','TolX',1e-3); + doa_param.lut_roll = cfg.fcs{1}{1}.roll(rline); for src_idx = 1:cfg.Nsrc doa_param.src_limits{src_idx} = doa_param.doa_constraints(src_idx).init_src_limits/180*pi; @@ -1692,7 +1929,12 @@ % Apply pseudoinverse and estimate power for each source Nsv2{1} = 'theta'; Nsv2{2} = doa(:)'; - [~,A] = cfg.sv_fh(Nsv2,doa_param.fc,doa_param.y_pc,doa_param.z_pc); + sv_opt_arg = []; + sv_opt_arg.theta = doa(:); + sv_arg = []; + sv_arg = {cfg.wfs.fc*sqrt(cfg.sv_dielectric), y_pos{ml_idx}, z_pos{ml_idx}, sv_opt_arg, cfg.lut, cfg.fcs{1}{1}.roll(rline)}; + [~,A] = cfg.sv_fh(sv_arg{:}); +% [~,A] = cfg.sv_fh(Nsv2,doa_param.fc,doa_param.y_pc,doa_param.z_pc); Weights = (A'*A)\A'; S_hat = Weights*dataSample.'; P_hat = mean(abs(S_hat).^2,2); @@ -2002,7 +2244,6 @@ tmp_doa_step_L = kk_std * doa_step_L; tmp_doa_step_R = kk_std * doa_step_R; - param.array.prior.model = 1; cfg.prior.model = 1; elseif 0 % 2) Fixed wide bounds @@ -2013,7 +2254,6 @@ tmp_doa_step_L = 5*pi/pi; tmp_doa_step_R = 5*pi/180; - param.array.prior.model = 2; cfg.prior.model = 2; elseif 0 % 3) Looser bounds when doa_step < certain threshold @@ -2043,7 +2283,6 @@ tmp_doa_step_L = a2_L*doa_step_L; tmp_doa_step_R = a2_R*doa_step_R; - param.array.prior.model = 3; cfg.prior.model = 3; elseif 0 % 4) Continuous bounding replacemnt for 3) (John's idea 1) @@ -2107,7 +2346,6 @@ tmp_doa_step_L = doa_step_geom_L; tmp_doa_step_R = doa_step_geom_R; - param.array.prior.model = 4; cfg.prior.model = 4; elseif 1 % 5) Geomtry-based bounding (John's idea 2) @@ -2168,7 +2406,6 @@ tmp_doa_step_L = kk_std * doa_step_L; tmp_doa_step_R = kk_std * doa_step_R; - param.array.prior.model = 5; cfg.prior.model = 5; if 0 figure; @@ -2599,7 +2836,12 @@ % Apply pseudoinverse and estimate power for each source Nsv2{1} = 'theta'; Nsv2{2} = doa(:)'; - [~,A] = cfg.sv_fh(Nsv2,doa_param.fc,doa_param.y_pc,doa_param.z_pc); + sv_opt_arg = []; + sv_opt_arg.theta = doa(:); + sv_arg = []; + sv_arg = {cfg.wfs.fc*sqrt(cfg.sv_dielectric), y_pos{ml_idx}, z_pos{ml_idx}, sv_opt_arg, cfg.lut, cfg.fcs{1}{1}.roll(rline)}; + [~,A] = cfg.sv_fh(sv_arg{:}); +% [~,A] = cfg.sv_fh(Nsv2,doa_param.fc*sqrt(cfg.sv_dielectric),doa_param.y_pc,doa_param.z_pc); % k = 4*pi*doa_param.fc/c; % A = sqrt(1/length(doa_param.y_pc))*exp(1i*k*(doa_param.y_pc*sin(doa(:)).' - doa_param.z_pc*cos(doa(:)).')); Weights = (A'*A)\A'; @@ -3151,8 +3393,11 @@ g = vertcat(1,zeros(size(keep_surf_doas))); % Convert to radians for array_proc_sv surf_doas_rad = keep_surf_doas*pi/180; - sv_fh_arg = {'theta'}; - sv_fh_arg{2} = [theta_desired(des_idx), surf_doas_rad(:)']; % array_proc_sv breaks if this is a column vector -- fix this! + theta_desired_rad = theta_desired(des_idx).*pi/180; +% sv_fh_arg = {'theta'}; +% sv_fh_arg{2} = [theta_desired(des_idx), surf_doas_rad(:)']; % array_proc_sv breaks if this is a column vector -- fix this! + sv_opt_arg = []; + sv_opt_arg.theta = [theta_desired_rad, surf_doas_rad(:)']; % array_proc_sv breaks if this is a column vector -- fix this! % Estimate power for ml_idx = 1:length(cfg.fcs) @@ -3162,9 +3407,14 @@ z_pos{ml_idx}(wf_adc_idx,1) = cfg.fcs{ml_idx}{wf_adc_idx}.pos(3,rline); end % Determine Steering Vectors for target and interference - [~,A] = cfg.sv_fh(sv_fh_arg,cfg.wfs.fc,y_pos{ml_idx},z_pos{ml_idx}); + sv_arg = []; + sv_arg = {cfg.wfs.fc*sqrt(cfg.sv_dielectric), y_pos{ml_idx}, z_pos{ml_idx}, sv_opt_arg, cfg.lut, cfg.fcs{1}{1}.roll(rline)}; + [~,A] = cfg.sv_fh(sv_arg{:}); +% [~,A] = cfg.sv_fh(sv_fh_arg,cfg.wfs.fc*sqrt(cfg.sv_dielectric),y_pos{ml_idx},z_pos{ml_idx}); % Apply pseudoinverse to g + warning off w = A * inv(A'*A) *g; + warning on w = w ./ sqrt(w'*w); sv_gn{ml_idx} = w; @@ -3190,7 +3440,7 @@ surf_power(des_idx) = surf_power(des_idx) ... + mean(abs(dataSample*conj(wgeo)).^2); end - surf_power(des_idx) = surf_power(des_idx) / length(din); + surf_power(des_idx) = surf_power(des_idx) / length(din); end end % Store the power estimated from each source @@ -3319,7 +3569,7 @@ for idx = 1:length(cfg.method) if cfg.method(idx) < DOA_METHOD_THRESHOLD m = array_proc_method_str(cfg.method(idx)); - Sarray.(m) = reshape(Sarray.(m),param.array.Nsv,Nt_out); + Sarray.(m) = reshape(Sarray.(m),cfg.Nsv,Nt_out); end end @@ -3392,7 +3642,7 @@ if 0 % Degug (for S-MAP only): plot lower/upper bound and left/rignt mean figure();clf; - uniform_theta = linspace(param.array.doa_constraints(1).src_limits(1),param.array.doa_constraints(1).src_limits(2),501); + uniform_theta = linspace(cfg.doa_constraints(1).src_limits(1),cfg.doa_constraints(1).src_limits(2),501); img_plot = interp1(tout.music.tomo.theta*180/pi,tout.music.tomo.img(:,:,line_idx).',uniform_theta).'; imagesc(uniform_theta,1:size(tout.music.tomo.img,1),10*log10(abs(img_plot))) hold on @@ -3422,7 +3672,7 @@ h5 = plot([bounds_l(:,2) bounds_r(:,2)],1:size(bounds_l,1),'-.g','LineWidth',1.5); h2 = plot(theta_plot,1:size(theta_plot,1),'-y','LineWidth',1.5); - xlim([param.array.doa_constraints(1).src_limits(1) param.array.doa_constraints(1).src_limits(2)]) + xlim([cfg.doa_constraints(1).src_limits(1) cfg.doa_constraints(1).src_limits(2)]) ylim([first_NoNaN_rbin last_NoNaN_rbin] + [-5 +5]) % ylim([first_NoNaN_rbin-5 3105]) % ylim([first_NoNaN_rbin-5 1970]) diff --git a/cresis-toolbox/processing/array_proc_sv.m b/cresis-toolbox/processing/array_proc_sv.m index c0b04c01..78708a2d 100644 --- a/cresis-toolbox/processing/array_proc_sv.m +++ b/cresis-toolbox/processing/array_proc_sv.m @@ -1,161 +1,126 @@ -function [theta,sv] = array_proc_sv(Nsv, fc, yAnt, zAnt, roll, LUT, rx_paths) -% [theta,sv] = array_proc_sv(Nsv, fc, yAnt, zAnt, roll, LUT, rx_paths) -% -% Function specified in sv_fh column of array worksheet of param -% spreadsheet for use with array_proc. -% -% Nsv = number of steering vectors to create -% fc = center frequency (Hz) -% yAnt,zAnt = phase centers from lever arm (note this includes tx and -% rx positions, so nyquist sampling is quarter wavelength for the -% phase centers because of two way propagation... i.e. k = 4*pi*fc/c) -% These are Ny by 1 vectors -% roll = roll information (aircraft attitude) -% LUT = measured steering vectors lookup table -% rx_paths = active receive adcs -% -% sv = steering vector of size Ny by Nsv -% theta = incidence angle vector of size 1 by Nsv, defined by atan2(ky,kz) -% -% y is increasing to the right -% z is increasing downwards -% sv = sqrt(1/length(yAnt)) * exp(1i*(-zAnt*kz + yAnt*ky)); -% A positive ky with a positive y (y points to the left) implies a positive -% phase for the steering vector. This means the measurement is closer to -% the target the bigger y gets (i.e. the more to the left you go) -% and therefore positive ky implies a target from the left. Positive ky -% corresponds to positive theta. -% kz is always positive. A positive z is always moving away from the target. -% -% Author: John Paden - -% Decide ideal or measured steering vectors generation -ni = nargin; -% Creation of linear steering vector for 2D arbitrary array -c = 2.997924580003452e+08; % physical_constants too slow -% Wavenumber for two way propagation -k = 4*pi*fc/c; - -if iscell(Nsv) - if strcmpi(Nsv{1},'theta') - theta = Nsv{2}; - - % shape doas into row vector - theta = theta(:).'; - - if ni == 4 % Ideal generation - ky = k*sin(theta); - kz = k*cos(theta); - - elseif ni > 4 % Measured generation - - if 1 - theta_lut = theta - roll; - sv_corr = (interp1(LUT.bins,LUT.sv_real,theta_lut,'linear','extrap') + 1i*interp1(LUT.bins,LUT.sv_imag,theta_lut,'linear','extrap')).'; - - ky = k*sin(theta); - kz = k*cos(theta); - sv = sqrt(1/length(yAnt)) * exp(1i*(-zAnt*kz + yAnt*ky)); - sv = sv .* sv_corr; - - else - theta = theta + roll; - ky = k*sin(theta); - kz = k*cos(theta); - range = -45:1:45; % Default sv correction table range - theta_deg = round(theta*180/pi); - sv = zeros(size(yAnt,1),length(theta_deg)); - if length(theta_deg) > length(yAnt) - angle_all = zeros(size(sv)); - % angle_all(:,1) = -zAnt*kz(1) + yAnt*ky(1); - % angle_all(:,end) = -zAnt*kz(end) + yAnt*ky(end); - sv(:,1) = sqrt(1/length(yAnt)) * exp(1i*(-zAnt*kz(1) + yAnt*ky(1))); - sv(:,end) = sqrt(1/length(yAnt)) * exp(1i*(-zAnt*kz(end) + yAnt*ky(end))); - - idx1 = find(theta_deg >= range(1),1,'first'); - idx2 = find(theta_deg <= range(end),1,'last'); - - [~,~,roll_idx] = intersect(theta_deg(idx1:idx2),range); - % tmp = zeros(length(yAnt),length(roll_idx)); - tmp = conj(LUT.sv_table(rx_paths,roll_idx)); - % angle_all(:,idx1:idx2) = angle(conj(LUT.sv_table(rx_paths,roll_idx))); - sv(:,idx1:idx2) = sqrt(1/length(yAnt)) * abs(tmp) .* exp(1i * angle(tmp)); - - % for ant_idx = 1:length(yAnt) - angle_all = interp1([theta_deg(1) range theta_deg(end)], [angle(sv(:,1)) angle(sv(:,idx1:idx2)) angle(sv(:,end))].', theta_deg(1):theta_deg(end)).'; - % angle_all = interp1([theta_deg(1) range theta_deg(end)], [angle_all(:,1) angle_all(:,idx1:idx2) angle_all(:,end)].', theta_deg(1):theta_deg(end)).'; - % mag_tmp = interp1([theta_deg(1) range theta_deg(end)], [abs(sv(ant_idx,1)) abs(sv(ant_idx,idx1:idx2)) abs(sv(ant_idx,end))], theta_deg(1):theta_deg(end)); - sv = sqrt(1/length(yAnt)) .* exp(1i * angle_all); - % end - - else - % Individual sv generation, no use in array_proc, just for - % integrity - for theta_idx = 1:length(theta_deg) - tmp = zeros(length(yAnt),1); - if (theta_deg(theta_idx) >= range(1) & theta_deg(theta_idx) <= range(end)) - roll_idx = find(theta_deg(theta_idx) == range); - tmp = conj(LUT.sv_table(:,roll_idx)); - sv(:,theta_idx) = sqrt(1/length(yAnt)) * abs(tmp) .* exp(1i * angle(tmp)); - else - sv(:,theta_idx) = sqrt(1/length(yAnt)) * exp(1i*(-zAnt*kz(theta_idx) + yAnt*ky(theta_idx))); - end - end - - end - end - - return; - end - end - -else - - % Choose equally spaced y-dimension (cross-track) wavenumbers - dNsv = 2/Nsv; - ky = dNsv *k* [0 : floor((Nsv-1)/2), -floor(Nsv/2) : -1]; - - % Determine z-dimension (elevation) dimension wavenumbers for each ky - kz = sqrt(k^2 - ky.^2); - - % Calculate the angle of arrival for each ky - theta = atan2(ky,kz); - % - % if ni > 4 % Measured generation - % [theta,sort_idx] = sort(theta,'ascend'); - % theta = theta + roll; - % ky = k*sin(theta); - % kz = k*cos(theta); - % range = -45:1:45; % Default sv correction table range - % theta_deg = round(theta*180/pi); - % sv = zeros(size(yAnt,1),length(theta_deg)); - % - % angle_all = zeros(size(sv)); - % angle_all(:,1) = -zAnt*kz(1) + yAnt*ky(1); - % angle_all(:,end) = -zAnt*kz(end) + yAnt*ky(end); - % idx1 = find(theta_deg >= range(1),1,'first'); - % idx2 = find(theta_deg <= range(end),1,'last'); - % - % [~,~,roll_idx] = intersect(theta_deg(idx1:idx2),range); - % % tmp = zeros(length(yAnt),length(roll_idx)); - % % tmp = conj(LUT.sv_table(rx_paths,roll_idx)); - % angle_all(:,idx1:idx2) = interp1(range, angle(conj(LUT.sv_table(rx_paths,roll_idx))).',theta_deg(idx1:idx2)).' - % % angle_all(:,idx1:idx2) = angle(conj(LUT.sv_table(rx_paths,roll_idx))); - % % sv(:,idx1:idx2) = sqrt(1/length(yAnt)) * abs(tmp) .* exp(1i * angle(tmp)); - % - % % for ant_idx = 1:length(yAnt) - % angle_all = interp1([theta_deg(1) theta_deg(idx1:idx2) theta_deg(end)], [angle_all(:,1) angle_all(:,idx1:idx2) angle_all(:,end)].', theta_deg(1):theta_deg(end)).'; - % % mag_tmp = interp1([theta_deg(1) range theta_deg(end)], [abs(sv(ant_idx,1)) abs(sv(ant_idx,idx1:idx2)) abs(sv(ant_idx,end))], theta_deg(1):theta_deg(end)); - % sv = sqrt(1/length(yAnt)) .* exp(1i * angle_all); - % return; - % end - -end -if nargout < 2 - return; -end - -% Take the outer product of the antenna positions with the trig(theta) -% to create 2D matrix. Normalize the steering vector lengths. -sv = sqrt(1/length(yAnt)) * exp(1i*(-zAnt*kz + yAnt*ky)); -% Equivalent: sv = sqrt(1/length(yAnt)) * exp(1i*k*(-zAnt*cos(theta) + yAnt*sin(theta))); +function [theta,sv] = array_proc_sv(fc, yAnt, zAnt, sv_arg, LUT, LUT_roll) +% [theta,sv] = array_proc_sv(fc, yAnt, zAnt, sv_arg, LUT, LUT_roll) +% +% Function specified in sv_fh column of array worksheet of param +% spreadsheet for use with array_proc. +% +% sv_arg: Several Options: +% sv_arg is a struct: +% .theta: theta vector (radians) of angles at which steering vectors +% will be made. +% sv_arg is numeric scalar: +% Contains the number of uniformily sampled in wavenumber space +% steering vectors that will be created. +% fc: center frequency (Hz) +% To adjust the dielectric of the steering vectors, pass in the effective +% center frequency after considering the dielectric. For example: +% fc_effective = fc_actual*sqrt(relative_dielectric) +% yAnt,zAnt: phase centers from lever arm (note this includes tx and +% rx positions, so nyquist sampling is quarter wavelength for the +% phase centers because of two way propagation... i.e. k = 4*pi*fc/c) +% These are Ny by 1 vectors +% LUT_roll: roll information (aircraft attitude) +% LUT: measured steering vectors lookup table (relative to zero-roll) +% +% sv = steering vector of size Ny by Nsv +% theta = incidence angle vector of size 1 by Nsv, defined by atan2(ky,kz), +% positive theta is left looking +% +% Useful points to understand the coordinate system: +% * y points to the left +% * z points upwards +% * theta is zero at nadir and increases to the left (theta is positive for +% targets on the left). Theta is restricted to -90 to +90. +% * k is the wavenumber (4*pi/lambda) where lambda = c/fc where c is the +% speed of light and fc is the center frequency +% * ky = k*sin(theta), ky is positive for targets on the left +% * kz = k*cos(theta), kz is always positive in our field of view +% * The expected wave is exp(1i*-k*R) where R is the distance to the target +% and k is the wavenumber. As an antenna is moved toward the target, its +% phase should become more positive because R is getting smaller and if an +% antenna is moved away from the target, its phase should be more negative. +% * Moving up, or in the positive z direction, causes the phase to be more +% negative. +% * Moving left, or in the positive y direction, causes the phase to be +% more positive for targets on the left since theta and ky are positive on +% the left). +% * Steering vectors are defined as: sv = sqrt(1/length(yAnt)) * +% exp(1i*(-zAnt*kz + yAnt*ky)); +% * We see that increasing "zAnt" always leads to a more negative phase +% * We see that increasing "yAnt" (i.e. moving it to the left) causes a +% more positive phase when ky is positive (i.e. target is on the left). +% * TO DO: The sign of theta should be switched so that left targets have a +% negative value since this would be more natural for plotting. This change +% would require many changes through out the code (especially for +% tomographic processing code) and has not been done yet. +% +% Author: John Paden + +%% Initialization +if ~exist('LUT','var') + LUT = []; +end + +if ~exist('LUT_roll','var') + LUT_roll = []; +end + +%% Calculate k +% Creation of linear steering vector for 2D arbitrary array +c = 2.997924580003452e+08; % physical_constants too slow +% Wavenumber for two way propagation +k = 4*pi*fc/c; + +%% Calculate ky, kz, and theta +if isstruct(sv_arg) + theta = sv_arg.theta; + + % shape doas into row vector + theta = theta(:).'; + + ky = k*sin(theta); + kz = k*cos(theta); + +else + Nsv = sv_arg; + % Choose equally spaced y-dimension (cross-track) wavenumbers + dNsv = 2/Nsv; + ky = dNsv *k* [0 : floor((Nsv-1)/2), -floor(Nsv/2) : -1]; + + % Determine z-dimension (elevation) dimension wavenumbers for each ky + kz = sqrt(k^2 - ky.^2); + + % Calculate the angle of arrival for each ky + theta = atan2(ky,kz); + +end +%% Return now if only theta is needed +if nargout < 2 + return; +end + +%% Create nominal sv table +% Take the outer product of the antenna positions with the trig(theta) +% to create 2D matrix. Normalize the steering vector lengths. +sv = sqrt(1/length(yAnt)) * exp(1i*(-zAnt*kz + yAnt*ky)); +% Equivalent: sv = sqrt(1/length(yAnt)) * exp(1i*k*(-zAnt*cos(theta) + yAnt*sin(theta))); + +%% Apply sv correction + +if ~isempty(LUT) && ~isempty(LUT_roll) + theta_lut = theta - LUT_roll; + sv_real = real(LUT.sv); + sv_imag = imag(LUT.sv); + sv_corr = (interp1(LUT.doa, sv_real, theta_lut,'linear','extrap') + 1i*interp1(LUT.doa, sv_imag,theta_lut,'linear','extrap')).'; + if any(any(isnan(sv_corr))) + nan_mask = isnan(sv_corr); + badmask = cumsum(nan_mask,1) | cumsum(nan_mask,1,'reverse'); + sv_corr(badmask) = 1*exp(1i*0); + end + + +% sv = sqrt(1/length(yAnt)) * exp(1i*(-zAnt*kz + yAnt*ky)); + sv = sv .* sv_corr; +end + + diff --git a/cresis-toolbox/processing/array_task.m b/cresis-toolbox/processing/array_task.m index 3270b64c..ee40b382 100644 --- a/cresis-toolbox/processing/array_task.m +++ b/cresis-toolbox/processing/array_task.m @@ -38,8 +38,8 @@ tmp_param = param; tmp_param.cmd.frms = max(1,param.load.frm-1) : min(length(frames.frame_idxs),param.load.frm+1); -if strcmpi(param.array.surf_layer.source, 'surfData') - surf_dir = ct_filename_out(param,'','surfData_sar'); +if strcmpi(param.array.surf_layer.source, 'surf_sar') + surf_dir = ct_filename_out(param,'','surf_sar'); fn_name = sprintf('Data_%s_%03.0f.mat',param.day_seg,param.load.frm); fn = fullfile(surf_dir,fn_name); surf_layer = tomo.surfdata(fn,param); @@ -183,7 +183,7 @@ if dTsys ~= 0 % Positive dTsys means Tsys > Tsys_old and we should reduce the % time delay to all targets by dTsys. - sar_data.(data_field_name) = ifft(bsxfun(@times,fft(sar_data.(data_field_name)),exp(1i*2*pi*sar_data.wfs(wf).freq*dTsys))); + sar_data.(data_field_name) = ifft(bsxfun(@times,fft(sar_data.(data_field_name),[],1),exp(1i*2*pi*sar_data.wfs(wf).freq*dTsys)),[],1); end % Concatenate data (resample in fast-time if needed since @@ -257,6 +257,7 @@ .* exp(1i*( ... param.radar.wfs(wf).chan_equal_deg(param.radar.wfs(wf).rx_paths(adc)) ... - sar_data.param_sar.radar.wfs(wf).chan_equal_deg(param.radar.wfs(wf).rx_paths(adc)) )/180*pi); + end % Correct any changes in Tsys @@ -266,7 +267,7 @@ if dTsys ~= 0 % Positive dTsys means Tsys > Tsys_old and we should reduce the % time delay to all targets by dTsys. - sar_data.(data_field_name) = ifft(bsxfun(@times,fft(sar_data.(data_field_name)),exp(1i*2*pi*sar_data.wfs(wf).freq*dTsys))); + sar_data.(data_field_name) = ifft(bsxfun(@times,fft(sar_data.(data_field_name),[],1),exp(1i*2*pi*sar_data.wfs(wf).freq*dTsys)),[],1); end % Concatenate data (handle situation of time axis of previous @@ -276,10 +277,8 @@ % Load next chunk data % =============================================================== - % If on the first chunk of the frame, then look at the previous - % frame. Special cases (like frm == 1 and empty - % param.load.prev_frm_num_chunks are handled by letting the file - % search fail). + % If on the last chunk of the frame, then look at the next + % frame. Special case when frm > number-of-frames checked. if param.load.chunk_idx == param.load.num_chunks load_frm = param.load.frm+1; load_chunk_idx = 1; @@ -298,7 +297,7 @@ sprintf('wf_%02.0f_adc_%02.0f_chk_%03.0f', wf, adc, load_chunk_idx),'','.mat'); end - if ~next_chunk_failed_flag && ~isempty(sar_type_fn) + if ~next_chunk_failed_flag && ~isempty(sar_type_fn) && load_frm <= length(frames.frame_idxs) % If file exists, then load it sar_data = load(sar_type_fn{1}); @@ -333,7 +332,7 @@ if dTsys ~= 0 % Positive dTsys means Tsys > Tsys_old and we should reduce the % time delay to all targets by dTsys. - sar_data.(data_field_name) = ifft(bsxfun(@times,fft(sar_data.(data_field_name)),exp(1i*2*pi*sar_data.wfs(wf).freq*dTsys))); + sar_data.(data_field_name) = ifft(bsxfun(@times,fft(sar_data.(data_field_name),[],1),exp(1i*2*pi*sar_data.wfs(wf).freq*dTsys)),[],1); end % Concatenate data (resample in fast-time if needed since @@ -360,11 +359,11 @@ if prev_chunk_failed_flag % Remove prev chunk data + lat = lat(num_prev_chunk_rlines+1:end); + lon = lon(num_prev_chunk_rlines+1:end); + elev = elev(num_prev_chunk_rlines+1:end); for ml_idx = 1:length(ml_list) data{ml_idx} = data{ml_idx}(:,num_prev_chunk_rlines+1:end,:,:,:); - lat = lat(num_prev_chunk_rlines+1:end); - lon = lon(num_prev_chunk_rlines+1:end); - elev = elev(num_prev_chunk_rlines+1:end); for wf_adc = 1:size(wf_adc_list,1) fcs{ml_idx}{wf_adc}.origin = fcs{ml_idx}{wf_adc}.origin(:,num_prev_chunk_rlines+1:end); fcs{ml_idx}{wf_adc}.x = fcs{ml_idx}{wf_adc}.x(:,num_prev_chunk_rlines+1:end); @@ -384,11 +383,11 @@ if next_chunk_failed_flag % Remove next chunk data + lat = lat(1:end-num_next_chunk_rlines); + lon = lon(1:end-num_next_chunk_rlines); + elev = elev(1:end-num_next_chunk_rlines); for ml_idx = 1:length(ml_list) data{ml_idx} = data{ml_idx}(:,1:end-num_next_chunk_rlines,:,:,:); - lat = lat(1:end-num_next_chunk_rlines); - lon = lon(1:end-num_next_chunk_rlines); - elev = elev(1:end-num_next_chunk_rlines); for wf_adc = 1:size(wf_adc_list,1) fcs{ml_idx}{wf_adc}.origin = fcs{ml_idx}{wf_adc}.origin(:,1:end-num_next_chunk_rlines); fcs{ml_idx}{wf_adc}.x = fcs{ml_idx}{wf_adc}.x(:,1:end-num_next_chunk_rlines); @@ -408,10 +407,10 @@ %% Process: Fast-time oversampling % ======================================================================= - if isfield(param.array,'ft_over_sample') && ~isempty(param.array.ft_over_sample) + if param.array.ft_over_sample ~= 1 % param.array.ft_over_sample should be a positive integer for ml_idx = 1:length(data) - data{ml_idx} = interpft(data{ml_idx},size(data{ml_idx},1) * param.array.ft_over_sample); + data{ml_idx} = interpft_memeff(data{ml_idx},size(data{ml_idx},1) * param.array.ft_over_sample); end for wf = 1:length(sar_data.wfs) sar_data.wfs(wf).fs = sar_data.wfs(wf).fs * param.array.ft_over_sample; @@ -422,11 +421,12 @@ sar_data.wfs(wf).freq = sar_data.wfs(wf).fc ... + sar_data.wfs(wf).df * ifftshift( -floor(sar_data.wfs(wf).Nt/2) : floor((sar_data.wfs(wf).Nt-1)/2) ).'; end + Time = sar_data.wfs(wf_base).time; end param.array_proc.chan_equal = chan_equal; param.array_proc.fcs = fcs; - + %% Process: WBDCM Setup % ======================================================================= % Setup fields for wideband space-time doa estimator passed to @@ -484,7 +484,7 @@ Hwin = sar_data.param_sar.csarp.ft_wind(length(sar_data.wfs(wf).time)); % Convert to time-domain and over-sample by Mt % - Take real to remove rounding errors that result in imag part - Hwin = interpft(real(ifft(ifftshift(Hwin).^2)), Mt*length(Hwin)); + Hwin = interpft_memeff(real(ifft(ifftshift(Hwin).^2)), Mt*length(Hwin)); % Store the impulse response and corresponding time axis for % passing to array_proc % - Ensure we grab enough samples of the impulse response so that @@ -518,18 +518,18 @@ param.array_proc.surface = surf_layer.twtt*ones(size(param.array_proc.lines)); else - if strcmpi(param.array.surf_layer.source, 'surfData') + if strcmpi(param.array.surf_layer.source, 'surf_sar') % If surf_layer source is surfData (twtt from DEM), just grab values % for the chunk - surf_index = surf_layer.get_index('top twtt'); - icemask_index = surf_layer.get_index('ice_mask'); - theta_frm = repmat(surf_layer.theta,1,numel(surf_layer.gps_time)); + surf_index = surf_layer.get_index({'top twtt'}); + icemask_index = surf_layer.get_index({'ice_mask'}); + theta_frm = repmat(surf_layer.x(:,1),1,numel(surf_layer.gps_time)); gps_frm = repmat(surf_layer.gps_time,numel(surf_layer.theta),1); - theta_chunk = repmat(surf_layer.theta,1,numel(fcs{1}{1}.gps_time)); + theta_chunk = repmat(surf_layer.x(:,1),1,numel(fcs{1}{1}.gps_time)); gps_chunk = repmat(fcs{1}{1}.gps_time,numel(surf_layer.theta),1); param.array_proc.surface = interp_finite(interp2(gps_frm, theta_frm,surf_layer.surf(surf_index).y,gps_chunk,theta_chunk)); param.array_proc.ice_mask = ... - interp_finite(interp2(gps_frm,theta_frm,surf_layer.surf(icemask_index).y,gps_chunk,theta_chunk,'nearest')); + interp_finite(interp2(gps_frm,theta_frm,surf_layer.surf(icemask_index).y,gps_chunk,theta_chunk,'nearest')); param.array_proc.surface_theta = surf_layer.theta; else % source = 'layerData' @@ -622,19 +622,19 @@ elseif length(surf_layer.gps_time) == 1; % Handle special case 2: gps time is length 1, repeat twtt over rlines param.array_proc.surface = surf_layer.twtt*ones(size(rlines)); - elseif strcmpi(param.array.surf_layer.source,'surfData') + elseif strcmpi(param.array.surf_layer.source,'surf_sar') % If surf_layer source is surfData (twtt from DEM), just grab values % for the chunk - surf_index = surf_layer.get_index('top twtt'); - icemask_index = surf_layer.get_index('ice_mask'); - theta_frm = repmat(surf_layer.theta,1,numel(surf_layer.gps_time)); - gps_frm = repmat(surf_layer.gps_time,numel(surf_layer.theta),1); - theta_chunk = repmat(surf_layer.theta,1,numel(fcs{1}{1}.gps_time)); - gps_chunk = repmat(fcs{1}{1}.gps_time,numel(surf_layer.theta),1); - param.array_proc.surface = interp_finite(interp2(gps_frm, theta_frm,surf_layer.surf(surf_index).y,gps_chunk,theta_chunk)); + surf_index = surf_layer.get_index({'top twtt'}); + icemask_index = surf_layer.get_index({'ice mask'}); + param.array_proc.surface_theta = surf_layer.surf(surf_index).x(:,1); + theta_frm = repmat(param.array_proc.surface_theta,1,numel(surf_layer.gps_time)); + gps_frm = repmat(surf_layer.gps_time,length(param.array_proc.surface_theta),1); + theta_chunk = repmat(param.array_proc.surface_theta,1,numel(fcs{1}{1}.gps_time)); + gps_chunk = repmat(fcs{1}{1}.gps_time,length(param.array_proc.surface_theta),1); + param.array_proc.surface = interp_finite(interp2(gps_frm, theta_frm,surf_layer.surf(surf_index).y,gps_chunk,theta_chunk),nan); param.array_proc.ice_mask = ... - interp_finite(interp2(gps_frm,theta_frm,surf_layer.surf(icemask_index).y,gps_chunk,theta_chunk,'nearest')); - param.array_proc.surface_theta = surf_layer.theta; + interp_finite(interp2(gps_frm,theta_frm,surf_layer.surf(icemask_index).y,gps_chunk,theta_chunk,'nearest'),nan); else % Path for layer param.array_proc.surface = interp_finite(interp1(surf_layer.gps_time, ... @@ -644,6 +644,82 @@ % end + % Load Look Up Table + if strcmpi(param.array.sv_model,'lookup_table') + % To be updated. Currently LUT is stored by day_seg. Intent is to have + % one LUT for the season in the analysis folder. Once estimate_sv_lut + % has been updated, delete first conditional and only use the statement + % after the else below. + + % lut_fn is either passed in as a fullfile by the user or built from + % the default paths in the input checking of array proc. + % If the user doesn't specify the fullfile, they must specify + % at a minimum the following: + % param.array.lut_type: choices are either 'training' or 'process' + % param.array.lut_dayseg: if user specifies the process lut type, + % lut_dayseg can be blank. + % param.array.lut_path: defaults to the CSARP_array_manifold folder + % param.array.sv_model: 'lookup_table' + % param.array.lut_method: choices are 'evd' for principal + % eigenvector, 'mmse' for a weiner filter type, and 'ave' for + % coherent averaged steering vector. + lut_fn = param.array.lut_fn; + + load(lut_fn,'sv','doa','param_array_manifold'); + + if isfield(param.array,'lut_method') & ~isempty(param.array.lut_method) + if length(sv) > 1 + sv_idx = find(strcmpi({sv.method},param.array.lut_method)==true); + sv = sv(sv_idx).manifold; + end + end + + % Check dimensionality + Nsv = length(doa); + N1 = size(sv,1); + if N1 < Nsv + sv = transpose(sv); + end + + lut_sv = nan(length(doa),size(param.array.imgs{img}{1}(:,1),1)); + + wf = param.array.imgs{img}{1}(1,1); + for wf_adc = 1:size(param.array.imgs{img}{1}(:,1),1) + wf = param.array.imgs{img}{1}(wf_adc,1); + adc = param.array.imgs{img}{1}(wf_adc,2); + rx = param.radar.wfs(wf).rx_paths(adc); + rx_lut_index = find(param_array_manifold.rx_list == rx); + if ~isempty(rx_lut_index) + lut_sv(:,wf_adc) = sv(:,rx_lut_index); + end + end + param.array_proc.lut.sv = lut_sv; + param.array_proc.lut.doa = doa; + clear sv doa + +% if strcmpi(param.array.lut_type,'process') +% lut_dir = ct_fileparts(ct_filename_out(param,param.array.lut_path)); +% +% % Does not support subband luts +% if regexp(param.array.sv_lut_path,'analysis') +% lut_fn = fullfile(ct_filename_out(param,param.array.sv_lut_path,[],1),'lut.mat'); +% elseif isfield(param.array, 'lut_day_seg') && ~isempty(param.array.lut_day_seg) +% lut_dir = fullfile(ct_filename_out(param,param.array.sv_lut_path,[],1),param.array.lut_day_seg); +% lut_fn = fullfile(lut_dir, sprintf('lut_%s.mat',param.array.lut_day_seg)); +% else +% lut_dir = ct_filename_out(param,param.array.sv_lut_path); +% lut_fn = fullfile(lut_dir, sprintf('lut_%s.mat',param.day_seg)); +% end + % if strcmpi(param.array.sv_lut_path,'estimate_sv_lut') + % lut_dir = ct_filename_out(param,param.array.sv_lut_path); + % lut_fn = fullfile(lut_dir, sprintf('lut_%s.mat',param.day_seg)); + % else + % lut_fn = fullfile(ct_filename_out(param,param.array.sv_lut_path,[],1),'lut.mat'); + % end + + else + param.array_proc.lut = []; + end % Array Processing Function Call [param,dout] = array_proc(param,data); end @@ -692,17 +768,18 @@ else file_version = '1'; end + file_type = 'array_tmp'; if ~param.array.tomo_en % Do not save tomographic 3D-image ct_save('-v7.3',array_fn,'Data','Latitude','Longitude','Elevation','GPS_time', ... 'Surface','Bottom','Time','param_array','param_records', ... - 'param_sar', 'Roll', 'Pitch', 'Heading', 'file_version'); + 'param_sar', 'Roll', 'Pitch', 'Heading', 'file_type', 'file_version'); else % Save tomographic 3D-image Tomo = dout.tomo; ct_save('-v7.3',array_fn,'Tomo','Data','Latitude','Longitude','Elevation','GPS_time', ... 'Surface','Bottom','Time','param_array','param_records', ... - 'param_sar', 'Roll', 'Pitch', 'Heading', 'file_version'); + 'param_sar', 'Roll', 'Pitch', 'Heading', 'file_type', 'file_version'); end end diff --git a/cresis-toolbox/processing/basic_file_loader.m b/cresis-toolbox/processing/basic_file_loader.m index 8f4583f8..cb546649 100644 --- a/cresis-toolbox/processing/basic_file_loader.m +++ b/cresis-toolbox/processing/basic_file_loader.m @@ -9,29 +9,33 @@ % param,defaults from default_radar_params_SEASON_NAME_RADAR_NAME % % OPTIONAL INPUTS: (any missing inputs will be prompted for so none required) -% param.file_search_mode: string which may be one of the following: +% param.config.file_search_mode: string which may be one of the following: % 'default','default+s','default+1','specific','last_file','segment','map' -% param.base_dir_search: cell array of base directories to search for files +% param.config.base_dir_search: cell array of base directories to search for files % in -% param.img: wf-adc array, Nx2 matrix where first column is the waveform, +% param.config.img: wf-adc array, Nx2 matrix where first column is the waveform, % second column is the adc, and each of the N rows specifies a channel to % load -% param.recs: 1x2 vector where first entry is the first record to load +% param.config.recs: 1x2 vector where first entry is the first record to load % from the file (zero indexed) and the second entry is how many records % to load (inf loads all records) % % Author: John Paden -if ~isfield(param,'file_search_mode') - param.file_search_mode = ''; +if ~isfield(param,'config') + param.config = []; end -if ~isfield(param,'base_dir_search') - param.base_dir_search = {''}; +if ~isfield(param.config,'file_search_mode') || isempty(param.config.file_search_mode) + param.config.file_search_mode = ''; end -if ~isfield(param,'multiselect') - param.multiselect = false; +if ~isfield(param.config,'base_dir_search') || isempty(param.config.base_dir_search) + param.config.base_dir_search = {''}; +end + +if ~isfield(param.config,'multiselect') || isempty(param.config.multiselect) + param.config.multiselect = false; end [output_dir,radar_type,radar_name] = ct_output_dir(param.radar_name); @@ -44,8 +48,8 @@ default = defaults{1}; global g_basic_file_loader_search; -if isempty(param.file_search_mode) ... - || all(~strcmpi(param.file_search_mode,{'default','default+s','default+1','specific','last_file','segment','map'})) +if isempty(param.config.file_search_mode) ... + || all(~strcmpi(param.config.file_search_mode,{'default','default+s','default+1','specific','last_file','segment','map'})) fprintf('\nSelect file search mode:\n'); fprintf(' 1: default (loads whatever was last loaded)'); if strcmpi(g_basic_file_loader_search,'default') @@ -77,55 +81,55 @@ file_search_mode = input('Selection (1-5): '); try if isempty(file_search_mode) && ~isempty(g_basic_file_loader_search) - param.file_search_mode = g_basic_file_loader_search; + param.config.file_search_mode = g_basic_file_loader_search; done = true; elseif file_search_mode==1 - param.file_search_mode = 'default'; + param.config.file_search_mode = 'default'; done = true; elseif file_search_mode==2 - param.file_search_mode = 'specific'; + param.config.file_search_mode = 'specific'; done = true; elseif file_search_mode==3 - param.file_search_mode = 'last_file'; + param.config.file_search_mode = 'last_file'; done = true; elseif file_search_mode==4 - param.file_search_mode = 'segment'; + param.config.file_search_mode = 'segment'; done = true; elseif file_search_mode==5 - param.file_search_mode = 'map'; + param.config.file_search_mode = 'map'; done = true; end end end end -if any(strcmpi(param.file_search_mode,{'specific','last_file','segment','map'})) - g_basic_file_loader_search = param.file_search_mode; +if any(strcmpi(param.config.file_search_mode,{'specific','last_file','segment','map'})) + g_basic_file_loader_search = param.config.file_search_mode; end %% Determine which file to load -if ~strncmpi(param.file_search_mode,'default',length('default')) - good_mask = logical(zeros(size(param.base_dir_search))); - for base_dir_idx = 1:length(param.base_dir_search) - base_dir = param.base_dir_search{base_dir_idx}; +if ~strncmpi(param.config.file_search_mode,'default',length('default')) + good_mask = logical(zeros(size(param.config.base_dir_search))); + for base_dir_idx = 1:length(param.config.base_dir_search) + base_dir = param.config.base_dir_search{base_dir_idx}; if exist(base_dir,'dir') good_mask(base_dir_idx) = true; end end - param.base_dir_search = param.base_dir_search(good_mask); + param.config.base_dir_search = param.config.base_dir_search(good_mask); global g_basic_file_loader_base_dir; base_dir = []; while isempty(base_dir) default_base_dir_idx = []; % If no default exists, then set default to first in list - if isempty(g_basic_file_loader_base_dir) && ~isempty(param.base_dir_search) - g_basic_file_loader_base_dir = param.base_dir_search{1}; + if isempty(g_basic_file_loader_base_dir) && ~isempty(param.config.base_dir_search) + g_basic_file_loader_base_dir = param.config.base_dir_search{1}; end % Print options to user fprintf('\n'); - for base_dir_idx = 1:length(param.base_dir_search) - fprintf(' %d: %s', base_dir_idx, param.base_dir_search{base_dir_idx}); - if strcmp(param.base_dir_search{base_dir_idx},g_basic_file_loader_base_dir) + for base_dir_idx = 1:length(param.config.base_dir_search) + fprintf(' %d: %s', base_dir_idx, param.config.base_dir_search{base_dir_idx}); + if strcmp(param.config.base_dir_search{base_dir_idx},g_basic_file_loader_base_dir) fprintf(' *'); default_base_dir_idx = base_dir_idx; end @@ -139,7 +143,7 @@ fprintf('\n'); % Get input from user try - if ~isempty(param.base_dir_search) + if ~isempty(param.config.base_dir_search) base_dir_idx = input(sprintf('Choose base directory [%d]: ',default_base_dir_idx)); else % Automatically choose custom if no base directories exist @@ -148,9 +152,9 @@ if isempty(base_dir_idx) base_dir_idx = default_base_dir_idx; end - if base_dir_idx <= length(param.base_dir_search) - base_dir = param.base_dir_search{base_dir_idx}; - elseif base_dir_idx == length(param.base_dir_search)+1 + if base_dir_idx <= length(param.config.base_dir_search) + base_dir = param.config.base_dir_search{base_dir_idx}; + elseif base_dir_idx == length(param.config.base_dir_search)+1 % Custom selected while ~exist(base_dir,'dir') base_dir = input('Enter custom directory path: ','s'); @@ -163,15 +167,14 @@ end g_basic_file_loader_base_dir = base_dir; - if strcmpi(param.file_search_mode,'last_file') + if strcmpi(param.config.file_search_mode,'last_file') global g_basic_file_loader_selection; - board_folder_name = defaults{1}.board_folder_name; - adc = defaults{1}.records.file.adcs(1); - board = adc_to_board(param.radar_name,adc); - board_folder_name = regexprep(board_folder_name,'%02d',sprintf('%02.0f',adc)); - board_folder_name = regexprep(board_folder_name,'%d',sprintf('%.0f',adc)); - board_folder_name = regexprep(board_folder_name,'%b',sprintf('%.0f',board)); - fns = get_filenames(fullfile(base_dir,board_folder_name),'','','.bin',struct('recursive',true)); + + adc = param.config.img(1,2); + [board,board_idx,~] = wf_adc_to_board(param,param.config.img(1,:)); + board_folder_name = param.records.file.board_folder_name; + board_folder_name = regexprep(board_folder_name,'%b',param.records.file.boards{board_idx}); + fns = get_filenames(fullfile(base_dir,board_folder_name),param.records.file.prefix,'','.bin',struct('recursive',true)); if isempty(fns) error('No data files in: %s\n', base_dir); end @@ -201,15 +204,12 @@ end g_basic_file_loader_selection = fn_idx; - elseif strcmpi(param.file_search_mode,'specific') - board_folder_name = defaults{1}.board_folder_name; - adc = defaults{1}.records.file.adcs(1); - board = adc_to_board(param.radar_name,adc); - board_folder_name = regexprep(board_folder_name,'%02d',sprintf('%02.0f',adc)); - board_folder_name = regexprep(board_folder_name,'%d',sprintf('%.0f',adc)); - board_folder_name = regexprep(board_folder_name,'%b',sprintf('%.0f',board)); - adc_dir = fullfile(base_dir,board_folder_name); - + elseif strcmpi(param.config.file_search_mode,'specific') + adc = param.config.img(1,2); + [board,board_idx,~] = wf_adc_to_board(param,param.config.img(1,:)); + board_folder_name = param.records.file.board_folder_name; + board_folder_name = regexprep(board_folder_name,'%b',param.records.file.boards{board_idx}); + data_fns = get_filenames(fullfile(base_dir,board_folder_name),param.records.file.prefix,'','.bin'); fn = ''; fprintf('\n'); while ~exist(fn,'file') @@ -220,14 +220,14 @@ [~,fn] = fileparts(g_basic_file_loader_fn); end - fn = get_filename(adc_dir,'',fn,'',struct('recursive',true)); + fn = get_filename(fullfile(base_dir,board_folder_name),'',fn,'',struct('recursive',true)); g_basic_file_loader_fns = {fn}; end end - elseif strcmpi(param.file_search_mode,'segment') + elseif strcmpi(param.config.file_search_mode,'segment') - if any(defaults{1}.records.file.version == [9 10 103 412]) + if any(param.records.file.version == [9 10 103 412]) % Arena based systems system_xml_fns = get_filenames(fullfile(base_dir),'','','system.xml',struct('recursive',true)); clear settings; @@ -240,25 +240,25 @@ else % NI based systems % Prepare inputs - xml_version = defaults{1}.xml_version; + xml_version = param.config.cresis.config_version; cresis_xml_mapping; [settings,settings_enc] = read_ni_xml_directory(base_dir,xml_file_prefix,false); end % Get the files that match the first wf-adc pair requested - if any(defaults{1}.records.file.version == [9 10 103 412]) + if any(param.records.file.version == [9 10 103 412]) % Arena based systems - wf = param.img(1,1); - adc = param.img(1,2); + wf = param.config.img(1,1); + adc = param.config.img(1,2); found = false; - for board_idx = 1:length(defaults{1}.records.arena.data_map) - for profile_idx = 1:size(defaults{1}.records.arena.data_map{board_idx},1) - if defaults{1}.records.arena.data_map{board_idx}(profile_idx,3) == wf ... - && defaults{1}.records.arena.data_map{board_idx}(profile_idx,4) == adc + for board_idx = 1:length(param.records.arena.data_map) + for profile_idx = 1:size(param.records.arena.data_map{board_idx},1) + if param.records.arena.data_map{board_idx}(profile_idx,3) == wf ... + && param.records.arena.data_map{board_idx}(profile_idx,4) == adc board_idx = board_idx; - board = defaults{1}.records.file.boards(board_idx); - mode_latch = defaults{1}.records.arena.data_map{board_idx}(profile_idx,1); - subchannel = defaults{1}.records.arena.data_map{board_idx}(profile_idx,2); + board = param.records.file.boards(board_idx); + mode_latch = param.records.arena.data_map{board_idx}(profile_idx,1); + subchannel = param.records.arena.data_map{board_idx}(profile_idx,2); found = true; break; end @@ -268,11 +268,12 @@ end end if ~found - error('wf-adc pair (%d,%d) was not found in defaults{1}.records.arena.data_map. Update default_radar_params_*.m or param.img.', wf, adc); + error('wf-adc pair (%d,%d) was not found in param.records.arena.data_map. Update default_radar_params_*.m or param.config.img.', wf, adc); end % Get raw data files associated with this directory - board_folder_name = defaults{1}.records.file.board_folder_name; - board_folder_name = regexprep(board_folder_name,'%b',sprintf('%.0f',board)); + [board,board_idx,~] = wf_adc_to_board(param,param.config.img(1,:)); + board_folder_name = param.records.file.board_folder_name; + board_folder_name = regexprep(board_folder_name,'%b',param.records.file.boards{board_idx}); fn_datenums = []; data_fns = get_filenames(fullfile(base_dir,board_folder_name),'','','.dat',struct('recursive',true,'regexp','[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]')); @@ -289,13 +290,11 @@ else % NI based systems - board_folder_name = defaults{1}.board_folder_name; - adc = defaults{1}.records.file.adcs(1); - board = adc_to_board(param.radar_name,adc); - board_folder_name = regexprep(board_folder_name,'%02d',sprintf('%02.0f',adc)); - board_folder_name = regexprep(board_folder_name,'%d',sprintf('%.0f',adc)); - board_folder_name = regexprep(board_folder_name,'%b',sprintf('%.0f',board)); - data_fns = get_filenames(fullfile(base_dir,board_folder_name),defaults{1}.data_file_prefix,'','.bin'); + adc = param.config.img(1,2); + [board,board_idx,~] = wf_adc_to_board(param,param.config.img(1,:)); + board_folder_name = param.records.file.board_folder_name; + board_folder_name = regexprep(board_folder_name,'%b',param.records.file.boards{board_idx}); + data_fns = get_filenames(fullfile(base_dir,board_folder_name),param.records.file.prefix,'','.bin'); % Get the date information out of the filename fn_datenums = []; for data_fn_idx = 1:length(data_fns) @@ -307,7 +306,7 @@ % Print out settings for each segment (XML file) fprintf('\nData segments:\n'); for set_idx = 1:length(settings) - if any(defaults{1}.records.file.version == [9 10 103 412]) + if any(param.records.file.version == [9 10 103 412]) fprintf(' %d: %s (%d seq, %d files)\n', set_idx, ... settings(set_idx).psc_config_name, length(settings(set_idx).psc.mode), length(settings(set_idx).fns{board_idx})); @@ -347,7 +346,7 @@ end end - if any(defaults{1}.records.file.version == [9 10 103 412]) + if any(param.records.file.version == [9 10 103 412]) data_fns = settings(set_idx).fns{board_idx}; else if set_idx < length(settings) @@ -383,13 +382,13 @@ end end - elseif strcmpi(param.file_search_mode,'map') + elseif strcmpi(param.config.file_search_mode,'map') basic_file_loader_map; end g_basic_file_loader_fn = fn; % Load sequential files if multiselect is turned on - if param.multiselect && any(strcmpi(param.file_search_mode,{'last_file','specific'})) + if param.config.multiselect && any(strcmpi(param.config.file_search_mode,{'last_file','specific'})) global g_num_files; if isempty(g_num_files) g_num_files = 1; @@ -410,9 +409,9 @@ end else - if strcmp(param.file_search_mode,'default') + if strcmp(param.config.file_search_mode,'default') fn = g_basic_file_loader_fn; - elseif strcmp(param.file_search_mode,'default+s') + elseif strcmp(param.config.file_search_mode,'default+s') % Get the next file in g_basic_file_loader_fns if any(strcmpi(radar_name,{'mcords3','mcords5'})) file_idx = find(strcmp(g_basic_file_loader_fn, g_basic_file_loader_fns)); @@ -426,7 +425,7 @@ error('Not supported for %s',param.radar_name); end - elseif strcmp(param.file_search_mode,'default+1') + elseif strcmp(param.config.file_search_mode,'default+1') % Get the next file number after the last loaded file [fn_dir,fn_name,fn_ext] = fileparts(g_basic_file_loader_fn); cur_file_idx = str2double(fn_name(end-3:end)); @@ -441,7 +440,7 @@ % Load the data (disable if you have already loaded) clear data; clear num_rec; -if any(defaults{1}.records.file.version == [9 10 103 412]) +if any(param.records.file.version == [9 10 103 412]) error('Not supported yet.'); elseif strcmpi(radar_name,'mcords') @@ -468,15 +467,14 @@ % basic_remove_mcords_digital_errors; elseif any(strcmpi(radar_name,{'mcords2','mcords3'})) - if ~isfield(param,'img') + if ~isfield(param.config,'img') fprintf('Enter wf-adc pair matrix. The wf-adc pair matrix is an Nx2 matrix\n'); fprintf('where the first column is the waveform, the second column is the adc,\n'); fprintf('and each row represents a channel to be loaded.\n'); - fprintf('Valid ADCs: '); fprintf('%d ', default.records.file.adcs); fprintf('\n'); - param.img = []; - while size(param.img,1) < 1 || size(param.img,2) ~= 2 + param.config.img = []; + while size(param.config.img,1) < 1 || size(param.config.img,2) ~= 2 try - param.img = input('Wf-adc pairs: '); + param.config.img = input('Wf-adc pairs: '); end end end @@ -489,7 +487,7 @@ % Map ADCs to board numbers fn_start = fn; % Store this first file away since fn gets overwritten below for board = 0:3 - if any(board == floor((param.img(:,2)-1)/4)) + if any(board == floor((param.config.img(:,2)-1)/4)) get_adcs = board*4 + (1:4); [fn_dir,fn_name] = fileparts(fn_start); @@ -506,12 +504,12 @@ end for get_adc_idx = 1:length(get_adcs) adc = get_adcs(get_adc_idx); - wf_adc_idx_matches = find(param.img(:,2) == adc); + wf_adc_idx_matches = find(param.config.img(:,2) == adc); if isempty(wf_adc_idx_matches) continue; end for wf_adc_idx = wf_adc_idx_matches(:).' - wf = param.img(wf_adc_idx,1); + wf = param.config.img(wf_adc_idx,1); fprintf(' Loading wf %d, adc %d\n', wf, adc); if ~exist('num_rec','var') % Since each file may have slightly different numbers of @@ -543,6 +541,7 @@ settings_enc = settings_enc(settings_idx); default = default_radar_params_settings_match(defaults,settings); + default = merge_structs(param,default); %% Format settings into pc_param if isfield(settings,'DDC_Ctrl') @@ -577,7 +576,7 @@ dt = 1/hdr.fs; Nt = size(data,1); clear pc_param; - pc_param.img = param.img; + pc_param.img = param.config.img; pc_param.DDC_mode = DDC_mode; pc_param.DDC_freq = DDC_freq; pc_param.f0 = f0; @@ -652,20 +651,19 @@ file_idx = 1; epri_intersect = []; - if ~isfield(param,'img') + if ~isfield(param.config,'img') fprintf('Enter wf-adc pair matrix. The wf-adc pair matrix is an Nx2 matrix\n'); fprintf('where the first column is the waveform, the second column is the adc,\n'); fprintf('and each row represents a channel to be loaded.\n'); - fprintf('Valid ADCs: '); fprintf('%d ', default.records.file.adcs); fprintf('\n'); - param.img = []; - while size(param.img,1) < 1 || size(param.img,2) ~= 2 + param.config.img = []; + while size(param.config.img,1) < 1 || size(param.config.img,2) ~= 2 try - param.img = input('Wf-adc pairs: '); + param.config.img = input('Wf-adc pairs: '); end end end - if ~isfield(param,'recs') + if ~isfield(param.config,'recs') start_rec = []; while length(start_rec) ~= 1 try @@ -684,11 +682,11 @@ end end end - param.recs = [start_rec num_rec]; + param.config.recs = [start_rec num_rec]; end % adcs: a list of the adcs that we are loading - adcs = unique(param.img(:,2)); + adcs = unique(param.config.img(:,2)); fn_start = fn; % Store this first file away since fn gets overwritten below for adc = reshape(adcs,[1 length(adcs)]) @@ -705,10 +703,12 @@ end fprintf('Loading file %s\n', fn); % Load the data file + load_param = param.config.header_load_param; + load_param.recs = param.config.recs; if strcmp(radar_name,'mcords4') - [hdr,data_tmp] = basic_load_mcords4(fn,struct('clk',default.radar.fs/4,'recs',param.recs)); + [hdr,data_tmp] = basic_load_mcords4(fn,load_param); else - [hdr,data_tmp] = basic_load_mcords5(fn,struct('clk',default.radar.fs,'recs',param.recs,'presum_bug_fixed',1)); + [hdr,data_tmp] = basic_load_mcords5(fn,load_param); end % Remove extra records to help reduce total memory usage % if isfield(param,'rlines') && ~isempty(param.rlines) @@ -719,13 +719,13 @@ % hdr.epri = hdr.epri(param.rlines); % end % Map each of the read waveforms needed into the correct output - for wf_adc_idx = 1:size(param.img,1) - % wf,adc: pair of values for this entry in param.img - wf = param.img(wf_adc_idx,1); + for wf_adc_idx = 1:size(param.config.img,1) + % wf,adc: pair of values for this entry in param.config.img + wf = param.config.img(wf_adc_idx,1); if wf > length(data_tmp) error('Requested waveform (%d) is larger than the number of waveforms in the file (%d in the file).', wf, length(data_tmp)); end - if adc == abs(param.img(wf_adc_idx,2)); + if adc == abs(param.config.img(wf_adc_idx,2)); % This pair needs to be loaded, insert into output array... handle % mismatched EPRIs using intersect function. Throw away any records % that do not have matching EPRI in all channels. @@ -777,7 +777,8 @@ settings_enc = settings_enc(settings_idx); default = default_radar_params_settings_match(defaults,settings); - + default = merge_structs(param,default); + %% Format settings into pc_param DDC_freq = double(settings.DDC_Ctrl.NCO_freq)*1e6; DDC_mode = double(settings.DDC_Ctrl.DDC_sel.Val); @@ -799,8 +800,17 @@ hdr.BW_noise = 175e6; end atten = double(settings.DDS_Setup.Waveforms(wf).Attenuator_1(1) + settings.DDS_Setup.Waveforms(wf).Attenuator_2(1)); - hdr.rx_gain = default.radar.rx_gain - atten; - t0 = hdr.wfs(wf).t0 + default.radar.Tadc_adjust; + hdr.rx_gain = param.config.cresis.rx_gain_dB - atten; + + if isfield(default.radar.wfs,'Tadc_adjust') + if length(default.radar.wfs) >= wf + t0 = hdr.wfs(wf).t0 + default.radar.wfs(wf).Tadc_adjust; + else + t0 = hdr.wfs(wf).t0 + default.radar.wfs(1).Tadc_adjust; + end + else + t0 = hdr.wfs(wf).t0 + default.radar.Tadc_adjust; + end tukey = settings.DDS_Setup.RAM_Taper; for wf = 1:length(settings.DDS_Setup.Waveforms) @@ -812,7 +822,7 @@ dt = 1/hdr.fs; Nt = size(data,1); clear pc_param; - pc_param.img = param.img; + pc_param.img = param.config.img; pc_param.DDC_mode = DDC_mode; pc_param.DDC_freq = DDC_freq; pc_param.f0 = f0; @@ -826,7 +836,7 @@ finfo = fname_info_mcords2(fn); [year,month,day] = datevec(finfo.datenum); - hdr.radar_time = utc_to_gps(datenum_to_epoch(datenum(year,month,day,0,0,hdr.utc_time_sod))) + default.vectors.gps.time_offset; + hdr.radar_time = utc_to_gps(datenum_to_epoch(datenum(year,month,day,0,0,hdr.utc_time_sod))) + default.records.gps.time_offset; %% Read GPS files in this directory param.day_seg = sprintf('%04d%02d%02d_01',year,month,day); diff --git a/cresis-toolbox/processing/basic_file_loader_map.m b/cresis-toolbox/processing/basic_file_loader_map.m index eee3ab1b..aa91032c 100644 --- a/cresis-toolbox/processing/basic_file_loader_map.m +++ b/cresis-toolbox/processing/basic_file_loader_map.m @@ -1,14 +1,12 @@ %% Read GPS files in this directory -adc = defaults{1}.records.file.adcs(1); -board = adc_to_board(param.radar_name,adc); -adc_folder_name = defaults{1}.adc_folder_name; -adc_folder_name = regexprep(adc_folder_name,'%02d',sprintf('%02.0f',adc)); -adc_folder_name = regexprep(adc_folder_name,'%d',sprintf('%.0f',adc)); -adc_folder_name = regexprep(adc_folder_name,'%b',sprintf('%.0f',board)); -data_fns = get_filenames(fullfile(base_dir,adc_folder_name),defaults{1}.data_file_prefix,'','.bin'); +adc = param.img(1,2); +[board,board_idx,profile] = wf_adc_to_board(param,param.img(1,:)); +board_folder_name = param.records.file.board_folder_name; +board_folder_name = regexprep(board_folder_name,'%b',param.records.file.boards{board_idx}); +data_fns = get_filenames(fullfile(base_dir,board_folder_name),param.records.file.prefix,'','.bin'); if isempty(data_fns) - error('No data files exist in %s\n', fullfile(base_dir,adc_folder_name)); + error('No data files exist in %s\n', fullfile(base_dir,board_folder_name)); end finfo = fname_info_mcords2(data_fns{1}); @@ -115,11 +113,11 @@ hdr_gps_time = []; for match_idx = settings(set_idx).match_idxs(1:end-1) try - hdr = defaults{1}.header_load_func(data_fns{match_idx},defaults{1}.header_load_params); + hdr = param.config.header_load_func(data_fns{match_idx},param.config.header_load_param); end finfo = fname_info_mcords2(data_fns{match_idx}); [year,month,day] = datevec(finfo.datenum); - hdr_gps_time(end+1) = utc_to_gps(datenum_to_epoch(datenum(year,month,day,0,0,hdr.utc_time_sod))) + defaults{1}.vectors.gps.time_offset; + hdr_gps_time(end+1) = utc_to_gps(datenum_to_epoch(datenum(year,month,day,0,0,hdr.utc_time_sod))) + param.records.gps.time_offset; end hdr_gps_time = utc_to_gps(hdr_gps_time); @@ -138,11 +136,19 @@ fig_number = h_geotiff.h_fig.Number; fn = ''; while isempty(fn) - fprintf('\nPress enter when done selecting files in figure %d\n\n', fig_number); + fprintf('\nPress any key in the command window (not the figure window!) when done selecting files in figure %d\n\n', fig_number); pause; - g_basic_file_loader_fns = h_geotiff.get_selection(); - if ~isempty(g_basic_file_loader_fns) - fn = g_basic_file_loader_fns{1}; + [~,g_basic_file_loader_fns] = h_geotiff.get_selection(); + % Check each segment (settings) to see if any files are selected + for set_idx = 1:length(g_basic_file_loader_fns) + if ~isempty(g_basic_file_loader_fns{set_idx}) + % If any segment has a file selected, then only return files from + % that segment. We do not allow files from multiple segments + % (settings files) to be selected. + g_basic_file_loader_fns = g_basic_file_loader_fns{set_idx}; + fn = g_basic_file_loader_fns{1}; + break; + end end end diff --git a/cresis-toolbox/processing/basic_motion_comp.m b/cresis-toolbox/processing/basic_motion_comp.m index 6437ec0a..1163349f 100644 --- a/cresis-toolbox/processing/basic_motion_comp.m +++ b/cresis-toolbox/processing/basic_motion_comp.m @@ -126,21 +126,17 @@ % ================================================================== % Leverarm and attitude compensation % ================================================================== - - % For readability we change the units of lat/lon from degrees to radians - lat = lat/180*pi; - lon = lon/180*pi; % ----------------------------------------------- % Create flight coordinate system % Determine local up vector at starting point - physical_constants; - [x_ecef,y_ecef,z_ecef] = geodetic2ecef(lat,lon,elev,WGS84.ellipsoid); - [x_local,y_local,z_local] = ecef2lv(x_ecef([1 end]),y_ecef([1 end]),z_ecef([1 end]), ... - lat(1),lon(1),elev(1),WGS84.ellipsoid); - [x_ecef_up,y_ecef_up,z_ecef_up] = lv2ecef(x_local(1),y_local(1),z_local(1)+1, ... - lat(1),lon(1),elev(1),WGS84.ellipsoid); + physical_constants; % Load WGS84.spheroid + [x_ecef,y_ecef,z_ecef] = geodetic2ecef(WGS84.spheroid,lat,lon,elev); + [x_local,y_local,z_local] = ecef2enu(x_ecef([1 end]),y_ecef([1 end]),z_ecef([1 end]), ... + lat(1),lon(1),elev(1),WGS84.spheroid); + [x_ecef_up,y_ecef_up,z_ecef_up] = enu2ecef(x_local(1),y_local(1),z_local(1)+1, ... + lat(1),lon(1),elev(1),WGS84.spheroid); local_up = [x_ecef_up-x_ecef(1); y_ecef_up-y_ecef(1); z_ecef_up-z_ecef(1)]; local_up = local_up / sqrt(dot(local_up,local_up)); % origin @@ -191,8 +187,8 @@ % Assume that flightpath is ideal straight-line and only correct % for attitude (ECEF) [mid_phase_center(1,1),mid_phase_center(2,1),mid_phase_center(3,1)] ... - = lv2ecef(mid_phase_center(2),mid_phase_center(1),-mid_phase_center(3), ... - lat(rline),lon(rline),elev(rline),WGS84.ellipsoid); + = enu2ecef(mid_phase_center(2),mid_phase_center(1),-mid_phase_center(3), ... + lat(rline),lon(rline),elev(rline),WGS84.spheroid); else % Determine ideal straight-line flight path phase center position % (ECEF) @@ -200,18 +196,18 @@ [x_ecef(rline) - fcs_origin(1); y_ecef(rline) - fcs_origin(2); z_ecef(rline) - fcs_origin(3)]); % Convert from ECEF to Geodetic [ideal(1,rline),ideal(2,rline),ideal(3,rline)] ... - = ecef2geodetic(ideal(1,rline),ideal(2,rline),ideal(3,rline),WGS84.ellipsoid); + = ecef2geodetic(WGS84.spheroid, ideal(1,rline),ideal(2,rline),ideal(3,rline)); % Determine mid_phase_center offset from ideal trajectory (ECEF) [mid_phase_center(1),mid_phase_center(2),mid_phase_center(3)] ... - = lv2ecef(mid_phase_center(2),mid_phase_center(1),-mid_phase_center(3), ... - ideal(1,rline),ideal(2,rline),ideal(3,rline),WGS84.ellipsoid); + = enu2ecef(mid_phase_center(2),mid_phase_center(1),-mid_phase_center(3), ... + ideal(1,rline),ideal(2,rline),ideal(3,rline),WGS84.spheroid); ideal(:,rline) = mid_phase_center; end % Convert from NED to ECEF [phase_center(1),phase_center(2),phase_center(3)] ... - = lv2ecef(phase_center(2),phase_center(1),-phase_center(3), ... - lat(rline),lon(rline),elev(rline),WGS84.ellipsoid); + = enu2ecef(phase_center(2),phase_center(1),-phase_center(3), ... + lat(rline),lon(rline),elev(rline),WGS84.spheroid); % Determine offset in ECEF coordinate system phase_center = mid_phase_center - phase_center; @@ -228,9 +224,7 @@ end if param.type == 5 - [ideal(1,:),ideal(2,:),ideal(3,:)] = ecef2geodetic(ideal(1,:),ideal(2,:),ideal(3,:),WGS84.ellipsoid); - ideal(1,:) = ideal(1,:)*180/pi; - ideal(2,:) = ideal(2,:)*180/pi; + [ideal(1,:),ideal(2,:),ideal(3,:)] = ecef2geodetic(WGS84.spheroid,ideal(1,:),ideal(2,:),ideal(3,:)); end return; diff --git a/cresis-toolbox/processing/basic_noise_analysis.m b/cresis-toolbox/processing/basic_noise_analysis.m index c7ad70d3..26e7bc7f 100644 --- a/cresis-toolbox/processing/basic_noise_analysis.m +++ b/cresis-toolbox/processing/basic_noise_analysis.m @@ -23,23 +23,38 @@ function basic_noise_analysis(param,defaults) %% basic_noise_analysis preparation [fn_dir fn_name] = fileparts(fn); -if ~isfield(param,'seg') || isempty(param.seg) - param.seg = ''; + +if ~isfield(param,'basic_noise_analysis') + param.basic_noise_analysis = []; +end + +if ~isfield(param.basic_noise_analysis,'seg') || isempty(param.basic_noise_analysis.seg) + param.basic_noise_analysis.seg = ''; +end + +if ~isfield(param.basic_noise_analysis,'psd_en') || isempty(param.basic_noise_analysis.psd_en) + param.basic_noise_analysis.psd_en = true; +end + +if ~isfield(param.basic_noise_analysis,'pdf_en') || isempty(param.basic_noise_analysis.pdf_en) + param.basic_noise_analysis.pdf_en = false; end -if ~isfield(param,'noise_burst_removal') || isempty(param.noise_burst_removal) - param.noise_burst_removal = 0; + +if ~isfield(param.basic_noise_analysis,'noise_burst_removal') || isempty(param.basic_noise_analysis.noise_burst_removal) + param.basic_noise_analysis.noise_burst_removal = 0; end + clear data_tmp; default_noise_rbins = 1:size(data,1); -if ~isfield(param,'noise_rbins') || isempty(param.noise_rbins) +if ~isfield(param.basic_noise_analysis,'noise_rbins') || isempty(param.basic_noise_analysis.noise_rbins) noise_rbins = default_noise_rbins; -elseif param.noise_rbins(end) > size(data,1) - noise_rbins = param.noise_rbins(1):size(data,1); -elseif param.noise_rbins(1) <= 0 - noise_rbins = size(data,1)+param.noise_rbins; +elseif param.basic_noise_analysis.noise_rbins(end) > size(data,1) + noise_rbins = param.basic_noise_analysis.noise_rbins(1):size(data,1); +elseif param.basic_noise_analysis.noise_rbins(1) <= 0 + noise_rbins = size(data,1)+param.basic_noise_analysis.noise_rbins; else - noise_rbins = param.noise_rbins(1):param.noise_rbins(end); + noise_rbins = param.basic_noise_analysis.noise_rbins(1):param.basic_noise_analysis.noise_rbins(end); end noise_rbins = intersect(noise_rbins,default_noise_rbins); @@ -77,55 +92,55 @@ function basic_noise_analysis(param,defaults) fprintf('2: Choose the red box (low altitude)\n'); fprintf('Custom: Enter with square brackets two numbers to choose the start\n'); fprintf(' and stop bin for the noise region. For example "[3400 %d]".\n', 3400+Nt_noise); -param.noise_rbins = []; -while any(size(param.noise_rbins) ~= [1 2]) +param.basic_noise_analysis.noise_rbins = []; +while any(size(param.basic_noise_analysis.noise_rbins) ~= [1 2]) try - param.noise_rbins = input(sprintf('[%d]: ', default_box)); - if isempty(param.noise_rbins) + param.basic_noise_analysis.noise_rbins = input(sprintf('[%d]: ', default_box)); + if isempty(param.basic_noise_analysis.noise_rbins) if noise_power_highalt > noise_power_lowalt - param.noise_rbins = box_lowalt; + param.basic_noise_analysis.noise_rbins = box_lowalt; else - param.noise_rbins = box_highalt; + param.basic_noise_analysis.noise_rbins = box_highalt; end - elseif length(param.noise_rbins) == 1 - if param.noise_rbins == 2 - param.noise_rbins = box_lowalt; + elseif length(param.basic_noise_analysis.noise_rbins) == 1 + if param.basic_noise_analysis.noise_rbins == 2 + param.basic_noise_analysis.noise_rbins = box_lowalt; else - param.noise_rbins = box_highalt; + param.basic_noise_analysis.noise_rbins = box_highalt; end end - param.noise_rbins = sort(param.noise_rbins); + param.basic_noise_analysis.noise_rbins = sort(param.basic_noise_analysis.noise_rbins); end end default_noise_rbins = 1:size(data,1); -if ~isfield(param,'noise_rbins') || isempty(param.noise_rbins) +if ~isfield(param.basic_noise_analysis,'noise_rbins') || isempty(param.basic_noise_analysis.noise_rbins) noise_rbins = default_noise_rbins; -elseif param.noise_rbins(end) > size(data,1) - noise_rbins = param.noise_rbins(1):size(data,1); -elseif param.noise_rbins(1) <= 0 - noise_rbins = size(data,1)+param.noise_rbins; +elseif param.basic_noise_analysis.noise_rbins(end) > size(data,1) + noise_rbins = param.basic_noise_analysis.noise_rbins(1):size(data,1); +elseif param.basic_noise_analysis.noise_rbins(1) <= 0 + noise_rbins = size(data,1)+param.basic_noise_analysis.noise_rbins; else - noise_rbins = param.noise_rbins(1):param.noise_rbins(end); + noise_rbins = param.basic_noise_analysis.noise_rbins(1):param.basic_noise_analysis.noise_rbins(end); end noise_rbins = intersect(noise_rbins,default_noise_rbins); %% Convert from quantization to voltage @ ADC and Additional software presums for wf_adc = 1:size(data,3) - wf = abs(param.img(wf_adc,1)); + wf = abs(param.config.img(wf_adc,1)); % Quantization to voltage data(:,:,wf_adc) = data(:,:,wf_adc) ... - * default.radar.adc_full_scale/2^default.radar.adc_bits ... + * default.radar.Vpp_scale/2^default.radar.adc_bits ... * 2^hdr.wfs(abs(wf)).bit_shifts / hdr.wfs(wf).presums; % Software presums - data(:,1:floor(size(data,2)/param.presums),wf_adc) = fir_dec(data(:,:,wf_adc),param.presums); + data(:,1:floor(size(data,2)/param.config.presums),wf_adc) = fir_dec(data(:,:,wf_adc),param.config.presums); end -data = data(:,1:floor(size(data,2)/param.presums),:); +data = data(:,1:floor(size(data,2)/param.config.presums),:); %% Noise Burst Removal -if param.noise_burst_removal +if param.basic_noise_analysis.noise_burst_removal noise_burst_removal; end @@ -136,30 +151,38 @@ function basic_noise_analysis(param,defaults) [output_dir,radar_type,radar_name] = ct_output_dir(param.radar_name); if strcmp(radar_name,'mcords5') && isfield(hdr,'DDC') && hdr.DDC(1) >= 2 % Add 3 dB for IQ combination - fprintf('Expected ADC noise floor @ ADC %.1f dBm\n', lp((default.radar.adc_full_scale/2/sqrt(2))^2/50,1)+30 - default.radar.adc_SNR_dB + 3 ); - fprintf('Expected Rx noise floor @ ADC %.1f dBm\n', lp(BoltzmannConst*290*hdr.BW_noise*default.radar.noise_figure*10^(hdr.rx_gain/10),1) + 3 +30); + fprintf('Expected ADC noise floor @ ADC %.1f dBm\n', lp((default.radar.Vpp_scale/2/sqrt(2))^2/50,1)+30 - param.config.adc_SNR_dB + 3 ); + fprintf('Expected Rx noise floor @ ADC %.1f dBm\n', lp(BoltzmannConst*290*hdr.BW_noise*param.config.noise_figure*10^(hdr.rx_gain/10),1) + 3 +30); else - fprintf('Expected ADC noise floor @ ADC %.1f dBm\n', lp((default.radar.adc_full_scale/2/sqrt(2))^2/50,1)+30 - default.radar.adc_SNR_dB ); - fprintf('Expected Rx noise floor @ ADC %.1f dBm\n', lp(BoltzmannConst*290*hdr.BW_noise*default.radar.noise_figure*10^(hdr.rx_gain/10),1)+30); + fprintf('Expected ADC noise floor @ ADC %.1f dBm\n', lp((default.radar.Vpp_scale/2/sqrt(2))^2/50,1)+30 - param.config.adc_SNR_dB ); + fprintf('Expected Rx noise floor @ ADC %.1f dBm\n', lp(BoltzmannConst*290*hdr.BW_noise*param.config.noise_figure*10^(hdr.rx_gain/10),1)+30); end fprintf('All powers are compensated to mimic no presums.\n'); fprintf('Noise power (dBm) at each ADC rx input and relative to 50 ohm (dB):\n') noise_power_dBm = zeros(1,size(data,3)); default_noise_50ohm = zeros(1,size(data,3)); for wf_adc = 1:size(data,3) - wf = abs(param.img(wf_adc,1)); - adc = abs(param.img(wf_adc,2)); + wf = abs(param.config.img(wf_adc,1)); + adc = abs(param.config.img(wf_adc,2)); noise_power_dBm(wf_adc) = lp(mean(mean(abs(data(noise_rbins,:,wf_adc)).^2/50, 1), 2) ... - * hdr.wfs(wf).presums * param.presums, 1) + 30; - default_noise_50ohm(wf_adc) = default.noise_50ohm(default.radar.rx_paths(adc)); + * hdr.wfs(wf).presums * param.config.presums, 1) + 30; + if isfield(default.radar.wfs,'rx_paths') + if length(default.radar.wfs) >= wf + default_noise_50ohm(wf_adc) = param.config.noise_50ohm(default.radar.wfs(wf).rx_paths(adc)); + else + default_noise_50ohm(wf_adc) = param.config.noise_50ohm(default.radar.wfs(1).rx_paths(adc)); + end + else + default_noise_50ohm(wf_adc) = param.config.noise_50ohm(default.radar.rx_paths(adc)); + end end -fprintf('wf-adc\t'); fprintf('%2.0f-%2.0f\t', param.img.'); fprintf('\n'); +fprintf('wf-adc\t'); fprintf('%2.0f-%2.0f\t', param.config.img.'); fprintf('\n'); fprintf('Noise \t'); fprintf('%+5.1f\t', noise_power_dBm); fprintf('\n'); fprintf('Rel \t'); fprintf('%+5.1f\t', noise_power_dBm - default_noise_50ohm); fprintf('\n'); %% Quantization analysis % ===================================================================== -if param.pdf_en +if param.basic_noise_analysis.pdf_en for wf_adc = 1:size(data,3) figure(wf_adc); clf; plot(real(data(:,1,wf_adc)),'.'); @@ -196,7 +219,7 @@ function basic_noise_analysis(param,defaults) %% Power Spectrum % ===================================================================== -if param.psd_en +if param.basic_noise_analysis.psd_en plot_combined_psd = true; combined_psd_cmap = hsv(size(data,3)); combined_psd_legend = {}; @@ -205,10 +228,10 @@ function basic_noise_analysis(param,defaults) end for wf_adc = 1:size(data,3) - wf = abs(param.img(wf_adc,1)); - adc = abs(param.img(wf_adc,2)); + wf = abs(param.config.img(wf_adc,1)); + adc = abs(param.config.img(wf_adc,2)); - fir_data = fir_dec(data(noise_rbins(1):noise_rbins(end),:,wf_adc),param.presums); + fir_data = fir_dec(data(noise_rbins(1):noise_rbins(end),:,wf_adc),param.config.presums); if strcmp(radar_name,'mcords5') && isfield(hdr,'DDC') && hdr.DDC(1) >= 2 dt = pc_param.time(2) - pc_param.time(1); @@ -219,7 +242,7 @@ function basic_noise_analysis(param,defaults) figure(400+adc); clf; set(400+adc,'WindowStyle','docked','NumberTitle','off','Name',sprintf('M%d',wf_adc)); plot(freq/1e6, lp(mean(abs(fftshift(fft(fir_data),1)).^2*2^2 / 50,2)/size(fir_data,1)) + 30) - title(sprintf('MeanFFT adc%d ave%d %s/%s', adc, param.presums, param.seg, fn_name),'Interpreter','none'); + title(sprintf('MeanFFT adc%d ave%d %s/%s', adc, param.config.presums, param.basic_noise_analysis.seg, fn_name),'Interpreter','none'); ylabel('Relative power (dB)'); xlabel('Frequency (MHz)'); grid on; @@ -231,7 +254,7 @@ function basic_noise_analysis(param,defaults) figure(300+adc); clf; set(300+adc,'WindowStyle','docked','NumberTitle','off','Name',sprintf('FFT%d',wf_adc)); imagesc([], freq/1e6, lp(fftshift(fft(fir_data),1)) + 30 + 10*log10(2^2/50/size(fir_data,1)) ) - title(sprintf('Freq-space adc%d ave%d %s/%s', adc, param.presums, param.seg, fn_name),'Interpreter','none'); + title(sprintf('Freq-space adc%d ave%d %s/%s', adc, param.config.presums, param.basic_noise_analysis.seg, fn_name),'Interpreter','none'); xlabel('Range line'); ylabel('Frequency (MHz)'); h = colorbar; @@ -258,7 +281,7 @@ function basic_noise_analysis(param,defaults) figure(300+adc); clf; set(300+adc,'WindowStyle','docked','NumberTitle','off','Name',sprintf('FFT%d',wf_adc)); imagesc([], freq/1e6, lp(fft(fir_data)) + 30 + 10*log10(2^2/50/size(fir_data,1)) ) - title(sprintf('Freq-space adc%d ave%d %s/%s', adc, param.presums, param.seg, fn_name),'Interpreter','none'); + title(sprintf('Freq-space adc%d ave%d %s/%s', adc, param.config.presums, param.basic_noise_analysis.seg, fn_name),'Interpreter','none'); xlabel('Range line'); ylabel('Frequency (MHz)'); if fc<(freq(1)+freq(end))/2 @@ -272,7 +295,7 @@ function basic_noise_analysis(param,defaults) figure(400+adc); clf; set(400+adc,'WindowStyle','docked','NumberTitle','off','Name',sprintf('M%d',wf_adc)); plot(freq/1e6, lp(mean(abs(fft(fir_data)).^2*2^2 / 50,2)/size(fir_data,1)) + 30) - title(sprintf('MeanFFT adc%d ave%d %s/%s', adc, param.presums, param.seg, fn_name),'Interpreter','none'); + title(sprintf('MeanFFT adc%d ave%d %s/%s', adc, param.config.presums, param.basic_noise_analysis.seg, fn_name),'Interpreter','none'); ylabel('Relative power (dB)'); xlabel('Frequency (MHz)'); if fc<(freq(1)+freq(end))/2 @@ -288,10 +311,12 @@ function basic_noise_analysis(param,defaults) end end end + link_figures(300+(1:size(data,3))); + link_figures(400+(1:size(data,3))); end if plot_combined_psd legend(h_psd_axes,combined_psd_legend,'location','best') - title(sprintf('PSD All ave%d %s/%s', param.presums, param.seg, fn_name),'Interpreter','none','parent',h_psd_axes); + title(sprintf('PSD All ave%d %s/%s', param.config.presums, param.basic_noise_analysis.seg, fn_name),'Interpreter','none','parent',h_psd_axes); end %% Done diff --git a/cresis-toolbox/processing/basic_rx_chan_equalization.m b/cresis-toolbox/processing/basic_rx_chan_equalization.m index 3cab955d..e5462872 100644 --- a/cresis-toolbox/processing/basic_rx_chan_equalization.m +++ b/cresis-toolbox/processing/basic_rx_chan_equalization.m @@ -44,39 +44,39 @@ function basic_rx_chan_equalization(param,defaults) if file_idx == 1 fn = fns{1}; else - param.file_search_mode = 'default+s'; + param.config.file_search_mode = 'default+s'; [data,fn,settings,default,gps,hdr,pc_param,settings_enc] = basic_file_loader(param,defaults); end - if size(data,2) < param.presums + if size(data,2) < param.config.presums error('Not enough data in file to process. Choose a different file.'); end %% Convert from quantization to voltage @ ADC - wf = abs(param.img(1,1)); + wf = abs(param.config.img(1,1)); data = data ... - * default.radar.adc_full_scale/2^default.radar.adc_bits ... + * default.radar.Vpp_scale/2^default.radar.adc_bits ... * 2^hdr.wfs(abs(wf)).bit_shifts / hdr.wfs(wf).presums; %% Additional software presums for wf_adc = 1:size(data,3) - data(:,1:floor(size(data,2)/param.presums),wf_adc) = fir_dec(data(:,:,wf_adc),param.presums); + data(:,1:floor(size(data,2)/param.config.presums),wf_adc) = fir_dec(data(:,:,wf_adc),param.config.presums); end - data = data(:,1:floor(size(data,2)/param.presums),:); - hdr.radar_time = fir_dec(hdr.radar_time,param.presums); - hdr.gps_time = fir_dec(hdr.gps_time,param.presums); - hdr.lat = fir_dec(hdr.lat,param.presums); - hdr.lon = fir_dec(hdr.lon,param.presums); - hdr.elev = fir_dec(hdr.elev,param.presums); - hdr.roll = fir_dec(hdr.roll,param.presums); - hdr.pitch = fir_dec(hdr.pitch,param.presums); - hdr.heading = fir_dec(hdr.heading,param.presums); + data = data(:,1:floor(size(data,2)/param.config.presums),:); + hdr.radar_time = fir_dec(hdr.radar_time,param.config.presums); + hdr.gps_time = fir_dec(hdr.gps_time,param.config.presums); + hdr.lat = fir_dec(hdr.lat,param.config.presums); + hdr.lon = fir_dec(hdr.lon,param.config.presums); + hdr.elev = fir_dec(hdr.elev,param.config.presums); + hdr.roll = fir_dec(hdr.roll,param.config.presums); + hdr.pitch = fir_dec(hdr.pitch,param.config.presums); + hdr.heading = fir_dec(hdr.heading,param.config.presums); %% Pulse compression [pc_signal,pc_time,pc_freq] = pulse_compress(data,pc_param); %% Track surface - ml_data = lp(fir_dec(abs(pc_signal(:,:,param.ref_wf_adc)).^2,ones(1,5)/5,1)); - good_time_bins = find(pc_time > pc_param.Tpd*1.1 & pc_time > default.basic_surf_track_min_time); + ml_data = lp(fir_dec(abs(pc_signal(:,:,param.config.ref_wf_adc)).^2,ones(1,5)/5,1)); + good_time_bins = find(pc_time > pc_param.Tpd*1.1 & pc_time > param.config.basic_surf_track_min_time); [max_value,surf_bin] = max(ml_data(good_time_bins,:)); surf_bin = surf_bin + good_time_bins(1)-1; @@ -111,8 +111,17 @@ function basic_rx_chan_equalization(param,defaults) else param.mocomp_type = 4; end - param.tx_weights = double(settings.DDS_Setup.Ram_Amplitude(logical(default.tx_DDS_mask))); - param.rx_paths = {}; param.rx_paths{wf} = default.radar.rx_paths; + param.tx_weights = double(settings.DDS_Setup.Ram_Amplitude(logical(param.config.tx_DDS_mask))); + param.rx_paths = {}; + if isfield(default.radar.wfs,'rx_paths') + if length(default.radar.wfs) >= wf + param.rx_paths{wf} = default.radar.wfs(wf).rx_paths; + else + param.rx_paths{wf} = default.radar.wfs(1).rx_paths; + end + else + param.rx_paths{wf} = default.radar.rx_paths; + end param.lever_arm_fh = @lever_arm; param.combine_channels = false; @@ -143,9 +152,9 @@ function basic_rx_chan_equalization(param,defaults) end %% Print Results -for wf_adc = 1:size(param.img,1) - wf = abs(param.img(wf_adc,1)); - adc = param.img(wf_adc,2); +for wf_adc = 1:size(param.config.img,1) + wf = abs(param.config.img(wf_adc,1)); + adc = param.config.img(wf_adc,2); rx_path(wf_adc) = param.rx_paths{wf}(adc); end [rx_path_sort,rx_path_sort_idxs] = sort(rx_path); @@ -155,8 +164,8 @@ function basic_rx_chan_equalization(param,defaults) sw_version = current_software_version; fprintf(' mocomp:%d, wf/adc:%d/%d method:"%s" bins:%d-%d git-hash:%s (%s)\n', ... - param.mocomp_type, param.img(param.ref_wf_adc,1), ... - param.img(param.ref_wf_adc,2), param.delay.method, param.rbins(1), param.rbins(end), ... + param.mocomp_type, param.config.img(param.config.ref_wf_adc,1), ... + param.config.img(param.config.ref_wf_adc,2), param.config.delay.method, param.rbins(1), param.rbins(end), ... sw_version.rev, sw_version.cur_date_time); fprintf('td settings\n'); for file_idx = 1:num_files @@ -186,8 +195,8 @@ function basic_rx_chan_equalization(param,defaults) fprintf('Rx Path\n'); for wf_adc = rx_path_sort_idxs - wf = abs(param.img(wf_adc,1)); - adc = param.img(wf_adc,2); + wf = abs(param.config.img(wf_adc,1)); + adc = param.config.img(wf_adc,2); fprintf('%d\t', param.rx_paths{wf}(adc)); end fprintf('\n'); diff --git a/cresis-toolbox/processing/basic_tx_chan_equalization.m b/cresis-toolbox/processing/basic_tx_chan_equalization.m index 8ce5dcee..33b7a57e 100644 --- a/cresis-toolbox/processing/basic_tx_chan_equalization.m +++ b/cresis-toolbox/processing/basic_tx_chan_equalization.m @@ -21,54 +21,54 @@ function basic_tx_chan_equalization(param,defaults) tstart = tic; % .plot_en = flag to enable plots -param.plot_en = true; +param.basic_tx_chan_equalization.plot_en = true; % .caxis = Color axis limits (leave empty first time since this causes it to use the % defaults). -param.caxis = []; -%param.caxis = [50 120]; +param.basic_tx_chan_equalization.caxis = []; +%param.basic_tx_chan_equalization.caxis = [50 120]; % .ylim = Leave empty the first time (it just uses the default limits then) -param.ylim = []; -%param.ylim = [1 500]; +param.basic_tx_chan_equalization.ylim = []; +%param.basic_tx_chan_equalization.ylim = [1 500]; % .xlim = Leave empty the first time (it just uses the default limits then) % These limits are in range lines post presumming -param.xlim = []; -%param.xlim = [200 450]; +param.basic_tx_chan_equalization.xlim = []; +%param.basic_tx_chan_equalization.xlim = [200 450]; % .rlines = 1x2 vector specifying range lines to process (to select all % range lines to the end, set second element to inf). These are range % lines post presumming -param.rlines = [1 inf]; +param.basic_tx_chan_equalization.rlines = [1 inf]; % .snr_threshold = SNR threshold in dB (range lines exceeding this % SNR for every transmit waveform are included in the estimate, % if even just one waveform does not meet the threshold the % range line is not used) -param.snr_threshold = 10; +param.basic_tx_chan_equalization.snr_threshold = 10; -% param.presums = Integer containing number of presums (coherent +% param.config.presums = Integer containing number of presums (coherent % averaging or stacking) to do -param.presums = 10; +param.config.presums = 10; % param.noise_bins_offsets = 1x2 vector specifying bins relative to peak % to use in estimating the noise power (usually some range before the % peak is used): -% param.noise_rbins = min(surf_bin)+param.noise_rbins_rel(1) : min(surf_bin)+param.noise_rbins_rel(2); -param.noise_rbins_rel = [-20 -10]; +% param.noise_rbins = min(surf_bin)+param.basic_tx_chan_equal.noise_rbins_rel(1) : min(surf_bin)+param.basic_tx_chan_equal.noise_rbins_rel(2); +param.basic_tx_chan_equal.noise_rbins_rel = [-20 -10]; -% param.ref_bins = 1x2 vector specifying bins relative to peak to use in +% param.basic_tx_chan_equal.ref_bins = 1x2 vector specifying bins relative to peak to use in % correlation -param.ref_bins = [-2 2]; -% param.search_bins = 1x2 vector specifying max search range to use when +param.basic_tx_chan_equal.ref_bins = [-2 2]; +% param.basic_tx_chan_equal.search_bins = 1x2 vector specifying max search range to use when % looking for the best correlation (this is to ensure that each output % correlation value has full support... i.e. no roll off effect) -param.search_bins = [-15 15]; -% param.Mt = amount to oversample the correlation -param.Mt = 100; +param.basic_tx_chan_equal.search_bins = [-15 15]; +% param.basic_tx_chan_equal.Mt = amount to oversample the correlation +param.basic_tx_chan_equal.Mt = 100; -param.img = defaults{1}.txequal.img; +param.config.img = param.config.txequal.img; %% Get the mode to run global g_basic_tx_chan_equalization_mode; @@ -123,7 +123,7 @@ function basic_tx_chan_equalization(param,defaults) update_phase = false; % AFTER UPDATING DELAY, ENABLE AMP/PHASE UPDATE and DISABLE DELAY end -param.recs = [0 inf]; +param.config.recs = [0 inf]; [data,fn,settings,default,gps,hdr,pc_param,settings_enc] = basic_file_loader(param,defaults); global g_basic_file_loader_fns; fns = g_basic_file_loader_fns; @@ -136,55 +136,55 @@ function basic_tx_chan_equalization(param,defaults) if file_idx == 1 fn = fns{1}; else - param.file_search_mode = 'default+s'; + param.config.file_search_mode = 'default+s'; [data,fn,settings,default,gps,hdr,pc_param,settings_enc] = basic_file_loader(param,defaults); end - if size(data,2) < param.presums + if size(data,2) < param.config.presums error('Not enough data in file to process. Choose a different file.'); end - param.ref_wf_adc = default.txequal.ref_wf_adc; + ref_wf_adc = param.config.txequal.ref_wf_adc; [fn_dir fn_name] = fileparts(fn); %% Convert from quantization to voltage @ ADC - wf = abs(param.img(1,1)); + wf = abs(param.config.img(1,1)); data = data ... - * default.radar.adc_full_scale/2^default.radar.adc_bits ... + * default.radar.Vpp_scale/2^default.radar.adc_bits ... * 2^hdr.wfs(abs(wf)).bit_shifts / hdr.wfs(wf).presums; %% Additional software presums for wf_adc = 1:size(data,3) - data(:,1:floor(size(data,2)/param.presums),wf_adc) = fir_dec(data(:,:,wf_adc),param.presums); - end - data = data(:,1:floor(size(data,2)/param.presums),:); - hdr.radar_time = fir_dec(hdr.radar_time,param.presums); - hdr.gps_time = fir_dec(hdr.gps_time,param.presums); - hdr.lat = fir_dec(hdr.lat,param.presums); - hdr.lon = fir_dec(hdr.lon,param.presums); - hdr.elev = fir_dec(hdr.elev,param.presums); - hdr.roll = fir_dec(hdr.roll,param.presums); - hdr.pitch = fir_dec(hdr.pitch,param.presums); - hdr.heading = fir_dec(hdr.heading,param.presums); + data(:,1:floor(size(data,2)/param.config.presums),wf_adc) = fir_dec(data(:,:,wf_adc),param.config.presums); + end + data = data(:,1:floor(size(data,2)/param.config.presums),:); + hdr.radar_time = fir_dec(hdr.radar_time,param.config.presums); + hdr.gps_time = fir_dec(hdr.gps_time,param.config.presums); + hdr.lat = fir_dec(hdr.lat,param.config.presums); + hdr.lon = fir_dec(hdr.lon,param.config.presums); + hdr.elev = fir_dec(hdr.elev,param.config.presums); + hdr.roll = fir_dec(hdr.roll,param.config.presums); + hdr.pitch = fir_dec(hdr.pitch,param.config.presums); + hdr.heading = fir_dec(hdr.heading,param.config.presums); %% Pulse compression - [pc_signal,pc_time] = pulse_compress(data,pc_param); + [pc_signal,pc_time,pc_freq] = pulse_compress(data,pc_param); %% Track surface - ml_data = lp(fir_dec(abs(pc_signal(:,:,param.ref_wf_adc)).^2,ones(1,5)/5,1)); - good_time_bins = find(pc_time > pc_param.Tpd*default.basic_surf_track_Tpd_factor & pc_time > default.basic_surf_track_min_time); + ml_data = lp(fir_dec(abs(pc_signal(:,:,ref_wf_adc)).^2,ones(1,5)/5,1)); + good_time_bins = find(pc_time > pc_param.Tpd*param.config.basic_surf_track_Tpd_factor & pc_time > param.config.basic_surf_track_min_time); [max_value,surf_bin] = max(ml_data(good_time_bins,:)); surf_bin = surf_bin + good_time_bins(1)-1; param.noise_rlines = 1:size(ml_data,2); - param.noise_rbins = min(surf_bin)+param.noise_rbins_rel(1) : min(surf_bin)+param.noise_rbins_rel(2); + param.noise_rbins = min(surf_bin)+param.basic_tx_chan_equal.noise_rbins_rel(1) : min(surf_bin)+param.basic_tx_chan_equal.noise_rbins_rel(2); param.noise_rbins = param.noise_rbins(param.noise_rbins >= 1); - param.rlines = 1:size(ml_data,2); + param.basic_tx_chan_equalization.rlines = 1:size(ml_data,2); param.rbins= min(surf_bin)-30 : max(surf_bin)+30; if (all(surf_bin==surf_bin(1)) || isempty(param.noise_rbins)) ... - && default.basic_surf_track_Tpd_factor > -inf - warning('DEBUG: Check surface tracker since surface is at the same range bin for every range line (this is normal for lab tests but not flight test). May need to adjust param.rbins and param.rlines to ensure maximum signal in the window is the nadir surface return. Ensure param.noise_bins and param.noise_rlines enclose a region with appropriate values for the background noise. Run dbcont after setting these parameters correctly.'); + && param.config.basic_surf_track_Tpd_factor > -inf + warning('DEBUG: Check surface tracker since surface is at the same range bin for every range line (this is normal for lab tests but not flight test). May need to adjust param.rbins and param.basic_tx_chan_equalization.rlines to ensure maximum signal in the window is the nadir surface return. Ensure param.noise_bins and param.noise_rlines enclose a region with appropriate values for the background noise. Run dbcont after setting these parameters correctly.'); fprintf('\n'); figure; imagesc(ml_data); @@ -200,18 +200,17 @@ function basic_tx_chan_equalization(param,defaults) for chan = 1:size(tmp,3) data{chan} = tmp(:,:,chan); end - param.wf_mapping = default.txequal.wf_mapping; + param.wf_mapping = param.config.txequal.wf_mapping; param.bad_chan_mask = param.wf_mapping == 0; - param.ref_chan = param.ref_wf_adc; - Hwindow_desired = default.txequal.Hwindow_desired; - max_DDS_amp = default.txequal.max_DDS_amp; - time_delay_desired = default.txequal.time_delay_desired; - phase_desired = default.txequal.phase_desired; + Hwindow_desired = param.config.txequal.Hwindow_desired; + max_DDS_amp = param.config.txequal.max_DDS_amp; + time_delay_desired = param.config.txequal.time_delay_desired; + phase_desired = param.config.txequal.phase_desired; time{1} = pc_time; rbins = param.rbins; - rlines = param.rlines; + rlines = param.basic_tx_chan_equalization.rlines; - xml_version = default.xml_version; + xml_version = param.config.cresis.config_version; cresis_xml_mapping; % .DDS_start_mag = current DDS waveform attenuation in dB or DDS counts @@ -232,11 +231,29 @@ function basic_tx_chan_equalization(param,defaults) % THIS OFTEN NEEDS TO BE SET param.DDS_start_phase_units = 'deg'; - out_xml_fn_dir = param.out_xml_fn_dir; - + out_xml_fn_dir = param.basic_tx_chan_equal.out_xml_fn_dir; + + %% Hack to apply time delay, amplitude and phase. + % ======================================================================= + if 0 + t_delay_hack = zeros(size(data)); + phase_hack = zeros(size(data)); + amp_hack = ones(size(data)); + + % Usually enter the difference between the Original and Mean here: +% t_delay_hack = ([-4.19 -5.07 -3.85 -3.43 -5.39 -4.53 -5.31 -4.64]-[-7.35 -5.57 -2.39 -3.43 0.97 -0.75 -5.91 -4.64])*1e-9; +% phase_hack = (-[-2.6 0.0 41.5 39.4 8.2 -50.4 -51.2 -46.2]+[-138.4 145.2 -33.0 0.0 -163.3 -82.7 111.3 -85.6])/180*pi; +% amp_hack = [1426 3350 2957 3952 4000 3548 2615 273]./[443 894 1502 1751 1605 4000 902 462]; + + for chan = 1:length(data) + data{chan} = (amp_hack(chan) .* exp(1i*phase_hack(chan))) ... + * ifft(fft(data{chan}) .* exp(-1i*2*pi*pc_freq*t_delay_hack(chan))); + end + end + %% Echogram plots % ======================================================================= - if param.plot_en + if param.basic_tx_chan_equalization.plot_en for chan = 1:length(data) if param.wf_mapping(chan) ~= 0 figure(chan); clf; set(gcf,'WindowStyle','docked','NumberTitle','off','Name',sprintf('E %d',chan)); @@ -248,14 +265,14 @@ function basic_tx_chan_equalization(param,defaults) grid on; h = colorbar; set(get(h,'YLabel'),'String','Relative power (dB)'); - if ~isempty(param.caxis) - caxis(param.caxis); + if ~isempty(param.basic_tx_chan_equalization.caxis) + caxis(param.basic_tx_chan_equalization.caxis); end - if ~isempty(param.ylim) - ylim(param.ylim); + if ~isempty(param.basic_tx_chan_equalization.ylim) + ylim(param.basic_tx_chan_equalization.ylim); end - if ~isempty(param.xlim) - xlim(param.xlim); + if ~isempty(param.basic_tx_chan_equalization.xlim) + xlim(param.basic_tx_chan_equalization.xlim); end end end @@ -265,12 +282,12 @@ function basic_tx_chan_equalization(param,defaults) %% Surface tracker % ======================================================================= % Incoherent along-track filtering - surf_data = filter2(ones(1,6),abs(data{param.ref_chan}.^2)); + surf_data = filter2(ones(1,6),abs(data{ref_wf_adc}.^2)); % Simple max search to find surface [surf_vals surf_bins] = max(surf_data(rbins,rlines)); surf_bins = rbins(1)-1 + surf_bins; - if param.plot_en + if param.basic_tx_chan_equalization.plot_en for chan = 1:length(data) if param.wf_mapping(chan) ~= 0 figure(chan); @@ -283,13 +300,13 @@ function basic_tx_chan_equalization(param,defaults) %% Noise power estimate and SNR threshold % ======================================================================= - noise_power = mean(mean(abs(data{abs(param.wf_mapping(param.ref_chan))}(param.noise_rbins,rlines)).^2)); + noise_power = mean(mean(abs(data{abs(param.wf_mapping(ref_wf_adc))}(param.noise_rbins,rlines)).^2)); %% Extract delay (using oversampled cross correlation), phase and amplitude % differences between channels % ======================================================================= - ref_bins = param.ref_bins(1):param.ref_bins(2); - search_bins = param.search_bins(1)+param.ref_bins(1) : param.search_bins(2)+param.ref_bins(2); + ref_bins = param.basic_tx_chan_equal.ref_bins(1):param.basic_tx_chan_equal.ref_bins(2); + search_bins = param.basic_tx_chan_equal.search_bins(1)+param.basic_tx_chan_equal.ref_bins(1) : param.basic_tx_chan_equal.search_bins(2)+param.basic_tx_chan_equal.ref_bins(2); zero_padding_offset = length(search_bins) - length(ref_bins); Hcorr_wind = hanning(length(ref_bins)); clear tx_phases tx_powers peak_val peak_offset; @@ -308,16 +325,16 @@ function basic_tx_chan_equalization(param,defaults) % Gets the time offset relative to the reference channel (a postive % offset means that the channel leads the reference channel) [corr_out,lags] = xcorr(data{chan}(surf_bins(rline_idx)+search_bins,rline), ... - data{param.ref_chan}(surf_bins(rline_idx)+ref_bins,rline) .* Hcorr_wind); - corr_int = interpft(corr_out,param.Mt*length(corr_out)); + data{ref_wf_adc}(surf_bins(rline_idx)+ref_bins,rline) .* Hcorr_wind); + corr_int = interpft(corr_out,param.basic_tx_chan_equal.Mt*length(corr_out)); [peak_val(chan,rline_idx) peak_offset(chan,rline_idx)] = max(corr_int); - peak_offset(chan,rline_idx) = (peak_offset(chan,rline_idx)-1)/param.Mt+1 ... + peak_offset(chan,rline_idx) = (peak_offset(chan,rline_idx)-1)/param.basic_tx_chan_equal.Mt+1 ... + ref_bins(1) + search_bins(1) - 1 - zero_padding_offset; end end end tx_snr = tx_powers ./ noise_power; - good_meas = lp(tx_snr) > param.snr_threshold; + good_meas = lp(tx_snr) > param.basic_tx_chan_equalization.snr_threshold; good_rlines = zeros(size(rlines)); good_rlines(sum(good_meas(~param.bad_chan_mask,:)) == sum(~param.bad_chan_mask)) = 1; good_rlines = logical(good_rlines); @@ -338,7 +355,7 @@ function basic_tx_chan_equalization(param,defaults) wf = abs(param.wf_mapping(chan)); ref_time = peak_offset_time(chan,:); - if chan == param.ref_chan + if chan == ref_wf_adc median_mask = ones(size(good_rlines)); ref_time_mean(chan) = 0; else @@ -352,18 +369,18 @@ function basic_tx_chan_equalization(param,defaults) fprintf('TX %d: %4.2f ns (%4.2f ns)\n', chan, 1e9*ref_time_mean(chan), ... 1e9*std(ref_time(good_rlines & median_mask))); - if param.plot_en + if param.basic_tx_chan_equalization.plot_en figure(120+chan); clf; set(gcf,'WindowStyle','docked','NumberTitle','off','Name',sprintf('Time %d',chan)); plot(ref_time); xlabel('Range line'); ylabel('Relative time (sec)'); end ref_time(~(good_rlines & median_mask)) = NaN; - if param.plot_en + if param.basic_tx_chan_equalization.plot_en hold on; plot(ref_time,'ro'); hold off; - title(sprintf('Relative Time (%d to ref %d)', chan, param.ref_chan)); + title(sprintf('Relative Time (%d to ref %d)', chan, ref_wf_adc)); ylim([min(min(ref_time),-1e-11) max(1e-11,max(ref_time))]); end end @@ -389,9 +406,9 @@ function basic_tx_chan_equalization(param,defaults) for chan = 1:length(param.wf_mapping) if param.wf_mapping(chan) ~= 0 wf = abs(param.wf_mapping(chan)); - ref_power = tx_powers(chan,:)./tx_powers(param.ref_chan,:); + ref_power = tx_powers(chan,:)./tx_powers(ref_wf_adc,:); - if chan == param.ref_chan || update_mode == 4 + if chan == ref_wf_adc || update_mode == 4 median_mask = ones(size(good_rlines)); delta_power(chan) = 0; else @@ -405,18 +422,18 @@ function basic_tx_chan_equalization(param,defaults) fprintf('TX %d: %4.2f dB (%4.2f dB), desired %4.2f\n', chan, delta_power(chan), ... lp(std(ref_power(good_rlines & median_mask))), 20*log10(Hwindow_desired(chan))); - if param.plot_en + if param.basic_tx_chan_equalization.plot_en figure(10+chan); clf; set(gcf,'WindowStyle','docked','NumberTitle','off','Name',sprintf('Pow %d',chan)); plot(lp(ref_power,1)); xlabel('Range line'); ylabel('Relative power (dB)'); end ref_power(~(good_rlines & median_mask)) = NaN; - if param.plot_en + if param.basic_tx_chan_equalization.plot_en hold on; plot(lp(ref_power,1),'ro'); hold off; - title(sprintf('Relative Power (%d to ref %d)', chan, param.ref_chan)); + title(sprintf('Relative Power (%d to ref %d)', chan, ref_wf_adc)); end end end @@ -443,13 +460,13 @@ function basic_tx_chan_equalization(param,defaults) for chan = 1:length(param.wf_mapping) if param.wf_mapping(chan) ~= 0 wf = abs(param.wf_mapping(chan)); - ref_phase = tx_phases(chan,:).*conj(tx_phases(param.ref_chan,:)); + ref_phase = tx_phases(chan,:).*conj(tx_phases(ref_wf_adc,:)); ref_values_real = real(ref_phase); ref_values_imag = imag(ref_phase); median_mask = ones(size(good_rlines)); ref_phase_mean(chan) = mean(ref_phase); -% if chan == param.ref_chan +% if chan == ref_wf_adc % median_mask = ones(size(good_rlines)); % ref_phase_mean(chan) = 1; % else @@ -472,7 +489,7 @@ function basic_tx_chan_equalization(param,defaults) fprintf('WF %d: relative phase: %1.3f radians, %3.1f deg\n', chan, ... angle(ref_phase_mean(chan)), angle(ref_phase_mean(chan))*180/pi); - if param.plot_en + if param.basic_tx_chan_equalization.plot_en figure(20+chan); clf; set(gcf,'WindowStyle','docked','NumberTitle','off','Name',sprintf('Ang %d',chan)); plot(angle(ref_phase)*180/pi); xlabel('Range line'); @@ -480,11 +497,11 @@ function basic_tx_chan_equalization(param,defaults) ylim([-180 180]); end ref_phase(~(good_rlines & median_mask)) = NaN; - if param.plot_en + if param.basic_tx_chan_equalization.plot_en hold on; plot(angle(ref_phase)*180/pi,'ro'); hold off; - title(sprintf('Relative Phase (%d to ref %d)', chan, param.ref_chan)); + title(sprintf('Relative Phase (%d to ref %d)', chan, ref_wf_adc)); end else ref_phase_mean(chan) = 1; @@ -518,9 +535,9 @@ function basic_tx_chan_equalization(param,defaults) end if update_mode == 1 mean_error = mean(results.DDS_time_error,1); - if default.txequal.remove_linear_phase_en + if param.config.txequal.remove_linear_phase_en mean_error = detrend(mean_error); - mean_error = mean_error - mean_error(param.ref_chan); + mean_error = mean_error - mean_error(ref_wf_adc); fprintf('%*s',fn_length,'Mean Error (slope removed)'); else fprintf('%*s',fn_length,'Mean Error'); @@ -531,7 +548,7 @@ function basic_tx_chan_equalization(param,defaults) fprintf('\n'); fprintf('%s',' '*ones(1,fn_length)); for wf = 1:size(mean_error,2) - if abs(mean_error(wf)) <= default.txequal.time_validation(wf) + if abs(mean_error(wf)) <= param.config.txequal.time_validation(wf) fprintf('\tPASS'); else fprintf('\tFAIL'); @@ -540,10 +557,10 @@ function basic_tx_chan_equalization(param,defaults) fprintf('\n'); fprintf('%s',' '*ones(1,fn_length)); for wf = 1:size(mean_error,2) - if abs(mean_error(wf)) <= default.txequal.time_validation(wf) + if abs(mean_error(wf)) <= param.config.txequal.time_validation(wf) fprintf('\t'); else - fprintf('\t%.1f>%.1f', abs(mean_error(wf))*1e9, default.txequal.time_validation(wf)*1e9); + fprintf('\t%.1f>%.1f', abs(mean_error(wf))*1e9, param.config.txequal.time_validation(wf)*1e9); end end fprintf('\n'); @@ -597,7 +614,7 @@ function basic_tx_chan_equalization(param,defaults) fprintf('\n'); fprintf('%s',' '*ones(1,fn_length)); for wf = 1:size(mean_error,2) - if abs(mean_error(wf)) <= default.txequal.amp_validation(wf); + if abs(mean_error(wf)) <= param.config.txequal.amp_validation(wf); fprintf('\tPASS'); else fprintf('\tFAIL'); @@ -606,10 +623,10 @@ function basic_tx_chan_equalization(param,defaults) fprintf('\n'); fprintf('%s',' '*ones(1,fn_length)); for wf = 1:size(mean_error,2) - if abs(mean_error(wf)) <= default.txequal.amp_validation(wf); + if abs(mean_error(wf)) <= param.config.txequal.amp_validation(wf); fprintf('\t'); else - fprintf('\t%.0f>%.0f', abs(mean_error(wf)), default.txequal.amp_validation(wf)); + fprintf('\t%.0f>%.0f', abs(mean_error(wf)), param.config.txequal.amp_validation(wf)); end end fprintf('\n'); @@ -667,7 +684,7 @@ function basic_tx_chan_equalization(param,defaults) end if update_mode == 1 mean_error = mean(results.DDS_phase_error,1); - if default.txequal.remove_linear_phase_en + if param.config.txequal.remove_linear_phase_en [~,tmp] = max(fft(mean_error,100*length(mean_error))); tmp = tmp - 1; mean_error = mean_error .* exp(-1i*2*pi*tmp/100*(0:length(mean_error)-1)/length(mean_error)); fprintf('%*s',fn_length,'Mean Error (slope removed)'); @@ -675,14 +692,14 @@ function basic_tx_chan_equalization(param,defaults) fprintf('%*s',fn_length,'Mean Error'); end mean_error = 180/pi*angle(mean_error); - mean_error = mean_error - mean_error(param.ref_chan); + mean_error = mean_error - mean_error(ref_wf_adc); for wf = 1:size(mean_error,2) fprintf('\t%.1f', mean_error(wf)); end fprintf('\n'); fprintf('%s',' '*ones(1,fn_length)); for wf = 1:size(mean_error,2) - if abs(mean_error(wf)) <= default.txequal.phase_validation(wf); + if abs(mean_error(wf)) <= param.config.txequal.phase_validation(wf); fprintf('\tPASS'); else fprintf('\tFAIL'); @@ -691,10 +708,10 @@ function basic_tx_chan_equalization(param,defaults) fprintf('\n'); fprintf('%s',' '*ones(1,fn_length)); for wf = 1:size(mean_error,2) - if abs(mean_error(wf)) <= default.txequal.phase_validation(wf); + if abs(mean_error(wf)) <= param.config.txequal.phase_validation(wf); fprintf('\t'); else - fprintf('\t%.0f>%.0f', abs(mean_error(wf)), default.txequal.phase_validation(wf)); + fprintf('\t%.0f>%.0f', abs(mean_error(wf)), param.config.txequal.phase_validation(wf)); end end fprintf('\n'); @@ -713,14 +730,14 @@ function basic_tx_chan_equalization(param,defaults) end fprintf('\n'); fprintf('Mean'); - final_DDS_phase = angle(mean(exp(j*results.DDS_phase),1))*180/pi; + final_DDS_phase = angle(mean(exp(j*(results.DDS_phase-results.DDS_phase(ref_wf_adc))),1))*180/pi; for wf = 1:size(results.DDS_time,2) fprintf('\t%.1f', final_DDS_phase(:,wf)); end fprintf('\n'); fprintf('Median'); for wf = 1:size(results.DDS_time,2) - fprintf('\t%.1f', final_DDS_phase(wf) + angle(mean(exp(j*results.DDS_phase(:,wf)) .* exp(-j*final_DDS_phase(wf)/180*pi),1))*180/pi); + fprintf('\t%.1f', final_DDS_phase(wf) + angle(mean(exp(j*(results.DDS_phase(:,wf)-results.DDS_phase(ref_wf_adc))) .* exp(-j*final_DDS_phase(wf)/180*pi),1))*180/pi); end fprintf('\n'); fprintf('Stdev'); @@ -733,7 +750,7 @@ function basic_tx_chan_equalization(param,defaults) fprintf('Delay Compensated Mean'); final_DDS_phase_comp = final_DDS_phase + 360*(final_DDS_time/1e9 ... - param.DDS_start_time)*(pc_param.f0+pc_param.f1)/2; - final_DDS_phase_comp = 180/pi*angle(exp(j*(final_DDS_phase_comp - final_DDS_phase_comp(param.ref_chan))/180*pi)); + final_DDS_phase_comp = 180/pi*angle(exp(j*(final_DDS_phase_comp - final_DDS_phase_comp(ref_wf_adc))/180*pi)); final_DDS_phase_comp(logical(param.bad_chan_mask)) = 0; for wf = 1:size(results.DDS_time,2) fprintf('\t%.1f', final_DDS_phase_comp(:,wf)); @@ -817,7 +834,7 @@ function basic_tx_chan_equalization(param,defaults) % Tx 1 is bit 0, tx 2 is bit 1, tx 3 is bit 2, ... tx_mask = settings_enc.sys.DDSZ5FSetup.Waveforms(wf).TXZ20Mask; tx_mask = fliplr(dec2bin(tx_mask,8))-'0'; - tx_mask = tx_mask | default.txequal.wf_mapping==0; + tx_mask = tx_mask | param.config.txequal.wf_mapping==0; tx_mask = bin2dec(char(fliplr(tx_mask+'0'))); settings_enc.sys.DDSZ5FSetup.Waveforms(wf).TXZ20Mask = uint8(tx_mask); end @@ -837,16 +854,17 @@ function basic_tx_chan_equalization(param,defaults) %% Write RSS Arena XML config file if isfield(default,'arena') && ~strcmpi(param.season_name,'2017_Antarctica_Basler') % Create arena parameter structure - arena = struct('version','1'); - arena.awg = default.arena.awg; - arena.dacs = default.arena.dacs; - arena.dacs_sampFreq = default.arena.dacs_sampFreq; - arena.dacs_internal_delay = default.arena.dacs_internal_delay; - arena.dacs_start_delay = default.arena.dacs_start_delay; - arena.zeropimods = default.arena.zeropimods; - arena.TTL_time = default.arena.TTL_time; - arena.TTL_names = default.arena.TTL_names; - arena.TTL_states = default.arena.TTL_states; + arena = default.arena; +% arena = struct('version','1'); +% arena.awg = default.arena.awg; +% arena.dacs = default.arena.dacs; +% arena.dacs_sampFreq = default.arena.dacs_sampFreq; +% arena.dacs_internal_delay = default.arena.dacs_internal_delay; +% arena.dacs_start_delay = default.arena.dacs_start_delay; +% arena.zeropimods = default.arena.zeropimods; +% arena.TTL_time = default.arena.TTL_time; +% arena.TTL_names = default.arena.TTL_names; +% arena.TTL_states = default.arena.TTL_states; % Ensure non-negative delays min_delay = inf; for wf = 1:length(settings_enc.sys.DDSZ5FSetup.Waveforms) @@ -860,11 +878,14 @@ function basic_tx_chan_equalization(param,defaults) arena.wfs(wf).name = ''; arena.wfs(wf).tukey = settings_enc.sys.DDSZ5FSetup.RAMZ20Taper; arena.wfs(wf).enabled = fliplr(~logical(dec2bin(settings_enc.sys.DDSZ5FSetup.Waveforms(wf).TXZ20Mask(1),8)-'0')); - arena.wfs(wf).scale = double(settings_enc.sys.DDSZ5FSetup.RamZ20Amplitude) .* default.arena.max_tx ./ default.txequal.max_DDS_amp; - arena.wfs(wf).fc = (settings_enc.sys.DDSZ5FSetup.Waveforms(wf).StartZ20Freq ... - + settings_enc.sys.DDSZ5FSetup.Waveforms(wf).StopZ20Freq)/2; - arena.wfs(wf).BW = abs(settings_enc.sys.DDSZ5FSetup.Waveforms(wf).StopZ20Freq ... - - settings_enc.sys.DDSZ5FSetup.Waveforms(wf).StartZ20Freq); + %arena.wfs(wf).scale = double(settings_enc.sys.DDSZ5FSetup.RamZ20Amplitude) .* param.config.max_tx ./ param.config.txequal.max_DDS_amp; + arena.wfs(wf).scale = double(settings_enc.sys.DDSZ5FSetup.RamZ20Amplitude) .* arena.max_tx ./ max_DDS_amp; + arena.wfs(wf).f0 = settings_enc.sys.DDSZ5FSetup.Waveforms(wf).StartZ20Freq; + arena.wfs(wf).f1 = settings_enc.sys.DDSZ5FSetup.Waveforms(wf).StopZ20Freq; +% arena.wfs(wf).fc = (settings_enc.sys.DDSZ5FSetup.Waveforms(wf).StartZ20Freq ... +% + settings_enc.sys.DDSZ5FSetup.Waveforms(wf).StopZ20Freq)/2; +% arena.wfs(wf).BW = abs(settings_enc.sys.DDSZ5FSetup.Waveforms(wf).StopZ20Freq ... +% - settings_enc.sys.DDSZ5FSetup.Waveforms(wf).StartZ20Freq); arena.wfs(wf).delay = settings_enc.sys.DDSZ5FSetup.Waveforms(wf).Delay - min_delay; arena.wfs(wf).phase = settings_enc.sys.DDSZ5FSetup.Waveforms(wf).PhaseZ20Offset; arena.wfs(wf).Tpd = double(settings_enc.sys.DDSZ5FSetup.Waveforms(wf).LenZ20Mult) ... @@ -873,24 +894,49 @@ function basic_tx_chan_equalization(param,defaults) end % Create XML document - xml_doc = write_arena_xml([],'init',arena); - xml_doc = write_arena_xml(xml_doc,'ctu_0013',arena); - xml_doc = write_arena_xml(xml_doc,'dac-ad9129_0014',arena); - xml_doc = write_arena_xml(xml_doc,'dac-ad9129_0014_waveform',arena); - xml_doc = write_arena_xml(xml_doc,'psc_0001',arena); - xml_doc = write_arena_xml(xml_doc,'subsystems',arena); - - out_str = xmlwrite(xml_doc); + xml_param = param; + xml_param.wfs = arena.wfs; + xml_param.prf = 1/arena.PRI; + xml_param.arena = arena; + xml_param.arena.adc = []; + xml_param.board_map = {}; + + [~,xml_param.arena.psc_name] = ct_fileparts(out_xml_fn); + xml_param.arena.fn = fullfile(param.basic_tx_chan_equal.arena_base_dir,[xml_param.arena.psc_name '.xml']); + + [doc,xml_param] = write_arena_xml([],xml_param); + + % Create XML document + out_str = xmlwrite(doc); out_str = ['' out_str(find(out_str==10,1):end)]; - [~,rss_fn_name] = ct_fileparts(out_xml_fn); - rss_fn = fullfile(param.rss_base_dir,[rss_fn_name '.xml']); - fprintf('\nWriting %s\n', rss_fn); - if ~exist(param.rss_base_dir,'dir') - mkdir(param.rss_base_dir); + arena_fn_dir = fileparts(xml_param.arena.fn); + if ~exist(arena_fn_dir,'dir') + mkdir(arena_fn_dir); end - fid = fopen(rss_fn,'w'); + fprintf(' Writing Arena XML: %s\n', xml_param.arena.fn); + fid = fopen(xml_param.arena.fn,'w'); fwrite(fid,out_str,'char'); fclose(fid); + +% [doc,param] = write_arena_xml(doc,param); +% xml_doc = write_arena_xml([],'init',arena); +% xml_doc = write_arena_xml(xml_doc,'ctu_0013',arena); +% xml_doc = write_arena_xml(xml_doc,'dac-ad9129_0014',arena); +% xml_doc = write_arena_xml(xml_doc,'dac-ad9129_0014_waveform',arena); +% xml_doc = write_arena_xml(xml_doc,'psc_0001',arena); +% xml_doc = write_arena_xml(xml_doc,'subsystems',arena); +% +% out_str = xmlwrite(xml_doc); +% out_str = ['' out_str(find(out_str==10,1):end)]; +% [~,rss_fn_name] = ct_fileparts(out_xml_fn); +% rss_fn = fullfile(param.rss_base_dir,[rss_fn_name '.xml']); +% fprintf('\nWriting %s\n', rss_fn); +% if ~exist(param.rss_base_dir,'dir') +% mkdir(param.rss_base_dir); +% end +% fid = fopen(rss_fn,'w'); +% fwrite(fid,out_str,'char'); +% fclose(fid); end end diff --git a/cresis-toolbox/processing/burst_noise_bad_samples.m b/cresis-toolbox/processing/burst_noise_bad_samples.m new file mode 100644 index 00000000..66649b38 --- /dev/null +++ b/cresis-toolbox/processing/burst_noise_bad_samples.m @@ -0,0 +1,38 @@ +function bad_samples = burst_noise_bad_samples(data_signal,data_noise,test_metric,wfs) +% bad_samples = burst_noise_bad_samples(data_signal,data_noise,test_metric,wfs) +% +% Burst noise bad samples function example for UTIG 2023/01/20 flight +% +% To be used with analysis.m "burst_noise" command as in: +% params.analysis.cmd{1}.threshold_fh{1} = @burst_noise_bad_samples; + +bad_samples = false(size(test_metric)); +tt = test_metric > 25; + +%% ESTIMATE TIME DELAY AND PHASE FROM OVERSAMPLED PEAK +peak_idxs = find(tt); +if isempty(peak_idxs) + return; +end +% h_plot = plot(h_axes,NaN,NaN,'x','linewidth',4,'markersize',15); +idx = 1; +while ~isempty(idx) + start_idx = peak_idxs(idx); +% x = floor((start_idx-1)/size(tt,1))+1; +% y = start_idx-(x-1)*size(tt,1); +% set(h_plot,'xdata',x,'ydata',y); + zero_idxs = 0; + stop_idx = start_idx; + while zero_idxs < 14 && stop_idx < numel(tt) + stop_idx = stop_idx + 1; + if tt(stop_idx) == 0 + zero_idxs = zero_idxs + 1; + end + end + % REMOVE the signal (either by zero samples or subtraction) + % Zero out peak + bad_samples(start_idx:stop_idx) = true; +% set(h_image,'cdata',lp(ff)); + % Skip peak idxs associate with this peak + idx = find(peak_idxs > stop_idx,1); +end diff --git a/cresis-toolbox/processing/burst_noise_corr.m b/cresis-toolbox/processing/burst_noise_corr.m new file mode 100644 index 00000000..7db8d321 --- /dev/null +++ b/cresis-toolbox/processing/burst_noise_corr.m @@ -0,0 +1,24 @@ +function data_signal = burst_noise_corr(raw_data,wfs,burst_noise_sample_fn) +% data_signal = burst_noise_corr(raw_data,wfs,burst_noise_sample_fn) +% +% Burst noise correlation function example for UTIG 2023/01/20 flight +% +% To be used with analysis.m "burst_noise" command as in: +% params.analysis.cmd{1}.signal_fh{1} = @(raw_data,wfs) analysis_burst_corr(raw_data,wfs,'/scratch/metadata/2022_Antarctica_BaslerMKB/burst_noise_sample.mat'); +% +% Capture of the burst_noise_sample uses run_load_data.m +% +% aa = double(data{2}(2285:2300,1937,1)); +% % BASEBAND SIGNAL +% [B,A] = butter(2,11/25); +% Nt = length(aa); +% bb = filtfilt(B,A,exp(j*2*pi*-10/50*(0:Nt-1).') .* aa); +% % FFT AND CONJUGATE +% Nt = size(data{2},1); +% cc = conj(fft(bb,Nt)); +% burst_noise_sample = cc; +% save('/scratch/metadata/2022_Antarctica_BaslerMKB/burst_noise_sample.mat','burst_noise_sample','-v7.3') + +load(burst_noise_sample_fn,'burst_noise_sample'); +[B,A] = butter(2,11/25); +data_signal = lp(ifft(fft(filtfilt(B,A,double(raw_data) .* exp(1i*2*pi*-10/50*(0:size(raw_data,1)-1).'))) .* burst_noise_sample)); diff --git a/cresis-toolbox/processing/check_surface.m b/cresis-toolbox/processing/check_surface.m index 7c6bf786..d2485b1c 100644 --- a/cresis-toolbox/processing/check_surface.m +++ b/cresis-toolbox/processing/check_surface.m @@ -2,26 +2,37 @@ function check_surface(param,param_override) % check_surface(param,param_override) % % 1. Loads coincident LIDAR data if it exists +% % 2. Loads DTU sea surface DEM and arctic/antarctica land DEM, combines -% these two DEMS taking land DEM over sea surface DEM. +% these two DEMS taking land DEM over sea surface DEM. +% % 3. Combines LIDAR data and DEM data, taking LIDAR data over DEM data. -% Uses elevation to interpolate where data are not available. +% Uses elevation to interpolate where data are not available. +% % 4. Estimates Tadc_adjust or t_ref error by comparing radar surface from -% the specified layer source and the LIDAR/DEM combination. -% This error should be subtracted from param.radar.wfs.Tadc_adjust for pulsed systems. -% This error should be added to param.radar.wfs.t_ref for deramp systems. -% 5. Estimates GPS offset by comparing radar surface and LIDAR/DEM. This offset -% should be added to param.records.gps.time_offset. +% the specified layer source and the LIDAR/DEM combination. +% * The error is calculated as the correction that needs to be applied. +% In other words if the radar surface twtt is too large, then the error +% is reported as a negative number. +% * This error should be added to param.radar.wfs.Tadc_adjust for pulsed +% systems. +% * This error should be added to param.radar.wfs.t_ref for deramp +% systems. +% +% 5. Estimates GPS offset by comparing radar surface to LIDAR and/or DEM +% surface. This offset should be added to the current +% param.records.gps.time_offset in the parameter spreadsheet. See wiki: +% https://gitlab.com/openpolarradar/opr/-/wikis/System-Time-Delay#updating-gps-offset +% % 6. For deramp systems, uses the LIDAR/DEM data to determine the Nyquist -% zone and sets the records.settings.nyquist_zone based on this. -% The second decimal mask in frames.proc_mode is also set to one for -% frames that will be outside max_nyquist_zone. +% zone and sets the records.settings.nyquist_zone based on this. The second +% decimal mask in frames.proc_mode is also set to one for frames that will +% be outside max_nyquist_zone. % % See run_check_surface.m for how to run. % % cat /N/dcwan/projects/cresis/output/ct_tmp/check_surface/snow/2017_Greenland_P3/*.txt % -% % Author: John Paden % ===================================================================== @@ -149,12 +160,10 @@ function check_surface(param,param_override) % ========================================================================= % Load records file -records_fn = ct_filename_support(param,'','records'); -records = load(records_fn); +records = records_load(param); % Load frames file -frames_fn = ct_filename_support(param,'','frames'); -load(frames_fn); +frames = frames_load(param); % ========================================================================= %% Load in ocean mask, land DEM, and sea surface DEM @@ -169,9 +178,6 @@ function check_surface(param,param_override) gdem_str = sprintf('%s:%s:%s',param.radar_name,param.season_name,param.day_seg); if ~strcmpi(gdem_str,gdem.name) - % Load records file - records_fn = ct_filename_support(param,'','records'); - records = load(records_fn); gdem.set_vector(records.lat,records.lon,gdem_str); end @@ -186,17 +192,6 @@ function check_surface(param,param_override) % lidar) layers = opsLoadLayers(param,layer_params); -% Ensure that layer gps times are monotonically increasing -for lay_idx = 1:length(layers) - layers_fieldnames = fieldnames(layers(lay_idx)); - [~,unique_idxs] = unique(layers(lay_idx).gps_time); - for field_idx = 1:length(layers_fieldnames)-1 - if ~isempty(layers(lay_idx).(layers_fieldnames{field_idx})) - layers(lay_idx).(layers_fieldnames{field_idx}) = layers(lay_idx).(layers_fieldnames{field_idx})(unique_idxs); - end - end -end - % Throw out low quality radar data layers(radar_idx).twtt(layers(radar_idx).quality==3) = NaN; @@ -470,7 +465,7 @@ function check_surface(param,param_override) set(h_fig(4),'name','GPS'); h_axes(4) = axes('parent',h_fig(4)); plot(h_axes(4),-lags*dt,ref_corr) -xlabel(h_axes(4),'GPS lag (sec)'); +xlabel(h_axes(4),'Radar''s time lag relative to actual time (sec)'); ylabel(h_axes(4),'Cross correlation'); grid(h_axes(4),'on'); fig_fn = ct_filename_ct_tmp(param,'',param.(mfilename).debug_out_dir,'gps'); @@ -502,16 +497,16 @@ function check_surface(param,param_override) nz(nzmax(param.radar.nz_valid)) = max(param.radar.nz_valid); - if isfield(records.settings,'nyquist_zone') - original_nz = records.settings.nyquist_zone; + if isfield(records,'nyquist_zone_sig') + original_nz = records.nyquist_zone_sig; else original_nz = nan(size(records.gps_time)); end - records.settings.nyquist_zone = interp1(layers(radar_idx).gps_time,nz,records.gps_time,'nearest','extrap'); + records.nyquist_zone_sig = interp1(layers(radar_idx).gps_time,nz,records.gps_time,'nearest','extrap'); if param.check_surface.save_records_en - save(records_fn,'-append','-struct','records','settings'); - records_aux_files_create(records_fn,false); + records_fn = ct_filename_support(param,'','records'); + ct_save(records_fn,'-append','-struct','records','nyquist_zone_sig'); end clf(h_fig(5)); @@ -540,7 +535,7 @@ function check_surface(param,param_override) % Find the new t_ref value BW = diff(param.radar.wfs(wf).BW_window); dt = 1/BW; - t_ref_new = param.radar.wfs(wf).t_ref - param.check_surface.radar_twtt_offset - round(nanmedian(twtt_error)/dt)*dt; + t_ref_new = param.radar.wfs(wf).t_ref + param.check_surface.radar_twtt_offset + round(nanmedian(twtt_error)/dt)*dt; else % Find the new Tadc_adjust (called t_ref_new to match deramp) value t_ref_new = param.radar.wfs(wf).Tadc_adjust + param.check_surface.radar_twtt_offset + round(nanmedian(twtt_error)*1e10)/1e10; @@ -558,7 +553,7 @@ function check_surface(param,param_override) 'Std error', ... 'Max error', ... 'Mean error all', ... - 'Median error all', '#records', 'GPS lag', 'Default NZ', 't_ref', 'DEM'); + 'Median error all', '#records', 'GPS lag', 'Default NZ', 't_ref_or_Tadc_adjust', 'DEM'); fclose(fid); txt_fn = [ct_filename_ct_tmp(param,'',param.(mfilename).debug_out_dir,'time') '.txt']; @@ -573,7 +568,7 @@ function check_surface(param,param_override) 1e9*nanstd(twtt_error), ... 1e9*nanmax(abs(twtt_error-mean_offset)), ... 1e9*nanmean(twtt_error_all), ... - 1e9*nanmedian(twtt_error_all), numel(recs), -lags(peak_idx)*dt, default_nz, 1e9*t_ref_new, dem_source); + 1e9*nanmedian(twtt_error_all), numel(recs), lags(peak_idx)*dt, default_nz, 1e9*t_ref_new, dem_source); fclose(fid); fprintf(1,'%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n', ... @@ -582,14 +577,14 @@ function check_surface(param,param_override) 'Std error', ... 'Max error', ... 'Mean error all', ... - 'Median error all', '#records', 'GPS lag', 'Default NZ', 't_ref', 'DEM'); + 'Median error all', '#records', 'GPS lag', 'Default NZ', 't_ref_or_Tadc_adjust', 'DEM'); fprintf(1,'%s\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%d\t%.1f\t%.0f\t%.12g\t%s\n', ... param.day_seg, 1e9*mean_offset, ... 1e9*nanmedian(twtt_error), ... 1e9*nanstd(twtt_error), ... 1e9*nanmax(abs(twtt_error-mean_offset)), ... 1e9*nanmean(twtt_error_all), ... - 1e9*nanmedian(twtt_error_all), numel(recs), -lags(peak_idx)*dt, default_nz, 1e9*t_ref_new, dem_source); + 1e9*nanmedian(twtt_error_all), numel(recs), lags(peak_idx)*dt, default_nz, 1e9*t_ref_new, dem_source); fprintf('All twtt times are in ns\n'); % ===================================================================== @@ -604,7 +599,7 @@ function check_surface(param,param_override) fprintf(' %d specularity records\n', length(spec_gps_time)); records = load(ct_filename_support(param,'','records')); along_track = geodetic_to_along_track(records.lat,records.lon); - load(ct_filename_support(param,'','frames')); + frames = frames_load(param); record = []; spec_frm = []; diff --git a/cresis-toolbox/processing/collate_burst_noise.m b/cresis-toolbox/processing/collate_burst_noise.m new file mode 100644 index 00000000..49da3e08 --- /dev/null +++ b/cresis-toolbox/processing/collate_burst_noise.m @@ -0,0 +1,370 @@ +% function collate_burst_noise(param,param_override) +% collate_burst_noise(param,param_override) +% +% Collects analysis.m results from burst noise tracking (burst_noise +% command) and creates files for removing the burst noise during data +% loading. +% Loads all the burst_noise_* files and creates burst_noise_simp_* files that +% are pre-filtered for speed and saves as netcdf so that subsets of the +% files can be loaded efficiently. +% +% Example: +% See run_collate_burst_noise for how to run. +% +% Authors: John Paden + +%% General Setup +% ===================================================================== +param = merge_structs(param, param_override); + +fprintf('=====================================================================\n'); +fprintf('%s: %s (%s)\n', mfilename, param.day_seg, datestr(now)); +fprintf('=====================================================================\n'); + +%% Input checks +% ===================================================================== + +if ~isfield(param,'collate_burst_noise') || isempty(param.collate_burst_noise) + param.collate_burst_noise = []; +end + +% param.collate_burst_noise.imgs: Check this first since other input checks +% are dependent on its value. +if ~isfield(param.analysis,'imgs') || isempty(param.analysis.imgs) + param.analysis.imgs = {[1 1]}; +end +if ~isfield(param.collate_burst_noise,'imgs') || isempty(param.collate_burst_noise.imgs) + param.collate_burst_noise.imgs = 1:length(param.analysis.imgs); +end + +if ~isfield(param.collate_burst_noise,'cmd_idx') || isempty(param.collate_burst_noise.cmd_idx) + param.collate_burst_noise.cmd_idx = 1; +end + +if ~isfield(param.collate_burst_noise,'bit_mask') || isempty(param.collate_burst_noise.bit_mask) + % These are the records.bit_mask bits that will be set by collate_burst_noise + param.collate_burst_noise.bit_mask = 4; +end + +if ~isfield(param.collate_burst_noise,'bit_mask_clear') || isempty(param.collate_burst_noise.bit_mask_clear) + % These are the records.bit_mask bits that will be cleared/reset by collate_burst_noise + param.collate_burst_noise.bit_mask_clear = 12; +end + +if ~isfield(param.collate_burst_noise,'debug_plots') + param.collate_burst_noise.debug_plots = {'bn_plot'}; +end +enable_visible_plot = any(strcmp('visible',param.collate_burst_noise.debug_plots)); +enable_bn_plot = any(strcmp('bn_plot',param.collate_burst_noise.debug_plots)); +if ~isempty(param.collate_burst_noise.debug_plots) + h_fig = get_figures(3,enable_visible_plot); +end + +if ~isfield(param.collate_burst_noise,'debug_max_plot_size') || isempty(param.collate_burst_noise.debug_max_plot_size) + param.collate_burst_noise.debug_max_plot_size = 50e6; +end + +if ~isfield(param.(mfilename),'debug_out_dir') || isempty(param.(mfilename).debug_out_dir) + param.(mfilename).debug_out_dir = mfilename; +end +debug_out_dir = param.(mfilename).debug_out_dir; + +if ~isfield(param.collate_burst_noise,'filt_length') || isempty(param.collate_burst_noise.filt_length) + param.collate_burst_noise.filt_length = 1; +end + +if ~isfield(param.collate_burst_noise,'filt_threshold') || isempty(param.collate_burst_noise.filt_threshold) + param.collate_burst_noise.filt_threshold = 0.5; +end + +if ~isfield(param.collate_burst_noise,'in_path') || isempty(param.collate_burst_noise.in_path) + param.collate_burst_noise.in_path = 'analysis'; +end + +if ~isfield(param.collate_burst_noise,'max_bad_waveforms') || isempty(param.collate_burst_noise.max_bad_waveforms) + param.collate_burst_noise.max_bad_waveforms = 1; +end + +if ~isfield(param.collate_burst_noise,'out_path') || isempty(param.collate_burst_noise.out_path) + param.collate_burst_noise.out_path = param.collate_burst_noise.in_path; +end + +if ~isfield(param.collate_burst_noise,'test_wf_adcs') || isempty(param.collate_burst_noise.test_wf_adcs) + param.collate_burst_noise.test_wf_adcs = []; +end + +if ~isfield(param.collate_burst_noise,'threshold_fh') || isempty(param.collate_burst_noise.threshold_fh) + for img = 1:max(param.collate_burst_noise.imgs) + param.collate_burst_noise.threshold_fh{img} = []; + end +end + +if ~isfield(param.collate_burst_noise,'wf_adcs') || isempty(param.collate_burst_noise.wf_adcs) + param.collate_burst_noise.wf_adcs = {}; +end +if ~isempty(param.collate_burst_noise.wf_adcs) && ~iscell(param.collate_burst_noise.wf_adcs) + wf_adcs = param.collate_burst_noise.wf_adcs; + param.collate_burst_noise.wf_adcs = {}; + for img = 1:max(param.collate_burst_noise.imgs) + param.collate_burst_noise.wf_adcs{img} = wf_adcs; + end +end + +records = records_load(param); +new_bit_mask = zeros(size(records.bit_mask)); +[~,frm_id,~] = get_frame_id(param,records.gps_time); + +for img = param.collate_burst_noise.imgs + + if isempty(param.collate_burst_noise.wf_adcs) + wf_adcs = 1:size(param.analysis.imgs{img},1); + else + wf_adcs = param.collate_burst_noise.wf_adcs{img}; + end + for wf_adc = wf_adcs + wf = param.analysis.imgs{img}(wf_adc,1); + adc = param.analysis.imgs{img}(wf_adc,2); + fprintf('Processing wf %d adc %d\t%s\n', wf, adc, datestr(now,'yyyymmdd_HHMMSS')); + + % Determine the boards for this wf-adc pair, find the time axis + param.load.imgs = {[wf adc]}; + [wfs,states] = data_load_wfs(param, records); + + %% Load the burst noise file + % =================================================================== + % For the detection of burst noise in a particular wf-adc channel, any + % number of wf-adc channels can be loaded, but the default is to just + % load the wf-adc channel in question + if isempty(param.collate_burst_noise.test_wf_adcs) ... + || length(param.collate_burst_noise.test_wf_adcs) < img ... + || isempty(param.collate_burst_noise.test_wf_adcs{img}) ... + || length(param.collate_burst_noise.test_wf_adcs{img}) < wf_adc ... + || isempty(param.collate_burst_noise.test_wf_adcs{img}{wf_adc}) + param.collate_burst_noise.test_wf_adcs{img}{wf_adc} = [wf adc]; + end + % Load each of the wf-adc channels that are to be loaded to detect + % noise in the current wf-adc channel. + noise = cell(size(param.collate_burst_noise.test_wf_adcs{img}{wf_adc},1),1); + for tmp_wf_adc = 1:size(param.collate_burst_noise.test_wf_adcs{img}{wf_adc},1) + tmp_wf = param.collate_burst_noise.test_wf_adcs{img}{wf_adc}(tmp_wf_adc,1); + tmp_adc = param.collate_burst_noise.test_wf_adcs{img}{wf_adc}(tmp_wf_adc,2); + fn_dir = fileparts(ct_filename_out(param,param.collate_burst_noise.in_path)); + fn = fullfile(fn_dir,sprintf('burst_noise_%s_wf_%d_adc_%d.mat', param.day_seg, tmp_wf, tmp_adc)); + fprintf('Loading %s (%s)\n', fn, datestr(now)); + noise{tmp_wf_adc} = load(fn); + end + cmd = noise{1}.param_analysis.analysis.cmd{param.collate_burst_noise.cmd_idx}; + + % Optionally update bad_recs field + if ~isempty(param.collate_burst_noise.threshold_fh{img}) + bad_recs = find(param.collate_burst_noise.threshold_fh{img}{wf_adc}(noise,wfs)); + else + bad_recs = noise{1}.bad_recs; + end + + % noise.bad_recs: a list of all the bad records for each burst noise + % event, there will be multiple entries for a record if there are + % multiple burst noise events in that record + bad_recs_unique = unique(bad_recs); + + % Optional along-track filtering to handle missed detections and false + % alarms when there are groups of burst noise + if param.collate_burst_noise.filt_length > 1 + % Create a mask of all the bad records + bad_recs_unique_filt = zeros(1,size(records.bit_mask,2)); + bad_recs_unique_filt(bad_recs_unique) = 1; + % Filter the mask with a boxcar filter and then threshold the output + bad_recs_unique_filt = fir_dec(bad_recs_unique_filt,ones(1,param.collate_burst_noise.filt_length),1) ... + > param.collate_burst_noise.filt_length*param.collate_burst_noise.filt_threshold; + % Store the new filtered result back into bad_recs_unique + bad_recs_unique = find(bad_recs_unique_filt); + end + + % Force all wf-adc pairs to be considered bad + if bitand(param.collate_burst_noise.bit_mask,8) + + % Add the specific section to be blanked out + % ------------------------------------------------------------------- + % burst_noise_table is a 3xNb table where Nb is the number of burst + % noise detections. Each column corresponds to one burst. + % + % burst_noise_table(1,:): The record that the burst occurs in. + % + % burst_noise_table(2,:): The start time of the burst + % + % burst_noise_table(3,:): The stop of the burst + if 0 + burst_noise_table = zeros(3,length(bad_recs_unique)); + else + burst_noise_table = zeros(3,0); + end + for col = 1:length(bad_recs_unique) + fasttime0 = -inf; + fasttime1 = inf; + if ~isempty(noise{1}.bad_bins) + mask = noise{1}.bad_recs == bad_recs_unique(col); + if 0 + % Combine all burst detections into one and mask out everything + % from the first detection bin to the last detection bin. + fasttime0 = wfs(wf).time_raw(min(noise{1}.bad_bins(mask))); + fasttime1 = wfs(wf).time_raw(max(noise{1}.bad_bins(mask))); + burst_noise_table(:,col) = [bad_recs_unique(col) fasttime0 fasttime1].'; + else + % Parse detections into individual events + bad_bins = noise{1}.bad_bins(mask); + while ~isempty(bad_bins) + cur_bad_bin = bad_bins(1); + fasttime0 = wfs(wf).time_raw(cur_bad_bin); + bad_bins = bad_bins(2:end); + while ~isempty(bad_bins) && bad_bins(1) == cur_bad_bin + 1 + cur_bad_bin = bad_bins(1); + bad_bins = bad_bins(2:end); + end + fasttime1 = wfs(wf).time_raw(cur_bad_bin); + burst_noise_table(:,end+1) = [bad_recs_unique(col) fasttime0 fasttime1].'; + end + end + end + end + + else + % Sub-rangeline/bin specific burst noise not being used so + % burst_noise_table is empty + burst_noise_table = []; + end + + % Combine all boards: Some wf-adc pairs result in multiple waveforms + % being loaded (e.g. for IQ on transmit or separate IQ channels or + % zero-pi mod sequences that are stored separately) + boards = cell2mat({states.board_idx}); + + % Set bits (usually bit 2 (value 4) or bit 3 (value 8)) to true for the bad records + new_bit_mask(boards,bad_recs_unique) = bitor(param.collate_burst_noise.bit_mask,new_bit_mask(boards,bad_recs_unique)); + + %% Plot + % ===================================================================== + if enable_bn_plot + %% Plot: Figure 1 + clf(h_fig(1)); + set(h_fig(1), 'name', 'burst_noise rec-bin'); + h_axes(1) = axes('parent',h_fig(1)); + % Unfiltered bad samples/records + if isempty(noise{1}.bad_bins) + plot(noise{1}.bad_recs, ones(size(noise{1}.bad_recs)), 'bx', 'parent', h_axes(1)); + else + plot(noise{1}.bad_recs, noise{1}.bad_bins, 'bx', 'parent', h_axes(1)); + end + % Filtered bad records + hold(h_axes(1),'on'); + bad_recs_mask = zeros(size(records.gps_time)); + bad_recs_mask(bad_recs_unique) = 1; + plot(bad_recs_mask, 'r-', 'parent', h_axes(1)); + title(h_axes(1), sprintf('%s wf %d adc %d',regexprep(param.day_seg,'_','\\_'), wf, adc)); + xlabel(h_axes(1), 'Record'); + ylabel(h_axes(1), 'Range bin'); + + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('burst_rec_bin_wf_%02d_adc_%02d',wf,adc)) '.jpg']; + fprintf('Saving %s %s\n', datestr(now), fig_fn); + fig_fn_dir = fileparts(fig_fn); + if ~exist(fig_fn_dir,'dir') + mkdir(fig_fn_dir); + end + ct_saveas(h_fig(1),fig_fn); + if 2*ct_whos(noise{1}.bad_bins) < param.collate_burst_noise.debug_max_plot_size + fig_fn(end-2:end) = 'fig'; + fprintf('Saving %s %s\n', datestr(now), fig_fn); + ct_saveas(h_fig(1),fig_fn); + end + + %% Plot: Figure 2 + clf(h_fig(2)); + set(h_fig(2), 'name', 'burst_noise waveforms'); + h_axes(2) = axes('parent',h_fig(2)); + for block_idx = 1:length(noise{1}.bad_waveforms) + num_bad_waveforms = size(noise{1}.bad_waveforms{block_idx},2); + plot(lp(noise{1}.bad_waveforms{block_idx}(:,1:min(param.collate_burst_noise.max_bad_waveforms,num_bad_waveforms))), 'parent', h_axes(2)); + hold(h_axes(2),'on'); + end + title(h_axes(2), sprintf('%s wf %d adc %d',regexprep(param.day_seg,'_','\\_'), wf, adc)); + xlabel(h_axes(2), 'Range bin'); + ylabel(h_axes(2), 'Relative power (dB)'); + + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('burst_waveform_wf_%02d_adc_%02d',wf,adc)) '.jpg']; + fprintf('Saving %s %s\n', datestr(now), fig_fn); + fig_fn_dir = fileparts(fig_fn); + if ~exist(fig_fn_dir,'dir') + mkdir(fig_fn_dir); + end + ct_saveas(h_fig(2),fig_fn); + if ct_whos(noise{1}.bad_waveforms) < param.collate_burst_noise.debug_max_plot_size + fig_fn(end-2:end) = 'fig'; + fprintf('Saving %s %s\n', datestr(now), fig_fn); + ct_saveas(h_fig(2),fig_fn); + end + + %% Plot: Figure 3 and 4 + if ~isempty(noise{1}.test_metric) + clf(h_fig(3)); + set(h_fig(3), 'name', 'burst_noise test_metric'); + h_axes(3) = subplot(2,1,1,'parent',h_fig(3)); + plot(noise{1}.test_metric, '.-', 'parent', h_axes(3)); + grid(h_axes(3),'on'); + title(h_axes(3), sprintf('%s wf %d adc %d',regexprep(param.day_seg,'_','\\_'), wf, adc)); + xlabel(h_axes(3), 'Record'); + ylabel(h_axes(3), 'test_metric output', 'interpreter','none'); + h_axes(4) = subplot(2,1,2,'parent',h_fig(3)); + if length(noise{1}.bad_recs) == length(noise{1}.test_metric) + plot(frm_id(noise{1}.bad_recs), noise{1}.test_metric, '.-', 'parent', h_axes(4)); + else + plot(frm_id, noise{1}.test_metric, '.-', 'parent', h_axes(4)); + end + grid(h_axes(4),'on'); + xlabel(h_axes(4), 'Frame'); + ylabel(h_axes(4), 'test_metric output', 'interpreter','none'); + linkaxes(h_axes([1 3]),'x'); + axis(h_axes([3 4]),'tight'); + + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('burst_test_metric_wf_%02d_adc_%02d',wf,adc)) '.jpg']; + fprintf('Saving %s %s\n', datestr(now), fig_fn); + fig_fn_dir = fileparts(fig_fn); + if ~exist(fig_fn_dir,'dir') + mkdir(fig_fn_dir); + end + ct_saveas(h_fig(3),fig_fn); + fig_fn(end-2:end) = 'fig'; + fprintf('Saving %s %s\n', datestr(now), fig_fn); + ct_saveas(h_fig(3),fig_fn); + end + end + + if enable_visible_plot && ~isempty(bad_recs_unique) + % Bring plots to front + for h_fig_idx = 1:length(h_fig) + figure(h_fig(h_fig_idx)); + end + % Enter debug mode + keyboard + end + + if ~isempty(burst_noise_table) + %% Save the result + % ===================================================================== + out_fn_dir = fileparts(ct_filename_out(param,param.collate_burst_noise.out_path, '')); + out_fn = fullfile(out_fn_dir,sprintf('burst_%s_wf_%d_adc_%d.mat', param.day_seg, wf, adc)); + fprintf('Saving %s (%s)\n', out_fn, datestr(now)); + ct_save(out_fn,'burst_noise_table'); + end + end +end + +%% Update records file +records.bit_mask = records.bit_mask - bitand(param.collate_burst_noise.bit_mask_clear,records.bit_mask) + uint8(new_bit_mask); +records_fn = ct_filename_support(param,'','records'); +fprintf('Saving %s %s\n', datestr(now), records_fn); +ct_save(records_fn,'-append','-struct','records','bit_mask'); + +if ~enable_visible_plot + try + delete(h_fig); + end +end diff --git a/cresis-toolbox/processing/collate_coh_noise.m b/cresis-toolbox/processing/collate_coh_noise.m index 19c32115..94bf3a77 100644 --- a/cresis-toolbox/processing/collate_coh_noise.m +++ b/cresis-toolbox/processing/collate_coh_noise.m @@ -36,11 +36,41 @@ if ~isfield(param.collate_coh_noise,'imgs') || isempty(param.collate_coh_noise.imgs) param.collate_coh_noise.imgs = 1:length(param.analysis.imgs); end +if ~isnumeric(param.collate_coh_noise.imgs) + error('param.collate_coh_noise.imgs must be a numeric integer array with indices into param.analysis.imgs. Default is 1:length(param.analysis.imgs).'); +end +% cmd_idx: which command index to pull values from param.analysis.cmd{cmd_idx} if ~isfield(param.collate_coh_noise,'cmd_idx') || isempty(param.collate_coh_noise.cmd_idx) param.collate_coh_noise.cmd_idx = 1; end +% debug_max_size: maximum size allowed for .fig files to be saved in +% cn_plot mode. .jpg files will always be saved. +if ~isfield(param.(mfilename),'debug_max_size') || isempty(param.(mfilename).debug_max_size) + param.(mfilename).debug_max_size = 0; +end + +% debug_out_dir: the output directory for the debug plots and text files, +% default is to place in ct_tmp under the name of the present m-file +if ~isfield(param.(mfilename),'debug_out_dir') || isempty(param.(mfilename).debug_out_dir) + param.(mfilename).debug_out_dir = mfilename; +end +debug_out_dir = param.(mfilename).debug_out_dir; + +% debug_plots: cell array of strings that enable functionality, the +% possible strings are {'cn_plot','reuse','threshold_plot','visible'}. The +% default setting is {'cn_plot','threshold_plot'}. +% 'threshold_plot': enable plotting of threshold plots (should always be +% enabled if threshold might be used). +% 'visible': displays figures and enters debug mode to allow reviewing for +% each wf-adc pair +% 'cn_plot': enable plotting of coherent noise plots (should always be +% enabled). +% 'reuse': causes the reuse file to be loaded if it exists rather than +% rebuilding the coherent noise. Typical use would be to create all the +% collate_coh_noise files without visible enabled and then enable both +% visible and reuse to review all the files. if ~isfield(param.collate_coh_noise,'debug_plots') || isempty(param.collate_coh_noise.debug_plots) param.collate_coh_noise.debug_plots = {'cn_plot','threshold_plot'}; end @@ -48,20 +78,41 @@ enable_threshold_plot = any(strcmp('threshold_plot',param.collate_coh_noise.debug_plots)); enable_cn_plot = any(strcmp('cn_plot',param.collate_coh_noise.debug_plots)); enable_reuse_files = any(strcmp('reuse',param.collate_coh_noise.debug_plots)); +enable_debug_plot = any(strcmp('debug',param.collate_coh_noise.debug_plots)); if ~isempty(param.collate_coh_noise.debug_plots) h_fig = get_figures(5,enable_visible_plot); end -if ~isfield(param.(mfilename),'debug_out_dir') || isempty(param.(mfilename).debug_out_dir) - param.(mfilename).debug_out_dir = mfilename; +if isfield(param.collate_coh_noise,'dft_corr_length') + error('Change field name param.collate_coh_noise.dft_corr_length to dft_corr_time and specify an entry for each image to be processed.'); end -debug_out_dir = param.(mfilename).debug_out_dir; -if isfield(param.collate_coh_noise,'dft_corr_length') - error('Change field name param.collate_coh_noise.dft_corr_length to dft_corr_time.'); +if ~isfield(param.collate_coh_noise,'dft_corr_time') + param.collate_coh_noise.dft_corr_time = {}; +end +if isnumeric(param.collate_coh_noise.dft_corr_time) + param.collate_coh_noise.dft_corr_time = {param.collate_coh_noise.dft_corr_time}; +end +for img = param.collate_coh_noise.imgs + if length(param.collate_coh_noise.dft_corr_time) < img + param.collate_coh_noise.dft_corr_time{img} = inf; + end +end + +% distance_weight: default is to match the setting during coh_noise +% analysis or false if the setting is absent; if true, then the coherent +% averaging is weighted by the distance travelled. This is primarily useful +% for ground based traverses where long stops can bias the coherent noise +% estimate. Setting this to true will reduce the affect the long stops have +% on the estimated coherent noise mean. +% NOTE: Only the dft method uses this field. +if ~isfield(param.collate_coh_noise,'distance_weight') || isempty(param.collate_coh_noise.distance_weight) + param.collate_coh_noise.distance_weight = {}; end -if ~isfield(param.collate_coh_noise,'dft_corr_time') || isempty(param.collate_coh_noise.dft_corr_time) - param.collate_coh_noise.dft_corr_time = inf; +for img = param.collate_coh_noise.imgs + if length(param.collate_coh_noise.distance_weight) < img + param.collate_coh_noise.distance_weight{img} = []; + end end if ~isfield(param.collate_coh_noise,'firdec_fs') || isempty(param.collate_coh_noise.firdec_fs) @@ -80,14 +131,29 @@ param.collate_coh_noise.in_path = 'analysis'; end +if ~isfield(param.collate_coh_noise,'min_samples') || isempty(param.collate_coh_noise.min_samples) + param.collate_coh_noise.min_samples = 0.5; +end + if ~isfield(param.collate_coh_noise,'method') || isempty(param.collate_coh_noise.method) for img = param.collate_coh_noise.imgs param.collate_coh_noise.method{img} = 'dft'; end end +if ~iscell(param.collate_coh_noise.method) + error('param.collate_coh_noise.method is not a cell array. It must be a cell array of strings containing the method for each image to be processed.'); +end +if isempty(param.collate_coh_noise.method) + error('In param.collate_coh_noise.method cell array of strings, a collate coh noise method must be specified for each image. For example {''dft'',''dft''} for images 1 and 2.'); +end +for img = param.collate_coh_noise.imgs + if length(param.collate_coh_noise.method) < img + param.collate_coh_noise.method{img} = param.collate_coh_noise.method{1}; + end +end if ~isfield(param.collate_coh_noise,'out_path') || isempty(param.collate_coh_noise.out_path) - param.collate_coh_noise.out_path = 'analysis'; + param.collate_coh_noise.out_path = param.collate_coh_noise.in_path; end if ~isfield(param.collate_coh_noise,'threshold_en') || isempty(param.collate_coh_noise.threshold_en) @@ -116,19 +182,25 @@ param.collate_coh_noise.threshold_ylims = []; end +% wf_adcs: list of wf-adc pairs to use from the wf-adc pair lists in +% param.analysis.imgs. This is a cell array where each cell corresponds to +% the image with the same index so that different wf-adc pairs can be +% pulled from each image. If left empty, then all wf-adc pairs are +% processed for all images. if ~isfield(param.collate_coh_noise,'wf_adcs') || isempty(param.collate_coh_noise.wf_adcs) param.collate_coh_noise.wf_adcs = []; end if ~isempty(param.collate_coh_noise.wf_adcs) && ~iscell(param.collate_coh_noise.wf_adcs) wf_adcs = param.collate_coh_noise.wf_adcs; - for img = 1:length(param.collate_coh_noise.imgs) + param.collate_coh_noise.wf_adcs = {}; + for img = 1:max(param.collate_coh_noise.imgs) param.collate_coh_noise.wf_adcs{img} = wf_adcs; end end for img = param.collate_coh_noise.imgs - if isempty(param.collate_coh_noise.wf_adcs) + if isempty(param.collate_coh_noise.wf_adcs) || length(param.collate_coh_noise.wf_adcs) < img wf_adcs = 1:size(param.analysis.imgs{img},1); else wf_adcs = param.collate_coh_noise.wf_adcs{img}; @@ -146,7 +218,7 @@ fprintf('Loading %s\n', reuse_fn); load(reuse_fn, 'param_collate_coh_noise'); if strcmpi( param_collate_coh_noise.method{img}, param.collate_coh_noise.method{img} ) ... - && param_collate_coh_noise.dft_corr_time == param.collate_coh_noise.dft_corr_time ... + && param_collate_coh_noise.dft_corr_time(img) == param.collate_coh_noise.dft_corr_time(img) ... && param_collate_coh_noise.firdec_fs{img} == param.collate_coh_noise.firdec_fs{img} ... && strcmpi( func2str(param_collate_coh_noise.firdec_fcutoff{img}), func2str(param.collate_coh_noise.firdec_fcutoff{img}) )... && param_collate_coh_noise.threshold_en == param.collate_coh_noise.threshold_en ... @@ -157,6 +229,16 @@ noise.fc = fc; noise.param_records = param_records; noise.param_analysis = param_analysis; + Nt = size(dft_noise,1); + time = start_bin*noise.dt + noise.dt*(0:Nt-1).'; + Tpd = param.radar.wfs(wf).Tpd; + if enable_threshold + coh_noise_est = sort(max_filt1(cn_before.',15),2); + coh_noise_est = lp(coh_noise_est(:,round(0.05*end)),2); + if Nt > 101 + coh_noise_est = sgolayfilt(coh_noise_est,3,101); + end + end reuse_success = 1; else reuse_success = 0; @@ -176,9 +258,7 @@ noise = load(fn); cmd = noise.param_analysis.analysis.cmd{param.collate_coh_noise.cmd_idx}; - if ~isfield(cmd,'min_samples') || isempty(cmd.min_samples) - cmd.min_samples = 0.5*cmd.block_ave; - end + min_samples = param.collate_coh_noise.min_samples*cmd.block_ave; % Each processed block has a constant start and stop bin, but between % blocks the start/stop range bin may change. @@ -202,7 +282,7 @@ Nx = length(noise.gps_time); recs = noise.param_analysis.analysis.block_size/2 + noise.param_analysis.analysis.block_size * (0:Nx-1); - Nx_dft = round(Nx / param.collate_coh_noise.dft_corr_time); + Nx_dft = round(Nx / param.collate_coh_noise.dft_corr_time{img}); if Nx_dft<1 Nx_dft = 1; end @@ -211,7 +291,11 @@ dft_freqs = dft_freqs(dft_freqs_idxs); dft_noise = zeros(Nt, Nx_dft, 'single'); + % Average time step in slow time/GPS time. If there is only one + % record, then diff(noise.gps_time) returns an empty matrix and + % median returns NaN. This condition is captured below. dgps_time = median(diff(noise.gps_time)); + dx = max(1,round(1/(dgps_time * param.collate_coh_noise.firdec_fs{img}))); dec_idxs = 1:dx:Nx; firdec_gps_time = noise.gps_time(dec_idxs); @@ -244,13 +328,13 @@ if size(noise.coh_ave_samples{block_idx},1) == 0 % This block was all bad data so it has fast time dimension % of zero. We handle this case separately. - coh_bin(block_start(block_idx)+(0:block_size(block_idx)-1)) = tmp; + coh_bin(block_start(block_idx)+(0:block_size(block_idx)-1)) = NaN; if enable_threshold coh_bin_mag(block_start(block_idx)+(0:block_size(block_idx)-1)) = NaN; end else tmp = noise.coh_ave{block_idx}(bin-start_bins(block_idx)+1,:); - tmp(noise.coh_ave_samples{block_idx}(bin-start_bins(block_idx)+1,:) < cmd.min_samples) = NaN; + tmp(noise.coh_ave_samples{block_idx}(bin-start_bins(block_idx)+1,:) < min_samples) = NaN; coh_bin(block_start(block_idx)+(0:block_size(block_idx)-1)) = tmp; if enable_threshold coh_bin_mag(block_start(block_idx)+(0:block_size(block_idx)-1)) = noise.coh_ave_mag{block_idx}(bin-start_bins(block_idx)+1,:); @@ -274,15 +358,28 @@ if enable_threshold cn_before_mag(:,bin_idx) = coh_bin_mag; if size(coh_bin_mag,2) < param.collate_coh_noise.threshold_fir_dec - threshold(bin_idx) = lp(mean(abs(coh_bin_mag).^2,2),1); + threshold(bin_idx) = lp(nanmean(abs(coh_bin_mag).^2,2),1); else threshold(bin_idx) = min(lp(fir_dec(abs(coh_bin_mag).^2,param.collate_coh_noise.threshold_fir_dec),1),[],2); end end + if isempty(param.collate_coh_noise.distance_weight{img}) + if isfield(cmd,'distance_weight') + param.collate_coh_noise.distance_weight{img} = cmd.distance_weight; + else + param.collate_coh_noise.distance_weight{img} = false; + end + end if strcmpi(param.collate_coh_noise.method{img},'dft') for dft_idx = 1:length(dft_freqs) mf = exp(1i*2*pi/Nx * dft_freqs(dft_idx) .* (0:Nx-1)); - dft_noise(bin_idx,dft_idx) = nanmean(conj(mf).*coh_bin); + if ~param.collate_coh_noise.distance_weight{img} || sum(noise.along_track) < 10 + dft_noise(bin_idx,dft_idx) = nanmean(conj(mf).*coh_bin); + else + distance_weight = noise.along_track./sum(noise.along_track); + distance_weight = distance_weight ./ sum(distance_weight); + dft_noise(bin_idx,dft_idx) = nansum(bsxfun(@times, conj(mf).*coh_bin, distance_weight)); + end coh_bin = coh_bin - dft_noise(bin_idx,dft_idx) * mf; end elseif strcmpi(param.collate_coh_noise.method{img},'firdec') @@ -296,12 +393,15 @@ firdec_noise(bin_idx,:) = 0; else % Positive cutoff frequency: Regular FIR filter + if ~isfinite(dgps_time) + error('dgps_time is not finite, use dft method instead. This is probably because length(noise.gps_time) < 2 because the segment is too short. length(noise.gps_time) = %d', length(noise.gps_time)); + end B = tukeywin(round(1/(fcutoff*dgps_time)/2)*2+1,0.5).'; B = B / sum(B); - firdec_noise(bin_idx,:) = nan_fir_dec(coh_bin,B,dx); + firdec_noise(bin_idx,:) = interp_finite(nan_fir_dec(coh_bin,B,dx),0); end %firdec_noise(bin_idx,isnan(firdec_noise(bin_idx,:))) = 0; - noise_est = interp_finite(interp1(firdec_gps_time,firdec_noise(bin_idx,:),noise.gps_time),0); + noise_est = interp1(firdec_gps_time,firdec_noise(bin_idx,:),noise.gps_time); coh_bin = coh_bin - noise_est; end if enable_cn_plot @@ -319,19 +419,42 @@ time = start_bin*noise.dt + noise.dt*(0:Nt-1).'; Tpd = param.radar.wfs(wf).Tpd; if enable_threshold + coh_noise_est = sort(max_filt1(cn_before.',15),2); + coh_noise_est = lp(coh_noise_est(:,round(0.05*(end-1))+1),2); + if Nt > 101 + coh_noise_est = sgolayfilt(coh_noise_est,3,101); + end + end + if enable_threshold + % For debugging and plotting, save the original threshold value + % When debugging, run "threshold = orig_threshold;" to test + % different threshold values. orig_threshold = threshold; if ~isempty(param.collate_coh_noise.threshold_eval) if numel(param.collate_coh_noise.threshold_eval) < img error('If param.collate_coh_noise.threshold_eval is specified, there must be a cell entry for each image. Image %d cannot be found since numel(param.collate_coh_noise.threshold_eval)=%d',img,numel(param.collate_coh_noise.threshold_eval)); end + cmd_str = param.collate_coh_noise.threshold_eval; + if iscell(cmd_str) + % Not a string, so threshold_eval is specified on a per image basis + cmd_str = cmd_str{img}; + if iscell(cmd_str) + % Not a string, so threshold_eval is specified on a per wf-adc pair basis + cmd_str = cmd_str{wf_adc}; + end + end %figure(100); plot(threshold); hold on; % Examples: % param.collate_coh_noise.threshold_eval{img} = 'threshold(time>Tpd+0.85e-6 & threshold>-110) = -100; threshold(time<=Tpd+0.85e-6) = inf;' % param.collate_coh_noise.threshold_eval{img} = 'threshold(time>Tpd+2.3e-6 & threshold>-130) = -110; threshold(time<=Tpd+2.3e-6) = threshold(time<=Tpd+2.3e-6)+20;'; % param.collate_coh_noise.threshold_eval{img} = 'threshold = max(min(-100,threshold + 20),10*log10(abs(dft_noise(:,1)).^2)+6);'; % param.collate_coh_noise.threshold_eval{img} = 'threshold = max(min(nt,threshold+6),max_filt1(10*log10(abs(dft_noise(:,1)).^2)+15-1e6*(time>(Tpd+1.2e-6)),5));'; - eval(param.collate_coh_noise.threshold_eval{img}); + % To update threshold in debug mode run: + % threshold = orig_threshold; + % cmd_str = param.collate_coh_noise.threshold_eval{img}; % Fill in string with new threshold_eval command + % eval(cmd_str); + eval(cmd_str); end end @@ -358,6 +481,7 @@ if enable_threshold reuse.orig_threshold = orig_threshold; reuse.threshold = threshold; + reuse.cn_before_mag = cn_before_mag; end reuse.file_version = '1'; @@ -398,7 +522,11 @@ mkdir(fig_fn_dir); end ct_saveas(h_fig(1),fig_fn); - + if numel(cn_before)*16 < param.(mfilename).debug_max_size + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('coh_fft_wf_%02d_adc_%02d',wf,adc)) '.fig']; + fprintf('Saving %s\n', fig_fn); + ct_saveas(h_fig(1),fig_fn); + end %cn_before(bsxfun(@gt,lp(cn_before,2),threshold)) = NaN; clf(h_fig(2)); @@ -412,7 +540,11 @@ fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('coh_wf_%02d_adc_%02d',wf,adc)) '.jpg']; fprintf('Saving %s\n', fig_fn); ct_saveas(h_fig(2),fig_fn); - + if numel(cn_before)*16 < param.(mfilename).debug_max_size + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('coh_wf_%02d_adc_%02d',wf,adc)) '.fig']; + fprintf('Saving %s\n', fig_fn); + ct_saveas(h_fig(1),fig_fn); + end %cn_before(bsxfun(@gt,lp(cn_before,2),threshold)) = NaN; clf(h_fig(3)); @@ -425,7 +557,11 @@ fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('coh_phase_wf_%02d_adc_%02d',wf,adc)) '.jpg']; fprintf('Saving %s\n', fig_fn); ct_saveas(h_fig(3),fig_fn); - + if numel(cn_before)*16 < param.(mfilename).debug_max_size + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('coh_phase_wf_%02d_adc_%02d',wf,adc)) '.fig']; + fprintf('Saving %s\n', fig_fn); + ct_saveas(h_fig(1),fig_fn); + end clf(h_fig(4)); set(h_fig(4), 'name', 'collate_coh_noise After'); @@ -438,6 +574,11 @@ fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('coh_after_wf_%02d_adc_%02d',wf,adc)) '.jpg']; fprintf('Saving %s\n', fig_fn); ct_saveas(h_fig(4),fig_fn); + if numel(cn_before)*16 < param.(mfilename).debug_max_size + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('coh_after_wf_%02d_adc_%02d',wf,adc)) '.fig']; + fprintf('Saving %s\n', fig_fn); + ct_saveas(h_fig(1),fig_fn); + end linkaxes(h_axes(2:4)); end @@ -451,7 +592,8 @@ grid(h_axes(5), 'on'); plot(h_axes(5), threshold, 'LineStyle', '--') plot(h_axes(5), lp(abs(dft_noise(:,1)).^2,1)) - legend(h_axes(5), 'Original', 'Modified', 'DC Noise', 'location', 'best') + plot(h_axes(5), coh_noise_est); + legend(h_axes(5), 'Original', 'Modified', 'DC Noise', 'Coh Noise Est', 'location', 'best') xlabel(h_axes(5), 'Range bin'); ylabel(h_axes(5), 'Relative power (dB)'); title(h_axes(5), sprintf('Threshold %s wf %d adc %d',regexprep(param.day_seg,'_','\\_'), wf, adc)); @@ -465,15 +607,25 @@ mkdir(fig_fn_dir); end ct_saveas(h_fig(5),fig_fn); + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('threshold_wf_%02d_adc_%02d',wf,adc)) '.fig']; + fprintf('Saving %s\n', fig_fn); + ct_saveas(h_fig(5),fig_fn); end if enable_visible_plot + % frm_id + fprintf('\nDebug breakpoint:\n The variable "frm_id" maps the "Block" to the frame that the block occurs in. The fast-time axis is stored in the "time" variable. \n\n'); + [~,frm_id] = get_frame_id(param,noise.gps_time); + % Bring plots to front for h_fig_idx = 1:length(h_fig) figure(h_fig(h_fig_idx)); end - % Enter debug mode - keyboard + + if enable_debug_plot + % Enter debug mode + keyboard + end end %% Create the simplified output @@ -483,7 +635,6 @@ noise_simp.fc = noise.fc; noise_simp.recs = recs; noise_simp.start_bin = start_bin; - noise_simp.datestr = datestr(now); noise_simp.param_collate_coh_noise = param; if strcmpi(param.collate_coh_noise.method{img},'dft') noise_simp.dft_freqs = dft_freqs; @@ -495,7 +646,8 @@ noise_simp.param_records = noise.param_records; noise_simp.param_analysis = noise.param_analysis; if enable_threshold - noise_simp.threshold = threshold; + noise_simp.threshold = threshold; + noise_simp.threshold_time = time; end if param.ct_file_lock @@ -511,60 +663,7 @@ out_fn = fullfile(out_fn_dir,sprintf('coh_noise_simp_%s_wf_%d_adc_%d.mat', param.day_seg, wf, adc)); fprintf('Saving %s (%s)\n', out_fn, datestr(now)); ct_save(out_fn,'-struct','noise_simp'); - - % case 'custom2' - % - % coh_ave_RFI = noise.coh_ave(regime_mask,:).'; - % - % % Remove RFI - % B = [ones(1,11), 0, ones(1,11)]; A = 1; B = B / sum(B); - % threshold = fir_dec(abs(coh_ave_RFI).^2, B, 1); - % mask = lp(coh_ave_RFI) > lp(threshold) + 20; - % coh_ave_RFI(mask) = 0; - % for rbin = 1:size(coh_ave_RFI,1) - % coh_ave_RFI(rbin,:) = interp_finite(coh_ave_RFI(rbin,:).',0).'; - % end - % - % coh_ave = coh_ave_RFI; - % Mx = 1; - % Nx = size(coh_ave,2); - % Nx_cutoff = round(cmd.Wn*Nx); - % - % median_coh_ave = median(abs(coh_ave),2); - % mask = bsxfun(@gt,abs(coh_ave),abs(median_coh_ave*cmd.threshold)); - % - % coh_ave(mask) = NaN; - % - % coh_ave_hpf = coh_ave_RFI; - % for fc = -round(cmd.Wn*Nx):round(cmd.Wn*Nx) - % coh_ave = coh_ave_hpf; - % coh_ave(mask) = NaN; - % coh_ave_mod = bsxfun(@times,coh_ave,exp(-1i*2*pi*fc/Nx*(0:Nx-1))); - % mean_coh_ave = nanmean(coh_ave_mod,2); - % mean_coh_ave = kron(mean_coh_ave, exp(1i*2*pi*fc/Nx*(0:Nx-1))); - % coh_ave_hpf = coh_ave_hpf - mean_coh_ave; - % end - % - % if 0 - % figure(1); clf; - % imagesc(lp(coh_ave_hpf)) - % title('Should have coherent noise removed if working well.') - % caxis([0 100]); - % - % figure(2); clf; - % imagesc(lp(noise.coh_ave(regime_mask,:).')) - % caxis([0 100]); - % - % figure(3); clf; - % imagesc(mask) - % - % keyboard - % end - % - % % Store data (1 - HPF = LPF) - % noise.coh_ave(regime_mask,:) = coh_ave_RFI.'-coh_ave_hpf.'; - % - + end end diff --git a/cresis-toolbox/processing/collate_coh_noise_load.m b/cresis-toolbox/processing/collate_coh_noise_load.m new file mode 100644 index 00000000..dcd31d6b --- /dev/null +++ b/cresis-toolbox/processing/collate_coh_noise_load.m @@ -0,0 +1,63 @@ +function noise = collate_coh_noise_load(param,wf,adc) +% noise = collate_coh_noise_load(param,wf,adc) +% +% Loads collate_coh_noise output file. +% +% Authors: John Paden + +if isempty(param) + error('param is empty and should contain a radar parameter spreadsheet structure or a collate_coh_noise filename.'); +end + +if ~exist('wf','var') || isempty(wf) + wf = 1; +end + +if ~exist('adc','var') || isempty(adc) + adc = 1; +end + +if ischar(param) + noise_fn = param; + param = []; +elseif isstruct(param) + noise_fn_dir = fileparts(ct_filename_out(param,param.radar.wfs(wf).coh_noise_arg.fn, '')); + noise_fn = fullfile(noise_fn_dir,sprintf('coh_noise_simp_%s_wf_%d_adc_%d.mat', param.day_seg, wf, adc)); +else + error('param must be a string or struct.'); +end +noise = load(noise_fn); + +update_collate_coh_noise_file = false; +if ~isfield(noise,'param_collate_coh_noise') || isempty(noise.param_collate_coh_noise) + update_collate_coh_noise_file = true; +elseif ~isfield(noise.param_collate_coh_noise.collate_coh_noise,'method') || isempty(noise.param_collate_coh_noise.collate_coh_noise.method) + update_collate_coh_noise_file = true; +end +if ~isfield(noise,'param_records') || isempty(noise.param_records) + update_collate_coh_noise_file = true; +end +if isfield(noise,'coh_noise') && isfield(noise,'coh_noise_gps_time') + update_collate_coh_noise_file = true; +end +for wf = 1:length(noise.param_analysis.radar.wfs) + if ~isfield(noise.param_analysis.radar.wfs(wf),'system_dB') + update_collate_coh_noise_file = true; + end +end + +if update_collate_coh_noise_file + if isempty(param) + if isfield(noise,'param_collate_coh_noise') + param = noise.param_collate_coh_noise; + else + error('A param structure must be passed in instead of collate_coh_noise filename to use the old file format. After updating, the filename may be used with collate_coh_noise_load.'); + end + end + warning('Old collate_coh_noise file format. collate_coh_noise_update.m being run to update collate_coh_noise file.'); + param.collate_coh_noise_update.wfs = wf; + param.collate_coh_noise_update.rx_paths{wf} = nan(1,adc); + param.collate_coh_noise_update.rx_paths{wf}(adc) = true; % Actual value does not matter as long as ~isnan + collate_coh_noise_update(param,[]); + noise = load(param,[]); +end diff --git a/cresis-toolbox/processing/collate_coh_noise_update.m b/cresis-toolbox/processing/collate_coh_noise_update.m new file mode 100644 index 00000000..623a78eb --- /dev/null +++ b/cresis-toolbox/processing/collate_coh_noise_update.m @@ -0,0 +1,120 @@ +function collate_coh_noise_update(param,param_override) +% collate_coh_noise_update(param,param_override) +% +% Update collate_coh_noise files +% +% Example: +% See run_collate_coh_noise_update for how to run. +% +% Authors: John Paden + +%% General Setup +% ===================================================================== +param = merge_structs(param, param_override); + +fprintf('=====================================================================\n'); +fprintf('%s: %s (%s)\n', mfilename, param.day_seg, datestr(now)); +fprintf('=====================================================================\n'); + +%% Input check +% ===================================================================== + +if ~isfield(param,'collate_coh_noise_update') + param.collate_coh_noise_update = []; +end + +% param.collate_coh_noise_update.wfs: numeric vector of waveforms to be +% updated. Default is to do all waveforms: 1:length(param.radar.wfs) +if ~isfield(param.collate_coh_noise_update,'wfs') + param.collate_coh_noise_update.wfs = 1:length(param.radar.wfs); +end + +% param.collate_coh_noise_update.rx_paths: vector of length equal to the +% number of adcs to update. The entries in the vector do not matter except +% whether or not they are NaN. ADCs corresponding to NaN entries will be +% skipped. Non-NaN will be updated. For example if there are six ADCs and +% only the fourth ADC should be processed, then two example field values +% would work: +% [NaN NaN NaN 1 NaN NaN] +% [NaN NaN NaN 1] +% Since only the fourth entry is ~isnan(), only the fourth ADC will be +% updated. +if ~isfield(param.collate_coh_noise_update,'rx_paths') + for wf = param.collate_coh_noise_update.wfs + % Make sure param.radar.wfs.rx_paths exists + if ~isfield(param.radar.wfs,'rx_paths') + param.radar.wfs.rx_paths = 1; + end + param.collate_coh_noise_update.rx_paths{wf} = param.radar.wfs(wf).rx_paths; + end +end + +%% Update files if needed +% ===================================================================== +if cluster_job_check() + error('collate_coh_noise_update may not be called from cluster_job (gRadar.cluster.is_cluster_job is currently set to true). To remove this error, run collate_coh_noise_update on:\n %s\n %s\n %s', param.radar_name, param.season_name, param.day_seg); +end + +% Loop through each waveform for this segment +for wf = param.collate_coh_noise_update.wfs + % Loop through each adc for this segment + for adc = 1:length(param.collate_coh_noise_update.rx_paths{wf}) + if ~isfinite(param.collate_coh_noise_update.rx_paths{wf}(adc)) + % NaN in rx_paths means the adc is not attached to anything + continue; + end + + % Create filename + noise_fn_dir = fileparts(ct_filename_out(param,param.radar.wfs(wf).coh_noise_arg.fn, '')); + noise_fn = fullfile(noise_fn_dir,sprintf('coh_noise_simp_%s_wf_%d_adc_%d.mat', param.day_seg, wf, adc)); + + % Check for existence + if ~exist(noise_fn,'file') + warning('File does not exist so skipping: %s', noise_fn); + continue; + end + + % Load file + fprintf(' Load coh_noise: %s (%s)\n', noise_fn, datestr(now)); + noise = load(noise_fn); + + % Check to see if missing any version 1 fields + update_collate_coh_noise_file = false; + if ~isfield(noise,'param_collate_coh_noise') || isempty(noise.param_collate_coh_noise) + update_collate_coh_noise_file = true; + warning('Missing field. Updated file will be saved.'); + noise.param_collate_coh_noise = noise.param_collate; + noise = rmfield(noise,'param_collate'); + end + if ~isfield(noise,'param_records') || isempty(noise.param_records) + update_collate_coh_noise_file = true; + warning('Missing field. Updated file will be saved.'); + noise.param_records = param; + end + if ~isfield(noise.param_collate_coh_noise.collate_coh_noise,'method') || isempty(noise.param_collate_coh_noise.collate_coh_noise.method) + update_collate_coh_noise_file = true; + warning('Missing field. Updated file will be saved.'); + noise.param_collate_coh_noise.collate_coh_noise.method = 'dft'; + end + if isfield(noise,'coh_noise') && isfield(noise,'coh_noise_gps_time') + update_collate_coh_noise_file = true; + warning('Missing field. Updated file will be saved.'); + noise.firdec_gps_time = noise.coh_noise_gps_time; + noise.firdec_noise = noise.coh_noise; + noise = rmfield(noise,{'coh_noise','coh_noise_gps_time'}); + end + if ~isfield(noise.param_analysis.radar.wfs(wf),'system_dB') + update_collate_coh_noise_file = true; + warning('Missing field. Updated file will be saved.'); + % Update each entry with system_dB default value of 0 + for tmp_wf = 1:length(param.radar.wfs) + noise.param_analysis.radar.wfs(tmp_wf).system_dB = 0; + end + end + + % Save result if missing any version 1 fields + if update_collate_coh_noise_file + ct_save(noise_fn,'-struct','noise') + end + end +end diff --git a/cresis-toolbox/processing/collate_deconv.m b/cresis-toolbox/processing/collate_deconv.m index f451b303..523cd95a 100644 --- a/cresis-toolbox/processing/collate_deconv.m +++ b/cresis-toolbox/processing/collate_deconv.m @@ -71,6 +71,11 @@ function collate_deconv(param,param_override) param.collate_deconv.abs_metric = [58 5 -25 -35 inf inf]; end +% param.collate_deconv.bad_gps_times: specify ranges of gps-times to +% exclude. Nrng by 2 numeric matrix. Nrng is the number of ranges to +% exclude. The first column is the start time of the range and the second +% column is the end time of the range. bad_gps_times has no effect on +% waveforms that are selected via gps_times. if ~isfield(param.collate_deconv,'bad_gps_times') || isempty(param.collate_deconv.bad_gps_times) param.collate_deconv.bad_gps_times = []; end @@ -99,6 +104,13 @@ function collate_deconv(param,param_override) end debug_ylim = param.collate_deconv.debug_ylim; +% decimate_table_sec: decimation spacing in seconds (default 10 sec), an entry +% in the deconvolution waveform mapping will be made every decimate_table_sec +% seconds. +if ~isfield(param.collate_deconv,'decimate_table_sec') || isempty(param.collate_deconv.decimate_table_sec) + param.collate_deconv.decimate_table_sec = 10; +end + if ~isfield(param.collate_deconv,'f0') || isempty(param.collate_deconv.f0) % Default is no limits on lower frequency param.collate_deconv.f0 = -inf; @@ -113,6 +125,15 @@ function collate_deconv(param,param_override) param.collate_deconv.gps_time_penalty = 1/(10*24*3600); end +% gps_times is a Ngt by 3 matrix where the number of rows, Ngt, is the +% number of gps times to specifically consider in the final deconvolution +% mapping. If gps_times is empty (Ngt == 0), then all waveforms are +% considered. The first column specifies the GPS time of the deconvolution +% waveform to use (the deconvolution waveform closest to this GPS time will +% be used), the second column and third column specify the range of GPS +% times that this waveform will be considered for. To consider a waveform +% for all times then the 2nd and 3rd columns should be -inf and +inf +% respectively. if ~isfield(param.collate_deconv,'gps_times') || isempty(param.collate_deconv.gps_times) param.collate_deconv.gps_times = []; end @@ -125,6 +146,33 @@ function collate_deconv(param,param_override) param.collate_deconv.in_path = 'analysis'; end +if ~isfield(param.collate_deconv,'magnitude_only') || isempty(param.collate_deconv.magnitude_only) + param.collate_deconv.magnitude_only = []; +end + +if ~isfield(param.collate_deconv.magnitude_only,'en') || isempty(param.collate_deconv.magnitude_only.en) + param.collate_deconv.magnitude_only.en = false; +end + +if ~isfield(param.collate_deconv.magnitude_only,'f_cutoff') || isempty(param.collate_deconv.magnitude_only.f_cutoff) + param.collate_deconv.magnitude_only.f_cutoff = 0.1; +end + +% metric_mode: scalar integer or a string containing one of three allowed +% strings: +% * "each" means that the deconvolution will be done for each sample with +% the averaged sample waveform itself (DEFAULT mode) +% * "best" means that the deconvolution will be done by the waveform with +% the best score (requires stage one to have been run) +% * "final" means that the deconvolution will be done with the waveform +% determined in the final deconvolution file (requires stage two to have +% been run) +% * integer: scalar integer which determines the index of he waveform that +% will be used for the deconvolution +if ~isfield(param.collate_deconv,'metric_mode') || isempty(param.collate_deconv.metric_mode) + param.collate_deconv.metric_mode = 'each'; +end + if ~isfield(param.collate_deconv,'metric_weights') || isempty(param.collate_deconv.metric_weights) param.collate_deconv.metric_weights = [0.5 0 3 5 0 0]; end @@ -183,10 +231,21 @@ function collate_deconv(param,param_override) param.collate_deconv.stage_two_en = true; end +if ~isfield(param.collate_deconv,'surf_layer') || isempty(param.collate_deconv.surf_layer) + param.collate_deconv.surf_layer.name = 'surface'; + param.collate_deconv.surf_layer.source = 'layerdata'; +end + if ~isfield(param.collate_deconv,'threshold') || isempty(param.collate_deconv.threshold) param.collate_deconv.threshold = -inf; end +% twtt_penalty: This is generally not relevant for non-deramp systems since +% the twtt should not affect the impulse response. However, for deramp +% systems and the way that pulse compression and deskew are done, the twtt +% affects the impulse reponse since sidelobes are affected by the relative +% time displacement between the imperfect reference-deramp and imperfect RF +% signal. if ~isfield(param.collate_deconv,'twtt_penalty') || isempty(param.collate_deconv.twtt_penalty) if strcmpi(radar_type,'deramp') param.collate_deconv.twtt_penalty = 1e6; @@ -248,7 +307,7 @@ function collate_deconv(param,param_override) % If no wf-adc pairs specified, then do them all. wf_adcs = 1:size(param.analysis.imgs{img},1); else - wf_adcs = param.collate_deconv.wf_adcs; + wf_adcs = param.collate_deconv.wf_adcs{img}; end for wf_adc = wf_adcs %% Stage 1: Load specular analysis file @@ -256,6 +315,7 @@ function collate_deconv(param,param_override) wf = param.analysis.imgs{img}(wf_adc,1); adc = param.analysis.imgs{img}(wf_adc,2); + % spec: Load specular file from analysis specular outputs fn_dir = fileparts(ct_filename_out(param,param.collate_deconv.in_path, '')); fn = fullfile(fn_dir,sprintf('specular_%s_wf_%d_adc_%d.mat', param.day_seg, wf, adc)); fprintf('Loading %s img %d wf %d adc %d\n %s\n', param.day_seg, img, wf, adc, fn); @@ -290,6 +350,10 @@ function collate_deconv(param,param_override) fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_peakiness_wf_%02d_adc_%02d',param.collate_deconv.out_path,wf,adc)) '.jpg']; fprintf('Saving %s\n', fig_fn); ct_saveas(h_fig(1),fig_fn); + + if any(strcmp('visible',param.collate_deconv.debug_plots)) + keyboard + end end if isempty(spec.deconv_gps_time) @@ -297,6 +361,11 @@ function collate_deconv(param,param_override) continue; end + % Load surface layer + layer = opsLoadLayers(param,param.collate_deconv.surf_layer); + spec.surface = interp_finite(interp1(layer.gps_time, layer.twtt, spec.gps_time)); + spec.deconv_twtt = interp_finite(interp1(layer.gps_time, layer.twtt, spec.deconv_gps_time)); + %% Stage 1: Preallocation deconv = []; deconv.gps_time = []; @@ -340,14 +409,51 @@ function collate_deconv(param,param_override) continue; end + % Load best and final metric_mode information + % =================================================================== + if strcmpi(param.collate_deconv.metric_mode,'best') + out_fn = fullfile(fn_dir,sprintf('deconv_lib_%s_wf_%d_adc_%d.mat', param.day_seg, wf, adc)); + if ~exist(out_fn,'file') + error('param.collate_deconv.metric_mode == "best" except there is no deconv_lib file. Run stage one first with metric_mode set to "each".'); + end + fprintf(' Loading best deconv info in %s\n', out_fn); + best_deconv = load(out_fn,'metric','rec'); + + % Compute the score + best_score = nansum(bsxfun(@times, param.collate_deconv.metric_weights(:), bsxfun(@minus, param.collate_deconv.abs_metric(:), best_deconv.metric))); + best_score(:,any(isnan(best_deconv.metric))) = -inf; + + % Make score adjustments for param.collate_deconv.rec_adjustments + for rec_idx = 1:size(param.collate_deconv.rec_adjustments{img},1) + [rec_offset,score_idx] = min(abs(best_deconv.rec-param.collate_deconv.rec_adjustments{img}(rec_idx,1))); + if rec_offset > cmd.rlines + warning('Record offset to closest waveform to rec_adjustments{%d}(%d,1) is larger than the STFT interval used in analysis spec. This may mean that the record in rec_adjustments{%d}(%d,1) is incorrect.',img,rec_idx,img,rec_idx); + else + score(score_idx) = score(score_idx) + param.collate_deconv.rec_adjustments{img}(rec_idx,2); + end + end + + [~,best_rline] = max(best_score); + clear best_deconv best_score; + + elseif strcmpi(param.collate_deconv.metric_mode,'final') + out_fn = fullfile(fn_dir,sprintf('deconv_%s_wf_%d_adc_%d.mat', param.day_seg, wf, adc)); + if ~exist(out_fn,'file') + error('param.collate_deconv.metric_mode == "final" except there is no deconv file. Run stage two first.'); + end + fprintf(' Loading final deconv info in %s\n', out_fn); + final_deconv = load(out_fn); + end + % Stage 1: Loop to analyze each waveform % =================================================================== for rline = 1:length(spec.deconv_gps_time) %% Stage 1: Impulse response - if interp1(spec.gps_time,spec.peakiness,spec.deconv_gps_time(rline)) < param.collate_deconv.threshold - % Waveform peakiness is too low + if interp1(spec.gps_time,spec.peakiness,spec.deconv_gps_time(rline)) < param.collate_deconv.threshold ... + || isnan(spec.deconv_fc(rline)) || isnan(spec.deconv_t0(rline)) || isnan(spec.dt) + % Waveform peakiness is too low OR NaN in deconv_fc or dt deconv.metric(:,rline) = nan(6,1); [~,match_idx] = min(abs(spec.gps_time - spec.deconv_gps_time(rline))); deconv.gps_time(rline) = spec.gps_time(match_idx); @@ -362,224 +468,61 @@ function collate_deconv(param,param_override) deconv.ref_nonnegative{rline} = []; deconv.ref_negative{rline} = []; deconv.ref_mult_factor(rline) = NaN; + deconv.radiometric_error_dB(rline) = NaN; deconv.impulse_response{rline} = []; continue; end - % h: impulse response - h = spec.deconv_mean{rline}; - - % Estimate SNR as a function of range bin - SNR = lp(abs(h).^2 ./ spec.deconv_std{rline}.^2 * cmd.rlines,1); - - % Time gate signal according to cmd.rlines - Ntg = param.collate_deconv.rbins{img}(end)-param.collate_deconv.rbins{img}(1)+1; - Htg = tukeywin(param.collate_deconv.rbins{img}(end)*2+1,0.2); - h_nonnegative = h(1:1+param.collate_deconv.rbins{img}(end)) .* Htg(end-param.collate_deconv.rbins{img}(end):end); - Htg = tukeywin(-param.collate_deconv.rbins{img}(1)*2,0.2); - h_negative = h(end+1+param.collate_deconv.rbins{img}(1):end) .* Htg(1:-param.collate_deconv.rbins{img}(1)); - - %% Stage 1: Plot cmd.rlines and param.collate_deconv.rbins{img} - if any(strcmp('rbins',param.collate_deconv.debug_plots)) ... - && (isempty(param.collate_deconv.debug_rlines) || any(rline==param.collate_deconv.debug_rlines)) - figure(h_fig(1)); clf(h_fig(1)); - set(h_fig(1),'Name','Impulse response falling edge'); - h_axes = axes('parent',h_fig(1)); - max_dm = max(lp(spec.deconv_mean{rline})); - plot(h_axes(1), lp(spec.deconv_mean{rline}) - max_dm) - hold(h_axes(1),'on'); - [max_val,max_idx] = max(lp(spec.deconv_sample{rline})); - plot(h_axes(1), circshift(lp(spec.deconv_sample{rline}) - max_val,[-max_idx 1])) - plot(h_axes(1), lp(spec.deconv_std{rline},2) - lp(cmd.rlines) - max_dm) - h_filled = [h_nonnegative; zeros(length(spec.deconv_sample{rline})-length(h_nonnegative)-length(h_negative),1); h_negative]; - plot(h_axes(1), lp(h_filled) - max_dm) - xlabel(h_axes(1), 'Range bin'); - ylabel(h_axes(1), 'Relative power (dB)'); - title(h_axes(1), sprintf('Impulse response falling edge (rline %d of %d)',rline, length(spec.deconv_gps_time))); - legend(h_axes(1), 'mean','sample','std','h','location','best'); - xlim(h_axes(1), [1 2*param.collate_deconv.rbins{img}(2)]); - ylim(h_axes(1), [-debug_ylim 0]); - grid(h_axes(1), 'on'); - - figure(h_fig(2)); clf(h_fig(2)); - set(h_fig(2),'Name','Impulse response rising edge'); - h_axes(2) = axes('parent',h_fig(2)); - max_dm = max(lp(spec.deconv_mean{rline})); - plot(h_axes(2), lp(spec.deconv_mean{rline}) - max_dm) - hold(h_axes(2),'on'); - [max_val,max_idx] = max(lp(spec.deconv_sample{rline})); - plot(h_axes(2), circshift(lp(spec.deconv_sample{rline}) - max_val,[-max_idx 1])) - plot(h_axes(2), lp(spec.deconv_std{rline},2) - lp(cmd.rlines) - max_dm) - h_filled = [h_nonnegative; zeros(length(spec.deconv_sample{rline})-length(h_nonnegative)-length(h_negative),1); h_negative]; - plot(h_axes(2), lp(h_filled) - max_dm) - xlabel(h_axes(2), 'Range bin'); - ylabel(h_axes(2), 'Relative power (dB)'); - title(h_axes(2), sprintf('Impulse response rising edge (rline %d of %d)',rline, length(spec.deconv_gps_time))); - legend(h_axes(2), 'mean','sample','std','h','location','best'); - Nt = length(spec.deconv_mean{rline}); - xlim(h_axes(2), [Nt+2*param.collate_deconv.rbins{img}(1) Nt]); - ylim(h_axes(2), [-debug_ylim 0]); - grid(h_axes(2), 'on'); - - figure(h_fig(3)); clf(h_fig(3)); - set(h_fig(3),'Name','SNR falling edge'); - h_axes(3) = axes('parent',h_fig(3)); - plot(h_axes(3),SNR) - hold(h_axes(3),'on'); - plot(h_axes(3),[1 length(SNR)], 20*[1 1],'k--'); - plot(h_axes(3),param.collate_deconv.rbins{img}(2)*[1 1], [0 debug_ylim],'k--'); - title(h_axes(3),sprintf('SNR falling edge (rline %d of %d)',rline, length(spec.deconv_gps_time))); - xlabel(h_axes(3),'Range bin'); - ylabel(h_axes(3),'SNR (dB)'); - ylim(h_axes(3), [0 debug_ylim]); - grid(h_axes(3), 'on'); - - figure(h_fig(4)); clf(h_fig(4)); - set(h_fig(4),'Name','SNR rising edge'); - h_axes(4) = axes('parent',h_fig(4)); - plot(h_axes(4),SNR) - hold(h_axes(4),'on'); - plot(h_axes(4),[1 length(SNR)], 20*[1 1],'k--'); - plot(h_axes(4),(Nt+param.collate_deconv.rbins{img}(1))*[1 1], [0 debug_ylim],'k--'); - title(h_axes(4),sprintf('SNR rising edge (rline %d of %d)',rline, length(spec.deconv_gps_time))); - xlabel(h_axes(4),'Range[-200 150] bin'); - ylabel(h_axes(4),'SNR (dB)'); - ylim(h_axes(4), [0 debug_ylim]); - grid(h_axes(4), 'on'); - - linkaxes(h_axes([1 3]),'x'); - linkaxes(h_axes([2 4]),'x'); - - keyboard - end - - %% Stage 1: Test deconvolution - - % Create frequency axis - Nt = length(spec.deconv_sample{rline}); - - % Get sample rline parameters - fc = spec.deconv_fc(rline); - t0 = spec.deconv_t0(rline); - dt = spec.dt; - time = t0 + dt*(0:Nt-1).'; - - % Adjust deconvolution signal to match sample rline - h_filled = [h_nonnegative; zeros(length(spec.deconv_sample{rline})-length(h_nonnegative)-length(h_negative),1); h_negative]; - deconv_Nt = length(h_filled); - % Is dt different? Error - if dt ~= spec.dt - error('The fast time sample spacing of the data (%g) does not match the deconvolution waveform sampling (%g).',dt,spec.dt); - end - % Is fc different? Multiply time domain by exp(1i*2*pi*dfc*deconv_time) - dfc = fc - spec.deconv_fc(rline); - if dfc/fc > 1e-6 - deconv_time = t0 + dt*(0:Nt-1).'; - h_filled = h_filled .* exp(1i*2*pi*dfc*deconv_time); - end - % Take FFT of deconvolution impulse response - h_filled = fft(h_filled); - - % Create inverse filter relative to window - df = 1/(Nt*dt); - freq = spec.deconv_fc(rline) + df * ifftshift(-floor(Nt/2) : floor((Nt-1)/2)).'; - freq = fftshift(freq); - Nt_shorten = find(param.collate_deconv.f0 <= freq,1); - Nt_shorten(2) = length(freq) - find(param.collate_deconv.f1 >= freq,1,'last'); - Nt_Hwind = Nt - sum(Nt_shorten); - Hwind = deconv.ref_window(Nt_Hwind); - Hwind_filled = ifftshift([zeros(Nt_shorten(1),1); Hwind; zeros(Nt_shorten(end),1)]); - h_filled_inverse = Hwind_filled ./ h_filled; - - % Normalize so that reflection is 0 dB (i.e. we assume this - % specular target is a perfect reflector) at this range, R when - % voltage is scaled R. + % Find peak of specular return + radiometric_measured = max(lp(spec.deconv_sample{rline})); + % Calculate the range R for debugging. Usually the specular + % reflection is 0 dB (i.e. the specular target is a perfect + % reflector) and we expect the voltage to be 1/R after corrections + % are applied when loading data (e.g. system_dB, adc_gains_dB, + % etc.). R = interp1(spec.gps_time,spec.surface,spec.deconv_gps_time(rline)) * c/2; - % Apply deconvolution with unnormalized filter - h_deconvolved = ifft(fft(spec.deconv_sample{rline}) .* h_filled_inverse); - % Oversample to get a good measurement of the peak value - h_deconvolved = interpft(h_deconvolved,Mt*Nt); - - % Oversample sample signal by the same amount as the deconvolved - % signal - h_sample = interpft(spec.deconv_sample{rline},Mt*Nt); - - % Scale so that the peak value is 1/R - h_mult_factor = max(abs(h_sample)) / max(abs(h_deconvolved)); - h_filled_inverse = h_filled_inverse * h_mult_factor; - h_deconvolved = h_deconvolved * h_mult_factor; - - % Find maximum values and indices for the deconvolved and - % undeconvolved signals. - [deconv_max_val,deconv_max_idx] = max(h_deconvolved); - [max_val,max_idx] = max(h_sample); + % Assuming specular return is water with 0 dB reflection + % coefficient and no roughness + radiometric_expected = lp(1/R^2); + deconv.radiometric_error_dB(rline) = radiometric_measured-radiometric_expected; - %% Stage 1: Plot h_deconvolved, f0, f1, SL_guard_bins - if any(strcmp('deconv',param.collate_deconv.debug_plots)) ... - && (isempty(param.collate_deconv.debug_rlines) || any(rline==param.collate_deconv.debug_rlines)) - - comp_bins = param.collate_deconv.rbins{img}(2)+1:2*param.collate_deconv.rbins{img}(2); - comp_bins = Nt+2*param.collate_deconv.rbins{img}(1) : Nt+param.collate_deconv.rbins{img}; - dnoise = lp(mean(abs(h_sample(comp_bins)).^2) ./ mean(abs(h_deconvolved(comp_bins)).^2)); - dsignal = lp(max(abs(h_sample).^2) ./ max(abs(h_deconvolved).^2)); - dSNR = dsignal - dnoise; - - fprintf('SNR loss: %.1f dB\n', dSNR) - fprintf('Peak magnitude: %.1f (dB)\n', lp(max_val./deconv_max_val)); - fprintf('Peak angle: %.1f (deg)\n', angle(max_val./deconv_max_val) * 180/pi); - fprintf('Index offset: %d\n', mod(max_idx - deconv_max_idx + Nt*Mt/2, Nt*Mt)-Nt*Mt/2); - - bins_Mt = 0:1/Mt:Nt-1/Mt; - - figure(h_fig(1)); clf(h_fig(1)); - set(h_fig(1),'Name','Impulse response falling edge'); - h_axes = axes('parent',h_fig(1)); - [max_val,max_idx] = max(lp(h_sample)); - plot(h_axes(1), bins_Mt, circshift(lp(h_sample) - max_val,[-max_idx 1])) - hold(h_axes(1),'on'); - plot(h_axes(1), bins_Mt, circshift(lp(h_deconvolved) - max_val + dsignal,[-max_idx 1])) - plot(h_axes(1), param.collate_deconv.SL_guard_bins*[1 1], [-debug_ylim 0], 'k--'); - xlabel(h_axes(1), 'Range bin'); - ylabel(h_axes(1), 'Relative power (dB)'); - title(h_axes(1), 'Impulse response falling edge'); - legend(h_axes(1), 'sample','deconvolved','location','best'); - xlim(h_axes(1), [0 2*param.collate_deconv.rbins{img}(2)]); - ylim(h_axes(1), [-debug_ylim 0]); - grid(h_axes(1), 'on'); + if strcmpi(param.collate_deconv.metric_mode,'final') + deconv_map_idx = interp1(final_deconv.map_gps_time,final_deconv.map_idxs,spec.deconv_gps_time(rline),'nearest','extrap'); - figure(h_fig(2)); clf(h_fig(2)); - set(h_fig(2),'Name','Impulse response rising edge'); - h_axes(2) = axes('parent',h_fig(2)); - [max_val,max_idx] = max(lp(h_sample)); - plot(h_axes(2), fliplr(bins_Mt+1), circshift(lp(h_sample) - max_val,[-max_idx 1])) - hold(h_axes(2),'on'); - plot(h_axes(2), fliplr(bins_Mt+1), circshift(lp(h_deconvolved) - max_val + dsignal,[-max_idx 1])) - plot(h_axes(2), (1+param.collate_deconv.SL_guard_bins)*[1 1], [-debug_ylim 0], 'k--'); - xlabel(h_axes(2), 'Range bin'); - ylabel(h_axes(2), 'Relative power (dB)'); - title(h_axes(2), 'Impulse response rising edge'); - legend(h_axes(2), 'sample','deconvolved','location','best'); - xlim(h_axes(2), [0 -2*param.collate_deconv.rbins{img}(1)]); - ylim(h_axes(2), [-debug_ylim 0]); - grid(h_axes(2), 'on'); + % Get the reference function + h_nonnegative = final_deconv.ref_nonnegative{deconv_map_idx}; + h_negative = final_deconv.ref_negative{deconv_map_idx}; - figure(h_fig(3)); clf(h_fig(3)); - set(h_fig(3),'Name','Transfer function'); - h_axes(3) = axes('parent',h_fig(3)); - plot(h_axes(3), lp(h_filled)) - hold(h_axes(3),'on'); - plot(h_axes(3), lp(Hwind_filled,1) - max(lp(Hwind_filled,1)) + max(lp(h_filled))) - plot(h_axes(3), lp(h_filled_inverse) - max(lp(h_filled_inverse)) + max(lp(h_filled))) - xlabel(h_axes(3), 'Frequency bin'); - ylabel(h_axes(3), 'Relative power (dB)'); - title(h_axes(3), 'Transfer function'); - legend(h_axes(3), 'sample','window','inverse','location','best'); - grid(h_axes(3), 'on'); + else + if strcmpi(param.collate_deconv.metric_mode,'each') + % Use this range line to generate the metrics for this waveform. + % h: impulse response + h = spec.deconv_mean{rline}; + elseif strcmpi(param.collate_deconv.metric_mode,'best') + % Use the best range line to generate the metrics for this + % waveform. + % h: impulse response + h = spec.deconv_mean{best_rline}; + elseif isnumeric(param.collate_deconv.metric_mode) + % Manually select the range line to use to generate the metrics + % for this waveform. + % h: impulse response + h = spec.deconv_mean{param.collate_deconv.metric_mode}; + end - keyboard + % Time gate signal according to cmd.rlines + Htg = tukeywin(param.collate_deconv.rbins{img}(end)*2+1,0.2); + h_nonnegative = h(1:1+param.collate_deconv.rbins{img}(end)) .* Htg(end-param.collate_deconv.rbins{img}(end):end); + Htg = tukeywin(-param.collate_deconv.rbins{img}(1)*2,0.2); + h_negative = h(end+1+param.collate_deconv.rbins{img}(1):end) .* Htg(1:-param.collate_deconv.rbins{img}(1)); end + rbins_plot = any(strcmp('rbins',param.collate_deconv.debug_plots)) ... + && (isempty(param.collate_deconv.debug_rlines) || any(rline==param.collate_deconv.debug_rlines)); + deconv_plot = any(strcmp('deconv',param.collate_deconv.debug_plots)) ... + && (isempty(param.collate_deconv.debug_rlines) || any(rline==param.collate_deconv.debug_rlines)); + [h_mult_factor,h_deconvolved] = collate_deconv_ascope(param,spec,deconv,h_fig,img,wf_adc,h_nonnegative,h_negative,rline,rbins_plot,deconv_plot); %% Stage 1: Metrics h_metric = abs(h_deconvolved).^2; % Convert to power @@ -638,7 +581,6 @@ function collate_deconv(param,param_override) deconv.metric(:,rline) = [peak, main_lobe, peak_sidelobe_falling_edge, peak_sidelobe_rising_edge, ... integrated_sidelobe_falling_edge, integrated_sidelobe_rising_edge]; - %% Stage 1: Deconvolution for storage [~,match_idx] = min(abs(spec.gps_time - spec.deconv_gps_time(rline))); deconv.gps_time(rline) = spec.gps_time(match_idx); @@ -781,7 +723,7 @@ function collate_deconv(param,param_override) fprintf(fid,'Peak\tML\tPSL FE\tPSL RE\tISL FE\tISL RE\n'); fprintf(fid,'%.1f\t', param.collate_deconv.abs_metric); fprintf('\n'); fprintf(fid,'Metric ( '); fprintf(fid_error,'Red Failed '); fprintf(fid,')\n'); - fprintf(fid,'INDEX\tFRM\tREC\tPeak\tML\tPSL FE\tPSL RE\tISL FE\tISL RE\tPASS\tSCORE\tTWTT\n'); + fprintf(fid,'INDEX\tFRM\tREC\tPeak\tML\tPSL FE\tPSL RE\tISL FE\tISL RE\tPASS\tSCORE\tTWTT\tGPS_TIME\tGPS_TIME\tRADIOMETRIC\n'); for rline = 1:length(deconv.gps_time) fprintf(fid,'%d\t',rline); fprintf(fid,'%.02f\t',floor(deconv.frm(rline)*100)/100); @@ -821,14 +763,64 @@ function collate_deconv(param,param_override) [~,twtts_idx] = min(abs(deconv.twtt(rline)-twtts)); fprintf(fid,'%.3g ', twtts(twtts_idx)); end + + % Print gps_time + fprintf(fid, '\t%.14g\t%s', deconv.gps_time(rline), datestr(epoch_to_datenum(deconv.gps_time(rline)),'yyyy-mm-dd HH:MM:SS.FFF')); + + % Print radiometric error (dB) + fprintf(fid, '\t%.1f', deconv.radiometric_error_dB(rline)); + fprintf(fid,'\n'); end end fclose(fid); fprintf('Metric table: %s\n', diary_fn); + + if any(strcmp('visible',param.collate_deconv.debug_plots)) + keyboard + end + end + + %% Stage 1: Plot the best waveform + rbins_plot = any(strcmp('rbins_best',param.collate_deconv.debug_plots)); + deconv_plot = any(strcmp('deconv_best',param.collate_deconv.debug_plots)); + if rbins_plot || deconv_plot + % Compute the score + score = nansum(bsxfun(@times, param.collate_deconv.metric_weights(:), bsxfun(@minus, param.collate_deconv.abs_metric(:), deconv.metric))); + score(:,any(isnan(deconv.metric))) = -inf; + + % Make score adjustments for param.collate_deconv.rec_adjustments + for rec_idx = 1:size(param.collate_deconv.rec_adjustments{img},1) + [rec_offset,score_idx] = min(abs(deconv.rec-param.collate_deconv.rec_adjustments{img}(rec_idx,1))); + if rec_offset > cmd.rlines + warning('Record offset to closest waveform to rec_adjustments{%d}(%d,1) is larger than the STFT interval used in analysis spec. This may mean that the record in rec_adjustments{%d}(%d,1) is incorrect.',img,rec_idx,img,rec_idx); + else + score(score_idx) = score(score_idx) + param.collate_deconv.rec_adjustments{img}(rec_idx,2); + end + end + + % Find the best waveform + [max_val,rline] = max(score); + + % h: impulse response + h = spec.deconv_mean{rline}; + % Time gate signal according to cmd.rlines + Htg = tukeywin(param.collate_deconv.rbins{img}(end)*2+1,0.2); + h_nonnegative = h(1:1+param.collate_deconv.rbins{img}(end)) .* Htg(end-param.collate_deconv.rbins{img}(end):end); + Htg = tukeywin(-param.collate_deconv.rbins{img}(1)*2,0.2); + h_negative = h(end+1+param.collate_deconv.rbins{img}(1):end) .* Htg(1:-param.collate_deconv.rbins{img}(1)); + + % Plot waveform + [h_mult_factor,h_deconvolved] = collate_deconv_ascope(param,spec,deconv,h_fig,img,wf_adc,h_nonnegative,h_negative,rline,rbins_plot,deconv_plot); end %% Stage 1: Save results + if ~strcmpi(param.collate_deconv.metric_mode,'each') + % Outputs should not be saved if metric_mode is not "each" since + % all other metric_mode's are for debugging. + param.collate_deconv.stage_two_en = false; + continue + end fn_dir = fileparts(ct_filename_out(param,param.collate_deconv.out_path, '')); if ~exist(fn_dir,'dir') mkdir(fn_dir); @@ -837,8 +829,12 @@ function collate_deconv(param,param_override) fprintf('Saving %s img %d wf %d adc %d\n %s\n', param.day_seg, img, wf, adc, out_fn); ct_file_lock_check(out_fn,2); if 0 + % For debugging: quick hand manipulation of the results before + % saving. Specifically setup to remove certain waveforms from the + % deconv_lib file. param.collate_deconv.bad_gps_times field should + % be used in the end. good_mask = true(size(deconv.gps_time)); - good_mask(1:7) = false; + good_mask(1:7) = false; % Manually select which waveforms to remove deconv.gps_time = deconv.gps_time(good_mask); deconv.lat = deconv.lat(good_mask); deconv.lon = deconv.lon(good_mask); @@ -872,7 +868,7 @@ function collate_deconv(param,param_override) % If no wf-adc pairs specified, then do them all. wf_adcs = 1:size(param.analysis.imgs{img},1); else - wf_adcs = param.collate_deconv.wf_adcs; + wf_adcs = param.collate_deconv.wf_adcs{img}; end for wf_adc = wf_adcs wf = param.analysis.imgs{img}(wf_adc,1); @@ -940,19 +936,16 @@ function collate_deconv(param,param_override) end if isempty(deconv_lib) || isempty(deconv_lib.gps_time) - warning('There are no deconvolution waveforms in the files loaded!!!\nSpecify other cmd.day_seg to load or remake current day_seg files with lower metric thresholds.'); + warning(sprintf('There are no deconvolution waveforms in the files loaded!!!\nSpecify other cmd.day_seg to load or remake current day_seg files with lower metric thresholds.')); continue end %% Stage 2: 3. Load surface using opsLoadLayers % To determine which waveforms are needed - layer_params = []; - layer_params.name = 'surface'; - layer_params.source = 'layerData'; - layer = opsLoadLayers(param,layer_params); + layer = opsLoadLayers(param,param.collate_deconv.surf_layer); %% Stage 2: 4. Decimate layer - decim_idxs = get_equal_alongtrack_spacing_idxs(layer.gps_time,10); + decim_idxs = get_equal_alongtrack_spacing_idxs(layer.gps_time,param.collate_deconv.decimate_table_sec); layer.gps_time = layer.gps_time(decim_idxs); layer.twtt = layer.twtt(decim_idxs); layer.lat = layer.lat(decim_idxs); @@ -983,10 +976,40 @@ function collate_deconv(param,param_override) % Score with twtt penalty and time constant penalty term d_twtt = layer.twtt(rline) - deconv_lib.twtt; d_gps_time = layer.gps_time(rline) - deconv_lib.gps_time; - %adjusted_score = min_score + score .* exp(-abs(param.collate_deconv.twtt_penalty*d_twtt).^2) ... - % .* exp(-abs(param.collate_deconv.gps_time_penalty*d_gps_time).^2); adjusted_score = min_score + score - (100-100*exp(-abs(param.collate_deconv.twtt_penalty*d_twtt).^2)) ... - (50-50*exp(-abs(param.collate_deconv.gps_time_penalty*d_gps_time).^2)); + if ~isempty(param.collate_deconv.gps_times) + % The user has specifies a list of gps_times to use rather than + % using the default best waveform search method. + gps_times_mask = false(size(deconv_lib.gps_time)); + for gps_times_idx = 1:size(param.collate_deconv.gps_times,1) + if param.collate_deconv.gps_times(gps_times_idx,2) <= layer.gps_time(rline) ... + && param.collate_deconv.gps_times(gps_times_idx,3) > layer.gps_time(rline) + [~,min_idx] = min(abs(param.collate_deconv.gps_times(gps_times_idx,1) - deconv_lib.gps_time)); + gps_times_mask(min_idx) = true; + end + end + if all(~gps_times_mask) + % If this record does not have any waveforms specified by + % gps_times field, then revert to the default best waveform + % search method: + gps_times_mask = true(size(deconv_lib.gps_time)); + % Remove bad regions + for gps_times_idx = 1:size(param.collate_deconv.bad_gps_times,1) + gps_times_mask(deconv_lib.gps_time >= param.collate_deconv.bad_gps_times(gps_times_idx,1) ... + & deconv_lib.gps_time <= param.collate_deconv.bad_gps_times(gps_times_idx,2)) = false; + end + end + else + gps_times_mask = true(size(deconv_lib.gps_time)); + % Remove bad regions + for gps_times_idx = 1:size(param.collate_deconv.bad_gps_times,1) + gps_times_mask(deconv_lib.gps_time >= param.collate_deconv.bad_gps_times(gps_times_idx,1) ... + & deconv_lib.gps_time <= param.collate_deconv.bad_gps_times(gps_times_idx,2)) = false; + end + + end + adjusted_score(~gps_times_mask) = NaN; [max_score(rline),max_idx(rline)] = max(adjusted_score); unadjusted_score(rline) = min_score + score(max_idx(rline)); @@ -1041,6 +1064,8 @@ function collate_deconv(param,param_override) %% Stage 2: 8. Plot final results if any(strcmp('final',param.collate_deconv.debug_plots)) + [~,map_frm] = get_frame_id(param,final.map_gps_time); + % TWTT Figure % =================================================================== clf(h_fig(1)); @@ -1050,15 +1075,15 @@ function collate_deconv(param,param_override) legend_str = {}; h_plot = []; for idx = 1:length(final.gps_time) - h_plot(idx+1) = plot(h_axes(1), find(final.map_idxs==idx), final.twtt(final.map_idxs(final.map_idxs==idx)),'.'); + h_plot(idx+1) = plot(h_axes(1), map_frm(final.map_idxs==idx), final.twtt(final.map_idxs(final.map_idxs==idx)),'.'); hold(h_axes(1),'on'); legend_str{idx+1} = sprintf('%d',idx); end - h_plot(1) = plot(h_axes(1), final.map_twtt, 'k', 'LineWidth',2); + h_plot(1) = plot(h_axes(1), map_frm, final.map_twtt, 'k', 'LineWidth',2); legend_str{1} = 'TWTT'; - xlabel(h_axes(1), 'Block'); + xlabel(h_axes(1), 'Frame'); ylabel(h_axes(1), 'Two way travel time (\mus)'); title(h_axes(1), ['TWTT ' regexprep(param.day_seg,'_','\\_')]); legend(h_axes(1), h_plot, legend_str,'location','best'); @@ -1084,16 +1109,16 @@ function collate_deconv(param,param_override) legend_str = {}; h_plot = []; for idx = 1:length(final.gps_time) - h_plot(idx) = plot(h_axes(2), find(final.map_idxs==idx), final.max_score(find(final.map_idxs==idx)),'.'); + h_plot(idx) = plot(h_axes(2), map_frm(final.map_idxs==idx), final.max_score(find(final.map_idxs==idx)),'.'); hold(h_axes(2),'on'); legend_str{idx} = sprintf('%d',idx); end for idx = 1:length(final.gps_time) - h_new_plot = plot(h_axes(2), find(final.map_idxs==idx), final.unadjusted_score(find(final.map_idxs==idx)),'.'); + h_new_plot = plot(h_axes(2), map_frm(final.map_idxs==idx), final.unadjusted_score(find(final.map_idxs==idx)),'.'); set(h_new_plot, 'Color', get(h_plot(idx),'Color')) end - xlabel(h_axes(2), 'Block'); + xlabel(h_axes(2), 'Frame'); ylabel(h_axes(2), 'Score'); title(h_axes(2), ['Score ' regexprep(param.day_seg,'_','\\_')]); legend(h_axes(2), h_plot, legend_str,'location','best'); @@ -1116,6 +1141,7 @@ function collate_deconv(param,param_override) h_axes(4) = subplot(5,1,3:5,'parent',h_fig(3)); legend_str = {}; + max_val_overall = -inf; for idx = 1:length(final.gps_time) % Get the reference function h_nonnegative = final.ref_nonnegative{idx}; @@ -1139,6 +1165,10 @@ function collate_deconv(param,param_override) h_filled_phase = unwrap(h_filled_phase)*180/pi; h_filled_phase = h_filled_phase - h_filled_phase(fc_idx); + if max(h_filled_lp) > max_val_overall + max_val_overall = max(h_filled_lp); + end + if max(freq) > 2e9 freq_scale = 1e9; else @@ -1148,9 +1178,13 @@ function collate_deconv(param,param_override) hold(h_axes(3),'on'); plot(h_axes(4), freq/freq_scale, h_filled_phase); hold(h_axes(4),'on'); - legend_str{idx} = sprintf('%d %s_%03.2f %4.0f %4.2fus',idx, ... - final.map_day_seg{idx},final.frm(idx),round(lp(final.ref_nonnegative{idx}(1),2)), ... - round(final.twtt(idx)*1e7)/10); + radiometric_error_dB = lp(1/(c/2*final.twtt(idx)).^2) - lp(final.ref_nonnegative{idx}(1)); + legend_str{idx} = sprintf('%d %s_%03.0f %7.0f %4.0fdB %4.1fus',idx, ... + final.map_day_seg{idx},floor(final.frm(idx)),final.rec(idx),radiometric_error_dB, ... + final.twtt(idx)*1e6); + end + if isfinite(max_val_overall) + ylim(h_axes(3), max_val_overall + [-31 1]); end if freq_scale == 1e9 @@ -1160,7 +1194,7 @@ function collate_deconv(param,param_override) end ylabel(h_axes(3), 'Relative power (dB)'); ylabel(h_axes(4), 'Relative angle (deg)'); - title(h_axes(3), regexprep(sprintf('%s (Legend idx:frm:peak:twtt)', param.day_seg),'_','\\_')); + title(h_axes(3), regexprep(sprintf('%s (Legend idx:frm:rec:radio:twtt)', param.day_seg),'_','\\_')); grid(h_axes(3), 'on'); grid(h_axes(4), 'on'); h_legend = legend(h_axes(3), legend_str, 'location', 'northeastoutside', 'interpreter','none'); @@ -1169,10 +1203,10 @@ function collate_deconv(param,param_override) pos4 = get(h_axes(4),'Position'); set(h_axes(4),'Position',[pos4(1:2) pos3(3) pos4(4)]); - fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_transfer_func_wf_%02d_adc_%02d',param.collate_deconv.out_path,wf,adc)) '.fig']; + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_final_transfer_func_wf_%02d_adc_%02d',param.collate_deconv.out_path,wf,adc)) '.fig']; fprintf('Saving %s\n', fig_fn); ct_saveas(h_fig(3),fig_fn); - fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_transfer_func_wf_%02d_adc_%02d',param.collate_deconv.out_path,wf,adc)) '.jpg']; + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_final_transfer_func_wf_%02d_adc_%02d',param.collate_deconv.out_path,wf,adc)) '.jpg']; fprintf('Saving %s\n', fig_fn); ct_saveas(h_fig(3),fig_fn); end diff --git a/cresis-toolbox/processing/collate_deconv_ascope.m b/cresis-toolbox/processing/collate_deconv_ascope.m new file mode 100644 index 00000000..386f95fc --- /dev/null +++ b/cresis-toolbox/processing/collate_deconv_ascope.m @@ -0,0 +1,277 @@ +function [h_mult_factor,h_deconvolved] = collate_deconv_ascope(param,spec,deconv,h_fig,img,wf_adc,h_nonnegative,h_negative,rline,rbins_plot,deconv_plot) +% [h_mult_factor,h_deconvolved] = collate_deconv_ascope(param,spec,deconv,h_fig,img,wf_adc,h_nonnegative,h_negative,rline,rbins_plot,deconv_plot) +% +% Support function for collate_deconv. Only called from this function. + +c = 2.997924580003452e+08; +Mt = param.collate_deconv.Mt; +cmd = spec.param_analysis.analysis.cmd{param.collate_deconv.cmd_idx}; +debug_out_dir = param.collate_deconv.debug_out_dir; +debug_ylim = param.collate_deconv.debug_ylim; +wf = param.analysis.imgs{img}(wf_adc,1); +adc = param.analysis.imgs{img}(wf_adc,2); + +% h: impulse response +h = spec.deconv_mean{rline}; + +% Estimate SNR as a function of range bin +SNR = lp(abs(h).^2 ./ spec.deconv_std{rline}.^2 * cmd.rlines,1); + +%% Stage 1: Plot cmd.rlines and param.collate_deconv.rbins{img} +if rbins_plot + clf(h_fig(1)); + set(h_fig(1),'Name','Impulse response falling edge'); + h_axes = axes('parent',h_fig(1)); + max_dm = max(lp(spec.deconv_mean{rline})); + plot(h_axes(1), lp(spec.deconv_mean{rline}) - max_dm) + hold(h_axes(1),'on'); + [max_val,max_idx] = max(lp(spec.deconv_sample{rline})); + plot(h_axes(1), circshift(lp(spec.deconv_sample{rline}) - max_val,[-max_idx 1])) + plot(h_axes(1), lp(spec.deconv_std{rline},2) - lp(cmd.rlines) - max_dm) + h_filled = [h_nonnegative; zeros(length(spec.deconv_sample{rline})-length(h_nonnegative)-length(h_negative),1); h_negative]; + plot(h_axes(1), lp(h_filled) - max_dm) + xlabel(h_axes(1), 'Range bin'); + ylabel(h_axes(1), 'Relative power (dB)'); + title(h_axes(1), sprintf('Impulse response falling edge (rline %d of %d)',rline, length(spec.deconv_gps_time))); + legend(h_axes(1), 'mean','sample','std','h','location','best'); + xlim(h_axes(1), [1 2*param.collate_deconv.rbins{img}(2)]); + ylim(h_axes(1), [-debug_ylim 0]); + grid(h_axes(1), 'on'); + + clf(h_fig(2)); + set(h_fig(2),'Name','Impulse response rising edge'); + h_axes(2) = axes('parent',h_fig(2)); + max_dm = max(lp(spec.deconv_mean{rline})); + plot(h_axes(2), lp(spec.deconv_mean{rline}) - max_dm) + hold(h_axes(2),'on'); + [max_val,max_idx] = max(lp(spec.deconv_sample{rline})); + plot(h_axes(2), circshift(lp(spec.deconv_sample{rline}) - max_val,[-max_idx 1])) + plot(h_axes(2), lp(spec.deconv_std{rline},2) - lp(cmd.rlines) - max_dm) + h_filled = [h_nonnegative; zeros(length(spec.deconv_sample{rline})-length(h_nonnegative)-length(h_negative),1); h_negative]; + plot(h_axes(2), lp(h_filled) - max_dm) + xlabel(h_axes(2), 'Range bin'); + ylabel(h_axes(2), 'Relative power (dB)'); + title(h_axes(2), sprintf('Impulse response rising edge (rline %d of %d)',rline, length(spec.deconv_gps_time))); + legend(h_axes(2), 'mean','sample','std','h','location','best'); + Nt = length(spec.deconv_mean{rline}); + xlim(h_axes(2), [Nt+2*param.collate_deconv.rbins{img}(1) Nt]); + ylim(h_axes(2), [-debug_ylim 0]); + grid(h_axes(2), 'on'); + + clf(h_fig(3)); + set(h_fig(3),'Name','SNR falling edge'); + h_axes(3) = axes('parent',h_fig(3)); + plot(h_axes(3),SNR) + hold(h_axes(3),'on'); + plot(h_axes(3),[1 length(SNR)], 20*[1 1],'k--'); + plot(h_axes(3),param.collate_deconv.rbins{img}(2)*[1 1], [0 debug_ylim],'k--'); + title(h_axes(3),sprintf('SNR falling edge (rline %d of %d)',rline, length(spec.deconv_gps_time))); + xlabel(h_axes(3),'Range bin'); + ylabel(h_axes(3),'SNR (dB)'); + ylim(h_axes(3), [0 debug_ylim]); + grid(h_axes(3), 'on'); + + clf(h_fig(4)); + set(h_fig(4),'Name','SNR rising edge'); + h_axes(4) = axes('parent',h_fig(4)); + plot(h_axes(4),SNR) + hold(h_axes(4),'on'); + plot(h_axes(4),[1 length(SNR)], 20*[1 1],'k--'); + plot(h_axes(4),(Nt+param.collate_deconv.rbins{img}(1))*[1 1], [0 debug_ylim],'k--'); + title(h_axes(4),sprintf('SNR rising edge (rline %d of %d)',rline, length(spec.deconv_gps_time))); + xlabel(h_axes(4),'Range bin'); + ylabel(h_axes(4),'SNR (dB)'); + ylim(h_axes(4), [0 debug_ylim]); + grid(h_axes(4), 'on'); + + linkaxes(h_axes([1 3]),'x'); + linkaxes(h_axes([2 4]),'x'); + + if any(strcmp('visible',param.collate_deconv.debug_plots)) + keyboard + end +end + +%% Stage 1: Test deconvolution + +% Create frequency axis +Nt = length(spec.deconv_sample{rline}); + +% Get sample rline parameters +fc = spec.deconv_fc(rline); +t0 = spec.deconv_t0(rline); +dt = spec.dt; +time = t0 + dt*(0:Nt-1).'; + +% Adjust deconvolution signal to match sample rline +h_filled = [h_nonnegative; zeros(length(spec.deconv_sample{rline})-length(h_nonnegative)-length(h_negative),1); h_negative]; +deconv_Nt = length(h_filled); +% Is dt different? Error +if dt ~= spec.dt + error('The fast time sample spacing of the data (%g) does not match the deconvolution waveform sampling (%g).',dt,spec.dt); +end +% Is fc different? Multiply time domain by exp(1i*2*pi*dfc*deconv_time) +dfc = fc - spec.deconv_fc(rline); +if dfc/fc > 1e-6 + deconv_time = t0 + dt*(0:Nt-1).'; + h_filled = h_filled .* exp(1i*2*pi*dfc*deconv_time); +end +% Take FFT of deconvolution impulse response +h_filled = fft(h_filled); + +% Create inverse filter relative to window +df = 1/(Nt*dt); +freq = spec.deconv_fc(rline) + df * ifftshift(-floor(Nt/2) : floor((Nt-1)/2)).'; +freq = fftshift(freq); +Nt_shorten = find(param.collate_deconv.f0 <= freq,1); +Nt_shorten(2) = length(freq) - find(param.collate_deconv.f1 >= freq,1,'last'); +Nt_Hwind = Nt - sum(Nt_shorten); +Hwind = deconv.ref_window(Nt_Hwind); +Hwind_filled = ifftshift([zeros(Nt_shorten(1),1); Hwind; zeros(Nt_shorten(end),1)]); +if ~param.collate_deconv.magnitude_only.en + % Apply regular inverse filter (default) + h_filled_inverse = Hwind_filled ./ h_filled; +else + % Apply filtered magnitude-only correction + h_filled_abs_filt = sgolayfilt(double(abs(h_filled)),2,round(param.collate_deconv.magnitude_only.f_cutoff*Nt_Hwind)*2+1); + if 0 + % For debugging + clf; + plot(lp(h_filled)) + hold on + plot(lp(h_filled_abs_filt)); + keyboard + end + h_filled_inverse = Hwind_filled ./ h_filled_abs_filt; +end + +% Apply deconvolution with unnormalized filter +h_deconvolved = ifft(fft(spec.deconv_sample{rline}) .* h_filled_inverse); +% Oversample to get a good measurement of the peak value +h_deconvolved = interpft(h_deconvolved,Mt*Nt); + +% Create ideal response (follow same steps as h_deconvolved) +h_ideal = deconv.ref_window(Nt_Hwind); +h_ideal = ifftshift([zeros(Nt_shorten(1),1); h_ideal; zeros(Nt_shorten(end),1)]); +h_ideal = ifft(h_ideal); +h_ideal = interpft(h_ideal,Mt*Nt); + +% Oversample sample signal by the same amount as the deconvolved +% signal +h_sample = interpft(spec.deconv_sample{rline},Mt*Nt); + +% Scale so that the peak value matches between deconvolved and +% sample (this h_mult_factor is used in data_pulse_compress) +h_mult_factor = max(abs(h_sample)) / max(abs(h_deconvolved)); +h_filled_inverse = h_filled_inverse * h_mult_factor; +h_deconvolved = h_deconvolved * h_mult_factor; + +% Find maximum values and indices for the deconvolved and +% undeconvolved signals. +[ideal_max_val,ideal_max_idx] = max(h_ideal); +[deconv_max_val,deconv_max_idx] = max(h_deconvolved); +[max_val,max_idx] = max(h_sample); + +%% Stage 1: Plot h_deconvolved, f0, f1, SL_guard_bins +if deconv_plot + + comp_bins = param.collate_deconv.rbins{img}(2)+1:2*param.collate_deconv.rbins{img}(2); + comp_bins = Nt+2*param.collate_deconv.rbins{img}(1) : Nt+param.collate_deconv.rbins{img}; + dnoise = lp(mean(abs(h_sample(comp_bins)).^2) ./ mean(abs(h_deconvolved(comp_bins)).^2)); + dsignal = lp(max(abs(h_sample).^2) ./ max(abs(h_deconvolved).^2)); + dSNR = dsignal - dnoise; + + fprintf('Deconvolved response relative to regular response:\n'); + fprintf(' SNR loss: %.1f dB\n', dSNR) + fprintf(' Peak magnitude offset: %.1f (dB)\n', lp(max_val./deconv_max_val)); + fprintf(' Peak angle offset: %.1f (deg)\n', angle(max_val./deconv_max_val) * 180/pi); + fprintf(' Index offset: %d\n', mod(max_idx - deconv_max_idx + Nt*Mt/2, Nt*Mt)-Nt*Mt/2); + + bins_Mt = 0:1/Mt:Nt-1/Mt; + + clf(h_fig(1)); + set(h_fig(1),'Name','Impulse response falling edge'); + h_axes = axes('parent',h_fig(1)); + [max_val,max_idx] = max(lp(h_sample,2)); + plot(h_axes(1), bins_Mt, circshift(lp(h_sample) - max_val,[-max_idx 1])) + hold(h_axes(1),'on'); + plot(h_axes(1), bins_Mt, circshift(lp(h_deconvolved) - max_val + dsignal,[-max_idx 1])) + plot(h_axes(1), bins_Mt, circshift(lp(h_ideal) - lp(ideal_max_val,2),[-ideal_max_idx 1])) + plot(h_axes(1), param.collate_deconv.SL_guard_bins*[1 1], [-debug_ylim 0], 'k--'); + xlabel(h_axes(1), 'Range bin'); + ylabel(h_axes(1), 'Relative power (dB)'); + title(h_axes(1), sprintf('Impulse response falling edge (frm %d-rec %d-rline %d) %d-%d',floor(deconv.frm(rline)),deconv.rec(rline),rline,wf,adc)); + legend(h_axes(1), 'sample','deconvolved','ideal','location','best'); + xlim(h_axes(1), [0 2*param.collate_deconv.rbins{img}(2)]); + ylim(h_axes(1), [-debug_ylim 0]); + grid(h_axes(1), 'on'); + + clf(h_fig(2)); + set(h_fig(2),'Name','Impulse response rising edge'); + h_axes(2) = axes('parent',h_fig(2)); + [max_val,max_idx] = max(lp(h_sample,2)); + plot(h_axes(2), fliplr(bins_Mt+1), circshift(lp(h_sample) - max_val,[-max_idx 1])) + hold(h_axes(2),'on'); + plot(h_axes(2), fliplr(bins_Mt+1), circshift(lp(h_deconvolved) - max_val + dsignal,[-max_idx 1])) + plot(h_axes(2), fliplr(bins_Mt+1), circshift(lp(h_ideal) - lp(ideal_max_val,2),[-ideal_max_idx 1])) + plot(h_axes(2), (1+param.collate_deconv.SL_guard_bins)*[1 1], [-debug_ylim 0], 'k--'); + xlabel(h_axes(2), 'Range bin'); + ylabel(h_axes(2), 'Relative power (dB)'); + title(h_axes(2), sprintf('Impulse response rising edge (frm %d-rec %d-rline %d) %d-%d',floor(deconv.frm(rline)),deconv.rec(rline),rline,wf,adc)); + legend(h_axes(2), 'sample','deconvolved','ideal','location','best'); + xlim(h_axes(2), [0 -2*param.collate_deconv.rbins{img}(1)]); + ylim(h_axes(2), [-debug_ylim 0]); + grid(h_axes(2), 'on'); + + clf(h_fig(3)); + set(h_fig(3),'Name','Transfer function'); + h_axes(3) = axes('parent',h_fig(3)); + plot(h_axes(3), lp(h_filled)) + hold(h_axes(3),'on'); + plot(h_axes(3), lp(Hwind_filled,1) - max(lp(Hwind_filled,1)) + max(lp(h_filled))) + plot(h_axes(3), lp(h_filled_inverse) - max(lp(h_filled_inverse)) + max(lp(h_filled))) + xlabel(h_axes(3), 'Frequency bin'); + ylabel(h_axes(3), 'Relative power (dB)'); + % Title gives frame, record, and range line for this deconvolution + % waveform. It also gives the radiometric error. + title(h_axes(3), sprintf('Transfer function (frm %d-rec %d-rline %d) %.1f/%.1f dB %d-%d',floor(deconv.frm(rline)),deconv.rec(rline),rline,deconv.radiometric_error_dB(rline),max(deconv.radiometric_error_dB),wf,adc)); + legend(h_axes(3), 'sample','window','inverse','location','best'); + grid(h_axes(3), 'on'); + + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_falling_wf_%02d_adc_%02d',param.collate_deconv.out_path,wf,adc)) '.fig']; + fprintf('Saving %s\n', fig_fn); + fig_fn_dir = fileparts(fig_fn); + if ~exist(fig_fn_dir,'dir') + mkdir(fig_fn_dir); + end + ct_saveas(h_fig(1),fig_fn); + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_falling_wf_%02d_adc_%02d',param.collate_deconv.out_path,wf,adc)) '.jpg']; + fprintf('Saving %s\n', fig_fn); + ct_saveas(h_fig(1),fig_fn); + + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_rising_wf_%02d_adc_%02d',param.collate_deconv.out_path,wf,adc)) '.fig']; + fprintf('Saving %s\n', fig_fn); + fig_fn_dir = fileparts(fig_fn); + if ~exist(fig_fn_dir,'dir') + mkdir(fig_fn_dir); + end + ct_saveas(h_fig(2),fig_fn); + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_rising_wf_%02d_adc_%02d',param.collate_deconv.out_path,wf,adc)) '.jpg']; + fprintf('Saving %s\n', fig_fn); + ct_saveas(h_fig(2),fig_fn); + + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_transfer_func_wf_%02d_adc_%02d',param.collate_deconv.out_path,wf,adc)) '.fig']; + fprintf('Saving %s\n', fig_fn); + fig_fn_dir = fileparts(fig_fn); + if ~exist(fig_fn_dir,'dir') + mkdir(fig_fn_dir); + end + ct_saveas(h_fig(3),fig_fn); + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_transfer_func_wf_%02d_adc_%02d',param.collate_deconv.out_path,wf,adc)) '.jpg']; + fprintf('Saving %s\n', fig_fn); + ct_saveas(h_fig(3),fig_fn); + + if any(strcmp('visible',param.collate_deconv.debug_plots)) + keyboard + end +end diff --git a/cresis-toolbox/processing/update_collate_deconv.m b/cresis-toolbox/processing/collate_deconv_update.m similarity index 75% rename from cresis-toolbox/processing/update_collate_deconv.m rename to cresis-toolbox/processing/collate_deconv_update.m index f4de85b2..56385bbe 100644 --- a/cresis-toolbox/processing/update_collate_deconv.m +++ b/cresis-toolbox/processing/collate_deconv_update.m @@ -1,11 +1,11 @@ -function update_collate_deconv(param,param_override) -% update_collate_deconv(param,param_override) +function collate_deconv_update(param,param_override) +% collate_deconv_update(param,param_override) % % This scripts updates the collate_deconv output files. It allows one to % delete, add, and update waveforms to the final deconv file. % % Example: -% See run_update_collate_deconv.m to run. +% See run_collate_deconv_update.m to run. % % Author: John Paden @@ -24,84 +24,84 @@ function update_collate_deconv(param,param_override) [output_dir,radar_type,radar_name] = ct_output_dir(param.radar_name); -% param.update_collate_deconv structure +% param.collate_deconv_update structure % ========================================================================= -if ~isfield(param.update_collate_deconv,'abs_metric') || isempty(param.update_collate_deconv.abs_metric) - param.update_collate_deconv.abs_metric = [58 5 -25 -35 inf inf]; +if ~isfield(param.collate_deconv_update,'abs_metric') || isempty(param.collate_deconv_update.abs_metric) + param.collate_deconv_update.abs_metric = [58 5 -25 -35 inf inf]; end -if ~isfield(param.update_collate_deconv,'debug_plots') - param.update_collate_deconv.debug_plots = {'final'}; - % param.update_collate_deconv.debug_plots = {'final','visible'}; +if ~isfield(param.collate_deconv_update,'debug_plots') + param.collate_deconv_update.debug_plots = {'final'}; + % param.collate_deconv_update.debug_plots = {'final','visible'}; end -if ~isfield(param.update_collate_deconv,'delete_existing') || isempty(param.update_collate_deconv.delete_existing) - param.update_collate_deconv.delete_existing = false; +if ~isfield(param.collate_deconv_update,'delete_existing') || isempty(param.collate_deconv_update.delete_existing) + param.collate_deconv_update.delete_existing = false; end -if ~isfield(param.update_collate_deconv,'gps_time_penalty') || isempty(param.update_collate_deconv.gps_time_penalty) - param.update_collate_deconv.gps_time_penalty = 1/(10*24*3600); +if ~isfield(param.collate_deconv_update,'gps_time_penalty') || isempty(param.collate_deconv_update.gps_time_penalty) + param.collate_deconv_update.gps_time_penalty = 1/(10*24*3600); end if ~isfield(param.analysis,'imgs') || isempty(param.analysis.imgs) param.analysis.imgs = {[1 1]}; end -if ~isfield(param.update_collate_deconv,'imgs') || isempty(param.update_collate_deconv.imgs) - param.update_collate_deconv.imgs = 1:length(param.analysis.imgs); +if ~isfield(param.collate_deconv_update,'imgs') || isempty(param.collate_deconv_update.imgs) + param.collate_deconv_update.imgs = 1:length(param.analysis.imgs); end -if ~isfield(param.update_collate_deconv,'in_path') || isempty(param.update_collate_deconv.in_path) - param.update_collate_deconv.in_path = 'analysis'; +if ~isfield(param.collate_deconv_update,'in_path') || isempty(param.collate_deconv_update.in_path) + param.collate_deconv_update.in_path = 'analysis'; end -if ~isfield(param.update_collate_deconv,'metric_weights') || isempty(param.update_collate_deconv.metric_weights) - param.update_collate_deconv.metric_weights = [0.5 0 3 5 0 0]; +if ~isfield(param.collate_deconv_update,'metric_weights') || isempty(param.collate_deconv_update.metric_weights) + param.collate_deconv_update.metric_weights = [0.5 0 3 5 0 0]; end -error_mask = isinf(param.update_collate_deconv.abs_metric) & param.update_collate_deconv.metric_weights ~= 0; +error_mask = isinf(param.collate_deconv_update.abs_metric) & param.collate_deconv_update.metric_weights ~= 0; if any(error_mask) warning('Fields set to inf in abs_metric must be set to 0 in metric_weights. Setting these to 0 now.'); - param.update_collate_deconv.metric_weights(error_mask) = 0; + param.collate_deconv_update.metric_weights(error_mask) = 0; end -if ~isfield(param.update_collate_deconv,'min_score') || isempty(param.update_collate_deconv.min_score) - param.update_collate_deconv.min_score = -10; +if ~isfield(param.collate_deconv_update,'min_score') || isempty(param.collate_deconv_update.min_score) + param.collate_deconv_update.min_score = -10; end -if ~isfield(param.update_collate_deconv,'out_path') || isempty(param.update_collate_deconv.out_path) - param.update_collate_deconv.out_path = param.update_collate_deconv.in_path; +if ~isfield(param.collate_deconv_update,'out_path') || isempty(param.collate_deconv_update.out_path) + param.collate_deconv_update.out_path = param.collate_deconv_update.in_path; end -if ~isfield(param.update_collate_deconv,'twtt_penalty') || isempty(param.update_collate_deconv.twtt_penalty) +if ~isfield(param.collate_deconv_update,'twtt_penalty') || isempty(param.collate_deconv_update.twtt_penalty) if strcmpi(radar_type,'deramp') - param.update_collate_deconv.twtt_penalty = 1e6; + param.collate_deconv_update.twtt_penalty = 1e6; else % No twtt dependence - param.update_collate_deconv.twtt_penalty = 1e-6; + param.collate_deconv_update.twtt_penalty = 1e-6; end end -if ~isfield(param.update_collate_deconv,'wf_adcs') || isempty(param.update_collate_deconv.wf_adcs) - param.update_collate_deconv.wf_adcs = []; +if ~isfield(param.collate_deconv_update,'wf_adcs') || isempty(param.collate_deconv_update.wf_adcs) + param.collate_deconv_update.wf_adcs = []; end % Other Setup % ========================================================================= -if ~isempty(param.update_collate_deconv.debug_plots) - h_fig = get_figures(3,any(strcmp('visible',param.update_collate_deconv.debug_plots)),'collate_deconv'); +if ~isempty(param.collate_deconv_update.debug_plots) + h_fig = get_figures(3,any(strcmp('visible',param.collate_deconv_update.debug_plots)),'collate_deconv'); end %% Process commands % ===================================================================== tmp_param = param; -for img = param.update_collate_deconv.imgs +for img = param.collate_deconv_update.imgs - if isempty(param.update_collate_deconv.wf_adcs) + if isempty(param.collate_deconv_update.wf_adcs) % If no wf-adc pairs specified, then do them all. wf_adcs = 1:size(param.analysis.imgs{img},1); else - wf_adcs = param.update_collate_deconv.wf_adcs; + wf_adcs = param.collate_deconv_update.wf_adcs; end for wf_adc = wf_adcs wf = param.analysis.imgs{img}(wf_adc,1); @@ -109,13 +109,14 @@ function update_collate_deconv(param,param_override) % Load input % =================================================================== - fn_dir = fileparts(ct_filename_out(param,param.update_collate_deconv.in_path, '')); + fn_dir = fileparts(ct_filename_out(param,param.collate_deconv_update.in_path, '')); fn = fullfile(fn_dir,sprintf('deconv_%s_wf_%d_adc_%d.mat', param.day_seg, wf, adc)); fprintf('\n==============================================================\n'); fprintf('Loading %s img %d wf %d adc %d\n %s\n', param.day_seg, img, wf, adc, fn); - if param.update_collate_deconv.delete_existing + if param.collate_deconv_update.delete_existing delete(fn); end + % Load existing waveforms for this segment if exist(fn,'file') deconv = load(fn,'param_collate_deconv','param_analysis','param_records'); if ~isfield(deconv,'param_collate_deconv') || isempty(deconv.param_collate_deconv) ... @@ -138,18 +139,18 @@ function update_collate_deconv(param,param_override) % Execute update commands % =================================================================== - for cmd_idx = 1:length(param.update_collate_deconv.cmd) - switch lower(param.update_collate_deconv.cmd{cmd_idx}.method) + for cmd_idx = 1:length(param.collate_deconv_update.cmd) + switch lower(param.collate_deconv_update.cmd{cmd_idx}.method) case 'delete' - fprintf('%s:', param.update_collate_deconv.cmd{cmd_idx}.method); - fprintf(' %d', param.update_collate_deconv.cmd{cmd_idx}.idxs); + fprintf('%s:', param.collate_deconv_update.cmd{cmd_idx}.method); + fprintf(' %d', param.collate_deconv_update.cmd{cmd_idx}.idxs); fprintf('\n'); if new_file error('Delete not allowed on segments that do not have a deconv file or any waveforms loaded.'); end - deconv_idxs = setdiff(1:length(deconv_lib.gps_time), param.update_collate_deconv.cmd{cmd_idx}.idxs); + deconv_idxs = setdiff(1:length(deconv_lib.gps_time), param.collate_deconv_update.cmd{cmd_idx}.idxs); deconv_lib.elev = deconv_lib.elev(deconv_idxs); deconv_lib.fc = deconv_lib.fc(deconv_idxs); deconv_lib.frm = deconv_lib.frm(deconv_idxs); @@ -170,27 +171,27 @@ function update_collate_deconv(param,param_override) deconv_lib.twtt = deconv_lib.twtt(deconv_idxs); case {'add','replace'} - for seg_idx = 1:length(param.update_collate_deconv.cmd{cmd_idx}.day_seg) - fprintf('%s: ', param.update_collate_deconv.cmd{cmd_idx}.method); - fprintf(' (%s,', param.update_collate_deconv.cmd{cmd_idx}.day_seg{seg_idx}); - fprintf(' %d', param.update_collate_deconv.cmd{cmd_idx}.idxs{seg_idx}); + for seg_idx = 1:length(param.collate_deconv_update.cmd{cmd_idx}.day_seg) + fprintf('%s: ', param.collate_deconv_update.cmd{cmd_idx}.method); + fprintf(' (%s,', param.collate_deconv_update.cmd{cmd_idx}.day_seg{seg_idx}); + fprintf(' %d', param.collate_deconv_update.cmd{cmd_idx}.idxs{seg_idx}); fprintf(')'); end fprintf('\n'); - for seg_idx = 1:length(param.update_collate_deconv.cmd{cmd_idx}.day_seg) - if ~isfield(param.update_collate_deconv.cmd{cmd_idx},'wf_adcs_map') ... - || numel(param.update_collate_deconv.cmd{cmd_idx}.wf_adcs_map) < seg_idx ... - || isempty(param.update_collate_deconv.cmd{cmd_idx}.wf_adcs_map{seg_idx}) + for seg_idx = 1:length(param.collate_deconv_update.cmd{cmd_idx}.day_seg) + if ~isfield(param.collate_deconv_update.cmd{cmd_idx},'wf_adcs_map') ... + || numel(param.collate_deconv_update.cmd{cmd_idx}.wf_adcs_map) < seg_idx ... + || isempty(param.collate_deconv_update.cmd{cmd_idx}.wf_adcs_map{seg_idx}) wf_map = wf; adc_map = adc; else - wf_map = param.update_collate_deconv.cmd{cmd_idx}.wf_adcs_map{seg_idx}{img}(wf_adc,1); - adc_map = param.update_collate_deconv.cmd{cmd_idx}.wf_adcs_map{seg_idx}{img}(wf_adc,2); + wf_map = param.collate_deconv_update.cmd{cmd_idx}.wf_adcs_map{seg_idx}{img}(wf_adc,1); + adc_map = param.collate_deconv_update.cmd{cmd_idx}.wf_adcs_map{seg_idx}{img}(wf_adc,2); end - tmp_param.day_seg = param.update_collate_deconv.cmd{cmd_idx}.day_seg{seg_idx}; - fn_dir = fileparts(ct_filename_out(tmp_param,param.update_collate_deconv.in_path, '')); + tmp_param.day_seg = param.collate_deconv_update.cmd{cmd_idx}.day_seg{seg_idx}; + fn_dir = fileparts(ct_filename_out(tmp_param,param.collate_deconv_update.in_path, '')); fn = fullfile(fn_dir,sprintf('deconv_%s_wf_%d_adc_%d.mat', tmp_param.day_seg, wf_map, adc_map)); fprintf(' Loading %s img %d wf %d adc %d\n %s\n', tmp_param.day_seg, img, wf_map, adc_map, fn); tmp_deconv{cmd_idx}{seg_idx} = load(fn); @@ -216,23 +217,22 @@ function update_collate_deconv(param,param_override) deconv_lib.ref_nonnegative = []; deconv_lib.roll = []; deconv_lib.twtt = []; - - deconv.param_collate_deconv = tmp_deconv{cmd_idx}{seg_idx}.param_collate_deconv; - deconv.param_analysis = tmp_deconv{cmd_idx}{seg_idx}.param_analysis; - deconv.param_records = tmp_deconv{cmd_idx}{seg_idx}.param_records; end if isfield(deconv_lib,'dt') && abs(tmp_deconv{cmd_idx}{seg_idx}.dt - deconv_lib.dt)/deconv_lib.dt > 1e-6 error('Time bins do not align %g ~= deconv_lib.dt == %g', tmp_deconv{cmd_idx}{seg_idx}.dt, deconv_lib.dt); end + deconv.param_collate_deconv = tmp_deconv{cmd_idx}{seg_idx}.param_collate_deconv; + deconv.param_analysis = tmp_deconv{cmd_idx}{seg_idx}.param_analysis; + deconv.param_records = tmp_deconv{cmd_idx}{seg_idx}.param_records; % Make sure selected waveforms to add/replace exist - param.update_collate_deconv.cmd{cmd_idx}.idxs{seg_idx} ... - = intersect(param.update_collate_deconv.cmd{cmd_idx}.idxs{seg_idx}, ... + param.collate_deconv_update.cmd{cmd_idx}.idxs{seg_idx} ... + = intersect(param.collate_deconv_update.cmd{cmd_idx}.idxs{seg_idx}, ... 1:length(tmp_deconv{cmd_idx}{seg_idx}.gps_time)); - new_idxs = param.update_collate_deconv.cmd{cmd_idx}.idxs{seg_idx}; - if strcmpi(param.update_collate_deconv.cmd{cmd_idx}.method,'add') + new_idxs = param.collate_deconv_update.cmd{cmd_idx}.idxs{seg_idx}; + if strcmpi(param.collate_deconv_update.cmd{cmd_idx}.method,'add') deconv_lib.elev = [deconv_lib.elev tmp_deconv{cmd_idx}{seg_idx}.elev(new_idxs)]; deconv_lib.fc = [deconv_lib.fc tmp_deconv{cmd_idx}{seg_idx}.fc(new_idxs)]; deconv_lib.frm = [deconv_lib.frm tmp_deconv{cmd_idx}{seg_idx}.frm(new_idxs)]; @@ -295,8 +295,8 @@ function update_collate_deconv(param,param_override) layer.elev = layer.elev(decim_idxs); % 4. Compare results to metric - pass = bsxfun(@lt,deconv_lib.metric,param.update_collate_deconv.abs_metric(:)); - score = nansum(bsxfun(@times, param.update_collate_deconv.metric_weights(:), bsxfun(@minus, param.update_collate_deconv.abs_metric(:), deconv_lib.metric))); + pass = bsxfun(@lt,deconv_lib.metric,param.collate_deconv_update.abs_metric(:)); + score = nansum(bsxfun(@times, param.collate_deconv_update.metric_weights(:), bsxfun(@minus, param.collate_deconv_update.abs_metric(:), deconv_lib.metric))); score(:,any(isnan(deconv_lib.metric))) = nan; % 5. Find best score at each point along the flight track @@ -307,16 +307,16 @@ function update_collate_deconv(param,param_override) % Score with twtt penalty and time constant penalty term d_twtt = layer.twtt(rline) - deconv_lib.twtt; d_gps_time = layer.gps_time(rline) - deconv_lib.gps_time; - %adjusted_score = min_score + score .* exp(-abs(param.update_collate_deconv.twtt_penalty*d_twtt).^2) ... - % .* exp(-abs(param.update_collate_deconv.gps_time_penalty*d_gps_time).^2); - adjusted_score = min_score + score - (100-100*exp(-abs(param.update_collate_deconv.twtt_penalty*d_twtt).^2)) ... - - (50-50*exp(-abs(param.update_collate_deconv.gps_time_penalty*d_gps_time).^2)); + %adjusted_score = min_score + score .* exp(-abs(param.collate_deconv_update.twtt_penalty*d_twtt).^2) ... + % .* exp(-abs(param.collate_deconv_update.gps_time_penalty*d_gps_time).^2); + adjusted_score = min_score + score - (100-100*exp(-abs(param.collate_deconv_update.twtt_penalty*d_twtt).^2)) ... + - (50-50*exp(-abs(param.collate_deconv_update.gps_time_penalty*d_gps_time).^2)); [max_score(rline),max_idx(rline)] = max(adjusted_score); unadjusted_score(rline) = min_score + score(max_idx(rline)); end - if any(max_score < param.update_collate_deconv.min_score) - warning('Score is too low for %d of %d blocks of range lines.', sum(max_score < param.update_collate_deconv.min_score), length(max_score)); + if any(max_score < param.collate_deconv_update.min_score) + warning('Score is too low for %d of %d blocks of range lines.', sum(max_score < param.collate_deconv_update.min_score), length(max_score)); end [max_idxs,~,max_idxs_mapping] = unique(max_idx); @@ -364,7 +364,7 @@ function update_collate_deconv(param,param_override) end % 7. Plot final deconv waveforms - if any(strcmp('final',param.update_collate_deconv.debug_plots)) + if any(strcmp('final',param.collate_deconv_update.debug_plots)) % TWTT Figure % =================================================================== clf(h_fig(1)); @@ -388,14 +388,14 @@ function update_collate_deconv(param,param_override) legend(h_axes(1), h_plot, legend_str,'location','best'); grid(h_axes(1), 'on'); - fig_fn = [ct_filename_ct_tmp(param,'','collate_deconv',sprintf('%s_twtt_wf_%02d_adc_%02d',param.update_collate_deconv.out_path,wf,adc)) '.fig']; + fig_fn = [ct_filename_ct_tmp(param,'','collate_deconv',sprintf('%s_twtt_wf_%02d_adc_%02d',param.collate_deconv_update.out_path,wf,adc)) '.fig']; fprintf('Saving %s\n', fig_fn); fig_fn_dir = fileparts(fig_fn); if ~exist(fig_fn_dir,'dir') mkdir(fig_fn_dir); end ct_saveas(h_fig(1),fig_fn); - fig_fn = [ct_filename_ct_tmp(param,'','collate_deconv',sprintf('%s_twtt_wf_%02d_adc_%02d',param.update_collate_deconv.out_path,wf,adc)) '.jpg']; + fig_fn = [ct_filename_ct_tmp(param,'','collate_deconv',sprintf('%s_twtt_wf_%02d_adc_%02d',param.collate_deconv_update.out_path,wf,adc)) '.jpg']; fprintf('Saving %s\n', fig_fn); ct_saveas(h_fig(1),fig_fn); @@ -423,10 +423,10 @@ function update_collate_deconv(param,param_override) legend(h_axes(2), h_plot, legend_str,'location','best'); grid(h_axes(2), 'on'); - fig_fn = [ct_filename_ct_tmp(param,'','collate_deconv',sprintf('%s_score_wf_%02d_adc_%02d',param.update_collate_deconv.out_path,wf,adc)) '.fig']; + fig_fn = [ct_filename_ct_tmp(param,'','collate_deconv',sprintf('%s_score_wf_%02d_adc_%02d',param.collate_deconv_update.out_path,wf,adc)) '.fig']; fprintf('Saving %s\n', fig_fn); ct_saveas(h_fig(2),fig_fn); - fig_fn = [ct_filename_ct_tmp(param,'','collate_deconv',sprintf('%s_score_wf_%02d_adc_%02d',param.update_collate_deconv.out_path,wf,adc)) '.jpg']; + fig_fn = [ct_filename_ct_tmp(param,'','collate_deconv',sprintf('%s_score_wf_%02d_adc_%02d',param.collate_deconv_update.out_path,wf,adc)) '.jpg']; fprintf('Saving %s\n', fig_fn); ct_saveas(h_fig(2),fig_fn); @@ -440,6 +440,7 @@ function update_collate_deconv(param,param_override) h_axes(4) = subplot(5,1,3:5,'parent',h_fig(3)); legend_str = {}; + max_val_overall = -inf; for idx = 1:length(final.gps_time) % Get the reference function h_nonnegative = final.ref_nonnegative{idx}; @@ -463,6 +464,10 @@ function update_collate_deconv(param,param_override) h_filled_phase = unwrap(h_filled_phase)*180/pi; h_filled_phase = h_filled_phase - h_filled_phase(fc_idx); + if max(h_filled_lp) > max_val_overall + max_val_overall = max(h_filled_lp); + end + if max(freq) > 2e9 freq_scale = 1e9; else @@ -472,9 +477,13 @@ function update_collate_deconv(param,param_override) hold(h_axes(3),'on'); plot(h_axes(4), freq/freq_scale, h_filled_phase); hold(h_axes(4),'on'); - legend_str{idx} = sprintf('%d %s_%03.2f %4.0f %4.2fus',idx, ... - final.map_day_seg{idx},final.frm(idx),round(lp(final.ref_nonnegative{idx}(1),2)), ... - round(final.twtt(idx)*1e7)/10); + radiometric_error_dB = lp(1/(c*final.twtt(idx)).^2) - lp(final.ref_nonnegative{idx}(1)); + legend_str{idx} = sprintf('%d %s_%03.0f %7.0f %4.0fdB %4.1fus',idx, ... + final.map_day_seg{idx},floor(final.frm(idx)),final.rec(idx),radiometric_error_dB, ... + final.twtt(idx)*1e6); + end + if isfinite(max_val_overall) + ylim(h_axes(3), max_val_overall + [-31 1]); end if freq_scale == 1e9 @@ -484,7 +493,7 @@ function update_collate_deconv(param,param_override) end ylabel(h_axes(3), 'Relative power (dB)'); ylabel(h_axes(4), 'Relative angle (deg)'); - title(h_axes(3), regexprep(sprintf('%s (Legend idx:frm:peak:twtt)', param.day_seg),'_','\\_')); + title(h_axes(3), regexprep(sprintf('%s (Legend idx:frm:rec:radio:twtt)', param.day_seg),'_','\\_')); grid(h_axes(3), 'on'); grid(h_axes(4), 'on'); h_legend = legend(h_axes(3), legend_str, 'location', 'northeastoutside', 'interpreter','none'); @@ -493,17 +502,17 @@ function update_collate_deconv(param,param_override) pos4 = get(h_axes(4),'Position'); set(h_axes(4),'Position',[pos4(1:2) pos3(3) pos4(4)]); - fig_fn = [ct_filename_ct_tmp(param,'','collate_deconv',sprintf('%s_transfer_func_wf_%02d_adc_%02d',param.update_collate_deconv.out_path,wf,adc)) '.fig']; + fig_fn = [ct_filename_ct_tmp(param,'','collate_deconv',sprintf('%s_final_transfer_func_wf_%02d_adc_%02d',param.collate_deconv_update.out_path,wf,adc)) '.fig']; fprintf('Saving %s\n', fig_fn); ct_saveas(h_fig(3),fig_fn); - fig_fn = [ct_filename_ct_tmp(param,'','collate_deconv',sprintf('%s_transfer_func_wf_%02d_adc_%02d',param.update_collate_deconv.out_path,wf,adc)) '.jpg']; + fig_fn = [ct_filename_ct_tmp(param,'','collate_deconv',sprintf('%s_final_transfer_func_wf_%02d_adc_%02d',param.collate_deconv_update.out_path,wf,adc)) '.jpg']; fprintf('Saving %s\n', fig_fn); ct_saveas(h_fig(3),fig_fn); end % Save outputs % =================================================================== - fn_dir = fileparts(ct_filename_out(param,param.update_collate_deconv.out_path, '')); + fn_dir = fileparts(ct_filename_out(param,param.collate_deconv_update.out_path, '')); if ~exist(fn_dir,'dir') mkdir(fn_dir); end diff --git a/cresis-toolbox/processing/collate_equal.m b/cresis-toolbox/processing/collate_equal.m index 70b8d087..e0ba3612 100644 --- a/cresis-toolbox/processing/collate_equal.m +++ b/cresis-toolbox/processing/collate_equal.m @@ -22,34 +22,68 @@ param.collate_equal = []; end +% chan_eq_en: logical scalar. Default is true. If true, channel +% equalization with current parameter spreadsheet values is applied. +% Includes Tadc_adjust, Tsys, chan_equal_dB, and chan_equal_deg. if ~isfield(param.collate_equal,'chan_eq_en') || isempty(param.collate_equal.chan_eq_en) param.collate_equal.chan_eq_en = true; end +% .cmd_idx: positive scalar. Index into the analysis.cmd cell array. +% Specifies which analysis output files will be loaded for equalization. if ~isfield(param.collate_equal,'cmd_idx') || isempty(param.collate_equal.cmd_idx) param.collate_equal.cmd_idx = 1; end -cmd = param.analysis.cmd{param.collate_equal.cmd_idx}; + +% .debug_out_dir: string containing the output folder name to use for the +% debug outputs. This is input to ct_filename_ct_tmp(). +if ~isfield(param.collate_equal,'debug_out_dir') || isempty(param.collate_equal.debug_out_dir) + param.collate_equal.debug_out_dir = 'collate_equal'; +end +debug_out_dir = param.collate_equal.debug_out_dir; + +% .debug_out_fn: string containing a word to insert into the output file +% name to identify files for this specific run. +if ~isfield(param.collate_equal,'debug_out_fn') || isempty(param.collate_equal.debug_out_fn) + param.collate_equal.debug_out_fn = 'analysis'; +end if ~isfield(param.collate_equal,'debug_plots') param.collate_equal.debug_plots = {'before_comp','after_comp','surf','final','visible','comp_image'}; end enable_visible_plot = any(strcmp('visible',param.collate_equal.debug_plots)); +% .delay: strucure controlling how the equalization coefficients are +% estimated. if ~isfield(param.collate_equal,'delay') || isempty(param.collate_equal.delay) param.collate_equal.delay = []; end +% .delay.method: string containing the method name. Default is +% 'xcorr_complex'. if ~isfield(param.collate_equal.delay,'method') || isempty(param.collate_equal.delay.method) param.collate_equal.delay.method = 'xcorr_complex'; end +% .delay.Mt: Positive scalar integer. Fast-time oversampling to use in +% equalization. Default is 64. if ~isfield(param.collate_equal.delay,'Mt') || isempty(param.collate_equal.delay.Mt) param.collate_equal.delay.Mt = 64; end +% .delay.ref_bins: Array of scalar integers. Default is [-20 20]. Only the +% first and last entry are used in the array. This specifies the relative +% to zero_surf_bin range to use for correlation methods for the +% param.collate_equal.ref wf_adc pair. For example, [-20 20] specifies 20 +% bins before the zero_surf_bin to 20 bins after the zero_surf_bin (so 41 +% total). if ~isfield(param.collate_equal.delay,'ref_bins') || isempty(param.collate_equal.delay.ref_bins) - param.collate_equal.delay.ref_bins = -20:20; + param.collate_equal.delay.ref_bins = [-20 20]; end +% .delay.search_bins: Array of scalar integers. Default is [-7 7]. Only +% the first and last entry are used in the array. This specifies the +% relative to zero_surf_bin range to use for correlation methods. For +% example, [-7 7] specifies 7 bins before the zero_surf_bin to 7 bins +% after the zero_surf_bin (so 15 total). if ~isfield(param.collate_equal.delay,'search_bins') || isempty(param.collate_equal.delay.search_bins) - param.collate_equal.delay.search_bins = -7:7; + param.collate_equal.delay.search_bins = [-7 7]; end if ~isfield(param.collate_equal,'img_lists') || isempty(param.collate_equal.img_lists) @@ -62,35 +96,58 @@ param.collate_equal.in_path = 'analysis'; end +% motion_comp_en: logical scalar. Default is true. If true, motion +% compensation is applied for the squint direction (nadir). if ~isfield(param.collate_equal,'motion_comp_en') || isempty(param.collate_equal.motion_comp_en) param.collate_equal.motion_comp_en = true; end +% out_path: string containing the output path for the equalization results. +% Passed to ct_filename_out. Default is 'equal' for CSARP_equal. if ~isfield(param.collate_equal,'out_path') || isempty(param.collate_equal.out_path) - param.collate_equal.out_path = 'analysis'; + param.collate_equal.out_path = 'equal'; end +% ref: positive scalar integer. Specifies the index into the wf_adc list +% for each image that should be used as the reference channel. if ~isfield(param.collate_equal,'ref') || isempty(param.collate_equal.ref) param.collate_equal.ref = 1; end +% retrack_en: logical scalar. Default is true. Retracks the layer that +% determines where in the analysis waveform extracted data the equalization +% coefficients will be derived. if ~isfield(param.collate_equal,'retrack_en') || isempty(param.collate_equal.retrack_en) param.collate_equal.retrack_en = true; end +% rlines: array of positive integers. Default is empty. Defines which range +% lines will be used to estimate the equalization coefficients. If left +% empty, then all range lines are used. if ~isfield(param.collate_equal,'rlines') || isempty(param.collate_equal.rlines) param.collate_equal.rlines = []; end +% wf_adcs: cell array of wf_adc index lists that correpond to entries in +% param.collate_equal.img_lists. For each image that equalization is run +% on, indexes to specific wf_adc pairs can be specified so that not all +% wf-adc pairs are equalized. The default is empty. If left empty or +% undefined for a particular img_lists entry, then all wf-adc pairs are +% equalized. if ~isfield(param.collate_equal,'wf_adcs') || isempty(param.collate_equal.wf_adcs) param.collate_equal.wf_adcs = []; end +% zero_surf_bin: positive integer scalar. Default is empty. Normally should +% be left empty unless there is an error in the timing. This specifies the +% bin to be used for the equalization process. If empty, the zero_surf_bin +% is automatically determined from the analysis waveform start_time custom +% fields. if ~isfield(param.collate_equal,'zero_surf_bin') || isempty(param.collate_equal.zero_surf_bin) param.collate_equal.zero_surf_bin = []; end -% Other Setup +%% Other Setup % ========================================================================= physical_constants; @@ -98,6 +155,8 @@ h_fig = get_figures(3,enable_visible_plot); end +%% img Loop --------------------------------------------------------------- +% ========================================================================= for img_lists_idx = 1:length(param.collate_equal.img_lists) %% Load waveform data % ===================================================================== @@ -106,7 +165,7 @@ for sub_img_idx = 1:length(param.collate_equal.img_lists{img_lists_idx}) sub_img = param.collate_equal.img_lists{img_lists_idx}(sub_img_idx); - if isempty(param.collate_equal.wf_adcs) + if isempty(param.collate_equal.wf_adcs) || isempty(param.collate_equal.wf_adcs{img_lists_idx}{sub_img_idx}) wf_adcs = 1:size(param.analysis.imgs{sub_img},1); else wf_adcs = param.collate_equal.wf_adcs{img_lists_idx}{sub_img_idx}; @@ -121,7 +180,7 @@ fn = fullfile(fn_dir,sprintf('waveform_%s_wf_%d_adc_%d.mat', param.day_seg, wf, adc)); fprintf('Loading %s (%s)\n', fn, datestr(now)); waveform = load(fn); - if wf_adc == wf_adcs(1) + if sub_img_idx == 1 && wf_adc == wf_adcs(1) gps_time = waveform.gps_time; lat = waveform.lat; lon = waveform.lon; @@ -131,6 +190,7 @@ heading = waveform.heading; time_rng = waveform.time_rng; wf_data = waveform.wf_data; + layer_nan_mask = waveform.layer_nan_mask; else gps_time(end+1,:) = waveform.gps_time; lat(end+1,:) = waveform.lat; @@ -141,12 +201,14 @@ heading(end+1,:) = waveform.heading; time_rng(:,:,end+1) = waveform.time_rng; wf_data(:,:,end+1) = waveform.wf_data; + layer_nan_mask(end+1,:) = waveform.layer_nan_mask; end end end dt = waveform.dt; fc = waveform.fc; ref_wf_adc_idx = param.collate_equal.ref; + cmd = waveform.param_analysis.analysis.cmd{param.collate_equal.cmd_idx}; % Taper off end of record to reduce circular convolution effects that may % show up during time delay compensation. @@ -198,17 +260,18 @@ plot_bins = zero_surf_bin; clf(h_fig(1)); + set(h_fig(1),'Name','Power-Phase-Roll'); pos = get(h_fig(1),'Position'); set(h_fig(1),'Position',[pos(1:2) 700 800]); h_axes = subplot(3,1,1,'parent',h_fig(1)); - plot(h_axes(1), lp(wf_data(plot_bins,:,debug_wf_adc_idx) ./ wf_data(plot_bins,:,ref_wf_adc_idx)).','.') + plot(h_axes(1), db(wf_data(plot_bins,:,debug_wf_adc_idx) ./ wf_data(plot_bins,:,ref_wf_adc_idx)).','.') grid(h_axes(1),'on'); title(h_axes(1),sprintf('Compare wf-adc pair %d to %d',debug_wf_adc_idx,ref_wf_adc_idx)); ylabel(h_axes(1),'Relative power (dB)'); h_axes(2) = subplot(3,1,2,'parent',h_fig(1)); plot(h_axes(2), 180/pi*angle(wf_data(plot_bins,:,debug_wf_adc_idx) .* conj(wf_data(plot_bins,:,ref_wf_adc_idx)) ).','.') grid(h_axes(2),'on'); - ylabel(h_axes(2),'Relative angle (deg)'); + ylabel(h_axes(2),'Relative phase (deg)'); h_axes(3) = subplot(3,1,3,'parent',h_fig(1)); plot(h_axes(3),180/pi*roll.'); grid(h_axes(3),'on'); @@ -216,8 +279,9 @@ xlabel(h_axes(3),'Range line'); clf(h_fig(2)); + set(h_fig(2),'Name','Echogram-Surface'); h_axes(4) = subplot(3,1,1:2,'parent',h_fig(2)); - imagesc(lp(wf_data(:,:,ref_wf_adc_idx)),'parent',h_axes(4)); + imagesc(db(wf_data(:,:,ref_wf_adc_idx)),'parent',h_axes(4)); ylabel(h_axes(4), 'Relative range bin'); h_axes(5) = subplot(3,1,3,'parent',h_fig(2)); plot(h_axes(5), time_rng(:,:).'*3e8/2); @@ -226,6 +290,7 @@ grid(h_axes(5), 'on'); clf(h_fig(3)); + set(h_fig(3),'Name','Relative Angle'); h_axes(6) = axes('parent',h_fig(3)); h_plot = zeros(1,Nc); legend_str = cell(1,Nc); @@ -265,25 +330,25 @@ linkaxes(h_axes,'x'); xlim(h_axes(1), [1 size(wf_data,2)]); - fig_fn = [ct_filename_ct_tmp(param,'','collate_equal',sprintf('%s_before_single_img_%02d',param.collate_equal.out_path,img)) '.fig']; + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_before_single_img_%02d',param.collate_equal.debug_out_fn,img)) '.fig']; fprintf('Saving %s\n', fig_fn); fig_fn_dir = fileparts(fig_fn); if ~exist(fig_fn_dir,'dir') mkdir(fig_fn_dir); end ct_saveas(h_fig(1),fig_fn); - fig_fn = [ct_filename_ct_tmp(param,'','collate_equal',sprintf('%s_before_single_img_%02d',param.collate_equal.out_path,img)) '.jpg']; + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_before_single_img_%02d',param.collate_equal.debug_out_fn,img)) '.jpg']; fprintf('Saving %s\n', fig_fn); ct_saveas(h_fig(1),fig_fn); - fig_fn = [ct_filename_ct_tmp(param,'','collate_equal',sprintf('%s_before_surface_img_%02d',param.collate_equal.out_path,img)) '.jpg']; + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_before_surface_img_%02d',param.collate_equal.debug_out_fn,img)) '.jpg']; fprintf('Saving %s\n', fig_fn); ct_saveas(h_fig(2),fig_fn); - fig_fn = [ct_filename_ct_tmp(param,'','collate_equal',sprintf('%s_before_all_img_%02d',param.collate_equal.out_path,img)) '.fig']; + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_before_all_img_%02d',param.collate_equal.debug_out_fn,img)) '.fig']; fprintf('Saving %s\n', fig_fn); ct_saveas(h_fig(3),fig_fn); - fig_fn = [ct_filename_ct_tmp(param,'','collate_equal',sprintf('%s_before_all_img_%02d',param.collate_equal.out_path,img)) '.jpg']; + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_before_all_img_%02d',param.collate_equal.debug_out_fn,img)) '.jpg']; fprintf('Saving %s\n', fig_fn); ct_saveas(h_fig(3),fig_fn); @@ -296,18 +361,19 @@ %% Retrack surface % ===================================================================== - if param.collate_equal.retrack_en - ml_data = fir_dec(abs(wf_data(:,:,ref_wf_adc_idx)).^2,ones(1,5)/5,1); + ml_data = fir_dec(abs(wf_data(:,:,ref_wf_adc_idx)).^2,ones(1,5)/5,1); + if ~param.collate_equal.retrack_en + surf_bin = zero_surf_bin*ones(1,Nx); + else surf_param = param; - surf_param.cmd.frms = 1; + surf_param.layer_tracker.frms = 1; surf_param.qlook.surf.min_bin = time(1); surf_param.qlook.surf.threshold_noise_rng = [0 (time(1)-time(zero_surf_bin))*2/3 (time(1)-time(zero_surf_bin))*1/3]; surf_param.qlook.surf.threshold_rel_max = -9; surf_param.qlook.surf.max_rng = [0 0]; - surf_param.qlook.surf.en = true; - surf_param.layer_tracker.echogram_source = struct('Data',ml_data,'Time',time,'GPS_time',gps_time(ref_wf_adc_idx,:),'Latitude',lat(ref_wf_adc_idx,:),'Longitude',lon(ref_wf_adc_idx,:),'Elevation',elev(ref_wf_adc_idx,:)); - surf_bin = layer_tracker(surf_param,[]); + surf_param.layer_tracker.echogram_source = struct('Data',ml_data,'Time',time,'GPS_time',gps_time(ref_wf_adc_idx,:),'Latitude',lat(ref_wf_adc_idx,:),'Longitude',lon(ref_wf_adc_idx,:),'Elevation',elev(ref_wf_adc_idx,:),'Roll',roll(ref_wf_adc_idx,:)); + surf_bin = layer_tracker_task(surf_param); surf_bin = round(interp1(time,1:length(time),surf_bin)); surf_bin = surf_bin + 1; @@ -325,8 +391,9 @@ if any(strcmp('surf',param.collate_equal.debug_plots)) clf(h_fig(1)); + set(h_fig(1),'Name','Echogram'); h_axes = axes('parent',h_fig(1)); - imagesc(lp(ml_data),'parent',h_axes(1)); + imagesc(db(ml_data,'power'),'parent',h_axes(1)); colormap(h_axes(1),1-gray(256)); hold(h_axes(1),'on'); plot(h_axes(1), surf_bin); @@ -335,14 +402,15 @@ if param.collate_equal.retrack_en % Check to make sure surface is flat clf(h_fig(2)); + set(h_fig(2),'Name','Echogram Surface Corrected'); h_axes(2) = axes('parent',h_fig(2)); - imagesc(lp(fir_dec(abs(wf_data(:,:,ref_wf_adc_idx)).^2,ones(1,5)/5,1)),'parent',h_axes(2)); + imagesc(db(fir_dec(abs(wf_data(:,:,ref_wf_adc_idx)).^2,ones(1,5)/5,1),'power'),'parent',h_axes(2)); colormap(h_axes(2),1-gray(256)); linkaxes(h_axes); end - fig_fn = [ct_filename_ct_tmp(param,'','collate_equal',sprintf('%s_track1_img_%02d',param.collate_equal.out_path,img)) '.jpg']; + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_track1_img_%02d',param.collate_equal.debug_out_fn,img)) '.jpg']; fprintf('Saving %s\n', fig_fn); fig_fn_dir = fileparts(fig_fn); if ~exist(fig_fn_dir,'dir') @@ -350,9 +418,13 @@ end ct_saveas(h_fig(1),fig_fn); - fig_fn = [ct_filename_ct_tmp(param,'','collate_equal',sprintf('%s_track2_img_%02d',param.collate_equal.out_path,img)) '.jpg']; - fprintf('Saving %s\n', fig_fn); - ct_saveas(h_fig(2),fig_fn); + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_track2_img_%02d',param.collate_equal.debug_out_fn,img)) '.jpg']; + if param.collate_equal.retrack_en + fprintf('Saving %s\n', fig_fn); + ct_saveas(h_fig(2),fig_fn); + elseif exist(fig_fn,'file') + delete(fig_fn); + end if enable_visible_plot keyboard @@ -469,14 +541,16 @@ end if any(strcmp('comp_image',param.collate_equal.debug_plots)) - h_comp_fig = get_figures(Nc,true,'comp_image'); + % These windows are not saved so they are always displayed even in + % visible is not set in debug_plots. + h_comp_fig = get_figures(Nc,true,[mfilename '_comp_image']); set(h_comp_fig,'WindowStyle','docked') for wf_adc = 1:Nc clf(h_comp_fig(wf_adc)); wf = wf_adc_list(wf_adc,1); adc = wf_adc_list(wf_adc,2); h_axes(wf_adc) = axes('parent',h_comp_fig(wf_adc)); - imagesc(lp(wf_data(:,:,wf_adc)),'Parent',h_axes(end), 'parent', h_axes(wf_adc)); + imagesc(db(wf_data(:,:,wf_adc)),'Parent',h_axes(end), 'parent', h_axes(wf_adc)); title(h_axes(wf_adc),sprintf('wf %d adc %d', wf, adc)); xlabel(h_axes(wf_adc),'Range line'); ylabel(h_axes(wf_adc),'Range bin'); @@ -492,10 +566,11 @@ plot_bins = zero_surf_bin; clf(h_fig(1)); + set(h_fig(1),'Name','Power-Phase-Roll'); pos = get(h_fig(1),'Position'); set(h_fig(1),'Position',[pos(1:2) 700 800]); h_axes = subplot(3,1,1,'parent',h_fig(1)); - plot(h_axes(1), lp(wf_data(plot_bins,:,debug_wf_adc_idx) ./ wf_data(plot_bins,:,ref_wf_adc_idx)).','.') + plot(h_axes(1), db(wf_data(plot_bins,:,debug_wf_adc_idx) ./ wf_data(plot_bins,:,ref_wf_adc_idx)).','.') grid(h_axes(1),'on'); title(h_axes(1),sprintf('Compare wf-adc pair %d to %d',debug_wf_adc_idx,ref_wf_adc_idx)); ylabel(h_axes(1),'Relative power (dB)'); @@ -510,8 +585,9 @@ xlabel(h_axes(3),'Range line'); clf(h_fig(2)); + set(h_fig(2),'Name','Echogram-Surface'); h_axes(4) = subplot(3,1,1:2,'parent',h_fig(2)); - imagesc(lp(wf_data(:,:,ref_wf_adc_idx)),'parent',h_axes(4)); + imagesc(db(wf_data(:,:,ref_wf_adc_idx)),'parent',h_axes(4)); ylabel(h_axes(4), 'Relative range bin'); h_axes(5) = subplot(3,1,3,'parent',h_fig(2)); plot(h_axes(5), time_rng(:,:).'*3e8/2); @@ -520,6 +596,7 @@ grid(h_axes(5), 'on'); clf(h_fig(3)); + set(h_fig(3),'Name','Relative Phase'); h_axes(6) = subplot(3,1,1:2,'parent',h_fig(3)); h_plot = zeros(1,Nc); legend_str = cell(1,Nc); @@ -568,25 +645,25 @@ xlim(h_axes(1), [1 size(wf_data,2)]); end - fig_fn = [ct_filename_ct_tmp(param,'','collate_equal',sprintf('%s_after_single_img_%02d',param.collate_equal.out_path,img)) '.fig']; + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_after_single_img_%02d',param.collate_equal.debug_out_fn,img)) '.fig']; fprintf('Saving %s\n', fig_fn); fig_fn_dir = fileparts(fig_fn); if ~exist(fig_fn_dir,'dir') mkdir(fig_fn_dir); end ct_saveas(h_fig(1),fig_fn); - fig_fn = [ct_filename_ct_tmp(param,'','collate_equal',sprintf('%s_after_single_img_%02d',param.collate_equal.out_path,img)) '.jpg']; + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_after_single_img_%02d',param.collate_equal.debug_out_fn,img)) '.jpg']; fprintf('Saving %s\n', fig_fn); ct_saveas(h_fig(1),fig_fn); - fig_fn = [ct_filename_ct_tmp(param,'','collate_equal',sprintf('%s_after_surface_img_%02d',param.collate_equal.out_path,img)) '.jpg']; + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_after_surface_img_%02d',param.collate_equal.debug_out_fn,img)) '.jpg']; fprintf('Saving %s\n', fig_fn); ct_saveas(h_fig(2),fig_fn); - fig_fn = [ct_filename_ct_tmp(param,'','collate_equal',sprintf('%s_after_all_img_%02d',param.collate_equal.out_path,img)) '.fig']; + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_after_all_img_%02d',param.collate_equal.debug_out_fn,img)) '.fig']; fprintf('Saving %s\n', fig_fn); ct_saveas(h_fig(3),fig_fn); - fig_fn = [ct_filename_ct_tmp(param,'','collate_equal',sprintf('%s_after_all_img_%02d',param.collate_equal.out_path,img)) '.jpg']; + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_after_all_img_%02d',param.collate_equal.debug_out_fn,img)) '.jpg']; fprintf('Saving %s\n', fig_fn); ct_saveas(h_fig(3),fig_fn); @@ -628,20 +705,20 @@ [corr_int,lags] = xcorr(abs(in), abs(ref_in) .* Hcorr_wind); elseif delay_method == 1 % Time delay: threshold method - threshold = lp(mean(abs(wf_data(param.collate_equal.noise_bin,:,ref_wf_adc_idx)).^2)) + 30; + threshold = db(mean(abs(wf_data(param.collate_equal.noise_bin,:,ref_wf_adc_idx)).^2),'power') + 30; in = interpft(wf_data(:,rline,wf_adc), Mt*Nt); ref_in = interpft(wf_data(:,rline,ref_wf_adc_idx), Mt*Nt); - in_bin = find(lp(in)>threshold,1); - ref_in_bin = find(lp(ref_in)>threshold,1); + in_bin = find(db(in)>threshold,1); + ref_in_bin = find(db(ref_in)>threshold,1); if ~isempty(in_bin) && ~isempty(ref_in_bin) % If the threshold was exceeded, then we use this range line peak_offset(wf_adc,rline) = (in_bin - ref_in_bin)/Mt; if abs(peak_offset(wf_adc,rline)) > 5 figure(1); clf; - plot(lp(in)); + plot(db(in)); hold on; - plot(lp(ref_in),'r'); + plot(db(ref_in),'r'); keyboard end @@ -672,7 +749,7 @@ legend_str = cell(1,Nc); plot_mode = [0 0 0; hsv(7)]; for wf_adc = 1:Nc - h_plot(wf_adc) = plot(h_axes,lp(ct_smooth(peak_val(wf_adc,:),0.01)), ... + h_plot(wf_adc) = plot(h_axes,db(ct_smooth(abs(peak_val(wf_adc,:)).^2,0.01),'power'), ... 'Color', plot_mode(mod(wf_adc-1,length(plot_mode))+1,:), ... 'LineStyle','none','Marker', '.'); hold(h_axes, 'on'); @@ -734,28 +811,28 @@ set(h_fig(2),'Position',[pos{2}(1:2) 700 pos{2}(4)]); set(h_fig(3),'Position',[pos{3}(1:2) 700 pos{3}(4)]); - fig_fn = [ct_filename_ct_tmp(param,'','collate_equal',sprintf('%s_amp_img_%02d',param.collate_equal.out_path,img)) '.fig']; + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_amp_img_%02d',param.collate_equal.debug_out_fn,img)) '.fig']; fprintf('Saving %s\n', fig_fn); fig_fn_dir = fileparts(fig_fn); if ~exist(fig_fn_dir,'dir') mkdir(fig_fn_dir); end ct_saveas(h_fig(1),fig_fn); - fig_fn = [ct_filename_ct_tmp(param,'','collate_equal',sprintf('%s_amp_img_%02d',param.collate_equal.out_path,img)) '.jpg']; + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_amp_img_%02d',param.collate_equal.debug_out_fn,img)) '.jpg']; fprintf('Saving %s\n', fig_fn); ct_saveas(h_fig(1),fig_fn); - fig_fn = [ct_filename_ct_tmp(param,'','collate_equal',sprintf('%s_phase_img_%02d',param.collate_equal.out_path,img)) '.fig']; + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_phase_img_%02d',param.collate_equal.debug_out_fn,img)) '.fig']; fprintf('Saving %s\n', fig_fn); ct_saveas(h_fig(2),fig_fn); - fig_fn = [ct_filename_ct_tmp(param,'','collate_equal',sprintf('%s_phase_img_%02d',param.collate_equal.out_path,img)) '.jpg']; + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_phase_img_%02d',param.collate_equal.debug_out_fn,img)) '.jpg']; fprintf('Saving %s\n', fig_fn); ct_saveas(h_fig(2),fig_fn); - fig_fn = [ct_filename_ct_tmp(param,'','collate_equal',sprintf('%s_time_img_%02d',param.collate_equal.out_path,img)) '.fig']; + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_time_img_%02d',param.collate_equal.debug_out_fn,img)) '.fig']; fprintf('Saving %s\n', fig_fn); ct_saveas(h_fig(3),fig_fn); - fig_fn = [ct_filename_ct_tmp(param,'','collate_equal',sprintf('%s_time_img_%02d',param.collate_equal.out_path,img)) '.jpg']; + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_time_img_%02d',param.collate_equal.debug_out_fn,img)) '.jpg']; fprintf('Saving %s\n', fig_fn); ct_saveas(h_fig(3),fig_fn); @@ -818,12 +895,14 @@ equal.peak_val{wf} = peak_val; equal.Tsys_offset{wf} = nanmean(peak_offset(:,rlines),2)*dt; - equal.chan_equal_deg_offset{wf} = angle(nanmean(peak_val(:,rlines),2)) * 180/pi; - equal.chan_equal_dB_offset{wf} = lp(nanmean(abs(peak_val(:,rlines)).^2,2),1); + peak_val_masked = peak_val; + peak_val_masked(abs(peak_offset(:,rlines) - nanmedian(peak_offset(:,rlines),2))>1) = NaN; + equal.chan_equal_deg_offset{wf} = angle(nanmean(peak_val_masked(:,rlines),2)) * 180/pi; + equal.chan_equal_dB_offset{wf} = db(nanmean(abs(peak_val_masked(:,rlines)).^2,2),'power'); equal.Tsys_offset_std{wf} = nanstd(peak_offset(:,rlines),[],2)*dt; equal.chan_equal_deg_offset_std{wf} = angle(nanstd(peak_val(:,rlines),[],2)) * 180/pi; - equal.chan_equal_dB_offset_std{wf} = lp(nanstd(abs(peak_val(:,rlines)).^2,[],2),1); + equal.chan_equal_dB_offset_std{wf} = db(nanstd(abs(peak_val(:,rlines)).^2,[],2),'power'); equal.Tsys_offset{wf} = reshape(equal.Tsys_offset{wf},[1 Nc]); equal.chan_equal_deg_offset{wf} = reshape(equal.chan_equal_deg_offset{wf},[1 Nc]); @@ -863,7 +942,7 @@ if any(strcmp('final',param.collate_equal.debug_plots)) sw_version = current_software_version; - diary_fn = [ct_filename_ct_tmp(param,'','collate_equal',sprintf('%s_table_img_%02d',param.collate_equal.out_path,img)) '.txt']; + diary_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_table_img_%02d_wf_%02d',param.collate_equal.debug_out_fn,img,wf)) '.txt']; fid = fopen(diary_fn,'wb'); for fid = [1 fid] if fid == 1; fid_error = 2; else fid_error = fid; end; @@ -912,7 +991,7 @@ %% Save Results % ========================================================================= - equal_fn_dir = ct_filename_out(param,'equal','',1); + equal_fn_dir = ct_filename_out(param,param.collate_equal.out_path,'',1); if ~exist(equal_fn_dir) mkdir(equal_fn_dir); end diff --git a/cresis-toolbox/processing/collate_est_nz_records.m b/cresis-toolbox/processing/collate_est_nz_records.m index 2ce76727..217f1939 100644 --- a/cresis-toolbox/processing/collate_est_nz_records.m +++ b/cresis-toolbox/processing/collate_est_nz_records.m @@ -64,13 +64,11 @@ function collate_est_nz_records(params_en,param_override) continue; end - try - recs_fn = ct_filename_support(params_en{idx},'','records'); - recs_old = load(recs_fn); - catch - fprintf('Missing records file: %s\n',recs_fn); + if ~exist(records_fn,'file') + warning('Missing records file: %s',records_fn); continue; end + recs_old = records_load(params_en{idx}); if length(recs_old.settings.nyquist_zone) == length(testt.coh_wf.records_mask_set) && testt.coh_wf.sets == length(params_en{idx}.collate_nz_est.nz_table) nyquist_zone_hw = NaN(1,length(testt.coh_wf.records_mask_set)); @@ -79,19 +77,16 @@ function collate_est_nz_records(params_en,param_override) end end - - records_mask = ones(1,length(testt.coh_wf.records_mask_set),'uint8'); - records_mask(isnan(nyquist_zone_hw)) = 0; + bit_mask = zeros(1,length(testt.coh_wf.records_mask_set),'uint8'); + % isnan(nyquist_zone_hw) implies a bad record??? + bit_mask(isnan(nyquist_zone_hw)) = bitor(1,bit_mask(isnan(nyquist_zone_hw))); %% UPDATE RECORDS - recs_old.settings.nyquist_zone_hw = nyquist_zone_hw; - recs_old.settings.records_mask = records_mask; - fprintf('Saving records file %s (%s)\n',recs_fn,datestr(now)); -% ct_file_lock_check(recs_fn,3); - ct_save(recs_fn,'-struct','recs_old'); - fprintf('Creating auxiliary records files %s (%s)\n',recs_fn,datestr(now)); - records_aux_files_create(recs_fn); + recs_old.nyquist_zone_hw = nyquist_zone_hw; + recs_old.bit_mask = bit_mask; + fprintf('Saving records file %s (%s)\n',records_fn,datestr(now)); + ct_save(records_fn,'-struct','recs_old','bit_mask','nyquist_zone_hw'); %% FIGURES diff --git a/cresis-toolbox/processing/collate_gain.m b/cresis-toolbox/processing/collate_gain.m index 1f794e28..7469e0f6 100644 --- a/cresis-toolbox/processing/collate_gain.m +++ b/cresis-toolbox/processing/collate_gain.m @@ -1,12 +1,22 @@ % function collate_gain(param,param_override) % -% Collects waveform results from run_analysis.m specified in analysis sheet -% Requires the param.analysis.cmd{1} is waveform -% Generates plots to verify fast-time gain for specified/all wf-adc pairs -% Normalized gain (dB) vs relative time for wf-adc pair +% Analyzes waveform results from run_analysis_gain.m. These waveforms +% should be collected with a continuous single frequency constant modulus +% input into the wf-adc receivers of interest. The amplitude should be low +% enough that the receiver does not saturate. The control/gain settings of +% the receiver should follow regular operation typically since the main +% purpose of the script is to measure the receiver gain as a function of +% fast-time. +% +% Debug plots and a gain curve that is usable by data_load.m are generated +% for each wf-adc pair that is specified. +% +% Analysis waveform should be run without pulse compression (i.e. +% param.analysis.pulse_comp = false). % % Example: -% See run_collate_gain for how to run. +% +% See run_collate_gain for how to run. % % Authors: John Paden, Hara Madhav Talasila @@ -23,407 +33,228 @@ %% Input Checks % ========================================================================= -if ~isfield(param.analysis, 'enable_visible_plot') - param.analysis.enable_visible_plot = 0; +if ~isfield(param,'collate_gain') || isempty(param.collate_gain) + param.collate_gain = []; end -if ~isfield(param.analysis, 'raw_plot_en') - param.analysis.raw_plot_en = 0; +% .debug_out_dir: string containing the output folder name to use for the +% debug outputs. This is input to ct_filename_ct_tmp(). +if ~isfield(param.collate_gain,'debug_out_dir') || isempty(param.collate_gain.debug_out_dir) + param.collate_gain.debug_out_dir = 'collate_gain'; end +debug_out_dir = param.collate_gain.debug_out_dir; -if ~isfield(param.analysis, 'ftg_plot_en') - param.analysis.ftg_plot_en = 1; +% .debug_out_fn: string containing a word to insert into the output file +% name to identify files for this specific run. +if ~isfield(param.collate_gain,'debug_out_fn') || isempty(param.collate_gain.debug_out_fn) + param.collate_gain.debug_out_fn = 'collate_gain'; end -wf_dir = ct_filename_out(param,param.analysis.cmd{1}.out_path,'',1); -if ~exist(wf_dir, 'dir') - fprintf('Empty directory --> run_analysis results (%s)\n',wf_dir); - return; +% .debug_plots: cell array of strings containing commands for which debug +% plots to create. +if ~isfield(param.collate_gain,'debug_plots') + param.collate_gain.debug_plots = {'input_check','gain','combined_plot','visible'}; end - -out_dir = ct_filename_ct_tmp(param,'','waveform','gain'); -if ~exist(out_dir, 'dir') - mkdir(out_dir); +enable_visible_plot = any(strcmp('visible',param.collate_gain.debug_plots)); + +% .echo_filt_args_coherent: 1x2 vector holding the echo_filt.m filter +% arguments [ROW COL] for filtering the coherent data. Default is [1 101] +% meaning no fast-time coherent averages and 101 along-track averages. +% Coherent averaging is the first operation that is applied and is used to +% reduce the noise. +if ~isfield(param.collate_gain,'echo_filt_args_coherent') || isempty(param.collate_gain.echo_filt_args_coherent) + param.collate_gain.echo_filt_args_coherent = [1 101]; end -%% Collate the ftg reults -% ========================================================================= - -% For figure handles -fig_count = 0; -leg=[]; -for idx =1:length(param.analysis.imgs) - fig_count = fig_count + size(param.analysis.imgs{idx},1); +% .echo_filt_args_incoherent: 1x2 vector holding the echo_filt.m filter +% arguments [ROW COL] for filtering the incoherent data. This filter is +% applied after coherent filtering, power detection, and the slow-time +% mean. Default is [5 1] (5 fast-time averages, 1 along-track average). +% Along-track averages have no effect. +if ~isfield(param.collate_gain,'echo_filt_args_incoherent') || isempty(param.collate_gain.echo_filt_args_incoherent) + param.collate_gain.echo_filt_args_incoherent = [1 1]; end -h_fig = get_figures(fig_count+1,param.analysis.enable_visible_plot); -fig_idx=0; -comb_plot = fig_count+1; -clf(h_fig(comb_plot)); -if param.analysis.raw_plot_en - fig2_count = 0; - leg2=[]; - for idx =1:length(param.analysis.imgs) - fig2_count = fig2_count + size(param.analysis.imgs{idx},1); - end - h_fig2 = get_figures(fig2_count+1,param.analysis.enable_visible_plot); - fig2_idx=0; - comb_plot2 = fig2_count+1; - clf(h_fig2(comb_plot2)); +% .imgs: which images and wf-adc pairs to load for collating. The default +% is to set it equal to the current param.analysis.imgs value. +if ~isfield(param.collate_gain,'imgs') || isempty(param.collate_gain.imgs) + param.collate_gain.imgs = param.analysis.imgs; end -% The big loop for each img-wf_adc pair -for img = 1:length(param.analysis.imgs) - for wf_adc = 1:size(param.analysis.imgs{img},1) - wf = param.analysis.imgs{img}(wf_adc,1); - adc = param.analysis.imgs{img}(wf_adc,2); - fig_idx = fig_idx +1; - %% Load the waveform file - % ===================================================================== - dd = load(fullfile(wf_dir, sprintf('waveform_%s_wf_%d_adc_%d.mat',param.day_seg,wf,adc))); - % Initialization - Nt = size(dd.wf_data,1); - r_lines = size(dd.wf_data,2); -% if size(dd.time_rng,1) == 2 && size(dd.time_rng,2) == r_lines && all(all(diff(dd.time_rng,1,2)))==0 -% time_axis = dd.time_rng(1,1):dd.dt:dd.time_rng(2,1); -% else -% fprintf('Time rannge not equal in all range lines'); -% continue; -% end - -% [time_zero, time_zero_idx] = min(abs(time_axis)); - - time_axis = dd.param_analysis.radar.wfs(wf).time; - time_zero_idx = find(time_axis==0); - time_axis_length = time_axis(end)-time_axis(1); - - - TTL_start = param.radar.wfs(wf).TTL_start * 1/param.radar.TTL_clock ; - TTL_length = param.radar.wfs(wf).TTL_length * 1/param.radar.TTL_clock ; - TTL_end = (param.radar.wfs(wf).TTL_start + param.radar.wfs(wf).TTL_length) * 1/param.radar.TTL_clock ; +% .in_path: argument to ct_filename_out to control which analysis outputs +% the function will try to read in. Default is 'analysis_gain' for +% CSARP_analysis_gain. +if ~isfield(param.collate_gain,'in_path') || isempty(param.collate_gain.in_path) + param.collate_gain.in_path = 'analysis_gain'; +end +wf_dir = fileparts(ct_filename_out(param,param.collate_gain.in_path)); - rec_start = param.radar.wfs(1).record_start*1/param.radar.fs ; - rec_stop = param.radar.wfs(1).record_stop*1/param.radar.fs ; - rec_length = rec_stop-rec_start; - - % Collate - mean_wf_data = mean(abs(dd.wf_data),2); - gain_dB = lp(mean_wf_data,2); - - low_Tlim = TTL_end - rec_start - dd.param_analysis.radar.wfs(wf).Tadc_adjust... - -dd.param_analysis.radar.wfs(wf).time_correction; - upper_Tlim = rec_length*0.8; - low_idxs = find(dd.param_analysis.radar.wfs(wf).time>low_Tlim); - upper_idxs = find(dd.param_analysis.radar.wfs(wf).time>upper_Tlim); - - Gain_trunc = mean_wf_data(low_idxs(1):upper_idxs(1)); - Gain = ( Gain_trunc ./ max(Gain_trunc) ); - Time = dd.param_analysis.radar.wfs(wf).time(low_idxs(1):upper_idxs(1)); - % Gain, Gain_trunc and Time have same size - % Gain_raw and time_axis ( or dd.param_analysis.radar.wfs(wf).time) have same size - - % Below equation works. - % In data_load, 1./Inf = zero before low_Tlim -% Gain_raw = [Gain(1)*Inf*ones(low_idxs(1)-1,1); Gain; Gain(end)*ones(upper_idxs(end)-upper_idxs(1),1) ]; +% out_path: string containing the output path for the collate_gain/fast +% time gain results. Passed to ct_filename_out. Default is 'gain' for +% CSARP_gain. +if ~isfield(param.collate_gain,'out_path') || isempty(param.collate_gain.out_path) + param.collate_gain.out_path = 'gain'; +end - % Below equation doesnot work. - % In data_load, 1./Gain = amplified noise before low_Tlim -% Gain_raw = [mean_wf_data(1:low_idxs(1)-1)./max(Gain_trunc); Gain; Gain(end)*ones(upper_idxs(end)-upper_idxs(1),1) ]; +%% Debug Figure Setup +% ========================================================================= - Gain_raw = [ones(low_idxs(1)-1,1); Gain; Gain(end)*ones(upper_idxs(end)-upper_idxs(1),1) ]; +% Create figure handles +h_fig = get_figures(1,enable_visible_plot); +clf(h_fig(1)); +set(h_fig(1),'WindowStyle','docked'); + +%% wf-adc Loop +% ===================================================================== +fig_idx = 0; +all_time = {}; +all_wf_dB = {}; +all_legend = {}; +for img = 1:length(param.collate_gain.imgs) + for wf_adc = 1:size(param.collate_gain.imgs{img},1) + % Process one wf-adc pair at a time + % --------------------------------------------------------------------- + wf = param.collate_gain.imgs{img}(wf_adc,1); + adc = param.collate_gain.imgs{img}(wf_adc,2); + fig_idx = fig_idx +1; + %% wf-adc Loop: Load Waveform File + % ===================================================================== + gain_wf_data = load(fullfile(wf_dir, sprintf('waveform_%s_wf_%d_adc_%d.mat',param.day_seg,wf,adc))); - % dB - gain_max_dB = lp(max(Gain_trunc),2); - norm_gain_dB = gain_dB-gain_max_dB; + time = gain_wf_data.param_analysis.radar.wfs(wf).time_raw; + gain_wf_data.wf_data = echo_filt(gain_wf_data.wf_data,param.collate_gain.echo_filt_args_coherent); + mean_wf_raw_dB = 10*log10(mean(abs(gain_wf_data.wf_data).^2,2)); + mean_wf = mean(abs(gain_wf_data.wf_data).^2,2); + mean_wf = echo_filt(mean_wf,param.collate_gain.echo_filt_args_incoherent); + mean_wf_dB = 10*log10(mean_wf); + max_gain_dB = max(mean_wf_dB(isfinite(mean_wf_dB))); - % To save and use - param_analysis = dd.param_analysis; - param_records = dd.param_records; - param_ftg = param; - %% Generate the plots - % ===================================================================== - cur_fig = h_fig(fig_idx); - clf(cur_fig); - if param.analysis.enable_visible_plot - figure(cur_fig); % Brings the current figure to the top - end - h_axes = axes('parent',cur_fig); - plot(h_axes, time_axis/1e-6,mean_wf_data, 'LineWidth', 1) - hold(h_axes, 'on'); - plot(h_axes, Time/1e-6, Gain_trunc,'--', 'LineWidth', 3); - xlabel(h_axes,'Fast Time (relative), us'); - ylabel(h_axes,'Gain'); - colors = get(h_axes,'ColorOrder'); - grid(h_axes, 'on'); - axis(h_axes, 'tight'); - line(h_axes, [low_Tlim low_Tlim]/1e-6, h_axes.YLim, 'Color','g', 'LineWidth',1); - line(h_axes, [upper_Tlim upper_Tlim]/1e-6, h_axes.YLim, 'Color','g', 'LineWidth',1); - legend(h_axes,sprintf('[%d-%d] G_m_a_x=%.2f',wf,adc,gain_max_dB), 'Location', 'South'); - hold(h_axes, 'off'); - if ~param_analysis.radar.wfs(wf).gain_en - title(h_axes,sprintf('For [ %d-%d ] Fast Time Gain Curve ORIGINAL',wf,adc)); - else - title(h_axes,sprintf('For [ %d-%d ] Fast Time Gain Curve CORRECTED',wf,adc)); - end - - %% Save the plots - if ~dd.param_analysis.radar.wfs(wf).gain_en - saveas(cur_fig, fullfile(out_dir,sprintf('gain_wf_%d_adc_%d.fig',wf,adc))); - saveas(cur_fig, fullfile(out_dir,sprintf('gain_wf_%d_adc_%d.jpg',wf,adc))); - else - saveas(cur_fig, fullfile(out_dir,sprintf('gain_wf_%d_adc_%d_2.fig',wf,adc))); - saveas(cur_fig, fullfile(out_dir,sprintf('gain_wf_%d_adc_%d_2.jpg',wf,adc))); - end - - if ~param_analysis.radar.wfs(wf).gain_en - file_version = '1'; - file_type = 'gain'; - ct_save( fullfile(out_dir,sprintf('gain_wf_%d_adc_%d',wf,adc)), '-v7.3', ... - 'Gain', 'Time', 'low_Tlim', 'upper_Tlim', 'low_idxs', 'upper_idxs',... - 'param_ftg', 'param_analysis', 'param_records', 'Gain_raw', 'time_axis', 'file_type', 'file_version'); - else - fprintf('Data was compensated for ftg. Therefore, new ftg datafile is not saved for wf-adc %d-%d\n',wf,adc); - end - %% Generate the RAW plots + %% wf-adc Loop: Input-Check Plot % ===================================================================== - if param.analysis.raw_plot_en - fig2_idx = fig2_idx +1; - cur_fig = h_fig2(fig2_idx); - clf(cur_fig); - if param.analysis.enable_visible_plot - figure(cur_fig); % Brings the current figure to the top + if any(strcmp('input_check',param.collate_gain.debug_plots)) + clf(h_fig(1)); + set(h_fig(1),'Name',sprintf('%d: Data %d-%d',h_fig(1).Number, wf,adc)); + set(h_fig(1),'WindowStyle','docked'); + h_axes = subplot(1,2,1,'parent',h_fig(1)); + imagesc(h_axes(1), [], time*1e6, mean_wf_raw_dB) + grid(h_axes(1),'on'); + xlabel(h_axes(1),'Range lines'); + ylabel(h_axes(1),'Time ({\mu}s)'); + h_axes(2) = subplot(1,2,2,'parent',h_fig(1)); + plot(h_axes(2), time*1e6, mean_wf_dB, 'LineWidth', 2); + grid(h_axes(2),'on'); + xlabel(h_axes(2),'Time ({\mu}s)'); + ylabel(h_axes(2),'Normalized power (dB)'); + title(h_axes(2),sprintf('Max gain %.1f dB\n', max_gain_dB)); + legend(h_axes(2), 'Raw'); + if enable_visible_plot + keyboard; end - h_axes = axes('parent',cur_fig); - plot(h_axes, time_axis/1e-6,gain_dB,'LineWidth',2); - hold(h_axes, 'on'); - xlabel(h_axes,'Fast Time (relative), us'); - ylabel(h_axes,'Gain, dB'); - colors = get(h_axes,'ColorOrder'); - grid(h_axes, 'on'); - axis(h_axes, 'tight'); - hold(h_axes, 'off'); - if ~param_analysis.radar.wfs(wf).gain_en - title(h_axes,sprintf('For [ %d-%d ] Fast Time Gain Curve ORIGINAL',wf,adc)); - else - title(h_axes,sprintf('For [ %d-%d ] Fast Time Gain Curve CORRECTED',wf,adc)); + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_input_check_%02d_%02d',param.collate_gain.debug_out_fn,wf,adc)) '.fig']; + fprintf('Saving %s\n', fig_fn); + fig_fn_dir = fileparts(fig_fn); + if ~exist(fig_fn_dir,'dir') + mkdir(fig_fn_dir); end - %% Save the RAW plots - saveas(cur_fig, fullfile(out_dir,sprintf('ftg2_wf_%d_adc_%d.fig',wf,adc))); - saveas(cur_fig, fullfile(out_dir,sprintf('ftg2_wf_%d_adc_%d.jpg',wf,adc))); + ct_saveas(h_fig(1),fig_fn); + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_input_check_%02d_%02d',param.collate_gain.debug_out_fn,wf,adc)) '.jpg']; + fprintf('Saving %s\n', fig_fn); + ct_saveas(h_fig(1),fig_fn); end - %% Generate the COMBINED plot + %% wf-adc Loop: Process Gain Waveform % ===================================================================== - cur_fig = h_fig(comb_plot); - if param.analysis.enable_visible_plot - figure(cur_fig); % Brings the current figure to the top + + % Force relative gain to be constant after user defined end time + time_end_idx = find(time>=param.collate_gain.time_end{img},1); + if ~isempty(time_end_idx) + mean_wf_dB(time_end_idx+1:end) = mean_wf_dB(time_end_idx); end - h_axes = gca(cur_fig); - plot(h_axes, time_axis/1e-6,norm_gain_dB,'LineWidth',2); - hold(h_axes, 'on'); - leg{fig_idx} = sprintf('[%d-%d] G_m_a_x=%.2f',wf,adc,gain_max_dB) ; - - %% Generate the COMBINED RAW plot + + % Force relative gain to be constant after user defined end time + time_start_idx = find(time<=param.collate_gain.time_start{img},1,'last'); + if ~isempty(time_start_idx) + mean_wf_dB(1:time_start_idx-1) = mean_wf_dB(time_start_idx); + end + + % Normalize gain to maximum gain + mean_wf_raw_dB = mean_wf_raw_dB - max_gain_dB; + mean_wf_dB = mean_wf_dB - max_gain_dB; + + % Force relative gain to be no less than user defined minimum relative + % gain + mean_wf_dB(mean_wf_dB < param.collate_gain.min_gain_dB{img}) = NaN; + mean_wf_dB = interp_finite(mean_wf_dB,0); + + %% wf-adc Loop: Fast Gain Plot % ===================================================================== - if param.analysis.raw_plot_en - cur_fig = h_fig2(comb_plot2); - if param.analysis.enable_visible_plot - figure(cur_fig); % Brings the current figure to the top + if any(strcmp('gain',param.collate_gain.debug_plots)) + plot(h_axes(2), time*1e6, mean_wf_raw_dB, 'LineWidth', 2); + hold(h_axes(2),'on'); + grid(h_axes(2),'on'); + plot(h_axes(2), time*1e6, mean_wf_dB, 'LineWidth', 2); + legend(h_axes(2), 'Raw', 'Final'); + if enable_visible_plot + keyboard; + end + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_%02d_%02d',param.collate_gain.debug_out_fn,wf,adc)) '.fig']; + fprintf('Saving %s\n', fig_fn); + fig_fn_dir = fileparts(fig_fn); + if ~exist(fig_fn_dir,'dir') + mkdir(fig_fn_dir); end - h_axes = gca(cur_fig); - plot(h_axes, time_axis/1e-6,gain_dB,'LineWidth',2); - hold(h_axes, 'on'); - leg2{fig2_idx} = sprintf('[%d-%d] G_m_a_x=%.2f',wf,adc,gain_max) ; + ct_saveas(h_fig(1),fig_fn); + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_%02d_%02d',param.collate_gain.debug_out_fn,wf,adc)) '.jpg']; + fprintf('Saving %s\n', fig_fn); + ct_saveas(h_fig(1),fig_fn); end + all_time{end+1} = time; + all_wf_dB{end+1} = mean_wf_dB; + all_legend{end+1} = sprintf('%d-%d',wf,adc); - % To see the progress of current segment - if wf_adc == 1 - fprintf('wf-adc %d-%d',wf,adc); - else - fprintf(' %d-%d',wf,adc); + %% wf-adc Loop: Save Results + % ========================================================================= + gain_fn_dir = ct_filename_out(param,param.collate_gain.out_path,'',1); + if ~exist(gain_fn_dir) + mkdir(gain_fn_dir); end - + gain_fn = fullfile(gain_fn_dir,sprintf('gain_%s_%02d_%02d.mat',param.day_seg,wf,adc)); + gain = []; + gain.time = time; + gain.gain = mean_wf_dB; + gain.param_equal = param; + gain.in_fn = dir(fn); + gain.sw_version = current_software_version; + gain.file_version = '1'; + gain.file_type = 'gain'; + fprintf('Saving %s\n', gain_fn); + ct_save(gain_fn,'-struct','gain'); end - fprintf('\n'); end -%% Wrap up the COMBINED plot -% ========================================================================= -cur_fig = h_fig(comb_plot); -if param.analysis.enable_visible_plot - figure(cur_fig); % Brings the current figure to the top -end -h_axes = gca(cur_fig); -colors = get(h_axes,'ColorOrder'); -xlabel(h_axes,'Fast Time (relative), us'); -ylabel(h_axes,'Normalized Gain'); -grid(h_axes, 'on'); -axis(h_axes, 'tight'); -hold(h_axes, 'off'); -if ~param_analysis.radar.wfs(wf).gain_en - title(h_axes,sprintf('Combined Fast Time Gain Curve ORIGINAL')); -else - title(h_axes,sprintf('Combined Fast Time Gain Curve CORRECTED')); -end -legend(h_axes,leg, 'Location', 'South'); - -%% Save the combined plot plot -if ~dd.param_analysis.radar.wfs(wf).gain_en - saveas(cur_fig, fullfile(out_dir,sprintf('gain_combined_ch_%d.fig',adc))); - saveas(cur_fig, fullfile(out_dir,sprintf('gain_combined_ch_%d.jpg',adc))); -else - saveas(cur_fig, fullfile(out_dir,sprintf('gain_combined_ch_%d_2.fig',adc))); - saveas(cur_fig, fullfile(out_dir,sprintf('gain_combined_ch_%d_2.jpg',adc))); -end - -%% Wrap up the COMBINED RAW plot -% ========================================================================= -if param.analysis.raw_plot_en - cur_fig = h_fig2(comb_plot2); - if param.analysis.enable_visible_plot - figure(cur_fig); % Brings the current figure to the top +%% Plot All Waveforms Combined +if any(strcmp('combined_plot',param.collate_gain.debug_plots)) + clf(h_fig(1)); + set(h_fig(1),'visible','on'); + h_axes = axes('parent',h_fig(1)); + for idx = 1:length(all_time) + plot(h_axes, all_time{idx}*1e6, all_wf_dB{idx}); + hold(h_axes,'on'); end - h_axes = gca(cur_fig); - colors = get(h_axes,'ColorOrder'); - xlabel(h_axes,'Fast Time (relative), us'); - ylabel(h_axes,'Gain, dB'); - grid(h_axes, 'on'); - axis(h_axes, 'tight'); - hold(h_axes, 'off'); - if ~param_analysis.radar.wfs(wf).gain_en - title(h_axes,sprintf('Combined Fast Time Gain Curve ORIGINAL')); - else - title(h_axes,sprintf('Combined Fast Time Gain Curve CORRECTED')); + grid(h_axes,'on'); + xlabel(h_axes,'Time ({\mu}s)'); + ylabel(h_axes,'Normalized power (dB)'); + legend(h_axes, all_legend); + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_combined',param.collate_gain.debug_out_fn)) '.fig']; + fprintf('Saving %s\n', fig_fn); + fig_fn_dir = fileparts(fig_fn); + if ~exist(fig_fn_dir,'dir') + mkdir(fig_fn_dir); end - legend(h_axes,leg2, 'Location', 'South'); - - %% Save the combined RAW plot plot - saveas(cur_fig, fullfile(out_dir,sprintf('ftg2_combined_ch_%d.fig',adc))); - saveas(cur_fig, fullfile(out_dir,sprintf('ftg2_combined_ch_%d.jpg',adc))); + ct_saveas(h_fig(1),fig_fn); + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('%s_combined',param.collate_gain.debug_out_fn)) '.jpg']; + fprintf('Saving %s\n', fig_fn); + ct_saveas(h_fig(1),fig_fn); end - -fprintf('=============================================================\n'); -fprintf('%s: %s (%s)\n', param.season_name, param.day_seg, datestr(now)); -fprintf('Output directory -->\n %s\n',out_dir); -fprintf('=============================================================\n'); - -% - - -%% - -%Surprise! -% plot_ftg; -if param.analysis.ftg_plot_en -% script plot_ftg -% -% Plots results from run_collate_ftg -% -% Authors: John Paden, Hara Madhav Talasila - -%% USER SETTINGS -% ========================================================================= - -% param_override = []; -% param_sheet_name = 'rds_param_2019_Greenland_P3.xls'; -% param_fn = ct_filename_param(param_sheet_name); -% params = read_param_xls(param_fn,'',{'analysis_gain' 'analysis'}); - -% param_override.analysis.enable_visible_plot = 0; % Set 1/0; Default: 1 - -%% Automated Section -% ========================================================================= -% -% % Input checking -% global gRadar; -% if exist('param_override','var') -% param_override = merge_structs(gRadar,param_override); -% else -% param_override = gRadar; -% end - -% Process each of the segments -for param_idx =param_idx - param = params(param_idx); - if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic - continue; - end - %% General Setup - % ======================================================================= - -% param = merge_structs(param, param_override); - physical_constants; - - fprintf('=============================================================\n'); - fprintf('%s: %s (%s)\n', param.season_name, param.day_seg, datestr(now)); - fprintf('=============================================================\n'); - %% Input Checks - % ======================================================================= - - if ~isfield(param.analysis, 'enable_visible_plot') - param.analysis.enable_visible_plot = 1; - end - - ftg_dir = ct_filename_ct_tmp(param,'','waveform','gain'); - if ~exist(ftg_dir, 'dir') - fprintf('Empty directory --> run_collate_ftg (%s)\n',ftg_dir); - return; - end - %% PLOT - % For figure handles - leg=[]; - leg_idx=0; - color_idx=0; - h_fig = get_figures(1,param.analysis.enable_visible_plot); - cur_fig = h_fig(1); - clf(cur_fig); - if param.analysis.enable_visible_plot - figure(cur_fig); % Brings the current figure to the top - end - h_axes = axes('parent',cur_fig); - hold(h_axes, 'on'); - colors = get(h_axes,'ColorOrder'); - dd={}; - % The loop for each img-wf_adc pair - for img = 1:length(param.analysis.imgs) - for wf_adc = 1:size(param.analysis.imgs{img},1) - wf = param.analysis.imgs{img}(wf_adc,1); - adc = param.analysis.imgs{img}(wf_adc,2); - %% Load the ftg file - % ===================================================================== - dd{wf,adc} = load(fullfile(ftg_dir, sprintf('gain_wf_%d_adc_%d.mat',wf,adc))); - color_idx = color_idx+1; - plot(h_axes, dd{wf,adc}.Time/1e-6,dd{wf,adc}.Gain,'LineWidth', 2, 'Color', colors(color_idx,:)); - leg_idx = leg_idx+1; - leg{leg_idx} = sprintf('[%d-%d] Truncated',wf,adc) ; - plot(h_axes, dd{wf,adc}.param_analysis.radar.wfs(wf).time/1e-6, dd{wf,adc}.Gain_raw,':', 'LineWidth', 2, 'Color', colors(color_idx,:)); - leg_idx = leg_idx+1; - leg{leg_idx} = sprintf('[%d-%d] Extrapolated',wf,adc) ; - line(h_axes, [dd{wf,adc}.low_Tlim dd{wf,adc}.low_Tlim]/1e-6, [dd{wf,adc}.Gain(1), 0.00],'Color', 'k', 'LineWidth',1); - leg_idx = leg_idx+1; - leg{leg_idx} = sprintf('[%d-%d] Limits',wf,adc) ; - line(h_axes, [dd{wf,adc}.upper_Tlim dd{wf,adc}.upper_Tlim]/1e-6, [dd{wf,adc}.Gain(end), 0.94],'Color', 'g', 'LineWidth',2); - leg_idx = leg_idx+1; - leg{leg_idx} = sprintf('[%d-%d] Limits',wf,adc) ; - - end - end - xlabel(h_axes,'Fast Time (relative), us'); - ylabel(h_axes,'Normalized Gain'); - colors = get(h_axes,'ColorOrder'); - grid(h_axes, 'on'); - axis(h_axes, 'tight'); - - - legend(h_axes,leg, 'Location', 'South'); - hold(h_axes, 'off'); - title(h_axes,sprintf('For chan [ %d ] Fast Time Gain Curve',adc)); - - %% Save the plots - saveas(cur_fig, fullfile(ftg_dir,sprintf('gain_norm_chan_%d.fig',adc))); - saveas(cur_fig, fullfile(ftg_dir,sprintf('gain_norm_chan_%d.jpg',adc))); -end - -end % if param.analysis.ftg_plot_en \ No newline at end of file diff --git a/cresis-toolbox/processing/collate_snapshots.m b/cresis-toolbox/processing/collate_snapshots.m new file mode 100644 index 00000000..a0635805 --- /dev/null +++ b/cresis-toolbox/processing/collate_snapshots.m @@ -0,0 +1,321 @@ +function collate_snapshots(param,param_override) +% collate_snapshots(param,param_override) +% +% +% THIS FUNCTION CULLS SNAPSHOTS CONTAINING SINGLE TARGETS - FIRST STEP OF +% LUT GENERATION +% +% Function that takes snapshots, pulls out calibration pixels and stores +% with corresponding surface ARRIVAL ANGLES (not surface intersection +% angles, i.e. roll-corrected). Currently culls by number of targets and +% signal to clutter ratio. +% +% TO DO: +% 1. ADD FUNCTIONALITY TO STORE MASK IN A MEANINGFULL WAY FOR BROWSING +% IN THE PICKER, +% 2. Future capability may allow for MOE +% 3. Does it make more sense to store the outputs by rx? +% +% param: struct with processing parameters +% -- OR -- +% function handle to script with processing parameters +% param_override: parameters in this struct will override parameters +% in param. This struct must also contain the gRadar fields. +% Typically global gRadar; param_override = gRadar; +% +% Example: +% See run_collate_snapshots.m for how to run this function directly. +% This function may be called from the run_master.m script using the +% param spreadsheet and the cmd.generic column. +% +% Authors: Theresa Moore +% +% See also: collate_snapshots.m, run_collate_snapshots.m, array_proc.m +% +%% General Setup +% ===================================================================== + +param = merge_structs(param, param_override); + +fprintf('=====================================================================\n'); +fprintf('%s: %s (%s)\n', mfilename, param.day_seg, datestr(now)); +fprintf('=====================================================================\n'); + +%% Input checks +% ===================================================================== +if ~isfield(param,mfilename) || isempty(param.(mfilename)) + param.(mfilename) = []; +end + +if ~isfield(param.collate_snapshots,'scr_threshold') || isempty(param.collate_snapshots.scr_threshold) + param.collate_snapshots.scr_threshold = 15; +end + +if ~isfield(param.collate_snapshots,'out_path') || isempty(param.collate_snapshots.out_path) + param.collate_snapshots.out_path = 'collate_snapshots'; +end + +if ~isfield(param.collate_snapshots,'in_path') || isempty(param.collate_snapshots.in_path) + param.collate_snapshots.in_path = 'snapshot'; +end + +if ~isfield(param.collate_snapshots,'layer_params') || isempty(param.collate_snapshots.layer_params) + param.collate_snapshots.layer_params.name = 'surface'; + param.collate_snapshots.layer_params.source = 'layerdata'; + param.collate_snapshots.layer_params.layer_path = 'layer'; +end + +if ~isfield(param.collate_snapshots.layer_params,'layer_path') || isempty(param.collate_snapshots.layer_params.layer_path) + param.collate_snapshots.layer_params.layer_path = 'layer'; +end + +if ~isfield(param.collate_snapshots,'remove_multiple') || isempty(param.collate_snapshots.remove_multiple) + param.collate_snapshots.remove_multiple = true; + param.collate_snapshots.multiple_guard_bins = 2; +end + +if ~isfield(param.collate_snapshots,'multiple_guard_bins') || isempty(param.collate_snapshots.multiple_guard_bins) + param.collate_snapshots.multiple_guard_bins = 2; +end + +if ~isfield(param.collate_snapshots,'img_list') + param.collate_snapshots.img_list = []; +end + +if ~isfield(param.collate_snapshots,'plot_enable') || isempty(param.collate_snapshots.plot_enable) + param.collate_snapshots.dtheta = 1; +end + +if ~isfield(param.collate_snapshots,'debug') || isempty(param.collate_snapshots.debug) + param.collate_snapshots.debug = 0; +end + +if ~isfield(param.collate_snapshots,'same_doa_guard') || isempty(param.collate_snapshots.same_doa_guard) + param.collate_snapshots.same_doa_guard = 5; % degrees +end + +%% collate_snapshots: load support files and loop on frames +% Load frames file +frames = frames_load(param); + +param.cmd.frms = frames_param_cmd_frms(param,frames); + +for frm_idx = 1:length(param.cmd.frms) + frm = param.cmd.frms(frm_idx); + % fprintf('Collate snapshots %s_%03d \n', param.day_seg,frm) + fprintf('\n\n'); + fprintf('collate snapshots: %s_%03d (%s)\n', param.day_seg,frm, datestr(now)); + fprintf('---------------------------------------------------------------------\n'); + + % Building input and output directories for the frame + snp_dir = ct_filename_out(param,param.collate_snapshots.in_path); + sv_offset_dir = ct_filename_out(param,param.collate_snapshots.out_path); + + % Check for existence of layerdata + layer_fn = fullfile(ct_filename_out(param,param.collate_snapshots.layer_params.layer_path,''),sprintf('Data_%s_%03d.mat', param.day_seg,frm)); + + % If surface layer doesn't exist, disable the flag to remove the multiple + if ~(exist(layer_fn,'file') == 2) + param.collate_snapshots.remove_multiple = false; + fprintf('Layer file does not exist: %s,\n', layer_fn); + fprintf('Surface multiple not removed\n'); + layer = []; + else + % Load in surface + surf_param = param.collate_snapshots.layer_params; + fprintf('Loading surface: %s (%s)\n', layer_fn, datestr(now)); + layer = opsLoadLayers(param,surf_param); + end + + + %% collate_snapthost: loop over images in the list + for img_idx = 1:length(param.collate_snapshots.img_list) + % Initialize variables + Tomo = []; + Time = []; + Elevation = []; + GPS_time = []; + Heading = []; + Latitude = []; + Longitude = []; + Roll = []; + Heading =[]; + Latitude =[]; + Longitude =[]; + Pitch =[]; + Roll=[]; + param_array =[]; + param_records=[]; + param_sar = []; + file_version=[]; + + + % Building the input filename + img= param.collate_snapshots.img_list(img_idx); + file = sprintf('Data_img_%02d_%s_%03d.mat',img,param.day_seg,frm); + fn = fullfile(snp_dir,file); + + % Ensure file exists + if exist(fn,'file') == 2 + fprintf('Loading snapshot file: %s (%s)\n', fn, datestr(now)); + load(fn,'Tomo','Time','Elevation','GPS_time','Heading','Latitude','Longitude','Pitch','Roll','param_array','param_records','param_sar','file_version'); + [Nt, Nc, Nx] = size(Tomo.img); + + %% collate_snapshots: reformat raw snapshot outputs + Tomo.img = permute(Tomo.img,[2 1 3]); + Tomo.power = permute(Tomo.power,[2 1 3]); + Tomo.surf_theta = permute(Tomo.surf_theta,[2 1 3]); + Tomo.surf_ice_mask = permute(Tomo.surf_ice_mask,[2 1 3]); + Tomo.two_src_mask = squeeze(sum(~isnan(Tomo.surf_theta),1))==2; + Tomo.one_src_mask = squeeze(sum(~isnan(Tomo.surf_theta),1))==1; + Tomo.surf_ice_mask = squeeze(any(Tomo.surf_ice_mask,1)); + Tomo.surf_ice_mask = cumsum(Tomo.surf_ice_mask,1) >= 1; + Tomo.surf_multiple_mask = true(size(Tomo.surf_ice_mask)); + Tomo.Nsrc = squeeze(sum(~isnan(Tomo.surf_theta))); + Tomo.edge_mask = false(size(Tomo.Nsrc)); + %% collate_snapshots: create pixel culling + + % Surface multiple mask + if ~isempty(layer) + surface = interp_finite(interp1(layer.gps_time,layer.twtt,GPS_time)); + multiple = 2*surface; + rlines = 1:Nx; + bad_rlines = rlines(multiple > Time(end)); + good_rlines = rlines(multiple < Time(end)); +% Tomo.surf_multiple_mask(:,multiple < Time(end)) = false; + for rline_idx = 1:length(good_rlines) + rline = good_rlines(rline_idx); +% surf_bin = round(interp_finite(interp1(Time,1:Nt,surface(rline_idx)))); +% surf_bins = surf_bin + [-param.collate_snapshots.multiple_guard_bins:param.collate_snapshots.multiple_guard_bins]; + multiple_bin = round(interp_finite(interp1(Time,1:Nt,multiple(rline)))); + multiple_bins = multiple_bin + [-param.collate_snapshots.multiple_guard_bins:param.collate_snapshots.multiple_guard_bins]; + if min(multiple_bins) > 1 & max(multiple_bins) < Nt + Tomo.surf_multiple_mask(multiple_bins,rline) = false; + end +% +% if min(surf_bins) > 1 & max(surf_bins) < Nt +% Tomo.surf_multiple_mask(surf_bins,rline_idx) = false; +% end + end + end + + % Assign nans to -inf + Tomo.power(isnan(Tomo.power)) = -inf; + + % Sort powers in descending order + [Tomo.power,sort_idxs] = sort(Tomo.power,1,'descend'); +% Tomo.power = 10*log10(abs(Tomo.power)); + + % Convert surface angle to DOA + Tomo.surf_doa = bsxfun(@minus,Tomo.surf_theta,permute(Roll*180/pi,[1 3 2])); + + % Maximum number of sources + Ndoa_max = size(Tomo.surf_doa,1); + + % Sort indexes for the sources + doa_sort_idxs = bsxfun(@plus,bsxfun(@plus,sort_idxs,Ndoa_max*(0:Nt-1)),permute(Ndoa_max*Nt*(0:Nx-1),[1 3 2])); + + % Source incidence angles of clutter and calibration angles + Tomo.surf_theta = Tomo.surf_theta(doa_sort_idxs); + Tomo.surf_doa = Tomo.surf_doa(doa_sort_idxs); + + Tomo.backlobe_mask = squeeze(sum(abs(Tomo.surf_doa) > 90,1) >=1); + + % Same doa mask + Tomo.same_doa_mask = squeeze(abs(Tomo.surf_doa(1,:,:)-Tomo.surf_doa(2,:,:)) < param.collate_snapshots.same_doa_guard); + + % Fast time mask + Tpd = param.radar.wfs(img).Tpd; + Tomo.edge_mask = repmat(Time >= Time(end) - Tpd/2,1,Nx); + + % Final culling mask +% Tomo.mask = Tomo.two_src_mask & ~Tomo.surf_ice_mask & (Tomo.power_mask | Tomo.same_doa_mask) & Tomo.surf_multiple_mask; +% Tomo.mask = Tomo.two_src_mask & ~Tomo.surf_ice_mask & Tomo.power_mask & ~Tomo.same_doa_mask & Tomo.surf_multiple_mask; +% Tomo.mask = (Tomo.one_src_mask | Tomo.two_src_mask) & ~Tomo.surf_ice_mask & ~Tomo.same_doa_mask & Tomo.surf_multiple_mask & ~Tomo.backlobe_mask; + Tomo.mask = (Tomo.one_src_mask | Tomo.two_src_mask) & ~Tomo.surf_ice_mask & ~Tomo.same_doa_mask & Tomo.surf_multiple_mask & ~Tomo.edge_mask; + % If only one source (not likely) store nans + if Ndoa_max == 1 + Tomo.power(2,:,:) = nan(size(Tomo.mask)); + Tomo.surf_doa(2,:,:) = nan(size(Tomo.mask)); + Tomo.surf_theta(2,:,:) = nan(size(Tomo.mask)); + end + + Nsrc_max = min([2 Ndoa_max]); + + %% colate_snapshots: create simplified output + + % Create a list of the rxs + wf = param_array.array_proc.imgs{1}(1,1); + adc_list = param_array.array_proc.imgs{1}(:,2); + rx_list = param.radar.wfs(wf).rx_paths(adc_list); + + + % Pick out the snapshots, power, surface incidence angles, cal doas, + % power of good pixels + snapshots.sv_list = Tomo.img(:,Tomo.mask); + snapshots.surf_doas = Tomo.surf_doa(1:Nsrc_max,Tomo.mask); + snapshots.surf_thetas = Tomo.surf_theta(1:Nsrc_max,Tomo.mask); + snapshots.power = Tomo.power(1:Nsrc_max,Tomo.mask); + snapshots.Nsrc = Tomo.Nsrc(Tomo.mask).'; + + % Pick out gps time and twtt of the culled pixels + twtt_matrix = repmat(Time,1,Nx); + gps_time_matrix = repmat(GPS_time,Nt,1); + elev_matrix = repmat(Elevation,Nt,1); + pitch_matrix = repmat(Pitch,Nt,1); + heading_matrix = repmat(Heading,Nt,1); + roll_matrix = repmat(Roll,Nt,1); + lat_matrix = repmat(Latitude,Nt,1); + lon_matrix = repmat(Longitude,Nt,1); + + snapshots.twtt = twtt_matrix(Tomo.mask); + snapshots.gps_time = gps_time_matrix(Tomo.mask); + snapshots.elev = elev_matrix(Tomo.mask); + snapshots.lat = lat_matrix(Tomo.mask); + snapshots.lon = lon_matrix(Tomo.mask); + snapshots.pitch = pitch_matrix(Tomo.mask); + snapshots.heading = heading_matrix(Tomo.mask); + snapshots.roll = roll_matrix(Tomo.mask); + + snapshots.twtt = snapshots.twtt(:).'; + snapshots.gps_time = snapshots.gps_time(:).'; + snapshots.elev = snapshots.elev(:).'; + snapshots.lat = snapshots.lat(:).'; + snapshots.lon = snapshots.lon(:).'; + snapshots.pitch = snapshots.pitch(:).'; + snapshots.heading = snapshots.heading(:).'; + snapshots.roll = snapshots.roll(:).'; + snapshots.rx_list = rx_list; + + snapshots.param_collate_snapshots = param.collate_snapshots; + snapshots.param_array = param_array; + snapshots.param_sar = param_sar; + snapshots.param_records = param.records; + snapshots.file_version = file_version; + snapshots.file_type = 'snapshots'; + + %% Save the result + % ===================================================================== + out_fn_dir = ct_filename_out(param,param.collate_snapshots.out_path,''); + if ~isdir(out_fn_dir) + mkdir(out_fn_dir) + end + out_fn = fullfile(out_fn_dir,sprintf('snapshots_simp_img_%02d_%s_%03d.mat', img, param.day_seg, frm)); + fprintf('Saving %s (%s)\n', out_fn, datestr(now)); + ct_save(out_fn,'-struct','snapshots'); + end + end +end + + % For more than one source, threshold highestst +% if Ndoa_max > 1 +% % Threshold +% Tomo.power_mask = squeeze(Tomo.power(1,:,:) > Tomo.power(2,:,:) + param.collate_snapshots.scr_threshold); +% % Tomo.scr = nan(size(Tomo.power_mask)); +% % Tomo.scr = squeeze(Tomo.power(1,:,:) - Tomo.power(2,:,:)); +% else +% Tomo.power_mask = true(size(squeeze(Tomo.power))); +% % Tomo.scr = nan(size(Tomo.power_mask)); +% end + + diff --git a/cresis-toolbox/processing/combine_wf_chan.m b/cresis-toolbox/processing/combine_wf_chan.m deleted file mode 100644 index 657aebc5..00000000 --- a/cresis-toolbox/processing/combine_wf_chan.m +++ /dev/null @@ -1,449 +0,0 @@ -function ctrl_chain = combine_wf_chan(param,param_override) -% ctrl_chain = combine_wf_chan(param,param_override) -% -% This script combines the receive channels and outputs the result -% for each waveform. It also combines the waveforms. It takes in -% f-k files one directory at a time and: -% 1. combines the receive channels -% 2. concatenates the results -% 3. square-law detects the data, abs()^2 -% 4. takes incoherent averages (multilooks data) -% 5. saves the result in a new directory -% 6. Combines the waveforms -% -% The assumption is that the directories in the input_path are named -% using the following convention: -% PROC-TYPE-STRING_data_#{_SUBAPERTURE-STRING} -% where -% PROC-TYPE-STRING can be 'fk','tdbp', or 'pc' for f-k migrated,time domain -% back projected,and pulse compressed respectively ('fk' and tdbp supported) -% _data_ is always present -% #, \d+: one or more numbers -% _SUBAPERTURE-STRING, {_[mp]\d\.\d}: optional subaperture string -% Examples: -% fk_data_01_01: f-k migrated, frame 1, subaperture 1 -% fk_data_04_02: f-k migrated, frame 4, subaperture 2 -% fk_data_01_03: f-k migrated, frame 1, subaperture 3 -% pc_data_01: pulse compressed only, frame 1 -% -% param = struct with processing parameters -% -- OR -- -% function handle to script with processing parameters -% param_override = parameters in this struct will override parameters -% in param. This struct must also contain the gRadar fields. -% Typically global gRadar; param_override = gRadar; -% -% Authors: John Paden -% -% See also: run_master.m, master.m, run_combine_wf_chan.m, combine_wf_chan.m, -% combine_wf_chan_task.m - -%% General Setup -% ===================================================================== -param = merge_structs(param, param_override); - -fprintf('=====================================================================\n'); -fprintf('%s: %s (%s)\n', mfilename, param.day_seg, datestr(now)); -fprintf('=====================================================================\n'); - -%% Input Checks -% ===================================================================== - -if ~isfield(param.combine,'frm_types') || isempty(param.combine.frm_types) - param.combine.frm_types = {-1,-1,-1,-1,-1}; -end - -% Remove frames that do not exist from param.cmd.frms list -load(ct_filename_support(param,'','frames')); % Load "frames" variable -if ~isfield(param.cmd,'frms') || isempty(param.cmd.frms) - param.cmd.frms = 1:length(frames.frame_idxs); -end -[valid_frms,keep_idxs] = intersect(param.cmd.frms, 1:length(frames.frame_idxs)); -if length(valid_frms) ~= length(param.cmd.frms) - bad_mask = ones(size(param.cmd.frms)); - bad_mask(keep_idxs) = 0; - warning('Nonexistent frames specified in param.cmd.frms (e.g. frame "%g" is invalid), removing these', ... - param.cmd.frms(find(bad_mask,1))); - param.cmd.frms = valid_frms; -end - -if ~isfield(param.combine,'img_comb_layer_params') - param.combine.img_comb_layer_params = []; -end - -if ~isfield(param.combine,'trim_time') - param.combine.trim_time = true; -end - -if ~isfield(param.csarp,'pulse_comp') || isempty(param.csarp.pulse_comp) - param.csarp.pulse_comp = 1; -end - -if ~isfield(param.csarp,'presums') || isempty(param.csarp.presums) - param.csarp.presums = 1; -end - -if ~isfield(param.combine,'in_path') || isempty(param.combine.in_path) - param.combine.in_path = 'out'; -end - -if ~isfield(param.combine,'array_path') || isempty(param.combine.array_path) - param.combine.array_path = 'out'; -end - -if ~isfield(param.combine,'out_path') || isempty(param.combine.out_path) - param.combine.out_path = param.combine.method; -end - -if ~isfield(param.csarp,'out_path') || isempty(param.csarp.out_path) - param.csarp.out_path = 'out'; -end - -if ~isfield(param.combine,'presums') || isempty(param.combine.presums) - if ~isfield(param.csarp,'presums') || isempty(param.csarp.presums) - param.combine.presums = 1; - else - param.combine.presums = param.csarp.presums; - end -end - -if ~isfield(param.combine,'sar_type') || isempty(param.combine.sar_type) - if ~isfield(param.csarp,'sar_type') || isempty(param.csarp.sar_type) - param.combine.sar_type = 'fk'; - else - param.combine.sar_type = param.csarp.sar_type; - end -end - -if strcmpi(param.combine.sar_type,'f-k') - error('Deprecated sar_type name. Change param.combine.sar_type from ''f-k'' to ''fk'' in your parameters (or remove parameter since ''fk'' is the default mode).'); -end - -if ~isfield(param.combine,'chunk_len') || isempty(param.combine.chunk_len) - if ~isfield(param.csarp,'chunk_len') || isempty(param.csarp.chunk_len) - error('param.combine.chunk_len or param.csarp.chunk_len must be defined'); - else - param.combine.chunk_len = param.csarp.chunk_len; - end -end - -% Handles multilooking syntax: -% {{[1 1],[1 2],[1 3],[1 4],[1 5]},{[2 1],[2 2],[2 3],[2 4],[2 5]}} -% If the image is a cell array it describes multilooking across apertures -if ~iscell(param.combine.imgs{1}) - % No special multilooking, reformat old syntax to new multilooking syntax - for img = 1:length(param.combine.imgs) - param.combine.imgs{img} = {param.combine.imgs{img}}; - end -end - -for img = 1:length(param.combine.imgs) - for ml_idx = 1:length(param.combine.imgs{img}) - % Imaginary image indices is for IQ combining during raw data load - % which we do not need here. - param.combine.imgs{img}{ml_idx} = abs(param.combine.imgs{img}{ml_idx}); - end -end - -if ~isfield(param.combine,'out_path') || isempty(param.combine.out_path) - param.combine.out_path = param.combine.method; -end - -%% Setup processing -% ===================================================================== - -% Get the standard radar name -[~,~,radar_name] = ct_output_dir(param.radar_name); - -% Create output directory path -combine_out_dir = ct_filename_out(param, param.combine.out_path); - -% Load records file -records_fn = ct_filename_support(param,'','records'); -records = load(records_fn); -% Apply presumming -if param.csarp.presums > 1 - records.lat = fir_dec(records.lat,param.csarp.presums); - records.lon = fir_dec(records.lon,param.csarp.presums); - records.elev = fir_dec(records.elev,param.csarp.presums); - records.roll = fir_dec(records.roll,param.csarp.presums); - records.pitch = fir_dec(records.pitch,param.csarp.presums); - records.heading = fir_dec(records.heading,param.csarp.presums); - records.gps_time = fir_dec(records.gps_time,param.csarp.presums); - records.surface = fir_dec(records.surface,param.csarp.presums); -end -% Along-track -along_track_approx = geodetic_to_along_track(records.lat,records.lon,records.elev); - -% Preload layer for image combine if it is specified -if isempty(param.combine.img_comb_layer_params) - layers = []; -else - param_load_layers = param; - param_load_layers.cmd.frms = param.cmd.frms; - layers = opsLoadLayers(param_load_layers,param.combine.img_comb_layer_params); -end - -%% Collect waveform information into one structure -% - This is used to break the frame up into chunks -% ===================================================================== -if strcmpi(radar_name,'mcrds') - wfs = load_mcrds_wfs(records.settings, param, ... - records.param_records.records.file.adcs, param.csarp); -elseif any(strcmpi(radar_name,{'acords','hfrds','hfrds2','mcords','mcords2','mcords3','mcords4','mcords5','mcrds','seaice','accum2'})) - wfs = load_mcords_wfs(records.settings, param, ... - records.param_records.records.file.adcs, param.csarp); -elseif any(strcmpi(radar_name,{'icards'}))% add icards---qishi - wfs = load_icards_wfs(records.settings, param, ... - records.param_records.records.file.adcs, param.csarp); -elseif any(strcmpi(radar_name,{'snow','kuband','snow2','kuband2','snow3','kuband3','kaband3','snow5'})) - error('Not supported'); - wfs = load_fmcw_wfs(records.settings, param, ... - records.param_records.records.file.adcs, param.csarp); - for wf=1:length(wfs) - wfs(wf).time = param.csarp.time_of_full_support; - wfs(wf).freq = 1; - end -end - -%% Create and setup the cluster batch -% ===================================================================== -ctrl = cluster_new_batch(param); -cluster_compile({'combine_wf_chan_task.m','combine_wf_chan_combine_task.m'},ctrl.cluster.hidden_depend_funs,ctrl.cluster.force_compile,ctrl); - -total_num_sam = []; -if any(strcmpi(radar_name,{'acords','hfrds','hfrds2','mcords','mcords2','mcords3','mcords4','mcords5','mcrds','seaice','accum2'})) - for img = 1:length(param.combine.imgs) - wf = abs(param.combine.imgs{img}{1}(1,1)); - total_num_sam(img) = wfs(wf).Nt_raw; - end - cpu_time_mult = 6e-8; - mem_mult = 8; - -elseif any(strcmpi(radar_name,{'snow','kuband','snow2','kuband2','snow3','kuband3','kaband3','snow5','snow8'})) - for img = 1:length(param.combine.imgs) - wf = abs(param.combine.imgs{img}{1}(1,1)); - total_num_sam(img) = 32000; - end - cpu_time_mult = 8e-8; - mem_mult = 64; - -else - error('radar_name %s not supported yet.', radar_name); - -end - -%% Loop through all the frame directories and process the SAR chunks -% ===================================================================== -sparam.argsin{1} = param; % Static parameters -sparam.task_function = 'combine_wf_chan_task'; -sparam.num_args_out = 1; -prev_frm_num_chunks = []; -for frm_idx = 1:length(param.cmd.frms); - frm = param.cmd.frms(frm_idx); - - if ct_proc_frame(frames.proc_mode(frm),param.combine.frm_types) - fprintf('%s %s_%03i (%i of %i) (%s)\n', sparam.task_function, param.day_seg, frm, frm_idx, length(param.cmd.frms), datestr(now)); - skip_frame = false; - else - fprintf('Skipping frame %s_%03i (no process frame)\n', param.day_seg, frm); - skip_frame = true; - end - - % Temporary output directory - combine_tmp_dir = fullfile(ct_filename_out(param, param.combine.array_path), ... - sprintf('%s_%03d', param.combine.method, frm)); - - % Current frame goes from the start record specified in the frames file - % to the record just before the start record of the next frame. For - % the last frame, the stop record is just the last record in the segment. - start_rec = ceil(frames.frame_idxs(frm)/param.csarp.presums); - if frm < length(frames.frame_idxs) - stop_rec = ceil((frames.frame_idxs(frm+1)-1)/param.csarp.presums); - else - stop_rec = length(records.gps_time); - end - - % Determine length of the frame - frm_dist = along_track_approx(stop_rec) - along_track_approx(start_rec); - - % Determine number of chunks and range lines per chunk - num_chunks = round(frm_dist / param.combine.chunk_len); - - %% Process each chunk (unless it is a skip frame) - for chunk_idx = 1:num_chunks*~skip_frame - % Prepare task inputs - % ================================================================= - dparam = []; - dparam.argsin{1}.load.frm = frm; - dparam.argsin{1}.load.chunk_idx = chunk_idx; - dparam.argsin{1}.load.num_chunks = num_chunks; - dparam.argsin{1}.load.prev_frm_num_chunks = prev_frm_num_chunks; - - % Create success condition - % ================================================================= - dparam.success = ''; - for img = 1:length(param.combine.imgs) - out_fn_name = sprintf('img_%02d_chk_%03d.mat',img,chunk_idx); - out_fn{img} = fullfile(combine_tmp_dir,out_fn_name); - if img == 1 - dparam.success = cat(2,dparam.success, ... - sprintf('if ~exist(''%s'',''file'')', out_fn{img})); - else - dparam.success = cat(2,dparam.success, ... - sprintf(' || ~exist(''%s'',''file'')', out_fn{img})); - end - if ~ctrl.cluster.rerun_only && exist(out_fn{img},'file') - delete(out_fn{img}); - end - end - dparam.success = cat(2,dparam.success,sprintf('\n')); - if 0 - % Enable this check if you want to open each output file to make - % sure it is not corrupt. - for img = 1:length(param.combine.imgs) - out_fn_name = sprintf('img_%02d_chk_%03d.mat',img,chunk_idx); - out_fn{img} = fullfile(combine_tmp_dir,out_fn_name); - dparam.success = cat(2,dparam.success, ... - sprintf(' load(''%s'');\n', out_fn{img})); - end - end - success_error = 64; - dparam.success = cat(2,dparam.success, ... - sprintf(' error_mask = bitor(error_mask,%d);\n', success_error)); - dparam.success = cat(2,dparam.success,sprintf('end;\n')); - - % Rerun only mode: Test to see if we need to run this task - % ================================================================= - dparam.notes = sprintf('%s:%s:%s %s_%03d (%d of %d)/%d of %d', ... - sparam.task_function, param.radar_name, param.season_name, param.day_seg, frm, frm_idx, length(param.cmd.frms), ... - chunk_idx, num_chunks); - if ctrl.cluster.rerun_only - % If we are in rerun only mode AND the get heights task success - % condition passes without error, then we do not run the task. - error_mask = 0; - eval(dparam.success); - if ~error_mask - fprintf(' Already exists [rerun_only skipping]: %s (%s)\n', ... - dparam.notes, datestr(now)); - continue; - end - end - - % Create task - % ================================================================= - - % CPU Time and Memory estimates: - % Nx*total_num_sam*K where K is some manually determined multiplier. - Nx = stop_rec - start_rec + 1; - dparam.cpu_time = 0; - dparam.mem = 0; - for img = 1:length(param.combine.imgs) - dparam.cpu_time = dparam.cpu_time + 10 + size(param.combine.imgs{img},1)*Nx*total_num_sam(img)*cpu_time_mult; - dparam.mem = max(dparam.mem,250e6 + size(param.combine.imgs{img},1)*Nx*total_num_sam(img)*mem_mult); - end - - ctrl = cluster_new_task(ctrl,sparam,dparam,'dparam_save',0); - - end - prev_frm_num_chunks = num_chunks; -end - -ctrl = cluster_save_dparam(ctrl); - -ctrl_chain = {ctrl}; - -%% Create and setup the combine batch -% ===================================================================== -ctrl = cluster_new_batch(param); - -if any(strcmpi(radar_name,{'acords','hfrds','hfrds2','mcords','mcords2','mcords3','mcords4','mcords5','mcrds','seaice','accum2'})) - cpu_time_mult = 6e-8; - mem_mult = 8; - -elseif any(strcmpi(radar_name,{'snow','kuband','snow2','kuband2','snow3','kuband3','kaband3','snow5','snow8'})) - cpu_time_mult = 100e-8; - mem_mult = 24; -end - -sparam = []; -sparam.argsin{1} = param; % Static parameters -sparam.task_function = 'combine_wf_chan_combine_task'; -sparam.num_args_out = 1; -sparam.cpu_time = 60; -sparam.mem = 0; -fprintf('%s %s (%s)\n', sparam.task_function, param.day_seg, datestr(now)); -% Add up all records being processed and find the most records in a frame -Nx = 0; -Nx_max = 0; -for frm = param.cmd.frms - % recs: Determine the records for this frame - if frm < length(frames.frame_idxs) - Nx_frm = (along_track_approx(frames.frame_idxs(frm+1)) - along_track_approx(frames.frame_idxs(frm))) / param.csarp.sigma_x; - else - Nx_frm = (along_track_approx(length(records.gps_time)) - along_track_approx(frames.frame_idxs(frm) + 1)) / param.csarp.sigma_x; - end - Nx_frm = round(Nx_frm); - if Nx_frm > Nx_max - Nx_max = Nx_frm; - end - Nx = Nx + Nx_frm; -end -% Account for averaging -for img = 1:length(param.combine.imgs) - sparam.cpu_time = sparam.cpu_time + (Nx*total_num_sam(img)*cpu_time_mult); - if isempty(param.combine.img_comb) - % Individual images, so need enough memory to hold the largest image - sparam.mem = max(sparam.mem,250e6 + Nx_max*total_num_sam(img)*mem_mult); - else - % Images combined into one so need enough memory to hold all images - sparam.mem = 250e6 + Nx*sum(total_num_sam)*mem_mult; - end -end -sparam.notes = sprintf('%s:%s:%s %s combine frames', ... - sparam.task_function, param.radar_name, param.season_name, param.day_seg); - -% Create success condition -success_error = 64; -sparam.success = ''; -for frm = param.cmd.frms - if numel(param.combine.imgs) > 1 - % More than one image: Check for individual image files - for img = 1:length(param.combine.imgs) - out_fn_name = sprintf('Data_img_%02d_%s_%03d.mat',img,param.day_seg,frm); - out_fn = fullfile(combine_out_dir,out_fn_name); - sparam.success = cat(2,sparam.success, ... - sprintf(' error_mask = bitor(error_mask,%d*~exist(''%s'',''file''));\n', success_error, out_fn)); - if ~ctrl.cluster.rerun_only && exist(out_fn,'file') - delete(out_fn); - end - end - % If combining set, then check for combined file - if ~isempty(param.combine.img_comb) - out_fn_name = sprintf('Data_%s_%03d.mat',param.day_seg,frm); - out_fn = fullfile(combine_out_dir,out_fn_name); - sparam.success = cat(2,sparam.success, ... - sprintf(' error_mask = bitor(error_mask,%d*~exist(''%s'',''file''));\n', success_error, out_fn)); - if ~ctrl.cluster.rerun_only && exist(out_fn,'file') - delete(out_fn); - end - end - else - % Only one image: Check for combined file - out_fn_name = sprintf('Data_%s_%03d.mat',param.day_seg,frm); - out_fn = fullfile(combine_out_dir,out_fn_name); - sparam.success = cat(2,sparam.success, ... - sprintf(' error_mask = bitor(error_mask,%d*~exist(''%s'',''file''));\n', success_error, out_fn)); - if ~ctrl.cluster.rerun_only && exist(out_fn,'file') - delete(out_fn); - end - end -end - -ctrl = cluster_new_task(ctrl,sparam,[]); - -ctrl_chain{end+1} = ctrl; - -fprintf('Done %s\n', datestr(now)); - -return; diff --git a/cresis-toolbox/processing/combine_wf_chan_combine_task.m b/cresis-toolbox/processing/combine_wf_chan_combine_task.m deleted file mode 100644 index 4096f6ee..00000000 --- a/cresis-toolbox/processing/combine_wf_chan_combine_task.m +++ /dev/null @@ -1,218 +0,0 @@ -function success = combine_wf_chan_combine_task(param) -% success = combine_wf_chan_combine_task(param) -% -% This script combines the receive channels and outputs the result -% for each waveform. It also combines the waveforms. It takes in -% f-k files one directory at a time and: -% 1. combines the receive channels -% 2. concatenates the results -% 3. square-law detects the data, abs()^2 -% 4. takes incoherent averages (multilooks data) -% 5. saves the result in a new directory -% 6. Combines the waveforms -% -% The assumption is that the directories in the input_path are named -% using the following convention: -% PROC-TYPE-STRING_data_#{_SUBAPERTURE-STRING} -% where -% PROC-TYPE-STRING can be 'fk','tdbp', or 'pc' for f-k migrated,time domain -% back projected,and pulse compressed respectively ('fk' and tdbp supported) -% _data_ is always present -% #, \d+: one or more numbers -% _SUBAPERTURE-STRING, {_[mp]\d\.\d}: optional subaperture string -% Examples: -% fk_data_01_01: f-k migrated, frame 1, subaperture 1 -% fk_data_04_02: f-k migrated, frame 4, subaperture 2 -% fk_data_01_03: f-k migrated, frame 1, subaperture 3 -% pc_data_01: pulse compressed only, frame 1 -% -% param = struct with processing parameters -% -- OR -- -% function handle to script with processing parameters -% param_override = parameters in this struct will override parameters -% in param. This struct must also contain the gRadar fields. -% Typically global gRadar; param_override = gRadar; -% -% Authors: John Paden -% -% See also: run_master.m, master.m, run_combine_wf_chan.m, combine_wf_chan.m, -% combine_wf_chan_task.m - -%% Setup processing -% ===================================================================== - -% Create output directory path -combine_out_dir = ct_filename_out(param, param.combine.out_path); -if ~exist(combine_out_dir,'dir') - mkdir(combine_out_dir); -end - -load(ct_filename_support(param,'','frames')); % Load "frames" variable - -% Load records file -records_fn = ct_filename_support(param,'','records'); -records = load(records_fn); -% Apply presumming -if param.csarp.presums > 1 - records.lat = fir_dec(records.lat,param.csarp.presums); - records.lon = fir_dec(records.lon,param.csarp.presums); - records.elev = fir_dec(records.elev,param.csarp.presums); - records.roll = fir_dec(records.roll,param.csarp.presums); - records.pitch = fir_dec(records.pitch,param.csarp.presums); - records.heading = fir_dec(records.heading,param.csarp.presums); - records.gps_time = fir_dec(records.gps_time,param.csarp.presums); - records.surface = fir_dec(records.surface,param.csarp.presums); -end -along_track_approx = geodetic_to_along_track(records.lat,records.lon,records.elev); - -%% Combine chunks into each frame -% ===================================================================== -for frm_idx = 1:length(param.cmd.frms); - frm = param.cmd.frms(frm_idx); - - if ct_proc_frame(frames.proc_mode(frm),param.combine.frm_types) - fprintf('combine %s_%03i (%i of %i) %s\n', param.day_seg, frm, frm_idx, length(param.cmd.frms), datestr(now,'HH:MM:SS')); - else - fprintf('Skipping frame %s_%03i (no process frame)\n', param.day_seg, frm); - continue; - end - - % Temporary output directory for uncombined array processed images - array_fn_dir = fullfile(ct_filename_out(param, param.combine.array_path), ... - sprintf('%s_%03d', param.combine.method, frm)); - - % Current frame goes from the start record specified in the frames file - % to the record just before the start record of the next frame. For - % the last frame, the stop record is just the last record in the segment. - start_rec = ceil(frames.frame_idxs(frm)/param.csarp.presums); - if frm < length(frames.frame_idxs) - stop_rec = ceil((frames.frame_idxs(frm+1)-1)/param.csarp.presums); - else - stop_rec = length(records.gps_time); - end - - % Determine length of the frame - frm_dist = along_track_approx(stop_rec) - along_track_approx(start_rec); - - % Determine number of chunks for this frame - num_chunks = round(frm_dist / param.combine.chunk_len); - - %% Loop through all the images - for img = 1:length(param.combine.imgs) - %% Loop through all the chunks and combine into an image - Latitude = []; - Longitude = []; - Elevation = []; - Roll = []; - Pitch = []; - Heading = []; - GPS_time = []; - Surface = []; - Bottom = []; - Data = []; - Topography = []; - for chunk_idx = 1:num_chunks - array_fn = fullfile(array_fn_dir, sprintf('img_%02d_chk_%03d.mat', img, chunk_idx)); - tmp = load(array_fn); - Time = tmp.Time; - Latitude = [Latitude double(tmp.Latitude)]; - Longitude = [Longitude double(tmp.Longitude)]; - Elevation = [Elevation double(tmp.Elevation)]; - Roll = [Roll double(tmp.Roll)]; - Pitch = [Pitch double(tmp.Pitch)]; - Heading = [Heading double(tmp.Heading)]; - GPS_time = [GPS_time tmp.GPS_time]; - Surface = [Surface double(tmp.Surface)]; - Bottom = [Bottom double(tmp.Bottom)]; - Data = [Data tmp.Data]; - param_records = tmp.param_records; - param_csarp = tmp.param_csarp; - if chunk_idx == 1 - param_combine = tmp.param_combine; - param_combine.array_param.fcs{1}{1}.x = tmp.param_combine.array_param.fcs{1}{1}.x(:,tmp.param_combine.array_param.lines); - param_combine.array_param.fcs{1}{1}.y = tmp.param_combine.array_param.fcs{1}{1}.y(:,tmp.param_combine.array_param.lines); - param_combine.array_param.fcs{1}{1}.z = tmp.param_combine.array_param.fcs{1}{1}.z(:,tmp.param_combine.array_param.lines); - param_combine.array_param.fcs{1}{1}.origin = tmp.param_combine.array_param.fcs{1}{1}.origin(:,tmp.param_combine.array_param.lines); - else - % Concatenate the fcs field - param_combine.array_param.fcs{1}{1}.x = [param_combine.array_param.fcs{1}{1}.x tmp.param_combine.array_param.fcs{1}{1}.x(:,tmp.param_combine.array_param.lines)]; - param_combine.array_param.fcs{1}{1}.y = [param_combine.array_param.fcs{1}{1}.y tmp.param_combine.array_param.fcs{1}{1}.y(:,tmp.param_combine.array_param.lines)]; - param_combine.array_param.fcs{1}{1}.z = [param_combine.array_param.fcs{1}{1}.z tmp.param_combine.array_param.fcs{1}{1}.z(:,tmp.param_combine.array_param.lines)]; - param_combine.array_param.fcs{1}{1}.origin = [param_combine.array_param.fcs{1}{1}.origin tmp.param_combine.array_param.fcs{1}{1}.origin(:,tmp.param_combine.array_param.lines)]; - end - if isfield(tmp,'Topography') - % 3D-surface is present so concatenate it too - % Topography = cat(3,Topography,tmp.Topography); - % Concatenate all the fields under struct Topography: valR, bins, val, freq - % and img. - fields = fieldnames(tmp.Topography); - if chunk_idx == 1 - for field_idx = 1:length(fields) - Topography.(fields{field_idx}) = tmp.Topography.(fields{field_idx}); - end - else - for field_idx = 1:length(fields) - max_dim = length(size(tmp.Topography.(fields{field_idx}))); - Topography.(fields{field_idx}) = cat(max_dim,Topography.(fields{field_idx}),tmp.Topography.(fields{field_idx})); - end - end - - end - end - - % ===================================================================== - % Save output - if length(param.combine.imgs) == 1 - out_fn = fullfile(combine_out_dir, sprintf('Data_%s_%03d.mat', ... - param.day_seg, frm)); - else - out_fn = fullfile(combine_out_dir, sprintf('Data_img_%02d_%s_%03d.mat', ... - img, param.day_seg, frm)); - end - fprintf(' Writing output to %s\n', out_fn); - if isempty(Topography) - % Do not save 3D surface - save('-v7.3',out_fn,'Time','Latitude','Longitude', ... - 'Elevation','GPS_time','Data','Surface','Bottom', ... - 'param_combine','param_records','param_csarp', ... - 'Roll', 'Pitch', 'Heading'); - else - % Save 3D surface - save('-v7.3',out_fn,'Topography','Time','Latitude', ... - 'Longitude','Elevation','GPS_time','Data','Surface','Bottom', ... - 'param_combine','param_records','param_csarp', ... - 'Roll', 'Pitch', 'Heading'); - end - end - - if isempty(param.combine.img_comb) - % No image combining is required - continue; - end - - if length(param.combine.img_comb) ~= 3*(length(param.combine.imgs)-1) - warning('param.combine.img_comb not the right length. There should be 3 entries for each image combination interface ([Tpd second image for surface saturation, -inf for second image blank, Tpd first image to avoid roll off] is typical). Set correctly here and update param spreadsheet before dbcont.'); - keyboard - end - - %% Combine images - if isempty(param.combine.img_comb_layer_params) - layers.gps_time = GPS_time; - layers.twtt = interp_finite(Surface,0); - end - param.load.frm = frm; - [Data, Time] = img_combine(param, 'combine', layers); - - %% Save output - out_fn = fullfile(combine_out_dir, sprintf('Data_%s_%03d.mat', ... - param.day_seg, frm)); - fprintf(' Writing output to %s\n', out_fn); - save('-v7.3',out_fn,'Time','Latitude','Longitude', ... - 'Elevation','GPS_time','Data','Surface','Bottom', ... - 'param_combine','param_records','param_csarp', ... - 'Roll', 'Pitch', 'Heading'); -end - -fprintf('%s done %s\n', mfilename, datestr(now)); - -success = true; diff --git a/cresis-toolbox/processing/combine_wf_chan_combine_task_ollie.m b/cresis-toolbox/processing/combine_wf_chan_combine_task_ollie.m deleted file mode 100644 index b32c26a9..00000000 --- a/cresis-toolbox/processing/combine_wf_chan_combine_task_ollie.m +++ /dev/null @@ -1,110 +0,0 @@ -function combine_wf_chan_combine_task_ollie(static_param_file_name) -% combine_wf_chan_combine_task_ollie(static_param_file_name) -% -% This functions is compiled and used for manual job submission to Slurm on -% Ollie using batch_combine_2.sh. -% -load(static_param_file_name,'static_param'); -param=static_param; - -if ~isfield(param.combine,'frm_types') || isempty(param.combine.frm_types) - param.combine.frm_types = {-1,-1,-1,-1,-1}; -end - -% Remove frames that do not exist from param.cmd.frms list -load(ct_filename_support(param,'','frames')); % Load "frames" variable -if ~isfield(param.cmd,'frms') || isempty(param.cmd.frms) - param.cmd.frms = 1:length(frames.frame_idxs); -end -[valid_frms,keep_idxs] = intersect(param.cmd.frms, 1:length(frames.frame_idxs)); -if length(valid_frms) ~= length(param.cmd.frms) - bad_mask = ones(size(param.cmd.frms)); - bad_mask(keep_idxs) = 0; - warning('Nonexistent frames specified in param.cmd.frms (e.g. frame "%g" is invalid), removing these', ... - param.cmd.frms(find(bad_mask,1))); - param.cmd.frms = valid_frms; -end - -if ~isfield(param.combine,'img_comb_layer_params') - param.combine.img_comb_layer_params = []; -end - -if ~isfield(param.combine,'trim_time') - param.combine.trim_time = true; -end - -if ~isfield(param.csarp,'pulse_comp') || isempty(param.csarp.pulse_comp) - param.csarp.pulse_comp = 1; -end - -if ~isfield(param.csarp,'presums') || isempty(param.csarp.presums) - param.csarp.presums = 1; -end - -if ~isfield(param.combine,'in_path') || isempty(param.combine.in_path) - param.combine.in_path = 'out'; -end - -if ~isfield(param.combine,'array_path') || isempty(param.combine.array_path) - param.combine.array_path = 'out'; -end - -if ~isfield(param.combine,'out_path') || isempty(param.combine.out_path) - param.combine.out_path = param.combine.method; -end - -if ~isfield(param.csarp,'out_path') || isempty(param.csarp.out_path) - param.csarp.out_path = 'out'; -end - -if ~isfield(param.combine,'presums') || isempty(param.combine.presums) - if ~isfield(param.csarp,'presums') || isempty(param.csarp.presums) - param.combine.presums = 1; - else - param.combine.presums = param.csarp.presums; - end -end - -if ~isfield(param.combine,'sar_type') || isempty(param.combine.sar_type) - if ~isfield(param.csarp,'sar_type') || isempty(param.csarp.sar_type) - param.combine.sar_type = 'fk'; - else - param.combine.sar_type = param.csarp.sar_type; - end -end - -if strcmpi(param.combine.sar_type,'f-k') - error('Deprecated sar_type name. Change param.combine.sar_type from ''f-k'' to ''fk'' in your parameters (or remove parameter since ''fk'' is the default mode).'); -end - -if ~isfield(param.combine,'chunk_len') || isempty(param.combine.chunk_len) - if ~isfield(param.csarp,'chunk_len') || isempty(param.csarp.chunk_len) - error('param.combine.chunk_len or param.csarp.chunk_len must be defined'); - else - param.combine.chunk_len = param.csarp.chunk_len; - end -end - -% Handles multilooking syntax: -% {{[1 1],[1 2],[1 3],[1 4],[1 5]},{[2 1],[2 2],[2 3],[2 4],[2 5]}} -% If the image is a cell array it describes multilooking across apertures -if ~iscell(param.combine.imgs{1}) - % No special multilooking, reformat old syntax to new multilooking syntax - for img = 1:length(param.combine.imgs) - param.combine.imgs{img} = {param.combine.imgs{img}}; - end -end - -for img = 1:length(param.combine.imgs) - for ml_idx = 1:length(param.combine.imgs{img}) - % Imaginary image indices is for IQ combining during raw data load - % which we do not need here. - param.combine.imgs{img}{ml_idx} = abs(param.combine.imgs{img}{ml_idx}); - end -end - -if ~isfield(param.combine,'out_path') || isempty(param.combine.out_path) - param.combine.out_path = param.combine.method; -end - -success = combine_wf_chan_combine_task(param); \ No newline at end of file diff --git a/cresis-toolbox/processing/combine_wf_chan_task.m b/cresis-toolbox/processing/combine_wf_chan_task.m deleted file mode 100644 index 455df78a..00000000 --- a/cresis-toolbox/processing/combine_wf_chan_task.m +++ /dev/null @@ -1,562 +0,0 @@ -function success = combine_wf_chan_task(param) -% success = combine_wf_chan_task(param) -% -% Task/job running on cluster that is called from combine. This -% function is generally not called directly. -% -% param.combine = structure controlling the processing -% Fields used in this function (and potentially array_proc.m) -% .imgs = images to process -% .method = 'csarp-combined', 'standard', 'mvdr', 'music', etc. -% .three_dim.en = boolean, enable 3-D mode -% .Nsv = used to compute sv -% .sv = steering vector function handle (empty for default) -% .in_path = input path string -% .out_path = output path string -% -% Fields used by array_proc.m only -% .window -% .bin_rng -% .rline_rng -% .dbin -% .dline -% .first_line = required to stitch multiple chunks together -% .chan_equal -% .freq_rng -% -% success = boolean which is true when the function executes properly -% if a task fails before it can return success, then success will be -% empty -% -% Authors: John Paden -% -% See also: combine, array_proc - -% ===================================================================== -% Preparation - -% speed of light, wgs84 ellipsoid -physical_constants; - -% Rename for code readability -sar_type = param.combine.sar_type; -data_field_name = sprintf('%s_data', sar_type); - -% Input directory (SAR SLC images) -in_fn_dir = fullfile(ct_filename_out(param, param.combine.in_path), ... - sprintf('%s_data_%03d_01_01', sar_type, param.load.frm)); - -% Temporary output directory for uncombined array processed images -combine_tmp_dir = fullfile(ct_filename_out(param, param.combine.array_path), ... - sprintf('%s_%03d', param.combine.method, param.load.frm)); - -% ===================================================================== -% Process data -% ===================================================================== -for img = 1:length(param.combine.imgs) - ml_list = param.combine.imgs{img}; - - %% Convert old syntax to new multilooking across SLC images syntax - % SLC = single look complex SAR echogram - if ~iscell(ml_list) - ml_list = {ml_list}; - end - wf_base = ml_list{1}(1,1); - - % ------------------------------------------------------------------- - %% Load data - % wf_adc_list = {[1 1],[1 2],[1 3],[1 4],[1 5]} - % This will multilook these 5 SLC images (e.g. add incoherently in the case - % of periodogram) - % wf_adc_list = {[1 1; 1 2; 1 3; 1 4; 1 5]} - % This will coherently add these 5 SLC images - if strcmpi(param.combine.method,'csarp-combined') - % CSARP-combined has already coherently combined every image in each - % image list, so we only grab the first image. Multilooking across - % SLCs IS NOT supported for this method! - ml_list = {ml_list{1}(1,:)}; - end - if ~isfield(param.combine,'subaps') || isempty(param.combine.subaps) - % If SAR sub-apertures not set, we assume that there is just one - % subaperture to be passed in for each multilook input - for ml_idx = 1:length(ml_list) - param.combine.subaps{ml_idx} = [1]; - end - end - if ~isfield(param.combine,'subbnds') || isempty(param.combine.subbnds) - % If subbands not set, we assume that there is just one - % subaperture to be passed in for each multilook input - for ml_idx = 1:length(ml_list) - param.combine.subbnds{ml_idx} = [1]; - end - end - clear fcs chan_equal data lat lon elev; - num_next_rlines = 0; - prev_chunk_failed_flag = false; - next_chunk_failed_flag = false; - % num_prev_chunk_rlines: number of range lines to load from the previous chunk - num_prev_chunk_rlines = max(-param.combine.rline_rng); - % num_next_chunk_rlines: number of range lines to load from the previous chunk - num_next_chunk_rlines = max(param.combine.rline_rng); - % out_rlines: output range lines from SAR processor for the current chunk - sar_out_rlines = []; - for ml_idx = 1:length(ml_list) - wf_adc_list = ml_list{ml_idx}; - for wf_adc_idx = 1:size(wf_adc_list,1) - wf = wf_adc_list(wf_adc_idx,1); - adc = wf_adc_list(wf_adc_idx,2); - for subap = param.combine.subaps{ml_idx} - for subbnd = param.combine.subbnds{ml_idx} - in_fn_dir(end-4:end-3) = sprintf('%02d',subap); - in_fn_dir(end-1:end) = sprintf('%02d',subbnd); - - % Load previous chunk data - % =============================================================== - - % If on the first chunk of the frame, then look at the previous - % frame. Special cases (like frm == 1 and empty - % param.load.prev_frm_num_chunks are handled by letting the file - % search fail). - if param.load.chunk_idx == 1 - load_frm = param.load.frm-1; - load_chunk_idx = param.load.prev_frm_num_chunks; - else - load_frm = param.load.frm; - load_chunk_idx = param.load.chunk_idx - 1; - end - - % Create the filename - in_fn_dir(end-8:end-6) = sprintf('%03d',load_frm); - if strcmpi(param.combine.method,'csarp-combined') - [sar_type_fn,status] = get_filenames(in_fn_dir, ... - sprintf('img_%02.0f_chk_%03.0f', img, load_chunk_idx),'','.mat'); - else - [sar_type_fn,status] = get_filenames(in_fn_dir, ... - sprintf('wf_%02.0f_adc_%02.0f_chk_%03.0f', wf, adc, load_chunk_idx),'','.mat'); - end - - if ~prev_chunk_failed_flag && ~isempty(sar_type_fn) - % If file exists, then load it - sar_data = load(sar_type_fn{1}); - rlines = 1:num_prev_chunk_rlines; - - if size(sar_data.(data_field_name),2) >= num_prev_chunk_rlines - if subap == param.combine.subaps{ml_idx}(1) && subbnd == param.combine.subbnds{ml_idx}(1) - fcs{ml_idx}{wf_adc_idx}.origin(:,rlines) = sar_data.fcs.origin(:,end-num_prev_chunk_rlines+1:end); - fcs{ml_idx}{wf_adc_idx}.x(:,rlines) = sar_data.fcs.x(:,end-num_prev_chunk_rlines+1:end); - fcs{ml_idx}{wf_adc_idx}.y(:,rlines) = sar_data.fcs.y(:,end-num_prev_chunk_rlines+1:end); - fcs{ml_idx}{wf_adc_idx}.z(:,rlines) = sar_data.fcs.z(:,end-num_prev_chunk_rlines+1:end); - fcs{ml_idx}{wf_adc_idx}.roll(rlines) = sar_data.fcs.roll(end-num_prev_chunk_rlines+1:end); - fcs{ml_idx}{wf_adc_idx}.pitch(rlines) = sar_data.fcs.pitch(end-num_prev_chunk_rlines+1:end); - fcs{ml_idx}{wf_adc_idx}.heading(rlines) = sar_data.fcs.heading(end-num_prev_chunk_rlines+1:end); - fcs{ml_idx}{wf_adc_idx}.gps_time(rlines) = sar_data.fcs.gps_time(end-num_prev_chunk_rlines+1:end); - fcs{ml_idx}{wf_adc_idx}.surface(rlines) = sar_data.fcs.surface(end-num_prev_chunk_rlines+1:end); - fcs{ml_idx}{wf_adc_idx}.bottom(rlines) = sar_data.fcs.bottom(end-num_prev_chunk_rlines+1:end); - fcs{ml_idx}{wf_adc_idx}.pos(:,rlines) = sar_data.fcs.pos(:,end-num_prev_chunk_rlines+1:end); - end - - data{ml_idx}(:,rlines,subap,subbnd,wf_adc_idx) = sar_data.(data_field_name)(:,end-num_prev_chunk_rlines+1:end); - lat(rlines) = sar_data.lat(:,end-num_prev_chunk_rlines+1:end); - lon(rlines) = sar_data.lon(:,end-num_prev_chunk_rlines+1:end); - elev(rlines) = sar_data.elev(:,end-num_prev_chunk_rlines+1:end); - else - prev_chunk_failed_flag = true; - end - else - % If file does not exist, then give up loading previous chunk - % data - prev_chunk_failed_flag = true; - end - - % Load current chunk data - % =============================================================== - load_frm = param.load.frm; - load_chunk_idx = param.load.chunk_idx; - in_fn_dir(end-8:end-6) = sprintf('%03d',load_frm); - if strcmpi(param.combine.method,'csarp-combined') - [sar_type_fn,status] = get_filenames(in_fn_dir, ... - sprintf('img_%02.0f_chk_%03.0f', img, load_chunk_idx),'','.mat'); - else - [sar_type_fn,status] = get_filenames(in_fn_dir, ... - sprintf('wf_%02.0f_adc_%02.0f_chk_%03.0f', wf, adc, load_chunk_idx),'','.mat'); - end - sar_data = load(sar_type_fn{1}); - num_rlines = size(sar_data.(data_field_name),2); - rlines = num_prev_chunk_rlines + (1:num_rlines); - - if subap == param.combine.subaps{ml_idx}(1) && subbnd == param.combine.subbnds{ml_idx}(1) - fcs{ml_idx}{wf_adc_idx}.Lsar = sar_data.fcs.Lsar; - fcs{ml_idx}{wf_adc_idx}.gps_source = sar_data.fcs.gps_source; - fcs{ml_idx}{wf_adc_idx}.origin(:,rlines) = sar_data.fcs.origin; - fcs{ml_idx}{wf_adc_idx}.x(:,rlines) = sar_data.fcs.x; - fcs{ml_idx}{wf_adc_idx}.y(:,rlines) = sar_data.fcs.y; - fcs{ml_idx}{wf_adc_idx}.z(:,rlines) = sar_data.fcs.z; - fcs{ml_idx}{wf_adc_idx}.roll(rlines) = sar_data.fcs.roll; - fcs{ml_idx}{wf_adc_idx}.pitch(rlines) = sar_data.fcs.pitch; - fcs{ml_idx}{wf_adc_idx}.heading(rlines) = sar_data.fcs.heading; - fcs{ml_idx}{wf_adc_idx}.gps_time(rlines) = sar_data.fcs.gps_time; - fcs{ml_idx}{wf_adc_idx}.surface(rlines) = sar_data.fcs.surface; - fcs{ml_idx}{wf_adc_idx}.bottom(rlines) = sar_data.fcs.bottom; - fcs{ml_idx}{wf_adc_idx}.pos(:,rlines) = sar_data.fcs.pos; - fcs{ml_idx}{wf_adc_idx}.squint = sar_data.fcs.squint; - if isfield(sar_data.fcs,'type') - fcs{ml_idx}{wf_adc_idx}.type = sar_data.fcs.type; - fcs{ml_idx}{wf_adc_idx}.filter = sar_data.fcs.filter; - else - fcs{ml_idx}{wf_adc_idx}.type = []; - fcs{ml_idx}{wf_adc_idx}.filter = []; - end - sar_out_rlines = sar_data.out_rlines; - - chan_equal{ml_idx}(wf_adc_idx) = 10.^(param.radar.wfs(wf).chan_equal_dB(param.radar.wfs(wf).rx_paths(adc))/20) ... - .* exp(j*param.radar.wfs(wf).chan_equal_deg(param.radar.wfs(wf).rx_paths(adc))/180*pi); - end - - data{ml_idx}(:,rlines,subap,subbnd,wf_adc_idx) = sar_data.(data_field_name); - lat(rlines) = sar_data.lat; - lon(rlines) = sar_data.lon; - elev(rlines) = sar_data.elev; - - % Load next chunk data - % =============================================================== - - % If on the first chunk of the frame, then look at the previous - % frame. Special cases (like frm == 1 and empty - % param.load.prev_frm_num_chunks are handled by letting the file - % search fail). - if param.load.chunk_idx == param.load.num_chunks - load_frm = param.load.frm+1; - load_chunk_idx = 1; - else - load_frm = param.load.frm; - load_chunk_idx = param.load.chunk_idx + 1; - end - - % Create the filename - in_fn_dir(end-8:end-6) = sprintf('%03d',load_frm); - if strcmpi(param.combine.method,'csarp-combined') - [sar_type_fn,status] = get_filenames(in_fn_dir, ... - sprintf('img_%02.0f_chk_%03.0f', img, load_chunk_idx),'','.mat'); - else - [sar_type_fn,status] = get_filenames(in_fn_dir, ... - sprintf('wf_%02.0f_adc_%02.0f_chk_%03.0f', wf, adc, load_chunk_idx),'','.mat'); - end - - if ~next_chunk_failed_flag && ~isempty(sar_type_fn) - % If file exists, then load it - sar_data = load(sar_type_fn{1}); - rlines = num_prev_chunk_rlines + num_rlines + (1:num_next_chunk_rlines); - - if size(sar_data.(data_field_name),2) >= num_next_chunk_rlines - if subap == param.combine.subaps{ml_idx}(1) && subbnd == param.combine.subbnds{ml_idx}(1) - fcs{ml_idx}{wf_adc_idx}.origin(:,rlines) = sar_data.fcs.origin(:,1:num_next_chunk_rlines); - fcs{ml_idx}{wf_adc_idx}.x(:,rlines) = sar_data.fcs.x(:,1:num_next_chunk_rlines); - fcs{ml_idx}{wf_adc_idx}.y(:,rlines) = sar_data.fcs.y(:,1:num_next_chunk_rlines); - fcs{ml_idx}{wf_adc_idx}.z(:,rlines) = sar_data.fcs.z(:,1:num_next_chunk_rlines); - fcs{ml_idx}{wf_adc_idx}.roll(rlines) = sar_data.fcs.roll(1:num_next_chunk_rlines); - fcs{ml_idx}{wf_adc_idx}.pitch(rlines) = sar_data.fcs.pitch(1:num_next_chunk_rlines); - fcs{ml_idx}{wf_adc_idx}.heading(rlines) = sar_data.fcs.heading(1:num_next_chunk_rlines); - fcs{ml_idx}{wf_adc_idx}.gps_time(rlines) = sar_data.fcs.gps_time(1:num_next_chunk_rlines); - fcs{ml_idx}{wf_adc_idx}.surface(rlines) = sar_data.fcs.surface(1:num_next_chunk_rlines); - fcs{ml_idx}{wf_adc_idx}.bottom(rlines) = sar_data.fcs.bottom(1:num_next_chunk_rlines); - fcs{ml_idx}{wf_adc_idx}.pos(:,rlines) = sar_data.fcs.pos(:,1:num_next_chunk_rlines); - end - - data{ml_idx}(:,rlines,subap,subbnd,wf_adc_idx) = sar_data.(data_field_name)(:,1:num_next_chunk_rlines); - lat(rlines) = sar_data.lat(:,1:num_next_chunk_rlines); - lon(rlines) = sar_data.lon(:,1:num_next_chunk_rlines); - elev(rlines) = sar_data.elev(:,1:num_next_chunk_rlines); - else - next_chunk_failed_flag = true; - end - else - % If file does not exist, then give up loading previous chunk - % data - next_chunk_failed_flag = true; - end - - end - end - end - end - - if prev_chunk_failed_flag - % Remove prev chunk data - for ml_idx = 1:length(ml_list) - data{ml_idx} = data{ml_idx}(:,num_prev_chunk_rlines+1:end,:,:,:); - lat = lat(num_prev_chunk_rlines+1:end); - lon = lon(num_prev_chunk_rlines+1:end); - elev = elev(num_prev_chunk_rlines+1:end); - for wf_adc_idx = 1:size(wf_adc_list,1) - fcs{ml_idx}{wf_adc_idx}.origin = fcs{ml_idx}{wf_adc_idx}.origin(:,num_prev_chunk_rlines+1:end); - fcs{ml_idx}{wf_adc_idx}.x = fcs{ml_idx}{wf_adc_idx}.x(:,num_prev_chunk_rlines+1:end); - fcs{ml_idx}{wf_adc_idx}.y = fcs{ml_idx}{wf_adc_idx}.y(:,num_prev_chunk_rlines+1:end); - fcs{ml_idx}{wf_adc_idx}.z = fcs{ml_idx}{wf_adc_idx}.z(:,num_prev_chunk_rlines+1:end); - fcs{ml_idx}{wf_adc_idx}.roll = fcs{ml_idx}{wf_adc_idx}.roll(num_prev_chunk_rlines+1:end); - fcs{ml_idx}{wf_adc_idx}.pitch = fcs{ml_idx}{wf_adc_idx}.pitch(num_prev_chunk_rlines+1:end); - fcs{ml_idx}{wf_adc_idx}.heading = fcs{ml_idx}{wf_adc_idx}.heading(num_prev_chunk_rlines+1:end); - fcs{ml_idx}{wf_adc_idx}.gps_time = fcs{ml_idx}{wf_adc_idx}.gps_time(num_prev_chunk_rlines+1:end); - fcs{ml_idx}{wf_adc_idx}.surface = fcs{ml_idx}{wf_adc_idx}.surface(num_prev_chunk_rlines+1:end); - fcs{ml_idx}{wf_adc_idx}.bottom = fcs{ml_idx}{wf_adc_idx}.bottom(num_prev_chunk_rlines+1:end); - fcs{ml_idx}{wf_adc_idx}.pos = fcs{ml_idx}{wf_adc_idx}.pos(:,num_prev_chunk_rlines+1:end); - end - end - num_prev_chunk_rlines = 0; - end - - if next_chunk_failed_flag - % Remove next chunk data - for ml_idx = 1:length(ml_list) - data{ml_idx} = data{ml_idx}(:,1:end-num_next_chunk_rlines,:,:,:); - lat = lat(1:end-num_next_chunk_rlines); - lon = lon(1:end-num_next_chunk_rlines); - elev = elev(1:end-num_next_chunk_rlines); - for wf_adc_idx = 1:size(wf_adc_list,1) - fcs{ml_idx}{wf_adc_idx}.origin = fcs{ml_idx}{wf_adc_idx}.origin(:,1:end-num_next_chunk_rlines); - fcs{ml_idx}{wf_adc_idx}.x = fcs{ml_idx}{wf_adc_idx}.x(:,1:end-num_next_chunk_rlines); - fcs{ml_idx}{wf_adc_idx}.y = fcs{ml_idx}{wf_adc_idx}.y(:,1:end-num_next_chunk_rlines); - fcs{ml_idx}{wf_adc_idx}.z = fcs{ml_idx}{wf_adc_idx}.z(:,1:end-num_next_chunk_rlines); - fcs{ml_idx}{wf_adc_idx}.roll = fcs{ml_idx}{wf_adc_idx}.roll(1:end-num_next_chunk_rlines); - fcs{ml_idx}{wf_adc_idx}.pitch = fcs{ml_idx}{wf_adc_idx}.pitch(1:end-num_next_chunk_rlines); - fcs{ml_idx}{wf_adc_idx}.heading = fcs{ml_idx}{wf_adc_idx}.heading(1:end-num_next_chunk_rlines); - fcs{ml_idx}{wf_adc_idx}.gps_time = fcs{ml_idx}{wf_adc_idx}.gps_time(1:end-num_next_chunk_rlines); - fcs{ml_idx}{wf_adc_idx}.surface = fcs{ml_idx}{wf_adc_idx}.surface(1:end-num_next_chunk_rlines); - fcs{ml_idx}{wf_adc_idx}.bottom = fcs{ml_idx}{wf_adc_idx}.bottom(1:end-num_next_chunk_rlines); - fcs{ml_idx}{wf_adc_idx}.pos = fcs{ml_idx}{wf_adc_idx}.pos(:,1:end-num_next_chunk_rlines); - end - end - num_next_chunk_rlines = 0; - end - - if isfield(param.combine,'ft_over_sample') && ~isempty(param.combine.ft_over_sample) - % param.combine.ft_over_sample should be a positive integer - for ml_idx = 1:length(data) - data{ml_idx} = interpft(data{ml_idx},size(data{ml_idx},1) * param.combine.ft_over_sample); - end - for wf = 1:length(sar_data.wfs) - sar_data.wfs(wf).fs = sar_data.wfs(wf).fs * param.combine.ft_over_sample; - sar_data.wfs(wf).dt = 1/sar_data.wfs(wf).fs; - sar_data.wfs(wf).Nt = sar_data.wfs(wf).Nt * param.combine.ft_over_sample; - sar_data.wfs(wf).df = sar_data.wfs(wf).fs / sar_data.wfs(wf).Nt; - sar_data.wfs(wf).time = sar_data.wfs(wf).time(1) + sar_data.wfs(wf).dt*(0:sar_data.wfs(wf).Nt-1).'; - sar_data.wfs(wf).freq = sar_data.wfs(wf).fc ... - + sar_data.wfs(wf).df * ifftshift( -floor(sar_data.wfs(wf).Nt/2) : floor((sar_data.wfs(wf).Nt-1)/2) ).'; - end - end - - % Setup fields for wideband space-time doa estimator passed to - % array_proc. This section does the following: - % 1). Computes the impulse response used to model the amplitude term of - % the space time correlation matrix, - if strcmpi(param.combine.method,'wideband') - % ------------------------------------------------------------------- - % Compute maximum propagation delay across the array (this is used to - % compute impulse response needed for covariance model (see below)) - - lever_arm_param.season_name = param.season_name; - lever_arm_param.radar_name = ct_output_dir(param.radar_name); - lever_arm_param.gps_source = sar_data.param_records.gps_source; - % Iterate through all wf-adc pairs and create a list of phase centers - % that are being used to create this image. - phase_centers = []; - for ml_idx = 1:length(ml_list) - wf_adc_list = ml_list{ml_idx}; - for wf_adc_idx = 1:size(wf_adc_list,1) - wf = wf_adc_list(wf_adc_idx,1); - adc = wf_adc_list(wf_adc_idx,2); - % Add phase center for the wf-adc pair to the list - phase_centers(:,end+1) = lever_arm(lever_arm_param, ... - param.radar.wfs(wf).tx_weights, param.radar.wfs(wf).rx_paths(adc)); - end - end - % Find the two phase centers separated by the maximum distance - pc_dist = zeros(size(phase_centers,2)); - for pc_idx = 1:size(phase_centers,2) - for pc_comp_idx = pc_idx+1:size(phase_centers,2) - pc_dist(pc_idx,pc_comp_idx) ... - = sqrt(sum(abs(phase_centers(:,pc_idx) - phase_centers(:,pc_comp_idx)).^2)); - end - end - max_array_dim = max(max(pc_dist)); - % Convert maximum distance to propagation time in free space - tau_max = 2*max_array_dim/c; - - % Check the value for W from param.combine.W (widening factor) - W_ideal = 1 + ceil(tau_max / sar_data.wfs(wf).dt); - if param.combine.W ~= W_ideal - warning('param.combine.W is %d, but should be %d', param.combine.W, W_ideal); - end - - % Compute impulse response - % ------------------------------------------------------------------- - % NOTE: The impulse response is computed based on the fast time - % window used in CSARP. This could be generalized for a measured - % impulse response vector but this is not yet supported. - - Mt = 10; % Over-sampling factor - % Create frequency-domain fast-time window identical to what was - % used for pulse compression - Hwin = sar_data.param_csarp.csarp.ft_wind(length(sar_data.wfs(wf).time)); - % Convert to time-domain and over-sample by Mt - % - Take real to remove rounding errors that result in imag part - Hwin = interpft(real(ifft(ifftshift(Hwin).^2)), Mt*length(Hwin)); - % Store the impulse response and corresponding time axis for - % passing to array_proc - % - Ensure we grab enough samples of the impulse response so that - % array_proc is always happy. - Hwin_num_samp = 2 * Mt * (W_ideal + param.combine.W); - param.combine.imp_resp.vals ... - = fftshift(Hwin([1:1+Hwin_num_samp, end-Hwin_num_samp+1:end])); - param.combine.imp_resp.time_vec ... - = sar_data.wfs(wf).dt/Mt * (-Hwin_num_samp:Hwin_num_samp); - end - - if strcmpi(param.combine.method,'csarp-combined') - %% csarp.m already combined channels, just incoherently average - - % Number of fast-time samples in the data - Nt = size(data{1},1); - - param.combine.bins = numel(param.combine.bin_rng)/2+0.5 : param.combine.dbin ... - : Nt-(numel(param.combine.bin_rng)/2-0.5); - - param.combine.lines = param.combine.rlines(1,chunk_idx): param.combine.dline ... - : min(param.combine.rlines(2,chunk_idx),size(data{1},2)-max(param.combine.rline_rng)); - - % Perform incoherent averaging - Hfilter2 = ones(length(param.combine.bin_rng),length(param.combine.rline_rng)); - Hfilter2 = Hfilter2 / numel(Hfilter2); - tomo.val = filter2(Hfilter2, abs(data{1}).^2); - - tomo.val = tomo.val(param.combine.bins,param.combine.lines); - - param.combine.chan_equal = chan_equal; - param.combine.fcs = fcs; - array_param = param.combine; - - else - %% Regular array processing operation - - % Load bin restriction layers if specified - if isfield(param.combine,'bin_restriction') && ~isempty(param.combine.bin_restriction) - ops_param = param; - % Invalid frames will be ignored by opsLoadLayers so we can ignore edge cases - ops_param.cmd.frms = param.load.frm + (-1:1); - param.combine.bin_restriction = opsLoadLayers(ops_param, param.combine.bin_restriction); - for idx=1:2 - % Interpolate layers onto GPS time of loaded data - param.combine.bin_restriction(idx).bin = interp1(param.combine.bin_restriction(idx).gps_time, ... - param.combine.bin_restriction(idx).twtt, sar_data.fcs.gps_time); - % Convert from two way travel time to range bins - param.combine.bin_restriction(idx).bin = interp1(sar_data.wfs(wf).time, ... - 1:length(sar_data.wfs(wf).time), param.combine.bin_restriction(idx).bin); - % Ensure there is a value everywhere - param.combine.bin_restriction(idx).bin = interp_finite(param.combine.bin_restriction(idx).bin); - end - end - - if isfield(param.combine,'doa_constraints') && ~isempty(param.combine.doa_constraints) - % If DOA constraints are specified, they must be specified for - % every signal. For no constraints, choose 'fixed' method and - % [-pi/2 pi/2] for init_src_limits and src_limits. - for src_idx = 1:param.combine.Nsig - % Load layers for each DOA constraint that needs it - doa_res = param.combine.doa_constraints(src_idx); - switch (doa_res.method) - case {'layerleft','layerright'} - ops_param = param; - % Invalid frames will be ignored by opsLoadLayers so we can ignore edge cases - ops_param.cmd.frms = param.load.frm + (-1:1); - param.combine.doa_constraints(src_idx).layer = opsLoadLayers(ops_param, doa_res.params); - % Interpolate layers onto GPS time of loaded data - param.combine.doa_constraints(src_idx).layer.twtt = interp1(param.combine.doa_constraints(src_idx).layer.gps_time, ... - param.combine.doa_constraints(src_idx).layer.twtt, sar_data.fcs.gps_time); - % Ensure there is a value everywhere - param.combine.doa_constraints(src_idx).layer.twtt = interp_finite(param.combine.doa_constraints(src_idx).layer.twtt); - end - end - end - - param.combine.chan_equal = chan_equal; - param.combine.fcs = fcs; - array_param = param.combine; - array_param.wfs = sar_data.wfs(wf); - array_param.imgs = param.combine.imgs{img}; - if ~iscell(array_param.imgs) - array_param.imgs = {array_param.imgs}; - end - - % Determine output range lines so that chunks will be seamless (this is - % necessary because the output is decimated and the decimation may not - % align with chunk lengths) - first_rline = find(~mod(sar_out_rlines-1,param.combine.dline),1); - array_param.rlines = num_prev_chunk_rlines + (first_rline : param.combine.dline : length(sar_out_rlines)); - array_param.rlines = array_param.rlines([1 end]); - - % Array Processing Function Call - [array_param,tomo] = array_proc(array_param,data); - param.combine.bins = array_param.bins; - param.combine.lines = array_param.lines; - end - - %% Debugging - % ------------------------------------------------------------------- - if 0 - figure(1); clf; - %imagesc(incfilt(mean(data{1},3),10,4)); - imagesc(10*log10(local_detrend(abs(mean(data{1},3)).^2,[40 100],[10 4],3))); - title(sprintf('Waveform %d: Boxcar window', wf)); - colorbar - grid on; - figure(2); clf; - %imagesc(10*log10(tomo.val)); - imagesc(10*log10(local_detrend(tomo.val,[20 50],[5 2],3))); - title(sprintf('Waveform %d: Tomography', wf)); - colorbar - grid on; - keyboard - end - - %% Save results - % ----------------------------------------------------------------------- - array_fn = fullfile(combine_tmp_dir, sprintf('img_%02d_chk_%03d.mat', img, param.load.chunk_idx)); - fprintf(' Saving combine_wf_chan data %s (%s)\n', array_fn, datestr(now)); - combine_tmp_dir = fileparts(array_fn); - if ~exist(combine_tmp_dir,'dir') - mkdir(combine_tmp_dir); - end - - Latitude = lat(1,array_param.lines); - Longitude = lon(1,array_param.lines); - Elevation = elev(1,array_param.lines); - GPS_time = fcs{1}{1}.gps_time(array_param.lines); - Surface = fcs{1}{1}.surface(array_param.lines); - Bottom = fcs{1}{1}.bottom(array_param.lines); - Roll = fcs{1}{1}.roll(array_param.lines); - Pitch = fcs{1}{1}.pitch(array_param.lines); - Heading = fcs{1}{1}.heading(array_param.lines); - Data = tomo.val; - Time = sar_data.wfs(wf_base).time(array_param.bins); - param_records = sar_data.param_records; - param_csarp = sar_data.param_csarp; - array_param.sv = []; % Set this to empty because it is so large - param_combine = param; - param_combine.array_param = array_param; - if ~param.combine.three_dim.en - % Do not save 3D-image - save('-v7.3',array_fn,'Data','Latitude','Longitude','Elevation','GPS_time', ... - 'Surface','Bottom','Time','param_combine','param_records', ... - 'param_csarp', 'Roll', 'Pitch', 'Heading'); - else - % Save 3D-image in file - Topography = tomo; - save('-v7.3',array_fn,'Topography','Data','Latitude','Longitude','Elevation','GPS_time', ... - 'Surface','Bottom','Time','param_combine','param_records', ... - 'param_csarp', 'Roll', 'Pitch', 'Heading'); - end - -end - -fprintf('%s done %s\n', mfilename, datestr(now)); - -success = true; diff --git a/cresis-toolbox/processing/combine_wf_chan_task_ollie.m b/cresis-toolbox/processing/combine_wf_chan_task_ollie.m deleted file mode 100644 index ffaf0aa7..00000000 --- a/cresis-toolbox/processing/combine_wf_chan_task_ollie.m +++ /dev/null @@ -1,128 +0,0 @@ -function combine_wf_chan_task_ollie(static_param_file_name,dynamic_param_file_name,frm,chunk_id) -% combine_wf_chan_task_ollie(static_param_file_name,dynamic_param_file_name,frm,chunk_id) -% -% This functions is compiled and used for manual job submission to Slurm on -% Ollie using batch_combine.sh. -% -if (ischar(frm)) - frm=str2num(frm); -end -if (ischar(chunk_id)) - chunk_id=str2num(chunk_id); -end - -load(static_param_file_name,'static_param'); -param=static_param; -load(dynamic_param_file_name,'dynamic_param'); -param.load.frm = frm; -param.load.chunk_idx = chunk_id; -param.load.num_chunks = dynamic_param.frms.(['frm',num2str(frm)]).num_chunks; -if (frm>1) - param.load.prev_frm_num_chunks = dynamic_param.frms.(['frm',num2str(frm-1)]).num_chunks; -else - param.load.prev_frm_num_chunks = []; -end - -clear frm chunk_id; - -if ~isfield(param.combine,'frm_types') || isempty(param.combine.frm_types) - param.combine.frm_types = {-1,-1,-1,-1,-1}; -end - -% Remove frames that do not exist from param.cmd.frms list -load(ct_filename_support(param,'','frames')); % Load "frames" variable -if ~isfield(param.cmd,'frms') || isempty(param.cmd.frms) - param.cmd.frms = 1:length(frames.frame_idxs); -end -[valid_frms,keep_idxs] = intersect(param.cmd.frms, 1:length(frames.frame_idxs)); -if length(valid_frms) ~= length(param.cmd.frms) - bad_mask = ones(size(param.cmd.frms)); - bad_mask(keep_idxs) = 0; - warning('Nonexistent frames specified in param.cmd.frms (e.g. frame "%g" is invalid), removing these', ... - param.cmd.frms(find(bad_mask,1))); - param.cmd.frms = valid_frms; -end - -if ~isfield(param.combine,'img_comb_layer_params') - param.combine.img_comb_layer_params = []; -end - -if ~isfield(param.combine,'trim_time') - param.combine.trim_time = true; -end - -if ~isfield(param.csarp,'pulse_comp') || isempty(param.csarp.pulse_comp) - param.csarp.pulse_comp = 1; -end - -if ~isfield(param.csarp,'presums') || isempty(param.csarp.presums) - param.csarp.presums = 1; -end - -if ~isfield(param.combine,'in_path') || isempty(param.combine.in_path) - param.combine.in_path = 'out'; -end - -if ~isfield(param.combine,'array_path') || isempty(param.combine.array_path) - param.combine.array_path = 'out'; -end - -if ~isfield(param.combine,'out_path') || isempty(param.combine.out_path) - param.combine.out_path = param.combine.method; -end - -if ~isfield(param.csarp,'out_path') || isempty(param.csarp.out_path) - param.csarp.out_path = 'out'; -end - -if ~isfield(param.combine,'presums') || isempty(param.combine.presums) - if ~isfield(param.csarp,'presums') || isempty(param.csarp.presums) - param.combine.presums = 1; - else - param.combine.presums = param.csarp.presums; - end -end - -if ~isfield(param.combine,'sar_type') || isempty(param.combine.sar_type) - if ~isfield(param.csarp,'sar_type') || isempty(param.csarp.sar_type) - param.combine.sar_type = 'fk'; - else - param.combine.sar_type = param.csarp.sar_type; - end -end - -if strcmpi(param.combine.sar_type,'f-k') - error('Deprecated sar_type name. Change param.combine.sar_type from ''f-k'' to ''fk'' in your parameters (or remove parameter since ''fk'' is the default mode).'); -end - -if ~isfield(param.combine,'chunk_len') || isempty(param.combine.chunk_len) - if ~isfield(param.csarp,'chunk_len') || isempty(param.csarp.chunk_len) - error('param.combine.chunk_len or param.csarp.chunk_len must be defined'); - else - param.combine.chunk_len = param.csarp.chunk_len; - end -end - -% Handles multilooking syntax: -% {{[1 1],[1 2],[1 3],[1 4],[1 5]},{[2 1],[2 2],[2 3],[2 4],[2 5]}} -% If the image is a cell array it describes multilooking across apertures -if ~iscell(param.combine.imgs{1}) - % No special multilooking, reformat old syntax to new multilooking syntax - for img = 1:length(param.combine.imgs) - param.combine.imgs{img} = {param.combine.imgs{img}}; - end -end - -for img = 1:length(param.combine.imgs) - for ml_idx = 1:length(param.combine.imgs{img}) - % Imaginary image indices is for IQ combining during raw data load - % which we do not need here. - param.combine.imgs{img}{ml_idx} = abs(param.combine.imgs{img}{ml_idx}); - end -end - -if ~isfield(param.combine,'out_path') || isempty(param.combine.out_path) - param.combine.out_path = param.combine.method; -end - -[success] = combine_wf_chan_task(param); \ No newline at end of file diff --git a/cresis-toolbox/processing/create_vectors_from_records.m b/cresis-toolbox/processing/create_vectors_from_records.m index 4d733ac8..d1bc7050 100644 --- a/cresis-toolbox/processing/create_vectors_from_records.m +++ b/cresis-toolbox/processing/create_vectors_from_records.m @@ -74,8 +74,7 @@ function create_vectors_from_records(param,param_override) % Get the files % ===================================================================== -records_fn = ct_filename_support(param,'','records'); -records = load(records_fn); +records = records_load(param); if ~isfield(param.vectors.file,'adc') || isempty(param.vectors.file.adc) adc_idx = 1; diff --git a/cresis-toolbox/processing/csarp.m b/cresis-toolbox/processing/csarp.m deleted file mode 100644 index d14a76b0..00000000 --- a/cresis-toolbox/processing/csarp.m +++ /dev/null @@ -1,522 +0,0 @@ -function ctrl_chain = csarp(param,param_override) -% ctrl_chain = csarp(param,param_override) -% -% SAR processor function which breaks up the frame into chunks -% which are processed by csarp_task.m -% -% param = struct with processing parameters -% -- OR -- -% function handle to script with processing parameters -% param_override = parameters in this struct will override parameters -% in param. This struct must also contain the gRadar fields. -% Typically global gRadar; param_override = gRadar; -% -% Authors: William Blake, John Paden -% -% Example: -% See run_csarp.m for how to run this function directly. -% Normally this function is called from master.m using the param spreadsheet. -% -% See also: run_master.m, master.m, run_csarp.m, csarp.m, -% csarp_task.m - -%% General Setup -% ===================================================================== -param = merge_structs(param, param_override); - -fprintf('=====================================================================\n'); -fprintf('%s: %s (%s)\n', mfilename, param.day_seg, datestr(now)); -fprintf('=====================================================================\n'); - -%% Input Checks -% ===================================================================== - -% Get speed of light, dielectric of ice constants -physical_constants; - -if ~isfield(param.csarp,'frm_types') || isempty(param.csarp.frm_types) - param.csarp.frm_types = {-1,-1,-1,-1,-1}; -end - -% Remove frames that do not exist from param.cmd.frms list -load(ct_filename_support(param,'','frames')); % Load "frames" variable -if ~isfield(param.cmd,'frms') || isempty(param.cmd.frms) - param.cmd.frms = 1:length(frames.frame_idxs); -end -[valid_frms,keep_idxs] = intersect(param.cmd.frms, 1:length(frames.frame_idxs)); -if length(valid_frms) ~= length(param.cmd.frms) - bad_mask = ones(size(param.cmd.frms)); - bad_mask(keep_idxs) = 0; - warning('Nonexistent frames specified in param.cmd.frms (e.g. frame "%g" is invalid), removing these', ... - param.cmd.frms(find(bad_mask,1))); - param.cmd.frms = valid_frms; -end - -if ~isfield(param.csarp,'out_path') || isempty(param.csarp.out_path) - param.csarp.out_path = 'out'; -end - -if ~isfield(param.csarp,'coord_path') || isempty(param.csarp.coord_path) - param.csarp.coord_path = param.csarp.out_path; -end - -if ~isfield(param.csarp,'ground_based') || isempty(param.csarp.ground_based) - param.csarp.ground_based = false; -end - -if ~isfield(param.csarp,'lever_arm_fh') || isempty(param.csarp.lever_arm_fh) - param.csarp.lever_arm_fh = []; -end - -if ~isfield(param.csarp,'sar_type') || isempty(param.csarp.sar_type) - param.csarp.sar_type = 'fk'; -end - -if strcmpi(param.csarp.sar_type,'f-k') - error('Deprecated sar_type name. Change param.csarp.sar_type from ''f-k'' to ''fk'' in your parameters (or remove parameter since ''fk'' is the default mode).'); -end - -if ~isfield(param.csarp,'pulse_comp') || isempty(param.csarp.pulse_comp) - param.csarp.pulse_comp = true; -end - -if ~isfield(param.csarp,'ft_dec') || isempty(param.csarp.ft_dec) - param.csarp.ft_dec = true; -end - -if ~isfield(param.csarp,'ft_wind_time') || isempty(param.csarp.ft_wind_time) - param.csarp.ft_wind_time = []; -end - -if ~isfield(param.csarp,'start_eps') || isempty(param.csarp.start_eps) - param.csarp.start_eps = er_ice; -end - -if ~isfield(param.csarp,'time_of_full_support') || isempty(param.csarp.time_of_full_support) - param.csarp.time_of_full_support = inf; -end - -if ~isfield(param.csarp,'force_one_wf_adc_pair_per_job') || isempty(param.csarp.force_one_wf_adc_pair_per_job) - param.csarp.force_one_wf_adc_pair_per_job = false; -end - -if ~isfield(param.csarp,'combine_rx') || isempty(param.csarp.combine_rx) - param.csarp.combine_rx = false; -end - -if ~isfield(param.csarp,'Lsar') - param.csarp.Lsar = []; -end - -if ~isfield(param.csarp.Lsar,'agl') - param.csarp.Lsar.agl = 500; -end - -if ~isfield(param.csarp.Lsar,'thick') - param.csarp.Lsar.thick = 1000; -end - -if ~isfield(param.csarp,'trim_vals') || isempty(param.csarp.trim_vals) - param.csarp.trim_vals = [0 0]; -end - -if isfield(param.csarp,'coh_noise_removal') - error('csarp.coh_noise_removal is deprecated. Please use the field name csarp.coh_noise_method or remove this column if not using coherent noise removal.'); -end - -if ~isfield(param.csarp,'coh_noise_method') || isempty(param.csarp.coh_noise_method) - param.csarp.coh_noise_method = 0; -end - -if ~isfield(param.csarp,'coh_noise_arg') - param.csarp.coh_noise_arg = []; -end - -if ~isfield(param.csarp,'pulse_rfi') || isempty(param.csarp.pulse_rfi) - param.csarp.pulse_rfi.en = 0; -end - -if ~isfield(param.csarp.mocomp,'filter') - param.csarp.mocomp.filter = ''; -end - -if ~isfield(param.csarp,'surf_filt_dist') || isempty(param.csarp.surf_filt_dist) - param.csarp.surf_filt_dist = 3e3; - warning('Surface filtering is not set. Using default value: %.0f m.',param.csarp.surf_filt_dist); -end - -if ~isfield(param.csarp,'presums') - param.csarp.presums = 1; -end - -if ~isfield(param.csarp,'deconvolution') || isempty(param.csarp.deconvolution) - param.csarp.deconvolution = 0; -end - -if ~isfield(param.csarp,'psd_smooth') || isempty(param.csarp.psd_smooth) - param.csarp.psd_smooth = 0; -end - -% Do not apply channel equalization during csarp combine unless receivers -% are being combined at this stage (csarp-combined method) -if ~param.csarp.combine_rx - for wf = 1:length(param.radar.wfs) - param.radar.wfs(wf).chan_equal_dB(:) = 0; - param.radar.wfs(wf).chan_equal_deg(:) = 0; - end -end - -%% Setup processing -% ===================================================================== - -% Get the standard radar name -[~,~,radar_name] = ct_output_dir(param.radar_name); - -% Load records file -records_fn = ct_filename_support(param,'','records'); -records = load(records_fn); -% Apply presumming -if param.csarp.presums > 1 - records.lat = fir_dec(records.lat,param.csarp.presums); - records.lon = fir_dec(records.lon,param.csarp.presums); - records.elev = fir_dec(records.elev,param.csarp.presums); - records.roll = fir_dec(records.roll,param.csarp.presums); - records.pitch = fir_dec(records.pitch,param.csarp.presums); - records.heading = fir_dec(records.heading,param.csarp.presums); - records.gps_time = fir_dec(records.gps_time,param.csarp.presums); - records.surface = fir_dec(records.surface,param.csarp.presums); -end -% Along-track -along_track_approx = geodetic_to_along_track(records.lat,records.lon,records.elev); - -% SAR output directory -csarp_out_dir = ct_filename_out(param, param.csarp.out_path); -csarp_coord_dir = ct_filename_out(param, param.csarp.coord_path); - -ctrl_chain = {}; - -% Get version information out of the coherent noise file -if any(param.csarp.coh_noise_method == [17 19]) - - cdf_fn_dir = fileparts(ct_filename_out(param,param.csarp.coh_noise_arg{4}, '')); - cdf_fn = fullfile(cdf_fn_dir,sprintf('coh_noise_simp_%s.nc', param.day_seg)); - - tmp = netcdf_to_mat(cdf_fn,[],'^sw_version.*'); - param.csarp.coh_noise_version = tmp.sw_version; - tmp = netcdf_to_mat(cdf_fn,[],'^param_collate.*'); - param.csarp.coh_noise_params = tmp.param_collate; -end - -%% Collect waveform information into one structure -% - This is used to break the frame up into chunks -% ===================================================================== -if strcmpi(radar_name,'mcrds') - wfs = load_mcrds_wfs(records.settings, param, ... - records.param_records.records.file.adcs, param.csarp); -elseif any(strcmpi(radar_name,{'acords','hfrds','hfrds2','mcords','mcords2','mcords3','mcords4','mcords5','mcrds','seaice','accum2'})) - wfs = load_mcords_wfs(records.settings, param, ... - records.param_records.records.file.adcs, param.csarp); -elseif any(strcmpi(radar_name,{'icards'}))% add icards---qishi - wfs = load_icards_wfs(records.settings, param, ... - records.param_records.records.file.adcs, param.csarp); -elseif any(strcmpi(radar_name,{'snow','kuband','snow2','kuband2','snow3','kuband3','kaband3','snow5'})) - error('Not supported'); - wfs = load_fmcw_wfs(records.settings, param, ... - records.param_records.records.file.adcs, param.csarp); - for wf=1:length(wfs) - wfs(wf).time = param.csarp.time_of_full_support; - wfs(wf).freq = 1; - end -end - -%% Create the SAR coordinate system (used for structuring processing) -% ===================================================================== - -sar_fn = fullfile(csarp_coord_dir,'sar_coord.mat'); -fprintf('csarp_sar_coord_task %s (%s):\n %s\n', datestr(now), param.day_seg, sar_fn); -if exist(sar_fn,'file') - sar_fields = {'Lsar','gps_source','gps_time_offset','type','sigma_x','presums','version','along_track'}; - sar = load(sar_fn,sar_fields{:}); - for sar_field = sar_fields - if ~isfield(sar,sar_field{1}) - error('SAR coordinates file missing %s field. Either delete file or correct file and then rerun.', sar_field{1}); - end - end -end -Lsar = c/wfs(1).fc*(param.csarp.Lsar.agl+param.csarp.Lsar.thick/sqrt(er_ice))/(2*param.csarp.sigma_x); -if ~exist(sar_fn,'file') ... - || sar.Lsar ~= Lsar ... - || ~strcmpi(sar.gps_source,records.gps_source) ... - || sar.gps_time_offset ~= records.param_records.vectors.gps.time_offset ... - || sar.type ~= param.csarp.mocomp.type ... - || sar.sigma_x ~= param.csarp.sigma_x ... - || sar.presums ~= param.csarp.presums ... - || sar.version ~= 1.0 - %% SAR coordinates file does not exist or needs to be recreated - - ctrl = cluster_new_batch(param); - - if any(strcmpi(radar_name,{'acords','hfrds','hfrds2','mcords','mcords2','mcords3','mcords4','mcords5','mcrds','seaice','accum2'})) - cpu_time_mult = 6e-3; - mem_mult = 64; - - elseif any(strcmpi(radar_name,{'snow','kuband','snow2','kuband2','snow3','kuband3','kaband3','snow5','snow8'})) - cpu_time_mult = 100e-8; - mem_mult = 64; - end - - sparam = []; - sparam.argsin{1} = param; % Static parameters - sparam.task_function = 'csarp_sar_coord_task'; - sparam.num_args_out = 1; - Nx = numel(records.gps_time); - sparam.cpu_time = 60 + Nx*cpu_time_mult; - sparam.mem = 250e6 + Nx*mem_mult; - sparam.notes = sprintf('%s:%s:%s %s', ... - sparam.task_function, param.radar_name, param.season_name, param.day_seg); - - % Create success condition - success_error = 64; - sparam.success = ... - sprintf('error_mask = bitor(error_mask,%d*~exist(''%s'',''file''));\n', success_error, sar_fn); - if ~ctrl.cluster.rerun_only && exist(sar_fn,'file') - delete(sar_fn); - end - - ctrl = cluster_new_task(ctrl,sparam,[]); - - ctrl_chain{end+1} = ctrl; - -else - %% SAR coordinates file exists and only needs surface updated - surf_idxs = get_equal_alongtrack_spacing_idxs(sar.along_track,param.csarp.sigma_x); - if surf_idxs(end) ~= length(sar.along_track) - surf_idxs(end+1) = length(sar.along_track); - end - - surf = sgolayfilt(records.surface(surf_idxs),3,round(param.csarp.surf_filt_dist / median(diff(sar.along_track(surf_idxs)))/2)*2+1); - sar.surf_pp = spline(sar.along_track(surf_idxs),surf); - - fprintf(' Updating with new surface values from records.\n'); - save(sar_fn,'-append','-struct','sar','surf_pp'); -end - -%% Create imgs_list which divides up images into tasks -% ===================================================================== -% imgs_list: cell array of images -% imgs_list{img}: cell array of wf-adc pair lists PER task -% imgs_list{img}{task}: array of wf-adc pairs to be processed by a task -imgs_list = {}; -if param.csarp.combine_rx - % One SAR image with all wf-adc pairs - for img = 1:length(param.csarp.imgs) - imgs_list{1}{img} = param.csarp.imgs{img}; - end - -else - % One SAR image per wf-adc pair - - if ~param.csarp.force_one_wf_adc_pair_per_job - % All SAR images from the same data files per task - for img = 1:length(param.csarp.imgs) - for wf_adc = 1:size(param.csarp.imgs{img},1) - adc = abs(param.csarp.imgs{img}(wf_adc,2)); - [board,board_idx] = adc_to_board(radar_name,adc); - if length(imgs_list) < board_idx - imgs_list{board_idx} = []; - end - if length(imgs_list{board_idx}) < img - imgs_list{board_idx}{img} = []; - end - imgs_list{board_idx}{img}(end+1,:) = param.csarp.imgs{img}(wf_adc,:); - end - end - - else - % One SAR image per task - for img = 1:length(param.csarp.imgs) - for wf_adc = 1:length(param.csarp.imgs{img}) - imgs_list{end+1}{1}(1,:) = param.csarp.imgs{img}(wf_adc,:); - end - end - end -end - -%% Create and setup the cluster batch -% ===================================================================== -ctrl = cluster_new_batch(param); -cluster_compile({'csarp_task.m','csarp_sar_coord_task'},ctrl.cluster.hidden_depend_funs,ctrl.cluster.force_compile,ctrl); - -total_num_sam = {}; -if any(strcmpi(radar_name,{'acords','hfrds','hfrds2','mcords','mcords2','mcords3','mcords4','mcords5','mcrds','seaice','accum2'})) - for imgs_idx = 1:length(imgs_list) - for img = 1:length(imgs_list{imgs_idx}) - wf = abs(imgs_list{imgs_idx}{img}(1,1)); - total_num_sam{imgs_idx}{img} = wfs(wf).Nt_raw; - end - end - cpu_time_mult = 150e-8/100; - mem_mult = 8; - -elseif any(strcmpi(radar_name,{'snow','kuband','snow2','kuband2','snow3','kuband3','kaband3','snow5','snow8'})) - for imgs_idx = 1:length(imgs_list) - for img = 1:length(imgs_list{imgs_idx}) - wf = abs(imgs_list{imgs_idx}{img}(1,1)); - total_num_sam{imgs_idx}{img} = 32000; - end - end - cpu_time_mult = 8e-8; - mem_mult = 64; - -else - error('radar_name %s not supported yet.', radar_name); - -end - -%% Setup tasks to SAR process each frame -% ===================================================================== -retry_fields = {}; -sparam.argsin{1} = param; % Static parameters -sparam.task_function = 'csarp_task'; -sparam.num_args_out = 1; -for frm_idx = 1:length(param.cmd.frms) - frm = param.cmd.frms(frm_idx); - if ct_proc_frame(frames.proc_mode(frm),param.csarp.frm_types) - fprintf('%s %s_%03i (%i of %i) (%s)\n', sparam.task_function, param.day_seg, frm, frm_idx, length(param.cmd.frms), datestr(now)); - else - fprintf('Skipping frame %s_%03i (no process frame)\n', param.day_seg, frm); - continue; - end - - % Current frame goes from the start record specified in the frames file - % to the record just before the start record of the next frame. For - % the last frame, the stop record is just the last record in the segment. - start_rec = ceil(frames.frame_idxs(frm)/param.csarp.presums); - if frm < length(frames.frame_idxs) - stop_rec = ceil((frames.frame_idxs(frm+1)-1)/param.csarp.presums); - else - stop_rec = length(records.gps_time); - end - - % Determine length of the frame - frm_dist = along_track_approx(stop_rec) - along_track_approx(start_rec); - - % Determine number of chunks and range lines per chunk - num_chunks = round(frm_dist / param.csarp.chunk_len); - - % Estimate number of input range lines per chunk - num_rlines_per_chunk = round((stop_rec-start_rec) / num_chunks); - - for chunk_idx = 1:num_chunks - % Setup dynamic params - % ===================================================================== - dparam.argsin{1}.load.frm = frm; - dparam.argsin{1}.load.chunk_idx = chunk_idx; - dparam.argsin{1}.load.num_chunks = num_chunks; - if chunk_idx == num_chunks - dparam.argsin{1}.load.recs = [start_rec + num_rlines_per_chunk*(chunk_idx-1), stop_rec]; - else - dparam.argsin{1}.load.recs = start_rec + num_rlines_per_chunk*(chunk_idx-1) + [0, num_rlines_per_chunk-1]; - end - - for imgs_idx = 1:length(imgs_list) - if isempty(imgs_list{imgs_idx}) - continue; - end - dparam.argsin{1}.load.imgs = imgs_list{imgs_idx}; - sub_band_idx = 1; - dparam.argsin{1}.load.sub_band_idx = sub_band_idx; - - % Create a list of waveform/adc pairs for displaying to the string. - wf_adc_str = ''; - for img = 1:length(dparam.argsin{1}.load.imgs) - for wf_adc_idx = 1:size(dparam.argsin{1}.load.imgs{img},1) - wf = abs(dparam.argsin{1}.load.imgs{img}(wf_adc_idx,1)); - adc = abs(dparam.argsin{1}.load.imgs{img}(wf_adc_idx,2)); - if isempty(wf_adc_str) - wf_adc_str = [wf_adc_str sprintf('wf,adc: %d,%d',wf,adc)]; - else - wf_adc_str = [wf_adc_str sprintf(' %d,%d',wf,adc)]; - end - end - end - - % Create success condition - % ================================================================= - dparam.success = ''; - success_error = 64; - for subap = 1:length(param.csarp.sub_aperture_steering) - out_fn_dir = fullfile(csarp_out_dir, ... - sprintf('%s_data_%03d_%02d_%02d',param.csarp.sar_type,frm, ... - subap, sub_band_idx)); - for img = 1:length(dparam.argsin{1}.load.imgs) - if param.csarp.combine_rx - out_fn = fullfile(out_fn_dir,sprintf('img_%02d_chk_%03d.mat',img,chunk_idx)); - dparam.success = cat(2,dparam.success, ... - sprintf(' error_mask = bitor(error_mask,%d*~exist(''%s'',''file''));\n', success_error, out_fn)); - if ~ctrl.cluster.rerun_only && exist(out_fn,'file') - delete(out_fn); - end - else - for wf_adc = 1:size(dparam.argsin{1}.load.imgs{img},1) - wf = abs(dparam.argsin{1}.load.imgs{img}(wf_adc,1)); - adc = abs(dparam.argsin{1}.load.imgs{img}(wf_adc,2)); - out_fn = fullfile(out_fn_dir,sprintf('wf_%02d_adc_%02d_chk_%03d.mat',wf,adc,chunk_idx)); - dparam.success = cat(2,dparam.success, ... - sprintf(' error_mask = bitor(error_mask,%d*~exist(''%s'',''file''));\n', success_error, out_fn)); - if ~ctrl.cluster.rerun_only && exist(out_fn,'file') - delete(out_fn); - end - end - end - end - end - - % Rerun only mode: Test to see if we need to run this task - % ================================================================= - dparam.notes = sprintf('%s:%s:%s %s_%03d (%d of %d)/%d of %d %s %.0f to %.0f recs', ... - sparam.task_function, param.radar_name, param.season_name, param.day_seg, frm, frm_idx, length(param.cmd.frms), ... - chunk_idx, num_chunks, wf_adc_str, (dparam.argsin{1}.load.recs(1)-1)*param.csarp.presums+1, ... - dparam.argsin{1}.load.recs(2)*param.csarp.presums); - if ctrl.cluster.rerun_only - % If we are in rerun only mode AND the get heights task success - % condition passes without error, then we do not run the task. - error_mask = 0; - eval(dparam.success); - if ~error_mask - fprintf(' Already exists [rerun_only skipping]: %s (%s)\n', ... - dparam.notes, datestr(now)); - continue; - end - end - - % Create task - % ================================================================= - - % CPU Time and Memory estimates: - % Nx*total_num_sam*K where K is some manually determined multiplier. - Nx = diff(dparam.argsin{1}.load.recs); - dparam.cpu_time = 0; - dparam.mem = 250e6; - for img = 1:length(dparam.argsin{1}.load.imgs) - if strcmpi(param.csarp.sar_type,'fk') - dparam.cpu_time = dparam.cpu_time + 10 + Nx*log2(Nx)*total_num_sam{imgs_idx}{img}*log2(total_num_sam{imgs_idx}{img})*size(dparam.argsin{1}.load.imgs{img},1)*cpu_time_mult; - dparam.mem = dparam.mem + Nx*total_num_sam{imgs_idx}{img}*size(dparam.argsin{1}.load.imgs{img},1)*mem_mult; - elseif strcmpi(param.csarp.sar_type,'tdbp') - dparam.cpu_time = ctrl.cluster.max_time_per_job - 40; - end - end - - ctrl = cluster_new_task(ctrl,sparam,dparam,'dparam_save',0); - end - end -end - -ctrl = cluster_save_dparam(ctrl); - -ctrl_chain{end+1} = ctrl; - -fprintf('Done %s\n', datestr(now)); - -return; - diff --git a/cresis-toolbox/processing/csarp_sar_coord_task.m b/cresis-toolbox/processing/csarp_sar_coord_task.m deleted file mode 100644 index 1993d7df..00000000 --- a/cresis-toolbox/processing/csarp_sar_coord_task.m +++ /dev/null @@ -1,130 +0,0 @@ -function [success] = csarp_sar_coord_task(param) -% [success] = csarp_sar_coord_task(param) -% -% Creates the SAR coordinate system for the entire segment. -% -% Authors: John Paden -% -% Example: -% See run_csarp.m for how to run this function directly. -% Normally this function is called from master.m using the param spreadsheet. -% -% See also: run_master.m, master.m, run_csarp.m, csarp.m, -% csarp_task.m - -%% Setup processing -% ===================================================================== - -% Get speed of light, dielectric of ice constants -physical_constants; -wgs84 = wgs84Ellipsoid('meters'); - -% Get the standard radar name -[~,~,radar_name] = ct_output_dir(param.radar_name); - -% Load records file -records_fn = ct_filename_support(param,'','records'); -records = load(records_fn); -% Apply presumming -if param.csarp.presums > 1 - records.lat = fir_dec(records.lat,param.csarp.presums); - records.lon = fir_dec(records.lon,param.csarp.presums); - records.elev = fir_dec(records.elev,param.csarp.presums); - records.roll = fir_dec(records.roll,param.csarp.presums); - records.pitch = fir_dec(records.pitch,param.csarp.presums); - records.heading = fir_dec(records.heading,param.csarp.presums); - records.gps_time = fir_dec(records.gps_time,param.csarp.presums); - records.surface = fir_dec(records.surface,param.csarp.presums); -end - -% SAR output directory -csarp_coord_dir = ct_filename_out(param, param.csarp.coord_path); - -%% Collect waveform information into one structure -% - This is used to break the frame up into chunks -% ===================================================================== -if strcmpi(radar_name,'mcrds') - wfs = load_mcrds_wfs(records.settings, param, ... - records.param_records.records.file.adcs, param.csarp); -elseif any(strcmpi(radar_name,{'acords','hfrds','hfrds2','mcords','mcords2','mcords3','mcords4','mcords5','seaice','accum2'})) - wfs = load_mcords_wfs(records.settings, param, ... - records.param_records.records.file.adcs, param.csarp); -elseif any(strcmpi(radar_name,{'icards'}))% add icards---qishi - wfs = load_icards_wfs(records.settings, param, ... - records.param_records.records.file.adcs, param.csarp); -elseif any(strcmpi(radar_name,{'snow','kuband','snow2','kuband2','snow3','kuband3','kaband3','snow5'})) - error('Not supported'); - wfs = load_fmcw_wfs(records.settings, param, ... - records.param_records.records.file.adcs, param.csarp); - for wf=1:length(wfs) - wfs(wf).time = param.csarp.time_of_full_support; - wfs(wf).freq = 1; - end -end - -%% Create the along-track axes (used for structuring processing) -% This cannot be done in advance or asynchronously because it controls how -% the data will be broken into chunks. It usually only needs to be run one -% time though. -% -% sar_ref: -% .orig: 3xNx matrix (3 by Nx, Nx = output positions) -% .x: 3xNx matrix, along-track unit vector -% .z: 3xNx matrix, up unit vector (orthgonal to x) -% Note that y, left vector, can be computed as cross of z and x -% .along_track: 1 by Nx vector -% .pp: spline interpolation kernel for -% ===================================================================== - -Lsar = c/wfs(1).fc*(param.csarp.Lsar.agl+param.csarp.Lsar.thick/sqrt(er_ice))/(2*param.csarp.sigma_x); - -trajectory_param = struct('gps_source',records.gps_source, ... - 'season_name',param.season_name,'radar_name',param.radar_name,'rx_path', 0, ... - 'tx_weights', [], 'lever_arm_fh', param.csarp.lever_arm_fh); - -ref = trajectory_with_leverarm(records,trajectory_param); - -along_track = geodetic_to_along_track(ref.lat,ref.lon,ref.elev,Lsar); -output_along_track = along_track(1) : param.csarp.sigma_x : along_track(end); - -surf_idxs = get_equal_alongtrack_spacing_idxs(along_track,param.csarp.sigma_x); -if surf_idxs(end) ~= length(along_track) - surf_idxs(end+1) = length(along_track); -end - -surf = sgolayfilt(records.surface(surf_idxs),3,round(param.csarp.surf_filt_dist / median(diff(along_track(surf_idxs)))/2)*2+1); - -SAR_coord_param.type = param.csarp.mocomp.type; -SAR_coord_param.squint = [0 0 -1].'; -SAR_coord_param.Lsar = Lsar; -fcs = SAR_coord_system(SAR_coord_param,ref,ref,along_track,output_along_track); - -sar = []; -sar.version = 1.0; -sar.Lsar = Lsar; -sar.gps_source = records.gps_source; -sar.gps_time_offset = records.param_records.vectors.gps.time_offset; -sar.type = param.csarp.mocomp.type; -sar.sigma_x = param.csarp.sigma_x; -sar.presums = param.csarp.presums; -sar.along_track = along_track; -sar.surf_pp = spline(along_track(surf_idxs),surf); -sar.origin = fcs.origin; -sar.x = fcs.x; -sar.z = fcs.z; -sar.roll = fcs.roll; -sar.pitch = fcs.pitch; -sar.heading = fcs.heading; -sar.gps_time = fcs.gps_time; - -sar_fn = fullfile(csarp_coord_dir,'sar_coord.mat'); -sar_fn_dir = fileparts(sar_fn); -if ~exist(sar_fn_dir,'dir') - mkdir(sar_fn_dir); -end -fprintf('Saving SAR coord %s (%s)\n', sar_fn, datestr(now)); -save(sar_fn,'-v7','-struct','sar','version','Lsar','gps_source','gps_time_offset','type','sigma_x','presums','surf_pp','along_track','origin','x','z','roll','pitch','heading','gps_time'); - -fprintf('%s done %s\n', mfilename, datestr(now)); - -success = true; diff --git a/cresis-toolbox/processing/csarp_task.m b/cresis-toolbox/processing/csarp_task.m deleted file mode 100644 index 5684bc88..00000000 --- a/cresis-toolbox/processing/csarp_task.m +++ /dev/null @@ -1,1293 +0,0 @@ -function [success] = csarp_task(param) -% [success] = csarp_task(param) -% -% SAR process a chunk of data (a frame is divided into chunks -% to keep the memory requirements low enough). This function -% is called from csarp.m. -% -% param = structure controlling the SAR processor -% .debug_level = scalar integer, debugging level -% .radar = structured used by load_mcords_hdr -% -% .load = structure containing info about what data to load -% .records_fn = records filename -% .recs = 2 element vector containing the start and stop records -% .imgs = cell vector of images to load, each image is Nx2 array of -% wf/adc pairs -% NOTE: wfs/ads are not indices into anything, they are the absolute -% waveform/adc numbers. The records file will be loaded to decode -% which index each wf/adc belongs to. -% -% .proc = structure about which frame to process and how it is broken -% into chunks -% .frm = only used to determine the file name of the output -% .output_along_track_offset = along-track offset from first input -% record to the first output range line -% .output_along_track_Nx = length of output in along-track -% -% .csarp = structure containing SAR processing parameters -% .file = struct with input file information -% .base_dir: string, e.g. '/cresis/data3/MCoRDS/2011_Greenland_P3/' -% .adc_folder_name = string, e.g. '20110507/board%b/seg_01' -% .file_prefix = string, e.g. '' -% .out_path = output path string -% .combine_rx = boolean, combine channels before SAR processing -% .coh_noise_removal = boolean, slow-time DC removal -% .lever_arm_fh = string containing function name to lever arm -% .mocomp = struct for motion compensation -% .en = boolean, apply motion compensation -% .type = see motion_comp.m for details -% .uniform_en = boolean, resample data to uniform sampling in along-track -% .sar_type = string, 'fk' or 'tdbp' -% .sigma_x = along-track output sample spacing (meters) -% .sub_aperture_steering = vector of doppler centroids to process to -% (i.e. subapertures) normalized to the doppler bandwidth -% .st_wind = function handle for slow time decimation -% .start_eps = epsilon to use for sub-surface -% -% Fields used by load_mcords_data.m (see that function for details) -% .pulse_rfi.en -% .pulse_rfi.inc_ave -% .pulse_rfi.thresh_scale -% .trim_vals -% .pulse_comp -% .ft_dec -% .ft_wind -% .ft_wind_time -% .radar.rx_path.chan_equal -% .radar.rx_path.td -% -% success = boolean which is true when the function executes properly -% if a task fails before it can return success, then success will be -% empty -% -% Authors: William Blake, John Paden -% -% See also: csarp.m -% - -%% Initialization and checking arguments - -% Get speed of light, dielectric of ice constants -physical_constants; -wgs84 = wgs84Ellipsoid('meters'); - -global g_data; g_data = []; - -[output_dir,radar_type,radar_name] = ct_output_dir(param.radar_name); - -records_fn = ct_filename_support(param,'','records'); -records = load(records_fn,'settings','param_records'); - -if param.csarp.combine_rx && param.csarp.mocomp.en - warning('CSARP motion compensation mode must be 0 for combine_rx (setting to 0)'); - param.csarp.mocomp.en = 0; -end - -% SAR output directory -csarp_out_dir = ct_filename_out(param, param.csarp.out_path); -csarp_coord_dir = ct_filename_out(param, param.csarp.coord_path); - -% Load SAR coordinate system -sar_fn = fullfile(csarp_coord_dir,'sar_coord.mat'); -sar = load(sar_fn,'version','Lsar','gps_source','type','sigma_x','presums','along_track','surf_pp'); - -% Determine output range lines -output_along_track = 0 : param.csarp.sigma_x : sar.along_track(end); -start_x = sar.along_track(param.load.recs(1)); -stop_x = sar.along_track(param.load.recs(2)); -out_rlines = find(output_along_track >= start_x & output_along_track <= stop_x); -output_along_track = output_along_track(out_rlines); - -%% Collect waveform information into one structure -% - This is used to break the frame up into chunks -% ===================================================================== -if strcmpi(radar_name,'mcrds') - wfs = load_mcrds_wfs(records.settings, param, ... - records.param_records.records.file.adcs, param.csarp); -elseif any(strcmpi(radar_name,{'acords','hfrds','hfrds2','mcords','mcords2','mcords3','mcords4','mcords5','seaice','accum2'})) - wfs = load_mcords_wfs(records.settings, param, ... - records.param_records.records.file.adcs, param.csarp); -elseif any(strcmpi(radar_name,{'icards'}))% add icards---qishi - wfs = load_icards_wfs(records.settings, param, ... - records.param_records.records.file.adcs, param.csarp); -elseif any(strcmpi(radar_name,{'snow','kuband','snow2','kuband2','snow3','kuband3','kaband3','snow5'})) - error('Not supported'); - wfs = load_fmcw_wfs(records.settings, param, ... - records.param_records.records.file.adcs, param.csarp); - for wf=1:length(wfs) - wfs(wf).time = param.csarp.time_of_full_support; - wfs(wf).freq = 1; - end -elseif strcmpi(radar_name,'sim'); -end - -%% Determine chunk overlap to ensure full support -% ===================================================================== -% Determine overlap of chunks from the range to furthest target -times = cell2mat({wfs.time}.'); -max_time = min(max(times),param.csarp.time_of_full_support); - -% wavelength (m) -wf = abs(param.load.imgs{1}(1,1)); -lambda = c/wfs(wf).fc; -% twtt to surface (sec) -surf_time = ppval(sar.surf_pp, start_x); surf_time = min(max_time,surf_time); -% effective in air max range (m) -max_range = ((max_time-surf_time)/sqrt(param.csarp.start_eps) + surf_time) * c/2; -% chunk overlap (m) -chunk_overlap_start = (max_range*lambda)/(2*param.csarp.sigma_x) / 2; -% chunk_overlap_start = max_range/sqrt((2*param.csarp.sigma_x/lambda)^2-1); - -% twtt to surface (sec) -surf_time = ppval(sar.surf_pp, stop_x); surf_time = min(max_time,surf_time); -% effective in air max range (m) -max_range = ((max_time-surf_time)/sqrt(param.csarp.start_eps) + surf_time) * c/2; -chunk_overlap_stop = (max_range*lambda)/(2*param.csarp.sigma_x) / 2; -% chunk_overlap_stop = max_range/sqrt((2*param.csarp.sigma_x/lambda)^2-1); - -% These are the records which will be used -cur_recs = [find(sar.along_track > start_x-chunk_overlap_start,1) ... - find(sar.along_track < stop_x+chunk_overlap_stop, 1, 'last')]; -param.load.recs = cur_recs; - -%% Determine zero padding to prevent circular convolution aliasing -% ===================================================================== -param.load.start_zero_pad = floor((sar.along_track(cur_recs(1)) - (start_x-chunk_overlap_start)) / param.csarp.sigma_x); -param.load.stop_zero_pad = floor((stop_x+chunk_overlap_stop - sar.along_track(cur_recs(2))) / param.csarp.sigma_x); - -%% Prepare trajectory information -% ========================================================================= - -% Create along-track vectors (output rline 1 is zero/origin) -along_track = sar.along_track(param.load.recs(1):param.load.recs(end)); - -% Create FCS: SAR (flight) coordinate system -fcs = []; -fcs.Lsar = sar.Lsar; -fcs.gps_source = sar.gps_source; - -% Slow, but memory efficient way to load SAR coordinate system -tmp = load(sar_fn,'origin'); -fcs.origin = tmp.origin(:,out_rlines); -tmp = load(sar_fn,'x'); -fcs.x = tmp.x(:,out_rlines); -tmp = load(sar_fn,'z'); -fcs.z = tmp.z(:,out_rlines); -fcs.y = cross(fcs.z,fcs.x); -% fcs.pos: to be added for each individual SAR image -tmp = load(sar_fn,'roll'); -fcs.roll = tmp.roll(1,out_rlines); -tmp = load(sar_fn,'pitch'); -fcs.pitch = tmp.pitch(1,out_rlines); -tmp = load(sar_fn,'heading'); -fcs.heading = tmp.heading(1,out_rlines); -tmp = load(sar_fn,'gps_time'); -fcs.gps_time = tmp.gps_time(1,out_rlines); - -fcs.surface = ppval(sar.surf_pp,output_along_track); -fcs.bottom = NaN*ones(size(fcs.surface)); - -%% Load record information -% ===================================================================== -load_param = param; -load_param.load.recs = [(param.load.recs(1)-1)*param.csarp.presums+1, ... - param.load.recs(2)*param.csarp.presums]; -orig_records = records_aux_files_read(records_fn,load_param.load.recs); -all_records = orig_records; -%Decimate orig_records and ref according to presums -if param.csarp.presums > 1 - orig_records.lat = fir_dec(orig_records.lat,param.csarp.presums); - orig_records.lon = fir_dec(orig_records.lon,param.csarp.presums); - orig_records.elev = fir_dec(orig_records.elev,param.csarp.presums); - orig_records.roll = fir_dec(orig_records.roll,param.csarp.presums); - orig_records.pitch = fir_dec(orig_records.pitch,param.csarp.presums); - orig_records.heading = fir_dec(orig_records.heading,param.csarp.presums); - orig_records.gps_time = fir_dec(orig_records.gps_time,param.csarp.presums); - orig_records.surface = fir_dec(orig_records.surface,param.csarp.presums); -end -old_param_records = orig_records.param_records; -old_param_records.gps_source = orig_records.gps_source; - -%% Get the new surface -% ===================================================================== -if isfield(param.csarp,'surface_src') && ~isempty(param.csarp.surface_src) - - % Get the generic layer data path - layer_path = fullfile(ct_filename_out(param,param.csarp.surface_src,'',0)); - - % Load the current frame - layer_fn = fullfile(layer_path,sprintf('Data_%s_%03d.mat',param.day_seg,param.proc.frm)); - layer = load(layer_fn); - new_surface_gps_time = layer.GPS_time; - new_surface = layer.layerData{1}.value{2}.data; - % Get the previous frame if necessary - if orig_records.gps_time(1) < new_surface_gps_time(1)-1 - layer_fn = fullfile(layer_path,sprintf('Data_%s_%03d.mat',param.day_seg,param.proc.frm-1)); - if exist(layer_fn,'file') - layer = load(layer_fn); - new_surface_gps_time = [layer.GPS_time new_surface_gps_time]; - new_surface = [layer.layerData{1}.value{2}.data new_surface]; - end - end - % Get the next frame if necessary - if orig_records.gps_time(end) > new_surface_gps_time(end)+1 - layer_fn = fullfile(layer_path,sprintf('Data_%s_%03d.mat',param.day_seg,param.proc.frm+1)); - if exist(layer_fn,'file') - layer = load(layer_fn); - new_surface_gps_time = [new_surface_gps_time layer.GPS_time]; - new_surface = [new_surface layer.layerData{1}.value{2}.data]; - end - end - % Since layer files may have overlapping data, sort it - [new_surface_gps_time new_surface_idxs] = sort(new_surface_gps_time); - new_surface = new_surface(new_surface_idxs); - - % Do the interpolation and overwrite the orig_records.surface variable - new_surface = interp1(new_surface_gps_time,new_surface,orig_records.gps_time,'linear','extrap'); - orig_records.surface = new_surface; -end - -%% Load waveforms and record data size -% ========================================================================= -if strcmpi(radar_name,'mcrds') - [wfs,rec_data_size] = load_mcrds_wfs(orig_records.settings, param, ... - 1:max(old_param_records.records.file.adcs), param.csarp); - load_param.load.rec_data_size = rec_data_size; -elseif any(strcmpi(radar_name,{'acords','hfrds','hfrds2','mcords','mcords2','mcords3','mcords4','mcords5','seaice','accum2'})) - [wfs,rec_data_size] = load_mcords_wfs(orig_records.settings, param, ... - 1:max(old_param_records.records.file.adcs), param.csarp); - load_param.load.rec_data_size = rec_data_size; -elseif any(strcmpi(radar_name,{'icards'}))% add icards----qishi - [wfs,rec_data_size] = load_icards_wfs(orig_records.settings, param, ... - 1:max(old_param_records.records.file.adcs), param.csarp); - load_param.load.rec_data_size = rec_data_size; -elseif any(strcmpi(radar_name,{'snow','kuband','snow2','kuband2','snow3','kuband3','kaband3','snow5'})) - wfs = load_fmcw_wfs(orig_records.settings, param, ... - 1:max(old_param_records.records.file.adcs), param.csarp); -end -load_param.wfs = wfs; - -%% Collect record file information required for using raw data loaders -% - Performs mapping between param.adcs and the records file contents -% - Translates filenames from relative to absolute -% - Makes filenames have the correct filesep -% ===================================================================== - -% Create a list of unique receivers required by the imgs list -param.load.adcs = []; -for idx = 1:length(param.load.imgs) - param.load.adcs = unique(cat(2, ... - abs(param.load.imgs{idx}(:,2)).', param.load.adcs)); -end - -if any(strcmpi(radar_name,{'hfrds','hfrds2','icards','mcords','mcords2','mcords3','mcords4','mcords5','seaice','accum2'})) - % adc_headers: the actual adc headers that were loaded - if ~isfield(old_param_records.records.file,'adc_headers') || isempty(old_param_records.records.file.adc_headers) - old_param_records.records.file.adc_headers = old_param_records.records.file.adcs; - end - - % boards_headers: the boards that the actual adc headers were loaded from - boards_headers = adc_to_board(param.radar_name,old_param_records.records.file.adcs); - - for idx = 1:length(param.load.adcs) - % adc: the specific ADC we would like to load - adc = param.load.adcs(idx); - % adc_idx: the records file index for this adc - adc_idx = find(old_param_records.records.file.adcs == adc); - if isempty(adc_idx) - error('ADC %d not present in records file\n', adc); - end - - % board: the board associated with the ADC we would like to load - board = adc_to_board(param.radar_name,adc); - % board_header: the board headers that we will use with this ADC - board_header = adc_to_board(param.radar_name,old_param_records.records.file.adc_headers(adc_idx)); - % board_idx: the index into the records board list to use - board_idx = find(board_header == boards_headers); - - % Just get the file-information for the records we need - load_param.load.file_idx{idx} = relative_rec_num_to_file_idx_vector( ... - load_param.load.recs,orig_records.relative_rec_num{board_idx}); - load_param.load.offset{idx} = orig_records.offset(board_idx,:); - file_idxs = unique(load_param.load.file_idx{idx}); - - % Recognize if first record is really from previous file and it is a - % valid record (i.e. offset does not equal -2^31) - if sign(load_param.load.offset{idx}(1)) < 0 && load_param.load.offset{idx}(1) ~= -2^31 - file_idxs = [file_idxs(1)-1 file_idxs]; - end - - % Just copy the filenames we need - load_param.load.filenames{idx}(file_idxs) = orig_records.relative_filename{board_idx}(file_idxs); - - % Modify filename according to channel - for file_idx = 1:length(load_param.load.filenames{idx}) - if any(strcmpi(radar_name,{'mcords5'})) - load_param.load.filenames{idx}{file_idx}(9:10) = sprintf('%02d',board); - end - end - - filepath = get_segment_file_list(param,adc); - - % Convert relative file paths into absolute file paths if required, - % also corrects filesep (\ and /) - for file_idx = 1:length(load_param.load.filenames{idx}) - load_param.load.filenames{idx}{file_idx} ... - = fullfile(filepath,load_param.load.filenames{idx}{file_idx}); - end - end - load_param.load.file_version = param.records.file_version; - load_param.load.wfs = orig_records.settings.wfs; -elseif strcmpi(radar_name,'mcrds') - load_param.load.offset = orig_records.offset; - load_param.load.file_rec_offset = orig_records.relative_rec_num; - load_param.load.filenames = orig_records.relative_filename; - base_dir = ct_filename_data(param,param.vectors.file.base_dir); - adc_folder_name = param.vectors.file.adc_folder_name; - load_param.load.filepath = fullfile(base_dir, adc_folder_name); - load_param.load.wfs = orig_records.settings.wfs; - load_param.load.wfs_records = orig_records.settings.wfs_records; -elseif strcmpi(radar_name,'acords') - load_param.load.offset = orig_records.offset; - load_param.load.file_rec_offset = orig_records.relative_rec_num; - load_param.load.filenames = orig_records.relative_filename; - base_dir = ct_filename_data(param,param.vectors.file.base_dir); - adc_folder_name = param.vectors.file.adc_folder_name; - load_param.load.filepath = fullfile(base_dir, adc_folder_name); - load_param.load.wfs = orig_records.settings.wfs; - load_param.load.wfs_records = orig_records.settings.wfs_records; -elseif strcmpi(radar_name,'icards')% add icards---qishi - load_param.load.offset = orig_records.offset; - load_param.load.file_rec_offset = orig_records.relative_rec_num; - load_param.load.filenames = orig_records.relative_filename; - base_dir = ct_filename_data(param,param.vectors.file.base_dir); - adc_folder_name = param.vectors.file.adc_folder_name; - load_param.load.filepath = fullfile(base_dir, adc_folder_name); - load_param.load.wfs = orig_records.settings.wfs; - load_param.load.wfs_records = orig_records.settings.wfs_records; -elseif any(strcmpi(radar_name,{'snow','kuband','snow2','kuband2','snow3','kuband3','kaband3','snow5'})) - % Determine which ADC boards are supported and which ones were actually loaded - if ~isfield(old_param_records.records.file,'adc_headers') || isempty(old_param_records.records.file.adc_headers) - old_param_records.records.file.adc_headers = old_param_records.records.file.adcs; - end - boards = adc_to_board(param.radar_name,old_param_records.records.file.adcs); - boards_headers = adc_to_board(param.radar_name,old_param_records.records.file.adc_headers); - - for idx = 1:length(param.load.adcs) - adc = param.load.adcs(idx); - adc_idx = find(old_param_records.records.file.adcs == param.load.adcs(idx)); - if isempty(adc_idx) - error('ADC %d not present in records file\n', param.load.adcs(idx)); - end - - % Just get the file-information for the records we need - board = adc_to_board(param.radar_name,adc); - actual_board_idx = find(board == boards); - board_idx = find(old_param_records.records.file.adc_headers(actual_board_idx) == boards_headers); - load_param.load.file_idx{idx} = relative_rec_num_to_file_idx_vector( ... - load_param.load.recs,orig_records.relative_rec_num{board_idx}); - load_param.load.offset{idx} = orig_records.offset(board_idx,:); - file_idxs = unique(load_param.load.file_idx{idx}); - - % Recognize if first record is really from previous file and it is a - % valid record (i.e. offset does not equal -2^31) - if sign(load_param.load.offset{idx}(1)) < 0 && load_param.load.offset{idx}(1) ~= -2^31 - file_idxs = [file_idxs(1)-1 file_idxs]; - end - - % Just copy the filenames we need - load_param.load.filenames{idx}(file_idxs) = orig_records.relative_filename{board_idx}(file_idxs); - - % Modify filename according to channel - for file_idx = 1:length(load_param.load.filenames{idx}) - if ~isequal(old_param_records.records.file.adc_headers,old_param_records.records.file.adcs) - load_param.load.filenames{idx}{file_idx}(9:10) = sprintf('%02d',board); - end - end - - filepath = get_segment_file_list(param,adc); - - % Convert relative file paths into absolute file paths if required, - % also corrects filesep (\ and /) - for file_idx = 1:length(load_param.load.filenames{idx}) - load_param.load.filenames{idx}{file_idx} ... - = fullfile(filepath,load_param.load.filenames{idx}{file_idx}); - end - end - load_param.load.file_version = param.records.file_version; - -else - error('Radar name %s not supported', param.radar_name); -end - -%% Setup control parameters for loading data -% ===================================================================== - -load_param.load.adcs = param.load.adcs; - -load_param.proc.pulse_comp = param.csarp.pulse_comp; -load_param.proc.ft_dec = param.csarp.ft_dec; -load_param.proc.ft_wind = param.csarp.ft_wind; -load_param.proc.ft_wind_time = param.csarp.ft_wind_time; -load_param.proc.presums = param.csarp.presums; -load_param.proc.combine_rx = param.csarp.combine_rx; -load_param.proc.pulse_rfi = param.csarp.pulse_rfi; -load_param.proc.trim_vals = param.csarp.trim_vals; -load_param.proc.coh_noise_method = param.csarp.coh_noise_method; -load_param.proc.coh_noise_arg = param.csarp.coh_noise_arg; - -load_param.surface = orig_records.surface; -if strcmpi(radar_name,'acords') - load_param.load.file_version = param.records.file_version; -end - -%% Load and Pulse Compress Data -% ===================================================================== -% Load data into g_data using load_mcords_data -load_param.load.imgs = param.load.imgs; - -if strcmpi(radar_name,'mcords') - % if strcmpi(param.season_name,'mcords_simulator') - % load_param.fn = get_filename(base_dir,'','','mat'); - % load_simulated_data(load_param); - % else - load_mcords_data(load_param); - % end -elseif any(strcmpi(radar_name,{'hfrds','hfrds2','mcords2','mcords3','mcords4','mcords5','seaice'})) - load_mcords2_data(load_param,all_records); -elseif strcmpi(radar_name,'accum2') - load_accum2_data(load_param); -elseif strcmpi(radar_name,'acords') - load_acords_data(load_param); -elseif strcmpi(radar_name,'mcrds') - if isfield(orig_records,'adc_phase_corr_deg') && isfield(param.radar,'adc_phase_corr_en') && param.radar.adc_phase_corr_en - load_param.adc_phase_corr_deg = orig_records.adc_phase_corr_deg; - else - load_param.adc_phase_corr_deg = zeros(length(load_param.surface),max(orig_records.param_records.records.file.adcs)); - end - load_mcrds_data(load_param); -elseif strcmpi(radar_name,'icards') - load_icards_data(load_param,param); -elseif any(strcmpi(radar_name,{'snow','kuband','snow2','kuband2','snow3','kuband3','kaband3','snow5'})) - load_param.proc.elev_correction = 0;%param.csarp.elev_correction; - load_param.proc.deconvolution = param.csarp.deconvolution; - load_param.proc.psd_smooth = param.csarp.psd_smooth; - load_param.radar_name = param.radar_name; - load_param.season_name = param.season_name; - load_param.day_seg = param.day_seg; - load_param.load.tmp_path = param.tmp_path; - load_param.out_path = param.out_path; - [img_time,img_valid_rng,img_deconv_filter_idx,img_freq] = load_fmcw_data(load_param,orig_records); - valid_rng = img_valid_rng{1}; - deconv_filter_idx = img_deconv_filter_idx{1}; - for wf = 1:length(wfs) - wfs(wf).time = img_time{1}; - wfs(wf).freq = img_freq{1}; - end -elseif strcmpi(radar_name,'sim'); - -end - -%% Prepare reference trajectory information -% ========================================================================= - -% Load reference trajectory -trajectory_param = struct('gps_source',orig_records.gps_source, ... - 'season_name',param.season_name,'radar_name',param.radar_name,'rx_path', 0, ... - 'tx_weights', [], 'lever_arm_fh', param.csarp.lever_arm_fh); -ref = trajectory_with_leverarm(orig_records,trajectory_param); - - -% Resample reference trajectory at output positions -% 1. Convert ref trajectory to ecef -ecef = zeros(3,size(ref.lat,2)); -[ecef(1,:) ecef(2,:) ecef(3,:)] = geodetic2ecef(ref.lat/180*pi, ref.lon/180*pi, ref.elev, WGS84.ellipsoid); -% 2. Resample based on input and output along track -ecef = interp1(along_track,ecef.',output_along_track,'linear','extrap').'; -% 3. Convert ecef to geodetic -[lat,lon,elev] = ecef2geodetic(ecef(1,:), ecef(2,:), ecef(3,:), WGS84.ellipsoid); -lat = lat*180/pi; -lon = lon*180/pi; -%% Remove coherent noise -% ========================================================================= -if param.csarp.coh_noise_method && ~any(strcmpi(radar_name,{'kuband','snow','kuband2','snow2','kuband3','kaband3','snow3','snow5'})) - - if param.csarp.coh_noise_method == 3 && isempty(param.csarp.coh_noise_arg) - param.csarp.coh_noise_arg = 255; - end - - % DC and near-DC REMOVAL - if param.csarp.ground_based - % Only remove coherent noise from sections which are moving - vel = diff(along_track) ./ diff(orig_records.gps_time); - good_mask = vel > median(vel)/5; - % figure(1); clf; - % plot(vel); - % hold on; - % vel(~good_mask) = NaN; - % plot(vel,'r'); - % hold off; - for idx=1:length(g_data) - % Transpose for faster memory access - g_data{idx} = permute(g_data{idx},[2 1 3]); - for rbin = 1:size(g_data{idx},2) - for wf_adc_idx = 1:size(g_data{idx},3) - if param.csarp.coh_noise_method == 1 - g_data{idx}(:,rbin,wf_adc_idx) = g_data{idx}(:,rbin,wf_adc_idx) - mean(g_data{idx}(good_mask,rbin,wf_adc_idx)); - else - error('param.csarp.coh_noise_method %d not supported.',param.csarp.coh_noise_method); - end - end - end - % Undo transpose - g_data{idx} = permute(g_data{idx},[2 1 3]); - end - else - % Remove only the DC Doppler component - for idx=1:length(g_data) - for wf_adc_idx = 1:size(g_data{idx},3) - if param.csarp.coh_noise_method == 1 - g_data{idx}(:,:,wf_adc_idx) = bsxfun(@minus, g_data{idx}(:,:,wf_adc_idx), ... - mean(g_data{idx}(:,:,wf_adc_idx),2)); - elseif param.csarp.coh_noise_method == 3 - g_data{idx}(:,:,wf_adc_idx) = bsxfun(@minus, g_data{idx}(:,:,wf_adc_idx), ... - fir_dec(g_data{idx}(:,:,wf_adc_idx),hanning(param.csarp.coh_noise_arg).'/(param.csarp.coh_noise_arg/2+0.5),1)); - end - end - end - end -end - -%% Main loop to process each image -% ========================================================================= -for img_idx = 1:length(load_param.load.imgs) - if param.csarp.combine_rx - % Receivers combined, so just run the first wf/adc pair - imgs_list = load_param.load.imgs{1}(1,:); - else - % Receivers processed individually, so get information for all wf/adc pairs. - imgs_list = load_param.load.imgs{img_idx}; - end - - %% Loop to process each wf-adc pair - for wf_adc_idx = 1:size(imgs_list,1) - % Processing loop - % Runs once for combine_rx = true - % Runs for each wf/adc pair in image if combine_rx = false - - wf = abs(imgs_list(wf_adc_idx,1)); - adc = abs(imgs_list(wf_adc_idx,2)); - - %% Compute trajectory, SAR phase center and coordinate system - if isempty(param.csarp.lever_arm_fh) - records = orig_records; - else - if ~param.csarp.combine_rx - % Create actual trajectory - trajectory_param = struct('gps_source',orig_records.gps_source, ... - 'season_name',param.season_name,'radar_name',param.radar_name, ... - 'rx_path', wfs(wf).rx_paths(adc), ... - 'tx_weights', wfs(wf).tx_weights, 'lever_arm_fh', param.csarp.lever_arm_fh); - records = trajectory_with_leverarm(orig_records,trajectory_param); - else - trajectory_param = struct('gps_source',orig_records.gps_source, ... - 'season_name',param.season_name,'radar_name',param.radar_name, ... - 'rx_path', wfs(wf).rx_paths(adc), ... - 'tx_weights', wfs(wf).tx_weights, 'lever_arm_fh', param.csarp.lever_arm_fh); - for tmp_wf_adc_idx = 2:size(load_param.load.imgs{1},1) - tmp_wf = abs(load_param.load.imgs{1}(tmp_wf_adc_idx,1)); - tmp_adc = abs(load_param.load.imgs{1}(tmp_wf_adc_idx,2)); - trajectory_param.rx_path(tmp_wf_adc_idx) = wfs(tmp_wf).rx_paths(tmp_adc); - end - records = trajectory_with_leverarm(orig_records,trajectory_param); - end - end - - % Create fcs.pos: phase center in the flight coordinate system - % 1. Convert phase center trajectory to ecef - ecef = zeros(3,size(records.lat,2)); - [ecef(1,:),ecef(2,:),ecef(3,:)] ... - = geodetic2ecef(wgs84,records.lat,records.lon,records.elev); - - % 2. Use the fcs to convert ecef to fcs coordinates and store in fcs.pos - Nx = size(fcs.origin,2); - good_rline = logical(ones(1,Nx)); - for out_rline = 1:Nx - % For this output range line determine the input range lines - rlines_in = find(along_track >= output_along_track(out_rline)-fcs.Lsar/2 ... - & along_track <= output_along_track(out_rline)+fcs.Lsar/2); - if isempty(rlines_in) - % Sometimes there are gaps in the data, we will use neighboring points - % to fill in these gaps - good_rline(out_rline) = 0; - continue; - end - fcs.pos(:,out_rline) = [fcs.x(:,out_rline) fcs.y(:,out_rline) fcs.z(:,out_rline)] \ (mean(ecef(:,rlines_in),2) - fcs.origin(:,out_rline)); - end - if ~any(good_rline) - error('Data gap extends across entire chunk. Consider breaking segment into two at this gap or increasing the chunk size.'); - end - - % 3. Fill in any missing lines - fcs.pos(:,~good_rline) = interp1(output_along_track(good_rline),fcs.pos(:,good_rline).',output_along_track(~good_rline).','linear','extrap').'; - - - %% SAR Processing - if strcmpi(param.csarp.sar_type,'fk') - % fk migration overview - % - % 1. Loop for each subaperture (repeat steps 2-4 for each aperture) - % - % 2. Motion compensation required before taking FFT - % g_data (raw with coherent noise optionally removed) - % --> g_data (motion compensated ft-fft) - % - % 3. Uniform re-sampling - % a. uniform_en = false, Assume data is uniformly sampled, apply fft in slow time and - % decimate in this domain by dropping doppler bins outside window - % g_data (motion compensated ft-fft) - % --> g_data (slow time ft/st-fft) [only done on the first loop] - % --> data (decimated ft/st-fft) - % b. uniform_en = true, Spatial filter to decimated axis and then take fft - % g_data (motion compensated ft-fft) - % --> data (decimated ft-fft) - % --> data (decimated ft/st-fft) - % - % 4. Regular fk migration for each subaperture - - g_data{img_idx}(:,:,wf_adc_idx) = fft(g_data{img_idx}(:,:,wf_adc_idx),[],1); - - fcs.squint = [0 0 -1].'; - %fcs.squint = fcs.squint ./ sqrt(dot(fcs.squint,fcs.squint)); - - %% Motion Compensation for fk migration - if param.csarp.mocomp.en - % Determine the required motion compensation (drange and dx) - % Positive drange means the the range will be made longer, time delay - % will be made larger, and phase will be more negative - % Positive dx means that the data will be shifted forward (i.e. it - % currently lags behind) - fcs.type = param.csarp.mocomp.type; - fcs.filter = param.csarp.mocomp.filter; - [drange,dx] = sar_motion_comp(fcs,records,ref,along_track,output_along_track); - - % Time shift data in the frequency domain - dtime = 2*drange/c; - for rline = 1:size(g_data{img_idx},2) - g_data{img_idx}(:,rline,wf_adc_idx) = g_data{img_idx}(:,rline,wf_adc_idx) ... - .*exp(-1i*2*pi*wfs(wf).freq*dtime(rline)); - end - - along_track_mc = along_track + dx; - else - along_track_mc = along_track; - end - - % output_along_track: these are the output values from csarp - % output_along_track_pre/post: these are the buffers to keep the - % data from wrapping around in slow-time (i.e. linear convolution - % vs. circular convolution) - % BUG!: At the beginning and end of a segment there is no data and - % the buffer is not added... need to fix. - output_along_track_pre = fliplr(output_along_track(1)-param.csarp.sigma_x:-param.csarp.sigma_x:along_track(1)); - if isempty(output_along_track_pre) - output_along_track_pre = [output_along_track(1)-param.csarp.sigma_x*(param.load.start_zero_pad:-1:1), output_along_track_pre]; - else - output_along_track_pre = [output_along_track_pre(1)-param.csarp.sigma_x*(param.load.start_zero_pad:-1:1), output_along_track_pre]; - end - output_along_track_post = output_along_track(end)+param.csarp.sigma_x:param.csarp.sigma_x:along_track(end); - if isempty(output_along_track_post) - output_along_track_post = [output_along_track_post, output_along_track(end)+param.csarp.sigma_x*(1:param.load.stop_zero_pad)]; - else - output_along_track_post = [output_along_track_post, output_along_track_post(end)+param.csarp.sigma_x*(1:param.load.stop_zero_pad)]; - end - output_along_track_full = [output_along_track_pre output_along_track output_along_track_post]; - - %% Prepare subaperture variables - num_subapertures = length(param.csarp.sub_aperture_steering); - if mod(num_subapertures,2) ~= 1 - error('Number of subapertures must be even'); - end - if any(param.csarp.sub_aperture_steering ~= -(num_subapertures-1)/4 : 0.5 : (num_subapertures-1)/4) - error('param.csarp.sub_aperture_steering must be of the form -N:0.5:N'); - end - proc_oversample = (1+num_subapertures)/2; % Oversampling rate - proc_sigma_x = param.csarp.sigma_x / proc_oversample; - proc_along_track = output_along_track_full(1) ... - + proc_sigma_x * (0:length(output_along_track_full)*proc_oversample-1); - - %% Uniform resampling and subaperture selection for fk migration - if param.csarp.mocomp.uniform_en - % Uniformly resample data in slow-time onto output along-track - data = arbitrary_resample(g_data{img_idx}(:,:,wf_adc_idx), ... - along_track_mc,proc_along_track, struct('filt_len', ... - proc_sigma_x*16,'dx',proc_sigma_x,'method','sinc')); - data = fft(data,[],2); - - else - % Assume data is already uniformly sampled in slow-time - % There are lots of approximations in this section... it's a bit - % of a hack. - x_lin = linspace(along_track(1), ... - along_track(end),length(along_track)); - - % Create kx (along-track wavenumber) axis of input data - kx = gen_kx(x_lin); - - % Create kx axis of output (desired) data - kx_desired = gen_kx(output_along_track_full); - - % Take slow-time FFT and decimate the data onto the desired - % output along track positions by selecting just the doppler - % bins that correspond to this - - g_data{img_idx}(:,:,wf_adc_idx) = fft(g_data{img_idx}(:,:,wf_adc_idx),[],2); - filt_idx = kx < max(kx_desired) & kx > min(kx_desired); - % Since kx and kx_desired won't match up perfectly, we may have - % to append a few more doppler bins to get the numbers to line - % up. - if length(output_along_track_full) - sum(filt_idx) == 1 - filt_idx(find(filt_idx==0,1)) = 1; - elseif length(output_along_track_full) - sum(filt_idx) == 2 - filt_idx(find(filt_idx==0,1)) = 1; - filt_idx(find(filt_idx==0,1,'last')) = 1; - end - filt_len = length(find(filt_idx)); - filt_idx = filt_idx.'; - filt_idx = find(filt_idx); - kx_sa = kx(filt_idx); - [kx_sa,kx_idxs] = sort(kx_sa); - filt_idx = filt_idx(kx_idxs); - filt_idx = ifftshift(filt_idx); - data = g_data{img_idx}(:,filt_idx,wf_adc_idx); - end - - %% fk migration - if param.csarp.mocomp.en - eps_r = perm_profile(mean(records.surface + dtime),wfs(wf).time,'constant',param.csarp.start_eps); - else - eps_r = perm_profile(mean(records.surface),wfs(wf).time,'constant',param.csarp.start_eps); - end - - kx = gen_kx(proc_along_track); - fk_data_ml = fk_migration(data,wfs(wf).time,wfs(wf).freq,kx,eps_r,param); - fk_data_ml = fk_data_ml(:,1+length(output_along_track_pre):end-length(output_along_track_post),:); - - if 0 - % DEBUG code - figure(1); clf; - for subap = 1:num_subapertures - imagesc(lp(fk_data_ml(300:700,:,subap))) - title(sprintf('%d',subap )); - pause(1); - end - figure(1); clf; - imagesc(lp(mean(abs(fk_data_ml(300:700,:,:)).^2,3))); - figure(2); clf; - imagesc(lp(mean(abs(fk_data_ml(300:700,:,1:6)).^2,3))); - figure(3); clf; - imagesc(lp(mean(abs(fk_data_ml(300:700,:,end-5:end)).^2,3))); - end - - if param.csarp.mocomp.en - %% Undo motion compensation - % Resample dtime to fk migration output - dtime = interp1(orig_records.gps_time,dtime,fcs.gps_time); - - % Time shift data in the frequency domain - fk_data_ml = fft(fk_data_ml,[],1); - fk_data_ml = fk_data_ml.*exp(1i*2*pi*repmat(wfs(wf).freq, [1,size(fk_data_ml,2),size(fk_data_ml,3)]) ... - .*repmat(dtime, [size(fk_data_ml,1),1,size(fk_data_ml,3)])); - fk_data_ml = ifft(fk_data_ml,[],1); - end - - %% Save Radar data - for subap = 1:num_subapertures - % Create output path - out_fn_dir = fullfile(ct_filename_out(param, ... - param.csarp.out_path), ... - sprintf('%s_data_%03d_%02d_%02d',param.csarp.sar_type,param.load.frm, ... - subap, param.load.sub_band_idx)); - if ~exist(out_fn_dir,'dir') - mkdir(out_fn_dir); - end - - % Save - param_records = old_param_records; - param_csarp = param; - if param.csarp.combine_rx - out_fn = fullfile(out_fn_dir,sprintf('img_%02d_chk_%03d.mat', img_idx, param.load.chunk_idx)); - else - out_fn = fullfile(out_fn_dir,sprintf('wf_%02d_adc_%02d_chk_%03d.mat', wf, adc, param.load.chunk_idx)); - end - fk_data = fk_data_ml(:,:,subap); - fprintf(' Saving %s (%s)\n', out_fn, datestr(now)); - save('-v6',out_fn,'fk_data','fcs','lat','lon','elev','out_rlines','wfs','param_csarp','param_records'); - end - - elseif strcmpi(param.csarp.sar_type,'tdbp_old') - % time domain backporjection overview - data = g_data{img_idx}(:,:,wf_adc_idx); - - % set up SAR coordinate system - [x_ecef, y_ecef, z_ecef] = geodetic2ecef(records.lat*pi/180, records.lon*pi/180, records.elev, WGS84.ellipsoid); - records.lon_ref = mean(records.lon); - records.lat_ref = mean(records.lat); - records.elev_ref = mean(records.elev); - [x_enu,y_enu,z_enu] = ecef2lv(x_ecef, y_ecef, z_ecef, records.lat_ref*pi/180, records.lon_ref*pi/180, records.elev_ref, WGS84.ellipsoid); -% along_track = [0 cumsum(sqrt(diff(x_enu).^2 + diff(y_enu).^2))]; - SAR_coord_param.phase_center = [x_enu;y_enu;z_enu]; - SAR_coord_param.Lsar = Lsar; - SAR_coord_param.along_track = along_track; - SAR_coord_param.output_along_track_idxs = param.proc.output_along_track_idxs; -% if strcmpi(param.season_name,'mcords_simulator') % for 20110708_01_001 simulated data -% param.proc.output_along_track_idxs = [fliplr([4632:-10:0]),[4642:10:length(along_track)]]; -% SAR_coord_param.output_along_track_idxs = param.proc.output_along_track_idxs; -% end - SAR_coord_param.wfs = wfs(wf); - - % surface tracker - % two methods to get ice surface: param.surf_source 1/2 - % 1: from get_heights; 2:from laser data; - param.surf_source = 1; - if param.surf_source == 1 - surfTimes = records.surface; - elseif param.surf_source == 2 - param.laser_surface = 1; - param.laser_data_fn = '/cresis/projects/metadata/2008_Greenland_TO_icessn/2008_Greenland/080801a_icessn_nadir0seg'; - param.laser_data_fn = '/cresis/projects/metadata/2008_Greenland_TO_icessn/2008_Greenland/080707_icessn_nadir0seg'; - fid = fopen(param.laser_data_fn); - [laser_data_tmp] = textscan(fid,'%f%f%f%f%f%f%f%f%f%f%f'); - fclose(fid); - Year = 2008; - Mon = 7; - Day = 7; - laser_data.gps_time = (datenum(Year,Mon,Day)-datenum(1970,1,1))*86400 + laser_data_tmp{1}; - laser_data.surf_elev = laser_data_tmp{4}; - laser_data.surf_elev = interp1(laser_data.gps_time,laser_data.surf_elev,records.gps_time); - surfTimes = 2*(records.elev-laser_data.surf_elev)/c; - clear laser_data_tmp; - end - - SAR_coord_param.surf = zeros(3,length(along_track)); - SAR_coord_param.surf(1,:) = x_enu; - SAR_coord_param.surf(2,:) = y_enu; - SAR_coord_param.surf(3,:) = z_enu - surfTimes*c/2; - SAR_coord_param.surf_p = polyfit(along_track,SAR_coord_param.surf(3,:),10); - SAR_coord_param.surf(3,:) = polyval(SAR_coord_param.surf_p,along_track); - N = length(SAR_coord_param.surf_p); - surfSlope = SAR_coord_param.surf_p(N-1); - x_pwr = along_track; - for ii = N-3:-1:1 - surfSlope = surfSlope + (N-ii)*SAR_coord_param.surf_p(ii)*along_track.*x_pwr; - x_pwr = x_pwr.*along_track; - end - surfSlope = surfSlope + 2*SAR_coord_param.surf_p(N-2)*along_track; - surfSlope(abs(surfSlope)<0.0001*pi/180) = 0; - SAR_coord_param.surfAngle = atan(surfSlope); - % SAR_coord_param.surfNormal = zeros(3,length(along_track)); - % SAR_coord_param.surfNormal(1,:) = cos(atan(surfSlope)+pi/2); - % SAR_coord_param.surfNormal(3,:) = sin(atan(surfSlope)+pi/2); - % SAR_coord_param.surfNormalAngle = atan(surfSlope)+pi/2; - SAR_coord_param.surfBins = round((2*(z_enu-SAR_coord_param.surf(3,:))/c-wfs(wf).time(1))/wfs(wf).dt) + 1; - - n = size(data,1); - m = length(param.proc.output_along_track_idxs); - SAR_coord_param.pixel = zeros(3,n,m); - SAR_coord_param.pixel(1,:,:) = repmat(x_enu(param.proc.output_along_track_idxs),n,1); - SAR_coord_param.pixel(2,:,:) = repmat(y_enu(param.proc.output_along_track_idxs),n,1); - eta_ice = sqrt(er_ice); - for line = 1:m - out_idx = param.proc.output_along_track_idxs(line); - SAR_coord_param.pixel(3,1:SAR_coord_param.surfBins(out_idx),line) = z_enu(out_idx) - wfs(wf).time(1)*c/2 - ... - [(0:SAR_coord_param.surfBins(out_idx)-1)'*wfs(wf).dt*c/2]; - SAR_coord_param.pixel(3,SAR_coord_param.surfBins(out_idx)+1,line) = ... - SAR_coord_param.pixel(3,SAR_coord_param.surfBins(out_idx),line) -... - c*(surfTimes(out_idx)-wfs(wf).time(SAR_coord_param.surfBins(out_idx)))/2 -... - (wfs(wf).time(SAR_coord_param.surfBins(out_idx)+1)-surfTimes(out_idx))*c/(2*eta_ice); - SAR_coord_param.pixel(3,SAR_coord_param.surfBins(out_idx)+2:n,line) = ... - SAR_coord_param.pixel(3,SAR_coord_param.surfBins(out_idx)+1,line) - ... - (1:n-SAR_coord_param.surfBins(out_idx)-1)'*wfs(wf).dt*c/(2*eta_ice); - end - SAR_coord_param.h = SAR_coord_param.phase_center(3,:)-SAR_coord_param.surf(3,:); - SAR_coord_param.h_mean = mean(SAR_coord_param.h); - Lsar_surf = c/wfs(wf).fc*SAR_coord_param.h_mean/(2*param.csarp.sigma_x); - SAR_coord_param.HbeamWidth = atan(0.5*Lsar_surf/SAR_coord_param.h_mean); - - tdbp_param = SAR_coord_param; - clear SAR_coord_param; - tdbp_param.proc.Nfft = 2^ceil(log2(length(wfs(wf).time_raw))); - tdbp_param.proc.skip_surf = param.csarp.skip_surf; - tdbp_param.proc.start_range_bin_above_surf = param.csarp.start_range_bin_above_surf; - tdbp_param.proc.start_range_bin = param.csarp.start_range_bin; - tdbp_param.proc.end_range_bin = param.csarp.end_range_bin; - tdbp_param.refraction_method = param.csarp.refraction_method; - tdbp_param.fc = wfs(wf).fc; - tdbp_param.time = wfs(wf).time; - tdbp_param.c = c; - tdbp_param.eta_ice = eta_ice; - tdbp_param.st_wind = param.csarp.st_wind; - tdbp_param.sub_aperture_steering = param.csarp.sub_aperture_steering; - - fcs.squint = [0 0 -1].'; - tdbp_data0 = tdbp(tdbp_param,data); - - for subap = 1:size(tdbp_data0,3) % save each subaperture data to its own folder - % Create output path - out_path = fullfile(ct_filename_out(param,param.csarp.out_path, 'CSARP_out'),... - sprintf('tdbp_data_%03d_%02d_%02d',param.proc.frm,subap, param.proc.sub_band_idx)); - if ~exist(out_path,'dir') - mkdir(out_path); - end - - % Create filename - % - Hack: multiple receivers are named with the first receiver in the list - out_fn = sprintf('wf_%02d_adc_%02d_chk_%03d', wf, adc, param.csarp.chunk_id); - out_full_fn = fullfile(out_path,[out_fn '.mat']); - - % Save - fprintf(' Saving output %s\n', out_full_fn); - param_records = old_param_records; - param_csarp = param; - param_csarp.tdbp = tdbp_param; - tdbp_data = tdbp_data0(:,:,subap); - save('-v6',out_full_fn,'tdbp_data','fcs','lat','lon','elev','wfs','param_csarp','param_records','tdbp_param'); - end - elseif strcmpi(param.csarp.sar_type,'mltdp') - fcs.squint = [0 0 -1].'; - [B,A] = butter(4,0.1); - - % Force elevation to be smooth (might be required for refraction) - smoothed_elevation = filtfilt(B,A,records.elev); - smoothed_ref_elevation = filtfilt(B,A,ref.elev); - - % Fit surface to polynomial to force it to be smooth (required for refraction) - % - Fit is done with special x-axis to prevent bad conditioning - smoothed_surface = filtfilt(B,A,records.surface); - sz_poly_order = 11; - xfit = linspace(-1,1,length(smoothed_surface)); - smoothed_surface = polyval(polyfit(xfit,smoothed_surface,sz_poly_order),xfit); - if 0 % set to 1 for surface fit over whole frame - smoothed_elevation = interp1(param.proc.along_track_frm,param.proc.smoothed_elevation,param.proc.along_track,'linear','extrap'); - smoothed_surface = interp1(param.proc.along_track_frm,param.proc.smoothed_surface,param.proc.along_track,'linear','extrap'); - end - - data = g_data{img_idx}(:,:,wf_adc_idx); - % options for processing window in fast time - surfBins_at_output = round((smoothed_surface-wfs(wf).time(1))/wfs(wf).dt)+1; - if isempty(param.csarp.skip_surf) - param.scarp.skip_surf = 0; % default value - end - if isempty(param.csarp.start_range_bin_above_surf) - param.csarp.start_range_bin_above_surf = 5; % default value - end - if param.csarp.skip_surf - if isempty(param.csarp.start_range_bin) - param.csarp.start_range_bin = max(surfBins_at_output) + 5; % default value - end - else - param.csarp.start_range_bin = min(surfBins_at_output) - param.csarp.start_range_bin_above_surf; % default value - end - if isempty(param.csarp.end_range_bin) - param.csarp.end_range_bin = size(data,1); % default value - end -% if strcmpi(param.season_name,'mcords_simulator') % for 20110708_01_001 simulated data -% output_along_track(463) = along_track(4632); -% output_along_track(1:462) = output_along_track(463)-[462:-1:1]*param.csarp.sigma_x; -% output_along_track(464:925) = output_along_track(463)+[1:462]*param.csarp.sigma_x; -% end - mltdp_data0 = ml_tdp(data,wfs(wf).fc,wfs(wf).time,along_track, smoothed_ref_elevation, ... - smoothed_elevation,smoothed_surface,output_along_track,Lsar, ... - length(param.csarp.sub_aperture_steering),param.csarp.start_range_bin,param.csarp.end_range_bin,param.csarp.start_eps); - - for subap = 1:size(mltdp_data0,3) % save each subaperture data to its own folder - % Create output path - out_path = fullfile(ct_filename_out(param,param.csarp.out_path, 'CSARP_out'),... - sprintf('mltdp_data_%03d_%02d_%02d',param.proc.frm,subap, param.proc.sub_band_idx)); - if ~exist(out_path,'dir') - mkdir(out_path); - end - - % Create filename - % - Hack: multiple receivers are named with the first receiver in the list - out_fn = sprintf('wf_%02d_adc_%02d_chk_%03d',wf, adc,param.csarp.chunk_id); - out_full_fn = fullfile(out_path,[out_fn '.mat']); - - fprintf(' Saving output %s\n', out_full_fn); - param_records = old_param_records; - param_csarp = param; - mltdp_data = mltdp_data0(:,:,subap); - save('-v6',out_full_fn,'mltdp_data','fcs','lat','lon','elev','wfs','param_csarp','param_records'); - end - elseif strcmpi(param.csarp.sar_type,'tdbp') - %% Time Domain Processor - % time domain backporjection overview - data = g_data{img_idx}(:,:,wf_adc_idx); - -% fcs_phase_centers = SAR_coord_system(SAR_coord_param,records,ref,along_track,along_track); - fcs_phase_center_idxs = interp1(output_along_track,1:length(output_along_track),along_track,'nearest'); - if isnan(fcs_phase_center_idxs(1)) - fcs_phase_center_idxs(1:find(~isnan(fcs_phase_center_idxs),1)-1) = 1; - end - if isnan(fcs_phase_center_idxs(end)) - fcs_phase_center_idxs(find(~isnan(fcs_phase_center_idxs),1,'last')+1:end) = length(output_along_track); - end - for fcs_idx = 1:length(fcs_phase_center_idxs) - fcs_phase_centers.x(:,fcs_idx) = fcs.x(:,fcs_phase_center_idxs(fcs_idx)); - end - - records.lon_ref = mean(records.lon); - records.lat_ref = mean(records.lat); - records.elev_ref = mean(records.elev); - - % set up SAR coordinate system - [x_ecef, y_ecef, z_ecef] = geodetic2ecef(records.lat*pi/180, records.lon*pi/180, records.elev, WGS84.ellipsoid); - - SAR_coord_param.phase_center = [x_ecef;y_ecef;z_ecef]; - SAR_coord_param.Lsar = sar.Lsar; - a1 = along_track(1); - SAR_coord_param.along_track = along_track-a1; - % Should be in c++ indices. - SAR_coord_param.output_along_track = output_along_track-a1; - SAR_coord_param.output_pos = fcs.origin; - SAR_coord_param.wfs = wfs(wf); - - % surface tracker - % two methods to get ice surface: param.surf_source 1/2 - % 1: from get_heights; 2:from laser data; - param.surf_source = 1; - if param.surf_source == 1 - surfTimes = records.surface; - elseif param.surf_source == 2 - param.laser_surface = 1; - param.laser_data_fn = '/cresis/projects/metadata/2008_Greenland_TO_icessn/2008_Greenland/080801a_icessn_nadir0seg'; - param.laser_data_fn = '/cresis/projects/metadata/2008_Greenland_TO_icessn/2008_Greenland/080707_icessn_nadir0seg'; - fid = fopen(param.laser_data_fn); - [laser_data_tmp] = textscan(fid,'%f%f%f%f%f%f%f%f%f%f%f'); - fclose(fid); - Year = 2008; - Mon = 7; - Day = 7; - laser_data.gps_time = (datenum(Year,Mon,Day)-datenum(1970,1,1))*86400 + laser_data_tmp{1}; - laser_data.surf_elev = laser_data_tmp{4}; - laser_data.surf_elev = interp1(laser_data.gps_time,laser_data.surf_elev,records.gps_time); - surfTimes = 2*(records.elev-laser_data.surf_elev)/c; - clear laser_data_tmp; - end - - for i = 1:3 - fcs_phase_centers.x(i,:) = interp1(output_along_track,fcs.x(i,:),along_track); - fcs_phase_centers.z(i,:) = interp1(output_along_track,fcs.z(i,:),along_track); - end - idx1 = find(~isnan(fcs_phase_centers.x(1,:)),1)-1; - idx2 = find(~isnan(fcs_phase_centers.x(1,:)),1,'last')+1; - for i = 1:3 - fcs_phase_centers.x(i,1:idx1) = fcs_phase_centers.x(i,idx1+1); - fcs_phase_centers.x(i,idx2:end) = fcs_phase_centers.x(i,idx2-1); - fcs_phase_centers.z(i,1:idx1) = fcs_phase_centers.z(i,idx1+1); - fcs_phase_centers.z(i,idx2:end) = fcs_phase_centers.z(i,idx2-1); - end - -% SAR_coord_param.surf = zeros(3,length(along_track)); -% SAR_coord_param.surf(1,:) = x_ecef +surfTimes*c/2 .* fcs_phase_centers.z(1,:); -% SAR_coord_param.surf(2,:) = y_ecef + surfTimes*c/2 .* fcs_phase_centers.z(2,:); -% SAR_coord_param.surf(3,:) = z_ecef + surfTimes*c/2 .* fcs_phase_centers.z(3,:); - - surfTimes = sgolayfilt(records.surface,3,round(param.csarp.surf_filt_dist / median(diff(along_track))/2)*2+1); -% surfTimes = sar.surf_pp.coefs(out_rlines,end); -% SAR_coord_param.surf_poly = sar.surf_pp.coefs(out_rlines,:).'*c/2; - SAR_coord_param.surf_along_track = -surfTimes*c/2; - surf_poly = spline(along_track,SAR_coord_param.surf_along_track); - SAR_coord_param.surf_poly = surf_poly.coefs.'; - SAR_coord_param.surf_line = polyfit(along_track,SAR_coord_param.surf_along_track,1); - - [~,surf_max_idx] = max(SAR_coord_param.surf_along_track); - if surf_max_idx==length(SAR_coord_param.surf_along_track); - surf_max_idx = surf_max_idx-1; - elseif surf_max_idx==1; - surf_max_idx = 2; - end - surf_max_poly = SAR_coord_param.surf_poly(:,surf_max_idx); - surf_der = polyval([length(surf_max_poly)-1:-1:1].'.*surf_max_poly(1:end-1),0); - if surf_der==0 - surf_max = surf_max_poly(end); - elseif surf_der<0 - surf_max_idx = surf_max_idx-1; - surf_max_poly = SAR_coord_param.surf_poly(:,surf_max_idx); - end - surf_at_max = roots((length(surf_max_poly)-1:-1:1).'.*surf_max_poly(1:end-1)); - surf_at_max = surf_at_max(surf_at_max>0 & surf_at_max=wfs(wf).time(1) - tdbp_param.end_time = param.csarp.end_time; - t_idx = interp1(wfs(wf).time,1:length(wfs(wf).time),param.csarp.end_time,'next'); - SAR_coord_param.pixel = SAR_coord_param.pixel(:,1:t_idx,:); - end - end -% - if isfield(param.csarp,'start_time') && ~isempty(param.csarp.start_time) - if param.csarp.start_time<=wfs(wf).time(end) && param.csarp.start_time>=wfs(wf).time(1) - tdbp_param.start_time = param.csarp.start_time; - t_idx = interp1(wfs(wf).time,1:length(wfs(wf).time),param.csarp.start_time,'previous'); - SAR_coord_param.pixel = SAR_coord_param.pixel(:,t_idx:end,:); - end - end - - % number of entries in library - N_lib = 32; - - dt = wfs(wf).dt; - - bw = wfs(wf).f1 - wfs(wf).f0; - - % sub-time bin step given number of entries - dts = dt/N_lib; - - matched_sig_lib = zeros(wfs(wf).Nt,N_lib); - mid = ceil(wfs(wf).Nt/2+1); - % loop through delays and directly create matched signal response - % implements marginal time shift in signal to find envelope. - t = ((1:size(matched_sig_lib,1)).'-mid)*dt; - for del_idx = 0:(N_lib-1) - matched_sig_lib(:,del_idx+1) = sinc((t-dts*del_idx)*bw); - end - % Filter contains ~97.5% of area under sinc^2 curve - matched_sig_lib = matched_sig_lib(mid+(-ceil(3.94/bw/dt):ceil(3.94/bw/dt)),:); - - tdbp_param = SAR_coord_param; - clear SAR_coord_param; - - tdbp_param.fc = wfs(wf).fc; - tdbp_param.t0 = t0; - tdbp_param.dt = dt; - tdbp_param.matched_sig_lib = matched_sig_lib; - - tdbp_param.fcs_x = fcs_phase_centers.x; - - tdbp_param.st_wind = param.csarp.st_wind; - tdbp_param.k_window = 1; - - tdbp_param.n0 = 1; - tdbp_param.n1 = eta_ice; - - if isfield(param.csarp,'refraction_flag'); - tdbp_param.refraction_flag = param.csarp.refraction_flag; - end - - kx_bw = abs(c/wfs(wf).fc)/param.csarp.sigma_x; - - num_subapertures = length(param.csarp.sub_aperture_steering); - tdbp_data0 = []; - for subap = 1:num_subapertures - - kx0 = param.csarp.sub_aperture_steering(subap); - kx_support_limits = asin(kx0+kx_bw*[-1,1]/2); - - tdbp_param.kx_support_limits = kx_support_limits; - - fprintf('Beginning SAR Processing...\n'); - tdbp_data0(:,:,subap) = sar_proc_task(tdbp_param,double(data)); - end - - for subap = 1:size(tdbp_data0,3) % save each subaperture data to its own folder - % Create output path - out_path = fullfile(ct_filename_out(param,param.csarp.out_path, 'CSARP_out'),... - sprintf('tdbp_data_%03d_%02d_%02d',param.load.frm,subap, param.load.sub_band_idx)); - if ~exist(out_path,'dir') - mkdir(out_path); - end - - % Create filename - % - Hack: multiple receivers are named with the first receiver in the list - out_fn = sprintf('wf_%02d_adc_%02d_chk_%03d', wf, adc, param.load.chunk_idx); - out_full_fn = fullfile(out_path,[out_fn '.mat']); - - % Save - fprintf(' Saving output %s\n', out_full_fn); - param_records = old_param_records; - param_csarp = param; - param_csarp.tdbp = tdbp_param; - tdbp_data = tdbp_data0(:,:,subap); - save('-v7.3',out_full_fn,'tdbp_data','fcs','lat','lon','elev','wfs','param_csarp','param_records','tdbp_param'); - end - end - end -end - -fprintf('%s done %s\n', mfilename, datestr(now)); - -success = true; - -return diff --git a/cresis-toolbox/processing/csarp_task_ollie.m b/cresis-toolbox/processing/csarp_task_ollie.m deleted file mode 100644 index bce7baff..00000000 --- a/cresis-toolbox/processing/csarp_task_ollie.m +++ /dev/null @@ -1,29 +0,0 @@ -function [success] = csarp_task_ollie(static_param_file_name,dynamic_param_file_name,frm,chunk,wf,adc) -% [success] = csarp_task_ollie(static_param_file_name,dynamic_param_file_name,frm,chunk,wf,adc) - -% Test -if (ischar(frm)) - frm=str2num(frm); -end -if (ischar(chunk)) - chunk=str2num(chunk); -end -if (ischar(wf)) - wf=str2num(wf); -end -if (ischar(adc)) - adc=str2num(adc); -end - -load(static_param_file_name,'static_param'); -param=static_param; -load(dynamic_param_file_name,'dynamic_param'); -param.load.frm = frm; -param.load.chunk_idx = chunk; -param.load.num_chunks = dynamic_param.frms.(['frm',num2str(frm)]).num_chunks; -param.load.recs = dynamic_param.frms.(['frm',num2str(frm)]).chunks.(['chunk',num2str(chunk)]).recs; -param.load.imgs = {[wf adc]}; -param.load.sub_band_idx = 1; - -clear frm chunk wf adc; -[success] = csarp_task(param); \ No newline at end of file diff --git a/cresis-toolbox/processing/data_load.m b/cresis-toolbox/processing/data_load.m index 74ddad6b..af5a7aa1 100644 --- a/cresis-toolbox/processing/data_load.m +++ b/cresis-toolbox/processing/data_load.m @@ -7,6 +7,8 @@ wfs = param.radar.wfs; +[output_dir,radar_type,radar_name] = ct_output_dir(param.radar_name); + %% Preallocate data % =================================================================== total_rec = param.load.recs(end)-param.load.recs(1)+1; @@ -110,8 +112,9 @@ % Find the file index for each record. If records.offset has extra % entries, get file_idxs for these too. - file_idxs = relative_rec_num_to_file_idx_vector( ... - param.load.recs+[0 size(records.offset,2)-(1+diff(param.load.recs))],records.relative_rec_num{board_idx}); + tmp_recs = param.load.recs+[0 size(records.offset,2)-(1+diff(param.load.recs))]; + file_idxs = relative_rec_num_to_file_idx_vector(tmp_recs(1):tmp_recs(end), ... + records.relative_rec_num{board_idx}); if param.records.file.version == 413 fn_name = records.relative_filename{1}{1}; @@ -135,6 +138,23 @@ rec = total_rec+1; + elseif param.records.file.version == 1000 + % for the full simulator + raw_data_file = load(param.sim.out_fn_raw_data); % has raw_data and param + wf = 1; + img = 1; + data = raw_data_file.raw_data; + hdr.bad_rec{img} = raw_data_file.hdr.bad_rec{img}; + hdr.nyquist_zone_hw{img} = raw_data_file.hdr.nyquist_zone_hw{img}; + hdr.nyquist_zone_signal{img} = raw_data_file.hdr.nyquist_zone_signal{img}; + hdr.DDC_dec{img} = raw_data_file.hdr.DDC_dec{img}; + hdr.DDC_freq{img} = raw_data_file.hdr.DDC_freq{img}; + hdr.Nt{img} = raw_data_file.hdr.Nt{img}; + hdr.t0_raw{img} = raw_data_file.hdr.t0_raw{img}; + hdr.t_ref{img} = raw_data_file.hdr.t_ref{img}; + + rec = total_rec+1; + else rec = 1; end @@ -203,9 +223,9 @@ adc = state.adc(ai); wf = state.wf(ai); img = state.img(ai); - mode_latch = state.mode(ai); + mode_latch = state.mode{ai}; weight = state.weight(ai); - subchannel = state.subchannel(ai); + subchannel = state.subchannel{ai}; % Determine which file has the current record file_idx = file_idxs(rec); @@ -283,7 +303,7 @@ if isempty(file_data_last_file) % Record offset is negative and so represents a record that % bridges into the next file - file_data_last_file = file_data(end+records.offset(board_idx,rec)+1:end); + file_data_last_file = file_data(end+max(-end,records.offset(board_idx,rec))+1:end); break else file_data_last_file = []; @@ -298,15 +318,48 @@ % file_data memory block rec_offset = records.offset(board_idx,rec) - file_data_offset; - % record_mode==1: Find the size of the current record + % record_mode==1: Find the size of the current record. This is used + % with the arena which has a dynamic record where the ordering of + % the contents of each record can change. If the sub-header fields + % have digital errors in them, especially in the length field, this + % can cause large erroneous reads from the file that break stuff. + % This field prevents the large erroneous reads. This code + % currently assumes that all records are the same length; the + % individual header-waveforms inside a record can all be different + % in size, but their sizes need to always add up to the same value + % for this code to work in all cases. if rec < length(file_idxs) && file_idxs(rec+1) == file_idxs(rec) % Next record is in this file, rec size is set to the offset to % the next record - rec_size = records.offset(board_idx,rec+1) - records.offset(board_idx,rec); + start_rec = -1; + test_rec = rec-1; + while start_rec < 0 && test_rec < size(records.offset,2) + test_rec = test_rec + 1; + if records.offset(board_idx,test_rec) ~= -2^31 + start_rec = test_rec; + end + end + next_rec = -1; + test_rec = start_rec; + while next_rec < 0 && test_rec < size(records.offset,2) + test_rec = test_rec + 1; + if records.offset(board_idx,test_rec) ~= -2^31 + next_rec = test_rec; + end + end + if next_rec ~= -1 + % Found two good records to estimate the maximum record size + % from. + rec_size = records.offset(board_idx,next_rec) - records.offset(board_idx,start_rec); + else + % Did not find two good records, so choose a very loose value + % for the maximum record size + rec_size = (length(file_data)+file_data_offset) - records.offset(board_idx,rec); + end else % Next record is in the next file, rec_size is set to the rest of % the data in this file - rec_size = (length(file_data)-file_data_offset) - records.offset(board_idx,rec); + rec_size = (length(file_data)+file_data_offset) - records.offset(board_idx,rec); end % Process all wf-adc pairs in this record @@ -315,8 +368,9 @@ adc = state.adc(ai); wf = state.wf(ai); img = state.img(ai); - mode_latch = state.mode(ai); - subchannel = state.subchannel(ai); + mode_latch = state.mode{ai}; + subchannel = state.subchannel{ai}; + mode_latch_subchannel = mode_latch*2^8 + subchannel; % Create unique number for each mode/subchannel pair (modes and subchannels limited to 0-255/8-bits) % Read in headers for this waveform % --------------------------------------------------------------- @@ -464,19 +518,32 @@ nyquist_zone_hw{img}(num_accum(ai)+1) = bitand(file_data(wf_hdr_offset+34),3); elseif any(param.records.file.version == [3 5 7]) nyquist_zone_hw{img}(num_accum(ai)+1) = file_data(wf_hdr_offset+45); + elseif any(param.records.file.version == [4]) + nyquist_zone_hw{img}(num_accum(ai)+1) = 0; else % nyquist_zone_hw defaults to 1 for all other file versions % (ideally this is overridden by - % records.settings.nyquist_zone) + % records.nyquist_zone_sig) nyquist_zone_hw{img}(num_accum(ai)+1) = 1; end + % Map any hardware nyquist_zones >= 4 to [0 1 2 3] + nyquist_zone_hw{img}(num_accum(ai)+1) = mod(nyquist_zone_hw{img}(num_accum(ai)+1),4); + elseif any(param.records.file.version == [415]) + Nt{img}(num_accum(ai)+1) = 3200; end - if isfield(records.settings,'nyquist_zone_hw') && ~isnan(records.settings.nyquist_zone_hw(rec)) - nyquist_zone_hw{img}(num_accum(ai)+1) = records.settings.nyquist_zone_hw(rec); + if isfield(records,'nyquist_zone_hw') && ~isnan(records.nyquist_zone_hw(rec)) +% if nyquist_zone_hw{img}(num_accum(ai)+1) ~= records.nyquist_zone_hw(rec) +% fprintf('Overwriting nz fro rec:%d from records\n',rec); +% end + nyquist_zone_hw{img}(num_accum(ai)+1) = records.nyquist_zone_hw(rec); end + % For the records generated using old data_load + % Map any hardware nyquist_zones >= 4 to [0 1 2 3] + nyquist_zone_hw{img}(num_accum(ai)+1) = mod(nyquist_zone_hw{img}(num_accum(ai)+1),4); + nyquist_zone_signal{img}(num_accum(ai)+1) = nyquist_zone_hw{img}(1); - if isfield(records.settings,'nyquist_zone') && ~isnan(records.settings.nyquist_zone(rec)) - nyquist_zone_signal{img}(num_accum(ai)+1) = records.settings.nyquist_zone(rec); + if isfield(records,'nyquist_zone_sig') && ~isnan(records.nyquist_zone_sig(rec)) + nyquist_zone_signal{img}(num_accum(ai)+1) = records.nyquist_zone_sig(rec); end % Extract waveform for this wf-adc pair @@ -516,12 +583,45 @@ tmp_data{adc,wf}(8:8:length(tmp)) = tmp(8:8:end); end + case 2 + % UTIG MARFA/HICARS fixed length + start_bin = 1+rec_offset + wfs(wf).offset + wfs(wf).time_raw_trim(1)*wfs(wf).sample_size; + if file_data(1+rec_offset+12800+68+5) == 2 + % file_data(12800+68+6) choff (channel offset) field + % indicates that adc 0 and 1 come first + if adc == 2 + start_bin = start_bin + 6400; + elseif adc == 3 + start_bin = start_bin + 12800 + 138; + elseif adc == 4 + start_bin = start_bin + 19200 + 138; + end + else + % file_data(12800+68+6) choff (channel offset) field + % indicates that adc 3 and 4 come first + if adc == 4 + start_bin = start_bin + 6400; + elseif adc == 1 + start_bin = start_bin + 12800 + 138; + elseif adc == 2 + start_bin = start_bin + 19200 + 138; + end + end + stop_bin = start_bin + (1+wfs(wf).complex)*Nt{img}(1)*wfs(wf).sample_size-1; + + tmp_data{adc,wf} = single(swapbytes(typecast(file_data(start_bin : stop_bin), wfs(wf).sample_type))); + case 1 - % Read in RSS dynamic record + % Read in Arena dynamic record sub_rec_offset = 0; missed_wf_adc = true; while sub_rec_offset < rec_size total_offset = rec_offset + sub_rec_offset; + if length(file_data) < total_offset+72 + % Unexpected end of file, so we missed the record + missed_wf_adc = true; + break; + end if swap_bytes_en radar_header_type = mod(swapbytes(typecast(file_data(total_offset+(9:12)),'uint32')),2^31); % Ignore MSB radar_header_len = double(swapbytes(typecast(file_data(total_offset+(13:16)),'uint32'))); @@ -541,18 +641,73 @@ end radar_profile_length = double(typecast(file_data(total_offset+radar_header_len+(21:24)),'uint32')); end - if any(radar_header_type == [5 16 23]) - if mode_latch == typecast(file_data(total_offset+17),'uint8') ... - && subchannel == typecast(file_data(total_offset+18),'uint8') - % This matches the mode and subchannel that we need - is_IQ = 0; - if length(file_data) < total_offset+24+radar_header_len+radar_profile_length - % Unexpected end of file, so we missed the record + if any(radar_header_type == [5 16 23 45 8194]) + if radar_header_type == 45 || radar_header_type == 8194 + profile = double(typecast(file_data(total_offset+19),'uint8')); +% mode2 = double(typecast(file_data(total_offset+17),'uint8')); +% subchannel2 = double(typecast(file_data(total_offset+18),'uint8')); + mode = param.records.data_map{board_idx}(param.records.data_map{board_idx}(:,1)==profile,2); + subchannel = param.records.data_map{board_idx}(param.records.data_map{board_idx}(:,1)==profile,3); +% fprintf('%2d %d %d %d %d\n', profile, mode, subchannel, mode2, subchannel2); + else + mode = double(typecast(file_data(total_offset+17),'uint8')); + subchannel = double(typecast(file_data(total_offset+18),'uint8')); + end + if swap_bytes_en + radar_profile_format = swapbytes(typecast(file_data(total_offset+radar_header_len+(17:20)),'uint32')); + if radar_profile_format == 196608 && radar_header_type ~= 8194 % 0x30000 + radar_profile_length = radar_profile_length*8; + end + else + radar_profile_format = typecast(file_data(total_offset+radar_header_len+(17:20)),'uint32'); + if radar_profile_format == 196608 && radar_header_type ~= 8194 % 0x30000 + radar_profile_length = radar_profile_length*8; + end + end + if length(file_data) < total_offset+24+radar_header_len+radar_profile_length + % Unexpected end of file, so we missed the record + missed_wf_adc = true; + break; + end + if swap_bytes_en + if swapbytes(typecast(file_data(total_offset+(1:8)),'uint64')) ~= 9187343241983295488 + % This record has a bad frame sync header so probably + % contains bad data. missed_wf_adc = true; break; end + if length(file_data) >= total_offset+24+radar_header_len+radar_profile_length+8 ... + && swapbytes(typecast(file_data(total_offset+ 24 + radar_header_len + radar_profile_length +(1:8)),'uint64')) ~= 9187343241983295488 + % This record has a bad frame sync header so probably + % contains bad data. + missed_wf_adc = true; + break; + end + else + if typecast(file_data(total_offset+(1:8)),'uint64') ~= 9187343241983295488 + % This record has a bad frame sync header so probably + % contains bad data. + missed_wf_adc = true; + break; + end + if length(file_data) >= total_offset+24+radar_header_len+radar_profile_length+8 ... + && typecast(file_data(total_offset+ 24 + radar_header_len + radar_profile_length +(1:8)),'uint64') ~= 9187343241983295488 + % This record has a bad frame sync header so probably + % contains bad data. + missed_wf_adc = true; + break; + end + end + found_mode_match = false; + for mode_idx = 1:length(mode) + if any(mode_latch_subchannel == mode(mode_idx)*2^8 + subchannel(mode_idx)) + found_mode_match = true; + end + end + if found_mode_match + % This matches the mode and subchannel that we need + is_IQ = 0; if swap_bytes_en - radar_profile_format = swapbytes(typecast(file_data(total_offset+radar_header_len+(17:20)),'uint32')); switch radar_profile_format case 0 % 0x00000 tmp_data{adc,wf} = single(swapbytes(typecast(file_data(total_offset+24+radar_header_len+(1:radar_profile_length)),'int16'))); @@ -567,7 +722,6 @@ is_IQ = 1; end else - radar_profile_format = typecast(file_data(total_offset+radar_header_len+(17:20)),'uint32'); switch radar_profile_format case 0 % 0x00000 tmp_data{adc,wf} = single(typecast(file_data(total_offset+24+radar_header_len+(1:radar_profile_length)),'int16')); @@ -667,18 +821,30 @@ % the loop. data{img}(1) = data{img}(1) + 1i; end - data{img}(1:Nt{img}(1),out_rec,wf_adc) = ... - data{img}(1:Nt{img}(1),out_rec,wf_adc) + state.weight(ai)*state.data{ai} / num_accum(ai); + if state.reset_sum(ai) + data{img}(1:Nt{img}(1),out_rec,wf_adc) = state.weight(ai)*state.data{ai} / num_accum(ai); + else + data{img}(1:Nt{img}(1),out_rec,wf_adc) = ... + data{img}(1:Nt{img}(1),out_rec,wf_adc) + state.weight(ai)*state.data{ai} / num_accum(ai); + end data{img}(Nt{img}(1)+1:end,out_rec,wf_adc) = wfs(wf).bad_value; hdr.nyquist_zone_hw{img}(out_rec) = nyquist_zone_hw{img}(1); hdr.nyquist_zone_signal{img}(out_rec) = nyquist_zone_signal{img}(1); hdr.DDC_dec{img}(out_rec) = DDC_dec{img}(1); - hdr.DDC_freq{img}(out_rec) = DDC_freq{img}(1); + if isfinite(wfs(wf).DDC_freq) + hdr.DDC_freq{img}(out_rec) = wfs(wf).DDC_freq; + else + hdr.DDC_freq{img}(out_rec) = DDC_freq{img}(1); + end hdr.Nt{img}(out_rec) = Nt{img}(1); hdr.t0_raw{img}(out_rec) = t0{img}(1); hdr.t_ref{img}(out_rec) = t_ref{img}(1); - + if any(param.records.file.version == [9]) + hdr.t_ref{img}(out_rec) = wfs(wf).t_ref; +% hdr.t0_raw{img}(out_rec) = wfs(wf).t0_raw; +% hdr.DDC_freq{img}(out_rec) = wfs(wf).DDC_freq; + end if any(isfinite(wfs(wf).blank)) % Blank data % - Blank is larger of two numbers passed in through radar worksheet blank parameter: @@ -705,6 +871,8 @@ end end +%% Corrections +% ========================================================================= if ~param.load.raw_data for img = 1:length(param.load.imgs) for wf_adc = 1:size(data{img},3) @@ -713,9 +881,25 @@ % Apply channel compensation, presum normalization, and constant % receiver gain compensation - chan_equal = 10.^(param.radar.wfs(wf).chan_equal_dB(param.radar.wfs(wf).rx_paths(adc))/20) ... - .* exp(1i*param.radar.wfs(wf).chan_equal_deg(param.radar.wfs(wf).rx_paths(adc))/180*pi); - mult_factor = single(wfs(wf).quantization_to_V(adc)/10.^(wfs(wf).adc_gains_dB(adc)/20)/chan_equal); + if strcmpi(radar_type,'deramp') + chan_equal = 1; + else + chan_equal = 10.^(param.radar.wfs(wf).chan_equal_dB(param.radar.wfs(wf).rx_paths(adc))/20) ... + .* exp(1i*param.radar.wfs(wf).chan_equal_deg(param.radar.wfs(wf).rx_paths(adc))/180*pi); + end + + if length(wfs(wf).system_dB) == 1 + % Only a single number is provided for system_dB so apply it to all + % receiver paths + mult_factor = single(wfs(wf).quantization_to_V(adc) ... + / (10.^(wfs(wf).adc_gains_dB(adc)/20) * chan_equal ... + * 10.^(wfs(wf).system_dB/20))); + else + % A number is provided for each receiver path for system_dB + mult_factor = single(wfs(wf).quantization_to_V(adc) ... + / (10.^(wfs(wf).adc_gains_dB(adc)/20) * chan_equal ... + * 10.^(wfs(wf).system_dB(param.radar.wfs(wf).rx_paths(adc))/20))); + end data{img}(:,:,wf_adc) = mult_factor * data{img}(:,:,wf_adc); % Compensate for receiver gain applied before ADC quantized the signal @@ -725,19 +909,31 @@ % - IMPORTANT: Only works for radars with constant length records. % Apply fast-time varying gain if enabled if wfs(wf).gain_en - gain_fn = fullfile(param.radar.gain_dir,sprintf('gain_wf_%d_adc_%d.mat',wf,adc)); + gain_dir = ct_filename_out(param, wfs(wf).gain_dir, 'analysis',1); + gain_fn = fullfile(gain_dir,sprintf('gain_wf_%d_adc_%d.mat',wf,adc)); if exist(gain_fn, 'file') ftg = load(gain_fn); - fprintf('Applying fast time gain compensation %d-%d\n',wf,adc); - corr_Time = ftg.param_analysis.radar.wfs(wf).time... % Actual Time axis - + (ftg.param_analysis.radar.wfs(wf).Tadc_adjust - 1*param.radar.wfs(wf).Tadc_adjust)... % Difference in Tadc_adjust - -1*ftg.param_analysis.radar.wfs(wf).time_correction; - % +-ftg.param_analysis.radar.wfs(wf).time(1); - % plot(wfs(wf).time_raw(1:hdr.Nt{img}(1))/1e-6,data{img}(:,1,wf_adc)); - % temp_data = bsxfun(@times,data{img}(:,:,wf_adc),interp1(corr_Time, 1./ftg.Gain_raw, wfs(wf).time_raw(1:hdr.Nt{img}(1)), 'linear','extrap')); - % hold on; plot(wfs(wf).time_raw(1:hdr.Nt{img}(1))/1e-6,temp_data(:,1),'--'); - data{img}(:,:,wf_adc) = bsxfun(@times,data{img}(:,:,wf_adc),interp1(corr_Time, 1./ftg.Gain_raw, wfs(wf).time_raw(1:hdr.Nt{img}(1)), 'linear','extrap')); - % hold on; plot(wfs(wf).time_raw(1:hdr.Nt{img}(1))/1e-6,data{img}(:,1,wf_adc)) + fprintf('Applying fast time gain compensation %d-%d\n %s\n',wf,adc,gain_fn); + corr_Time = ftg.param_collate_gain.radar.wfs(wf).time... % Actual Time axis + + (ftg.param_collate_gain.radar.wfs(wf).Tadc_adjust - 1*param.radar.wfs(wf).Tadc_adjust)... % Difference in Tadc_adjust + -1*ftg.param_collate_gain.radar.wfs(wf).time_correction; + if 0 + figure(1); + if img == 1 && wf_adc == 1 + clf; + end + plot(lp(mean(abs(data{img}(:,:,wf_adc)).^2,2))); + grid on; + keyboard + end + data{img}(:,:,wf_adc) = bsxfun(@times,data{img}(:,:,wf_adc),interp1(corr_Time, ftg.Gain_raw, wfs(wf).time_raw(1:hdr.Nt{img}(1)), 'linear','extrap')); + if 0 + hold on; + plot(lp(mean(abs(data{img}(:,:,wf_adc)).^2,2))); + figure(2); clf; + imagesc(lp(data{img}(:,:,wf_adc))); + keyboard + end else error(sprintf('Fast-time Gain compensation file not found:\n %s\nPlease run run_collate_ftg.m.',gain_fn)); end @@ -763,6 +959,64 @@ data{img}(1:hdr.Nt{img}(1),:,wf_adc) = data{img}(1:hdr.Nt{img}(1),:,wf_adc) ... .*interp1(reshape(chan_equal.gps_time,[numel(chan_equal.gps_time) 1]),chan_equal.chan_equal,records.gps_time,'linear','extrap').'; end + + % Remove burst noise + if wfs(wf).burst.en + % Load the burst noise file + noise_fn_dir = fileparts(ct_filename_out(param,wfs(wf).burst.fn, '')); + noise_fn = fullfile(noise_fn_dir,sprintf('burst_%s_wf_%d_adc_%d.mat', param.day_seg, wf, adc)); + fprintf(' Loading burst noise: %s (%s)\n', noise_fn, datestr(now)); + if ~exist(noise_fn,'file') + warning('data_load:burst:missing_file', ... + 'Burst noise file not found\t%s\t%s\n', noise_fn, datestr(now,'yyyymmdd_HHMMSS')); + else + burst = load(noise_fn); + % burst.burst_noise_table is created and described in + % collate_burst_noise.m + % --------------------------------------------------------------- + % burst_noise_table is a 3xNb table where Nb is the number of + % burst noise detections. Each column corresponds to one burst. + % + % burst_noise_table(1,:): The record that the burst occurs in. + % + % burst_noise_table(2,:): The start time of the burst + % + % burst_noise_table(3,:): The stop of the burst + + % start_idx to stop_idx: burst noise present in the current data + % block + start_idx = find(burst.burst_noise_table(1,:) >= param.load.recs(1),1); + if ~isempty(start_idx) + stop_idx = find(burst.burst_noise_table(1,:) <= param.load.recs(end),1,'last'); + if ~isempty(stop_idx) + % Loop through each burst noise incident in this block of + % data + for idx = start_idx:stop_idx + % Convert absolute record into relative index into loaded + % data arrays. + rec_rel = burst.burst_noise_table(1,idx) - param.load.recs(1) + 1; + + % start_bin to stop_bin: burst noise present in the current + % record + dt = wfs(wf).time_raw(2)-wfs(wf).time_raw(1); + start_bin = max(1, ... + round((burst.burst_noise_table(2,idx)-wfs(wf).time_raw(1))/dt)); + stop_bin = min(hdr.Nt{img}(rec_rel), ... + round((burst.burst_noise_table(3,idx)-wfs(wf).time_raw(1))/dt)); + + if start_bin == 1 && stop_bin == hdr.Nt{img}(rec_rel) + data{img}(:,rec_rel,wf_adc) = wfs(wf).bad_value; + hdr.bad_rec{img}(1,rec_rel,wf_adc) = 1; + elseif start_bin <= hdr.Nt{img}(rec_rel) && stop_bin >= 1 + data{img}(start_bin:stop_bin,rec_rel,wf_adc) = wfs(wf).bad_value; + end + end + end + end + + end + end % End burst noise removal + end end end @@ -774,6 +1028,7 @@ hdr.surface = fir_dec(records.surface,param.load.presums); %% Create trajectories +% ========================================================================= % Create reference trajectory (rx_path == 0, tx_weights = []) trajectory_param = struct('gps_source',records.gps_source, ... diff --git a/cresis-toolbox/processing/data_load_wfs.m b/cresis-toolbox/processing/data_load_wfs.m index a8afa8eb..d9c7de13 100644 --- a/cresis-toolbox/processing/data_load_wfs.m +++ b/cresis-toolbox/processing/data_load_wfs.m @@ -8,34 +8,50 @@ %% Build raw data loading "states" structure % ========================================================================= +if ~iscell(param.load.imgs{1}) + % param.load.imgs is not using multilooking syntax; reformat + % param.load.imgs non-multilooking syntax to multilooking syntax to + % ensure param.load.imgs is always in the multilooking syntax. This + % makes coding easier since the format is always the same. + for img = 1:length(param.load.imgs) + param.load.imgs{img} = {param.load.imgs{img}}; + end +end + % Create list of unique boards to load board_list = []; board_idxs = []; for img = 1:length(param.load.imgs) - for wf_adc = 1:size(param.load.imgs{img},1) - [board_list(end+1),board_idxs(end+1)] = wf_adc_to_board(param,param.load.imgs{img}(wf_adc,:)); - - % Follow wfs(wf).next field links for additional wf-adc pairsboards to load - wf = param.load.imgs{img}(wf_adc,1); - adc = param.load.imgs{img}(wf_adc,2); - if ~isfield(param.radar.wfs(wf),'next') || size(param.radar.wfs(wf).next,1) < adc || isnan(param.radar.wfs(wf).next(adc,1)) - % if next is not specified, then this wf-adc pair is the last to be - % loaded in the next chain. - param.radar.wfs(wf).next(adc,1:2) = [NaN NaN]; - end - next = param.radar.wfs(wf).next(adc,1:2); - while ~isnan(next) - wf = next(1); - adc = next(2); + for ml_idx = 1:length(param.load.imgs{img}) + for wf_adc = 1:size(param.load.imgs{img}{ml_idx},1) + [board_list(end+1),board_idxs(end+1)] = wf_adc_to_board(param,param.load.imgs{img}{ml_idx}(wf_adc,:)); + + % Follow wfs(wf).next field links for additional wf-adc pairsboards to load + wf = param.load.imgs{img}{ml_idx}(wf_adc,1); + adc = param.load.imgs{img}{ml_idx}(wf_adc,2); if ~isfield(param.radar.wfs(wf),'next') || size(param.radar.wfs(wf).next,1) < adc || isnan(param.radar.wfs(wf).next(adc,1)) % if next is not specified, then this wf-adc pair is the last to be % loaded in the next chain. - param.radar.wfs(wf).next(adc,1:2) = [NaN NaN]; + if isfield(param.radar.wfs(wf),'next') && size(param.radar.wfs(wf).next,1) < adc + param.radar.wfs(wf).next(end+1:adc,1:2) = NaN; + else + param.radar.wfs(wf).next(adc,1:2) = NaN; + end end - % Determine which board this wf-adc pair comes from - [board_list(end+1),board_idxs(end+1)] = wf_adc_to_board(param,[wf adc]); - % Follow the wfs(wf).next field link next = param.radar.wfs(wf).next(adc,1:2); + while ~isnan(next) + wf = next(1); + adc = next(2); + if ~isfield(param.radar.wfs(wf),'next') || size(param.radar.wfs(wf).next,1) < adc || isnan(param.radar.wfs(wf).next(adc,1)) + % if next is not specified, then this wf-adc pair is the last to be + % loaded in the next chain. + param.radar.wfs(wf).next(adc,1:2) = [NaN NaN]; + end + % Determine which board this wf-adc pair comes from + [board_list(end+1),board_idxs(end+1)] = wf_adc_to_board(param,[wf adc]); + % Follow the wfs(wf).next field link + next = param.radar.wfs(wf).next(adc,1:2); + end end end end @@ -51,86 +67,98 @@ states(state_idx).board_idx = board_idxs(state_idx); states(state_idx).adc = []; states(state_idx).wf = []; - states(state_idx).mode = []; - states(state_idx).subchannel = []; + states(state_idx).mode = {}; + states(state_idx).subchannel = {}; states(state_idx).wf_adc = []; states(state_idx).img = []; + states(state_idx).ml_idx = []; states(state_idx).weight = []; states(state_idx).next = []; + states(state_idx).reset_sum = []; end for img = 1:length(param.load.imgs) % For each image img - for wf_adc = 1:size(param.load.imgs{img},1) % For ach wf-adc pair - wf = param.load.imgs{img}(wf_adc,1); - adc = param.load.imgs{img}(wf_adc,2); - - [board,~,profile] = wf_adc_to_board(param,param.load.imgs{img}(wf_adc,:)); - % Determine if this wf,adc is from the current board - if board ~= states(state_idx).board - continue; - end - if any(param.records.file.version == [9 10 103 412]) - mode_latch = profile(1); - subchannel = profile(2); - else - mode_latch = 0; - subchannel = 0; - end - - if ~isfield(param.radar.wfs(wf),'weight') || length(param.radar.wfs(wf).weight) < adc - % if weight is not specified, then this wf-adc pair is loaded with a - % weight of one - param.radar.wfs(wf).weight(adc) = 1; - end - % Create wf_adc_sum list from weight/next commands - states(state_idx).adc(end+1) = adc; - states(state_idx).wf(end+1) = wf; - states(state_idx).mode(end+1) = mode_latch; - states(state_idx).subchannel(end+1) = subchannel; - states(state_idx).wf_adc(end+1) = wf_adc; - states(state_idx).img(end+1) = img; - states(state_idx).weight(end+1) = param.radar.wfs(wf).weight(adc); - next = param.radar.wfs(wf).next(adc,1:2); - while ~isnan(next(1)) - wf = next(1); - adc = next(2); - if ~isfield(param.radar.wfs(wf),'weight') || length(param.radar.wfs(wf).weight) < adc - % if weight is not specified, then this wf-adc pair is loaded with a - % weight of one - param.radar.wfs(wf).weight(adc) = 1; + for ml_idx = 1:length(param.load.imgs{img}) + for wf_adc = 1:size(param.load.imgs{img}{ml_idx},1) % For ach wf-adc pair + wf = param.load.imgs{img}{ml_idx}(wf_adc,1); + adc = param.load.imgs{img}{ml_idx}(wf_adc,2); + + [board,~,profile] = wf_adc_to_board(param,param.load.imgs{img}{ml_idx}(wf_adc,:)); + % Determine if this wf,adc is from the current board + if board ~= states(state_idx).board + continue; end - % Determine which board this wf-adc pair comes from - [board,~,profile] = wf_adc_to_board(param,[wf adc]); - next_state_idx = find(boards == board,1); if any(param.records.file.version == [9 10 103 412]) - mode_latch = profile(1); - subchannel = profile(2); + mode_latch = profile{1}(:,1); + subchannel = profile{1}(:,2); else mode_latch = 0; subchannel = 0; end - % Add wf-adc pair to states list - if next_state_idx > length(states) || ~isfield(states(next_state_idx),'board') ... - || isempty(states(state_idx).board) - % Create new state if not already created. - states(next_state_idx).board = boards(next_state_idx); - states(next_state_idx).board_idx = board_idxs(next_state_idx); - states(next_state_idx).adc = []; - states(next_state_idx).wf = []; - states(next_state_idx).mode = []; - states(next_state_idx).subchannel = []; - states(next_state_idx).wf_adc = []; - states(next_state_idx).img = []; - states(next_state_idx).weight = []; - states(next_state_idx).next = []; + + if ~isfield(param.radar.wfs(wf),'weight') + % if weight is not specified, then this wf-adc pair is loaded with a + % weight of one + param.radar.wfs(wf).weight(adc) = 1; + elseif length(param.radar.wfs(wf).weight) < adc + param.radar.wfs(wf).weight(end+1:adc) = 1; end - states(next_state_idx).adc(end+1) = adc; - states(next_state_idx).wf(end+1) = wf; - states(next_state_idx).mode(end+1) = mode_latch; - states(next_state_idx).subchannel(end+1) = subchannel; - states(next_state_idx).wf_adc(end+1) = wf_adc; - states(next_state_idx).img(end+1) = img; - states(next_state_idx).weight(end+1) = param.radar.wfs(wf).weight(adc); + % Create wf_adc_sum list from weight/next commands + states(state_idx).adc(end+1) = adc; + states(state_idx).wf(end+1) = wf; + states(state_idx).mode{end+1} = mode_latch; + states(state_idx).subchannel{end+1} = subchannel; + states(state_idx).wf_adc(end+1) = wf_adc; + states(state_idx).img(end+1) = img; + states(state_idx).ml_idx(end+1) = ml_idx; + states(state_idx).weight(end+1) = param.radar.wfs(wf).weight(adc); + states(state_idx).reset_sum(end+1) = true; next = param.radar.wfs(wf).next(adc,1:2); + while ~isnan(next(1)) + wf = next(1); + adc = next(2); + if ~isfield(param.radar.wfs(wf),'weight') || length(param.radar.wfs(wf).weight) < adc + % if weight is not specified, then this wf-adc pair is loaded with a + % weight of one + param.radar.wfs(wf).weight(adc) = 1; + end + % Determine which board this wf-adc pair comes from + [board,~,profile] = wf_adc_to_board(param,[wf adc]); + next_state_idx = find(boards == board,1); + if any(param.records.file.version == [9 10 103 412]) + mode_latch = profile{1}(:,1); + subchannel = profile{1}(:,2); + else + mode_latch = 0; + subchannel = 0; + end + % Add wf-adc pair to states list + if next_state_idx > length(states) || ~isfield(states(next_state_idx),'board') ... + || isempty(states(state_idx).board) + % Create new state if not already created. + states(next_state_idx).board = boards(next_state_idx); + states(next_state_idx).board_idx = board_idxs(next_state_idx); + states(next_state_idx).adc = []; + states(next_state_idx).wf = []; + states(next_state_idx).mode = {}; + states(next_state_idx).subchannel = {}; + states(next_state_idx).wf_adc = []; + states(next_state_idx).img = []; + states(next_state_idx).ml_idx = []; + states(next_state_idx).weight = []; + states(next_state_idx).next = []; + states(next_state_idx).reset_sum = []; + end + states(next_state_idx).adc(end+1) = adc; + states(next_state_idx).wf(end+1) = wf; + states(next_state_idx).mode{end+1} = mode_latch; + states(next_state_idx).subchannel{end+1} = subchannel; + states(next_state_idx).wf_adc(end+1) = wf_adc; + states(next_state_idx).img(end+1) = img; + states(next_state_idx).ml_idx(end+1) = ml_idx; + states(next_state_idx).weight(end+1) = param.radar.wfs(wf).weight(adc); + states(next_state_idx).reset_sum(end+1) = false; + next = param.radar.wfs(wf).next(adc,1:2); + end end end end @@ -156,12 +184,16 @@ % bad_value: Value to use for bad records (is also the value used to fill % in unused parts of the data matrix when the number of range bins is % allowed to vary on a range line to range line basis). The default value - % is 0. Typical values are 0 or NaN. If NaN, consider using "nan_dec" - % option in qlook. + % is 0 for non-deramp systems and NaN for deramp systems. Typical values + % are 0 or NaN. If NaN, consider using "nan_dec" option in qlook. if isfield(param.radar.wfs(wf),'bad_value') && ~isempty(param.radar.wfs(wf).bad_value) wfs(wf).bad_value = param.radar.wfs(wf).bad_value; else - wfs(wf).bad_value = 0; + if strcmpi(radar_type,'deramp') + wfs(wf).bad_value = NaN; + else + wfs(wf).bad_value = 0; + end end if isfield(param.radar.wfs(wf),'DDC_dec') && ~isempty(param.radar.wfs(wf).DDC_dec) @@ -321,6 +353,8 @@ end if any(param.records.file.version == [405 406 410]) % acords and mcrds wfs(wf).Nt_raw = records.settings.wfs(1).wfs(wf).num_sam(1) - sum(wfs(wf).time_raw_trim); + elseif any(param.records.file.version == [415]) + wfs(wf).Nt_raw = 3200; elseif isfield(records.settings.wfs,'num_sam') if numel(records.settings.wfs) >= wf wfs(wf).Nt_raw = records.settings.wfs(wf).num_sam(1); @@ -362,6 +396,11 @@ else wfs(wf).gain_dir = ''; end + if isfield(param.radar.wfs(wf),'nz_complex') && ~isempty(param.radar.wfs(wf).nz_complex) + wfs(wf).nz_complex = param.radar.wfs(wf).nz_complex; + else + wfs(wf).nz_complex = false; + end if isfield(param.radar.wfs(wf),'nz_trim') && ~isempty(param.radar.wfs(wf).nz_trim) wfs(wf).nz_trim = param.radar.wfs(wf).nz_trim; else @@ -424,20 +463,26 @@ wfs(wf).coh_noise_arg.A_coh_noise = 1; end end + if isfield(param.radar.wfs(wf),'burst') && ~isempty(param.radar.wfs(wf).burst) + wfs(wf).burst = param.radar.wfs(wf).burst; + else + wfs(wf).burst = []; + end + if ~isfield(wfs(wf).burst,'en') || isempty(wfs(wf).burst.en) + wfs(wf).burst.en = false; + end + if ~isfield(wfs(wf).burst,'fn') || isempty(wfs(wf).burst.fn) + wfs(wf).burst.fn = 'analysis'; + end if isfield(param.radar.wfs(wf),'deconv') && ~isempty(param.radar.wfs(wf).deconv) wfs(wf).deconv = param.radar.wfs(wf).deconv; else wfs(wf).deconv = []; - param.radar.wfs(wf).deconv = []; end - if isfield(param.radar.wfs(wf).deconv,'en') && ~isempty(param.radar.wfs(wf).deconv.en) - wfs(wf).deconv.en = param.radar.wfs(wf).deconv.en; - else + if ~isfield(wfs(wf).deconv,'en') || isempty(wfs(wf).deconv.en) wfs(wf).deconv.en = false; end - if isfield(param.radar.wfs(wf).deconv,'fn') && ~isempty(param.radar.wfs(wf).deconv.fn) - wfs(wf).deconv.fn = param.radar.wfs(wf).deconv.fn; - else + if ~isfield(wfs(wf).deconv,'fn') || isempty(wfs(wf).deconv.fn) wfs(wf).deconv.fn = 'analysis'; end % Per wf-adc pair amplitude equalization @@ -469,18 +514,23 @@ else wfs(wf).ref_fn = ''; end - + if isfield(param.radar.wfs(wf),'tx_weights') && ~isempty(param.radar.wfs(wf).tx_weights) wfs(wf).tx_weights = param.radar.wfs(wf).tx_weights; else wfs(wf).tx_weights = 1; end + if isfield(param.radar.wfs(wf),'system_dB') && ~isempty(param.radar.wfs(wf).system_dB) + wfs(wf).system_dB = param.radar.wfs(wf).system_dB; + else + wfs(wf).system_dB = 0; + end if isfield(param.radar.wfs(wf),'adc_gains_dB') && ~isempty(param.radar.wfs(wf).adc_gains_dB) wfs(wf).adc_gains_dB = param.radar.wfs(wf).adc_gains_dB; else wfs(wf).adc_gains_dB = 0; end - + %% Compute supporting variables % ======================================================================= wfs(wf).chirp_rate = (wfs(wf).f1-wfs(wf).f0) / wfs(wf).Tpd; @@ -516,7 +566,7 @@ end wfs(wf).quantization_to_V ... = param.radar.Vpp_scale * 2.^wfs(wf).bit_shifts ... - / (2^param.radar.adc_bits*wfs(wf).presums); + ./ (2^param.radar.adc_bits*wfs(wf).presums); if strcmpi(radar_type,'deramp') %% FMCW: Create time and frequency axis information @@ -555,7 +605,7 @@ % --------------------------------------------------------------------- dt = 1/wfs(wf).fs_raw; wfs(wf).time_raw = wfs(wf).t0_raw + dt*(0:wfs(wf).Nt_raw-1).'; - + nz0 = floor((wfs(wf).f0-wfs(wf).DDC_freq)/wfs(wf).fs_raw*2); nz1 = floor((wfs(wf).f1-wfs(wf).DDC_freq)/wfs(wf).fs_raw*2); @@ -566,7 +616,7 @@ % requirements for cluster processing before data loading happens. if isfield(param.radar.wfs(wf),'complex') ... && ~isempty(param.radar.wfs(wf).complex) - wfs(wf).complex = param.radar.wfs(wf).complex; + wfs(wf).complex = param.radar.wfs(wf).complex; else if nz0 == nz1 && wfs(wf).DDC_dec == 1 && wfs(wf).DDC_freq == 0 % Assume real sampling since signal does not cross Nyquist boundary @@ -577,7 +627,7 @@ wfs(wf).complex = true; end end - + if ~wfs(wf).complex if mod(nz0,2) % Negative frequencies first since this is an odd Nyquist zone @@ -618,7 +668,7 @@ wfs(wf).freq_raw = wfs(wf).freq_raw - floor((wfs(wf).freq_raw - (wfs(wf).fc-wfs(wf).fs_raw/2))/wfs(wf).fs_raw)*wfs(wf).fs_raw; end end - + % Pulse compressed data is applied to complex base banded raw data % with f_LO = wfs(wf).fc-wfs(wf).DDC_freq @@ -710,7 +760,7 @@ % Apply time correction so that start time will be a multiple of % the sampling frequency of the radar wfs(wf).ref{adc} = wfs(wf).ref{adc} .* exp(1i*2*pi*wfs(wf).freq_pc*wfs(wf).time_correction); - + % Normalize reference function so that it is an estimator % -- Accounts for pulse duration differences time_domain_ref = ifft(wfs(wf).ref{adc}); @@ -731,8 +781,8 @@ % 6 snow4, kuband4 (NI based, Spring 2015 NRL version of DDC code, multichannel) % 7 snow5 (NI based, AWI version of DDC code, multichannel, multiwaveform). Note this is a standard file type loaded by basic_load.m % 8 snow8 (NI based, Keysight waveform generator). Some files start with snow4 from the test flight. -% 9 snow9 (RSS based, Arena Snow Radar). -% 10 snow10 (RSS based, Arena Helicopter Snow Radar). +% 9 snow9 (Arena based, Arena Snow Radar). +% 10 snow10 (Arena based, Arena Helicopter Snow Radar). % 11 data_v11 (NI based, Mini-Snow Radar). % 101 accum (Leuschen/Ledford based) % 102 accum2 (Sundance, Paden/Rink based) @@ -747,7 +797,7 @@ % 409 icards (?/Akins Linux PCI card based, introduced 1993 Greenland P3) % 410 mcrds (Akins based, introduced 2006 Greenland P3, multichannel/waveforms) % 411 hfrds (Leuschen eval board based, 2013 Antarctica G1XB???, 2016 Greenland G1XB) -% 412 hfrds2 (RSS based, Arena HF Sounder, 2016 Greenland TOdtu). +% 412 hfrds2 (Arena based, Arena HF Sounder, 2016 Greenland TOdtu). % % wfs(wf).wf_offset: bytes of data before this data waveform % wfs(wf).num_sam: bytes of data in this record @@ -755,17 +805,17 @@ for wf = 1:length(param.radar.wfs) switch param.records.file.version - + case {1,2,3,4,5,7,8,11} - if param.records.file.version == 1 + if param.records.file.version == 1 HEADER_SIZE = 32; WF_HEADER_SIZE = 0; elseif param.records.file.version == 2 HEADER_SIZE = 40; WF_HEADER_SIZE = 0; elseif param.records.file.version == 4 - HEADER_SIZE = 32; - WF_HEADER_SIZE = 4; + HEADER_SIZE = 32; + WF_HEADER_SIZE = 4; else HEADER_SIZE = 0; WF_HEADER_SIZE = 48; @@ -787,6 +837,18 @@ + wfs(wf).sample_size*wfs(wf).adc_per_board*records.settings.wfs(wf-1).num_sam ... + HEADER_SIZE + WF_HEADER_SIZE; end + + case {415} + % UTIG MARFA/HICARS 60 MHz + HEADER_SIZE = 0; + WF_HEADER_SIZE = 0; + wfs(wf).wf_header_size = WF_HEADER_SIZE; + wfs(wf).record_mode = 2; + wfs(wf).complex = 0; + wfs(wf).sample_size = 2; + wfs(wf).adc_per_board = 4; + wfs(wf).sample_type = 'int16'; + wfs(wf).offset = HEADER_SIZE + WF_HEADER_SIZE; case {9,10,103,412} wfs(wf).record_mode = 1; @@ -874,7 +936,7 @@ end end -end +end % if param.load.file_version == 402 || param.load.file_version == 403 % HEADER_SIZE = 32; @@ -923,4 +985,4 @@ % sample_type = 'int32'; % HACK FIX LATER % boards = unique(param.records.wf_adc_boards(1,param.load.adcs)); % end -% +% diff --git a/cresis-toolbox/processing/data_merge_combine.m b/cresis-toolbox/processing/data_merge_combine.m index efde73ca..70a1a95b 100644 --- a/cresis-toolbox/processing/data_merge_combine.m +++ b/cresis-toolbox/processing/data_merge_combine.m @@ -52,6 +52,6 @@ %% Combine images into a single image % ========================================================================= for img = 1:length(param.load.imgs) - data{img} = mean(data{img},3); + data{img} = nanmean(data{img},3); end end diff --git a/cresis-toolbox/processing/data_pulse_compress.m b/cresis-toolbox/processing/data_pulse_compress.m index 5f28a4c2..6b32300f 100644 --- a/cresis-toolbox/processing/data_pulse_compress.m +++ b/cresis-toolbox/processing/data_pulse_compress.m @@ -30,11 +30,14 @@ nz_prev = NaN; nz_signal_prev = NaN; for rec = 1:size(data{img},2) + if hdr.bad_rec{img}(rec) + continue; + end rx = param.radar.wfs(wf).rx_paths(adc); nz = double(hdr.nyquist_zone_hw{img}(rec)); % Check and load measured filter response for corresponding Nyquist Zone if ~(nz==nz_prev) - prepulse_fn = fullfile(ct_filename_out(param,'analysis','',1),... + prepulse_fn = fullfile(ct_filename_out(param,prepulse_H.dir,'',1),... sprintf('%s_rx_%d_nz_%d.mat', param.radar.wfs(wf).prepulse_H.fn, rx, nz)); prepulse = load(prepulse_fn); nz_signal_prev = NaN; @@ -69,7 +72,6 @@ end % rec loop end % INVERSE FILTER if strcmpi ends here - extra_delay = zeros(1,size(data{img},2)); if strcmpi(wfs(wf).prepulse_H.type,'NI_DDC_2019') if 0 % Test code to find each DDC filter delay @@ -132,6 +134,7 @@ plot(angle(ref8)) end + data_complex_hack = false; % for rec = 1:size(data{img},2) freq_axis = ifftshift(-floor(hdr.Nt{img}(rec)/2):floor((hdr.Nt{img}(rec)-1)/2)).'; if hdr.DDC_dec{img}(rec) == 2 @@ -143,7 +146,6 @@ data{img}(1:hdr.Nt{img}(rec),rec,wf_adc) = exp(1i*-pi/2)*hdr.DDC_dec{img}(rec) ... * data{img}(1:hdr.Nt{img}(rec),rec,wf_adc); % Delay of 100/4 = 25 relative to DDC_dec==2 - extra_delay(rec) = 25; data{img}(1:hdr.Nt{img}(rec),rec,wf_adc) = ifft(fft(data{img}(1:hdr.Nt{img}(rec),rec,wf_adc)) ... .* exp(1i*2*pi*25*freq_axis/hdr.Nt{img}(rec))); elseif hdr.DDC_dec{img}(rec) == 8 @@ -151,7 +153,6 @@ data{img}(1:hdr.Nt{img}(rec),rec,wf_adc) = exp(1i*-pi/2)*hdr.DDC_dec{img}(rec) ... * data{img}(1:hdr.Nt{img}(rec),rec,wf_adc); % Delay of (100+2*100)/8 = 37.5 relative to DDC_dec==2 - extra_delay(rec) = 37.5; data{img}(1:hdr.Nt{img}(rec),rec,wf_adc) = ifft(fft(data{img}(1:hdr.Nt{img}(rec),rec,wf_adc)) ... .* exp(1i*2*pi*37.5*freq_axis/hdr.Nt{img}(rec))); elseif hdr.DDC_dec{img}(rec) == 16 @@ -162,6 +163,18 @@ data{img}(1:hdr.Nt{img}(rec),rec,wf_adc) = ifft(fft(data{img}(1:hdr.Nt{img}(rec),rec,wf_adc)) ... .* exp(1i*2*pi*43.75*freq_axis/hdr.Nt{img}(rec))); end + if ~isreal(data{img}) && ~data_complex_hack && size(data{img},1) > 0 + data_complex_hack = true; + % Temporarily add an imaginary part to the first value in the + % matrix so that Matlab will know right away that this matrix + % is complex and won't have to search through the entire + % matrix to find this out. We remove this value at the end of + % the loop. + data{img}(1) = data{img}(1) + 1i; + end + end + if data_complex_hack + data{img}(1) = data{img}(1) - 1i; end end end @@ -172,39 +185,39 @@ %% Coherent noise: Analysis Load % =================================================================== if strcmpi(wfs(wf).coh_noise_method,'analysis') - noise_fn_dir = fileparts(ct_filename_out(param,wfs(wf).coh_noise_arg.fn, '')); + noise_fn_dir = fileparts(ct_filename_out(param,param.radar.wfs(wf).coh_noise_arg.fn, '')); noise_fn = fullfile(noise_fn_dir,sprintf('coh_noise_simp_%s_wf_%d_adc_%d.mat', param.day_seg, wf, adc)); - - fprintf(' Load coh_noise: %s (%s)\n', noise_fn, datestr(now)); - noise = load(noise_fn); - if ~isfield(noise,'param_collate_coh_noise') || isempty(noise.param_collate_coh_noise) - fprintf('\n\nTHIS IS A HACK... THIS NOISE FILE SHOULD BE UPDATED.\n\n'); - noise.param_collate_coh_noise = noise.param_collate; - noise = rmfield(noise,'param_collate'); - save(noise_fn,'-struct','noise') - end + fprintf(' Loading coherent noise: %s (%s)\n', noise_fn, datestr(now)); + noise = collate_coh_noise_load(param,wf,adc); param.collate_coh_noise.param_collate = noise.param_collate_coh_noise; param.collate_coh_noise.param_analysis = noise.param_analysis; - if ~isfield(noise,'param_records') || isempty(noise.param_records) - fprintf('\n\nTHIS IS A HACK... THIS NOISE FILE SHOULD BE UPDATED.\n\n'); - noise.param_records = param; - save(noise_fn,'-struct','noise') - end param.collate_coh_noise.param_records = noise.param_records; - if ~isfield(noise.param_collate_coh_noise.collate_coh_noise,'method') || isempty(noise.param_collate_coh_noise.collate_coh_noise.method) - fprintf('\n\nTHIS IS A HACK... THIS NOISE FILE SHOULD BE UPDATED.\n\n'); - noise.param_collate_coh_noise.collate_coh_noise.method = 'dft'; - save(noise_fn,'-struct','noise') - end cmd = noise.param_analysis.analysis.cmd{noise.param_collate_coh_noise.collate_coh_noise.cmd_idx}; noise.Nx = length(noise.gps_time); - if strcmpi(noise.param_collate_coh_noise.collate_coh_noise.method,'dft') + % Find the matching image index that includes this wf-adc pair. + match_found = false; + for collate_coh_noise_img = 1:length(noise.param_analysis.analysis.imgs) + for match_wf_adc = 1:size(noise.param_analysis.analysis.imgs{collate_coh_noise_img},1) + if noise.param_analysis.analysis.imgs{collate_coh_noise_img}(match_wf_adc,1) == wf ... + && noise.param_analysis.analysis.imgs{collate_coh_noise_img}(match_wf_adc,2) == adc + match_found = true; + break; + end + end + if match_found + break; + end + end + if ~match_found + error('Could not find matching wf-adc pair in noise.param_analysis.analysis.imgs and therefore the collate_coh_noise.method cannot be verified.'); + end + if strcmpi(noise.param_collate_coh_noise.collate_coh_noise.method{collate_coh_noise_img},'dft') coh_noise = noise.dft_noise; noise = rmfield(noise,'dft_noise'); - elseif strcmpi(noise.param_collate_coh_noise.collate_coh_noise.method,'firdec') + elseif strcmpi(noise.param_collate_coh_noise.collate_coh_noise.method{collate_coh_noise_img},'firdec') coh_noise = noise.firdec_noise; noise = rmfield(noise,'firdec_noise'); end @@ -216,26 +229,56 @@ % when the coherent noise was loaded and estimated. coh_noise = coh_noise * 10.^((noise.param_analysis.radar.wfs(wf).adc_gains_dB(adc)-wfs(wf).adc_gains_dB(adc))/20); - % Adjust the coherent noise Tsys, chan_equal_dB, chan_equal_deg for - % changes relative to when the coherent noise was loaded and - % estimated. - coh_noise = coh_noise * 10.^(( ... - noise.param_analysis.radar.wfs(wf).chan_equal_dB(param.radar.wfs(wf).rx_paths(adc)) ... - - param.radar.wfs(wf).chan_equal_dB(param.radar.wfs(wf).rx_paths(adc)) )/20) ... - .* exp(1i*( ... - noise.param_analysis.radar.wfs(wf).chan_equal_deg(param.radar.wfs(wf).rx_paths(adc)) ... - - param.radar.wfs(wf).chan_equal_deg(param.radar.wfs(wf).rx_paths(adc)) )/180*pi); + % Adjust coherent noise for changes in system_dB + if length(wfs(wf).system_dB) == 1 + system_dB = wfs(wf).system_dB; + % Only a single number is provided for system_dB so apply it to all + % receiver paths + else + % A number is provided for each receiver path for system_dB + system_dB = wfs(wf).system_dB(param.radar.wfs(wf).rx_paths(adc)); + end + if length(noise.param_analysis.radar.wfs(wf).system_dB) == 1 + system_dB_noise = noise.param_analysis.radar.wfs(wf).system_dB; + % Only a single number is provided for system_dB so apply it to all + % receiver paths + else + % A number is provided for each receiver path for system_dB + system_dB_noise = noise.param_analysis.radar.wfs(wf).system_dB(param.radar.wfs(wf).rx_paths(adc)); + end + coh_noise = coh_noise * 10.^((system_dB_noise-system_dB)/20); - % Correct any changes in Tsys - Tsys = param.radar.wfs(wf).Tsys(param.radar.wfs(wf).rx_paths(adc)); - Tsys_old = noise.param_analysis.radar.wfs(wf).Tsys(param.radar.wfs(wf).rx_paths(adc)); - dTsys = Tsys-Tsys_old; noise.Nt = size(coh_noise,1); noise.freq = noise.fc + 1/(noise.dt*noise.Nt) * ifftshift(-floor(noise.Nt/2):floor((noise.Nt-1)/2)).'; - if dTsys ~= 0 - % Positive dTsys means Tsys > Tsys_old and we should reduce the - % time delay to all targets by dTsys. - coh_noise = ifft(bsxfun(@times, fft(coh_noise), exp(1i*2*pi*noise.freq*dTsys))); + if ~strcmpi(radar_type,'deramp') + % Adjust the coherent noise Tsys, chan_equal_dB, chan_equal_deg for + % changes relative to when the coherent noise was loaded and + % estimated. + coh_noise = coh_noise * 10.^(( ... + noise.param_analysis.radar.wfs(wf).chan_equal_dB(param.radar.wfs(wf).rx_paths(adc)) ... + - param.radar.wfs(wf).chan_equal_dB(param.radar.wfs(wf).rx_paths(adc)) )/20) ... + .* exp(1i*( ... + noise.param_analysis.radar.wfs(wf).chan_equal_deg(param.radar.wfs(wf).rx_paths(adc)) ... + - param.radar.wfs(wf).chan_equal_deg(param.radar.wfs(wf).rx_paths(adc)) )/180*pi); + + % Tadc_adjust changes do not matter since they do not affect the data + % (only the time axis is affected). + + % Correct any changes in Tsys + Tsys = param.radar.wfs(wf).Tsys(param.radar.wfs(wf).rx_paths(adc)); + Tsys_old = noise.param_analysis.radar.wfs(wf).Tsys(param.radar.wfs(wf).rx_paths(adc)); + if strcmpi(radar_type,'pulsed') + time_correction = param.radar.wfs(wf).time_correction; + time_correction_old = noise.param_analysis.radar.wfs(wf).time_correction; + dTsys = Tsys-Tsys_old + time_correction-time_correction_old; + else + dTsys = Tsys-Tsys_old; + end + if dTsys ~= 0 + % Positive dTsys means Tsys > Tsys_old and we should reduce the + % time delay to all targets by dTsys. + coh_noise = ifft(bsxfun(@times, fft(coh_noise), exp(1i*2*pi*noise.freq*dTsys))); + end end recs = interp1(noise.gps_time, noise.recs, hdr.gps_time, 'linear', 'extrap'); @@ -313,7 +356,7 @@ plot(lp(coh_noise)) end - if strcmpi(noise.param_collate_coh_noise.collate_coh_noise.method,'dft') + if strcmpi(noise.param_collate_coh_noise.collate_coh_noise.method{collate_coh_noise_img},'dft') cn.data = zeros([size(coh_noise,1) numel(recs)],'single'); for dft_idx = 1:length(noise.dft_freqs) % mf: matched filter @@ -323,19 +366,19 @@ cn.data(bin,:) = cn.data(bin,:)-coh_noise(bin,dft_idx) * mf; end end - elseif strcmpi(noise.param_collate_coh_noise.collate_coh_noise.method,'firdec') + elseif strcmpi(noise.param_collate_coh_noise.collate_coh_noise.method{collate_coh_noise_img},'firdec') % Interpolate coherent noise onto current data's gps time - if all(hdr.gps_time>noise.fir_dec_gps_time(end)) + if all(hdr.gps_time>noise.firdec_gps_time(end)) % All current data's gps time is after the coherent noise % estimates gps time cn.data = -repmat(coh_noise(:,end),[1 length(hdr.gps_time)]); - elseif all(hdr.gps_time 1e-6 - error('wfs(%d).BW_window must be an integer multiple of two times the wfs(wf).chirp rate divided by sampling frequency.'); + error('wfs(%d).BW_window must be an integer multiple of two times the wfs(wf).chirp rate divided by sampling frequency. See BW_window_gen.m for help.'); end % Remove rounding errors Nt_desired = round(Nt_desired); @@ -708,7 +764,7 @@ q end - %% Pulse compress: IF->Delay + %% Pulse compress Deramp: IF->Delay % ============================================================= if 0 % ENABLE_FOR_DEBUG_FREQ_MAP @@ -722,40 +778,56 @@ hdr.DDC_freq{img}(rec) = 95e6; end - % nz: Nyquist zone containing signal spectrum (just renaming - % variable for convenience). The assumption is that the - % signal does not cross nyquist zones. - nz = hdr.nyquist_zone_signal{img}(rec); - - % f_nz0: Lowest frequency in terms of ADC input frequency of the - % nyquist zone which contains the signal - f_nz0 = wfs(wf).fs_raw * floor(nz/2); - - % freq_raw: Frequency axis of raw data assuming that raw signal - % spectrum is restricted to the frequency range [(N-1)*fs N*fs] - % where N is chosen so that the selected nyquist zone lies - % within this frequency range. - freq_raw = f_nz0 + mod(hdr.DDC_freq{img}(rec) ... - + df_raw*ifftshift(-floor(Nt_raw_trim/2):floor((Nt_raw_trim-1)/2)).', wfs(wf).fs_raw); - freq_raw_valid = freq_raw; - - % conjugate_bins: logical mask indicating which bins are - % conjugated, this is also used to determine how frequencies - % are wrapped in the nyquist zone when real only sampling is - % used (for DFT there are 1 or 2 bins which are real-only and - % these are marked to be conjugated by using >= and <=; since - % conjugation of these real only bins makes no difference the - % only reason to do this is because of the nyquist zone - % wrapping) - conjugate_bins = ~(freq_raw_valid >= nz*wfs(wf).fs_raw/2 ... - & freq_raw_valid <= (1+nz)*wfs(wf).fs_raw/2); - - % freq_raw_valid: modified to handle wrapping at Nyquist - % boundaries - if mod(nz,2) - freq_raw_valid(conjugate_bins) = nz*wfs(wf).fs_raw - freq_raw_valid(conjugate_bins); + if wfs(wf).nz_complex + % The signal can cross nyquist zones because it is complex. + % Currently the code assumes that the signal is represented + % in complex baseband in the first nyquist zone. + nz = 0; + + freq_raw = hdr.DDC_freq{img}(rec) ... + + df_raw*ifftshift(-floor(Nt_raw_trim/2):floor((Nt_raw_trim-1)/2)).'; + freq_raw_valid = freq_raw; + + conjugate_bins = false(size(freq_raw_valid)); + else - freq_raw_valid(conjugate_bins) = (nz+1)*wfs(wf).fs_raw - freq_raw_valid(conjugate_bins); + + % nz: Nyquist zone containing signal spectrum (just renaming + % variable for convenience). The assumption is that the + % signal does not cross nyquist zones. + nz = hdr.nyquist_zone_signal{img}(rec); + + % f_nz0: Lowest frequency in terms of ADC input frequency of the + % nyquist zone which contains the signal + f_nz0 = wfs(wf).fs_raw * floor(nz/2); + + % freq_raw: Frequency axis of raw data assuming that raw signal + % spectrum is restricted to the frequency range [(N-1)*fs N*fs] + % where N is chosen so that the selected nyquist zone lies + % within this frequency range. + freq_raw = f_nz0 + mod(hdr.DDC_freq{img}(rec) ... + + df_raw*ifftshift(-floor(Nt_raw_trim/2):floor((Nt_raw_trim-1)/2)).', wfs(wf).fs_raw); + freq_raw_valid = freq_raw; + + % conjugate_bins: logical mask indicating which bins are + % conjugated, this is also used to determine how frequencies + % are wrapped in the nyquist zone when real only sampling is + % used (for DFT there are 1 or 2 bins which are real-only and + % these are marked to be conjugated by using >= and <=; since + % conjugation of these real only bins makes no difference the + % only reason to do this is because of the nyquist zone + % wrapping) + conjugate_bins = ~(freq_raw_valid >= nz*wfs(wf).fs_raw/2 ... + & freq_raw_valid <= (1+nz)*wfs(wf).fs_raw/2); + + % freq_raw_valid: modified to handle wrapping at Nyquist + % boundaries + if mod(nz,2) + freq_raw_valid(conjugate_bins) = nz*wfs(wf).fs_raw - freq_raw_valid(conjugate_bins); + else + freq_raw_valid(conjugate_bins) = (nz+1)*wfs(wf).fs_raw - freq_raw_valid(conjugate_bins); + end + end % freq_raw_valid: reduce rounding errors so that unique will @@ -794,32 +866,46 @@ ylabel('Frequency (Hz)'); end - %% Pulse compress: IF->Delay (Coh Noise) + %% Pulse compress Deramp: IF->Delay (Coh Noise) if strcmpi(wfs(wf).coh_noise_method,'analysis') % ============================================================= cn.df_raw = wfs(wf).fs_raw/hdr.DDC_dec{img}(rec)/Nt_raw_trim; - % nz: Nyquist zone containing signal spectrum (just renaming - % variable for convenience). The assumption is that the - % signal does not cross nyquist zones. - cn.nz = double(hdr.nyquist_zone_hw{img}(rec)); - - % f_nz0: Lowest frequency in terms of ADC input frequency of the - % nyquist zone which contains the signal - cn.f_nz0 = wfs(wf).fs_raw * floor(cn.nz/2); - - cn.freq_raw = cn.f_nz0 + mod(hdr.DDC_freq{img}(rec) ... - + cn.df_raw*ifftshift(-floor(Nt_raw_trim/2):floor((Nt_raw_trim-1)/2)).', wfs(wf).fs_raw); - cn.freq_raw_valid = cn.freq_raw; - - cn.conjugate_bins = ~(cn.freq_raw_valid >= cn.nz*wfs(wf).fs_raw/2 ... - & cn.freq_raw_valid <= (1+cn.nz)*wfs(wf).fs_raw/2); - - if mod(cn.nz,2) - cn.freq_raw_valid(cn.conjugate_bins) = cn.nz*wfs(wf).fs_raw - cn.freq_raw_valid(cn.conjugate_bins); + if wfs(wf).nz_complex + % The signal can cross nyquist zones because it is complex. + % Currently the code assumes that the signal is represented + % in complex baseband in the first nyquist zone. + cn.nz = 0; + + cn.freq_raw = hdr.DDC_freq{img}(rec) ... + + cn.df_raw*ifftshift(-floor(Nt_raw_trim/2):floor((Nt_raw_trim-1)/2)).'; + cn.freq_raw_valid = cn.freq_raw; + + cn.conjugate_bins = false(size(cn.freq_raw_valid)); + else - cn.freq_raw_valid(cn.conjugate_bins) = (cn.nz+1)*wfs(wf).fs_raw - cn.freq_raw_valid(cn.conjugate_bins); + % nz: Nyquist zone containing signal spectrum (just renaming + % variable for convenience). The assumption is that the + % signal does not cross nyquist zones. + cn.nz = double(hdr.nyquist_zone_hw{img}(rec)); + + % f_nz0: Lowest frequency in terms of ADC input frequency of the + % nyquist zone which contains the signal + cn.f_nz0 = wfs(wf).fs_raw * floor(cn.nz/2); + + cn.freq_raw = cn.f_nz0 + mod(hdr.DDC_freq{img}(rec) ... + + cn.df_raw*ifftshift(-floor(Nt_raw_trim/2):floor((Nt_raw_trim-1)/2)).', wfs(wf).fs_raw); + cn.freq_raw_valid = cn.freq_raw; + + cn.conjugate_bins = ~(cn.freq_raw_valid >= cn.nz*wfs(wf).fs_raw/2 ... + & cn.freq_raw_valid <= (1+cn.nz)*wfs(wf).fs_raw/2); + + if mod(cn.nz,2) + cn.freq_raw_valid(cn.conjugate_bins) = cn.nz*wfs(wf).fs_raw - cn.freq_raw_valid(cn.conjugate_bins); + else + cn.freq_raw_valid(cn.conjugate_bins) = (cn.nz+1)*wfs(wf).fs_raw - cn.freq_raw_valid(cn.conjugate_bins); + end end cn.freq_raw_valid = cn.df_raw*round(cn.freq_raw_valid/cn.df_raw); @@ -838,7 +924,7 @@ freq_axes_changed = false; % Reset state - %% Pulse compress: Time axis + %% Pulse compress Deramp: Time axis % Convert IF frequency to time delay and account for reference % deramp time offset, hdr.t_ref @@ -864,7 +950,7 @@ deskew_shift = 1i*2*pi*(0:Nt_raw_trim-1).'/Nt_raw_trim; time_correction_freq = exp(1i*2*pi*(freq-fc)*time_correction); - %% Pulse compress: Time axis (Coh Noise) + %% Pulse compress Deramp: Time axis (Coh Noise) if strcmpi(wfs(wf).coh_noise_method,'analysis') % Convert IF frequency to time delay and account for reference @@ -951,6 +1037,7 @@ else window_start_idx = find(f_rf >= wfs(wf).BW_window(1),1); end + f_rf = wfs(wf).f0 + wfs(wf).chirp_rate*(time_raw_no_trim - wfs(wf).td_mean); window_start_idx = window_start_idx_norm; H_idxs = window_start_idx : window_start_idx+Nt_raw_trim-1; if 0 @@ -969,10 +1056,10 @@ end - %% Pulse compress: FFT and Deskew + %% Pulse compress Deramp: FFT, Deskew, Coh Noise Removal % Window and DFT (raw deramped time to regular time) - NCO_time = hdr.t0_raw{1}(rec) + wfs(wf).DDC_NCO_delay + (H_idxs(:)-1) /(wfs(wf).fs_raw/hdr.DDC_dec{img}(rec)); + NCO_time = hdr.t0_raw{1}(rec) + wfs(wf).Tadc_adjust + wfs(wf).DDC_NCO_delay + (H_idxs(:)-1) /(wfs(wf).fs_raw/hdr.DDC_dec{img}(rec)); tmp = fft(data{img}(H_idxs,rec,wf_adc) ... .* exp(1i*2*pi*DDC_freq_adjust*NCO_time) ... @@ -1022,9 +1109,9 @@ tmp = tmp(cn.unique_idxs); tmp(cn.conjugate_unique) = conj(tmp(cn.conjugate_unique)); if wfs(wf).f0 > wfs(wf).f1 - tmp = fft(tmp .* exp(-1i*2*pi*(fc-min(cn.freq))*cn.time)); + tmp = fft(tmp .* exp(-1i*2*pi*(fc-f_rf(H_idxs(1)))*cn.time)); else - tmp = fft(conj(tmp) .* exp(-1i*2*pi*(fc-min(cn.freq))*cn.time)); + tmp = fft(conj(tmp) .* exp(-1i*2*pi*(fc-f_rf(H_idxs(1)))*cn.time)); end tmp = tmp .* cn.time_correction_freq; tmp = ifft(tmp); @@ -1067,11 +1154,11 @@ % Undo tmp = tmp .* time_correction; tmp = tmp ./ cn.time_correction_freq; if wfs(wf).f0 > wfs(wf).f1 - % Undo tmp = fft(tmp .* exp(-1i*2*pi*(fc-min(freq))*time)); - tmp = ifft(tmp) .* exp(1i*2*pi*(fc-min(freq))*time); + % Undo tmp = fft(tmp .* exp(-1i*2*pi*(fc-f_rf(H_idxs(1)))*time)); + tmp = ifft(tmp) .* exp(1i*2*pi*(fc-f_rf(H_idxs(1)))*time); else - % Undo tmp = fft(conj(tmp) .* exp(-1i*2*pi*(fc-min(freq))*time)); - tmp = conj(ifft(tmp)) .* exp(1i*2*pi*(fc-min(freq))*time); + % Undo tmp = fft(conj(tmp) .* exp(-1i*2*pi*(fc-f_rf(H_idxs(1)))*time)); + tmp = conj(ifft(tmp)) .* exp(1i*2*pi*(fc-f_rf(H_idxs(1)))*time); end % Undo tmp = tmp(unique_idxs); tmp = tmp(cn.return_idxs); @@ -1082,9 +1169,9 @@ tmp = tmp(unique_idxs); tmp(conjugate_unique) = conj(tmp(conjugate_unique)); if wfs(wf).f0 > wfs(wf).f1 - tmp = fft(tmp .* exp(-1i*2*pi*(fc-min(freq))*time)); + tmp = fft(tmp .* exp(-1i*2*pi*(fc-f_rf(H_idxs(1)))*time)); else - tmp = fft(conj(tmp) .* exp(-1i*2*pi*(fc-min(freq))*time)); + tmp = fft(conj(tmp) .* exp(-1i*2*pi*(fc-f_rf(H_idxs(1)))*time)); end tmp = tmp .* time_correction_freq; tmp = ifft(tmp); @@ -1123,7 +1210,7 @@ % % Therefore, only a circular shift is required to complex baseband % the data. - tmp = fft(tmp .* exp(-1i*2*pi*(fc-min(freq))*time)); + tmp = fft(tmp .* exp(-1i*2*pi*(fc-f_rf(H_idxs(1)))*time)); else % Positive chirp: the initial DFT causes a frequency domain reversal % which flips the frqeuency domain so that the RF frequency mapping @@ -1134,7 +1221,7 @@ % % The frequency reversal and conjugation are fixed by conjugating the % signal before the FFT. - tmp = fft(conj(tmp) .* exp(-1i*2*pi*(fc-min(freq))*time)); + tmp = fft(conj(tmp) .* exp(-1i*2*pi*(fc-f_rf(H_idxs(1)))*time)); end % Modulate the raw data to adjust the start time to always be a @@ -1167,24 +1254,26 @@ else tmp = tmp(1 : end-1); end - hdr.Nt{img}(rec) = length(tmp); - data{img}(1:hdr.Nt{img}(rec),rec,wf_adc) = tmp; + if wf_adc == 1 + new_Nt(rec) = length(tmp); + end + data{img}(1:new_Nt(rec),rec,wf_adc) = tmp; end if 0 % ENABLE_FOR_DEBUG figure(1); clf; Mt = 10; - data_oversampled = interpft(data{img}(1:hdr.Nt{img}(rec),:,wf_adc), hdr.Nt{img}(rec)*Mt); + data_oversampled = interpft(data{img}(1:new_Nt(rec),:,wf_adc), new_Nt(rec)*Mt); [~,idx] = max(data_oversampled); - time_oversampled = time(1) + dt/Mt* (0:hdr.Nt{img}(rec)*Mt-1).'; + time_oversampled = time(1) + dt/Mt* (0:new_Nt(rec)*Mt-1).'; plot((time_oversampled(idx).' - tds)/dt) grid on; xlabel('Record'); ylabel('Time error (\Delta_t)'); figure(2); clf; - phase_sim = max(data{img}(1:hdr.Nt{img}(rec),:,wf_adc)); + phase_sim = max(data{img}(1:new_Nt(rec),:,wf_adc)); [~,ref_idx] = min(abs(tds-wfs(wf).td_mean)); plot(angle(phase_sim./phase_sim(ref_idx)),'+-'); hold on @@ -1197,6 +1286,8 @@ grid on; end + %% Pulse compress Deramp: Corrections, Constant Nt + % Create a matrix of data with constant time rows, fill invalid samples with NaN if wf_adc == 1 if all(isnan(hdr.t0{img})) @@ -1206,7 +1297,7 @@ dt = 1; else idx_start = min(round(hdr.t0{img}/dt)); - wfs(wf).Nt = max(round(hdr.t0{img}/dt) + hdr.Nt{img})-idx_start; + wfs(wf).Nt = max(round(hdr.t0{img}/dt) + new_Nt)-idx_start; end hdr.time{img} = idx_start*dt + dt*(0:wfs(wf).Nt-1).'; fc = sum(wfs(wf).BW_window)/2; @@ -1214,6 +1305,7 @@ df = 1/T; hdr.freq{img} = fc + df * ifftshift(-floor(wfs(wf).Nt/2) : floor((wfs(wf).Nt-1)/2)).'; end + % Method of copying to make this more efficient for very large % complex (real/imag) arrays. Lots of small matrix operations on % huge complex matrices is very slow in matlab. Real only matrices @@ -1221,6 +1313,7 @@ blocks = round(linspace(1,size(data{img},2)+1,8)); blocks = unique(blocks); for block = 1:length(blocks)-1 rlines = blocks(block) : blocks(block+1)-1; + reD = real(data{img}(:,rlines,wf_adc)); imD = imag(data{img}(:,rlines,wf_adc)); for rec = 1:length(rlines) @@ -1230,10 +1323,10 @@ cur_idx_stop = 0; else cur_idx_start = round(hdr.t0{img}(rlines(rec))/dt) - idx_start + 1; - cur_idx_stop = round(hdr.t0{img}(rlines(rec))/dt) - idx_start + hdr.Nt{img}(rlines(rec)); + cur_idx_stop = round(hdr.t0{img}(rlines(rec))/dt) - idx_start + new_Nt(rlines(rec)); end - reD(cur_idx_start : cur_idx_stop,rec) = reD(1:hdr.Nt{img}(rlines(rec)),rec); + reD(cur_idx_start : cur_idx_stop,rec) = reD(1:new_Nt(rlines(rec)),rec); reD(1:cur_idx_start-1,rec) = wfs(wf).bad_value; reD(cur_idx_stop+1 : wfs(wf).Nt,rec) = wfs(wf).bad_value; end @@ -1244,14 +1337,37 @@ cur_idx_stop = 0; else cur_idx_start = round(hdr.t0{img}(rlines(rec))/dt) - idx_start + 1; - cur_idx_stop = round(hdr.t0{img}(rlines(rec))/dt) - idx_start + hdr.Nt{img}(rlines(rec)); + cur_idx_stop = round(hdr.t0{img}(rlines(rec))/dt) - idx_start + new_Nt(rlines(rec)); end - imD(cur_idx_start : cur_idx_stop,rec) = imD(1:hdr.Nt{img}(rlines(rec)),rec); + imD(cur_idx_start : cur_idx_stop,rec) = imD(1:new_Nt(rlines(rec)),rec); imD(1:cur_idx_start-1,rec) = wfs(wf).bad_value; imD(cur_idx_stop+1 : wfs(wf).Nt,rec) = wfs(wf).bad_value; end data{img}(1:wfs(wf).Nt,rlines,wf_adc) = reD(1:wfs(wf).Nt,:) + 1i*imD(1:wfs(wf).Nt,:); + + % Corrections: + % Apply wf-adc specific channel equalization (for multichannel + % systems). For pulsed systems this is taken care of in + % data_load.m in corrections. + chan_equal = 10.^(param.radar.wfs(wf).chan_equal_dB(param.radar.wfs(wf).rx_paths(adc))/20) ... + .* exp(1i*param.radar.wfs(wf).chan_equal_deg(param.radar.wfs(wf).rx_paths(adc))/180*pi); + data{img}(1:wfs(wf).Nt,rlines,wf_adc) = chan_equal .* data{img}(1:wfs(wf).Nt,rlines,wf_adc); + + % Corrections: + % Apply wf-adc specific system time delay (for multichannel + % systems). For pulsed systems this is taken care of in + % data_load_wfs.m where the reference function is created. + Tsys = param.radar.wfs(wf).Tsys(param.radar.wfs(wf).rx_paths(adc)); + if Tsys ~= 0 + % Positive Tsys means the time delay to the target is too large + % and we should reduce the time delay to all targets by Tsys. + data{img}(1:wfs(wf).Nt,rlines,wf_adc) ... + = ifft(bsxfun(@times, ... + fft(data{img}(1:wfs(wf).Nt,rlines,wf_adc),[],1), ... + exp(1i*2*pi*hdr.freq{img}*Tsys)),[],1); + end + end clear reD imD; @@ -1260,6 +1376,7 @@ end else + %% No pulse compress if wf_adc == 1 if strcmpi(radar_type,'pulsed') hdr.time{img} = wfs(wf).time_raw; @@ -1267,7 +1384,7 @@ elseif strcmpi(radar_type,'deramp') % Time axis is not valid if DDC or time offset changes - hdr.time{img} = hdr.t0_raw{img}(1) + hdr.DDC_dec{1}(1)/wfs(wf).fs_raw*(0:hdr.Nt{img}-1).'; + hdr.time{img} = hdr.t0_raw{img}(1) + wfs(wf).Tadc_adjust + hdr.DDC_dec{1}(1)/wfs(wf).fs_raw*(0:hdr.Nt{img}-1).'; % Frequency is not valid df = wfs(wf).fs_raw / hdr.Nt{img}(1); hdr.freq{img} = df*(0:hdr.Nt{img}-1).'; @@ -1286,7 +1403,7 @@ if wfs(wf).coh_noise_arg.DC_remove_en data{img}(1:wfs(wf).Nt,:,wf_adc) = bsxfun(@minus, data{img}(1:wfs(wf).Nt,:,wf_adc), ... - mean(data{img}(1:wfs(wf).Nt,:,wf_adc),2)); + nanmean(data{img}(1:wfs(wf).Nt,:,wf_adc),2)); end if length(wfs(wf).coh_noise_arg.B_coh_noise) > 1 @@ -1311,7 +1428,7 @@ end if strcmpi(radar_type,'pulsed') if strcmpi(wfs(wf).coh_noise_method,'analysis') - if strcmpi(noise.param_collate_coh_noise.collate_coh_noise.method,'dft') + if strcmpi(noise.param_collate_coh_noise.collate_coh_noise.method{collate_coh_noise_img},'dft') for dft_idx = 1:length(noise.dft_freqs) % mf: matched filter % coh_noise(bin,dft_idx): Coefficient for the matched filter @@ -1321,10 +1438,10 @@ end end - elseif strcmpi(noise.param_collate_coh_noise.collate_coh_noise.method,'firdec') + elseif strcmpi(noise.param_collate_coh_noise.collate_coh_noise.method{collate_coh_noise_img},'firdec') blocks = round(linspace(1,size(data{img},2)+1,8)); blocks = unique(blocks); - rel_gps_time = single(noise.fir_dec_gps_time - noise.fir_dec_gps_time(1)); - rel_gps_time_interp = single(hdr.gps_time - noise.fir_dec_gps_time(1)); + rel_gps_time = single(noise.firdec_gps_time - noise.firdec_gps_time(1)); + rel_gps_time_interp = single(hdr.gps_time - noise.firdec_gps_time(1)); for block = 1:length(blocks)-1 rlines = blocks(block) : blocks(block+1)-1; @@ -1370,7 +1487,7 @@ hold on; plot(lp(afterf(:,rline))); end - + %% Deconvolution % =================================================================== if param.load.pulse_comp == 1 && wfs(wf).deconv.en && wfs(wf).Nt > 0 @@ -1402,7 +1519,9 @@ df = hdr.freq{img}(2)-hdr.freq{img}(1); BW = df * wfs(wf).Nt; deconv_dfc = deconv_fc - fc; - hdr.freq{img} = mod(hdr.freq{img} + deconv_dfc-wfs(wf).BW_window(1), BW)+wfs(wf).BW_window(1); + if wf_adc == 1 + new_deconv_hdr_freq = mod(hdr.freq{img} + deconv_dfc-wfs(wf).BW_window(1), BW)+wfs(wf).BW_window(1); + end for unique_idxs_idx = 1:length(unique_idxs) % deconv_mask: Create logical mask corresponding to range lines that use this deconv waveform @@ -1478,11 +1597,51 @@ end + %% Nulling unsteady Doppler spikes for specified range bins (for example, 20181011_02) + % .DSN, a parameter structure to control the Doppler spike nulling + % .en, 0 or 1 to disable or enable the nulling + % .rbin_clusters, N by 2 array, N is the number of range bin + % clusters, the first and the second collumns specify the start and stop range bin respectively for each range bin cluster + % .threshold, Doppler threshold in dB above the mean of local Doppler signals + % .surf_threshold, surface threshold in dB above the mean of local signals + if isfield(wfs(wf),'DSN') && wfs(wf).DSN.en + for rcluster = 1:size(wfs(wf).DSN.rbin_clusters,1) + for rbin = wfs(wf).DSN.rbin_clusters(rcluster,1):min(size(data{1},1),wfs(wf).DSN.rbin_clusters(rcluster,2)) + good_rline_idxs = ~isnan(data{1}(rbin,:)); + tmp = data{1}(rbin,good_rline_idxs); + thresholding_idxs = find(lp(tmp)>mean(lp(tmp))+wfs(wf).DSN.surf_threshold); + if ~isempty(thresholding_idxs) + continue % skipping surface removes most part of noise in general without nulling artifact +% tmp(thresholding_idxs) = 0; % thresholding surface signals + end + tmp = fft(tmp); + tmp_m = mean(lp(tmp)); + tmp_spikes = lp(tmp)-tmp_m; + spike_idxs = find(tmp_spikes>wfs(wf).DSN.threshold); + if length(spike_idxs) >0 + for spike_idx = 1:length(spike_idxs) + tmp(spike_idxs(spike_idx)) = 10^(-tmp_spikes(spike_idxs(spike_idx))/20)*tmp(spike_idxs(spike_idx)); + end + data{1}(rbin,good_rline_idxs) = ifft(tmp); + end + end + end + end + + end + + % Update frequency axis for deconv + if param.load.pulse_comp == 1 && wfs(wf).deconv.en && wfs(wf).Nt > 0 + hdr.freq{img} = new_deconv_hdr_freq; + end + % Update record length field + if param.load.pulse_comp == 1 && strcmpi(radar_type,'deramp') + hdr.Nt{img} = new_Nt; end if param.load.pulse_comp == 1 % Check if any good records, skip truncation if not - if any(~hdr.bad_rec{img}(1,:,wf_adc)) + if any(~hdr.bad_rec{img}(1,:)) data{img} = data{img}(1:wfs(wf).Nt,:,:); end end diff --git a/cresis-toolbox/processing/data_resample.m b/cresis-toolbox/processing/data_resample.m index bd28088c..4b4443b7 100644 --- a/cresis-toolbox/processing/data_resample.m +++ b/cresis-toolbox/processing/data_resample.m @@ -24,7 +24,7 @@ end end if ~found - error('Resample will result in non-aligned data samples.'); + error('Resample values [%d %d] will result in non-aligned data samples.', pq(1,1), pq(1,2)); end offset = test_bin - start_bin; if offset > 20 diff --git a/cresis-toolbox/processing/default_radar_params_settings_match.m b/cresis-toolbox/processing/default_radar_params_settings_match.m index 8121ca87..8abeb459 100644 --- a/cresis-toolbox/processing/default_radar_params_settings_match.m +++ b/cresis-toolbox/processing/default_radar_params_settings_match.m @@ -7,7 +7,15 @@ found = false; if isfield(settings,'XML_File_Path') + % MCoRDS NI settings_fn = settings.XML_File_Path{1}.values{1}; +elseif ischar(settings) || isa(settings,'java.lang.String') + % Arena + settings_fn = char(settings); +else + settings_fn = ''; +end +if ~isempty(settings_fn) for default_idx = 1:length(defaults) if ~isempty(regexp(settings_fn, defaults{default_idx}.config_regexp)) default = defaults{default_idx}; @@ -15,12 +23,10 @@ break; end end -else - settings_fn = ''; end while ~found - warning('Did not find a matching set of default parameters for %s.', settings_fn); + warning('Did not find a matching set of default parameters for "%s".', settings_fn); for default_idx = 1:length(defaults) fprintf(' (%d) %s\n', default_idx, defaults{default_idx}.name); end diff --git a/cresis-toolbox/processing/frames_check.m b/cresis-toolbox/processing/frames_check.m index d76d4a2f..b048763a 100644 --- a/cresis-toolbox/processing/frames_check.m +++ b/cresis-toolbox/processing/frames_check.m @@ -1,7 +1,11 @@ -function frames_create(param) -% frames_create(param) +function frames_check(param,param_override) +% frames_check(param,param_override) % % Checks the fields in the frames files for potential errors. +% Use run_all_frames_check to run on all seasons. +% +% To concatenate and look at all the outputs in Linux: +% grep "^" `find /cresis/snfs1/dataproducts/ct_data/ct_tmp/frames_check -iname "*output*" -print | sort` | less % % param = parameter structure indicating which radar and season to load. % This causes the standard parameter spreadsheet filename to be created @@ -14,33 +18,52 @@ function frames_create(param) % param = []; % param.radar_name = 'mcrds'; % param.season_name = '2009_Antarctica_TO'; -% frames_create(param) +% frames_check(param) % % param = []; -% param.radar_name = 'icards'; -% param.season_name = '1993_Greenland_P3'; -% frames_create(param) +% param.radar_name = 'rds'; +% param.season_name = '2018_Greenland_P3'; +% frames_check(param) % % param = []; % param.radar_name = 'snow3'; % param.season_name = '2013_Antarctica_Basler'; -% frames_create(param) +% frames_check(param) +% +% Author: John Paden +% +% See also: check_data_products, frames_check, gps_check, records_check +% run_all_frames_check, run_all_gps_check, run_all_records_check + +%% frames_check ========================================================== + +%% Input checks +global gRadar; +if exist('param_override','var') + param_override = merge_structs(gRadar,param_override); +else + param_override = gRadar; +end if ischar(param) - % param is a string containing the filename - - records_fn = ct_filename_support(param,'','records'); - records = load(param,'param_records'); - - frames_fn = ct_filename_support(records.param_records,'','frames'); - + %% param is a string containing the filename + % ======================================================================= + records_fn = param; + records = records_load(records_fn,'param_records'); if ~isfield(records,'param_records') error('This mode only supported for new records files with param_records field.'); end - records_check_support_func(param,records.param_records); + + param = merge_structs(records.param_records,param_override); + + frames_fn = ct_filename_support(param,'','frames'); + + frames_create_support_func(frames_fn,param); elseif isstruct(param) - % param is a struct indicating which radar/season to check (checks all segments) + %% param is a struct + % ======================================================================= + % struct indicates radar/season to check (checks all segments) [output_dir,radar_type,radar_name] = ct_output_dir(param.radar_name); param_fn = ct_filename_param(sprintf('%s_param_%s.xls', output_dir, param.season_name)); @@ -48,23 +71,24 @@ function frames_create(param) for param_idx = 1:length(params) param = params(param_idx); - - records_fn = ct_filename_support(param,'','records'); + param = merge_structs(param,param_override); if ~isempty(regexpi(param.cmd.notes,'do not process')) continue; end - % DEBUG OPTION TO JUST CHECK RECORDS BASED ON GENERIC COLUMN IN SPREADSHEET + frames_fn = ct_filename_support(param,'','frames'); + + % DEBUG OPTION TO JUST CHECK FRAMES BASED ON GENERIC COLUMN IN SPREADSHEET % Uses the generic column of the parameter spreadsheet to determine which segments % to check. %if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic % continue; %end - fprintf('%s\tChecking\t%s\n', param.day_seg, records_fn); + fprintf('%s\tChecking\t%s\n', param.day_seg, frames_fn); - frames_create_support_func(records_fn,param); + frames_create_support_func(frames_fn,param); end else @@ -73,46 +97,66 @@ function frames_create(param) end -function frames_create_support_func(records_fn,param) -% frames_create_support_func(records_fn,param) +%% frames_create_support_func ============================================= +function frames_create_support_func(frames_fn,param) +% frames_create_support_func(frames_fn,param) % % Support function which does the actual checking of the frames -if ~exist(records_fn,'file') - fprintf(2,'%s\tno_records_file\t%s\n', param.day_seg, records_fn); - return; +%% input checks +% ========================================================================= + +command_window_out_fn = ct_filename_ct_tmp(param,'','frames_check', 'output.txt'); +command_window_out_fn_dir = fileparts(command_window_out_fn); +if ~exist(command_window_out_fn_dir,'dir') + mkdir(command_window_out_fn_dir); end +fid = fopen(command_window_out_fn,'wb'); +fprintf(' Console output: %s\n', command_window_out_fn); +fprintf(fid, '%s\n', param.day_seg); + +% fixable_error: Set to true if a fixable error in which case the frames +% file will be updated at the end of this function. +fixable_error = false; -frames_fn = ct_filename_support(param,'','frames'); if ~exist(frames_fn,'file') - fprintf(2,'%s\tno_frames_file\t%s\n', param.day_seg, records_fn); + fprintf(2,'%s\tno_frames_file\t%s\n', param.day_seg, frames_fn); + fprintf(fid,'%s\tno_frames_file\t%s!!!\n', param.day_seg, frames_fn); + fclose(fid); return; end -records = load(records_fn,'lat','lon'); +records_fn = ct_filename_support(param,'','frames'); +if ~exist(records_fn,'file') + fprintf(2,'%s\tno_records_file\t%s\n', param.day_seg, records_fn); + fprintf(fid,'%s\tno_records_file\t%s!!!\n', param.day_seg, records_fn); + fclose(fid); + return; +end + +records = records_load(param,'lat','lon'); along_track = geodetic_to_along_track(records.lat,records.lon); -load(frames_fn); +frames = frames_load(param); if frames.frame_idxs(1) ~= 1 fprintf(2,'%s\tframe_idxs_1_not_1\t%d instead of 1\n', param.day_seg, frames.frame_idxs(1)); + fprintf(fid,'%s\tframe_idxs_1_not_1\t%d instead of 1\n', param.day_seg, frames.frame_idxs(1)); end +%% Check each field +% ========================================================================= + if any(frames.frame_idxs > length(records.lat)) fprintf(2,'%s\tframe_idxs_too_large\tindex %d > number of records %d\n', ... param.day_seg, max(frames.frame_idxs), length(records.lat)); + fprintf(fid,'%s\tframe_idxs_too_large\tindex %d > number of records %d!!!\n', ... + param.day_seg, max(frames.frame_idxs), length(records.lat)); end -fixable_error = false; - -if length(frames.frame_idxs) ~= length(frames.nyquist_zone) - fprintf(2,'%s\tnyquist_zone_length_mismatch\tlength %d does not match frame_idxs length %d\n', ... - param.day_seg, length(frames.nyquist_zone), length(frames.frame_idxs)); +if isfield(frames,'nyquist_zone') + fprintf(2,'%s\tnyquist_zone_remove\n', param.day_seg); + frames = rmfield(frames,'nyquist_zone'); fixable_error = true; - if length(frames.frame_idxs) > length(frames.nyquist_zone) - frames.nyquist_zone(end+1 : length(frames.frame_idxs)) = NaN; - else - frames.nyquist_zone = frames.nyquist_zone(1:length(frames.frame_idxs)); - end end if length(frames.frame_idxs) ~= length(frames.proc_mode) @@ -126,6 +170,9 @@ function frames_create_support_func(records_fn,param) end end +%% Check frame along-track +% ========================================================================= + [output_dir,radar_type,radar_name] = ct_output_dir(param.radar_name); if any(strcmpi(output_dir,'rds')) default_frame_len = 50000; @@ -149,22 +196,36 @@ function frames_create_support_func(records_fn,param) % Check length of frame if length(frames.frame_idxs) == 1 && frame_len < default_frame_len/10 fprintf(2,'%s\tsingle_frame_too_short\t%g km < %g km for frame \t%d\n', param.day_seg, frame_len/1e3, 1/5*default_frame_len/1e3, frm); + fprintf(fid,'%s\tsingle_frame_too_short\t%g km < %g km for frame \t%d\n', param.day_seg, frame_len/1e3, 1/5*default_frame_len/1e3, frm); elseif length(frames.frame_idxs) > 1 && frame_len < default_frame_len/5 fprintf(2,'%s\ttoo_short\t%g km < %g km for frame \t%d\n', param.day_seg, frame_len/1e3, 1/5*default_frame_len/1e3, frm); + fprintf(fid,'%s\ttoo_short\t%g km < %g km for frame \t%d\n', param.day_seg, frame_len/1e3, 1/5*default_frame_len/1e3, frm); end - if frame_len > 2*default_frame_len + if frame_len > 5*default_frame_len + % Add "!!!" to the end of string to indicate that this frame should be + % fixed. + fprintf(2,'%s\ttoo_long\t%g km > %g km for frame \t%d\n', param.day_seg, frame_len/1e3, 5*default_frame_len/1e3, frm); + fprintf(fid,'%s\ttoo_long\t%g km > %g km for frame \t%d!!!\n', param.day_seg, frame_len/1e3, 5*default_frame_len/1e3, frm); + elseif frame_len > 2*default_frame_len fprintf(2,'%s\ttoo_long\t%g km > %g km for frame \t%d\n', param.day_seg, frame_len/1e3, 2*default_frame_len/1e3, frm); + fprintf(fid,'%s\ttoo_long\t%g km > %g km for frame \t%d\n', param.day_seg, frame_len/1e3, 2*default_frame_len/1e3, frm); end % Check number of records in frame if frame_num_recs < default_min_rec fprintf(2,'%s\tFrame has very few records\t%d\t%d\n', param.day_seg, frame_num_recs, frm); + fprintf(fid, '%s\tFrame has very few records\t%d\t%d\n', param.day_seg, frame_num_recs, frm); end end +%% Save updates/cleanup +% ========================================================================= + if 1 && fixable_error fprintf('Save %s\n', frames_fn); - save(frames_fn,'frames'); + ct_save(frames_fn,'frames'); end +fclose(fid); + end diff --git a/cresis-toolbox/processing/get_echogram_stats_single.m b/cresis-toolbox/processing/get_echogram_stats_single.m deleted file mode 100644 index 2f77201e..00000000 --- a/cresis-toolbox/processing/get_echogram_stats_single.m +++ /dev/null @@ -1,123 +0,0 @@ -function [layer_SNR] = get_echogram_stats_single(img,layer_row,noise_bin_rng) -% [layer_SNR] = get_echogram_stats_single(img,layer_row,noise_bin_rng) -% -% Returns signal to noise ratio for layer. Signal power is estimated by -% taking the value at img(layer_row(col),col). Noise power is found by -% taking the average of all the pixels that fall in -% img(layer_row(col)+noise_bin_rng(1):end-noise_bin_rng(end),col) and -% estimating the power of those pixels. If there are no pixels available -% for the noise power, then layer_SNR will return all NaN. -% -% INPUTS: -% -% img: -% Assumed to be log power image of size Nt by Nx -% -% layer_row: -% 1 by Nx vector such that layer_row(col) indicates the row in img for the -% layer for column "col". -% -% noise_bin_rng: -% 2 element vector that specifies the row range of the pixels to calculate -% the noise power with. The first number is the number of pixels after the -% layer to start at and the second number is the number of pixels to -% ignore at the bottom of the image. For example, the bins that will be -% used to calculated the noise for column, col, are: -% noise_bins = layer_row(col) + noise_bin_rng(1) : Nt - noise_bin_rng(end) -% -% OUTPUTS: -% -% layer_SNR: -% 1 by Nx vector such that layer_SNR(col) indicates the signal to noise -% ratio (SNR) in dB for the layer for column "col". -% -% EXAMPLE: -% -% % Load data -% fn = '/N/dcwan/projects/cresis/output/rds/2018_Greenland_P3/CSARP_standard/20180405_01/Data_20180405_01_014.mat'; -% mdata = load(fn); -% % Convert data to log scale -% img = 10*log10(mdata.Data); -% -% % Load layer -% fn = '/N/dcwan/projects/cresis/output/rds/2018_Greenland_P3/CSARP_layerData/20180405_01/Data_20180405_01_014.mat'; -% lay = load(fn); -% % Convert layer two way travel time to rows -% layer_row = interp1(mdata.Time,1:length(mdata.Time),lay.layerData{2}.value{2}.data); -% layer_row = round(interp1(lay.GPS_time,layer_row,mdata.GPS_time,'linear','extrap')); -% -% % For noise, use bins starting 400 bins after the layer and stopping 50 -% % bins from the end of each column. -% noise_bin_rng = [400 50]; -% -% % Calculate and plot SNR for the layer -% layer_SNR = get_echogram_stats_single(img,layer_row,noise_bin_rng); -% plot(layer_SNR); grid on; xlabel('Column'),ylabel('SNR (dB)'); -% -% Author: John Paden - -if 0 - % Example/Test - - % Load data - fn = '/N/dcwan/projects/cresis/output/rds/2018_Greenland_P3/CSARP_standard/20180405_01/Data_20180405_01_014.mat'; - mdata = load(fn); - % Convert data to log scale - img = 10*log10(mdata.Data); - - % Load layer - fn = '/N/dcwan/projects/cresis/output/rds/2018_Greenland_P3/CSARP_layerData/20180405_01/Data_20180405_01_014.mat'; - lay = load(fn); - % Convert layer two way travel time to rows - layer_row = interp1(mdata.Time,1:length(mdata.Time),lay.layerData{2}.value{2}.data); - layer_row = round(interp1(lay.GPS_time,layer_row,mdata.GPS_time,'linear','extrap')); - - % For noise, use bins starting 400 bins after the layer and stopping 50 - % bins from the end of each column. - noise_bin_rng = [400 50]; - - % Calculate and plot SNR for the layer - layer_SNR = get_echogram_stats_single(img,layer_row,noise_bin_rng); - plot(layer_SNR); - -end - -%% Input Check -if ~exist('noise_bin_rng','var') || isempty(noise_bin_rng) - % Default is to use all pixels below the layer - noise_bin_rng = [0 0]; -end - -%% Setup -Nt = size(img,1); -Nx = size(img,2); -layer_SNR = zeros(1,Nx); -signal_bin_rng = 0; -noise_bin_rng(end) = max(0,noise_bin_rng(end)); -noise_mask = false(size(img)); % memory inefficient, but easy - -%% Check the signal and noise for each column (range line) of the img -for rline = 1:Nx - signal_rows = max(1,layer_row(rline) + signal_bin_rng(1)) : min(Nt,layer_row(rline) + signal_bin_rng(end)); - - % Put the signal into layer_SNR: - if isempty(signal_rows) - % No valid signal bins for this column - layer_SNR(rline) = NaN; - else - layer_SNR(rline) = max(img(signal_rows,rline)); - end - - noise_rows = max(1,layer_row(rline) + noise_bin_rng(1)) : min(Nt,Nt - noise_bin_rng(end)); - - noise_mask(noise_rows,rline) = true; -end - -%% Final calculation of SNR -if any(noise_mask(:)) - % Divide signal in layer_SNR by average noise power - layer_SNR = layer_SNR - 10*log10(mean(10.^(img(noise_mask)/10))); -else - % No valid noise pixels found - layer_SNR(:) = NaN; -end diff --git a/cresis-toolbox/processing/img_combine.m b/cresis-toolbox/processing/img_combine.m index 1644f276..5b0bb352 100644 --- a/cresis-toolbox/processing/img_combine.m +++ b/cresis-toolbox/processing/img_combine.m @@ -2,8 +2,23 @@ % [Data, Time] = img_combine(param, param_mode, layers, data_in) % % Blends and combines together individual echogram image files -% "Data_img_II_*" into a combined datafile "Data_*" according to the -% param structure. +% "Data_img_II_*" into a combined datafile "Data_*" according to the param +% structure. Combining of different images is done in the vertical or +% fast-time dimension (e.g. for low gain images with high gain images). +% +% ------------------ +% | | +% | IMG 1 | +% | | +% ------------------ +% | | +% | ... | +% | | +% ------------------ +% | | +% | IMG N | +% | | +% ------------------ % % param: parameter structure usually loaded from parameter spreadsheet with % read_param_xls.m. The parameter structure will either look at the @@ -65,51 +80,12 @@ % % Authors: John Paden, Victor Berger % -% See also: run_update_img_combine.m, update_img_combine.m +% See also: run_img_combine_update.m, img_combine_update.m %% Input checks % ========================================================================= -[~,radar_type,~] = ct_output_dir(param.radar_name); - -if ~isfield(param.(param_mode), 'img_comb_weights') || isempty(param.(param_mode).img_comb_weights) - param.(param_mode).img_comb_weights = []; -end -if ~isfield(param.(param_mode), 'img_comb_mult') || isempty(param.(param_mode).img_comb_mult) - param.(param_mode).img_comb_mult = inf; -end -if ~isfield(param.(param_mode), 'img_comb_bins') || isempty(param.(param_mode).img_comb_bins) - param.(param_mode).img_comb_bins = 0; -end -if ~isfield(param.(param_mode), 'img_comb_weights_mode') || isempty(param.(param_mode).img_comb_weights_mode) - param.(param_mode).img_comb_weights_mode = ''; -end -if ~isfield(param.(param_mode), 'img_comb_trim') || isempty(param.(param_mode).img_comb_trim) - if strcmpi(radar_type,'deramp') - param.(param_mode).img_comb_trim = [0 0 0 inf]; - else - % Set relative trim to be 50% support level or higher for pulse compression - % Set absolute trim to be >= 0 time - if iscell(param.(param_mode).imgs{1}) - % Multilook format (param_mode is 'array') - wf_adc_list = param.(param_mode).imgs{1}{1}; - else - wf_adc_list = param.(param_mode).imgs{1}; - end - wf_first = wf_adc_list(1,1); - if iscell(param.(param_mode).imgs{end}) - % Multilook format (param_mode is 'array') - wf_adc_list = param.(param_mode).imgs{end}{1}; - else - wf_adc_list = param.(param_mode).imgs{end}; - end - wf_last = wf_adc_list(1,1); - param.(param_mode).img_comb_trim = [param.radar.wfs(wf_first).Tpd/2 -param.radar.wfs(wf_last).Tpd/2 0 inf]; - end -end -if ~isfield(param.(param_mode),'img_comb_layer_params') || isempty(param.(param_mode).img_comb_layer_params) - param.(param_mode).img_comb_layer_params = []; -end +param = img_combine_input_check(param, param_mode); %% Setup processing % ========================================================================= @@ -133,6 +109,7 @@ error('param.%s.img_comb not the right length. Since it is not empty, there should be 3 entries for each image combination interface ([Tpd second image for surface saturation, -inf for second image blank, Tpd first image to avoid roll off] is typical). Set correctly here and update param spreadsheet before dbcont.', param_mode); end end +Time = []; for img = 1:num_imgs if length(param.(param_mode).imgs) == 1 img_fn = fullfile(img_fn_dir, sprintf('Data_%s_%03d.mat', ... @@ -144,8 +121,8 @@ %% Combine a pair of images: image "img" with image "img-1" % Data, Time => combined result - % append.Data, append.Time => new data to append - if img == 1 + % appended.Data, appended.Time => new data to appended + if isempty(Time) if exist('data_in','var') %Overwrite data Data = data_in.Data{img}; Time = data_in.Time{img}; @@ -153,25 +130,44 @@ else load(img_fn,'Data','Time','GPS_time'); end + if length(Time) == 1 + % Force length==1 in fast time to just be empty to simplify data + % handling later. The idea is that a length 1 range line is useless + % anyway so we might as well simplify things by making it zero length. + Time = []; + Data = Data([],:); + end if ~isempty(param.(param_mode).img_comb_weights) Data = Data*10.^(param.(param_mode).img_comb_weights(img)/10); end - first_idx = find(Time >= Time(1)+param.(param_mode).img_comb_trim(1) ... + if isempty(Time) + Time1 = NaN; + else + Time1 = Time(1); + end + first_idx = find(Time >= Time1+param.(param_mode).img_comb_trim(1) ... & Time >= param.(param_mode).img_comb_trim(3),1,'first'); if ~isempty(first_idx) Time = Time(first_idx:end); Data = Data(first_idx:end,:); else - error('Zero range bin length images not supported.'); + Time = []; + Data = Data([],:); end if img == num_imgs - last_idx = find(Time <= Time(end)+param.(param_mode).img_comb_trim(2) ... + if isempty(Time) + TimeE = NaN; + else + TimeE = Time(end); + end + last_idx = find(Time <= TimeE+param.(param_mode).img_comb_trim(2) ... & Time <= param.(param_mode).img_comb_trim(4),1,'last'); if ~isempty(last_idx) Time = Time(1:last_idx); Data = Data(1:last_idx,:); else - error('Zero range bin length images not supported.'); + Time = []; + Data = Data([],:); end end Surface = zeros(size(GPS_time)); @@ -184,18 +180,26 @@ else if exist('data_in','var') %Overwrite data - append.Data = data_in.Data{img}; - append.Time = data_in.Time{img}; + appended.Data = data_in.Data{img}; + appended.Time = data_in.Time{img}; else - append = load(img_fn,'Time','Data'); + appended = load(img_fn,'Time','Data'); + end + if length(appended.Time) < 2 + continue; end if img == num_imgs - last_idx = find(append.Time <= append.Time(end)+param.(param_mode).img_comb_trim(2) ... - & append.Time <= param.(param_mode).img_comb_trim(4),1,'last'); + if isempty(appended.Time) + TimeE = NaN; + else + TimeE = appended.Time(end); + end + last_idx = find(appended.Time <= TimeE+param.(param_mode).img_comb_trim(2) ... + & appended.Time <= param.(param_mode).img_comb_trim(4),1,'last'); if ~isempty(last_idx) - append.Time = append.Time(1:last_idx); - append.Data = append.Data(1:last_idx,:); + appended.Time = appended.Time(1:last_idx); + appended.Data = appended.Data(1:last_idx,:); else error('Zero range bin length images not supported.'); end @@ -204,8 +208,8 @@ % Interpolate image N onto already loaded data (assumption is that image % N-1 always comes before image N) dt = Time(2)-Time(1); - newTime = (Time(1) : dt : append.Time(end)).'; - append.Data = interp1(append.Time,append.Data,newTime,'linear',0); + newTime = (Time(1) : dt : appended.Time(end)).'; + appended.Data = interp1(appended.Time,appended.Data,newTime,'linear',0); % Determine guard at end of image 1 that will not be used blend_bins = param.(param_mode).img_comb_bins; @@ -234,12 +238,12 @@ % Estimate difference if strcmpi(param.(param_mode).img_comb_weights_mode,'auto') - newData = zeros(size(append.Data),'single'); + newData = zeros(size(appended.Data),'single'); difference = NaN*zeros(1,size(newData,2)); for rline = 1:size(newData,2) trans_bins = img_bins(1,rline)+1:img_bins(2,rline); - if trans_bins <= size(append.Data,1) - difference(rline) = mean(Data(trans_bins,rline) ./ append.Data(trans_bins,rline)); + if trans_bins <= size(appended.Data,1) + difference(rline) = mean(Data(trans_bins,rline) ./ appended.Data(trans_bins,rline)); end end difference = nanmedian(difference); @@ -250,15 +254,15 @@ end % Combine images - newData = zeros(size(append.Data),'single'); + newData = zeros(size(appended.Data),'single'); for rline = 1:size(newData,2) trans_bins = img_bins(1,rline)+1:img_bins(2,rline); weights = 0.5+0.5*cos(pi*linspace(0,1,length(trans_bins)).'); - if trans_bins <= size(append.Data,1) + if trans_bins <= size(appended.Data,1) newData(:,rline) = [Data(1:img_bins(1,rline),rline); ... weights.*Data(trans_bins,rline) ... - + difference*(1-weights).*append.Data(trans_bins,rline); ... - difference*append.Data(img_bins(2,rline)+1:end,rline)]; + + difference*(1-weights).*appended.Data(trans_bins,rline); ... + difference*appended.Data(img_bins(2,rline)+1:end,rline)]; else newData(:,rline) = Data(1:size(newData,1),rline); end diff --git a/cresis-toolbox/processing/check_img_combine.m b/cresis-toolbox/processing/img_combine_check.m similarity index 92% rename from cresis-toolbox/processing/check_img_combine.m rename to cresis-toolbox/processing/img_combine_check.m index 01636220..5204ef45 100644 --- a/cresis-toolbox/processing/check_img_combine.m +++ b/cresis-toolbox/processing/img_combine_check.m @@ -1,5 +1,5 @@ -function mdata = check_img_combine(data_fns, rlines) -% mdata = check_img_combine(data_fns, rlines) +function mdata = img_combine_check(data_fns, rlines) +% mdata = img_combine_check(data_fns, rlines) % % Function for checking the gains of different images including those % formed with img_combine. Note that this function automatically searches @@ -9,17 +9,17 @@ % % % The following loads Data_20160413_17_005, Data_img_01_20160413_17_005, % Data_img_02_20160413_17_005, Data_img_03_20160413_17_005: -% mdata = check_img_combine('/cresis/snfs1/dataproducts/ct_data/rds/2016_Greenland_Polar6/CSARP_qlook/20160413_17/Data_20160413_17_005'); +% mdata = img_combine_check('/cresis/snfs1/dataproducts/ct_data/rds/2016_Greenland_Polar6/CSARP_qlook/20160413_17/Data_20160413_17_005'); % % % The following compares different processing types: -% mdata = check_img_combine({'/N/dcwan/projects/cresis/output/snow/2017_Greenland_P3/CSARP_deconv/20170309_01/Data_20170309_01_158.mat', ... +% mdata = img_combine_check({'/N/dcwan/projects/cresis/output/snow/2017_Greenland_P3/CSARP_deconv/20170309_01/Data_20170309_01_158.mat', ... % '/N/dcwan/projects/cresis/output/snow/2017_Greenland_P3/CSARP_qlook/20170309_01/Data_20170309_01_158.mat', ... % '/N/dcwan/projects/cresis/output/snow/2017_Greenland_P3/CSARP_qlook_uwb/20170309_01/Data_20170309_01_158.mat', ... % '/N/dcwan/projects/cresis/output/snow/2017_Greenland_P3/CSARP_qlook_kuband/20170309_01/Data_20170309_01_158.mat'},372); % % Author: John Paden % -% See also: check_data_products.m, check_img_combine.m, +% See also: check_data_products.m, img_combine_check.m, % run_check_data_products.m %% Setup diff --git a/cresis-toolbox/processing/img_combine_input_check.m b/cresis-toolbox/processing/img_combine_input_check.m new file mode 100644 index 00000000..b5b387ff --- /dev/null +++ b/cresis-toolbox/processing/img_combine_input_check.m @@ -0,0 +1,57 @@ +function param = img_combine_input_check(param, param_mode) +% param = img_combine_input_check(param, param_mode) +% +% Support function for img_combine. Checks inputs and adds missing fields +% to parameter structure. Also called by qlook.m and array.m to ensure the +% img_comb fields are populated. +% +% param: parameter structure +% +% param_mode: string containing the name of the field in the param +% structure that contains the img_comb_* fields. Usually "qlook" or "array". + +[~,radar_type,~] = ct_output_dir(param.radar_name); + +if ~isfield(param.(param_mode), 'img_comb_weights') || isempty(param.(param_mode).img_comb_weights) + param.(param_mode).img_comb_weights = []; +end +if ~isfield(param.(param_mode), 'img_comb_mult') || isempty(param.(param_mode).img_comb_mult) + param.(param_mode).img_comb_mult = inf; +end +if ~isfield(param.(param_mode), 'img_comb_bins') || isempty(param.(param_mode).img_comb_bins) + param.(param_mode).img_comb_bins = 0; +end +if ~isfield(param.(param_mode), 'img_comb_weights_mode') || isempty(param.(param_mode).img_comb_weights_mode) + param.(param_mode).img_comb_weights_mode = ''; +end +if ~isfield(param.(param_mode), 'img_comb_trim') || isempty(param.(param_mode).img_comb_trim) + if strcmpi(radar_type,'deramp') + param.(param_mode).img_comb_trim = [0 0 0 inf]; + else + % Set relative trim to be 50% support level or higher for pulse compression + % Set absolute trim to be >= 0 time + if iscell(param.(param_mode).imgs{1}) + % Multilook format (param_mode is 'array') + wf_adc_list = param.(param_mode).imgs{1}{1}; + else + wf_adc_list = param.(param_mode).imgs{1}; + end + wf_first = wf_adc_list(1,1); + if isempty(param.(param_mode).img_comb) + % Only the first image is used when there is no combining + wf_last = wf_first; + else + if iscell(param.(param_mode).imgs{end}) + % Multilook format (param_mode is 'array') + wf_adc_list = param.(param_mode).imgs{end}{1}; + else + wf_adc_list = param.(param_mode).imgs{end}; + end + wf_last = wf_adc_list(1,1); + end + param.(param_mode).img_comb_trim = [param.radar.wfs(wf_first).Tpd/2 -param.radar.wfs(wf_last).Tpd/2 0 inf]; + end +end +if ~isfield(param.(param_mode),'img_comb_layer_params') || isempty(param.(param_mode).img_comb_layer_params) + param.(param_mode).img_comb_layer_params = []; +end diff --git a/cresis-toolbox/processing/update_img_combine.m b/cresis-toolbox/processing/img_combine_update.m similarity index 68% rename from cresis-toolbox/processing/update_img_combine.m rename to cresis-toolbox/processing/img_combine_update.m index 2762f1ca..d5dc2586 100644 --- a/cresis-toolbox/processing/update_img_combine.m +++ b/cresis-toolbox/processing/img_combine_update.m @@ -1,11 +1,11 @@ -function update_img_combine(param, param_override) -% update_img_combine(param, param_override) +function img_combine_update(param, param_override) +% img_combine_update(param, param_override) % % Function for running img_combine on echogram files. Works with % get_heights and combine files. % % param = struct with processing parameters -% .update_img_combine: struct with processing parameters +% .img_combine_update: struct with processing parameters % .mode: string containing 'get_heights' or 'combine' depending on which % one generated the echogram files % param_override = parameters in this struct will override parameters @@ -13,17 +13,19 @@ function update_img_combine(param, param_override) % Typically global gRadar; param_override = gRadar; % % Example: -% See run_update_img_combine.m for how to run this function directly. +% See run_img_combine_update.m for how to run this function directly. % This function may be called from the run_master.m script using the % param spreadsheet and the cmd.generic column. % % Authors: John Paden -% See also: run_update_img_combine.m, update_img_combine.m +% See also: run_img_combine_update.m, img_combine_update.m %% General Setup % ===================================================================== -param = merge_structs(param, param_override); +if exist('param_override','var') + param = merge_structs(param, param_override); +end fprintf('=====================================================================\n'); fprintf('%s: %s (%s)\n', mfilename, param.day_seg, datestr(now)); @@ -32,26 +34,12 @@ function update_img_combine(param, param_override) %% Input checks % ===================================================================== -mode = param.update_img_combine.mode; - -if ~isfield(param.(mode),'img_comb_mult') || isempty(param.(mode).img_comb_mult) - param.(mode).img_comb_mult = inf; -end - -if ~isfield(param.(mode),'img_comb_weights') || isempty(param.(mode).img_comb_weights) - param.(mode).img_comb_weights = []; -end - -if ~isfield(param.(mode),'img_comb_weights_mode') || isempty(param.(mode).img_comb_weights_mode) - param.(mode).img_comb_weights_mode = ''; -end +mode = param.img_combine_update.mode; -if ~isfield(param.(mode),'img_comb_bins') || isempty(param.(mode).img_comb_bins) - param.(mode).img_comb_bins = 1; -end +param = img_combine_input_check(param,mode); % Load frames file -load(ct_filename_support(param,'','frames')); +frames = frames_load(param); % If no frames specified, then do all frames if isempty(param.cmd.frms) @@ -92,6 +80,12 @@ function update_img_combine(param, param_override) fprintf('%s %s (%s)\n', mfilename, out_fn, datestr(now)); param_mode_str = sprintf('param_%s',mode); + if ~exist(out_fn,'file') + tmp_out_fn = fullfile(out_path, sprintf('Data_img_01_%s_%03d.mat', ... + param.day_seg, frm)); + warning('Combined file does not exist, so using img_01 file as the base for the combined file.'); + copyfile(tmp_out_fn,out_fn); + end load(out_fn,param_mode_str,'Surface','GPS_time'); if isempty(param.(mode).img_comb) @@ -109,7 +103,19 @@ function update_img_combine(param, param_override) [Data, Time] = img_combine(param, mode, layers); % Update parameter structure with new parameters - cmd = sprintf('%s.(mode) = param.(mode);', param_mode_str); + cmd = sprintf('%s.(mode).img_comb = param.(mode).img_comb;', param_mode_str); + eval(cmd); + cmd = sprintf('%s.(mode).img_comb_weights = param.(mode).img_comb_weights;', param_mode_str); + eval(cmd); + cmd = sprintf('%s.(mode).img_comb_mult = param.(mode).img_comb_mult;', param_mode_str); + eval(cmd); + cmd = sprintf('%s.(mode).img_comb_bins = param.(mode).img_comb_bins;', param_mode_str); + eval(cmd); + cmd = sprintf('%s.(mode).img_comb_weights_mode = param.(mode).img_comb_weights_mode;', param_mode_str); + eval(cmd); + cmd = sprintf('%s.(mode).img_comb_trim = param.(mode).img_comb_trim;', param_mode_str); + eval(cmd); + cmd = sprintf('%s.(mode).img_comb_layer_params = param.(mode).img_comb_layer_params;', param_mode_str); eval(cmd); % Save output diff --git a/cresis-toolbox/processing/layer_create.m b/cresis-toolbox/processing/layer_create.m new file mode 100644 index 00000000..b883bbcf --- /dev/null +++ b/cresis-toolbox/processing/layer_create.m @@ -0,0 +1,85 @@ +function layer_create(param,param_override) +% layer_create(param,param_override) +% +% The layer_create function performs one of three actions depending on the +% "update_mode" field: +% * It can be used to make layer files when they do not exist +% * It can be used to blank out layer information in layer files +% * It can be used to update GPS information in layer files (e.g. if the +% records file is updated with new GPS information with records_update.m) +% +% The layer files are used by the imb.picker.m program as well as the data +% processing. +% +% This function should be run from run_layer_create.m (that script sets up +% all the control variables). +% +% Author: John Paden +% +% See also: layer_create, run_layer_create + +param = merge_structs(param,param_override); + +dbstack_info = dbstack; +fprintf('=====================================================================\n'); +fprintf('%s: %s (%s)\n', dbstack_info(1).name, param.day_seg, datestr(now,'HH:MM:SS')); +fprintf('=====================================================================\n'); + +%% Input Checks + +if ~isfield(param.layer_file_make,'update_mode') || isempty(param.layer_file_make.update_mode) + param.layer_file_make.update_mode = 0; + % In all modes, if file does not exist, then a new file is created. If a + % file exists, how it is updated is controlled by the update_mode field. + % The update_mode field is a nonnegative integer scalar indicating: + % + % 0: no updates are made, layer file is created only if no layer file + % exists or if there were problems with an existing layer file. This is + % the default setting. + % + % 1: layer file is overwritten with new information (blank layer file) + % and old layer data is lost + % + % 2: Old GPS time field is adjusted according to changes in + % param.records.gps.time_offset and then the position and layer + % information is reinterpolated from the old GPS time field to the new + % GPS time field. +end + +if ~isfield(param.layer_file_make,'out_path') || isempty(param.layer_file_make.out_path) + param.layer_file_make.out_path = 'layer'; +end + +%% Make or update layer files +layers = layerdata(param,param.layer_file_make.out_path); +if param.layer_file_make.update_mode == 1 + if isempty(param.cmd.frms) + layers.delete_all(); + else + layers.delete_layer_file(param.cmd.frms); + end +end +if param.layer_file_make.update_mode == 2 + layers.update_gps_all(); +end +layers.check_all(); +layer_names = layers.get_layer_names(); +surface_idx = find(strcmp('surface',layer_names)); +if isempty(surface_idx) + layer_organizer.lyr_name = {'surface'}; + layer_organizer.lyr_group_name = {'standard'}; + layers.insert_layers(layer_organizer); + layers.update_layer(1:length(layers.frames.frame_idxs),'surface',layers.frames.gps_time([1 end]),[NaN NaN]); +else + obj.layer_organizer.lyr_group_name{surface_idx} = 'standard'; +end +bottom_idx = find(strcmp('bottom',layer_names)); +if isempty(bottom_idx) + layer_organizer.lyr_name = {'bottom'}; + layer_organizer.lyr_group_name = {'standard'}; + layers.insert_layers(layer_organizer); + layers.update_layer(1:length(layers.frames.frame_idxs),'bottom',layers.frames.gps_time([1 end]),[NaN NaN]); +else + obj.layer_organizer.lyr_group_name{bottom_idx} = 'standard'; +end +layers.save(); diff --git a/cresis-toolbox/processing/lever_arm.m b/cresis-toolbox/processing/lever_arm.m index 9fa130a6..13054fb3 100644 --- a/cresis-toolbox/processing/lever_arm.m +++ b/cresis-toolbox/processing/lever_arm.m @@ -1,36 +1,53 @@ function [phase_center] = lever_arm(param, tx_weights, rxchannel) % [phase_center] = lever_arm(param, tx_weights, rxchannel) % -% Returns lever arm position for antenna phase center. +% Returns lever arm position for antenna phase center. See remarks below +% to understand the coordinate system for output "phase_center". % -% param = parameter struct -% .season_name = string containing the season name (e.g. 2011_Greenland_TO) -% .radar_name = string containing the radar name (e.g. snow2) -% .gps_source = string from GPS file using format SOURCE-VERSION -% Only the source is used (e.g. ATM-final_20120303) -% tx_weights = transmit amplitude weightings (from the radar worksheet of -% the parameter spreadsheet) -% These are amplitude weights, not power weights. -% rxchannel = receive channel to return phase_center for (scalar, -% positive integer) -% Setting rxchannel to 0, causes the "reference" position to be returned. -% This is usually the position of one of the center receive elements -% and equal weights on all transmitters. +% ========================================================================= +% INPUTS: +% +% param: parameter struct +% +% .season_name: string containing the season name (e.g. 2011_Greenland_TO) +% +% .radar_name: string containing the radar name (e.g. snow2) +% +% .gps_source: string from GPS file using format SOURCE-VERSION (e.g. +% ATM-final_20120303). The SOURCE portion is used to determine which lever +% arm set to use for a particular dataset since some field seasons have +% more than one GPS source. +% +% tx_weights: transmit amplitude weightings (from the radar worksheet of +% the parameter spreadsheet) These are amplitude weights, not power +% weights. +% +% rxchannel: receive channel to return phase_center for (scalar, positive +% integer) Setting rxchannel to 0, causes the "reference" position to be +% returned. This is usually the position of one of the center receive +% elements and equal weights on all transmitters. +% +% ========================================================================= +% OUTPUTS: +% +% phase_center: lever arm to each phase center specified by tx_weights and +% rxchannel. See remarks below to understand the coordinate system for +% output "phase_center". % -% phase_center = lever arm to each phase center specified by -% tx_weights and rxchannel % % ========================================================================= % REMARKS: % % 1). Lever arm refers to a (3 x 1) vector that expresses the position of % each phase center relative to the position that the GPS trajectory -% was processed to. The basis for the vector is the coordinate -% system of the plane's body (Xb, Yb, Zb). This is a righthanded, -% orthogonal system that agrees with aerospace convention. +Xb points -% from the plane's center of gravity towards its nose. +Yb points from -% the plane's center of gravity along the right wing. +Zb points from -% the plane's center of gravity down towards the Earth's surface. +% was processed to (this is often the GPS antenna or IMU measurement +% center). The basis for the vector is the coordinate system of the +% plane's body (Xb, Yb, Zb). This is a righthanded, orthogonal system +% that agrees with aerospace convention. +Xb points from the plane's +% center of gravity towards its nose. +Yb points from the plane's +% center of gravity along the right wing. +Zb points from the plane's +% center of gravity down towards the Earth's surface (i.e. increasing +% Zb points downwards!). % % 2). The lever arm of the Nth receive channel is defined using the % following syntax: @@ -78,6 +95,30 @@ % For the full simulator, remove 'sim' at the end($) of param.season_name param.season_name = regexprep(param.season_name,'sim$','','ignorecase'); +if any(strcmpi(param.season_name,{'2022_Greenland_X6'})) + gps.x = 0; + gps.y = 0; + gps.z = 0; +end + +if any(strcmpi(param.season_name,{'2022_Greenland_Vapor'})) + gps.x = 0; + gps.y = 0; + gps.z = 0; +end + +if any(strcmpi(param.season_name,{'2021_Arctic_Vanilla'})) + gps.x = 0; + gps.y = 0; + gps.z = 0; +end + +if any(strcmpi(param.season_name,{'2019_SouthDakota_N1KU','2020_SouthDakota_N1KU'})) + gps.x = 0; + gps.y = 0; + gps.z = 0; +end + if any(strcmpi(param.season_name,{'2019_Arctic_GV','2019_Antarctica_GV'})) %... % && any(strcmpi(gps_source,{'nmea'})) && any(strcmpi(gps_source,{'atm-field'})) % warning('ACTUAL LEVER ARM ACTUAL LEVER ARM NEEDS TO BE DETERMINED'); @@ -96,7 +137,6 @@ end if (strcmpi(param.season_name,'2019_Antarctica_Ground') && any(strcmpi(gps_source,{'arena','cresis'}))) -% warning('ACTUAL LEVER ARM ACTUAL LEVER ARM NEEDS TO BE DETERMINED'); % Platform: Ground based sled % gps.x = 0; @@ -104,6 +144,30 @@ gps.z = 0; end +if (strcmpi(param.season_name,'2022_Antarctica_BaslerMKB') && any(strcmpi(gps_source,{'arena','cresis','novatelraw','utig'}))) + % Platform: Airborne Radar Kenn Borek Air Basler call sign MKB (COLDEX 1, COLDEX 2, UTIG) + % + gps.x = 0; + gps.y = 0; + gps.z = 0; +end + +if any(strcmpi(param.season_name,{'2022_Antarctica_Ground','2023_Greenland_Ground'})) && any(strcmpi(gps_source,{'arena','cresis'})) + % Platform: Ground based sled (EAGER 1, EAGER 2) + % + gps.x = 0; + gps.y = 0; + gps.z = 0; +end + +if (strcmpi(param.season_name,'2022_Antarctica_GroundGHOST') && any(strcmpi(gps_source,{'arena','cresis'}))) + % Platform: Ground based sled (GHOST 1, GHOST 2) + % + gps.x = 0; + gps.y = 0; + gps.z = 0; +end + if any(strcmpi(param.season_name,{'2019_Greenland_TO'})) %... % && any(strcmpi(gps_source,{'nmea'})) % warning('ACTUAL LEVER ARM ACTUAL LEVER ARM NEEDS TO BE DETERMINED'); @@ -114,8 +178,8 @@ gps.z = 0; end -if any(strcmpi(param.season_name,{'2018_Alaska_SO','2019_Alaska_SO'})) ... - && any(strcmpi(gps_source,{'nmea','ualidar'})) +if any(strcmpi(param.season_name,{'2018_Alaska_SO','2021_Alaska_SO'})) ... + && any(strcmpi(gps_source,{'nmea','lidar'})) % The snow radar shared the same GPS antenna with the lidar of the univ. of Fairbanks % Emily measured the positions of the snow radar rx and tx antennas relative to the GPS antenna gps.x = 0; @@ -131,11 +195,10 @@ gps.z = 1300; end -if (any(strcmpi(param.season_name,{'2018_Antarctica_TObas','2019_Antarctica_TObas'})) && any(strcmpi(gps_source,{'arena','bas'}))) - % See (strcmpi(param.season_name,'2018_Antarctica_TObas') && strcmpi(gps_source,'arena')) - % +if (any(strcmpi(param.season_name,{'2018_Antarctica_TObas'})) && any(strcmpi(gps_source,{'arena','bas'}))) % 2018 Antarctica TObas (Jan-Feb 2019) GPS data are processed to the IMAR - % gravimeter. + % gravimeter. The origin that we use here is the aft GPS antenna above + % the radar system. % % From Tom Jordan at BAS: % My best solution is that the IMAR solution to the GPS above the radar is as follows (all in m): @@ -143,8 +206,6 @@ % Y (Positive port) 0.1116 % Z (Positive up) 1.4762 % - % - % % Aircraft: British Antarctic Survey (BAS) VP-FBL % % Carl Robinson at BAS Aug 2018: Though we do have measurements for @@ -180,6 +241,43 @@ gps.z = 1.4762; % Gravimeter was below (down is positive-z) the GPS end +if (any(strcmpi(param.season_name,{'2019_Antarctica_TObas'})) && any(strcmpi(gps_source,{'bas','bas_imu_to_gps'}))) + % 2019 Antarctica TObas (Dec 2019-Jan 2020) GPS data are processed to the + % IMAR gravimeter for some flights and forward GPS antenna for some + % flights. This handles the data that are stored relative to the IMAR + % gravimeter. + % + % From Tom Jordan at BAS. + + % T06 flight (before merging with GNSS only data), T07 flight + gps.x = -0.0538; % Gravimeter was just behind the antenna + gps.y = 0.349; % Gravimeter was on the right side of the plane/GPS + gps.z = 1.4803; % Gravimeter was below (down is positive-z) the GPS +end + +if (any(strcmpi(param.season_name,{'2019_Antarctica_TObas'})) && any(strcmpi(gps_source,{'arena','bas_gnss'}))) + % 2019 Antarctica TObas (Dec 2019-Jan 2020) GPS data are processed to the + % IMAR gravimeter for some flights and forward GPS antenna for some + % flights. This handles the data that are stored relative to the forward + % GPS antenna. + % + % From Tom Jordan at BAS. + + % End of T06 flight when IMU failed, all January flights + gps.x = 0; % Forward GPS antenna + gps.y = 0; % Forward GPS antenna + gps.z = 0; % Forward GPS antenna +end + +if (any(strcmpi(param.season_name,{'2022_Greenland_Ground'})) && any(strcmpi(gps_source,{'arena','cresis'}))) + % GPS antenna is mounted almost on the radar antenna phase center + % Need to update exactly what the vertical offset is. + + gps.x = 0; + gps.y = 0; + gps.z = 0; +end + if (strcmpi(param.season_name,'2016_Greenland_TOdtu') && strcmpi(gps_source,'dtu')) % =========================================================================== % All antenna positions measurements are relative to GPS antenna @@ -292,6 +390,13 @@ gps.z = 0; end +if any(strcmpi(param.season_name,{'2015_Antarctica_Ground','2015_Greenland_Ground'})) + % NMEA data only and unknown lever arm + gps.x = 0; + gps.y = 0; + gps.z = 0; +end + if (strcmpi(param.season_name,'2013_Antarctica_Basler') && strcmpi(gps_source,'cresis')) % Absolute position of IMU for radar systems % For 2013: @@ -351,6 +456,47 @@ gps.z = 0; end +if (strcmpi(param.season_name,'2022_Greenland_P3') && any(strcmpi(gps_source,{'ARENA','CReSIS_GNSS'}))) + % Trajectory is referenced to the GNSS antenna, so gps struct contains + % the offset to the GNSS antenna + % + % Very rough sanity check: + % GPS antenna connector is just aft of 305" flight station (308.5" flight + % station). + % IMU is aft of the GPS antenna by ~48". +308.5 = 356.5" flight station (IMU to GPS Y = +48") + % IMU is ~24" (below floor) + 94" (above floor) = 118" (IMU to GPS Z = +118") + % IMU is starboard of GPS antenna by about 24" (IMU to GPS X = -24") + % + % Inertial Explorer lever arm offset (IMU to GPS): + % SETIMUTOANTOFFSET -0.430 1.310 3.498 0.2 0.2 0.2 + % + % Coordinates from Emily Arnold/Brad Schroeder CAD model, Aaron Paden CAD model, John Paden + % Name FS (X) BL (Y) WL (Z) Notes + % GPS 318.4 0.0 214.0 w/ Inertial Explorer Flight Station Correction + % IMU 370.0 17.9 77.9 This is the measurement center of the IMU + % Viv 1 354.0 -19.4 68.5 Antenna numbering F-B and L-R + % Viv2 354.0 -0.1 68.5 + % Viv3 354.0 4.9 68.5 + % Viv4 354.0 20.0 68.5 + % Viv5 386.1 -19.4 68.2 + % Viv6 386.1 -13.7 68.2 + % Viv7 386.1 9.2 68.2 + % Viv8 386.1 16.8 68.2 + % Horn1 359.2 -12.3 72.9 + % Horn2 391.2 -2.4 72.6 + gps.x = -318.4*2.54/100; + gps.y = 0.0*2.54/100; + gps.z = -214.0*2.54/100; +end + +if (strcmpi(param.season_name,'2022_Greenland_P3') && any(strcmpi(gps_source,{'CReSIS'}))) + % Trajectory is of the IMU, so gps struct contains the offset to the IMU + % See initial GPS entry for notes. + gps.x = -370.0*2.54/100; + gps.y = 17.9*2.54/100; + gps.z = -77.9*2.54/100; +end + if (strcmpi(param.season_name,'2019_Greenland_P3') && any(strcmpi(gps_source,{'ATM','NMEA','DMS','novatel'}))) ... || (strcmpi(param.season_name,'2018_Greenland_P3') && any(strcmpi(gps_source,{'ATM','NMEA','DMS'}))) ... || (strcmpi(param.season_name,'2017_Antarctica_P3') && any(strcmpi(gps_source,{'ATM','NMEA','DMS'}))) ... @@ -476,7 +622,7 @@ end if (strcmpi(param.season_name,'2009_Antarctica_DC8') && strcmpi(gps_source,'DMS')) ... - || (strcmpi(param.season_name,'2010_Antarctica_DC8') && strcmpi(gps_source,'DMSATM')) + || (strcmpi(param.season_name,'2010_Antarctica_DC8') && any(strcmpi(gps_source,{'DMSATM','DMSATM_20101026'}))) ... % Absolute position of ATM antenna % For 2009: % DMS data are processed to the GPS antenna. @@ -620,7 +766,19 @@ end -if (any(strcmpi(param.season_name,{'2015_Greenland_Polar6','2016_Greenland_Polar6','2017_Arctic_Polar5','2017_Antarctica_Polar6','2018_Greenland_Polar6','2019_Antarctica_Polar6','2019_Arctic_Polar6','2020_Arctic_Polar6'})) && any(strcmpi(gps_source,{'AWI','NMEA'}))) +if (any(strcmpi(param.season_name, ... + {'2015_Greenland_Polar6', ... + '2016_Greenland_Polar6', ... + '2017_Arctic_Polar5', ... + '2017_Antarctica_Polar6', ... + '2018_Greenland_Polar6', ... + '2019_Antarctica_Polar6', ... + '2019_Arctic_Polar6', ... + '2020_Arctic_Polar6', ... + '2021_Greenland_Polar5', ... + '2022_Greenland_Polar5', ... + '2022_Antarctica_Polar5', ... + })) && any(strcmpi(gps_source,{'AWI','NMEA'}))) % Measurements are from Richard Hale Aug 12, 2015 for RDS and Aug 15, % 2015 for Snow Radar. Measurements are made relative to the AWI Aft % Science GPS antenna known as ST5. @@ -692,9 +850,9 @@ %% Accumulation Radar % ========================================================================= -if (any(strcmpi(param.season_name,{'2018_Antarctica_TObas','2019_Antarctica_TObas'})) && strcmpi(radar_name,'accum')) +if (any(strcmpi(param.season_name,{'2018_Antarctica_TObas'})) && strcmpi(radar_name,'accum')) % See GPS section for 2018_Antarctica_TObas for details: - % 3+5/8" from back of antenna box + % 3+5/8" from back/aft-side of antenna box % 9+15/16" from right/starboard side of lid (used edge of lid) % 64+1/16" below to the lid of the antenna % @@ -738,6 +896,85 @@ end end +if (any(strcmpi(param.season_name,{'2019_Antarctica_TObas'})) && strcmpi(radar_name,'accum')) + % See GPS section for 2019_Antarctica_TObas for details: + % -2.8274 from forward-side of antenna box + % -0.1757 from right/starboard side of lid (used edge of lid) + % -1.7068 below to the lid of the antenna + % + % The offset from the outer back right top corner of the box to the + % center of the aperture of each of the antennas is: + % (reference is aft, starboard, top) + % + % With the box (outer surfaces) as reference, the measurements (in + % inches) are the following: + % Element 1 (starboard): (x,y,z) = (6.8125, 1.69885,10.04). + % Element 2 (next to starboard): (x,y,z) = (6.8125, 6.44885, 10.04) + % Element 3 (next to port): (x,y,z) = (6.8125,11.19885, 10.04) + % Element 4 (port): (x,y,z) = (6.8125, 15.94885, 10.04) + % The thickness of each antenna element is 0.125 in. + % + % The bars were 0.75 in. thick. This decreases the z-position of all + % elements by 0.75 in. + + % Accumulation antenna + LArx = []; + LArx(1,:) = ( (-2.8274/0.0254 - 6.8125) + [0 0 0 0])*0.0254 - gps.x; % m + LArx(2,:) = ( (+0.1757/0.0254) - [15.9489 11.1989 6.4489 1.6988])*0.0254 - gps.y; % m + LArx(3,:) = ( (+1.7068/0.0254 - 0.75 + 10.04) + [0 0 0 0])*0.0254 - gps.z; % m + + LArx = mean(LArx,2); % Combine all 4 elements into a single element + + LAtx = []; + LAtx(1,:) = ( (-2.8274/0.0254 - 6.8125) + [0 0 0 0])*0.0254 - gps.x; % m + LAtx(2,:) = ( (+0.1757/0.0254) - [15.9489 11.1989 6.4489 1.6988])*0.0254 - gps.y; % m + LAtx(3,:) = ( (+1.7068/0.0254 - 0.75 + 10.04) + [0 0 0 0])*0.0254 - gps.z; % m + + LAtx = mean(LAtx,2); % Combine all 4 elements into a single element + + if strcmpi(gps_source,'bas_imu_to_gps') + % Special code for doing IMU to GPS lever arm rather than to radar + LArx = [-gps.x; -gps.y; -gps.z]; + LAtx = [-gps.x; -gps.y; -gps.z]; + end + + if ~exist('rxchannel','var') || isempty(rxchannel) + rxchannel = 1; + end + + if rxchannel == 0 + rxchannel = 1; + tx_weights = ones(1,size(LAtx,2)); + end +end + +if (any(strcmpi(param.season_name,{'2022_Greenland_Ground'})) && strcmpi(radar_name,'accum')) + + % Accumulation antenna + LArx = []; + LArx(1,:) = ( 0*0.0254 ) - gps.x; % m + LArx(2,:) = ( 0*0.0254 ) - gps.y; % m + LArx(3,:) = ( 0*0.0254 ) - gps.z; % m + + LArx = mean(LArx,2); % Combine all elements into a single element + + LAtx = []; + LAtx(1,:) = ( 0*0.0254 ) - gps.x; % m + LAtx(2,:) = ( 0*0.0254 ) - gps.y; % m + LAtx(3,:) = ( 0*0.0254 ) - gps.z; % m + + LAtx = mean(LAtx,2); % Combine all elements into a single element + + if ~exist('rxchannel','var') || isempty(rxchannel) + rxchannel = 1; + end + + if rxchannel == 0 + rxchannel = 1; + tx_weights = ones(1,size(LAtx,2)); + end +end + if (strcmpi(param.season_name,'2009_antarctica_TO') && strcmpi(radar_name,'accum')) ... || (strcmpi(param.season_name,'2011_antarctica_TO') && strcmpi(radar_name,'accum')) % Accumulation antenna @@ -789,13 +1026,13 @@ % along-track elements are combined using in cabin power combiners. % Each of the four combined channels are individually transmitted and % received on. - LArx(1,:) = (-433.3*0.0254 + [0 0 0 0]) - gps.x; % m - LArx(2,:) = (0 + [-0.39 -0.13 0.13 0.39]) - gps.y; % m - LArx(3,:) = (-72.5*0.0254 + [0 0 0 0]) - gps.z; % m + LArx(1,:) = (-433.3*0.0254 + [0 0 0 0 0]) - gps.x; % m + LArx(2,:) = (0 + [-0.39 -0.13 0.13 0.39 0]) - gps.y; % m + LArx(3,:) = (-72.5*0.0254 + [0 0 0 0 0]) - gps.z; % m - LArx(1,:) = (-433.3*0.0254 + [0 0 0 0]) - gps.x; % m - LAtx(2,:) = (0 + [-0.39 -0.13 0.13 0.39]) - gps.y; % m - LAtx(3,:) = (-72.5*0.0254 + [0 0 0 0]) - gps.z; % m + LAtx(1,:) = (-433.3*0.0254 + [0 0 0 0 0]) - gps.x; % m + LAtx(2,:) = (0 + [-0.39 -0.13 0.13 0.39 0]) - gps.y; % m + LAtx(3,:) = (-72.5*0.0254 + [0 0 0 0 0]) - gps.z; % m if ~exist('rxchannel','var') || isempty(rxchannel) rxchannel = 1:4; @@ -859,6 +1096,27 @@ end end +if any(strcmpi(param.season_name,{'2015_Antarctica_Ground','2015_Greenland_Ground'})) && strcmpi(radar_name,'accum') + % Accumulation antenna + % NMEA data only and unknown lever arm + LArx(1,:) = ([0]) - gps.x; % m + LArx(2,:) = ([0]) - gps.y; % m + LArx(3,:) = ([0]) - gps.z; % m + + LAtx(1,:) = ([0]) - gps.x; % m + LAtx(2,:) = ([0]) - gps.y; % m + LAtx(3,:) = ([0]) - gps.z; % m + + if ~exist('rxchannel','var') || isempty(rxchannel) + rxchannel = 1; + end + + if rxchannel == 0 + rxchannel = 1; + tx_weights = ones(1,size(LAtx,2)); + end +end + if (strcmpi(param.season_name,'2013_Antarctica_Ground') && strcmpi(radar_name,'accum')) % Accumulation antenna LArx(1,:) = ([0 0 0 0 0 0]) - gps.x; % m @@ -899,6 +1157,54 @@ end end +if (strcmpi(param.season_name,'2022_Antarctica_BaslerMKB') && strcmpi(radar_name,'accum')) + % Platform: Airborne Radar Kenn Borek Air Basler call sign MKB (COLDEX 1, COLDEX 2, UTIG) + % These values need to be updated with actual values. + + % Measurements, X,Y,Z are in aircraft coordinates, not IMU coordinates + %LArx(1,1:16) = 1.5859; + LArx(1,1:16) = -1.5; % GUESS + LArx(2,1:16) = [-9.2102*(0:15)] * 2.54/100; + LArx(2,1:16) = LArx(2,1:16) - mean(LArx(2,1:16)); + LArx(3,1:16) = -3.4609; % GUESS + warning('This file needs to be updated with actual values for 2022.'); + + LAtx = LArx(:,1:16); + + if ~exist('rxchannel','var') || isempty(rxchannel) + rxchannel = 1:16; + end + + % Amplitude (not power) weightings for transmit side. + if rxchannel == 0 + rxchannel = 8; + tx_weights = ones(1,size(LAtx,2)); + end +end + +if any(strcmpi(param.season_name,{'2022_Antarctica_Ground','2023_Greenland_Ground'})) && strcmpi(radar_name,'accum') + % Sled antennas EAGER 1 and EAGER 2 + % + % Primary GPS antenna: GPS positions are relative to primary which is in the center of the radar antenna array. + % Secondary GPS antenna: 22.5" forward and 12" right of the primary. Align information will be relative to this. + + % GPS Antenna to Antenna phase center + LArx = [0 0 -4 % along-track polarization/H-polarization + 0 0 -4 % cross-track polarization/V-polarization + ].' * 2.54/100; + + LAtx = LArx(:,:); + + if ~exist('rxchannel','var') || isempty(rxchannel) + rxchannel = 1; + end + + if rxchannel == 0 + rxchannel = 1; + tx_weights = ones(1,size(LAtx,2)); + end +end + % ========================================================================= %% Ka-band @@ -1156,6 +1462,82 @@ % ========================================================================= %% Radar Depth Sounder % ========================================================================= +if any(strcmpi(param.season_name,{'2022_Greenland_X6'})) && strcmpi(radar_name,'rds') + % X,Y,Z are in aircraft coordinates relative to GPS antenna + % Undetermined, temporarily set to zeros + LArx = [0 0 0].'; + LAtx = [0 0 0].'; + if ~exist('rxchannel','var') || isempty(rxchannel) + rxchannel = 1; + end + + if rxchannel == 0 + rxchannel = 1; + tx_weights = ones(1,size(LAtx,2)); + end +end + +if any(strcmpi(param.season_name,{'2022_Greenland_Vapor'})) && strcmpi(radar_name,'rds') + % X,Y,Z are in aircraft coordinates relative to GPS antenna + % Undetermined, temporarily set to zeros + LArx = [0 0 0].'; + LAtx = [0 0 0].'; + if ~exist('rxchannel','var') || isempty(rxchannel) + rxchannel = 1; + end + + if rxchannel == 0 + rxchannel = 1; + tx_weights = ones(1,size(LAtx,2)); + end +end + +if (strcmpi(param.season_name,'2022_Antarctica_BaslerMKB') && strcmpi(radar_name,'rds')) + % Platform: Airborne Radar Kenn Borek Air Basler call sign MKB (COLDEX 1, COLDEX 2, UTIG) + % These values need to be updated with actual values. + + % Measurements, X,Y,Z are in aircraft coordinates, not IMU coordinates + LArx = []; + LArx(1,1:2) = [-1.5 -1.5]; % GUESS + LArx(2,1:2) = [-9 9]; % GUESS + LArx(3,1:2) = [1.5 1.5]; % GUESS + warning('This file needs to be updated with actual values for 2022.'); + + LAtx = [-1.5; 0; 1.5]; + + if ~exist('rxchannel','var') || isempty(rxchannel) + rxchannel = 1:2; + end + + % Amplitude (not power) weightings for transmit side. + if rxchannel == 0 + rxchannel = 1; + tx_weights = ones(1,size(LAtx,2)); + end +end + +if (strcmpi(param.season_name,'2022_Antarctica_GroundGHOST') && strcmpi(radar_name,'rds')) + % Sled antennas GHOST 1 + % + % Primary GPS antenna: GPS positions are relative to primary which is in the center of the radar antenna array. + % Secondary GPS antenna: 6*30 = 180" (WILD GUESS!!!!) right of the primary. Align information will be relative to this. + + % GPS Antenna to Antenna phase center + LArx = [zeros(1,6); + 28*[0 1 2 3 4 5]; + zeros(1,6)] * 2.54/100; + + LAtx = LArx(:,:); + + if ~exist('rxchannel','var') || isempty(rxchannel) + rxchannel = 4; + end + + if rxchannel == 0 + rxchannel = 4; + tx_weights = ones(1,size(LAtx,2)); + end +end if any(strcmpi(param.season_name,{'2019_Antarctica_GV'})) ... && strcmpi(radar_name,'rds') @@ -1315,16 +1697,18 @@ if (strcmpi(param.season_name,'2019_Antarctica_Ground') && strcmpi(radar_name,'rds')) % Sled antennas % Center elements left to right - LArx = [0 -64.4623 10.5 - 0 -46.0371 10.5 - 0 -27.6119 10.5 - 0 -9.1867 10.5 - 0 9.2337 10.5 - 0 27.6637 10.5 - 0 46.0889 10.5 - 0 64.5141 10.5].' * 2.54/100; - LAtx = LArx(:,[1 2 7 8]); + % GPS Antenna to Antenna ports (top side of antenna glass to bottom center of GPS antenna) + LArx = [-18.475 -119.94 14.596 + -18.475 -101.57 14.596 + -18.475 -83.24 14.596 + -18.475 -64.87 14.596 + -18.475 -46.54 14.596 + -18.475 -28.17 14.596 + -18.475 -9.84 14.596 + -18.475 8.53 14.596].' * 2.54/100; + + LAtx = LArx(:,[1 7 2 8]); if ~exist('rxchannel','var') || isempty(rxchannel) rxchannel = 4; @@ -1336,6 +1720,36 @@ end end +if (strcmpi(param.season_name,'2022_Antarctica_Ground') && strcmpi(radar_name,'rds')) + % Sled antennas + % Center elements left to right + + % GPS Antenna to Antenna ports (top side of antenna glass to bottom center of GPS antenna) + LArx = [0 -83.24 0 % along-track polarization/H-polarization + 0 -64.87 0 + 0 -46.54 0 + 0 -28.17 0 + 0 -9.84 0 + 0 8.53 0 + 0 -83.24 0 % cross-track polarization/V-polarization + 0 -64.87 0 + 0 -46.54 0 + 0 -28.17 0 + 0 -9.84 0 + 0 8.53 0].' * 2.54/100; + + LAtx = LArx(:,:); + + if ~exist('rxchannel','var') || isempty(rxchannel) + rxchannel = 3; + end + + if rxchannel == 0 + rxchannel = 3; + tx_weights = ones(1,size(LAtx,2)); + end +end + if (strcmpi(param.season_name,'2016_Greenland_TOdtu') && strcmpi(radar_name,'rds')) % X,Y,Z are in aircraft coordinates relative to GPS antenna LArx(1,:) = [-110.2]*2.54/100; @@ -1379,7 +1793,10 @@ end % Only for 24ch configuration -if (any(strcmpi(param.season_name,{'2015_Greenland_Polar6','2016_Greenland_Polar6','2017_Antarctica_Polar6'})) && strcmpi(radar_name,'rds')) +if (any(strcmpi(param.season_name,{'2015_Greenland_Polar6', ... + '2016_Greenland_Polar6', ... + '2017_Antarctica_Polar6', ... + })) && strcmpi(radar_name,'rds')) % See notes in GPS section % Center elements left to right @@ -1430,7 +1847,11 @@ end % Only for 8ch configuration -if (any(strcmpi(param.season_name,{'2018_Greenland_Polar6','2019_Antarctica_Polar6'})) && strcmpi(radar_name,'rds')) +if (any(strcmpi(param.season_name,{'2018_Greenland_Polar6', ... + '2019_Antarctica_Polar6', ... + '2021_Greenland_Polar5', ... + '2022_Greenland_Polar5', ... + })) && strcmpi(radar_name,'rds')) % See notes in GPS section % Center elements left to right @@ -1514,7 +1935,9 @@ end if (strcmpi(param.season_name,'2017_Antarctica_Basler') && strcmpi(radar_name,'rds')) - % See notes in GPS section + % These values need to be updated with actual values. They appear to not + % be using the total station survey values that Craig from ATM provided + % and the offsets from those that Richard Hale provided. % Measurements, X,Y,Z are in aircraft coordinates, not IMU coordinates LArx(1,1:8) = 1.5859; @@ -1535,72 +1958,8 @@ end end -if (strcmpi(param.season_name,'2019_Greenland_P3') && strcmpi(radar_name,'rds')) - % IMPORTANT NOTE: - % - % CHANNELS 4 and 5 WERE SWAPPED. - % - % This lever arm is the same as all the other P3 OIB campaigns except - % that channels 4 and 5 are swapped. - - % Offsets from the ground plane (based on the CAD model) - if 1 - XYZ_offset = ... - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0; ... - 1.8, 0.4, 1.5, 0, -1.5, -0.4, -1.8, 2, 2, 2.1, 2.1, -2.1, -2.1, -2, -2; ... - -15.4, -14.8, -16.5, -13.3, -16.5, -14.8, -15.4, -16.7, -16.7, -16.7, -16.7, -16.7, -16.7, -16.7, -16.7]; - else - XYZ_offset = zeros(3,15); - end - - % Center elements left to right (in inches) - LArx(:,1) = [-587.7 -88.6 -72.8]; - LArx(:,2) = [-587.7 -58.7 -71]; - LArx(:,3) = [-587.7 -30.4 -69.2]; - LArx(:,4) = [-587.7 0 -68.1]; - LArx(:,5) = [-587.7 30.4 -69.2]; - LArx(:,6) = [-587.7 58.7 -71]; - LArx(:,7) = [-587.7 88.6 -72.8]; - % Left outer elements, left to right (in inches) - LArx(:,8) = [-586.3 -549.2 -128.7]; - LArx(:,9) = [-586.3 -520.6 -125.2]; - LArx(:,10) = [-586.3 -491.2 -121.6]; - LArx(:,11) = [-586.3 -462.2 -118.1]; - % Right outer elements, left to right (in inches) - LArx(:,12) = [-586.3 462.2 -118.1]; - LArx(:,13) = [-586.3 491.2 -121.6]; - LArx(:,14) = [-586.3 520.6 -125.2]; - LArx(:,15) = [-586.3 549.2 -128.7]; - - % Add offsets from ground plane - LArx = LArx + XYZ_offset; - - % Convert to meters units and add gps trajectory position - LArx(1,:) = LArx(1,:)*0.0254 - gps.x; - LArx(2,:) = LArx(2,:)*0.0254 - gps.y; - LArx(3,:) = LArx(3,:)*0.0254 - gps.z; - - % SWAP CHANNELS - LArx = LArx(:,[1 2 3 5 4 6 7]); - - LAtx = LArx(:,1:7); - - % TEST EQUALIZATION -% LAtx(2,1:3) = LAtx(2,1:3)-30*0.0254; -% LAtx(2,4:7) = LAtx(2,4:7)+30*0.0254; - - if ~exist('rxchannel','var') || isempty(rxchannel) - rxchannel = 1:15; - end - - % Amplitude (not power) weightings for transmit side. - if rxchannel == 0 - rxchannel = 4; - tx_weights = ones(1,size(LAtx,2)); - end -end - -if (strcmpi(param.season_name,'2018_Greenland_P3') && strcmpi(radar_name,'rds')) ... +if (strcmpi(param.season_name,'2019_Greenland_P3') && strcmpi(radar_name,'rds')) ... + || (strcmpi(param.season_name,'2018_Greenland_P3') && strcmpi(radar_name,'rds')) ... || (strcmpi(param.season_name,'2017_Antarctica_P3') && strcmpi(radar_name,'rds')) ... || (strcmpi(param.season_name,'2017_Greenland_P3') && strcmpi(radar_name,'rds')) ... || (strcmpi(param.season_name,'2014_Greenland_P3') && strcmpi(radar_name,'rds')) ... @@ -1959,6 +2318,62 @@ % ========================================================================= %% Snow Radar % ========================================================================= +if any(strcmpi(param.season_name,{'2021_Arctic_Vanilla'})) && strcmpi(radar_name,'snow') + % X,Y,Z are in aircraft coordinates relative to GPS antenna + % Undetermined, temporarily set to zeros + LArx = [0 0 0].'; + LAtx = [0 0 0].'; + + if ~exist('rxchannel','var') || isempty(rxchannel) + rxchannel = 1; + end + + % Amplitude (not power) weightings for transmit side. + if rxchannel == 0 + rxchannel = 1; + tx_weights = ones(1,size(LAtx,2)); + end +end + +if any(strcmpi(param.season_name,{'2019_SouthDakota_N1KU'})) ... + && strcmpi(radar_name,'snow') + % X,Y,Z are in aircraft coordinates relative to GPS antenna + LArx = [0.3827 -1.2155 -0.9425].'; + + LAtx = [0.3771 1.7367 -0.9409].'; + + if ~exist('rxchannel','var') || isempty(rxchannel) + rxchannel = 1; + end + + % Amplitude (not power) weightings for transmit side. + if rxchannel == 0 + rxchannel = 1; + tx_weights = ones(1,size(LAtx,2)); + end +end + +if any(strcmpi(param.season_name,{'2020_SouthDakota_N1KU'})) ... + && strcmpi(radar_name,'snow') + % X,Y,Z are in aircraft coordinates relative to GPS antenna + % + LAtx = [0.5486 0.2423 -1.6352].'; + + LArx = [0.2952 -2.1810 0.7222].'; + LArx(:,2) = [0.3836 -1.6054 1.0586].'; + LArx(:,3) = [0.4637 -1.0808 1.3995].'; + LArx(:,4) = [0.4647 0.5980 1.3891].'; + + if ~exist('rxchannel','var') || isempty(rxchannel) + rxchannel = 1; + end + + % Amplitude (not power) weightings for transmit side. + if rxchannel == 0 + rxchannel = 1; + tx_weights = ones(1,size(LAtx,2)); + end +end if any(strcmpi(param.season_name,{'2019_Arctic_GV','2019_Antarctica_GV'})) ... && strcmpi(radar_name,'snow') @@ -1983,7 +2398,7 @@ end end -if any(strcmpi(param.season_name,{'2018_Alaska_SO','2019_Alaska_SO'})) ... +if any(strcmpi(param.season_name,{'2018_Alaska_SO','2021_Alaska_SO'})) ... && strcmpi(radar_name,'snow') % X,Y,Z are in aircraft coordinates relative to GPS antenna LArx(1,1) = -0.288; @@ -2026,7 +2441,14 @@ end end -if (any(strcmpi(param.season_name,{'2015_Greenland_Polar6','2016_Greenland_Polar6','2017_Arctic_Polar5','2018_Greenland_Polar6','2019_Arctic_Polar6','2020_Arctic_Polar6'})) && strcmpi(radar_name,'snow')) +if (any(strcmpi(param.season_name,{ ... + '2015_Greenland_Polar6', ... + '2016_Greenland_Polar6', ... + '2017_Arctic_Polar5', ... + '2018_Greenland_Polar6', ... + '2019_Arctic_Polar6', ... + '2020_Arctic_Polar6', ... + '2022_Antarctica_Polar5'})) && strcmpi(radar_name,'snow')) % See notes in GPS section LArx(1,1:2) = -[95.5 95.5]*2.54/100; @@ -2143,6 +2565,40 @@ end end +if (strcmpi(param.season_name,'2022_Greenland_P3') && strcmpi(radar_name,'snow')) + % See initial GPS entry for notes. + % + % General layout (forward antennas first): + % Tx1 TxHV Rx3 Rx4 Tx2 + % Rx1 Rx2 RxHV Rx5 Rx6 + + % Vivaldi Rx (left to right) followed by rx horn antenna V/H + % Vivaldi antennas 3 and 4 are in the front row, all other receive + % antennas are in the back row. + % rx_path order: + % [Rx1 Rx2 Rx3 Rx4 Rx5 Rx6 RxV RxH] + LArx(1,:) = [-386.1 -386.1 -354.0 -354.0 -386.1 -386.1 -391.2 -391.2]*0.0254 - gps.x; % m + LArx(2,:) = [ -19.4 -13.7 -0.1 4.9 9.2 16.8 -2.4 -2.4]*0.0254 - gps.y; % m + LArx(3,:) = [ -68.2 -68.2 -68.5 -68.5 -68.2 -68.2 -72.6 -72.6]*0.0254 - gps.z; % m + + % Vivaldi Tx (left to right) followed by tx horn antenna V/H + % All transmitters are in the front row + % tx_path order: + % [Tx1 Tx2 TxV TxH] + LAtx(1,:) = [-354.0 -354.0 -359.2 -359.2]*0.0254 - gps.x; % m + LAtx(2,:) = [ -19.4 20.0 -12.3 -12.3]*0.0254 - gps.y; % m + LAtx(3,:) = [ -68.5 -68.5 -72.9 -72.9]*0.0254 - gps.z; % m + + if ~exist('rxchannel','var') || isempty(rxchannel) + rxchannel = 1; + end + + if rxchannel == 0 + rxchannel = 1; + tx_weights = [1 0 0 0]; + end +end + if (strcmpi(param.season_name,'2019_Greenland_P3') && strcmpi(radar_name,'snow')) ... || (strcmpi(param.season_name,'2018_Greenland_P3') && strcmpi(radar_name,'snow')) ... || (strcmpi(param.season_name,'2017_Antarctica_P3') && strcmpi(radar_name,'snow')) ... @@ -2192,6 +2648,15 @@ LAtx(1,:) = [335.6]*0.0254; % m LAtx(2,:) = [18.9]*0.0254; % m LAtx(3,:) = [146]*0.0254; % m + + if ~exist('rxchannel','var') || isempty(rxchannel) + rxchannel = 1; + end + + if rxchannel == 0 + rxchannel = 1; + tx_weights = ones(1,size(LAtx,2)); + end end if (strcmpi(param.season_name,'2010_Antarctica_DC8') && strcmpi(radar_name,'snow')) ... diff --git a/cresis-toolbox/processing/load_data.m b/cresis-toolbox/processing/load_data.m index 223a0b0e..d79312bb 100644 --- a/cresis-toolbox/processing/load_data.m +++ b/cresis-toolbox/processing/load_data.m @@ -12,7 +12,6 @@ % .radar_name = name of radar string % .season_name = name of season string % .day_seg = day-segment string -% .records_fn = filename of records file % .recs = records to load (one indexed) % .imgs = cell vector of images to load, each image is Nx2 array of % wf/adc pairs @@ -58,7 +57,9 @@ % ===================================================================== if ~isfield(param.load_data,'bit_mask') || isempty(param.load_data.bit_mask) - param.load_data.bit_mask = 1; % Only skip records marked as bad in records.bit_mask + % Remove bad records (bit_mask==1), leave stationary records + % (bit_mask==2), and remove bad records (bit_mask==4) + param.load_data.bit_mask = 1 + 4; end if ~isfield(param.load_data,'combine_rx') || isempty(param.load_data.combine_rx) @@ -93,10 +94,17 @@ if ~isfield(param.load_data,'resample') || isempty(param.load_data.resample) param.load_data.resample = [1 1; 1 1]; end -if size(param.load_data.resample,1) == 1 - param.load_data.resample(2,1:2) = [1 1]; +if numel(param.load_data.resample) == 2 + param.load_data.resample = [param.load_data.resample(1) param.load_data.resample(2); 1 1]; end +if ~isfield(param.load_data,'surf_layer') || isempty(param.load_data.surf_layer) + param.load_data.surf_layer.name = 'surface'; + param.load_data.surf_layer.source = 'layerdata'; +end +% Never check for the existence of files +param.load_data.surf_layer.existence_check = false; + if ~isfield(param.load_data,'trim') || isempty(param.load_data.trim) param.load_data.trim = [0 0]; end @@ -113,12 +121,31 @@ %% Load records file % ===================================================================== -records_fn = ct_filename_support(param,'','records'); -records = records_aux_files_read(records_fn,param.load.recs); +records = records_load(param,param.load.recs); +if param.load.recs(2) == inf + param.load.recs(2) = param.load.recs(1) + length(records.gps_time)-1; +end old_param_records = records.param_records; [output_dir,radar_type,radar_name] = ct_output_dir(param.radar_name); +%% Load surface layer +% ========================================================================= +frames = frames_load(param); + +tmp_param = param; +% Determine which frames have the records that are needed +frms = find(param.load.recs(1) >= frames.frame_idxs,1,'last') : find(param.load.recs(2) >= frames.frame_idxs,1,'last'); +tmp_param.cmd.frms = max(1,min(frms)-1) : min(length(frames.frame_idxs),max(frms)+1); +surf_layer = opsLoadLayers(tmp_param,param.load_data.surf_layer); +if isempty(surf_layer.gps_time) || all(isnan(surf_layer.gps_time)) + records.surface = zeros(size(records.gps_time)); +elseif length(surf_layer.gps_time) == 1; + records.surface = surf_layer.twtt*ones(size(records.gps_time)); +else + records.surface = interp_finite(interp1(surf_layer.gps_time,surf_layer.twtt,records.gps_time),0); +end + %% Collect waveform information into one structure % ===================================================================== [wfs,states] = data_load_wfs(param,records); diff --git a/cresis-toolbox/processing/make_layer_files.m b/cresis-toolbox/processing/make_layer_files.m deleted file mode 100644 index 5d307d14..00000000 --- a/cresis-toolbox/processing/make_layer_files.m +++ /dev/null @@ -1,65 +0,0 @@ -function make_layer_files(param,param_override) -% make_layer_files(param,param_override) -% -% Makes layer files for the picker.m program. This should be run -% from run_make_layer_files.m (that script sets up all the control -% variables). -% -% Author: John Paden -% -% See also: run_make_layer_files - -param = merge_structs(param,param_override); - -dbstack_info = dbstack; -fprintf('=====================================================================\n'); -fprintf('%s: %s (%s)\n', dbstack_info(1).name, param.day_seg, datestr(now,'HH:MM:SS')); -fprintf('=====================================================================\n'); - -%% Input Checks - -if ~isfield(param.make_layer_files,'update_mode') || isempty(param.make_layer_files.update_mode) - param.make_layer_files.update_mode = 0; - % In all modes, if file does not exist, then a new file is created. If a - % file exists, how it is updated is controlled by the update_mode field. - % The update_mode field is a nonnegative integer scalar indicating: - % - % 0: no updates are made, layer file is created only if no layer file - % exists or if there were problems with an existing layer file. This is - % the default setting. - % - % 1: layer file is overwritten with new information (blank layer file) - % and old layer data is lost - % - % 2: Old GPS time field is adjusted according to changes in - % param.records.gps.time_offset and then the position and layer - % information is reinterpolated from the old GPS time field to the new - % GPS time field. -end - -if ~isfield(param.make_layer_files,'out_path') || isempty(param.make_layer_files.out_path) - param.make_layer_files.out_path = 'layer'; -end - -%% Make or update layer files -layers = layerdata(param,param.make_layer_files.out_path); -if param.make_layer_files.update_mode == 1 - if isempty(param.cmd.frms) - layers.delete_all(); - else - layers.delete_layer_file(param.cmd.frms); - end -end -if param.make_layer_files.update_mode == 2 - layers.update_gps_all(); -end -layers.check_all(); -if isempty(layers.layer_organizer.lyr_name) - layer_organizer = []; - layer_organizer.lyr_name = {'surface','bottom'}; - layer_organizer.lyr_group_name = {'standard','standard'}; - layers.insert_layers(layer_organizer); - layers.update_layer(1:length(layers.frames.frame_idxs),'surface',layers.frames.gps_time([1 end]),[0 0]); - layers.update_layer(1:length(layers.frames.frame_idxs),'bottom',layers.frames.gps_time([1 end]),[NaN NaN]); -end -layers.save(); diff --git a/cresis-toolbox/processing/mle_cost_function.m b/cresis-toolbox/processing/mle_cost_function.m index e71102d4..b62e369f 100644 --- a/cresis-toolbox/processing/mle_cost_function.m +++ b/cresis-toolbox/processing/mle_cost_function.m @@ -28,11 +28,41 @@ % See also: array_proc.m, mle_compute_cost.m, mle_initialization.m % ========================================================================= -c = 2.997924580003452e+08; % physical_constants too slow -if ~isfield(param,'proj_mtx_update') +%% mle_cost_function: Input checks +% ========================================================================= +if ~isfield(param,'sv_fh') || isempty(param.sv_fh) + param.sv_fh = @array_proc_sv; +end + +if ~isfield(param,'sv_dielectric') || isempty(param.sv_dielectric) + param.sv_dielectric =1; +end + +if ~isfield(param,'lut') || isempty(param.lut) + param.lut = []; +end + +if ~isfield(param,'lut_roll') || isempty(param.lut_roll) + param.lut_roll = []; +end + +if ~isfield(param,'doa_seq') || isempty(param.doa_seq) + param.doa_seq = false; +end + +if ~isfield(param,'apriori') || isempty(param.apriori) + param.apriori.en = false; + param.apriori.theta_range = []; + param.apriori.mean_doa = []; + param.apriori.var_doa = []; +end + +if ~isfield(param,'proj_mtx_update') || isempty(param.proj_mtx_update) param.proj_mtx_update = false; end + +c = 2.997924580003452e+08; % physical_constants too slow % Force theta to be a row vector in preparation for inner product theta = theta(:).'; @@ -42,13 +72,9 @@ % Setup steering vectors for the fixed theta_eval = [param.theta_fixed(:).',theta]; theta_eval = theta_eval(:).'; % make theta have the right dimensions - Nsv2{1} = 'theta'; - Nsv2{2} = theta_eval; - [~,SVs] = array_proc_sv(Nsv2,param.fc,param.y_pc,param.z_pc); -% k = 4*pi*param.fc/c; -% ky = k*sin(theta_eval).'; -% kz = k*cos(theta_eval).'; -% SVs = (1/sqrt(length(param.y_pc)))*exp(1i*(param.y_pc*ky - param.z_pc*kz)); + sv_opt_arg.theta = theta_eval; + sv_arg ={param.fc*param.sv_dielectric, param.y_pc, param.z_pc, sv_opt_arg, param.lut, param.lut_roll}; + [~,SVs] = param.sv_fh(sv_arg{:}); A = SVs(:,1:numel(param.theta_fixed)); % Nc x (Nsrc - 1) C = SVs(:,numel(param.theta_fixed)+1:end); % Nc x 1 (always) Pa = A* inv(A' * A) * A'; % Nc x Nc @@ -57,21 +83,19 @@ L = trace(B'*param.Rxx*B); else - DCM = param.Rxx; + Rxx = param.Rxx; M = param.Nsrc; - Nsv2{1} = 'theta'; - Nsv2{2} = theta; - [~,A] = array_proc_sv(Nsv2,param.fc,param.y_pc,param.z_pc); -% k = 4*pi*param.fc/c; -% A = sqrt(1/length(param.y_pc)) * exp(1i*k*(-param.z_pc*cos(theta) + param.y_pc*sin(theta))); + sv_opt_arg.theta = theta; + sv_arg={param.fc*sqrt(param.sv_dielectric),param.y_pc,param.z_pc, sv_opt_arg, param.lut, param.lut_roll}; + [~,A] = param.sv_fh(sv_arg{:}); Pa = A * inv(A'*A) * A'; if param.doa_seq && param.apriori.en - L = -(M*size(A,1)) * log(abs(sum(sum((eye(size(Pa))-Pa) .* DCM.')))); + L = -(M*size(A,1)) * log(abs(sum(sum((eye(size(Pa))-Pa) .* Rxx.')))); else - L = abs(sum(sum(Pa .* DCM.'))); + L = abs(sum(sum(Pa .* Rxx.'))); end -% L = -abs(trace((eye(size(Pa))-Pa)*DCM)); % Mohanad -% L = abs(trace(Pa*DCM)); % Wax +% L = -abs(trace((eye(size(Pa))-Pa)*Rxx)); % Mohanad +% L = abs(trace(Pa*Rxx)); % Wax end if param.doa_seq && param.apriori.en diff --git a/cresis-toolbox/processing/mle_initialization.m b/cresis-toolbox/processing/mle_initialization.m index 95e5f6a2..9429f72d 100644 --- a/cresis-toolbox/processing/mle_initialization.m +++ b/cresis-toolbox/processing/mle_initialization.m @@ -1,5 +1,5 @@ -function out = mle_initialization(DCM,param) -% out = mle_initialization(DCM,param) +function out = mle_initialization(Rxx,param) +% out = mle_initialization(Rxx,param) % % Function used to initialize maximum likelihood stimator called in % array_proc.m. Uses the alternating projection approach to initialize. @@ -8,7 +8,7 @@ % refine that estimate. % % Inputs: -% DCM = data covariance matrix +% Rxx = data covariance matrix % % param = control struction containing the following fields: % .Nsig = number of sources specified by array_param.Nsrc field, @@ -26,7 +26,7 @@ % .theta = coarse grid used to evaluate cost function. The number of % elements in S.theta is set by the Nsv field in the combine % spreadsheet. S.theta is the FFTSHIFTED theta vector output -% by the array_proc_sv call in array_proc. +% by the array_proc_sv call in array_proc. RADIANS % .search_type = string indicating search type. 'grid' for grid search % (most expensive) or 'ap' for alternating projection search % .theta_guard = double scalar that gives the minimum separation @@ -39,6 +39,37 @@ % % See Also: array_proc.m, mle_cost_function.m, mle_compute_cost.cpp % ========================================================================= + +%% mle_initialization: Input Checks +% ========================================================================= +if ~isfield(param,'sv_dielectric') || isempty(param.sv_dielectric) + param.sv_dielectric = 1; +end + +if ~isfield(param,'sv_fh') || isempty(param.sv_fh) + param.sv_fh = @array_proc_sv; +end + +if ~isfield(param,'lut') || isempty(param.lut) + param.lut = []; +end + +if ~isfield(param,'lut_roll') || isempty(param.lut_roll) + param.lut_roll = []; +end + +if ~isfield(param,'search_type') || isempty(param.search_type) + param.search_type = 'grid'; +end + +if ~isfield(param,'doa_seq') || isempty(param.doa_seq) + param.doa_seq = false; +end + +if ~isfield(param,'apriori') || isempty(param.apriori) + param.apriori.en = false; +end + c = 2.997924580003452e+08; % physical_constants too slow k = 4*pi*param.fc/c; @@ -86,15 +117,17 @@ if good % Evaluate cost function - Nsv2{1} = 'theta'; - Nsv2{2} = theta.'; - [~,A] = array_proc_sv(Nsv2,param.fc,param.y_pc,param.z_pc); + sv_opt_arg.theta = theta.'; + sv_arg = {param.fc*sqrt(param.sv_dielectric),param.y_pc,param.z_pc, sv_opt_arg, param.lut, param.lut_roll}; + [~,A] = param.sv_fh(sv_arg{:}); + +% [~,A] = array_proc_sv(Nsv2,param.fc*param.sv_dielectric,param.y_pc,param.z_pc); % A = sqrt(1/length(param.y_pc)) * exp(1i*k*(-param.z_pc*cos(theta).' + param.y_pc*sin(theta).')); Pa = A * inv(A'*A) * A'; if param.doa_seq && param.apriori.en - J(idx) = -(M*size(A,1)) * log(abs(sum(sum((eye(size(Pa))-Pa) .* DCM.')))); + J(idx) = -(M*size(A,1)) * log(abs(sum(sum((eye(size(Pa))-Pa) .* Rxx.')))); else - J(idx) = abs(sum(sum(Pa .* DCM.'))); + J(idx) = abs(sum(sum(Pa .* Rxx.'))); end % Incorporate the a priori pdf if available @@ -154,11 +187,11 @@ % Initialize first source % ------------------------------------------------------------------------- - B = Cb ./ (repmat(sqrt(sum(abs(Cb).^2,1)),size(DCM,1),1)); + B = Cb ./ (repmat(sqrt(sum(abs(Cb).^2,1)),size(Rxx,1),1)); L = zeros(size(B,2),1); for theta_idx = 1:size(B,2) b = B(:,theta_idx); - L(theta_idx) = abs(b'*DCM*b); + L(theta_idx) = abs(b'*Rxx*b); end [~,max_idx] = max(L); @@ -209,10 +242,13 @@ % Setup steering vectors theta_eval = [out(:).',search_theta]; % search range has same meaning as theta_i in [Ziskind and Wax] - k = 4*pi*param.fc/c; - ky = k*sin(theta_eval); - kz = k*cos(theta_eval); - SVs = sqrt(1/length(param.y_pc)) * exp(1i*(-param.z_pc*kz + param.y_pc*ky)); + sv_opt_arg.theta = theta_eval; + sv_arg = {param.fc*sqrt(param.sv_dielectric),param.y_pc,param.z_pc, sv_opt_arg, param.lut, param.lut_roll}; + [~,SVs] = param.sv_fh(sv_arg{:}); +% k = 4*pi*param.fc/c; +% ky = k*sin(theta_eval); +% kz = k*cos(theta_eval); +% SVs = sqrt(1/length(param.y_pc)) * exp(1i*(-param.z_pc*kz + param.y_pc*ky)); A = SVs(:,1:numel(out)); C = SVs(:,numel(out)+1:end); Pa = A * inv(A'*A) * A'; @@ -220,7 +256,7 @@ B = bsxfun(@rdivide, Cb, sqrt(sum(abs(Cb).^2,1))); % Evaluate likelihood function - L = abs(diag(B'*DCM*B)); + L = abs(diag(B'*Rxx*B)); % Maximize cost function [~,max_idx] = max(L); diff --git a/cresis-toolbox/processing/music_cost_function.m b/cresis-toolbox/processing/music_cost_function.m index 9ef49ad1..831e2475 100644 --- a/cresis-toolbox/processing/music_cost_function.m +++ b/cresis-toolbox/processing/music_cost_function.m @@ -23,19 +23,34 @@ % See also: array_proc.m, music_initialization.m % ========================================================================= +%% music_cost_function: Input checks +% ========================================================================= + +if ~isfield(param,'sv_fh')||isempty(param.sv_fh) + param.sv_fh = @array_proc_sv; +end + +if ~isfield(param,'sv_dielectric') || isempty(param.sv_dielectric) + param.sv_dielectric = 1; +end + +if ~isfield(param,'lut') || isempty(param.lut) + param.lut = []; +end + +if ~isfield(param,'lut_roll') || isempty(param.lut_roll) + param.lut_roll = []; +end -%% Evaluate steering vectors at test value theta +%% music_cost_function: Steering vector evaluation % ========================================================================= % % NOTE: Currently only ideal steering vectors are supported. This could be % improved by passing in LUT through the param structure. c = 2.997924580003452e+08; % physical_constants too slow -k = 4*pi*param.fc/c; -ky = k*sin(theta); -kz = k*cos(theta); -ky = ky(:).'; -kz = kz(:).'; -SV = sqrt(1/length(param.y_pc))*exp(1i*(-param.z_pc*kz + param.y_pc*ky)); +sv_opt_arg.theta = theta; +sv_arg = {param.fc*sqrt(param.sv_dielectric),param.y_pc,param.z_pc, sv_opt_arg, param.lut, param.lut_roll}; +[~,SV] = param.sv_fh(sv_arg{:}); %% Compute Cost % ========================================================================= diff --git a/cresis-toolbox/processing/music_initialization.m b/cresis-toolbox/processing/music_initialization.m index be699d58..96f0801e 100644 --- a/cresis-toolbox/processing/music_initialization.m +++ b/cresis-toolbox/processing/music_initialization.m @@ -29,8 +29,6 @@ % See also: array_proc.m, music_cost_function.m % ========================================================================= -c = 2.997924580003452e+08; % physical_constants too slow - [V,D] = eig(Rxx); eigenVals = diag(D); [eigenVals, eigenValIdxs] = sort(real(eigenVals),'descend'); diff --git a/cresis-toolbox/processing/nan_fir_dec.m b/cresis-toolbox/processing/nan_fir_dec.m index e0323fe6..f09f290d 100644 --- a/cresis-toolbox/processing/nan_fir_dec.m +++ b/cresis-toolbox/processing/nan_fir_dec.m @@ -107,7 +107,7 @@ out_data(:,out_idx) = nansum(bsxfun(@times, data(:,idx_rng(1):idx_rng(2)), Bfilter.*dphase),2); end nan_normalize = sum(Bfilter,2) ./ sum(bsxfun(@times,Bfilter,~isnan(data(:,idx_rng(1):idx_rng(2)))),2); - nan_normalize(~isfinite(nan_normalize) | nan_normalize>=nan_normalize_threshold) = NaN; + nan_normalize(~isfinite(nan_normalize) | nan_normalize>nan_normalize_threshold) = NaN; out_data(:,out_idx) = out_data(:,out_idx) .* nan_normalize; else Bfilter_tmp = Bfilter(filter_rng(1):filter_rng(2)); @@ -121,7 +121,7 @@ Bfilter_tmp.*dphase), 2); end nan_normalize = sum(Bfilter_tmp,2) ./ sum(bsxfun(@times,Bfilter_tmp,~isnan(data(:,idx_rng(1):idx_rng(2)))),2); - nan_normalize(~isfinite(nan_normalize) | nan_normalize>=nan_normalize_threshold) = NaN; + nan_normalize(~isfinite(nan_normalize) | nan_normalize>nan_normalize_threshold) = NaN; out_data(:,out_idx) = out_data(:,out_idx) .* nan_normalize; if renormalize_en diff --git a/cresis-toolbox/processing/phase_lla.m b/cresis-toolbox/processing/phase_lla.m index 9353302a..f68c41a3 100644 --- a/cresis-toolbox/processing/phase_lla.m +++ b/cresis-toolbox/processing/phase_lla.m @@ -90,21 +90,14 @@ -% WGS84 ellipsoid parameters -semimajor = 6378137.0; -e2 = 0.00669437999013; -ellipsoid = [semimajor, sqrt(e2)]; - +physical_constants; % Load WGS84.spheroid % Convert inertials and geodetic lat & lon from degress to radians. phi0 = roll; theta0 = pitch; ci0 = heading; -lat0 = deg2rad(lat); -lon0 = deg2rad(lon); - % Preallocate memory for phase_lat, phase_lon, phase_elev vectors. phase_lat = zeros(1,length(elev)); phase_lon = zeros(1,length(elev)); @@ -128,17 +121,13 @@ PhaseENU = [PhaseEND(2), PhaseEND(1), -1*PhaseEND(3)]; % Calculate Phase Center in terms of ECEF coordinate system. - [PhaseECEF_x, PhaseECEF_y, PhaseECEF_z] = lv2ecef(PhaseENU(1), PhaseENU(2), PhaseENU(3), lat0(ind), lon0(ind), elev(ind), ellipsoid); + [PhaseECEF_x, PhaseECEF_y, PhaseECEF_z] ... + = enu2ecef(PhaseENU(1), PhaseENU(2), PhaseENU(3), lat(ind), lon(ind), elev(ind), WGS84.spheroid); % Calculate Phase Center in terms of geodetic latitude(radians), % geodetic longitude (radians) and elevation (height above the % ellipsoid in meters). - [lat_rad lon_rad phase_elev(ind)] = ecef2geodetic(PhaseECEF_x, PhaseECEF_y, PhaseECEF_z, ellipsoid); - - % Convert Phase Center geodetic latitude and longitude from degrees to - % radians. - phase_lat(ind) = rad2deg(lat_rad); - phase_lon(ind) = rad2deg(lon_rad); + [phase_lat(ind) phase_lon(ind) phase_elev(ind)] = ecef2geodetic(WGS84.spheroid, PhaseECEF_x, PhaseECEF_y, PhaseECEF_z); end return; diff --git a/cresis-toolbox/processing/qlook.m b/cresis-toolbox/processing/qlook.m index 863f4d6a..defd1dac 100644 --- a/cresis-toolbox/processing/qlook.m +++ b/cresis-toolbox/processing/qlook.m @@ -5,10 +5,11 @@ % surface, and (optionally) stores the surface to a layer data destination % (default is layerData). % -% param = struct with processing parameters -% param_override = parameters in this struct will override parameters -% in param. This struct must also contain the gRadar fields. -% Typically global gRadar; param_override = gRadar; +% param: struct with processing parameters +% +% param_override: parameters in this struct will override parameters in +% param. This struct must also contain the gRadar fields. Typically global +% gRadar; param_override = gRadar; % % Example: % See run_qlook.m for how to run this function directly. @@ -21,28 +22,23 @@ %% General Setup % ===================================================================== -param = merge_structs(param, param_override); +if exist('param_override','var') + param = merge_structs(param, param_override); +end fprintf('=====================================================================\n'); fprintf('%s: %s (%s)\n', mfilename, param.day_seg, datestr(now)); fprintf('=====================================================================\n'); +% Get the standard radar name and radar type +[~,radar_type,radar_name] = ct_output_dir(param.radar_name); + %% Input Checks: cmd % ===================================================================== % Remove frames that do not exist from param.cmd.frms list frames = frames_load(param); -if ~isfield(param.cmd,'frms') || isempty(param.cmd.frms) - param.cmd.frms = 1:length(frames.frame_idxs); -end -[valid_frms,keep_idxs] = intersect(param.cmd.frms, 1:length(frames.frame_idxs)); -if length(valid_frms) ~= length(param.cmd.frms) - bad_mask = ones(size(param.cmd.frms)); - bad_mask(keep_idxs) = 0; - warning('Nonexistent frames specified in param.cmd.frms (e.g. frame "%g" is invalid), removing these', ... - param.cmd.frms(find(bad_mask,1))); - param.cmd.frms = valid_frms; -end +param.cmd.frms = frames_param_cmd_frms(param,frames); %% Input Checks: records % ===================================================================== @@ -58,6 +54,12 @@ %% Input Checks: qlook % ===================================================================== +if ~isfield(param.qlook,'bit_mask') || isempty(param.qlook.bit_mask) + % Remove bad records (bit_mask==1), leave stationary records + % (bit_mask==2), and remove bad records (bit_mask==4) + param.qlook.bit_mask = 1 + 4; +end + if ~isfield(param.qlook,'block_size') || isempty(param.qlook.block_size) error('param.qlook.block_size must be specified. This is the number of range lines or records to process at a time.'); end @@ -77,7 +79,7 @@ error('param.qlook.B_filter must be odd length.'); end param.qlook.B_filter = param.qlook.B_filter(:).'; % Must be row vector -if abs(sum(param.qlook.B_filter)-1) > 1e4*eps +if abs(sum(param.qlook.B_filter)-1) > 1e4*eps % Ensure filter weights sum to 1 to preserve radiometry param.qlook.B_filter = param.qlook.B_filter / sum(param.qlook.B_filter); end @@ -98,6 +100,8 @@ error('param.qlook.img_comb not the right length. Since it is not empty, there should be 3 entries for each image combination interface ([Tpd second image for surface saturation, -inf for second image blank, Tpd first image to avoid roll off] is typical).'); end +param = img_combine_input_check(param,'qlook'); + % Incoherent decimation (inc_dec, inc_B_filter) input check % Setting inc_dec = 0: returns coherent data % Setting inc_dec = 1: returns power detected data with no decimation @@ -116,7 +120,7 @@ error('param.qlook.inc_B_filter must be odd length.'); end param.qlook.inc_B_filter = param.qlook.inc_B_filter(:).'; % Must be row vector -if abs(sum(param.qlook.inc_B_filter)-1) > 1e4*eps +if abs(sum(param.qlook.inc_B_filter)-1) > 1e4*eps % Ensure filter weights sum to 1 to preserve radiometry param.qlook.inc_B_filter = param.qlook.inc_B_filter / sum(param.qlook.inc_B_filter); end @@ -130,18 +134,34 @@ [~,out_path_dir] = fileparts(param.qlook.out_path); % nan_fir_dec: if true, function uses the slower nan_fir_dec function on -% the data instead of fir_dec for the dec and inc_dec functions. +% the data instead of fir_dec for the dec and inc_dec functions. Default is +% true for deramp systems and false for non-deramp systems. if ~isfield(param.qlook,'nan_dec') || isempty(param.qlook.nan_dec) - param.qlook.nan_dec = false; + if strcmpi(radar_type,'deramp') + param.qlook.nan_dec = true; + else + param.qlook.nan_dec = false; + end +end + +if ~isfield(param.qlook,'nan_dec_normalize_threshold') || isempty(param.qlook.nan_dec_normalize_threshold) + param.qlook.nan_dec_normalize_threshold = 2; end if ~isfield(param.qlook,'presums') || isempty(param.qlook.presums) param.qlook.presums = 1; end +if ~isfield(param.qlook,'radiometric_corr_dB') || isempty(param.qlook.radiometric_corr_dB) + param.qlook.radiometric_corr_dB = NaN; +end + if ~isfield(param.qlook,'resample') || isempty(param.qlook.resample) param.qlook.resample = [1 1; 1 1]; end +if numel(param.qlook.resample) == 2 + param.qlook.resample = [param.qlook.resample(1) param.qlook.resample(2); 1 1]; +end if ~isfield(param.qlook,'surf') || isempty(param.qlook.surf) param.qlook.surf.en = false; @@ -161,28 +181,26 @@ %% Setup Processing % ===================================================================== -% Get the standard radar name -[~,~,radar_name] = ct_output_dir(param.radar_name); - % Load records file -records_fn = ct_filename_support(param,'','records'); -if ~exist(records_fn) - error('You must run create the records file before running anything else:\n %s', records_fn); -end -records = load(records_fn); +records = records_load(param); % Quick look radar echogram output directory out_fn_dir = ct_filename_out(param, param.qlook.out_path); tmp_out_fn_dir_dir = ct_filename_out(param, param.qlook.out_path,'qlook_tmp'); +%% Collect waveform information into one structure +% - This is used to break the frame up into chunks +% ===================================================================== +[wfs,~] = data_load_wfs(setfield(param,'load',struct('imgs',{param.qlook.imgs})),records); +param.radar.wfs = merge_structs(param.radar.wfs,wfs); + %% Setup cluster % ===================================================================== ctrl = cluster_new_batch(param); cluster_compile({'qlook_task.m','qlook_combine_task.m'},ctrl.cluster.hidden_depend_funs,ctrl.cluster.force_compile,ctrl); total_num_sam = []; -[wfs,~] = data_load_wfs(setfield(param,'load',struct('imgs',{param.qlook.imgs})),records); -if any(strcmpi(radar_name,{'acords','hfrds','hfrds2','mcords','mcords2','mcords3','mcords4','mcords5','mcords6','mcrds','rds','seaice','accum2','accum3'})) +if any(strcmpi(radar_name,{'acords','hfrds','hfrds2','mcords','mcords2','mcords3','mcords4','mcords5','mcords6','mcrds','rds','seaice','accum2','accum3','accum'})) for img = 1:length(param.qlook.imgs) wf = abs(param.qlook.imgs{img}(1,1)); total_num_sam(img) = wfs(wf).Nt_raw; @@ -194,6 +212,11 @@ total_num_sam = 32000 * ones(size(param.qlook.imgs)); cpu_time_mult = 8e-8; mem_mult = 64; + +elseif strcmpi(radar_name,'snow9') + total_num_sam = 45000 * ones(size(param.qlook.imgs)); + cpu_time_mult = 8e-8; + mem_mult = 64; else error('radar_name %s not supported yet.', radar_name); @@ -263,6 +286,11 @@ block_size = param.qlook.block_size(1); blocks = 1:block_size:length(recs)-0.5*block_size; if isempty(blocks) + warning('%s: The frame is smaller than usual. For best results, it is often best to have frames with a number of records that is more than or equal to half the block size (%d), but there are only %d records in this frame. If this is unexpected it may 1) be due to GPS errors since these can create artifically long paths over very short time periods; 2) the frame generation process needs to be revised with run_frames_create to make the frames longer; or 3) the param.qlook.block_size=%d needs to be reduced. qlook will try to process the frame, but it may fail.', mfilename, 0.5*block_size, length(recs), block_size); + if length(recs) < 50 + warning('This frame is unusually short (<50 records). This is usually caused by a too short data segment (in which case it should be marked "do not process" in the param.cmd.notes field and not enabled) or an error in preprocessing, GPS, or records generation. Unless you know the frame is very short (e.g. lab calibration measurement where only a very small amount of data was needed), it is probably best to dbquit and fix the situation. Type "dbcont" to continue and ignore the issue. qlook processing may fail for this frame.'); + keyboard + end blocks = 1; end @@ -335,13 +363,18 @@ dparam.cpu_time = 0; dparam.mem = 250e6; for img = 1:length(param.qlook.imgs) + wf = abs(param.qlook.imgs{img}(1,1)); + adc = abs(param.qlook.imgs{img}(1,2)); dparam.cpu_time = dparam.cpu_time + 10 + Nx*size(param.qlook.imgs{img},1)*total_num_sam(img)*log2(total_num_sam(img))*cpu_time_mult; + data_pulse_compress_mult = 1; if isfield(param.radar.wfs(wf),'deconv') ... && isfield(param.radar.wfs(wf).deconv,'en') && any(param.radar.wfs(wf).deconv.en) - dparam.mem = dparam.mem + Nx*size(param.qlook.imgs{img},1)*total_num_sam(img)*mem_mult*1.7; - else - dparam.mem = dparam.mem + Nx*size(param.qlook.imgs{img},1)*total_num_sam(img)*mem_mult; + data_pulse_compress_mult = data_pulse_compress_mult + 0.7; + end + if strcmpi(param.radar.wfs(wf).coh_noise_method,'analysis') + data_pulse_compress_mult = data_pulse_compress_mult + 0.7; end + dparam.mem = dparam.mem + Nx*size(param.qlook.imgs{img},1)*total_num_sam(img)*mem_mult*data_pulse_compress_mult; end ctrl = cluster_new_task(ctrl,sparam,dparam,'dparam_save',0); @@ -394,7 +427,7 @@ ctrl.cluster.type = 'debug'; end -if any(strcmpi(radar_name,{'acords','hfrds','hfrds2','mcords','mcords2','mcords3','mcords4','mcords5','mcords6','mcrds','rds','seaice','accum2','accum3'})) +if any(strcmpi(radar_name,{'acords','hfrds','hfrds2','mcords','mcords2','mcords3','mcords4','mcords5','mcords6','mcrds','rds','seaice','accum2','accum3','accum'})) cpu_time_mult = 12e-7; mem_mult = 24; @@ -428,14 +461,16 @@ % Account for averaging Nx_max = Nx_max / param.qlook.dec / max(1,param.qlook.inc_dec); Nx = Nx / param.qlook.dec / max(1,param.qlook.inc_dec); + +records_var = whos('records'); for img = 1:length(param.qlook.imgs) sparam.cpu_time = sparam.cpu_time + (Nx*total_num_sam(img)*cpu_time_mult); if isempty(param.qlook.img_comb) % Individual images, so need enough memory to hold the largest image - sparam.mem = max(sparam.mem,250e6 + Nx_max*total_num_sam(img)*mem_mult); + sparam.mem = max(sparam.mem,350e6 + records_var.bytes + Nx_max*total_num_sam(img)*mem_mult); else % Images combined into one so need enough memory to hold all images - sparam.mem = 250e6 + Nx*sum(total_num_sam)*mem_mult; + sparam.mem = 350e6 + records_var.bytes + Nx*sum(total_num_sam)*mem_mult; end end if param.qlook.surf.en diff --git a/cresis-toolbox/processing/qlook_combine_task.m b/cresis-toolbox/processing/qlook_combine_task.m index a8e9efc6..0ad0dec1 100644 --- a/cresis-toolbox/processing/qlook_combine_task.m +++ b/cresis-toolbox/processing/qlook_combine_task.m @@ -19,15 +19,16 @@ % ===================================================================== frames = frames_load(param); - % Load records file -records_fn = ct_filename_support(param,'','records'); -records = load(records_fn); +records = records_load(param); % Quick look radar echogram output directory out_dir = ct_filename_out(param, param.qlook.out_path); tmp_out_dir = ct_filename_out(param, param.qlook.out_path, 'qlook_tmp'); +% Radiometric correction (dB) +radiometric_corr_dB = param.qlook.radiometric_corr_dB; + %% Loop through all the frames: combine and surface track % ===================================================================== [output_dir,radar_type] = ct_output_dir(param.radar_name); @@ -126,7 +127,7 @@ time_vector_changed = false; if block_idx == 1 Time = tmp.Time; - elseif any(size(Time) ~= size(tmp.Time)) || any(Time ~= tmp.Time) + elseif ~isequal(Time,tmp.Time) % Determine the new time axis % Note that even though time axis is aligned with multiples of % dt, there will be rounding errors which need to be dealt with @@ -210,7 +211,7 @@ % Truncate deramp data to nonnegative time if this is going to be the % final combined file if (length(param.qlook.imgs) == 1 || (img == 1 && isempty(param.qlook.img_comb))) ... - && Time(1) < 0 && strcmpi(radar_type,'deramp') + && ~isempty(Time) && Time(1) < 0 && strcmpi(radar_type,'deramp') good_bins = Time>=0; Time = Time(good_bins); Data = Data(good_bins,:); @@ -243,13 +244,17 @@ file_version = '1'; end file_type = 'qlook'; - Data = single(Data); + if isnan(radiometric_corr_dB) + Data = single(Data); + else + Data = single(Data * 10^(radiometric_corr_dB/10)); + end if isempty(custom) - save('-v7.3',out_fn,'Time','Latitude','Longitude', ... + ct_save(out_fn,'Time','Latitude','Longitude', 'radiometric_corr_dB', ... 'Elevation','Roll','Pitch','Heading','GPS_time','Data','Surface', ... 'param_qlook','param_records','file_version','file_type'); else - save('-v7.3',out_fn,'Time','Latitude','Longitude', ... + ct_save(out_fn,'Time','Latitude','Longitude', 'radiometric_corr_dB', ... 'Elevation','Roll','Pitch','Heading','GPS_time','Data','Surface', ... 'param_qlook','param_records','file_version','file_type','custom'); end @@ -265,9 +270,14 @@ [Data_Surface, Time_Surface] = img_combine(img_combine_param, 'qlook', surf_layer); surf_param = param; - surf_param.cmd.frms = frm; - surf_param.layer_tracker.echogram_source = struct('Data',Data_Surface,'Time',Time_Surface,'GPS_time',GPS_time,'Latitude',Latitude,'Longitude',Longitude,'Elevation',Elevation); - Surface = layer_tracker(surf_param,[]); + surf_param.layer_tracker.frms = frm; + surf_param.layer_tracker.echogram_source = struct('Data',Data_Surface,'Time',Time_Surface,'GPS_time',GPS_time,'Latitude',Latitude,'Longitude',Longitude,'Elevation',Elevation,'Roll',Roll); + if length(Time_Surface) < 2 + % This frame has all bad records, so surface tracking cannot be completed. + Surface = nan(size(GPS_time)); + else + Surface = layer_tracker_task(surf_param); + end end %% Save combined image output @@ -294,7 +304,7 @@ % No images were combined, no img_comb_trim needs to be done, % therefore the data will remain unchanged and we can just update the % Surface variable and mark the file_version complete. - save(out_fn,'-append','Surface','file_version'); + ct_save(out_fn,'-append','Surface','file_version'); end else @@ -305,15 +315,16 @@ surf_layer.gps_time = GPS_time; surf_layer.twtt = Surface; [Data, Time] = img_combine(img_combine_param, 'qlook', surf_layer); + file_type = 'qlook'; if isempty(custom) - save('-v7.3',out_fn,'Time','Latitude','Longitude', ... + ct_save(out_fn,'Time','Latitude','Longitude', 'radiometric_corr_dB', ... 'Elevation','Roll','Pitch','Heading','GPS_time','Data','Surface', ... - 'param_qlook','param_records','file_version'); + 'param_qlook','param_records','file_version','file_type'); else - save('-v7.3',out_fn,'Time','Latitude','Longitude', ... + ct_save(out_fn,'Time','Latitude','Longitude', 'radiometric_corr_dB', ... 'Elevation','Roll','Pitch','Heading','GPS_time','Data','Surface', ... - 'param_qlook','param_records','file_version','custom'); + 'param_qlook','param_records','file_version','file_type','custom'); end end @@ -374,12 +385,11 @@ copy_param.layer_source.source = 'echogram'; copy_param.layer_source.echogram_source = param.qlook.out_path; copy_param.layer_source.existence_check = false; + copy_param.layer_source.existence_warning = false; copy_param.layer_source.echogram_source_img = echogram_source_img; copy_param.layer_dest = param.qlook.surf_layer; - if strcmpi(param.qlook.surf_layer.source,'layerdata') - copy_param.layer_dest.echogram_source = param.qlook.out_path; - end + copy_param.layer_dest.group_name = 'standard'; copy_param.layer_dest.existence_check = false; copy_param.layer_dest.echogram_source_img = echogram_source_img; diff --git a/cresis-toolbox/processing/qlook_task.m b/cresis-toolbox/processing/qlook_task.m index 54cc7b11..6471824e 100644 --- a/cresis-toolbox/processing/qlook_task.m +++ b/cresis-toolbox/processing/qlook_task.m @@ -7,7 +7,6 @@ % param = struct controlling the loading, processing, surface tracking, % and quick look generation % .load = structure for which records to load -% .records_fn = filename of records file % .recs = current records % .imgs = cell vector of images to load, each image is Nx2 array of % wf/adc pairs @@ -42,10 +41,14 @@ % .roll_correction = boolean, whether or not to apply roll phase correction % .lever_arm_fh = string containing function name % .elev_correction = boolean, whether or not to apply elevation phase correction -% .B_filter = double vector, FIR filter coefficients to apply before +% .B_filter: double vector, FIR filter coefficients to apply before % decimating, this function loads data before and after this frame -% (if available) to avoid transients at the beginning and end -% .dec = positive integer, decimation rate +% (if available) to avoid transients at the beginning and end of the +% processing block. This filter controls how coherent averaging is done +% (AKA stacking, presumming, or unfocused SAR). Default value is a +% boxcar filter of length equal to the decimation rate or +% ones(1,qlook.dec)/qlook.dec. +% .dec: positive integer, decimation rate % .inc_B_filter: double vector, FIR filter coefficients to apply before % incoherent average decimation. If not defined or empty, then % inc_B_filter is set to ones(1,inc_dec)/inc_dec. @@ -87,7 +90,6 @@ %% Load records file % ========================================================================= -records_fn = ct_filename_support(param,'','records'); % Adjust the load records to account for filtering and decimation. Care is % taken to ensure that when blocks and frames are concatenated together, @@ -116,21 +118,45 @@ % Determine number of records before/after the start/stop output record % that are needed for the filtering start_buffer_ps = (length(param.qlook.inc_B_filter)-1)/2*param.qlook.dec + (length(param.qlook.B_filter)-1)/2; -stop_buffer_ps = (length(param.qlook.inc_B_filter)-1)/2*param.qlook.dec + (length(param.qlook.B_filter)-1)/2; +stop_buffer_ps = start_buffer_ps; + +% If do Doppler spikes nulling, set the default parameters for the equivalent adaptive notch filter +% at the end of data_pulse_compress.m. These parameters include the extra buffers at the start and +% end of the data blocks in terms of a factor of the data block size to avoid bounary artifact +% from fft and ifft transforms, and range bins to filt through, thresholds for Doppler noise and surface signals. +% (see more detailed descriptions in data_pulse_compress.m) +if isfield(param.radar.wfs,'DSN') && param.radar.wfs.DSN.en + if ~isfield(param.radar.wfs.DSN,'rbin_clusters') || isempty(param.radar.wfs.DSN.rbin_clusters) + param.radar.wfs.DSN.rbin_clusters = [1,inf]; + end + if ~isfield(param.radar.wfs.DSN,'theshold') || isempty(param.radar.wfs.DSN.threshold) + param.radar.wfs.DSN.threshold = 10; + end + if ~isfield(param.radar.wfs.DSN,'surf_theshold') || isempty(param.radar.wfs.DSN.threshold) + param.radar.wfs.DSN.surf_threshold = 20; + end + if ~isfield(param.radar.wfs.DSN,'block_overlap_factor') || isempty(param.radar.wfs.DSN.block_overlap_factor) + param.radar.wfs.DSN.block_overlap_factor = 0.1; + end + start_buffer_ps = start_buffer_ps + round(param.radar.wfs.DSN.block_overlap_factor*param.qlook.block_size); + stop_buffer_ps = start_buffer_ps; +end % Adjust start_buffer_ps in case at the beginning of the segment start_buffer_ps = start_buffer_ps - max(0,(1- (output_recs_ps(1) - start_buffer_ps) )); % These are the input records (in presummed record counts) input_recs_ps(1) = output_recs_ps(1) - start_buffer_ps; -input_recs_ps(2) = output_recs_ps(end) + stop_buffer_ps + 1; +% input_recs_ps(2) = output_recs_ps(end) + stop_buffer_ps + 1; +input_recs_ps(2) = output_recs_ps(end) + stop_buffer_ps; % These are the input records in raw record counts param.load.recs(1) = param.qlook.presums * (input_recs_ps(1) - 1) + 1; -param.load.recs(2) = param.qlook.presums * input_recs_ps(2); +% param.load.recs(2) = param.qlook.presums * input_recs_ps(2); +param.load.recs(2) = param.qlook.presums * (input_recs_ps(2)-1) + 1; % Load the records -records = records_aux_files_read(records_fn,param.load.recs); +records = records_load(param,param.load.recs); % Adjust stop_buffer_ps and loaded records in case at the end of the segment stop_buffer_ps = stop_buffer_ps + floor(length(records.gps_time)/param.qlook.presums) - (diff(input_recs_ps)+1); @@ -173,12 +199,11 @@ % ========================================================================= param.load.raw_data = false; param.load.presums = param.qlook.presums; -param.load.bit_mask = 1; % Skip bad records marked in records.bit_mask +param.load.bit_mask = param.qlook.bit_mask; % Skip bad records marked in records.bit_mask [hdr,data] = data_load(param,records,states); param.load.pulse_comp = true; [hdr,data,param] = data_pulse_compress(param,hdr,data); - param.load.motion_comp = param.qlook.motion_comp; param.load.combine_rx = true; [hdr,data] = data_merge_combine(param,hdr,data); @@ -216,9 +241,7 @@ nanmask = isnan(data{img}); data{img}(nanmask) = 0; data{img} = fft(data{img},[],1); - for wf_adc = 1:size(param.load.imgs{img},1) - data{img} = data{img} .* exp(1j*2*pi*freq*relative_td); - end + data{img} = data{img} .* exp(1j*2*pi*freq*relative_td); data{img} = ifft(data{img},[],1); % Relative time delay to reference time delay in time bin units @@ -237,7 +260,7 @@ if param.qlook.nan_dec data{img} = nan_fir_dec(data{img}, param.qlook.B_filter, ... - param.qlook.dec, rline0, Nidxs); + param.qlook.dec, rline0, Nidxs, [], [], param.qlook.nan_dec_normalize_threshold); else data{img} = fir_dec(data{img}, param.qlook.B_filter, ... param.qlook.dec, rline0, Nidxs); @@ -251,9 +274,7 @@ nanmask = isnan(data{img}); data{img}(nanmask) = 0; data{img} = fft(data{img},[],1); - for wf_adc = 1:size(param.load.imgs{img},1) - data{img} = data{img} .* exp(-1j*2*pi*freq*relative_td); - end + data{img} = data{img} .* exp(-1j*2*pi*freq*relative_td); data{img} = ifft(data{img},[],1); % Relative time delay to reference time delay in time bin units @@ -309,6 +330,7 @@ % Remove elevation variations if param.load.motion_comp % Relative time delay to reference time delay in time bin units + dt = hdr.time{img}(2) - hdr.time{img}(1); relative_bins = round( relative_td / dt); % Add zero padding (NaNs) data{img} = [data{img}; nan(zero_pad,size(data{img},2))]; @@ -320,7 +342,7 @@ if param.qlook.nan_dec data{img} = nan_fir_dec(abs(data{img}).^2, param.qlook.inc_B_filter, ... - param.qlook.inc_dec, rline0, Nidxs); + param.qlook.inc_dec, rline0, Nidxs, [], [], param.qlook.nan_dec_normalize_threshold); else data{img} = fir_dec(abs(data{img}).^2, param.qlook.inc_B_filter, ... param.qlook.inc_dec, rline0, Nidxs); diff --git a/cresis-toolbox/processing/radiometric_calibration.m b/cresis-toolbox/processing/radiometric_calibration.m index cdfe4cc3..f1617474 100644 --- a/cresis-toolbox/processing/radiometric_calibration.m +++ b/cresis-toolbox/processing/radiometric_calibration.m @@ -32,15 +32,9 @@ function radiometric_calibration(param, param_override) physical_constants; % Load frames file -load(ct_filename_support(param,'','frames')); +frames = frames_load(param); % Load records file -records_fn = ct_filename_support(param,'','records'); -records_ver = load(records_fn,'ver'); -if isfield(records_ver,'ver') - records = load(records_fn); -else - load(records_fn, 'records'); -end +records = records_load(param); global g_data; g_data = []; diff --git a/cresis-toolbox/processing/records_aux_files_create.m b/cresis-toolbox/processing/records_aux_files_create.m index fdff26fe..4cbcffee 100644 --- a/cresis-toolbox/processing/records_aux_files_create.m +++ b/cresis-toolbox/processing/records_aux_files_create.m @@ -1,4 +1,6 @@ function records_aux_files_create(records_fn,print_flag) +% DEPRECATED +% % records_aux_files_create(records_fn,print_flag) % % Creates the auxilliary files to the main records file. These files @@ -15,35 +17,4 @@ function records_aux_files_create(records_fn,print_flag) % See also: records_aux_files_create, records_aux_files_read, % records_create_sync -if ~exist('print_flag','var') || isempty(print_flag) - print_flag = true; -end - -% ===================================================================== -% Create records netcdf file -% -- This file allows quick loading of specific records rather than -% having to load the entire file. -% ===================================================================== - -[path name] = fileparts(records_fn); -cdf_fn = fullfile(path, sprintf('%s.nc', name)); -if print_flag - fprintf(' Creating file %s\n from %s\n', cdf_fn, records_fn); -end -netcdf_from_mat(cdf_fn,records_fn); - -return; - - -% ===================================================================== -% ===================================================================== -% Examples -% ===================================================================== -% ===================================================================== - -records_path = '/cresis/scratch1/mdce/csarp_support/records/mcords/2009_Antarctica_DC8/'; -records_fns = get_filenames(records_path,'records','','.mat'); - -for file_idx = 1:length(records_fns) - records_aux_files_create(records_fns{file_idx}); -end +error('This function is deprecated. Just use ct_save() since the NetCDF file is no longer used.'); diff --git a/cresis-toolbox/processing/records_aux_files_read.m b/cresis-toolbox/processing/records_aux_files_read.m index e0ba3c75..35215dda 100644 --- a/cresis-toolbox/processing/records_aux_files_read.m +++ b/cresis-toolbox/processing/records_aux_files_read.m @@ -1,4 +1,6 @@ function [records] = records_aux_files_read(records_fn,recs) +% DEPRECATED +% % [records] = records_aux_files_read(records_fn,recs) % % Reads specific fields out of the auxilliary netcdf file for processing. @@ -18,63 +20,4 @@ % See also: records_aux_files_create, records_aux_files_read, % records_create_sync -if ~exist('recs') - recs = [1 inf]; -end - -[path name] = fileparts(records_fn); -cdf_fn = fullfile(path, sprintf('%s.nc', name)); - -finfo = ncinfo(cdf_fn); -num_recs = finfo.Variables(find(strcmp('lat',{finfo.Variables.Name}))).Size(2); -if recs(2) == inf || recs(2) > num_recs; - % Determine number of records and set recs(1) to this - recs(2) = num_recs; -end -if recs(2) < recs(1) - error('Requested records beyond the end of the records file or requested zero records: recs(1)=%d > recs(2)=%d. There are %d records in the records file: %s.', recs(1), recs(2), recs(2), records_fn); -end - -records.lat = ncread(cdf_fn,'lat',[1 recs(1)],[1 recs(2)-recs(1)+1]); -records.lon = ncread(cdf_fn,'lon',[1 recs(1)],[1 recs(2)-recs(1)+1]); -records.elev = ncread(cdf_fn,'elev',[1 recs(1)],[1 recs(2)-recs(1)+1]); -records.roll = ncread(cdf_fn,'roll',[1 recs(1)],[1 recs(2)-recs(1)+1]); -records.pitch = ncread(cdf_fn,'pitch',[1 recs(1)],[1 recs(2)-recs(1)+1]); -records.heading = ncread(cdf_fn,'heading',[1 recs(1)],[1 recs(2)-recs(1)+1]); -records.gps_time = ncread(cdf_fn,'gps_time',[1 recs(1)],[1 recs(2)-recs(1)+1]); -try - records.settings.nyquist_zone = ncread(cdf_fn,'settings(1).nyquist_zone',[1, recs(1)],[1, recs(2)-recs(1)+1]); -end -try - records.settings.nyquist_zone_hw = ncread(cdf_fn,'settings(1).nyquist_zone_hw',[1, recs(1)],[1, recs(2)-recs(1)+1]); -end - -% Get one more record for the offset field (this is helpful when loading -% records with an unknown size). -if recs(2) < num_recs; - recs(2) = recs(2) + 1; -end -records.offset = ncread(cdf_fn,'offset',[1 recs(1)],[inf recs(2)-recs(1)+1]); - -try - % bit 0 is bad record (e.g. corrupt header or data) - % bit 1 is stationary record (e.g. stationary ground collection, used by sar.m/load_data.m and records_bit_mask.m) - records.bit_mask = ncread(cdf_fn,'bit_mask',[1 recs(1)],[inf recs(2)-recs(1)+1]); -catch - % Default is no bit flags set - records.bit_mask = zeros(size(records.offset)); -end - -tmp = netcdf_to_mat(cdf_fn,[],'^gps_source$'); -records.gps_source = tmp.gps_source; - -tmp = netcdf_to_mat(cdf_fn,[],'^relative_rec_num'); -records.relative_rec_num = tmp.relative_rec_num; -tmp = netcdf_to_mat(cdf_fn,[],'^relative_filename'); -records.relative_filename = tmp.relative_filename; - -tmp = netcdf_to_mat(cdf_fn,[],'^param_records'); -records.param_records = tmp.param_records; - -tmp = netcdf_to_mat(cdf_fn,[],'^settings(1\).wfs('); -records.settings.wfs = tmp.settings.wfs; +error('This function is deprecated. Use records_load.m instead because NetCDF file is no longer used.'); diff --git a/cresis-toolbox/processing/records_check.m b/cresis-toolbox/processing/records_check.m index 3eb07299..9ff14584 100644 --- a/cresis-toolbox/processing/records_check.m +++ b/cresis-toolbox/processing/records_check.m @@ -2,6 +2,10 @@ function records_check(param,param_override) % records_check(param,param_override) % % Checks the fields in the records files for potential errors. +% Use run_all_records_check to run on all seasons. +% +% To concatenate and look at all the outputs in Linux: +% grep "^" `find /cresis/snfs1/dataproducts/ct_data/ct_tmp/records_check -iname "*output*" -print | sort` | less % % param = parameter structure indicating which radar and season to load. % This causes the standard parameter spreadsheet filename to be created @@ -12,21 +16,28 @@ function records_check(param,param_override) % % Example: % param = []; -% param.radar_name = 'mcrds'; -% param.season_name = '2009_Antarctica_TO'; +% param.radar_name = 'rds'; +% param.season_name = '2009_Antarctica_DC8'; % records_check(param) % % param = []; -% param.radar_name = 'mcords2'; +% param.radar_name = 'rds'; % param.season_name = '2011_Greenland_P3'; % records_check(param) % % param = []; -% param.radar_name = 'snow3'; +% param.radar_name = 'snow'; % param.season_name = '2013_Antarctica_Basler'; % records_check(param) +% +% Author: John Paden +% +% See also: check_data_products, frames_check, gps_check, records_check +% run_all_frames_check, run_all_gps_check, run_all_records_check + +%% records_check ========================================================== -% Input checking +%% Input checks global gRadar; if exist('param_override','var') param_override = merge_structs(gRadar,param_override); @@ -34,8 +45,10 @@ function records_check(param,param_override) param_override = gRadar; end +old_time = [inf -inf]; if ischar(param) - % param is a string containing the filename + %% param is a string containing the filename + % ======================================================================= records_fn = param; records = load(records_fn,'param_records'); if ~isfield(records,'param_records') @@ -43,16 +56,18 @@ function records_check(param,param_override) end param = merge_structs(records.param_records,param_override); - records_check_support_func(records_fn,param,-inf); + fprintf('%d of %d: Checking records %s\n', 1, 1, records_fn); + records_check_support_func(records_fn,param,old_time); elseif isstruct(param) - % param is a struct indicating which radar/season to check (checks all segments) + %% param is a struct + % ======================================================================= + % struct indicates radar/season to check (checks all segments) [output_dir,radar_type] = ct_output_dir(param.radar_name); param_fn = ct_filename_param(sprintf('%s_param_%s.xls', output_dir, param.season_name)); params = read_param_xls(param_fn); - old_time = [inf -inf]; for param_idx = 1:length(params) param = params(param_idx); @@ -82,13 +97,29 @@ function records_check(param,param_override) end +%% records_check_support_func ============================================= function old_time = records_check_support_func(records_fn,param,old_time) % records_check_support_func(records_fn,param,old_time) % % Support function which does the actual checking of the records +%% input checks +% ========================================================================= + +% Setup records_check output file +command_window_out_fn = ct_filename_ct_tmp(param,'','records_check', 'output.txt'); +command_window_out_fn_dir = fileparts(command_window_out_fn); +if ~exist(command_window_out_fn_dir,'dir') + mkdir(command_window_out_fn_dir); +end +fid = fopen(command_window_out_fn,'wb'); +fprintf(' Console output: %s\n', command_window_out_fn); +fprintf(fid, '%s\n', param.day_seg); + if ~exist(records_fn,'file') warning('Missing records file %s\n', records_fn); + fprintf(fid,' Missing records file %s!!!\n', records_fn); + fclose(fid); return; end @@ -99,7 +130,7 @@ function records_check(param,param_override) % along_track_skip_limit: positive numeric scalar, default 2000, units meters, threshold for % maximum allowed along-track skip if ~isfield(param.records_check,'along_track_skip_limit') || isempty(param.records_check.along_track_skip_limit) - param.records_check.along_track_skip_limit = 20; + param.records_check.along_track_skip_limit = 2000; end % gps_source_check: logical scalar, default true, if true prints non-final GPS source error @@ -138,33 +169,40 @@ function records_check(param,param_override) % threshold is how far the platform can move once it enters a stationary % state and still be considered stationary if ~isfield(param.records_check,'stationary_threshold_alongtrack') || isempty(param.records_check.stationary_threshold_alongtrack) - param.records_check.stationary_threshold_alongtrack = 0.1; + param.records_check.stationary_threshold_alongtrack = [0.1 10]; end -% vel_threshold: postive numeric scalar, default 10, units m/s, threshold +% vel_threshold: positive numeric scalar, default 10, units m/s, threshold % for bad velocity if ~isfield(param.records_check,'vel_threshold') || isempty(param.records_check.vel_threshold) param.records_check.vel_threshold = 10; end -records = load(records_fn); +records = records_load(param); if isfield(records,'records') warning('Old records file format, skipping'); + fprintf(fid,' Old records file format, skipping!!!\n'); + fclose(fid); return; end if param.records_check.gps_source_check && isempty(regexpi(records.gps_source,'final')) warning('GPS source is %s and not final\n', records.gps_source); + fprintf(fid,' GPS source is %s and not final!!!\n', records.gps_source); end +%% mcrds specific checks +% ========================================================================= if strcmpi(records.radar_name,'mcrds') if length(records.raw.wfs) > 1 warning('Waveform headers change during records'); + fprintf(fid,' Waveform headers change during records!!!\n'); end if isfield(records,'adc_phase_corr_deg') - fprintf(' This record has adc_phase_corr_deg field\n'); + warning('This record has adc_phase_corr_deg field'); + fprintf(fid,' This record has adc_phase_corr_deg field!!!\n'); end radar_clock_delta = records.raw.radar_time - records.raw.comp_time; @@ -173,6 +211,7 @@ function records_check(param,param_override) if any(abs(radar_clock_delta) > 0.5) warning('Radar and computer clocks show too large delta'); + fprintf(fid,' Radar and computer clocks show too large delta!!!\n'); plot(radar_clock_delta) keyboard; end @@ -185,21 +224,27 @@ function records_check(param,param_override) end if records.raw.wfs(settings_idx).Waveform(wf).SampleDelay(1) ~= records.raw.wfs(1).Waveform(wf).SampleDelay(1) warning('SampleDelay not consistent'); + fprintf(fid,' SampleDelay not consistent!!!\n'); end if records.raw.wfs(settings_idx).Waveform(wf).NumberSamples(1) ~= records.raw.wfs(1).Waveform(wf).NumberSamples(1) warning('NumberSamples not consistent'); + fprintf(fid,' NumberSamples not consistent!!!\n'); end if param.radar.wfs(wf).Tpd ~= records.raw.wfs(settings_idx).Waveform(wf).PulseDuration warning('%d: Pulse duration does not match',settings_idx); + fprintf(fid,' %d: Pulse duration does not match!!!\n',settings_idx); end if param.radar.wfs(wf).f0 ~= records.raw.wfs(settings_idx).Waveform(wf).StartFrequency warning('%d: Start frequency does not match',settings_idx); + fprintf(fid,' %d: Start frequency does not match!!!\n',settings_idx); end if param.radar.wfs(wf).f1 ~= records.raw.wfs(settings_idx).Waveform(wf).StopFrequency warning('%d: Stop frequency does not match',settings_idx); + fprintf(fid,' %d: Stop frequency does not match!!!\n',settings_idx); end if 20*log10(param.radar.wfs(wf).adc_gains(1)) ~= 70.0-records.raw.wfs(settings_idx).Waveform(wf).RxAttenuation warning('%d: Receiver attenuation does not match',settings_idx); + fprintf(fid,' %d: Receiver attenuation does not match!!!\n',settings_idx); end end end @@ -209,61 +254,96 @@ function records_check(param,param_override) end +%% velocity +% ========================================================================= along_track = geodetic_to_along_track(records.lat,records.lon,records.elev); - vel = diff(along_track) ./ diff(records.gps_time); if param.records_check.ground if any(vel > param.records_check.vel_threshold) warning('Large velocity (%g). All below %g m/s.', min(vel), max(vel), param.records_check.vel_threshold); + fprintf(fid,' Large velocity (%g). All below %g m/s.\n', min(vel), max(vel), param.records_check.vel_threshold); end else - if any(vel < param.records_check.vel_threshold | vel > 300) + if any(vel < param.records_check.vel_threshold) if all(vel < param.records_check.vel_threshold) - warning('Small (%g) or large velocity (%g). All below %g m/s.', min(vel), max(vel), param.records_check.vel_threshold); + warning('Small velocity detected (min %g max %g). All below %g m/s.', min(vel), max(vel), param.records_check.vel_threshold); + fprintf(fid,' Small velocity detected (min %g max %g). All below %g m/s.\n', min(vel), max(vel), param.records_check.vel_threshold); else - warning('Small (%g) or large velocity (%g). Some above %g m/s.', min(vel), max(vel), param.records_check.vel_threshold); + warning('Small velocity detected (min %g max %g). Some above %g m/s.', min(vel), max(vel), param.records_check.vel_threshold); + fprintf(fid,' Small velocity detected (min %g max %g). Some above %g m/s.\n', min(vel), max(vel), param.records_check.vel_threshold); end end + if any(vel > 300) + warning('Large velocity detected (min %g max %g).', min(vel), max(vel)); + fprintf(fid,' Large velocity detected (min %g max %g)!!!\n', min(vel), max(vel)); + end end +%% lat, lon, elev +% ========================================================================= if any(records.lat >= 90 | records.lat <= -90) warning('lat out of bounds'); + fprintf(fid,' lat out of bounds!!!'); end if any(records.lon >= 360 | records.lon <= -360) warning('lon out of bounds'); + fprintf(fid,' lon out of bounds!!!'); end if any(records.elev >= 40000 | records.elev <= -10000) warning('elev out of bounds'); + fprintf(fid,' elev out of bounds!!!'); end +%% roll, pitch, heading +% ========================================================================= if any(records.roll >= param.records_check.roll_limit/180*pi | records.roll <= -param.records_check.roll_limit/180*pi) - warning('roll > %g deg: max %.1f min %.1f', param.records_check.roll_limit, max(records.roll)*180/pi, min(records.roll)*180/pi); + warning('abs(roll) > %g deg: max %.1f min %.1f', param.records_check.roll_limit, max(records.roll)*180/pi, min(records.roll)*180/pi); + fprintf(fid,' abs(roll) > %g deg: max %.1f min %.1f', param.records_check.roll_limit, max(records.roll)*180/pi, min(records.roll)*180/pi); end if any(records.pitch >= 25/180*pi | records.pitch <= -25/180*pi) - warning('pitch > 25 deg: max %.1f min %.1f', max(records.pitch)*180/pi, min(records.pitch)*180/pi); + warning('abs(pitch) > 25 deg: max %.1f min %.1f', max(records.pitch)*180/pi, min(records.pitch)*180/pi); + fprintf(fid,' abs(pitch) > 25 deg: max %.1f min %.1f', max(records.pitch)*180/pi, min(records.pitch)*180/pi); end if any(records.heading >= 2*pi | records.heading <= -2*pi) - warning('heading out of bounds'); + warning('abs(heading) > 2*pi'); + fprintf(fid,' abs(heading) > 2*pi'); end +%% gps_time +% ========================================================================= + if old_time(2) > records.gps_time(1) warning('records out of order (meaning that the previous segment has a gps time (%s to %s) that is greater than the start of this segment (%s to %s), but the current segment is listed later in the parameters and segments must be listed chronologically)', ... datestr(epoch_to_datenum(old_time(1))), datestr(epoch_to_datenum(old_time(end))), ... datestr(epoch_to_datenum(records.gps_time(1))), datestr(epoch_to_datenum(records.gps_time(end)))); + fprintf(fid,' records out of order (meaning that the previous segment has a gps time (%s to %s) that is greater than the start of this segment (%s to %s), but the current segment is listed later in the parameters and segments must be listed chronologically)!!!\n', ... + datestr(epoch_to_datenum(old_time(1))), datestr(epoch_to_datenum(old_time(end))), ... + datestr(epoch_to_datenum(records.gps_time(1))), datestr(epoch_to_datenum(records.gps_time(end)))); end +% old_time: output argument used to ensure segments are in chronological order +old_time = records.gps_time([1 end]); + nonmonotonic_records = diff(records.gps_time) < 0; if any(nonmonotonic_records) warning('time not monotonically increasing: First non-monotonic record %d of %d total, %d total records.', ... find(nonmonotonic_records,1), sum(nonmonotonic_records), length(records.gps_time)); + fprintf(fid,' time not monotonically increasing: First non-monotonic record %d of %d total, %d total records!!!', ... + find(nonmonotonic_records,1), sum(nonmonotonic_records), length(records.gps_time)); +end + +if any(isnan(records.gps_time)) + warning('NaN in records.gps_time'); + fprintf(fid,' NaN in records.gps_time!!!'); end if any(isnan(records.lat) | isnan(records.lon) | isnan(records.elev) | isnan(records.roll) | isnan(records.pitch) | isnan(records.heading)) - warning('NaN in record'); + warning('NaN in record trajectory/attitude fields'); + fprintf(fid,' NaN in record trajectory/attitude fields!!!'); end diff_time = diff(records.gps_time); @@ -276,21 +356,40 @@ function records_check(param,param_override) fprintf(' Gap is at file index %d to %d of %d to %d files\n', find(records.relative_rec_num{1} > jump_idx,1), ... find(records.relative_rec_num{1} > jump_idx+1,1), 1, length(records.relative_rec_num{1})); end + fprintf(fid,' Time gap greater than limit %.1f sec, max skip found is %.1f sec!!!', ... + param.records_check.gps_time_skip_limit, max(diff_time)); + fprintf(fid,' Time gaps in seconds: '); fprintf(fid,'%f\t', diff_time(jump_idxs)); fprintf(fid,'\n'); + for jump_idx = jump_idxs + fprintf(fid,' Gap is at file index %d to %d of %d to %d files\n', find(records.relative_rec_num{1} > jump_idx,1), ... + find(records.relative_rec_num{1} > jump_idx+1,1), 1, length(records.relative_rec_num{1})); + end end +%% along_track gaps +% ========================================================================= diff_along_track = diff(along_track); if any(diff_along_track > param.records_check.along_track_skip_limit) jump_idxs = find(diff_along_track > param.records_check.along_track_skip_limit); - warning('Along-track gap greater than limit %.1f km, max skip found is is %.1f km', ... + warning('Along-track gap greater than limit %.1f m, max skip found is %.1f m', ... param.records_check.along_track_skip_limit, max(diff_along_track)); - fprintf('Along-track gaps in seconds: '); fprintf('%f\t', diff_along_track(jump_idxs)); fprintf('\n'); + fprintf('Along-track gaps: '); fprintf('%f\t', diff_along_track(jump_idxs)); fprintf('\n'); for jump_idx = jump_idxs fprintf(' Gap is at file index %d to %d of %d to %d files\n', find(records.relative_rec_num{1} > jump_idx,1), ... find(records.relative_rec_num{1} > jump_idx+1,1), 1, length(records.relative_rec_num{1})); end + fprintf(fid,' Along-track gap greater than limit %.1f m, max skip found is %.1f m!!!', ... + param.records_check.along_track_skip_limit, max(diff_along_track)); + fprintf(fid,' Along-track gaps: '); fprintf(fid,'%f\t', diff_along_track(jump_idxs)); fprintf(fid,'\n'); + for jump_idx = jump_idxs + fprintf(fid,' Gap is at file index %d to %d of %d to %d files\n', find(records.relative_rec_num{1} > jump_idx,1), ... + find(records.relative_rec_num{1} > jump_idx+1,1), 1, length(records.relative_rec_num{1})); + end end -% Regenerate without elevation +%% stationary data +% ========================================================================= +% along_track: generate without elevation since elevation errors can +% accumulate and the platform can appear to be moving when it is not along_track = geodetic_to_along_track(records.lat,records.lon); rec = 1; cur_along_track = along_track(rec); @@ -328,8 +427,16 @@ function records_check(param,param_override) datestr(epoch_to_datenum(stationary_list(idx).gps_time(1))), ... datestr(epoch_to_datenum(stationary_list(idx).gps_time(end)))); end + fprintf(fid,' Stationary data exists.'); + for idx = 1:length(stationary_list) + fprintf(fid,' Stationary %.1f sec from %s to %s\n', stationary_list(idx).duration, ... + datestr(epoch_to_datenum(stationary_list(idx).gps_time(1))), ... + datestr(epoch_to_datenum(stationary_list(idx).gps_time(end)))); + end end -old_time = records.gps_time([1 end]); +%% cleanup +% ========================================================================= +fclose(fid); end diff --git a/cresis-toolbox/processing/records_create.m b/cresis-toolbox/processing/records_create.m index 5a27affd..5e306dd4 100644 --- a/cresis-toolbox/processing/records_create.m +++ b/cresis-toolbox/processing/records_create.m @@ -11,17 +11,11 @@ function records_create(param,param_override) % in the support directories for the specific radar in .mat form for % quicker access. % -% Output file contains: -% hdr: structure with the following fields -% .utc_time_sod: vector of corrected utc times -% .seconds: vector -% .fraction: vector -% % Inputs: -% param = struct with processing parameters +% param: struct with processing parameters % -- OR -- % function handle to script with processing parameters -% param_override = parameters in this struct will override parameters +% param_override: parameters in this struct will override parameters % in param. This struct must also contain the gRadar fields. % Typically global gRadar; param_override = gRadar; % @@ -43,10 +37,22 @@ function records_create(param,param_override) [output_dir,radar_type,radar_name] = ct_output_dir(param.radar_name); +% param.records.arena: structure controlling records creation of +% arena-based digital systems +if ~isfield(param.records,'arena') + param.records.arena = []; +end + +% param.records.arena.total_presums: override total_presums value that is +% normally read in from the config.xml file +if ~isfield(param.records.arena,'total_presums') + param.records.arena.total_presums = []; +end + % boards: List of subdirectories containing the files for each board (a % board is a data stream stored to disk and often contains the data stream % from multiple ADCs) -if any(param.records.file.version == [1:5 8 11 101:102 405:406 409:411 413 414]) +if any(param.records.file.version == [1:5 8 11 101:102 405:406 409:411 413 414 415 420]) if ~isfield(param.records.file,'boards') || isempty(param.records.file.boards) % Assume a single channel system param.records.file.boards = {''}; @@ -88,7 +94,7 @@ function records_create(param,param_override) % to be sent and discarded (not used for presumming) param.records.presum_mode = 1; elseif any(param.records.file.version == [407 408]) - error('The param.records.presum_mode must be specified. Set to 0 if not using the 8-channel 1 GSPS DDS by Ledford/Leuschen. Set to 1 if it was used.');; + error('The param.records.presum_mode must be specified. If using the new waveform generator (Arena-based), this field should be 0. If using the old waveform generator, this field should be set to 1. The old waveform generator is an 8-channel 1 GSPS DDS by Ledford/Leuschen (not Arena-based).'); else param.records.presum_mode = 0; end @@ -106,10 +112,15 @@ function records_create(param,param_override) end end +% records.frames: structure that controls the frame generation after the +% records file is created. if ~isfield(param.records,'frames') || isempty(param.records.frames) param.records.frames = []; end +% records.frames.mode: 0, 1, or 2. Default is 0. 0: do nothing with frames, +% 1: autogenerate frames if they do not exist and then manually edit the +% frames, 2: autogenerate the frames if ~isfield(param.records.frames,'mode') || isempty(param.records.frames.mode) param.records.frames.mode = 0; end @@ -164,6 +175,12 @@ function records_create(param,param_override) board_hdrs{board_idx}.file_idx = zeros([0 0],'int32'); board_hdrs{board_idx}.offset = zeros([0 0],'int32'); + elseif any(param.records.file.version == [415]) + board_hdrs{board_idx}.file_idx = zeros([0 0],'int32'); + board_hdrs{board_idx}.radar_time = zeros([0 0],'double'); + board_hdrs{board_idx}.comp_time = zeros([0 0],'double'); + board_hdrs{board_idx}.offset = zeros([0 0],'int32'); + else % NI, Rink, Paden, Leuschen, and Ledford systems board_hdrs{board_idx}.seconds = zeros([0 0],'uint32'); @@ -204,6 +221,13 @@ function records_create(param,param_override) fprintf(' %i/%i %s (%s)\n', ... file_idx,length(file_idxs), fn, datestr(now,'HH:MM:SS')); + if any(param.records.file.version == [415]) + % Files may be empty, these are skipped in preprocessing + if dir_info.bytes == 0 + continue + end + end + % Load temporary files tmp_hdr_fn = ct_filename_ct_tmp(rmfield(param,'day_seg'),'','headers', ... fullfile(adc_folder_name, [fn_name '.mat'])); @@ -224,7 +248,12 @@ function records_create(param,param_override) board_hdrs{board_idx}.file_size(cur_idx + (1:length(hdr_tmp.mode_latch))) = dir_info.bytes; board_hdrs{board_idx}.file_idx(cur_idx + (1:length(hdr_tmp.mode_latch))) = file_num; - + + if isfield(hdr_tmp,'profile') + board_hdrs{board_idx}.profile(cur_idx + (1:length(hdr_tmp.mode_latch))) = hdr_tmp.profile; + else + board_hdrs{board_idx}.profile(cur_idx + (1:length(hdr_tmp.mode_latch))) = nan(size(hdr_tmp.mode_latch)); + end board_hdrs{board_idx}.mode_latch(cur_idx + (1:length(hdr_tmp.mode_latch))) = hdr_tmp.mode_latch; board_hdrs{board_idx}.offset(cur_idx + (1:length(hdr_tmp.mode_latch))) = hdr_tmp.offset; board_hdrs{board_idx}.subchannel(cur_idx + (1:length(hdr_tmp.mode_latch))) = hdr_tmp.subchannel; @@ -242,6 +271,14 @@ function records_create(param,param_override) board_hdrs{board_idx}.offset(end+1:end+length(hdr_tmp.gps_time)) = 1:length(hdr_tmp.gps_time); wfs = hdr_tmp.wfs; + elseif any(param.records.file.version == [415]) + % UTIG RDS 60 MHZ MARFA/HICARS + board_hdrs{board_idx}.radar_time(end+1:end+length(hdr_tmp.radar_time)) = hdr_tmp.radar_time; + board_hdrs{board_idx}.comp_time(end+1:end+length(hdr_tmp.radar_time)) = datenum_to_epoch(hdr_tmp.comp_time); + board_hdrs{board_idx}.offset(end+1:end+length(hdr_tmp.radar_time)) = int32(hdr_tmp.offset); + board_hdrs{board_idx}.file_idx(end+1:end+length(hdr_tmp.radar_time)) = file_num; + wfs = []; + else % NI, Rink, Paden, Leuschen, and Ledford systems if isempty(hdr_tmp.seconds) @@ -255,7 +292,7 @@ function records_create(param,param_override) board_hdrs{board_idx}.file_idx(end+1:end+length(hdr_tmp.seconds)) = file_num; board_hdrs{board_idx}.offset(end+1:end+length(hdr_tmp.seconds)) = int32(hdr_tmp.offset); - if any(param.records.file.version == [1:8 11 102 401:404 407:408]) + if any(param.records.file.version == [1:8 11 102 401:404 407:408 420]) % Ledford, Rink and NI systems have EPRI field board_hdrs{board_idx}.epri(end+1:end+length(hdr_tmp.seconds)) = hdr_tmp.epri; end @@ -285,7 +322,7 @@ function records_create(param,param_override) % Drop the last record of the last file since it is generally not a % complete record and there is no additional file to load which % contains the remainder of the record. - if any(param.records.file.version == [1:8 11 102 401:404 407:408]) + if any(param.records.file.version == [1:8 11 102 401:404 407:408 420]) board_hdrs{board_idx}.epri = board_hdrs{board_idx}.epri(1:end-1); end if param.records.file.version == 8 @@ -318,7 +355,7 @@ function records_create(param,param_override) % ===================================================================== if size(param.records.data_map{board_idx},2) == 4 % No Profile Processor Digital System (use mode_latch,subchannel instead) - % Each row of param.records.data_map{board_idx} = [mode_latch channel wf adc] + % Each row of param.records.data_map{board_idx} = [mode_latch subchannel wf adc] % Get the first row (which is always the EPRI row) epri_mode = param.records.data_map{board_idx}(1,1); @@ -326,12 +363,15 @@ function records_create(param,param_override) mask = board_hdrs{board_idx}.mode_latch == epri_mode & board_hdrs{board_idx}.subchannel == epri_subchannel; else - error('Profile mode not supported.'); % Profile Processor Digital System - % Each row of param.records.data_map{board_idx} = [profile wf adc] + % Each row of param.records.data_map{board_idx} = [profile mode_latch subchannel wf adc] + + % Get the first row (which is always the EPRI row) epri_profile = param.records.data_map{board_idx}(1,1); + epri_mode = param.records.data_map{board_idx}(1,2); + epri_subchannel = param.records.data_map{board_idx}(1,3); - mask = profile == epri_profile; + mask = board_hdrs{board_idx}.profile == epri_profile; end % Data before first PPS reset may be incorrectly time tagged so we do not % use it @@ -339,16 +379,20 @@ function records_create(param,param_override) if board_hdrs{board_idx}.pps_ftime_cntr_latch(first_reset) > 10e6 mask(1:first_reset) = 0; end - epri_pris = board_hdrs{board_idx}.profile_cntr_latch(mask); - + %% Correct EPRI/Arena: Find EPRI jumps and mask out % ===================================================================== - jump_idxs = find( abs(diff(double(epri_pris))/configs.total_presums - 1) > 0.1); + if isempty(param.records.arena.total_presums) + total_presums = configs.total_presums; + else + total_presums = param.records.arena.total_presums; + end + jump_idxs = find( abs(diff(double(epri_pris))/total_presums - 1) > 0.1); bad_mask = zeros(size(epri_pris)); for jump_idx = jump_idxs - jump = (epri_pris(jump_idx+1)-epri_pris(jump_idx))/configs.total_presums - 1; + jump = (epri_pris(jump_idx+1)-epri_pris(jump_idx))/total_presums - 1; fprintf('jump_idx: %d, jump: %d\n', jump_idx, jump); fprintf('epri_pris: %d to %d\n', epri_pris(jump_idx), epri_pris(jump_idx+1)); if jump < -0.1 @@ -396,15 +440,33 @@ function records_create(param,param_override) % ===================================================================== mask = zeros(size(board_hdrs{board_idx}.pps_cntr_latch)); mask(epri_pri_idxs) = 1; + found_board = false; for idx = 1:length(epri_pri_idxs)-1 %if ~mod(idx-1,1000000) % fprintf('%d\n', idx); %end for pri_idx = epri_pri_idxs(idx):epri_pri_idxs(idx+1)-1 + + if ~found_board + for configs_board_idx = 1:size(configs.adc,1) + if isfield(configs.adc{configs_board_idx,board_hdrs{board_idx}.mode_latch(pri_idx)+1,board_hdrs{board_idx}.subchannel(pri_idx)+1},'name') ... + && strcmpi(configs.adc{configs_board_idx,board_hdrs{board_idx}.mode_latch(pri_idx)+1,board_hdrs{board_idx}.subchannel(pri_idx)+1}.name, boards{board_idx}); + found_board = true; + break; + end + end + if ~found_board + error('boards(%d)=%s not found in configs file\n', board_idx, boards(board_idx)); + end + end + + if mod(board_hdrs{board_idx}.mode_latch(pri_idx),2) + board_hdrs{board_idx}.mode_latch(pri_idx)=board_hdrs{board_idx}.mode_latch(pri_idx)-1; + end if board_hdrs{board_idx}.mode_latch(pri_idx) >= size(configs.adc,2) ... || board_hdrs{board_idx}.subchannel(pri_idx) >= size(configs.adc,3) ... - || ~isfield(configs.adc{board_idx,board_hdrs{board_idx}.mode_latch(pri_idx)+1,board_hdrs{board_idx}.subchannel(pri_idx)+1},'rg') ... - || isempty(configs.adc{board_idx,board_hdrs{board_idx}.mode_latch(pri_idx)+1,board_hdrs{board_idx}.subchannel(pri_idx)+1}.rg) + || ~isfield(configs.adc{configs_board_idx,board_hdrs{board_idx}.mode_latch(pri_idx)+1,board_hdrs{board_idx}.subchannel(pri_idx)+1},'rg') ... + || isempty(configs.adc{configs_board_idx,board_hdrs{board_idx}.mode_latch(pri_idx)+1,board_hdrs{board_idx}.subchannel(pri_idx)+1}.rg) fprintf('Bad record %d\n', pri_idx); mask(pri_idx) = 0; break; @@ -454,15 +516,42 @@ function records_create(param,param_override) board = boards(board_idx); for map_idx = 1:size(param.records.data_map{board_idx},1) - wf = param.records.data_map{board_idx}(map_idx,3); - % adc = param.records.data_map{board_idx}(map_idx,4); - mode_latch = param.records.data_map{board_idx}(map_idx,1); - subchannel = param.records.data_map{board_idx}(map_idx,2); + if size(param.records.data_map{board_idx},2) == 4 + % No Profile Processor Digital System (use mode_latch,subchannel instead) + % Each row of param.records.data_map{board_idx} = [mode_latch channel wf adc] + + wf = param.records.data_map{board_idx}(map_idx,3); + % adc = param.records.data_map{board_idx}(map_idx,4); + mode_latch = param.records.data_map{board_idx}(map_idx,1); + subchannel = param.records.data_map{board_idx}(map_idx,2); + + else + % Profile mode + % Each row of param.records.data_map{board_idx} = [profile mode_latch channel wf adc] + + wf = param.records.data_map{board_idx}(map_idx,4); + % adc = param.records.data_map{board_idx}(map_idx,5); + mode_latch = param.records.data_map{board_idx}(map_idx,2); + subchannel = param.records.data_map{board_idx}(map_idx,3); + profile = param.records.data_map{board_idx}(map_idx,1); + end - wfs(wf).num_sam = configs.adc{board_idx,mode_latch+1,subchannel+1}.num_sam; + found_board = false; + for configs_board_idx = 1:size(configs.adc,1) + if isfield(configs.adc{configs_board_idx,mode_latch+1,subchannel+1},'name') ... + && strcmpi(configs.adc{configs_board_idx,mode_latch+1,subchannel+1}.name, boards{board_idx}); + found_board = true; + break; + end + end + if ~found_board + error('boards(%d)=%s not found in configs file\n', board_idx, boards(board_idx)); + end + + wfs(wf).num_sam = configs.adc{configs_board_idx,mode_latch+1,subchannel+1}.num_sam; wfs(wf).bit_shifts = param.radar.wfs(wf).bit_shifts; wfs(wf).t0 = param.radar.wfs(wf).Tadc; - wfs(wf).presums = configs.adc{board_idx,mode_latch+1,subchannel+1}.presums; + wfs(wf).presums = configs.adc{configs_board_idx,mode_latch+1,subchannel+1}.presums; end end @@ -470,6 +559,9 @@ function records_create(param,param_override) % UTUA RDS systems % BAS RDS systems +elseif any(param.records.file.version == [415]) + % UTIG RDS systems + else % NI, Rink, Paden, Leuschen, and Ledford systems @@ -481,15 +573,20 @@ function records_create(param,param_override) %% Correct time, sync GPS data, and save records records_create_sync; +%% Create reference trajectory file +if param.records.gps.en + records_reference_trajectory(param); +end + %% Create frames % param.records.frames.mode == 0: Do nothing -if param.records.frames.mode == 1 +if param.records.frames.mode == 1 % Autogenerate if needed and then manually edit frames_fn = ct_filename_support(param,'','frames'); if ~exist(frames_fn,'file') frames_autogenerate(param,param_override); end obj = frames_create(param,param_override); -elseif param.records.frames.mode >= 2 +elseif param.records.frames.mode >= 2 % Autogenerate the frames file frames_autogenerate(param,param_override); end diff --git a/cresis-toolbox/processing/records_create_epri_estimate.m b/cresis-toolbox/processing/records_create_epri_estimate.m index 04a4d59e..4fa6edf7 100644 --- a/cresis-toolbox/processing/records_create_epri_estimate.m +++ b/cresis-toolbox/processing/records_create_epri_estimate.m @@ -11,7 +11,7 @@ % .vectors.gps.utc_time_halved % .radar.fs % .radar_name -% .records.presum_bug_fixed +% .records.presum_mode % file_idxs: indexes into fns filename list corresponding to this segment % fns: cell vector of filenames % adc_folder_name: only required for using temporary files @@ -22,8 +22,8 @@ % % Author: John Paden -if ~isfield(param.records,'presum_bug_fixed') || isempty(param.records.presum_bug_fixed) - param.records.presum_bug_fixed = false; +if ~isfield(param.records,'presum_mode') || isempty(param.records.presum_mode) + param.records.presum_mode = 1; end [output_dir,radar_type,radar_name] = ct_output_dir(param.radar_name); @@ -118,7 +118,7 @@ struct('clk',param.radar.fs/4,'first_byte',first_byte)); elseif strcmp(radar_name,'mcords5') [first_file,tmp] = basic_load_mcords5(fns{file_idxs(init_EPRI_file_idx)}, ... - struct('clk',param.radar.fs,'first_byte',first_byte,'presum_bug_fixed',param.records.presum_bug_fixed)); + struct('clk',param.radar.fs,'first_byte',first_byte,'presum_mode',param.records.presum_mode)); % Check for the temporary file [~,fn_name] = fileparts(fns{file_idxs(init_EPRI_file_idx)}); diff --git a/cresis-toolbox/processing/records_create_sync.m b/cresis-toolbox/processing/records_create_sync.m index 6462f8fd..02feb0f3 100644 --- a/cresis-toolbox/processing/records_create_sync.m +++ b/cresis-toolbox/processing/records_create_sync.m @@ -131,7 +131,8 @@ offsets = 0:diff_epri(board_idx); for offset_idx = 1:length(offsets) offset = offsets(offset_idx); - offset_scores{board_idx}(offset_idx) = sum(mod((epri_pri_idxs - offset)/diff_epri(board_idx),1) ~= 0); + GUARD = 4; + offset_scores{board_idx}(offset_idx) = sum(abs(mod((epri_pri_idxs - offset)/diff_epri(board_idx),1)) > GUARD); if offset_scores{board_idx}(offset_idx) < min_score min_score = offset_scores{board_idx}(offset_idx); best_offset = offset; @@ -148,7 +149,7 @@ if min_score > 0 warning('%d of %d records show slipped EPRI values.', min_score, length(epri_pri_idxs)); if length(good_offsets{board_idx}) > 1 - warning('An ADC link error probably occured since PRI-numbers do not fall a regular interval. Special generation of the EPRI vector is now enabled.') + warning('An ADC link error probably occured or incorrect pulse sequence radar settings were used since PRI-numbers do not follow a regular interval. Special generation of the EPRI vector is now enabled.') special_epri_generation = true; end end @@ -201,7 +202,24 @@ records.raw.profile_cntr_latch = nan(size(epri)); records.raw.rel_time_cntr_latch = nan(size(epri)); for board_idx = 1:length(boards) - [~,out_idxs,in_idxs] = intersect(epri,board_hdrs{board_idx}.profile_cntr_latch); + if GUARD == 0 + [~,out_idxs,in_idxs] = intersect(epri,board_hdrs{board_idx}.profile_cntr_latch); + else + out_idxs = nan(size(epri)); + in_idxs = nan(size(epri)); + out_idx = 0; + for idx = 1:length(epri) + [offset,min_idx] = min(abs(epri(idx) - board_hdrs{board_idx}.profile_cntr_latch)); + if offset < GUARD + out_idx = out_idx + 1; + out_idxs(out_idx) = idx; + in_idxs(out_idx) = min_idx; + end + end + out_idxs = out_idxs(1:out_idx); + in_idxs = in_idxs(1:out_idx); + end + fprintf('Board %d is missing %d of %d records.\n', board_idx, length(epri)-length(out_idxs), length(epri)); % offset: Missing records filled in with -2^31 @@ -278,6 +296,9 @@ %% Align/UTUA RDS: Nothing required %% Align/BAS RDS: Nothing required +elseif any(param.records.file.version == [415]) + %% Align/UTIG RDS MARFA/HICARS: Nothing required + else %% Align/CReSIS: Create output EPRI vector min_epri = inf; @@ -337,7 +358,7 @@ records.raw.seconds = nan(size(epri)); records.raw.fraction = nan(size(epri)); if param.records.file.version == 8 - records.settings.nyquist_zone = nan(size(epri)); + records.settings.nyquist_zone_sig = nan(size(epri)); records.settings.waveform_ID = nan(size(epri)); end for board_idx = 1:length(boards) @@ -358,11 +379,18 @@ % Time stamps are assumed to be the same from each board so each board % just writes all of its time stamps to the output records fields. records.raw.epri(out_idxs) = board_hdrs{board_idx}.epri(in_idxs); - records.raw.seconds(out_idxs) = board_hdrs{board_idx}.seconds(in_idxs) ... - + max(param.records.gps.time_offset) - param.records.gps.time_offset(board_idx); + if board_idx > 1 && length(param.records.gps.time_offset) == 1 + % Old parameter spreadsheet format only contained a single entry for + % all boards in param.records.gps.time_offset + records.raw.seconds(out_idxs) = board_hdrs{board_idx}.seconds(in_idxs) ... + + max(param.records.gps.time_offset) - param.records.gps.time_offset; + else + records.raw.seconds(out_idxs) = board_hdrs{board_idx}.seconds(in_idxs) ... + + max(param.records.gps.time_offset) - param.records.gps.time_offset(board_idx); + end records.raw.fraction(out_idxs) = board_hdrs{board_idx}.fraction(in_idxs); if param.records.file.version == 8 - records.settings.nyquist_zone(out_idxs) = board_hdrs{board_idx}.nyquist_zone(in_idxs); + records.settings.nyquist_zone_sig(out_idxs) = board_hdrs{board_idx}.nyquist_zone(in_idxs); records.settings.waveform_ID(out_idxs) = board_hdrs{board_idx}.waveform_ID(in_idxs); end end @@ -411,7 +439,14 @@ radar_time = board_hdrs{board_idx}.gps_time; comp_time = []; -elseif any(param.records.file.version == [1 2 3 4 5 6 7 8 11 101 403 407 408]) +elseif any(param.records.file.version == [415]) + %% Radar time: UTIG RDS + % Nothing to be done + board_idx = 1; + radar_time = board_hdrs{board_idx}.radar_time; + comp_time = board_hdrs{board_idx}.comp_time; + +elseif any(param.records.file.version == [1 2 3 4 5 6 7 8 11 101 403 407 408 420]) %% Radar time: CReSIS if 0 @@ -496,8 +531,6 @@ %% Correlate GPS with radar data % =================================================================== -fprintf('Loading GPS data (%s)\n', datestr(now)); - if param.records.gps.en records = records_create_sync_gps(param,records,radar_time,comp_time); @@ -533,7 +566,6 @@ % .heading % .gps_time % .gps_source -records.surface = NaN*zeros(size(records.lat)); records.relative_filename = []; records.relative_rec_num = []; records.offset = []; @@ -546,6 +578,7 @@ end records.offset(board_idx,:) = board_hdrs{board_idx}.offset; end +records.bit_mask = zeros(size(records.offset),'uint8'); records.radar_name = param.radar_name; if param.ct_file_lock records.file_version = '1L'; @@ -566,10 +599,3 @@ fprintf('Saving records file %s (%s)\n',records_fn,datestr(now)); ct_file_lock_check(records_fn,3); ct_save(records_fn,'-v7.3','-struct','records'); - -%% Create record aux files for faster loading times -% ===================================================================== - -fprintf('Creating auxiliary records files %s (%s)\n',records_fn,datestr(now)); -records_aux_files_create(records_fn); - diff --git a/cresis-toolbox/processing/records_reference_trajectory.m b/cresis-toolbox/processing/records_reference_trajectory.m new file mode 100644 index 00000000..dd656a97 --- /dev/null +++ b/cresis-toolbox/processing/records_reference_trajectory.m @@ -0,0 +1,65 @@ +function records_reference_trajectory(param,param_override) +% records_reference_trajectory(param,param_override) +% +% Function for creating reference trajectory files (ref_YYYYMMDD_SS.mat) in +% CSARP_reference_trajectory. +% +% The reference trajectory files can then be loaded with +% records_reference_trajectory_load.m. +% +% Inputs: +% param: struct with processing parameters +% -- OR -- +% function handle to script with processing parameters +% param_override: parameters in this struct will override parameters +% in param. This struct must also contain the gRadar fields. +% Typically global gRadar; param_override = gRadar; +% +% Authors: John Paden +% +% See also: records_reference_trajectory.m, +% records_reference_trajectory_load.m, run_records_reference_trajectory.m, +% run_all_records_reference_trajectory.m + +%% General Setup +% ===================================================================== +if exist('param_override','var') + param = merge_structs(param, param_override); +end + +fprintf('=====================================================================\n'); +fprintf('%s: %s (%s)\n', mfilename, param.day_seg, datestr(now)); +fprintf('=====================================================================\n'); + +%% Load records +% ===================================================================== + +fprintf('Loading records\t%s\n', datestr(now,'yyyymmdd_HHMMSS')); +records = records_load(param); + +%% Create reference trajectory using records and lever arm +% ===================================================================== + +fprintf('Creating reference trajectory\t%s\n', datestr(now,'yyyymmdd_HHMMSS')); +% Create reference trajectory (rx_path == 0, tx_weights = []). Update +% the records field with this information. +trajectory_param = struct('gps_source',records.gps_source, ... + 'season_name',param.season_name,'radar_name',param.radar_name,'rx_path', 0, ... + 'tx_weights', [], 'lever_arm_fh', param.radar.lever_arm_fh); +[records,lever_arm_val] = trajectory_with_leverarm(records,trajectory_param); + +%% Save output +% ===================================================================== + +ref_fn_dir = ct_filename_out(param,'reference_trajectory','',1); +if ~exist(ref_fn_dir,'dir') + mkdir(ref_fn_dir); +end +ref_fn = fullfile(ref_fn_dir,sprintf('ref_%s.mat', param.day_seg)); +fprintf('Saving reference trajectory\t%s\t%s\n', ref_fn, datestr(now,'yyyymmdd_HHMMSS')); +records.lever_arm_val = lever_arm_val; +records.lever_arm_fh = param.radar.lever_arm_fh; +records.file_version = '1'; +records.file_type = 'reference_trajectory'; +records.sw_version = current_software_version; +ct_save(ref_fn,'-struct','records','gps_time','lat','lon','elev','gps_source','lever_arm_val','lever_arm_fh','file_version','file_type','sw_version'); diff --git a/cresis-toolbox/processing/records_reference_trajectory_load.m b/cresis-toolbox/processing/records_reference_trajectory_load.m new file mode 100644 index 00000000..26ee83f5 --- /dev/null +++ b/cresis-toolbox/processing/records_reference_trajectory_load.m @@ -0,0 +1,70 @@ +function ref = records_reference_trajectory_load(param,records) +% ref = records_reference_trajectory_load(param,records) +% +% Loads the reference trajectory using the input records structure. First +% it checks to see if the reference trajectory file exists in +% CSARP_reference_trajectory/ref_YYYYMMDD_SS.mat and loads from there if +% the file exists. If the file does not exist, then the reference +% trajectory is created. +% +% The reference trajectory is the trajectory of the sensor's nominal phase +% center location (usually the center antenna element). +% +% param: radar parameter structure usually read from read_param_xls.m, only +% the file paths will be used. +% +% records: loaded records file usually from records_load.m. If this +% argument is not passed in, then the records will be loaded and the +% reference trajectory fields will be added to it. +% +% ref: structure containing the reference phase center trajectory for the +% radar. These fields will be added to the records structure that is passed +% in. Fields are the same as records file but are the reference antenna +% phase center: gps_time, lat, lon, elev. +% +% Author: John Paden +% +% See also: records_reference_trajectory.m, +% records_reference_trajectory_load.m, run_records_reference_trajectory.m, +% run_all_records_reference_trajectory.m + +% If records file not passed in, then load it +if ~exist('records','var') || isempty(records) + records = records_load(param); +end + +% Check to see if reference trajectory file exists +ref_fn_dir = ct_filename_out(param,'reference_trajectory','',1); +ref_fn = fullfile(ref_fn_dir,sprintf('ref_%s.mat', param.day_seg)); +if exist(ref_fn,'file') + % If reference trajectory file exists, load and use it (this is much + % faster usually than creating a new reference trajectory) + ref_traj = load(ref_fn); + if strcmp(ref_traj.gps_source,records.gps_source) + if ~isempty(param.radar.lever_arm_fh) + trajectory_param = struct('gps_source',records.gps_source, ... + 'season_name',param.season_name,'radar_name',param.radar_name,'rx_path', 0, ... + 'tx_weights', [], 'lever_arm_fh', param.radar.lever_arm_fh); + [lever_arm_val] = param.radar.lever_arm_fh(trajectory_param, [], 0); + end + if isempty(param.radar.lever_arm_fh) || sum(abs(ref_traj.lever_arm_val-lever_arm_val)) < 1e-6 + ref = records; + ref.lat = ref_traj.lat; + ref.lon = ref_traj.lon; + ref.elev = ref_traj.elev; + ref.param_records.radar.lever_arm_fh = ref_traj.lever_arm_fh; + return; + else + end + else + warning('records_reference_trajectory_load:gps_source','Reference trajectory file exists, but gps_source does not match. Run records_reference_trajectory.m to recreate: %s', ref_fn); + end +end + +% No usable reference trajectory file, so we have to create the reference +% trajectory. +trajectory_param = struct('gps_source',records.gps_source, ... + 'season_name',param.season_name,'radar_name',param.radar_name,'rx_path', 0, ... + 'tx_weights', [], 'lever_arm_fh', param.radar.lever_arm_fh); +ref = trajectory_with_leverarm(records,trajectory_param); +ref.param_records.radar.lever_arm_fh = param.radar.lever_arm_fh; diff --git a/cresis-toolbox/processing/records_remove_zero_blocks.m b/cresis-toolbox/processing/records_remove_zero_blocks.m index f6333884..c685cf97 100644 --- a/cresis-toolbox/processing/records_remove_zero_blocks.m +++ b/cresis-toolbox/processing/records_remove_zero_blocks.m @@ -26,8 +26,7 @@ zero_blocks_fn = ct_filename_support(param,'','zero_blocks'); zero_blocks = load(zero_blocks_fn); - records_fn = ct_filename_support(param,'','records'); - records = load(records_fn); + records = records_load(param); records_mask = zeros(size(records.lat)); frames_fn = ct_filename_support(param,'','frames'); @@ -94,7 +93,6 @@ end movefile(records_fn,'/tmp/records/'); save(records_fn,'-struct','records'); - records_aux_files_create(records_fn); if ~exist('/tmp/frames/','dir') mkdir('/tmp/frames/'); diff --git a/cresis-toolbox/processing/relative_rec_num_to_file_idx_vector.m b/cresis-toolbox/processing/relative_rec_num_to_file_idx_vector.m index 3af0aa0c..15fa067b 100644 --- a/cresis-toolbox/processing/relative_rec_num_to_file_idx_vector.m +++ b/cresis-toolbox/processing/relative_rec_num_to_file_idx_vector.m @@ -6,9 +6,9 @@ file_idx = zeros(size(recs)); cur_file_idx = 1; -for rec_idx = 1:(recs(end)-recs(1)+1) +for rec_idx = 1:length(recs) while cur_file_idx+1 <= length(relative_rec_num) ... - && relative_rec_num(cur_file_idx+1) <= recs(1)+rec_idx-1 + && relative_rec_num(cur_file_idx+1) <= recs(rec_idx) cur_file_idx = cur_file_idx + 1; end file_idx(rec_idx) = cur_file_idx; diff --git a/cresis-toolbox/processing/run_all_frames_check.m b/cresis-toolbox/processing/run_all_frames_check.m new file mode 100644 index 00000000..f29d1205 --- /dev/null +++ b/cresis-toolbox/processing/run_all_frames_check.m @@ -0,0 +1,16 @@ +% script run_all_frames_check +% +% Script for running records check on many seasons at once. +% +% Author: John Paden +% +% See also: check_data_products, frames_check, gps_check, records_check +% run_all_frames_check, run_all_gps_check, run_all_records_check + +run_all; + +for param_fns_idx = 1:length(param_fns) + param_fn = param_fns{param_fns_idx}; + param = regexp(param_fn,'(?\w+)_param_(?\w+)','names'); + frames_check(param); +end diff --git a/cresis-toolbox/processing/run_all_records_check.m b/cresis-toolbox/processing/run_all_records_check.m new file mode 100644 index 00000000..c5d4eb54 --- /dev/null +++ b/cresis-toolbox/processing/run_all_records_check.m @@ -0,0 +1,16 @@ +% script run_all_records_check +% +% Script for running records check on many seasons at once. +% +% Author: John Paden +% +% See also: check_data_products, frames_check, gps_check, records_check +% run_all_frames_check, run_all_gps_check, run_all_records_check + +run_all; + +for param_fns_idx = 1:length(param_fns) + param_fn = param_fns{param_fns_idx}; + param = regexp(param_fn,'(?\w+)_param_(?\w+)','names'); + records_check(param); +end diff --git a/cresis-toolbox/processing/run_all_records_reference_trajectory.m b/cresis-toolbox/processing/run_all_records_reference_trajectory.m new file mode 100644 index 00000000..ca3d3aec --- /dev/null +++ b/cresis-toolbox/processing/run_all_records_reference_trajectory.m @@ -0,0 +1,103 @@ +% script run_all_records_reference_trajectory +% +% Script for running records_reference_trajectory on many seasons at once. +% +% Author: John Paden +% +% See also: records_reference_trajectory.m, +% records_reference_trajectory_load.m, run_records_reference_trajectory.m, +% run_all_records_reference_trajectory.m + +%% User Setup +% ===================================================================== + +param_override = []; + +param_override.records_reference_trajectory.force_update = false; + +run_all; + +%% Automated Section +% ===================================================================== + +% Merge gRadar into param_override +global gRadar; +if exist('param_override','var') + param_override = merge_structs(gRadar,param_override); +else + param_override = gRadar; +end + +%% Seasons +for param_fns_idx = 1:length(param_fns) + %% Seasons: Load each season + param_fn = ct_filename_param(param_fns{param_fns_idx}); + params = read_param_xls(param_fn); + + if 1 + % Select all segments + params = ct_set_params(params,'cmd.generic',1); + params = ct_set_params(params,'cmd.generic',0,'cmd.notes','do not process'); + else + % Debug: Run a specific segment + params = ct_set_params(params,'cmd.generic',0); + params = ct_set_params(params,'cmd.generic',1,'day_seg','20120330_04'); + end + + %% Seasons: Segments + for param_idx = 1:length(params) + param = params(param_idx); + if isfield(param.cmd,'generic') && ~iscell(param.cmd.generic) && ~ischar(param.cmd.generic) && param.cmd.generic + %% Seasons: Segments: Input Checks + % ===================================================================== + + param = merge_structs(param, param_override); + + if ~isfield(param,'records_reference_trajectory') || isempty(param.records_reference_trajectory) + param.records_reference_trajectory = []; + end + + if ~isfield(param.records_reference_trajectory,'force_update') || isempty(param.records_reference_trajectory.force_update) + param.records_reference_trajectory.force_update = true; + end + + %% Seasons: Segments: Decide to create ref trajectory + % Check for the existence of the reference trajectory file + ref_fn_dir = ct_filename_out(param,'reference_trajectory','',1); + ref_fn = fullfile(ref_fn_dir,sprintf('ref_%s.mat', param.day_seg)); + + if param.records_reference_trajectory.force_update || ~exist(ref_fn,'file') + %% Seasons: Segments: Create new reference trajectory + records_reference_trajectory(param,[]); + + else + %% Seasons: Segments: Perform debug operations + + if 1 + fprintf('Reference_trajectory_up_to_date\t%s\t%s\n', ref_fn, datestr(now,'yyyymmdd_HHMMSS')); + elseif 0 + % DEBUG: Ensures lever_arm information in file + mat_vars = whos('-file',ref_fn); + if any(strcmp({mat_vars.name},'lever_arm_fh')) + fprintf('Reference_trajectory_up_to_date\t%s\t%s\n', ref_fn, datestr(now,'yyyymmdd_HHMMSS')); + else + fprintf('Updating reference trajectory\t%s\t%s\n', ref_fn, datestr(now,'yyyymmdd_HHMMSS')); + fprintf(' Loading_records\t%s\n', datestr(now,'yyyymmdd_HHMMSS')); + records = records_load(param,'gps_source'); + + trajectory_param = struct('gps_source',records.gps_source, ... + 'season_name',param.season_name,'radar_name',param.radar_name,'rx_path', 0, ... + 'tx_weights', [], 'lever_arm_fh', param.radar.lever_arm_fh); + + [lever_arm_val] = param.radar.lever_arm_fh(trajectory_param, [], 0); + + records.lever_arm_val = lever_arm_val; + records.lever_arm_fh = param.radar.lever_arm_fh; + fprintf(' Saving_updated_reference_trajectory\t%s\n', datestr(now,'yyyymmdd_HHMMSS')); + ct_save(ref_fn,'-append','-struct','records','lever_arm_val','lever_arm_fh'); + end + end + end + end + end +end diff --git a/cresis-toolbox/processing/run_basic_rx_chan_equalization.m b/cresis-toolbox/processing/run_basic_rx_chan_equalization.m index 0d265548..60964550 100644 --- a/cresis-toolbox/processing/run_basic_rx_chan_equalization.m +++ b/cresis-toolbox/processing/run_basic_rx_chan_equalization.m @@ -10,32 +10,37 @@ %% RDS: MCORDS5 if strcmpi(radar_setup,'MCORDS5') - [param,defaults] = default_radar_params_2018_Greenland_Polar6_mcords; + [param,defaults] = default_radar_params_2022_Greenland_Polar5_rds; % .file_search_mode: Specify how to search for a file: 'last_file', - % 'specific', 'default', or empty to be asked - param.file_search_mode = ''; + % 'specific', 'default', 'segment', 'map', or empty to be asked + param.config.file_search_mode = 'segment'; - % .base_dir_search: cell vector of paths to search for data files - param.base_dir_search = {'D:\awi\','/cresis/snfs1/scratch/2016_Germany_AWI_tests/AWI_ICE_bak/test_flight','/mnt/AWI_SSD0/1604261202/UWB/'}; + % .config.base_dir_search: cell vector of paths to search for data files + param.config.base_dir_search = {'\\192.168.1.100\d\awi','C:\rds\2022_Greenland_Polar5\20220512\','D:\awi\','/cresis/snfs1/scratch/2016_Germany_AWI_tests/AWI_ICE_bak/test_flight','/mnt/AWI_SSD0/1604261202/UWB/'}; - % .img: wf-adc pair list which specifies which waveform-adc pairs to - % analyze + % .config.img: wf-adc pair list which specifies which waveform-adc pairs + % to analyze %wf = 1; adcs = 1:24; ref = 12; % Waveform 1, All ADCs wf = 1; adcs = 1:8; ref = 4; % Waveform 1, Left subarray %wf = 1; adcs = 9:16; ref = 4; % Waveform 1, Center subarray %wf = 1; adcs = 17:24; ref = 4; % Waveform 1, Right subarray - param.img = cat(2,wf*ones(length(adcs),1),adcs.'); param.ref_wf_adc = ref; + param.config.img = cat(2,wf*ones(length(adcs),1),adcs.'); + + % .config.ref_wf_adc: index into config.img wf-adc pair list to use as + % the reference for equalization + param.config.ref_wf_adc = ref; - % .recs: two element vector specifying which records/range-lines to load - % [start_record num_records] - param.recs = [0 inf]; + % .config.recs: two element vector specifying which records/range-lines + % to load [start_record num_records] + param.config.recs = [0 inf]; - % .presums: Number of additional software presums (coherent averaging) to do - param.presums = 20; + % .config.presums: Number of additional software presums (coherent + % averaging) to do + param.config.presums = 20; - % .delay: the method used to calculate delay between the channels - param.delay = struct('method','xcorr_complex','ref_bins',-20:20,'search_bins',-7:7,'Mt',64) + % .config.delay: the method used to calculate delay between the channels + param.config.delay = struct('method','xcorr_complex','ref_bins',-20:20,'search_bins',-7:7,'Mt',64) end %% RDS: MCORDS5_P3 (Accumulation Radar) diff --git a/cresis-toolbox/processing/run_basic_tx_chan_equalization.m b/cresis-toolbox/processing/run_basic_tx_chan_equalization.m index 1427f685..85c7f96a 100644 --- a/cresis-toolbox/processing/run_basic_tx_chan_equalization.m +++ b/cresis-toolbox/processing/run_basic_tx_chan_equalization.m @@ -9,23 +9,24 @@ %% RDS: MCORDS5 if strcmpi(radar_setup,'MCORDS5') - [param,defaults] = default_radar_params_2018_Greenland_Polar6_mcords; + [param,defaults] = default_radar_params_2022_Greenland_Polar5_rds; % .file_search_mode: Specify how to search for a file: 'last_file', % 'specific', 'default', 'segment', 'map', or empty to be asked - param.file_search_mode = 'segment'; - + param.config.file_search_mode = 'segment'; + % .base_dir_search: cell vector of paths to search for data files - param.base_dir_search = {'D:\awi\','/cresis/snfs1/scratch/2016_Germany_AWI_tests/AWI_ICE_bak/test_flight','/mnt/AWI_SSD0/1604261202/UWB/'}; + param.config.base_dir_search = {'\\192.168.1.100\d\awi','C:\rds\2022_Greenland_Polar5\20220512\','D:\awi\','/cresis/snfs1/scratch/2016_Germany_AWI_tests/AWI_ICE_bak/test_flight','/mnt/AWI_SSD0/1604261202/UWB/'}; % out_xml_fn_dir = String containing the directory where the new XML file % will be placed if ispc - param.out_xml_fn_dir = 'C:\waveforms\'; - param.rss_base_dir = 'C:\Users\Administrator\Desktop\Arena_Shared\configs\'; + param.basic_tx_chan_equal.out_xml_fn_dir = 'C:\waveforms\'; + param.basic_tx_chan_equal.arena_base_dir = 'C:\Users\Administrator\Desktop\Arena_Shared\configs\'; + param.basic_tx_chan_equal.arena_base_dir = 'C:\arena_waveforms\'; else - param.out_xml_fn_dir = '~/waveforms/'; - param.rss_base_dir = '~/rss_waveforms/'; + param.basic_tx_chan_equal.out_xml_fn_dir = '~/waveforms/'; + param.basic_tx_chan_equal.arena_base_dir = '~/arena_waveforms/'; end elseif strcmpi(radar_setup,'MCORDS3') diff --git a/cresis-toolbox/processing/run_check_surface.m b/cresis-toolbox/processing/run_check_surface.m index 32ae849b..52c1af49 100644 --- a/cresis-toolbox/processing/run_check_surface.m +++ b/cresis-toolbox/processing/run_check_surface.m @@ -38,3 +38,11 @@ check_surface(param,param_override); end end + +% Output all results as table +checksur_dir = fileparts(ct_filename_ct_tmp(params(1),'','check_surface','*.txt')); +try + system(sprintf('cat %s/*.txt',checksur_dir)) +catch ME + warning('Output all results as table failed. %s', ME.getReport); +end diff --git a/cresis-toolbox/processing/run_collate_burst_noise.m b/cresis-toolbox/processing/run_collate_burst_noise.m new file mode 100644 index 00000000..8e2edfae --- /dev/null +++ b/cresis-toolbox/processing/run_collate_burst_noise.m @@ -0,0 +1,64 @@ +% script run_collate_burst_noise +% +% Runs collate_burst_noise +% +% Authors: John Paden + +%% USER SETTINGS +% ========================================================================= + +param_override = []; + +% params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls'),'',{'analysis_burst','analysis'}); +params = read_param_xls(ct_filename_param('rds_param_2018_Greenland_P3.xls'),'',{'analysis_burst','analysis'}); +% params = read_param_xls(ct_filename_param('rds_param_2019_Greenland_P3.xls'),'',{'analysis_burst','analysis'}); +% params = read_param_xls(ct_filename_param('rds_param_2019_Antarctica_Ground.xls'),'',{'analysis_burst','analysis'}); + +param_override.collate_burst_noise.in_path = 'analysis_burst'; + +% param_override.collate_burst_noise.debug_plots = {}; +param_override.collate_burst_noise.debug_plots = {'bn_plot'}; +% param_override.collate_burst_noise.debug_plots = {'visible','bn_plot'}; % <== CHOOSE to debug + +cmd_method = 'generic'; +rds_settings; + +if 0 + % For debugging, use this to select a specific image and wf_adc to + % collate instead of doing them all + for img=1:6 + param_override.collate_burst_noise.wf_adcs{img} = []; + param_override.collate_burst_noise.wf_adcs{img} = [1 2 3 4 12 13 14 15]; + param_override.collate_burst_noise.wf_adcs{img} = [2 13]; + end + param_override.collate_burst_noise.imgs = [3 4]; +end + +%% Automated Section +% ===================================================================== + +% Input checking +global gRadar; +if exist('param_override','var') + param_override = merge_structs(gRadar,param_override); +else + param_override = gRadar; +end + +% Process each of the segments +for param_idx = 1:length(params) + param = params(param_idx); + if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic + continue; + end + %collate_burst_noise(param,param_override); + collate_burst_noise + if 0 + % Debug code to print out which frames are affected by burst noise + records = records_load(param,'bit_mask','gps_time'); + bad_rec = any(bitand(records.bit_mask,param.collate_burst_noise.bit_mask)); + [~,frm_id,~] = get_frame_id(param,records.gps_time); + param.day_seg + unique(floor(frm_id(bad_rec))) + end +end diff --git a/cresis-toolbox/processing/run_collate_coh_noise.m b/cresis-toolbox/processing/run_collate_coh_noise.m index b65dbec6..4c057983 100644 --- a/cresis-toolbox/processing/run_collate_coh_noise.m +++ b/cresis-toolbox/processing/run_collate_coh_noise.m @@ -17,12 +17,12 @@ if 1 % Near-DC removal - param_override.collate_coh_noise.method = 'firdec'; - param_override.collate_coh_noise.firdec_fcutoff = @(t) 1/30; % Update coherent noise estimate every 30 seconds + param_override.collate_coh_noise.method = {'firdec'}; + param_override.collate_coh_noise.firdec_fcutoff = {@(t) 1/30}; % Update coherent noise estimate every 30 seconds param_override.collate_coh_noise.firdec_fs = 1/7.5; % Should update about 4 times as often as the estimate: 30/4 = 7.5 else % DC removal when dft_corr_time set to inf - param_override.collate_coh_noise.method = 'dft'; + param_override.collate_coh_noise.method = {'dft'}; param_override.collate_coh_noise.dft_corr_time = inf; end @@ -60,7 +60,6 @@ end % Process each of the segments -ctrl_chain = {}; for param_idx = 1:length(params) param = params(param_idx); if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic diff --git a/cresis-toolbox/processing/run_collate_coh_noise_update.m b/cresis-toolbox/processing/run_collate_coh_noise_update.m new file mode 100644 index 00000000..04215bb1 --- /dev/null +++ b/cresis-toolbox/processing/run_collate_coh_noise_update.m @@ -0,0 +1,46 @@ +% script run_collate_coh_noise_update +% +% Runs collate_coh_noise_update +% +% Authors: John Paden + +%% USER SETTINGS +% ========================================================================= + +param_override = []; + +params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls'),'',{'analysis_noise','analysis'}); + +% Enable a specific segment +params = ct_set_params(params,'cmd.generic',0); +params = ct_set_params(params,'cmd.generic',1,'day_seg','20140401_03'); + +% Update the collate_coh_noise file location +for param_idx = 1:length(params) + for wf = 1:length(params(param_idx).radar.wfs) + % params(param_idx).radar.wfs(wf).coh_noise_arg.fn = 'analysis'; + params(param_idx).radar.wfs(wf).coh_noise_arg.fn = 'analysis_threshold'; + end +end + +%% Automated Section +% ===================================================================== + +% Input checking +global gRadar; +if exist('param_override','var') + param_override = merge_structs(gRadar,param_override); +else + param_override = gRadar; +end + +% Process each of the segments +for param_idx = 1:length(params) + param = params(param_idx); + if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic + continue; + end + collate_coh_noise_update(param,param_override); +% collate_coh_noise_update + +end diff --git a/cresis-toolbox/processing/run_collate_deconv.m b/cresis-toolbox/processing/run_collate_deconv.m index 585a1436..0e8fa48c 100644 --- a/cresis-toolbox/processing/run_collate_deconv.m +++ b/cresis-toolbox/processing/run_collate_deconv.m @@ -6,25 +6,64 @@ %% USER SETTINGS % ========================================================================= - -param_override = []; +clear param_override; params = read_param_xls(ct_filename_param('snow_param_2017_Greenland_P3.xls'),'',{'analysis_spec' 'analysis'}); % params = read_param_xls(ct_filename_param('rds_param_2018_Greenland_P3.xls'),'',{'analysis_spec' 'analysis'}); - % params = ct_set_params(params,'cmd.generic',1); % params = ct_set_params(params,'cmd.generic',0,'cmd.notes','Do not process'); params = ct_set_params(params,'cmd.generic',0); -params = ct_set_params(params,'cmd.generic',1,'day_seg','20170410_01'); +params = ct_set_params(params,'cmd.generic',1,'day_seg','20180322_04'); params = ct_set_params(params,'collate_deconv.f0',2.85e9); params = ct_set_params(params,'collate_deconv.f1',7.5e9); +params = ct_set_params(params,'collate_deconv.rbins',{[-100 100]}); params = ct_set_params(params,'collate_deconv.abs_metric',[58 9.8 -25 -35 inf inf]); params = ct_set_params(params,'collate_deconv.SL_guard_bins',10); -param_override.collate_deconv.out_dir = 'analysis'; +% STEP 1: Check peakiness to ensure that enough waveforms qualify. If +% peakiness threshold has to be adjusted to let more waveforms in, then +% analysis spec must be run again. +param_override.collate_deconv.debug_plots = {'peakiness','metric','visible'}; param_override.collate_deconv.stage_two_en = false; + +% STEP 2: Use the "metric" table output to choose debug_rlines +%param_override.collate_deconv.debug_plots = {'metric','visible'}; param_override.collate_deconv.stage_two_en = false; + +% STEP 3: To evaluate individual waveforms, set debug_rlines to these +% waveforms and enable rbins (evaluate SNR for the rbins setting you have +% chosen) and/or deconv (evaluate the SL_guard_bins, abs_metric, and +% sidelobe suppression achieved): +%param_override.collate_deconv.debug_rlines = [1]; +%param_override.collate_deconv.debug_plots = {'deconv','metric','visible'}; param_override.collate_deconv.stage_two_en = false; +%param_override.collate_deconv.debug_plots = {'rbins','deconv','metric','visible'}; param_override.collate_deconv.stage_two_en = false; +% STEP 3 ALTERNATE: To evaluate just the best individual waveform, just enable +% "rbins_best" and/or "deconv_best" +% param_override.collate_deconv.debug_plots = {'deconv_best','metric','visible'}; param_override.collate_deconv.stage_two_en = false; +% param_override.collate_deconv.debug_plots = {'deconv_best','metric'}; param_override.collate_deconv.stage_two_en = false; + +% STEP 4: Once rbins are set and waveforms appear to be deconvolving well, +% run stage one and stage two (recommend disabling "visible" if many +% segments or wf_adc pairs). +%param_override.collate_deconv.debug_plots = {'deconv_best','metric','final','visible'}; +%param_override.collate_deconv.debug_plots = {'deconv_best','metric','final'}; + +if 0 + % For debugging, use this to test specific waveforms for the whole + % segment. + param_override.collate_deconv.metric_mode = 'each'; % Default mode + %param_override.collate_deconv.metric_mode = 'best'; param_override.collate_deconv.stage_two_en = false; + %param_override.collate_deconv.metric_mode = 'final'; param_override.collate_deconv.stage_two_en = false; + %param_override.collate_deconv.metric_mode = 1; param_override.collate_deconv.stage_two_en = false; +end + +if 0 + % For debugging, use this to select specific images and wf_adc pairs to + % collate instead of doing them all + param_override.collate_deconv.wf_adcs = {[1],[1],[1]}; + param_override.collate_deconv.imgs = [1]; +end %% Automated Section % ===================================================================== diff --git a/cresis-toolbox/processing/run_collate_deconv_update.m b/cresis-toolbox/processing/run_collate_deconv_update.m new file mode 100644 index 00000000..6f039318 --- /dev/null +++ b/cresis-toolbox/processing/run_collate_deconv_update.m @@ -0,0 +1,64 @@ +% Script run_collate_deconv_update.m +% +% Runs collate_deconv_update.m +% +% Author: John Paden + +%% USER SETTINGS +% ========================================================================= + +param_override = []; + +params = read_param_xls(ct_filename_param('accum_param_2019_Antarctica_TObas.xls'),'',{'analysis_spec' 'analysis'}); +% params = read_param_xls(ct_filename_param('snow_param_2017_Greenland_P3.xls'),'',{'analysis_spec' 'analysis'}); + +params = ct_set_params(params,'cmd.generic',0); +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20170327_01'); +params = ct_set_params(params,'cmd.generic',1,'day_seg','20200127_01'); +param_override.collate_deconv_update.cmd = {}; +% param_override.collate_deconv_update.cmd{end+1}.method = 'delete'; +% param_override.collate_deconv_update.cmd{end}.idxs = [1 2 3]; +param_override.collate_deconv_update.cmd{end+1}.method = 'replace'; +% param_override.collate_deconv_update.cmd{end}.day_seg = {'20170323_02'} +% param_override.collate_deconv_update.cmd{end}.day_seg = {'20170320_01'} +param_override.collate_deconv_update.cmd{end}.day_seg = {'20200128_01'}; +param_override.collate_deconv_update.cmd{end}.idxs = {[1]}; + +% 2-18 GHz Deconvolution Settings (3 sets) +% params = ct_set_params(params,'analysis.cmd{1}.abs_metric',[58 4.5 -25 -35 inf inf]); +% param_override.collate_deconv_update.in_dir = 'analysis_uwb'; + +% params = ct_set_params(params,'analysis.cmd{1}.abs_metric',[58 9.8 -25 -35 inf inf]); +% param_override.collate_deconv_update.in_dir = 'analysis'; + +% params = ct_set_params(params,'analysis.cmd{1}.abs_metric',[58 24 -25 -28 inf inf]); +% param_override.collate_deconv_update.in_dir = 'analysis_kuband'; + +% 2-8 GHz Deconvolution Settings +% params = ct_set_params(params,'analysis.cmd{1}.abs_metric',[65 4.5 -25 -35 inf inf]); +param_override.collate_deconv_update.in_dir = 'analysis'; + +% param_override.collate_deconv_update.debug_plots = {'final','visible'}; +param_override.collate_deconv_update.debug_plots = {'final'}; + +%% Automated Section +% ===================================================================== + +% Input checking +global gRadar; +if exist('param_override','var') + param_override = merge_structs(gRadar,param_override); +else + param_override = gRadar; +end + +% Process each of the segments +for param_idx = 1:length(params) + param = params(param_idx); + if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic + continue; + end + collate_deconv_update(param,param_override); + %collate_deconv_update + +end diff --git a/cresis-toolbox/processing/run_collate_snapshots.m b/cresis-toolbox/processing/run_collate_snapshots.m new file mode 100644 index 00000000..306dd951 --- /dev/null +++ b/cresis-toolbox/processing/run_collate_snapshots.m @@ -0,0 +1,72 @@ +% script run_collate_snapshots.m +% +% Script for running collate_snapshots.m. Takes in raw snapshots and +% prepares them for calibration +% +% Authors: Theresa Moore +% +% See also: collate_snapshots.m + +%% User Setup +% ===================================================================== +param_override = []; + +params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls')); +params = ct_set_params(params,'cmd.generic',0); +params = ct_set_params(params,'cmd.generic',1,'day_seg','20140506_01'); +% params = ct_set_params(params,'cmd.frms',[3:5 34:36 40:46]); +params = ct_set_params(params,'cmd.frms',[14:16]); + +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20140401_03'); +% params = ct_set_params(params,'cmd.frms',[1:5 15:16 34:35 41 42]); + +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20140325_07'); +% params = ct_set_params(params,'cmd.frms',[1:5]); + +params = ct_set_params(params,'collate_snapshots.in_path','snapshot_air'); +params = ct_set_params(params,'collate_snapshots.img_list',[1 2 3]); +params = ct_set_params(params,'collate_snapshots.remove_multiple',true); + +% params = ct_set_params(params,'measure_sv_offsets.out_path',' +% params = ct_set_params(params,'measure_sv_offsets.gamma', +% Syntax for running a specific segment and frame by overriding parameter spreadsheet values +%params = read_param_xls(ct_filename_param('rds_param_2009_Antarctica_TO.xls'),'20091228_01'); +%params = ct_set_params(params,'cmd.generic',0); +%params = ct_set_params(params,'cmd.generic',1,'day_seg','20091224_01|20091228|20091229|20100101_02|20100103|20100104_01|20100112'); +%params = ct_set_params(params,'cmd.frms',[]); + +% Set override parameters using ct_set_params, by setting param_override, or by setting FUNCTION_params variable which eventually is copied to param_override.FUNCTION). +% These two examples for setting input parameters are equivalent. +% Example 1 +% params = ct_set_params(params,'FUNCTION.param1','first parameters'); +% params = ct_set_params(params,'FUNCTION.param2',[2]); +% Example 2 +% param_override.FUNCTION.param1 = 'first parameters'; +% param_override.FUNCTION.param2 = [2]; + + + +%param_override.sched.type = 'no scheduler'; % Example to override default cluster settings + +%% Automated Section +% ===================================================================== + +% Input checking +global gRadar; +if exist('param_override','var') + param_override = merge_structs(gRadar,param_override); +else + param_override = gRadar; +end + + +% Process each of the segments +for param_idx = 1:length(params) + param = params(param_idx); + if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic + continue; + end + collate_snapshots(param,param_override); +end + +% Post process code (only include if necessary) \ No newline at end of file diff --git a/cresis-toolbox/processing/run_combine_wf_chan.m b/cresis-toolbox/processing/run_combine_wf_chan.m deleted file mode 100644 index 75be6bb6..00000000 --- a/cresis-toolbox/processing/run_combine_wf_chan.m +++ /dev/null @@ -1,83 +0,0 @@ -% script run_combine_wf_chan -% -% Script for running combine_wf_chan (usually just used for debugging). -% -% Authors: John Paden -% -% See also: run_master.m, master.m, run_combine_wf_chan.m, combine_wf_chan.m, -% combine_wf_chan_task.m - -%% User Setup -% ===================================================================== -param_override = []; - -params = read_param_xls(ct_filename_param('accum_param_2018_Greenland_P3.xls'),''); - -% Syntax for running a specific segment and frame by overriding parameter spreadsheet values -%params = read_param_xls(ct_filename_param('rds_param_2016_Antarctica_DC8.xls'),'20161024_05'); -% params = ct_set_params(params,'cmd.combine_wf_chan',0); -% params = ct_set_params(params,'cmd.combine_wf_chan',1,'day_seg','20161102_03'); -% params = ct_set_params(params,'cmd.frms',6); - -params = ct_set_params(params,'cmd.combine_wf_chan',0); -params = ct_set_params(params,'cmd.combine_wf_chan',1,'day_seg','20180405_01'); -params = ct_set_params(params,'cmd.frms',[115]); -% params = ct_set_params(params,'cmd.csarp',1,'day_seg','20161024_05'); -% params = ct_set_params(params,'combine.in_path','out_cn'); -% params = ct_set_params(params,'combine.array_path','out_cn'); -% params = ct_set_params(params,'combine.out_path','standard_cn'); - -% params = ct_set_params(params,'cmd.combine_wf_chan',0); -% params = ct_set_params(params,'cmd.combine_wf_chan',1,'day_seg','20161107_02'); -% params = ct_set_params(params,'cmd.combine',1,'day_seg','20161024_05'); -% params = ct_set_params(params,'cmd.frms',2,'day_seg','20161107_02'); -% params = ct_set_params(params,'combine.out_path','paden_standard'); -% params = ct_set_params(params,'combine.in_path','paden_out'); -% params = ct_set_params(params,'combine.array_path','paden_out'); - -dbstop if error; -% param_override.cluster.type = 'torque'; -% param_override.cluster.type = 'matlab'; -param_override.cluster.type = 'debug'; -%param_override.cluster.rerun_only = true; -param_override.cluster.desired_time_per_job = 0*60; -% param_override.cluster.cpu_time_mult = 2; -% param_override.cluster.mem_mult = 2; -% param_override.cluster.max_jobs_active = 1; -% param_override.cluster.qsub_submit_arguments = '-q debug -m n -l nodes=1:ppn=1:dcwan:dc2,pmem=%dmb,walltime=%d:00'; - -%% Automated Section -% ===================================================================== - -% Input checking -global gRadar; -if exist('param_override','var') - param_override = merge_structs(gRadar,param_override); -else - param_override = gRadar; -end - -% Process each of the segments -ctrl_chain = {}; -for param_idx = 1:length(params) - param = params(param_idx); - if param.cmd.combine_wf_chan - ctrl_chain{end+1} = combine_wf_chan(param,param_override); - end -end - -cluster_print_chain(ctrl_chain); - -[chain_fn,chain_id] = cluster_save_chain(ctrl_chain); - -% Potentially stop and inspect cluster_print_chain output to adjust -% cluster control parameters before running or to run the next lines on a -% different computer (the save/load functions are for this purpose). - -return -%ctrl_chain = cluster_set_chain(ctrl_chain,'cluster.desired_time_per_job',5*60); -%ctrl_chain = cluster_set_chain(ctrl_chain,'cluster.cpu_time_mult',2); -%ctrl_chain = cluster_set_chain(ctrl_chain,'cluster.mem_mult',2); - -[ctrl_chain,chain_fn] = cluster_load_chain([],chain_id); -ctrl_chain = cluster_run(ctrl_chain); diff --git a/cresis-toolbox/processing/run_csarp.m b/cresis-toolbox/processing/run_csarp.m deleted file mode 100644 index faba5ad7..00000000 --- a/cresis-toolbox/processing/run_csarp.m +++ /dev/null @@ -1,68 +0,0 @@ -% script run_csarp -% -% Script for running csarp (usually just used for debugging). -% -% Authors: John Paden -% -% See also: run_master.m, master.m, run_csarp.m, csarp.m, -% csarp_task.m - -%% User Setup -% ===================================================================== -param_override = []; - -params = read_param_xls(ct_filename_param('rds_param_2016_Antarctica_DC8.xls'),''); - -% Syntax for running a specific segment and frame by overriding parameter spreadsheet values -%params = read_param_xls(ct_filename_param('rds_param_2016_Antarctica_DC8.xls'),'20161024_05'); -params = ct_set_params(params,'cmd.csarp',0); -params = ct_set_params(params,'cmd.csarp',1,'day_seg','20161022_03'); -% params = ct_set_params(params,'cmd.csarp',1,'day_seg','20161024_05'); -params = ct_set_params(params,'csarp.out_path','paden_out'); - -dbstop if error; -param_override.cluster.type = 'torque'; -% param_override.cluster.type = 'matlab'; -% param_override.cluster.type = 'debug'; -%param_override.cluster.rerun_only = true; -param_override.cluster.desired_time_per_job = 120*60; -% param_override.cluster.cpu_time_mult = 2; -% param_override.cluster.mem_mult = 2; -% param_override.cluster.max_jobs_active = 1; -% param_override.cluster.qsub_submit_arguments = '-q debug -m n -l nodes=1:ppn=1:dcwan:dc2,pmem=%dmb,walltime=%d:00'; - -%% Automated Section -% ===================================================================== - -% Input checking -global gRadar; -if exist('param_override','var') - param_override = merge_structs(gRadar,param_override); -else - param_override = gRadar; -end - -% Process each of the segments -ctrl_chain = {}; -for param_idx = 1:length(params) - param = params(param_idx); - if param.cmd.csarp - ctrl_chain{end+1} = csarp(param,param_override); - end -end - -cluster_print_chain(ctrl_chain); - -[chain_fn,chain_id] = cluster_save_chain(ctrl_chain); - -% Potentially stop and inspect cluster_print_chain output to adjust -% cluster control parameters before running or to run the next lines on a -% different computer (the save/load functions are for this purpose). - -return -%ctrl_chain = cluster_set_chain(ctrl_chain,'cluster.desired_time_per_job',5*60); -%ctrl_chain = cluster_set_chain(ctrl_chain,'cluster.cpu_time_mult',2); -%ctrl_chain = cluster_set_chain(ctrl_chain,'cluster.mem_mult',2); - -[ctrl_chain,chain_fn] = cluster_load_chain([],chain_id); -ctrl_chain = cluster_run(ctrl_chain); diff --git a/cresis-toolbox/processing/run_img_combine_update.m b/cresis-toolbox/processing/run_img_combine_update.m new file mode 100644 index 00000000..dbdd8b08 --- /dev/null +++ b/cresis-toolbox/processing/run_img_combine_update.m @@ -0,0 +1,45 @@ +% script run_img_combine_update +% +% Script for running img_combine_update +% +% Authors: John Paden +% +% See also: run_img_combine_update.m, img_combine_update.m + +%% User Setup +% ===================================================================== +params = read_param_xls(ct_filename_param('rds_param_2019_Antarctica_Ground.xls'),''); +params = ct_set_params(params,'cmd.generic',0); +params = ct_set_params(params,'cmd.generic',1,'day_seg','20200107_01'); + + +mode = 'array'; % <== OFTEN CHANGED (qlook or array) + +img_combine_update_param.out_path = 'standard'; % <== OFTEN CHANGED (no default) +img_combine_update_param.img_comb_mult = inf; % <== OFTEN CHANGED (inf default) +img_combine_update_param.img_comb_bins = 10; % <== OFTEN CHANGED (1 default) +img_combine_update_param.img_comb_layer_params = struct('name','surface','source','layerdata','layerdata_source','layer');% <== OFTEN CHANGED + +%% Automated Section +% ===================================================================== + +param_override = []; +param_override.(mode) = img_combine_update_param; +param_override.img_combine_update.mode = mode; + +% Input checking +global gRadar; +if exist('param_override','var') + param_override = merge_structs(gRadar,param_override); +else + param_override = gRadar; +end + +% Process each of the segments +for param_idx = 1:length(params) + param = params(param_idx); + if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic + continue; + end + img_combine_update(param,param_override); +end diff --git a/cresis-toolbox/processing/run_layer_create.m b/cresis-toolbox/processing/run_layer_create.m new file mode 100644 index 00000000..c156f8d1 --- /dev/null +++ b/cresis-toolbox/processing/run_layer_create.m @@ -0,0 +1,90 @@ +% script run_layer_create +% +% Calls the layer_create function +% +% Author: John Paden +% +% See also: layer_create, run_layer_create + +%% User Settings +% ========================================================================= +param_override = []; + +% Parameters spreadsheet to use for updating +% params = read_param_xls(ct_filename_param('rds_param_2010_Greenland_DC8.xls')); +% params = read_param_xls(ct_filename_param('rds_param_2010_Greenland_P3.xls')); +% params = read_param_xls(ct_filename_param('rds_param_2011_Greenland_P3.xls')); +% params = read_param_xls(ct_filename_param('rds_param_2012_Greenland_P3.xls')); +% params = read_param_xls(ct_filename_param('rds_param_2013_Greenland_P3.xls')); +% params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls')); +% params = read_param_xls(ct_filename_param('rds_param_2015_Greenland_C130.xls')); +% params = read_param_xls(ct_filename_param('rds_param_2016_Greenland_P3.xls')); +params = read_param_xls(ct_filename_param('rds_param_2017_Greenland_P3.xls')); +% params = read_param_xls(ct_filename_param('snow_param_2014_Greenland_P3.xls')); +% params = read_param_xls(ct_filename_param('snow_param_2016_Greenland_P3.xls')); + +params = ct_set_params(params,'cmd.generic',0); +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20160501_01'); +% params = ct_set_params(params,'cmd.generic',0,'cmd.notes','do not process'); +% params = ct_set_params(params,'cmd.frms',[]); + +params = ct_set_params(params,'cmd.generic',1); +params = ct_set_params(params,'cmd.generic',0,'cmd.notes','do not process'); +params = ct_set_params(params,'cmd.frms',[]); + +% .out_path: string containing file path where layer files will be stored +% to; string is passed to ct_filename_out to form the file path. +% Default is 'layer'. +param_override.layer_file_make.out_path = 'layer'; +% param_override.layer_file_make.out_path = 'CSARP_post/layer'; + +% param_override.layer_file_make.update_mode +% 0: no updates are made, layer file is created only if no layer file +% exists or if there were problems with an existing layer file. This is +% the default setting. +% 1: layer file is overwritten with new information (blank layer file) +% and old layer data is lost +% 2: Old GPS time field is adjusted according to changes in +% param.records.gps.time_offset and then the position and layer +% information is reinterpolated from the old GPS time field to the new +% GPS time field. +param_override.layer_file_make.update_mode = 0; +% param_override.layer_file_make.update_mode = 1; +% param_override.layer_file_make.update_mode = 2; + + +%% Automated section +% ========================================================================= + +global gRadar; + +% Input checking +if exist('param_override','var') + param_override = merge_structs(gRadar,param_override); +else + param_override = gRadar; +end + +%% Make layer data files for each of the enabled segments +failed_segments = []; +for param_idx = 1:length(params) + param = params(param_idx); + if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic + continue; + end + try + layer_create(param,param_override); + %layer_create + catch ME + failed_segments(end+1).param_idx = param_idx; + failed_segments(end).report = ME.getReport; + failed_segments(end).message = ME.message; + %keyboard + end + fprintf(' Complete (%s)\n', datestr(now)); +end + +for failed_idx = 1:length(failed_segments) + fprintf('%s: %s\n', params(failed_segments(failed_idx).param_idx).day_seg, ... + failed_segments(failed_idx).message); +end diff --git a/cresis-toolbox/processing/run_load_data.m b/cresis-toolbox/processing/run_load_data.m index 41012815..1e3d034b 100644 --- a/cresis-toolbox/processing/run_load_data.m +++ b/cresis-toolbox/processing/run_load_data.m @@ -19,9 +19,7 @@ param = read_param_xls(ct_filename_param('rds_param_2018_Antarctica_Ground.xls'),'20181014_02'); % Determine which records you want to load: - frames_fn = ''; - frames_fn = ct_filename_support(param,frames_fn,'frames'); - load(frames_fn); + frames = frames_load(param); frm = 1; param.load_data.recs = frames.frame_idxs(frm) + 0 + [0 0]; @@ -30,7 +28,7 @@ param.load_data.imgs = {[2 5]}; param.load_data.pulse_comp = false; param.load_data.raw_data = false; - param.load_data.ft_wind = @hanning; + %param.load_data.ft_wind = @hanning; param.load_data.combine_rx = false; % Load data @@ -84,9 +82,7 @@ param = read_param_xls(ct_filename_param('rds_param_2016_Greenland_Polar6.xls'),'20160413_04'); % Determine which records you want to load: - frames_fn = ''; - frames_fn = ct_filename_support(param,frames_fn,'frames'); - load(frames_fn); + frames = frames_load(param); frm = 1; param.load_data.recs = frames.frame_idxs(frm) - 1 + [10000 10250]; @@ -105,7 +101,7 @@ %% Load data [hdr,data] = load_data(param); - + %% Print out DC values and create DC adjust files if 0 adc_mean = []; @@ -135,7 +131,7 @@ end end end - + %% Plot data img = 1; wf_adc_idx = 1; @@ -175,9 +171,7 @@ param = read_param_xls(param_fn,'20100324_01'); % Determine which records you want to load: - frames_fn = ''; - frames_fn = ct_filename_support(param,frames_fn,'frames'); - load(frames_fn); + frames = frames_load(param); frm = 10; param.load_data.recs = frames.frame_idxs(frm) - 1 + [4001 8000]; @@ -239,9 +233,7 @@ param = read_param_xls(param_fn,'20110317_01'); % Determine which records you want to load: - frames_fn = ''; - frames_fn = ct_filename_support(param,frames_fn,'frames'); - load(frames_fn); + frames = frames_load(param); frm = 2; param.load_data.recs = frames.frame_idxs(frm) - 1 + [5000 17000]; @@ -457,9 +449,7 @@ fprintf('Processing segment %s (%i of %i)\n', param.day_seg, idx, length(params)); % Determine which records you want to load: - frames_fn = ''; - frames_fn = ct_filename_support(param,frames_fn,'frames'); - load(frames_fn); % GET NUMBER OF FRAMES HERE (look at variable named "frames") + frames = frames_load(param); if isempty(param.cmd.frms) frms = 1:length(frames.frame_idxs); @@ -521,5 +511,184 @@ end end end + +elseif run_example == 6 + % ======================================================================= + % Setup loading parameters for example 6 + % - Examines BW_window (coherent ave of elev compensated raw data) + % ======================================================================= + try hm; end + try user_window_style = get(0,'DefaultFigureWindowStyle'); + set(0,'DefaultFigureWindowStyle','docked'); end + + switch 3 % Check multiple specular surfaces + case 1 + param = read_param_xls(ct_filename_param('snow_param_2017_Greenland_P3.xls'),'20170407_02'); % 2-18 + param.load_data.recs = 9124619+ [500 900]; % [-1000 +1000] frm = 486; + pick_index = 13263; % used to remove phase variation + case 2 + param = read_param_xls(ct_filename_param('snow_param_2017_Greenland_P3.xls'),'20170323_02'); % 2-8 + frames = load(ct_filename_support(param,'','frames')); + frm = 363; + param.load_data.recs = frames.frame_idxs(frm) + [2600 3100]; % [1200 3400] + pick_index = 9200; % used to remove phase variation + case 3 + param = read_param_xls(ct_filename_param('snow_param_2017_Greenland_P3.xls'),'20170410_01'); + % 2-8 rx saturated? operator switched from 2-18 to 2-8 for this segment + frames = load(ct_filename_support(param,'','frames')); + frm = 3; + param.load_data.recs = frames.frame_idxs(frm) + [10300 11200]; + pick_index = 10363; % used to remove phase variation + end + + % param to load raw data + param = ct_set_params(param,'radar.wfs(1).deconv.en',false); + param.radar.wfs(1).coh_noise_method = 'analysis'; + param.radar.wfs(1).coh_noise_arg.fn = 'analysis'; + param.load_data.imgs = {[1 1]}; + param.load_data.pulse_comp = false; + param.load_data.raw_data = true; + param.load_data.ft_wind = @hanning; + param.load_data.combine_rx = false; + + % Load data + [hdr,data] = load_data(param); + % hdr.surface = sgolayfilt(hdr.surface, 2, 201); % smoothen surface in some cases + img = 1; + wf_adc_idx = 1; + wf = param.load_data.imgs{img}(wf_adc_idx,1); + adc = param.load_data.imgs{img}(wf_adc_idx,2); + data{img} = bsxfun(@minus,data{img},mean(data{1},2)); % DC removal + + % radar params + c = physical_constants('c'); + f0 = param.radar.wfs(1).f0; + f1 = param.radar.wfs(1).f1; + Tpd = param.radar.wfs(1).Tpd; + chirp_rate = (f1-f0) / Tpd; + t_ref = param.radar.wfs(1).t_ref; + fs = param.radar.fs; + IF_nz = [0:3]; + IF_cutoffs = [0 : 0.5 : 2]' * fs; % Hz + range_gates = IF_cutoffs / chirp_rate *c/2; % meter + td = hdr.surface; % twtt (radar, surface) + R = td *c/2; + nz = max(IF_nz(any(bsxfun(@ge,R,range_gates) ,2))); % assuming only one nz + f_beat = chirp_rate*(td-t_ref); + f_beat = abs(f_beat-round(f_beat/fs)*fs); % real apparent frequency + time = hdr.time{1}; + elev = hdr.records{1}.elev; % aircraft position rel to WGS-84 + elev_td = elev *2/c; % twtt (radar, WGS-84) + time2freq_xaxis = ( (time + t_ref) *chirp_rate + f0 )/1e9; % freq in GHz + + % Baseband the data + data_f{img} = fft(data{img}); + try + data_f{img}(round(end/2+1:end),:) = 0; + catch + fprintf('data_f error\n'); + Nt = size(data_f{img},1); + data_f{img}(round(Nt/2+1:Nt),:) = 0; + end + data{img} = ifft(data_f{img}); + + % Pre-compensation PLOTS + for compressing_this = 1 + + fig_id = 10; fig_h = figure(fig_id); clf(fig_id); + aa(1) = subplot(1,2,1); + imagesc([],hdr.time{img}/1e-6, lp(abs(data{img}(:,:,wf_adc_idx)).^2/2/50)+30); + grid on; colorbar; ylabel('Fast-time, us'); xlabel('rlines'); + title('Pre-compensation Time-space domain (dBm signal)'); + + fig_id = 20; fig_h = figure(fig_id); clf(fig_id); + bb(1) = subplot(1,2,1); + plot_data = lp(data_f{img}); + [~,max_idxs] = max(plot_data(2:end/2,:)); + max_idxs = max_idxs+1; % add 1 if max from 2nd rbin + imagesc([],hdr.freq{img}/1e6,plot_data); hold on; clear plot_data data_f; + plot(f_beat/1e6,'.-'); + plot(hdr.freq{img}(max_idxs)/1e6,'.-'); + grid on; ylabel('Freq, MHz'); xlabel('rlines'); zoom on; + colorbar;legend('fb','max'); + title('Pre-compensation lp( FT(data) )'); + + fig_id = 30; fig_h = figure(fig_id); clf(fig_id); + cc(1) = subplot(1,2,1); + imagesc([],hdr.time{img}/1e-6,angle(data{img})); hold on; + grid on; ylabel('Fast-time, us'); xlabel('rlines'); zoom on; + colorbar; + title('Pre-compensation angle(data)'); + + fig_id = 123; fig_h = figure(fig_id); clf(fig_id); hold on; + mean_data = mean(data{img},2); + dd(1) = subplot(2,2,1); + plot(time2freq_xaxis, real(mean_data)); grid on; title('Pre-compensation mean'); + xlabel('Freq, in GHz'); ylabel('Voltage, V'); + dd(2) = subplot(2,2,3); + plot(time2freq_xaxis, lp(mean_data)); grid on; title('Pre-compensation lp(mean)'); + xlabel('Freq, in GHz'); ylabel('Magnitude, dB'); + linkaxes(dd,'x'); zoom on; + clear mean_data; + + end + + % Compensation + ttt = (td(1)+elev_td-elev_td(1)) - td(1); %first order + ttt2 = -(td(1)+elev_td-elev_td(1)).^2 + td(1)^2; % second order + phase_comp = +2*pi*chirp_rate*time*(ttt) + 2*pi*f0*(ttt) + pi*chirp_rate*(ttt2); + data{img} = data{img} .* exp(-1i* (-1)^nz *( phase_comp )); + if 1 % enable for additional phase correction, uses row specified by pick_index + data{img} = fir_dec(data{img}, ones(1,11),1); + filt_phase = angle(data{img}); + pick_phase = filt_phase(pick_index,:); + data{img} = bsxfun(@times,data{img}, exp(-1i*pick_phase)); + end + data_f{img} = fft(data{img}); + + clear phase_comp; + + % Post-compensation PLOTS + for compressing_this = 1 + + fig_id = 10; fig_h = figure(fig_id); %clf(fig_id); + aa(2) = subplot(1,2,2); + imagesc([],hdr.time{img}/1e-6, lp(abs(data{img}(:,:,wf_adc_idx)).^2/2/50)+30); + grid on; colorbar; ylabel('Fast-time, us'); xlabel('rlines'); + title('Post-compensation Time-space domain (dBm signal)'); + + fig_id = 20; fig_h = figure(fig_id); %clf(fig_id); + bb(2) = subplot(1,2,2); + plot_data = lp(data_f{img}); + [~,max_idxs] = max(plot_data(2:end/2,:)); + max_idxs = max_idxs+1; % add 1 if max from 2nd rbin + imagesc([],hdr.freq{img}/1e6,plot_data); hold on; clear plot_data data_f; + plot(f_beat/1e6,'.-'); + plot(hdr.freq{img}(max_idxs)/1e6,'.-'); + grid on; ylabel('Freq, MHz'); xlabel('rlines'); zoom on; + colorbar;legend('fb','max'); + title('Post-compensation lp( FT(data) )'); + + fig_id = 30; fig_h = figure(fig_id); %clf(fig_id); + cc(2) = subplot(1,2,2); + imagesc([],hdr.time{img}/1e-6,angle(data{img})); hold on; + grid on; ylabel('Fast-time, us'); xlabel('rlines'); zoom on; + colorbar; + title('Post-compensation angle(data)'); + + fig_id = 123; fig_h = figure(fig_id); % clf(fig_id); hold on; + mean_data = mean(data{img},2); + dd(3) = subplot(2,2,2); + plot(time2freq_xaxis, real(mean_data)); grid on; title('Post-compensation mean'); + xlabel('Freq, in GHz'); ylabel('Voltage, V'); + dd(4) = subplot(2,2,4); + plot(time2freq_xaxis, lp(mean_data)); grid on; title('Post-compensation lp(mean)'); + xlabel('Freq, in GHz'); ylabel('Magnitude, dB'); + linkaxes(dd,'x'); zoom on; + clear mean_data; + + end + try linkaxes([aa]); linkaxes([bb]); linkaxes([cc]); end + try set(0,'DefaultFigureWindowStyle',user_window_style); end end \ No newline at end of file diff --git a/cresis-toolbox/processing/run_records_create.m b/cresis-toolbox/processing/run_records_create.m index f8fd2977..d19c0961 100644 --- a/cresis-toolbox/processing/run_records_create.m +++ b/cresis-toolbox/processing/run_records_create.m @@ -1,6 +1,6 @@ % script run_records_create % -% Script for running run_records_create (usually just used for debugging). +% Script for running records_create. % % Authors: John Paden % diff --git a/cresis-toolbox/processing/run_records_reference_trajectory.m b/cresis-toolbox/processing/run_records_reference_trajectory.m new file mode 100644 index 00000000..7f9e6111 --- /dev/null +++ b/cresis-toolbox/processing/run_records_reference_trajectory.m @@ -0,0 +1,47 @@ +% script run_records_reference_trajectory +% +% Script for running records_reference_trajectory +% +% Authors: John Paden +% +% See also: records_reference_trajectory.m, +% records_reference_trajectory_load.m, run_records_reference_trajectory.m, +% run_all_records_reference_trajectory.m + +%% User Setup +% ===================================================================== +param_override = []; + +params = read_param_xls(ct_filename_param('rds_param_2018_Greenland_P3.xls')); + +if 1 + % Example to run a specific segment and frame by overriding parameter spreadsheet values + params = ct_set_params(params,'cmd.generic',0); + params = ct_set_params(params,'cmd.generic',1,'day_seg','20180419_01'); + +elseif 0 + % Example to run all segments + params = ct_set_params(params,'cmd.generic',1); + params = ct_set_params(params,'cmd.generic',0,'cmd.notes','do not process'); +end + +dbstop if error; + +%% Automated Section +% ===================================================================== + +% Input checking +global gRadar; +if exist('param_override','var') + param_override = merge_structs(gRadar,param_override); +else + param_override = gRadar; +end + +% Process each of the segments +for param_idx = 1:length(params) + param = params(param_idx); + if isfield(param.cmd,'generic') && ~iscell(param.cmd.generic) && ~ischar(param.cmd.generic) && param.cmd.generic + records_reference_trajectory(param,param_override); + end +end diff --git a/cresis-toolbox/processing/run_sar_equal.m b/cresis-toolbox/processing/run_sar_equal.m new file mode 100644 index 00000000..97f487fd --- /dev/null +++ b/cresis-toolbox/processing/run_sar_equal.m @@ -0,0 +1,200 @@ +% script run_sar_equal +% +% Example for running sar_equal.m +% +% Author: John Paden + +warning('This is an example file, copy to personal directory, rename, and remove this warning/return to use'); + +%% User Settings +% ======================================================================= + +param_override = []; params = []; + +if 0 + % 2011 EGIG line + params = read_param_xls(ct_filename_param('rds_param_2011_Greenland_P3.xls'),'20110426_11'); + + params.cmd.frms = [5]; + params.cmd.generic = true; + + if 0 + % Waveform 1 equalization + param_override.sar_equal.imgs = {[1 2],[1 3],[1 4],[1 5],[1 6],[1 7],[1 8]}; + param_override.sar_equal.start_times = struct('name','surface','eval',struct('cmd','s=s-1.5e-6;')); + + elseif 1 + % Waveform 1 to waveform 2 + param_override.sar_equal.imgs = {[2*ones(7,1) (2:8)'],[1*ones(7,1) (2:8)']}; + param_override.sar_equal.start_time = struct('name','surface','eval',struct('cmd','s=s+10e-6;')); + param_override.sar_equal.delay.ref_bins = [-30 31]; + param_override.sar_equal.delay.search_bins = [-24 25]; + param_override.sar_equal.debug_out_dir = 'sar_equal_wf1'; + param_override.sar_equal.coh_ave = 10; + + param_override.radar.wfs(1).Tsys = params.radar.wfs(1).Tsys - 5.8045e-09; + param_override.radar.wfs(1).chan_equal_deg = params.radar.wfs(1).chan_equal_deg + -26.8951 + 47.4778; + param_override.radar.wfs(2).Tsys = params.radar.wfs(2).Tsys; + param_override.radar.wfs(2).chan_equal_deg = params.radar.wfs(2).chan_equal_deg; + + end + +elseif 0 + % 2012 Summit line + params = read_param_xls(ct_filename_param('rds_param_2012_Greenland_P3.xls'),'20120330_03'); + + params.cmd.frms = 8; + params.cmd.generic = true; + + if 0 + % Waveform 1 equalization + param_override.sar_equal.imgs = {[1 2],[1 3],[1 4],[1 5],[1 6],[1 7],[1 8]}; + param_override.sar_equal.start_times = struct('name','surface','eval',struct('cmd','s=s-1.5e-6;')); + + elseif 1 + % Waveform 1 to waveform 2 + param_override.sar_equal.imgs = {[2*ones(7,1) (2:8)'],[1*ones(7,1) (2:8)']}; + param_override.sar_equal.start_time = struct('name','surface','eval',struct('cmd','s=s+10e-6;')); + param_override.sar_equal.delay.ref_bins = [-30 31]; + param_override.sar_equal.delay.search_bins = [-24 25]; + param_override.sar_equal.debug_out_dir = 'sar_equal_wf1'; + param_override.sar_equal.coh_ave = 10; + + param_override.radar.wfs(1).Tsys = params.radar.wfs(1).Tsys -5.65956e-09; + param_override.radar.wfs(1).chan_equal_deg = params.radar.wfs(1).chan_equal_deg + 5.09989; + param_override.radar.wfs(2).Tsys = params.radar.wfs(2).Tsys; + param_override.radar.wfs(2).chan_equal_deg = params.radar.wfs(2).chan_equal_deg; + end + +elseif 0 + % 2012 EGIG line + params = read_param_xls(ct_filename_param('rds_param_2012_Greenland_P3.xls'),'20120411_02'); + + params.cmd.frms = [9 10]; + params.cmd.frms = 9; + params.cmd.generic = true; + + if 0 + % Waveform 1 equalization + param_override.sar_equal.imgs = {[1 2],[1 3],[1 4],[1 5],[1 6],[1 7],[1 8]}; + param_override.sar_equal.start_times = struct('name','surface','eval',struct('cmd','s=s-1.5e-6;')); + + elseif 1 + % Waveform 1 to waveform 2 + param_override.sar_equal.imgs = {[2*ones(7,1) (2:8)'],[1*ones(7,1) (2:8)']}; + param_override.sar_equal.start_time = struct('name','surface','eval',struct('cmd','s=s+10e-6;')); + param_override.sar_equal.delay.ref_bins = [-30 31]; + param_override.sar_equal.delay.search_bins = [-24 25]; + param_override.sar_equal.debug_out_dir = 'sar_equal_wf1'; + param_override.sar_equal.coh_ave = 10; + + param_override.radar.wfs(1).Tsys = params.radar.wfs(1).Tsys -5.65956e-09; + param_override.radar.wfs(1).chan_equal_deg = params.radar.wfs(1).chan_equal_deg + 5.09989; + param_override.radar.wfs(2).Tsys = params.radar.wfs(2).Tsys; + param_override.radar.wfs(2).chan_equal_deg = params.radar.wfs(2).chan_equal_deg; + end + +elseif 1 + % 2014 low altitude above ground level frames to measure wf 1-wf 2 + % overlap + if 0 + params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls'),'20140502_01'); + params.cmd.frms = [31 32 33 34]; % frame 31 is mostly too high AGL + elseif 1 + params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls'),'20140410_01'); + params.cmd.frms = 57; + elseif 0 + params.cmd.frms = [41]; + end + params.cmd.generic = true; + + if 0 + % Waveform 1 equalization +% param_override.sar_equal.imgs = {[1 2],[1 3],[1 4],[1 5],[1 6],[1 7],[1 8]}; +% param_override.sar_equal.start_time = struct('name','surface','eval',struct('cmd','s=s-1.5e-6;')); + param_override.sar_equal.imgs = {[2 2],[2 3],[2 4],[2 5],[2 6],[2 7],[2 8]}; + param_override.sar_equal.imgs = {[3 2],[3 3],[3 4],[3 5],[3 6],[3 7],[3 8]}; + param_override.sar_equal.start_time = struct('name','surface','eval',struct('cmd','s=s+10e-6;')); + + elseif 0 + % Waveform 1 to waveform 2 + param_override.sar_equal.imgs = {[2*ones(7,1) (2:8)'],[1*ones(7,1) (2:8)']}; + param_override.sar_equal.start_time = struct('name','surface','eval',struct('cmd','s=s+3e-6;')); + param_override.sar_equal.delay.ref_bins = [-19 20]; + param_override.sar_equal.delay.search_bins = [-13 14]; + param_override.sar_equal.debug_out_dir = 'sar_equal_wf1'; + param_override.sar_equal.coh_ave = 10; + + elseif 1 + % Waveform 2 to waveform 3 + param_override.sar_equal.imgs = {[2*ones(7,1) (2:8)'],[3*ones(7,1) (2:8)']}; + param_override.sar_equal.start_time = struct('name','surface','eval',struct('cmd','s=s+10e-6;')); + param_override.sar_equal.delay.ref_bins = [-30 31]; + param_override.sar_equal.delay.search_bins = [-24 25]; + param_override.sar_equal.debug_out_dir = 'sar_equal_wf3'; + param_override.sar_equal.coh_ave = 10; + end + + param_override.radar.wfs(1).Tsys = params.radar.wfs(1).Tsys + 5.86577e-09; + param_override.radar.wfs(1).chan_equal_deg = params.radar.wfs(1).chan_equal_deg + -94.9225; + param_override.radar.wfs(2).Tsys = params.radar.wfs(2).Tsys; + param_override.radar.wfs(2).chan_equal_deg = params.radar.wfs(2).chan_equal_deg; + param_override.radar.wfs(3).Tsys = params.radar.wfs(3).Tsys + 1.16604e-08; + param_override.radar.wfs(3).chan_equal_deg = params.radar.wfs(3).chan_equal_deg + 52.7387 + -98.5571; + param_override.sar_equal.sar_load.debug_level = 2; + +elseif 1 + % 2014 EGIG line + params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls'),'20140410_01'); + + params.cmd.frms = [57]; + params.cmd.generic = true; + + if 0 + % Waveform 1 equalization + param_override.sar_equal.imgs = {[1 2],[1 3],[1 4],[1 5],[1 6],[1 7],[1 8]}; + param_override.sar_equal.start_times = struct('name','surface','eval',struct('cmd','s=s-1.5e-6;')); + + elseif 0 + % Waveform 1 to waveform 2 + % NOT VALID, ALTITUDE AGL IS TOO LARGE + + elseif 1 + % Waveform 2 to waveform 3 + param_override.sar_equal.imgs = {[2*ones(7,1) (2:8)'],[3*ones(7,1) (2:8)']}; + param_override.sar_equal.start_time = struct('name','surface','eval',struct('cmd','s=s+10e-6;')); + param_override.sar_equal.delay.ref_bins = [-30 31]; + param_override.sar_equal.delay.search_bins = [-24 25]; + param_override.sar_equal.debug_out_dir = 'sar_equal_wf3'; + param_override.sar_equal.coh_ave = 1; + end + + param_override.radar.wfs(1).Tsys = params.radar.wfs(1).Tsys + 5.86577e-09; + param_override.radar.wfs(1).chan_equal_deg = params.radar.wfs(1).chan_equal_deg + -94.9225; + param_override.radar.wfs(2).Tsys = params.radar.wfs(2).Tsys; + param_override.radar.wfs(2).chan_equal_deg = params.radar.wfs(2).chan_equal_deg; + param_override.radar.wfs(3).Tsys = params.radar.wfs(3).Tsys + 1.16604e-08; + param_override.radar.wfs(3).chan_equal_deg = params.radar.wfs(3).chan_equal_deg + 52.7387 + -98.5571; +end + +%% Automated Section +% ===================================================================== + +% Input checking +global gRadar; +if exist('param_override','var') + param_override = merge_structs(gRadar,param_override); +else + param_override = gRadar; +end + +% Process each of the segments +ctrl_chain = {}; +for param_idx = 1:length(params) + param = params(param_idx); + if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic + continue; + end + %sar_equal(param,param_override); + sar_equal; +end diff --git a/cresis-toolbox/processing/run_sar_equal_post.m b/cresis-toolbox/processing/run_sar_equal_post.m new file mode 100644 index 00000000..80f96b5e --- /dev/null +++ b/cresis-toolbox/processing/run_sar_equal_post.m @@ -0,0 +1,270 @@ +% script run_sar_equal_post +% +% Example for running sar_equal_post.m +% +% Author: John Paden + +warning('This is an example file, copy to personal directory, rename, and remove this warning/return to use'); + +%% User Settings +% ======================================================================= + +param_override = []; params = []; + +if 0 + % 2011 EGIG line + params = read_param_xls(ct_filename_param('rds_param_2011_Greenland_P3.xls'),'20110426_11'); + + params.cmd.frms = [5]; + params.cmd.generic = true; + + % Waveform 1 to waveform 2 + param_override.sar_equal_post.imgs = {[2*ones(7,1) (2:8)'],[1*ones(7,1) (2:8)']}; + param_override.sar_equal_post.debug_in_dir = 'sar_equal_wf1'; + +elseif 0 + % 2012 EGIG line + params = read_param_xls(ct_filename_param('rds_param_2012_Greenland_P3.xls'),'20120411_02'); + + params.cmd.frms = [9 10]; + params.cmd.generic = true; + + % Waveform 1 to waveform 2 + param_override.sar_equal_post.imgs = {[2*ones(7,1) (2:8)'],[1*ones(7,1) (2:8)']}; + param_override.sar_equal_post.debug_in_dir = 'sar_equal_wf1'; + +elseif 1 + % 2014 low altitude above ground level frames to measure wf 1-wf 2 + % overlap + params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls'),'20140502_01'); + + params.cmd.frms = [31 32 33 34]; % frame 31 is mostly too high AGL + params.cmd.generic = true; + + if 0 + % Waveform 1 to waveform 2 + param_override.sar_equal_post.imgs = {[2*ones(7,1) (2:8)'],[1*ones(7,1) (2:8)']}; + param_override.sar_equal_post.debug_in_dir = 'sar_equal_wf1'; + else + % Waveform 2 to waveform 3 + param_override.sar_equal_post.imgs = {[2*ones(7,1) (2:8)'],[3*ones(7,1) (2:8)']}; + param_override.sar_equal_post.debug_in_dir = 'sar_equal_wf3'; + end + +elseif 0 + % 2014 EGIG line + params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls'),'20140410_01'); + + params.cmd.frms = [57]; + params.cmd.generic = true; + + % Waveform 2 to waveform 3 + param_override.sar_equal_post.imgs = {[2*ones(7,1) (2:8)'],[3*ones(7,1) (2:8)']}; + param_override.sar_equal_post.debug_in_dir = 'sar_equal_wf3'; +end + +%% Automated Section +% ===================================================================== + +% Input checking +global gRadar; +if exist('param_override','var') + param_override = merge_structs(gRadar,param_override); +else + param_override = gRadar; +end + +% Process each of the segments +ctrl_chain = {}; +for param_idx = 1:length(params) + param = params(param_idx); + if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic + continue; + end + %sar_equal_output{param_idx} = sar_equal_post(param,param_override); + sar_equal_post; sar_equal_output{param_idx} = sar_equal_output; +end + +%% Setup plots + +h_fig = get_figures(3,enable_visible_plot); +for fig_num = 1:3 + clf(h_fig(fig_num)); + h_axes(fig_num) = axes('parent',h_fig(fig_num)); +end + +%% Loop Segments +output_idx = 0; +for param_idx = 1:length(params) + param = params(param_idx); + if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic + continue; + end + + %% Segments: Loop Frames + for frm_idx = 1:length(param.cmd.frms) + frm = param.cmd.frms(frm_idx); + + output_idx = output_idx + 1; + + % Recompute means + + % plot(metadata.fcs{1}{1}.gps_time, metadata.fcs{1}{1}.surface); hold on; + % plot(gps_time(img,:), surface(img,:)); + % plot(gps_time(img,:), abs(dsurface(img,:))); + + %% Segments: Frames: Compute mean + + [offset_ave,mask] = mean_without_outliers(peak_offset, 1, 0.2, 2); + + peak_val_mean = peak_val; + peak_val_mean(mask) = NaN; + peak_val_mean = nanmean(peak_val_mean, 2); + + Tsys_deg = angle(exp(1i*2*pi*offset_ave*dt * fc))*180/pi + Tsys = -offset_ave*dt + chan_equal_dB = db(nanmean(abs(peak_val_mean).^2, 2),'power') + chan_equal_deg = angle(peak_val_mean)*180/pi + + + + %% Segments: Frames: Store recomputed means + Tsys(output_idx) = sar_equal_output{param_idx}{frm}.Tsys; + Tsys_deg(output_idx) = sar_equal_output{param_idx}{frm}.Tsys_deg; + chan_equal_deg(output_idx) = sar_equal_output{param_idx}{frm}.chan_equal_deg; + chan_equal_dB(output_idx) = sar_equal_output{param_idx}{frm}.chan_equal_dB; + + frm_id = sprintf('%s_%03d\n', param.day_seg, frm); + frm_ids{output_idx} = frm_id; + +% plot(h_axes(1), sar_equal_output{param_idx}{frm} +% hold on; + end +end + +return + +%% Compute mean + +[offset_ave,mask] = mean_without_outliers(peak_offset, 1, 0.2, 2); + +peak_val_mean = peak_val; +peak_val_mean(mask) = NaN; +peak_val_mean = nanmean(peak_val_mean, 2); + +Tsys_deg = angle(exp(1i*2*pi*offset_ave*dt * fc))*180/pi +Tsys = -offset_ave*dt +chan_equal_dB = db(nanmean(abs(peak_val_mean).^2, 2),'power') +chan_equal_deg = angle(peak_val_mean)*180/pi + +%% Print results +new_wfs = param.radar.wfs; +for img = 1:length(param.sar_equal.imgs) + wf = param.sar_equal.imgs{img}(1,1); + adc = param.sar_equal.imgs{img}(1,2); + + if img == ref_img + fprintf('%% img %d wf %d adc %d (reference image)\n', img, wf, adc); + fprintf(' Tsys = old_Tsys + %g;\n', 0); + fprintf(' chan_equal_deg = old_chan_equal_deg + %g;\n', 0); + fprintf(' chan_equal_dB = old_chan_equal_dB + %g;\n', 0); + else + fprintf('%% img %d wf %d adc %d\n', img, wf, adc); + fprintf(' Tsys = Tsys + %g;\n', Tsys(img)); + fprintf(' chan_equal_deg = chan_equal_deg + %g;\n', chan_equal_deg(img) + Tsys_deg(img)); + fprintf(' chan_equal_dB = chan_equal_dB + %g;\n', chan_equal_dB(img)); + fprintf(' chan_equal_deg = chan_equal_deg + %g; %% Use this if not changing Tsys\n', chan_equal_deg(img)); + + for wf_adc = 1:size(param.sar_equal.imgs{img},1) + wf = param.sar_equal.imgs{img}(wf_adc,1); + adc = param.sar_equal.imgs{img}(wf_adc,2); + new_wfs(wf).Tsys(adc) = new_wfs(wf).Tsys(adc) + Tsys(img); + new_wfs(wf).chan_equal_deg(adc) = new_wfs(wf).chan_equal_deg(adc) + chan_equal_deg(img) + Tsys_deg(img); + new_wfs(wf).chan_equal_dB(adc) = new_wfs(wf).chan_equal_dB(adc) + chan_equal_dB(img); + end + end +end +for wf = 1:length(new_wfs) + fprintf('param_override.radar.wfs(%d).Tsys = %s;\n', wf, mat2str_generic(new_wfs(wf).Tsys)); + fprintf('param_override.radar.wfs(%d).chan_equal_deg = %s;\n', wf, mat2str_generic(new_wfs(wf).chan_equal_deg)); + fprintf('param_override.radar.wfs(%d).chan_equal_dB = %s;\n', wf, mat2str_generic(new_wfs(wf).chan_equal_dB)); +end + +%% Plot results +clf(h_fig(1)); +clf(h_fig(2)); +clf(h_fig(3)); +h_axes(1) = axes('parent',h_fig(1)); +h_axes(2) = axes('parent',h_fig(2)); +h_axes(3) = axes('parent',h_fig(3)); +legend_str = cell(size(data)); +for img = 1:length(param.sar_equal.imgs) + wf = param.sar_equal.imgs{img}(1,1); + adc = param.sar_equal.imgs{img}(1,2); + legend_str{img} = sprintf('img %d wf %d adc %d',img, wf, adc); + + plot(h_axes(1),angle(peak_val(img,:))*180/pi) + grid(h_axes(1),'on'); + hold(h_axes(1),'on'); + xlabel(h_axes(1),'Range line'); + ylabel(h_axes(1),'Angle (deg)'); + + plot(h_axes(2),db(peak_val(img,:))) + grid(h_axes(2),'on'); + hold(h_axes(2),'on'); + xlabel(h_axes(2),'Range line'); + ylabel(h_axes(2),'Relative power (dB)'); + + plot(h_axes(3),peak_offset(img,:),'.') + grid(h_axes(3),'on'); + hold(h_axes(3),'on'); + xlabel(h_axes(3),'Range line'); + ylabel(h_axes(3),'Time delay (bins)'); +end +legend(h_axes(1),legend_str); +legend(h_axes(2),legend_str); +legend(h_axes(3),legend_str); + +fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('angle_%02d_adc_%02d',wf,adc)) sprintf('_%03d.jpg',frm)]; +fprintf('Saving %s\n', fig_fn); +fig_fn_dir = fileparts(fig_fn); +if ~exist(fig_fn_dir,'dir') + mkdir(fig_fn_dir); +end +ct_saveas(h_fig(1),fig_fn); +fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('angle_%02d_adc_%02d',wf,adc)) sprintf('_%03d.fig',frm)]; +fprintf('Saving %s\n', fig_fn); +ct_saveas(h_fig(1),fig_fn); + +fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('abs_%02d_adc_%02d',wf,adc)) sprintf('_%03d.jpg',frm)]; +fprintf('Saving %s\n', fig_fn); +fig_fn_dir = fileparts(fig_fn); +if ~exist(fig_fn_dir,'dir') + mkdir(fig_fn_dir); +end +ct_saveas(h_fig(2),fig_fn); +fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('abs_%02d_adc_%02d',wf,adc)) sprintf('_%03d.fig',frm)]; +fprintf('Saving %s\n', fig_fn); +ct_saveas(h_fig(3),fig_fn); + +fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('td_%02d_adc_%02d',wf,adc)) sprintf('_%03d.jpg',frm)]; +fprintf('Saving %s\n', fig_fn); +fig_fn_dir = fileparts(fig_fn); +if ~exist(fig_fn_dir,'dir') + mkdir(fig_fn_dir); +end +ct_saveas(h_fig(3),fig_fn); +fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('td_%02d_adc_%02d',wf,adc)) sprintf('_%03d.fig',frm)]; +fprintf('Saving %s\n', fig_fn); +ct_saveas(h_fig(3),fig_fn); + +%% Save outputs +mat_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('sar_equal_wf_%02d_adc_%02d',wf,adc)) sprintf('_%03d.mat',frm)]; +fprintf('Saving %s\n', mat_fn); +mat_fn_dir = fileparts(mat_fn); +if ~exist(mat_fn_dir,'dir') + mkdir(mat_fn_dir); +end +param_sar_equal = param; +ct_save(mat_fn,'peak_offset','peak_val','Tsys','Tsys_deg','chan_equal_dB','chan_equal_deg','gps_time','surface','param_sar_equal'); + diff --git a/cresis-toolbox/processing/run_update_collate_deconv.m b/cresis-toolbox/processing/run_update_collate_deconv.m deleted file mode 100644 index 0b18073a..00000000 --- a/cresis-toolbox/processing/run_update_collate_deconv.m +++ /dev/null @@ -1,61 +0,0 @@ -% Script run_update_collate_deconv.m -% -% Runs update_collate_deconv.m -% -% Author: John Paden - -%% USER SETTINGS -% ========================================================================= - -param_override = []; - -params = read_param_xls(ct_filename_param('snow_param_2017_Greenland_P3.xls'),'',{'analysis_spec' 'analysis'}); - -params = ct_set_params(params,'cmd.generic',0); -params = ct_set_params(params,'cmd.generic',1,'day_seg','20170327_01'); -param_override.update_collate_deconv.cmd = {}; -% param_override.update_collate_deconv.cmd{end+1}.method = 'delete'; -% param_override.update_collate_deconv.cmd{end}.idxs = [1 2 3]; -param_override.update_collate_deconv.cmd{end+1}.method = 'replace'; -param_override.update_collate_deconv.cmd{end}.day_seg = {'20170323_02'} -% param_override.update_collate_deconv.cmd{end}.day_seg = {'20170320_01'} -param_override.update_collate_deconv.cmd{end}.idxs = {[1]}; - -% 2-18 GHz Deconvolution Settings (3 sets) -% params = ct_set_params(params,'analysis.cmd{1}.abs_metric',[58 4.5 -25 -35 inf inf]); -% param_override.update_collate_deconv.in_dir = 'analysis_uwb'; - -% params = ct_set_params(params,'analysis.cmd{1}.abs_metric',[58 9.8 -25 -35 inf inf]); -% param_override.update_collate_deconv.in_dir = 'analysis'; - -% params = ct_set_params(params,'analysis.cmd{1}.abs_metric',[58 24 -25 -28 inf inf]); -% param_override.update_collate_deconv.in_dir = 'analysis_kuband'; - -% 2-8 GHz Deconvolution Settings -params = ct_set_params(params,'analysis.cmd{1}.abs_metric',[65 4.5 -25 -35 inf inf]); -param_override.update_collate_deconv.in_dir = 'analysis'; - -% param_override.update_collate_deconv.debug_plots = {'final','visible'}; -param_override.update_collate_deconv.debug_plots = {'final'}; - -%% Automated Section -% ===================================================================== - -% Input checking -global gRadar; -if exist('param_override','var') - param_override = merge_structs(gRadar,param_override); -else - param_override = gRadar; -end - -% Process each of the segments -for param_idx = 1:length(params) - param = params(param_idx); - if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic - continue; - end - update_collate_deconv(param,param_override); - %update_collate_deconv - -end diff --git a/cresis-toolbox/processing/run_update_img_combine.m b/cresis-toolbox/processing/run_update_img_combine.m deleted file mode 100644 index 19470c9c..00000000 --- a/cresis-toolbox/processing/run_update_img_combine.m +++ /dev/null @@ -1,45 +0,0 @@ -% script run_update_img_combine -% -% Script for running update_img_combine -% -% Authors: John Paden -% -% See also: run_update_img_combine.m, update_img_combine.m - -%% User Setup -% ===================================================================== -params = read_param_xls(ct_filename_param('accum_param_2018_Antarctica_TObas.xls'),''); -params = ct_set_params(params,'cmd.generic',0); -params = ct_set_params(params,'cmd.generic',1,'day_seg','20190201_01'); - - -mode = 'qlook'; % <== OFTEN CHANGED (qlook or array) - -update_img_combine_param.out_path = 'qlook'; -update_img_combine_param.img_comb_mult = inf; % <== OFTEN CHANGED (inf default) -update_img_combine_param.img_comb_bins = 1; % <== OFTEN CHANGED (1 default) -update_img_combine_param.img_comb_layer_params = struct('name','surface','source','layerdata','layerdata_source','layerData');% <== OFTEN CHANGED - -%% Automated Section -% ===================================================================== - -param_override = []; -param_override.(mode) = update_img_combine_param; -param_override.update_img_combine.mode = mode; - -% Input checking -global gRadar; -if exist('param_override','var') - param_override = merge_structs(gRadar,param_override); -else - param_override = gRadar; -end - -% Process each of the segments -for param_idx = 1:length(params) - param = params(param_idx); - if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic - continue; - end - update_img_combine(param,param_override); -end diff --git a/cresis-toolbox/processing/run_update_layerdata_format.m b/cresis-toolbox/processing/run_update_layerdata_format.m deleted file mode 100644 index d6fb02c7..00000000 --- a/cresis-toolbox/processing/run_update_layerdata_format.m +++ /dev/null @@ -1,48 +0,0 @@ -% script run_update_layerdata_format -% -% Script for running update_layerdata_format.m -% -% Authors: Jilu Li -% -% See also: update_layerdata_format.m, update_layerdata_format_task.m - -%% User Setup -% ===================================================================== -param_override = []; - -params = read_param_xls(ct_filename_param('rds_param_2012_Greenland_P3.xls')); -params = ct_set_params(params,'cmd.generic',0); -params = ct_set_params(params,'cmd.generic',1,'day_seg','20120330_01'); -% params = ct_set_params(params,'cmd.generic',1,'day_seg','20120330_01|20120330_02|20120330_03'); -params = ct_set_params(params,'cmd.frms',[1,2]); - -param_override.frame_overlap_removal = true; -% param_override.cluster.type = 'debug'; -% param_override.cluster.type = 'matlab'; -param_override.cluster.type = 'torque'; -param_override.cluster.rerun_only = false; - -%% Automated Section -% ===================================================================== - -% Input checking -global gRadar; -if exist('param_override','var') - param_override = merge_structs(gRadar,param_override); -else - param_override = gRadar; -end - -% Process each of the segments -ctrl_chain = {}; -for param_idx = 1:length(params) - param = params(param_idx); - if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic - continue; - end - ctrl_chain{end+1} = update_layerdata_format(param,param_override); -end - -cluster_print_chain(ctrl_chain); - -[chain_fn,chain_id] = cluster_save_chain(ctrl_chain); \ No newline at end of file diff --git a/cresis-toolbox/processing/rx_chan_equal.m b/cresis-toolbox/processing/rx_chan_equal.m new file mode 100644 index 00000000..68ba3dcf --- /dev/null +++ b/cresis-toolbox/processing/rx_chan_equal.m @@ -0,0 +1,544 @@ +function [td_out,amp_out,phase_out,full_out] = rx_chan_equal(data,param,hdr) +% [td_out,amp_out,phase_out,full_out] = rx_chan_equal(data,param,hdr) +% +% Inputs: +% data = Nt by Nx by Nc single/double matrix +% Nt = fast time +% Nx = slow time +% Nc = cross-track channels +% param = structure controlling receiver channel equalization +% .img = single wf-adc list (Nc by 2 matrix) +% List of wf-adc pairs. +% [wf adc; wf adc; ... ; wf adc]; +% hdr.img(wf_adc_idx,:) corresponds to data(:,:,wf_adc_idx) +% .lever_arm_fh = function handle to lever arm code +% no default supplied, but not required if param.mocomp_type equals 0 +% .mocomp_type = scalar integer (see param.type in motion_comp.m) +% .td = time delay correction for each wf_adc pair (Nc by 1 vector) +% e.g. -12 ns implies channel is delayed by 12 ns so 12 ns of delay +% will be removed (targets will move to a closer range) +% default is all zeros, units of seconds +% .phase = phase correction for each wf_adc pair (Nc by 1 vector) +% e.g. 35 deg implies channel leads by 35 deg so -35 deg phase will be +% applied (target phases will have -35 deg phase from original) +% default is all zeros, units of degrees (angle(voltage)*180/pi) +% .amp = amplitude correction for each wf_adc pair (Nc by 1 vector) +% e.g. 2 dB implies channel is 2 dB larger so -2 dB amplitude adjustment +% will be applied (targets will be 2 dB smaller) +% default is all zeros, units of log power (20*log(voltage)) +% .plot_en = enable plotting of outputs +% default is false +% .delay: struct describing how the time delay between channels is found +% .method: string containing the system time delay method. The options +% are: +% 'xcorr_complex': Finds time delay between channels by finding the +% lag of the peak of the cross correlation. This is the default. +% 'peak': Finds time delay between channels by finding the offset +% between the peak values. +% .ref_bins: Only required for the xcorr_complex delay method. Sets the +% bins to use from the reference wf-adc channel. +% [-20 20] uses 20 bins before and after the surface +% Default is [-20 20]. +% .search_bins: Depends on the delay method used. The options +% are: +% 'xcorr_complex': For the channel being compared to the reference, +% [-7 7] uses 7 bins before and after the surface. +% 'peak': [-7 7] searches 7 bins backward and forward from the +% surface bin for the peak. +% Default is [-7 7]. +% .Mt: over-sampling factor to use in determining delay. Default is 64. +% hdr = structure describing data +% .{lat,lon,elev,roll,pitch,heading} +% 1 by Nx vectors +% Not required if param.mocomp_mode is 0 or 1 +% lat,lon in degrees +% elev in meters +% roll, pitch, heading in radians +% .time = double Nt by 1 vector, fast-time axis of data, seconds +% +% Outputs: +% td_out = recommended hdr.td +% phase_out = recommended hdr.phase +% amp_out = recommended hdr.amp +% +% Authors: Logan Smith, John Paden, Jilu Li + +%% Input arguments and general setup +% ====================================================================== + +% rlines,rbins: Format inputs to select region to find max signal +param.noise_rlines = round(param.noise_rlines); +param.noise_rbins= round(param.noise_rbins); +param.noise_rlines = sort(param.noise_rlines); +param.noise_rbins = sort(param.noise_rbins); +if param.noise_rbins(1) < 1 + param.noise_rbins(1) = 1; +end +if param.noise_rbins(end) > size(data,1) + param.noise_rbins(end) = size(data,1); +end +if param.noise_rlines(1) < 1 + param.noise_rlines(1) = 1; +end +if param.noise_rlines(end) > size(data,2) + param.noise_rlines(end) = size(data,2); +end +noise_rlines = param.noise_rlines(1):param.noise_rlines(end); +noise_rbins = param.noise_rbins(1):param.noise_rbins(end); + +param.rlines = round(param.rlines); +param.rbins= round(param.rbins); +param.rlines = sort(param.rlines); +param.rbins = sort(param.rbins); +if param.rbins(1) < 1 + param.rbins(1) = 1; +end +if param.rbins(end) > size(data,1) + param.rbins(end) = size(data,1); +end +if param.rlines(1) < 1 + param.rlines(1) = 1; +end +if param.rlines(end) > size(data,2) + param.rlines(end) = size(data,2); +end +rlines = param.rlines(1):param.rlines(end); +rbins = param.rbins(1):param.rbins(end); + +% ref_idx: index into data that will be the reference +ref_idx = param.config.ref_wf_adc; + +% colors: used for plotting +colors = {'k.','r.','y.','g.','c.','b.','m.','kx','rx','yx','gx','cx','bx','mx','ko','ro','yo','go','co','bo','mo','k+','r+','y+'}; + +% ave_fh: averaging function handle +ave_fh = param.averaging_fh; + +% freq,time: extract frequency and time axes +freq = param.freq; +time = param.time; + +if ~isfield(param.config,'delay') + param.config.delay = []; +end + +if ~isfield(param.config.delay,'method') || isempty(param.config.delay.method) + param.config.delay.method = 'xcorr_complex'; +end + +%% Setup code for estimation equalization coefficients +switch(param.config.delay.method) + case 'threshold' + delay_method = 1; + case 'xcorr_complex' + delay_method = 2; + case 'xcorr_magnitude' + delay_method = 3; + case 'peak' + delay_method = 4; + otherwise + error('delay.method %d is not supported', param.analysis.surf.delay.method); +end + +if ~isfield(param.config.delay,'ref_bins') || isempty(param.config.delay.ref_bins) + param.config.delay.ref_bins = -20:20; +end + +if ~isfield(param.config.delay,'search_bins') || isempty(param.config.delay.search_bins) + param.config.delay.search_bins = -7:7; +end + +if ~isfield(param.config.delay,'Mt') || isempty(param.config.delay.Mt) + param.config.delay.Mt = 64; +end + +if ~isfield(param,'td') || isempty(param.td) + param.td = zeros(size(data,3),1); +end + +if ~isfield(param,'amp') || isempty(param.amp) + param.amp = zeros(size(data,3),1); +end + +if ~isfield(param,'phase') || isempty(param.phase) + param.phase = zeros(size(data,3),1); +end + +if ~isfield(param,'coherent_noise_removal') || isempty(param.coherent_noise_removal) + param.coherent_noise_removal = false; +end + +if ~isfield(param,'multilook') || isempty(param.multilook) + param.multilook = ones(1,7)/7; +end + +if ~isfield(param,'plot_en') || isempty(param.plot_en) + param.plot_en = false; +end + +clear full_out; + +%% Prepare surface data +% ====================================================================== +if param.coherent_noise_removal + data = data - repmat(mean(data,2), [1 size(data,2) 1]); +end + +%% Motion compensation +% ====================================================================== +mocomp_param.type = param.mocomp_type; +mocomp_param.tx_weights = param.tx_weights; +mocomp_param.season_name = param.season_name; +mocomp_param.radar_name = param.radar_name; +mocomp_param.gps_source = hdr.gps_source; +for wf_adc_idx = 1:size(data,3) + wf = abs(param.config.img(wf_adc_idx,1)); + adc = param.config.img(wf_adc_idx,2); + mocomp_param.rx = param.rx_paths{wf}(adc); + + % drange = change in range (positive is longer range) + drange = basic_motion_comp(mocomp_param,param.lever_arm_fh,hdr.roll, ... + hdr.pitch,hdr.heading,hdr.lat,hdr.lon,hdr.elev); + % dtime = Convert to time (in air), positive is longer range/time-delay + dtime = 2*drange/3e8; + Nt = size(data,1); + Nx = size(data,2); + if 0 + figure(1); clf; + plot(dtime*1e9); + title('adc %d'); + pause; + end + % Time shift data in the frequency domain + % A positive dtime implies a larger negative phase delay (since + % delay is negative/lagging phase) + data(:,:,wf_adc_idx) = ifft(fft(data(:,:,wf_adc_idx)) ... + .*exp(-1i*2*pi*repmat(freq,1,Nx).*repmat(dtime,Nt,1))); +end + +num_chan = size(data,3); + +%% Apply time correction +% Time delay is removed (positive moves targets closer in range) +% ======================================================================= +w = 2*pi*freq; +for wf_adc_idx = 1:size(data,3) + wf = abs(param.config.img(wf_adc_idx,1)); + adc = param.config.img(wf_adc_idx,2); + data(:,:,wf_adc_idx) = ifft(fft(data(:,:,wf_adc_idx)).*exp(1i*repmat(w,1,size(data,2))*param.td(param.rx_paths{wf}(adc)))); +end + +%% Apply amplitude and phase correction +% Amp/phase are DIVIDED out as opposed to being multiplied +% ======================================================================= + +for wf_adc_idx = 1:size(data,3) + wf = abs(param.config.img(wf_adc_idx,1)); + adc = param.config.img(wf_adc_idx,2); + data(:,:,wf_adc_idx) = data(:,:,wf_adc_idx) ./ (10^(param.amp(param.rx_paths{wf}(adc))/20).*exp(1i*param.phase(param.rx_paths{wf}(adc))/180*pi)); +end +ascope_check_flag = 0; % compare average ascopes before and after equalization +if ascope_check_flag + data_combine = sum(data,3); + figure(4);plot(mean(lp(data_combine),2),'r');grid;legend('no eq','with eq');hold off; + disp('press any key to continue') + pause +end + +%% Surface tracker +% ======================================================================= +if param.combine_channels + surf_data = mean(data,3); +else + surf_data = data(:,:,ref_idx); +end +surf_data = fir_dec(abs(surf_data).^2,param.multilook,1); + +[surf_vals surf_bins] = max(surf_data(rbins,rlines)); +surf_bins = rbins(1)-1 + surf_bins; + +surface_tracker_check_flag = 1; +if surface_tracker_check_flag + % Debug code for checking surface tracker + figure(1000); clf; + %imagesc([],rbins,lp(data(rbins,:,ref_idx))); + imagesc([],rbins,lp(surf_data(rbins,:))); + colormap(1-gray(256)); + hold on; + plot(rlines, surf_bins,'r-.'); + hold off; + xlabel('Record'); + ylabel('Range bin'); + title('Echogram with surface track result'); +end + +if 0 + % Check that all channels are good + sig_power = zeros(size(data,3),length(rlines)); + for rline_idx = 1:length(rlines) + rline = rlines(rline_idx); + sig_power(:,rline_idx) = data(surf_bins(rline_idx),rline,:); + end +else + % Check only that the reference channel is good + sig_power = zeros(1,length(rlines)); + for rline_idx = 1:length(rlines) + rline = rlines(rline_idx); + sig_power(:,rline_idx) = data(surf_bins(rline_idx),rline,ref_idx); + end +end + +noise_power = mean(mean(abs(data(noise_rbins,noise_rlines)).^2)); +good_rlines = find(all(lp(sig_power.') > lp(noise_power) + param.snr_threshold,2)); +data = data(:,good_rlines,:); +surf_bins = surf_bins(good_rlines); +rlines = 1:length(good_rlines); + +data_check_flag = 0; +if data_check_flag + lp(noise_power) + h_axes = []; + for h_fig = 1:size(data,3) + figure(h_fig+100); clf; + set(h_fig+100,'WindowStyle','docked'); + imagesc(lp(data(:,:,h_fig))); + colorbar; + hold on; + plot(surf_bins); + hold off; + h_axes(end+1) = gca; + xlabel('Record'); + ylabel('Range bin'); + h = colorbar; + set(get(h,'YLabel'),'String','Relative power (dB)'); + title(sprintf('wf_adc pair %d',h_fig),'interpreter','none') + ylim([max(1,rbins(1)-10) min(size(data,1),rbins(end)+10)]); + end + linkaxes(h_axes,'xy'); +end + +if ascope_check_flag + data_combine = sum(data,3); + figure(4);plot(mean(lp(data_combine),2));hold on; +end + +%% Cross correlation to determine recommended time, phase, and amplitude offsets +% ======================================================================= +ref_bins = param.config.delay.ref_bins(1) : param.config.delay.ref_bins(end); +search_bins = param.config.delay.search_bins(1) : param.config.delay.search_bins(end); +zero_padding_offset = length(search_bins) - length(ref_bins); +Hcorr_wind = hanning(length(ref_bins)); +clear peak_val peak_offset; + +peak_val = zeros(size(data,3),length(rlines)); +peak_offset = zeros(size(data,3),length(rlines)); + +if delay_method == 2 + %% Cross correlation method with complex data + for adc_idx = 1:size(data,3) + for rline_idx = 1:length(rlines) + rline = rlines(rline_idx); + [corr_out,lags] = xcorr(data(surf_bins(rline_idx)+search_bins,rline,adc_idx), ... + data(surf_bins(rline_idx)+ref_bins,rline,ref_idx) .* Hcorr_wind); + corr_int = interpft(corr_out,param.config.delay.Mt*length(corr_out)); + [peak_val(adc_idx,rline_idx) peak_offset(adc_idx,rline_idx)] = max(corr_int); + peak_val(adc_idx,rline_idx) = abs(max(data(surf_bins(rline_idx)+search_bins,rline,adc_idx))) ... + .*exp(1i*angle(peak_val(adc_idx,rline_idx))); + peak_offset(adc_idx,rline_idx) = (peak_offset(adc_idx,rline_idx)-1)/param.config.delay.Mt+1 ... + + ref_bins(1) + search_bins(1) - 1 - zero_padding_offset; + end + end + + peak_offset = peak_offset - repmat(peak_offset(ref_idx,:),[size(peak_offset,1),1]); + dt = (time(2)-time(1)); + +elseif delay_method == 3 + %% Cross correlation method with magnitude data + error('Not finished'); + +elseif delay_method == 3 + %% Threshold method + error('Not finished'); + +elseif delay_method == 4 + %% Peak method + for rline_idx = 1:length(rlines) + rline = rlines(rline_idx); + data_int = interpft(data(surf_bins(rline_idx)+search_bins,rline,ref_idx),param.config.delay.Mt*length(search_bins)); + [peak_val(ref_idx,rline_idx),peak_offset(ref_idx,rline_idx)] = max(data_int); + end + + adc_idxs = 1:size(data,3); + adc_idxs(adc_idxs==ref_idx) = []; + for adc_idx = adc_idxs + for rline_idx = 1:length(rlines) + rline = rlines(rline_idx); + data_int = interpft(data(surf_bins(rline_idx)+search_bins,rline,adc_idx),param.config.delay.Mt*length(search_bins)); + [peak_val(adc_idx,rline_idx),peak_offset(adc_idx,rline_idx)] = max(data_int); + peak_offset(adc_idx,rline_idx) = peak_offset(adc_idx,rline_idx) - peak_offset(ref_idx,rline_idx); + peak_val(adc_idx,rline_idx) = abs(peak_val(adc_idx,rline_idx)).*exp(1i*angle(data_int(peak_offset(ref_idx,rline_idx)))); + end + end + peak_offset = peak_offset / param.config.delay.Mt; + peak_offset(ref_idx,:) = 0; + dt = (time(2)-time(1)); +else + error('method not supported'); +end + +%% Roll Estimation +param.roll_est.bin_rng = 0; +param.roll_est.rline_rng = -5:5; +param.roll_est.Nsig = 1; +y_offset = zeros(size(data,3),1); +z_offset = zeros(size(data,3),1); +lever_arm_param.season_name = param.season_name; +lever_arm_param.radar_name = ct_output_dir(param.radar_name); +lever_arm_param.gps_source = hdr.gps_source; +for wf_adc_idx = 1:size(data,3) + wf = abs(param.config.img(wf_adc_idx,1)); + adc = param.config.img(wf_adc_idx,2); + mocomp_param.rx = param.rx_paths{wf}(adc); + phase_center = lever_arm(lever_arm_param, param.tx_weights, mocomp_param.rx); + y_offset(wf_adc_idx) = -phase_center(2); + z_offset(wf_adc_idx) = -phase_center(3); +end + +% [theta,sv] = array_proc_sv(256,param.freq(1), y_offset, z_offset); +[theta,sv] = array_proc_sv(param.freq(1), y_offset, z_offset, 256); + +theta = fftshift(theta); +sv = fftshift(sv,2); + +Nt = size(data,1); +Nx = size(data,2); +Nc = size(data,3); +roll_est_val = []; +roll_est_theta = []; +for rline = 1:length(rlines) + bin = surf_bins(rline); + + if rline+param.roll_est.rline_rng(1) < 1 + start_rline_rng = 1-rline; + else + start_rline_rng = param.roll_est.rline_rng(1); + end + if rline+param.roll_est.rline_rng(end) > Nx + stop_rline_rng = Nx-rline; + else + stop_rline_rng = param.roll_est.rline_rng(end); + end + rline_rng = start_rline_rng : stop_rline_rng; + + if bin+param.roll_est.bin_rng(1) < 1 + bin_rng = 1-bin : param.roll_est.bin_rng(end); + elseif bin+param.roll_est.bin_rng(end) > Nt + bin_rng = param.roll_est.bin_rng(1) : Nt-bin; + else + bin_rng = param.roll_est.bin_rng; + end + + dataSample = data(bin+bin_rng,rline+rline_rng,:); + dataSample = reshape(dataSample,[length(bin_rng)*length(rline_rng) Nc]).'; + + Rxx = 1/size(dataSample,1) * (dataSample * dataSample'); + [V,D] = eig(Rxx); + eigenVals = diag(D); + [eigenVals noiseIdxs] = sort(eigenVals); + noiseIdxs = noiseIdxs(1:end-param.roll_est.Nsig); + music_pattern = mean(abs(sv'*V(:,noiseIdxs)).^2,2); + [roll_est_val(rline),roll_est_idx] = min(music_pattern); + roll_est_theta(rline) = theta(roll_est_idx); +end +full_out.roll_est_theta = roll_est_theta; +full_out.roll_est_val = roll_est_val; +full_out.gps_time = hdr.gps_time(rlines); + +%% Calculate corrections +% ======================================================================= + +ref_val = zeros(size(data,3),1); +ref_val_pow = zeros(size(data,3),1); +full_out.peak_ref = zeros(size(peak_val)); +full_out.peak_offset = peak_offset; +for wf_adc_idx = 1:size(data,3) + full_out.peak_ref(wf_adc_idx,:) = peak_val(wf_adc_idx,:)./peak_val(ref_idx,:); + ref_val(wf_adc_idx) = ave_fh(peak_val(wf_adc_idx,:)./peak_val(ref_idx,:),2); + ref_val_pow(wf_adc_idx) = ave_fh(abs(peak_val(wf_adc_idx,:)./peak_val(ref_idx,:)).^2,2); +end + +td_out = param.td; +amp_out = param.amp; +phase_out = param.phase; +for wf_adc_idx = 1:size(data,3) + wf = abs(param.config.img(wf_adc_idx,1)); + adc = param.config.img(wf_adc_idx,2); + td_out(param.rx_paths{wf}(adc)) = param.td(param.rx_paths{wf}(adc)) + ave_fh(peak_offset(wf_adc_idx,:))*dt; + amp_out(param.rx_paths{wf}(adc)) = param.amp(param.rx_paths{wf}(adc)) + lp(ref_val_pow(wf_adc_idx),1); + phase_out(param.rx_paths{wf}(adc)) = param.phase(param.rx_paths{wf}(adc)) + angle(ref_val(wf_adc_idx))*180/pi; +end + +%% Optional printing and plotting +% ======================================================================= + +if param.plot_en + figure(1); clf; + ha1 = axes; hold on; + figure(2); clf; + ha2 = axes; hold on; + figure(3); clf; + subplot(3,1,1); + plot(hdr.roll(good_rlines)*180/pi) + ylabel('roll (deg)'); + subplot(3,1,2:3); + plot([]); ha3 = gca; hold on; + clear leg_cell; + for wf_adc_idx = 1:size(data,3) + wf = abs(param.config.img(wf_adc_idx,1)); + adc = param.config.img(wf_adc_idx,2); + rx = param.rx_paths{wf}(adc); + plot(ha1,peak_offset(wf_adc_idx,:) - peak_offset(ref_idx,:),[colors{mod(wf_adc_idx-1,length(colors))+1}]) + plot(ha2,lp(peak_val(wf_adc_idx,:)./peak_val(ref_idx,:),2),[colors{mod(wf_adc_idx-1,length(colors))+1}]) + plot(ha3,180/pi*angle(peak_val(wf_adc_idx,:)./peak_val(ref_idx,:)),[colors{mod(wf_adc_idx-1,length(colors))+1}]) + leg_cell{wf_adc_idx} = sprintf('rx %d',rx); + end + for wf_adc_idx = 1:size(data,3) + plot(ha1,[1 length(rlines)],[ave_fh(peak_offset(wf_adc_idx,:) - peak_offset(ref_idx,:)) ave_fh(peak_offset(wf_adc_idx,:) - peak_offset(ref_idx,:))],colors{mod(wf_adc_idx-1,length(colors))+1}) + plot(ha2,[1 length(rlines)],[lp(ref_val_pow(wf_adc_idx),1) lp(ref_val_pow(wf_adc_idx),1)],colors{mod(wf_adc_idx-1,length(colors))+1}) + plot(ha3,[1 length(rlines)],180/pi*[angle(ave_fh(peak_val(wf_adc_idx,:)./peak_val(ref_idx,:))) angle(ave_fh(peak_val(wf_adc_idx,:)./peak_val(ref_idx,:)))],colors{mod(wf_adc_idx-1,length(colors))+1}) + end + title(ha1,'index offset') + title(ha2,'power offset') + title(ha3,'phase offset') + legend(ha1,leg_cell); + legend(ha2,leg_cell); + legend(ha3,leg_cell); + hold(ha1,'off') + hold(ha2,'off') + hold(ha3,'off') + drawnow; + + fprintf('Peak offset indices:\n'); + fprintf('%.1f\t', ave_fh(peak_offset(1:end-1,:),2)); + fprintf('%.1f', ave_fh(peak_offset(end,:),2)); + fprintf('\n'); + + fprintf('Recommended td settings (ns):\n'); + fprintf('%.1f\t', td_out(1:end-1)*1e9); + fprintf('%.1f', td_out(end)*1e9); + fprintf('\n'); + + fprintf('Recommended amp settings (dB):\n'); + fprintf('%.1f\t', amp_out(1:end-1)); + fprintf('%.1f', amp_out(end)); + fprintf('\n'); + + fprintf('Recommended phase settings (deg):\n'); + fprintf('%.1f\t', phase_out(1:end-1)); + fprintf('%.1f', phase_out(end)); + fprintf('\n'); + +end + +return; + diff --git a/cresis-toolbox/processing/sar.m b/cresis-toolbox/processing/sar.m index f837580d..a2ab5dc1 100644 --- a/cresis-toolbox/processing/sar.m +++ b/cresis-toolbox/processing/sar.m @@ -22,7 +22,9 @@ %% General Setup % ===================================================================== -param = merge_structs(param, param_override); +if exist('param_override','var') + param = merge_structs(param, param_override); +end fprintf('=====================================================================\n'); fprintf('%s: %s (%s)\n', mfilename, param.day_seg, datestr(now)); @@ -49,8 +51,9 @@ end if ~isfield(param.sar,'bit_mask') || isempty(param.sar.bit_mask) - % Remove bad records (bit_mask==1) and stationary records (bit_mask==2) - param.sar.bit_mask = 3; + % Remove bad records (bit_mask==1), remove stationary records + % (bit_mask==2), and remove bad records (bit_mask==4) + param.sar.bit_mask = 1 + 2 + 4; end if ~isfield(param.sar,'combine_rx') || isempty(param.sar.combine_rx) @@ -292,8 +295,8 @@ sparam.cpu_time = 60 + Nx*cpu_time_mult; records_var = whos('records'); sparam.mem = 250e6 + records_var.bytes*mem_mult; - sparam.notes = sprintf('%s:%s:%s %s', ... - sparam.task_function, param.radar_name, param.season_name, param.day_seg); + sparam.notes = sprintf('%s %s:%s:%s %s', ... + sparam.task_function, param.sar.out_path, param.radar_name, param.season_name, param.day_seg); % Create success condition success_error = 64; @@ -390,7 +393,7 @@ for wf_adc = 1:size(param.sar.imgs{img},1) wf = param.sar.imgs{img}(wf_adc,1); adc = param.sar.imgs{img}(wf_adc,2); - [board,board_idx,profile] = wf_adc_to_board(param,[wf adc]); + [board,board_idx,~] = wf_adc_to_board(param,[wf adc]); if length(imgs_list) < board_idx imgs_list{board_idx} = {}; @@ -509,6 +512,7 @@ % Estimate number of input range lines per chunk num_rlines_per_chunk = round((stop_rec-start_rec) / num_chunks); + cur_rec = start_rec; for chunk_idx = 1:num_chunks % Setup dynamic params % ===================================================================== @@ -520,6 +524,14 @@ else dparam.argsin{1}.load.recs = start_rec + num_rlines_per_chunk*(chunk_idx-1) + [0, num_rlines_per_chunk-1]; end + if chunk_idx == num_chunks + dparam.argsin{1}.load.recs = [cur_rec, stop_rec]; + else + dparam.argsin{1}.load.recs = cur_rec-1+[find(along_track_approx(cur_rec:end)-along_track_approx(start_rec) >= (chunk_idx-1)*param.sar.chunk_len,1), ... + find(along_track_approx(cur_rec:end)-along_track_approx(start_rec) < chunk_idx*param.sar.chunk_len,1,'last')]; + end + dparam.argsin{1}.load.recs + cur_rec = dparam.argsin{1}.load.recs(2)+1; for imgs_idx = 1:length(imgs_list) if isempty(imgs_list{imgs_idx}) @@ -601,8 +613,8 @@ % Rerun only mode: Test to see if we need to run this task % ================================================================= - dparam.notes = sprintf('%s:%s:%s %s_%03d (%d of %d)/%d of %d %s %.0f to %.0f recs', ... - sparam.task_function, param.radar_name, param.season_name, param.day_seg, frm, frm_idx, length(param.cmd.frms), ... + dparam.notes = sprintf('%s %s:%s:%s %s_%03d (%d of %d)/%d of %d %s %.0f to %.0f recs', ... + sparam.task_function, param.sar.out_path, param.radar_name, param.season_name, param.day_seg, frm, frm_idx, length(param.cmd.frms), ... chunk_idx, num_chunks, wf_adc_str, (dparam.argsin{1}.load.recs(1)-1)*param.sar.presums+1, ... dparam.argsin{1}.load.recs(2)*param.sar.presums); if ctrl.cluster.rerun_only @@ -685,8 +697,8 @@ wf = tmp_dparam.argsin{1}.load.imgs{1}(1,1); adc = tmp_dparam.argsin{1}.load.imgs{1}(1,2); wf_adc_str = sprintf('%d,%d', wf, adc); - tmp_dparam.notes = sprintf('%s:%s:%s %s_%03d (%d of %d)/%d of %d %s %.0f to %.0f recs', ... - sparam.task_function, param.radar_name, param.season_name, param.day_seg, frm, frm_idx, length(param.cmd.frms), ... + tmp_dparam.notes = sprintf('%s %s:%s:%s %s_%03d (%d of %d)/%d of %d %s %.0f to %.0f recs', ... + sparam.task_function, param.sar.out_path, param.radar_name, param.season_name, param.day_seg, frm, frm_idx, length(param.cmd.frms), ... chunk_idx, num_chunks, wf_adc_str, (dparam.argsin{1}.load.recs(1)-1)*param.sar.presums+1, ... dparam.argsin{1}.load.recs(2)*param.sar.presums); diff --git a/cresis-toolbox/processing/sar_coord_task.m b/cresis-toolbox/processing/sar_coord_task.m index 3b9471d9..23b1b2f3 100644 --- a/cresis-toolbox/processing/sar_coord_task.m +++ b/cresis-toolbox/processing/sar_coord_task.m @@ -23,8 +23,7 @@ [~,~,radar_name] = ct_output_dir(param.radar_name); % Load records file -records_fn = ct_filename_support(param,'','records'); -records = load(records_fn); +records = records_load(param); % Apply presumming if param.sar.presums > 1 records.lat = fir_dec(records.lat,param.sar.presums); @@ -100,9 +99,9 @@ if length(surf_idxs) < frame_length if mod(length(surf_idxs),2) == 0 % Even length(surf_idxs), but sgolayfilt filter must be odd length - surf = sgolayfilt(records.surface(surf_idxs),3,length(surf_idxs)-1); + surf = sgolayfilt(records.surface(surf_idxs),min(3,length(surf_idxs)-2),length(surf_idxs)-1); else - surf = sgolayfilt(records.surface(surf_idxs),3,length(surf_idxs)); + surf = sgolayfilt(records.surface(surf_idxs),min(3,length(surf_idxs)-1),length(surf_idxs)); end else surf = sgolayfilt(records.surface(surf_idxs),3,frame_length); diff --git a/cresis-toolbox/processing/sar_equal.m b/cresis-toolbox/processing/sar_equal.m new file mode 100644 index 00000000..f49f5ed0 --- /dev/null +++ b/cresis-toolbox/processing/sar_equal.m @@ -0,0 +1,640 @@ +% function sar_equal(param,param_override) +% sar_equal(param,param_override) +% +% Perform equalization using SAR data. Since images are what are compared, +% the images can be made up of individual wf-adc pairs and/or groups of +% wf-adc pairs. This program is similar to collate_equal.m in that it +% assumes the target is at nadir and motion compensation is done for the +% nadir direction of arrival. Any residual time delay, phase, or amplitude +% differences are assumed to be system effects and the average of these are +% the new equalization coefficients that are recommended. +% +% This function was created for measuring the time and phase offset between +% waveforms. In this mode the imgs field generally combines all the adc's +% together for each waveform to increase the SNR and reduce side clutter. +% +% Use run_sar_equal.m to run sar_equal.m. After running, use +% run_sar_equal_post.m to combine results from different segments and +% frames to find the overall mean and trends. +% +% Inputs +% ========================================================================= +% +% param: struct with processing parameters +% +% param_override: parameters in this struct will override parameters in +% param. This struct must also contain the gRadar fields. Typically global +% gRadar; param_override = gRadar; +% +% Outputs +% ========================================================================= +% +% No outputs besides debug plots and the output file that are stored in the +% ct_tmp output directory specified by param.sar_equal.debug_out_dir. +% +% Author: John Paden +% +% See also: run_sar_equal.m, run_sar_equal_post.m, run_sar_load.m, sar_equal.m, +% sar_equal_post.m, sar_load.m + +%% General Setup +% ===================================================================== +param = merge_structs(param, param_override); + +fprintf('=====================================================================\n'); +fprintf('%s: %s (%s)\n', mfilename, param.day_seg, datestr(now)); +fprintf('=====================================================================\n'); + +%% Input checking +% ===================================================================== + +% .coh_ave: Positive scalar integer. Slow-time averaging/stacking. Default +% is 1. +if ~isfield(param.sar_equal,'coh_ave') || isempty(param.sar_equal.coh_ave) + param.sar_equal.coh_ave = 1; +end + +% .debug_out_dir: string containing the ct_tmp output folder name to use +% for the debug outputs. This is input to ct_filename_ct_tmp(). +if ~isfield(param.sar_equal,'debug_out_dir') || isempty(param.sar_equal.debug_out_dir) + param.sar_equal.debug_out_dir = 'sar_equal'; +end +debug_out_dir = param.sar_equal.debug_out_dir; + +% debug_plots: cell array of debug plots to enable +if ~isfield(param.sar_equal,'debug_plots') + param.sar_equal.debug_plots = {'final','visible'}; +end +enable_visible_plot = any(strcmp('visible',param.sar_equal.debug_plots)); + +% .delay: strucure controlling how the equalization coefficients are +% estimated. +if ~isfield(param.sar_equal,'delay') || isempty(param.sar_equal.delay) + param.sar_equal.delay = []; +end +% .delay.method: string containing the method name. Default is +% 'xcorr_complex'. +if ~isfield(param.sar_equal.delay,'method') || isempty(param.sar_equal.delay.method) + param.sar_equal.delay.method = 'xcorr_complex'; +end +% .delay.Mt: Positive scalar integer. Fast-time oversampling to use in +% equalization. Default is 64. +if ~isfield(param.sar_equal.delay,'Mt') || isempty(param.sar_equal.delay.Mt) + param.sar_equal.delay.Mt = 64; +end +% .delay.ref_bins: Array of scalar integers. Default is [-19 20]. Only the +% first and last entry are used in the array. This specifies the relative +% to zero_surf_bin range to use for correlation methods for the +% param.sar_equal.ref wf_adc pair. For example, [-19 20] specifies 19 +% bins before the zero_surf_bin to 20 bins after the zero_surf_bin (so 40 +% total). +if ~isfield(param.sar_equal.delay,'ref_bins') || isempty(param.sar_equal.delay.ref_bins) + param.sar_equal.delay.ref_bins = [-19 20]; +end +% .delay.search_bins: Array of scalar integers. Default is [-13 14]. Only +% the first and last entry are used in the array. This specifies the +% relative to zero_surf_bin range to use for correlation methods. For +% example, [-13 14] specifies 13 bins before the zero_surf_bin to 14 bins +% after the zero_surf_bin (so 28 total). +if ~isfield(param.sar_equal.delay,'search_bins') || isempty(param.sar_equal.delay.search_bins) + param.sar_equal.delay.search_bins = [-13 14]; +end + +% imgs: cell array of wf_adc pair lists, images to load, equalization will +% be performed between the reference image specified by ref +if ~isfield(param.sar_equal,'imgs') || isempty(param.sar_equal.imgs) + error('param.sar_equal.imgs must be set.'); +end + +% ref_img: positive scalar integer. Index of reference image in +% param.sar_equal.imgs list. Default is 1. +if ~isfield(param.sar_equal,'ref_img') || isempty(param.sar_equal.ref_img) + param.sar_equal.ref_img = 1; +end +ref_img = param.sar_equal.ref_img; + +% sar_load: structure array for param.sar_load. fields which will be copied +% to param.sar_load when loading the SAR data. +if ~isfield(param.sar_equal,'sar_load') || isempty(param.sar_equal.sar_load) + param.sar_equal.sar_load = []; +end + +% start_time: See analysis waveform cmd start_time field for options. +if ~isfield(param.sar_equal,'start_time') || isempty(param.sar_equal.start_time) + error('param.sar_equal.start_time must be set.'); +end + +%% Other Setup +% ========================================================================= + +physical_constants; + +if ~isempty(param.sar_equal.debug_plots) + h_fig = get_figures(5,enable_visible_plot); +end + +% Find the first wf-adc pair for the last image, the output file names +% are formed with these. +for img = 1:length(param.sar_equal.imgs) + if img ~= ref_img + fn_wf = param.sar_equal.imgs{img}(1,1); + fn_adc = param.sar_equal.imgs{img}(1,2); + end +end + +%% Loop Frames: load each frame one at a time +% ===================================================================== +for frm_idx = 1:length(param.cmd.frms) + frm = param.cmd.frms(frm_idx); + + %% Load SAR for frame + % ===================================================================== + + param_load = param; + param_load.sar_load = param.sar_equal.sar_load; + param_load.sar_load.frms = frm; + param_load.sar_load.chunk = {}; + param_load.sar_load.subap = []; + param_load.sar_load.imgs = param.sar_equal.imgs; + param_load.sar_load.combine_channels = 0; + param_load.sar_load.incoherent = 0; + param_load.sar_load.combine_imgs = 0; + param_load.sar_load.detrend.cmd = 0; + + [data,metadata] = sar_load(param_load); + + ref_wf = param.sar_equal.imgs{ref_img}(1,1); + ref_adc = param.sar_equal.imgs{ref_img}(1,2); + + %% Channel equalization + % ===================================================================== + for img = 1:length(param.sar_equal.imgs) + for wf_adc = 1:size(data{img},3) + wf = param.sar_equal.imgs{img}(wf_adc,1); + adc = param.sar_equal.imgs{img}(wf_adc,2); + chan_equal = 10.^((param.radar.wfs(wf).chan_equal_dB(param.radar.wfs(wf).rx_paths(adc)) ... + - metadata.param_sar.radar.wfs(wf).chan_equal_dB(param.radar.wfs(wf).rx_paths(adc)) )/20) ... + .* exp(1i*( ... + param.radar.wfs(wf).chan_equal_deg(param.radar.wfs(wf).rx_paths(adc)) ... + - metadata.param_sar.radar.wfs(wf).chan_equal_deg(param.radar.wfs(wf).rx_paths(adc)) )/180*pi); + data{img}(:,:,wf_adc) = data{img}(:,:,wf_adc) ./ chan_equal; + chan_equal + end + end + + %% Motion compensation + % ======================================================================= + param.array.squint = [0; 0; -1]; + for img = 1:length(param.sar_equal.imgs) + % Determine number of range bins + Nt = size(data{img},1); + % Determine number of range lines + Nx = size(data{img},2); + + for wf_adc = 1:size(data{img},3) + wf = param.sar_equal.imgs{img}(wf_adc,1); + adc = param.sar_equal.imgs{img}(wf_adc,2); + % %Interested in finding the magnitude of the projection of y in the z + % %direction [y_inz = dot(y,z)/mag(z)] + % zmag = sqrt(dot(fcs{img}{wf_adc}.z,fcs{img}{wf_adc}.z)); + % dTsys_mocomp = dot(fcs{img}{wf_adc}.z,fcs{img}{wf_adc}.y)./zmag; + % %Compensate for desired squint angle + % dTsys_mocomp = dTsys_mocomp/cos(squint_angle); + + % Calculate complex baseband frequencies + df = metadata.param_sar.radar.wfs(wf).freq(2)-metadata.param_sar.radar.wfs(wf).freq(1); + fc = metadata.param_sar.radar.wfs.fc; + % freq: double vector. Carrier band frequency axis since we are + % correcting for the envelope and phase. + freq = fc + df * ifftshift( -floor(Nt/2) : floor((Nt-1)/2) ).'; + + % Correct any changes in Tsys + Tsys = param.radar.wfs(wf).Tsys(param.radar.wfs(wf).rx_paths(adc)); + Tsys_old = metadata.param_sar.radar.wfs(wf).Tsys(param.radar.wfs(wf).rx_paths(adc)); + dTsys = Tsys-Tsys_old; + for rline = 1:Nx + % range_offset: dot product between squint direction and phase + % center offset. If the phase center was closer to the target, then + % the phase will be more positive (less negative). + range_offset = dot(param.array.squint,[0 metadata.fcs{img}{wf_adc}.pos(2,rline) metadata.fcs{img}{wf_adc}.pos(3,rline)]); + % time_offset: convert range offset to two way travel time + time_offset = range_offset * 2/c; + % If time offset is non-zero, apply time shift in frequency domain + if time_offset+dTsys ~= 0 + % 1. Positive dTsys means Tsys > Tsys_old and the time delay is + % too large. We compensate by reducing the time delay to all + % targets by dTsys. Reducing time delay means making the phase + % more positive. + % + % 2. Positive time_offset means the range is less; to undo this, + % the phase is made more negative to increase the range and + % time_delay. + data{img}(:,rline,wf_adc) = ifft(bsxfun(@times,fft(data{img}(:,rline,wf_adc),[],1), ... + exp(-1j*2*pi*freq*(dTsys+time_offset)))); + end + end + end + end + if 0 + % Debug plots + rline = 1; % <== SPECIFY WHICH RANGE LINE TO PLOT + colors_cell = {'k','r','y','g','b','c','m'}; + h_plot = []; + legend_str = {}; + figure; clf; + for img = 1:length(data) + for wf_adc = 1:size(data{img},3) + wf = param.sar_equal.imgs{img}(wf_adc,1); + adc = param.sar_equal.imgs{img}(wf_adc,2); + rline = max(1,min(rline,size(data{img},2))); + h_tmp = plot(metadata.wfs(wf).time, squeeze(imag(data{img}(:,rline,wf_adc))), colors_cell{1+mod(img-1,length(colors_cell))}); + if wf_adc == 1 + legend_str{img} = sprintf('Img %d wf %d adc %d', img, wf, adc); + h_plot(img) = h_tmp; + end + hold on; + end + end + legend(h_plot, legend_str); + end + + %% Combine wf-adc channels + % ======================================================================= + for img = 1:length(param.sar_equal.imgs) + data{img} = mean(data{img},3); + end + + %% Determine fast time bin for equalization (reference) + % ======================================================================= + time = metadata.param_sar.radar.wfs(ref_wf).time; + dt = time(2)-time(1); + t0 = time(1); + fc = metadata.param_sar.radar.wfs(ref_wf).fc; + Tpd = metadata.param_sar.radar.wfs(ref_wf).Tpd; + if isnumeric(param.sar_equal.start_time) + ref_start_bin = find(time>=param.sar_equal.start_time,1)*ones(1,size(data{ref_img},2)); + if isempty(ref_start_bin) + error('No time (%g-%g) is >= param.sar_equal.start_time (%g).', time(1), time(end), param.sar_equal.start_time); + end + elseif isstruct(param.sar_equal.start_time) + param.sar_equal.start_time.eval.Tpd = Tpd; + param.sar_equal.start_time.eval.dt = dt; + param.sar_equal.start_time.eval.Tstart = time(1); + param.sar_equal.start_time.eval.Tend = time(end); + layers = opsLoadLayers(param,param.sar_equal.start_time); + layers.twtt = interp_finite(layers.twtt,0); + layers.twtt = interp1(layers.gps_time, layers.twtt, metadata.fcs{ref_img}{1}.gps_time); + ref_start_bin = round(interp1(time, 1:length(time), layers.twtt,'linear','extrap')); + ref_start_bin = interp_finite(ref_start_bin); + ref_start_bin = min(max(1,ref_start_bin),size(data{ref_img},1)); + elseif ischar(param.sar_equal.start_time) + es = []; + es.Tpd = Tpd; + es.dt = dt; + es.Tstart = time(1); + es.Tend = time(end); + s = 0; + eval(param.sar_equal.start_time); + ref_start_bin = find(time>=s,1)*ones(1,size(data{ref_img},2)); + if isempty(ref_start_bin) + error('No time (%g-%g) is >= param.sar_equal.start_time (%g).', time(1), time(end), param.sar_equal.start_time); + end + end + + %% Equalization setup + % ===================================================================== + + % Setup for estimating equalization coefficients + switch(param.sar_equal.delay.method) + case 'threshold' + delay_method = 1; + case 'xcorr_complex' + delay_method = 2; + case 'xcorr_magnitude' + delay_method = 3; + otherwise + error('delay.method %d is not supported', param.sar_equal.delay.method); + end + + Mt = param.sar_equal.delay.Mt; + + search_bins = param.sar_equal.delay.search_bins(1) : param.sar_equal.delay.search_bins(end); + ref_bins = param.sar_equal.delay.ref_bins(1) : param.sar_equal.delay.ref_bins(end); + + if delay_method == 2 + Hcorr_wind = boxcar(length(ref_bins)); + else + Hcorr_wind = boxcar(Mt*length(ref_bins)); + end + zero_padding_offset = length(search_bins) - length(ref_bins); + + peak_val = nan(length(param.sar_equal.imgs),floor(Nx/param.sar_equal.coh_ave)); + peak_offset = nan(length(param.sar_equal.imgs),floor(Nx/param.sar_equal.coh_ave)); + gps_time = nan(length(param.sar_equal.imgs),floor(Nx/param.sar_equal.coh_ave)); + surface = nan(length(param.sar_equal.imgs),floor(Nx/param.sar_equal.coh_ave)); + dsurface = nan(length(param.sar_equal.imgs),floor(Nx/param.sar_equal.coh_ave)); + + %% Cross-correlation to extract time delay, phase, and amplitude offsets + % ===================================================================== + for img = 1:length(param.sar_equal.imgs) + if img == ref_img + continue; + end + % Determine number of range bins + Nt = size(data{img},1); + + wf_adc = 1; + wf = param.sar_equal.imgs{img}(wf_adc,1); + adc = param.sar_equal.imgs{img}(wf_adc,2); + + %% Determine fast time bin for equalization + % ======================================================================= + time = metadata.param_sar.radar.wfs(wf).time; + dt = time(2)-time(1); + t0 = time(1); + fc = metadata.param_sar.radar.wfs(wf).fc; + Tpd = metadata.param_sar.radar.wfs(wf).Tpd; + if isnumeric(param.sar_equal.start_time) + start_bin = find(time>=param.sar_equal.start_time,1)*ones(1,size(data{img},2)); + if isempty(start_bin) + error('No time (%g-%g) is >= param.sar_equal.start_time (%g).', time(1), time(end), param.sar_equal.start_time); + end + elseif isstruct(param.sar_equal.start_time) + param.sar_equal.start_time.eval.Tpd = Tpd; + param.sar_equal.start_time.eval.dt = dt; + param.sar_equal.start_time.eval.Tstart = time(1); + param.sar_equal.start_time.eval.Tend = time(end); + start_bin = round(interp1(time, 1:length(time), layers.twtt,'linear','extrap')); + start_bin = interp_finite(start_bin); + start_bin = min(max(1,start_bin),size(data{img},1)); + elseif ischar(param.sar_equal.start_time) + es = []; + es.Tpd = Tpd; + es.dt = dt; + es.Tstart = time(1); + es.Tend = time(end); + s = 0; + eval(param.sar_equal.start_time); + start_bin = find(time>=s,1)*ones(1,size(data{img},2)); + if isempty(start_bin) + error('No time (%g-%g) is >= param.sar_equal.start_time (%g).', time(1), time(end), param.sar_equal.start_time); + end + end + + %% Plot echogram images + % ======================================================================= + if 1 + wf = param.sar_equal.imgs{img}(1,1); + adc = param.sar_equal.imgs{img}(1,2); + clf(h_fig(4)); + h_axes(4) = axes('parent',h_fig(4)); + title(sprintf('img %d wf %d adc %s',img, wf, adc),'parent',h_axes(4)); + imagesc(db(data{img}),'parent',h_axes(4)); + grid(h_axes(4),'on'); + xlabel(h_axes(4),'Range line'); + ylabel(h_axes(4),'Range bin'); + h_color = colorbar(h_axes(4)); + hold(h_axes(4),'on'); + plot(start_bin,'parent',h_axes(4)) + plot(start_bin+search_bins(1),'parent',h_axes(4)) + plot(start_bin+search_bins(end),'parent',h_axes(4),'Color',[0 1 0]) + + clf(h_fig(5)); + h_axes(5) = axes('parent',h_fig(5)); + title(sprintf('img %d wf %d adc %s',img, wf, adc),'parent',h_axes(5)); + imagesc(db(data{ref_img}),'parent',h_axes(5)); + grid(h_axes(5),'on'); + xlabel(h_axes(5),'Range line'); + ylabel(h_axes(5),'Range bin'); + h_color = colorbar(h_axes(5)); + hold(h_axes(5),'on'); + plot(ref_start_bin,'parent',h_axes(5)) + plot(ref_start_bin+ref_bins(1),'parent',h_axes(5)) + plot(ref_start_bin+ref_bins(end),'parent',h_axes(5),'Color',[0 1 0]) + end + + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('img_wf_%02d_adc_%02d',wf,adc)) sprintf('_%03d.jpg',frm)]; + fprintf('Saving %s\n', fig_fn); + fig_fn_dir = fileparts(fig_fn); + if ~exist(fig_fn_dir,'dir') + mkdir(fig_fn_dir); + end + ct_saveas(h_fig(4),fig_fn); + + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('ref_img_wf_%02d_adc_%02d',ref_wf,ref_adc)) sprintf('_%03d.jpg',frm)]; + fprintf('Saving %s\n', fig_fn); + fig_fn_dir = fileparts(fig_fn); + if ~exist(fig_fn_dir,'dir') + mkdir(fig_fn_dir); + end + ct_saveas(h_fig(5),fig_fn); + + %% Cross-correlation to extract time delay, phase, and amplitude offsets + % ===================================================================== + rline_out = 0; + corr_out_accum = 0; + coh_ave = 0; + for rline = 1:Nx + if ~mod(rline-1,10^floor(log10(Nx)-1)) + fprintf(' Estimating rline %d of %d (%s)\n', rline, Nx, datestr(now)); + end + if start_bin(rline) + search_bins(1) < 1 ... + || start_bin(rline) + search_bins(end) > size(data{img},1) ... + || ref_start_bin(rline)+ref_bins(1) < 1 ... + || ref_start_bin(rline)+ref_bins(end) > size(data{ref_img},1) + continue; + end + in = data{img}(start_bin(rline) + search_bins,rline); + ref_in = data{ref_img}(ref_start_bin(rline)+ref_bins,rline); + if 0 + figure(1); clf; + plot(db(in)) + hold on + plot(db(ref_in)) + grid on; + end + % Notes: + % xcorr zero pads the end of the shorter waveform + % If "in" lags behind "ref_in", then the lag will be positive. + % If "in" leads ahead of "ref_in", then the lag will be negative. + [corr_out,lags0] = xcorr(in, ref_in .* Hcorr_wind); + lags = lags0(1) + search_bins(1) - ref_bins(1) + (0:length(lags0)-1); + if 0 + figure(1); clf; + plot(lags,db(corr_out)) + grid on; + end + corr_out_accum = corr_out_accum + corr_out; + coh_ave = coh_ave + 1; + if coh_ave >= param.sar_equal.coh_ave + corr_int = interpft(corr_out_accum,Mt*length(corr_out_accum)); + lags_Mt = lags(1) + 1/Mt*(0:Mt*length(lags)-1); + if 0 + figure(1); clf; + plot(lags_Mt,db(corr_int)) + grid on; + end + corr_out_accum = 0; + coh_ave = 0; + rline_out = rline_out + 1; + [peak_val(img,rline_out) peak_offset(img,rline_out)] = max(corr_int); + peak_offset(img,rline_out) = lags_Mt(peak_offset(img,rline_out)); + + + % IMPORTANT: WE SHOULD STORE AMPLITUDE SEPARATELY!!!!!!!!!!! + % Using the max method is good, but the max method is not good for + % phase averaging + % + % PHASE SHOULD BE KEPT with real peak amplitudes for the purposes of + % averaging + + + + + + peak_val(img,rline_out) = abs( ... + max(data{img}(start_bin(rline) + search_bins,rline)) ... + ./ max(data{ref_img}(ref_start_bin(rline)+ref_bins,rline))) ... + .* exp(1i*angle(peak_val(img,rline_out))); + gps_time(img,rline_out) = metadata.fcs{1}{1}.gps_time(rline); + surface(img,rline_out) = metadata.fcs{1}{1}.surface(rline); + if rline < Nx + dsurface(img,rline_out) = metadata.fcs{1}{1}.surface(rline+1) - metadata.fcs{1}{1}.surface(rline); + else + dsurface(img,rline_out) = metadata.fcs{1}{1}.surface(rline) - metadata.fcs{1}{1}.surface(rline-1); + end + + end + end + + end + + % plot(metadata.fcs{1}{1}.gps_time, metadata.fcs{1}{1}.surface); hold on; + % plot(gps_time(img,:), surface(img,:)); + % plot(gps_time(img,:), abs(dsurface(img,:))); + + %% Compute mean + + [offset_ave,mask] = mean_without_outliers(peak_offset, 1, 0.2, 2); + + peak_val_mean = peak_val; + peak_val_mean(mask) = NaN; + peak_val_mean = nanmean(peak_val_mean, 2); + + Tsys_deg = angle(exp(1i*2*pi*offset_ave*dt * fc))*180/pi + Tsys = -offset_ave*dt + chan_equal_dB = db(nanmean(abs(peak_val_mean).^2, 2),'power') + chan_equal_deg = angle(peak_val_mean)*180/pi + + %% Print results + new_wfs = param.radar.wfs; + for img = 1:length(param.sar_equal.imgs) + wf = param.sar_equal.imgs{img}(1,1); + adc = param.sar_equal.imgs{img}(1,2); + + if img == ref_img + fprintf('%% img %d wf %d adc %d (reference image)\n', img, wf, adc); + fprintf(' Tsys = old_Tsys + %g;\n', 0); + fprintf(' chan_equal_deg = old_chan_equal_deg + %g;\n', 0); + fprintf(' chan_equal_dB = old_chan_equal_dB + %g;\n', 0); + else + fprintf('%% img %d wf %d adc %d\n', img, wf, adc); + fprintf(' Tsys = Tsys + %g;\n', Tsys(img)); + fprintf(' chan_equal_deg = chan_equal_deg + %g;\n', chan_equal_deg(img) + Tsys_deg(img)); + fprintf(' chan_equal_dB = chan_equal_dB + %g;\n', chan_equal_dB(img)); + fprintf(' chan_equal_deg = chan_equal_deg + %g; %% Use this if not changing Tsys\n', chan_equal_deg(img)); + + for wf_adc = 1:size(param.sar_equal.imgs{img},1) + wf = param.sar_equal.imgs{img}(wf_adc,1); + adc = param.sar_equal.imgs{img}(wf_adc,2); + new_wfs(wf).Tsys(adc) = new_wfs(wf).Tsys(adc) + Tsys(img); + new_wfs(wf).chan_equal_deg(adc) = new_wfs(wf).chan_equal_deg(adc) + chan_equal_deg(img) + Tsys_deg(img); + new_wfs(wf).chan_equal_dB(adc) = new_wfs(wf).chan_equal_dB(adc) + chan_equal_dB(img); + end + end + end + for wf = 1:length(new_wfs) + fprintf('param_override.radar.wfs(%d).Tsys = %s;\n', wf, mat2str_generic(new_wfs(wf).Tsys)); + fprintf('param_override.radar.wfs(%d).chan_equal_deg = %s;\n', wf, mat2str_generic(new_wfs(wf).chan_equal_deg)); + fprintf('param_override.radar.wfs(%d).chan_equal_dB = %s;\n', wf, mat2str_generic(new_wfs(wf).chan_equal_dB)); + end + + %% Plot results + for fig_num = 1:3 + clf(h_fig(fig_num)); + h_axes(fig_num) = axes('parent',h_fig(fig_num)); + title(h_axes(fig_num), sprintf('ref\_img %d ref\_wf %d ref\_adc %d',ref_img, ref_wf, ref_adc)); + end + legend_str = {}; + for img = 1:length(param.sar_equal.imgs) + if img ~= ref_img + wf = param.sar_equal.imgs{img}(1,1); + adc = param.sar_equal.imgs{img}(1,2); + legend_str{end+1} = sprintf('img %d wf %d adc %d',img, wf, adc); + + plot(h_axes(1),angle(peak_val(img,:))*180/pi) + grid(h_axes(1),'on'); + hold(h_axes(1),'on'); + xlabel(h_axes(1),'Range line'); + ylabel(h_axes(1),'Angle (deg)'); + + plot(h_axes(2),db(peak_val(img,:))) + grid(h_axes(2),'on'); + hold(h_axes(2),'on'); + xlabel(h_axes(2),'Range line'); + ylabel(h_axes(2),'Relative power (dB)'); + + plot(h_axes(3),peak_offset(img,:),'.') + grid(h_axes(3),'on'); + hold(h_axes(3),'on'); + xlabel(h_axes(3),'Range line'); + ylabel(h_axes(3),'Time delay (bins)'); + end + end + legend(h_axes(1),legend_str); + legend(h_axes(2),legend_str); + legend(h_axes(3),legend_str); + + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('angle_%02d_adc_%02d',fn_wf,fn_adc)) sprintf('_%03d.jpg',frm)]; + fprintf('Saving %s\n', fig_fn); + fig_fn_dir = fileparts(fig_fn); + if ~exist(fig_fn_dir,'dir') + mkdir(fig_fn_dir); + end + ct_saveas(h_fig(1),fig_fn); + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('angle_%02d_adc_%02d',fn_wf,fn_adc)) sprintf('_%03d.fig',frm)]; + fprintf('Saving %s\n', fig_fn); + ct_saveas(h_fig(1),fig_fn); + + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('abs_%02d_adc_%02d',fn_wf,fn_adc)) sprintf('_%03d.jpg',frm)]; + fprintf('Saving %s\n', fig_fn); + fig_fn_dir = fileparts(fig_fn); + if ~exist(fig_fn_dir,'dir') + mkdir(fig_fn_dir); + end + ct_saveas(h_fig(2),fig_fn); + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('abs_%02d_adc_%02d',fn_wf,fn_adc)) sprintf('_%03d.fig',frm)]; + fprintf('Saving %s\n', fig_fn); + ct_saveas(h_fig(3),fig_fn); + + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('td_%02d_adc_%02d',fn_wf,fn_adc)) sprintf('_%03d.jpg',frm)]; + fprintf('Saving %s\n', fig_fn); + fig_fn_dir = fileparts(fig_fn); + if ~exist(fig_fn_dir,'dir') + mkdir(fig_fn_dir); + end + ct_saveas(h_fig(3),fig_fn); + fig_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('td_%02d_adc_%02d',fn_wf,fn_adc)) sprintf('_%03d.fig',frm)]; + fprintf('Saving %s\n', fig_fn); + ct_saveas(h_fig(3),fig_fn); + + %% Save outputs + mat_fn = [ct_filename_ct_tmp(param,'',debug_out_dir,sprintf('sar_equal_wf_%02d_adc_%02d',fn_wf,fn_adc)) sprintf('_%03d.mat',frm)]; + fprintf('Saving %s\n', mat_fn); + mat_fn_dir = fileparts(mat_fn); + if ~exist(mat_fn_dir,'dir') + mkdir(mat_fn_dir); + end + param_sar_equal = param; + ct_save(mat_fn,'peak_offset','peak_val','Tsys','Tsys_deg','chan_equal_dB','chan_equal_deg','gps_time','surface','param_sar_equal'); +end diff --git a/cresis-toolbox/processing/sar_equal_post.m b/cresis-toolbox/processing/sar_equal_post.m new file mode 100644 index 00000000..13234a47 --- /dev/null +++ b/cresis-toolbox/processing/sar_equal_post.m @@ -0,0 +1,52 @@ +% function sar_equal_output = sar_equal_post(param,param_override) +% sar_equal_output = sar_equal_post(param,param_override) +% +% Support function for sar_equal.m. Run this function from +% run_sar_equal_post.m after run_sar_equal.m has been run. + +%% General Setup +% ===================================================================== +param = merge_structs(param, param_override); + +fprintf('=====================================================================\n'); +fprintf('%s: %s (%s)\n', mfilename, param.day_seg, datestr(now)); +fprintf('=====================================================================\n'); + +%% Input checking +% ===================================================================== + +% .debug_in_dir: string containing the ct_tmp output folder name where +% sar_equal stored the debug outputs. This is input to +% ct_filename_ct_tmp(). +if ~isfield(param.sar_equal_post,'debug_in_dir') || isempty(param.sar_equal_post.debug_in_dir) + param.sar_equal_post.debug_in_dir = 'sar_equal'; +end +debug_in_dir = param.sar_equal_post.debug_in_dir; + +%% Other Setup +% ========================================================================= + +physical_constants; + +% Find the first wf-adc pair for the last image, the input file name +% is formed with these. +for img = 1:length(param.sar_equal_post.imgs) + if img ~= ref_img + fn_wf = param.sar_equal_post.imgs{img}(1,1); + fn_adc = param.sar_equal_post.imgs{img}(1,2); + end +end + +%% Loop Frames: load each frame one at a time +% ========================================================================= +sar_equal_output = {}; +for frm_idx = 1:length(param.cmd.frms) + frm = param.cmd.frms(frm_idx); + + %% Loop Frames: Load + + mat_fn = [ct_filename_ct_tmp(param,'',debug_in_dir,sprintf('sar_equal_wf_%02d_adc_%02d',fn_wf,fn_adc)) sprintf('_%03d.mat',frm)]; + fprintf('Loading %s\n', mat_fn); + sar_equal_output{frm} = load(mat_fn); + +end diff --git a/cresis-toolbox/processing/sar_motion_comp.m b/cresis-toolbox/processing/sar_motion_comp.m index d154bc6e..69286908 100644 --- a/cresis-toolbox/processing/sar_motion_comp.m +++ b/cresis-toolbox/processing/sar_motion_comp.m @@ -1,5 +1,5 @@ -function [drange,dx] = sar_motion_comp(fcs,gps,ref,along_track,output_along_track) -% [drange,dx] = sar_motion_comp(fcs,gps,ref,along_track,output_along_track) +function [drange,dx,dy] = sar_motion_comp(fcs,gps,ref,along_track,output_along_track) +% [drange,dx,dy] = sar_motion_comp(fcs,gps,ref,along_track,output_along_track) % % Returns drange and mean(dx) that provides motion compensation for Doppler % domain SAR processors for the positions in the gps structure relative @@ -69,23 +69,24 @@ %% For each output range line position, find the closest range lines to it % The polynomial fitting for a particular output will only update drange % and dx values for input range lines closest to it. -start_input = zeros(size(output_along_track)); -stop_input = zeros(size(output_along_track)); -cur_input = 1; -for rline = 1:length(output_along_track)-1 - break_threshold = 0.5*(output_along_track(rline)+output_along_track(rline+1)); - start_input(rline) = cur_input; - stop_input(rline) = find(along_track<=break_threshold,1,'last'); - cur_input = stop_input(rline)+1; -end -start_input(end) = cur_input; -stop_input(end) = length(along_track); +% start_input = zeros(size(output_along_track)); +% stop_input = zeros(size(output_along_track)); +% cur_input = 1; +% for rline = 1:length(output_along_track)-1 +% break_threshold = 0.5*(output_along_track(rline)+output_along_track(rline+1)); +% start_input(rline) = cur_input; +% stop_input(rline) = find(along_track<=break_threshold,1,'last'); +% cur_input = stop_input(rline)+1; +% end +% start_input(end) = cur_input; +% stop_input(end) = length(along_track); %% Perform fitting to input data and determine the offset in % drange and dx to that fitting (fitting is usually a polynomial fit of % some kind). drange = zeros(size(along_track)); dx = zeros(size(along_track)); +dy = zeros(size(along_track)); dx_out = output_along_track(2)-output_along_track(1); for rline = 1:length(output_along_track) rlines_fit = find(along_track >= output_along_track(rline)-fcs.Lsar/2 ... @@ -94,51 +95,57 @@ continue; end if rline == 1 - rlines_in = [1:rlines_fit(1)-1 rlines_fit]; + rlines_in = find(along_track < 0.5*(output_along_track(rline)+output_along_track(rline+1)) ); elseif rline == length(output_along_track) - rlines_in = [rlines_fit rlines_fit(end)+1:length(along_track)]; + rlines_in = find(along_track >= 0.5*(output_along_track(rline-1)+output_along_track(rline)) ); else - rlines_in = rlines_fit; + rlines_in = find(along_track >= 0.5*(output_along_track(rline-1)+output_along_track(rline)) ... + & along_track < 0.5*(output_along_track(rline)+output_along_track(rline+1)) ); end - - % Determine the closest range lines for this output range line - closest_rlines = start_input(rline):stop_input(rline); - if isempty(closest_rlines) + if isempty(rlines_in) continue; end - fit_ecef = []; - if fcs.type == 4 || fcs.type == 5 - %% Compensation to savitsky golay - fit_ecef = sv_ecef(:,closest_rlines); - elseif fcs.type == 3 - %% Compensation to line fit to reference (param.type == 3) - fit_ecef(1,:) = polyval(polyfit(1:size(ref_ecef,2),ref_ecef(1,:),1),closest_rlines); - fit_ecef(2,:) = polyval(polyfit(1:size(ref_ecef,2),ref_ecef(2,:),1),closest_rlines); - fit_ecef(3,:) = polyval(polyfit(1:size(ref_ecef,2),ref_ecef(3,:),1),closest_rlines); - elseif fcs.type == 2 - %% Compensation to piece-wise line fit to reference (param.type == 2) - fit_ecef(1,:) = polyval(polyfit(rlines_fit,ref_ecef(1,rlines_fit),1),closest_rlines); - fit_ecef(2,:) = polyval(polyfit(rlines_fit,ref_ecef(2,rlines_fit),1),closest_rlines); - fit_ecef(3,:) = polyval(polyfit(rlines_fit,ref_ecef(3,rlines_fit),1),closest_rlines); - elseif fcs.type == 1 - %% Compensation to reference (param.type == 1) - fit_ecef = ref_ecef(:,closest_rlines); - else - error('Motion compensation type must be 1, 2, or 3'); - end + % Determine the closest range lines for this output range line +% closest_rlines = start_input(rline):stop_input(rline); +% if isempty(closest_rlines) +% continue; +% end + +% fit_ecef = []; +% if fcs.type == 4 || fcs.type == 5 +% %% Compensation to savitsky golay +% fit_ecef = sv_ecef(:,closest_rlines); +% elseif fcs.type == 3 +% %% Compensation to line fit to reference (fcs.type == 3) +% fit_ecef(1,:) = polyval(polyfit(1:size(ref_ecef,2),ref_ecef(1,:),1),closest_rlines); +% fit_ecef(2,:) = polyval(polyfit(1:size(ref_ecef,2),ref_ecef(2,:),1),closest_rlines); +% fit_ecef(3,:) = polyval(polyfit(1:size(ref_ecef,2),ref_ecef(3,:),1),closest_rlines); +% elseif fcs.type == 2 +% %% Compensation to piece-wise line fit to reference (fcs.type == 2) +% fit_ecef(1,:) = polyval(polyfit(rlines_fit,ref_ecef(1,rlines_fit),1),closest_rlines); +% fit_ecef(2,:) = polyval(polyfit(rlines_fit,ref_ecef(2,rlines_fit),1),closest_rlines); +% fit_ecef(3,:) = polyval(polyfit(rlines_fit,ref_ecef(3,rlines_fit),1),closest_rlines); +% elseif fcs.type == 1 +% %% Compensation to reference (fcs.type == 1) +% fit_ecef = ref_ecef(:,closest_rlines); +% else +% error('Motion compensation type must be 1, 2, or 3'); +% end % The following operation computes the linear transformation matrix: % fcs_T = [fcs.x(:,out_rline) fcs.y(:,out_rline) fcs.z(:,out_rline)]; % Uses the "A\B" (help slash) nomenclature to inverse it: % fcs_Tinv = inv(fcs_T); - gps_fcs = [fcs.x(:,rline) fcs.y(:,rline) fcs.z(:,rline)] \ (ecef(:,closest_rlines) - repmat(fcs.origin(:,rline),[1 length(closest_rlines)])); - ref_fcs = [fcs.x(:,rline) fcs.y(:,rline) fcs.z(:,rline)] \ (fit_ecef - repmat(fcs.origin(:,rline),[1 length(closest_rlines)])); + gps_fcs = [fcs.x(:,rline) fcs.y(:,rline) fcs.z(:,rline)] \ (ecef(:,rlines_in) - repmat(fcs.origin(:,rline),[1 length(rlines_in)])); + ref_fcs = [fcs.x(:,rline) fcs.y(:,rline) fcs.z(:,rline)] \ (ref_ecef(:,rlines_in) - repmat(fcs.origin(:,rline),[1 length(rlines_in)])); diff_fcs = gps_fcs - ref_fcs; %% Calculate drange and dx for each input and SAR output - drange(closest_rlines) = dot(repmat(fcs.squint,[1 size(gps_fcs,2)]),diff_fcs); - dx(closest_rlines) = dot(repmat([1 0 0].',[1 size(gps_fcs,2)]),diff_fcs); + drange(rlines_in) = dot(repmat(fcs.squint,[1 size(gps_fcs,2)]),diff_fcs); + dx(rlines_in) = dot(repmat([1 0 0].',[1 size(gps_fcs,2)]),diff_fcs); + dx(rlines_in) = diff_fcs(1,:); + dy(rlines_in) = gps_fcs(2,:); end if fcs.type == 5 diff --git a/cresis-toolbox/processing/sar_task.m b/cresis-toolbox/processing/sar_task.m index c9e81de4..32f2db6c 100644 --- a/cresis-toolbox/processing/sar_task.m +++ b/cresis-toolbox/processing/sar_task.m @@ -10,7 +10,6 @@ % .radar = structured used by load_mcords_hdr % % .load = structure containing info about what data to load -% .records_fn = records filename % .recs = 2 element vector containing the start and stop records % .imgs = cell vector of images to load, each image is Nx2 array of % wf/adc pairs @@ -74,8 +73,7 @@ [output_dir,radar_type,radar_name] = ct_output_dir(param.radar_name); -records_fn = ct_filename_support(param,'','records'); -records = load(records_fn,'settings','param_records'); +records = records_load(param,'settings','param_records'); if param.sar.combine_rx && param.sar.mocomp.en warning('SAR motion compensation mode must be 0 for combine_rx (setting to 0)'); @@ -88,7 +86,7 @@ % Load SAR coordinate system sar_fn = fullfile(sar_coord_dir,'sar_coord.mat'); -sar = load(sar_fn,'version','Lsar','gps_source','type','sigma_x','presums','along_track','surf_pp'); +sar = load(sar_fn,'file_version','Lsar','gps_source','type','sigma_x','presums','along_track','surf_pp'); % Determine output range lines output_along_track = 0 : param.sar.sigma_x : sar.along_track(end); @@ -180,7 +178,7 @@ % ========================================================================= recs = [(param.load.recs(1)-1)*param.sar.presums+1, ... param.load.recs(2)*param.sar.presums]; -records = records_aux_files_read(records_fn,recs); +records = records_load(param,recs); % Decimate records according to presums if param.sar.presums > 1 records.lat = fir_dec(records.lat,param.sar.presums); @@ -319,7 +317,15 @@ end % 3. Fill in any missing lines - fcs.pos(:,~good_rline) = interp1(output_along_track(good_rline),fcs.pos(:,good_rline).',output_along_track(~good_rline).','linear','extrap').'; + if length(good_rline) > 1 + fcs.pos(:,~good_rline) = interp1(output_along_track(good_rline),fcs.pos(:,good_rline).',output_along_track(~good_rline).','linear','extrap').'; + elseif length(good_rline) == 1 + fcs.pos(1,~good_rline) = fcs.pos(1,good_rline); + fcs.pos(2,~good_rline) = fcs.pos(2,good_rline); + fcs.pos(3,~good_rline) = fcs.pos(3,good_rline); + else + error('good_rline is empty; no good data?'); + end %% SAR Processing @@ -445,6 +451,11 @@ % of a hack. x_lin = linspace(along_track(1), ... along_track(end),length(along_track)); + dx = mean(diff(x_lin)); + pre_x_lin = fliplr(x_lin(1)-dx:-dx:output_along_track_full(1)); + post_x_lin = x_lin(end)+dx:dx:output_along_track_full(end); + x_lin = [pre_x_lin, x_lin, post_x_lin]; + fk_data = [zeros(size(fk_data,1),length(pre_x_lin)), fk_data, zeros(size(fk_data,1),length(post_x_lin))]; % Create kx (along-track wavenumber) axis of input data kx = gen_kx(x_lin); diff --git a/cresis-toolbox/processing/sim_doa_task.m b/cresis-toolbox/processing/sim_doa_task.m index 6665f95a..550265f0 100644 --- a/cresis-toolbox/processing/sim_doa_task.m +++ b/cresis-toolbox/processing/sim_doa_task.m @@ -13,15 +13,22 @@ % SV params % Setup theta grid and steering vectors for multi-dimensional search % algorithms +% [param.method.theta, param.method.SV] ... +% = array_proc_sv(param.method.Nsv,param.src.fc, param.src.y_pc, param.src.z_pc); [param.method.theta, param.method.SV] ... - = array_proc_sv(param.method.Nsv,param.src.fc, param.src.y_pc, param.src.z_pc); + = array_proc_sv(param.src.fc*sqrt(param.src.sv_dielectric), param.src.y_pc, param.src.z_pc, param.method.Nsv, [],[]); + param.method.theta = fftshift(param.method.theta); param.method.SV = fftshift(param.method.SV,2); % Setup theta grid and steering vectors for 1-dimensional search % algorithms (e.g. MUSIC) +% [param.method.OneD_theta, param.method.OneD_SV] ... +% = array_proc_sv(param.method.OneD_Nsv,param.src.fc, param.src.y_pc, param.src.z_pc); + [param.method.OneD_theta, param.method.OneD_SV] ... - = array_proc_sv(param.method.OneD_Nsv,param.src.fc, param.src.y_pc, param.src.z_pc); + = array_proc_sv(param.src.fc* sqrt(param.src.sv_dielectric), param.src.y_pc, param.src.z_pc, param.method.OneD_Nsv,[],[]); + param.method.OneD_theta = fftshift(param.method.OneD_theta); param.method.OneD_SV = fftshift(param.method.OneD_SV,2); @@ -30,6 +37,7 @@ doa_nb_1d_param = []; doa_nb_1d_param.fs = param.src.fs; doa_nb_1d_param.fc = param.src.fc; +doa_nb_1d_param.sv_dielectric = param.src.sv_dielectric; doa_nb_1d_param.src_limits = param.method.src_limits; doa_nb_1d_param.theta_guard = param.method.theta_guard; doa_nb_1d_param.y_pc = param.src.y_pc; @@ -44,6 +52,7 @@ doa_nb_nd_param.doa_seq = false; doa_nb_nd_param.fs = param.src.fs; doa_nb_nd_param.fc = param.src.fc; +doa_nb_nd_param.sv_dielectric = param.src.sv_dielectric; doa_nb_nd_param.src_limits = param.method.src_limits; doa_nb_nd_param.theta_guard = param.method.theta_guard; doa_nb_nd_param.y_pc = param.src.y_pc; @@ -58,6 +67,7 @@ doa_wb_td_param= []; doa_wb_td_param.fs = param.src.fs; doa_wb_td_param.fc = param.src.fc; +doa_wb_td_param.sv_dielectric = param.src.sv_dielectric; doa_wb_td_param.src_limits = param.method.src_limits; doa_wb_td_param.theta_guard = param.method.theta_guard; doa_wb_td_param.y_pc = param.src.y_pc; @@ -73,6 +83,7 @@ doa_wb_fd_param.doa_seq = false; doa_wb_fd_param.fs = param.src.fs; doa_wb_fd_param.fc = param.src.fc; +doa_wb_fd_param.sv_dielectric = param.src.sv_dielectric; doa_wb_fd_param.src_limits = param.method.src_limits; doa_wb_fd_param.theta_guard = param.method.theta_guard; doa_wb_fd_param.y_pc = param.src.y_pc; @@ -140,26 +151,45 @@ rng(rng_args(run_idx)); %% Test/Run Loop: Create simulated data for each simulation run - [Data,DCM,imp_response,DCM_fd] = sim.doa_wideband_data(param); - + % Hack to test position errors +% tmp_param = []; +% tmp_param = param; +% c = 2.997924580003452e+08; +% lambda = c / param.src.fc; +% +% sigmay = (0.5)*lambda*0.05; +% er_ypc = sigmay* randn(size(param.src.y_pc)); +% er_ypc(2) = 0; +% sigmaz = sigmay; +% er_zpc = sigmaz* randn(size(param.src.z_pc)); +% er_zpc(2) = 0; +% tmp_param.src.y_pc = tmp_param.src.y_pc + er_ypc; +% tmp_param.src.z_pc = tmp_param.src.z_pc + er_zpc; +% [Data,Rxx,imp_response,Rxx_fd] = sim.doa_wideband_data(tmp_param); +% tmp_param = []; + + [Data,Rxx,imp_response,Rxx_fd] = sim.doa_wideband_data(param); + %% Test/Run Loop: Set up estimation parameters for each method if isfield(param,'Nsig_tmp') && ~isempty(param.Nsig_tmp) % For model order estimation simulation. - DCM_runs{run_idx} = DCM; + Rxx_runs{run_idx} = Rxx; end doa_wb_td_param.h = conv(imp_response.vals,imp_response.vals,'same'); doa_wb_td_param.h = doa_wb_td_param.h ./ max(abs(doa_wb_td_param.h)); doa_wb_td_param.t0 = imp_response.time_vec(1); doa_wb_td_param.dt = mean(diff(imp_response.time_vec)); - doa_wb_td_param.DCM = DCM; + doa_wb_td_param.Rxx = Rxx; - DCM_nb_idxs = (param.method.wb_td.widening_factor-1)/2*length(param.src.y_pc) + (1:length(param.src.y_pc)); - DCM_nb = DCM(DCM_nb_idxs,DCM_nb_idxs); - doa_nb_1d_param.Rxx = DCM_nb; - doa_nb_nd_param.Rxx = DCM_nb; + Rxx_nb_idxs = (param.method.wb_td.widening_factor-1)/2*length(param.src.y_pc) + (1:length(param.src.y_pc)); + Rxx_nb = Rxx(Rxx_nb_idxs,Rxx_nb_idxs); +% doa_nb_1d_param.Rxx = Rxx_nb; +% doa_nb_nd_param.Rxx = Rxx_nb; - doa_wb_fd_param.Rxx = DCM_fd; + doa_nb_1d_param.Rxx = Rxx_nb; + doa_nb_nd_param.Rxx = Rxx_nb; + doa_wb_fd_param.Rxx = Rxx_fd; % Debug plots of impulse response if 0 @@ -176,28 +206,28 @@ switch method case MUSIC_DOA_METHOD % MUSIC method: DOA initialization and estimation - doa0 = sort(music_initialization(DCM_nb,doa_nb_1d_param)); + doa0 = sort(music_initialization(Rxx_nb,doa_nb_1d_param)); [doa,Jval,exitflag,OUTPUT,~,~,HESSIAN] = ... fmincon(@(theta_hat) music_cost_function(theta_hat,doa_nb_1d_param), doa0,[],[],[],[],LB,UB,doa_nonlcon_fh,doa_nb_1d_param.options); case MLE_METHOD % MLE method: DOA initialization and estimation - doa0 = sort(mle_initialization(DCM_nb,doa_nb_nd_param)); + doa0 = sort(mle_initialization(Rxx_nb,doa_nb_nd_param)); [doa,Jval,exitflag,OUTPUT,~,~,HESSIAN] = ... fmincon(@(theta_hat) mle_cost_function(theta_hat,doa_nb_nd_param), doa0,[],[],[],[],LB,UB,doa_nonlcon_fh,doa_nb_nd_param.options); case DCM_METHOD % WB DCM method: DOA initialization and estimation - doa0 = sort(wb_initialization(DCM,doa_wb_td_param)); + doa0 = sort(wb_initialization(Rxx,doa_wb_td_param)); [doa,Jval,exitflag,OUTPUT,~,~,HESSIAN] = ... fmincon(@(theta_hat) wb_cost_function(theta_hat,doa_wb_td_param), doa0,[],[],[],[],LB,UB,doa_nonlcon_fh,doa_wb_td_param.options); case WBMLE_METHOD % WBMLE method: DOA initialization and estimation - doa0 = sort(wbmle_initialization(DCM_fd,doa_wb_fd_param)); + doa0 = sort(wbmle_initialization(Rxx_fd,doa_wb_fd_param)); [doa,Jval,exitflag,OUTPUT,~,~,HESSIAN] = ... fmincon(@(theta_hat) wbmle_cost_function(theta_hat,doa_wb_fd_param), doa0,[],[],[],[],LB,UB,doa_nonlcon_fh,doa_wb_fd_param.options); diff --git a/cresis-toolbox/processing/slope_tracker.m b/cresis-toolbox/processing/slope_tracker.m index 5888fc9d..4fe6d90f 100644 --- a/cresis-toolbox/processing/slope_tracker.m +++ b/cresis-toolbox/processing/slope_tracker.m @@ -57,10 +57,9 @@ function slope_tracker(param,param_override) physical_constants; % Load frames file -load(ct_filename_support(param,'','frames')); +frames = frames_load(param); % Load records file -records_fn = ct_filename_support(param,'','records'); -records = load(records_fn); +records = records_load(param); global g_data; g_data = []; @@ -614,7 +613,6 @@ function slope_tracker(param,param_override) % Store surface information to the records file save(records_fn,'-APPEND','-struct','records','surface'); - records_aux_files_create(records_fn); end return; diff --git a/cresis-toolbox/processing/slope_tracker_task.m b/cresis-toolbox/processing/slope_tracker_task.m index 4a916e06..72223149 100644 --- a/cresis-toolbox/processing/slope_tracker_task.m +++ b/cresis-toolbox/processing/slope_tracker_task.m @@ -7,7 +7,6 @@ % param = struct controlling the loading, processing, surface tracking, % and quick look generation % .load = structure for which records to load -% .records_fn = filename of records file % .recs = current records % .imgs = cell vector of images to load, each image is Nx2 array of % wf/adc pairs @@ -83,8 +82,6 @@ physical_constants; surfTimes = []; -records_fn = ct_filename_support(param,'','records'); - if ~isfield(param.slope,'elev_correction') || isempty(param.slope.elev_correction) param.slope.elev_correction = false; end @@ -122,7 +119,7 @@ if simple_firdec load_param.load.recs(1) = param.load.recs(1); load_param.load.recs(2) = param.load.recs(2); - records = records_aux_files_read(records_fn,load_param.load.recs); + records = records_load(param,load_param.load.recs); old_param_records = records.param_records; else if mod(length(param.slope.B_filter)-1,2) @@ -132,7 +129,7 @@ start_buffer = min(filter_order/2,param.load.recs(1)-1); load_param.load.recs(1) = param.load.recs(1)-start_buffer; load_param.load.recs(2) = param.load.recs(2)+filter_order/2; - records = records_aux_files_read(records_fn,load_param.load.recs); + records = records_load(param,load_param.load.recs); old_param_records = records.param_records; stop_buffer = filter_order/2 - ((load_param.load.recs(2)-load_param.load.recs(1)+1) ... - length(records.lat)); @@ -201,15 +198,6 @@ 1:max(old_param_records.records.file.adcs), param.slope); load_param.load.rec_data_size = rec_data_size; elseif any(strcmpi(param.radar_name,{'snow','kuband','snow2','kuband2','snow3','kuband3'})) - [path name] = fileparts(records_fn); - cdf_fn = fullfile(path, sprintf('%s.nc', name)); - try - records.settings.nyquist_zone = ncread(cdf_fn,'settings(1).nyquist_zone',[1 load_param.load.recs(1)],[1 1]); - end - - try - records.settings.loopback_mode = ncread(cdf_fn,'settings(1).loopback_mode',[1 load_param.load.recs(1)],[1 1]); - end wfs_idx = find(records.settings.wfs_records <= load_param.load.recs(1),1,'last'); records.settings.wfs = records.settings.wfs(wfs_idx).wfs; wfs = load_fmcw_wfs(records.settings, param, ... @@ -246,7 +234,7 @@ board = adc_to_board(param.radar_name,adc); board_idx = find(board == boards); load_param.load.file_idx{idx} = relative_rec_num_to_file_idx_vector( ... - load_param.load.recs,records.relative_rec_num{board_idx}); + load_param.load.recs(1):load_param.load.recs(end),records.relative_rec_num{board_idx}); load_param.load.offset{idx} = records.offset(board_idx,:); file_idxs = unique(load_param.load.file_idx{idx}); diff --git a/cresis-toolbox/processing/tomo_collate_task.m b/cresis-toolbox/processing/tomo_collate_task.m index 7d3e568a..98fbe499 100644 --- a/cresis-toolbox/processing/tomo_collate_task.m +++ b/cresis-toolbox/processing/tomo_collate_task.m @@ -14,7 +14,7 @@ % in param. % % See also: tomo.run_collate, tomo.collate, tomo_collate_task, -% tomo.fuse_images, tomo.add_icemask_surfacedem, tomo.create_surfdata, +% tomo.fuse_images, tomo.add_icemask_surfacedem, tomo.track_surface, % % Author: John Paden, Jordan Sprick, and Mingze Xu @@ -39,7 +39,7 @@ mdata = load(combined_fn); end - mdata = tomo.add_icemask_surfacedem(param, mdata); + mdata = tomo.add_dem_icemask(param, mdata); fprintf(' Done (%s)\n', datestr(now)); end @@ -58,7 +58,7 @@ mdata = load(combined_fn); end - tomo.create_surfdata(param,mdata); + tomo.track_surface(param,mdata); fprintf(' Done (%s)\n', datestr(now)); end diff --git a/cresis-toolbox/processing/trajectory_with_leverarm.m b/cresis-toolbox/processing/trajectory_with_leverarm.m index 84e49e49..3f65f18b 100644 --- a/cresis-toolbox/processing/trajectory_with_leverarm.m +++ b/cresis-toolbox/processing/trajectory_with_leverarm.m @@ -1,5 +1,5 @@ -function gps = trajectory_with_leverarm(gps,param) -% gps = trajectory_with_leverarm(gps,param) +function [gps,lever_arm_val] = trajectory_with_leverarm(gps,param) +% [gps,lever_arm_val] = trajectory_with_leverarm(gps,param) % % Takes a struct, gps, and updates the fields lat, lon, elev, roll, % pitch, heading with the lever arm offset. All other fields of the struct @@ -23,14 +23,15 @@ % Author: John Paden if isempty(param.lever_arm_fh) + warning('No lever arm function is available. The reference trajectory is most likely incorrect without a lever arm defined. Normally this is defined in param.radar.lever_arm_fh and set to @lever_arm_fh.'); return; end -physical_constants; +physical_constants; % Load WGS84.spheroid % Get the phase center of the receiver in question in BCS -[base_phase_center] = param.lever_arm_fh(param, param.tx_weights, param.rx_path); -base_phase_center = mean(base_phase_center,2); +[lever_arm_val] = param.lever_arm_fh(param, param.tx_weights, param.rx_path); +lever_arm_val = mean(lever_arm_val,2); phase_center = zeros(3,length(gps.roll)); for rline = 1:length(gps.roll) @@ -42,18 +43,14 @@ % Middle and receiver phase centers with roll or roll/pitch only % compensation in NED coordinate system - phase_center(:,rline) = rotation_mat * base_phase_center; + phase_center(:,rline) = rotation_mat * lever_arm_val; % Convert from NED to ECEF (lv2ecef input is ENU, so we transpose the first % two inputs and negate the third) [phase_center(1,rline),phase_center(2,rline),phase_center(3,rline)] ... - = lv2ecef(phase_center(2,rline),phase_center(1,rline),-phase_center(3,rline), ... - gps.lat(rline)/180*pi,gps.lon(rline)/180*pi,gps.elev(rline),WGS84.ellipsoid); + = enu2ecef(phase_center(2,rline),phase_center(1,rline),-phase_center(3,rline), ... + gps.lat(rline),gps.lon(rline),gps.elev(rline),WGS84.spheroid); end % Convert from ECEF to geodetic -[gps.lat,gps.lon,gps.elev] = ecef2geodetic(phase_center(1,:),phase_center(2,:),phase_center(3,:),WGS84.ellipsoid); -gps.lat = gps.lat*180/pi; -gps.lon = gps.lon*180/pi; - -return; +[gps.lat,gps.lon,gps.elev] = ecef2geodetic(WGS84.spheroid, phase_center(1,:),phase_center(2,:),phase_center(3,:)); diff --git a/cresis-toolbox/processing/update_layerdata_format.m b/cresis-toolbox/processing/update_layerdata_format.m deleted file mode 100644 index 43887078..00000000 --- a/cresis-toolbox/processing/update_layerdata_format.m +++ /dev/null @@ -1,180 +0,0 @@ -function ctrl_chain = update_layerdata_format(param,param_override) -% update_layerdata_format(param,param_override) -% -% This function updates layer data files of an old version (0) to a new version (1), and save -% the updated layer data files to a layer data destination (default is CSARP_layer). -% -% param = struct with processing parameters -% param_override = parameters in this struct will override parameters -% in param. This struct must also contain the gRadar fields. -% Typically global gRadar; param_override = gRadar; -% -% Example: -% See run_update_layerdata_format.m for how to run this function directly. -% This function may be called from master.m using the param spreadsheet and -% the cmd.generic column. -% -% Authors: Jilu Li -% -% See also: run_master.m, master.m, run_update_layerdata_format.m, update_layerdata_formatk.m, -% update_layerdata_format_task.m - -%% General Setup -% ===================================================================== - -param = merge_structs(param, param_override); - -fprintf('=====================================================================\n'); -fprintf('%s: %s (%s)\n', mfilename, param.day_seg, datestr(now)); -fprintf('=====================================================================\n'); - -%% Input checks -% ===================================================================== -if ~isfield(param,'frame_overlap_removal') || isempty(param.frame_overlap_removal) - param.frame_overlap_removal = true; -end - -if ~isfield(param,'layerdata_source') || isempty(param.layerdata_source) - param.layerdata_source = 'layerData'; -end - -if ~isfield(param,'file_version_old') || isempty(param.file_version_old) - param.file_version_old = '0'; -end - -if ~isfield(param,'layerdata_dest') || isempty(param.layerdata_dest) - param.layerdata_dest = 'layer'; -end - -if ~isfield(param,'file_version_new') || isempty(param.file_version_new) - param.file_version_new = '1'; -end - - -% Load frames file -load(ct_filename_support(param,'','frames')); - -% If no frames specified, then do all frames -if isempty(param.cmd.frms) - param.cmd.frms = 1:length(frames.frame_idxs); -end - -% Remove frames that do not exist from param.cmd.frms list -[valid_frms,keep_idxs] = intersect(param.cmd.frms, 1:length(frames.frame_idxs)); -if length(valid_frms) ~= length(param.cmd.frms) - bad_mask = ones(size(param.cmd.frms)); - bad_mask(keep_idxs) = 0; - warning('Nonexistent frames specified in param.cmd.frms (e.g. frame "%g" is invalid), removing these', ... - param.cmd.frms(find(bad_mask,1))); - param.cmd.frms = valid_frms; -end - -% Set up output path -output_path = ct_filename_out(param,param.layerdata_dest,''); -if ~exist(output_path,'dir') - mkdir(output_path); -end - -%% Setup Processing -% ===================================================================== - -% Get the standard radar name -[~,~,radar_name] = ct_output_dir(param.radar_name); - -% Load records file -records_fn = ct_filename_support(param,'','records'); -if ~exist(records_fn) - error('You must run create the records file before running anything else:\n %s', records_fn); -end -records = load(records_fn); - -out_fn_dir = ct_filename_out(param, param.layerdata_dest); - -%% Create and setup the cluster batch -% ===================================================================== -ctrl = cluster_new_batch(param); -cluster_compile({'update_layerdata_format_task.m'},ctrl.cluster.hidden_depend_funs,ctrl.cluster.force_compile,ctrl); -cpu_time_mult = 8e-8; -mem_mult = 64; - -ctrl_chain = {}; - -%% Create task -% ===================================================================== -% -% For each frame load REC_BLOCK_SIZE records at a time (code groups -% by file index, but has to watch negative offset values which imply -% the record starts in a previous file and carries over into the next) -% --> The last block can range from 0.5 to 1.5 * REC_BLOCK_SIZE -% ===================================================================== - -% Set the task's static parameters -sparam.argsin{1} = param; % Static parameters -sparam.task_function = 'update_layerdata_format_task'; -sparam.num_args_out = 1; -for frm_idx = 1:length(param.cmd.frms) - frm = param.cmd.frms(frm_idx); - - % rerun_only==true checks - if ctrl.cluster.rerun_only - update_file_success = {}; - out_fn = fullfile(out_fn_dir, sprintf('Data_%s_%03d.mat',param.day_seg, frm)); - update_file_success{end+1} = out_fn; - end - - % recs: Determine the records for this frame - if frm < length(frames.frame_idxs) - recs = frames.frame_idxs(frm):frames.frame_idxs(frm+1)-1; - recs = [frames.frame_idxs(frm),frames.frame_idxs(frm+1)-1]; - else - recs = frames.frame_idxs(frm):length(records.gps_time); - recs = [frames.frame_idxs(frm),length(records.gps_time)]; - end - - % Set the task's dynamic parameters - % ================================================================= - dparam = []; - dparam.argsin{1}.load.frm = frm; - dparam.argsin{1}.load.recs = recs; - - % Create success condition - % ================================================================= - dparam.file_success = {}; - out_fn_name = sprintf('Data_%s_%03d.mat', param.day_seg, frm); - out_fn = fullfile(out_fn_dir,out_fn_name); - dparam.file_success{end+1} = out_fn; - if ~ctrl.cluster.rerun_only && exist(out_fn,'file') - delete(out_fn); - end - - % Rerun only mode: Test to see if we need to run this task - % ================================================================= - dparam.notes = sprintf('%s:%s:%s:%s %s_%03d (%d of %d)/%d of %d', ... - sparam.task_function, param.radar_name, param.season_name, param.layerdata_dest, param.day_seg, frm, frm_idx, length(param.cmd.frms)); - if ctrl.cluster.rerun_only - % If we are in rerun only mode AND the layer data file success - % condition passes without error then we do not run the task. - if ~cluster_file_success(dparam.file_success) || ~cluster_file_success(update_file_success) - fprintf(' Already exists [rerun_only skipping]: %s (%s)\n', ... - dparam.notes, datestr(now)); - continue; - end - end - - % Create a cluster task for each frame - % ================================================================= - - % CPU Time and Memory estimates: - % Nx*total_num_sam*K where K is some manually determined multiplier. - Nx = recs(end)-recs(1)+1; - dparam.cpu_time = 0; - dparam.mem = 250e6; - dparam.cpu_time = dparam.cpu_time + 10 + Nx*1000*log2(1000)*cpu_time_mult; - dparam.mem = dparam.mem + Nx*1000*mem_mult; - ctrl = cluster_new_task(ctrl,sparam,dparam,'dparam_save',0); -end - -ctrl = cluster_save_dparam(ctrl); - -ctrl_chain{end+1} = ctrl; - diff --git a/cresis-toolbox/processing/update_layerdata_format_task.m b/cresis-toolbox/processing/update_layerdata_format_task.m deleted file mode 100644 index ab1bd486..00000000 --- a/cresis-toolbox/processing/update_layerdata_format_task.m +++ /dev/null @@ -1,83 +0,0 @@ -function [success] = update_layerdata_format_task(param) -% [success] = update_layerdate_format_task(param) -% -% Cluster task for updateing layer data format. -% -% param = struct controlling the updates. -% .load = structure for which records to load -% .records_fn = filename of records file -% .recs = current records -% .imgs = cell vector of images to load, each image is Nx2 array of -% wf/adc pairs -% NOTE: wfs/adc pairs are not indices into anything, they are the absolute -% waveform/adc numbers. The records file will be loaded to decode -% which index each wf/adc belongs to. -% .debug_level = debug level (scalar integer) -% -% .proc = structure containing information about framing -% .frm = only used to determine the filename of the output -% -% success = boolean which is true when the function executes properly -% if a task fails before it can return success, then success will be -% empty -% -% Author: Jilu Li -% -% See also update_layerdata_format.m - - -%% Load the records -% ========================================================================= -records_fn = ct_filename_support(param,'','records'); -records = read_records_aux_files(records_fn,param.load.recs); - - - -%% Load the old layer data file of the frame -frm = param.load.frm; -layer_fn = fullfile(ct_filename_out(param,param.layerdata_source,''), ... - sprintf('Data_%s_%03d.mat', param.day_seg, frm)); -lay = load(layer_fn); - -% Remove data that is not contained within frame boundaries (old version -% layer data file contains overlapped data at boundaries) -if param.frame_overlap_removal - frm_mask = logical(zeros(size(lay.GPS_time))); - frm_mask(lay.GPS_time >= records.gps_time(1) & lay.GPS_time <= records.gps_time(end)) = true; -else - frm_mask = logical(ones(size(lay.GPS_time))); -end - -layer.GPS_time = lay.GPS_time(frm_mask); -layer.Pitch = interp1(records.gps_time,records.pitch,layer.GPS_time); -layer.Roll = interp1(records.gps_time,records.roll,layer.GPS_time); -layer.Heading = interp1(records.gps_time,records.heading,layer.GPS_time); -layer.file_version = param.file_version_new; - -if strcmp(param.file_version_old,'0') - for lyr_idx = 1:length(lay.layerData) - layer.id(lyr_idx,1) = lyr_idx; - layer.twtt(lyr_idx,:) = lay.layerData{lyr_idx}.value{2}.data(frm_mask); - layer.quality(lyr_idx,:) = uint8(lay.layerData{lyr_idx}.quality(frm_mask)); - layer.type(lyr_idx,:) = uint32(ones(size(lay.layerData{lyr_idx}.value{2}.data(frm_mask)))); - idxs_manual = find(isfinite(lay.layerData{lyr_idx}.value{1}.data(frm_mask))); - if ~isempty(idxs_manual) - layer.type(lyr_idx,idxs_manual) = uint32(0); - end - end -else -end - -%% Save the new layer data -% ========================================================================= -layer_fn = fullfile(ct_filename_out(param,param.layerdata_dest,''), ... - sprintf('Data_%s_%03d.mat', param.day_seg, frm)); -fprintf(' Save %s\n', layer_fn); -ct_save(layer_fn,'-v7.3', '-struct', 'layer'); - -%% Done -% ========================================================================= - -fprintf('%s done %s\n', mfilename, datestr(now)); - -success = true; diff --git a/cresis-toolbox/processing/wb_cost_function.m b/cresis-toolbox/processing/wb_cost_function.m index f96dfa29..da4f8bff 100644 --- a/cresis-toolbox/processing/wb_cost_function.m +++ b/cresis-toolbox/processing/wb_cost_function.m @@ -49,7 +49,7 @@ h = param.h; uy = sin(theta).'; % make directional sines and cosines into row vecs uz = cos(theta).'; -tau = (2/c)*(param.y_pc*uy - param.z_pc*uz); +tau = (2/c*param.sv_dielectric)*(param.y_pc*uy - param.z_pc*uz); J = wb_compute_cost(tau, param.DCM, param.fc, param.fs, h, t0,dt); J = 10*log10(abs(J)); diff --git a/cresis-toolbox/processing/wb_initialization.m b/cresis-toolbox/processing/wb_initialization.m index c730150b..8f64f7f4 100644 --- a/cresis-toolbox/processing/wb_initialization.m +++ b/cresis-toolbox/processing/wb_initialization.m @@ -97,8 +97,8 @@ % Evaluate cost function uy = sin(theta).'; uz = cos(theta).'; - tau = (2/c)*(param.y_pc*uy - param.z_pc*uz); - val = wb_compute_cost(tau,DCM,param.fc, param.fs, h,t0,dt); + tau = (2/c)*(param.y_pc*uy - param.z_pc*uz)*param.sv_dielectric; + val = wb_compute_cost(tau, DCM, param.fc, param.fs, h, t0, dt); J(idx) = 10*log10(abs(val)); end @@ -143,7 +143,7 @@ theta = search_theta(idx); uy = sin(theta).'; uz = cos(theta).'; - tau = (2/c)*(param.y_pc*uy - param.z_pc*uz); + tau = (2/c)*(param.y_pc*uy - param.z_pc*uz)*param.sv_dielectric; val = wb_compute_cost(tau, DCM, param.fc, param.fs, h, t0, dt); J(idx) = 10*log10(abs(val)); end @@ -207,7 +207,7 @@ search_theta = mean(param.src_limits{src_idx}); end - % Evaluate cost cunction over coarse grid whose step size is set by + % Evaluate cost function over coarse grid whose step size is set by % array_param.Nsv J = NaN(size(search_theta)); for idx = 1:length(search_theta); @@ -215,8 +215,8 @@ theta = [theta_i;theta0]; uy = sin(theta).'; uz = cos(theta).'; - tau = (2/c)*(param.y_pc*uy - param.z_pc*uz); - val = wb_compute_cost(tau,DCM,param.fc, param.fs, h,t0,dt); + tau = (2/c)*(param.y_pc*uy - param.z_pc*uz)*param.sv_dielectric; + val = wb_compute_cost(tau, DCM, param.fc, param.fs, h, t0, dt); J(idx) = 10*log10(abs(val)); end @@ -239,5 +239,3 @@ out = theta0; end - -return; diff --git a/cresis-toolbox/processing/wbmle_cost_function.m b/cresis-toolbox/processing/wbmle_cost_function.m index 49e63ce9..4b2f1d90 100644 --- a/cresis-toolbox/processing/wbmle_cost_function.m +++ b/cresis-toolbox/processing/wbmle_cost_function.m @@ -43,7 +43,7 @@ % Setup steering vectors for the fixed theta_eval = [param.theta_fixed(:).',theta]; theta_eval = theta_eval(:).'; % make theta have the right dimensions - k = 4*pi*param.fc/c; + k = 4*pi*param.fc*param.sv_dielectric/c; ky = k*sin(theta_eval).'; kz = k*cos(theta_eval).'; SVs = (1/sqrt(length(param.y_pc)))*exp(1i*(param.y_pc*ky - param.z_pc*kz)); @@ -57,7 +57,7 @@ else DCM = param.Rxx; Nk = param.nb_filter_banks; - k = 4*pi*(param.fc + param.fs*[0:floor((Nk-1)/2), -floor(Nk/2):-1]/Nk)/c; + k = 4*pi*(param.fc + param.fs*[0:floor((Nk-1)/2), -floor(Nk/2):-1]/Nk)/c*param.sv_dielectric; L = 0; for k_idx = 1:Nk diff --git a/cresis-toolbox/processing/wbmle_initialization.m b/cresis-toolbox/processing/wbmle_initialization.m index 1bf2f57c..3b8355c3 100644 --- a/cresis-toolbox/processing/wbmle_initialization.m +++ b/cresis-toolbox/processing/wbmle_initialization.m @@ -42,7 +42,7 @@ physical_constants Nk = param.nb_filter_banks; -k = 4*pi*(param.fc + param.fs*[0:floor((Nk-1)/2), -floor(Nk/2):-1]/Nk)/c; +k = 4*pi*(param.fc + param.fs*[0:floor((Nk-1)/2), -floor(Nk/2):-1]/Nk)/c*param.sv_dielectric; if isfield(param,'search_type') && strcmpi(param.search_type,'grid') %% Perform N-dimensional grid search diff --git a/cresis-toolbox/processing/wf_adc_to_board.m b/cresis-toolbox/processing/wf_adc_to_board.m index 53d36445..0e9fd549 100644 --- a/cresis-toolbox/processing/wf_adc_to_board.m +++ b/cresis-toolbox/processing/wf_adc_to_board.m @@ -30,29 +30,44 @@ % NI systems with 4 adcs per board board = unique(floor((wf_adc_list(:,2).'-1)/4)); board_idx = board+1; - profile = []; + profile = {}; elseif any(param.records.file.version == [410]) % MCRDS with all adcs on one board board = 1; board_idx = 1; - profile = []; + profile = {}; elseif any(param.records.file.version == [9 10 103 412]) % RSS which uses param.records.data_map board = zeros(1,size(wf_adc_list,1)); - profile = zeros(size(wf_adc_list,1),2); + profile = cell(size(wf_adc_list,1),1); for wf_adc = 1:size(wf_adc_list,1) found = false; for board_idx = 1:length(param.records.data_map) for profile_idx = 1:size(param.records.data_map{board_idx},1) - wf = param.records.data_map{board_idx}(profile_idx,3); % processing wf - adc = param.records.data_map{board_idx}(profile_idx,4); % processing adc - if wf_adc_list(wf_adc,1) == wf && wf_adc_list(wf_adc,2) == adc - board(wf_adc) = board_idx; - profile(wf_adc,1) = param.records.data_map{board_idx}(profile_idx,1); % mode - profile(wf_adc,2) = param.records.data_map{board_idx}(profile_idx,2); % subchannel - found = true; + if size(param.records.data_map{board_idx},2) == 4 + wf = param.records.data_map{board_idx}(profile_idx,3); % processing wf + adc = param.records.data_map{board_idx}(profile_idx,4); % processing adc + if wf_adc_list(wf_adc,1) == wf && wf_adc_list(wf_adc,2) == adc + board(wf_adc) = board_idx; + % Add an entry (this approach allows multiple mode,subchannel + % combinations to be lumped into a single wf/adc pair). + profile{wf_adc}(end+1,1) = param.records.data_map{board_idx}(profile_idx,1); % mode + profile{wf_adc}(end,2) = param.records.data_map{board_idx}(profile_idx,2); % subchannel + found = true; + end + else + wf = param.records.data_map{board_idx}(profile_idx,4); % processing wf + adc = param.records.data_map{board_idx}(profile_idx,5); % processing adc + if wf_adc_list(wf_adc,1) == wf && wf_adc_list(wf_adc,2) == adc + board(wf_adc) = board_idx; + % Add an entry (this approach allows multiple mode,subchannel + % combinations to be lumped into a single wf/adc pair). + profile{wf_adc}(end+1,1) = param.records.data_map{board_idx}(profile_idx,2); % mode + profile{wf_adc}(end,2) = param.records.data_map{board_idx}(profile_idx,3); % subchannel + found = true; + end end end end @@ -67,7 +82,7 @@ % NI system if isfield(param.records,'data_map') && ~isempty(param.records.data_map) board = zeros(1,size(wf_adc_list,1)); - profile = zeros(size(wf_adc_list,1),2); + profile = cell(size(wf_adc_list,1),1); for wf_adc = 1:size(wf_adc_list,1) found = false; for board_idx = 1:length(param.records.data_map) @@ -78,14 +93,14 @@ && ( isnan(adc) || wf_adc_list(wf_adc,2) == adc ) board(wf_adc) = board_idx; if isnan(param.records.data_map{board_idx}(profile_idx,1)) - profile(wf_adc,1) = wf_adc_list(wf_adc,1); % hardware wf matches processing wf + profile{wf_adc}(1,1) = wf_adc_list(wf_adc,1); % hardware wf matches processing wf else - profile(wf_adc,1) = param.records.data_map{board_idx}(profile_idx,1); % hardware wf + profile{wf_adc}(1,1) = param.records.data_map{board_idx}(profile_idx,1); % hardware wf end if isnan(param.records.data_map{board_idx}(profile_idx,2)) - profile(wf_adc,2) = wf_adc_list(wf_adc,2); % hardware adc matches processing adc + profile{wf_adc}(1,2) = wf_adc_list(wf_adc,2); % hardware adc matches processing adc else - profile(wf_adc,2) = param.records.data_map{board_idx}(profile_idx,2); % hardware adc + profile{wf_adc}(1,2) = param.records.data_map{board_idx}(profile_idx,2); % hardware adc end found = true; end @@ -100,20 +115,26 @@ else board = unique(wf_adc_list(:,2).'); board_idx = board; - profile = []; + profile = {}; end elseif any(param.records.file.version == [413]) % UTUA HFRDS board = 1; board_idx = board; - profile = []; + profile = {}; elseif any(param.records.file.version == [414]) % BAS Matlab RDS board = 12*(wf_adc_list(:,1)-1) + wf_adc_list(:,2); board_idx = ones(size(board)); - profile = []; + profile = {}; + +elseif any(param.records.file.version == [415]) + % BAS Matlab RDS + board = 1; + board_idx = ones(size(board)); + profile = {}; else % All other systems @@ -130,14 +151,14 @@ && ( isnan(adc) || wf_adc_list(wf_adc,2) == adc ) board(wf_adc) = board_idx; if isnan(param.records.data_map{board_idx}(profile_idx,1)) - profile(wf_adc,1) = wf_adc_list(wf_adc,1); % hardware wf matches processing wf + profile{wf_adc}(1,1) = wf_adc_list(wf_adc,1); % hardware wf matches processing wf else - profile(wf_adc,1) = param.records.data_map{board_idx}(profile_idx,1); % hardware wf + profile{wf_adc}(1,1) = param.records.data_map{board_idx}(profile_idx,1); % hardware wf end if isnan(param.records.data_map{board_idx}(profile_idx,2)) - profile(wf_adc,2) = wf_adc_list(wf_adc,2); % hardware adc matches processing adc + profile{wf_adc}(1,2) = wf_adc_list(wf_adc,2); % hardware adc matches processing adc else - profile(wf_adc,2) = param.records.data_map{board_idx}(profile_idx,2); % hardware adc + profile{wf_adc}(1,2) = param.records.data_map{board_idx}(profile_idx,2); % hardware adc end found = true; end @@ -152,7 +173,7 @@ else board = unique(wf_adc_list(:,2).'); board_idx = board; - profile = []; + profile = {}; end end diff --git a/cresis-toolbox/processing/write_arena_xml.m b/cresis-toolbox/processing/write_arena_xml.m index c115691b..d9b8a0bf 100644 --- a/cresis-toolbox/processing/write_arena_xml.m +++ b/cresis-toolbox/processing/write_arena_xml.m @@ -19,6 +19,47 @@ % See also: read_arena_xml, read_cresis_xml, write_arena_xml, write_ni_xml, % write_radar_xml +%% Input checks +for wf = 1:length(param.wfs) + if ~isfield(param.wfs(wf),'f0') || isempty(param.wfs(wf).f0) + param.wfs(wf).f0 = param.f0; + end + if ~isfield(param.wfs(wf),'f1') || isempty(param.wfs(wf).f1) + param.wfs(wf).f1 = param.f1; + end + if (~isfield(param.wfs(wf),'DDC_freq') || isempty(param.wfs(wf).DDC_freq)) && isfield(param,'DDC_freq') + param.wfs(wf).DDC_freq = param.DDC_freq; + end + if ~isfield(param.wfs(wf),'tukey') || isempty(param.wfs(wf).tukey) + param.wfs(wf).tukey = param.tukey; + end + if ~isfield(param.wfs(wf),'Tpd') || isempty(param.wfs(wf).Tpd) + param.wfs(wf).Tpd = param.Tpd; + end + if ~isfield(param.wfs(wf),'zeropimods') || isempty(param.wfs(wf).zeropimods) + param.wfs(wf).zeropimods = param.zeropimods; + end + if ~isfield(param.wfs,'name') + param.wfs(wf).name = ''; + end + if ~isfield(param.wfs(wf),'phase') || isempty(param.wfs(wf).phase) + param.wfs(wf).phase = param.phase; + end + if ~isfield(param.wfs(wf),'delay') || isempty(param.wfs(wf).delay) + param.wfs(wf).delay = param.delay; + end + if (~isfield(param.wfs(wf),'tx_enable') || isempty(param.wfs(wf).tx_enable)) && isfield(param.config,'tx_enable') + param.wfs(wf).tx_enable = param.config.tx_enable; + end + if (~isfield(param.wfs(wf),'scale') || isempty(param.wfs(wf).scale)) && isfield(param,'scale') + param.wfs(wf).scale = param.scale; + end + if any(param.wfs(wf).scale > param.config.max_tx) + error('Tx weights param.wfs(%d).scale=%s > param.config.max_tx=%s for waveform %d', ... + wf, mat2str_generic(param.wfs(wf).scale), mat2str_generic(param.config.max_tx), wf); + end +end + %% Initialization wfs = param.wfs; @@ -46,132 +87,280 @@ %% Determine modes/sequences for each waveform total_modes = 0; +total_indexes = 0; for wf = 1:length(wfs) zeropimods = wfs(wf).zeropimods(:).'; - - % Sequence of modes goes: - % For length(zeropimods) == 1 - % If wf == 1 - % 1. Phase is zeropimods(1) - % - % For length(zeropimods) == N > 1 - % 1. Phase is zeropimods(1), eprireset integrator - % 2. Phase is zeropimods(2) - % ... - % N. Phase is zeropimods(N) - % N+1. Phase is zeropimods(1) - - if numel(zeropimods) == 1 - % This waveform has only one phase in its zero_pi_mode sequence. - zeropimod = zeropimods(1); + + if strcmpi(arena.psc.type,'psc_0001') + % psc_0001 + % ===================================================================== + % Does not support "next" field - % We use integrator reset, so more than one presum requires two - % modes, one to reset and one to determine the number of presums - if wfs(wf).presums > 1 - num_modes = 2; - wfs(wf).reset = false(1,num_modes); - wfs(wf).reset(1) = true; - wfs(wf).next = total_modes+[1 2]; - wfs(wf).repeat_to = [0 total_modes+1]; - wfs(wf).repeat_count = [0 wfs(wf).presums/numel(zeropimods)-1]; - else - % No presumming so no reset mode is required - num_modes = 1; + if numel(zeropimods) == 1 + % This waveform has only one phase in its zero_pi_mode sequence. + zeropimod = zeropimods(1); + + if wf == 1 && wfs(wf).presums > 1 + % Need to add an EPRI mode since this is the first waveform and + % there is more than 1 presum + num_modes = 1; + num_indexes = 1; + wfs(wf).next = total_indexes+1; + wfs(wf).repeat_to = total_indexes; + wfs(wf).repeat_count = wfs(wf).presums/numel(zeropimods)-1; + else + num_modes = 2; + num_indexes = 2; + wfs(wf).next = total_indexes+[1 2]; + wfs(wf).repeat_to = total_indexes+1; + wfs(wf).repeat_count = wfs(wf).presums/numel(zeropimods)-2; + end + wfs(wf).modes = total_modes+(0:num_modes-1); + wfs(wf).epri = false(1,num_modes); wfs(wf).reset = false(1,num_modes); - wfs(wf).next = total_modes+[1]; - wfs(wf).repeat_to = 0; - wfs(wf).repeat_count = 0; - end - if zeropimod==180 || zeropimod==270 - wfs(wf).tx_invert = true(1,num_modes); - else - wfs(wf).tx_invert = false(1,num_modes); - end - if zeropimod==180 || zeropimod==270 - wfs(wf).adc_zeropi = true(1,num_modes); - else - wfs(wf).adc_zeropi = false(1,num_modes); - end - if zeropimod==90 || zeropimod==270 - wfs(wf).zeropiphase = 90*ones(1,num_modes); + if zeropimod==180 || zeropimod==270 + wfs(wf).adc_zeropi = true(1,num_modes); + else + wfs(wf).adc_zeropi = false(1,num_modes); + end + if any(find(strcmpi('dac-ad9129_0014',{arena.dac.type}))) + % This DAC does not support tx_invert so a new waveform must be + % loaded for each zero pi phase + wfs(wf).zeropiphase = zeropimod*ones(1,num_modes); + wfs(wf).tx_invert = false(1,num_modes); + else + if zeropimod==180 || zeropimod==270 + wfs(wf).tx_invert = true(1,num_modes); + else + wfs(wf).tx_invert = false(1,num_modes); + end + if zeropimod==90 || zeropimod==270 + wfs(wf).zeropiphase = 90*ones(1,num_modes); + else + wfs(wf).zeropiphase = zeros(1,num_modes); + end + end + else - wfs(wf).zeropiphase = zeros(1,num_modes); - end - - else - if mod(wfs(wf).presums,numel(zeropimods)) - error('The number of presums, wfs(wf).presums = %d, must be a multiple of numel(wfs(wf).zeropimods) = %d.', wfs(wf).presums, numel(wfs(wf).zeropimods)); + % This waveform has more than one phase in the zero_pi_mode sequence. + if mod(wfs(wf).presums,numel(zeropimods)) + error('The number of presums, wfs(wf).presums = %d, must be a multiple of numel(wfs(wf).zeropimods) = %d.', wfs(wf).presums, numel(wfs(wf).zeropimods)); + end + + wfs(wf).tx_invert = false(1,numel(zeropimods)); + wfs(wf).adc_zeropi = false(1,numel(zeropimods)); + wfs(wf).zeropiphase = zeros(1,numel(zeropimods)); + for zeropimod_idx = 1:numel(zeropimods) + zeropimod = zeropimods(zeropimod_idx); + if zeropimod==180 || zeropimod==270 + wfs(wf).adc_zeropi(zeropimod_idx) = true; + else + wfs(wf).adc_zeropi(zeropimod_idx) = false; + end + if any(find(strcmpi('dac-ad9129_0014',{arena.dac.type}))) + % This DAC does not support tx_invert so a new waveform must be + % loaded for each zero pi phase + wfs(wf).zeropiphase(zeropimod_idx) = zeropimod; + wfs(wf).tx_invert(zeropimod_idx) = false; + else + if zeropimod==180 || zeropimod==270 + wfs(wf).tx_invert(zeropimod_idx) = true; + else + wfs(wf).tx_invert(zeropimod_idx) = false; + end + if zeropimod==90 || zeropimod==270 + wfs(wf).zeropiphase(zeropimod_idx) = 90; + else + wfs(wf).zeropiphase(zeropimod_idx) = 0; + end + end + end + + if wf == 1 && wfs(wf).presums / numel(zeropimods) > 1 + % We cycle through the zero-pi modes more than once and the very + % first mode needs to be the EPRI mode so we have to add an extra + % index/mode to handle this special case. We therefore create a + % one-time pass through the zero pi modes the first of which is + % marked as the EPRI mode and the second and later passes through + % the zero pi modes are not EPRI modes. + num_modes = numel(zeropimods)+1; + num_indexes = 2*numel(zeropimods); + wfs(wf).tx_invert = wfs(wf).tx_invert([1:end, 1:end]); + wfs(wf).adc_zeropi = wfs(wf).adc_zeropi([1:end, 1:end]); + wfs(wf).zeropiphase = wfs(wf).zeropiphase([1:end, 1:end]); + wfs(wf).next = total_indexes+[1:2*numel(zeropimods)]; + wfs(wf).repeat_to = zeros(1,2*numel(zeropimods)); + wfs(wf).repeat_to(end) = total_indexes+numel(zeropimods); + wfs(wf).repeat_count = zeros(1,2*numel(zeropimods)); + wfs(wf).repeat_count(end) = wfs(wf).presums/numel(zeropimods)-2; + wfs(wf).modes = total_modes+[0:numel(zeropimods) 1:numel(zeropimods)-1]; + wfs(wf).epri = false(1,2*numel(zeropimods)); + wfs(wf).reset = false(1,2*numel(zeropimods)); + + else + % This waveform does not require separate passes through the + % zero-pi modes for EPRI and non-EPRI modes. + num_modes = numel(zeropimods); + num_indexes = numel(zeropimods); + wfs(wf).next = total_indexes+(1:num_modes); + wfs(wf).repeat_to = zeros(1,num_modes); + wfs(wf).repeat_to(end) = total_indexes; + wfs(wf).repeat_count = zeros(1,num_modes); + wfs(wf).repeat_count(end) = wfs(wf).presums/numel(zeropimods)-1; + wfs(wf).modes = total_modes+(0:num_modes-1); + wfs(wf).epri = false(1,num_modes); + wfs(wf).reset = false(1,num_modes); + end + end - - wfs(wf).tx_invert = false(1,numel(zeropimods)); - wfs(wf).adc_zeropi = false(1,numel(zeropimods)); - wfs(wf).zeropiphase = zeros(1,numel(zeropimods)); - for zeropimod_idx = 1:numel(zeropimods) - zeropimod = zeropimods(zeropimod_idx); - if zeropimod==180 || zeropimod==270 - wfs(wf).tx_invert(zeropimod_idx) = true; + + elseif strcmpi(arena.psc.type,'psc_0003') + % psc_0003 + % ===================================================================== + % Sequence of modes goes: + % For length(zeropimods) == 1 + % If wf == 1 + % 1. Phase is zeropimods(1) + % + % For length(zeropimods) == N > 1 + % 1. Phase is zeropimods(1), eprireset integrator + % 2. Phase is zeropimods(2) + % ... + % N. Phase is zeropimods(N) + % N+1. Phase is zeropimods(1) + + if numel(zeropimods) == 1 + % This waveform has only one phase in its zero_pi_mode sequence. + zeropimod = zeropimods(1); + + % We use integrator reset, so more than one presum requires two + % modes, one to reset and one to determine the number of presums + if wfs(wf).presums > 1 + num_modes = 2; + wfs(wf).reset = false(1,num_modes); + wfs(wf).reset(1) = true; + wfs(wf).next = total_modes+[1 2]; + wfs(wf).repeat_to = [0 total_modes+1]; + wfs(wf).repeat_count = [0 wfs(wf).presums/numel(zeropimods)-1]; else - wfs(wf).tx_invert(zeropimod_idx) = false; + % No presumming so no reset mode is required + num_modes = 1; + wfs(wf).reset = false(1,num_modes); + wfs(wf).next = total_modes+[1]; + wfs(wf).repeat_to = 0; + wfs(wf).repeat_count = 0; end if zeropimod==180 || zeropimod==270 - wfs(wf).adc_zeropi(zeropimod_idx) = true; + wfs(wf).adc_zeropi = true(1,num_modes); else - wfs(wf).adc_zeropi(zeropimod_idx) = false; + wfs(wf).adc_zeropi = false(1,num_modes); end - if zeropimod==90 || zeropimod==270 - wfs(wf).zeropiphase(zeropimod_idx) = 90; + if any(find(strcmpi('dac-ad9129_0014',{arena.dac.type}))) + % This DAC does not support tx_invert so a new waveform must be + % loaded for each zero pi phase + wfs(wf).zeropiphase = zeropimod*ones(1,num_modes); + wfs(wf).tx_invert = false(1,num_modes); else - wfs(wf).zeropiphase(zeropimod_idx) = 0; + if zeropimod==180 || zeropimod==270 + wfs(wf).tx_invert = true(1,num_modes); + else + wfs(wf).tx_invert = false(1,num_modes); + end + if zeropimod==90 || zeropimod==270 + wfs(wf).zeropiphase = 90*ones(1,num_modes); + else + wfs(wf).zeropiphase = zeros(1,num_modes); + end end - end - - if wfs(wf).presums / numel(zeropimods) > 1 - % We use integrator reset, so more than one cycle through its zero pi - % mode sequence requires a reset and a non-reset version of the first - % zero pi mode in the sequence to reset the integrator. Two modes are - % created for the first mode (except that the first one will be reset - % and the second one will not be reset). - num_modes = 1+numel(zeropimods); - wfs(wf).tx_invert = wfs(wf).tx_invert([1 1 2:end]); - wfs(wf).adc_zeropi = wfs(wf).adc_zeropi([1 1 2:end]); - wfs(wf).zeropiphase = wfs(wf).zeropiphase([1 1 2:end]); - wfs(wf).next = total_modes+[2 2:num_modes]; - wfs(wf).repeat_to = zeros(1,num_modes); - wfs(wf).repeat_to(end) = total_modes+1; - wfs(wf).repeat_count = zeros(1,num_modes); - wfs(wf).repeat_count(end) = wfs(wf).presums/numel(zeropimods)-1; - + else - % This waveform has only one cycle through zero pi mode sequence, so - % we don't need a reset and non-reset mode for the first zero pi - % mode in the sequence. - num_modes = numel(zeropimods); - wfs(wf).next = total_modes+(1:num_modes); - wfs(wf).repeat_to = zeros(1,num_modes); - wfs(wf).repeat_count = zeros(1,num_modes); + % This waveform has more than one phase in the zero_pi_mode sequence. + if mod(wfs(wf).presums,numel(zeropimods)) + error('The number of presums, wfs(wf).presums = %d, must be a multiple of numel(wfs(wf).zeropimods) = %d.', wfs(wf).presums, numel(wfs(wf).zeropimods)); + end + + wfs(wf).tx_invert = false(1,numel(zeropimods)); + wfs(wf).adc_zeropi = false(1,numel(zeropimods)); + wfs(wf).zeropiphase = zeros(1,numel(zeropimods)); + for zeropimod_idx = 1:numel(zeropimods) + zeropimod = zeropimods(zeropimod_idx); + if zeropimod==180 || zeropimod==270 + wfs(wf).adc_zeropi(zeropimod_idx) = true; + else + wfs(wf).adc_zeropi(zeropimod_idx) = false; + end + if any(find(strcmpi('dac-ad9129_0014',{arena.dac.type}))) + % This DAC does not support tx_invert so a new waveform must be + % loaded for each zero pi phase + wfs(wf).zeropiphase(zeropimod_idx) = zeropimod; + wfs(wf).tx_invert(zeropimod_idx) = false; + else + if zeropimod==180 || zeropimod==270 + wfs(wf).tx_invert(zeropimod_idx) = true; + else + wfs(wf).tx_invert(zeropimod_idx) = false; + end + if zeropimod==90 || zeropimod==270 + wfs(wf).zeropiphase(zeropimod_idx) = 90; + else + wfs(wf).zeropiphase(zeropimod_idx) = 0; + end + end + end + + if wfs(wf).presums / numel(zeropimods) > 1 + % We use integrator reset, so more than one cycle through its zero pi + % mode sequence requires a reset and a non-reset version of the first + % zero pi mode in the sequence to reset the integrator. Two modes are + % created for the first mode (except that the first one will be reset + % and the second one will not be reset). + num_modes = 1+numel(zeropimods); + wfs(wf).tx_invert = wfs(wf).tx_invert([1 1 2:end]); + wfs(wf).adc_zeropi = wfs(wf).adc_zeropi([1 1 2:end]); + wfs(wf).zeropiphase = wfs(wf).zeropiphase([1 1 2:end]); + wfs(wf).next = total_modes+[2 2:num_modes]; + wfs(wf).repeat_to = zeros(1,num_modes); + wfs(wf).repeat_to(end) = total_modes+1; + wfs(wf).repeat_count = zeros(1,num_modes); + wfs(wf).repeat_count(end) = wfs(wf).presums/numel(zeropimods)-1; + + else + % This waveform has only one cycle through zero pi mode sequence, so + % we don't need a reset and non-reset mode for the first zero pi + % mode in the sequence. + num_modes = numel(zeropimods); + wfs(wf).next = total_modes+(1:num_modes); + wfs(wf).repeat_to = zeros(1,num_modes); + wfs(wf).repeat_count = zeros(1,num_modes); + end + % A reset is always required with zero pi modulation, because there is + % always more than one pulse being averaged. + wfs(wf).reset = false(1,num_modes); + wfs(wf).reset(1) = true; + end - % A reset is always required with zero pi modulation, because there is - % always more than one pulse being averaged. - wfs(wf).reset = false(1,num_modes); - wfs(wf).reset(1) = true; - + + wfs(wf).modes = total_modes+(0:num_modes-1); + wfs(wf).epri = false(1,num_modes); + num_indexes = num_modes; + + else + error('param.arena.psc.type must be psc_0001 or psc_0003. The correct version depends on the hardware firmware.'); end - - wfs(wf).modes = total_modes+(0:num_modes-1); - + % If this is the first waveform, then the epri is true for the first % mode of this waveform. - wfs(wf).epri = false(1,num_modes); if wf == 1 wfs(wf).epri(1) = true; end - + % If the last waveform, then the next should go to the first mode if wf == length(wfs) wfs(wf).next(end) = 0; end - + total_modes = total_modes + num_modes; + total_indexes = total_indexes + num_indexes; end @@ -230,7 +419,11 @@ %% ADC: adc-ad9680_0017 -adc_idxs = find(strcmpi('adc-ad9680_0017',{arena.adc.type})); +if isempty(arena.adc) + adc_idxs = []; +else + adc_idxs = find(strcmpi('adc-ad9680_0017',{arena.adc.type})); +end param.data_rate = 0; for adc_idx = adc_idxs @@ -429,7 +622,7 @@ config.setAttribute('type',arena.ctu.type); child = doc.createElement('name'); config.appendChild(child); - child.appendChild(doc.createTextNode('ctu')); + child.appendChild(doc.createTextNode(arena.ctu.name)); child = doc.createElement('description'); config.appendChild(child); child.appendChild(doc.createTextNode('')); @@ -550,10 +743,10 @@ config = doc.createElement('config'); configs.appendChild(config); - config.setAttribute('type',arena.ctu_type); + config.setAttribute('type',arena.ctu.type); child = doc.createElement('name'); config.appendChild(child); - child.appendChild(doc.createTextNode('ctu')); + child.appendChild(doc.createTextNode(arena.ctu.name)); child = doc.createElement('description'); config.appendChild(child); child.appendChild(doc.createTextNode('')); @@ -571,27 +764,28 @@ child = doc.createElement('numSegments'); config.appendChild(child); child.appendChild(doc.createTextNode(sprintf('%d',size(arena.TTL_states{1},2)))); - num_modes = 0; for wf = 1:length(arena.wfs) segment_times = [arena.TTL_time(1:2) wfs(wf).Tpd*1e6+arena.TTL_time(3) arena.PRI*1e6]; - if wf == 1 - wf_modes = length(arena.zeropimods)+1; - segment_states_idx = [1 2*ones(size(arena.zeropimods))]; - else - wf_modes = length(arena.zeropimods); - segment_states_idx = [2*ones(size(arena.zeropimods))]; - end + + modes = wfs(wf).modes; + [~,unique_idxs] = unique(modes); + + for mode_idx = unique_idxs(:).' + mode_latch = modes(mode_idx); - for mode = 0:wf_modes-1 child = doc.createElement('mode'); config.appendChild(child); child.appendChild(doc.createTextNode('')); grandchild = doc.createElement('id'); child.appendChild(grandchild); - grandchild.appendChild(doc.createTextNode(sprintf('%d',num_modes+mode))); + grandchild.appendChild(doc.createTextNode(sprintf('%d',mode_latch))); grandchild = doc.createElement('segmentTimes'); child.appendChild(grandchild); segment_time_str = [sprintf('%g', segment_times(1)), sprintf(' %g', segment_times(2:end))]; grandchild.appendChild(doc.createTextNode(segment_time_str)); grandchild = doc.createElement('segmentStates'); child.appendChild(grandchild); - idx = segment_states_idx(mode+1); + if wfs(wf).epri(mode_idx) == 1 + idx = 1; + else + idx = 2; + end segment_state_str = lower(dec2hex(bin2dec(char(arena.TTL_states{idx}(end:-1:1,1).'+48)),8)); for state_idx = 2:size(arena.TTL_states{idx},2) segment_state_str = cat(2,segment_state_str, ' ', ... @@ -599,7 +793,6 @@ end grandchild.appendChild(doc.createTextNode(segment_state_str)); end - num_modes = num_modes + wf_modes; end child = doc.createElement('pps'); config.appendChild(child); @@ -666,9 +859,11 @@ for wf = 1:length(wfs) Tpd = round(wfs(wf).Tpd*1e6); + modes = wfs(wf).modes; + [~,unique_idxs] = unique(modes); - for mode_idx = 1:length(modes) + for mode_idx = unique_idxs(:).' mode_latch = modes(mode_idx); mode_xml = doc.createElement('mode'); config.appendChild(mode_xml); @@ -689,11 +884,11 @@ child = doc.createElement('config'); mode_xml.appendChild(child); child.setAttribute('type','dac-ad9129_0012_waveform'); - if wfs(wf).tx_enable(dac_idx) + if wfs(wf).enabled(dac_idx) if wfs(wf).zeropiphase(mode_idx) == 0 - child.appendChild(doc.createTextNode(sprintf('waveformCh%d_%s_%dus',dac_idx,wfs(wf).name,Tpd) )); + child.appendChild(doc.createTextNode(sprintf('waveformCh%d_%s_%dus',dac_idx-1,wfs(wf).name,Tpd) )); else - child.appendChild(doc.createTextNode(sprintf('waveformCh%d_%s_%dus_%.0fdeg',dac_idx,wfs(wf).name,Tpd,wfs(wf).zeropiphase) )); + child.appendChild(doc.createTextNode(sprintf('waveformCh%d_%s_%dus_%.0fdeg',dac_idx-1,wfs(wf).name,Tpd,wfs(wf).zeropiphase(mode_idx)) )); end else child.appendChild(doc.createTextNode('No_Tx')); @@ -707,7 +902,7 @@ for dac_idx = dac_idxs dac = arena.dac(dac_idx); - tx_idx = find(strcmpi(dac.name,param.tx_map)); + tx_idx = find(strcmpi(dac.name,param.config.tx_map)); system = doc.getFirstChild; configs = system.getFirstChild; @@ -719,8 +914,8 @@ fc = (wfs(wf).f0+wfs(wf).f1)/2; %Nt = round(wfs(wf).Tpd * dac.dacClk/8)*8; Nt = 256; - BW = abs(wfs(wf).f1-wfs(wf).f0); - %scale = wfs(wf).tx_weights(tx_idx); + BW = wfs(wf).f1-wfs(wf).f0; + %scale = wfs(wf).scale(tx_idx); scale = 0; zeropiphase = 0; alpha = wfs(wf).tukey; @@ -768,9 +963,9 @@ fc = (wfs(wf).f0+wfs(wf).f1)/2; Nt = round(wfs(wf).Tpd * dac.dacClk/8)*8; - BW = abs(wfs(wf).f1-wfs(wf).f0); + BW = wfs(wf).f1-wfs(wf).f0; if wfs(wf).tx_enable(tx_idx) - scale = wfs(wf).tx_weights(tx_idx); + scale = wfs(wf).scale(tx_idx); else scale = 0; end @@ -782,9 +977,9 @@ child = doc.createElement('name'); config.appendChild(child); if wfs(wf).zeropiphase(mode_idx) == 0 - child.appendChild(doc.createTextNode(sprintf('waveformCh%d_%s_%dus',dac_idx,wfs(wf).name,Tpd) )); + child.appendChild(doc.createTextNode(sprintf('waveformCh%d_%s_%dus',dac_idx-1,wfs(wf).name,Tpd) )); else - child.appendChild(doc.createTextNode(sprintf('waveformCh%d_%s_%dus_%.0fdeg',dac_idx,wfs(wf).name,Tpd,zeropiphase) )); + child.appendChild(doc.createTextNode(sprintf('waveformCh%d_%s_%dus_%.0fdeg',dac_idx-1,wfs(wf).name,Tpd,zeropiphase) )); end child = doc.createElement('description'); config.appendChild(child); @@ -824,156 +1019,156 @@ %% DAC: dac-ad9129_0014 dac_idxs = find(strcmpi('dac-ad9129_0014',{arena.dac.type})); for dac_idx = dac_idxs - + dac = arena.dac(dac_idx); system = doc.getFirstChild; configs = system.getFirstChild; - - for dac = arena.dacs - config = doc.createElement('config'); configs.appendChild(config); - config.setAttribute('type',dac.type); - - child = doc.createElement('name'); config.appendChild(child); - child.appendChild(doc.createTextNode(sprintf('dacConfig%d',dac))); - - child = doc.createElement('description'); config.appendChild(child); - child.appendChild(doc.createTextNode('')); + + config = doc.createElement('config'); configs.appendChild(config); + config.setAttribute('type',dac.type); + + child = doc.createElement('name'); config.appendChild(child); + child.appendChild(doc.createTextNode(sprintf('%s',dac.name))); + + child = doc.createElement('description'); config.appendChild(child); + child.appendChild(doc.createTextNode('')); + + for wf = 1:length(arena.wfs) + Tpd = round(wfs(wf).Tpd*1e6); + delay = arena.dacs_start_delay - arena.dacs_internal_delay; + if delay < 0 + error('Delay "arena.dacs_start_delay - arena.dacs_internal_delay" must be nonnegative: %g.', delay); + end + + modes = wfs(wf).modes; + [~,unique_idxs] = unique(modes); - num_modes = 0; - for wf = 1:length(arena.wfs) - zeropimods = wfs(wf).zeropimods(:).'; - Tpd = round(wfs(wf).Tpd*1e6); - delay = arena.dacs_start_delay - arena.dacs_internal_delay; - if delay < 0 - error('Delay "arena.dacs_start_delay - arena.dacs_internal_delay" must be nonnegative: %g.', delay); - end - if wf == 1 - wf_modes = 1+length(zeropimods); + for mode_idx = unique_idxs(:).' + mode_latch = modes(mode_idx); + + child = doc.createElement('mode'); config.appendChild(child); + child.appendChild(doc.createTextNode('')); + grandchild = doc.createElement('id'); child.appendChild(grandchild); + grandchild.appendChild(doc.createTextNode(sprintf('%d',mode_latch))); + grandchild = doc.createElement('enabled'); child.appendChild(grandchild); + grandchild.appendChild(doc.createTextNode('1')); + grandchild = doc.createElement('delay'); child.appendChild(grandchild); + grandchild.appendChild(doc.createTextNode(sprintf('%.6f',delay))); + grandchild = doc.createElement('vDelayEnabled'); child.appendChild(grandchild); + grandchild.appendChild(doc.createTextNode('0')); + grandchild = doc.createElement('vDelay'); child.appendChild(grandchild); + grandchild.appendChild(doc.createTextNode('0.000000')); + grandchild = doc.createElement('config'); child.appendChild(grandchild); + if wfs(wf).enabled(dac_idx) + grandchild.appendChild(doc.createTextNode(sprintf('waveformCh%d_%s_%dus_%.0fdeg',dac_idx-1,wfs(wf).name,Tpd,wfs(wf).zeropiphase(mode_idx)) )); else - wf_modes = length(zeropimods); + grandchild.appendChild(doc.createTextNode('No_Tx')); end - - for mode = 0:wf_modes-1 - child = doc.createElement('mode'); config.appendChild(child); - child.appendChild(doc.createTextNode('')); - grandchild = doc.createElement('id'); child.appendChild(grandchild); - grandchild.appendChild(doc.createTextNode(sprintf('%d',num_modes+mode))); - grandchild = doc.createElement('enabled'); child.appendChild(grandchild); - grandchild.appendChild(doc.createTextNode('1')); - grandchild = doc.createElement('delay'); child.appendChild(grandchild); - grandchild.appendChild(doc.createTextNode(sprintf('%.6f',delay))); - grandchild = doc.createElement('vDelayEnabled'); child.appendChild(grandchild); - grandchild.appendChild(doc.createTextNode('0')); - grandchild = doc.createElement('vDelay'); child.appendChild(grandchild); - grandchild.appendChild(doc.createTextNode('0.000000')); - grandchild = doc.createElement('config'); child.appendChild(grandchild); - if wfs(wf).enabled(dac+1) - grandchild.appendChild(doc.createTextNode(sprintf('waveformCh%d_%s_%dus_%.0f',dac,wfs(wf).name,Tpd,zeropimods(1+mod(mode,length(zeropimods))) ))); - else - grandchild.appendChild(doc.createTextNode('No_Tx')); - end - grandchild.setAttribute('type','dac-ad9129_0014_waveform'); - end - num_modes = num_modes + wf_modes; + grandchild.setAttribute('type','dac-ad9129_0014_waveform'); end end + end %% DAC Waveforms: dac-ad9129_0014 dac_idxs = find(strcmpi('dac-ad9129_0014',{arena.dac.type})); -for dac_idx = dac_idxs - +if ~isempty(dac_idxs) + dac_idx = dac_idxs(1); + dac = arena.dac(dac_idx); - tx_idx = find(strcmpi(dac.name,param.tx_map)); - + tx_idx = find(strcmpi(dac.name,param.config.tx_map)); + system = doc.getFirstChild; configs = system.getFirstChild; - - wf = 1; dac_idx = 1; dac = arena.dacs(dac_idx); zeropimod = 0; - fs = arena.dacs_sampFreq(dac_idx); - fc = wfs(wf).fc; - BW = wfs(wf).BW; - Tpd = wfs(wf).Tpd; - alpha = wfs(wf).tukey; - equal.delay = wfs(wf).delay; - equal.phase = wfs(wf).phase; - equal.scale = wfs(wf).scale; - Nt = round((wfs(wf).Tpd+equal.delay/1e9) * fs); - - config = doc.createElement('config'); configs.appendChild(config); - config.setAttribute('type',sprintf('%s_waveform',dac.type)); - - child = doc.createElement('name'); config.appendChild(child); - child.appendChild(doc.createTextNode('No_Tx')); - - child = doc.createElement('description'); config.appendChild(child); - child.appendChild(doc.createTextNode('')); - - child = doc.createElement('sampFreq'); config.appendChild(child); - child.appendChild(doc.createTextNode(sprintf('%f',fs/1e6))); - - child = doc.createElement('pulse'); config.appendChild(child); - grandchild = doc.createElement('name'); child.appendChild(grandchild); - grandchild.appendChild(doc.createTextNode('Pulse')); - grandchild = doc.createElement('centerFreq'); child.appendChild(grandchild); - grandchild.appendChild(doc.createTextNode(sprintf('%f',fc(dac_idx)/1e6))); - grandchild = doc.createElement('bandwidth'); child.appendChild(grandchild); - grandchild.appendChild(doc.createTextNode(sprintf('%f',BW(dac_idx)/1e6))); - grandchild = doc.createElement('initialDelay'); child.appendChild(grandchild); - grandchild.appendChild(doc.createTextNode(sprintf('%f',equal.delay(dac_idx)*1e-3))); - grandchild = doc.createElement('initialPhase'); child.appendChild(grandchild); - grandchild.appendChild(doc.createTextNode(sprintf('%f',equal.phase(dac_idx)+zeropimod))); - grandchild = doc.createElement('afterPulseDelay'); child.appendChild(grandchild); - grandchild.appendChild(doc.createTextNode('1.000000')); - grandchild = doc.createElement('taper'); child.appendChild(grandchild); - grandchild.appendChild(doc.createTextNode('Tukey')); - grandchild = doc.createElement('alpha'); child.appendChild(grandchild); - grandchild.appendChild(doc.createTextNode(sprintf('%f',alpha))); - grandchild = doc.createElement('scale'); child.appendChild(grandchild); - grandchild.appendChild(doc.createTextNode('0.000000')); - grandchild = doc.createElement('numPoints'); child.appendChild(grandchild); - grandchild.appendChild(doc.createTextNode(sprintf('%d',Nt(dac_idx)))); - grandchild = doc.createElement('Filename'); child.appendChild(grandchild); - grandchild.appendChild(doc.createTextNode('')); - - waveform_names = {}; - for wf = 1:length(arena.wfs) - fc = wfs(wf).fc; - BW = wfs(wf).BW; + + if dac_idx == 1 + wf = 1; dac_idx = 1; dac = arena.dac(dac_idx); zeropiphase = 0; + fs = arena.dac(dac_idx).dacClk; + fc = (wfs(wf).f0+wfs(wf).f1)/2; + BW = wfs(wf).f1 - wfs(wf).f0; Tpd = wfs(wf).Tpd; alpha = wfs(wf).tukey; equal.delay = wfs(wf).delay; equal.phase = wfs(wf).phase; equal.scale = wfs(wf).scale; Nt = round((wfs(wf).Tpd+equal.delay/1e9) * fs); - - %[0.63 ] - % 0.1652 0.326800 0.511500 0.63 0.6300 0.511500 0.3268 0.1652 - % chebwin(8,30) - - for dac_idx=1:length(arena.dacs) - dac = arena.dacs(dac_idx); - fs = arena.dacs_sampFreq(dac_idx); - for zeropimod = wfs(wf).zeropimods(:).' - new_waveform_name = sprintf('waveformCh%d_%s_%.0fus_%.0f',dac,wfs(wf).name,Tpd*1e6,zeropimod); + + config = doc.createElement('config'); configs.appendChild(config); + config.setAttribute('type',sprintf('%s_waveform',dac.type)); + + child = doc.createElement('name'); config.appendChild(child); + child.appendChild(doc.createTextNode('No_Tx')); + + child = doc.createElement('description'); config.appendChild(child); + child.appendChild(doc.createTextNode('')); + + child = doc.createElement('sampFreq'); config.appendChild(child); + child.appendChild(doc.createTextNode(sprintf('%f',fs/1e6))); + + child = doc.createElement('pulse'); config.appendChild(child); + grandchild = doc.createElement('name'); child.appendChild(grandchild); + grandchild.appendChild(doc.createTextNode('Pulse')); + grandchild = doc.createElement('centerFreq'); child.appendChild(grandchild); + grandchild.appendChild(doc.createTextNode(sprintf('%f',fc(dac_idx)/1e6))); + grandchild = doc.createElement('bandwidth'); child.appendChild(grandchild); + grandchild.appendChild(doc.createTextNode(sprintf('%f',BW(dac_idx)/1e6))); + grandchild = doc.createElement('initialDelay'); child.appendChild(grandchild); + grandchild.appendChild(doc.createTextNode(sprintf('%f',equal.delay(dac_idx)*1e-3))); + grandchild = doc.createElement('initialPhase'); child.appendChild(grandchild); + grandchild.appendChild(doc.createTextNode(sprintf('%f',equal.phase(dac_idx)+zeropiphase))); + grandchild = doc.createElement('afterPulseDelay'); child.appendChild(grandchild); + grandchild.appendChild(doc.createTextNode('1.000000')); + grandchild = doc.createElement('taper'); child.appendChild(grandchild); + grandchild.appendChild(doc.createTextNode('Tukey')); + grandchild = doc.createElement('alpha'); child.appendChild(grandchild); + grandchild.appendChild(doc.createTextNode(sprintf('%f',alpha))); + grandchild = doc.createElement('scale'); child.appendChild(grandchild); + grandchild.appendChild(doc.createTextNode('0.000000')); + grandchild = doc.createElement('numPoints'); child.appendChild(grandchild); + grandchild.appendChild(doc.createTextNode(sprintf('%d',Nt(dac_idx)))); + grandchild = doc.createElement('Filename'); child.appendChild(grandchild); + grandchild.appendChild(doc.createTextNode('')); + end + + waveform_names = {}; + for wf = 1:length(arena.wfs) + + for dac_idx = dac_idxs + fs = arena.dac(dac_idx).dacClk; + + fc = (wfs(wf).f0+wfs(wf).f1)/2; + BW = wfs(wf).f1 - wfs(wf).f0; + Tpd = wfs(wf).Tpd; + alpha = wfs(wf).tukey; + equal.delay = wfs(wf).delay; + equal.phase = wfs(wf).phase; + equal.scale = wfs(wf).scale; + Nt = round((wfs(wf).Tpd+equal.delay/1e9) * fs); + + %[0.63 ] + % 0.1652 0.326800 0.511500 0.63 0.6300 0.511500 0.3268 0.1652 + % chebwin(8,30) + + for zeropiphase = unique(wfs(wf).zeropiphase) + new_waveform_name = sprintf('waveformCh%d_%s_%.0fus_%.0fdeg',dac_idx-1,wfs(wf).name,Tpd*1e6,zeropiphase); if any(strcmpi(new_waveform_name,waveform_names)) continue; end waveform_names{end+1} = new_waveform_name; - + config = doc.createElement('config'); configs.appendChild(config); config.setAttribute('type',sprintf('%s_waveform',dac.type)); - + child = doc.createElement('name'); config.appendChild(child); child.appendChild(doc.createTextNode(waveform_names{end})); - + child = doc.createElement('description'); config.appendChild(child); child.appendChild(doc.createTextNode('')); - + child = doc.createElement('sampFreq'); config.appendChild(child); child.appendChild(doc.createTextNode(sprintf('%f',fs/1e6))); - + child = doc.createElement('pulse'); config.appendChild(child); grandchild = doc.createElement('name'); child.appendChild(grandchild); grandchild.appendChild(doc.createTextNode('Pulse')); @@ -984,7 +1179,7 @@ grandchild = doc.createElement('initialDelay'); child.appendChild(grandchild); grandchild.appendChild(doc.createTextNode(sprintf('%f',equal.delay(dac_idx)*1e-3))); grandchild = doc.createElement('initialPhase'); child.appendChild(grandchild); - grandchild.appendChild(doc.createTextNode(sprintf('%f',equal.phase(dac_idx)+zeropimod))); + grandchild.appendChild(doc.createTextNode(sprintf('%f',equal.phase(dac_idx)+zeropiphase))); grandchild = doc.createElement('afterPulseDelay'); child.appendChild(grandchild); grandchild.appendChild(doc.createTextNode('1.000000')); grandchild = doc.createElement('taper'); child.appendChild(grandchild); @@ -1000,11 +1195,15 @@ end end end - + end %% DAQ -daq_idxs = find(strcmpi('daq_0001',{arena.daq.type})); +if isfield(arena,'daq') && ~isempty(arena.daq) + daq_idxs = find(strcmpi('daq_0001',{arena.daq.type})); +else + daq_idxs = []; +end for daq_idx = daq_idxs daq = arena.daq(daq_idx); @@ -1082,6 +1281,79 @@ end +%% PSC: psc_0001 +% Primary Sequence Controller +if strcmpi(arena.psc.type,'psc_0001') + system = doc.getFirstChild; + configs = system.getFirstChild; + + config = doc.createElement('config'); configs.appendChild(config); + config.setAttribute('type',arena.psc.type); + + child = doc.createElement('name'); config.appendChild(child); + child.appendChild(doc.createTextNode( sprintf('psc_%s',arena.psc_name) )); + + child = doc.createElement('description'); config.appendChild(child); + child.appendChild(doc.createTextNode('')); + +% child = doc.createElement('extAsyncTrigSelect'); config.appendChild(child); +% child.appendChild(doc.createTextNode('0')); +% +% child = doc.createElement('holdOnStartup'); config.appendChild(child); +% child.appendChild(doc.createTextNode('0')); +% +% child = doc.createElement('interruptibleOnStartup'); config.appendChild(child); +% child.appendChild(doc.createTextNode('0')); +% +% child = doc.createElement('extJumpToIndex'); config.appendChild(child); +% child.appendChild(doc.createTextNode('0')); +% +% child = doc.createElement('modulus'); config.appendChild(child); +% child.appendChild(doc.createTextNode('0')); + + for wf = 1:length(wfs) + for mode_idx = 1:numel(wfs(wf).modes) + mode = wfs(wf).modes(mode_idx); + repeat_to = wfs(wf).repeat_to(mode_idx); + repeat_count = wfs(wf).repeat_count(mode_idx); + next = wfs(wf).next(mode_idx); + sequence = doc.createElement('sequence'); config.appendChild(sequence); + sequence.setAttribute('type','primary'); +% child = doc.createElement('marker'); sequence.appendChild(child); +% child.appendChild(doc.createTextNode('0')); +% child = doc.createElement('hold'); sequence.appendChild(child); +% child.appendChild(doc.createTextNode('0')); + child = doc.createElement('mode'); sequence.appendChild(child); + child.appendChild(doc.createTextNode(sprintf('%d',mode))); + child = doc.createElement('name'); sequence.appendChild(child); + if wfs(wf).epri(mode_idx) + field = 'EPRI'; + else + field = 'PRI'; + end + zeropiphase = wfs(wf).zeropiphase(mode_idx); + if wfs(wf).tx_invert(mode_idx) + zeropiphase = zeropiphase + 180; + end + psc_name = sprintf('%.0fus, %s, %d',wfs(wf).Tpd*1e6, field, zeropiphase); + child.appendChild(doc.createTextNode(psc_name)); + child = doc.createElement('period'); sequence.appendChild(child); + child.appendChild(doc.createTextNode(sprintf('%g',1/param.prf*1e6))); +% child = doc.createElement('next'); sequence.appendChild(child); +% child.appendChild(doc.createTextNode(sprintf('%d',next))); + child = doc.createElement('repeatTo'); sequence.appendChild(child); + child.appendChild(doc.createTextNode(sprintf('%d',repeat_to))); + child = doc.createElement('repeatCount'); sequence.appendChild(child); + child.appendChild(doc.createTextNode(sprintf('%d',repeat_count))); + child = doc.createElement('interruptible'); sequence.appendChild(child); + child.appendChild(doc.createTextNode('0')); + end + end + + child = doc.createElement('interruptEna'); config.appendChild(child); + child.appendChild(doc.createTextNode('0')); +end + %% PSC: psc_0003 % Primary Sequence Controller if strcmpi(arena.psc.type,'psc_0003') @@ -1135,13 +1407,10 @@ else field = 'PRI'; end - zeropiphase = 0; - if wfs(wf).tx_invert + zeropiphase = wfs(wf).zeropiphase(mode_idx); + if wfs(wf).tx_invert(mode_idx) zeropiphase = zeropiphase + 180; end - if wfs(wf).zeropiphase - zeropiphase = zeropiphase + 90; - end psc_name = sprintf('%.0fus, %s, %d',wfs(wf).Tpd*1e6, field, zeropiphase); child.appendChild(doc.createTextNode(psc_name)); child = doc.createElement('period'); sequence.appendChild(child); @@ -1227,7 +1496,7 @@ child = doc.createElement('config'); subsystem_doc.appendChild(child); child.appendChild(doc.createTextNode('daq0')); child.setAttribute('type',arena.daq.type); - elseif strcmpi(subsystem.name,'ARENA-CTU') + elseif ~isempty(regexpi(subsystem.name,'CTU')) child = doc.createElement('config'); subsystem_doc.appendChild(child); child.appendChild(doc.createTextNode( sprintf('psc_%s',arena.psc_name) )); child.setAttribute('type',arena.psc.type); @@ -1258,6 +1527,19 @@ child.appendChild(doc.createTextNode(mezz_name)); child.setAttribute('type',config_type); + elseif strcmpi(config_type,'ctu_0013') + + mezz = doc.createElement('subSystem'); subsystem_doc.appendChild(mezz); + + child = doc.createElement('name'); mezz.appendChild(child); + child.appendChild(doc.createTextNode(mezz_name)); + +% child = doc.createElement('fanSpeeds'); mezz.appendChild(child); +% child.appendChild(doc.createTextNode('20 20 20')); + child = doc.createElement('config'); mezz.appendChild(child); + child.appendChild(doc.createTextNode(mezz_name)); + child.setAttribute('type',config_type); + elseif strcmpi(config_type,'dac-ad9129_0012') mezz = doc.createElement('subSystem'); subsystem_doc.appendChild(mezz); @@ -1291,6 +1573,36 @@ child.appendChild(doc.createTextNode('')); child.setAttribute('type','ctu_0012'); + elseif strcmpi(config_type,'dac-ad9129_0014') + + mezz = doc.createElement('subSystem'); subsystem_doc.appendChild(mezz); + + child = doc.createElement('name'); mezz.appendChild(child); + child.appendChild(doc.createTextNode(mezz_name)); + + % Find the DAC in the list + dac_idx = find(strcmpi(mezz_name,{arena.dac.name})); + if isempty(dac_idx) + error('Cannot find subsystem dac %s in dac list.', mezz_name); + end + dac = arena.dac(dac_idx); + + child = doc.createElement('disableSync'); mezz.appendChild(child); + child.appendChild(doc.createTextNode('0')); + child = doc.createElement('dacClk'); mezz.appendChild(child); + child.appendChild(doc.createTextNode(sprintf('%g',dac.dacClk/1e6))); + child = doc.createElement('mixMode'); mezz.appendChild(child); + child.appendChild(doc.createTextNode('0')); + child = doc.createElement('desiredAlignMin'); mezz.appendChild(child); + child.appendChild(doc.createTextNode(sprintf('%g',dac.desiredAlignMin))); + child = doc.createElement('desiredAlignMax'); mezz.appendChild(child); + child.appendChild(doc.createTextNode(sprintf('%g',dac.desiredAlignMax))); +% child = doc.createElement('dcoPhase'); mezz.appendChild(child); +% child.appendChild(doc.createTextNode(sprintf('%g',dac.dcoPhase))); + child = doc.createElement('config'); mezz.appendChild(child); + child.appendChild(doc.createTextNode(mezz_name)); + child.setAttribute('type',config_type); + elseif strcmpi(config_type,'adc-ad9680_0017') mezz = doc.createElement('subSystem'); subsystem_doc.appendChild(mezz); diff --git a/cresis-toolbox/processing/write_cresis_xml.m b/cresis-toolbox/processing/write_cresis_xml.m index b7e5a74d..8e4bad4f 100644 --- a/cresis-toolbox/processing/write_cresis_xml.m +++ b/cresis-toolbox/processing/write_cresis_xml.m @@ -586,7 +586,10 @@ if ~exist(out_xml_fn_dir,'dir') mkdir(out_xml_fn_dir); end -fid = fopen(out_xml_fn,'w'); +[fid,msg] = fopen(out_xml_fn,'w'); +if fid == -1 + error(msg); +end fprintf(fid,'\n'); fprintf(fid,'\n'); write_ni_xml_object(settings_enc,fid,true,struct('array_list','Waveforms','enum_list','DDCZ20sel')); @@ -604,10 +607,10 @@ end % Create arena parameter structure - arena = struct('version','1'); - arena.awg = param.arena.awg; - arena.dacs = param.arena.dacs; - arena.dacs_sampFreq = param.arena.dacs_sampFreq; + arena = param.arena; +% arena.awg = param.arena.awg; + arena.dac = param.arena.dac; +% arena.dacs_sampFreq = param.arena.dacs_sampFreq; arena.dacs_internal_delay = param.arena.dacs_internal_delay; arena.dacs_start_delay = param.arena.dacs_start_delay; arena.zeropimods = param.arena.zeropimods; @@ -628,36 +631,46 @@ arena.wfs(wf).tukey = settings_enc.sys.DDSZ5FSetup.RAMZ20Taper; arena.wfs(wf).enabled = fliplr(~logical(dec2bin(settings_enc.sys.DDSZ5FSetup.Waveforms(wf).TXZ20Mask(1),8)-'0')); arena.wfs(wf).scale = double(settings_enc.sys.DDSZ5FSetup.RamZ20Amplitude) .* param.arena.max_tx ./ max_DDS_amp; - arena.wfs(wf).fc = (settings_enc.sys.DDSZ5FSetup.Waveforms(wf).StartZ20Freq ... - + settings_enc.sys.DDSZ5FSetup.Waveforms(wf).StopZ20Freq)/2; - arena.wfs(wf).BW = abs(settings_enc.sys.DDSZ5FSetup.Waveforms(wf).StopZ20Freq ... - - settings_enc.sys.DDSZ5FSetup.Waveforms(wf).StartZ20Freq); + arena.wfs(wf).f0 = settings_enc.sys.DDSZ5FSetup.Waveforms(wf).StartZ20Freq; + arena.wfs(wf).f1 = settings_enc.sys.DDSZ5FSetup.Waveforms(wf).StopZ20Freq; +% arena.wfs(wf).fc = (settings_enc.sys.DDSZ5FSetup.Waveforms(wf).StartZ20Freq ... +% + settings_enc.sys.DDSZ5FSetup.Waveforms(wf).StopZ20Freq)/2; +% arena.wfs(wf).BW = abs(settings_enc.sys.DDSZ5FSetup.Waveforms(wf).StopZ20Freq ... +% - settings_enc.sys.DDSZ5FSetup.Waveforms(wf).StartZ20Freq); arena.wfs(wf).delay = settings_enc.sys.DDSZ5FSetup.Waveforms(wf).Delay - min_delay; arena.wfs(wf).phase = settings_enc.sys.DDSZ5FSetup.Waveforms(wf).PhaseZ20Offset; arena.wfs(wf).Tpd = double(settings_enc.sys.DDSZ5FSetup.Waveforms(wf).LenZ20Mult) ... * settings_enc.sys.DDSZ5FSetup.BaseZ20Len; arena.wfs(wf).presums = settings_enc.sys.DDSZ5FSetup.Waveforms(wf).Presums; end - - % Create XML document - doc = write_arena_xml([],'init',arena); - doc = write_arena_xml(doc,'ctu_0013',arena); - doc = write_arena_xml(doc,'dac-ad9129_0014',arena); - doc = write_arena_xml(doc,'dac-ad9129_0014_waveform',arena); - doc = write_arena_xml(doc,'psc_0001',arena); - doc = write_arena_xml(doc,'subsystems',arena); - - out_str = xmlwrite(doc); - out_str = ['' out_str(find(out_str==10,1):end)]; - [~,rss_fn_name] = fileparts(param.fn); - rss_fn = fullfile(param.rss_base_dir,[rss_fn_name '.xml']); - if ~exist(param.rss_base_dir,'dir') - mkdir(param.rss_base_dir); - end - fprintf(' Writing RSS: %s\n', rss_fn); - fid = fopen(rss_fn,'w'); - fwrite(fid,out_str,'char'); - fclose(fid); + + % Create XML document + xml_param = param; + xml_param.wfs = arena.wfs; + xml_param.prf = 1/arena.PRI; + xml_param.arena = arena; + xml_param.arena.adc = []; + xml_param.board_map = {}; + + [~,xml_param.arena.psc_name] = ct_fileparts(out_xml_fn); + xml_param.arena.fn = fullfile(param.arena_base_dir,[xml_param.arena.psc_name '.xml']); + + [doc,xml_param] = write_arena_xml([],xml_param); + + % Create XML document + out_str = xmlwrite(doc); + out_str = ['' out_str(find(out_str==10,1):end)]; + arena_fn_dir = fileparts(xml_param.arena.fn); + if ~exist(arena_fn_dir,'dir') + mkdir(arena_fn_dir); + end + fprintf(' Writing Arena XML: %s\n', xml_param.arena.fn); + [fid,msg] = fopen(xml_param.arena.fn,'w'); + if fid == -1 + error(msg); + end + fwrite(fid,out_str,'char'); + fclose(fid); end diff --git a/cresis-toolbox/rss/arena_convert_range.m b/cresis-toolbox/rss/arena_convert_range.m index ffc77c9a..cdc7cf29 100644 --- a/cresis-toolbox/rss/arena_convert_range.m +++ b/cresis-toolbox/rss/arena_convert_range.m @@ -1,4 +1,44 @@ function nums = arena_convert_range(str) -% Convert 0:2,4,5,7:10 to [0 1 2 3 4 7 8 9 10] +% Converts arena string from a list of ranges or a binary mask into a list +% of numbers. +% +% Examples +% ========================================================================= +% +% List of ranges: +% +% arena_convert_range('0:2 4 5 7:10') +% ans = +% 0 1 2 4 5 7 8 9 10 +% arena_convert_range('0:2,4,5,7:10') +% ans = +% 0 1 2 4 5 7 8 9 10 +% +% Binary Mask: +% +% arena_convert_range('00000X0X') +% ans = +% 0 1 4 5 +% arena_convert_range('0000010X') +% ans = +% 4 5 +% arena_convert_range('000xx10x') +% ans = +% 4 5 12 13 20 21 28 29 -nums = eval(['[' str(:).' ']']); +str = upper(str(:).'); + +if any(str=='X') || all(str=='0' | str=='1' & length(str) > 1) + % Binary mask + num_X = length(str(str == 'X')); + nums = zeros(1,2^num_X); + for idx = 0:2^num_X-1 + tmp_str = str; + tmp_str(str=='X') = dec2bin(idx,num_X); + nums(1+idx) = bin2dec(tmp_str); + end + +else + % Mode ranges + nums = eval(['[' str ']']); +end diff --git a/cresis-toolbox/rss/arena_packet_strip_tcp_mex.cpp b/cresis-toolbox/rss/arena_packet_strip_tcp_mex.cpp index 15352f0f..08bef623 100644 --- a/cresis-toolbox/rss/arena_packet_strip_tcp_mex.cpp +++ b/cresis-toolbox/rss/arena_packet_strip_tcp_mex.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include "mat.h" #include "mex.h" @@ -231,6 +232,7 @@ mexFunction( int nlhs, } hdr = hdr_tmp; } + unsigned int hdr_type = (*((unsigned int *)(hdr+8))) & 0x7FFFFFFF; memcpy(hdr+num_hdr*header_size,last_bytes,*last_bytes_len); memcpy(hdr+num_hdr*header_size+*last_bytes_len,data,header_size-*last_bytes_len); //mexPrintf("%d: %lld %d 0x%016llx\n", __LINE__, idx, num_hdr+1, ((unsigned long long int *)(hdr+num_hdr*header_size))[0]); @@ -276,6 +278,7 @@ mexFunction( int nlhs, num_expected_original = *num_expected; //mexPrintf("%d: %d\n", __LINE__, *num_expected); int num_expected_bins = *num_expected; + mexPrintf("%d: %x\n", __LINE__, profile_data_format); switch (profile_data_format) { case 0x00000000: @@ -295,16 +298,20 @@ mexFunction( int nlhs, break; case 0x00020000: case 0x00030000: - // num_expected is in units of bytes - // num_expected_bins needs to be adjusted for 8 byte bins: - // >>3 = /8, 2 IQ channels, 4 bynum_expectedte samples or 2*4 = 8 - num_expected_bins = num_expected_bins >> 3; - //*num_expected = *num_expected << 3; // This line replaced the above line for temporary hack + // num_expected is in units of samples, needs to be in units of bytes + // num_expected_bins is in units of samples, no adjustment needed + // <<3 = *8, 2 IQ channels, 4 byte samples or 2*4 = 8 + if (hdr_type==45) { + // GHOST radar header + *num_expected = *num_expected << 3; + } else { + num_expected_bins = num_expected_bins >> 3; + } break; } if (num_expected_bins < *min_num_expected || num_expected_bins > *max_num_expected) { - mexPrintf("%d: %d Bad Record Length %d\n", __LINE__, ((unsigned int *)(hdr+num_hdr*header_size))[0], *num_expected); + mexPrintf("%d: %d Bad Record Length %d outside of specified min-max range of %d to %d.\n", __LINE__, ((unsigned int *)(hdr+num_hdr*header_size))[0], num_expected_bins, *min_num_expected, *max_num_expected); *num_expected = *default_num_expected; } *num_expected = *num_expected + *length_field_offset + 4- *last_bytes_len; @@ -372,6 +379,8 @@ mexFunction( int nlhs, } hdr = hdr_tmp; } + unsigned int hdr_type = (*((unsigned int *)(data+idx+8))) & 0x7FFFFFFF; + memcpy(hdr+num_hdr*header_size,(void*)(data+idx),header_size); // Uncomment next line for debugging //mexPrintf("%d: %lld %d 0x%016llx\n", __LINE__, idx, num_hdr+1, ((unsigned long long int *)(hdr+num_hdr*header_size))[0]); @@ -394,10 +403,8 @@ mexFunction( int nlhs, profile_data_format = (*(int *)(data + idx - 4)); *num_expected = (*(int *)(data + idx)); num_expected_original = *num_expected; - idx += 4 + *num_expected; // Skip to the end of the record - // Uncomment next line for debugging - //mexPrintf("%d: expect %d bytes, new record expected at %d of %d bytes\n", __LINE__, *num_expected, idx, file_size); int num_expected_bins = *num_expected; + //mexPrintf("%d: %x\n", __LINE__, profile_data_format); switch (profile_data_format) { case 0x00000000: @@ -417,16 +424,23 @@ mexFunction( int nlhs, break; case 0x00020000: case 0x00030000: - // num_expected is in units of bytes - // num_expected_bins needs to be adjusted for 8 byte bins: - // >>3 = /8, 2 IQ channels, 4 byte samples or 2*4 = 8 - num_expected_bins = num_expected_bins >> 3; - //*num_expected = *num_expected << 3; // This line replaced the above line for temporary hack + // num_expected is in units of samples, needs to be in units of bytes + // num_expected_bins is in units of samples, no adjustment needed + // <<3 = *8, 2 IQ channels, 4 byte samples or 2*4 = 8 + if (hdr_type==45) { + // GHOST radar header + *num_expected = *num_expected << 3; + } else { + num_expected_bins = num_expected_bins >> 3; + } break; } + idx += 4 + *num_expected; // Skip to the end of the record + // Uncomment next line for debugging + //mexPrintf("%d: expect %d bytes, new record expected at %d of %d bytes\n", __LINE__, *num_expected, idx, file_size); if (num_expected_bins < *min_num_expected || num_expected_bins > *max_num_expected) { - mexPrintf("%d: %d Bad Record Length %d outside of specified min-max range.\n", __LINE__, ((unsigned int *)(hdr+num_hdr*header_size))[0], num_expected_bins); + mexPrintf("%d: %d Bad Record Length %d outside of specified min-max range of %d to %d.\n", __LINE__, ((unsigned int *)(hdr+num_hdr*header_size))[0], num_expected_bins, *min_num_expected, *max_num_expected); *num_expected = *default_num_expected; } } @@ -440,7 +454,7 @@ mexFunction( int nlhs, __LINE__, last_record_offset, idx, num_expected_original); // Uncomment next seven lines for debugging mexPrintf("Bytes around the expected location of the sync:\n"); - for (int debug_idx=0; debug_idx<16; debug_idx++) + for (ptrdiff_t debug_idx=0; debug_idxuint64'); if tmp ~= arena_frame_sync - fprintf('Lost lock state at byte %d\n', ftell(fid)-8); + fprintf('Lost lock state at byte %d record %d/%d wf/adc %d/%d\n', ftell(fid)-8, rec_in, rec, wf, adc); + % Remove the last record + data{adc,wf} = data{adc,wf}(:,1:end-1); + hdr_fieldnames = fieldnames(hdr{adc,wf}); + for field_idx = 1:length(hdr_fieldnames) + hdr{adc,wf}.(hdr_fieldnames{field_idx}) = hdr{adc,wf}.(hdr_fieldnames{field_idx})(1:end-1); + end lock_state = 0; continue; elseif feof(fid) break; + else + frame_sync_offset = ftell(fid)-8; end end @@ -127,12 +170,40 @@ new_hdr = basic_load_arena_ku0001(fid); subchannel = new_hdr.subchannel; mode = new_hdr.mode; + elseif hdr_type == ghost_ku0001_radar_header_type + new_hdr = basic_load_arena_ghost_ku0001(fid); + if isempty(param.processor_subchannel) + new_hdr.subchannel = 0; + new_hdr.mode = new_hdr.processor; + else + new_hdr.subchannel = param.processor_subchannel(new_hdr.processor+1); + new_hdr.mode = param.processor_mode(new_hdr.processor+1); + end + subchannel = new_hdr.subchannel; + mode = new_hdr.mode; + profile = new_hdr.processor; + elseif hdr_type == coldex_arena5xx_radar_header_type + new_hdr = basic_load_arena_ghost_ku0001(fid); + if isempty(param.processor_subchannel) + new_hdr.subchannel = 0; + new_hdr.mode = new_hdr.processor; + else + new_hdr.subchannel = param.processor_subchannel(new_hdr.processor+1); + new_hdr.mode = param.processor_mode(new_hdr.processor+1); + end + subchannel = new_hdr.subchannel; + mode = new_hdr.mode; + profile = new_hdr.processor; + %fprintf('%3d %10d %4d %4d\n', new_hdr.processor, new_hdr.profile_cntr_latch, subchannel, mode); else subchannel = 0; mode = fread(fid,1,'uint8'); fseek(fid,hdr_len-1,0); new_hdr = struct(); end + hdr_debug.mode(end+1) = mode; + hdr_debug.subchannel(end+1) = subchannel; + hdr_debug.profile(end+1) = profile; %% Read in data profile_type = fread(fid,1,'uint32'); @@ -148,7 +219,11 @@ tmp = fread(fid,profile_len/4,'int32'); isIQ = 1; case 196608 % 0x30000 - tmp = fread(fid,profile_len/4,'float32'); + if hdr_type == ghost_ku0001_radar_header_type + tmp = fread(fid,profile_len*2,'float32'); + else + tmp = fread(fid,profile_len/4,'float32'); + end isIQ = 1; otherwise error('Unsupported profile type %d.', profile_type); @@ -164,7 +239,7 @@ rec = size(data{adc,wf},2)+1; end if rec >= 0 - hdr{adc,wf}.frame_sync(rec) = ftell(fid)-8; + hdr{adc,wf}.frame_sync(rec) = frame_sync_offset; hdr{adc,wf}.hdr_type(rec) = hdr_type; hdr{adc,wf}.hdr_len(rec) = hdr_len; @@ -178,11 +253,12 @@ data{adc,wf}(:,rec) = tmp; end else - warning('Invalid record size %d at %d record %d. Expected %d.', ... + warning('Invalid record size (size %d) at file byte %d on record number %d. Expected size %d.', ... length(tmp), ftell(fid), rec, 2*size(data{adc,wf},1)); end end catch ME + ME.getReport rec = rec - 1; hdr{adc,wf}.frame_sync = hdr{adc,wf}.frame_sync(1:rec); break; @@ -200,6 +276,7 @@ hdr.mode = fread(fid,1,'uint8'); hdr.subchannel = fread(fid,1,'uint8'); +hdr.processor = NaN; fseek(fid,2,0); hdr.wg_delay_latch = fread(fid,1,'uint16'); fseek(fid,10,0); @@ -215,6 +292,8 @@ function hdr = basic_load_arena_doppler(fid) hdr.mode = fread(fid,1,'uint8'); +hdr.subchannel = NaN; +hdr.processor = NaN; hdr.decimation_ratio = fread(fid,1,'uint8'); hdr.num_pulses_in_burst = fread(fid,1,'uint8'); fseek(fid,5,0); @@ -240,6 +319,7 @@ tmp = fread(fid,1,'uint8'); hdr.subchannel = mod(tmp,16); hdr.data_channel = floor(tmp/16); +hdr.processor = NaN; fseek(fid,6,0); hdr.encoder = fread(fid,1,'uint32'); fseek(fid,4,0); @@ -258,6 +338,7 @@ tmp = fread(fid,1,'uint8'); hdr.subchannel = mod(tmp,16); hdr.data_channel = floor(tmp/16); +hdr.processor = NaN; fseek(fid,6,0); hdr.encoder = fread(fid,1,'uint32'); fseek(fid,4,0); @@ -267,3 +348,20 @@ hdr.pps_cntr_latch = fread(fid,1,'uint64'); end + +% ========================================================================= +%% Function for loading GHOST/ku0001 radar header +function hdr = basic_load_arena_ghost_ku0001(fid) + +hdr.mode = fread(fid,1,'uint8'); +tmp = fread(fid,1,'uint8'); +hdr.subchannel = mod(tmp,16); +hdr.data_channel = floor(tmp/16); +hdr.processor = fread(fid,1,'uint8'); +fseek(fid,5+8,0); % RESERVED for future use +hdr.rel_time_cntr_latch = fread(fid,1,'uint64'); +hdr.profile_cntr_latch = fread(fid,1,'uint64'); +hdr.pps_ftime_cntr_latch = fread(fid,1,'uint64'); +hdr.pps_cntr_latch = fread(fid,1,'uint64'); + +end diff --git a/cresis-toolbox/rss/read_arena_xml.m b/cresis-toolbox/rss/read_arena_xml.m index 59a77a62..adb7c1d8 100644 --- a/cresis-toolbox/rss/read_arena_xml.m +++ b/cresis-toolbox/rss/read_arena_xml.m @@ -94,41 +94,116 @@ %% PULSE SEQUENCE % ========================================================================= -% 1. Get the name of the CTU -% 1a. Get the CTU -expression = xpath.compile('//subSystem[starts-with(@type,"arenactu")]'); -ctu = expression.evaluate(doc,XPathConstants.NODESET); - -% 1b. Get the name -expression = xpath.compile('name'); -nodeList = expression.evaluate(ctu.item(0),XPathConstants.NODESET); -name = nodeList.item(0).getTextContent.toCharArray; -name = name(:).'; - -% 2. Search for the subSystem in the config XML -expression = xpath.compile(sprintf('//subSystem[name="%s"]',name)); -nodeList = expression.evaluate(doc_cfg,XPathConstants.NODESET); -match = nodeList.item(0); +if ~isempty(regexpi(configs.radar_name,'arena5xx', 'once')) + %% PULSE SEQUENCE: Arena 522 CTU + + % SYSTEM XML: + % + % ARENA5xxRF + % + % CONFIG XML: + % + % ARENA5xxRF + % psc1 + % + % + % + % psc1 -% 3. Get the config name and type for the CTU -expression = xpath.compile('config/@type'); -nodeList = expression.evaluate(match,XPathConstants.NODESET); -config_type = nodeList.item(0).getTextContent.toCharArray; -config_type = config_type(:).'; -expression = xpath.compile('config'); -nodeList = expression.evaluate(match,XPathConstants.NODESET); -config_name = nodeList.item(0).getTextContent.toCharArray; -config_name = config_name(:).'; + % Get the arena 522 subsystems + expression = xpath.compile('//subSystem[starts-with(@type,"arena5xxrf")]'); + arena522_list = expression.evaluate(doc,XPathConstants.NODESET); -configs.psc.name = name; -configs.psc.config_type = config_type; -configs.psc.config_name = config_name; + found_psc = false; + for arena522_idx = 1:arena522_list.getLength + arena522_syscfg = arena522_list.item(arena522_idx-1); + if isempty(arena522_syscfg) + continue; + end + + % Get the arena 522 subsystem name + expression = xpath.compile('name'); + nodeList = expression.evaluate(arena522_syscfg,XPathConstants.NODESET); + arena522_name = nodeList.item(0); + arena522_name = arena522_name.getTextContent.toCharArray; + arena522_name = arena522_name(:).'; + + % Find the configuration for this arena 522 + expression = xpath.compile(sprintf('//subSystem[name="%s"]', arena522_name)); + arena522_cfg = expression.evaluate(doc_cfg,XPathConstants.NODESET); + arena522_cfg = arena522_cfg.item(0); -expression = xpath.compile(sprintf('//configs/config[(@type="%s" and name="%s")]',config_type,config_name)); -nodeList = expression.evaluate(doc_cfg,XPathConstants.NODESET); -psc_cfg = nodeList.item(0); -if isempty(psc_cfg) - error('Could not find pulse sequence controller (psc) type "%s" and name "%s".', config_type, config_name); + % Get the Arena 522 CTU config name + config_type = 'psc_0008'; + expression = xpath.compile(sprintf('config[@type="%s"]', config_type)); + config_name = expression.evaluate(arena522_cfg,XPathConstants.NODESET); + config_name = config_name.item(0).getTextContent.toCharArray; + config_name = config_name(:).'; + + % Get the Arena 522 CTU configs + expression = xpath.compile(sprintf('//configs/config[@type="%s" and name="%s"]',config_type,config_name)); + psc_cfg = expression.evaluate(doc_cfg,XPathConstants.NODESET); + if ~isempty(psc_cfg) + psc_cfg = psc_cfg.item(0); + found_psc = true; + break; + end + end + if ~found_psc || psc_cfg.getLength == 0 + error('Did not find PSC configuration.'); + end + + configs.psc.config_type = config_type; + configs.psc.config_name = config_name; + +else + %% PULSE SEQUENCE: CTU Module or Arena 313 CTU + + % 1. Get the name of the CTU + % 1a. Get the CTU + if strcmpi(configs.radar_name,'ku0002') + expression = xpath.compile('//subSystem[starts-with(@type,"arena3xx")]'); + ctu = expression.evaluate(doc,XPathConstants.NODESET); + else + expression = xpath.compile('//subSystem[starts-with(@type,"arenactu")]'); + ctu = expression.evaluate(doc,XPathConstants.NODESET); + end + if ctu.getLength == 0 + error('Did not find CTU configuration.'); + end + + % 1b. Get the name + expression = xpath.compile('name'); + nodeList = expression.evaluate(ctu.item(0),XPathConstants.NODESET); + name = nodeList.item(0).getTextContent.toCharArray; + name = name(:).'; + + % 2. Search for the subSystem in the config XML + expression = xpath.compile(sprintf('//subSystem[name="%s"]',name)); + nodeList = expression.evaluate(doc_cfg,XPathConstants.NODESET); + match = nodeList.item(0); + + % 3. Get the config name and type for the CTU + expression = xpath.compile('config/@type'); + nodeList = expression.evaluate(match,XPathConstants.NODESET); + config_type = nodeList.item(0).getTextContent.toCharArray; + config_type = config_type(:).'; + expression = xpath.compile('config'); + nodeList = expression.evaluate(match,XPathConstants.NODESET); + config_name = nodeList.item(0).getTextContent.toCharArray; + config_name = config_name(:).'; + + configs.psc.name = name; + configs.psc.config_type = config_type; + configs.psc.config_name = config_name; + + expression = xpath.compile(sprintf('//configs/config[(@type="%s" and name="%s")]',config_type,config_name)); + nodeList = expression.evaluate(doc_cfg,XPathConstants.NODESET); + psc_cfg = nodeList.item(0); + if isempty(psc_cfg) + error('Could not find pulse sequence controller (psc) type "%s" and name "%s".', config_type, config_name); + end + end % 5. Read the pulse sequence @@ -270,10 +345,10 @@ configs.total_presums = total_presums; configs.psc.seq.mode_count = mode_count; -%% ADCS +%% ADCS/DIGRX % ========================================================================= % Get all the ADCs -expression = xpath.compile('//subSystem[starts-with(@type,"adc")]'); +expression = xpath.compile('//subSystem[contains(@type,"adc")]'); % expression = xpath.compile('//subSystem[@type="daq"]'); adcList = expression.evaluate(doc,XPathConstants.NODESET); @@ -315,6 +390,41 @@ nodeList = expression.evaluate(doc_cfg,XPathConstants.NODESET); adc_cfg = nodeList.item(0); + +% +% rds_digrx1_udp +% +% 0 +% 172.16.0.121 +% 55001 +% 8192 +% +% +% rds_digrx2_udp +% +% 0 +% 172.16.0.121 +% 55002 +% 8192 +% +% +% snow_udp +% +% 0 +% 172.16.0.121 +% 55000 +% 8192 +% +% +% 0 +% dataStream0 +% +% eth0 +% snow_udp +% +% + + % Get the datastream type expression = xpath.compile('dataStream/@type'); nodeList = expression.evaluate(match,XPathConstants.NODESET); @@ -341,7 +451,7 @@ expression = xpath.compile('dataOutput'); nodeList = expression.evaluate(match,XPathConstants.NODESET); if nodeList.getLength > 0 - configs.datastream_type = 'socket'; + configs.datastream_type = 'udp'; if 0 expression = xpath.compile('dataOutput/config'); nodeList = expression.evaluate(match,XPathConstants.NODESET); @@ -414,7 +524,7 @@ end ncoFreq = nodeList.item(0); ncoFreq = ncoFreq.getTextContent.toCharArray; - ncoFreq = str2double(ncoFreq(:).'); + ncoFreq = str2double(ncoFreq(:).') * 1e6; configs.adc{adc_idx,mode_latch+1,subchannel+1}.ncoFreq = ncoFreq; expression = xpath.compile('cicDecimation'); @@ -523,11 +633,798 @@ if num_bins < configs.min_num_bins configs.min_num_bins = num_bins; end - if num_bins > configs.max_num_bins - configs.max_num_bins = num_bins; + if num_bins > configs.max_num_bins + configs.max_num_bins = num_bins; + end + end + + elseif strcmpi(config_type,'rfsoc_adc_2002') + % ===================================================================== + % Arena 522 COLDEX + % ===================================================================== + + % Get the sampling frequencies in MHz + expression = xpath.compile('adcSampClk'); + nodeList = expression.evaluate(adc_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + sampFreq = nodeList.item(0); + sampFreq = sampFreq.getTextContent.toCharArray; + sampFreq = str2double(sampFreq(:).') * 1e6; + + expression = xpath.compile('ddcDecimation'); + nodeList = expression.evaluate(adc_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + ddcDecimation = nodeList.item(0); + ddcDecimation = ddcDecimation.getTextContent.toCharArray; + ddcDecimation = str2double(ddcDecimation(:).'); + + sampFreq = sampFreq / ddcDecimation; + + % Load each down converter settings + % ===================================================================== + expression = xpath.compile('ncoFreqs/ncoFreq'); + ncoFreq_nodeList = expression.evaluate(adc_cfg,XPathConstants.NODESET); + ncoFreq_list = []; + for ncoFreq_idx = 1:ncoFreq_nodeList.getLength + ncoFreq_cfg = ncoFreq_nodeList.item(ncoFreq_idx-1); + if isempty(ncoFreq_cfg) + continue; + end + + % + % + % 0:7 + % 0:255 + % 700 + % + % + + % Load the subchannel ID + expression = xpath.compile('subChannels'); + nodeList = expression.evaluate(ncoFreq_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + subchannel = nodeList.item(0); + subchannel = subchannel.getTextContent.toCharArray; + subchannel = arena_convert_range(subchannel); + ncoFreq_list(end+1).subchannel = subchannel; + + % Load the modes + expression = xpath.compile('modes'); + nodeList = expression.evaluate(ncoFreq_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + modes = nodeList.item(0); + modes = modes.getTextContent.toCharArray; + modes = arena_convert_range(modes); + ncoFreq_list(end).modes = modes; + + expression = xpath.compile('freq'); + nodeList = expression.evaluate(ncoFreq_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + ncoFreq = nodeList.item(0); + ncoFreq = ncoFreq.getTextContent.toCharArray; + ncoFreq = str2double(ncoFreq(:).') * 1e6; + ncoFreq_list(end).ncoFreq = ncoFreq; + end + + % Load each down converter settings + % ===================================================================== + expression = xpath.compile('dcSetting'); + subchannel_nodeList = expression.evaluate(adc_cfg,XPathConstants.NODESET); + subchannels = []; + for subchannel_idx = 1:subchannel_nodeList.getLength + subchannel_cfg = subchannel_nodeList.item(subchannel_idx-1); + if isempty(subchannel_cfg) + continue; + end + + % + % 0 + % 0 + % 0 + % 0 + % 3 + % RFSOC + % 0 + % 0 + % 0 + + % Load the subchannel ID + expression = xpath.compile('subChannels'); + nodeList = expression.evaluate(subchannel_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + subchannel = nodeList.item(0); + subchannel = subchannel.getTextContent.toCharArray; + subchannel = arena_convert_range(subchannel); + subchannels(end+1).subchannel = subchannel; + + % Load the modes + expression = xpath.compile('modes'); + nodeList = expression.evaluate(subchannel_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + modes = nodeList.item(0); + modes = modes.getTextContent.toCharArray; + modes = arena_convert_range(modes); + subchannels(end).modes = modes; + + expression = xpath.compile('adcChannel'); + nodeList = expression.evaluate(subchannel_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + adcChannel = nodeList.item(0); + adcChannel = adcChannel.getTextContent.toCharArray; + adcChannel = arena_convert_range(adcChannel); + subchannels(end).adcChannel = adcChannel; + + expression = xpath.compile('zeroPi'); + nodeList = expression.evaluate(subchannel_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + zeroPi = nodeList.item(0); + zeroPi = zeroPi.getTextContent.toCharArray; + zeroPi = str2double(zeroPi); + subchannels(end).zeroPi = zeroPi; + + expression = xpath.compile('adcDecimation'); + nodeList = expression.evaluate(subchannel_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + adcDecimation = nodeList.item(0); + adcDecimation = adcDecimation.getTextContent.toCharArray; + adcDecimation = str2double(adcDecimation); + subchannels(end).adcDecimation = adcDecimation+1; + + expression = xpath.compile('route'); + nodeList = expression.evaluate(subchannel_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + route = nodeList.item(0); + subchannels(end).route = route.getTextContent.toCharArray; + subchannels(end).route = subchannels(end).route(:).'; + + expression = xpath.compile('ncoFreq'); + nodeList = expression.evaluate(subchannel_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + ncoFreq = nodeList.item(0); + ncoFreq = ncoFreq.getTextContent.toCharArray; + ncoFreq = str2double(ncoFreq)*1e6; + subchannels(end).ncoFreq = ncoFreq; + + expression = xpath.compile('ncoPhase'); + nodeList = expression.evaluate(subchannel_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + ncoPhase = nodeList.item(0); + ncoPhase = ncoPhase.getTextContent.toCharArray; + ncoPhase = str2double(ncoPhase); + subchannels(end).ncoPhase = ncoPhase; + end + + % Load each intra-profile processing settings + % ===================================================================== + expression = xpath.compile('ippSetting'); + ipp_settings_nodeList = expression.evaluate(adc_cfg,XPathConstants.NODESET); + ipp_settings = []; + for ipp_settings_idx = 1:ipp_settings_nodeList.getLength + ipp_settings_cfg = ipp_settings_nodeList.item(ipp_settings_idx-1); + if isempty(ipp_settings_cfg) + continue; + end + + % + % 0:7 + % 0:3 + % 0:2047 + % 128 + % 0 + % 0 + % + % + % + % + + % Load the subchannel ID + expression = xpath.compile('subChannels'); + nodeList = expression.evaluate(ipp_settings_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + subchannel = nodeList.item(0); + subchannel = subchannel.getTextContent.toCharArray; + subchannel = arena_convert_range(subchannel); + ipp_settings(end+1).subchannel = subchannel; + + % Load the modes + expression = xpath.compile('modes'); + nodeList = expression.evaluate(ipp_settings_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + modes = nodeList.item(0); + modes = modes.getTextContent.toCharArray; + modes = arena_convert_range(modes); + ipp_settings(end).modes = modes; + + expression = xpath.compile('digRx_RG'); + nodeList = expression.evaluate(ipp_settings_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + digRx_RG = nodeList.item(0); + digRx_RG = digRx_RG.getTextContent.toCharArray; + digRx_RG = arena_convert_range(digRx_RG); + ipp_settings(end).digRx_RG = digRx_RG; + + num_bins = length(digRx_RG); + if num_bins < configs.min_num_bins + configs.min_num_bins = num_bins; + end + if num_bins > configs.max_num_bins + configs.max_num_bins = num_bins; + end + + end + + % Load each profile input + % ===================================================================== + expression = xpath.compile('profileProcessing/input'); + subchannel_nodeList = expression.evaluate(adc_cfg,XPathConstants.NODESET); + profileInput = []; + for subchannel_idx = 1:subchannel_nodeList.getLength + subchannel_cfg = subchannel_nodeList.item(subchannel_idx-1); + if isempty(subchannel_cfg) + continue; + end + + % + % -1 + % + % ch0_10us + % DigRx + % 2:3 + % 0 + % + + % Load the profile name + expression = xpath.compile('name'); + nodeList = expression.evaluate(subchannel_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + profile_input_name = nodeList.item(0); + profileInput(end+1).name = profile_input_name.getTextContent.toCharArray; + profileInput(end).name = profileInput(end).name(:).'; + + % Load the profile modes + expression = xpath.compile('profileMode'); + nodeList = expression.evaluate(subchannel_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + profileMode = nodeList.item(0); + profileMode = profileMode.getTextContent.toCharArray; + profileMode = arena_convert_range(profileMode); + profileInput(end).profileMode = profileMode; + + % Load the profile subchannels + expression = xpath.compile('profileSubChannel'); + nodeList = expression.evaluate(subchannel_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + profileSubChannel = nodeList.item(0); + profileSubChannel = profileSubChannel.getTextContent.toCharArray; + profileSubChannel = arena_convert_range(profileSubChannel(:).'); + profileInput(end).profileSubChannel = profileSubChannel; + end + + + % Load each profile processor + % ===================================================================== + expression = xpath.compile('profileProcessing/processor'); + subchannel_nodeList = expression.evaluate(adc_cfg,XPathConstants.NODESET); + profileProcessor = []; + for subchannel_idx = 1:subchannel_nodeList.getLength + subchannel_cfg = subchannel_nodeList.item(subchannel_idx-1); + if isempty(subchannel_cfg) + continue; + end + + % + % + % ch0 + % -1 + % ch0 + % Coherent Accum. + % ch0 + % 0 + % 0 + % 2048 + % 29 + % + + % Load the subchannel ID + expression = xpath.compile('name'); + nodeList = expression.evaluate(subchannel_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + profile_processor_name = nodeList.item(0); + profileProcessor(end+1).name = profile_processor_name.getTextContent.toCharArray; + profileProcessor(end).name = profileProcessor(end).name(:).'; + + % Load the subchannel ID + expression = xpath.compile('length'); + nodeList = expression.evaluate(subchannel_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + profile_length = nodeList.item(0); + profile_length = profile_length.getTextContent.toCharArray; + profile_length = str2double(profile_length(:).'); + profileProcessor(end).profile_length = profile_length; + + expression = xpath.compile('accumulations'); + nodeList = expression.evaluate(subchannel_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + accumulations = nodeList.item(0); + accumulations = accumulations.getTextContent.toCharArray; + accumulations = str2double(accumulations(:).'); + profileProcessor(end).accumulations = accumulations; + + end + + % Update adc configs for this integrator's modes and subchannel + for profile_idx = 1:length(profileInput) + for mode_latch = profileInput(profile_idx).profileMode + for subchannel = profileInput(profile_idx).profileSubChannel + + % Search for the particular mode/subchannel in the subchannels + found_mode = false; + for idx = 1:length(subchannels) + if any(subchannels(idx).modes == mode_latch) && any(subchannels(idx).subchannel == subchannel) + adcDecimation = subchannels(idx).adcDecimation; + ncoFreq = subchannels(idx).ncoFreq; + ncoPhase = subchannels(idx).ncoPhase; + found_mode = true; + break; + end + end + if ~found_mode + % Should not happen for a good config file + keyboard + end + + % Search for the particular mode/subchannel in the processing + found_mode = false; + for idx = 1:length(ipp_settings) + if any(ipp_settings(idx).modes == mode_latch) && any(ipp_settings(idx).subchannel == subchannel) + digRx_RG = ipp_settings(idx).digRx_RG; + found_mode = true; + break; + end + end + if ~found_mode + % Should not happen for a good config file + keyboard + end + + % Search for the particular mode/subchannel in the + found_mode = false; + for idx = 1:length(profileProcessor) + if strcmpi(profileProcessor(idx).name,profileInput(profile_idx).name) + profile_length = profileProcessor(idx).profile_length; + accumulations = profileProcessor(idx).accumulations; + found_mode = true; + break; + end + end + if ~found_mode + % Should not happen for a good config file + keyboard + end + + configs.adc{adc_idx,mode_latch+1,subchannel+1}.name = name; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.config_name = config_name; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.config_type = config_type; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.ncoPhase = ncoPhase; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.ncoFreq = ncoFreq_list(1).ncoFreq; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.ddc0NcoMode = 2; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.ddc1NcoMode = 2; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.ddc2NcoMode = 2; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.ddc3NcoMode = 2; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.ddc0NcoFreq = ncoFreq_list(1).ncoFreq; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.ddc1NcoFreq = ncoFreq_list(1).ncoFreq; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.ddc2NcoFreq = ncoFreq_list(1).ncoFreq; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.ddc3NcoFreq = ncoFreq_list(1).ncoFreq; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.cicDecimation = adcDecimation; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.adcMode = NaN; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.sampFreq = sampFreq; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.presums = accumulations; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.num_sam = profile_length; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.rg = sprintf('%d',digRx_RG(1)); + configs.adc{adc_idx,mode_latch+1,subchannel+1}.shiftLSB = 0; % 32 bit float + end + end + end + + elseif strcmpi(config_type,'adc-ad9684_002D') + % ===================================================================== + % GHOST RDS + % ===================================================================== + + % Get the sampling frequencies in MHz + expression = xpath.compile('sampFreq0'); + nodeList = expression.evaluate(adc_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + sampFreq0 = nodeList.item(0); + sampFreq0 = sampFreq0.getTextContent.toCharArray; + sampFreq0 = str2double(sampFreq0(:).') * 1e6; + + expression = xpath.compile('sampFreq1'); + nodeList = expression.evaluate(adc_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + sampFreq1 = nodeList.item(0); + sampFreq1 = sampFreq1.getTextContent.toCharArray; + sampFreq1 = str2double(sampFreq1(:).') * 1e6; + + % Load each subchannel + % ===================================================================== + expression = xpath.compile('subChannels/subChannel'); + subchannel_nodeList = expression.evaluate(adc_cfg,XPathConstants.NODESET); + subchannels = []; + for subchannel_idx = 1:subchannel_nodeList.getLength + subchannel_cfg = subchannel_nodeList.item(subchannel_idx-1); + if isempty(subchannel_cfg) + continue; + end + + % Load the subchannel ID + expression = xpath.compile('id'); + nodeList = expression.evaluate(subchannel_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + subchannel = nodeList.item(0); + subchannel = subchannel.getTextContent.toCharArray; + subchannel = str2double(subchannel(:).'); + subchannels(end+1).id = subchannel; + + % Load each subchannel's modes + expression = xpath.compile('mode'); + mode_nodeList = expression.evaluate(subchannel_cfg,XPathConstants.NODESET); + subchannels(end).modes = []; + for mode_idx = 1:mode_nodeList.getLength + mode_cfg = mode_nodeList.item(mode_idx-1); + if isempty(mode_cfg) + continue; + end + + expression = xpath.compile('id'); + nodeList = expression.evaluate(mode_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + modes = nodeList.item(0); + modes = modes.getTextContent.toCharArray; + modes = arena_convert_range(modes); + subchannels(end).modes(end+1).modes = modes; + + expression = xpath.compile('adcChannel'); + nodeList = expression.evaluate(mode_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + adcChannel = nodeList.item(0); + adcChannel = adcChannel.getTextContent.toCharArray; + adcChannel = arena_convert_range(adcChannel); + subchannels(end).modes(end).adcChannel = adcChannel; + + expression = xpath.compile('zeroPi'); + nodeList = expression.evaluate(mode_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + zeroPi = nodeList.item(0); + zeroPi = zeroPi.getTextContent.toCharArray; + zeroPi = str2double(zeroPi); + subchannels(end).modes(end).zeroPi = zeroPi; + + expression = xpath.compile('adcDecimation'); + nodeList = expression.evaluate(mode_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + adcDecimation = nodeList.item(0); + adcDecimation = adcDecimation.getTextContent.toCharArray; + adcDecimation = str2double(adcDecimation); + subchannels(end).modes(end).adcDecimation = adcDecimation; + + expression = xpath.compile('route'); + nodeList = expression.evaluate(mode_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + route = nodeList.item(0); + subchannels(end).modes(end).route = route.getTextContent.toCharArray; + subchannels(end).modes(end).route = subchannels(end).modes(end).route(:).'; + + expression = xpath.compile('ncoFreq'); + nodeList = expression.evaluate(mode_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + ncoFreq = nodeList.item(0); + ncoFreq = ncoFreq.getTextContent.toCharArray; + ncoFreq = str2double(ncoFreq)*1e6; + subchannels(end).modes(end).ncoFreq = ncoFreq; + + expression = xpath.compile('ncoPhase'); + nodeList = expression.evaluate(mode_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + ncoPhase = nodeList.item(0); + ncoPhase = ncoPhase.getTextContent.toCharArray; + ncoPhase = str2double(ncoPhase); + subchannels(end).modes(end).ncoPhase = ncoPhase; + + end + end + + % Load each processing chain + % ===================================================================== + expression = xpath.compile('processing/subChannel'); + subchannel_nodeList = expression.evaluate(adc_cfg,XPathConstants.NODESET); + processing = []; + for subchannel_idx = 1:subchannel_nodeList.getLength + subchannel_cfg = subchannel_nodeList.item(subchannel_idx-1); + if isempty(subchannel_cfg) + continue; + end + + % Load the subchannel ID + expression = xpath.compile('id'); + nodeList = expression.evaluate(subchannel_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + subchannel = nodeList.item(0); + subchannel = subchannel.getTextContent.toCharArray; + subchannel = str2double(subchannel(:).'); + processing(end+1).id = subchannel; + + % Load each subchannel's modes + expression = xpath.compile('mode'); + mode_nodeList = expression.evaluate(subchannel_cfg,XPathConstants.NODESET); + processing(end).modes = []; + for mode_idx = 1:mode_nodeList.getLength + mode_cfg = mode_nodeList.item(mode_idx-1); + if isempty(mode_cfg) + continue; + end + + expression = xpath.compile('id'); + nodeList = expression.evaluate(mode_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + modes = nodeList.item(0); + modes = modes.getTextContent.toCharArray; + modes = arena_convert_range(modes); + processing(end).modes(end+1).modes = modes; + + expression = xpath.compile('digRx_RG'); + nodeList = expression.evaluate(mode_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + digRx_RG = nodeList.item(0); + digRx_RG = digRx_RG.getTextContent.toCharArray; + digRx_RG = arena_convert_range(digRx_RG); + processing(end).modes(end).digRx_RG = digRx_RG; + + num_bins = length(digRx_RG); + if num_bins < configs.min_num_bins + configs.min_num_bins = num_bins; + end + if num_bins > configs.max_num_bins + configs.max_num_bins = num_bins; + end + + end + end + + % Load each profile input + % ===================================================================== + expression = xpath.compile('profileProcessing/input'); + subchannel_nodeList = expression.evaluate(adc_cfg,XPathConstants.NODESET); + profileInput = []; + for subchannel_idx = 1:subchannel_nodeList.getLength + subchannel_cfg = subchannel_nodeList.item(subchannel_idx-1); + if isempty(subchannel_cfg) + continue; + end + + % Load the profile name + expression = xpath.compile('name'); + nodeList = expression.evaluate(subchannel_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + profile_input_name = nodeList.item(0); + profileInput(end+1).name = profile_input_name.getTextContent.toCharArray; + profileInput(end).name = profileInput(end).name(:).'; + + % Load the profile modes + expression = xpath.compile('profileMode'); + nodeList = expression.evaluate(subchannel_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + profileMode = nodeList.item(0); + profileMode = profileMode.getTextContent.toCharArray; + profileMode = arena_convert_range(profileMode); + profileInput(end).profileMode = profileMode; + + % Load the profile subchannels + expression = xpath.compile('profileSubChannel'); + nodeList = expression.evaluate(subchannel_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + profileSubChannel = nodeList.item(0); + profileSubChannel = profileSubChannel.getTextContent.toCharArray; + profileSubChannel = arena_convert_range(profileSubChannel(:).'); + profileInput(end).profileSubChannel = profileSubChannel; + end + + + % Load each profile processor + % ===================================================================== + expression = xpath.compile('profileProcessing/processor'); + subchannel_nodeList = expression.evaluate(adc_cfg,XPathConstants.NODESET); + profileProcessor = []; + for subchannel_idx = 1:subchannel_nodeList.getLength + subchannel_cfg = subchannel_nodeList.item(subchannel_idx-1); + if isempty(subchannel_cfg) + continue; + end + + % Load the subchannel ID + expression = xpath.compile('name'); + nodeList = expression.evaluate(subchannel_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + profile_processor_name = nodeList.item(0); + profileProcessor(end+1).name = profile_processor_name.getTextContent.toCharArray; + profileProcessor(end).name = profileProcessor(end).name(:).'; + + % Load the subchannel ID + expression = xpath.compile('length'); + nodeList = expression.evaluate(subchannel_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + profile_length = nodeList.item(0); + profile_length = profile_length.getTextContent.toCharArray; + profile_length = str2double(profile_length(:).'); + profileProcessor(end).profile_length = profile_length; + + expression = xpath.compile('accumulations'); + nodeList = expression.evaluate(subchannel_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + accumulations = nodeList.item(0); + accumulations = accumulations.getTextContent.toCharArray; + accumulations = str2double(accumulations(:).'); + profileProcessor(end).accumulations = accumulations+1; + + end + + % Update adc configs for this integrator's modes and subchannel + for profile_idx = 1:length(profileInput) + for mode_latch = profileInput(profile_idx).profileMode + for subchannel = profileInput(profile_idx).profileSubChannel + + % Search for the particular mode/subchannel in the subchannels + found_mode = false; + for idx = 1:length(subchannels(subchannel+1).modes) + if any(subchannels(subchannel+1).modes(idx).modes == mode_latch) + ncoFreq = subchannels(subchannel+1).modes(idx).ncoFreq; + ncoPhase = subchannels(subchannel+1).modes(idx).ncoPhase; + found_mode = true; + break; + end + end + if ~found_mode + % Should not happen for a good config file + keyboard + end + + % Search for the particular mode/subchannel in the processing + found_mode = false; + for idx = 1:length(processing(subchannel+1).modes) + if any(processing(subchannel+1).modes(idx).modes == mode_latch) + digRx_RG = processing(subchannel+1).modes(idx).digRx_RG; + found_mode = true; + break; + end + end + if ~found_mode + % Should not happen for a good config file + keyboard + end + + % Search for the particular mode/subchannel in the + found_mode = false; + for idx = 1:length(profileProcessor) + if strcmpi(profileProcessor(idx).name,profileInput(profile_idx).name) + profile_length = profileProcessor(idx).profile_length; + accumulations = profileProcessor(idx).accumulations; + found_mode = true; + break; + end + end + if ~found_mode + % Should not happen for a good config file + keyboard + end + + decimation = 4; + + configs.adc{adc_idx,mode_latch+1,subchannel+1}.name = name; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.config_name = config_name; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.config_type = config_type; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.ncoPhase = ncoPhase; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.ncoFreq = ncoFreq; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.ddc0NcoMode = 2; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.ddc1NcoMode = 2; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.ddc2NcoMode = 2; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.ddc3NcoMode = 2; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.ddc0NcoFreq = ncoFreq; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.ddc1NcoFreq = ncoFreq; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.ddc2NcoFreq = ncoFreq; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.ddc3NcoFreq = ncoFreq; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.cicDecimation = decimation; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.adcMode = NaN; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.sampFreq = sampFreq0; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.presums = accumulations; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.num_sam = profile_length; + configs.adc{adc_idx,mode_latch+1,subchannel+1}.rg = sprintf('%d',digRx_RG(1)); + configs.adc{adc_idx,mode_latch+1,subchannel+1}.shiftLSB = 0; % 32 bit float + end end end - + elseif strcmpi(config_type,'adc-ad9680_0017') % ===================================================================== % BAS Accumulation Radar, Dome Fuji RDS @@ -647,7 +1544,7 @@ end ncoFreq = nodeList.item(0); ncoFreq = ncoFreq.getTextContent.toCharArray; - ncoFreq = str2double(ncoFreq(:).'); + ncoFreq = str2double(ncoFreq(:).') * 1e6; expression = xpath.compile('decimation'); nodeList = expression.evaluate(digRx_cfg,XPathConstants.NODESET); @@ -774,132 +1671,153 @@ %% CTU DIGITAL IO % ========================================================================= % Get all the CTUs -expression = xpath.compile('//subSystem[starts-with(@type,"ctu")]'); -ctuList = expression.evaluate(doc,XPathConstants.NODESET); +if strcmpi(configs.radar_name,'extsyncarena5xx') + % SYSTEM XML: + % + % ARENA5xxRF + % + % CONFIG XML: + % + % ARENA5xxRF + % + % dac + % ctu1 + % + % + % + % ctu1 -% 1. Get the name of the CTU -expression = xpath.compile('name'); -nodeList = expression.evaluate(ctuList.item(0),XPathConstants.NODESET); -name = nodeList.item(0).getTextContent.toCharArray; -name = name(:).'; + % Get the arena 522 subsystems + expression = xpath.compile('//subSystem[starts-with(@type,"arena5xxrf")]'); + arena522_list = expression.evaluate(doc,XPathConstants.NODESET); -% 2. Search for the subSystem in the config XML -expression = xpath.compile(sprintf('//subSystem/subSystem[name="%s"]',name)); -nodeList = expression.evaluate(doc_cfg,XPathConstants.NODESET); -match = nodeList.item(0); - -% 3. Get the config name and type for this CTU -expression = xpath.compile('config/@type'); -nodeList = expression.evaluate(match,XPathConstants.NODESET); -config_type = nodeList.item(0).getTextContent.toCharArray; -config_type = config_type(:).'; -expression = xpath.compile('config'); -nodeList = expression.evaluate(match,XPathConstants.NODESET); -config_name = nodeList.item(0).getTextContent.toCharArray; -config_name = config_name(:).'; - -configs.ctu.name = name; -configs.ctu.config_type = config_type; -configs.ctu.config_name = config_name; -configs.ctu.io = {}; - -% Get the config associated with this CTU -expression = xpath.compile(sprintf('//configs/config[(@type="%s" and name="%s")]',config_type,config_name)); -nodeList = expression.evaluate(doc_cfg,XPathConstants.NODESET); -ctu_cfg = nodeList.item(0); - -% Find the longest possible record size -if strcmpi(config_type,'ctu_0013') - % TOHFSounder - - % Get each subchannel - expression = xpath.compile('mode'); - mode_nodeList = expression.evaluate(ctu_cfg,XPathConstants.NODESET); - for mode_idx = 1:mode_nodeList.getLength - mode_cfg = mode_nodeList.item(mode_idx-1); - if isempty(mode_cfg) + found_ctu = false; + for arena522_idx = 1:arena522_list.getLength + arena522_syscfg = arena522_list.item(arena522_idx-1); + if isempty(arena522_syscfg) continue; end - expression = xpath.compile('id'); - nodeList = expression.evaluate(mode_cfg,XPathConstants.NODESET); - if nodeList.getLength < 1 || isempty(nodeList.item(0)) - continue; - end - mode_latch = nodeList.item(0); - mode_latch = mode_latch.getTextContent.toCharArray; - mode_latch = arena_convert_range(mode_latch); + % Get the arena 522 subsystem name + expression = xpath.compile('name'); + nodeList = expression.evaluate(arena522_syscfg,XPathConstants.NODESET); + arena522_name = nodeList.item(0); + arena522_name = arena522_name.getTextContent.toCharArray; + arena522_name = arena522_name(:).'; - expression = xpath.compile('segmentTimes'); - nodeList = expression.evaluate(mode_cfg,XPathConstants.NODESET); - if nodeList.getLength < 1 || isempty(nodeList.item(0)) - continue; - end - segmentTimes = nodeList.item(0); - segmentTimes = segmentTimes.getTextContent.toCharArray; - segmentTimes = segmentTimes(:).'; - configs.ctu.io{mode_latch+1}.segmentTimes = segmentTimes; + % Find the configuration for this arena 522 + expression = xpath.compile(sprintf('//subSystem[name="%s"]', arena522_name)); + arena522_cfg = expression.evaluate(doc_cfg,XPathConstants.NODESET); + arena522_cfg = arena522_cfg.item(0); + + % Get the Arena 522 CTU config name + config_type = 'rfsoc_ctu_4001'; + expression = xpath.compile(sprintf('//subSystem/config[@type="%s"]', config_type)); + config_name = expression.evaluate(arena522_cfg,XPathConstants.NODESET); + config_name = config_name.item(0).getTextContent; - expression = xpath.compile('segmentStates'); - nodeList = expression.evaluate(mode_cfg,XPathConstants.NODESET); - if nodeList.getLength < 1 || isempty(nodeList.item(0)) - continue; + % Get the Arena 522 CTU configs + expression = xpath.compile(sprintf('//configs/config[@type="%s" and name="%s"]',config_type,config_name)); + ctu_cfg = expression.evaluate(doc_cfg,XPathConstants.NODESET); + if ~isempty(ctu_cfg) + ctu_cfg = ctu_cfg.item(0); + found_ctu = true; + break; end - segmentStates = nodeList.item(0); - segmentStates = segmentStates.getTextContent.toCharArray; - segmentStates = segmentStates(:).'; - configs.ctu.io{mode_latch+1}.segmentStates = segmentStates; + end + if ~found_ctu || ctu_cfg.getLength == 0 + error('Did not find CTU configuration.'); end -elseif strcmpi(config_type,'ctu_001D') - - % Get each subchannel - expression = xpath.compile('mode'); - mode_nodeList = expression.evaluate(ctu_cfg,XPathConstants.NODESET); - for mode_idx = 1:mode_nodeList.getLength - mode_cfg = mode_nodeList.item(mode_idx-1); - if isempty(mode_cfg) - continue; - end - - expression = xpath.compile('id'); - nodeList = expression.evaluate(mode_cfg,XPathConstants.NODESET); - if nodeList.getLength < 1 || isempty(nodeList.item(0)) - continue; - end - modes = nodeList.item(0); - modes = modes.getTextContent.toCharArray; - modes = arena_convert_range(modes); - - expression = xpath.compile('segmentTimes'); - nodeList = expression.evaluate(mode_cfg,XPathConstants.NODESET); - if nodeList.getLength < 1 || isempty(nodeList.item(0)) - continue; - end - segmentTimes = nodeList.item(0); - segmentTimes = segmentTimes.getTextContent.toCharArray; - segmentTimes = segmentTimes(:).'; - for mode_latch = modes - configs.ctu.io{mode_latch+1}.segmentTimes = segmentTimes; - end - expression = xpath.compile('segmentStates'); - nodeList = expression.evaluate(mode_cfg,XPathConstants.NODESET); - if nodeList.getLength < 1 || isempty(nodeList.item(0)) - continue; - end - segmentStates = nodeList.item(0); - segmentStates = segmentStates.getTextContent.toCharArray; - segmentStates = segmentStates(:).'; - for mode_latch = modes - configs.ctu.io{mode_latch+1}.segmentStates = segmentStates; - end +else + if strcmpi(configs.radar_name,'ku0002') + expression = xpath.compile('//subSystem[starts-with(@type,"dac-ad9129_0032")]'); + else + expression = xpath.compile('//subSystem[starts-with(@type,"ctu")]'); end + ctuList = expression.evaluate(doc,XPathConstants.NODESET); + + % 1. Get the name of the CTU + expression = xpath.compile('name'); + nodeList = expression.evaluate(ctuList.item(0),XPathConstants.NODESET); + name = nodeList.item(0).getTextContent.toCharArray; + name = name(:).'; + + % 2. Search for the subSystem in the config XML + expression = xpath.compile(sprintf('//subSystem/subSystem[name="%s"]',name)); + nodeList = expression.evaluate(doc_cfg,XPathConstants.NODESET); + match = nodeList.item(0); + + % 3. Get the config name and type for this CTU + expression = xpath.compile('config/@type'); + nodeList = expression.evaluate(match,XPathConstants.NODESET); + config_type = nodeList.item(0).getTextContent.toCharArray; + config_type = config_type(:).'; + expression = xpath.compile('config'); + nodeList = expression.evaluate(match,XPathConstants.NODESET); + config_name = nodeList.item(0).getTextContent.toCharArray; + config_name = config_name(:).'; + + configs.ctu.name = name; + configs.ctu.config_type = config_type; + configs.ctu.config_name = config_name; + configs.ctu.io = {}; + + % Get the config associated with this CTU + expression = xpath.compile(sprintf('//configs/config[(@type="%s" and name="%s")]',config_type,config_name)); + nodeList = expression.evaluate(doc_cfg,XPathConstants.NODESET); + ctu_cfg = nodeList.item(0); end +% strcmpi(config_type,'rfsoc_ctu_4001'): Arena 522 +% strcmpi(config_type,'ctu_0013'): TOHFSounder +% strcmpi(config_type,'ctu_001D'): Many systems + +% Get each subchannel +expression = xpath.compile('mode'); +mode_nodeList = expression.evaluate(ctu_cfg,XPathConstants.NODESET); +for mode_idx = 1:mode_nodeList.getLength + mode_cfg = mode_nodeList.item(mode_idx-1); + if isempty(mode_cfg) + continue; + end + + expression = xpath.compile('id'); + nodeList = expression.evaluate(mode_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + modes = nodeList.item(0); + modes = modes.getTextContent.toCharArray; + modes = arena_convert_range(modes); + + expression = xpath.compile('segmentTimes'); + nodeList = expression.evaluate(mode_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + segmentTimes = nodeList.item(0); + segmentTimes = segmentTimes.getTextContent.toCharArray; + segmentTimes = segmentTimes(:).'; + for mode_latch = modes + configs.ctu.io{mode_latch+1}.segmentTimes = segmentTimes; + end + expression = xpath.compile('segmentStates'); + nodeList = expression.evaluate(mode_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + segmentStates = nodeList.item(0); + segmentStates = segmentStates.getTextContent.toCharArray; + segmentStates = segmentStates(:).'; + for mode_latch = modes + configs.ctu.io{mode_latch+1}.segmentStates = segmentStates; + end +end %% DACS/AWG % ========================================================================= -expression = xpath.compile('//subSystem[starts-with(@type,"dac")]'); +expression = xpath.compile('//subSystem[contains(@type,"dac")]'); dacList = expression.evaluate(doc,XPathConstants.NODESET); for dac_idx = 1:dacList.getLength @@ -943,8 +1861,8 @@ dac_cfg = nodeList.item(0); % Find the longest possible record size - if strcmpi(config_type,'dac-ad9129_0012') - % TOHFSounder + if strcmpi(config_type,'dac-ad9129_0012') || strcmpi(config_type,'dac-ad9129_0032') + % TOHFSounder, EAGER, MELT, Vanilla, AWI MCoRDS % Get each subchannel expression = xpath.compile('mode'); @@ -971,7 +1889,7 @@ end delay = nodeList.item(0); delay = delay.getTextContent.toCharArray; - delay = str2double(delay(:).'); + delay = str2double(delay(:).') * 1e-6; configs.dac{mode_latch+1}.delay = delay; % 3. Get the config name and type for this DAC waveform @@ -990,7 +1908,7 @@ dac_wf_cfg = nodeList.item(0); % Read DAC waveform parameters - if strcmpi(config_type,'dac-ad9129_0012_waveform') + if strcmpi(config_type,'dac-ad9129_0012_waveform') || strcmpi(config_type,'dac-ad9129_0032_waveform') % TOHFSounder expression = xpath.compile('name'); @@ -998,14 +1916,14 @@ name = nodeList.item(0); name = name.getTextContent.toCharArray; name = name(:).'; - configs.dac{tx_idx,mode_latch+1}.name = name; + configs.dac{tx_idx,mode_latch+1}.wfs{1}.name = name; expression = xpath.compile('sampFreq'); nodeList = expression.evaluate(dac_wf_cfg,XPathConstants.NODESET); sampFreq = nodeList.item(0); sampFreq = sampFreq.getTextContent.toCharArray; - sampFreq = str2double(sampFreq(:).'); - configs.dac{tx_idx,mode_latch+1}.sampFreq = sampFreq; + sampFreq = str2double(sampFreq(:).') * 1e6; + configs.dac{tx_idx,mode_latch+1}.wfs{1}.sampFreq = sampFreq; % Get each subchannel expression = xpath.compile('pulse'); @@ -1021,77 +1939,376 @@ name = nodeList.item(0); name = name.getTextContent.toCharArray; name = name(:).'; - configs.dac{tx_idx,mode_latch+1}.wfs{pulse_idx}.name = name; + configs.dac{tx_idx,mode_latch+1}.wfs{1}.pulse{pulse_idx}.name = name; expression = xpath.compile('centerFreq'); nodeList = expression.evaluate(pulse_cfg,XPathConstants.NODESET); centerFreq = nodeList.item(0); centerFreq = centerFreq.getTextContent.toCharArray; - centerFreq = str2double(centerFreq(:).'); - configs.dac{tx_idx,mode_latch+1}.wfs{pulse_idx}.centerFreq = centerFreq; + centerFreq = str2double(centerFreq(:).') * 1e6; + configs.dac{tx_idx,mode_latch+1}.wfs{1}.pulse{pulse_idx}.centerFreq = centerFreq; expression = xpath.compile('bandwidth'); nodeList = expression.evaluate(pulse_cfg,XPathConstants.NODESET); bandwidth = nodeList.item(0); bandwidth = bandwidth.getTextContent.toCharArray; - bandwidth = str2double(bandwidth(:).'); - configs.dac{tx_idx,mode_latch+1}.wfs{pulse_idx}.bandwidth = bandwidth; + bandwidth = str2double(bandwidth(:).') * 1e6; + configs.dac{tx_idx,mode_latch+1}.wfs{1}.pulse{pulse_idx}.bandwidth = bandwidth; expression = xpath.compile('initialDelay'); nodeList = expression.evaluate(pulse_cfg,XPathConstants.NODESET); initialDelay = nodeList.item(0); initialDelay = initialDelay.getTextContent.toCharArray; - initialDelay = str2double(initialDelay(:).'); - configs.dac{tx_idx,mode_latch+1}.wfs{pulse_idx}.initialDelay = initialDelay; + initialDelay = str2double(initialDelay(:).') * 1e-6; + configs.dac{tx_idx,mode_latch+1}.wfs{1}.pulse{pulse_idx}.initialDelay = initialDelay; + + expression = xpath.compile('initialPhase'); + nodeList = expression.evaluate(pulse_cfg,XPathConstants.NODESET); + initialPhase = nodeList.item(0); + initialPhase = initialPhase.getTextContent.toCharArray; + initialPhase = str2double(initialPhase(:).'); + configs.dac{tx_idx,mode_latch+1}.wfs{1}.pulse{pulse_idx}.initialPhase = initialPhase; + + expression = xpath.compile('afterPulseDelay'); + nodeList = expression.evaluate(pulse_cfg,XPathConstants.NODESET); + afterPulseDelay = nodeList.item(0); + afterPulseDelay = afterPulseDelay.getTextContent.toCharArray; + afterPulseDelay = str2double(afterPulseDelay(:).') * 1e-6; + configs.dac{tx_idx,mode_latch+1}.wfs{1}.pulse{pulse_idx}.afterPulseDelay = afterPulseDelay; + + expression = xpath.compile('taper'); + nodeList = expression.evaluate(pulse_cfg,XPathConstants.NODESET); + taper = nodeList.item(0); + taper = taper.getTextContent.toCharArray; + taper = taper(:).'; + configs.dac{tx_idx,mode_latch+1}.wfs{1}.pulse{pulse_idx}.taper = taper; + + expression = xpath.compile('alpha'); + nodeList = expression.evaluate(pulse_cfg,XPathConstants.NODESET); + alpha = nodeList.item(0); + alpha = alpha.getTextContent.toCharArray; + alpha = str2double(alpha(:).'); + configs.dac{tx_idx,mode_latch+1}.wfs{1}.pulse{pulse_idx}.alpha = alpha; + + expression = xpath.compile('scale'); + nodeList = expression.evaluate(pulse_cfg,XPathConstants.NODESET); + scale = nodeList.item(0); + scale = scale.getTextContent.toCharArray; + scale = str2double(scale(:).'); + configs.dac{tx_idx,mode_latch+1}.wfs{1}.pulse{pulse_idx}.scale = scale; + + expression = xpath.compile('numPoints'); + nodeList = expression.evaluate(pulse_cfg,XPathConstants.NODESET); + numPoints = nodeList.item(0); + numPoints = numPoints.getTextContent.toCharArray; + numPoints = str2double(numPoints(:).'); + configs.dac{tx_idx,mode_latch+1}.wfs{1}.pulse{pulse_idx}.numPoints = numPoints; + end + end + + + end + + elseif strcmpi(config_type,'dac-ad9783_002C') + % GHOST + + % Get each subchannel + expression = xpath.compile('mode'); + mode_nodeList = expression.evaluate(dac_cfg,XPathConstants.NODESET); + for mode_idx = 1:mode_nodeList.getLength + mode_cfg = mode_nodeList.item(mode_idx-1); + if isempty(mode_cfg) + continue; + end + + expression = xpath.compile('id'); + nodeList = expression.evaluate(mode_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + mode_latch = nodeList.item(0); + mode_latch = mode_latch.getTextContent.toCharArray; + mode_latch = arena_convert_range(mode_latch); + + expression = xpath.compile('delay'); + nodeList = expression.evaluate(mode_cfg,XPathConstants.NODESET); + if nodeList.getLength < 1 || isempty(nodeList.item(0)) + continue; + end + delay = nodeList.item(0); + delay = delay.getTextContent.toCharArray; + delay = str2double(delay(:).'); + configs.dac{mode_latch+1}.delay = delay; + + % 3. Get the config name and type for this DAC waveform + expression = xpath.compile('config/@type'); + nodeList = expression.evaluate(mode_cfg,XPathConstants.NODESET); + config_type = nodeList.item(0).getTextContent.toCharArray; + config_type = config_type(:).'; + + % Get each of the four DAC channel waveforms + expression = xpath.compile('config'); + wf_nodeList = expression.evaluate(mode_cfg,XPathConstants.NODESET); + for wf_idx = 1:wf_nodeList.getLength + wf_cfg = wf_nodeList.item(wf_idx-1); + if isempty(wf_cfg) + continue; + end + + config_name = wf_cfg.item(0).getTextContent.toCharArray; + config_name = config_name(:).'; + + % Get the config associated with this DAC waveform + expression = xpath.compile(sprintf('//configs/config[(@type="%s" and name="%s")]',config_type,config_name)); + nodeList = expression.evaluate(doc_cfg,XPathConstants.NODESET); + dac_wf_cfg = nodeList.item(0); + + expression = xpath.compile('name'); + nodeList = expression.evaluate(dac_wf_cfg,XPathConstants.NODESET); + name = nodeList.item(0); + name = name.getTextContent.toCharArray; + name = name(:).'; + configs.dac{tx_idx,mode_latch+1}.wfs{wf_idx}.name = name; + + expression = xpath.compile('sampFreq'); + nodeList = expression.evaluate(dac_wf_cfg,XPathConstants.NODESET); + sampFreq = nodeList.item(0); + sampFreq = sampFreq.getTextContent.toCharArray; + sampFreq = str2double(sampFreq(:).') * 1e6; + configs.dac{tx_idx,mode_latch+1}.wfs{wf_idx}.sampFreq = sampFreq; + + % Get each pulse + expression = xpath.compile('pulse'); + pulse_nodeList = expression.evaluate(dac_wf_cfg,XPathConstants.NODESET); + for pulse_idx = 1:pulse_nodeList.getLength + pulse_cfg = pulse_nodeList.item(pulse_idx-1); + if isempty(pulse_cfg) + continue; + end + + expression = xpath.compile('name'); + nodeList = expression.evaluate(pulse_cfg,XPathConstants.NODESET); + name = nodeList.item(0); + name = name.getTextContent.toCharArray; + name = name(:).'; + configs.dac{tx_idx,mode_latch+1}.wfs{wf_idx}.pulse{pulse_idx}.name = name; + + expression = xpath.compile('centerFreq'); + nodeList = expression.evaluate(pulse_cfg,XPathConstants.NODESET); + centerFreq = nodeList.item(0); + centerFreq = centerFreq.getTextContent.toCharArray; + centerFreq = str2double(centerFreq(:).') * 1e6; + configs.dac{tx_idx,mode_latch+1}.wfs{wf_idx}.pulse{pulse_idx}.centerFreq = centerFreq; + + expression = xpath.compile('bandwidth'); + nodeList = expression.evaluate(pulse_cfg,XPathConstants.NODESET); + bandwidth = nodeList.item(0); + bandwidth = bandwidth.getTextContent.toCharArray; + bandwidth = str2double(bandwidth(:).') * 1e6; + configs.dac{tx_idx,mode_latch+1}.wfs{wf_idx}.pulse{pulse_idx}.bandwidth = bandwidth; + + configs.dac{tx_idx,mode_latch+1}.wfs{wf_idx}.pulse{pulse_idx}.initialDelay = 0; expression = xpath.compile('initialPhase'); nodeList = expression.evaluate(pulse_cfg,XPathConstants.NODESET); initialPhase = nodeList.item(0); initialPhase = initialPhase.getTextContent.toCharArray; initialPhase = str2double(initialPhase(:).'); - configs.dac{tx_idx,mode_latch+1}.wfs{pulse_idx}.initialPhase = initialPhase; + configs.dac{tx_idx,mode_latch+1}.wfs{wf_idx}.pulse{pulse_idx}.initialPhase = initialPhase; expression = xpath.compile('afterPulseDelay'); nodeList = expression.evaluate(pulse_cfg,XPathConstants.NODESET); afterPulseDelay = nodeList.item(0); afterPulseDelay = afterPulseDelay.getTextContent.toCharArray; afterPulseDelay = str2double(afterPulseDelay(:).'); - configs.dac{tx_idx,mode_latch+1}.wfs{pulse_idx}.afterPulseDelay = afterPulseDelay; + configs.dac{tx_idx,mode_latch+1}.wfs{wf_idx}.pulse{pulse_idx}.afterPulseDelay = afterPulseDelay; expression = xpath.compile('taper'); nodeList = expression.evaluate(pulse_cfg,XPathConstants.NODESET); taper = nodeList.item(0); taper = taper.getTextContent.toCharArray; taper = taper(:).'; - configs.dac{tx_idx,mode_latch+1}.wfs{pulse_idx}.taper = taper; + configs.dac{tx_idx,mode_latch+1}.wfs{wf_idx}.pulse{pulse_idx}.taper = taper; expression = xpath.compile('alpha'); nodeList = expression.evaluate(pulse_cfg,XPathConstants.NODESET); alpha = nodeList.item(0); alpha = alpha.getTextContent.toCharArray; alpha = str2double(alpha(:).'); - configs.dac{tx_idx,mode_latch+1}.wfs{pulse_idx}.alpha = alpha; + configs.dac{tx_idx,mode_latch+1}.wfs{wf_idx}.pulse{pulse_idx}.alpha = alpha; expression = xpath.compile('scale'); nodeList = expression.evaluate(pulse_cfg,XPathConstants.NODESET); scale = nodeList.item(0); scale = scale.getTextContent.toCharArray; scale = str2double(scale(:).'); - configs.dac{tx_idx,mode_latch+1}.wfs{pulse_idx}.scale = scale; + configs.dac{tx_idx,mode_latch+1}.wfs{wf_idx}.pulse{pulse_idx}.scale = scale; expression = xpath.compile('numPoints'); nodeList = expression.evaluate(pulse_cfg,XPathConstants.NODESET); numPoints = nodeList.item(0); numPoints = numPoints.getTextContent.toCharArray; numPoints = str2double(numPoints(:).'); - configs.dac{tx_idx,mode_latch+1}.wfs{pulse_idx}.numPoints = numPoints; + configs.dac{tx_idx,mode_latch+1}.wfs{wf_idx}.pulse{pulse_idx}.numPoints = numPoints; end end + end + + elseif strcmpi(config_type,'rfsoc_dac_1002') + % ===================================================================== + % Arena 522 COLDEX + % ===================================================================== + + % + % awg0 + % + % 8960.000000 + % + % 10us + % 0 + % 0 + % 0 + % 2.000000 + % waveform1us ch4 + % + % + % waveform1us ch1 + % + % Pulse + % 700.000000 + % 50.000000 + % 0.000000 + % 0.000000 + % Tukey + % 0.100000 + % 1.000000 + % 8960 + % + expression = xpath.compile('sampFreq'); + sampFreq = expression.evaluate(dac_cfg,XPathConstants.NODESET); + sampFreq = sampFreq.item(0); + sampFreq = sampFreq.getTextContent.toCharArray; + sampFreq = str2double(sampFreq(:).')*1e6; + + expression = xpath.compile('waveformSetting'); + wf_setting_list = expression.evaluate(dac_cfg,XPathConstants.NODESET); + for wf_idx = 1:wf_setting_list.getLength + wf_setting = wf_setting_list.item(wf_idx-1); + if isempty(wf_setting) + continue; + end + + expression = xpath.compile('name'); + name = expression.evaluate(wf_setting,XPathConstants.NODESET); + name = name.item(0); + name = name.getTextContent.toCharArray; + name = name(:).'; + + expression = xpath.compile('channels'); + channels = expression.evaluate(wf_setting,XPathConstants.NODESET); + channels = channels.item(0); + channels = channels.getTextContent.toCharArray; + channels = arena_convert_range(channels(:).'); + + expression = xpath.compile('modes'); + modes = expression.evaluate(wf_setting,XPathConstants.NODESET); + modes = modes.item(0); + modes = modes.getTextContent.toCharArray; + modes = arena_convert_range(modes(:).'); + + expression = xpath.compile('invert'); + invert = expression.evaluate(wf_setting,XPathConstants.NODESET); + invert = invert.item(0); + invert = invert.getTextContent.toCharArray; + invert = str2double(invert(:).'); + + expression = xpath.compile('delay'); + delay = expression.evaluate(wf_setting,XPathConstants.NODESET); + delay = delay.item(0); + delay = delay.getTextContent.toCharArray; + delay = str2double(delay(:).'); + + expression = xpath.compile('waveform'); + waveform_name = expression.evaluate(wf_setting,XPathConstants.NODESET); + waveform_name = waveform_name.item(0); + waveform_name = waveform_name.getTextContent.toCharArray; + waveform_name = waveform_name(:).'; + expression = xpath.compile(sprintf('waveform[name="%s"]', waveform_name)); + wf_cfg = expression.evaluate(dac_cfg,XPathConstants.NODESET); + expression = xpath.compile('pulse'); + wf_cfg = expression.evaluate(wf_cfg.item(0),XPathConstants.NODESET); + wf_cfg = wf_cfg.item(0); + + expression = xpath.compile('centerFreq'); + centerFreq = expression.evaluate(wf_cfg,XPathConstants.NODESET); + centerFreq = centerFreq.item(0); + centerFreq = centerFreq.getTextContent.toCharArray; + centerFreq = str2double(centerFreq(:).') * 1e6; + + expression = xpath.compile('bandwidth'); + bandwidth = expression.evaluate(wf_cfg,XPathConstants.NODESET); + bandwidth = bandwidth.item(0); + bandwidth = bandwidth.getTextContent.toCharArray; + bandwidth = str2double(bandwidth(:).') * 1e6; + + expression = xpath.compile('initialDelay'); + initialDelay = expression.evaluate(wf_cfg,XPathConstants.NODESET); + initialDelay = initialDelay.item(0); + initialDelay = initialDelay.getTextContent.toCharArray; + initialDelay = str2double(initialDelay(:).'); + + expression = xpath.compile('initialPhase'); + initialPhase = expression.evaluate(wf_cfg,XPathConstants.NODESET); + initialPhase = initialPhase.item(0); + initialPhase = initialPhase.getTextContent.toCharArray; + initialPhase = str2double(initialPhase(:).'); + + expression = xpath.compile('taper'); + taper = expression.evaluate(wf_cfg,XPathConstants.NODESET); + taper = taper.item(0); + taper = taper.getTextContent.toCharArray; + taper = taper(:).'; + + expression = xpath.compile('alpha'); + alpha = expression.evaluate(wf_cfg,XPathConstants.NODESET); + alpha = alpha.item(0); + alpha = alpha.getTextContent.toCharArray; + alpha = str2double(alpha(:).'); + + expression = xpath.compile('scale'); + scale = expression.evaluate(wf_cfg,XPathConstants.NODESET); + scale = scale.item(0); + scale = scale.getTextContent.toCharArray; + scale = str2double(scale(:).'); + + expression = xpath.compile('numPoints'); + numPoints = expression.evaluate(wf_cfg,XPathConstants.NODESET); + numPoints = numPoints.item(0); + numPoints = numPoints.getTextContent.toCharArray; + numPoints = str2double(numPoints(:).'); + + % Set modes + pulse_idx = 1; + for mode_latch = modes(:).' + configs.dac{tx_idx,mode_latch+1}.delay = delay; + for channel = channels(:).' + configs.dac{tx_idx,mode_latch+1}.wfs{channel+1}.sampFreq = sampFreq; + configs.dac{tx_idx,mode_latch+1}.wfs{channel+1}.pulse{pulse_idx}.name = name; + configs.dac{tx_idx,mode_latch+1}.wfs{channel+1}.pulse{pulse_idx}.centerFreq = centerFreq; + configs.dac{tx_idx,mode_latch+1}.wfs{channel+1}.pulse{pulse_idx}.bandwidth = bandwidth; + configs.dac{tx_idx,mode_latch+1}.wfs{channel+1}.pulse{pulse_idx}.initialDelay = initialDelay; + configs.dac{tx_idx,mode_latch+1}.wfs{channel+1}.pulse{pulse_idx}.initialPhase = initialPhase; + configs.dac{tx_idx,mode_latch+1}.wfs{channel+1}.pulse{pulse_idx}.afterPulseDelay = 0; + configs.dac{tx_idx,mode_latch+1}.wfs{channel+1}.pulse{pulse_idx}.taper = taper; + configs.dac{tx_idx,mode_latch+1}.wfs{channel+1}.pulse{pulse_idx}.alpha = alpha; + configs.dac{tx_idx,mode_latch+1}.wfs{channel+1}.pulse{pulse_idx}.scale = scale; + configs.dac{tx_idx,mode_latch+1}.wfs{channel+1}.pulse{pulse_idx}.numPoints = numPoints; + end + end + end end + end - end - - diff --git a/cresis-toolbox/tracker/block_data.m b/cresis-toolbox/tracker/block_data.m new file mode 100644 index 00000000..8417c60c --- /dev/null +++ b/cresis-toolbox/tracker/block_data.m @@ -0,0 +1,551 @@ +% function status = block_data(param,param_override) +% status = block_data(param,param_override) +% +% param: Parameter structure from read_param_xls parameter spreadsheet +% +% param.block_data: Structure which controls the size of each block +% +% .block_size: number of columns in each block +% +% .block_overlap: the percentage of overlap between each block +% +% .debug_plot: Set to true for debug plots. +% +% .detrend_debug: Set to true for detrend debug plots. +% +% .bottom_pad : number of rows after the deepest layer +% +% .early_trunc: Truncate data immediately after surface flattening +% (before detrending and normalizing) +% +% .echo_path: Path to echogram data, typically an argument of +% ct_filename_out function e.g 'CSARP\standard' => +% ct_filename_out(param,'CSARP\standard'). +% +% .late_trunc: Truncate data after all data manipulation( i.e detrending +% and normalizing ) is done. +% +% .layer_params: opsLoadLayers.m structure describing which layers to load +% and store in the block files +% +% .layers_source: This specifies where the layer data is loaded from(e.g +% layerdata, records, lidar, etc). This forms a field of the layer_params +% struct passed into opsLoadLayers. See runOpsLoadLayers.m +% +% .layerdata_source: When layers_source is layerdata, this string +% specifies the layerdata (e.g layer_koenig, layer, post) to be loaded. +% This field is also one of the fields of the layer_params struct passed +% into opsLoadLayers. See runOpsLoadLayers.m +% +% .out_fn: Path where output blocks and files are saved. Currently, this +% is passed as an argument to ct_filename_tmp to save the outputs in KU +% user's scratch +% +% .post_detrend_filter_en: Enable/Disable filtering after detrending +% (before normalization) +% +% .pre_detrend_filter_en: Enable/Disable filtering before detrending +% +% .regexp: When layers_source is layerdata, all the layers with layer +% names that match this regular expression pattern are loaded. This field +% is also one of the fields of the layer_params struct passed into +% opsLoadLayers. See runOpsLoadLayers.m +% +% .surf_param: opsLoadLayers.m structure describing which surface to load +% and store in the preprocessing and in the block files +% +% .surface_flat_en: Enable/Disable surface flattening +% +% .surface_rel_layers_flat_en: Optional feature when surface filtering is +% enabled. Enable this feature to flatten the layers relative to the +% filtered surface. +% +% .surface_filter_len: Specifies the length of the filter for filtering +% the surface +% +% .top_gap: number of rows before the first layer +% +% .uncompress_en: Depending on the echogram data product used (e.g qlook, +% post), the echogram may be compressed. This flag when true uncompresses +% the compressed data using uncompress_echogram function prior to any +% processing. +% +% Authors: Ibikunle ( Adapted from John Paden's koenig_mat_loader ) +% +% See also: run_block_data.m, block_data.m, run_unblock_data.m, +% unblock_data.m + +%% General Setup +% ===================================================================== +param = merge_structs(param, param_override); + +fprintf('=====================================================================\n'); +fprintf('%s: %s (%s)\n', mfilename, param.day_seg, datestr(now)); +fprintf('=====================================================================\n'); + +[output_dir,radar_type,radar_name] = ct_output_dir(param.radar_name); + +physical_constants; + +%% Input Checks: cmd +% ========================================================================= + +% Remove frames that do not exist from param.cmd.frms list +frames = frames_load(param); +param.cmd.frms = frames_param_cmd_frms(param,frames); + +records = records_load(param); +ref = records_reference_trajectory_load(param,records); +along_track = geodetic_to_along_track(ref.lat,ref.lon,ref.elev); + +%% Input Checks: block_data +% ========================================================================= + +if ~isfield(param,'block_data') || isempty(param.block_data) + param.block_data = []; +end + +% block_data.block_along_track: scalar double, along-track length of each +% block in meters, default is 5000 m +if ~isfield(param.block_data,'block_along_track') || isempty(param.block_data.block_along_track) + param.block_data.block_along_track = 5e3; % Use default block size +end + +% block_data.block_Nx: scalar integer, number of columns in each output +% block, default is 256 +if ~isfield(param.block_data,'block_Nx') || isempty(param.block_data.block_Nx) + param.block_data.block_Nx = 256; % Use default block size +end + +% block_data.block_overlap: scalar double, percentage overlap for each +% block, default is 0.5, valid range is 0 to 1 +if ~isfield(param.block_data,'block_overlap') || isempty(param.block_data.block_overlap) + param.block_data.block_overlap = 0.5; +end +param.block_data.block_overlap = min(1,param.block_data.block_overlap); +param.block_data.block_overlap = max(0,param.block_data.block_overlap); + +% block_data.echo_img: scalar integer, image number, default is 0 (the +% combined image Data_YYYYMMDD_SS_FFF.mat). +if ~isfield(param.block_data,'echo_img') || isempty(param.block_data.echo_img) + param.block_data.echo_img = 0; % Use default echo_img +end + +% block_data.file: structure controlling file saving and which output files +% will be generated +if ~isfield(param.block_data,'file') || isempty(param.block_data.file) + param.block_data.file = []; +end +if ~isfield(param.block_data.file,'img_en') || isempty(param.block_data.file.img_en) + param.block_data.file.img_en = true; +end +if ~isfield(param.block_data.file,'layer_bin_en') || isempty(param.block_data.file.layer_bin_en) + param.block_data.file.layer_bin_en = true; +end +if ~isfield(param.block_data.file,'layer_mult_en') || isempty(param.block_data.file.layer_mult_en) + param.block_data.file.layer_mult_en = true; +end +if ~isfield(param.block_data.file,'layer_seg_en') || isempty(param.block_data.file.layer_seg_en) + param.block_data.file.layer_seg_en = true; +end +if ~isfield(param.block_data.file,'mat_en') || isempty(param.block_data.file.mat_en) + param.block_data.file.mat_en = true; +end + +% block_data.flatten: structure controlling echo_flatten.m +if ~isfield(param.block_data,'flatten') || isempty(param.block_data.flatten) + param.block_data.flatten = []; +end +if ~isfield(param.block_data.flatten,'resample_field') || isempty(param.block_data.flatten.resample_field) + param.block_data.flatten.resample_field = []; +end +if ~isfield(param.block_data.flatten,'interp_method') || isempty(param.block_data.flatten.interp_method) + param.block_data.flatten.interp_method = []; +end + +% Incoherent decimation (inc_dec, inc_B_filter) input check +% Setting inc_dec = 0: returns coherent data +% Setting inc_dec = 1: returns power detected data with no decimation +% Setting inc_dec > 1: decimates at the rate specified by inc_dec +if ~isfield(param.block_data,'inc_dec') || isempty(param.block_data.inc_dec) + param.block_data.inc_dec = 1; +end +if ~isfield(param.block_data,'inc_B_filter') || isempty(param.block_data.inc_B_filter) + if param.block_data.inc_dec == 0 || param.block_data.inc_dec == 1 + param.block_data.inc_B_filter = 1; + else + param.block_data.inc_B_filter = hanning(2*param.block_data.inc_dec+1); + end +end +if ~mod(length(param.block_data.inc_B_filter),2) + error('param.block_data.inc_B_filter must be odd length.'); +end +param.block_data.inc_B_filter = param.block_data.inc_B_filter(:).'; % Must be row vector +if abs(sum(param.block_data.inc_B_filter)-1) > 1e4*eps % Ensure filter weights sum to 1 to preserve radiometry + param.block_data.inc_B_filter = param.block_data.inc_B_filter / sum(param.block_data.inc_B_filter); +end + +% block_data.out_path: string specifying which output directory to put the +% block images in +if ~isfield(param.block_data,'out_path') || isempty(param.block_data.out_path) + param.block_data.out_path = 'block_data'; +end +out_fn_dir = ct_filename_out(param,param.block_data.out_path); + +% block_data.rows: structure controlling truncation in row dimension +if ~isfield(param.block_data,'rows') || isempty(param.block_data.rows) + param.block_data.rows = []; +end + +% block_data.rows.t0_pad: integer controlling how many rows of the image +% will be preserved beyond the top layer. Default is inf which will +% preserve all available rows on the top of the image. +if ~isfield(param.block_data.rows,'t0_pad') || isempty(param.block_data.rows.t0_pad) + param.block_data.rows.t0_pad = inf; +end + +% block_data.rows.t1_pad: integer controlling how many rows of the image +% will be preserved below the bottom layer. Default is inf which will +% preserve all available rows on the bottom of the image. +if ~isfield(param.block_data.rows,'t1_pad') || isempty(param.block_data.rows.t1_pad) + param.block_data.rows.t1_pad = inf; +end + +%% Create blocks +% ========================================================================= +dx = param.block_data.block_along_track*param.block_data.block_overlap; +X = param.block_data.block_along_track; +x0 = 0:dx:along_track(end)-X; +x1 = x0 + X; + +%% Determine which blocks to create +% ========================================================================= +block_mask = false(size(x0)); +for frm = param.cmd.frms + start_x = along_track(frames.frame_idxs(frm)); + if frm == length(frames.frame_idxs) + stop_x = along_track(end); + else + stop_x = along_track(frames.frame_idxs(frm+1)-1); + end + for block_idx = 1:length(x0) + if x0(block_idx) < stop_x && x1(block_idx) > start_x + block_mask(block_idx) = true; + end + end +end +% Add additional frames to start and end to account for blocks that extend +% before the first desired frame and past the last desired frame +frm_list = []; +for block_idx = find(block_mask) + for frm = 1:length(frames.frame_idxs) + start_x = along_track(frames.frame_idxs(frm)); + if frm == length(frames.frame_idxs) + stop_x = along_track(end); + else + stop_x = along_track(frames.frame_idxs(frm+1)-1); + end + if start_x < x1(block_idx) && stop_x > x0(block_idx) + frm_list(end+1) = frm; + end + end +end +frm_list = unique(frm_list); + +%% Load layers +% ========================================================================= +ops_param = param; +ops_param.cmd.frms = frm_list; +[layers,layer_params] = opsLoadLayers(ops_param, param.block_data.layer_params); + +%% Load surface layer +% ========================================================================= +ops_param = param; +ops_param.cmd.frms = frm_list; +[surf,surf_param] = opsLoadLayers(ops_param, param.block_data.surf_param); + +%% Block Loop +% ========================================================================= +frm_mask = false(size(frames.frame_idxs)); +echogram_fn_dir = ct_filename_out(param,param.block_data.echo_path); +mdata = {}; +for block_idx = find(block_mask) + + %% Block: Frames to load + % ========================================================================= + cat_data = []; + for frm = 1:length(frames.frame_idxs) + start_x = along_track(frames.frame_idxs(frm)); + if frm == length(frames.frame_idxs) + stop_x = along_track(end); + else + stop_x = along_track(frames.frame_idxs(frm+1)-1); + end + if start_x < x1(block_idx) && stop_x > x0(block_idx) + if ~frm_mask(frm) + frm_mask(frm) = true; + % Load frame into memory + if param.block_data.echo_img == 0 + echogram_fn_name = sprintf('Data_%s_%03d.mat', param.day_seg, frm); + else + echogram_fn_name = sprintf('Data_img_%02d_%s_%03d.mat', param.block_data.echo_img, param.day_seg, frm); + end + echogram_fn = fullfile(echogram_fn_dir,echogram_fn_name); + mdata{frm} = load_L1B(echogram_fn); + end + + % Concatenate data + cat_data = echo_concatenate(cat_data,mdata{frm}); + + else + if frm_mask(block_idx) + frm_mask(frm) = false; + % Remove frame from memory + mdata{frm} = []; + end + end + end + Nx_original = length(cat_data.GPS_time); + if ~isempty(param.block_data.surf_param) + cat_data.Surface = interp_finite(interp1(surf.gps_time,surf.twtt,cat_data.GPS_time),0); + end + + %% Block: Echogram layer flattening + % ======================================================================= + if isfield(param.block_data,'flatten') && ~isempty(param.block_data.flatten) + [cat_data.Data,resample_field] = echo_flatten(cat_data, ... + param.block_data.flatten.resample_field, false, ... + param.block_data.flatten.interp_method,[],true); + end + + %% Block: Incoherent filtering + % ======================================================================= + + % .inc_B_filter: double vector, FIR filter coefficients to apply before + % incoherent average decimation. If not defined or empty, then + % inc_B_filter is set to ones(1,inc_dec)/inc_dec. + % .inc_dec = integer scalar, number of incoherent averages to apply + % (also decimates by this number). If set to < 1, complex data are + % returned. Setting to 1 causes the data to be power detected (i.e. + % become incoherent), but no averaging is done. + param.block_data.inc_B_filter = ones(1,21)/21; + param.block_data.inc_dec = 10; + param.block_data.nan_dec_normalize_threshold = []; + param.block_data.nan_dec = false; + % Along-track incoherent filtering (multilooking) of data + if param.block_data.nan_dec + cat_data.Data = nan_fir_dec(abs(cat_data.Data).^2, param.block_data.inc_B_filter, ... + param.block_data.inc_dec, [], [], [], [], param.block_data.nan_dec_normalize_threshold); + else + cat_data.Data = fir_dec(abs(cat_data.Data).^2, param.block_data.inc_B_filter, ... + param.block_data.inc_dec); + end + % Account for filtering and decimation in remaining fields + cat_data.GPS_time = fir_dec(cat_data.GPS_time, param.block_data.inc_B_filter, ... + param.block_data.inc_dec); + cat_data.Latitude = fir_dec(cat_data.Latitude, param.block_data.inc_B_filter, ... + param.block_data.inc_dec); + cat_data.Longitude = fir_dec(cat_data.Longitude, param.block_data.inc_B_filter, ... + param.block_data.inc_dec); + cat_data.Elevation = fir_dec(cat_data.Elevation, param.block_data.inc_B_filter, ... + param.block_data.inc_dec); + cat_data.Roll = fir_dec(cat_data.Roll, param.block_data.inc_B_filter, ... + param.block_data.inc_dec); + cat_data.Pitch = fir_dec(cat_data.Pitch, param.block_data.inc_B_filter, ... + param.block_data.inc_dec); + cat_data.Heading = fir_dec(cat_data.Heading, param.block_data.inc_B_filter, ... + param.block_data.inc_dec); + cat_data.Surface = fir_dec(cat_data.Surface, param.block_data.inc_B_filter, ... + param.block_data.inc_dec); + resample_field = fir_dec(resample_field, param.block_data.inc_B_filter, ... + param.block_data.inc_dec); + + %% Block: Extract block + % ======================================================================= + start_rec = find(along_track >= x0(block_idx),1); + stop_rec = find(along_track <= x1(block_idx),1,'last'); + start_idx = find(cat_data.GPS_time >= records.gps_time(start_rec),1); + stop_idx = find(cat_data.GPS_time <= records.gps_time(stop_rec),1,'last'); + + dec_idxs = fir_dec(1:Nx_original, param.block_data.inc_B_filter, ... + param.block_data.inc_dec); + new_axis = linspace(dec_idxs(start_idx),dec_idxs(stop_idx),param.block_data.block_Nx); + + cat_data.Data = interp1(dec_idxs.',cat_data.Data.',new_axis.').'; + cat_data.Elevation = interp1(dec_idxs,cat_data.Elevation,new_axis); + cat_data.GPS_time = interp1(dec_idxs,cat_data.GPS_time,new_axis); + cat_data.Heading = interp1(dec_idxs,cat_data.Heading,new_axis); + cat_data.Latitude = interp1(dec_idxs,cat_data.Latitude,new_axis); + cat_data.Longitude = interp1(dec_idxs,cat_data.Longitude,new_axis); + cat_data.Roll = interp1(dec_idxs,cat_data.Roll,new_axis); + cat_data.Pitch = interp1(dec_idxs,cat_data.Pitch,new_axis); + cat_data.Surface = interp1(dec_idxs,cat_data.Surface,new_axis); + resample_field = interp1(dec_idxs.',resample_field.',new_axis).'; + + %% Block: Extract surface + % ======================================================================= + Nx = length(cat_data.GPS_time); + if isempty(param.block_data.surf_param) + surf_bin = interp1(cat_data.Time,1:length(cat_data.Time),cat_data.Surface); + else + % Loaded an update to the surface + cat_data.Surface = interp_finite(interp1(surf.gps_time,surf.twtt,cat_data.GPS_time),0); + twtt = interp1(surf.gps_time,surf.twtt,cat_data.GPS_time); + surf_bin = zeros(size(twtt)); + if isfield(param.block_data,'flatten') && ~isempty(param.block_data.flatten) + for rline = 1:Nx + % time_flat: vector of twtt associated with each pixel for the + % particular column + time_flat = interp1(1:length(cat_data.Time),cat_data.Time,resample_field(:,rline),'linear','extrap'); + surf_bin(rline) = interp1(time_flat,1:length(time_flat),twtt(rline),'linear','extrap'); + end + end + end + + %% Block: Extract layers + % ======================================================================= + layers_bin_bitmap = zeros(size(cat_data.Data),'uint8'); + layers_bitmap = zeros(size(cat_data.Data),'uint16'); + layers_segment_bitmap = zeros(size(cat_data.Data),'uint16'); + layers_vector = nan(length(layers),Nx); + for idx = 1:length(layers) + twtt = interp1(layers(idx).gps_time,layers(idx).twtt,cat_data.GPS_time); + twtt_bin = zeros(size(twtt)); + if isfield(param.block_data,'flatten') && ~isempty(param.block_data.flatten) + for rline = 1:Nx + % time_flat: vector of twtt associated with each pixel for the + % particular column + time_flat = interp1(1:length(cat_data.Time),cat_data.Time,resample_field(:,rline),'linear','extrap'); + twtt_bin(rline) = interp1(time_flat,1:length(time_flat),twtt(rline),'linear','extrap'); + end + else + twtt_bin = interp1(cat_data.Time, 1:length(cat_data.Time), twtt); + end + twtt_bin = round(twtt_bin); + + Nt = size(cat_data.Data,1); + good_idxs = find(isfinite(twtt_bin) & twtt_bin >= 1 & twtt_bin <= Nt); + layers_bin_bitmap(twtt_bin(good_idxs) + Nt*(good_idxs-1)) = 1; + layers_bitmap(twtt_bin(good_idxs) + Nt*(good_idxs-1)) = idx; + for col = 1:Nx + if isfinite(twtt_bin(col)) && twtt_bin(col) >= 1 && twtt_bin(col) <= Nt + layers_segment_bitmap(twtt_bin(col):end,col) = idx; + end + end + layers_vector(idx,good_idxs) = twtt_bin(good_idxs); + end + + min_layer = interp_finite(min(layers_vector),0) - param.block_data.rows.t0_pad; + max_layer = interp_finite(max(layers_vector),0) + param.block_data.rows.t1_pad; + min_layer = min(min_layer); + max_layer = max(max_layer); + min_layer = min(Nt,max(1,min_layer)); + max_layer = min(Nt,max(1,max_layer)); + + layers_bin_bitmap = layers_bin_bitmap(min_layer:max_layer,:); + layers_bitmap = layers_bitmap(min_layer:max_layer,:); + layers_segment_bitmap = layers_segment_bitmap(min_layer:max_layer,:); + layers_vector = layers_vector - min_layer + 1; + cat_data.Data = cat_data.Data(min_layer:max_layer,:); + resample_field = resample_field(min_layer:max_layer,:); + + %% Block: Log scaling data + % ======================================================================= + cat_data.Data = db(cat_data.Data); + + %% Block: Detrend + % ======================================================================= + if isfield(param.block_data,'detrend') && ~isempty(param.block_data.detrend) + param.block_data.detrend.layer_top = interp_finite(surf_bin,1); + param.block_data.detrend.layer_bottom = nan(size(surf_bin)); + cat_data.Data = echo_detrend(cat_data,param.block_data.detrend); + end + + %% Block: TBD (roll compensation, etc.) + % ======================================================================= + + %% Block: Normalization + % ======================================================================= + if isfield(param.block_data,'norm') && ~isempty(param.block_data.norm) + cat_data.Data = echo_norm(cat_data.Data,param.block_data.norm); + end + + %% Block: Save + % ======================================================================= + if ~exist(out_fn_dir,'dir') + mkdir(out_fn_dir); + end + + if param.block_data.file.img_en + out_fn_name = sprintf('img_%s_%04d.png',param.day_seg,block_idx); + out_fn = fullfile(out_fn_dir,out_fn_name); + fprintf('%s\t%s\n', out_fn, datestr(now,'yyyymmdd_HHMMSS')); + min_val = min(cat_data.Data(:)); + max_val = max(cat_data.Data(:)); + img_tmp = uint8((cat_data.Data-min_val)/(max_val-min_val)*255); + imwrite(uint8(img_tmp), out_fn); + clear img_tmp; + end + + if param.block_data.file.layer_bin_en + out_fn_name = sprintf('layer_bin_%s_%04d.png',param.day_seg,block_idx); + out_fn = fullfile(out_fn_dir,out_fn_name); + fprintf('%s\t%s\n', out_fn, datestr(now,'yyyymmdd_HHMMSS')); + imwrite(uint8(layers_bin_bitmap), out_fn); + end + + if param.block_data.file.layer_mult_en + out_fn_name = sprintf('layer_mult_%s_%04d.png',param.day_seg,block_idx); + out_fn = fullfile(out_fn_dir,out_fn_name); + fprintf('%s\t%s\n', out_fn, datestr(now,'yyyymmdd_HHMMSS')); + imwrite(uint8(layers_bitmap), out_fn); + end + + if param.block_data.file.layer_seg_en + out_fn_name = sprintf('layer_seg_%s_%04d.png',param.day_seg,block_idx); + out_fn = fullfile(out_fn_dir,out_fn_name); + fprintf('%s\t%s\n', out_fn, datestr(now,'yyyymmdd_HHMMSS')); + imwrite(uint8(layers_segment_bitmap), out_fn); + end + + if 0 + % Debug code to review images + % ===================================================================== + out_fn_name = sprintf('img_%s_%04d.png',param.day_seg,block_idx); + out_fn = fullfile(out_fn_dir,out_fn_name); + fprintf('%s\t%s\n', out_fn, datestr(now,'yyyymmdd_HHMMSS')); + img_test = imread(out_fn); + imagesc(img_test); + imshow(out_fn); + + out_fn_name = sprintf('layer_bin_%s_%04d.png',param.day_seg,block_idx); + out_fn = fullfile(out_fn_dir,out_fn_name); + fprintf('%s\t%s\n', out_fn, datestr(now,'yyyymmdd_HHMMSS')); + img_test = imread(out_fn); + imagesc(img_test); + + out_fn_name = sprintf('layer_mult_%s_%04d.png',param.day_seg,block_idx); + out_fn = fullfile(out_fn_dir,out_fn_name); + fprintf('%s\t%s\n', out_fn, datestr(now,'yyyymmdd_HHMMSS')); + img_test = imread(out_fn); + imagesc(img_test); + + out_fn_name = sprintf('layer_seg_%s_%04d.png',param.day_seg,block_idx); + out_fn = fullfile(out_fn_dir,out_fn_name); + fprintf('%s\t%s\n', out_fn, datestr(now,'yyyymmdd_HHMMSS')); + img_test = imread(out_fn); + imagesc(img_test); + end + + if ~param.block_data.file.mat_en + cat_data = rmfield(cat_data,'Data'); + end + out_fn_name = sprintf('%s_%04d.mat',param.day_seg,block_idx); + out_fn = fullfile(out_fn_dir,out_fn_name); + cat_data.param_block_data = param; + cat_data.layers_vector = layers_vector; + fprintf('%s\t%s\n', out_fn, datestr(now,'yyyymmdd_HHMMSS')); + ct_save(out_fn,'-struct','cat_data'); + +end +status = true; \ No newline at end of file diff --git a/cresis-toolbox/tracker/cluster_tmp_tuning.m b/cresis-toolbox/tracker/cluster_tmp_tuning.m new file mode 100644 index 00000000..d8eec3cc --- /dev/null +++ b/cresis-toolbox/tracker/cluster_tmp_tuning.m @@ -0,0 +1,173 @@ +%% Script run_cluster_LSM_tuning.m +% +% Runs the lsm 2D tracking algorithm +% on the desired dataset +% +% See also: LSM_tuning.m + +dbstack_info = dbstack; +fprintf('=====================================================================\n'); +fprintf('%s (%s)\n', dbstack_info(1).name, datestr(now,'HH:MM:SS')); +fprintf('=====================================================================\n'); + +%% General User Settings +% Tracking algorithms: lsm + +% params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls')); +% +% params = ct_set_params(params,'cmd.generic',0); +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20140313_08'); +% params = ct_set_params(params,'cmd.frms',[1]); + +% 23 segs +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20140313_07|20140313_08|20140313_10|20140325_07|20140331_02|20140401_04|20140405_01|20140409_02|20140410_01|20140412_01|20140412_02|20140412_04|20140415_04|20140415_05|20140416_01|20140421_01|20140501_01|20140506_01|20140507_01|20140520_04|20140520_05|20140521_01|20140508_01'); + +% %params = ct_set_params(params,'cmd.frms',[1]); +% for param_idx = 1:length(params) +% param = params(param_idx); +% if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) ... +% || ischar(param.cmd.generic) || ~param.cmd.generic +% continue; +% end +% end +% for param_idx = 1:length(params) +% param = params(param_idx); +% if isfield(param.cmd,'generic') && ~iscell(param.cmd.generic) && ~ischar(param.cmd.generic) && param.cmd.generic +% break; +% end +% end + +temp = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140516_01_20200416_181841_t032_lsm.mat'); +param= temp.param; +res_matrix_surface = zeros([length(param.cmd.frms),param.layer_tracker.track{1}.idx_reshape]); +res_matrix_bottom = zeros([length(param.cmd.frms),param.layer_tracker.track{1}.idx_reshape]); +layer_params = []; +idx = 0; +idx = idx + 1; +layer_params(idx).name = 'surface'; +layer_params(idx).source = 'layerdata'; +layer_params(idx).layerdata_source = 'layer'; +idx = idx+1; +layer_params(idx).name = 'bottom'; +layer_params(idx).source = 'layerdata'; +layer_params(idx).layerdata_source = 'layer'; +layers = opsLoadLayers(param,layer_params); +filename = '/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_layer_tracker_tmp/CSARP_layer_tune/20140516_01/'; +save_name = '/cresis/snfs1/scratch/anjali/cluster_tuning/'; +num_gt_isfinite_surf = 0; +num_isnan_surf = 0; +num_points_surf = 0; + +num_gt_isfinite_bot = 0; +num_isnan_bot = 0; +num_points_bot = 0; + +for track_idx = 1:3 + frm_idx = 1; + for frm = [40 41] + data_fn_dir = ct_filename_out(param, param.layer_tracker.echogram_source, ''); + data_fn_name = sprintf('Data_%s_%03d.mat',param.day_seg,frm); + data_fn = fullfile(data_fn_dir, data_fn_name); + data = load(data_fn); + frm_dir = sprintf('layer_tracker_%03d',frm); + surf = interp1(layers(1).gps_time,layers(1).twtt,data.GPS_time); + surf_bins = round(interp1(data.Time,1:length(data.Time),surf)); + bot = interp1(layers(2).gps_time,layers(2).twtt,data.GPS_time); + bot_bins = round(interp1(data.Time,1:length(data.Time),bot)); + dt = data.Time(2) - data.Time(1); + switch param.layer_tracker.track{track_idx}.method + case 'lsm' + + fname = sprintf('%s_%s.mat',param.layer_tracker.track{track_idx}.name,param.layer_tracker.track{track_idx}.method); + frame_fn_name = fullfile(filename,frm_dir,fname); + track_data = load(frame_fn_name); + for i = 1:16 + try + surf = interp1(track_data.gps_time,track_data.twtt(i,:),data.GPS_time); + catch ME + surf = interp_finite(interp1(track_data.gps_time,track_data.twtt(i,:),data.GPS_time),NaN); + end + surf_bins_itr = round(interp1(data.Time,1:length(data.Time),surf)); + num_gt_isfinite = sum(isfinite(surf_bins)); + num_gt_isfinite_surf = num_gt_isfinite_surf + num_gt_isfinite; + num_isnan = sum(isfinite(surf_bins) & ~isfinite(surf_bins_itr)); + num_isnan_surf = num_isnan_surf + num_isnan; + num_points = sum(abs(surf_bins-surf_bins_itr) < 5*dt); + num_points_surf = num_points_surf + num_points; +% for temp = 1:length(surf) +% if(isfinite(surf(temp)) && isnan(surf_bins_itr(temp))) +% num_isnan = num_isnan + 1; +% end +% if (abs(surf(temp)-surf_bins_itr(temp)) < 5*dt) +% num_points = num_points+1; +% end +% end + res_s.(sprintf('%s_%s',param.layer_tracker.track{track_idx}.method,param.layer_tracker.track{track_idx}.name)).res(frm_idx,param.layer_tracker.track{track_idx}.idx(i)) = nanmean(abs(surf_bins - surf_bins_itr)); + res_s.(sprintf('%s_%s',param.layer_tracker.track{track_idx}.method,param.layer_tracker.track{track_idx}.name)).num_gt_isfinite(frm_idx) = num_gt_isfinite; + res_s.(sprintf('%s_%s',param.layer_tracker.track{track_idx}.method,param.layer_tracker.track{track_idx}.name)).num_isnan(frm_idx) = num_isnan; + res_s.(sprintf('%s_%s',param.layer_tracker.track{track_idx}.method,param.layer_tracker.track{track_idx}.name)).num_points(frm_idx) = num_points; + end + + pos = 1; + for i = 17:32 + try + bot = interp1(track_data.gps_time,track_data.twtt(i,:),data.GPS_time); + catch ME + bot = interp_finite(interp1(track_data.gps_time,track_data.twtt(i,:),data.GPS_time),NaN); + end + bot_bins_itr = round(interp1(data.Time,1:length(data.Time),bot)); + num_gt_isfinite = sum(isfinite(bot_bins)); + num_gt_isfinite_bot = num_gt_isfinite_bot + num_gt_isfinite; + num_isnan = sum(isfinite(bot_bins) & ~isfinite(bot_bins_itr)); + num_isnan_bot = num_isnan_bot + num_isnan; + num_points = sum(abs(bot_bins-bot_bins_itr) < 5*dt); + num_points_bot = num_points_bot + num_points; +% for temp = 1:length(bot) +% if(isfinite(bot(temp)) && isnan(bot_bins_itr(temp))) +% num_isnan = num_isnan + 1; +% end +% if (abs(bot(temp)-bot_bins_itr(temp)) < 5*dt) +% num_points = num_points+1; +% end + + res_b.(sprintf('%s_%s',param.layer_tracker.track{track_idx}.method,param.layer_tracker.track{track_idx}.name)).res(frm_idx,param.layer_tracker.track{track_idx}.idx(i)) = nanmean(abs(bot_bins - bot_bins_itr)); + + res_b.(sprintf('%s_%s',param.layer_tracker.track{track_idx}.method,param.layer_tracker.track{track_idx}.name)).num_gt_isfinite(frm_idx) = num_gt_isfinite; + res_b.(sprintf('%s_%s',param.layer_tracker.track{track_idx}.method,param.layer_tracker.track{track_idx}.name)).num_isnan(frm_idx) = num_isnan; + res_b.(sprintf('%s_%s',param.layer_tracker.track{track_idx}.method,param.layer_tracker.track{track_idx}.name)).num_points(frm_idx) = num_points; + pos = pos+1; + end + case 'viterbi' + idx = 1; + layer_params_bot(idx).name = sprintf('%s_%s_bottom',param.layer_tracker.track{track_idx}.name,param.layer_tracker.track{track_idx}.method); + layer_params_bot(idx).source = param.layer_tracker.layer_params.source; + layer_params_bot(idx).layerdata_source = param.layer_tracker.layer_params.layerdata_source; + + case {'mcmc','stereo'} + idx = 1; + layer_params_surf(idx).name = sprintf('%s_%s_surface',param.layer_tracker.track{track_idx}.name,param.layer_tracker.track{track_idx}.method); + layer_params_surf(idx).source = param.layer_tracker.layer_params.source; + layer_params_surf(idx).layerdata_source = param.layer_tracker.layer_params.layerdata_source; + idx = idx + 1; + layer_params_bot(idx).name = sprintf('%s_%s_bottom',param.layer_tracker.track{track_idx}.name,param.layer_tracker.track{track_idx}.method); + layer_params_bot(idx).source = param.layer_tracker.layer_params.source; + layer_params_bot(idx).layerdata_source = param.layer_tracker.layer_params.layerdata_source; + + otherwise + idx = 1; + layer_params_surf(idx).name = sprintf('%s_%s_surface',param.layer_tracker.track{track_idx}.name,param.layer_tracker.track{track_idx}.method); + layer_params_surf(idx).source = param.layer_tracker.layer_params.source; + layer_params_surf(idx).layerdata_source = param.layer_tracker.layer_params.layerdata_source; + end + ctr = ctr+1; + end +% res_matrix_surface = res_matrix_surface + res_s.(sprintf('%s_%s',param.layer_tracker.track{track_idx}.method,param.layer_tracker.track{track_idx}.name)).res; +% res_matrix_bottom = res_matrix_bottom + res_b.(sprintf('%s_%s',param.layer_tracker.track{track_idx}.method,param.layer_tracker.track{track_idx}.name)).res; +end +[min_val_surf,min_idx1] = min(res_matrix_surface,[],1); +[min_val_surf,min_idx2] = min(min_val_surf,[],2); +min_idx1 = min_idx1(min_idx2); + +fprintf('hi'); + + diff --git a/cresis-toolbox/tracker/koenig_mat_loader.m b/cresis-toolbox/tracker/koenig_mat_loader.m index b30d151d..6651a365 100644 --- a/cresis-toolbox/tracker/koenig_mat_loader.m +++ b/cresis-toolbox/tracker/koenig_mat_loader.m @@ -7,13 +7,15 @@ %% User Settings % Lora Koenig -% base_dir = '/cresis/snfs1/dataproducts/public/data/temp/internal_layers/NASA_OIB_test_files/image_files/greenland_picks_final_2009-2012_20140602'; -% out_dir = '/cresis/snfs1/dataproducts/public/data/temp/internal_layers/NASA_OIB_test_files/image_files/greenland_picks_final_2009_2012_reformat/'; -% year_dir_type = 1; +base_dir = '/cresis/snfs1/dataproducts/public/data/temp/internal_layers/NASA_OIB_test_files/image_files/greenland_picks_final_2009-2012_20140602'; +out_dir = '/cresis/snfs1/dataproducts/public/data/temp/internal_layers/NASA_OIB_test_files/image_files/greenland_picks_final_2009_2012_reformat/'; +year_dir_type = 1; + % Lynn Montgomery -base_dir = '/cresis/snfs1/dataproducts/public/data/temp/internal_layers/NASA_OIB_test_files/image_files/Corrected_SEGL_picks_lnm_2009_2017'; -out_dir = '/cresis/snfs1/dataproducts/public/data/temp/internal_layers/NASA_OIB_test_files/image_files/Corrected_SEGL_picks_lnm_2009_2017_reformat/'; -year_dir_type = 2; +% base_dir = '/cresis/snfs1/dataproducts/public/data/temp/internal_layers/NASA_OIB_test_files/image_files/Corrected_SEGL_picks_lnm_2009_2017'; +% out_dir = '/cresis/snfs1/dataproducts/public/data/temp/internal_layers/NASA_OIB_test_files/image_files/Corrected_SEGL_picks_lnm_2009_2017_reformat/'; +% year_dir_type = 2; + block_size = 256; block_overlap = 6; debug_plot = false; @@ -29,6 +31,7 @@ % fns = get_filenames(base_dir, 'layers_','dec','.mat',struct('recursive',true)); old_day_seg = ''; +total_along_track = 0; % Keep track of how many line-km are processed for fn_idx = 1:length(fns) fn = fns{fn_idx}; [fn_dir,fn_name] = fileparts(fn); @@ -37,6 +40,10 @@ tmp = load(fn); + % Keep track of how many line-km are processed + along_track = geodetic_to_along_track(tmp.lat, tmp.lon); + total_along_track = total_track + along_track(end); + if debug_plot % Plot whole file figure(1); clf; diff --git a/cresis-toolbox/tracker/koenig_snow_layer_ingest.m b/cresis-toolbox/tracker/koenig_snow_layer_ingest.m index 0fb02bc6..b8c4fa9d 100644 --- a/cresis-toolbox/tracker/koenig_snow_layer_ingest.m +++ b/cresis-toolbox/tracker/koenig_snow_layer_ingest.m @@ -30,7 +30,7 @@ fns = get_filenames('/cresis/snfs1/dataproducts/metadata/koenig_snow_layers/output_mapdata_2012','mapdata','','.txt'); end -%% +%% Load Koenig Data physical_constants; surf_filter_len = 51; % Range lines to average surf twtt (must be odd integer) num_layers = 30; @@ -73,7 +73,7 @@ end end -% Get unique days +%% Get unique days days = cell(size(layer_data{1})); for idx = 1:length(layer_data{1}) if length(layer_data{1}{idx}) < 15 @@ -87,7 +87,7 @@ [days,~,map_idxs] = unique(days); map_idxs = map_idxs(1:end-1); -% Extract other fields +%% Extract other fields trace_idx = {}; lat = {}; lon = {}; @@ -133,7 +133,12 @@ end end -%% +%% Process Each Day +enable_visible_plot = false; +enable_debug_plot = false; +if enable_debug_plot + h_fig = get_figures(2,enable_visible_plot); +end params = read_param_xls(param_fn); for day_idx=13:13 %1:length(days) @@ -143,6 +148,7 @@ param_idxs = strmatch(days{day_idx},{params.day_seg}); + %% Process: Process Each Segment for param_idx = param_idxs(:).' param = params(param_idx); @@ -155,13 +161,14 @@ layer_params.source = 'layerdata'; end surf = opsLoadLayers(param,layer_params); + surf.twtt = interp_finite(surf.twtt); - % Filter surf data + %% Process:Segment Filter Surf Data surf.twtt_filtered = surf.twtt - surf.elev/(c/2); surf.twtt_filtered = fir_dec(surf.twtt_filtered,ones(1,surf_filter_len)/surf_filter_len,1); surf.twtt_filtered = surf.twtt_filtered + surf.elev/(c/2); - % Align layers to layerData + %% Process:Segment Sync Layers master = []; master.GPS_time = surf.gps_time; master.Latitude = surf.lat; @@ -187,83 +194,85 @@ master.layerData{lay_idx}.value{1}.data(surf.type == 1) ... = surf.twtt(surf.type==1); % Auto points - master.layerData{lay_idx}.value{2}.data = surf.twtt; + master.layerData{lay_idx}.value{2}.data = surf.twtt_filtered; master.layerData{lay_idx}.name = 'surface'; master.layerData{lay_idx}.quality = ones(size(master.GPS_time)); - % Ice bottom (all NaN) - master.layerData{lay_idx+1}.value{1}.data = nan(size(master.GPS_time)); - master.layerData{lay_idx+1}.value{2}.data = nan(size(master.GPS_time)); - master.layerData{lay_idx+1}.name = 'bottom'; - master.layerData{lay_idx+1}.quality = ones(size(master.GPS_time)); end if lay_idx > 1 - master.layerData{lay_idx+1}.value{1}.data = nan(size(master.GPS_time)); - master.layerData{lay_idx+1}.value{2}.data = surf.twtt_filtered + new_layer_twtt - surf_ref; - master.layerData{lay_idx+1}.name = sprintf('Koenig_%d', lay_idx); - master.layerData{lay_idx+1}.quality = ones(size(master.GPS_time)); + master.layerData{lay_idx}.value{1}.data = nan(size(master.GPS_time)); + master.layerData{lay_idx}.value{2}.data = surf.twtt_filtered + new_layer_twtt - surf_ref; + master.layerData{lay_idx}.name = sprintf('Koenig_%d', lay_idx); + master.layerData{lay_idx}.quality = ones(size(master.GPS_time)); end end - % Load records file associated with this segment - records_fn = ct_filename_support(param,'','records'); - records = load(records_fn); - % Load frames associated with this segment - frames_fn = ct_filename_support(param,'','frames'); - load(frames_fn); + %% Process:Segment Save Output + copy_param = []; + copy_param.copy_method = 'overwrite'; + copy_param.gaps_fill.method = 'preserve_gaps'; + copy_param.layer_source.existence_check = false; + copy_param.layer_source.source = 'custom'; + copy_param.layer_dest.source = 'layerdata'; + copy_param.layer_dest.layerdata_source = 'layer_koenig'; + copy_param.layer_dest.existence_check = false; - % Write layer information into layerData files - % update opsCopyLayers to create new layerData files if they do not exist - for frm = 1:length(frames.frame_idxs) - first_rec = frames.frame_idxs(frm); - if frm == length(frames.frame_idxs) - last_rec = length(records.gps_time); - else - last_rec = frames.frame_idxs(frm+1)-1; + copy_param.layer_source.gps_time = {}; + copy_param.layer_source.quality = {}; + copy_param.layer_source.twtt = {}; + copy_param.layer_source.type = {}; + copy_param.layer_dest.name = {}; + layer_str = ''; + deepest_layer_with_points = NaN; + if enable_debug_plot + clf(h_fig(1)); + h_axes(1) = axes('parent',h_fig(1)); + clf(h_fig(2)); + h_axes(2) = axes('parent',h_fig(2)); + end + for lay_idx = 1:num_layers-1 + copy_param.layer_source.gps_time{end+1} = surf.gps_time; + copy_param.layer_source.twtt{end+1} = master.layerData{lay_idx}.value{2}.data; + if any(~isnan(master.layerData{lay_idx}.value{2}.data)) + deepest_layer_with_points = lay_idx; + layer_str = [layer_str sprintf(' %d', lay_idx)]; end - frm_mask = master.GPS_time >= records.gps_time(first_rec) ... - & master.GPS_time <= records.gps_time(last_rec); - lay = []; - lay.GPS_time = master.GPS_time(frm_mask); - lay.Latitude = master.Latitude(frm_mask); - lay.Longitude = master.Longitude(frm_mask); - lay.Elevation = master.Elevation(frm_mask); - deepest_layer_with_points = 1; - for lay_idx = 1:num_layers - if lay_idx == 1 - % Copy surface and bottom - offsets = [0 1]; - else - % Copy snow radar layer - offsets = 1; - end - for o = offsets - % 1. Manual points - lay.layerData{lay_idx+o}.value{1}.data ... - = master.layerData{lay_idx+o}.value{1}.data(frm_mask); - % 2. Auto points - lay.layerData{lay_idx+o}.value{2}.data ... - = master.layerData{lay_idx+o}.value{2}.data(frm_mask); - % 3. Name - lay.layerData{lay_idx+o}.name = master.layerData{lay_idx+o}.name; - % 4. Quality - lay.layerData{lay_idx+o}.quality ... - = master.layerData{lay_idx+o}.quality(frm_mask); - end - if any(~isnan(lay.layerData{lay_idx+o}.value{2}.data)) - deepest_layer_with_points = lay_idx; - end + if lay_idx == 1 + copy_param.layer_source.quality{end+1} = surf.quality; + copy_param.layer_source.type{end+1} = surf.type; + copy_param.layer_dest.name{end+1} = sprintf('surface', lay_idx); + else + copy_param.layer_source.quality{end+1} = ones(size(surf.quality)); + copy_param.layer_source.type{end+1} = 2*ones(size(surf.type)); + copy_param.layer_dest.name{end+1} = sprintf('Koenig_%d', lay_idx); end - layer_fn_dir = ct_filename_out(param,'layerData',''); - if ~exist(layer_fn_dir,'dir') - mkdir(layer_fn_dir); + if enable_debug_plot + plot(copy_param.layer_source.gps_time{end}, copy_param.layer_source.twtt{end}, 'parent', h_axes(1)); + hold(h_axes(1), 'on'); + xlabel(h_axes(1), 'GPS time (ANSI-C sec since Jan 1, 1970)'); + ylabel(h_axes(1), 'Two way travel time (sec)'); + + plot(copy_param.layer_source.gps_time{end}, surf.elev - copy_param.layer_source.twtt{end}*c/2, 'parent', h_axes(2)); + hold(h_axes(2), 'on'); + xlabel(h_axes(2), 'GPS time (ANSI-C sec since Jan 1, 1970)'); + ylabel(h_axes(2), 'Elevation (m, WGS-84)'); end - layer_fn = fullfile(layer_fn_dir,sprintf('Data_%s_%03d.mat', param.day_seg, frm)); - save(layer_fn,'-v7.3','-struct','lay'); - if deepest_layer_with_points > 1 - fprintf('%d: %s\n', deepest_layer_with_points, layer_fn); + end + + fprintf('opsCopyLayers %s (%s)\n', param.day_seg, datestr(now)); + fprintf(' Layers:%s\n', layer_str); + fprintf(' Deepest\t%d\n', deepest_layer_with_points); + + if enable_visible_plot + % Bring plots to front + for h_fig_idx = 1:length(h_fig) + figure(h_fig(h_fig_idx)); end + % Enter debug mode + keyboard end + opsCopyLayers(param,copy_param); + end end diff --git a/cresis-toolbox/tracker/layer_tracker.m b/cresis-toolbox/tracker/layer_tracker.m new file mode 100644 index 00000000..870d7b45 --- /dev/null +++ b/cresis-toolbox/tracker/layer_tracker.m @@ -0,0 +1,490 @@ +function [ctrl_chain,param] = layer_tracker(param,param_override) +% [ctrl_chain,param] = layer_tracker(param,param_override) +% +% Check input parameters and create tracking tasks for running on a cluster +% with layer_tracker. See run_layer_tracker for an example of routine +% tracking of echograms. See run_layer_tracker_tune for an example of +% hyperparameter tuning to improve tracking parameters. The function +% "layer_tracker_task" does the actual tracking and +% "layer_tracker_combine_task" combines the tracking results and stores them +% into standard layer storage locations (either the OPS database or +% layerdata files). +% +% First stage temporary outputs stored in: +% /cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_layer_tracker_tmp/CSARP_layer/20140313_08/ +% Second stage output stored in any format supported by opsCopyLayers and +% input parameters control where the final combiend output goes. +% +% Comparing four different methods for example might store the outputs like this: +% layer_tracker_001/t001_lsm.mat, ..., layer_tracker_00N/t001_lsm.mat +% layer_tracker_001/t002_mcmc.mat, ..., layer_tracker_00N/t002_mcmc.mat +% layer_tracker_001/t003_stereo.mat, ..., layer_tracker_00N/t003_stereo.mat +% layer_tracker_001/t004_viterbi.mat, ..., layer_tracker_00N/t004_viterbi.mat +% Layers in the files (all combined into one file during combine): +% t001_lsm_surface_001, ..., t001_lsm_surface_016, t001_lsm_bottom_001, ..., t001_lsm_bottom_016 +% t002_mcmc_surface, t002_mcmc_bottom +% t003_stereo_surface, t003_stereo_bottom +% t004_viterbi_bottom +% +% Comparing the same method with four different sets of parameters: +% layer_tracker_001/t001_lsm.mat, ..., layer_tracker_00N/t001_lsm.mat +% layer_tracker_001/t002_lsm.mat, ..., layer_tracker_00N/t001_lsm.mat +% layer_tracker_001/t003_lsm.mat, ..., layer_tracker_00N/t001_lsm.mat +% layer_tracker_001/t004_lsm.mat, ..., layer_tracker_00N/t001_lsm.mat +% Layers in the files (all combined into one file during combine): +% t001_lsm_surface_001, ..., t001_lsm_surface_016, t001_lsm_bottom_001, ..., t001_lsm_bottom_016 +% t002_lsm_surface_001, ..., t002_lsm_surface_016, t002_lsm_bottom_001, ..., t002_lsm_bottom_016 +% t003_lsm_surface_001, ..., t003_lsm_surface_016, t003_lsm_bottom_001, ..., t003_lsm_bottom_016 +% t004_lsm_surface_001, ..., t004_lsm_surface_016, t004_lsm_bottom_001, ..., t004_lsm_bottom_016 +% +% Authors: Anjali Pare, John Paden +% +% See also: layer_tracker.m, layer_tracker_combine_task.m, +% layer_tracker_task.m, layer_tracker_input_check.m, +% layer_tracker_profile.m, run_layer_tracker.m, run_layer_tracker_tune.m + +%% General Setup +% ===================================================================== +param = merge_structs(param, param_override); + +fprintf('=====================================================================\n'); +fprintf('%s: %s (%s)\n', mfilename, param.day_seg, datestr(now)); +fprintf('=====================================================================\n'); + +%% Input Checks: cmd +% ===================================================================== + +% Remove frames that do not exist from param.cmd.frms list +frames = frames_load(param); +param.cmd.frms = frames_param_cmd_frms(param,frames); + +%% Input Checks: layer_tracker +% ===================================================================== + +layer_tracker_input_check; + +%% layer_tracker --------------------------------- +% =================================================================== +% =================================================================== + +%% layer_tracker: Setup Cluster +% =================================================================== + +ctrl = cluster_new_batch(param); +%cluster_compile({'layer_tracker_task'},ctrl.cluster.hidden_depend_funs,ctrl.cluster.force_compile,ctrl); +cluster_compile({'layer_tracker_task','layer_tracker_combine_task'},ctrl.cluster.hidden_depend_funs,ctrl.cluster.force_compile,ctrl); +ctrl_chain = {}; + +%% layer_tracker: Setup static parameters +sparam.argsin{1} = param; +sparam.task_function = 'layer_tracker_task'; +sparam.num_args_out = 1; +sparam.argsin{1}.load.echogram_img = param.layer_tracker.echogram_img; +sparam.cpu_time = 60; +sparam.mem = 500e6; +sparam.notes = ''; + +%% layer_tracker: Get all the frames to load +if param.layer_tracker.overlap > 0 + % If overlap with neighboring frames is requested, then grab frames one + % before and after all those that are requested. + frm_list = unique([param.cmd.frms(:).'-1, param.cmd.frms(:).', param.cmd.frms(:).'+1]); +else + frm_list = param.cmd.frms; +end + +%% layer_tracker: Setup cpu_time and mem cluster parameters +cpu_time_mult = zeros(size(param.layer_tracker.track)); +mem_mult = zeros(size(param.layer_tracker.track)); +for track_idx = 1:length(param.layer_tracker.track) + switch param.layer_tracker.track{track_idx}.method + case 'viterbi' + cpu_time_mult(track_idx) = 11e-6; + mem_mult(track_idx) = 64; + + case 'lsm' + cpu_time_mult(track_idx) = 5.5e-7*max(param.layer_tracker.track{track_idx}.lsm.storeIter); + mem_mult(track_idx) = 80; + + otherwise + cpu_time_mult(track_idx) = 11e-6; + mem_mult(track_idx) = 64; + end +end + +%% layer_tracker task loop +% ------------------------------------------------------------------------- +if strcmp(param.layer_tracker.layer_params.source,'ops') + tmp_out_fn_dir_dir = ct_filename_out(param,'ops','layer_tracker_tmp'); + param.layer_tracker.layer_params.layerdata_source = 'ops'; % Only used for stdout +else + tmp_out_fn_dir_dir = ct_filename_out(param,param.layer_tracker.layer_params.layerdata_source,'layer_tracker_tmp'); %/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_layer_tracker_tmp/CSARP_layer_tune_vit_seg4_NC/20140313_09 +end +mem_combine = 0; +cputime_combine = 0; +frm_idx = 1; + +gps_time = []; +lat = []; +lon = []; +frm_vector = []; +while frm_idx <= length(param.cmd.frms) + Nx = 0; + + %% layer_tracker task loop: Loop through block of frames + start_frm_idx = frm_idx; + frms = []; + subblock_idx = 1; + while subblock_idx <= param.layer_tracker.block_size_frms + if frm_idx > param.cmd.frms + break; + end + frm = param.cmd.frms(frm_idx); + % Check proc_mode from frames file that contains this frames type and + % make sure the user has specified to process this frame type + if ~ct_proc_frame(frames.proc_mode(frm),param.layer_tracker.frm_types) + fprintf('Skipping %s_%03i (no process frame)\n', param.day_seg, frm); + if subblock_idx == 1 + % No frames added to the block yet, so just keep going + frm_idx = frm_idx + 1; + continue; + else + break; + end + end + % Add frame to this block + frm_idx = frm_idx + 1; + frms(end+1) = frm; + + % Compute matrix size + % --------------------------------------------------------------------- + echo_fn = echo_filename(param,param.layer_tracker.echogram_source,frm,param.layer_tracker.echogram_img); + try + mdata = load(echo_fn, 'GPS_time','Time','Latitude','Longitude'); + if (subblock_idx==1) + max_time = mdata.Time(end); + min_time = mdata.Time(1); + else + if(max_time <= mdata.Time(end)) + max_time = mdata.Time(end); + end + if(min_time >= mdata.Time(1)) + min_time = mdata.Time(1); + end + end + Nx_frm = length(mdata.GPS_time); + Nx = Nx + Nx_frm; + + gps_time(end+(1:Nx_frm)) = mdata.GPS_time; + lat(end+(1:Nx_frm)) = mdata.Latitude; + lon(end+(1:Nx_frm)) = mdata.Longitude; + frm_vector(end+(1:Nx_frm)) = frm; + + catch ME + error('Failed to load echogram %s:\n %s', echo_fn, ME.getReport); + %mdata.Time = [0 1e-6]; + %min_time = 0; + %max_time = 0; + % keyboard % Uncomment for debugging why file loading failed + end + subblock_idx = subblock_idx + 1; + end + + % If tracking with frame overlap, then try loading frame that precedes and + % frame that follows as well. + if param.layer_tracker.overlap > 0 + if param.cmd.frms(start_frm_idx) > 1 + pre_frm = param.cmd.frms(start_frm_idx)-1; + echo_fn = echo_filename(param,param.layer_tracker.echogram_source,pre_frm,param.layer_tracker.echogram_img); + if exist(echo_fn,'file') + mdata = load(echo_fn, 'GPS_time','Time','Latitude','Longitude'); + if (subblock_idx==1) + max_time = mdata.Time(end); + min_time = mdata.Time(1); + else + if(max_time <= mdata.Time(end)) + max_time = mdata.Time(end); + end + if(min_time >= mdata.Time(1)) + min_time = mdata.Time(1); + end + end + Nx_frm = length(mdata.GPS_time); + + gps_time(end+(1:Nx_frm)) = mdata.GPS_time; + lat(end+(1:Nx_frm)) = mdata.Latitude; + lon(end+(1:Nx_frm)) = mdata.Longitude; + frm_vector(end+(1:Nx_frm)) = pre_frm; + end + end + + if frm < length(frames.frame_idxs) + post_frm = frm+1; + echo_fn = echo_filename(param,param.layer_tracker.echogram_source,post_frm,param.layer_tracker.echogram_img); + if exist(echo_fn,'file') + mdata = load(echo_fn, 'GPS_time','Time','Latitude','Longitude'); + if (subblock_idx==1) + max_time = mdata.Time(end); + min_time = mdata.Time(1); + else + if(max_time <= mdata.Time(end)) + max_time = mdata.Time(end); + end + if(min_time >= mdata.Time(1)) + min_time = mdata.Time(1); + end + end + Nx_frm = length(mdata.GPS_time); + + gps_time(end+(1:Nx_frm)) = mdata.GPS_time; + lat(end+(1:Nx_frm)) = mdata.Latitude; + lon(end+(1:Nx_frm)) = mdata.Longitude; + frm_vector(end+(1:Nx_frm)) = pre_frm; + end + end + + end + dt = mdata.Time(2) - mdata.Time(1); + Nt = 1 + (max_time-min_time)/dt; + + %% layer_tracker task loop: Loop through track structures + for track_idx = 1:param.layer_tracker.track_per_task:length(param.layer_tracker.track) + dparam = []; + dparam.file_success = {}; + dparam.argsin{1}.layer_tracker.frms = frms; + + tracks_in_task = track_idx:min(track_idx-1+param.layer_tracker.track_per_task,length(param.layer_tracker.track)); + + dparam.argsin{1}.layer_tracker.tracks_in_task = tracks_in_task; + + % File Success + % --------------------------------------------------------------------- + for track_idx = tracks_in_task + tmp_out_fn_name = sprintf('%s_%s.mat', param.layer_tracker.track{track_idx}.name, param.layer_tracker.track{track_idx}.method); + for frm = frms + tmp_out_fn = fullfile(tmp_out_fn_dir_dir,sprintf('layer_tracker_%03d', frm),tmp_out_fn_name); + dparam.file_success{end+1} = tmp_out_fn; + if ~ctrl.cluster.rerun_only && exist(tmp_out_fn,'file') + delete(tmp_out_fn); + end + end + end + + % Notes + % --------------------------------------------------------------------- + dparam.notes = sprintf('%s %s:%s:%s %s %s:%d-%d %s %d-%d (%d of %d)', ... + sparam.task_function, param.radar_name, param.season_name, ... + param.layer_tracker.echogram_source, param.layer_tracker.layer_params.layerdata_source, ... + param.layer_tracker.track{tracks_in_task(1)}.method, tracks_in_task([1 end]), param.day_seg, ... + dparam.argsin{1}.layer_tracker.frms([1 end]), start_frm_idx, length(param.cmd.frms)); + + % Rerun only check + % --------------------------------------------------------------------- + if ctrl.cluster.rerun_only + if ~cluster_file_success(dparam.file_success) + fprintf(' Already exists [rerun_only skipping]: %s (%s)\n', ... + dparam.notes, datestr(now)); + continue; + end + end + + % CPU time and memory + % --------------------------------------------------------------------- + dparam.cpu_time = sum(cpu_time_mult(tracks_in_task)) * Nx * Nt; + dparam.mem = 800e6 + max(mem_mult(tracks_in_task)) * Nx * Nt; + if strcmp(track.init.method,'dem') + % Add extra time and memory for DEM + dparam.cpu_time = dparam.cpu_time + 120; + dparam.mem = dparam.mem + 2e9; + end + mem_combine = mem_combine + 256*Nx*length(tracks_in_task); + cputime_combine = cputime_combine + 1e-2*Nx*length(tracks_in_task); + + % Create task + % --------------------------------------------------------------------- + ctrl = cluster_new_task(ctrl,sparam,dparam,'dparam_save',0); + end +end + +%% layer_tracker: Determine if crossovers, dem, and ice mask need to be loaded +crossover_en = false; +dem_en = false; +ice_mask_en = false; +for track_idx = 1:length(param.layer_tracker.track) + track = param.layer_tracker.track{track_idx}; + if track.crossover.en + crossover_en = true; + end + if isfield(track,'init') && strcmpi(track.init.method,'dem') + dem_en = true; + end + if track.ice_mask.en + ice_mask_en = true; + end +end +if param.layer_tracker.overlap > 0 + % Resort/unique since frames could have been added out of order and/or + % added multiple times because of overlapping frames + [frm_vector,unique_idxs] = unique(frm_vector); + gps_time = gps_time(unique_idxs); + lat = lat(unique_idxs); + lon = lon(unique_idxs); +end + +if crossover_en || dem_en || ice_mask_en + sparam.argsin{1}.layer_tracker.frm_vector = frm_vector; + sparam.argsin{1}.layer_tracker.gps_time = gps_time; + sparam.argsin{1}.layer_tracker.lat = lat; + sparam.argsin{1}.layer_tracker.lon = lon; +end + +%% layer_tracker: Load crossovers +if crossover_en + sys = ct_output_dir(param.radar_name); + + % Get the segment ID + ops_param = []; + ops_param.properties.search_str = param.day_seg; + ops_param.properties.location = param.post.ops.location; + ops_param.properties.season = param.season_name; + [status,ops_data_segment] = opsGetFrameSearch(sys,ops_param); + + % Get crossovers + ops_param = []; + ops_param.properties.location = param.post.ops.location; + ops_param.properties.lyr_name = track.crossover.name; + ops_param.properties.frame = {}; + ops_param.properties.segment_id = {}; + for frm = frm_list + ops_param.properties.frame{end+1} = [param.day_seg '_' sprintf('%03d',frm)]; + ops_param.properties.segment_id{end+1} = ops_data_segment.properties.segment_id; + end + [status,ops_crossovers] = opsGetCrossovers(sys,ops_param); + + % Reformat crossovers + ops_crossovers.properties.twtt = ops_crossovers.properties.twtt(:).'; + ops_crossovers.properties.source_point_path_id = double(ops_crossovers.properties.source_point_path_id(:).'); + ops_crossovers.properties.cross_point_path_id = double(ops_crossovers.properties.cross_point_path_id(:).'); + ops_crossovers.properties.source_elev = ops_crossovers.properties.source_elev.'; + ops_crossovers.properties.cross_elev = ops_crossovers.properties.cross_elev.'; + + % Get the GPS information for all the point path IDs + ops_param = []; + ops_param.properties.location = param.post.ops.location; + ops_param.properties.point_path_id = [ops_crossovers.properties.source_point_path_id ops_crossovers.properties.cross_point_path_id]; + [status,ops_data_gps_time] = opsGetPath(sys,ops_param); + + % Associate GPS time with crossover point path IDs + source_gps_time = zeros(size(ops_crossovers.properties.source_point_path_id)); + crossover_gps_time = zeros(size(ops_crossovers.properties.source_point_path_id)); + % Ensure crossover season is good + for idx = 1:length(ops_crossovers.properties.source_point_path_id) + match_idx = find(ops_crossovers.properties.source_point_path_id(idx) == ops_data_gps_time.properties.id,1); + source_gps_time(idx) = ops_data_gps_time.properties.gps_time(match_idx); + match_idx = find(ops_crossovers.properties.cross_point_path_id(idx) == ops_data_gps_time.properties.id,1); + crossover_gps_time(idx) = ops_data_gps_time.properties.gps_time(match_idx); + end + + % Update sparam + sparam.argsin{1}.layer_tracker.crossover.source_elev ... + = ops_crossovers.properties.source_elev; + sparam.argsin{1}.layer_tracker.crossover.cross_elev ... + = ops_crossovers.properties.cross_elev; + sparam.argsin{1}.layer_tracker.crossover.twtt ... + = ops_crossovers.properties.twtt; + sparam.argsin{1}.layer_tracker.crossover.season_name ... + = ops_crossovers.properties.season_name; + sparam.argsin{1}.layer_tracker.crossover.crossover_gps_time ... + = crossover_gps_time; + sparam.argsin{1}.layer_tracker.crossover.source_gps_time ... + = source_gps_time; +end + +%% layer_tracker: Load ocean mask, land DEM, sea surface DEM +if dem_en + global gdem; + if isempty(gdem) || ~isa(gdem,'dem_class') || ~isvalid(gdem) + gdem = dem_class(param,500); + end + gdem.set_res(500); + gdem.ocean_mask_mode = 'l'; + + gdem_str = sprintf('%s:%s:%s',param.radar_name,param.season_name,param.day_seg); + for frm = frm_list + gdem_str = cat(2,gdem_str,sprintf('_%03d',frm)); + end + if ~strcmpi(gdem_str,gdem.name) + gdem.set_vector(lat,lon,gdem_str); + end + [sparam.argsin{1}.layer_tracker.land_dem, ... + sparam.argsin{1}.layer_tracker.msl, ... + sparam.argsin{1}.layer_tracker.ocean_mask] = gdem.get_vector_dem(); +end + +%% layer_tracker: Load Ice Mask +if ice_mask_en + if strcmp(track.ice_mask.type,'bin') + mask = load(track.ice_mask.mat_fn,'R','X','Y','proj'); + [fid,msg] = fopen(track.ice_mask.fn,'r'); + if fid < 1 + fprintf('Could not open file %s\n', track.ice_mask.fn); + error(msg); + end + mask.mask = logical(fread(fid,[length(mask.Y),length(mask.X)],'uint8')); + fclose(fid); + else + [mask.mask,mask.R,~] = geotiffread(track.ice_mask.fn); + mask.proj = geotiffinfo(track.ice_mask.fn); + end + + [mask.x, mask.y] = projfwd(mask.proj, lat, lon); + mask.X = mask.R(3,1) + mask.R(2,1)*(1:size(mask.mask,2)); + mask.Y = mask.R(3,2) + mask.R(1,2)*(1:size(mask.mask,1)); + [mask.X,mask.Y] = meshgrid(mask.X,mask.Y); + ice_mask.mask = round(interp2(mask.X, mask.Y, double(mask.mask), mask.x, mask.y)); + ice_mask.mask(isnan(ice_mask.mask)) = 1; + sparam.argsin{1}.layer_tracker.ice_mask = ice_mask.mask; +end + +%% layer_tracker: Save cluster +ctrl = cluster_save_sparam(ctrl,sparam); +ctrl = cluster_save_dparam(ctrl); +ctrl_chain{end+1} = ctrl; +fprintf('Done %s\n',datestr(now)); + +%% layer_tracker_combine ------------------------- +% ========================================================================= +% ========================================================================= +ctrl = cluster_new_batch(param); + +sparam = []; +sparam.argsin{1} = param; +sparam.task_function = 'layer_tracker_combine_task'; +sparam.num_args_out = 1; + +records_fn = ct_filename_support(param,'','records'); +file_info = dir(records_fn); +sparam.cpu_time = 30 + cputime_combine; +sparam.mem = 500e6 + 1.5*mem_combine + 1.5*file_info.bytes; +sparam.notes = ''; + +if strcmp(param.layer_tracker.layer_params.source,'ops') + sparam.file_success = {}; +else + sparam.file_success = {}; + out_fn_dir = ct_filename_out(param,param.layer_tracker.layer_params.layerdata_source); + for frm = param.cmd.frms + % Check proc_mode from frames file that contains this frames type and + % make sure the user has specified to process this frame type + if ~ct_proc_frame(frames.proc_mode(frm),param.layer_tracker.frm_types) + continue; + end + out_fn = fullfile(out_fn_dir,sprintf('Data_%s_%03d.mat',param.day_seg,frm)); + sparam.file_success{end+1} = out_fn; + end +end + +ctrl = cluster_new_task(ctrl,sparam,[]); +ctrl_chain{end+1} = ctrl; +fprintf('Done %s\n',datestr(now)); + diff --git a/cresis-toolbox/tracker/layer_tracker_2D.m b/cresis-toolbox/tracker/layer_tracker_2D.m deleted file mode 100644 index 4943593e..00000000 --- a/cresis-toolbox/tracker/layer_tracker_2D.m +++ /dev/null @@ -1,475 +0,0 @@ -function ctrl_chain = layer_tracker_2D(param,param_override) -% set parameters and create tasks in this part. layer_tracker_2D_task will -% then do the actual processing and tuning -% look into sparam and dparam structs for create tasks and also block_size -% for create task for loop. Add settings for viterbi and lsm tracker -%% General Setup -% ===================================================================== -param = merge_structs(param, param_override); - -%% Input Checks -% ===================================================================== -% global gRadar -% param = merge_structs(gRadar,param); - -physical_constants; - -if ~isfield(param,'layer_tracker') || isempty(param.layer_tracker) - param.layer_tracker=[]; -end - -if ~isfield(param.layer_tracker,'debug_plots') || isempty(param.layer_tracker.debug_plots) - param.layer_tracker.debug_plots = {}; -end - -% if ~isempty(param.layer_tracker.debug_plots) -% h_fig = get_figures(1,true); -% end -% -param.layer_tracker.enable_debug_plot = any(strcmp('debug',param.layer_tracker.debug_plots)); - -% echogram_img: To choose an image besides the base (0) image -if ~isfield(param.layer_tracker,'echogram_img') || isempty(param.layer_tracker.echogram_img) - param.layer_tracker.echogram_img = 0; -end -%echogram_img = param.layer_tracker.echogram_img; - -% echogram source: location of echogram data used for tracking -if ~isfield(param.layer_tracker,'echogram_source') || isempty(param.layer_tracker.echogram_source) - error('An echogram source must be specified'); -end -%echogram_source = param.layer_tracker.echogram_source; - -if ~isfield(param.layer_tracker,'layer_params') || isempty(param.layer_tracker.layer_params) - param.layer_tracker.layer_params = []; -end -%layer_params = param.layer_tracker.layer_params; - -%% General Setup Continued -% ====================================================================== -% -if isstruct(param.layer_tracker.echogram_source) - fprintf(' %s: %s (%s)\n', mfilename, 'param.layer_tracker.echogram_source', datestr(now)); -else - fprintf('=====================================================================\n'); - fprintf('%s: %s (%s)\n', mfilename, param.day_seg, datestr(now)); - fprintf('=====================================================================\n'); -end - -%% Input Checks: track field -% ====================================================================== - -if ~isfield(param.layer_tracker,'track') || isempty(param.layer_tracker.track) - param.layer_tracker.track = []; -end - - -track = merge_structs(param.qlook.surf,param.layer_tracker.track); - -% profile: default is no profile, otherwise loads preset configuration -if ~isfield(param.layer_tracker.track,'profile') || isempty(param.layer_tracker.track.profile) - param.layer_tracker.track.profile = ''; -end - -track = merge_structs(layer_tracker_profile(param,track.profile), track); - -if ~isfield(track,'en') || isempty(track.en) - % If true, tracking will be done on this segment. If false, then no - % tracking is done. Default is true. - track.en = true; -end -if ~track.en - return; -end - -if ~isfield(track,'data_noise_en') || isempty(track.data_noise_en) - % If true, then a matrix data_noise will be created which will go through - % all the same operations as data except no "sidelobe" and no "feedthru" - % masking will be done. This is sometimes necessary to enable when - % sidelobe or feedthru remove too much data so that the noise estimatation - % process in the tracker does not function properly. Currently only - % tracker_threshold makes use of data_noise. - track.data_noise_en = false; -end - -% debug_time_guard: Vertical band around layer in debug plot -if ~isfield(track,'debug_time_guard') || isempty(track.debug_time_guard) - track.debug_time_guard = 50e-9; -end -param.layer_tracker.debug_time_guard = track.debug_time_guard; - -if ~isfield(track,'detrend') || (isempty(track.detrend) && ~ischar(track.detrend)) - track.detrend = 0; -end - -if ischar(track.detrend) - % Load detrend file generated by run_get_echogram_stats - % e.g. ct_tmp/echogram_stats/snow/2011_Greenland_P3/stats_20110329_02.mat - if isempty(track.detrend) - track.detrend = [ct_filename_ct_tmp(param,'','echogram_stats','stats') '.mat']; - end - track.detrend_strcut = load(track.detrend,'dt','bins','min_means'); - track.detrend_struct.time = track.detrend_struct.dt*track.detrend_struct.bins; -end - -if ~isfield(track,'feedthru') || isempty(track.feedthru) - track.feedthru = []; -end - -if ~isfield(track,'filter') || isempty(track.filter) - track.filter = [1 1]; -end -if length(track.filter) == 1 - warning('Deprecated surf.filter format. Should specify 2 element vector that specifies the multilooks in [cross-track along-track].'); - track.filter = [1 track.filter(1)]; -end -if any(mod(track.filter,2) == 0) - error('Surface filter lengths must be odd. layer_tracker.track.filter = [%d %d].', layer_tracker.track.filter); -end - -if ~isfield(track,'filter_trim') || isempty(track.filter_trim) - track.filter_trim = [0 0]; -end - -if ~isfield(track,'fixed_value') || isempty(track.fixed_value) - track.fixed_value = 0; -end - -if ~isfield(track,'init') || isempty(track.init) - track.init = []; -end -if ~isfield(track.init,'method') || isempty(track.init.method) - track.init.method = 'max'; -end -if ~isfield(track.init,'snake_rng') || isempty(track.init.snake_rng) - track.init.snake_rng = [-2e-7 2e-7]; -end -if ~isfield(track.init,'dem_layer') || isempty(track.init.dem_layer) - track.init.dem_layer = ''; -end -if ~any(strcmpi(track.init.method,{'max','nan','snake','medfilt','dem'})) - error('Unsupported surface init method %s. Options are max, nan, snake, medfilt, or dem. max is default.', track.init.method); -end -if ~isfield(track.init,'max_diff') || isempty(track.init.max_diff) - track.init.max_diff = inf; -end -if ~isfield(track.init,'max_diff_method') || isempty(track.init.max_diff_method) - if strcmpi(track.init.method,{'dem'}) || ~isempty(track.init.dem_layer) - % If the initial surface is from a dem or reference layer, the default - % method for outliers is to use merge_vectors to fill the outliers in. - track.init.max_diff_method = 'merge_vectors'; - else - % Otherwise the default method is just to interpolate between the good - % points that exist to fill the outliers in. - track.init.max_diff_method = 'interp_finite'; - end -end -if ~any(strcmpi(track.init.max_diff_method,{'merge_vectors','interp_finite'})) - error('Unsupported max diff method %s. Options are merge_vectors, interp_finite. The default is interp_finite unless a dem or reference layer is provided.', track.init.max_diff_method); -end - -if ~isfield(track,'max_bin') || isempty(track.max_bin) - track.max_bin = inf; -end - -if ~isfield(track,'max_rng') || isempty(track.max_rng) - track.max_rng = [0 0]; -end - -if ~isfield(track,'max_rng_units') || isempty(track.max_rng_units) - track.max_rng_units = 'time'; -end - -if ~isfield(track,'medfilt') || isempty(track.medfilt) - % Like medfilt1 except it handles the edges of the frame better - track.medfilt = 0; -end -if ~isfield(track,'medfilt_threshold') || isempty(track.medfilt_threshold) - % medfilt_threshold: the point is compared to the medfilt1 result and if - % the point is > medfilt_threshold from medfilt1, then the point is - % replaced with the medfilt1 result. The two extremes are: - % 0 makes medfilt act like medfilt1 - % inf effectively disables the medfilt operation - track.medfilt_threshold = 0; -end - -if ~isfield(track,'method') || isempty(track.method) - track.method = ''; -end - -if ~isfield(track,'min_bin') || isempty(track.min_bin) - track.min_bin = 0; -end - -if ~isfield(track,'prefilter_trim') || isempty(track.prefilter_trim) - track.prefilter_trim = [0 0]; -end - -if ~isfield(track,'sidelobe_rows') || isempty(track.sidelobe_rows) || ~isfield(track,'sidelobe_dB') || isempty(track.sidelobe_dB) - track.sidelobe_dB = []; - track.sidelobe_rows = []; -end - -if ~isfield(track,'snake_rng') || isempty(track.snake_rng) - track.snake_rng = [-2e-7 2e-7]; -end - -if ~isfield(track,'threshold') || isempty(track.threshold) - track.threshold = 15; -end - -if ~isfield(track,'threshold_noise_rng') || isempty(track.threshold_noise_rng) - track.threshold_noise_rng = [0 -inf -1]; -end - -param.layer_tracker.tracker = track; -%% Load in ocean mask, land DEM, and sea surface DEM -if isfield(track,'init') && strcmpi(track.init.method,'dem') - global gdem; - if isempty(gdem) || ~isa(gdem,'dem_class') || ~isvalid(gdem) - gdem = dem_class(param,500); - end - gdem.set_res(500); - gdem.ocean_mask_mode = 'l'; - - gdem_str = sprintf('%s:%s:%s',param.radar_name,param.season_name,param.day_seg); - if ~strcmpi(gdem_str,gdem.name) - % Load records file - records_fn = ct_filename_support(param,'','records'); - records = load(records_fn); - gdem.set_vector(records.lat,records.lon,gdem_str); - end -end - -%% Determine valid frames to process -if nargout ~= 1 - frames_fn = ct_filename_support(param,'','frames'); - if ~exist(frames_fn,'file') - warning('Cannot verify param.cmd.frms because frames file does not exist: %s.', frames_fn); - else - load(frames_fn); - if isempty(param.cmd.frms) - param.cmd.frms = 1:length(frames.frame_idxs); - end - % Remove frames that do not exist from param.cmd.frms list - [valid_frms,keep_idxs] = intersect(param.cmd.frms, 1:length(frames.frame_idxs)); - if length(valid_frms) ~= length(param.cmd.frms) - bad_mask = ones(size(param.cmd.frms)); - bad_mask(keep_idxs) = 0; - warning('Nonexistent frames specified in param.cmd.frms (e.g. frame "%g" is invalid), removing these', ... - param.cmd.frms(find(bad_mask,1))); - param.cmd.frms = valid_frms; - end - end -end - -% %% Load reference surface -% if isfield(track,'init') && isfield(track.init,'dem_layer') ... -% && ~isempty(track.init.dem_layer) -% layers = opsLoadLayers(param,track.init.dem_layer); -% -% % Ensure that layer gps times are monotonically increasing -% lay_idx = 1; -% layers_fieldnames = fieldnames(layers(lay_idx)); -% [~,unique_idxs] = unique(layers(lay_idx).gps_time); -% for field_idx = 1:length(layers_fieldnames)-1 -% if ~isempty(layers(lay_idx).(layers_fieldnames{field_idx})) -% layers(lay_idx).(layers_fieldnames{field_idx}) = layers(lay_idx).(layers_fieldnames{field_idx})(unique_idxs); -% end -% end -% end - - -%% Setup Processing -% ==================================================================== - -[~,~,radar_name] = ct_output_dir(param.radar_name); -records_fn = ct_filename_support(param,'','records'); -frames_fn = ct_filename_support(param,'','frames'); - -if ~exist(records_fn) - error('You must run create the records file before running anything else'); -end - -records = load(records_fn); -frames = load(frames_fn); - -if isempty(param.cmd.frms) - param.cmd.frms = 1:length(frames.frames.frame_idxs); -end -save_name = '/cresis/snfs1/scratch/anjali/CSARP_layerData_Anjali'; -dir_name1 = 'CSARP_layer_tracker_tmp'; -dir_name2 = 'CSARP_layer_tracker'; -out_fn_dir = ct_filename_out(param,param.layer_tracker.echogram_source,''); -tmp_out_fn_dir = fullfile(save_name, param.season_name, dir_name1, dir_name2,param.day_seg); - -%% Adding inputs to param - -param.layer_tracker.in_path = ct_filename_out(param,param.layer_tracker.echogram_source,''); -param.layer_tracker.img = 0; -param.layer_tracker.cmds = []; -layer_params_list = []; - -% Done in run_layer_tracker_2D param.layer_tracker.ice_mask_fn; -if strcmpi(param.layer_tracker.track.method,'viterbi') - %layer_params_list = {struct('name',{'bottom','bottom_viterbi'},'source',{'layerdata','layerdata'},'echogram_source',{'CSARP_post/standard','CSARP_post/standard'},'layerdata_source',{'layerData','layerData'})}; - layer_params_list = {struct('name',{'bottom'},'source',{'layerdata'},'echogram_source',{'CSARP_post/standard'},'layerdata_source',{'layerData'})}; - param.layer_tracker.cmds(end+1).layer_params = layer_params_list{1}; - -elseif any(strcmp(param.layer_tracker.track.method,'lsm')) || any(strcmp(param.layer_tracker.track.method,'stereo')) || any(strcmp(param.layer_tracker.track.method,'mcmc')) - layer_params_list = {struct('name','surface','source','layerdata','echogram_source','CSARP_post/standard','layerdata_source','layerData'),struct('name','bottom','source','layerdata','echogram_source','CSARP_post/standard','layerdata_source','layerData')}; - param.layer_tracker.cmds(end+1).layer_params = layer_params_list; - -else - - layer_params_list = {struct('name',{'surface'},'source',{'layerdata'},'echogram_source',{'CSARP_post/standard'},'layerdata_source',{'layerData'})}; - param.layer_tracker.cmds(end+1).layer_params = layer_params_list; -end - - -%% Set up Cluster -% =================================================================== - -ctrl = cluster_new_batch(param); -%cluster_compile({'layer_tracker_task'},ctrl.cluster.hidden_depend_funs,ctrl.cluster.force_compile,ctrl); -cluster_compile({'layer_tracker_task','layer_tracker_combine_task'},ctrl.cluster.hidden_depend_funs,ctrl.cluster.force_compile,ctrl); -ctrl_chain = {}; - -%% Create tasks -% sparam, dparam structs, frm loop for frms, ct_proc_frames, cluster_rerun, -% O/P directory, cluster task for block, success condition, create task -% -% sparam.argsin{1}.load.imgs = param.qlook.imgs; -% sparam, dparam, imgs for loop, success condition, rerun mode, create -% task -% -% sparam, dparam, new_task, save_dparam, save_chain, print, cleanup -tmp_out_fn = {}; -totalfrms = {}; -sparam.argsin{1} = param; -sparam.task_function = 'layer_temp_task'; -sparam.num_args_out = 1; -sparam.argsin{1}.load.echogram_img = param.layer_tracker.echogram_img; -sparam.cpu_time = 60; -sparam.mem = 500e6; -sparam.notes = ''; - -Nx = 0; -dparam = []; -dparam.file_success = {}; -dparam.argsin{1}.frm_nums = []; -param.filename = []; - - -for frm_idx = 1:param.layer_tracker.N - frm = param.cmd.frms(frm_idx); - - data_fn = fullfile(ct_filename_out(param,param.layer_tracker.echogram_source,''),sprintf('Data_%s_%03d.mat',param.day_seg,frm)); - data = load(data_fn, 'GPS_time','Time'); - if (frm_idx==1) - max_time = data.Time(end); - min_time = data.Time(1); - else - if(max_time <= data.Time(end)) - max_time = data.Time(end); - end - if(min_time >= data.Time(1)) - min_time = data.Time(1); - end - end - Nx = Nx + length(data.GPS_time); - dt = data.Time(2) - data.Time(1); -end - -Nt = min_time:dt:max_time; - -%% Create success condition -% =============================================================== - -% dparam.file_success = {}; -% out_fn = fullfile(out_fn_dir,sprintf('Data_%s_%03d.mat',param.day_seg,frm)); -% dparam.file_success{end+1} = out_fn; - -switch param.layer_tracker.track.method - case 'viterbi' - cpu_time_mult = 100; - mem_mult = 500e6; - - case 'lsm' - cpu_time_mult = 150; - mem_mult = 64; -end - -dparam.cpu_time = 7000*3; %cpu_time_mult * Nx * length(Nt); -dparam.mem = 500e6*3; %mem_mult * Nx * length(Nt); - -% if ~ctrl.cluster.rerun_only -% % Mark file for deletion -% ct_file_lock_check(out_fn,3); -% end -frm = 1; -%if (N == inf) -% for frm = 1:length(param.cmd.frms) -% out_fn = fullfile(out_fn_dir,sprintf('Data_%s_%03d.mat',param.day_seg,frm)); -% dparam.file_success{end+1} = out_fn; -% dparam.argsin{1}.frm_nums(end+1) = param.cmd.frms; -% dparam.notes = sprintf('%s:%s:%s:%s %s_%03d (%d of %d)',sparam.task_function,param.radar_name, param.season_name, out_fn_dir, param.day_seg, frm, frm_idx,length(param.cmd.frms)); -% end -% ctrl = cluster_new_task(ctrl,sparam,dparam,'dparam_save',0); -%else - %while frm <= length(frames.frames.frame_idxs) - while frm <= length(frames.frames.frame_idxs) - for i = 1:param.layer_tracker.N - if(frm <= 3) %length(frames.frames.frame_idxs)) - out_fn = fullfile(out_fn_dir,sprintf('Data_%s_%03d.mat',param.day_seg,1)); - dparam.notes = sprintf('%s:%s:%s:%s %s_%03d (%d of %d)',sparam.task_function,param.radar_name, param.season_name, out_fn_dir, param.day_seg, 1, 1,length(param.cmd.frms)); - dparam.file_success{end+1} = tmp_out_fn_dir; - dparam.argsin{1}.frm_nums(end+1) = frm; - param.filename{end+1} = fullfile(ct_filename_out(param,'layerdata',''),sprintf('Data_%s_%03d.mat', param.day_seg, frm)); - %totalfrms(end+1) = frm; - end - frm=frm+1; - end - frm_dir = sprintf('layer_tracker_%03d_%03d', dparam.argsin{1}.frm_nums(1),dparam.argsin{1}.frm_nums(end)); - if ~exist(fullfile(save_name,param.season_name,dir_name1,dir_name2,param.day_seg,frm_dir)) - mkdir(fullfile(save_name,param.season_name,dir_name1,dir_name2,param.day_seg,frm_dir)); - end - tmp_out_fn{end+1} = fullfile(save_name,param.season_name,dir_name1,dir_name2,param.day_seg,frm_dir,sprintf('layer_%s_%s',param.layer_tracker.track.method,param.layer_tracker.name)); - totalfrms{end+1} = [dparam.argsin{1}.frm_nums(1), dparam.argsin{1}.frm_nums(end)]; - %tmp_out_fn.(sprintf('number_frames_%d_%d',dparam.argsin{1}.frm_nums(1),dparam.argsin{1}.frm_nums(end))){end+1} = - ctrl = cluster_new_task(ctrl,sparam,dparam,'dparam_save',0); - dparam.file_success = {}; - - dparam.argsin{1}.frm_nums = []; - end - param.tmp_out_fn = tmp_out_fn; - param.totalfrms = totalfrms; - %dparam.argsin{1}.tmp = param; - - -%end -%% Rerun only mode: Test to see if we need to run this task -ctrl = cluster_save_dparam(ctrl); -ctrl_chain{end+1} = ctrl; -fprintf('Done %s\n',datestr(now)); - -%% Combine task -ctrl = cluster_new_batch(param); - -sparam = []; -sparam.argsin{1} = param; -sparam.task_function = 'layer_tracker_combine_task'; -sparam.num_args_out = 1; - -sparam.cpu_time = 7000*3; %cpu_time_mult * Nx * length(Nt); -sparam.mem = 500e6*3; %mem_mult * Nx * length(Nt); -sparam.notes = ''; -sparam.file_success = {}; -out_fn = fullfile(out_fn_dir,sprintf('Data_%s_%03d.mat',param.day_seg,frm)); -sparam.argsin{1}.temp_fn = tmp_out_fn_dir; -%sparam.file_success{end+1} = out_fn; - -ctrl = cluster_new_task(ctrl,sparam,[]); -ctrl_chain{end+1} = ctrl; -fprintf('Done %s\n',datestr(now)); - diff --git a/cresis-toolbox/tracker/layer_tracker_combine_task.m b/cresis-toolbox/tracker/layer_tracker_combine_task.m index 4a975632..6537162b 100644 --- a/cresis-toolbox/tracker/layer_tracker_combine_task.m +++ b/cresis-toolbox/tracker/layer_tracker_combine_task.m @@ -1,117 +1,159 @@ function success = layer_tracker_combine_task(param) -%function to combine layer data information received from the tracker. To -%store in layerData files. Use opsCopyLayers -global gRadar; -if (param.layer_tracker.save_ops_copy_layers) -layer_params = param.layer_tracker.cmds.layer_params; - for layer_idx = 1:length(layer_params) - TWTT{layer_idx} = [];%lay.layerData{layer_idx}.quality = []; -% lay.layerData{layer_idx}.value{2}.data =[]; - end +% success = layer_tracker_combine_task(param) +% +% Combines temporary layer data from layer_tracker_task.m and stores the +% output with opsCopyLayers. See run_layer_tracker.m. +% +% Authors: Anjali Pare, John Paden +% +% See also: layer_tracker.m, layer_tracker_combine_task.m, +% layer_tracker_task.m, layer_tracker_input_check.m, +% layer_tracker_profile.m, run_layer_tracker.m, run_layer_tracker_tune.m - GPS_time = []; - numFrms = []; -for idx = 1:length(param.tmp_out_fn) - tmp_out_fn = load(param.tmp_out_fn{idx}); - - %for frm = param.totalfrms{idx}(1):param.totalfrms{idx}(2) - for layer_idx = 1:length(layer_params) -% layer_param = layer_params(layer_idx); -% layer_fn = fullfile(ct_filename_out(param,layer_param{1}.source,''),sprintf('Data_%s_%03d.mat', param.day_seg, frm)); -% lay = load(layer_fn); -% temp = find(tmp_out_fn.indexes{layer_idx}==frm); -% tmp_out_fn.TWTT = interp1(1:length(tmp_out_fn.gps_time),tmp_out_fn.gps_time,tmp_out_fn.twtt{layer_idx}); - TWTT{layer_idx} = cat(2,TWTT{layer_idx},tmp_out_fn.twtt{layer_idx}); -% tmp_out_fn.TWTT = interp_finite(tmp_out_fn.TWTT,NaN); -% tmp_out_fn.GPS_time = tmp_out_fn.gps_time; -% lay.layerData{layer_idx}.quality = interp1(tmp_out_fn.gps_time(temp),ones(1,length(temp)),lay.GPS_time,'nearest'); -% lay.layerData{layer_idx}.value{2}.data = interp1(tmp_out_fn.gps_time(temp),tmp_out_fn.twtt{layer_idx}(length(temp)),lay.GPS_time); - %tmp_info = Ldata.tracker_data{layer_idx}.(sprintf('%s_%s_%03d',param.layer_tracker.track.method,param.layer_tracker.name,layer_idx)); - end - - %layerData = lay.layerData; - GPS_time = cat(2,GPS_time,tmp_out_fn.gps_time); - %save(layer_fn,'-append','layerData'); - %end - numFrms = cat(2,numFrms,(param.totalfrms{idx}(1):param.totalfrms{idx}(2))); - param.cmd.frms = numFrms; +%% Setup Processing +% ===================================================================== +frames = frames_load(param); + +% Create output directory path +if strcmp(param.layer_tracker.layer_params.source,'ops') + tmp_out_fn_dir_dir = ct_filename_out(param,'ops','layer_tracker_tmp'); +else + tmp_out_fn_dir_dir = ct_filename_out(param,param.layer_tracker.layer_params.layerdata_source,'layer_tracker_tmp'); end +layer_dest.name = {}; +layer_dest.age = {}; +layer_dest.age_source = {}; +layer_dest.desc = {}; +layer_dest.group_name = {}; +layer_source.twtt = []; +layer_source.gps_time = []; -% lay.GPS_time+1:lay.GPS_time*idx(which ranges from 1, 2, 3... end) -% for idx = 1:length(numfolders) -% layer_fn = fullfile(ct_filename_out(param,layer_param{1}.source,''),sprintf('Data_%s_%03d.mat', param.day_seg, frm)); -% lay = load(layer_fn); -% fname = fullfile(fpath,sprintf('layer_tracker_%03d_%03d',idx,idx),sprintf('layer_%s_%s',param.layer_tracker.track.method,param.layer_tracker.name)); -% Ldata = load(fname); -% -% end -% lay.layerData{layer_idx}.value{2}.data = interp1(tmp_info.GPS_time,tmp_info.layerData{layer_idx}.value{2}.data,lay.GPS_time); -% - keyboard; - copy_param = []; - copy_param.layer_source.existence_check = false; - copy_param.layer_dest.existence_check = false; +%% Track loop +% ===================================================================== +% Combine and Copy each tracking result +for track_idx = 1:length(param.layer_tracker.track) + track = param.layer_tracker.track{track_idx}; + %% Track: Load in all temporary files + % ===================================================================== + gps_time = []; + twtt = []; + for frm_idx = 1:length(param.cmd.frms) + frm = param.cmd.frms(frm_idx); + + if ct_proc_frame(frames.proc_mode(frm),param.layer_tracker.frm_types) + fprintf('layer_tracker_combine frame %s_%03i (%i of %i) %s\n', param.day_seg, frm, frm_idx, length(param.cmd.frms), datestr(now)); + else + fprintf('Skipping frame %s_%03i (no process frame)\n', param.day_seg, frm); + continue; + end + + tmp_out_fn_name = sprintf('t%03d_%s.mat', track_idx, track.method); + tmp_out_fn = fullfile(tmp_out_fn_dir_dir,sprintf('layer_tracker_%03d', frm),tmp_out_fn_name); + fprintf('Loading %s (%s)\n', tmp_out_fn, datestr(now)); + tmp = load(tmp_out_fn); + + Nx = length(tmp.gps_time); + gps_time(1,end+(1:Nx)) = tmp.gps_time; + try +% if size(tmp.twtt,1) ~= length(param.layer_tracker.track{track_idx}.lsm.storeIter) * 2 +% tmp.twtt(36,:) = tmp.twtt(35,:); +% end + twtt(:,end+(1:Nx)) = tmp.twtt; + catch ME + tmp.twtt(36,:) = tmp.twtt(35,:); + twtt(:,end+(1:Nx)) = tmp.twtt; + end + + end - % Set the source + %% Track: opsCopyLayer copy struct + % ===================================================================== + param.layer_tracker.copy_param.gaps_fill.method = 'preserve_gaps'; + copy_param = param.layer_tracker.copy_param; + copy_param.layer_source.existence_check = false; copy_param.layer_source.source = 'custom'; - copy_param.layer_dest.source = param.layer_tracker.track.layer_dest_source; - - if strcmp(copy_param.layer_dest.source, 'layerdata') - copy_param.layer_dest.layerdata_source = param.layer_tracker.track.layer_dest_layerdata_source; %layerData_test - copy_param.layer_dest.echogram_source = param.layer_tracker.track.layer_dest_echogram_source; %standard + copy_param.layer_source.gps_time = gps_time; + copy_param.layer_dest = param.layer_tracker.layer_params; + copy_param.layer_dest.existence_check = false; + if ~isfield(param.layer_tracker.track{track_idx},'layer_names') || isempty(param.layer_tracker.track{track_idx}.layer_names) + automated_name_en = true; + else + automated_name_en = false; + layer_dest.name = param.layer_tracker.track{track_idx}.layer_names; end - copy_param.copy_method = 'overwrite'; - - %if strcmpi(copy_param.layer_dest.name,'surface') + %% Track: Copy each layer + % ===================================================================== + for layer_idx = 1:size(twtt,1) + if automated_name_en + switch track.method + case 'lsm' + if layer_idx <= length(track.lsm.storeIter) + layer_dest.name{end+1} = sprintf('%s_%s_surface_%03d', ... + track.name,track.method,layer_idx); + layer_dest.desc{end+1} = sprintf('y = %d, dy = %d, iter = %d', ... + track.lsm.y,track.lsm.dy,layer_idx); + else + layer_dest.name{end+1} = sprintf('%s_%s_bottom_%03d', ... + track.name,track.method,layer_idx-length(track.lsm.storeIter)); + layer_dest.desc{end+1} = sprintf('y = %d, dy = %d, iter = %d', ... + track.lsm.y,track.lsm.dy,layer_idx-length(track.lsm.storeIter)); + end + case {'mcmc','stereo'} + if layer_idx == 1 + layer_dest.name{end+1} = sprintf('%s_%s_surface',track.name,track.method); + else + layer_dest.name{end+1} = sprintf('%s_%s_bottom',track.name,track.method); + end + case 'viterbi' + layer_dest.name{end+1} = sprintf('%s_%s_bottom',track.name,track.method); + layer_dest.desc{end+1} = sprintf('smoothness = %d', track.viterbi.transition_weight); + otherwise + layer_dest.name{end+1} = sprintf('%s_%s_surface',track.name,track.method); + end + end + + layer_source.twtt{end+1} = twtt(layer_idx,:); + layer_source.gps_time{end+1} = gps_time; - %else + % Update age, age_source, desc, and group_name if override values are + % given + if iscell(track.age) && length(track.age) >= layer_idx + layer_dest.age{end+1} = track.age{layer_idx}; + elseif ~isempty(track.age) + layer_dest.age{end+1} = track.age; + end + if iscell(track.age_source) && length(track.age_source) >= layer_idx + layer_dest.age_source{end+1} = track.age_source{layer_idx}; + elseif ~isempty(track.age_source) + layer_dest.age_source{end+1} = track.age_source; + end + if iscell(track.desc) && length(track.desc) >= layer_idx + layer_dest.desc{end+1} = track.desc{layer_idx}; + elseif ~isempty(track.desc) + layer_dest.desc{end+1} = track.desc; + end + if iscell(track.group_name) && length(track.group_name) >= layer_idx + layer_dest.group_name{end+1} = track.group_name{layer_idx}; + elseif ~isempty(track.group_name) + layer_dest.group_name{end+1} = track.group_name; + end - %end - param = merge_structs(param,gRadar); + end - keyboard; - %for surface - if strcmp(param.layer_tracker.track.method,'viterbi') - copy_param.layer_source.gps_time = GPS_time; - copy_param.layer_source.twtt = TWTT; - - copy_param.gaps_fill.method = 'preserve_gaps'; - copy_param.gaps_fill.method_args = [40 20]; - - copy_param.layer_dest.name = 'bottom';%param.layer_tracker.track.(sprintf('%s',param.layer_tracker.track.method)).lyrbot; - fprintf('opsCopyLayers %s (%s)\n', param.day_seg, datestr(now)); - opsCopyLayers(param,copy_param); - else - copy_param.layer_source.gps_time = GPS_time; - copy_param.layer_source.twtt = TWTT{1}; - - copy_param.gaps_fill.method = 'interp_finite'; - - copy_param.layer_dest.name = 'surface';%param.layer_tracker.track.(sprintf('%s',param.layer_tracker.track.method)).lyrtop; - fprintf('opsCopyLayers %s (%s)\n', param.day_seg, datestr(now)); - opsCopyLayers(param,copy_param); +end - % for bottom - copy_param.layer_source.gps_time = GPS_time; - copy_param.layer_source.twtt = TWTT{2}; +copy_param.layer_source.gps_time = layer_source.gps_time; +copy_param.layer_source.twtt = layer_source.twtt; +copy_param.layer_dest.name = layer_dest.name; +copy_param.layer_source.age = layer_dest.age; +copy_param.layer_source.age_source = layer_dest.age_source; +copy_param.layer_source.desc = layer_dest.desc; +copy_param.layer_source.group_name = layer_dest.group_name; - copy_param.gaps_fill.method = 'preserve_gaps'; - copy_param.gaps_fill.method_args = [40 20]; +fprintf('opsCopyLayers %s (%s)\n', param.day_seg, datestr(now)); +opsCopyLayers(param,copy_param); - copy_param.layer_dest.name = 'bottom';%param.layer_tracker.track.(sprintf('%s',param.layer_tracker.track.method)).lyrbot; - fprintf('opsCopyLayers %s (%s)\n', param.day_seg, datestr(now)); - opsCopyLayers(param,copy_param); - end - - fprintf('Complete (%s)\n', datestr(now)); - warning('on'); -end -success=true; -end -% surface and bottom can be named as the layer name. Change param.cmd.frms -% to get the number of frames that you are working with. Look into saving -% the layerData info and viewing the results in imb.picker or by plotting a -% graph. Group,description,source,echogram_source, set the propoerties by -% User. Look into other ways to save layername of the destination. +fprintf('Done (%s)\n', datestr(now)); +success = true; diff --git a/cresis-toolbox/tracker/layer_tracker_generate_combine_task.m b/cresis-toolbox/tracker/layer_tracker_generate_combine_task.m new file mode 100644 index 00000000..0ea1b007 --- /dev/null +++ b/cresis-toolbox/tracker/layer_tracker_generate_combine_task.m @@ -0,0 +1,256 @@ +function success = layer_tracker_generate_combine_task(param) +%% Setup plots +options = param.options; +ff = {}; +num_layers = 1; +for layer_idx = 1:num_layers + for param_idx = 1 + ff{param_idx} = load(param.save_names{layer_idx,param_idx}); + param = merge_structs(param,ff{param_idx}.params); + end +end + +num_layers = length(param.gt_layer_params); +layer_names = num_layers; +img_dir = param.img_dir_combine; + +for layer_idx = 1:num_layers +fprintf('\n=====All segements=====\n'); + foo{layer_idx} = []; + gps_time = []; + season_vect = []; + for hist_idx = 1:length(ff{1}.frame_error{layer_idx}) + range_per{layer_idx}{hist_idx} = []; + res_matrix{layer_idx}{hist_idx} = []; + end + percent_error{layer_idx} = zeros(1,(length(ff{1}.frame_error{layer_idx}))); + + for i = 1:length(ff) + foo{layer_idx} = [foo{layer_idx};ff{i}.res_matrix_all{layer_idx}]; + percent_error{layer_idx} = percent_error{layer_idx} + ff{i}.percent_error{layer_idx}; + gps_time = cat(2,gps_time,ff{i}.gps_time); + season_vect = cat(2,gps_time,ff{i}.gps_time); + for hist_idx = 1:length(ff{i}.frame_error{layer_idx}) + range_per{layer_idx}{hist_idx} = cat(2,range_per{layer_idx}{hist_idx},ff{i}.range_per{layer_idx}{hist_idx}); + res_matrix{layer_idx}{hist_idx} = cat(2,res_matrix{layer_idx}{hist_idx},ff{i}.res_matrix{layer_idx}{hist_idx}); + end + end + + + + points = []; + min_val = []; + res_matrix_all_frms = []; + res_matrix_all_frms{layer_idx} = nanmean(foo{layer_idx},1); + res_matrix_all_frms{layer_idx} = permute(res_matrix_all_frms{layer_idx},ff{1}.idx_matrix); + [min_val{layer_idx},i]=min(res_matrix_all_frms{layer_idx}(:)); + sizeMatrix = size(res_matrix_all_frms{layer_idx}); + for dim = 1:length(size(res_matrix_all_frms{layer_idx})) + points{layer_idx}(dim) = mod(floor((i-1)/prod(sizeMatrix(1:dim-1))),sizeMatrix(dim))+1; + end + if (length(size(res_matrix_all_frms{layer_idx})) < length(param.layer_tracker.track{1}.idx_reshape)) + points{layer_idx}(dim+1) = 1; + end + + if (options.percentage_correct.plot) + fprintf('\n=====Percentage correct all plots=====\n'); + + h_fig(1) = figure('visible','off','Name',param.tmp_name); + for hist_idx = 1:length(ff{1}.frame_error{layer_idx}) + + hold on; + plot(range_per{layer_idx}{hist_idx},'.-','DisplayName',sprintf('layer%d',hist_idx)); + xlabel('Frames'); + ylabel('Percentage correct (%)'); + xlim = ([1 length(param.cmd.frms)]); + title(sprintf('Percentage correct vs frames Layer:%d %s',layer_idx,param.season_name),'Interpreter', 'none'); + percent_error{layer_idx}(hist_idx) = percent_error{layer_idx}(hist_idx)/length(season_vect) * 100; + end + img_format = 'png'; + if(options.percentage_correct.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_percentage_correct_overall_%03d.%s',testname,param.season_name,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + + h_fig(1) = figure('visible','off','Name',param.tmp_name); + plot(percent_error{layer_idx}); + hold on; + plot(percent_error{layer_idx}, 'r*'); + ylabel('Percentage correct (%)'); + xlabel('Trackers'); + title(sprintf('Percentage correct vs trackers Layer:%d %s',layer_idx,param.season_name),'Interpreter', 'none'); + set(gca,'XTick',1:length(ff{1}.frame_error{layer_idx})); + img_format = 'png'; + if(options.percentage_correct.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_total_percent_correct_%03d.%s',testname,param.season_name,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + + end + + if (options.hist_generation.plot) + fprintf('\n=====Hist generation all plots=====\n'); + + h_fig(1) = figure('visible','off','Name',param.tmp_name); + for hist_idx = 1:length(ff{1}.frame_error{layer_idx}) + hold on; + [N,x] = hist(abs(res_matrix{layer_idx}{hist_idx})); + plot(x,N,'DisplayName',sprintf('layer%d',hist_idx)); + ylabel('Data points'); + xlabel('Absolute error (bins)'); + title(sprintf('Data points vs Absolute error Layer:%d %s',layer_idx,param.season_name),'Interpreter', 'none'); + end + hold off; + img_format = 'png'; + if(options.hist_generation.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_histogram_generation_overall_%03d.%s',testname,param.season_name,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + + end + + + if(options.season_vs_gps.plot) + fprintf('\n=====season plots=====\n'); + if(options.season_vs_gps.multiple_figs) + for track_idx = 1:length(ff{1}.frame_error{layer_idx}) + h_fig(1) = figure('visible','off','Name',param.tmp_name); + plot(gps_time,abs(res_matrix{layer_idx}{track_idx})); + ylabel('Absolute error (bins)'); + xlabel('GPS time'); + title(sprintf('%s Tracker: %d Layer: %d',param.season_name,track_idx,layer_idx),'Interpreter', 'none'); + + img_format = 'png'; + if(options.season_vs_gps.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_season_gps_%03d_%03d.%s',testname,param.season_name,track_idx,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + end + else + h_fig(1) = figure('visible','off','Name',param.tmp_name); + subplot_idx = 0; + for track_idx = 1:length(ff{1}.frame_error{layer_idx}) + subplot_idx = subplot_idx + 1; + + if(subplot_idx < 6) + subplot(3,2,subplot_idx) + plot(gps_time,abs(res_matrix{layer_idx}{track_idx})); + ylabel('Absolute error (bins)'); + xlabel('GPS time'); + title(sprintf('%s Tracker: %d Layer: %d',param.season_name, track_idx, layer_idx),'Interpreter', 'none'); + else + subplot(3,2,subplot_idx) + plot(gps_time,abs(res_matrix{layer_idx}{track_idx})); + ylabel('Absolute error (bins)'); + xlabel('GPS time'); + title(sprintf('%s Tracker: %d Layer: %d',param.season_name, track_idx, layer_idx),'Interpreter', 'none'); + img_format = 'png'; + if(options.season_vs_gps.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_season_gps_%03d_%03d.%s',testname,param.season_name,track_idx,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + h_fig(1) = figure('visible','off'); + subplot_idx = 0; + end + end + end + + end + + + if(options.absolute_error.plot) + fprintf('\n=====Absolute error all plots=====\n'); + if(strcmpi(param.layer_tracker.track{1}.method,'viterbi')) + res_mat = res_matrix_all_frms{layer_idx}; + points_new = points{layer_idx}; + x = 1:length(res_mat); % x tick + h_fig(1) = figure('visible','off','Name',param.tmp_name); + plot(x,res_mat); + hold on; + for i = 1:length(res_mat) + plot(x(i),res_mat(i), 'r*'); + end + plot(x(points_new(1)),res_mat(points_new(1)), 'ko','LineWidth',4); + set(gca,'XTick',x); + ylabel('Mean absolute error (bins)'); + xlabel('Trackers'); + title(sprintf('%s',param.season_name),'Interpreter', 'none'); + img_format = 'png'; + if(options.absolute_error.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%03d_%s_%s_absolute_error.%s',2,testname,param.season_name,img_format)); + ct_saveas(h_fig(1),img_fn); + + elseif(strcmpi(param.layer_tracker.track{1}.method,'lsm')) + data = []; + for dim = 1:length(param.layer_tracker.track{1}.idx_reshape) + temp = []; + for track_idx = 1:length(param.layer_tracker.track) + if isempty(temp) + temp = param.layer_tracker.track{track_idx}.(param.layer_tracker.track{track_idx}.method).(param.layer_tracker.track{track_idx}.idx_dim_name{dim}); + else + if ~any(temp == param.layer_tracker.track{track_idx}.(param.layer_tracker.track{track_idx}.method).(param.layer_tracker.track{track_idx}.idx_dim_name{dim})) + temp(end+1) = param.layer_tracker.track{track_idx}.(param.layer_tracker.track{track_idx}.method).(param.layer_tracker.track{track_idx}.idx_dim_name{dim}); + end + end + + end + data{dim}=temp; + end + res_mat = res_matrix_all_frms{layer_idx}; + temp_points = points{layer_idx}; + + A = nchoosek(1:length(param.layer_tracker.track{1}.idx_reshape),2); % changed from 2 + for id = 1:length(A) + min_points = setdiff(1:length(param.layer_tracker.track{1}.idx_reshape),A(id,:)); + min_idxs = temp_points(min_points); + res_matrix_tot = permute(res_mat,[A(id,:) setdiff(1:length(param.layer_tracker.track{1}.idx_reshape),A(id,:))]); + res_matrix_tot = res_matrix_tot(:,:,min_idxs(:)); + idx = A(id,:); + labelTickX = data{idx(1)}; % x tick label + labelTickY = data{idx(2)}; % y tick label + labelNameX = param.layer_tracker.track{1}.idx_dim_name{idx(1)}; % x label name + labelNameY = param.layer_tracker.track{1}.idx_dim_name{idx(2)}; % y label name + xTick = 1:length(labelTickX); % x tick + yTick = 1:length(labelTickY); % y tick + %% Generate Figure + h_fig(1) = figure('visible','off'); % figure is not displayed + imagesc(squeeze(res_matrix_tot')); + hold on; + plot(temp_points(idx(1)),temp_points(idx(2)),'x','LineWidth',4,'MarkerSize',10,'Color','white'); + + ylabel(labelNameY); + xlabel(labelNameX); + set(gca,'XTickLabel',labelTickX); + set(gca,'YTickLabel',labelTickY); + set(gca,'XTick',xTick); + set(gca,'YTick',yTick); + h_colorbar = colorbar(gca); + set(get(h_colorbar,'YLabel'),'String','Mean absolute error (rows)'); + img_format ='jpg'; + if(options.absolute_error.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_absolute_error_%s_vs_%s_%03d.%s',testname,param.season_name,labelNameX,labelNameY,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + + end + end + end +end +success = true; +end + diff --git a/cresis-toolbox/tracker/layer_tracker_generate_task.m b/cresis-toolbox/tracker/layer_tracker_generate_task.m new file mode 100644 index 00000000..d4cb1250 --- /dev/null +++ b/cresis-toolbox/tracker/layer_tracker_generate_task.m @@ -0,0 +1,345 @@ +function success = layer_tracker_generate_task(param) +params_all = []; +params_all{end+1} = param; + +%% Setup general +% set the range and indicate the layers to track + +options = param.options; + +gt_layer_params = []; +layer_params = []; +res_matrix = []; +res_list = []; +gps_time = []; +season_vect = []; +segment_vect = []; +frame_vect = []; +seg_idx = 0; +season_vect_frm = []; +segment_vect_frm = []; +frame_vect_frm = []; +range = options.set_range; +idx = 1; + +gt_layer_params = param.gt_layer_params; + +num_layers = length(gt_layer_params); +img_dir = param.fname; +for idx = 1:num_layers + foo{idx} = []; +end +%% Implementation + +season_idx = param.season_idx; +param_idx = param.param_idx; +for frm = param.cmd.frms + season_vect_frm(end+1) = season_idx; + segment_vect_frm(end+1) = param_idx; + frame_vect_frm(end+1) = frm; +end + +param_override = param.filename.param; +param = merge_structs(param,param_override); +total_length = 0; +layers = opsLoadLayers(param,gt_layer_params); +idx_matrix = [(1:length(param.layer_tracker.track{1}.idx_reshape))+1 1]; % used for resizing +for layer_names = 1:num_layers + res_matrix_all{layer_names} = zeros([length(param.cmd.frms),param.layer_tracker.track{1}.idx_reshape]); + res_mat_temp{layer_names} = cell([param.layer_tracker.track{1}.idx_reshape]); +end + +for layer_idx = 1:length(gt_layer_params) + if strcmpi(param.layer_tracker.track{1}.method,'lsm') + percent_error{layer_idx} = zeros(1,(length(param.layer_tracker.track)*length(param.layer_tracker.track{1}.lsm.storeIter))); + for iter_idx = 1:(length(param.layer_tracker.track)*length(param.layer_tracker.track{1}.lsm.storeIter)) + range_per{layer_idx}{iter_idx} = []; + end + elseif strcmpi(param.layer_tracker.track{1}.method,'viterbi') + percent_error{layer_idx} = zeros(1,(length(param.layer_tracker.track))); + for iter_idx = 1:(length(param.layer_tracker.track)) + range_per{layer_idx}{iter_idx} = []; + end + end +end + +for track_idx = 1:length(param.layer_tracker.track) + range_percent_all_frms.(sprintf('%s', param.layer_tracker.track{track_idx}.name)) = 0; + for frm_idx = 1:length(param.cmd.frms) + range_percent.(sprintf('%s_%03d', param.layer_tracker.track{track_idx}.name,frm_idx)) = 0; + prob_error.(sprintf('%s_%03d', param.layer_tracker.track{track_idx}.name,frm_idx)) = 0; + end +end +frm_idx = 0; +for frm = param.cmd.frms + frm_idx = frm_idx+1; + data_fn_dir = ct_filename_out(param, param.layer_tracker.echogram_source, ''); + data_fn_name = sprintf('Data_%s_%03d.mat',param.day_seg,frm); + data_fn = fullfile(data_fn_dir, data_fn_name); + data = load(data_fn, 'GPS_time', 'Time'); + gps_time = cat(2,gps_time,data.GPS_time); + dt = data.Time(2) - data.Time(1); + total_length = total_length + length(data.GPS_time); + idx_list = 1; + + frm_length = length(data.GPS_time); + for j = 1:length(data.GPS_time) + season_vect(end+1) = season_idx; + segment_vect(end+1) = param_idx; + frame_vect(end+1) = frm; + end + + for track_idx = 1:length(param.layer_tracker.track) + + for layer_idx = 1:length(gt_layer_params) + if strcmpi(param.layer_tracker.track{1}.method,'lsm') + for idx = 1:length(param.layer_tracker.track{track_idx}.lsm.storeIter) + layer_params(idx).name = sprintf('%s_%s_%s_%03d',param.layer_tracker.track{track_idx}.name,param.layer_tracker.track{track_idx}.method,gt_layer_params(layer_idx).name,idx); + layer_params(idx).source = param.layer_tracker.layer_params.source; + layer_params(idx).layerdata_source = param.layer_tracker.layer_params.layerdata_source; + end + else + layer_params(1).name = sprintf('%s_%s_%s',param.layer_tracker.track{track_idx}.name,param.layer_tracker.track{track_idx}.method,gt_layer_params(layer_idx).name); + layer_params(1).source = param.layer_tracker.layer_params.source; + layer_params(1).layerdata_source = param.layer_tracker.layer_params.layerdata_source; + end + ids = 1; + fprintf('\n=====Loading layers tracker: %d %d %s=====\n',track_idx,frm,datestr(now)); + layers_new = opsLoadLayers(param,layer_params); + fprintf('\n=====Loaded layers tracker: %d %d %s=====\n',track_idx,frm,datestr(now)); + + surf = interp1(layers(layer_idx).gps_time,layers(layer_idx).twtt,data.GPS_time); + surf_bins = round(interp1(data.Time,1:length(data.Time),surf)); + + for pos = 1:length(layers_new) + surf = interp1(layers_new(pos).gps_time,layers_new(pos).twtt,data.GPS_time); + surf_bins_itr = round(interp1(data.Time,1:length(data.Time),surf)); + res_matrix_all{layer_idx}(frm_idx,param.layer_tracker.track{track_idx}.idx(pos)) = nanmean(abs(surf_bins - surf_bins_itr)); + counter = 0; + for i = 1:length(surf_bins) + try + res_matrix{layer_idx}{param.layer_tracker.track{track_idx}.idx(pos)}(end+1) = surf_bins(i) - surf_bins_itr(i); + res_mat_temp{layer_idx}{param.layer_tracker.track{track_idx}.idx(pos)}(end+1) = surf_bins(i) - surf_bins_itr(i); + catch ME + res_matrix{layer_idx}{param.layer_tracker.track{track_idx}.idx(pos)}(i) = surf_bins(i) - surf_bins_itr(i); + res_mat_temp{layer_idx}{param.layer_tracker.track{track_idx}.idx(pos)}(i) = surf_bins(i) - surf_bins_itr(i); + end + if (abs(surf_bins(i) - surf_bins_itr(i)) <= range) + counter = counter + 1; + end + end + range_percent.(sprintf('%s_%03d', param.layer_tracker.track{track_idx}.name,frm)) = (counter/length(surf_bins))*100; + range_per{layer_idx}{param.layer_tracker.track{track_idx}.idx(pos)}(end+1) = (counter/length(surf_bins))*100; + percent_error{layer_idx}(param.layer_tracker.track{track_idx}.idx(pos)) = counter + percent_error{layer_idx}(param.layer_tracker.track{track_idx}.idx(pos)); + prob_error.(sprintf('%s_%03d', param.layer_tracker.track{track_idx}.name,frm)) = counter*100; + frame_error{layer_idx}{param.layer_tracker.track{track_idx}.idx(pos)}(frm_idx) = nanmean(abs(surf_bins - surf_bins_itr)); + try + range_percent_all_frms.(sprintf('%s', param.layer_tracker.track{track_idx}.name)) = range_percent_all_frms.(sprintf('%s', param.layer_tracker.track{track_idx}.name)) + counter; + catch ME + range_percent_all_frms.(sprintf('%s', param.layer_tracker.track{track_idx}.name)) = counter; + end + end + end + idx_list = idx_list + 1; + end +end + + +points = []; +min_val = []; +res_matrix_all_frms = []; +for layer_names = 1:num_layers + res_matrix_all_frms{layer_names} = nanmean(res_matrix_all{layer_names},1); + res_matrix_all_frms{layer_names} = permute(res_matrix_all_frms{layer_names},idx_matrix); + [min_val{layer_names},i]=min(res_matrix_all_frms{layer_names}(:)); + sizeMatrix = size(res_matrix_all_frms{layer_names}); + for dim = 1:length(size(res_matrix_all_frms{layer_names})) + points{layer_names}(dim) = mod(floor((i-1)/prod(sizeMatrix(1:dim-1))),sizeMatrix(dim))+1; + end + if (length(size(res_matrix_all_frms{layer_names})) < length(param.layer_tracker.track{1}.idx_reshape)) + points{layer_names}(dim+1) = 1; + end + if(param_idx==1) + foo{layer_names} = res_matrix_all{layer_names}; + else + foo{layer_names} = [foo{layer_names};res_matrix_all{layer_names}]; + end +end + +% per segment vs frame +if (options.segment_vs_frame.plot) + fprintf('\n=====Segment frame plots=====\n'); + for layer_idx = 1:length(gt_layer_params) + h_fig(1) = figure('visible','off','Name',param.tmp_name); + for fig_idx = 1:length(frame_error{layer_idx}) + hold on; + plot(frame_error{layer_idx}{fig_idx},'.-','DisplayName',sprintf('layer%d',fig_idx)); + labelNameX = 'Frames'; + labelNameY = 'Absolute error (bins)'; + ylabel(labelNameY); + xlabel(labelNameX); + xlim = ([1 length(param.cmd.frms)]); + title(sprintf('%s Layer: %d',param.day_seg,layer_idx),'Interpreter', 'none'); + end + + hold off; + img_format = 'png'; + if(options.segment_vs_frame.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_segment_vs_frame_%03d.%s',testname,param.day_seg,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + end +end + +if (options.percentage_correct.plot) + fprintf('\n=====Percentage plots=====\n'); + if (options.percentage_correct.multiple_plots) + for layer_idx = 1:length(gt_layer_params) + h_fig(1) = figure('visible','off','Name',param.tmp_name); + for hist_idx = 1:length(frame_error{layer_idx}) + hold on; + y = range_per{layer_idx}{hist_idx}(segment_vect_frm == param_idx); + plot(y,'.-','DisplayName',sprintf('layer%d',hist_idx)); + xlabel('Frames'); + ylabel('Percentage error'); + xlim = ([1 length(param.cmd.frms)]); + title(sprintf('%s Layer: %d',param.day_seg,layer_idx),'Interpreter', 'none'); + end + img_format = 'png'; + if(options.percentage_correct.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_percentage_correct_%03d.%s',testname,param.day_seg,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + end + end +end + +if (options.hist_generation.plot) + fprintf('\n=====Hist generation plots=====\n'); + if (options.hist_generation.multiple_plots) + for layer_idx = 1:length(gt_layer_params) + h_fig(1) = figure('visible','off','Name',param.tmp_name); + for hist_idx = 1:length(frame_error{layer_idx}) + hold on; + plot_hist = abs(res_matrix{layer_idx}{hist_idx}(segment_vect == param_idx)); + [N,x] = hist(plot_hist); + plot(x,N,'DisplayName',sprintf('layer%d',hist_idx)); + %legend('-DynamicLegend','Location','northeastoutside'); + ylabel('Data points'); + xlabel('Absolute error (bins)'); + title(sprintf('%s Layer: %d',param.day_seg,layer_idx),'Interpreter', 'none'); + end + hold off; + img_format = 'png'; + if(options.hist_generation.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_histogram_generation_%03d.%s',testname,param.day_seg,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + end + end +end + +if(options.absolute_error.plot) + fprintf('\n=====Absolute error plots=====\n'); + if (options.absolute_error.multiple_plots) + if(strcmpi(param.layer_tracker.track{1}.method,'viterbi')) + for layer_idx = 1:num_layers + res_mat = res_matrix_all_frms{layer_idx}; + points_new = points{layer_idx}; + x = 1:length(res_mat); % x tick + h_fig(1) = figure('visible','off','Name',param.tmp_name); + plot(x,res_mat); + hold on; + for i = 1:length(res_mat) + plot(x(i),res_mat(i), 'r*'); + end + plot(x(points_new(1)),res_mat(points_new(1)), 'ko','LineWidth',4); + set(gca,'XTick',x); + ylabel('Mean absolute error (bins)'); + xlabel('Trackers'); + title(sprintf('%s frames: %03d',param.day_seg,length(param.cmd.frms)),'Interpreter', 'none'); + end + img_format = 'png'; + if(options.absolute_error.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%03d_%s_%s_absolute_error.%s',2,testname,param.day_seg,img_format)); + ct_saveas(h_fig(1),img_fn); + + elseif(strcmpi(param.layer_tracker.track{1}.method,'lsm')) + data = []; + for dim = 1:length(param.layer_tracker.track{1}.idx_reshape) + temp = []; + for track_idx = 1:length(param.layer_tracker.track) + if isempty(temp) + temp = param.layer_tracker.track{track_idx}.(param.layer_tracker.track{track_idx}.method).(param.layer_tracker.track{track_idx}.idx_dim_name{dim}); + else + if ~any(temp == param.layer_tracker.track{track_idx}.(param.layer_tracker.track{track_idx}.method).(param.layer_tracker.track{track_idx}.idx_dim_name{dim})) + temp(end+1) = param.layer_tracker.track{track_idx}.(param.layer_tracker.track{track_idx}.method).(param.layer_tracker.track{track_idx}.idx_dim_name{dim}); + end + end + + end + data{dim}=temp; + end + for layer_idx = 1:num_layers + res_mat = res_matrix_all_frms{layer_idx}; + temp_points = points{layer_idx}; + + A = nchoosek(1:length(param.layer_tracker.track{1}.idx_reshape),2); % changed from 2 + for id = 1:length(A) + min_points = setdiff(1:length(param.layer_tracker.track{1}.idx_reshape),A(id,:)); + min_idxs = temp_points(min_points); + res_matrix_tot = permute(res_mat,[A(id,:) setdiff(1:length(param.layer_tracker.track{1}.idx_reshape),A(id,:))]); + res_matrix_tot = res_matrix_tot(:,:,min_idxs(:)); + idx = A(id,:); + labelTickX = data{idx(1)}; % x tick label + labelTickY = data{idx(2)}; % y tick label + labelNameX = param.layer_tracker.track{1}.idx_dim_name{idx(1)}; % x label name + labelNameY = param.layer_tracker.track{1}.idx_dim_name{idx(2)}; % y label name + xTick = 1:length(labelTickX); % x tick + yTick = 1:length(labelTickY); % y tick + %% Generate Figure + h_fig(1) = figure('visible','off'); % figure is not displayed + imagesc(squeeze(res_matrix_tot')); + hold on; + plot(temp_points(idx(1)),temp_points(idx(2)),'x','LineWidth',4,'MarkerSize',10,'Color','white'); + + ylabel(labelNameY); + xlabel(labelNameX); + set(gca,'XTickLabel',labelTickX); + set(gca,'YTickLabel',labelTickY); + set(gca,'XTick',xTick); + set(gca,'YTick',yTick); + h_colorbar = colorbar(gca); + set(get(h_colorbar,'YLabel'),'String','Mean absolute error (rows)'); + img_format ='jpg'; + if(options.absolute_error.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_absolute_error_%s_vs_%s_%03d.%s',testname,param.day_seg,labelNameX,labelNameY,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + + end + end + end + end + + +end +save_name = sprintf('%s/result.mat',img_dir); +params = param; +save(save_name,'res_matrix_all','frame_error','range_per','percent_error','season_vect','res_matrix','points','min_val','gps_time','params','idx_matrix','-v7.3'); + + +success = true; + + diff --git a/cresis-toolbox/tracker/layer_tracker_input_check.m b/cresis-toolbox/tracker/layer_tracker_input_check.m new file mode 100644 index 00000000..9ba23b2c --- /dev/null +++ b/cresis-toolbox/tracker/layer_tracker_input_check.m @@ -0,0 +1,474 @@ +% script layer_tracker_input_check +% +% Support function for layer_tracker.m and layer_tracker_task.m that checks +% the inputs. +% +% Authors: Anjali Pare, John Paden +% +% See also: layer_tracker.m, layer_tracker_combine_task.m, +% layer_tracker_task.m, layer_tracker_input_check.m, +% layer_tracker_profile.m, run_layer_tracker.m, run_layer_tracker_tune.m + +%% Input Checks: layer_tracker +% ===================================================================== + +physical_constants; + +% .layer_tracker: parameter structure controlling layer tracker +if ~isfield(param,'layer_tracker') || isempty(param.layer_tracker) + param.layer_tracker = []; +end + +% .block_size_frms: Number of data frames to load and process at a time. +% Default is 1 frame at a time. +if ~isfield(param.layer_tracker,'block_size_frms') || isempty(param.layer_tracker.block_size_frms) + param.layer_tracker.block_size_frms = 1; +end +param.layer_tracker.block_size_frms = min(length(param.cmd.frms), param.layer_tracker.block_size_frms); + +% .copy_param: Final output opsCopyLayer parameter struct +if ~isfield(param.layer_tracker,'copy_param') || isempty(param.layer_tracker.copy_param) + param.layer_tracker.copy_param = []; +end +% .copy_param.copy_method +if ~isfield(param.layer_tracker.copy_param,'copy_method') || isempty(param.layer_tracker.copy_param.copy_method) + param.layer_tracker.copy_param.copy_method = 'overwrite'; +end +% .copy_param.gaps_fill +if ~isfield(param.layer_tracker.copy_param,'gaps_fill') || isempty(param.layer_tracker.copy_param.gaps_fill) + param.layer_tracker.copy_param.gaps_fill = []; +end +% .copy_param.gaps_fill.method +if ~isfield(param.layer_tracker.copy_param.gaps_fill,'method') || isempty(param.layer_tracker.copy_param.gaps_fill.method) + param.layer_tracker.copy_param.gaps_fill.method = 'preserve_gaps'; +end + +% .debug_plots: cell array of strings +if ~isfield(param.layer_tracker,'debug_plots') || isempty(param.layer_tracker.debug_plots) + param.layer_tracker.debug_plots = {}; +end + +if ~isfield(param.layer_tracker,'debug_out_dir') || isempty(param.layer_tracker.debug_out_dir) + param.layer_tracker.debug_out_dir = 'layer_tracker'; +end +debug_out_dir = param.layer_tracker.debug_out_dir; + +% .echogram_img: To choose an image besides the base (0) image +if ~isfield(param.layer_tracker,'echogram_img') || isempty(param.layer_tracker.echogram_img) + param.layer_tracker.echogram_img = 0; +end + +% .echogram_source: string containing location of echogram data used for +% tracking +if ~isfield(param.layer_tracker,'echogram_source') || isempty(param.layer_tracker.echogram_source) + param.layer_tracker.echogram_source = 'qlook'; +end + +if ~isfield(param.layer_tracker,'frm_types') || isempty(param.layer_tracker.frm_types) + param.layer_tracker.frm_types = {-1,-1,-1,-1,-1}; +end + +% .layer_params: layerparams structure of where to store the output using +% opsCopyLayers.m +if ~isfield(param.layer_tracker,'layer_params') || isempty(param.layer_tracker.layer_params) + param.layer_tracker.layer_params = []; +end +if ~isfield(param.layer_tracker.layer_params,'source') || isempty(param.layer_tracker.layer_params.source) + param.layer_tracker.layer_params.source = 'layerdata'; +end +if ~any(strcmpi(param.layer_tracker.layer_params.source,{'ops','layerdata','lidar','records','echogram'})) + error('Unsupported output param.layer_tracker.layer_params.source = ''%s'', but must be one of ''ops'',''layerdata'',''lidar'',''records'',''echogram''.', param.layer_tracker.layer_params.source); +end +if ~isfield(param.layer_tracker.layer_params,'layerdata_source') || isempty(param.layer_tracker.layer_params.layerdata_source) + param.layer_tracker.layer_params.layerdata_source = 'layer'; +end + +% .overlap: nonnegative scalar integer indicating the number of range +% lines to load before and after the current set of frames in order to +% reduce edge effects (if data are not available because this is the first +% or last frame or the data files for the overlapping data are missing, +% then the overlap is set to zero for this side). +if ~isfield(param.layer_tracker,'overlap') || isempty(param.layer_tracker.overlap) + param.layer_tracker.overlap = 0; +end + +% .surf_layer: layer parameter structure for loading a layer with +% opsLoadLayers.m +if ~isfield(param.layer_tracker,'surf_layer') || isempty(param.layer_tracker.surf_layer) + param.layer_tracker.surf_layer = []; +end + +% .track_per_task: Positive integer specifying the number of tracks to +% process per cluster task. Default is inf which means each task will +% process all of the specified param.layer_tracker.track{:} commands. +if ~isfield(param.layer_tracker,'track_per_task') || isempty(param.layer_tracker.track_per_task) + param.layer_tracker.track_per_task = inf; +end + +%% Input Checks: layer_tracker.track field +% ====================================================================== + +if ~isfield(param.layer_tracker,'track') || isempty(param.layer_tracker.track) + param.layer_tracker.track = {}; +end +if isstruct(param.layer_tracker.track) + param.layer_tracker.track = {param.layer_tracker.track}; +end + +for track_idx = 1:length(param.layer_tracker.track) + + track = merge_structs(param.qlook.surf,param.layer_tracker.track{track_idx}); + + % profile: default is the profile named after the system (e.g. accum, + % kaband, kuband, rds, or snow), otherwise loads preset configuration + if ~isfield(track,'profile') || isempty(track.profile) + track.profile = ct_output_dir(param.radar_name); + end + + track = merge_structs(layer_tracker_profile(param,track.profile), track); + + if ~isfield(track,'en') || isempty(track.en) + % If true, tracking will be done on this segment. If false, then no + % tracking is done. Default is true. + track.en = true; + end + if ~track.en + continue; + end + + % .age: override age in output layer (see layerdata for field info) + if ~isfield(track,'age') || isempty(track.age) + track.age = []; + end + % .age_source: override age source in output layer (see layerdata for + % field info) + if ~isfield(track,'age_source') || isempty(track.age_source) + track.age_source = []; + end + + % .compress: double scalar (default is empty), empty matrix disables, + % compress values to a limited range from [0 to .compress] + if ~isfield(track,'compress') || isempty(track.compress) + track.compress = []; + end + + % .crossover: struct controlling crossover loading + if ~isfield(track,'crossover') || isempty(track.crossover) + track.crossover = []; + end + % .crossover.cutoff: scalar integer, default is 50, layer may go plus or + % minus this many bins from the crossover ground truth + if ~isfield(track.crossover,'cutoff') || isempty(track.crossover.cutoff) + track.crossover.cutoff = 50; + end + % .crossover.en: enable loading of crossovers + if ~isfield(track.crossover,'en') || isempty(track.crossover.en) + track.crossover.en = false; + end + % .crossover.gps_time_good_eval: function which returns good/bad based + % on crossover gps time. + if ~isfield(track.crossover,'gps_time_good_eval') || isempty(track.crossover.gps_time_good_eval) + track.crossover.gps_time_good_eval = @(x) true; + end + % .crossover.name: layer name to load crossovers for + if ~isfield(track.crossover,'name') || isempty(track.crossover.name) + track.crossover.name = 'bottom'; + end + % .crossover.season_names_bad: cell array of strings with bad seasons to + % not include in crossovers + if ~isfield(track.crossover,'season_names_bad') || isempty(track.crossover.season_names_bad) + track.crossover.season_names_bad = {}; + end + + if ~isfield(track,'data_noise_en') || isempty(track.data_noise_en) + % If true, then a matrix data_noise will be created which will go through + % all the same operations as data except no "sidelobe" and no "feedthru" + % masking will be done. This is sometimes necessary to enable when + % sidelobe or feedthru remove too much data so that the noise estimatation + % process in the tracker does not function properly. Currently only + % tracker_threshold makes use of data_noise. + track.data_noise_en = false; + end + + % .debug_time_guard: Vertical band around layer in debug plot + if ~isfield(track,'debug_time_guard') || isempty(track.debug_time_guard) + track.debug_time_guard = 50e-9; + end + param.layer_tracker.debug_time_guard = track.debug_time_guard; + + % .desc: override description in output layer (see layerdata for + % field info) + if ~isfield(track,'desc') || isempty(track.desc) + track.desc = []; + end + + % .detrend: optional detrending fields (NEED MORE DESCRIPTION HERE) + if ~isfield(track,'detrend') || isempty(track.detrend) + track.detrend = []; + end + + % .emphasize_last: structure controlling emphasize last bins, default is + % empty which disables + % + % .emphasize_last.threshold: scalar threshold value + % + % .emphasize_last.shift: scalar integer representing the number of bins + % to shift the emphasis scaling (usually a negative) + if ~isfield(track,'emphasize_last') || isempty(track.emphasize_last) + track.emphasize_last = []; + end + + if ~isfield(track,'feedthru') || isempty(track.feedthru) + track.feedthru = []; + end + + if ~isfield(track,'filter') || isempty(track.filter) + track.filter = [1 1]; + end + if length(track.filter) == 1 + warning('Deprecated surf.filter format. Should specify 2 element vector that specifies the multilooks in [cross-track along-track].'); + track.filter = [1 track.filter(1)]; + end + if any(mod(track.filter,2) == 0) + error('Surface filter lengths must be odd. layer_tracker.track.filter = [%d %d].', layer_tracker.track.filter); + end + + if ~isfield(track,'filter_trim') || isempty(track.filter_trim) + track.filter_trim = [0 0]; + end + + if ~isfield(track,'fixed_value') || isempty(track.fixed_value) + track.fixed_value = 0; + end + + % .ground_truth: struct controlling ground truth loading + if ~isfield(track,'ground_truth') || isempty(track.ground_truth) + track.ground_truth = []; + end + % .ground_truth.en: enable loading of ground truth + if ~isfield(track.ground_truth,'en') || isempty(track.ground_truth.en) + track.ground_truth.en = false; + end + % .ground_truth.layers: layer struct array to load ground truth from + if ~isfield(track.ground_truth,'layers') || isempty(track.ground_truth.layers) + track.ground_truth.layers = []; + end + % .ground_truth.cutoff: integer vector the same length as + % .ground_truth.layers, default is 100, layer may go plus or minus this + % many bins from the ground truth + if ~isfield(track.ground_truth,'cutoff') || isempty(track.ground_truth.cutoff) + track.ground_truth.cutoff = 50; + end + if length(track.ground_truth.cutoff) == 1 + % If only 1 element specified, then assume the same cutoff should be + % used for all ground truth layers. + track.ground_truth.cutoff = track.ground_truth.cutoff*ones(size(track.ground_truth.layers)); + end + if numel(track.ground_truth.layers) ~= numel(track.ground_truth.cutoff) + error('The number of ground truth layers must match the number of elements in the cutoff vector. numel(track.ground_truth.layers)=%d ~= numel(track.ground_truth.cutoff)=%d.',numel(track.ground_truth.layers),numel(track.ground_truth.cutoff)); + end + + % .group_name: override group name in output layer (see layerdata for + % field info) + if ~isfield(track,'group_name') || isempty(track.group_name) + track.group_name = []; + end + + % .ice_mask: struct controlling ice mask loading + if ~isfield(track,'ice_mask') || isempty(track.ice_mask) + track.ice_mask = []; + end + % .ice_mask.en: enable loading of ice mask + if ~isfield(track.ice_mask,'en') || isempty(track.ice_mask.en) + track.ice_mask.en = false; + end + + if ~isfield(track,'init') || isempty(track.init) + track.init = []; + end + if ~isfield(track.init,'method') || isempty(track.init.method) + track.init.method = 'max'; + end + if ~isfield(track.init,'snake_rng') || isempty(track.init.snake_rng) + track.init.snake_rng = [-2e-7 2e-7]; + end + if ~isfield(track.init,'dem_layer') || isempty(track.init.dem_layer) + track.init.dem_layer = ''; + end + if isfield(track.init.dem_layer,'source') ... + && ~any(strcmpi(track.init.dem_layer.source,{'ops','layerdata','lidar','records','echogram'})) + error('Unsupported output track.init.dem_layer.source = ''%s'', but must be one of ''ops'',''layerdata'',''lidar'',''records'',''echogram''.', track.init.dem_layer.source); + end + if ~any(strcmpi(track.init.method,{'max','nan','snake','medfilt','dem'})) + error('Unsupported surface init method %s. Options are max, nan, snake, medfilt, or dem. max is default.', track.init.method); + end + if ~isfield(track.init,'max_diff') || isempty(track.init.max_diff) + track.init.max_diff = inf; + end + if ~isfield(track.init,'max_diff_method') || isempty(track.init.max_diff_method) + if strcmpi(track.init.method,{'dem'}) || ~isempty(track.init.dem_layer) + % If the initial surface is from a dem or reference layer, the default + % method for outliers is to use merge_vectors to fill the outliers in. + track.init.max_diff_method = 'merge_vectors'; + else + % Otherwise the default method is just to interpolate between the good + % points that exist to fill the outliers in. + track.init.max_diff_method = 'interp_finite'; + end + end + if ~any(strcmpi(track.init.max_diff_method,{'merge_vectors','interp_finite'})) + error('Unsupported max diff method %s. Options are merge_vectors, interp_finite. The default is interp_finite unless a dem or reference layer is provided.', track.init.max_diff_method); + end + + if ~isfield(track,'lsm') || isempty(track.lsm) + track.lsm = []; + end + % lsm.use_mean_surf_en: logical scalar, default false. If true, tracker + % uses the mean surface to initialize lsm.y + if ~isfield(track.lsm,'use_mean_surf_en') || isempty(track.lsm.use_mean_surf_en) + track.lsm.use_mean_surf_en = false; + end + % lsm.y: initial surface layer bin + if ~isfield(track.lsm,'y') || isempty(track.lsm.y) + track.lsm.y = 1; + end + + if ~isfield(track,'max_bin') || isempty(track.max_bin) + track.max_bin = inf; + elseif isstruct(track.max_bin) + if ~isfield(track.max_bin,'existence_check') + track.max_bin.existence_check = false; + end + end + + % max_rng: two element numeric vector, default is [0 0]. After the layer + % is tracked, this allows the tracked layer to be updated with the + % maximum value in a window around the tracked layer. Specifies the range + % of bins [start stop] to search for a maximum relative to the tracked + % layer. Negative values are before the tracked layer and positive values + % are after the tracked layer. To not update the tracked layer set this + % to [0 0]. A common use is to use the threshold tracker and then follow + % this with a positive range to search for a peak after the treshold was + % exceeded. For example, [0 0.1e-6] with max_rng_units of 'time' would + % cause the layer to be updated to the location of the maximum value that + % occurs in the range from where the threshold was exceeded to 0.1e-6 + % seconds after it was exceeded. + if ~isfield(track,'max_rng') || isempty(track.max_rng) + track.max_rng = [0 0]; + end + + if ~isfield(track,'max_rng_filter') || isempty(track.max_rng_filter) + track.max_rng_filter = track.filter; + end + + % max_rng_units: string, default 'time'. Specifies the units of max_rng. + % Options are 'time' (units of seconds two way travel time) and 'bins' + % (range bins or rows). + if ~isfield(track,'max_rng_units') || isempty(track.max_rng_units) + track.max_rng_units = 'time'; + end + + if ~isfield(track,'medfilt') || isempty(track.medfilt) + % Like medfilt1 except it handles the edges of the frame better + track.medfilt = 0; + end + if ~isfield(track,'medfilt_threshold') || isempty(track.medfilt_threshold) + % medfilt_threshold: the point is compared to the medfilt1 result and if + % the point is > medfilt_threshold from medfilt1, then the point is + % replaced with the medfilt1 result. The two extremes are: + % 0 makes medfilt act like medfilt1 + % inf effectively disables the medfilt operation + track.medfilt_threshold = 0; + end + + if ~isfield(track,'method') || isempty(track.method) + track.method = ''; + end + + if ~isfield(track,'min_bin') || isempty(track.min_bin) + track.min_bin = 0; + elseif isstruct(track.min_bin) + if ~isfield(track.min_bin,'existence_check') + track.min_bin.existence_check = false; + end + end + + % .mult_suppress: struct controlling surface multiple suppression + if ~isfield(track,'mult_suppress') || isempty(track.mult_suppress) + track.mult_suppress = []; + end + % .mult_suppress.en: enable surface multiple suppression + if ~isfield(track.mult_suppress,'en') || isempty(track.mult_suppress.en) + track.mult_suppress.en = false; + end + % .mult_suppress.param: param field to pass to mult_suppress.m + if ~isfield(track.mult_suppress,'param') || isempty(track.mult_suppress.param) + track.mult_suppress.param = []; + end + + if ~isfield(track,'name') || isempty(track.name) + track.name = sprintf('t%03d', track_idx); + end + + % prefilter_trim: two element numeric vector of nonnegative numbers, + % default is [0 0]. The first values specifies how much to trim off the + % start of each range line before any processing occurs and the second + % number specifies how much to trim off the end of each range line. + % Useful, when it is known that there are artifacts at the start/stop of + % each range line that will confuse the tracker and the layer to be + % tracked does not go into these regions of the image. The default value + % [0 0] effectively disables the prefilter_trim since nothing will be + % trimmed off. + if ~isfield(track,'prefilter_trim') || isempty(track.prefilter_trim) + track.prefilter_trim = [0 0]; + end + + % prefilter_trim_units: string, default 'time'. Specifies the units of + % prefilter_trim. Options are 'time' (units of seconds two way travel + % time) and 'bins' (range bins or rows). + if ~isfield(track,'prefilter_trim_units') || isempty(track.prefilter_trim_units) + track.prefilter_trim_units = 'time'; + end + + if ~isfield(track,'sidelobe_rows') || isempty(track.sidelobe_rows) || ~isfield(track,'sidelobe_dB') || isempty(track.sidelobe_dB) + track.sidelobe_dB = []; + track.sidelobe_rows = []; + end + + if ~isfield(track,'smooth_sgolayfilt') || isempty(track.smooth_sgolayfilt) + track.smooth_sgolayfilt = []; + end + + if ~isfield(track,'snake_rng') || isempty(track.snake_rng) + track.snake_rng = [-2e-7 2e-7]; + end + + % .surf_suppress: structure controlling surface suppression. Default is + % []. Empty matrix disables surface suppression. + % + % .eval_cmd: evaluation + if ~isfield(track,'surf_suppress') || isempty(track.surf_suppress) + track.surf_suppress = []; + end + + if ~isfield(track,'threshold') || isempty(track.threshold) + track.threshold = 15; + end + + if ~isfield(track,'threshold_noise_rng') || isempty(track.threshold_noise_rng) + track.threshold_noise_rng = [0 -inf -1]; + end + + % viterbi: structure controlling operation of the viterbi method tracker + if ~isfield(track,'viterbi') || isempty(track.viterbi) + track.viterbi = []; + end + + % viterbi.transition_weight: controls the smoothness weighting, larger + % numbers cause the layer to be smoother + if ~isfield(track.viterbi,'transition_weight') || isempty(track.viterbi.transition_weight) + track.viterbi.transition_weight = 1; + end + + param.layer_tracker.track{track_idx} = track; +end diff --git a/cresis-toolbox/tracker/layer_tracker_profile.m b/cresis-toolbox/tracker/layer_tracker_profile.m new file mode 100644 index 00000000..1762b803 --- /dev/null +++ b/cresis-toolbox/tracker/layer_tracker_profile.m @@ -0,0 +1,261 @@ +function track = layer_tracker_profile(param,profile_str) +% track = layer_tracker_profile(param,profile_str) +% +% Returns a default set of tracking parameters for layer tracking with +% layer_tracker.m. +% +% param: parameter spreadsheet structure +% profile_str: optional string containing the profile name to load (default +% is "default" +% +% track: Parameters used to control the tracker. See layer_tracker.m. +% +% Example: +% Should only be used from layer_tracker.m +% +% Authors: Anjali Pare, John Paden +% +% See also: layer_tracker.m, layer_tracker_combine_task.m, +% layer_tracker_task.m, layer_tracker_input_check.m, +% layer_tracker_profile.m, run_layer_tracker.m, run_layer_tracker_tune.m + +%% Check input arguments +if ~exist('profile_str','var') || isempty(profile_str) + profile_str = 'default'; +end +track.profile = profile_str; + +%% Default track settings +track.data_noise_en = false; +track.detrend = []; +track.en = true; +track.init = []; +track.init.method = 'max'; +track.init.snake_rng = [-2e-7 2e-7]; +track.init.dem_layer = ''; +track.init.dem_offset = 0; +track.init.dem_layer_offset = 0; +track.init.max_diff = inf; +track.init.max_diff_method = 'interp_finite'; +track.filter = [1 1]; +track.filter_trim = [0 0]; +track.fixed_value = 0; +track.min_bin = 0; +track.max_bin = inf; +track.max_rng = [0 0]; +track.max_rng_units = 'time'; +track.medfilt = 0; +track.medfilt_threshold = 0; +track.method = 'threshold'; +track.prefilter_trim = [0 0]; +track.sidelobe_dB = []; +track.sidelobe_rows = []; +track.snake_rng = [-2e-7 2e-7]; +track.threshold = 15; +track.threshold_noise_rng = [0 -inf -1]; + +if strcmpi(profile_str,'default') + %% Default profile + +elseif strcmpi(profile_str,'ACCUM') + %% ACCUM profile + track.debug_time_guard = 2e-6; + track.filter = [3 3]; + track.filter_trim = [0 3]; + track.init.method = 'medfilt'; + track.init.medfilt = 11; + track.init.max_diff = 1e-6;%0.5e-6; + track.max_rng = [0 3]; + track.max_rng_units = 'bins'; + track.medfilt = 3;%11; + track.medfilt_threshold = 10;%30; + track.method = 'threshold'; + track.min_bin = 0;%1.6e-6; + track.threshold = 5;%10; + track.threshold_noise_rng = [0 -1e-6 -0.3e-6]; + track.threshold_rel_max = -9; + track.threshold_rng = 5; + +elseif strcmpi(profile_str,'KABAND') + %% KABAND profile + track.debug_time_guard = 50e-9; + track.min_bin = 0.1e-6; + track.prefilter_trim = [0 0]; + track.filter = [5 3]; + track.filter_trim = [10 10]; + track.init.method = 'medfilt'; + track.init.medfilt = 51; + track.init.max_diff = 0.3e-6; + track.max_rng = [0 9]; + track.max_rng_units = 'bins'; + track.medfilt = 11; + track.medfilt_threshold = 100; + track.method = 'threshold'; + track.threshold = 8; + track.threshold_noise_rng = [15e-9 -75e-9 -30e-9]; + track.threshold_rel_max = -9; + +elseif strcmpi(profile_str,'KUBAND') + %% KUBAND profile + track.debug_time_guard = 50e-9; + track.min_bin = 0.1e-6; + track.prefilter_trim = [0 0]; + track.filter = [5 3]; + track.filter_trim = [10 10]; + track.init.method = 'medfilt'; + track.init.medfilt = 51; + track.init.max_diff = 0.3e-6; + track.max_rng = [0 9]; + track.max_rng_units = 'bins'; + track.medfilt = 11; + track.medfilt_threshold = 100; + track.method = 'threshold'; + track.threshold = 8; + track.threshold_noise_rng = [15e-9 -75e-9 -30e-9]; + track.threshold_rel_max = -9; + +elseif strcmpi(profile_str,'RDS') + %% RDS profile + track.debug_time_guard = 2e-6; + track.filter = [1 5]; + track.init.method = 'medfilt'; + track.init.medfilt = 11; + track.init.max_diff = 0.5e-6; + track.max_rng = [0 1]; + track.max_rng_units = 'bins'; + track.medfilt = 11; + track.medfilt_threshold = 30; + track.method = 'threshold'; + track.min_bin = 1.6e-6; + track.threshold = 10; + track.threshold_noise_rng = [0 -2e-6 -0.2e-6]; + track.threshold_rel_max = -9; + track.threshold_rng = 5; + +elseif strcmpi(profile_str,'RDS_OIB') + %% RDS_OIB profile + track.debug_time_guard = 2e-6; + track.filter = [1 5]; + track.init.method = 'medfilt'; + track.init.medfilt = 11; + track.init.max_diff = 0.5e-6; + track.max_rng = [0 1]; + track.max_rng_units = 'bins'; + track.medfilt = 11; + track.medfilt_threshold = 30; + track.method = 'threshold'; + track.min_bin = 1.6e-6; + track.threshold = 10; + track.threshold_noise_rng = [0 -2e-6 -0.2e-6]; + track.threshold_rel_max = -9; + track.threshold_rng = 5; + +elseif strcmpi(profile_str,'SNOW') + %% SNOW profile + track.debug_time_guard = 50e-9; + track.min_bin = 0.1e-6; + track.prefilter_trim = [0 0]; + track.filter = [5 3]; + track.filter_trim = [10 10]; + track.init.method = 'medfilt'; + track.init.medfilt = 51; + track.init.max_diff = 0.3e-6; + track.max_rng = [0 9]; + track.max_rng_units = 'bins'; + track.medfilt = 11; + track.medfilt_threshold = 100; + track.method = 'threshold'; + track.threshold = 8; + track.threshold_noise_rng = [15e-9 -75e-9 -30e-9]; + track.threshold_rel_max = -9; + +elseif strcmpi(profile_str,'SNOW_FLAT_STEP1') + %% SNOW FLAT step 1 of 2 profile + % Use param.layer_tracker.overlap == 50 and track.layer_names = {'surface_flatten'} + track.layer_names = {'surface_flatten'}; + track.debug_time_guard = 50e-9; + track.min_bin = 0.1e-6; + track.prefilter_trim = [0 0]; + track.filter = [5 7]; % Use longer filter + track.filter_trim = [10 10]; + track.init.method = 'dem'; + track.init.dem_layer.name = 'surface'; + track.init.dem_layer.source = 'lidar'; + track.init.dem_layer.lidar_source = 'atm'; + track.init.dem_layer.lever_arm_en = true; + track.init.max_diff_method = 'merge_vectors'; + track.init.max_diff = 0.3e-6; + track.max_rng = [0 9]; + track.max_rng_units = 'bins'; + track.medfilt = 11; + track.medfilt_threshold = 0; % Median filter more tightly + track.method = 'threshold'; + track.threshold = 8; + track.threshold_noise_rng = [15e-9 -75e-9 -30e-9]; + track.threshold_rel_max = -21; % Allow for bright shallow layers + track.max_rng_filter = [5 3]; % During max_rng, use less filtered data to track the surface more closely + track.smooth_sgolayfilt = {3,41}; % Smooth layer + +elseif strcmpi(profile_str,'SNOW_FLAT_STEP2') + %% SNOW FLAT step 2 of 2 profile + track.debug_time_guard = 50e-9; + track.min_bin = 0.1e-6; + track.prefilter_trim = [0 0]; + track.min_bin = struct('name','surface_flatten','eval',struct('cmd','s=fir_dec(s,ones(1,41)/41,1)-150e-9;')); + track.max_bin = struct('name','surface_flatten','eval',struct('cmd','s=fir_dec(s,ones(1,41)/41,1)+80e-9;')); + track.filter = [1 19]; % Use longer filter because of surface flattening + track.filter_trim = [10 10]; + track.flatten = struct('name','surface_flatten'); + track.init.method = 'nan'; + track.init.dem_layer.name = 'surface_flatten'; + track.init.dem_layer.source = 'layerdata'; + track.init.max_diff_method = 'merge_vectors'; + track.max_rng = [0 9]; + track.max_rng_filter = [1 5]; % During max_rng, use less filtered data to track the surface more closely + track.max_rng_units = 'bins'; + if 0 + track.xcorr = echo_xcorr_profile('snow'); + track.method = 'viterbi'; + else + track.method = 'threshold'; + track.threshold = 8; + track.threshold_noise_rng = [15e-9 -75e-9 -30e-9]; + track.threshold_rel_max = -21; % Allow for bright shallow layers + end + +elseif strcmpi(profile_str,'SNOW_AWI') + %% SNOW_AWI profile + track.debug_time_guard = 50e-9; + track.min_bin = 0.1e-6; + track.prefilter_trim = [0 0]; + track.filter = [5 3]; + track.filter_trim = [10 10]; + track.init.method = 'medfilt'; + track.init.medfilt = 51; + track.init.max_diff = 0.3e-6; + track.max_rng = [0 9]; + track.max_rng_units = 'bins'; + track.medfilt = 11; + track.medfilt_threshold = 100; + track.method = 'threshold'; + track.threshold = 8; + track.threshold_noise_rng = [15e-9 -75e-9 -30e-9]; + track.threshold_rel_max = -9; + +elseif strcmpi(profile_str,'DEM_LIDAR') + %% DEM profile + track.debug_time_guard = 100e-9; + track.min_bin = 0e-6; + track.init.method = 'dem'; + track.init.dem_layer.name = 'surface'; + track.init.dem_layer.source = 'lidar'; + track.init.dem_layer.lidar_source = 'atm'; + track.init.dem_layer.lever_arm_en = true; + track.init.max_diff_method = 'merge_vectors'; + track.init.max_diff = 0e-6; % Force output layer to equal DEM + track.max_rng = [0 0]; + track.method = ''; % No data dependent tracking + +else + error('Invalid profile selected: %s\n', profile_str); +end diff --git a/cresis-toolbox/tracker/layer_tracker_stat_generator.m b/cresis-toolbox/tracker/layer_tracker_stat_generator.m new file mode 100644 index 00000000..8f402bf6 --- /dev/null +++ b/cresis-toolbox/tracker/layer_tracker_stat_generator.m @@ -0,0 +1,523 @@ +%% Script layer_tracker_tune_layerdata.m + +% Tuning script to work with layerdata files rather than temporary files +% Used to find best combination of paramters for given tracker method +% Enter layer names in the gt_layer_params(idx).name field +% set temp to the file location of the param structure used for tracking in run_layer_tracker.m script +% set save_name to store final tuning results +% Statistical data saved: +% number of points with finite values of ground truth of twtt (num_gt_infinite), +% number of points where ground truth twtt is finite and tracked data twtt is NaN (num_isnan) +% number of points where (tracked data twtt - ground truth twtt) < 5 * dt (num_points) +% nanmean of (tracked data twtt - ground truth twtt) (res_matrix) +% Authors: Anjali Pare, John Paden +% See layer_tracker_tune_plot.m to view 2 dimensional imagesc plots of the data. + +% Look at % correct. If we're close to the right ans 100 % of the time but +% not quite right 100 % of the time is worse than being right 90% of the +% time and completely wrong 10 % of the time. Need to correct in quality +% control. Check if you are outside of a wondow and label as bad. % correct +% should be exactly the same (not necesary). There could be offsets. +% 1-2 range bin. + +% Have fig option for all. Use hist instead of histogram. Histogram plots +% separate from % correct graphs. Enable option to generate best histogram +% plot. Plot as regular line using hist + +% What to plot, finding best combination, trying with LSM + +dbstack_info = dbstack; +fprintf('=====================================================================\n'); +fprintf('%s (%s)\n', dbstack_info(1).name, datestr(now,'HH:MM:SS')); +fprintf('=====================================================================\n'); + +img_dir = '/cresis/snfs1/scratch/anjali/cluster_tuning/stat_generator'; +if ~exist(img_dir,'dir') + mkdir(img_dir); +end +%% Setup +%20140516_01_20210107_163106_t005_viterbi +%20140313_08_20210107_162824_t005_viterbi +%temp = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140516_01_20201122_193803_t005_viterbi.mat'); + +% Example to run 1 season with 1 segment and frames 1:5 +params_all = []; +idx_segment = 1; +params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls'),'20140516_01'); +params(idx_segment).cmd.generic = 1; +params(idx_segment).cmd.frms = [40 41]; +params_all{end+1} = params; + +% Example to run 1 season with 2 segment and different frames +% params_all = []; +% idx_segment = 1; +% params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls'),'20140313_08|20140516_01'); +% params(idx_segment).cmd.generic = 1; +% params(idx_segment).cmd.frms = 1:2; +% idx_segment = idx_segment + 1; +% params(idx_segment).cmd.generic = 1; +% params(idx_segment).cmd.frms = 3:4; +% params_all{end+1} = params; + +% Example to run 2 season with 1 segment frames 1:5 +% params_all = []; +% idx_segment = 1; +% params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls'),'20140516_01'); +% params(idx_segment).cmd.generic = 1; +% params(idx_segment).cmd.frms = 1:5; +% params_all{end+1} = params; + +% idx_segment = 1; +% params = read_param_xls(ct_filename_param('rds_param_2014_Antarctica_DC8.xls'),'20141122_08'); +% idx_segment = idx_segment + 1; +% params(idx_segment).cmd.generic = 1; +% params(idx_segment).cmd.frms = 1:5; +% params_all{end+1} = params; + + +% Example to run all segments and all frames of a season +% params_all = []; +% idx_segment = 1; +% params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls')); +% params = ct_set_params(params,'cmd.generic',1); +% params(idx_segment).cmd.frms = []; + +temp = []; +temp{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140516_01_20201202_020912_t005_viterbi.mat'); +%temp{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140516_01_20210124_024502_t004_lsm.mat'); +% two segments +%temp{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140313_08_20210107_162824_t005_viterbi.mat'); +%temp{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140516_01_20210107_163106_t005_viterbi.mat'); + +% which plots are wanted +options.segment_vs_frame.plot = true; +options.segment_vs_frame.fig_format = false; +options.season_vs_gps.plot = true; +options.season_vs_gps.fig_format = true; +options.absolute_error.plot = true; +options.absolute_error.fig_format = false; +options.percentage_correct.plot = true; +options.percentage_correct.fig_format = false; +options.hist_generation.plot = true; +options.hist_generation.fig_format = false; + +save_name = '/cresis/snfs1/scratch/anjali/cluster_tuning/result_layer_tune_vit_s015'; % where to store tuning final result + +gt_layer_params = []; +layer_params = []; +res_matrix = []; +res_list = []; +gps_time = []; +num_layers = 1; +range = 10; +idx = 1; +%gt_layer_params(idx).name = 'surface'; +%idx = idx + 1; +gt_layer_params(idx).name = 'bottom'; % specify layer names + + +% run_all; % uncomment the seasons you want first +% params_all = []; +% for param_fns_idx = 1:length(param_fns) +% param_fn = param_fns{param_fns_idx}; +% param = regexp(param_fn,'(?\w+)_param_(?\w+)','names'); +% [output_dir,radar_type,radar_name] = ct_output_dir(param.radar_name); +% param_fn = ct_filename_param(sprintf('%s_param_%s.xls', output_dir, param.season_name)); +% params = read_param_xls(param_fn,'20140516_01'); +% +% params = ct_set_params(params,'cmd.generic',0); +% params = ct_set_params(params,'cmd.generic',1,'day_seg',segment_list{param_fns_idx}); +% params = ct_set_params(params,'cmd.frms',frm_list{param_fns_idx}); %% Specify specific frames (or leave empty/undefined to do all frames) +% params_all{end+1} = params; +% end +season_vect = []; +segment_vect = []; +frame_vect = []; + +seg_idx = 0; +for season_idx = 1:length(params_all) %seasons + params = params_all{season_idx}; + + for param_idx = 1:length(params) % segments + param = params(param_idx); + + if isfield(param.cmd,'generic') && ~iscell(param.cmd.generic) && ~ischar(param.cmd.generic) && param.cmd.generic + seg_idx = seg_idx + 1; + param_override = temp{seg_idx}.param; + param = merge_structs(param,param_override); + total_length = 0; + layers = opsLoadLayers(param,gt_layer_params); + idx_matrix = [(1:length(param.layer_tracker.track{1}.idx_reshape))+1 1]; % used for resizing + + for track_idx = 1:length(param.layer_tracker.track) + range_percent_all_frms.(sprintf('%s', param.layer_tracker.track{track_idx}.name)) = 0; + range_per{track_idx} = []; + percent_error(track_idx) = 0; + for frm_idx = 1:length(param.cmd.frms) + range_percent.(sprintf('%s_%03d', param.layer_tracker.track{track_idx}.name,frm_idx)) = 0; + prob_error.(sprintf('%s_%03d', param.layer_tracker.track{track_idx}.name,frm_idx)) = 0; + frame_error{track_idx}(frm_idx) = 0; + end + end + frm_idx = 0; + for frm = param.cmd.frms + frm_idx = frm_idx+1; + data_fn_dir = ct_filename_out(param, param.layer_tracker.echogram_source, ''); + data_fn_name = sprintf('Data_%s_%03d.mat',param.day_seg,frm); + data_fn = fullfile(data_fn_dir, data_fn_name); + data = load(data_fn, 'GPS_time', 'Time'); + gps_time = cat(2,gps_time,data.GPS_time); + dt = data.Time(2) - data.Time(1); + total_length = total_length + length(data.GPS_time); + idx_list = 1; + + frm_length = length(data.GPS_time); + for j = 1:length(data.GPS_time) + season_vect(end+1) = season_idx; + segment_vect(end+1) = param_idx; + frame_vect(end+1) = frm; + end + + for track_idx = 1:length(param.layer_tracker.track) + + for layer_idx = 1:length(gt_layer_params) + layer_params(layer_idx).name = sprintf('%s_%s_%s',param.layer_tracker.track{track_idx}.name,param.layer_tracker.track{track_idx}.method,gt_layer_params(layer_idx).name); + layer_params(layer_idx).source = param.layer_tracker.layer_params.source; + layer_params(layer_idx).layerdata_source = param.layer_tracker.layer_params.layerdata_source; + layers_new = opsLoadLayers(param,layer_params); + + surf = interp1(layers(layer_idx).gps_time,layers(layer_idx).twtt,data.GPS_time); + surf_bins = round(interp1(data.Time,1:length(data.Time),surf)); + + for pos = 1:length(layers_new) + surf = interp1(layers_new(pos).gps_time,layers_new(pos).twtt,data.GPS_time); + surf_bins_itr = round(interp1(data.Time,1:length(data.Time),surf)); + res_list(frm_idx,idx_list) = nanmean(abs(surf_bins - surf_bins_itr)); + res_matrix_all{layer_idx}(frm_idx,param.layer_tracker.track{track_idx}.idx(pos)) = nanmean(abs(surf_bins - surf_bins_itr)); + counter = 0; + for i = 1:length(surf_bins) + try + res_matrix{layer_idx}{param.layer_tracker.track{track_idx}.idx(pos)}(end+1) = surf_bins(i) - surf_bins_itr(i); + catch ME + res_matrix{layer_idx}{param.layer_tracker.track{track_idx}.idx(pos)}(i) = surf_bins(i) - surf_bins_itr(i); + end + if (abs(surf_bins(i) - surf_bins_itr(i)) <= range) + counter = counter + 1; + end + end + range_percent.(sprintf('%s_%03d', param.layer_tracker.track{track_idx}.name,frm)) = (counter/length(surf_bins))*100; + range_per{track_idx}(end+1) = (counter/length(surf_bins))*100; + percent_error(track_idx) = counter + percent_error(track_idx); + prob_error.(sprintf('%s_%03d', param.layer_tracker.track{track_idx}.name,frm)) = counter*100; + frame_error{track_idx}(frm_idx) = nanmean(abs(surf_bins - surf_bins_itr)); + try + range_percent_all_frms.(sprintf('%s', param.layer_tracker.track{track_idx}.name)) = range_percent_all_frms.(sprintf('%s', param.layer_tracker.track{track_idx}.name)) + counter; + catch ME + range_percent_all_frms.(sprintf('%s', param.layer_tracker.track{track_idx}.name)) = counter; + end + end + end + idx_list = idx_list + 1; + end + end + + %Histogram stuff + % hist_list = []; + % + % for hist_track_idx = 1:length(param.layer_tracker.track) + % prob_error_list{hist_track_idx} = []; + % range_percent_all_frms.(sprintf('%s', param.layer_tracker.track{hist_track_idx}.name)) = range_percent_all_frms.(sprintf('%s', param.layer_tracker.track{hist_track_idx}.name))/length(param.cmd.frms); + % end + % + % for hist_track_idx = 1:length(param.layer_tracker.track) + % for hist_frm_idx = param.cmd.frms + % hist_list(end+1) = range_percent.(sprintf('%s_%03d', param.layer_tracker.track{hist_track_idx}.name,hist_frm_idx)); + % prob_error.(sprintf('%s_%03d', param.layer_tracker.track{hist_track_idx}.name,hist_frm_idx)) = prob_error.(sprintf('%s_%03d', param.layer_tracker.track{hist_track_idx}.name,hist_frm_idx))/total_length; + % prob_error_list{hist_track_idx}(end+1) = prob_error.(sprintf('%s_%03d', param.layer_tracker.track{hist_track_idx}.name,hist_frm_idx)); + % end + % %figure; + % %h = histogram(hist_list); + % end + points = []; + min_val = []; + res_matrix_all_frms = []; + for layer_names = 1:num_layers + res_matrix_all_frms{layer_names} = nanmean(res_matrix_all{layer_names},1); + res_matrix_all_frms{layer_names} = permute(res_matrix_all_frms{layer_names},idx_matrix); + [min_val{layer_names},i]=min(res_matrix_all_frms{layer_names}(:)); + sizeMatrix = size(res_matrix_all_frms{layer_names}); + for dim = 1:length(size(res_matrix_all_frms{layer_names})) + points{layer_names}(dim) = mod(floor((i-1)/prod(sizeMatrix(1:dim-1))),sizeMatrix(dim))+1; + end + if (length(size(res_matrix_all_frms{layer_names})) < length(param.layer_tracker.track{1}.idx_reshape)) + points{layer_names}(dim+1) = 1; + end + end + + % per segment vs frame + if (options.segment_vs_frame.plot) + h_fig(1) = figure; + for fig_idx = 1:length(param.layer_tracker.track) + hold on; + plot(frame_error{fig_idx},'DisplayName',sprintf('tracker%d',fig_idx)); + legend('-DynamicLegend','Location','northeastoutside'); + labelNameX = 'frames'; + labelNameY = 'absolute error'; + ylabel(labelNameY); + xlabel(labelNameX); + title(sprintf('%s frames: %03d',param.day_seg,length(param.cmd.frms)),'Interpreter', 'none'); + end + hold off; + img_format = 'png'; + if(options.segment_vs_frame.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%03d_%s_%s_segment_vs_frame.%s',2,testname,param.day_seg,img_format)); + ct_saveas(h_fig(1),img_fn); + end + + if (options.percentage_correct.plot) + h_fig(1) = figure; + for hist_idx = 1:length(param.layer_tracker.track) + hold on; + plot(range_per{hist_idx},'DisplayName',sprintf('tracker%d',hist_idx)); + legend('-DynamicLegend','Location','northeastoutside'); + xlabel('Number of frames'); + ylabel('Percentage error'); + title(param.day_seg,'Interpreter', 'none'); + percent_error(hist_idx) = percent_error(hist_idx)/total_length * 100; + end + img_format = 'png'; + if(options.percentage_correct.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%03d_%s_%s_percentage_correct.%s',2,testname,param.day_seg,img_format)); + ct_saveas(h_fig(1),img_fn); + + if (options.hist_generation.plot) + h_fig(1) = figure; + % for hist_idx = 1:length(param.layer_tracker.track) + % hold on; + % plot(hist(range_per{hist_idx}),'DisplayName',sprintf('tracker%d',hist_idx)); + % legend('-DynamicLegend'); + % ylabel('Number of frames'); + % xlabel('Bins'); + % title(param.day_seg,'Interpreter', 'none'); + % percent_error(hist_idx) = percent_error(hist_idx)/total_length * 100; + % end + for hist_idx = 1:length(param.layer_tracker.track) + hold on; + [N,x] = hist(abs(res_matrix{1}{hist_idx})); + plot(x,N,'DisplayName',sprintf('tracker%d',hist_idx)); + legend('-DynamicLegend','Location','northeastoutside'); + ylabel('Data points'); + xlabel('Absolute error'); + title(sprintf('%s frames: %03d',param.day_seg,length(param.cmd.frms)),'Interpreter', 'none'); + end + hold off; + img_format = 'png'; + if(options.hist_generation.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%03d_%s_%s_histogram_generation.%s',2,testname,param.day_seg,img_format)); + ct_saveas(h_fig(1),img_fn); + end + + h_fig(1) = figure; + plot(percent_error); + hold on; + plot(percent_error, 'r*'); + ylabel('Percentage correct'); + xlabel('Trackers'); + title(param.day_seg,'Interpreter', 'none'); + set(gca,'XTick',1:length(param.layer_tracker.track)); + img_format = 'png'; + if(options.percentage_correct.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%03d_%s_%s_total_percent_correct.%s',2,testname,param.day_seg,img_format)); + ct_saveas(h_fig(1),img_fn); + end + + if(options.absolute_error.plot) + if(strcmpi(param.layer_tracker.track{1}.method,'viterbi')) + for layer_idx = 1:num_layers + res_mat = res_matrix_all_frms{layer_idx}; + points_new = points{layer_idx}; + x = 1:length(res_mat); % x tick + h_fig(1) = figure; + plot(x,res_mat); + hold on; + for i = 1:length(res_mat) + plot(x(i),res_mat(i), 'r*'); + end + plot(x(points_new(1)),res_mat(points_new(1)), 'ko','LineWidth',4); + set(gca,'XTick',x); + ylabel('Mean absolute error'); + xlabel('Trackers'); + title(sprintf('%s frames: %03d',param.day_seg,length(param.cmd.frms)),'Interpreter', 'none'); + end + img_format = 'png'; + if(options.absolute_error.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%03d_%s_%s_absolute_error.%s',2,testname,param.day_seg,img_format)); + ct_saveas(h_fig(1),img_fn); + elseif(strcmpi(param.layer_tracker.track{1}.method,'lsm')) + for layer_idx = 1:num_layers + res_mat = res_matrix_all_frms{layer_idx}; + points = points{layer_idx}; + + A = nchoosek(1:length(param.layer_tracker.track{1}.idx_reshape),2); % changed from 2 + for id = 1:length(A) + min_points = setdiff(1:length(param.layer_tracker.track{1}.idx_reshape),A(id,:)); + min_idxs = points(min_points); + res_matrix = permute(res_mat,[A(id,:) setdiff(1:length(param.layer_tracker.track{1}.idx_reshape),A(id,:))]); + res_matrix = res_matrix(:,:,min_idxs(:)); + idx = A(id,:); + labelTickX = data{idx(1)}; % x tick label + labelTickY = data{idx(2)}; % y tick label + labelNameX = param.layer_tracker.track{1}.idx_dim_name{idx(1)}; % x label name + labelNameY = param.layer_tracker.track{1}.idx_dim_name{idx(2)}; % y label name + xTick = 1:length(labelTickX); % x tick + yTick = 1:length(labelTickY); % y tick + %% Generate Figure + h_fig(1) = figure; % figure is not displayed + imagesc(squeeze(res_matrix')); + hold on; + plot(points(idx(1)),points(idx(2)),'x','LineWidth',4,'MarkerSize',10,'Color','white'); + + ylabel(labelNameY); + xlabel(labelNameX); + set(gca,'XTickLabel',labelTickX); + set(gca,'YTickLabel',labelTickY); + set(gca,'XTick',xTick); + set(gca,'YTick',yTick); + h_colorbar = colorbar(gca); + set(get(h_colorbar,'YLabel'),'String','Mean absolute error (rows)'); + img_format ='jpg'; + if(options.fig_generation) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%03d_%s_%s_absolute_error.%s',2,testname,param.day_seg,img_format)); + ct_saveas(h_fig(1),img_fn); + + end + end + end + + + end + + if(options.season_vs_gps.plot) + for track_idx = 1:length(param.layer_tracker.track) + h_fig(1) = figure; + plot(gps_time,abs(res_matrix{1}{track_idx})); + ylabel('Absolute error'); + xlabel('GPS time'); + title(sprintf('%s frames: %03d Tracker: %03d',param.season_name,length(param.cmd.frms), track_idx),'Interpreter', 'none'); + xlim([gps_time(1) gps_time(end)]); + + img_format = 'png'; + if(options.season_vs_gps.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%03d_%s_%s_season_gps_%03d.%s',2,testname,param.day_seg,track_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + end + end + + + end + end +end + +keyboard; + +%% Find Best Combination +for layer_names = 1:idx + res_matrix{layer_names} = zeros([length(param.cmd.frms),param.layer_tracker.track{1}.idx_reshape]); + num_gt_isfinite.(sprintf('%s',gt_layer_params(layer_names).name)) = 0; + num_isnan.(sprintf('%s',gt_layer_params(layer_names).name)) = 0; + num_points.(sprintf('%s',gt_layer_params(layer_names).name)) = 0; + num_layers = num_layers+1; +end + + +idx_matrix = [(1:length(param.layer_tracker.track{1}.idx_reshape))+1 1]; % used for resizing + +for track_idx = 1:length(param.layer_tracker.track) + frm_idx = 0; + for frm = param.cmd.frms + frm_idx = frm_idx+1; + data_fn_dir = ct_filename_out(param, param.layer_tracker.echogram_source, ''); + data_fn_name = sprintf('Data_%s_%03d.mat',param.day_seg,frm); + data_fn = fullfile(data_fn_dir, data_fn_name); + data = load(data_fn); + dt = data.Time(2) - data.Time(1); + for layer_idx = 1:length(gt_layer_params) + + layer_params(1).name = sprintf('%s_%s_%s',param.layer_tracker.track{track_idx}.name,param.layer_tracker.track{track_idx}.method,gt_layer_params(layer_idx).name); + layer_params(1).source = param.layer_tracker.layer_params.source; + layer_params(1).layerdata_source = param.layer_tracker.layer_params.layerdata_source; + layers_new = opsLoadLayers(param,layer_params); + + surf = interp1(layers(layer_idx).gps_time,layers(layer_idx).twtt,data.GPS_time); + surf_bins = round(interp1(data.Time,1:length(data.Time),surf)); + + for pos = 1:length(layers_new) + surf = interp1(layers_new(pos).gps_time,layers_new(pos).twtt,data.GPS_time); + surf_bins_itr = round(interp1(data.Time,1:length(data.Time),surf)); + num_gt_isfinite.(sprintf('%s',gt_layer_params(layer_idx).name)) = num_gt_isfinite.(sprintf('%s',gt_layer_params(layer_idx).name)) + sum(isfinite(surf_bins)); + num_isnan.(sprintf('%s',gt_layer_params(layer_idx).name)) = num_isnan.(sprintf('%s',gt_layer_params(layer_idx).name)) + (sum(isfinite(surf_bins) & ~isfinite(surf_bins_itr))); + num_points.(sprintf('%s',gt_layer_params(layer_idx).name)) = num_points.(sprintf('%s',gt_layer_params(layer_idx).name)) + (sum(abs(surf_bins-surf_bins_itr) < 5*dt)); + res_matrix{layer_idx}(frm_idx,param.layer_tracker.track{track_idx}.idx(pos)) = nanmean(abs(surf_bins - surf_bins_itr)); + counter = 0; + for i = 1:length(surf_bins) + if (abs(surf_bins(i) - surf_bins_itr(i)) <= range) + counter = counter + 1; + end + end + range_percent.(sprintf('%s_%03d', param.layer_tracker.track{track_idx}.name,frm))= (counter/length(surf_bins))*100; + end + end + end + counter = 0; + for i = 1:length(param.cmd.frms) + counter = counter + range_percent.(sprintf('%s_%03d', param.layer_tracker.track{track_idx}.name,frm)); + end + range_percent_all_frms.(sprintf('%s', param.layer_tracker.track{track_idx}.name)) = counter/length(param.cmd.frms); +end + +points = []; +min_val = []; +res_matrix_all_frms = []; +for layer_names = 1:num_layers + res_matrix_all_frms{layer_names} = nanmean(res_matrix{layer_names},1); + res_matrix_all_frms{layer_names} = permute(res_matrix_all_frms{layer_names},idx_matrix); + [min_val{layer_names},i]=min(res_matrix_all_frms{layer_names}(:)); + sizeMatrix = size(res_matrix_all_frms{layer_names}); + for dim = 1:length(size(res_matrix_all_frms{layer_names})) + points{layer_names}(dim) = mod(floor((i-1)/prod(sizeMatrix(1:dim-1))),sizeMatrix(dim))+1; + end + if (length(size(res_matrix_all_frms{layer_names})) < length(param.layer_tracker.track{1}.idx_reshape)) + points{layer_names}(dim+1) = 1; + end +end + +file_version = '1'; +file_type = 'layer_tracker_tuning'; + +fn_dir = fileparts(save_name); +if ~exist(fn_dir,'dir') + mkdir(fn_dir); +end + +save(save_name,'res_matrix','range_percent','range_percent_all_frms','num_isnan','num_points','num_gt_isfinite','points','min_val','res_matrix_all_frms','param','file_version','file_type'); diff --git a/cresis-toolbox/tracker/layer_tracker_stat_generator_temp.m b/cresis-toolbox/tracker/layer_tracker_stat_generator_temp.m new file mode 100644 index 00000000..64d9210f --- /dev/null +++ b/cresis-toolbox/tracker/layer_tracker_stat_generator_temp.m @@ -0,0 +1,912 @@ +%% Script layer_tracker_tune_layerdata.m +% % sign and percentage correct plot_best_segment_overall/segment % correct +% and mean absolute error. +% /cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140313_08_20210211_201501_t006_lsm.mat +% /cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140516_01_20210211_201509_t006_lsm.mat +% Tuning script to work with layerdata files rather than temporary files +% Used to find best combination of paramters for given tracker method +% Enter layer names in the gt_layer_params(idx).name field +% set temp to the file location of the param structure used for tracking in run_layer_tracker.m script +% set save_name to store final tuning results +% Statistical data saved: +% number of points with finite values of ground truth of twtt (num_gt_infinite), +% number of points where ground truth twtt is finite and tracked data twtt is NaN (num_isnan) +% number of points where (tracked data twtt - ground truth twtt) < 5 * dt (num_points) +% nanmean of (tracked data twtt - ground truth twtt) (res_matrix) +% Authors: Anjali Pare, John Paden +% See layer_tracker_tune_plot.m to view 2 dimensional imagesc plots of the data. + +% Look at % correct. If we're close to the right ans 100 % of the time but +% not quite right 100 % of the time is worse than being right 90% of the +% time and completely wrong 10 % of the time. Need to correct in quality +% control. Check if you are outside of a wondow and label as bad. % correct +% should be exactly the same (not necesary). There could be offsets. +% 1-2 range bin. + +% Have fig option for all. Use hist instead of histogram. Histogram plots +% separate from % correct graphs. Enable option to generate best histogram +% plot. Plot as regular line using hist + +% What to plot, finding best combination, trying with LSM +clear; +dbstack_info = dbstack; +fprintf('=====================================================================\n'); +fprintf('%s (%s)\n', dbstack_info(1).name, datestr(now,'HH:MM:SS')); +fprintf('=====================================================================\n'); + +%% Setup param +%20140516_01_20210107_163106_t005_viterbi +%20140313_08_20210107_162824_t005_viterbi +%temp = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140516_01_20201122_193803_t005_viterbi.mat'); + +% Example to run 1 season with 1 segment all frames +params_all = []; +idx_segment = 1; +params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls'),'20140421_01');%'20140313_09|20140415_05|20140421_01|20140514_01'); +params(idx_segment).cmd.generic = 1; +params(idx_segment).cmd.frms = []; +% idx_segment = idx_segment + 1; +% params(idx_segment).cmd.generic = 1; +% params(idx_segment).cmd.frms = []; +% idx_segment = idx_segment + 1; +% params(idx_segment).cmd.generic = 1; +% params(idx_segment).cmd.frms = []; +% idx_segment = idx_segment + 1; +% params(idx_segment).cmd.generic = 1; +% params(idx_segment).cmd.frms = []; +params_all{end+1} = params; + +% Example to run 1 season with 2 segment and different frames +% params_all = []; +% idx_segment = 1; +% params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls'),'20140313_08|20140516_01'); +% params(idx_segment).cmd.generic = 1; +% params(idx_segment).cmd.frms = 1:2; +% idx_segment = idx_segment + 1; +% params(idx_segment).cmd.generic = 1; +% params(idx_segment).cmd.frms = 3:4; +% params_all{end+1} = params; + +% Example to run 2 season with 1 segment frames 1:5 +% params_all = []; +% idx_segment = 1; +% params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls'),'20140516_01'); +% params(idx_segment).cmd.generic = 1; +% params(idx_segment).cmd.frms = 1:5; +% params_all{end+1} = params; + +% idx_segment = 1; +% params = read_param_xls(ct_filename_param('rds_param_2014_Antarctica_DC8.xls'),'20141122_08'); +% idx_segment = idx_segment + 1; +% params(idx_segment).cmd.generic = 1; +% params(idx_segment).cmd.frms = 1:5; +% params_all{end+1} = params; + + +% Example to run all segments and all frames of a season +% params_all = []; +% idx_segment = 1; +% params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls')); +% params = ct_set_params(params,'cmd.generic',1); +% params(idx_segment).cmd.frms = []; + +%% Setup filename +filename = []; +% one segment +%%filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140516_01_20201202_020912_t005_viterbi.mat'); +% filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140516_01_20210124_024502_t004_lsm.mat'); +%filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140516_01_20200921_121933_t063_lsm.mat'); +%filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140516_01_20210204_195538_t006_lsm.mat'); + + +% filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140313_08_20210211_201501_t006_lsm.mat'); +% filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140516_01_20210211_201509_t006_lsm.mat'); + + %filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140313_08_20210304_221148_t005_viterbi.mat'); + %filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140516_01_20210315_220253_t005_viterbi.mat'); +%filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140313_09_20210311_013103_t056_lsm.mat'); +%filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140415_05_20210311_013232_t056_lsm.mat'); +filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140421_01_20210311_014435_t056_lsm.mat'); +%filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140514_01_20210311_015110_t056_lsm.mat'); + % two segments +%filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140313_08_20210107_162824_t005_viterbi.mat'); +%filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140516_01_20210107_163106_t005_viterbi.mat'); + +%% Setup plots +% set fig_format to true to generate .fig plots +options.segment_vs_frame.plot = true; % absolute error vs frames (for each segment) +options.segment_vs_frame.fig_format = true; +options.season_vs_gps.plot = true; % absolute error vs gps time (for entire season) +options.season_vs_gps.fig_format = true; +options.season_vs_gps.multiple_figs = true; % choose this option if you separate plots generated for each layer. If set to false, 6 subplots will be plotted in one figure. +options.absolute_error.plot = true; % mean absolute error plots +options.absolute_error.fig_format = true; +options.absolute_error.multiple_plots = true; +options.percentage_correct.plot = true; % plot of percentage correct vs frames (for each segment) +options.percentage_correct.fig_format = true; +options.percentage_correct.multiple_plots = true; +options.hist_generation.plot = true; % histograms of data points vs absolute error +options.hist_generation.fig_format = true; +options.hist_generation.multiple_plots = true; + +%% Setup save +% Enter filename of where you want to store the images +img_dir = '/cresis/snfs1/scratch/anjali/cluster_tuning/lsm_test'; +if ~exist(img_dir,'dir') + mkdir(img_dir); +end + +%% Setup general +% set the range and indicate the layers to track + +gt_layer_params = []; +layer_params = []; +res_matrix = []; +res_list = []; +gps_time = []; +season_vect = []; +segment_vect = []; +frame_vect = []; +seg_idx = 0; +season_vect_frm = []; +segment_vect_frm = []; +frame_vect_frm = []; +range = 100; % set acceptable range for error +idx = 1; + +% If only bottom + %gt_layer_params(idx).name = 'bottom'; % specify layer names + +% If surface and bottom +foo{idx} = []; +gt_layer_params(idx).name = 'surface'; +idx = idx + 1; +foo{idx} = []; +gt_layer_params(idx).name = 'bottom'; % specify layer names + +num_layers = idx; + +%% Implementation + +for season_idx = 1:length(params_all) %seasons + params = params_all{season_idx}; + for param_idx = 1:length(params) % segments + param = params(param_idx); + for frm = param.cmd.frms + season_vect_frm(end+1) = season_idx; + segment_vect_frm(end+1) = param_idx; + frame_vect_frm(end+1) = frm; + end + end +end + +for season_idx = 1:length(params_all) %seasons + params = params_all{season_idx}; + for param_idx = 1:length(params) % segments + param = params(param_idx); + + if isfield(param.cmd,'generic') && ~iscell(param.cmd.generic) && ~ischar(param.cmd.generic) && param.cmd.generic + seg_idx = seg_idx + 1; + param_override = filename{seg_idx}.param; + param = merge_structs(param,param_override); + total_length = 0; + layers = opsLoadLayers(param,gt_layer_params); + idx_matrix = [(1:length(param.layer_tracker.track{1}.idx_reshape))+1 1]; % used for resizing + for layer_names = 1:idx + res_matrix_all{layer_names} = zeros([length(param.cmd.frms),param.layer_tracker.track{1}.idx_reshape]); + res_mat_temp{layer_names} = cell([param.layer_tracker.track{1}.idx_reshape]); + end + + if (param_idx == 1) + for layer_idx = 1:length(gt_layer_params) + if strcmpi(param.layer_tracker.track{1}.method,'lsm') + percent_error{layer_idx} = zeros(1,(length(param.layer_tracker.track)*length(param.layer_tracker.track{1}.lsm.storeIter))); + for iter_idx = 1:(length(param.layer_tracker.track)*length(param.layer_tracker.track{1}.lsm.storeIter)) + range_per{layer_idx}{iter_idx} = []; + end + elseif strcmpi(param.layer_tracker.track{1}.method,'viterbi') + percent_error{layer_idx} = zeros(1,(length(param.layer_tracker.track))); + for iter_idx = 1:(length(param.layer_tracker.track)) + range_per{layer_idx}{iter_idx} = []; + end + end + end + end + % for layer_idx = 1:length(gt_layer_params) + % percent_error{layer_idx} = zeros(1,(length(param.layer_tracker.track)*length(param.layer_tracker.track{1}.lsm.storeIter))); + % end + + for track_idx = 1:length(param.layer_tracker.track) + range_percent_all_frms.(sprintf('%s', param.layer_tracker.track{track_idx}.name)) = 0; + for frm_idx = 1:length(param.cmd.frms) + range_percent.(sprintf('%s_%03d', param.layer_tracker.track{track_idx}.name,frm_idx)) = 0; + prob_error.(sprintf('%s_%03d', param.layer_tracker.track{track_idx}.name,frm_idx)) = 0; + %frame_error{track_idx}(frm_idx) = 0; + end + end + frm_idx = 0; + for frm = param.cmd.frms + frm_idx = frm_idx+1; + data_fn_dir = ct_filename_out(param, param.layer_tracker.echogram_source, ''); + data_fn_name = sprintf('Data_%s_%03d.mat',param.day_seg,frm); + data_fn = fullfile(data_fn_dir, data_fn_name); + data = load(data_fn, 'GPS_time', 'Time'); + gps_time = cat(2,gps_time,data.GPS_time); + dt = data.Time(2) - data.Time(1); + total_length = total_length + length(data.GPS_time); + idx_list = 1; + + frm_length = length(data.GPS_time); + for j = 1:length(data.GPS_time) + season_vect(end+1) = season_idx; + segment_vect(end+1) = param_idx; + frame_vect(end+1) = frm; + end + + for track_idx = 1:length(param.layer_tracker.track) + + for layer_idx = 1:length(gt_layer_params) + if strcmpi(param.layer_tracker.track{1}.method,'lsm') + for idx = 1:length(param.layer_tracker.track{track_idx}.lsm.storeIter) + layer_params(idx).name = sprintf('%s_%s_%s_%03d',param.layer_tracker.track{track_idx}.name,param.layer_tracker.track{track_idx}.method,gt_layer_params(layer_idx).name,idx); + layer_params(idx).source = param.layer_tracker.layer_params.source; + layer_params(idx).layerdata_source = param.layer_tracker.layer_params.layerdata_source; + end + else + layer_params(1).name = sprintf('%s_%s_%s',param.layer_tracker.track{track_idx}.name,param.layer_tracker.track{track_idx}.method,gt_layer_params(layer_idx).name); + layer_params(1).source = param.layer_tracker.layer_params.source; + layer_params(1).layerdata_source = param.layer_tracker.layer_params.layerdata_source; + end + ids = 1; + sprintf('\n=====Loading layers tracker: %d=====\n',track_idx); + layers_new = opsLoadLayers(param,layer_params); + sprintf('\n=====Loaded layers tracker: %d=====\n',track_idx); + + surf = interp1(layers(layer_idx).gps_time,layers(layer_idx).twtt,data.GPS_time); + surf_bins = round(interp1(data.Time,1:length(data.Time),surf)); + + for pos = 1:length(layers_new) + surf = interp1(layers_new(pos).gps_time,layers_new(pos).twtt,data.GPS_time); + surf_bins_itr = round(interp1(data.Time,1:length(data.Time),surf)); + res_matrix_all{layer_idx}(frm_idx,param.layer_tracker.track{track_idx}.idx(pos)) = nanmean(abs(surf_bins - surf_bins_itr)); + counter = 0; + for i = 1:length(surf_bins) + try + res_matrix{layer_idx}{param.layer_tracker.track{track_idx}.idx(pos)}(end+1) = surf_bins(i) - surf_bins_itr(i); + res_mat_temp{layer_idx}{param.layer_tracker.track{track_idx}.idx(pos)}(end+1) = surf_bins(i) - surf_bins_itr(i); + catch ME + res_matrix{layer_idx}{param.layer_tracker.track{track_idx}.idx(pos)}(i) = surf_bins(i) - surf_bins_itr(i); + res_mat_temp{layer_idx}{param.layer_tracker.track{track_idx}.idx(pos)}(i) = surf_bins(i) - surf_bins_itr(i); + end + if (abs(surf_bins(i) - surf_bins_itr(i)) <= range) + counter = counter + 1; + end + end + range_percent.(sprintf('%s_%03d', param.layer_tracker.track{track_idx}.name,frm)) = (counter/length(surf_bins))*100; + range_per{layer_idx}{param.layer_tracker.track{track_idx}.idx(pos)}(end+1) = (counter/length(surf_bins))*100; + percent_error{layer_idx}(param.layer_tracker.track{track_idx}.idx(pos)) = counter + percent_error{layer_idx}(param.layer_tracker.track{track_idx}.idx(pos)); + prob_error.(sprintf('%s_%03d', param.layer_tracker.track{track_idx}.name,frm)) = counter*100; + frame_error{layer_idx}{param.layer_tracker.track{track_idx}.idx(pos)}(frm_idx) = nanmean(abs(surf_bins - surf_bins_itr)); + try + range_percent_all_frms.(sprintf('%s', param.layer_tracker.track{track_idx}.name)) = range_percent_all_frms.(sprintf('%s', param.layer_tracker.track{track_idx}.name)) + counter; + catch ME + range_percent_all_frms.(sprintf('%s', param.layer_tracker.track{track_idx}.name)) = counter; + end + end + end + idx_list = idx_list + 1; + end + end + + + points = []; + min_val = []; + res_matrix_all_frms = []; + for layer_names = 1:num_layers + res_matrix_all_frms{layer_names} = nanmean(res_matrix_all{layer_names},1); + res_matrix_all_frms{layer_names} = permute(res_matrix_all_frms{layer_names},idx_matrix); + [min_val{layer_names},i]=min(res_matrix_all_frms{layer_names}(:)); + sizeMatrix = size(res_matrix_all_frms{layer_names}); + for dim = 1:length(size(res_matrix_all_frms{layer_names})) + points{layer_names}(dim) = mod(floor((i-1)/prod(sizeMatrix(1:dim-1))),sizeMatrix(dim))+1; + end + if (length(size(res_matrix_all_frms{layer_names})) < length(param.layer_tracker.track{1}.idx_reshape)) + points{layer_names}(dim+1) = 1; + end + if(param_idx==1) + foo{layer_names} = res_matrix_all{layer_names}; + else + foo{layer_names} = [foo{layer_names};res_matrix_all{layer_names}]; + end + end + + % per segment vs frame + if (options.segment_vs_frame.plot) + fprintf('\n=====Segment frame plots=====\n'); + for layer_idx = 1:length(gt_layer_params) + h_fig(1) = figure('visible','off'); + for fig_idx = 1:length(frame_error{layer_idx}) + hold on; + %plot(frame_error{layer_idx}{fig_idx},'DisplayName',sprintf('tracker%d',fig_idx)); + %legend('-DynamicLegend','Location','northeastoutside'); + plot(frame_error{layer_idx}{fig_idx},'.-','DisplayName',sprintf('layer%d',fig_idx)); + labelNameX = 'Frames'; + labelNameY = 'Absolute error (bins)'; + ylabel(labelNameY); + xlabel(labelNameX); + xlim = ([1 length(param.cmd.frms)]); + % labelNameX = 'frames'; + % labelNameY = 'absolute error'; + % ylabel(labelNameY); + % xlabel(labelNameX); + % set(gca,'XTick',1:length(param.cmd.frms)); + % set(gca,'XTickLabel',[param.cmd.frms]) + title(sprintf('%s Layer: %d',param.day_seg,layer_idx),'Interpreter', 'none'); + end + + hold off; + img_format = 'png'; + if(options.segment_vs_frame.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_segment_vs_frame_%03d.%s',testname,param.day_seg,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + end + end + + if (options.percentage_correct.plot) + fprintf('\n=====Percentage plots=====\n'); + if (options.percentage_correct.multiple_plots) + for layer_idx = 1:length(gt_layer_params) + h_fig(1) = figure('visible','off'); + for hist_idx = 1:length(frame_error{layer_idx}) + % hold on; + % plot(range_per{layer_idx}{hist_idx},'DisplayName',sprintf('tracker%d',hist_idx)); + % legend('-DynamicLegend','Location','northeastoutside'); + % xlabel('Number of frames'); + % ylabel('Percentage error'); + % set(gca,'XTick',1:length(param.cmd.frms)); + % set(gca,'XTickLabel',[param.cmd.frms]); + hold on; + y = range_per{layer_idx}{hist_idx}(segment_vect_frm == param_idx); + plot(y,'.-','DisplayName',sprintf('layer%d',hist_idx)); + xlabel('Frames'); + ylabel('Percentage error'); + xlim = ([1 length(param.cmd.frms)]); + title(sprintf('%s Layer: %d',param.day_seg,layer_idx),'Interpreter', 'none'); + % percent_error{layer_idx}(hist_idx) = percent_error{layer_idx}(hist_idx)/length(season_vect) * 100; + end + img_format = 'png'; + if(options.percentage_correct.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_percentage_correct_%03d.%s',testname,param.day_seg,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + end + end + end + + if (options.hist_generation.plot) + fprintf('\n=====Hist generation plots=====\n'); + if (options.hist_generation.multiple_plots) + for layer_idx = 1:length(gt_layer_params) + h_fig(1) = figure('visible','off'); + for hist_idx = 1:length(frame_error{layer_idx}) + hold on; + plot_hist = abs(res_matrix{layer_idx}{hist_idx}(segment_vect == param_idx)); + [N,x] = hist(plot_hist); + plot(x,N,'DisplayName',sprintf('layer%d',hist_idx)); + %legend('-DynamicLegend','Location','northeastoutside'); + ylabel('Data points'); + xlabel('Absolute error (bins)'); + title(sprintf('%s Layer: %d',param.day_seg,layer_idx),'Interpreter', 'none'); + end + hold off; + img_format = 'png'; + if(options.hist_generation.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_histogram_generation_%03d.%s',testname,param.day_seg,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + end + end + end + + % for layer_idx = 1:length(gt_layer_params) + % h_fig(1) = figure('visible','off'); + % plot(percent_error{layer_idx}); + % hold on; + % plot(percent_error{layer_idx}, 'r*'); + % ylabel('Percentage correct'); + % xlabel('Trackers'); + % title(sprintf('%s frames: %d Layer: %d',param.day_seg,length(param.cmd.frms),layer_idx),'Interpreter', 'none'); + % set(gca,'XTick',1:length(frame_error{layer_idx})); + % img_format = 'png'; + % if(options.percentage_correct.fig_format) + % img_format = 'fig'; + % end + % testname = param.layer_tracker.layer_params.layerdata_source; + % img_fn = fullfile(img_dir,sprintf('%s_%s_total_percent_correct_%03d.%s',testname,param.day_seg,layer_idx,img_format)); + % ct_saveas(h_fig(1),img_fn); + % end + + if(options.absolute_error.plot) + fprintf('\n=====Absolute error plots=====\n'); + if (options.absolute_error.multiple_plots) + if(strcmpi(param.layer_tracker.track{1}.method,'viterbi')) + for layer_idx = 1:num_layers + res_mat = res_matrix_all_frms{layer_idx}; + points_new = points{layer_idx}; + x = 1:length(res_mat); % x tick + h_fig(1) = figure('visible','off'); + plot(x,res_mat); + hold on; + for i = 1:length(res_mat) + plot(x(i),res_mat(i), 'r*'); + end + plot(x(points_new(1)),res_mat(points_new(1)), 'ko','LineWidth',4); + set(gca,'XTick',x); + ylabel('Mean absolute error (bins)'); + xlabel('Trackers'); + title(sprintf('%s frames: %03d',param.day_seg,length(param.cmd.frms)),'Interpreter', 'none'); + end + img_format = 'png'; + if(options.absolute_error.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%03d_%s_%s_absolute_error.%s',2,testname,param.day_seg,img_format)); + ct_saveas(h_fig(1),img_fn); + + elseif(strcmpi(param.layer_tracker.track{1}.method,'lsm')) + data = []; + for dim = 1:length(param.layer_tracker.track{1}.idx_reshape) + temp = []; + for track_idx = 1:length(param.layer_tracker.track) + if isempty(temp) + temp = param.layer_tracker.track{track_idx}.(param.layer_tracker.track{track_idx}.method).(param.layer_tracker.track{track_idx}.idx_dim_name{dim}); + else + if ~any(temp == param.layer_tracker.track{track_idx}.(param.layer_tracker.track{track_idx}.method).(param.layer_tracker.track{track_idx}.idx_dim_name{dim})) + temp(end+1) = param.layer_tracker.track{track_idx}.(param.layer_tracker.track{track_idx}.method).(param.layer_tracker.track{track_idx}.idx_dim_name{dim}); + end + end + + end + data{dim}=temp; + end + for layer_idx = 1:num_layers + res_mat = res_matrix_all_frms{layer_idx}; + temp_points = points{layer_idx}; + + A = nchoosek(1:length(param.layer_tracker.track{1}.idx_reshape),2); % changed from 2 + for id = 1:length(A) + min_points = setdiff(1:length(param.layer_tracker.track{1}.idx_reshape),A(id,:)); + min_idxs = temp_points(min_points); + res_matrix_tot = permute(res_mat,[A(id,:) setdiff(1:length(param.layer_tracker.track{1}.idx_reshape),A(id,:))]); + res_matrix_tot = res_matrix_tot(:,:,min_idxs(:)); + idx = A(id,:); + labelTickX = data{idx(1)}; % x tick label + labelTickY = data{idx(2)}; % y tick label + labelNameX = param.layer_tracker.track{1}.idx_dim_name{idx(1)}; % x label name + labelNameY = param.layer_tracker.track{1}.idx_dim_name{idx(2)}; % y label name + xTick = 1:length(labelTickX); % x tick + yTick = 1:length(labelTickY); % y tick + %% Generate Figure + h_fig(1) = figure('visible','off'); % figure is not displayed + imagesc(squeeze(res_matrix_tot')); + hold on; + plot(temp_points(idx(1)),temp_points(idx(2)),'x','LineWidth',4,'MarkerSize',10,'Color','white'); + + ylabel(labelNameY); + xlabel(labelNameX); + set(gca,'XTickLabel',labelTickX); + set(gca,'YTickLabel',labelTickY); + set(gca,'XTick',xTick); + set(gca,'YTick',yTick); + h_colorbar = colorbar(gca); + set(get(h_colorbar,'YLabel'),'String','Mean absolute error (rows)'); + img_format ='jpg'; + if(options.absolute_error.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_absolute_error_%s_vs_%s_%03d.%s',testname,param.day_seg,labelNameX,labelNameY,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + + end + end + end + end + + + end + + % if(options.season_vs_gps.plot) + % if(options.season_vs_gps.multiple_figs) + % for layer_idx = 1:length(gt_layer_params) + % for track_idx = 1:length(frame_error{layer_idx}) + % h_fig(1) = figure; + % plot(gps_time,abs(res_matrix{layer_idx}{track_idx})); + % ylabel('Absolute error (bins)'); + % xlabel('GPS time'); + % title(sprintf('%s frames: %03d Tracker: %d Layer: %d',param.season_name,length(param.cmd.frms), track_idx, layer_idx),'Interpreter', 'none'); + % %xlim([gps_time(1) gps_time(end)]); + % + % img_format = 'png'; + % if(options.season_vs_gps.fig_format) + % img_format = 'fig'; + % end + % testname = param.layer_tracker.layer_params.layerdata_source; + % img_fn = fullfile(img_dir,sprintf('%s_%s_season_gps_%03d_%03d.%s',testname,param.day_seg,track_idx,layer_idx,img_format)); + % ct_saveas(h_fig(1),img_fn); + % end + % end + % else + % for layer_idx = 1:length(gt_layer_params) + % h_fig(1) = figure; + % subplot_idx = 0; + % for track_idx = 1:length(frame_error{layer_idx}) + % subplot_idx = subplot_idx + 1; + % if(subplot_idx < 6) + % subplot(3,2,subplot_idx) + % plot(gps_time,abs(res_matrix{layer_idx}{track_idx})); + % ylabel('Absolute error (bins)'); + % xlabel('GPS time'); + % title(sprintf('%s frames: %03d Tracker: %d Layer: %d',param.season_name,length(param.cmd.frms), track_idx, layer_idx),'Interpreter', 'none'); + % %xlim([gps_time(1) gps_time(end)]); + % else + % subplot(3,2,subplot_idx) + % plot(gps_time,abs(res_matrix{layer_idx}{track_idx})); + % ylabel('Absolute error (bins)'); + % xlabel('GPS time'); + % title(sprintf('%s Tracker: %d Layer: %d',param.season_name, track_idx, layer_idx),'Interpreter', 'none'); + % %xlim([gps_time(1) gps_time(end)]); + % img_format = 'png'; + % if(options.season_vs_gps.fig_format) + % img_format = 'fig'; + % end + % testname = param.layer_tracker.layer_params.layerdata_source; + % img_fn = fullfile(img_dir,sprintf('%s_%s_season_gps_%03d_%03d.%s',testname,param.day_seg,track_idx,layer_idx,img_format)); + % ct_saveas(h_fig(1),img_fn); + % h_fig(1) = figure; + % subplot_idx = 0; + % end + % + % end + % end + % end + % + % end + + save_name = sprintf('/cresis/snfs1/scratch/anjali/cluster_tuning/lsm_test/result_layer_tune_lsm_s%d',param_idx); + save(save_name,'res_matrix_all','frame_error','range_per','percent_error','season_vect','res_matrix','points','min_val','gps_time','param'); + end + end + fprintf('\n=====All segements=====\n'); + points = []; + min_val = []; + res_matrix_all_frms = []; + for layer_names = 1:num_layers + res_matrix_all_frms{layer_names} = nanmean(foo{layer_names},1); + res_matrix_all_frms{layer_names} = permute(res_matrix_all_frms{layer_names},idx_matrix); + [min_val{layer_names},i]=min(res_matrix_all_frms{layer_names}(:)); + sizeMatrix = size(res_matrix_all_frms{layer_names}); + for dim = 1:length(size(res_matrix_all_frms{layer_names})) + points{layer_names}(dim) = mod(floor((i-1)/prod(sizeMatrix(1:dim-1))),sizeMatrix(dim))+1; + end + if (length(size(res_matrix_all_frms{layer_names})) < length(param.layer_tracker.track{1}.idx_reshape)) + points{layer_names}(dim+1) = 1; + end + end + + if (options.percentage_correct.plot) + fprintf('\n=====Percentage correct all plots=====\n'); + %if (~options.percentage_correct.multiple_plots) + for layer_idx = 1:length(gt_layer_params) + h_fig(1) = figure('visible','off'); + for hist_idx = 1:length(frame_error{layer_idx}) + % hold on; + % plot(range_per{layer_idx}{hist_idx},'DisplayName',sprintf('tracker%d',hist_idx)); + % legend('-DynamicLegend','Location','northeastoutside'); + % xlabel('Number of frames'); + % ylabel('Percentage error'); + % set(gca,'XTick',1:length(param.cmd.frms)); + % set(gca,'XTickLabel',[param.cmd.frms]); + hold on; + plot(range_per{layer_idx}{hist_idx},'.-','DisplayName',sprintf('layer%d',hist_idx)); + xlabel('Frames'); + ylabel('Percentage correct (%)'); + xlim = ([1 length(param.cmd.frms)]); + title(sprintf('Percentage correct vs frames Layer:%d %s',layer_idx,param.season_name),'Interpreter', 'none'); + percent_error{layer_idx}(hist_idx) = percent_error{layer_idx}(hist_idx)/length(season_vect) * 100; + end + img_format = 'png'; + if(options.percentage_correct.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_percentage_correct_overall_%03d.%s',testname,param.season_name,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + end + + for layer_idx = 1:length(gt_layer_params) + h_fig(1) = figure('visible','off'); + plot(percent_error{layer_idx}); + hold on; + plot(percent_error{layer_idx}, 'r*'); + ylabel('Percentage correct (%)'); + xlabel('Trackers'); + title(sprintf('Percentage correct vs trackers Layer:%d %s',layer_idx,param.season_name),'Interpreter', 'none'); + set(gca,'XTick',1:length(frame_error{layer_idx})); + img_format = 'png'; + if(options.percentage_correct.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_total_percent_correct_%03d.%s',testname,param.season_name,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + end + %end + end + + if (options.hist_generation.plot) + fprintf('\n=====Hist generation all plots=====\n'); + %if (~options.hist_generation.multiple_plots) + for layer_idx = 1:length(gt_layer_params) + h_fig(1) = figure('visible','off'); + for hist_idx = 1:length(frame_error{layer_idx}) + hold on; + [N,x] = hist(abs(res_matrix{layer_idx}{hist_idx})); + plot(x,N,'DisplayName',sprintf('layer%d',hist_idx)); + %legend('-DynamicLegend','Location','northeastoutside'); + ylabel('Data points'); + xlabel('Absolute error (bins)'); + title(sprintf('Data points vs Absolute error Layer:%d %s',layer_idx,param.season_name),'Interpreter', 'none'); + end + hold off; + img_format = 'png'; + if(options.hist_generation.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_histogram_generation_overall_%03d.%s',testname,param.season_name,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + end + %end + end + + + if(options.season_vs_gps.plot) + fprintf('\n=====season plots=====\n'); + if(options.season_vs_gps.multiple_figs) + for layer_idx = 1:length(gt_layer_params) + for track_idx = 1:length(frame_error{layer_idx}) + h_fig(1) = figure('visible','off'); + plot(gps_time,abs(res_matrix{layer_idx}{track_idx})); + ylabel('Absolute error (bins)'); + xlabel('GPS time'); + title(sprintf('%s frames: %03d Tracker: %d Layer: %d',param.season_name,length(param.cmd.frms), track_idx, layer_idx),'Interpreter', 'none'); + %xlim([gps_time(1) gps_time(end)]); + + img_format = 'png'; + if(options.season_vs_gps.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_season_gps_%03d_%03d.%s',testname,param.season_name,track_idx,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + end + end + else + for layer_idx = 1:length(gt_layer_params) + h_fig(1) = figure('visible','off'); + subplot_idx = 0; + for track_idx = 1:length(frame_error{layer_idx}) + subplot_idx = subplot_idx + 1; + + if(subplot_idx < 6) + subplot(3,2,subplot_idx) + plot(gps_time,abs(res_matrix{layer_idx}{track_idx})); + ylabel('Absolute error (bins)'); + xlabel('GPS time'); + title(sprintf('%s frames: %03d Tracker: %d Layer: %d',param.season_name,length(param.cmd.frms), track_idx, layer_idx),'Interpreter', 'none'); + %xlim([gps_time(1) gps_time(end)]); + else + subplot(3,2,subplot_idx) + plot(gps_time,abs(res_matrix{layer_idx}{track_idx})); + ylabel('Absolute error (bins)'); + xlabel('GPS time'); + title(sprintf('%s Tracker: %d Layer: %d',param.season_name, track_idx, layer_idx),'Interpreter', 'none'); + %xlim([gps_time(1) gps_time(end)]); + img_format = 'png'; + if(options.season_vs_gps.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_season_gps_%03d_%03d.%s',testname,param.season_name,track_idx,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + h_fig(1) = figure('visible','off'); + subplot_idx = 0; + end + end + end + end + + end + + if(options.absolute_error.plot) + fprintf('\n=====Absolute error all plots=====\n'); + if(strcmpi(param.layer_tracker.track{1}.method,'viterbi')) + for layer_idx = 1:num_layers + res_mat = res_matrix_all_frms{layer_idx}; + points_new = points{layer_idx}; + x = 1:length(res_mat); % x tick + h_fig(1) = figure('visible','off'); + plot(x,res_mat); + hold on; + for i = 1:length(res_mat) + plot(x(i),res_mat(i), 'r*'); + end + plot(x(points_new(1)),res_mat(points_new(1)), 'ko','LineWidth',4); + set(gca,'XTick',x); + ylabel('Mean absolute error (bins)'); + xlabel('Trackers'); + title(sprintf('%s frames: %03d',param.day_seg,length(param.cmd.frms)),'Interpreter', 'none'); + end + img_format = 'png'; + if(options.absolute_error.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%03d_%s_%s_absolute_error.%s',2,testname,param.day_seg,img_format)); + ct_saveas(h_fig(1),img_fn); + + elseif(strcmpi(param.layer_tracker.track{1}.method,'lsm')) + data = []; + for dim = 1:length(param.layer_tracker.track{1}.idx_reshape) + temp = []; + for track_idx = 1:length(param.layer_tracker.track) + if isempty(temp) + temp = param.layer_tracker.track{track_idx}.(param.layer_tracker.track{track_idx}.method).(param.layer_tracker.track{track_idx}.idx_dim_name{dim}); + else + if ~any(temp == param.layer_tracker.track{track_idx}.(param.layer_tracker.track{track_idx}.method).(param.layer_tracker.track{track_idx}.idx_dim_name{dim})) + temp(end+1) = param.layer_tracker.track{track_idx}.(param.layer_tracker.track{track_idx}.method).(param.layer_tracker.track{track_idx}.idx_dim_name{dim}); + end + end + + end + data{dim}=temp; + end + for layer_idx = 1:num_layers + res_mat = res_matrix_all_frms{layer_idx}; + temp_points = points{layer_idx}; + + A = nchoosek(1:length(param.layer_tracker.track{1}.idx_reshape),2); % changed from 2 + for id = 1:length(A) + min_points = setdiff(1:length(param.layer_tracker.track{1}.idx_reshape),A(id,:)); + min_idxs = temp_points(min_points); + res_matrix_tot = permute(res_mat,[A(id,:) setdiff(1:length(param.layer_tracker.track{1}.idx_reshape),A(id,:))]); + res_matrix_tot = res_matrix_tot(:,:,min_idxs(:)); + idx = A(id,:); + labelTickX = data{idx(1)}; % x tick label + labelTickY = data{idx(2)}; % y tick label + labelNameX = param.layer_tracker.track{1}.idx_dim_name{idx(1)}; % x label name + labelNameY = param.layer_tracker.track{1}.idx_dim_name{idx(2)}; % y label name + xTick = 1:length(labelTickX); % x tick + yTick = 1:length(labelTickY); % y tick + %% Generate Figure + h_fig(1) = figure('visible','off'); % figure is not displayed + imagesc(squeeze(res_matrix_tot')); + hold on; + plot(temp_points(idx(1)),temp_points(idx(2)),'x','LineWidth',4,'MarkerSize',10,'Color','white'); + + ylabel(labelNameY); + xlabel(labelNameX); + set(gca,'XTickLabel',labelTickX); + set(gca,'YTickLabel',labelTickY); + set(gca,'XTick',xTick); + set(gca,'YTick',yTick); + h_colorbar = colorbar(gca); + set(get(h_colorbar,'YLabel'),'String','Mean absolute error (rows)'); + img_format ='jpg'; + if(options.absolute_error.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_absolute_error_%s_vs_%s_%03d.%s',testname,param.season_name,labelNameX,labelNameY,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + + end + end + end + + + end + +end + +keyboard; +% +% %% Find Best Combination +% for layer_names = 1:idx +% res_matrix{layer_names} = zeros([length(param.cmd.frms),param.layer_tracker.track{1}.idx_reshape]); +% num_gt_isfinite.(sprintf('%s',gt_layer_params(layer_names).name)) = 0; +% num_isnan.(sprintf('%s',gt_layer_params(layer_names).name)) = 0; +% num_points.(sprintf('%s',gt_layer_params(layer_names).name)) = 0; +% num_layers = num_layers+1; +% end +% +% +% idx_matrix = [(1:length(param.layer_tracker.track{1}.idx_reshape))+1 1]; % used for resizing +% +% for track_idx = 1:length(param.layer_tracker.track) +% frm_idx = 0; +% for frm = param.cmd.frms +% frm_idx = frm_idx+1; +% data_fn_dir = ct_filename_out(param, param.layer_tracker.echogram_source, ''); +% data_fn_name = sprintf('Data_%s_%03d.mat',param.day_seg,frm); +% data_fn = fullfile(data_fn_dir, data_fn_name); +% data = load(data_fn); +% dt = data.Time(2) - data.Time(1); +% for layer_idx = 1:length(gt_layer_params) +% +% layer_params(1).name = sprintf('%s_%s_%s',param.layer_tracker.track{track_idx}.name,param.layer_tracker.track{track_idx}.method,gt_layer_params(layer_idx).name); +% layer_params(1).source = param.layer_tracker.layer_params.source; +% layer_params(1).layerdata_source = param.layer_tracker.layer_params.layerdata_source; +% layers_new = opsLoadLayers(param,layer_params); +% +% surf = interp1(layers(layer_idx).gps_time,layers(layer_idx).twtt,data.GPS_time); +% surf_bins = round(interp1(data.Time,1:length(data.Time),surf)); +% +% for pos = 1:length(layers_new) +% surf = interp1(layers_new(pos).gps_time,layers_new(pos).twtt,data.GPS_time); +% surf_bins_itr = round(interp1(data.Time,1:length(data.Time),surf)); +% num_gt_isfinite.(sprintf('%s',gt_layer_params(layer_idx).name)) = num_gt_isfinite.(sprintf('%s',gt_layer_params(layer_idx).name)) + sum(isfinite(surf_bins)); +% num_isnan.(sprintf('%s',gt_layer_params(layer_idx).name)) = num_isnan.(sprintf('%s',gt_layer_params(layer_idx).name)) + (sum(isfinite(surf_bins) & ~isfinite(surf_bins_itr))); +% num_points.(sprintf('%s',gt_layer_params(layer_idx).name)) = num_points.(sprintf('%s',gt_layer_params(layer_idx).name)) + (sum(abs(surf_bins-surf_bins_itr) < 5*dt)); +% res_matrix{layer_idx}(frm_idx,param.layer_tracker.track{track_idx}.idx(pos)) = nanmean(abs(surf_bins - surf_bins_itr)); +% counter = 0; +% for i = 1:length(surf_bins) +% if (abs(surf_bins(i) - surf_bins_itr(i)) <= range) +% counter = counter + 1; +% end +% end +% range_percent.(sprintf('%s_%03d', param.layer_tracker.track{track_idx}.name,frm))= (counter/length(surf_bins))*100; +% end +% end +% end +% counter = 0; +% for i = 1:length(param.cmd.frms) +% counter = counter + range_percent.(sprintf('%s_%03d', param.layer_tracker.track{track_idx}.name,frm)); +% end +% range_percent_all_frms.(sprintf('%s', param.layer_tracker.track{track_idx}.name)) = counter/length(param.cmd.frms); +% end +% +% points = []; +% min_val = []; +% res_matrix_all_frms = []; +% for layer_names = 1:num_layers +% res_matrix_all_frms{layer_names} = nanmean(res_matrix{layer_names},1); +% res_matrix_all_frms{layer_names} = permute(res_matrix_all_frms{layer_names},idx_matrix); +% [min_val{layer_names},i]=min(res_matrix_all_frms{layer_names}(:)); +% sizeMatrix = size(res_matrix_all_frms{layer_names}); +% for dim = 1:length(size(res_matrix_all_frms{layer_names})) +% points{layer_names}(dim) = mod(floor((i-1)/prod(sizeMatrix(1:dim-1))),sizeMatrix(dim))+1; +% end +% if (length(size(res_matrix_all_frms{layer_names})) < length(param.layer_tracker.track{1}.idx_reshape)) +% points{layer_names}(dim+1) = 1; +% end +% end +% +% file_version = '1'; +% file_type = 'layer_tracker_tuning'; +% +% fn_dir = fileparts(save_name); +% if ~exist(fn_dir,'dir') +% mkdir(fn_dir); +% end +% +% save(save_name,'res_matrix','range_percent','range_percent_all_frms','num_isnan','num_points','num_gt_isfinite','points','min_val','res_matrix_all_frms','param','file_version','file_type'); diff --git a/cresis-toolbox/tracker/layer_tracker_task.m b/cresis-toolbox/tracker/layer_tracker_task.m index 478528f3..f7af80b5 100644 --- a/cresis-toolbox/tracker/layer_tracker_task.m +++ b/cresis-toolbox/tracker/layer_tracker_task.m @@ -1,115 +1,466 @@ -function success = layer_temp_task(param) -% layer_tracker_2D_task is used to set the parameters (options) for the -% different tracking algorithms. Look into calling the algorithms and -% running it on the cluster -success = {}; -%% Load reference surface -if any(strcmp(param.layer_tracker.track.method,'snake')) || any(strcmp(param.layer_tracker.track.method,'threshold'))|| any(strcmp(param.layer_tracker.track.method,'max')) || any(strcmp(param.layer_tracker.track.method,'fixed')) || any(strcmp(param.layer_tracker.track.method,'viterbi')) || any(strcmp(param.layer_tracker.track.method,'lsm')) || any(strcmp(param.layer_tracker.track.method,'stereo')) || any(strcmp(param.layer_tracker.track.method,'mcmc')) - - %intialising twtt, gps_time to save layer information - layer_params = param.layer_tracker.cmds.layer_params; - param.layer_tracker.fname = []; - if param.layer_tracker.track.track_data - for iter = 1:16 - for layers_idx = 1:length(layer_params) - twtt{iter,layers_idx} = []; - end +function success = layer_tracker_task(param) +% success = layer_tracker_task(param) +% +% layer_tracker_task is used to load and prepare the data and metadata and +% run the tracker. See run_layer_tracker.m. +% +% param: parameter structure from parameter spreadsheet +% param.layer_tracker.echogram_source: The normal cluster mode is for +% this to be a ct_filename_out compatible string. The normal +% qlook_combine_task mode is for this to be a structure and this function +% returns the tracked surface and does not save the result. +% +% success: +% In cluster mode: logical scalar, true when task completes successfully +% In qlook_combine_task mode: surface twtt corresponding to the +% param.layer_tracker.echogram_source.GPS_time. +% +% Authors: Anjali Pare, John Paden +% +% See also: layer_tracker.m, layer_tracker_combine_task.m, +% layer_tracker_task.m, layer_tracker_input_check.m, +% layer_tracker_profile.m, run_layer_tracker.m, run_layer_tracker_tune.m + +%% Input Checks: track field +% ===================================================================== + +physical_constants('c'); + +if isstruct(param.layer_tracker.echogram_source) + % echogram_source is the data structure + param.layer_tracker.track = {param.qlook.surf}; + layer_tracker_input_check; +end + +tracked_images_en = any(strcmp('tracked_images',param.layer_tracker.debug_plots)); +visible_en = any(strcmp('visible',param.layer_tracker.debug_plots)); +if tracked_images_en + h_fig = get_figures(1,visible_en); +end + +%% Create output directory paths +if strcmp(param.layer_tracker.layer_params.source,'ops') + tmp_out_fn_dir_dir = ct_filename_out(param,'ops','layer_tracker_tmp'); + param.layer_tracker.layer_params.layerdata_source = 'ops'; % Only used for stdout +else + tmp_out_fn_dir_dir = ct_filename_out(param,param.layer_tracker.layer_params.layerdata_source,'layer_tracker_tmp'); +end + +%% Load echogram data +if isstruct(param.layer_tracker.echogram_source) + % echogram_source is the radar image/data structure + % ======================================================================= + mdata = param.layer_tracker.echogram_source; + frm_str = {sprintf('%s_%03d',param.day_seg,param.layer_tracker.frms)}; + frm_start = 1; + frm_stop = length(mdata.GPS_time); + param.layer_tracker.tracks_in_task = 1; + dt = mdata.Time(2) - mdata.Time(1); + +else + % Load in the specified frames from files + % ======================================================================= + + % Create input directory paths + in_fn_dir = ct_filename_out(param,param.layer_tracker.echogram_source,''); + + mdata = []; + mdata.GPS_time = []; + mdata.Data = []; + mdata.Time = []; + mdata.Elevation=[]; + mdata.Latitude = []; + mdata.Longitude = []; + mdata.Roll = []; + % Keep track of start/stop index for each frame + frm_start = zeros(size(param.layer_tracker.frms)); + frm_stop = zeros(size(param.layer_tracker.frms)); + frm_str = cell(size(param.layer_tracker.frms)); + for frm_idx = 1:length(param.layer_tracker.frms) + frm = param.layer_tracker.frms(frm_idx); + + % Load the current frame + frm_str{frm_idx} = sprintf('%s_%03d',param.day_seg,frm); + if param.layer_tracker.echogram_img == 0 + data_fn = fullfile(in_fn_dir, sprintf('Data_%s.mat',frm_str{frm_idx})); + else + data_fn = fullfile(in_fn_dir, sprintf('Data_img_%02d_%s.mat',param.layer_tracker.echogram_img,frm_str{frm_idx})); end - else - for layer_idx = 1:length(layer_params) - twtt{layer_idx} = []; + fprintf('Loading %s (%s)\n', data_fn, datestr(now)); + if frm_idx == 1 + mdata = load_L1B(data_fn); + frm_start(frm_idx) = 1; + frm_stop(frm_idx) = length(mdata.GPS_time); + if length(mdata.Time) >= 2 + dt = mdata.Time(2) - mdata.Time(1); + else + dt = NaN; + end + + else + tmp_data = load_L1B(data_fn); + + Nx = length(tmp_data.GPS_time); + frm_start(frm_idx) = length(mdata.GPS_time)+1; + frm_stop(frm_idx) = length(mdata.GPS_time)+Nx; + + mdata.GPS_time(1,end+(1:Nx)) = tmp_data.GPS_time; + mdata.Elevation(1,end+(1:Nx)) = tmp_data.Elevation; + mdata.Latitude(1,end+(1:Nx)) = tmp_data.Latitude; + mdata.Longitude(1,end+(1:Nx)) = tmp_data.Longitude; + mdata.Roll(1,end+(1:Nx)) = tmp_data.Roll; + mdata.Surface(1,end+(1:Nx)) = tmp_data.Surface; + + % Handle time axis that changes from one frame to the next + min_time = min(tmp_data.Time(1),mdata.Time(1)); + if isnan(dt) && length(mdata.Time) >= 2 + dt = mdata.Time(2) - mdata.Time(1); + end + Time = (mdata.Time(1) - dt*round((mdata.Time(1) - min_time)/dt) ... + : dt : max(tmp_data.Time(end),mdata.Time(end))).'; + % Interpolate data to new time axes + warning off; + mdata.Data = interp1(mdata.Time,mdata.Data,Time); + tmp_data.Data = interp1(tmp_data.Time,tmp_data.Data,Time); + warning on; + mdata.Time = Time; + mdata.Data(:,end+(1:Nx)) = tmp_data.Data; end end - % - - % - gps_time = []; + overlap = [0 0]; + if param.layer_tracker.frms(1) > 1 && param.layer_tracker.overlap > 0 + frm = param.layer_tracker.frms(1)-1; + + % Load the previous frame + frm_str{frm_idx} = sprintf('%s_%03d',param.day_seg,frm); + if param.layer_tracker.echogram_img == 0 + data_fn = fullfile(in_fn_dir, sprintf('Data_%s.mat',frm_str{frm_idx})); + else + data_fn = fullfile(in_fn_dir, sprintf('Data_img_%02d_%s.mat',param.layer_tracker.echogram_img,frm_str{frm_idx})); + end + if exist(data_fn,'file') + fprintf('Loading previous frame %s (%s)\n', data_fn, datestr(now)); + tmp_data = load_L1B(data_fn); + + overlap(1) = min(param.layer_tracker.overlap,length(tmp_data.GPS_time)); + mdata.GPS_time = [tmp_data.GPS_time(end-overlap(1)+1:end) mdata.GPS_time]; + mdata.Elevation = [tmp_data.Elevation(end-overlap(1)+1:end) mdata.Elevation]; + mdata.Latitude = [tmp_data.Latitude(end-overlap(1)+1:end) mdata.Latitude]; + mdata.Longitude = [tmp_data.Longitude(end-overlap(1)+1:end) mdata.Longitude]; + mdata.Roll = [tmp_data.Roll(end-overlap(1)+1:end) mdata.Roll]; + mdata.Surface = [tmp_data.Surface(end-overlap(1)+1:end) mdata.Surface]; + + % Handle time axis that changes from one frame to the next + min_time = min(tmp_data.Time(1),mdata.Time(1)); + if isnan(dt) && length(mdata.Time) >= 2 + dt = mdata.Time(2) - mdata.Time(1); + end + Time = (mdata.Time(1) - dt*round((mdata.Time(1) - min_time)/dt) ... + : dt : max(tmp_data.Time(end),mdata.Time(end))).'; + % Interpolate data to new time axes + warning off; + mdata.Data = interp1(mdata.Time,mdata.Data,Time); + tmp_data.Data = interp1(tmp_data.Time,tmp_data.Data(:,end-overlap(1)+1:end),Time); + warning on; + mdata.Time = Time; + mdata.Data = [tmp_data.Data mdata.Data]; + end + end + frames = frames_load(param); + if param.layer_tracker.frms(end) <= length(frames.frame_idxs) && param.layer_tracker.overlap > 0 + frm = param.layer_tracker.frms(end)+1; + + % Load the next frame + frm_str{frm_idx} = sprintf('%s_%03d',param.day_seg,frm); + if param.layer_tracker.echogram_img == 0 + data_fn = fullfile(in_fn_dir, sprintf('Data_%s.mat',frm_str{frm_idx})); + else + data_fn = fullfile(in_fn_dir, sprintf('Data_img_%02d_%s.mat',param.layer_tracker.echogram_img,frm_str{frm_idx})); + end + if exist(data_fn,'file') + fprintf('Loading next frame %s (%s)\n', data_fn, datestr(now)); + tmp_data = load_L1B(data_fn); + + overlap(2) = min(param.layer_tracker.overlap,length(tmp_data.GPS_time)); + mdata.GPS_time(1,end+(1:overlap(2))) = tmp_data.GPS_time(1:overlap(2)); + mdata.Elevation(1,end+(1:overlap(2))) = tmp_data.Elevation(1:overlap(2)); + mdata.Latitude(1,end+(1:overlap(2))) = tmp_data.Latitude(1:overlap(2)); + mdata.Longitude(1,end+(1:overlap(2))) = tmp_data.Longitude(1:overlap(2)); + mdata.Roll(1,end+(1:overlap(2))) = tmp_data.Roll(1:overlap(2)); + mdata.Surface(1,end+(1:overlap(2))) = tmp_data.Surface(1:overlap(2)); + + % Handle time axis that changes from one frame to the next + min_time = min(tmp_data.Time(1),mdata.Time(1)); + if isnan(dt) && length(mdata.Time) >= 2 + dt = mdata.Time(2) - mdata.Time(1); + end + Time = (mdata.Time(1) - dt*round((mdata.Time(1) - min_time)/dt) ... + : dt : max(tmp_data.Time(end),mdata.Time(end))).'; + % Interpolate data to new time axes + warning off; + mdata.Data = interp1(mdata.Time,mdata.Data,Time); + tmp_data.Data = interp1(tmp_data.Time,tmp_data.Data(:,(1:overlap(2))),Time); + warning on; + mdata.Time = Time; + mdata.Data(:,end+(1:overlap(2))) = tmp_data.Data; + end + end + +end +mdata = echo_param_update(mdata,param); +Nx = size(mdata.Data,2); +if isnan(dt) + warning('This set of frame(s) does not have a fast-time axis because the data records were all bad.'); + mdata.Time = [0 1]; +end + +%% Track +% ========================================================================= +for track_idx = param.layer_tracker.tracks_in_task + track = param.layer_tracker.track{track_idx}; + orig_track = track; + layer_param = param; + % Load layer data from the frame before and after the current frame. + % opsLoadLayers will check to see if the frame exists or not so we don't + % need to worry about specifying frames that do not exist. + layer_param.cmd.frms = [param.layer_tracker.frms(1)-1 param.layer_tracker.frms param.layer_tracker.frms(end)+1]; - if isfield(param.layer_tracker.tracker,'init') && isfield(param.layer_tracker.tracker.init,'dem_layer') ... - && ~isempty(param.layer_tracker.tracker.init.dem_layer) - layers = opsLoadLayers(param,param.layer_tracker.tracker.init.dem_layer); + %% Track: Load reference surface + if isfield(track,'init') && isfield(track.init,'dem_layer') ... + && ~isempty(track.init.dem_layer) + layers = opsLoadLayers(layer_param,track.init.dem_layer); % Ensure that layer gps times are monotonically increasing lay_idx = 1; - layers_fieldnames = fieldnames(layers(lay_idx)); + layers_fieldnames = {'gps_time','twtt','elev','lat','lon','type','quality','frm'}; [~,unique_idxs] = unique(layers(lay_idx).gps_time); for field_idx = 1:length(layers_fieldnames)-1 if ~isempty(layers(lay_idx).(layers_fieldnames{field_idx})) layers(lay_idx).(layers_fieldnames{field_idx}) = layers(lay_idx).(layers_fieldnames{field_idx})(unique_idxs); end end + if length(layers(lay_idx).gps_time) < 2 + mdata.Surface = nan(size(mdata.GPS_time)); + else + mdata.Surface = interp_finite(interp1(layers(lay_idx).gps_time,layers(lay_idx).twtt,mdata.GPS_time), NaN); + end end - orig_track = param.layer_tracker.tracker; - %% - - save_name = '/cresis/snfs1/scratch/anjali/CSARP_layerData_Anjali'; - dir_name1 = 'CSARP_layer_tracker_tmp'; - dir_name2 = 'CSARP_layer_tracker'; - frm_dir = sprintf('layer_tracker_%03d_%03d', param.frm_nums(1),param.frm_nums(end)); - if ~exist(fullfile(save_name,param.season_name,dir_name1,dir_name2,param.day_seg,frm_dir)) - mkdir(fullfile(save_name,param.season_name,dir_name1,dir_name2,param.day_seg,frm_dir)); + %% Track: Load ocean mask, land DEM, sea surface DEM + if isfield(track,'init') && strcmpi(track.init.method,'dem') + if isstruct(param.layer_tracker.echogram_source) + % Only do this if it has not already been done in layer_tracker.m + global gdem; + if isempty(gdem) || ~isa(gdem,'dem_class') || ~isvalid(gdem) + gdem = dem_class(param,500); + end + gdem.set_res(500); + gdem.ocean_mask_mode = 'l'; + + gdem_str = sprintf('%s:%s:%s_%03d_%03d',param.radar_name,param.season_name,param.day_seg,param.layer_tracker.frms([1 end])); + if ~strcmpi(gdem_str,gdem.name) + gdem.set_vector(mdata.Latitude,mdata.Longitude,gdem_str); + end + end end - tmp_out_fn_dir = fullfile(save_name,param.season_name,dir_name1,dir_name2,param.day_seg,frm_dir); - - mdata = []; - mdata.GPS_time = []; - mdata.Data = []; - mdata.Time = []; - mdata.Elevation=[]; - mdata.Latitude = []; - mdata.Longitude = []; - mdata.data_length = []; - mdata.Bottom = []; - - for frm_idx = 1:length(param.frm_nums) - frm = param.frm_nums(frm_idx); - data_fn = fullfile(ct_filename_out(param,param.layer_tracker.echogram_source,''),sprintf('Data_%s_%03d.mat',param.day_seg,frm)); - lay=load(data_fn); - param.layer_tracker.fname{end+1} = data_fn; - mdata.GPS_time = cat(2,mdata.GPS_time,lay.GPS_time); - mdata.Data = cat(2,mdata.Data,lay.Data); - mdata.Elevation = cat(2,mdata.Elevation,lay.Elevation); - mdata.Latitude = cat(2,mdata.Latitude,lay.Latitude); - mdata.Longitude = cat(2, mdata.Longitude,lay.Longitude); - param.layer_tracker.tracker.frm = frm; - mdata.data_length = cat(2,mdata.data_length,length(lay.Bottom)); - mdata.Bottom = cat(2,mdata.Bottom,lay.Bottom); - end - mdata.Time = lay.Time; - orig_track = param.layer_tracker.tracker; - track = orig_track; - Nx = size(mdata.Data,2); - data = lp(mdata.Data); - - %% Track: Interpolate GIMP and Geoid + %% Track: Interpolate DEM, ocean mask, and mean sea level if strcmpi(track.init.method,'dem') - gdem.set_vector(mdata.Latitude,mdata.Longitude); - [land_dem,msl,ocean_mask] = gdem.get_vector_dem(); - + if isstruct(param.layer_tracker.echogram_source) + % If layer_tracker.m did not load dem: + gdem.set_vector(mdata.Latitude,mdata.Longitude); + [land_dem,msl,ocean_mask] = gdem.get_vector_dem(); + else + % If layer_tracker.m did load dem: + land_dem = interp1(param.layer_tracker.gps_time,param.layer_tracker.land_dem,mdata.GPS_time); + ocean_mask = interp1(param.layer_tracker.gps_time,double(param.layer_tracker.ocean_mask),mdata.GPS_time); + msl = interp1(param.layer_tracker.gps_time,param.layer_tracker.msl,mdata.GPS_time); + end + + ocean_mask = logical(ocean_mask); % Merge land surface and sea surface DEMs - track.dem = land_dem; + track.dem = double(land_dem); track.dem(ocean_mask) = msl(ocean_mask); track.dem = (mdata.Elevation - track.dem) / (c/2); track.dem = interp1(mdata.Time,1:length(mdata.Time),track.dem + track.init.dem_offset,'linear','extrap'); track.dem = interp_finite(track.dem,1); end - %% Track: Prefilter trim% - % Also set leading/following zeros or ~isfinite to NaN + %% Track: Ice mask calculation + if track.ice_mask.en + if isstruct(param.layer_tracker.echogram_source) + % If layer_tracker.m did not load ice_mask: + if strcmp(track.ice_mask.type,'bin') + mask = load(track.ice_mask.mat_fn,'R','X','Y','proj'); + [fid,msg] = fopen(track.ice_mask.fn,'r'); + if fid < 1 + fprintf('Could not open file %s\n', track.ice_mask.fn); + error(msg); + end + mask.maskmask = logical(fread(fid,[length(mask.Y),length(mask.X)],'uint8')); + fclose(fid); + else + [mask.maskmask,mask.R,~] = geotiffread(track.ice_mask.fn); + mask.proj = geotiffinfo(track.ice_mask.fn); + end + [mask.x, mask.y] = projfwd(mask.proj, mdata.Latitude, mdata.Longitude); + mask.X = mask.R(3,1) + mask.R(2,1)*(1:size(mask.maskmask,2)); + mask.Y = mask.R(3,2) + mask.R(1,2)*(1:size(mask.maskmask,1)); + [mask.X,mask.Y] = meshgrid(mask.X,mask.Y); + ice_mask.mask = round(interp2(mask.X, mask.Y, double(mask.maskmask), mask.x, mask.y)); + ice_mask.mask(isnan(ice_mask.mask)) = 1; + track.ice_mask = ice_mask.mask; + else + % If layer_tracker.m did load ice_mask: + track.ice_mask = interp1(param.layer_tracker.gps_time,param.layer_tracker.ice_mask,mdata.GPS_time); + end + else + track.ice_mask = ones(1,Nx); + end + + %% Track: Ground Truth + if track.ground_truth.en + ground_truth = opsLoadLayers(layer_param,track.ground_truth.layers); + % Convert ground_truths twtt to source twtt and then convert from twtt + % to rows/bins + cols = round(interp1(mdata.GPS_time,1:length(mdata.GPS_time), ground_truth.gps_time)); + rows = round(interp1(mdata.Time,1:length(mdata.Time), ground_truth.twtt)); + track.ground_truths = [cols; rows; track.ground_truth.cutoff*ones(size(cols))]; + track.ground_truths = track.ground_truths(:,isfinite(cols)); + else + track.ground_truths = zeros(3,0); + end + + %% Track: Crossover loading + if track.crossover.en + if isstruct(param.layer_tracker.echogram_source) + % Crossovers not loaded during layer_tracker.m + sys = ct_output_dir(param.radar_name); + ops_param = []; + ops_param.properties.search_str = frm_str{frm_idx}; + ops_param.properties.location = param.post.ops.location; + ops_param.properties.season = param.season_name; + [status,ops_data_segment] = opsGetFrameSearch(sys,ops_param); + if status == 1 + ops_param = []; + ops_param.properties.location = param.post.ops.location; + ops_param.properties.lyr_name = track.crossover.name; + ops_param.properties.frame = frm_str; + ops_param.properties.segment_id = ops_data_segment.properties.segment_id; + [status,ops_data] = opsGetCrossovers(sys,ops_param); + if status == 1 + ops_data.properties.twtt = ops_data.properties.twtt(:).'; + ops_data.properties.source_point_path_id = double(ops_data.properties.source_point_path_id(:).'); + ops_data.properties.cross_point_path_id = double(ops_data.properties.cross_point_path_id(:).'); + ops_data.properties.source_elev = ops_data.properties.source_elev.'; + ops_data.properties.cross_elev = ops_data.properties.cross_elev.'; + + % Get the OPS + ops_param = []; + ops_param.properties.location = param.post.ops.location; + ops_param.properties.point_path_id = [ops_data.properties.source_point_path_id ops_data.properties.cross_point_path_id]; + [status,ops_data_gps_time] = opsGetPath(sys,ops_param); + + source_gps_time = zeros(size(ops_data.properties.source_point_path_id)); + crossover_gps_time = zeros(size(ops_data.properties.source_point_path_id)); + good_mask = true(size(ops_data.properties.source_point_path_id)); + % Ensure crossover season is good + for idx = 1:length(ops_data.properties.source_point_path_id) + match_idx = find(ops_data.properties.source_point_path_id(idx) == ops_data_gps_time.properties.id,1); + source_gps_time(idx) = ops_data_gps_time.properties.gps_time(match_idx); + match_idx = find(ops_data.properties.cross_point_path_id(idx) == ops_data_gps_time.properties.id,1); + crossover_gps_time(idx) = ops_data_gps_time.properties.gps_time(match_idx); + if any(strcmp(ops_data.properties.season_name{idx}, track.crossover.season_names_bad)) + good_mask(idx) = false; + end + end + % Ensure crossover gps_time is good + good_mask = good_mask & track.crossover.gps_time_good_eval(crossover_gps_time); + + % Convert crossover twtt to source twtt and then convert from twtt + % to rows/bins + cols = round(interp1(mdata.GPS_time,1:length(mdata.GPS_time), source_gps_time(good_mask))); + rows = round(interp1(mdata.Time,1:length(mdata.Time), ops_data.properties.twtt(good_mask) ... + + (ops_data.properties.source_elev(good_mask) - ops_data.properties.cross_elev(good_mask))*2/c )); + track.crossovers = [cols; rows]; + track.crossovers(3,:) = track.viterbi.gt_cutoff; + end + end + + else + % Crossovers already loaded during layer_tracker.m + % Ensure season of the crossing line is good + good_mask = true(size(param.layer_tracker.crossover.twtt)); + for idx = 1:length(param.layer_tracker.crossover.twtt) + if any(strcmp(param.layer_tracker.crossover.season_name{idx}, track.crossover.season_names_bad)) + good_mask(idx) = false; + end + end + % Ensure gps_time of the crossing line is good + good_mask = good_mask & track.crossover.gps_time_good_eval(param.layer_tracker.crossover.crossover_gps_time); + + % Convert crossover twtt to source twtt and then convert from twtt + % to rows/bins + cols = round(interp1(mdata.GPS_time,1:length(mdata.GPS_time), param.layer_tracker.crossover.source_gps_time(good_mask))); + rows = round(interp1(mdata.Time,1:length(mdata.Time), param.layer_tracker.crossover.twtt(good_mask) ... + + (param.layer_tracker.crossover.source_elev(good_mask) - param.layer_tracker.crossover.cross_elev(good_mask))*2/c )); + % Crossover with NaN cols is a crossover that is not in this frame, + % remove any cols that fall outside of this frame + good_mask = isfinite(cols); + % Build crossover matrix, track.crossovers + track.crossovers = [cols(good_mask); rows(good_mask)]; + track.crossovers(3,:) = track.viterbi.gt_cutoff; + + end + + else + track.crossovers = zeros(2,0); + end + + %% Track: Multiple Suppression + if track.mult_suppress.en + data = lp(echo_mult_suppress(mdata,[],track.mult_suppress),1); + else + data = lp(mdata.Data,1); + end + + %% Track: Prefilter trim + % prefilter_trim: convert prefilter_trim to range bins + if strcmpi(track.prefilter_trim_units,'bins') + % Units of range bins (rows) + prefilter_trim = track.prefilter_trim; + else + % Units of time (default) + if isnan(dt) + prefilter_trim = [0 0]; + else + prefilter_trim = round(track.prefilter_trim/dt); + end + end for rline = 1:Nx - start_bin = find(isfinite(data(:,rline)),1); + % Step 1. Sets leading zeros or ~isfinite to NaN + start_bin = find(isfinite(data(:,rline)) & data(:,rline) ~= 0,1); if ~isempty(start_bin) - stop_bin = min(size(data,1), start_bin+track.prefilter_trim(1)-1); + % Found finite non-zero value + stop_bin = min(size(data,1), start_bin+prefilter_trim(1)-1); data(1:stop_bin,rline) = NaN; - end - stop_bin = find(isfinite(data(:,rline)),1,'last'); - if ~isempty(stop_bin)% - start_bin = max(1, stop_bin-track.prefilter_trim(2)+1); - data(start_bin:end,rline) = NaN; + + % Step 2. Sets leading/following zeros or ~isfinite to NaN + stop_bin = find(isfinite(data(:,rline)) & data(:,rline) ~= 0,1,'last'); + if ~isempty(stop_bin) + % Found finite non-zero value + start_bin = max(1, stop_bin-prefilter_trim(2)+1); + data(start_bin:end,rline) = NaN; + else + % No finite non-zero values in this range line + data(:,rline) = NaN; + end + + else + % No finite non-zero values in this range line + data(:,rline) = NaN; end end @@ -131,51 +482,16 @@ end %% Track: Detrend - if ischar(track.detrend) - % A detrend file was passed in - track.detrend_struct = detrend; - detrend_curve = interp_finite(interp1(detrend.time,interp_finite(detrend.min_means),mdata.Time),NaN); - if all(isnan(detrend_curve)) - error('Detrend curve is all NaN.'); - end - if 0 - % Debug - rline = 200; - figure(1); clf; - plot(data(:,rline)) - hold on - mean_power = nanmean(data,2); - plot(mean_power) - plot(detrend_curve); - keyboard - end - data = bsxfun(@minus,data,detrend_curve); - if track.data_noise_en - data_noise = bsxfun(@minus,data_noise,detrend_curve); - end - - elseif track.detrend > 0 - poly_x = (-size(data,1)/2+(1:size(data,1))).'; - mean_power = nanmean(data,2); - good_mask = isfinite(mean_power); - p = polyfit(poly_x(good_mask),mean_power(good_mask),track.detrend); - detrend_curve = polyval(p,poly_x); - detrend_curve(~good_mask) = NaN; - detrend_curve = interp_finite(detrend_curve,0); - if 0 - % Debug - rline = 200; - figure(1); clf; - plot(data(:,rline)) - hold on - plot(mean_power) - plot(detrend_curve); - keyboard - end - data = bsxfun(@minus,data,detrend_curve); - if track.data_noise_en - data_noise = bsxfun(@minus,data_noise,detrend_curve); - end + if isfield(track,'detrend') && ~isempty(track.detrend) + echo_detrend_data.Data = data; + echo_detrend_data.Time = mdata.Time; + echo_detrend_data.Surface = mdata.Surface; + data = echo_detrend(echo_detrend_data,track.detrend); + end + + %% Track: Fasttime Cross Correlation (xcorr) + if isfield(track,'xcorr') && ~isempty(track.xcorr) + data = echo_xcorr(data,track.xcorr); end %% Track: Sidelobe @@ -184,6 +500,38 @@ data(mask) = NaN; end + %% Track: Flatten + if isfield(track,'flatten') && ~isempty(track.flatten) + mdata_flatten = []; + mdata_flatten.param = param; + mdata_flatten.param.load.frm = param.layer_tracker.frms; + mdata_flatten.Data = data; + mdata_flatten.Time = mdata.Time; + mdata_flatten.GPS_time = mdata.GPS_time; + [data,resample_field] = echo_flatten(mdata_flatten,track.flatten); + % Debug to convert to twtt: resample_field = mdata.Time(1) + (resample_field-1)*dt; + % Flatten supporting fields + if strcmpi(track.init.method,'dem') + for rline = 1:Nx + track.dem(rline) = interp1(resample_field(:,rline),1:size(resample_field,1),track.dem(rline),'linear','extrap'); + end + for idx = 1:size(track.crossovers,2) + track.crossovers(2,idx) = interp1(resample_field(:,track.crossovers(1,idx)),1:size(resample_field,1),track.crossovers(2,idx),'linear','extrap'); + end + for idx = 1:size(track.ground_truths,2) + track.ground_truths(2,idx) = interp1(resample_field(:,track.ground_truths(1,idx)),1:size(resample_field,1),track.ground_truths(2,idx),'linear','extrap'); + end + end + end + + %% Track: Max Range Filter + if ~isequal(track.max_rng_filter,track.filter) + % Multilooking in cross-track/fast-time + data_max_rng = lp(nan_fir_dec(10.^(data/10).',ones(1,track.max_rng_filter(1))/track.max_rng_filter(1),1,[],[],[],[],2.0).'); + % Multilooking in along-track + data_max_rng = lp(nan_fir_dec(10.^(data_max_rng/10),ones(1,track.max_rng_filter(2))/track.max_rng_filter(2),1,[],[],[],[],2.0)); + end + %% Track: Filter if track.filter(1) ~= 1 % Multilooking in cross-track/fast-time @@ -209,6 +557,9 @@ if track.data_noise_en data_noise(start_bin:stop_bin,rline) = NaN; end + if ~isequal(track.max_rng_filter,track.filter) + data_max_rng(start_bin:stop_bin,rline) = NaN; + end end stop_bin = find(isfinite(data(:,rline)),1,'last'); if ~isempty(stop_bin) @@ -217,31 +568,123 @@ if track.data_noise_en data_noise(start_bin:stop_bin,rline) = NaN; end + if ~isequal(track.max_rng_filter,track.filter) + data_max_rng(start_bin:stop_bin,rline) = NaN; + end end end + + %% Track: Surface suppression + if ~isempty(track.surf_suppress) + time = mdata.Time; + for rline = 1:size(data,2) + surf = mdata.Surface(rline); + data(:,rline) = data(:,rline) + eval(track.surf_suppress.eval_cmd); + end + end + + %% Track: Compress values + if ~isempty(track.compress) + data = atan((data-track.compress/2)/2)*track.compress/pi+track.compress/2; + data(data<0) = 0; + end + + %% Track: Emphasize last + if ~isempty(track.emphasize_last) + last_metric = flipud(cumsum(flipud(data>track.emphasize_last.threshold))); + last_metric = 1 - last_metric ./ (max(last_metric)+1); + last_metric = circshift(last_metric,[track.emphasize_last.shift 0]); + data = data .* last_metric; + end %% Track: min_bin/max_bin + time to bins conversions - % Convert from two way travel time to bins - track.min_bin = find(mdata.Time >= orig_track.min_bin, 1); - track.max_bin = find(mdata.Time <= orig_track.max_bin, 1, 'last'); - dt = mdata.Time(2) - mdata.Time(1); - track.init.max_diff = orig_track.init.max_diff/dt; + if isnumeric(track.min_bin) + % Convert from two way travel time to bins + track.min_bin = ones(1,Nx) * find(mdata.Time >= orig_track.min_bin, 1); + elseif isstruct(track.min_bin) + % Load layer + track.min_bin = opsLoadLayers(layer_param, orig_track.min_bin); + % Interpolate min_bin_layer onto echogram GPS times + track.min_bin = layerdata.interp(mdata,track.min_bin); + track.min_bin = interp1(mdata.Time,1:length(mdata.Time),track.min_bin.twtt_ref); + end + + if isnumeric(track.max_bin) + % Convert from two way travel time to bins + track.max_bin = find(mdata.Time >= orig_track.max_bin, 1); + if isempty(track.max_bin) + track.max_bin = ones(1,Nx) * length(mdata.Time); + else + track.max_bin = ones(1,Nx) * track.max_bin; + end + + elseif isstruct(track.max_bin) + % Load layer + track.max_bin = opsLoadLayers(layer_param, orig_track.max_bin); + % Interpolate max_bin_layer onto echogram GPS times + track.max_bin = layerdata.interp(mdata,track.max_bin); + track.max_bin = interp1(mdata.Time,1:length(mdata.Time),track.max_bin.twtt_ref); + end + + if isfield(track,'flatten') && ~isempty(track.flatten) + % Flatten supporting fields + for rline = 1:Nx + track.min_bin(rline) = interp1(resample_field(:,rline),1:size(resample_field,1),track.min_bin(rline),'linear','extrap'); + track.max_bin(rline) = interp1(resample_field(:,rline),1:size(resample_field,1),track.max_bin(rline),'linear','extrap'); + end + end + track.min_bin(~isfinite(track.min_bin)) = 1; + track.max_bin(~isfinite(track.max_bin)) = size(data,1); + + % Fix invalid min_bin and max_bin by setting to the midpoint + invalid_min_max = track.max_bin < track.min_bin; + track.min_bin(invalid_min_max) = round((track.max_bin(invalid_min_max) + track.min_bin(invalid_min_max))/2); + track.max_bin(invalid_min_max) = track.min_bin(invalid_min_max); + + min_min_bin = round(max(1,min(track.min_bin))); + max_max_bin = round(min(size(data,1),max(track.max_bin))); + + if isnan(dt) + track.init.max_diff = inf; + else + track.init.max_diff = orig_track.init.max_diff/dt; + end if strcmpi(track.max_rng_units,'bins') track.max_rng = orig_track.max_rng(1) : orig_track.max_rng(end); else - track.max_rng = round(orig_track.max_rng(1)/dt) : round(orig_track.max_rng(end)/dt); + if isnan(dt) + track.max_rng = 0; + else + track.max_rng = round(orig_track.max_rng(1)/dt) : round(orig_track.max_rng(end)/dt); + end end - data = data(track.min_bin:track.max_bin,:); + data = data(min_min_bin:max_max_bin,:); if track.data_noise_en - data_noise = data_noise(track.min_bin:track.max_bin,:); + data_noise = data_noise(min_min_bin:max_max_bin,:); end + if ~isequal(track.max_rng_filter,track.filter) + data_max_rng = data_max_rng(min_min_bin:max_max_bin,:); + end + if isnan(dt) + track.zero_bin = 1; + else + track.zero_bin = floor(-mdata.Time(min_min_bin)/dt + 1); + end + + track.min_bin = track.min_bin - min_min_bin; + track.max_bin = track.max_bin - min_min_bin; + track.crossovers(2,:) = track.crossovers(2,:) - min_min_bin; - %% Track: Create Initial Surface + %% Track: Create Initial Layer if strcmpi(track.init.method,'dem') % Correct for min_bin removal - track.dem = track.dem - track.min_bin + 1; + track.dem = track.dem - min_min_bin + 1; elseif strcmp(track.init.method,'snake') - track.init.search_rng = round(orig_track.init.snake_rng(1)/dt) : round(orig_track.init.snake_rng(2)/dt); + if isnan(dt) + track.init.search_rng = 0; + else + track.init.search_rng = round(orig_track.init.snake_rng(1)/dt) : round(orig_track.init.snake_rng(2)/dt); + end track.dem = tracker_snake_simple(data,track.init); elseif strcmp(track.init.method,'nan') track.dem = nan(1,Nx); @@ -266,64 +709,81 @@ ops_layer{1}.quality(isnan(ops_layer{1}.quality)) = 1; lay = opsInterpLayersToMasterGPSTime(mdata,ops_layer,ref_interp_gaps_dist); dem_layer = lay.layerData{1}.value{2}.data; - dem_layer = interp1(mdata.Time,1:length(mdata.Time),dem_layer); + dem_layer = interp1(mdata.Time,1:length(mdata.Time),dem_layer)-min_min_bin+1; + if isfield(track,'flatten') && ~isempty(track.flatten) + % Flatten supporting fields + for rline = 1:Nx + dem_layer(rline) = interp1(resample_field(:,rline),1:size(resample_field,1),dem_layer(rline),'linear','extrap'); + end + end track.dem = merge_vectors(dem_layer, track.dem); end - %% Track: Tracking - - if strcmpi(track.method,'threshold') - track.threshold_noise_rng = round(orig_track.threshold_noise_rng/dt); - if track.data_noise_en - track.data_noise = data_noise; - else - track.data_noise = data; - end - [new_layer,new_quality] = tracker_threshold(data,track); - elseif strcmpi(track.method,'max') - new_layer = tracker_max(data,track); - new_quality = ones(1,Nx); - elseif strcmpi(track.method,'viterbi') - new_layer = tracker_viterbi(mdata,param); - new_quality = ones(1,Nx); - elseif strcmpi(track.method,'lsm') - if param.layer_tracker.track.track_data - %new_layer = tracker_lsm_track(mdata,param); - %new_quality = ones(1,Nx); - new_layer = load('/cresis/snfs1/projects/LSM_anjali/LSM_result_part9.mat'); - new_quality = ones(1,Nx); - else - new_layer = tracker_lsm(mdata,param); - new_quality = ones(1,Nx); - end - elseif strcmpi(track.method,'stereo') - [new_layer,big_matrix] = tracker_stereo(mdata,param); - new_quality = ones(1,Nx); - elseif strcmpi(track.method,'mcmc') - new_layer = tracker_mcmc(mdata,param); - new_quality = ones(1,Nx); - elseif strcmpi(track.method,'snake') - track.search_rng = round(orig_track.snake_rng(1)/dt) : round(orig_track.snake_rng(2)/dt); - new_layer = tracker_snake_simple(data,track); - new_quality = ones(1,Nx); - elseif strcmpi(track.method,'fixed') - new_layer = ones(size(mdata.GPS_time)) * track.fixed_value; - new_quality = ones(1,Nx); - elseif isempty(track.method) - new_layer = track.dem; - new_quality = ones(1,Nx); + %% Track: Normalization + if isfield(track,'norm') && ~isempty(track.norm) + data = echo_norm(data,track.norm); + end + + %% Track: Tracking + if strcmpi(track.method, 'lsm') && track.lsm.use_mean_surf_en + surf = interp_finite(interp1(param.layer_tracker.gt_params.gps_time,param.layer_tracker.gt_params.twtt,mdata.GPS_time)); + surf_bins = round(interp1(mdata.Time,1:length(mdata.Time),surf)); + track.lsm.y = mean(surf_bins); + end + + if strcmpi(track.method,'threshold') + if isnan(dt) + new_layers = nan(1,Nx); + new_quality = ones(1,Nx); + else + track.threshold_noise_rng = round(orig_track.threshold_noise_rng/dt); + if track.data_noise_en + track.data_noise = data_noise; else - error('Not a supported layer tracking method.'); + track.data_noise = data; end - - new_temp= new_layer; + [new_layers,new_quality] = tracker_threshold(data,track); + end + elseif strcmpi(track.method,'max') + new_layers = tracker_max(data,track); + new_quality = ones(1,Nx); + elseif strcmpi(track.method,'viterbi') + new_layers = tracker_viterbi(data,track); + new_quality = ones(1,Nx); + elseif strcmpi(track.method,'lsm') + new_layers = tracker_lsm(data,track); + new_quality = ones(1,Nx); + elseif strcmpi(track.method,'stereo') + [new_layers,big_matrix] = tracker_stereo(data,track); + new_quality = ones(1,Nx); + elseif strcmpi(track.method,'mcmc') + new_layers = tracker_mcmc(data,track); + new_quality = ones(1,Nx); + elseif strcmpi(track.method,'snake') + if isnan(dt) + new_layers = nan(1,Nx); + new_quality = ones(1,Nx); + else + track.search_rng = round(orig_track.snake_rng(1)/dt) : round(orig_track.snake_rng(2)/dt); + new_layers = tracker_snake_simple(data,track); + new_quality = ones(1,Nx); + end + elseif strcmpi(track.method,'fixed') + new_layers = ones(size(mdata.GPS_time)) * track.fixed_value; + new_quality = ones(1,Nx); + elseif isempty(track.method) + new_layers = track.dem; + new_quality = ones(1,Nx); + else + error('Not a supported layer tracking method.'); + end - for layer_idx = 1:length(layer_params) + % Trackers may output multiple layers (different rows in new_layers). For + % loop to post-process each of these layers. + for layer_idx = 1:size(new_layers,1) + new_layer = new_layers(layer_idx,:); - layer_param = layer_params(layer_idx); - - new_layer = new_temp(layer_idx,:); - track.init.max_diff = inf; + %% Track: max_diff new_layer(abs(new_layer - track.dem) > track.init.max_diff) = NaN; switch (track.init.max_diff_method) case 'merge_vectors' @@ -351,183 +811,132 @@ for rline = 1:Nx search_bins = round(new_layer(rline)) + track.max_rng; search_bins = search_bins(find(search_bins >= 1 & search_bins <= size(data,1))); - [~,offset] = max(data(search_bins,rline)); + if ~isequal(track.max_rng_filter,track.filter) + [~,offset] = max(data_max_rng(search_bins,rline)); + else + [~,offset] = max(data(search_bins,rline)); + end if ~isempty(offset) new_layer(rline) = search_bins(offset); end end end + %% Track: Smooth sgolayfilt + if ~isempty(track.smooth_sgolayfilt) + % This smooth filter is for smoothing the layer output + new_layer = sgolayfilt(new_layer,track.smooth_sgolayfilt{:}); + end + %% Track: Convert bins to twtt - Surface = interp1(1:length(mdata.Time), mdata.Time, new_layer + track.min_bin - 1,'linear','extrap'); - %Bottom = interp1(1:length(mdata.Time), mdata.Time, new_layer + track.min_bin - 1,'linear','extrap'); - %end x check if needed or not - % Some layer sources may not be "double", but we require that Surface be double: - Surface = double(Surface); - %Bottom = double(Bottom); + if isfield(track,'flatten') && ~isempty(track.flatten) + % Unflatten layers + for rline = 1:Nx + new_layer(rline) = interp1(1:size(resample_field,1),resample_field(:,rline),new_layer(rline)+min_min_bin-1,'linear','extrap'); + end + for rline = 1:Nx + track_dem(rline) = interp1(1:size(resample_field,1),resample_field(:,rline),track.dem(rline)+min_min_bin-1,'linear','extrap'); + end + if isnan(dt) + % Use "fake" dt of 1 when isnan(dt) + new_layer = mdata.Time(1) + (new_layer-1)*1; + track_dem = mdata.Time(1) + (track_dem-1)*1; + else + new_layer = mdata.Time(1) + (new_layer-1)*dt; + track_dem = mdata.Time(1) + (track_dem-1)*dt; + end + else + % Some layer sources may not be "double", but we require that layers be double type: + new_layer = interp1(1:length(mdata.Time), mdata.Time, new_layer + min_min_bin - 1,'linear','extrap'); + track_dem = interp1(1:length(mdata.Time), mdata.Time, track.dem + min_min_bin - 1,'linear','extrap'); + end + + %% Track: Remove overlap + if ~exist('overlap','var') + overlap = [0 0]; + end + overlap_rlines = 1+overlap(1) : Nx-overlap(2); + if any(overlap) > 0 + new_layer = new_layer(overlap_rlines); + new_quality = new_quality(overlap_rlines); + end %% Track: Debug plot - if ~isempty(param.layer_tracker.debug_plots) - h_fig = get_figures(1,true); - end - if param.layer_tracker.enable_debug_plot + if tracked_images_en || visible_en clf(h_fig(1)); - %set(h_fig(1),'name',sprintf('layer_tracker %s',data_fn_name)); + figure_name = sprintf('layer_tracker %s %d-%d layer %d',param.day_seg, param.layer_tracker.frms([1 end]), layer_idx); + set(h_fig(1),'name',figure_name); h_axes(1) = axes('parent',h_fig(1)); - imagesc([],mdata.Time,lp(mdata.Data), 'parent', h_axes(1)); + imagesc([],mdata.Time,lp(mdata.Data(:,overlap_rlines)), 'parent', h_axes(1)); colormap(h_axes(1), 1-gray(256)); hold(h_axes(1),'on'); - plot(h_axes(1),find(new_quality==1),Surface(new_quality==1),'g'); - plot(h_axes(1),find(new_quality==3),Surface(new_quality==3),'r'); + plot(h_axes(1),find(new_quality==1),new_layer(new_quality==1),'g.'); + plot(h_axes(1),find(new_quality==2|new_quality==3),new_layer(new_quality==2|new_quality==3),'r.'); if strcmpi(track.init.method,{'dem'}) || ~isempty(track.init.dem_layer) - plot(h_axes(1),interp1(1:length(mdata.Time),mdata.Time,track.dem+track.min_bin-1),'m--') - plot(h_axes(1),interp1(1:length(mdata.Time),mdata.Time, ... - track.dem+track.min_bin-1-track.init.max_diff),'r--') - plot(h_axes(1),interp1(1:length(mdata.Time),mdata.Time, ... - track.dem+track.min_bin-1+track.init.max_diff),'b--') + plot(h_axes(1),track_dem(:,overlap_rlines),'m--'); + plot(h_axes(1),track_dem(:,overlap_rlines)-orig_track.init.max_diff,'r--'); + plot(h_axes(1),track_dem(:,overlap_rlines)+orig_track.init.max_diff,'b--'); end hold(h_axes(1),'off'); if ~isempty(mdata.Time) - ylims = [max(mdata.Time(1),min(Surface)-track.debug_time_guard) min(mdata.Time(end),max(Surface)+track.debug_time_guard)]; - % if ylims(end)>ylims(1) - % ylim(h_axes(1),ylims); - % end - end - %title(h_axes(1),sprintf('%s',regexprep(data_fn_name,'_','\\_'))); - keyboard - end - - %% Track: Save - if param.layer_tracker.track.save_layerData - % if nargout == 1 - % return; - % end - % layer_params = param.layer_tracker.layer_params; - % for layer_idx = 1:length(layer_params) - % layer_param = layer_params(layer_idx); - try - lay_param.source = layer_param{1}.source; - lay_param.echogram_source = layer_param{1}.echogram_source; - lay_param.layerdata_source = layer_param{1}.layerdata_source; - catch ME - lay_param.source = layer_param.source; - lay_param.echogram_source = layer_param.echogram_source; - lay_param.layerdata_source = layer_param.layerdata_source; - end - - if strcmpi(lay_param.source,'echogram') - if isempty(lay_param.echogram_source) - lay_param.echogram_source = echogram_source; + ylims = [max(mdata.Time(1),min(new_layer)-track.debug_time_guard) min(mdata.Time(end),max(new_layer)+track.debug_time_guard)]; + if ylims(end)>ylims(1) + ylim(h_axes(1),ylims); end - - data_fn = fullfile(ct_filename_out(param,lay_param.echogram_source,''), ... - sprintf('Data_%s_%03d.mat', param.day_seg, frm)); - fprintf(' Saving %s (%s)\n', data_fn, datestr(now)); - %save(data_fn,'-append','Surface'); end - - if strcmpi(lay_param.source,'layerdata') - layer_fn = fullfile(ct_filename_out(param,lay_param.layerdata_source,''), ... - sprintf('Data_%s_%03d.mat', param.day_seg, frm)); - if ~exist(layer_fn,'file') - fprintf(' Create %s (%s)\n', layer_fn, datestr(now)); - - lay.GPS_time = mdata.GPS_time; - lay.Latitude = mdata.Latitude; - lay.Longitude = mdata.Longitude; - lay.Elevation = mdata.Elevation; - - lay.layerData{1}.quality = interp1(mdata.GPS_time,new_quality,lay.GPS_time,'nearest'); - lay.layerData{1}.value{1}.data = nan(size(lay.GPS_time)); - lay.layerData{1}.value{2}.data = interp1(mdata.GPS_time,Surface,lay.GPS_time); - lay.layerData{1}.value{2}.data = interp_finite(lay.layerData{1}.value{2}.data,NaN); - - lay.layerData{2}.quality = ones(size(lay.GPS_time)); - lay.layerData{2}.value{1}.data = nan(size(lay.GPS_time)); - lay.layerData{2}.value{2}.data = nan(size(lay.GPS_time)); - - layer_fn_dir = fileparts(layer_fn); - if ~exist(layer_fn_dir,'dir') - mkdir(layer_fn_dir); - end - %save(layer_fn,'-struct','lay'); - - else - % Load the layerData file - % lay = load(layer_fn); - % % Update the surface auto picks - % lay.layerData{layer_idx}.quality = interp1(mdata.GPS_time,new_quality,lay.GPS_time,'nearest'); - % lay.layerData{layer_idx}.value{2}.data = interp1(mdata.GPS_time,Surface,lay.GPS_time); - % %lay.layerData{1}.value{2}.data = interp1(mdata.GPS_time,new_temp.(sprintf('layer_%s_%03d',param.day_seg,frm)).top,lay.GPS_time); - % lay.layerData{layer_idx}.value{2}.data = interp_finite(lay.layerData{layer_idx}.value{2}.data,NaN); - % Append the new results back to the layerData file - fprintf(' Saving %s (%s)\n', layer_fn, datestr(now)); - %filename = fullfile(save_name,param.season_name,dir_name,param.day_seg,frm_dir, sprintf('layers_%s_%03d.mat',param.layer_tracker.track.method,layer_idx)); - filename = fullfile(save_name,param.season_name,dir_name1,dir_name2,param.day_seg,frm_dir,sprintf('layer_%s_%s',param.layer_tracker.track.method,param.layer_tracker.name)); - tmp_name = sprintf('%s_%s_%03d',param.layer_tracker.track.method,param.layer_tracker.name,layer_idx); - %quality{layer_idx} = interp1(mdata.GPS_time,new_quality,lay.GPS_time,'nearest'); - - % tmp.layerData{1} = cat(2,tmp.layerData{1},lay.layerData); - % tmp.GPS_time{1} = cat(2,tmp.GPS_time{1},lay.GPS_time); - % tmp.param_layer_tracker{1} = cat(2,tmp.tracker{1},param); - twtt{layer_idx} = cat(2,twtt{layer_idx},Surface); - - % tmp.layerData = lay.layerData; - % tmp.GPS_time = cat(2,tmp.GPS_time,lay.GPS_time); - % tmp.param_layer_tracker = param; - % if param.ct_file_lock - % tmp.file_version = '1L'; - % else - % tmp.file_version = '1'; - % end - % tracker_data{layer_idx} = struct(tmp_name,tmp); - %save(filename, '-struct','lay','layerData'); - %save(layer_fn,'-append','-struct','lay','layerData'); - end + title(h_axes(1),regexprep(figure_name,'_','\\_')); + if visible_en + fprintf('Debug plots: review "%s" and then run "dbcont" to continue.\n', figure_name); + keyboard end - if strcmpi(lay_param.source,'ops') - % Get all the frames for this segment - if any(strcmpi({layer_params.source},'ops')) - opsAuthenticate(param,false); - sys = ct_output_dir(param.radar_name); - ops_param = struct('properties',[]); - ops_param.properties.season = param.season_name; - ops_param.properties.segment = param.day_seg; - [status,ops_seg_data] = opsGetSegmentInfo(sys,ops_param); + if tracked_images_en + fig_fn = fullfile(ct_filename_ct_tmp(param,'',param.layer_tracker.debug_out_dir,''), ... + param.layer_tracker.layer_params.layerdata_source, ... + sprintf('%s_%s_l%03d_frm_%03d.jpg',track.name,track.method,layer_idx,param.layer_tracker.frms(1))); + fprintf('Saving %s\n', fig_fn); + fig_fn_dir = fileparts(fig_fn); + if ~exist(fig_fn_dir,'dir') + mkdir(fig_fn_dir); end - - % OPS query to get the point path ID's - ops_param = struct('properties',[]); - ops_param.properties.location = param.post.ops.location; - ops_param.properties.season = param.season_name; - ops_param.properties.start_gps_time = ops_seg_data.properties.start_gps_time(frm); - ops_param.properties.stop_gps_time = ops_seg_data.properties.stop_gps_time(frm); - - sys = ct_output_dir(param.radar_name); - [status,data] = opsGetPath(sys,ops_param); - - % Write the new layer information to these point path ID's - ops_param = struct('properties',[]); - ops_param.properties.point_path_id = data.properties.id; - ops_param.properties.twtt = interp_finite(interp1(mdata.GPS_time,Surface,data.properties.gps_time)); - ops_param.properties.type = 2*ones(size(ops_param.properties.twtt)); - ops_param.properties.quality = interp1(mdata.GPS_time,new_quality,data.properties.gps_time,'nearest'); - ops_param.properties.lyr_name = layer_param.name; - - opsCreateLayerPoints(sys,ops_param); + ct_saveas(h_fig(1),fig_fn); end - %end end + + new_layers(layer_idx,overlap_rlines) = new_layer; end + new_layers = new_layers(:,overlap_rlines); - gps_time = cat(2,gps_time,mdata.GPS_time); - param_layer_tracker = param; - if param.ct_file_lock - file_version = '1L'; + %% Track: Save + if isstruct(param.layer_tracker.echogram_source) + % Qlook mode + + % Return twtt in "success" variable + success = new_layers(:, frm_start(1):frm_stop(1)); + else - file_version = '1'; + % Cluster Mode + for frm_idx = 1:length(param.layer_tracker.frms) + frm = param.layer_tracker.frms(frm_idx); + + % Get just the current frames layer data + gps_time = mdata.GPS_time(frm_start(frm_idx)+overlap(1):frm_stop(frm_idx)+overlap(1)); + twtt = new_layers(:, frm_start(frm_idx):frm_stop(frm_idx)); + param_layer_tracker = param; + file_version = '1'; + file_type = 'layer_tracker'; + + tmp_out_fn_name = sprintf('t%03d_%s.mat', track_idx, track.method); + tmp_out_fn = fullfile(tmp_out_fn_dir_dir,sprintf('layer_tracker_%03d', frm),tmp_out_fn_name); + fprintf(' Saving %s (%s)\n', tmp_out_fn, datestr(now)); + tmp_out_fn_dir = fileparts(tmp_out_fn); + if ~exist(tmp_out_fn_dir,'dir') + mkdir(tmp_out_fn_dir); + end + ct_save(tmp_out_fn,'twtt','gps_time','param_layer_tracker','file_type','file_version'); + end + success = true; end - save(filename,'twtt','gps_time','param_layer_tracker','file_version');%,'indexes','twtt_test'); - success = true; -end \ No newline at end of file +end + +fprintf('Done %s\n',datestr(now)); diff --git a/cresis-toolbox/tracker/layer_tracker_tune.m b/cresis-toolbox/tracker/layer_tracker_tune.m new file mode 100644 index 00000000..f20f3efc --- /dev/null +++ b/cresis-toolbox/tracker/layer_tracker_tune.m @@ -0,0 +1,113 @@ +%% Script layer_tracker_tune.m + +% Used to find best combination of paramters for given tracker method +% Uses temporary saved files stored in CSARP_layer_tracker_tmp +% Enter layer names in the gt_layer_params(idx).name field +% set temp to the file location of the param structure used for tracking in run_layer_tracker.m script +% set save_name to store final tuning results +% Statistical data saved: +% number of points with finite values of ground truth of twtt (num_gt_infinite), +% number of points where ground truth twtt is finite and tracked data twtt is NaN (num_isnan) +% number of points where (tracked data twtt - ground truth twtt) < 5 * dt (num_points) +% nanmean of (tracked data twtt - ground truth twtt) (res_matrix) +% Authors: Anjali Pare, John Paden +% +% See layer_tracker_tune_plot.m to view 2 dimensional imagesc plots of the data. + + +dbstack_info = dbstack; +fprintf('=====================================================================\n'); +fprintf('%s (%s)\n', dbstack_info(1).name, datestr(now,'HH:MM:SS')); +fprintf('=====================================================================\n'); + +%% General User Settings + +temp = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140516_01_20200909_032808_t056_lsm.mat'); +param= temp.param; +filename = ct_filename_out(param,param.layer_tracker.layer_params.layerdata_source,'layer_tracker_tmp'); % where temporary files are saved +save_name = '/cresis/snfs1/scratch/anjali/cluster_tuning/resultTest3.mat'; % where to store tuning final result + +gt_layer_params = []; +res_matrix = []; +num_layers = 0; +idx = 1; +gt_layer_params(idx).name = 'surface'; +idx = idx + 1; +gt_layer_params(idx).name = 'bottom'; + +%% Find Best combination +%for layer_names = 1:length(gt_layer_names) +for layer_names = 1:idx + res_matrix{layer_names} = zeros([length(param.cmd.frms),param.layer_tracker.track{1}.idx_reshape]); + num_gt_isfinite.(sprintf('%s',gt_layer_params(layer_names).name)) = 0; + num_isnan.(sprintf('%s',gt_layer_params(layer_names).name)) = 0; + num_points.(sprintf('%s',gt_layer_params(layer_names).name)) = 0; + num_layers = num_layers+1; +end + +layers = opsLoadLayers(param,gt_layer_params); + +idx_matrix = [(1:length(param.layer_tracker.track{1}.idx_reshape))+1 1]; % used for resizing + +for track_idx = 1:length(param.layer_tracker.track) + frm_idx = 0; + for frm = param.cmd.frms + frm_idx = frm_idx+1; + data_fn_dir = ct_filename_out(param, param.layer_tracker.echogram_source, ''); + data_fn_name = sprintf('Data_%s_%03d.mat',param.day_seg,frm); + data_fn = fullfile(data_fn_dir, data_fn_name); + data = load(data_fn); + frm_dir = sprintf('layer_tracker_%03d',frm); + dt = data.Time(2) - data.Time(1); + + fname = sprintf('%s_%s.mat',param.layer_tracker.track{track_idx}.name,param.layer_tracker.track{track_idx}.method); + frame_fn_name = fullfile(filename,frm_dir,fname); + track_data = load(frame_fn_name); % temporary file data + ctr = 1; + layer_idx = 1; + pos = 0; + for i = 1:(temp.param.layer_tracker.track{track_idx}.idx_reshape(1)*num_layers) % Number of iterations + pos = pos + 1; + surf = interp1(layers(layer_idx).gps_time,layers(layer_idx).twtt,data.GPS_time); + surf_bins = round(interp1(data.Time,1:length(data.Time),surf)); + surf = interp1(track_data.gps_time,track_data.twtt(i,:),data.GPS_time); + surf_bins_itr = round(interp1(data.Time,1:length(data.Time),surf)); + num_gt_isfinite.(sprintf('%s',gt_layer_params(layer_idx).name)) = num_gt_isfinite.(sprintf('%s',gt_layer_params(layer_idx).name)) + sum(isfinite(surf_bins)); + num_isnan.(sprintf('%s',gt_layer_params(layer_idx).name)) = num_isnan.(sprintf('%s',gt_layer_params(layer_idx).name)) + (sum(isfinite(surf_bins) & ~isfinite(surf_bins_itr))); + num_points.(sprintf('%s',gt_layer_params(layer_idx).name)) = num_points.(sprintf('%s',gt_layer_params(layer_idx).name)) + (sum(abs(surf_bins-surf_bins_itr) < 5*dt)); + res_matrix{layer_idx}(frm_idx,param.layer_tracker.track{track_idx}.idx(pos)) = nanmean(abs(surf_bins - surf_bins_itr)); + + if mod(ctr,temp.param.layer_tracker.track{track_idx}.idx_reshape(1)) == 0 + layer_idx = layer_idx +1; + pos = 0; + end + ctr = ctr +1; + end + end +end + +points = []; +min_val = []; +res_matrix_all_frms = []; +for layer_names = 1:idx + res_matrix_all_frms{layer_names} = nanmean(res_matrix{layer_names},1); + res_matrix_all_frms{layer_names} = permute(res_matrix_all_frms{layer_names},idx_matrix); + [min_val{layer_names},i]=min(res_matrix_all_frms{layer_names}(:)); + sizeMatrix = size(res_matrix_all_frms{layer_names}); + + for dim = 1:length(size(res_matrix_all_frms{layer_names})) + points{layer_names}(dim) = mod(floor((i-1)/prod(sizeMatrix(1:dim-1))),sizeMatrix(dim))+1; + end +end + +file_version = '1'; +file_type = 'layer_tracker_tuning'; + +fn_dir = fileparts(save_name); +if ~exist(fn_dir,'dir') + mkdir(fn_dir); +end + +save(save_name,'res_matrix','num_isnan','num_points','num_gt_isfinite','points','min_val','res_matrix_all_frms','param','file_version','file_type'); + + diff --git a/cresis-toolbox/tracker/layer_tracker_tune_layerdata.m b/cresis-toolbox/tracker/layer_tracker_tune_layerdata.m new file mode 100644 index 00000000..447a3d1a --- /dev/null +++ b/cresis-toolbox/tracker/layer_tracker_tune_layerdata.m @@ -0,0 +1,161 @@ +%% Script layer_tracker_tune_layerdata.m + +% Tuning script to work with layerdata files rather than temporary files +% Used to find best combination of paramters for given tracker method +% Enter layer names in the gt_layer_params(idx).name field +% set temp to the file location of the param structure used for tracking in run_layer_tracker.m script +% set save_name to store final tuning results +% Statistical data saved: +% number of points with finite values of ground truth of twtt (num_gt_infinite), +% number of points where ground truth twtt is finite and tracked data twtt is NaN (num_isnan) +% number of points where (tracked data twtt - ground truth twtt) < 5 * dt (num_points) +% nanmean of (tracked data twtt - ground truth twtt) (res_matrix) +% Authors: Anjali Pare, John Paden +% See layer_tracker_tune_plot.m to view 2 dimensional imagesc plots of the data. + +dbstack_info = dbstack; +fprintf('=====================================================================\n'); +fprintf('%s (%s)\n', dbstack_info(1).name, datestr(now,'HH:MM:SS')); +fprintf('=====================================================================\n'); + +%% General User Settings + +temp = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140516_01_20200921_121933_t063_lsm.mat'); +param= temp.param; +save_name = '/cresis/snfs1/scratch/anjali/cluster_tuning/result_layer_tune_mean.mat'; % where to store tuning final result + +gt_layer_params = []; +layer_params = []; +res_matrix = []; +num_layers = 0; +idx = 1; +gt_layer_params(idx).name = 'surface'; +idx = idx + 1; +gt_layer_params(idx).name = 'bottom'; +layers = opsLoadLayers(param,gt_layer_params); + +%% Find Best Combination +for layer_names = 1:idx + res_matrix{layer_names} = zeros([length(param.cmd.frms),param.layer_tracker.track{1}.idx_reshape]); + num_gt_isfinite.(sprintf('%s',gt_layer_params(layer_names).name)) = 0; + num_isnan.(sprintf('%s',gt_layer_params(layer_names).name)) = 0; + num_points.(sprintf('%s',gt_layer_params(layer_names).name)) = 0; + num_layers = num_layers+1; +end + + +idx_matrix = [(1:length(param.layer_tracker.track{1}.idx_reshape))+1 1]; % used for resizing + +for track_idx = 1:length(param.layer_tracker.track) + frm_idx = 0; + for frm = param.cmd.frms + frm_idx = frm_idx+1; + data_fn_dir = ct_filename_out(param, param.layer_tracker.echogram_source, ''); + data_fn_name = sprintf('Data_%s_%03d.mat',param.day_seg,frm); + data_fn = fullfile(data_fn_dir, data_fn_name); + data = load(data_fn); + dt = data.Time(2) - data.Time(1); + for layer_idx = 1:length(gt_layer_params) + if strcmpi(param.layer_tracker.track{1}.method,'lsm') + for idx = 1:length(param.layer_tracker.track{track_idx}.lsm.storeIter) + layer_params(idx).name = sprintf('%s_%s_%s_%03d',param.layer_tracker.track{track_idx}.name,param.layer_tracker.track{track_idx}.method,gt_layer_params(layer_idx).name,idx); + layer_params(idx).source = param.layer_tracker.layer_params.source; + layer_params(idx).layerdata_source = param.layer_tracker.layer_params.layerdata_source; + end + else + layer_params(1).name = sprintf('%s_%s_%s_%03d',param.layer_tracker.track{track_idx}.name,param.layer_tracker.track{track_idx}.method,gt_layer_params(layer_idx).name,idx); + layer_params(1).source = param.layer_tracker.layer_params.source; + layer_params(1).layerdata_source = param.layer_tracker.layer_params.layerdata_source; + end + + layers_new = opsLoadLayers(param,layer_params); + + surf = interp1(layers(layer_idx).gps_time,layers(layer_idx).twtt,data.GPS_time); + surf_bins = round(interp1(data.Time,1:length(data.Time),surf)); + + for pos = 1:length(layers_new) + surf = interp1(layers_new(pos).gps_time,layers_new(pos).twtt,data.GPS_time); + surf_bins_itr = round(interp1(data.Time,1:length(data.Time),surf)); + num_gt_isfinite.(sprintf('%s',gt_layer_params(layer_idx).name)) = num_gt_isfinite.(sprintf('%s',gt_layer_params(layer_idx).name)) + sum(isfinite(surf_bins)); + num_isnan.(sprintf('%s',gt_layer_params(layer_idx).name)) = num_isnan.(sprintf('%s',gt_layer_params(layer_idx).name)) + (sum(isfinite(surf_bins) & ~isfinite(surf_bins_itr))); + num_points.(sprintf('%s',gt_layer_params(layer_idx).name)) = num_points.(sprintf('%s',gt_layer_params(layer_idx).name)) + (sum(abs(surf_bins-surf_bins_itr) < 5*dt)); + res_matrix{layer_idx}(frm_idx,param.layer_tracker.track{track_idx}.idx(pos)) = nanmean(abs(surf_bins - surf_bins_itr)); + + end + end + end +end + +points = []; +min_val = []; +res_matrix_all_frms = []; +if strcmpi(param.layer_tracker.track{1}.method,'lsm') + if param.layer_tracker.track{track_idx}.flag ~= 1 + for layer_names = 1:num_layers + res_matrix_all_frms{layer_names} = nanmean(res_matrix{layer_names},1); + res_matrix_all_frms{layer_names} = permute(res_matrix_all_frms{layer_names},idx_matrix); + [min_val{layer_names},i]=min(res_matrix_all_frms{layer_names}(:)); + sizeMatrix = size(res_matrix_all_frms{layer_names}); + for dim = 1:length(size(res_matrix_all_frms{layer_names})) + points{layer_names}(dim) = mod(floor((i-1)/prod(sizeMatrix(1:dim-1))),sizeMatrix(dim))+1; + end + if (length(size(res_matrix_all_frms{layer_names})) < length(param.layer_tracker.track{1}.idx_reshape)) + points{layer_names}(dim+1) = 1; + end + end + + else + res_matrix_mean = []; + min_val_mean = []; + points_mean = []; + for layer_names = 1:num_layers + res_matrix_all_frms{layer_names} = nanmean(res_matrix{layer_names},1); + res_matrix_all_frms{layer_names} = permute(res_matrix_all_frms{layer_names},idx_matrix); + + res_matrix_mean{layer_names} = squeeze(res_matrix_all_frms{layer_names}(:,:,1)); % since we want all the combinations of number of itertaions and dy for the surface mean + res_matrix_all_frms{layer_names}(:,:,1) = []; + + [min_val{layer_names},i]=min(res_matrix_all_frms{layer_names}(:)); + sizeMatrix = size(res_matrix_all_frms{layer_names}); + for dim = 1:length(size(res_matrix_all_frms{layer_names})) + points{layer_names}(dim) = mod(floor((i-1)/prod(sizeMatrix(1:dim-1))),sizeMatrix(dim))+1; + end + + [min_val_mean{layer_names},j]=min(res_matrix_mean{layer_names}(:)); + sizeMatrix = size(res_matrix_mean{layer_names}); + for dim = 1:length(size(res_matrix_mean{layer_names})) + points_mean{layer_names}(dim) = mod(floor((j-1)/prod(sizeMatrix(1:dim-1))),sizeMatrix(dim))+1; + end + if (length(size(res_matrix_all_frms{layer_names})) < length(param.layer_tracker.track{1}.idx_reshape)) + points{layer_names}(dim+1) = 1; + end + if (length(size(res_matrix_mean{layer_names})) < length(param.layer_tracker.track{1}.idx_reshape)) + points_mean{layer_names}(dim+1) = 1; + end + end + end +else + for layer_names = 1:num_layers + res_matrix_all_frms{layer_names} = nanmean(res_matrix{layer_names},1); + res_matrix_all_frms{layer_names} = permute(res_matrix_all_frms{layer_names},idx_matrix); + [min_val{layer_names},i]=min(res_matrix_all_frms{layer_names}(:)); + sizeMatrix = size(res_matrix_all_frms{layer_names}); + for dim = 1:length(size(res_matrix_all_frms{layer_names})) + points{layer_names}(dim) = mod(floor((i-1)/prod(sizeMatrix(1:dim-1))),sizeMatrix(dim))+1; + end + if (length(size(res_matrix_all_frms{layer_names})) < length(param.layer_tracker.track{1}.idx_reshape)) + points{layer_names}(dim+1) = 1; + end + end +end + +file_version = '1'; +file_type = 'layer_tracker_tuning'; + +fn_dir = fileparts(save_name); +if ~exist(fn_dir,'dir') + mkdir(fn_dir); +end + +save(save_name,'res_matrix','num_isnan','num_points','num_gt_isfinite','points','min_val','res_matrix_all_frms','param','file_version','file_type'); + diff --git a/cresis-toolbox/tracker/layer_tracker_tune_layerdata_vit.m b/cresis-toolbox/tracker/layer_tracker_tune_layerdata_vit.m new file mode 100644 index 00000000..7e3bcfbe --- /dev/null +++ b/cresis-toolbox/tracker/layer_tracker_tune_layerdata_vit.m @@ -0,0 +1,103 @@ +%% Script layer_tracker_tune_layerdata.m + +% Tuning script to work with layerdata files rather than temporary files +% Used to find best combination of paramters for given tracker method +% Enter layer names in the gt_layer_params(idx).name field +% set temp to the file location of the param structure used for tracking in run_layer_tracker.m script +% set save_name to store final tuning results +% Statistical data saved: +% number of points with finite values of ground truth of twtt (num_gt_infinite), +% number of points where ground truth twtt is finite and tracked data twtt is NaN (num_isnan) +% number of points where (tracked data twtt - ground truth twtt) < 5 * dt (num_points) +% nanmean of (tracked data twtt - ground truth twtt) (res_matrix) +% Authors: Anjali Pare, John Paden +% See layer_tracker_tune_plot.m to view 2 dimensional imagesc plots of the data. + +dbstack_info = dbstack; +fprintf('=====================================================================\n'); +fprintf('%s (%s)\n', dbstack_info(1).name, datestr(now,'HH:MM:SS')); +fprintf('=====================================================================\n'); + +%% General User Settings + +temp = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140516_01_20201122_193803_t005_viterbi.mat'); +param= temp.param; +save_name = '/cresis/snfs1/scratch/anjali/cluster_tuning/result_layer_tune_vit_s012'; % where to store tuning final result + +gt_layer_params = []; +layer_params = []; +res_matrix = []; +num_layers = 0; +idx = 1; +gt_layer_params(idx).name = 'bottom'; +layers = opsLoadLayers(param,gt_layer_params); + +%% Find Best Combination +for layer_names = 1:idx + res_matrix{layer_names} = zeros([length(param.cmd.frms),param.layer_tracker.track{1}.idx_reshape]); + num_gt_isfinite.(sprintf('%s',gt_layer_params(layer_names).name)) = 0; + num_isnan.(sprintf('%s',gt_layer_params(layer_names).name)) = 0; + num_points.(sprintf('%s',gt_layer_params(layer_names).name)) = 0; + num_layers = num_layers+1; +end + + +idx_matrix = [(1:length(param.layer_tracker.track{1}.idx_reshape))+1 1]; % used for resizing + +for track_idx = 1:length(param.layer_tracker.track) + frm_idx = 0; + for frm = param.cmd.frms + frm_idx = frm_idx+1; + data_fn_dir = ct_filename_out(param, param.layer_tracker.echogram_source, ''); + data_fn_name = sprintf('Data_%s_%03d.mat',param.day_seg,frm); + data_fn = fullfile(data_fn_dir, data_fn_name); + data = load(data_fn); + dt = data.Time(2) - data.Time(1); + for layer_idx = 1:length(gt_layer_params) + + layer_params(1).name = sprintf('%s_%s_%s',param.layer_tracker.track{track_idx}.name,param.layer_tracker.track{track_idx}.method,gt_layer_params(layer_idx).name); + layer_params(1).source = param.layer_tracker.layer_params.source; + layer_params(1).layerdata_source = param.layer_tracker.layer_params.layerdata_source; + layers_new = opsLoadLayers(param,layer_params); + + surf = interp1(layers(layer_idx).gps_time,layers(layer_idx).twtt,data.GPS_time); + surf_bins = round(interp1(data.Time,1:length(data.Time),surf)); + + for pos = 1:length(layers_new) + surf = interp1(layers_new(pos).gps_time,layers_new(pos).twtt,data.GPS_time); + surf_bins_itr = round(interp1(data.Time,1:length(data.Time),surf)); + num_gt_isfinite.(sprintf('%s',gt_layer_params(layer_idx).name)) = num_gt_isfinite.(sprintf('%s',gt_layer_params(layer_idx).name)) + sum(isfinite(surf_bins)); + num_isnan.(sprintf('%s',gt_layer_params(layer_idx).name)) = num_isnan.(sprintf('%s',gt_layer_params(layer_idx).name)) + (sum(isfinite(surf_bins) & ~isfinite(surf_bins_itr))); + num_points.(sprintf('%s',gt_layer_params(layer_idx).name)) = num_points.(sprintf('%s',gt_layer_params(layer_idx).name)) + (sum(abs(surf_bins-surf_bins_itr) < 5*dt)); + res_matrix{layer_idx}(frm_idx,param.layer_tracker.track{track_idx}.idx(pos)) = nanmean(abs(surf_bins - surf_bins_itr)); + + end + end + end +end + +points = []; +min_val = []; +res_matrix_all_frms = []; +for layer_names = 1:num_layers +res_matrix_all_frms{layer_names} = nanmean(res_matrix{layer_names},1); +res_matrix_all_frms{layer_names} = permute(res_matrix_all_frms{layer_names},idx_matrix); +[min_val{layer_names},i]=min(res_matrix_all_frms{layer_names}(:)); +sizeMatrix = size(res_matrix_all_frms{layer_names}); + for dim = 1:length(size(res_matrix_all_frms{layer_names})) + points{layer_names}(dim) = mod(floor((i-1)/prod(sizeMatrix(1:dim-1))),sizeMatrix(dim))+1; + end + if (length(size(res_matrix_all_frms{layer_names})) < length(param.layer_tracker.track{1}.idx_reshape)) + points{layer_names}(dim+1) = 1; + end +end + +file_version = '1'; +file_type = 'layer_tracker_tuning'; + +fn_dir = fileparts(save_name); +if ~exist(fn_dir,'dir') + mkdir(fn_dir); +end + +save(save_name,'res_matrix','num_isnan','num_points','num_gt_isfinite','points','min_val','res_matrix_all_frms','param','file_version','file_type'); diff --git a/cresis-toolbox/tracker/layer_tracker_tune_plot.m b/cresis-toolbox/tracker/layer_tracker_tune_plot.m new file mode 100644 index 00000000..6a5f1994 --- /dev/null +++ b/cresis-toolbox/tracker/layer_tracker_tune_plot.m @@ -0,0 +1,122 @@ +function success = layer_tracker_tune_plot +%% Function layer_tracker_tune_plot.m + +% Used to plot 2 dimensional imagesc plots of data matrix from layer_tracker_tune.m +% set filename to the file location of where result data matrix is stored in layer_tracker_plot.m script +% set number of total layers (num_layers) +% set the format image must be saved in (e.g. fig, jpg, png) +% Plots will be saved in same folder as specified in filename +% Plots will be saved in the filename formatted as +% Authors: Anjali Pare, John Paden +% +% See layer_tracker_tune.m +%% Set Input +% ===================================================================== +%filename = ct_filename_out(param,param.layer_tracker.layer_params.layerdata_source,'layer_tracker_tmp'); %where temporary files are saved +img_format = 'jpg'; +filename = '/cresis/snfs1/scratch/anjali/cluster_tuning/result_layer_tune_vit_s011.mat'; % where the matrix is saved. Images generated will be saved in the same folder +num_layers = 1; % enter number of layers +img_dir = fileparts(filename); +if ~exist(img_dir,'dir') + mkdir(img_dir); +end + +%% Generate Input for Figure +matrix_fn = load(filename); +param = matrix_fn.param; +data = []; +for dim = 1:length(param.layer_tracker.track{1}.idx_reshape) + temp = []; + for track = 1:length(param.layer_tracker.track) + if isempty(temp) + temp = param.layer_tracker.track{track}.(param.layer_tracker.track{track}.method).(param.layer_tracker.track{track}.idx_dim_name{dim}); + else + if ~any(temp == param.layer_tracker.track{track}.(param.layer_tracker.track{track}.method).(param.layer_tracker.track{track}.idx_dim_name{dim})) + temp(end+1) = param.layer_tracker.track{track}.(param.layer_tracker.track{track}.method).(param.layer_tracker.track{track}.idx_dim_name{dim}); + end + end + + end + data{dim}=temp; +end + + +for layer_idx = 1:num_layers + res_mat = matrix_fn.res_matrix_all_frms{layer_idx}; + points = matrix_fn.points{layer_idx}; + + A = nchoosek(1:length(param.layer_tracker.track{1}.idx_reshape),2); % changed from 2 + for id = 1:length(A) + min_points = setdiff(1:length(param.layer_tracker.track{1}.idx_reshape),A(id,:)); + min_idxs = points(min_points); + res_matrix = permute(res_mat,[A(id,:) setdiff(1:length(param.layer_tracker.track{1}.idx_reshape),A(id,:))]); + res_matrix = res_matrix(:,:,min_idxs(:)); + idx = A(id,:); + labelTickX = data{idx(1)}; % x tick label + labelTickY = data{idx(2)}; % y tick label + labelNameX = param.layer_tracker.track{1}.idx_dim_name{idx(1)}; % x label name + labelNameY = param.layer_tracker.track{1}.idx_dim_name{idx(2)}; % y label name + xTick = 1:length(labelTickX); % x tick + yTick = 1:length(labelTickY); % y tick + %% Generate Figure + h_fig(1) = figure('visible','off'); % figure is not displayed + imagesc(squeeze(res_matrix')); + hold on; + plot(points(idx(1)),points(idx(2)),'x','LineWidth',4,'MarkerSize',10,'Color','white'); + + ylabel(labelNameY); + xlabel(labelNameX); + set(gca,'XTickLabel',labelTickX); + set(gca,'YTickLabel',labelTickY); + set(gca,'XTick',xTick); + set(gca,'YTick',yTick); + h_colorbar = colorbar(gca); + set(get(h_colorbar,'YLabel'),'String','Mean absolute error (rows)'); + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%03d_%s_%s_%s.%s',layer_idx,testname,labelNameX,labelNameY,img_format)); + ct_saveas(h_fig(1),img_fn); + + end +end + +% if param.layer_tracker.track{1}.flag == 1 +% for layer_idx = 1:num_layers +% res_mat = matrix_fn.res_matrix_mean{layer_idx}; +% points = matrix_fn.points_mean{layer_idx}; +% +% A = nchoosek(1:length(param.layer_tracker.track{1}.idx_reshape),2); +% for id = 1:length(A) +% min_points = setdiff(1:length(param.layer_tracker.track{1}.idx_reshape),A(id,:)); +% min_idxs = points(min_points); +% res_matrix = permute(res_mat,[A(id,:) setdiff(1:length(param.layer_tracker.track{1}.idx_reshape),A(id,:))]); +% res_matrix = res_matrix(:,:,min_idxs(:)); +% idx = A(id,:); +% labelTickX = data{idx(1)}; % x tick label +% labelTickY = data{idx(2)}; % y tick label +% labelNameX = param.layer_tracker.track{1}.idx_dim_name{idx(1)}; % x label name +% labelNameY = param.layer_tracker.track{1}.idx_dim_name{idx(2)}; % y label name +% xTick = 1:length(labelTickX); % x tick +% yTick = 1:length(labelTickY); % y tick +% %% Generate Figure +% h_fig(1) = figure('visible','off'); % figure is not displayed +% imagesc(squeeze(res_matrix')); +% hold on; +% plot(points(idx(1)),points(idx(2)),'x','LineWidth',4,'MarkerSize',10,'Color','white'); +% +% ylabel(labelNameY); +% xlabel(labelNameX); +% set(gca,'XTickLabel',labelTickX); +% set(gca,'YTickLabel',labelTickY); +% set(gca,'XTick',xTick); +% set(gca,'YTick',yTick); +% h_colorbar = colorbar(gca); +% set(get(h_colorbar,'YLabel'),'String','Mean absolute error (rows)'); +% testname = param.layer_tracker.layer_params.layerdata_source; +% testname = 'check'; +% img_fn = fullfile(img_dir,sprintf('%03d_%s_%s_%s.%s',layer_idx,testname,labelNameX,labelNameY,img_format)); +% ct_saveas(h_fig(1),img_fn); +% +% end +% end +% end +end \ No newline at end of file diff --git a/cresis-toolbox/tracker/layer_tracker_tune_plot_vit.m b/cresis-toolbox/tracker/layer_tracker_tune_plot_vit.m new file mode 100644 index 00000000..8320a15f --- /dev/null +++ b/cresis-toolbox/tracker/layer_tracker_tune_plot_vit.m @@ -0,0 +1,99 @@ +function success = layer_tracker_tune_plot_vit +%% Function layer_tracker_tune_plot.m + +% Used to plot 2 dimensional imagesc plots of data matrix from layer_tracker_tune.m +% set filename to the file location of where result data matrix is stored in layer_tracker_plot.m script +% set number of total layers (num_layers) +% set the format image must be saved in (e.g. fig, jpg, png) +% Plots will be saved in same folder as specified in filename +% Plots will be saved in the filename formatted as +% Authors: Anjali Pare, John Paden +% +% See layer_tracker_tune.m +%% Set Input +% ===================================================================== +%filename = ct_filename_out(param,param.layer_tracker.layer_params.layerdata_source,'layer_tracker_tmp'); %where temporary files are saved +img_format = 'jpg'; +filename = '/cresis/snfs1/scratch/anjali/cluster_tuning/result_layer_tune_vit_s012.mat'; % where the matrix is saved. Images generated will be saved in the same folder +num_layers = 1; % enter number of layers +img_dir = fileparts(filename); +if ~exist(img_dir,'dir') + mkdir(img_dir); +end + +%% Generate Input for Figure +matrix_fn = load(filename); +param = matrix_fn.param; +data = []; +for dim = 1:length(param.layer_tracker.track{1}.idx_reshape) + temp = []; + for track = 1:length(param.layer_tracker.track) + if isempty(temp) + temp = param.layer_tracker.track{track}.(param.layer_tracker.track{track}.method).(param.layer_tracker.track{track}.idx_dim_name{dim}); + else + if ~any(temp == param.layer_tracker.track{track}.(param.layer_tracker.track{track}.method).(param.layer_tracker.track{track}.idx_dim_name{dim})) + temp(end+1) = param.layer_tracker.track{track}.(param.layer_tracker.track{track}.method).(param.layer_tracker.track{track}.idx_dim_name{dim}); + end + end + + end + data{dim}=temp; +end + + +for layer_idx = 1:num_layers + res_mat = matrix_fn.res_matrix_all_frms{layer_idx}; + points = matrix_fn.points{layer_idx}; + x = 1:length(res_mat); % x tick + figure; + plot(x,res_mat); + hold on; + for i = 1:length(res_mat) + plot(x(i),res_mat(i), 'r*'); + end + plot(x(points(1)),res_mat(points(1)), 'ko'); + set(gca,'XTick',x); +end + + +% if param.layer_tracker.track{1}.flag == 1 +% for layer_idx = 1:num_layers +% res_mat = matrix_fn.res_matrix_mean{layer_idx}; +% points = matrix_fn.points_mean{layer_idx}; +% +% A = nchoosek(1:length(param.layer_tracker.track{1}.idx_reshape),2); +% for id = 1:length(A) +% min_points = setdiff(1:length(param.layer_tracker.track{1}.idx_reshape),A(id,:)); +% min_idxs = points(min_points); +% res_matrix = permute(res_mat,[A(id,:) setdiff(1:length(param.layer_tracker.track{1}.idx_reshape),A(id,:))]); +% res_matrix = res_matrix(:,:,min_idxs(:)); +% idx = A(id,:); +% labelTickX = data{idx(1)}; % x tick label +% labelTickY = data{idx(2)}; % y tick label +% labelNameX = param.layer_tracker.track{1}.idx_dim_name{idx(1)}; % x label name +% labelNameY = param.layer_tracker.track{1}.idx_dim_name{idx(2)}; % y label name +% xTick = 1:length(labelTickX); % x tick +% yTick = 1:length(labelTickY); % y tick +% %% Generate Figure +% h_fig(1) = figure('visible','off'); % figure is not displayed +% imagesc(squeeze(res_matrix')); +% hold on; +% plot(points(idx(1)),points(idx(2)),'x','LineWidth',4,'MarkerSize',10,'Color','white'); +% +% ylabel(labelNameY); +% xlabel(labelNameX); +% set(gca,'XTickLabel',labelTickX); +% set(gca,'YTickLabel',labelTickY); +% set(gca,'XTick',xTick); +% set(gca,'YTick',yTick); +% h_colorbar = colorbar(gca); +% set(get(h_colorbar,'YLabel'),'String','Mean absolute error (rows)'); +% testname = param.layer_tracker.layer_params.layerdata_source; +% testname = 'check'; +% img_fn = fullfile(img_dir,sprintf('%03d_%s_%s_%s.%s',layer_idx,testname,labelNameX,labelNameY,img_format)); +% ct_saveas(h_fig(1),img_fn); +% +% end +% end +% end +end \ No newline at end of file diff --git a/cresis-toolbox/tracker/run_block_data.m b/cresis-toolbox/tracker/run_block_data.m new file mode 100644 index 00000000..b886219c --- /dev/null +++ b/cresis-toolbox/tracker/run_block_data.m @@ -0,0 +1,85 @@ +% run_block_data.m +% +%Script for running block_data +% +% Authors: Ibikunle, John Paden +% +% See also: run_block_data.m, block_data.m, run_unblock_data.m, +% unblock_data.m + +%% User Setup +% ===================================================================== +param_override = []; + +params = read_param_xls(ct_filename_param('snow_param_2016_Greenland_P3.xls')); + +% Syntax for running a specific segment and frame by overriding parameter spreadsheet values +%params = read_param_xls(ct_filename_param('rds_param_2009_Antarctica_TO.xls'),'20091228_01'); +params = ct_set_params(params,'cmd.generic',0); +params = ct_set_params(params,'cmd.generic',1,'day_seg','20160519_01'); +params = ct_set_params(params,'cmd.frms',[34:44]); + +param_override.block_data.block_along_track = 50e3; % Along-track length of each block +param_override.block_data.block_Nx = 256; % Number of samples in each block +param_override.block_data.block_overlap = 0.5; % Set the % of overlap between each block +param_override.block_data.rows.t0_pad = 50; +param_override.block_data.rows.t1_pad = 35; + +param_override.block_data.inc_B_filter = ones(1,41)/41; +param_override.block_data.inc_dec = 20; +param_override.block_data.nan_dec_normalize_threshold = []; +param_override.block_data.nan_dec = false; +param_override.block_data.norm.scale = [0.4 1]; +param_override.block_data.detrend.method = 'polynomial'; + +param_override.block_data.flatten.resample_field = []; +param_override.block_data.flatten.resample_field.name = 'surface'; +param_override.block_data.flatten.resample_field.source = 'layerdata'; +param_override.block_data.flatten.resample_field.layerdata_source = 'layer_overly2021'; +param_override.block_data.flatten.interp_method = []; + +param_override.block_data.echo_path = 'CSARP_post/qlook'; + +param_override.block_data.out_path = 'block_data'; + +param_override.block_data.surf_param = []; +param_override.block_data.surf_param.name = 'surface'; +param_override.block_data.surf_param.source = 'layerdata'; +param_override.block_data.surf_param.layerdata_source = 'layer_overly2021'; + +param_override.block_data.layer_params = []; +param_override.block_data.layer_params.name = 'surface'; +param_override.block_data.layer_params(1).source = 'layerdata'; +param_override.block_data.layer_params(1).layerdata_source = 'layer_overly2021'; +param_override.block_data.layer_params(2).regexp = 'snow.*'; +param_override.block_data.layer_params(2).source = 'layerdata'; +param_override.block_data.layer_params(2).layerdata_source = 'layer_overly2021'; + +param_override.block_data.file.img_en = true; +param_override.block_data.file.layer_bin_en = true; +param_override.block_data.file.layer_mult_en = true; +param_override.block_data.file.layer_seg_en = true; +param_override.block_data.file.mat_en = true; + +%% Automated Section +% ===================================================================== + +% Input checking +global gRadar; +if exist('param_override','var') + param_override = merge_structs(gRadar,param_override); +else + param_override = gRadar; +end + +% Process each of the segments +for param_idx = 1:length(params) + param = params(param_idx); + if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic + continue; + end + %block_data(param,param_override); + block_data + return +end + diff --git a/cresis-toolbox/tracker/run_block_data_rds.m b/cresis-toolbox/tracker/run_block_data_rds.m new file mode 100644 index 00000000..27af1a90 --- /dev/null +++ b/cresis-toolbox/tracker/run_block_data_rds.m @@ -0,0 +1,71 @@ + +% Script for running block_data +% +% Authors: fill in... +% +% See also: run_FUNCTION.m, run_unblock.m, + +%% User Setup +% ===================================================================== +param_override = []; + +params = read_param_xls(ct_filename_param('rds_param_2012_Greenland_P3.xls')); +% Syntax for running a specific segment and frame by overriding parameter spreadsheet values +%params = read_param_xls(ct_filename_param('rds_param_2009_Antarctica_TO.xls'),'20091228_01'); +params = ct_set_params(params,'cmd.generic',0); +params = ct_set_params(params,'cmd.generic',1,'day_seg','20120330_03'); +params = ct_set_params(params,'cmd.frms',[]); + +% Set override parameters using ct_set_params, by setting param_override, or by setting FUNCTION_params variable which eventually is copied to param_override.FUNCTION). + + +param_override.block_data = [] ; +param_override.block_data.block_size = 256; % Set the value for each block +param_override.block_data.block_overlap = 0.5; % Set the % of overlap between each block +param_override.block_data.top_gap = 100; % Number of rows of data before first layer + +param_override.block_data.bottom_pad = 100; % Number of rows of data used to pad after the last layer +param_override.block_data.filter_len = 1; + + +param_override.block_data.values.user_max = []; % Set to empty [] to use max finite value from data or set predetermined max values +param_override.block_data.values.user_min = []; % Set min valid finite value + +param_override.block_data.echo_path = 'CSARP_post/standard'; % Echogram source e.g rds uses 'CSARP\standard' => ct_filename_out(param,'CSARP\standard') + +param_override.block_data.out_fn ='macgregor_troubleshoot3'; % Specify desired output path i.e fn passed into ct_filename_tmp +%param_override.sched.type = 'no scheduler'; % Example to override default cluster settings + +% Paramaters of "layer_params" argument of OpsLoadLayers(param,"layer_params" +param_override.block_data.layers_name = 'layers'; +param_override.block_data.layers_source = 'layerData_ver1'; +param_override.block_data.layerdata_source = 'layerData_ver1'; + + +% param_override.block_data.layers_prefix = 'layers'; + +param_override.block_data.debug_plot = 0 ; % Debug plots (True or False) + + +%% Automated Section +% ===================================================================== + +% Input checking +global gRadar; +if exist('param_override','var') + param_override = merge_structs(gRadar,param_override); +else + param_override = gRadar; +end + + +% Process each of the segments +for param_idx = 1:length(params) + param = params(param_idx); + if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic + continue; + end + block_data(param,param_override); +end + +% Post process code (only include if necessary) \ No newline at end of file diff --git a/cresis-toolbox/tracker/run_cluster_lsm_tuning.m b/cresis-toolbox/tracker/run_cluster_lsm_tuning.m new file mode 100644 index 00000000..c3f24d60 --- /dev/null +++ b/cresis-toolbox/tracker/run_cluster_lsm_tuning.m @@ -0,0 +1,153 @@ +%% Script run_cluster_LSM_tuning.m +% +% Runs the lsm 2D tracking algorithm +% on the desired dataset +% +% See also: LSM_tuning.m + +dbstack_info = dbstack; +fprintf('=====================================================================\n'); +fprintf('%s (%s)\n', dbstack_info(1).name, datestr(now,'HH:MM:SS')); +fprintf('=====================================================================\n'); + +%% General User Settings +% Tracking algorithms: lsm + +% params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls')); +% +% params = ct_set_params(params,'cmd.generic',0); +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20140313_08'); +% params = ct_set_params(params,'cmd.frms',[1]); + +% 23 segs +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20140313_07|20140313_08|20140313_10|20140325_07|20140331_02|20140401_04|20140405_01|20140409_02|20140410_01|20140412_01|20140412_02|20140412_04|20140415_04|20140415_05|20140416_01|20140421_01|20140501_01|20140506_01|20140507_01|20140520_04|20140520_05|20140521_01|20140508_01'); + +% %params = ct_set_params(params,'cmd.frms',[1]); +% for param_idx = 1:length(params) +% param = params(param_idx); +% if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) ... +% || ischar(param.cmd.generic) || ~param.cmd.generic +% continue; +% end +% end +% for param_idx = 1:length(params) +% param = params(param_idx); +% if isfield(param.cmd,'generic') && ~iscell(param.cmd.generic) && ~ischar(param.cmd.generic) && param.cmd.generic +% break; +% end +% end +frm=1; +temp = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140516_01_20200416_002026_t032_lsm.mat'); +param= temp.param; +layer_params = []; +idx = 0; +idx = idx + 1; +layer_params(idx).name = 'surface'; +layer_params(idx).source = 'layerdata'; +layer_params(idx).layerdata_source = 'layer'; +idx = idx+1; +layer_params(idx).name = 'bottom'; +layer_params(idx).source = 'layerdata'; +layer_params(idx).layerdata_source = 'layer'; +layers = opsLoadLayers(param,layer_params); + +for track_idx = 1:length(param.layer_tracker.track) + for frm = param.cmd.frms + data_fn_dir = ct_filename_out(param, param.layer_tracker.echogram_source, ''); + data_fn_name = sprintf('Data_%s_%03d.mat',param.day_seg,frm); + data_fn = fullfile(data_fn_dir, data_fn_name); + data = load(data_fn); + + switch param.layer_tracker.track{track_idx}.method + case 'lsm' + for idx = 1:length(param.layer_tracker.track{track_idx}.lsm.storeIter) + layer_params_surf(idx).name = sprintf('%s_%s_surface_%03d',param.layer_tracker.track{track_idx}.name,param.layer_tracker.track{track_idx}.method,idx); + layer_params_surf(idx).source = param.layer_tracker.layer_params.source; + layer_params_surf(idx).layerdata_source = param.layer_tracker.layer_params.layerdata_source; + end + + for idx = 1:length(param.layer_tracker.track{track_idx}.lsm.storeIter) + layer_params_bot(idx).name = sprintf('%s_%s_bottom_%03d',param.layer_tracker.track{track_idx}.name,param.layer_tracker.track{track_idx}.method,idx); + layer_params_bot(idx).source = param.layer_tracker.layer_params.source; + layer_params_bot(idx).layerdata_source = param.layer_tracker.layer_params.layerdata_source; + end + + case 'viterbi' + idx = 1; + layer_params_bot(idx).name = sprintf('%s_%s_bottom',param.layer_tracker.track{track_idx}.name,param.layer_tracker.track{track_idx}.method); + layer_params_bot(idx).source = param.layer_tracker.layer_params.source; + layer_params_bot(idx).layerdata_source = param.layer_tracker.layer_params.layerdata_source; + + case {'mcmc','stereo'} + idx = 1; + layer_params_surf(idx).name = sprintf('%s_%s_surface',param.layer_tracker.track{track_idx}.name,param.layer_tracker.track{track_idx}.method); + layer_params_surf(idx).source = param.layer_tracker.layer_params.source; + layer_params_surf(idx).layerdata_source = param.layer_tracker.layer_params.layerdata_source; + idx = idx + 1; + layer_params_bot(idx).name = sprintf('%s_%s_bottom',param.layer_tracker.track{track_idx}.name,param.layer_tracker.track{track_idx}.method); + layer_params_bot(idx).source = param.layer_tracker.layer_params.source; + layer_params_bot(idx).layerdata_source = param.layer_tracker.layer_params.layerdata_source; + + otherwise + idx = 1; + layer_params_surf(idx).name = sprintf('%s_%s_surface',param.layer_tracker.track{track_idx}.name,param.layer_tracker.track{track_idx}.method); + layer_params_surf(idx).source = param.layer_tracker.layer_params.source; + layer_params_surf(idx).layerdata_source = param.layer_tracker.layer_params.layerdata_source; + end + + layers_surf = opsLoadLayers(param,layer_params_surf); + layers_bot = opsLoadLayers(param,layer_params_bot); + + + surf = interp_finite(interp1(layers(1).gps_time,layers(1).twtt,data.GPS_time)); + surf_bins = round(interp1(data.Time,1:length(data.Time),surf)); + bot = interp_finite(interp1(layers(2).gps_time,layers(2).twtt,data.GPS_time)); + bot_bins = round(interp1(data.Time,1:length(data.Time),bot)); + dt = data.Time(2) - data.Time(1); + for i = 1:length(layers_surf) + num_isnan= 0; + num_points = 0; + surf = interp_finite(interp1(layers_surf(i).gps_time,layers_surf(i).twtt,data.GPS_time)); + surf_bins_itr = round(interp1(data.Time,1:length(data.Time),surf)); + num_gt_isfinite = length(find(isfinite(surf))); + for temp = 1:length(surf) + if(isfinite(surf(temp)) && isnan(surf_bins_itr(temp))) + num_isnan = num_isnan + 1; + end + if (abs(surf(temp)-surf_bins_itr(temp)) < 5*dt) + num_points = num_points+1; + end + end + res_s.(sprintf('%s_%s',param.layer_tracker.track{track_idx}.method,param.layer_tracker.track{track_idx}.name)).res(frm,i) = nanmean(abs(surf_bins - surf_bins_itr)); + res_s.(sprintf('%s_%s',param.layer_tracker.track{track_idx}.method,param.layer_tracker.track{track_idx}.name)).num_gt_isfinite = num_gt_isfinite; + res_s.(sprintf('%s_%s',param.layer_tracker.track{track_idx}.method,param.layer_tracker.track{track_idx}.name)).num_isnan = num_isnan; + res_s.(sprintf('%s_%s',param.layer_tracker.track{track_idx}.method,param.layer_tracker.track{track_idx}.name)).num_points = num_points; + end + + for i = 1:length(layers_bot) + num_isnan= 0; + num_points = 0; + bot = interp_finite(interp1(layers_bot(i).gps_time,layers_bot(i).twtt,data.GPS_time)); + bot_bins_itr = round(interp1(data.Time,1:length(data.Time),bot)); + num_gt_isfinite = length(find(isfinite(bot))); + for temp = 1:length(bot) + if(isfinite(bot(temp)) && isnan(bot_bins_itr(temp))) + num_isnan = num_isnan + 1; + end + if (abs(bot(temp)-bot_bins_itr(temp)) < 5*dt) + num_points = num_points+1; + end + end + res_b.(sprintf('%s_%s',param.layer_tracker.track{track_idx}.method,param.layer_tracker.track{track_idx}.name)).res(frm,i) = nanmean(abs(bot_bins - bot_bins_itr)); + res_b.(sprintf('%s_%s',param.layer_tracker.track{track_idx}.method,param.layer_tracker.track{track_idx}.name)).num_gt_isfinite = num_gt_isfinite; + res_b.(sprintf('%s_%s',param.layer_tracker.track{track_idx}.method,param.layer_tracker.track{track_idx}.name)).num_isnan = num_isnan; + res_b.(sprintf('%s_%s',param.layer_tracker.track{track_idx}.method,param.layer_tracker.track{track_idx}.name)).num_points = num_points; + end + end +end + + + +fprintf('hi'); + + diff --git a/cresis-toolbox/tracker/run_layer_tracker.m b/cresis-toolbox/tracker/run_layer_tracker.m new file mode 100644 index 00000000..bb9791d1 --- /dev/null +++ b/cresis-toolbox/tracker/run_layer_tracker.m @@ -0,0 +1,458 @@ +% script run_layer_tracker +% +% Runs layer_tracker.m +% +% Authors: Anjali Pare, John Paden +% +% See also: layer_tracker.m, layer_tracker_combine_task.m, +% layer_tracker_task.m, layer_tracker_input_check.m, +% layer_tracker_profile.m, run_layer_tracker.m, run_layer_tracker_tune.m + +%% User Settings +% ---------------------------------------------------------------------- +param_override = []; + +% params = read_param_xls(ct_filename_param('accum_param_2018_Antarctica_TObas.xls')); +% params = read_param_xls(ct_filename_param('rds_param_2011_Greenland_P3.xls')); +params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls')); +% params = read_param_xls(ct_filename_param('rds_param_2018_Greenland_P3.xls')); +% params = read_param_xls(ct_filename_param('snow_param_2012_Greenland_P3.xls')); + +if 1 + % Example to run a specific segment or frame + params = ct_set_params(params,'cmd.generic',0); + params = ct_set_params(params,'cmd.generic',1,'day_seg','20140506_01'); + params = ct_set_params(params,'cmd.frms',[21]); +else + % Example to run all segments + params = ct_set_params(params,'cmd.generic',1); + params = ct_set_params(params,'cmd.generic',0,'cmd.notes','do not process'); +else + cmd_method = 'generic'; + rds_settings; +end + +% param_override.layer_tracker.debug_plots = {'tracked_images'}; % Uncomment to save jpg output files +param_override.layer_tracker.debug_plots = {'tracked_images','visible'}; % Uncomment to save jpg output files and debug +% param_override.layer_tracker.debug_plots = {'visible'}; % Uncomment for debugging + +param_override.layer_tracker.echogram_img = 0; % To choose an image besides the base (0) image +% echogram_source: location of echogram data used for tracking +param_override.layer_tracker.echogram_source = 'qlook'; +% param_override.layer_tracker.echogram_source = 'standard'; +% param_override.layer_tracker.echogram_source = 'CSARP_post/qlook'; +% param_override.layer_tracker.echogram_source = 'CSARP_post/mvdr'; +% param_override.layer_tracker.echogram_source = 'CSARP_post/standard'; + +param_override.layer_tracker.frm_types = {0,0,-1,-1,-1}; % Uncomment to only process "good" frames + +% layer_params: layerparams structure array of where to store the output using +% opsCopyLayers.m +param_override.layer_tracker.layer_params = []; layer_idx = 0; +% Uncomment to enable CSARP_layer layerdata storage +layer_idx = layer_idx + 1; +param_override.layer_tracker.layer_params(layer_idx).layerdata_source = 'layer'; +% Uncomment to enable CSARP_post/CSARP_layer layerdata storage +% layer_idx = layer_idx + 1; +% param_override.layer_tracker.layer_params(layer_idx).layerdata_source = 'CSARP_post/layer'; +% Uncomment to enable OPS storage +% layer_idx = layer_idx + 1; +% param_override.layer_tracker.layer_params(layer_idx).source = 'ops'; + +% block_size_frms: Number of frames to be loaded at a time (inf for all) +param_override.layer_tracker.block_size_frms = 1; + +% track_per_task: Number of tracks per task +param_override.layer_tracker.track_per_task = inf; + +% param_override.layer_tracker.surf_layer = struct('name','surface','source','layerdata','layerdata_source','layer'); + +%% param.layer_tracker.track options +track = []; + +% ========================================================================= +% NOTE ON USAGE: +% Enable one set of tracking parameters below +% ========================================================================= +track.en = true; +switch ct_output_dir(params(1).radar_name) + case 'rds' + %% RDS + + %% RDS: Surface tracking + if 1 + track.profile = 'rds'; + + track.layer_names = {'surface'}; + + % Override default filter settings for low AGL + if 0 + track.min_bin = 0.75e-6; + end + + % Trim bad data at start and end + if 0 + track.prefilter_trim = [0 3.5e-6]; + end + + % Override default filter settings for broad bandwidth + if 0 + track.max_rng = [0 10]; + track.max_rng_units = 'bins'; + end + + % Override default filter settings for rapidly changing elevation + if 0 + track.filter = [5 1]; + end + + % Override default filter settings + if 0 + track.filter = [3 31]; + track.filter_trim = [3 31]; + track.threshold = 7; + track.max_rng = [0 1]; + track.max_rng_units = 'bins'; + end + + % Use sidelobe rejection + if 0 + % run_get_echogram_stats output + sidelobe = load('/N/dcwan/projects/cresis/output/ct_tmp/echogram_stats/rds/2018_Greenland_P3/stats_20180421_01.mat','sidelobe_rows','sidelobe_dB','sidelobe_vals'); + track.sidelobe_rows = [sidelobe.sidelobe_rows(75:98)]; + track.sidelobe_dB = -(sidelobe.sidelobe_dB(75:98,1)-max(sidelobe.sidelobe_dB(:,1))+21); + track.sidelobe_dB(track.sidelobe_dB<9) = 9; + track.threshold_rel_max = -max(track.sidelobe_dB); + track.data_noise_en = true; + end + + % Use feedthrough rejection + if 0 + % run_get_echogram_stats output + feedthru = load('/N/dcwan/projects/cresis/output/ct_tmp/echogram_stats/rds/2018_Greenland_P3/stats_20180421_01.mat'); + track.feedthru.time = feedthru.dt*feedthru.bins; + track.feedthru.power_dB = feedthru.min_means+20; + bin_mask = track.feedthru.time<2e-6; + track.feedthru.time = track.feedthru.time(bin_mask); + track.feedthru.power_dB = track.feedthru.power_dB(bin_mask); + track.feedthru.power_dB(end) = -inf; + track.min_bin = 0.5e-6; + track.data_noise_en = true; + end + + % Use DEM and LIDAR for init + if 1 + track.init.method = 'dem'; + track.init.dem_offset = 0; + track.init.dem_layer.name = 'surface'; + track.init.dem_layer.source = 'lidar'; + track.init.dem_layer.lidar_source = 'atm'; + track.init.max_diff = 1e-6; + track.init.max_diff_method = 'merge_vectors'; + end + + % Use snake method for init + if 0 + track.init.method = 'snake'; + track.init.snake_rng = [-0.5e-6 0.5e-6]; + track.init.max_diff = 0.5e-6; + end + + % Override default method + if 0 + track.method = 'snake'; + track.snake_rng = [-0.15e-6 0.15e-6]; + end + end + + %% RDS: Surface tracking (DEM) + % Use DEM and LIDAR with no automated tracking (use this when the + % echogram data are bad and no tracking is possible) + if 0 + track.profile = 'RDS'; + track.layer_names = {'surface_dem'}; + + % Override default init method + track.init.method = 'dem'; + track.init.dem_offset = 0; + track.init.dem_layer = []; + track.init.max_diff = 0; + track.init.max_diff_method = 'merge_vectors'; + + track.method = ''; % Just use DEM surface + track.max_rng = [0 0]; + track.medfilt = []; + end + + %% RDS: Viterbi bottom + if 0 + track.method = 'viterbi'; + track.layer_names = {'bottom_viterbi'}; + + track.min_bin = struct('name','tomo_top'); + track.max_bin = struct('name','tomo_bottom'); + + track.crossover.en = true; + track.crossover.season_names_bad = {'2003_Greenland_P3', '2005_Greenland_P3', param.season_name}; % Bad seasons to not include + % track.crossover.gps_time_good_eval = @(x) true; % All cross overs are good + track.crossover.gps_time_good_eval = @(x) x > datenum_to_epoch(datenum('2010/01/01')); % Cross overs before this date are good + + if 0 + track.ice_mask.en = false; + elseif 1 + % Greenland + track.ice_mask.en = true; + track.ice_mask.type = 'geotiff'; + track.ice_mask.fn = ct_filename_gis([], fullfile('greenland','IceMask','GimpIceMask_90m_v1.1.tif')); + elseif 0 + % Canada + track.ice_mask.en = true; + track.ice_mask.type = 'bin'; + track.ice_mask.fn = ct_filename_gis([], fullfile('canada','ice_mask','03_rgi50_ArcticCanadaNorth','03_rgi50_ArcticCanadaNorth.bin')); + track.ice_mask.mat_fn = ct_filename_gis([], fullfile('canada','ice_mask','03_rgi50_ArcticCanadaNorth','03_rgi50_ArcticCanadaNorth.mat')); + elseif 0 + % Antarctica + track.ice_mask.en = true; + track.ice_mask.type = 'geotiff2'; + track.ice_mask.fn = ct_filename_gis([], fullfile('antarctica/DEM/BEDMAP2/original_data/bedmap2_tiff/bedmap2_icemask_grounded_and_shelves.tif')); + track.ice_mask.fn2 = ct_filename_gis([], fullfile('antarctica/DEM/BEDMAP2/original_data/bedmap2_tiff/bedmap2_rockmask.tif')); + end + track.init.dem_layer = struct('name','surface'); + + track.viterbi.transition_weight = 1; % Larger --> smoother + track.viterbi.gt_cutoff = 50; + + track.surf_suppress.eval_cmd = '6*log2(time/surf) - 15*max(0,1 - abs(time-surf)/2e-6);'; + track.mult_suppress.en = true; + track.init.max_diff = inf; + track.init.method = 'nan'; + track.detrend = []; + track.filter_trim = [0 120]; + track.norm.scale = [-40 90]; + track.xcorr = echo_xcorr_profile('short_unitstep'); + end + + %% RDS: MCMC bottom + if 0 + track.method = 'mcmc'; + track.layer_names = {'surface','bottom'}; + track.mcmc.alg = 'MCMC'; + track.init.max_diff = inf; + end + + %% RDS: LSM bottom + if 0 + track.method = 'lsm'; + track.layer_names = {'surface','bottom'}; + track.lsm.y = 220; + track.lsm.dy = 10; + track.lsm.storeIter = 400; + track.init.max_diff = inf; + track.detrend = []; + track.norm.scale = [-40 90]; + + end + + %% RDS: Stereo bottom + if 0 + track.method = 'stereo'; + track.layer_names = {'surface','bottom'}; + track.stereo.surfaceload = true; + track.stereo.crossoverload = true; + track.stereo.top_smooth = 1000; + track.stereo.bottom_smooth = 1000; + track.stereo.top_peak = 0.5; + track.stereo.bottom_peak = 0.5; + track.stereo.repulsion = 10; + track.stereo.alg = 'HMM'; + track.init.max_diff = inf; + end + + case 'accum' + %% ACCUM + + %% ACCUM: Surface tracking + if 1 + track.profile = 'accum'; + + track.layer_names = {'surface'}; + + % track.threshold = 5; + % track.threshold_rel_max = -9; + % track.init.method = 'nan'; % May be necessary if bottom is brighter than surface + + % Override default init method + if 0 + track.init.method = 'dem'; + track.init.dem_offset = 0; + track.init.dem_layer.name = 'surface'; + track.init.dem_layer.source = 'lidar'; + track.init.dem_layer.lidar_source = 'atm'; + track.init.max_diff = 0.3e-6; + end + + % Override default method + if 0 + track.method = 'snake'; + track.snake_rng = [-0.15e-6 0.15e-6]; + end + end + + %% ACCUM: Surface tracking (DEM) + % Use DEM and LIDAR with no automated tracking (use this when the + % echogram data are bad and no tracking is possible) + if 0 + track.profile = 'ACCUM'; + track.layer_names = {'surface_dem'}; + + % Override default init method + track.init.method = 'dem'; + track.init.dem_offset = 0; + track.init.dem_layer = []; + track.init.max_diff = 0; + track.init.max_diff_method = 'merge_vectors'; + + track.method = ''; % Just use DEM surface + track.max_rng = [0 0]; + track.medfilt = []; + end + + %% ACCUM: Viterbi bottom + if 1 + track.method = 'viterbi'; + track.layer_names = {'bottom'}; + + track.min_bin = struct('name','surface','eval',struct('cmd','s=s+1e-6;')); + % track.min_bin = struct('name','tomo_top'); + % track.max_bin = struct('name','tomo_bottom'); + + track.crossover.en = false; + track.crossover.season_names_bad = {'2003_Greenland_P3', '2005_Greenland_P3'}; % Bad seasons to not include + % track.crossover.gps_time_good_eval = @(x) true; % All cross overs are good + track.crossover.gps_time_good_eval = @(x) x > datenum_to_epoch(datenum('2010/01/01')); % Cross overs before this date are good + + if 1 + track.ice_mask.en = false; + elseif 0 + % Greenland + track.ice_mask.en = true; + track.ice_mask.type = 'geotiff'; + track.ice_mask.fn = ct_filename_gis([], fullfile('greenland','IceMask','GimpIceMask_90m_v1.1.tif')); + elseif 0 + % Canada + track.ice_mask.en = true; + track.ice_mask.type = 'bin'; + track.ice_mask.fn = ct_filename_gis([], fullfile('canada','ice_mask','03_rgi50_ArcticCanadaNorth','03_rgi50_ArcticCanadaNorth.bin')); + track.ice_mask.mat_fn = ct_filename_gis([], fullfile('canada','ice_mask','03_rgi50_ArcticCanadaNorth','03_rgi50_ArcticCanadaNorth.mat')); + elseif 0 + % Antarctica + track.ice_mask.en = true; + track.ice_mask.type = 'geotiff2'; + track.ice_mask.fn = ct_filename_gis([], fullfile('antarctica/DEM/BEDMAP2/original_data/bedmap2_tiff/bedmap2_icemask_grounded_and_shelves.tif')); + track.ice_mask.fn2 = ct_filename_gis([], fullfile('antarctica/DEM/BEDMAP2/original_data/bedmap2_tiff/bedmap2_rockmask.tif')); + end + track.init.dem_layer = struct('name','surface'); + + track.viterbi.transition_weight = 0.01; % Larger --> smoother + track.viterbi.gt_cutoff = 50; + + track.mult_suppress.en = true; + track.init.max_diff = inf; + track.init.method = 'nan'; + track.detrend = []; + track.filter_trim = [0 120]; + track.norm.scale = [-40 90]; + track.xcorr = echo_xcorr_profile('xlong_unitstep'); + end + + case {'snow','kuband','kaband'} + %% SNOW (also kaband, kuband) + + %% SNOW: Surface tracking + if 1 + track.profile = 'snow'; + + track.layer_names = {'surface'}; + + % Use sidelobe rejection + if 0 + % run_get_echogram_stats output + sidelobe = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/echogram_stats/snow/2011_Greenland_P3/stats_20110329_01.mat','sidelobe_rows','sidelobe_dB','sidelobe_vals'); + track.sidelobe_rows = [sidelobe.sidelobe_rows(1:194)]; + track.sidelobe_dB = [-sidelobe.sidelobe_dB(1:194,end)]-sidelobe.sidelobe_vals(end)+4.5; + track.threshold_rel_max = -max(track.sidelobe_dB); + end + + % Override default init method + if 0 + track.init.method = 'dem'; + track.init.dem_offset = 0; + track.init.dem_layer.name = 'surface'; + track.init.dem_layer.source = 'lidar'; + track.init.dem_layer.lidar_source = 'atm'; + track.init.max_diff = 0.3e-6; + track.init.max_diff_method = 'merge_vectors'; + elseif 0 + track.init.method = 'snake'; + track.init.snake_rng = [-15e-9 15e-9]; + track.init.max_diff = 0.3e-6; + track.method = 'snake'; + end + end + + %% SNOW: Surface tracking (DEM) + % Use DEM and LIDAR with no automated tracking (use this when the + % echogram data are bad and no tracking is possible) + if 0 + track.profile = 'snow'; + track.layer_names = {'surface_dem'}; + + % Override default init method + track.init.method = 'dem'; + track.init.dem_offset = 0; + track.init.dem_layer = []; + track.init.max_diff = 0; + track.init.max_diff_method = 'merge_vectors'; + + track.method = ''; % Just use DEM surface + track.max_rng = [0 0]; + track.medfilt = []; + end + +end + +param_override.layer_tracker.track = {track}; + +% dbstop if error; +% param_override.cluster.type = 'torque'; +% param_override.cluster.type = 'matlab'; +param_override.cluster.type = 'debug'; +% param_override.cluster.type = 'slurm'; +% param_override.cluster.rerun_only = true; +% param_override.cluster.desired_time_per_job = 240*60; +% param_override.cluster.cpu_time_mult = 2; +% param_override.cluster.mem_mult = 2; + +%% Automated Section +% ---------------------------------------------------------------------- + +% Input checking +global gRadar; +if exist('param_override','var') + param_override = merge_structs(gRadar,param_override); +else + param_override = gRadar; +end + +ctrl_chain = {}; +% Process each of the segments +for param_idx = 1:length(params) + param = params(param_idx); + if isfield(param.cmd,'generic') && ~iscell(param.cmd.generic) && ~ischar(param.cmd.generic) && param.cmd.generic + ctrl_chain{end+1} = layer_tracker(param,param_override); + end +end + +cluster_print_chain(ctrl_chain); + +[chain_fn,chain_id] = cluster_save_chain(ctrl_chain); diff --git a/cresis-toolbox/tracker/run_layer_tracker_2D.m b/cresis-toolbox/tracker/run_layer_tracker_2D.m deleted file mode 100644 index df1dde17..00000000 --- a/cresis-toolbox/tracker/run_layer_tracker_2D.m +++ /dev/null @@ -1,323 +0,0 @@ -% script run_layer_tracker_2D -% -% Runs layer_tracker_2D.m -% -% Need to set track_override.track_data to true if LSM tuning results are -% needed. Once the results are saved, it is possible to compare them, find -% the lowest combination (y, dy and iterations) and then run the LSM -% tracker with the tuned results. Set track_override.track_data to false in -% that case. - -%% User Settings -% ---------------------------------------------------------------------- -param_override = []; - -% params = read_param_xls(ct_filename_param('rds_param_2018_Greenland_P3.xls')); -params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls')); - -params = ct_set_params(params,'cmd.generic',0); -params = ct_set_params(params,'cmd.generic',1,'day_seg','20140313_08'); -params = ct_set_params(params,'cmd.frms',[1:3]); % Specify specific frames (or leave empty/undefined to do all frames) - -param_override.layer_tracker.debug_plots = {'debug'}; - -param_override.layer_tracker.echogram_img = 0; % To choose an image besides the base (0) image -% echogram_source: location of echogram data used for tracking -% param_override.layer_tracker.echogram_source = 'CSARP_post/qlook'; -param_override.layer_tracker.echogram_source = 'CSARP_post/standard'; - -% layer_params: structure of layer references of where to store the output -param_override.layer_tracker.layer_params = []; idx = 0; -% idx = idx + 1; -% param_override.layer_tracker.layer_params(idx).name = 'surface'; -% param_override.layer_tracker.layer_params(idx).source = 'echogram'; -% param_override.layer_tracker.layer_params(idx).echogram_source = 'deconv'; -idx = idx + 1; -param_override.layer_tracker.layer_params(idx).name = 'surface'; -param_override.layer_tracker.layer_params(idx).source = 'layerdata'; -param_override.layer_tracker.layer_params(idx).layerdata_source = 'layerData'; -param_override.layer_tracker.layer_params(idx).echogram_source = 'CSARP_post/standard'; -% idx = idx + 1; -% param_override.layer_tracker.layer_params(idx).name = 'surface'; -% param_override.layer_tracker.layer_params(idx).source = 'ops'; - -param_override.layer_tracker.N = 2; % no of frames to be loaded at a time -% Enter name for saving the data -param_override.layer_tracker.name = 'test_vit'; -param_override.layer_tracker.save_ops_copy_layers = false; - -track_override = []; -track_override.name = 'mvdr'; -track_override.debug = true; -track_override.ops_write = false; -track_override.save_layerData = true; -track_override.save_img = false; -track_override.save_add_f = false; -track_override.track_data = false; % for LSM tuning - - -%% Image saving options -% Only effective if options.save_img == true -track_override.save_img_format = '-djpeg'; -track_override.save_img_path = ''; - -%% OPS/layerData writing options -% Only effective if options.ops_write == true - -% Usually 'layerdata' or 'ops' -track_override.layer_dest_source = 'layerdata'; - -% If using 'layerdata': -track_override.layer_dest_layerdata_source = 'layerData_test'; % layerData file will be saved in this location -track_override.layer_dest_echogram_source = 'CSARP_post/standard'; % Only required if layerData files do not exist and need to be created -%% Additional file writing options -% Only effective if options.save_add_f == true -track_override.save_add_f_path = ''; - -%% Ice mask options -% if 1 % If using GeoTIFF file for ice mask -% track_override.binary_icemask = false; -% track_override.icemask_fn = 'greenland/IceMask/GimpIceMask_90m_v1.1.tif'; -% track_override.icemask_fn = ct_filename_gis([], track_override.icemask_fn); -% -% % Useful for Antarctica seasons: -% % track_override.icemask_fn = 'antarctica/DEM/BEDMAP2/original_data/bedmap2_tiff/bedmap2_icemask_grounded_and_shelves.tif'; -% % track_override.icemask2_fn = 'antarctica/DEM/BEDMAP2/original_data/bedmap2_tiff/bedmap2_rockmask.tif'; -% % if isfield(track_override, 'icemask2_fn') && ~isempty(track_override.icemask2_fn) -% % track_override.icemask2_fn = ct_filename_gis([],track_override.icemask2_fn); -% % end -% else -% track_override.binary_icemask = true; -% track_override.icemask_fn = '/cresis/snfs1/dataproducts/GIS_data/canada/ice_mask/03_rgi50_ArcticCanadaNorth/03_rgi50_ArcticCanadaNorth.bin'; -% [track_override.ice_mask_fn_dir,track_override.ice_mask_fn_name] = fileparts(track_override.icemask_fn); -% track_override.ice_mask_mat_fn = fullfile(track_override.ice_mask_fn_dir,[track_override.ice_mask_fn_name '.mat']); -% end - -if 1 % If using GeoTIFF file for ice mask - if strcmpi(params(1).post.ops.location,'arctic') - if 1 - % Greenland - track_override.binary_icemask = false; - track_override.icemask_fn = 'greenland/IceMask/GimpIceMask_90m_v1.1.tif'; - track_override.icemask_fn = ct_filename_gis([], track_override.icemask_fn); - else - % Canada - track_override.binary_icemask = true; - track_override.icemask_fn = '/cresis/snfs1/dataproducts/GIS_data/canada/ice_mask/03_rgi50_ArcticCanadaNorth/03_rgi50_ArcticCanadaNorth.bin'; - [track_override.ice_mask_fn_dir,track_override.ice_mask_fn_name] = fileparts(track_override.icemask_fn); - track_override.ice_mask_mat_fn = fullfile(track_override.ice_mask_fn_dir,[track_override.ice_mask_fn_name '.mat']); - end - else - % Useful for Antarctica seasons: - track_override.binary_icemask = false; - track_override.icemask_fn = ct_filename_gis([], 'greenland/IceMask/GimpIceMask_90m_v1.1.tif'); - end - -end - -%% Enable one set of parameters -track_override.en = true; -switch ct_output_dir(params(1).radar_name) - case 'rds' - % RDS - track_override.profile = 'rds_OIB'; - - % Override default filter settings - if 0 - track_override.filter = [3 3]; - track_override.filter_trim = [3 3]; - track_override.threshold = 10; - track_override.max_rng = [0 2]; - end - - % Use sidelobe rejection - if 0 - % run_get_echogram_stats output - sidelobe = load('/N/dcwan/projects/cresis/output/ct_tmp/echogram_stats/rds/2018_Greenland_P3/stats_20180421_01.mat','sidelobe_rows','sidelobe_dB','sidelobe_vals'); - track_override.sidelobe_rows = [sidelobe.sidelobe_rows(75:98)]; - track_override.sidelobe_dB = -(sidelobe.sidelobe_dB(75:98,1)-max(sidelobe.sidelobe_dB(:,1))+21); - track_override.sidelobe_dB(track_override.sidelobe_dB<9) = 9; - track_override.threshold_rel_max = -max(track_override.sidelobe_dB); - track_override.data_noise_en = true; - end - - % Use feedthrough rejection - if 0 - % run_get_echogram_stats output - feedthru = load('/N/dcwan/projects/cresis/output/ct_tmp/echogram_stats/rds/2018_Greenland_P3/stats_20180421_01.mat'); - track_override.feedthru.time = feedthru.dt*feedthru.bins; - track_override.feedthru.power_dB = feedthru.min_means+20; - bin_mask = track_override.feedthru.time<2e-6; - track_override.feedthru.time = track_override.feedthru.time(bin_mask); - track_override.feedthru.power_dB = track_override.feedthru.power_dB(bin_mask); - track_override.feedthru.power_dB(end) = -inf; - track_override.min_bin = 0.5e-6; - track_override.data_noise_en = true; - end - - % Override default init method - if 0 - track_override.init.method = 'dem'; - track_override.init.dem_offset = 0; - track_override.init.dem_layer.name = 'surface'; - track_override.init.dem_layer.source = 'lidar'; - track_override.init.dem_layer.lidar_source = 'atm'; - track_override.init.max_diff = 1e-6; - elseif 0 - track_override.init.method = 'snake'; - track_override.init.snake_rng = [-0.5e-6 0.5e-6]; - track_override.init.max_diff = 0.5e-6; - end - - % Override default method - if 1 - track_override.method = 'snake'; - track_override.snake_rng = [-0.15e-6 0.15e-6]; - end - - %% Viterbi - if 0 - %% Viterbi User Settings - track_override.method = 'viterbi'; - track_override.viterbi.crossoverload = true; - track_override.viterbi.layername = 'viterbi_bot'; %surface or bottom - track_override.viterbi.detrending = true; - track_override.viterbi.top_sup = false; - track_override.viterbi.mult_sup = false; - track_override.viterbi.custom_combine = false; - track_override.viterbi.DIM_matrix = fullfile('+tomo', 'Layer_tracking_2D_parameters_Matrix.mat'); - - track_override.viterbi.bottom_bin = -1; - track_override.viterbi.egt_weight = -1; - track_override.viterbi.mu_size = 31; - track_override.viterbi.mu = log10(exp(-(-(track_override.viterbi.mu_size-1)/2 : (track_override.viterbi.mu_size-1)/2).^4/1)); - track_override.viterbi.mu_thr = -30; - track_override.viterbi.mu(track_override.viterbi.mu < track_override.viterbi.mu_thr) = track_override.viterbi.mu_thr; - track_override.viterbi.mu = track_override.viterbi.mu - mean(track_override.viterbi.mu); - track_override.viterbi.sigma = sum(abs(track_override.viterbi.mu))/10*ones(1,track_override.viterbi.mu_size); - track_override.viterbi.smooth_var = inf; - track_override.viterbi.repulsion = 150000; - track_override.viterbi.smooth_weight = 1; - track_override.viterbi.ice_bin_thr = 10; - end - - %% MCMC - if 0 - %% MCMC User Settings - track_override.method = 'mcmc'; - track_override.mcmc.lyrtop = 'mcmc_top'; %layername, layer_dest.name - track_override.mcmc.lyrbot = 'mcmc_bot'; - track_override.mcmc.alg = 'MCMC'; - end - - %% LSM - if 0 - %% LSM User Settings - track_override.method = 'lsm'; - track_override.lsm.lyrtop = 'lsm_top'; %layername, layer_dest.name - track_override.lsm.lyrbot = 'lsm_bot'; - track_override.lsm.y = 220; % = '' for y = mean(SURF) - track_override.lsm.dy = 10; - track_override.lsm.numOuterIter = 350; - track_override.lsm.maxOuterIter = 50; - end - - %% Stereo - if 0 - %% Stereo User Settings - track_override.method = 'stereo'; - track_override.stereo.lyrtop = 'stereo_top'; %layername, layer_dest.name - track_override.stereo.lyrbot = 'stereo_bot'; - track_override.stereo.surfaceload = true; - track_override.stereo.crossoverload = true; - track_override.stereo.top_smooth = 1000; - track_override.stereo.bottom_smooth = 1000; - track_override.stereo.top_peak = 0.5; - track_override.stereo.bottom_peak = 0.5; - track_override.stereo.repulsion = 10; - track_override.stereo.alg = 'HMM'; - end - - %% Threshold - if 0 - track_override.method = 'threshold'; - end - - %% Fixed - if 0 - track_override.method = 'fixed'; - end - case 'accum' - % ACCUM - track_override.profile = 'ACCUM'; - - % Override default init method - if 0 - track_override.init.method = 'dem'; - track_override.init.dem_offset = 0; - track_override.init.dem_layer.name = 'surface'; - track_override.init.dem_layer.source = 'lidar'; - track_override.init.dem_layer.lidar_source = 'atm'; - track_override.init.max_diff = 0.3e-6; - end - - % Override default method - if 0 - track_override.method = 'snake'; - track_override.snake_rng = [-0.15e-6 0.15e-6]; - end - - case {'snow','kuband','kaband'} - % FMCW - track_override.profile = 'snow_AWI'; - - % Use sidelobe rejection - if 0 - % run_get_echogram_stats output - sidelobe = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/echogram_stats/snow/2011_Greenland_P3/stats_20110329_01.mat','sidelobe_rows','sidelobe_dB','sidelobe_vals'); - track_override.sidelobe_rows = [sidelobe.sidelobe_rows(1:194)]; - track_override.sidelobe_dB = [-sidelobe.sidelobe_dB(1:194,end)]-sidelobe.sidelobe_vals(end)+4.5; - track_override.threshold_rel_max = -max(track_override.sidelobe_dB); - end - - % Override default init method - if 0 - track_override.init.method = 'dem'; - track_override.init.dem_offset = 0; - track_override.init.dem_layer.name = 'surface'; - track_override.init.dem_layer.source = 'lidar'; - track_override.init.dem_layer.lidar_source = 'atm'; - track_override.init.max_diff = 0.3e-6; - elseif 0 - track_override.init.method = 'snake'; - track_override.init.snake_rng = [-15e-9 15e-9]; - track_override.init.max_diff = 0.3e-6; - end - -end -param_override.layer_tracker.track = track_override; - -%% Automated Section -% ---------------------------------------------------------------------- - -% Input checking -global gRadar; -if exist('param_override','var') - param_override = merge_structs(gRadar,param_override); -else - param_override = gRadar; -end - -ctrl_chain = {}; -% Process each of the segments -for param_idx = 1:length(params) - param = params(param_idx); - if isfield(param.cmd,'generic') && ~iscell(param.cmd.generic) && ~ischar(param.cmd.generic) && param.cmd.generic - - ctrl_chain{end+1} = layer_tracker_2D(param,param_override); - end -end - -cluster_print_chain(ctrl_chain); - -[chain_fn,chain_id] = cluster_save_chain(ctrl_chain); diff --git a/cresis-toolbox/tracker/run_layer_tracker_generate.m b/cresis-toolbox/tracker/run_layer_tracker_generate.m new file mode 100644 index 00000000..ce4eaf7c --- /dev/null +++ b/cresis-toolbox/tracker/run_layer_tracker_generate.m @@ -0,0 +1,238 @@ +% script run_layer_tracker_tune +% Runs layer_tracker.m for tuning layer tracking hyperparameters. +% +% Authors: Anjali Pare, John Paden +% +% See also: layer_tracker.m, layer_tracker_combine_task.m, +% layer_tracker_task.m, layer_tracker_profile.m, run_layer_tracker.m, +% run_layer_tracker_tune.m +% +% + +%% User Settings +% ---------------------------------------------------------------------- +param_override = []; +params_all = []; +idx_segment = 1; +params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls'),'20140313_09|20140415_05|20140421_01|20140516_01');%'20140313_09|20140415_05|20140421_01|20140514_01'); +params(idx_segment).cmd.generic = 1; +params(idx_segment).cmd.frms = []; +idx_segment = idx_segment + 1; +params(idx_segment).cmd.generic = 1; +params(idx_segment).cmd.frms = []; +idx_segment = idx_segment + 1; +params(idx_segment).cmd.generic = 1; +params(idx_segment).cmd.frms = []; +idx_segment = idx_segment + 1; +params(idx_segment).cmd.generic = 1; +params(idx_segment).cmd.frms = []; + +% params = ct_set_params(params,'cmd.generic',0); +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20140313_09|20140516_01');%20140415_05|20140421_01|');%'20140415_05|20140421_01|20140514_01|20140313_09'); +% params = ct_set_params(params,'cmd.frms',[]); % Specify specific frames (or leave empty/undefined to do all frames) +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20110331_02'); +% params = ct_set_params(params,'cmd.frms',19); % Specify specific frames (or leave empty/undefined to do all frames) + +param_override.layer_tracker.layer_params = []; +param_override.layer_tracker.layer_params.layerdata_source = 'layer_tune_vit_seg4_C'; +param_override.tmp_name = 'layer_tune_vit_mass_seg4_NoCross'; % Enter the name of the file where images will be saved + + +tracker_method = 'viterbi'; % Choose the tracking method + +%% param.layer_tracker.track options +options = []; +options.set_range = 50; % set acceptable range for error +options.segment_vs_frame.plot = true; % absolute error vs frames (for each segment) +options.segment_vs_frame.fig_format = true; +options.season_vs_gps.plot = true; % absolute error vs gps time (for entire season) +options.season_vs_gps.fig_format = true; +options.season_vs_gps.multiple_figs = true; % choose this option if you separate plots generated for each layer. If set to false, 6 subplots will be plotted in one figure. +options.absolute_error.plot = true; % mean absolute error plots +options.absolute_error.fig_format = true; +options.absolute_error.multiple_plots = true; +options.percentage_correct.plot = true; % plot of percentage correct vs frames (for each segment) +options.percentage_correct.fig_format = true; +options.percentage_correct.multiple_plots = true; +options.hist_generation.plot = true; % histograms of data points vs absolute error +options.hist_generation.fig_format = true; +options.hist_generation.multiple_plots = true; +param_override.options = options; + +seg_idx = 0; +filename = []; + +filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140313_09_20210520_171538_t005_viterbi.mat'); +filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140415_05_20210520_172629_t005_viterbi.mat'); +filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140421_01_20210520_181458_t005_viterbi.mat'); +filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140516_01_20210520_184444_t005_viterbi.mat'); +%filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140313_09_20210311_013103_t056_lsm.mat'); +% filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140415_05_20210311_013232_t056_lsm.mat'); +% filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140421_01_20210311_014435_t056_lsm.mat'); +% filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140516_01_20200909_032808_t056_lsm.mat'); +%filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140313_08_20210408_205951_t006_lsm.mat');%20140313_08_20210407_100556_t005_viterbi.mat'); +%filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140313_09_20210408_210034_t006_lsm.mat');%20140313_09_20210401_230350_t005_viterbi.mat'); + + +% filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140313_09_20210401_230350_t005_viterbi.mat'); +% filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140415_05_20210401_231542_t005_viterbi.mat'); +% filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140421_01_20210402_000547_t005_viterbi.mat'); +% filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140516_01_20210402_003509_t005_viterbi.mat'); +% + +% filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140313_09_20210331_232507_t005_viterbi.mat'); +% filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140415_05_20210331_233855_t005_viterbi.mat'); +% filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140421_01_20210401_004304_t005_viterbi.mat'); +% filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140516_01_20210401_012014_t005_viterbi.mat'); +% + +%param_override.filename = filename; +% param_override.layer_tracker.surf_layer = struct('name','surface','source','layerdata','layerdata_source','layer'); + +% param_override.layer_tracker.crossover_layer = struct('name','bottom','source','ops'); + +% If surface and bottom +% foo{idx} = []; +% gt_layer_params(idx).name = 'surface'; +% idx = idx + 1; +% foo{idx} = []; +% gt_layer_params(idx).name = 'bottom'; % specify layer names + +% dbstop if error; +param_override.cluster.type = 'torque'; +% param_override.cluster.type = 'matlab'; +% param_override.cluster.type = 'debug'; +% param_override.cluster.type = 'slurm'; +% param_override.cluster.rerun_only = true; +% param_override.cluster.desired_time_per_job = 240*60; +param_override.cluster.cpu_time_mult = 30; +param_override.cluster.mem_mult = 15; + +idx = 1; + +% Uncomment below for surface and bottom +% gt_layer_params = []; +% gt_layer_params(idx).name = 'surface'; +% idx = idx + 1; +gt_layer_params(idx).name = 'bottom'; % specify layer names +param_override.gt_layer_params = gt_layer_params; +layer_idx = length(gt_layer_params); + + +%% Automated Section +% ---------------------------------------------------------------------- + +% Input checking +global gRadar; +if exist('param_override','var') + param_override = merge_structs(gRadar,param_override); +else + param_override = gRadar; +end + +ctrl_chain = {}; +img_fn_dir = {}; +all_params = {}; +% Process each of the segments +for param_idx = 1:length(params) + param = params(param_idx); + if isfield(param.cmd,'generic') && ~iscell(param.cmd.generic) && ~ischar(param.cmd.generic) && param.cmd.generic + param = merge_structs(param, param_override); + frames = frames_load(param); + param.cmd.frms = frames_param_cmd_frms(param,frames); + seg_idx = seg_idx + 1; + param.filename = filename{seg_idx}; + all_params{end+1} = param.day_seg; + fprintf('=====================================================================\n'); + fprintf('%s: %s (%s)\n', mfilename, param.day_seg, datestr(now)); + fprintf('=====================================================================\n'); + + %% Set up Cluster + % =================================================================== + + ctrl = cluster_new_batch(param); + cluster_compile({'layer_tracker_generate_task','layer_tracker_generate_combine_task'},ctrl.cluster.hidden_depend_funs,ctrl.cluster.force_compile,ctrl); + + %% layer_tracker + % ========================================================================= + % ========================================================================= + + % Cluster setup + % ------------------------------------------------------------------------- + dparam = []; + sparam.argsin{1} = param; + sparam.task_function = 'layer_tracker_generate_task'; + sparam.argsin{1}.num_args_out = 1; + sparam.num_args_out = 1; + sparam.cpu_time = 100; + sparam.mem = 1000e6; + sparam.notes = ''; + sparam.file_success = {}; + + % end + + %% layer_tracker: Loop to create tasks + % ------------------------------------------------------------------------- + img_dir = '/cresis/snfs1/scratch/anjali/cluster_tuning/'; + img_dir_combine = sprintf('%s%s/',img_dir,param.tmp_name); + img_dir = sprintf('%s%s',img_dir_combine,param.day_seg); + if ~exist(img_dir,'dir') + mkdir(img_dir); + end + img_fn_dir{end+1} = img_dir; + dparam.argsin{1}.fname = img_dir; + dparam.argsin{1}.img_dir_combine = img_dir_combine; + dparam.argsin{1}.param_idx = param_idx; + dparam.argsin{1}.season_idx = 1; + dparam.num_args_out = 1; + sparam.file_success{end+1} = img_dir; + mem_combine = 0; + cputime_combine = 0; + frm_idx = 1; + idx = 1; + + + % Create task + % --------------------------------------------------------------------- + ctrl = cluster_new_task(ctrl,sparam,dparam,'dparam_save',0); + ctrl = cluster_save_dparam(ctrl); + ctrl_chain{end+1} = {ctrl}; + fprintf('Done %s\n',datestr(now)); + % ctrl_chain = {{ctrl}}; + end +end +cluster_print_chain(ctrl_chain); +[chain_fn,chain_id] = cluster_save_chain(ctrl_chain); + +ctrl_chain = {}; +ctrl = cluster_new_batch(param); + +sparam = []; +sparam.argsin{1} = param; +sparam.task_function = 'layer_tracker_generate_combine_task'; +sparam.num_args_out = 1; +sparam.file_success = {}; + +sparam.file_success{end+1} = img_dir; + +sparam.cpu_time = 500; +sparam.mem = 500e6; +sparam.notes = ''; + +out_fn_dir = ct_filename_out(param,'',param.layer_tracker.layer_params.layerdata_source); %/cresis/snfs1/dataproducts/ct_data/rds/2014_Greenland_P3/CSARP_layer_tune_vit_seg4_NC/20140313_09 +save_names = {}; +for i = 1:length(params) + for layer_idx = 1:length(gt_layer_params) + save_names{layer_idx,i} = sprintf('%s/result.mat',img_fn_dir{i}); + end +end +sparam.argsin{1}.save_names = save_names; + + +ctrl = cluster_new_task(ctrl,sparam,[]); +ctrl_chain{end+1} = {ctrl}; +fprintf('Done %s\n',datestr(now)); + + +cluster_print_chain(ctrl_chain); +[chain_fn,chain_id] = cluster_save_chain(ctrl_chain); \ No newline at end of file diff --git a/cresis-toolbox/tracker/run_layer_tracker_tune.m b/cresis-toolbox/tracker/run_layer_tracker_tune.m new file mode 100644 index 00000000..d8cb6483 --- /dev/null +++ b/cresis-toolbox/tracker/run_layer_tracker_tune.m @@ -0,0 +1,201 @@ +% script run_layer_tracker_tune +% Runs layer_tracker.m for tuning layer tracking hyperparameters. +% +% Authors: Anjali Pare, John Paden +% +% See also: layer_tracker.m, layer_tracker_combine_task.m, +% layer_tracker_task.m, layer_tracker_input_check.m, +% layer_tracker_profile.m, run_layer_tracker.m, run_layer_tracker_tune.m + +%% User Settings +% ---------------------------------------------------------------------- +param_override = []; + +% params = read_param_xls(ct_filename_param('rds_param_2011_Greenland_P3.xls')); +params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls')); +% params = read_param_xls(ct_filename_param('rds_param_2018_Greenland_P3.xls')) +params = ct_set_params(params,'cmd.generic',1);%|20140415_05|20140421_01|20140516_01');%'20140415_05|20140421_01|20140514_01|20140313_09'); +%params = ct_set_params(params,'cmd.generic',0,'day_seg','20140310_01'); +params = ct_set_params(params,'cmd.generic',0,'cmd.notes','Do not process'); +params = ct_set_params(params,'cmd.frms',[]); % Specify specific frames (or leave empty/undefined to do all frames) +% params = ct_set_params(params,'cmd.generic',1,'day_seg','20110331_02'); +% params = ct_set_params(params,'cmd.frms',19); % Specify specific frames (or leave empty/undefined to do all frames) + +param_override.layer_tracker.debug_plots = {}; +% param_override.layer_tracker.debug_plots = {'tracked_images'}; +% param_override.layer_tracker.debug_plots = {'tracked_images','visible'}; % Uncomment for debugging + +param_override.layer_tracker.echogram_img = 0; % To choose an image besides the base (0) image +% echogram_source: location of echogram data used for tracking +% param_override.layer_tracker.echogram_source = 'CSARP_post/qlook'; +% param_override.layer_tracker.echogram_source = 'CSARP_post/mvdr'; +param_override.layer_tracker.echogram_source = 'CSARP_post/standard'; + +% layer_params: layerparams structure of where to store the output using +% opsCopyLayers.m +param_override.layer_tracker.layer_params = []; +% Uncomment to enable layerdata storage +param_override.layer_tracker.layer_params.layerdata_source = 'layer_tune_vit_season'; +% Uncomment to enable OPS storage +% param_override.layer_tracker.layer_params.source = 'ops'; + +tracker_method = 'viterbi'; % Choose the tracking method +% block_size_frms: Number of frames to be loaded at a time +param_override.layer_tracker.block_size_frms = 1; + +% track_per_task: Number of tracks per task +param_override.layer_tracker.track_per_task = 1; + +%% param.layer_tracker.track options +if strcmpi(tracker_method,'lsm') + track_idx = 0; + y_values = [140:20:280];%[140:20:280]; % Enter 1 as the first element if surface mean is calculated + dy_values = [5 10 20 40 60 80 100]; + + for y_idx = 1:length(y_values) + y = y_values(y_idx); + + for dy_idx = 1:length(dy_values) + dy = dy_values(dy_idx); + track = []; + + %% Enable one set of parameters + track.en = true; + % RDS + track.profile = 'rds_OIB'; + + %% LSM User Settings + track.method = 'lsm'; + track.flag = 0; %to specify whether we want to consider mean of y + track.lsm.y = y; % = '' for y = mean(SURF) + track.lsm.dy = dy; + track.lsm.storeIter = [75:25:500]; + track.idx_dim_name = {'storeIter' 'dy' 'y'}; + track.idx_reshape = [length(track.lsm.storeIter) length(dy_values) length(y_values)]; + track.idx = length(dy_values)*length(track.lsm.storeIter)*(y_idx-1) ... + + length(track.lsm.storeIter)*(dy_idx-1) + (1:length(track.lsm.storeIter)); + track.init.max_diff = inf; + track.detrend = []; + track.norm.scale = [-40 90]; + + track_idx = track_idx + 1; + param_override.layer_tracker.track{track_idx} = track; + end + end +elseif strcmpi(tracker_method,'viterbi') + track_idx = 0; + transition_weight_values = [0.1 0.316 1 3.16 10]; % Enter the transition weight (smoothness) values to consider + + for tw_idx = 1:length(transition_weight_values) + transition_weight = transition_weight_values(tw_idx); + track = []; + + %% Enable one set of parameters + track.en = true; + % RDS + track.profile = 'rds_OIB'; + + %% Viterbi User Settings + track.method = 'viterbi'; + track.viterbi.transition_weight = transition_weight; + track.idx_dim_name = {'transition_weight'}; + track.idx_reshape = [length(track.viterbi.transition_weight)]; + track.idx = tw_idx; + %track.min_bin = struct('name','tomo_top'); + track.min_bin.name = 'surface'; + track.min_bin.eval.cmd = 's=s+0.5e-6;'; + track.max_bin = struct('name','tomo_bottom'); + track.crossover.en = false; + track.crossover.season_names_bad = {'2003_Greenland_P3', '2005_Greenland_P3'}; % Bad seasons to not include + % track.crossover.gps_time_good_eval = @(x) true; % All cross overs are good + track.crossover.gps_time_good_eval = @(x) x < datenum_to_epoch(datenum('2014/03/01')); % Cross overs before this date are good + if 0 + track.ice_mask.en = false; + elseif 1 + % Greenland + track.ice_mask.en = true; + track.ice_mask.type = 'geotiff'; + track.ice_mask.fn = ct_filename_gis([], fullfile('greenland','IceMask','GimpIceMask_90m_v1.1.tif')); + elseif 0 + % Canada + track.ice_mask.en = true; + track.ice_mask.type = 'bin'; + track.ice_mask.fn = ct_filename_gis([], fullfile('canada','ice_mask','03_rgi50_ArcticCanadaNorth','03_rgi50_ArcticCanadaNorth.bin')); + track.ice_mask.mat_fn = ct_filename_gis([], fullfile('canada','ice_mask','03_rgi50_ArcticCanadaNorth','03_rgi50_ArcticCanadaNorth.mat')); + elseif 0 + % Antarctica + track.ice_mask.en = true; + track.ice_mask.type = 'geotiff2'; + track.ice_mask.fn = ct_filename_gis([], fullfile('antarctica/DEM/BEDMAP2/original_data/bedmap2_tiff/bedmap2_icemask_grounded_and_shelves.tif')); + track.ice_mask.fn2 = ct_filename_gis([], fullfile('antarctica/DEM/BEDMAP2/original_data/bedmap2_tiff/bedmap2_rockmask.tif')); + end + track.init.dem_layer = struct('name','surface'); + track.viterbi.gt_cutoff = 50; + track.mult_suppress.en = true; + track.init.max_diff = inf; + track.detrend = []; + track.filter_trim = [0 120]; + track.norm.scale = [-40 90]; + track.xcorr = echo_xcorr_profile('short_unitstep'); + track.ground_truth.en = true; + track.ground_truth.layers.source = 'layerdata'; + track.ground_truth.layers.layerdata_source = 'layer'; + track.ground_truth.layers.name = 'bottom_mc'; + track.ground_truth.cutoff = 450; + track_idx = track_idx + 1; + param_override.layer_tracker.track{track_idx} = track; + end +end + +% param_override.layer_tracker.surf_layer = struct('name','surface','source','layerdata','layerdata_source','layer'); + +% param_override.layer_tracker.crossover_layer = struct('name','bottom','source','ops'); + +% dbstop if error; + param_override.cluster.type = 'torque'; +% param_override.cluster.type = 'matlab'; +% param_override.cluster.type = 'debug'; +% param_override.cluster.type = 'slurm'; +% param_override.cluster.rerun_only = true; +% param_ovserride.cluster.desired_time_per_job = 240*60; +param_override.cluster.cpu_time_mult = 2; +param_override.cluster.mem_mult = 2; + +%% Automated Section +% ---------------------------------------------------------------------- + +% Input checking +global gRadar; +if exist('param_override','var') + param_override = merge_structs(gRadar,param_override); +else + param_override = gRadar; +end + +ctrl_chain = {}; +% Process each of the segments +for param_idx = 1:length(params) + param = params(param_idx); + if isfield(param.cmd,'generic') && ~iscell(param.cmd.generic) && ~ischar(param.cmd.generic) && param.cmd.generic + if strcmpi(tracker_method,'lsm') + if track.flag == 1 + gt_layer_params = []; + idx = 1; + gt_layer_params(idx).name = 'surface'; + layers = opsLoadLayers(param,gt_layer_params); + param_override.layer_tracker.gt_params = layers; + end + end + [ctrl_chain{end+1},param] = layer_tracker(param,param_override); + % Since we are tuning, save the parameters + out_param_fn = [ct_filename_ct_tmp(param,'','layer_tracker','') ... + sprintf('_%s_t%03d_%s.mat',datestr(now,'yyyymmdd_HHMMSS'), ... + length(param.layer_tracker.track), param.layer_tracker.track{1}.method)]; + fprintf('Saving tuning parameters %s\n',out_param_fn); + ct_save(out_param_fn,'param'); + end +end + +cluster_print_chain(ctrl_chain); + +[chain_fn,chain_id] = cluster_save_chain(ctrl_chain); diff --git a/cresis-toolbox/tracker/run_unblock_data.m b/cresis-toolbox/tracker/run_unblock_data.m new file mode 100644 index 00000000..e279d16b --- /dev/null +++ b/cresis-toolbox/tracker/run_unblock_data.m @@ -0,0 +1,70 @@ +% Script for running unblock_data +% +% Authors: fill in... +% +% See also: run_FUNCTION.m, run_unblock.m, + +%% User Setup +% ===================================================================== +param_override = []; + +params = read_param_xls(ct_filename_param('snow_param_2012_Greenland_P3.xls')); +% Syntax for running a specific segment and frame by overriding parameter spreadsheet values +%params = read_param_xls(ct_filename_param('rds_param_2009_Antarctica_TO.xls'),'20091228_01'); +params = ct_set_params(params,'cmd.generic',0); +params = ct_set_params(params,'cmd.generic',1,'day_seg','20120330_04'); +params = ct_set_params(params,'cmd.frms',[]); + + +% Set override parameters using ct_set_params, by setting param_override, or by setting FUNCTION_params variable which eventually is copied to param_override.FUNCTION). + + +param_override.unblock_data = [] ; + +% block_data_param structure + +% Provide path to tracked layers and echogram path argument in ct_data +param_override.unblock_data.tracked_layers_dir_path = '/cresis/snfs1/scratch/ibikunle/ct_user_tmp/final/snow/2012_Greenland_P3/frames_001_243_20120330_04/image'; +param_override.unblock_data.echo_path = 'CSARP_post/qlook'; % Argument passed into ct_filename_out(param,argument) e.g 'qlook' or for rds--> argument = 'CSARP_post/standard' + +param_override.unblock_data.uncompress_en = 1; + +param_override.unblock_data.tracked_layers_search_str = 'image'; % The prefix of each NN layer .mat file +param_override.unblock_data.echo_search_str = 'Data_2012'; % search string for echogram file + +param_override.unblock_data.layer_dest_source = 'layerData'; % +param_override.unblock_data.layer_dest_layerdata_source = 'layerData_koenig_ibk'; % Argument to destination layerdata_source + + +param_override.unblock_data.dest_layer_prefix = 'Koenig_'; + +param_override.unblock_data.copy_method = 'fillgaps'; %'fill_gaps' or 'overwrite' +param_override.unblock_data.gaps_fill_method = 'preserve_gaps'; %'preserve_gaps'; %interp_finite +param_override.unblock_data.layer_source.existence_check = false; +param_override.unblock_data.layer_dest.existence_check = false; + + + +%% Automated Section +% ===================================================================== + +% Input checking +global gRadar; +if exist('param_override','var') + param_override = merge_structs(gRadar,param_override); +else + param_override = gRadar; +end + + +% Process each of the segments +for param_idx = 1:length(params) + param = params(param_idx); + if ~isfield(param.cmd,'generic') || iscell(param.cmd.generic) || ischar(param.cmd.generic) || ~param.cmd.generic + continue; + end + unblock_data(param,param_override); +end + +% Post process code (only include if necessary) + diff --git a/cresis-toolbox/tracker/stat_generator_mult_segs.m b/cresis-toolbox/tracker/stat_generator_mult_segs.m new file mode 100644 index 00000000..b4a350ad --- /dev/null +++ b/cresis-toolbox/tracker/stat_generator_mult_segs.m @@ -0,0 +1,192 @@ +%% Setup plots +% set fig_format to true to generate .fig plots +options.segment_vs_frame.plot = true; % absolute error vs frames (for each segment) +options.segment_vs_frame.fig_format = true; +options.season_vs_gps.plot = true; % absolute error vs gps time (for entire season) +options.season_vs_gps.fig_format = true; +options.season_vs_gps.multiple_figs = true; % choose this option if you separate plots generated for each layer. If set to false, 6 subplots will be plotted in one figure. +options.absolute_error.plot = true; % mean absolute error plots +options.absolute_error.fig_format = true; +options.absolute_error.multiple_plots = true; +options.percentage_correct.plot = true; % plot of percentage correct vs frames (for each segment) +options.percentage_correct.fig_format = true; +options.percentage_correct.multiple_plots = true; +options.hist_generation.plot = true; % histograms of data points vs absolute error +options.hist_generation.fig_format = true; +options.hist_generation.multiple_plots = true; +num_layers = 1; +fprintf('\n=====All segements=====\n'); + ff{1} = load('/cresis/snfs1/scratch/anjali/cluster_tuning/vit_fig1/result_layer_tune_lsm_s1'); + ff{2} = load('/cresis/snfs1/scratch/anjali/cluster_tuning/vit_fig1/result_layer_tune_lsm_s2'); + foo{1} = []; + gps_time = []; + season_vect = []; + for hist_idx = 1:length(ff{1}.frame_error{layer_idx}) + range_per{1}{hist_idx} = []; + res_matrix{1}{hist_idx} = []; + end + percent_error{layer_names} = zeros(1,(length(ff{1}.frame_error{layer_names}))); + + for i = 1:length(ff) + foo{layer_names} = [foo{layer_names};ff{i}.res_matrix_all{layer_names}]; + percent_error{layer_names} = percent_error{layer_names} + ff{i}.percent_error{layer_names}; + gps_time = cat(2,gps_time,ff{i}.gps_time); + season_vect = cat(2,gps_time,ff{i}.gps_time); + for hist_idx = 1:length(frame_error{layer_idx}) + range_per{1}{hist_idx} = cat(2,range_per{1}{hist_idx},ff{i}.range_per{1}{hist_idx}); + res_matrix{1}{hist_idx} = cat(2,res_matrix{1}{hist_idx},ff{i}.res_matrix{1}{hist_idx}); + end + end + + + + points = []; + min_val = []; + res_matrix_all_frms = []; + for layer_names = 1:num_layers + res_matrix_all_frms{layer_names} = nanmean(foo{layer_names},1); + res_matrix_all_frms{layer_names} = permute(res_matrix_all_frms{layer_names},idx_matrix); + [min_val{layer_names},i]=min(res_matrix_all_frms{layer_names}(:)); + sizeMatrix = size(res_matrix_all_frms{layer_names}); + for dim = 1:length(size(res_matrix_all_frms{layer_names})) + points{layer_names}(dim) = mod(floor((i-1)/prod(sizeMatrix(1:dim-1))),sizeMatrix(dim))+1; + end + if (length(size(res_matrix_all_frms{layer_names})) < length(param.layer_tracker.track{1}.idx_reshape)) + points{layer_names}(dim+1) = 1; + end + end + + if (options.percentage_correct.plot) + fprintf('\n=====Percentage correct all plots=====\n'); + %if (~options.percentage_correct.multiple_plots) + for layer_idx = 1:length(gt_layer_params) + h_fig(1) = figure;%('visible','off'); + for hist_idx = 1:length(frame_error{layer_idx}) + % hold on; + % plot(range_per{layer_idx}{hist_idx},'DisplayName',sprintf('tracker%d',hist_idx)); + % legend('-DynamicLegend','Location','northeastoutside'); + % xlabel('Number of frames'); + % ylabel('Percentage error'); + % set(gca,'XTick',1:length(param.cmd.frms)); + % set(gca,'XTickLabel',[param.cmd.frms]); + hold on; + plot(range_per{layer_idx}{hist_idx},'.-','DisplayName',sprintf('layer%d',hist_idx)); + xlabel('Frames'); + ylabel('Percentage correct (%)'); + xlim = ([1 length(param.cmd.frms)]); + title(sprintf('Percentage correct vs frames Layer:%d %s',layer_idx,param.season_name),'Interpreter', 'none'); + percent_error{layer_idx}(hist_idx) = percent_error{layer_idx}(hist_idx)/length(season_vect) * 100; + end + img_format = 'png'; + if(options.percentage_correct.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_percentage_correct_overall_%03d.%s',testname,param.season_name,layer_idx,img_format)); + %ct_saveas(h_fig(1),img_fn); + end + + for layer_idx = 1:length(gt_layer_params) + h_fig(1) = figure;%('visible','off'); + plot(percent_error{layer_idx}); + hold on; + plot(percent_error{layer_idx}, 'r*'); + ylabel('Percentage correct (%)'); + xlabel('Trackers'); + title(sprintf('Percentage correct vs trackers Layer:%d %s',layer_idx,param.season_name),'Interpreter', 'none'); + set(gca,'XTick',1:length(frame_error{layer_idx})); + img_format = 'png'; + if(options.percentage_correct.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_total_percent_correct_%03d.%s',testname,param.season_name,layer_idx,img_format)); + %ct_saveas(h_fig(1),img_fn); + end + %end + end + + if (options.hist_generation.plot) + fprintf('\n=====Hist generation all plots=====\n'); + %if (~options.hist_generation.multiple_plots) + for layer_idx = 1:length(gt_layer_params) + h_fig(1) = figure;%('visible','off'); + for hist_idx = 1:length(frame_error{layer_idx}) + hold on; + [N,x] = hist(abs(res_matrix{layer_idx}{hist_idx})); + plot(x,N,'DisplayName',sprintf('layer%d',hist_idx)); + %legend('-DynamicLegend','Location','northeastoutside'); + ylabel('Data points'); + xlabel('Absolute error (bins)'); + title(sprintf('Data points vs Absolute error Layer:%d %s',layer_idx,param.season_name),'Interpreter', 'none'); + end + hold off; + img_format = 'png'; + if(options.hist_generation.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_histogram_generation_overall_%03d.%s',testname,param.season_name,layer_idx,img_format)); + % ct_saveas(h_fig(1),img_fn); + end + %end + end + + + if(options.season_vs_gps.plot) + fprintf('\n=====season plots=====\n'); + if(options.season_vs_gps.multiple_figs) + for layer_idx = 1:length(gt_layer_params) + for track_idx = 1:length(frame_error{layer_idx}) + h_fig(1) = figure;%('visible','off'); + plot(gps_time,abs(res_matrix{layer_idx}{track_idx})); + ylabel('Absolute error (bins)'); + xlabel('GPS time'); + title(sprintf('%s frames: %03d Tracker: %d Layer: %d',param.season_name,length(param.cmd.frms), track_idx, layer_idx),'Interpreter', 'none'); + %xlim([gps_time(1) gps_time(end)]); + + img_format = 'png'; + if(options.season_vs_gps.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_season_gps_%03d_%03d.%s',testname,param.season_name,track_idx,layer_idx,img_format)); + %ct_saveas(h_fig(1),img_fn); + end + end + else + for layer_idx = 1:length(gt_layer_params) + h_fig(1) = figure('visible','off'); + subplot_idx = 0; + for track_idx = 1:length(frame_error{layer_idx}) + subplot_idx = subplot_idx + 1; + + if(subplot_idx < 6) + subplot(3,2,subplot_idx) + plot(gps_time,abs(res_matrix{layer_idx}{track_idx})); + ylabel('Absolute error (bins)'); + xlabel('GPS time'); + title(sprintf('%s frames: %03d Tracker: %d Layer: %d',param.season_name,length(param.cmd.frms), track_idx, layer_idx),'Interpreter', 'none'); + %xlim([gps_time(1) gps_time(end)]); + else + subplot(3,2,subplot_idx) + plot(gps_time,abs(res_matrix{layer_idx}{track_idx})); + ylabel('Absolute error (bins)'); + xlabel('GPS time'); + title(sprintf('%s Tracker: %d Layer: %d',param.season_name, track_idx, layer_idx),'Interpreter', 'none'); + %xlim([gps_time(1) gps_time(end)]); + img_format = 'png'; + if(options.season_vs_gps.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_season_gps_%03d_%03d.%s',testname,param.season_name,track_idx,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + h_fig(1) = figure('visible','off'); + subplot_idx = 0; + end + end + end + end + + end \ No newline at end of file diff --git a/cresis-toolbox/tracker/stat_generator_mult_segs_temp.m b/cresis-toolbox/tracker/stat_generator_mult_segs_temp.m new file mode 100644 index 00000000..f64db215 --- /dev/null +++ b/cresis-toolbox/tracker/stat_generator_mult_segs_temp.m @@ -0,0 +1,937 @@ +%% Script layer_tracker_tune_layerdata.m +% % sign and percentage correct plot_best_segment_overall/segment % correct +% and mean absolute error. +% /cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140313_08_20210211_201501_t006_lsm.mat +% /cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140516_01_20210211_201509_t006_lsm.mat +% Tuning script to work with layerdata files rather than temporary files +% Used to find best combination of paramters for given tracker method +% Enter layer names in the gt_layer_params(idx).name field +% set temp to the file location of the param structure used for tracking in run_layer_tracker.m script +% set save_name to store final tuning results +% Statistical data saved: +% number of points with finite values of ground truth of twtt (num_gt_infinite), +% number of points where ground truth twtt is finite and tracked data twtt is NaN (num_isnan) +% number of points where (tracked data twtt - ground truth twtt) < 5 * dt (num_points) +% nanmean of (tracked data twtt - ground truth twtt) (res_matrix) +% Authors: Anjali Pare, John Paden +% See layer_tracker_tune_plot.m to view 2 dimensional imagesc plots of the data. + +% Look at % correct. If we're close to the right ans 100 % of the time but +% not quite right 100 % of the time is worse than being right 90% of the +% time and completely wrong 10 % of the time. Need to correct in quality +% control. Check if you are outside of a wondow and label as bad. % correct +% should be exactly the same (not necesary). There could be offsets. +% 1-2 range bin. + +% Have fig option for all. Use hist instead of histogram. Histogram plots +% separate from % correct graphs. Enable option to generate best histogram +% plot. Plot as regular line using hist + +% What to plot, finding best combination, trying with LSM +clear; +dbstack_info = dbstack; +fprintf('=====================================================================\n'); +fprintf('%s (%s)\n', dbstack_info(1).name, datestr(now,'HH:MM:SS')); +fprintf('=====================================================================\n'); + +%% Setup param +%20140516_01_20210107_163106_t005_viterbi +%20140313_08_20210107_162824_t005_viterbi +%temp = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140516_01_20201122_193803_t005_viterbi.mat'); + + +% Example to run 1 season with 1 segment all frames +params_all = []; +idx_segment = 1; +params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls'),'20140313_08|20140516_01'); +params(idx_segment).cmd.generic = 1; +params(idx_segment).cmd.frms = []; +idx_segment = idx_segment + 1; +params(idx_segment).cmd.generic = 1; +params(idx_segment).cmd.frms = []; +% idx_segment = idx_segment + 1; +% params(idx_segment).cmd.generic = 1; +% params(idx_segment).cmd.frms = []; +% idx_segment = idx_segment + 1; +% params(idx_segment).cmd.generic = 1; +% params(idx_segment).cmd.frms = []; +params_all{end+1} = params; + +% Example to run 1 season with 2 segment and different frames +% params_all = []; +% idx_segment = 1; +% params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls'),'20140313_08|20140516_01'); +% params(idx_segment).cmd.generic = 1; +% params(idx_segment).cmd.frms = 1:2; +% idx_segment = idx_segment + 1; +% params(idx_segment).cmd.generic = 1; +% params(idx_segment).cmd.frms = 3:4; +% params_all{end+1} = params; + +% Example to run 2 season with 1 segment frames 1:5 +% params_all = []; +% idx_segment = 1; +% params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls'),'20140516_01'); +% params(idx_segment).cmd.generic = 1; +% params(idx_segment).cmd.frms = 1:5; +% params_all{end+1} = params; + +% idx_segment = 1; +% params = read_param_xls(ct_filename_param('rds_param_2014_Antarctica_DC8.xls'),'20141122_08'); +% idx_segment = idx_segment + 1; +% params(idx_segment).cmd.generic = 1; +% params(idx_segment).cmd.frms = 1:5; +% params_all{end+1} = params; + + +% Example to run all segments and all frames of a season +% params_all = []; +% idx_segment = 1; +% params = read_param_xls(ct_filename_param('rds_param_2014_Greenland_P3.xls')); +% params = ct_set_params(params,'cmd.generic',1); +% params(idx_segment).cmd.frms = []; + +%% Setup filename +filename = []; +% one segment +%%filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140516_01_20201202_020912_t005_viterbi.mat'); +% filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140516_01_20210124_024502_t004_lsm.mat'); +%filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140516_01_20200921_121933_t063_lsm.mat'); +%filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140516_01_20210204_195538_t006_lsm.mat'); + + +% filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140313_08_20210211_201501_t006_lsm.mat'); +% filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140516_01_20210211_201509_t006_lsm.mat'); + +filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140313_08_20210304_221148_t005_viterbi.mat'); +filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140516_01_20210315_220253_t005_viterbi.mat'); + +%filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140313_09_20210311_013103_t056_lsm.mat'); +%filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140415_05_20210311_013232_t056_lsm.mat'); +% filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140421_01_20210311_014435_t056_lsm.mat'); +% filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140514_01_20210311_015110_t056_lsm.mat'); + % two segments +%filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140313_08_20210107_162824_t005_viterbi.mat'); +%filename{end+1} = load('/cresis/snfs1/dataproducts/ct_data/ct_tmp/layer_tracker/rds/2014_Greenland_P3/20140516_01_20210107_163106_t005_viterbi.mat'); + +%% Setup plots +% set fig_format to true to generate .fig plots +options.segment_vs_frame.plot = true; % absolute error vs frames (for each segment) +options.segment_vs_frame.fig_format = true; +options.season_vs_gps.plot = true; % absolute error vs gps time (for entire season) +options.season_vs_gps.fig_format = true; +options.season_vs_gps.multiple_figs = true; % choose this option if you separate plots generated for each layer. If set to false, 6 subplots will be plotted in one figure. +options.absolute_error.plot = true; % mean absolute error plots +options.absolute_error.fig_format = true; +options.absolute_error.multiple_plots = true; +options.percentage_correct.plot = true; % plot of percentage correct vs frames (for each segment) +options.percentage_correct.fig_format = true; +options.percentage_correct.multiple_plots = true; +options.hist_generation.plot = true; % histograms of data points vs absolute error +options.hist_generation.fig_format = true; +options.hist_generation.multiple_plots = true; + +%% Setup save +% Enter filename of where you want to store the images +img_dir = '/cresis/snfs1/scratch/anjali/cluster_tuning/vit_fig1'; +if ~exist(img_dir,'dir') + mkdir(img_dir); +end + +%% Setup general +% set the range and indicate the layers to track + +gt_layer_params = []; +layer_params = []; +res_matrix = []; +res_list = []; +gps_time = []; +season_vect = []; +segment_vect = []; +frame_vect = []; +seg_idx = 0; +season_vect_frm = []; +segment_vect_frm = []; +frame_vect_frm = []; +range = 10; % set acceptable range for error +idx = 1; + +% If only bottom + %gt_layer_params(idx).name = 'bottom'; % specify layer names + +% If surface and bottom +Sfoo{idx} = []; +gt_layer_params(idx).name = 'bottom'; % specify layer names + +num_layers = idx; + +%% Implementation + +for season_idx = 1:length(params_all) %seasons + params = params_all{season_idx}; + for param_idx = 1:length(params) % segments + param = params(param_idx); + for frm = param.cmd.frms + season_vect_frm(end+1) = season_idx; + segment_vect_frm(end+1) = param_idx; + frame_vect_frm(end+1) = frm; + end + end +end + +for season_idx = 1:length(params_all) %seasons + params = params_all{season_idx}; + for param_idx = 1:length(params) % segments + param = params(param_idx); + + if isfield(param.cmd,'generic') && ~iscell(param.cmd.generic) && ~ischar(param.cmd.generic) && param.cmd.generic + gps_time = []; + season_vect = []; + segment_vect = []; + seg_idx = seg_idx + 1; + param_override = filename{seg_idx}.param; + param = merge_structs(param,param_override); + total_length = 0; + layers = opsLoadLayers(param,gt_layer_params); + idx_matrix = [(1:length(param.layer_tracker.track{1}.idx_reshape))+1 1]; % used for resizing + for layer_names = 1:idx + res_matrix_all{layer_names} = zeros([length(param.cmd.frms),param.layer_tracker.track{1}.idx_reshape]); + res_mat_temp{layer_names} = cell([param.layer_tracker.track{1}.idx_reshape]); + end + + %if (param_idx == 1) + for layer_idx = 1:length(gt_layer_params) + if strcmpi(param.layer_tracker.track{1}.method,'lsm') + percent_error{layer_idx} = zeros(1,(length(param.layer_tracker.track)*length(param.layer_tracker.track{1}.lsm.storeIter))); + for iter_idx = 1:(length(param.layer_tracker.track)*length(param.layer_tracker.track{1}.lsm.storeIter)) + range_per{layer_idx}{iter_idx} = []; + res_matrix{layer_idx}{iter_idx} = []; + + end + elseif strcmpi(param.layer_tracker.track{1}.method,'viterbi') + percent_error{layer_idx} = zeros(1,(length(param.layer_tracker.track))); + for iter_idx = 1:(length(param.layer_tracker.track)) + range_per{layer_idx}{iter_idx} = []; + res_matrix{layer_idx}{iter_idx} = []; + end + end + end + %end + % for layer_idx = 1:length(gt_layer_params) + % percent_error{layer_idx} = zeros(1,(length(param.layer_tracker.track)*length(param.layer_tracker.track{1}.lsm.storeIter))); + % end + + for track_idx = 1:length(param.layer_tracker.track) + range_percent_all_frms.(sprintf('%s', param.layer_tracker.track{track_idx}.name)) = 0; + for frm_idx = 1:length(param.cmd.frms) + range_percent.(sprintf('%s_%03d', param.layer_tracker.track{track_idx}.name,frm_idx)) = 0; + prob_error.(sprintf('%s_%03d', param.layer_tracker.track{track_idx}.name,frm_idx)) = 0; + %frame_error{track_idx}(frm_idx) = 0; + end + end + frm_idx = 0; + for frm = param.cmd.frms + frm_idx = frm_idx+1; + data_fn_dir = ct_filename_out(param, param.layer_tracker.echogram_source, ''); + data_fn_name = sprintf('Data_%s_%03d.mat',param.day_seg,frm); + data_fn = fullfile(data_fn_dir, data_fn_name); + data = load(data_fn, 'GPS_time', 'Time'); + gps_time = cat(2,gps_time,data.GPS_time); + dt = data.Time(2) - data.Time(1); + total_length = total_length + length(data.GPS_time); + idx_list = 1; + + frm_length = length(data.GPS_time); + for j = 1:length(data.GPS_time) + season_vect(end+1) = season_idx; + segment_vect(end+1) = param_idx; + frame_vect(end+1) = frm; + end + + for track_idx = 1:length(param.layer_tracker.track) + + for layer_idx = 1:length(gt_layer_params) + if strcmpi(param.layer_tracker.track{1}.method,'lsm') + for idx = 1:length(param.layer_tracker.track{track_idx}.lsm.storeIter) + layer_params(idx).name = sprintf('%s_%s_%s_%03d',param.layer_tracker.track{track_idx}.name,param.layer_tracker.track{track_idx}.method,gt_layer_params(layer_idx).name,idx); + layer_params(idx).source = param.layer_tracker.layer_params.source; + layer_params(idx).layerdata_source = param.layer_tracker.layer_params.layerdata_source; + end + else + layer_params(1).name = sprintf('%s_%s_%s',param.layer_tracker.track{track_idx}.name,param.layer_tracker.track{track_idx}.method,gt_layer_params(layer_idx).name); + layer_params(1).source = param.layer_tracker.layer_params.source; + layer_params(1).layerdata_source = param.layer_tracker.layer_params.layerdata_source; + end + ids = 1; + layers_new = opsLoadLayers(param,layer_params); + + surf = interp1(layers(layer_idx).gps_time,layers(layer_idx).twtt,data.GPS_time); + surf_bins = round(interp1(data.Time,1:length(data.Time),surf)); + + for pos = 1:length(layers_new) + surf = interp1(layers_new(pos).gps_time,layers_new(pos).twtt,data.GPS_time); + surf_bins_itr = round(interp1(data.Time,1:length(data.Time),surf)); + res_matrix_all{layer_idx}(frm_idx,param.layer_tracker.track{track_idx}.idx(pos)) = nanmean(abs(surf_bins - surf_bins_itr)); + counter = 0; + for i = 1:length(surf_bins) + try + res_matrix{layer_idx}{param.layer_tracker.track{track_idx}.idx(pos)}(end+1) = surf_bins(i) - surf_bins_itr(i); + res_mat_temp{layer_idx}{param.layer_tracker.track{track_idx}.idx(pos)}(end+1) = surf_bins(i) - surf_bins_itr(i); + catch ME + res_matrix{layer_idx}{param.layer_tracker.track{track_idx}.idx(pos)}(i) = surf_bins(i) - surf_bins_itr(i); + res_mat_temp{layer_idx}{param.layer_tracker.track{track_idx}.idx(pos)}(i) = surf_bins(i) - surf_bins_itr(i); + end + if (abs(surf_bins(i) - surf_bins_itr(i)) <= range) + counter = counter + 1; + end + end + range_percent.(sprintf('%s_%03d', param.layer_tracker.track{track_idx}.name,frm)) = (counter/length(surf_bins))*100; + range_per{layer_idx}{param.layer_tracker.track{track_idx}.idx(pos)}(end+1) = (counter/length(surf_bins))*100; + percent_error{layer_idx}(param.layer_tracker.track{track_idx}.idx(pos)) = counter + percent_error{layer_idx}(param.layer_tracker.track{track_idx}.idx(pos)); + prob_error.(sprintf('%s_%03d', param.layer_tracker.track{track_idx}.name,frm)) = counter*100; + frame_error{layer_idx}{param.layer_tracker.track{track_idx}.idx(pos)}(frm_idx) = nanmean(abs(surf_bins - surf_bins_itr)); + try + range_percent_all_frms.(sprintf('%s', param.layer_tracker.track{track_idx}.name)) = range_percent_all_frms.(sprintf('%s', param.layer_tracker.track{track_idx}.name)) + counter; + catch ME + range_percent_all_frms.(sprintf('%s', param.layer_tracker.track{track_idx}.name)) = counter; + end + end + end + idx_list = idx_list + 1; + end + end + + + points = []; + min_val = []; + res_matrix_all_frms = []; + for layer_names = 1:num_layers + res_matrix_all_frms{layer_names} = nanmean(res_matrix_all{layer_names},1); + res_matrix_all_frms{layer_names} = permute(res_matrix_all_frms{layer_names},idx_matrix); + [min_val{layer_names},i]=min(res_matrix_all_frms{layer_names}(:)); + sizeMatrix = size(res_matrix_all_frms{layer_names}); + for dim = 1:length(size(res_matrix_all_frms{layer_names})) + points{layer_names}(dim) = mod(floor((i-1)/prod(sizeMatrix(1:dim-1))),sizeMatrix(dim))+1; + end + if (length(size(res_matrix_all_frms{layer_names})) < length(param.layer_tracker.track{1}.idx_reshape)) + points{layer_names}(dim+1) = 1; + end + if(param_idx==1) + foo{layer_names} = res_matrix_all{layer_names}; + else + foo{layer_names} = [foo{layer_names};res_matrix_all{layer_names}]; + end + end + + % per segment vs frame + if (options.segment_vs_frame.plot) + fprintf('\n=====Segment frame plots=====\n'); + for layer_idx = 1:length(gt_layer_params) + h_fig(1) = figure('visible','off'); + for fig_idx = 1:length(frame_error{layer_idx}) + hold on; + %plot(frame_error{layer_idx}{fig_idx},'DisplayName',sprintf('tracker%d',fig_idx)); + %legend('-DynamicLegend','Location','northeastoutside'); + plot(frame_error{layer_idx}{fig_idx},'.-','DisplayName',sprintf('layer%d',fig_idx)); + labelNameX = 'Frames'; + labelNameY = 'Absolute error (bins)'; + ylabel(labelNameY); + xlabel(labelNameX); + xlim = ([1 length(param.cmd.frms)]); + % labelNameX = 'frames'; + % labelNameY = 'absolute error'; + % ylabel(labelNameY); + % xlabel(labelNameX); + % set(gca,'XTick',1:length(param.cmd.frms)); + % set(gca,'XTickLabel',[param.cmd.frms]) + title(sprintf('%s Layer: %d',param.day_seg,layer_idx),'Interpreter', 'none'); + end + + hold off; + img_format = 'png'; + if(options.segment_vs_frame.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_segment_vs_frame_%03d.%s',testname,param.day_seg,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + end + end + + if (options.percentage_correct.plot) + fprintf('\n=====Percentage plots=====\n'); + if (options.percentage_correct.multiple_plots) + for layer_idx = 1:length(gt_layer_params) + h_fig(1) = figure('visible','off'); + for hist_idx = 1:length(frame_error{layer_idx}) + % hold on; + % plot(range_per{layer_idx}{hist_idx},'DisplayName',sprintf('tracker%d',hist_idx)); + % legend('-DynamicLegend','Location','northeastoutside'); + % xlabel('Number of frames'); + % ylabel('Percentage error'); + % set(gca,'XTick',1:length(param.cmd.frms)); + % set(gca,'XTickLabel',[param.cmd.frms]); + hold on; + y = range_per{layer_idx}{hist_idx}(segment_vect_frm == param_idx); + plot(y,'.-','DisplayName',sprintf('layer%d',hist_idx)); + xlabel('Frames'); + ylabel('Percentage error'); + xlim = ([1 length(param.cmd.frms)]); + title(sprintf('%s Layer: %d',param.day_seg,layer_idx),'Interpreter', 'none'); + % percent_error{layer_idx}(hist_idx) = percent_error{layer_idx}(hist_idx)/length(season_vect) * 100; + end + img_format = 'png'; + if(options.percentage_correct.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_percentage_correct_%03d.%s',testname,param.day_seg,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + end + end + end + + if (options.hist_generation.plot) + fprintf('\n=====Hist generation plots=====\n'); + if (options.hist_generation.multiple_plots) + for layer_idx = 1:length(gt_layer_params) + h_fig(1) = figure('visible','off'); + for hist_idx = 1:length(frame_error{layer_idx}) + hold on; + plot_hist = abs(res_matrix{layer_idx}{hist_idx}(segment_vect == param_idx)); + [N,x] = hist(plot_hist); + plot(x,N,'DisplayName',sprintf('layer%d',hist_idx)); + %legend('-DynamicLegend','Location','northeastoutside'); + ylabel('Data points'); + xlabel('Absolute error (bins)'); + title(sprintf('%s Layer: %d',param.day_seg,layer_idx),'Interpreter', 'none'); + end + hold off; + img_format = 'png'; + if(options.hist_generation.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_histogram_generation_%03d.%s',testname,param.day_seg,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + end + end + end + + % for layer_idx = 1:length(gt_layer_params) + % h_fig(1) = figure('visible','off'); + % plot(percent_error{layer_idx}); + % hold on; + % plot(percent_error{layer_idx}, 'r*'); + % ylabel('Percentage correct'); + % xlabel('Trackers'); + % title(sprintf('%s frames: %d Layer: %d',param.day_seg,length(param.cmd.frms),layer_idx),'Interpreter', 'none'); + % set(gca,'XTick',1:length(frame_error{layer_idx})); + % img_format = 'png'; + % if(options.percentage_correct.fig_format) + % img_format = 'fig'; + % end + % testname = param.layer_tracker.layer_params.layerdata_source; + % img_fn = fullfile(img_dir,sprintf('%s_%s_total_percent_correct_%03d.%s',testname,param.day_seg,layer_idx,img_format)); + % ct_saveas(h_fig(1),img_fn); + % end + + if(options.absolute_error.plot) + fprintf('\n=====Absolute error plots=====\n'); + if (options.absolute_error.multiple_plots) + if(strcmpi(param.layer_tracker.track{1}.method,'viterbi')) + for layer_idx = 1:num_layers + res_mat = res_matrix_all_frms{layer_idx}; + points_new = points{layer_idx}; + x = 1:length(res_mat); % x tick + h_fig(1) = figure('visible','off'); + plot(x,res_mat); + hold on; + for i = 1:length(res_mat) + plot(x(i),res_mat(i), 'r*'); + end + plot(x(points_new(1)),res_mat(points_new(1)), 'ko','LineWidth',4); + set(gca,'XTick',x); + ylabel('Mean absolute error (bins)'); + xlabel('Trackers'); + title(sprintf('%s frames: %03d',param.day_seg,length(param.cmd.frms)),'Interpreter', 'none'); + end + img_format = 'png'; + if(options.absolute_error.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%03d_%s_%s_absolute_error.%s',2,testname,param.day_seg,img_format)); + ct_saveas(h_fig(1),img_fn); + + elseif(strcmpi(param.layer_tracker.track{1}.method,'lsm')) + data = []; + for dim = 1:length(param.layer_tracker.track{1}.idx_reshape) + temp = []; + for track_idx = 1:length(param.layer_tracker.track) + if isempty(temp) + temp = param.layer_tracker.track{track_idx}.(param.layer_tracker.track{track_idx}.method).(param.layer_tracker.track{track_idx}.idx_dim_name{dim}); + else + if ~any(temp == param.layer_tracker.track{track_idx}.(param.layer_tracker.track{track_idx}.method).(param.layer_tracker.track{track_idx}.idx_dim_name{dim})) + temp(end+1) = param.layer_tracker.track{track_idx}.(param.layer_tracker.track{track_idx}.method).(param.layer_tracker.track{track_idx}.idx_dim_name{dim}); + end + end + + end + data{dim}=temp; + end + for layer_idx = 1:num_layers + res_mat = res_matrix_all_frms{layer_idx}; + temp_points = points{layer_idx}; + + A = nchoosek(1:length(param.layer_tracker.track{1}.idx_reshape),2); % changed from 2 + for id = 1:length(A) + min_points = setdiff(1:length(param.layer_tracker.track{1}.idx_reshape),A(id,:)); + min_idxs = temp_points(min_points); + res_matrix_tot = permute(res_mat,[A(id,:) setdiff(1:length(param.layer_tracker.track{1}.idx_reshape),A(id,:))]); + res_matrix_tot = res_matrix_tot(:,:,min_idxs(:)); + idx = A(id,:); + labelTickX = data{idx(1)}; % x tick label + labelTickY = data{idx(2)}; % y tick label + labelNameX = param.layer_tracker.track{1}.idx_dim_name{idx(1)}; % x label name + labelNameY = param.layer_tracker.track{1}.idx_dim_name{idx(2)}; % y label name + xTick = 1:length(labelTickX); % x tick + yTick = 1:length(labelTickY); % y tick + %% Generate Figure + h_fig(1) = figure('visible','off'); % figure is not displayed + imagesc(squeeze(res_matrix_tot')); + hold on; + plot(temp_points(idx(1)),temp_points(idx(2)),'x','LineWidth',4,'MarkerSize',10,'Color','white'); + + ylabel(labelNameY); + xlabel(labelNameX); + set(gca,'XTickLabel',labelTickX); + set(gca,'YTickLabel',labelTickY); + set(gca,'XTick',xTick); + set(gca,'YTick',yTick); + h_colorbar = colorbar(gca); + set(get(h_colorbar,'YLabel'),'String','Mean absolute error (rows)'); + img_format ='jpg'; + if(options.absolute_error.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_absolute_error_%s_vs_%s_%03d.%s',testname,param.day_seg,labelNameX,labelNameY,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + + end + end + end + end + + + end + + % if(options.season_vs_gps.plot) + % if(options.season_vs_gps.multiple_figs) + % for layer_idx = 1:length(gt_layer_params) + % for track_idx = 1:length(frame_error{layer_idx}) + % h_fig(1) = figure; + % plot(gps_time,abs(res_matrix{layer_idx}{track_idx})); + % ylabel('Absolute error (bins)'); + % xlabel('GPS time'); + % title(sprintf('%s frames: %03d Tracker: %d Layer: %d',param.season_name,length(param.cmd.frms), track_idx, layer_idx),'Interpreter', 'none'); + % %xlim([gps_time(1) gps_time(end)]); + % + % img_format = 'png'; + % if(options.season_vs_gps.fig_format) + % img_format = 'fig'; + % end + % testname = param.layer_tracker.layer_params.layerdata_source; + % img_fn = fullfile(img_dir,sprintf('%s_%s_season_gps_%03d_%03d.%s',testname,param.day_seg,track_idx,layer_idx,img_format)); + % ct_saveas(h_fig(1),img_fn); + % end + % end + % else + % for layer_idx = 1:length(gt_layer_params) + % h_fig(1) = figure; + % subplot_idx = 0; + % for track_idx = 1:length(frame_error{layer_idx}) + % subplot_idx = subplot_idx + 1; + % if(subplot_idx < 6) + % subplot(3,2,subplot_idx) + % plot(gps_time,abs(res_matrix{layer_idx}{track_idx})); + % ylabel('Absolute error (bins)'); + % xlabel('GPS time'); + % title(sprintf('%s frames: %03d Tracker: %d Layer: %d',param.season_name,length(param.cmd.frms), track_idx, layer_idx),'Interpreter', 'none'); + % %xlim([gps_time(1) gps_time(end)]); + % else + % subplot(3,2,subplot_idx) + % plot(gps_time,abs(res_matrix{layer_idx}{track_idx})); + % ylabel('Absolute error (bins)'); + % xlabel('GPS time'); + % title(sprintf('%s Tracker: %d Layer: %d',param.season_name, track_idx, layer_idx),'Interpreter', 'none'); + % %xlim([gps_time(1) gps_time(end)]); + % img_format = 'png'; + % if(options.season_vs_gps.fig_format) + % img_format = 'fig'; + % end + % testname = param.layer_tracker.layer_params.layerdata_source; + % img_fn = fullfile(img_dir,sprintf('%s_%s_season_gps_%03d_%03d.%s',testname,param.day_seg,track_idx,layer_idx,img_format)); + % ct_saveas(h_fig(1),img_fn); + % h_fig(1) = figure; + % subplot_idx = 0; + % end + % + % end + % end + % end + % + % end + save_name = sprintf('/cresis/snfs1/scratch/anjali/cluster_tuning/vit_fig1/result_layer_tune_lsm_s%d',param_idx); + save(save_name,'res_matrix_all','frame_error','range_per','percent_error','season_vect','res_matrix','points','min_val','gps_time'); + end + end +% fprintf('\n=====All segements=====\n'); +% ff{1} = load('/cresis/snfs1/scratch/anjali/cluster_tuning/lsm_all_figs2/result_layer_tune_lsm_s1'); +% ff{2} = load('/cresis/snfs1/scratch/anjali/cluster_tuning/lsm_all_figs2/result_layer_tune_lsm_s2'); +% +% for hist_idx = 1:length(frame_error{layer_idx}) +% range_per{1}{hist_idx} = []; +% res_matrix{1}{hist_idx} = []; +% end +% percent_error{layer_names} = zeros(1,(length(frame_error{layer_names}))); +% +% for i = 1:length(ff) +% foo{layer_names} = [foo{layer_names};ff.res_matrix_all{layer_names}]; +% percent_error{layer_names} = zeros(1,(length(param.layer_tracker.track))); +% +% percent_error{layer_names} = percent_error{layer_names} + ff.percent_error{layer_names}; +% for hist_idx = 1:length(frame_error{layer_idx}) +% range_per{1}{hist_idx} = cat(2,range_per{1}{hist_idx},ff.range_per{1}{hist_idx}); +% res_matrix{1}{hist_idx} = cat(2,res_matrix{1}{hist_idx},ff.res_matrix{1}{hist_idx}); +% end +% end + + + points = []; + min_val = []; + res_matrix_all_frms = []; + for layer_names = 1:num_layers + res_matrix_all_frms{layer_names} = nanmean(foo{layer_names},1); + res_matrix_all_frms{layer_names} = permute(res_matrix_all_frms{layer_names},idx_matrix); + [min_val{layer_names},i]=min(res_matrix_all_frms{layer_names}(:)); + sizeMatrix = size(res_matrix_all_frms{layer_names}); + for dim = 1:length(size(res_matrix_all_frms{layer_names})) + points{layer_names}(dim) = mod(floor((i-1)/prod(sizeMatrix(1:dim-1))),sizeMatrix(dim))+1; + end + if (length(size(res_matrix_all_frms{layer_names})) < length(param.layer_tracker.track{1}.idx_reshape)) + points{layer_names}(dim+1) = 1; + end + end + + if (options.percentage_correct.plot) + fprintf('\n=====Percentage correct all plots=====\n'); + %if (~options.percentage_correct.multiple_plots) + for layer_idx = 1:length(gt_layer_params) + h_fig(1) = figure('visible','off'); + for hist_idx = 1:length(frame_error{layer_idx}) + % hold on; + % plot(range_per{layer_idx}{hist_idx},'DisplayName',sprintf('tracker%d',hist_idx)); + % legend('-DynamicLegend','Location','northeastoutside'); + % xlabel('Number of frames'); + % ylabel('Percentage error'); + % set(gca,'XTick',1:length(param.cmd.frms)); + % set(gca,'XTickLabel',[param.cmd.frms]); + hold on; + plot(range_per{layer_idx}{hist_idx},'.-','DisplayName',sprintf('layer%d',hist_idx)); + xlabel('Frames'); + ylabel('Percentage correct (%)'); + xlim = ([1 length(param.cmd.frms)]); + title(sprintf('Percentage correct vs frames Layer:%d %s',layer_idx,param.season_name),'Interpreter', 'none'); + percent_error{layer_idx}(hist_idx) = percent_error{layer_idx}(hist_idx)/length(season_vect) * 100; + end + img_format = 'png'; + if(options.percentage_correct.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_percentage_correct_overall_%03d.%s',testname,param.season_name,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + end + + for layer_idx = 1:length(gt_layer_params) + h_fig(1) = figure('visible','off'); + plot(percent_error{layer_idx}); + hold on; + plot(percent_error{layer_idx}, 'r*'); + ylabel('Percentage correct (%)'); + xlabel('Trackers'); + title(sprintf('Percentage correct vs trackers Layer:%d %s',layer_idx,param.season_name),'Interpreter', 'none'); + set(gca,'XTick',1:length(frame_error{layer_idx})); + img_format = 'png'; + if(options.percentage_correct.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_total_percent_correct_%03d.%s',testname,param.season_name,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + end + %end + end + + if (options.hist_generation.plot) + fprintf('\n=====Hist generation all plots=====\n'); + %if (~options.hist_generation.multiple_plots) + for layer_idx = 1:length(gt_layer_params) + h_fig(1) = figure('visible','off'); + for hist_idx = 1:length(frame_error{layer_idx}) + hold on; + [N,x] = hist(abs(res_matrix{layer_idx}{hist_idx})); + plot(x,N,'DisplayName',sprintf('layer%d',hist_idx)); + %legend('-DynamicLegend','Location','northeastoutside'); + ylabel('Data points'); + xlabel('Absolute error (bins)'); + title(sprintf('Data points vs Absolute error Layer:%d %s',layer_idx,param.season_name),'Interpreter', 'none'); + end + hold off; + img_format = 'png'; + if(options.hist_generation.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_histogram_generation_overall_%03d.%s',testname,param.season_name,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + end + %end + end + + + if(options.season_vs_gps.plot) + fprintf('\n=====season plots=====\n'); + if(options.season_vs_gps.multiple_figs) + for layer_idx = 1:length(gt_layer_params) + for track_idx = 1:length(frame_error{layer_idx}) + h_fig(1) = figure('visible','off'); + plot(gps_time,abs(res_matrix{layer_idx}{track_idx})); + ylabel('Absolute error (bins)'); + xlabel('GPS time'); + title(sprintf('%s frames: %03d Tracker: %d Layer: %d',param.season_name,length(param.cmd.frms), track_idx, layer_idx),'Interpreter', 'none'); + %xlim([gps_time(1) gps_time(end)]); + + img_format = 'png'; + if(options.season_vs_gps.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_season_gps_%03d_%03d.%s',testname,param.season_name,track_idx,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + end + end + else + for layer_idx = 1:length(gt_layer_params) + h_fig(1) = figure('visible','off'); + subplot_idx = 0; + for track_idx = 1:length(frame_error{layer_idx}) + subplot_idx = subplot_idx + 1; + + if(subplot_idx < 6) + subplot(3,2,subplot_idx) + plot(gps_time,abs(res_matrix{layer_idx}{track_idx})); + ylabel('Absolute error (bins)'); + xlabel('GPS time'); + title(sprintf('%s frames: %03d Tracker: %d Layer: %d',param.season_name,length(param.cmd.frms), track_idx, layer_idx),'Interpreter', 'none'); + %xlim([gps_time(1) gps_time(end)]); + else + subplot(3,2,subplot_idx) + plot(gps_time,abs(res_matrix{layer_idx}{track_idx})); + ylabel('Absolute error (bins)'); + xlabel('GPS time'); + title(sprintf('%s Tracker: %d Layer: %d',param.season_name, track_idx, layer_idx),'Interpreter', 'none'); + %xlim([gps_time(1) gps_time(end)]); + img_format = 'png'; + if(options.season_vs_gps.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_season_gps_%03d_%03d.%s',testname,param.season_name,track_idx,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + h_fig(1) = figure('visible','off'); + subplot_idx = 0; + end + end + end + end + + end + + if(options.absolute_error.plot) + fprintf('\n=====Absolute error all plots=====\n'); + if(strcmpi(param.layer_tracker.track{1}.method,'viterbi')) + for layer_idx = 1:num_layers + res_mat = res_matrix_all_frms{layer_idx}; + points_new = points{layer_idx}; + x = 1:length(res_mat); % x tick + h_fig(1) = figure('visible','off'); + plot(x,res_mat); + hold on; + for i = 1:length(res_mat) + plot(x(i),res_mat(i), 'r*'); + end + plot(x(points_new(1)),res_mat(points_new(1)), 'ko','LineWidth',4); + set(gca,'XTick',x); + ylabel('Mean absolute error (bins)'); + xlabel('Trackers'); + title(sprintf('Absolute error frames: %03d Layer:%d %s',length(param.cmd.frms),layer_idx,param.season_name),'Interpreter', 'none'); + end + img_format = 'png'; + if(options.absolute_error.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + %img_fn = fullfile(img_dir,sprintf('%s_%s_season_gps_%03d_%03d.%s',testname,param.season_name,track_idx,layer_idx,img_format)); + + img_fn = fullfile(img_dir,sprintf('%03d_%s_%s_absolute_error_%03d.%s',2,testname,param.season_name,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + + elseif(strcmpi(param.layer_tracker.track{1}.method,'lsm')) + data = []; + for dim = 1:length(param.layer_tracker.track{1}.idx_reshape) + temp = []; + for track_idx = 1:length(param.layer_tracker.track) + if isempty(temp) + temp = param.layer_tracker.track{track_idx}.(param.layer_tracker.track{track_idx}.method).(param.layer_tracker.track{track_idx}.idx_dim_name{dim}); + else + if ~any(temp == param.layer_tracker.track{track_idx}.(param.layer_tracker.track{track_idx}.method).(param.layer_tracker.track{track_idx}.idx_dim_name{dim})) + temp(end+1) = param.layer_tracker.track{track_idx}.(param.layer_tracker.track{track_idx}.method).(param.layer_tracker.track{track_idx}.idx_dim_name{dim}); + end + end + + end + data{dim}=temp; + end + for layer_idx = 1:num_layers + res_mat = res_matrix_all_frms{layer_idx}; + temp_points = points{layer_idx}; + + A = nchoosek(1:length(param.layer_tracker.track{1}.idx_reshape),2); % changed from 2 + for id = 1:length(A) + min_points = setdiff(1:length(param.layer_tracker.track{1}.idx_reshape),A(id,:)); + min_idxs = temp_points(min_points); + res_matrix_tot = permute(res_mat,[A(id,:) setdiff(1:length(param.layer_tracker.track{1}.idx_reshape),A(id,:))]); + res_matrix_tot = res_matrix_tot(:,:,min_idxs(:)); + idx = A(id,:); + labelTickX = data{idx(1)}; % x tick label + labelTickY = data{idx(2)}; % y tick label + labelNameX = param.layer_tracker.track{1}.idx_dim_name{idx(1)}; % x label name + labelNameY = param.layer_tracker.track{1}.idx_dim_name{idx(2)}; % y label name + xTick = 1:length(labelTickX); % x tick + yTick = 1:length(labelTickY); % y tick + %% Generate Figure + h_fig(1) = figure('visible','off'); % figure is not displayed + imagesc(squeeze(res_matrix_tot')); + hold on; + plot(temp_points(idx(1)),temp_points(idx(2)),'x','LineWidth',4,'MarkerSize',10,'Color','white'); + + ylabel(labelNameY); + xlabel(labelNameX); + set(gca,'XTickLabel',labelTickX); + set(gca,'YTickLabel',labelTickY); + set(gca,'XTick',xTick); + set(gca,'YTick',yTick); + h_colorbar = colorbar(gca); + set(get(h_colorbar,'YLabel'),'String','Mean absolute error (rows)'); + img_format ='jpg'; + if(options.absolute_error.fig_format) + img_format = 'fig'; + end + testname = param.layer_tracker.layer_params.layerdata_source; + img_fn = fullfile(img_dir,sprintf('%s_%s_absolute_error_%s_vs_%s_%03d.%s',testname,param.season_name,labelNameX,labelNameY,layer_idx,img_format)); + ct_saveas(h_fig(1),img_fn); + + end + end + end + + + end + +end + +keyboard; +% +% %% Find Best Combination +% for layer_names = 1:idx +% res_matrix{layer_names} = zeros([length(param.cmd.frms),param.layer_tracker.track{1}.idx_reshape]); +% num_gt_isfinite.(sprintf('%s',gt_layer_params(layer_names).name)) = 0; +% num_isnan.(sprintf('%s',gt_layer_params(layer_names).name)) = 0; +% num_points.(sprintf('%s',gt_layer_params(layer_names).name)) = 0; +% num_layers = num_layers+1; +% end +% +% +% idx_matrix = [(1:length(param.layer_tracker.track{1}.idx_reshape))+1 1]; % used for resizing +% +% for track_idx = 1:length(param.layer_tracker.track) +% frm_idx = 0; +% for frm = param.cmd.frms +% frm_idx = frm_idx+1; +% data_fn_dir = ct_filename_out(param, param.layer_tracker.echogram_source, ''); +% data_fn_name = sprintf('Data_%s_%03d.mat',param.day_seg,frm); +% data_fn = fullfile(data_fn_dir, data_fn_name); +% data = load(data_fn); +% dt = data.Time(2) - data.Time(1); +% for layer_idx = 1:length(gt_layer_params) +% +% layer_params(1).name = sprintf('%s_%s_%s',param.layer_tracker.track{track_idx}.name,param.layer_tracker.track{track_idx}.method,gt_layer_params(layer_idx).name); +% layer_params(1).source = param.layer_tracker.layer_params.source; +% layer_params(1).layerdata_source = param.layer_tracker.layer_params.layerdata_source; +% layers_new = opsLoadLayers(param,layer_params); +% +% surf = interp1(layers(layer_idx).gps_time,layers(layer_idx).twtt,data.GPS_time); +% surf_bins = round(interp1(data.Time,1:length(data.Time),surf)); +% +% for pos = 1:length(layers_new) +% surf = interp1(layers_new(pos).gps_time,layers_new(pos).twtt,data.GPS_time); +% surf_bins_itr = round(interp1(data.Time,1:length(data.Time),surf)); +% num_gt_isfinite.(sprintf('%s',gt_layer_params(layer_idx).name)) = num_gt_isfinite.(sprintf('%s',gt_layer_params(layer_idx).name)) + sum(isfinite(surf_bins)); +% num_isnan.(sprintf('%s',gt_layer_params(layer_idx).name)) = num_isnan.(sprintf('%s',gt_layer_params(layer_idx).name)) + (sum(isfinite(surf_bins) & ~isfinite(surf_bins_itr))); +% num_points.(sprintf('%s',gt_layer_params(layer_idx).name)) = num_points.(sprintf('%s',gt_layer_params(layer_idx).name)) + (sum(abs(surf_bins-surf_bins_itr) < 5*dt)); +% res_matrix{layer_idx}(frm_idx,param.layer_tracker.track{track_idx}.idx(pos)) = nanmean(abs(surf_bins - surf_bins_itr)); +% counter = 0; +% for i = 1:length(surf_bins) +% if (abs(surf_bins(i) - surf_bins_itr(i)) <= range) +% counter = counter + 1; +% end +% end +% range_percent.(sprintf('%s_%03d', param.layer_tracker.track{track_idx}.name,frm))= (counter/length(surf_bins))*100; +% end +% end +% end +% counter = 0; +% for i = 1:length(param.cmd.frms) +% counter = counter + range_percent.(sprintf('%s_%03d', param.layer_tracker.track{track_idx}.name,frm)); +% end +% range_percent_all_frms.(sprintf('%s', param.layer_tracker.track{track_idx}.name)) = counter/length(param.cmd.frms); +% end +% +% points = []; +% min_val = []; +% res_matrix_all_frms = []; +% for layer_names = 1:num_layers +% res_matrix_all_frms{layer_names} = nanmean(res_matrix{layer_names},1); +% res_matrix_all_frms{layer_names} = permute(res_matrix_all_frms{layer_names},idx_matrix); +% [min_val{layer_names},i]=min(res_matrix_all_frms{layer_names}(:)); +% sizeMatrix = size(res_matrix_all_frms{layer_names}); +% for dim = 1:length(size(res_matrix_all_frms{layer_names})) +% points{layer_names}(dim) = mod(floor((i-1)/prod(sizeMatrix(1:dim-1))),sizeMatrix(dim))+1; +% end +% if (length(size(res_matrix_all_frms{layer_names})) < length(param.layer_tracker.track{1}.idx_reshape)) +% points{layer_names}(dim+1) = 1; +% end +% end +% +% file_version = '1'; +% file_type = 'layer_tracker_tuning'; +% +% fn_dir = fileparts(save_name); +% if ~exist(fn_dir,'dir') +% mkdir(fn_dir); +% end +% +% save(save_name,'res_matrix','range_percent','range_percent_all_frms','num_isnan','num_points','num_gt_isfinite','points','min_val','res_matrix_all_frms','param','file_version','file_type'); diff --git a/cresis-toolbox/tracker/tracker_lsm.m b/cresis-toolbox/tracker/tracker_lsm.m index 2ca48396..3ea075cf 100644 --- a/cresis-toolbox/tracker/tracker_lsm.m +++ b/cresis-toolbox/tracker/tracker_lsm.m @@ -1,135 +1,40 @@ -function labels = tracker_lsm(data,param) -%% Automated Section -% ===================================================================== -% Create param structure array -% ===================================================================== -labels = []; -result.top = []; -result.bot = []; -global gRadar -% params = merge_structs(params,gRadar); - % Load frames file -% load(ct_filename_support(param,'','frames')); -% -% if isempty(param.cmd.frms) -% param.cmd.frms = 1:length(frames.frame_idxs); -% end -% -% data_fn_dir = ct_filename_out(param, param.layer_tracker.layer_params.echogram_source, ''); -% - %for frm = param.frm_nums -% - %fprintf('\nLSM: Running frame %s_%03d (%s)\n',param.day_seg, frm, datestr(now,'HH:MM:SS')); - frm = param.layer_tracker.tracker.frm; - lsm_tic = tic; - %for idx = 1:length(param.layer_tracker.fname) - data_fn = param.layer_tracker.fname(1); -% -% try -% data = data_struct.(sprintf('data_%s_%03d',param.day_seg,frm)); -% catch ME -% fprintf('\nProblem with frame %s_%03d, verify.\n' ,param.day_seg,frm); -% continue; -% end -% -% data_fn_name = sprintf('Data_%s_%03d.mat',param.day_seg,frm); -% data_fn = fullfile(data_fn_dir, data_fn_name); - %imds = datastore(data_fn, 'Type','image','FileExtensions', '.mat'); - %imds = datastore({data}, 'Type','image','FileExtensions', '.mat'); - obj = tomo.LSMObject({lp(data.Data)}); - %obj = tomo.LSMObject(imds.Files); - obj.setLSMOptions('y', param.layer_tracker.track.lsm.y, 'dy', param.layer_tracker.track.lsm.dy, 'outerIter', param.layer_tracker.track.lsm.numOuterIter); - - [flag, Labels.top, Labels.bot] = obj.runLSM(); - -% if flag == 0 -% warning('LSM failed to converge to two layers after %d inner iterations. Trying again with %d inner iterations...', 2*param.layer_tracker.track.lsm.numOuterIter, 2*param.layer_tracker.track.lsm.maxOuterIter); -% obj = tomo.LSMObject(imds.Files); -% obj.setLSMOptions('y', 240, 'dy', 10, 'outerIter', param.layer_tracker.track.lsm.maxOuterIter); -% [flag, Labels.top, Labels.bot] = obj.runLSM(); -% fprintf('\nDone: second runLSM() call. '); -% end -% - Labels.top.y = interp1(Labels.top.x, Labels.top.y, 1:length(data.Bottom), 'linear', 'extrap'); - Labels.bot.y = interp1(Labels.bot.x, Labels.bot.y, 1:length(data.Bottom), 'linear', 'extrap')'; - - lsm_toc = toc(lsm_tic); - - if param.layer_tracker.track.debug - figure; imagesc(lp(data.Data)); colormap(1 - gray(256)); hold on; - plot(Labels.top.y, 'g'); plot(Labels.bot.y, 'r'); - legend('Ice-surface', 'Ice-bottom'); - keyboard - end +function labels = tracker_lsm(data,track) +% labels = tracker_lsm(data,track) +% +% data: echogram image +% +% labels: 2*N_iter by Nx array of layers. First N_iter layers are the +% surface and the second N_iter layers are the bottom. - if param.layer_tracker.track.ops_write - warning('off') - %% Set write options - % Load labels into OPS using opsCopyLayers - copy_param = []; - copy_param.layer_source.existence_check = false; - copy_param.layer_dest.existence_check = false; - - % Set the source - copy_param.layer_source.source = 'custom'; - copy_param.layer_dest.source = param.layer_tracker.track.layer_dest_source; - - if strcmp(copy_param.layer_dest.source, 'layerdata') - copy_param.layer_dest.layerdata_source = param.layer_tracker.track.layer_dest_layerdata_source; - copy_param.layer_dest.echogram_source = param.layer_tracker.track.layer_dest_echogram_source; - end - - copy_param.copy_method = 'overwrite'; - - copy_param.gaps_fill.method = 'preserve_gaps'; - copy_param.gaps_fill.method_args = [40 20]; - - param = merge_structs(param,gRadar); - - %% Write surface layer - % Interpolate from row number to TWTT - data.TWTT = interp1(1:length(data.Time), data.Time, Labels.top.y); - copy_param.layer_source.gps_time = data.GPS_time; - copy_param.layer_source.twtt = data.TWTT; - - % Load labels into OPS using opsCopyLayers - copy_param.layer_dest.name = param.layer_tracker.track.lsm.lyrtop; - fprintf('\nopsCopyLayers %s (%s) [TOP]', param.day_seg, datestr(now)); - opsCopyLayers(param,copy_param); - - %% Write bottom layer - % Interpolate from row number to TWTT - data.TWTT = interp1(1:length(data.Time), data.Time, Labels.bot.y); - copy_param.layer_source.gps_time = data.GPS_time; - copy_param.layer_source.twtt = data.TWTT; - - % Load labels into OPS using opsCopyLayers - copy_param.layer_dest.name = param.layer_tracker.track.lsm.lyrbot; - fprintf('\nopsCopyLayers %s (%s) [BOTTOM]', param.day_seg, datestr(now)); - opsCopyLayers(param,copy_param); - - fprintf('\n Complete (%s)\n', datestr(now)); - warning('on'); - end - %result.top = cat(2,result.top,Labels.top.y); - %result.bot = cat(1,result.bot,Labels.bot.y); - -% labels.(sprintf('layer_%s_%03d',param.day_seg,frm)).top = ... -% Labels.top.y; -% labels.(sprintf('layer_%s_%03d',param.day_seg,frm)).bot = ... -% Labels.bot.y; -% labels.(sprintf('layer_%s_%03d',param.day_seg,frm)).toc = ... -% lsm_toc; - %end -% if param.layer_tracker.track.debug -% figure; imagesc(lp(data.Data)); colormap(1 - gray(256)); hold on; -% plot(result.top, 'g'); plot(result.bot, 'r'); -% legend('Ice-surface', 'Ice-bottom'); -% keyboard -% end +% Create LSM object and pass in data +obj = tomo.LSMObject_tuning({data}); + +% Set parameters +obj.setLSMOptions('y', track.lsm.y, ... + 'dy', track.lsm.dy, ... + 'storeIter', track.lsm.storeIter, ... + 'outerIter', max(track.lsm.storeIter)); -labels(1,:) = Labels.top.y; -labels(2,:) = Labels.bot.y'; - - %end -end \ No newline at end of file +% Run LSM +[flag, Labels.top, Labels.bot, matrix_x, matrix_y] = obj.runLSM(); + +% Interpolate contour outputs to image columns +% Nx: Number of image columns +Nx = size(data,2); +% N_iter: Number of iterations stored by LSM +N_iter = length(track.lsm.storeIter); +% Preallocate output +labels = nan(N_iter,Nx); + +labels_idx = 0; +for layer_idx = 1:2 % [surface bottom] + for iter_idx = 1:N_iter % Stored iterations + try + labels_idx = labels_idx + 1; + labels(labels_idx,:) = interp1(matrix_x(layer_idx,:,iter_idx),matrix_y(layer_idx,:,iter_idx),1:Nx,'linear','extrap'); + catch ME + warning('LSM contour failed to produce a layer for layer %d iteration %d.', layer_idx, iter_idx); + continue + end + end +end diff --git a/cresis-toolbox/tracker/tracker_sim.m b/cresis-toolbox/tracker/tracker_sim.m index 053de7bb..2b17129a 100644 --- a/cresis-toolbox/tracker/tracker_sim.m +++ b/cresis-toolbox/tracker/tracker_sim.m @@ -15,7 +15,7 @@ signal_power_fh = @(x) 15*exp(-x/200)-20; noise_power = -5; mean_layer_fh = @(x) 75*exp(-x/1500); -var_layer_fh = @(x) 0.3*exp(-x/1000); +var_layer_fh = @(x) 60*exp(-x/1000); var_spread_fh = @(x) 8*exp(-x/1000); var_spread_gauss = 1; @@ -63,7 +63,7 @@ %plot(layers(layer_idx,:)); hold on; % Filter layer - tmp = filtfilt(B,A,tmp)*corr_len; + tmp = filtfilt(B,A,tmp)*sqrt(corr_len); layers(layer_idx,:) = tmp(1+2*corr_len : end - 2*corr_len); %plot(layers(layer_idx,:)); hold on; @@ -116,7 +116,7 @@ * var_spread(layer_idx,rline,spread_idx); end - Data(:,rline) = Data(:,rline) + signal_power * sinc((bins - layers(layer_idx,rline) - var_spread(layer_idx,rline,spread_idx))/sinc_width).^2; + Data(:,rline) = Data(:,rline) + signal_power * sinc((bins - layers(layer_idx,rline) - var_spread(layer_idx,rline,spread_idx))/sinc_width); %plot(lp(Data(:,rline))); hold on end end diff --git a/cresis-toolbox/tracker/tracker_sim_cluster.m b/cresis-toolbox/tracker/tracker_sim_cluster.m new file mode 100644 index 00000000..4526dfb2 --- /dev/null +++ b/cresis-toolbox/tracker/tracker_sim_cluster.m @@ -0,0 +1,58 @@ +%% tracker_sim_cluster + + +global gRadar; + +param = gRadar; + +ctrl = cluster_new_batch(param); +ctrl.cluster.type = 'debug'; +% ctrl.cluster.type = 'torque'; + + +% cluster_compile({'tracker_sim_task.m'},[],0,ctrl);% This would need input dependencies +cluster_compile({'tracker_sim_task.m'},ctrl.cluster.hidden_depend_funs,ctrl.cluster.force_compile,ctrl); + + +cpu_time = 10000; +mem = 10e9; +% find out mem_mult and incorporate here + +block = true; + + +% Create task +% Confirm sparam from others (tomo, analysis, qlook etc) +sparam.argsin{1} = param; +sparam.task_function = 'tracker_sim_task'; + +sparam.num_args_out = 1; +sparam.argsin{1}.tracker_sim.num_layers = 10; % Number of layers +sparam.argsin{1}.tracker_sim.output_dir = 'fixedsnr_randomlayers'; % This should be changed + +repeat_task = 1000; + +dparam = []; + +dparam.cpu_time = cpu_time ; % * sparam.argsin{1}.tracker_sim.num_layers; +dparam.mem = mem ; + +for tmp = 1:repeat_task +% update dparam.argsin{tmp} + dparam.argsin{1}.tracker_sim.num_layers = randi([5 12]); + dparam.argsin{1}.tracker_sim.img_idx = tmp; + + ctrl = cluster_new_task(ctrl,sparam,dparam,'dparam_save',0); + +end + +% cluster_save_dparam +ctrl = cluster_save_dparam(ctrl); + +fprintf('Submitting %s \n', ctrl.batch_dir); +ctrl_chain = {{ctrl}}; +ctrl_chain = cluster_run(ctrl_chain,block); +[in,out] = cluster_print(ctrl_chain{1}{1}.batch_id,1,0); +% cluster_cleanup(ctrl); + + diff --git a/cresis-toolbox/tracker/tracker_sim_task.m b/cresis-toolbox/tracker/tracker_sim_task.m new file mode 100644 index 00000000..bd472c31 --- /dev/null +++ b/cresis-toolbox/tracker/tracker_sim_task.m @@ -0,0 +1,240 @@ +function [success] = tracker_sim_task(param) + + + +fprintf('Task is starting (%s)\n', datestr(now)); + +% script tracker_sim +% +% Script for simulating broadband sounder data over the ice sheet (e.g. +% like snow radar or kuband radar). This is currently just a prototype +% script. +% +% Author: John Paden + + +Nt = 1000; % Number of fast time samples +num_incoh_ave = 11; % No of incoherent averages +Nx = 256 * num_incoh_ave; % No of slow-time samples + +surf_bin = 200; + +rng('shuffle') + +% Function handlers +signal_power_fh = @(x) 15*exp(-x/3000)-20; +noise_power = -5; +mean_layer_fh = @(x) 75*exp(-x/1500); +var_layer_fh = @(x) 0.3*exp(-x/1000); + +var_spread_fh = @(x) 8*exp(-x/1000); +var_spread_gauss = 1; +var_spread_exp = 2; +corr_len = 200; +corr_len_spread = 0*num_incoh_ave; + +sinc_width = 2; + +num_layers = param.tracker_sim.num_layers; +% num_layers = 25; + +layer_power = 1.5*randn(1,num_layers+1); +layer_power(1) = layer_power(1) + 3; +layer_power(2) = layer_power(2) + 1; + +% ======================= +tracker_sim_tstart = tic; + +bins = (1:Nt).'; +rlines = 1:Nx; + + +% Preallocate layers matrix. Layers matrix contains the range bin or row +% for each layer. +layers = zeros(num_layers,Nx); + +for layer_idx = 1:num_layers + + if layer_idx == 1 + mean_thickness = mean_layer_fh(zeros(1,Nx)); + var_thickness = var_layer_fh(zeros(1,Nx)); + else + mean_thickness = mean_layer_fh(layers(layer_idx-1,:)); + var_thickness = var_layer_fh(layers(layer_idx-1,:)); + end + + [B,A] = butter(2,1/corr_len); + + %layer_spread = sqrt(var_thickness/2) * randn(1,Nx).^2; + var_thickness = [ones(1,2*corr_len)*var_thickness(1) var_thickness ones(1,2*corr_len)*var_thickness(end)]; + tmp = sqrt(var_thickness) .* randn(1,Nx + 4*corr_len); + layers(layer_idx,:) = tmp(1+2*corr_len : end - 2*corr_len); + %plot(layers(layer_idx,:)); hold on; + + % Filter layer + tmp = filtfilt(B,A,tmp)*corr_len; + layers(layer_idx,:) = tmp(1+2*corr_len : end - 2*corr_len); + %plot(layers(layer_idx,:)); hold on; + + % Add in mean thickness + layers(layer_idx,:) = layers(layer_idx,:) + mean_thickness; + %plot(layers(layer_idx,:)); hold on; + + % Set negative thickness to zero thickness + layers(layer_idx,layers(layer_idx,:) < 0) = 0; + %plot(layers(layer_idx,:)); hold on; + + % Sum thickness to previous layer + if layer_idx > 1 + layers(layer_idx,:) = layers(layer_idx-1,:) + layers(layer_idx,:); + end + +% plot(layers(layer_idx,:),'LineWidth',2); hold on; +end + +fprintf('Exiting the first loop: to be deleted afterwards \n'); + +% Append surface layer +layers = [zeros(1,Nx); layers]; +% Add in surface bin offset to all layers +layers = layers + surf_bin; + +% ======================= +% Convolution (sidelobes) +% figure(2); clf; +Data = zeros(Nt,Nx); + +% [B,A] = butter(2,1/corr_len_spread); + +num_spread = 100; + + +var_spread = (var_spread_gauss*randn(num_layers+1,Nx+4*corr_len_spread,num_spread) + var_spread_exp*randn(num_layers+1,Nx+4*corr_len_spread,num_spread).^2); +% var_spread = permute(filtfilt(B,A,permute(var_spread,[2 1 3])),[2 1 3]); +var_spread = var_spread(:,1+2*corr_len_spread:end-2*corr_len_spread,:); + +% Add layer_power fading along-track here + +% Add cross-track clutter and winter storm layers here + +warning('Test %d \n', num_layers); + +for layer_idx = 1:num_layers+1 +warning('Second Loop begin (%d of %d) (%s)\n', layer_idx, num_layers, datestr(now)); +whos +fprintf('%d',num_spread); + for rline = 1:Nx + for spread_idx = 1:num_spread + signal_power = (randn(1)+1i*randn(1))/sqrt(2) * 10.^((layer_power(layer_idx) + signal_power_fh(layers(layer_idx,rline)-surf_bin))/20); + % Write a comment + + if layer_idx == 1 + var_spread(layer_idx,rline,spread_idx) = sqrt(var_spread_fh(0)) ... + * var_spread(layer_idx,rline,spread_idx); + else + var_spread(layer_idx,rline,spread_idx) = sqrt(var_spread_fh(layers(layer_idx-1,rline))) ... + * var_spread(layer_idx,rline,spread_idx); + end + + Data(:,rline) = Data(:,rline) + signal_power * sinc((bins - layers(layer_idx,rline) - var_spread(layer_idx,rline,spread_idx))/sinc_width).^2; % This gave a sinc^2 plot + %plot(lp(Data(:,rline))); hold on + + end + end + toc(tracker_sim_tstart); + warning('Second Loop ends %d \n', layer_idx); +end +warning('Test Complete'); + + +% ======================= +% Noise (noise variance) +% Data = abs(Data).^2 + sum((10.^(noise_power/20/2)*randn(Nt,Nx,num_incoh_ave)).^2,3)/num_incoh_ave; +Data = abs(Data).^2 + sum((10.^(noise_power/20/2)*randn(Nt,Nx,1)).^2,3)/1; +H = hanning(2*num_incoh_ave+1).'; +H = H / sum(H); +Data = fir_dec(Data,H,num_incoh_ave); +layers = fir_dec(layers,H,num_incoh_ave); +Nx = Nx / num_incoh_ave; + + +% Create raster image +raster = zeros(size(Data)); +layers2 = round(layers); + +for layer_idx = 1:size(layers2,1) + for layer_col = 1:size(layers2,2) + tmp = layers2(layer_idx,layer_col); % This idx represents the column of the layer + if raster(tmp,layer_col) == 0 + raster(tmp,layer_col) = layer_idx; + end + end +end + +raster = uint8(raster); + +%% Create directories if they don't exist +base_m = fullfile(param.tmp_path,param.tracker_sim.output_dir); + +if ~exist('base_m', 'dir') + mkdir(base_m); +end + +base_layer_bin = fullfile(base_m,'layer_bin'); +base_layer = fullfile(base_m,'layer'); +base_image = fullfile(base_m,'image'); + +if ~exist('base_layer_bin', 'dir') + mkdir(base_layer_bin); +end + +if ~exist('base_layer', 'dir') + mkdir(base_layer); +end + +if ~exist('base_image', 'dir') + mkdir(base_image); +end + + +%% Save data and plots + +% Save Data.png +Data2 = lp(Data); +max_data = max(Data2(:)); +min_data = min(Data2(:)); +% Create and save .png +new_data = uint8(255*((Data2 - min_data)/(max_data-min_data))); + +fn_data_png = fullfile(base_image,sprintf('image_%06d.png',param.tracker_sim.img_idx)); +fprintf('Saving the .mat files %s \n',datestr(now)); +imwrite(new_data,fn_data_png); + +% Saving Data .mat +fn_data_mat = fullfile(base_image,sprintf('image_%06d.mat',param.tracker_sim.img_idx)); +fprintf('Saving %s (%s)\n', fn_data_mat, datestr(now)); +save(fn_data_mat,'-v7.3','Data'); + +% Save layer data as .png +fn_layer_png = fullfile(base_layer,sprintf('layer_%06d.png',param.tracker_sim.img_idx)); +fprintf('Saving %s (%s)\n', fn_layer_png, datestr(now)); +imwrite(raster,fn_layer_png); + +% Save layer data as .mat +fn_layer_mat = fullfile(base_layer,sprintf('layer_%06d.mat',param.tracker_sim.img_idx)); +fprintf('Saving %s (%s)\n', fn_layer_mat, datestr(now)); +save(fn_layer_mat,'-v7.3','raster','layers2'); + +% Save layer_binary data as .png +fn_layer_png = fullfile(base_layer_bin,sprintf('layer_binary_%06d.png',param.tracker_sim.img_idx)); +fprintf('Saving %s (%s)\n', fn_layer_png, datestr(now)); +imwrite(logical(raster),fn_layer_png); + +toc(tracker_sim_tstart); + +%% Done +% ========================================================================= + +fprintf(' done %s\n', datestr(now)); + +success = true; diff --git a/cresis-toolbox/tracker/tracker_snake_simple.m b/cresis-toolbox/tracker/tracker_snake_simple.m index 7fc65f49..1f8b9d99 100644 --- a/cresis-toolbox/tracker/tracker_snake_simple.m +++ b/cresis-toolbox/tracker/tracker_snake_simple.m @@ -31,12 +31,15 @@ if ~exist('surf','var') surf = struct(); end -if ~isfield(surf,'min_bin') - surf.min_bin = 1; +if ~isfield(surf,'min_bin') || isempty(surf.min_bin) + surf.min_bin = ones(1,size(data,2)); end -if ~isfield(surf,'max_bin') || isempty(surf.max_bin) || ~isfinite(surf.max_bin) - surf.max_bin = size(data,1); +surf.min_bin(~isfinite(surf.min_bin)) = 1; +if ~isfield(surf,'max_bin') || isempty(surf.max_bin) + surf.max_bin = size(data,1)*ones(1,size(data,2)); end +surf.max_bin(~isfinite(surf.max_bin)) = size(data,1); + if ~isfield(surf,'search_rng') surf.search_rng = -1:1; end diff --git a/cresis-toolbox/tracker/tracker_threshold.m b/cresis-toolbox/tracker/tracker_threshold.m index 1cba7688..94668b18 100644 --- a/cresis-toolbox/tracker/tracker_threshold.m +++ b/cresis-toolbox/tracker/tracker_threshold.m @@ -63,7 +63,6 @@ end surf.dem = round(surf.dem); - %% Determine the threshold value using range bins specified by noise_rng median_data = nan(1,size(data,2)); for rline=1:size(data,2) @@ -124,7 +123,7 @@ % particular range line (this is useful if the noise estimates are noisy) % surf.threshold_rel_max: Threshold in dB below the maximum value in the % rbin range -if ~isempty(surf.threshold_noise_dB) && all(isnan(median_data)) && surf.threshold_rel_max == -inf +if isempty(surf.threshold_noise_dB) && all(isnan(median_data)) && surf.threshold_rel_max == -inf warning('Insufficient information to generate threshold since all values are zero.'); surface = surf.dem; return diff --git a/cresis-toolbox/tracker/tracker_viterbi.m b/cresis-toolbox/tracker/tracker_viterbi.m index 5363b84e..56fc21c9 100644 --- a/cresis-toolbox/tracker/tracker_viterbi.m +++ b/cresis-toolbox/tracker/tracker_viterbi.m @@ -1,490 +1,44 @@ -function labels_wholeseg = tracker_viterbi(data_struct,param) -% function viterbi_tracker_2D (params, options, data_struct) -% ICE MASK OPTION, layername(444), image_path(420) -% Builds a large matrix containing multiple segments of 2D data and applies -% the Viterbi layer-tracking program on it. -% Loads crossovers from OPS to use as ground truth for the tracking. -% Other functionality includes multiple supression and -% a simple detrending technique. +function labels = tracker_viterbi(data,track) +% labels = tracker_viterbi(data,track) % -%% Automated Section -% ===================================================================== -% Create param structure array -% ===================================================================== - -global gRadar; -param = merge_structs(param,gRadar); +gt = [track.crossovers track.ground_truths]; physical_constants; -%% Check parameters and set to default if needed -for layer_idx = 1:length(param.layer_tracker.cmds.layer_params) - if isfield(param.layer_tracker.track.viterbi, 'bottom_bin') && ~isempty(param.layer_tracker.track.viterbi.bottom_bin) - bottom_bin = param.layer_tracker.track.viterbi.bottom_bin; - else - bottom_bin = -1; - end - - if isfield(param.layer_tracker.track.viterbi, 'egt_weight') && ~isempty(param.layer_tracker.track.viterbi.egt_weight) - egt_weight = param.layer_tracker.track.viterbi.egt_weight; - else - egt_weight = -1; - end - - if isfield(param.layer_tracker.track.viterbi, 'ice_bin_thr') && ~isempty(param.layer_tracker.track.viterbi.ice_bin_thr) - ice_bin_thr = param.layer_tracker.track.viterbi.ice_bin_thr; - else - ice_bin_thr = 10; - end - - if isfield(param.layer_tracker.track.viterbi, 'mu_size') && ~isempty(param.layer_tracker.track.viterbi.mu_size) - mu_size = param.layer_tracker.track.viterbi.mu_size; - else - mu_size = 31; - end - - if isfield(param.layer_tracker.track.viterbi, 'mu_thr') && ~isempty(param.layer_tracker.track.viterbi.mu_thr) - mu_thr = param.layer_tracker.track.viterbi.mu_thr; - else - mu_thr = -30; - end - - if isfield(param.layer_tracker.track.viterbi, 'mu') && ~isempty(param.layer_tracker.track.viterbi.mu) - mu = param.layer_tracker.track.viterbi.mu; - else - mu = log10(exp(-(-(mu_size-1)/2 : (mu_size-1)/2).^4/1)); - mu(mu < mu_thr) = mu_thr; - mu = mu - mean(mu); - end - - if isfield(param.layer_tracker.track.viterbi, 'repulsion') && ~isempty(param.layer_tracker.track.viterbi.repulsion) - repulsion = param.layer_tracker.track.viterbi.repulsion; - else - repulsion = 150000; - end - - if isfield(param.layer_tracker.track.viterbi, 'sigma') && ~isempty(param.layer_tracker.track.viterbi.sigma) - sigma = param.layer_tracker.track.viterbi.sigma; - else - sigma = sum(abs(mu))/10*ones(1, mu_size); - end - - if isfield(param.layer_tracker.track.viterbi, 'smooth_var') && ~isempty(param.layer_tracker.track.viterbi.smooth_var) - smooth_var = param.layer_tracker.track.viterbi.smooth_var; - else - smooth_var = Inf; - end - - if isfield(param.layer_tracker.track.viterbi, 'smooth_weight') && ~isempty(param.layer_tracker.track.viterbi.smooth_weight) - smooth_weight = param.layer_tracker.track.viterbi.smooth_weight; - else - smooth_weight = 5; - end - - if isfield(param.layer_tracker.track.viterbi, 'surf_layer_params') && ~isempty(param.layer_tracker.track.viterbi.surf_layer_params) - surf_layer_params = param.layer_tracker.track.viterbi.surf_layer_params; - else - surf_layer_params = []; - end - - if ~isfield(surf_layer_params, 'name') || ~isempty(surf_layer_params.name) - surf_layer_params.name = 'surface'; - end -% keyboard; - %% Distance-to-Ice-Margin model - if ~isfield(param.layer_tracker.track.viterbi, 'DIM_matrix') || isempty(param.layer_tracker.track.viterbi.DIM_matrix) - prob.DIM_means = [6.908407 14.603709 22.077745 29.333980 36.375879 43.206908 49.830531 56.250214 62.469423 68.491622 74.320277 79.958853 85.410815 90.679629 95.768760 100.681673 105.421833 109.992706 114.397756 118.640450 122.724252 126.652627 130.429042 134.056960 137.539848 140.881171 144.084393 147.152981 150.090399 152.900113 155.585588 158.150289 160.597681 162.931230 165.154401 167.270659 169.283470 171.196299 173.012610 174.735870 176.369543 177.917095 179.381991 180.767696 182.077676 183.315395 184.484320 185.587915 186.629645 187.612977 188.541374 189.418303 190.247228 191.031615 191.774929 192.480636 193.152200 193.793087 194.406763 194.996691 195.566338 196.119170 196.658650 197.188245 197.711419 198.231639 198.752368 199.277073 199.809219 200.352271 200.909693 201.484953 202.081514 202.702842 203.352402 204.033660 204.750081 205.505130 206.302272 207.144972 208.036696 208.980910 209.981077 211.040664 212.163136 213.351958 214.610596 215.942514 217.351178 218.840053 220.412604 222.072297 223.822597 225.666969 227.608878 229.651790 231.799170 234.054483 236.421194 238.902770]; - prob.DIM_vars = [19.032807 25.055615 28.640839 32.753438 36.144273 39.329838 42.688930 45.511082 49.036818 52.177650 55.028380 57.516627 60.519048 63.217206 65.211203 67.459337 69.609678 71.543557 73.182822 74.615772 75.628159 77.127086 78.155483 79.447090 80.011376 81.108576 81.618789 82.287856 82.979740 83.561585 84.281769 84.648076 85.290095 85.566969 86.052342 86.487424 86.675812 86.959733 87.181337 87.641261 87.674246 87.947628 87.895269 87.286380 87.202972 86.878606 87.151259 87.477659 88.049960 88.587946 88.515276 89.070799 88.756636 88.345201 87.754785 87.689382 87.240118 86.800999 86.164340 86.085916 85.803664 85.356194 85.831974 85.264038 85.222428 84.898093 84.652262 84.332790 84.249144 83.871931 83.552786 83.233334 82.842279 82.658637 82.008042 81.694151 81.421515 80.901673 80.885452 81.070003 80.524210 80.776716 80.320438 80.445820 80.085639 79.751146 79.557559 78.923447 78.522063 77.525973 77.426494 76.624448 76.855826 77.277564 76.777165 76.716292 75.970217 77.149291 76.900846 76.890210]; - gauss = @(x, mean, var)((1 / (sqrt(2 * pi * (var.^2)))) * (exp(-((x - mean).^2)/(2 * var.^ 2)))); - costm = ones(50, 100); - for t = 1:50 - for i = 1:100 - costm(t,i) = 10 * exp(-0.01 .* t) - 10 * exp(-0.01 * 50); - end - end - alpha = 0.025; - beta = 0.05; - - DIM_costmatrix = ones(1000, 100); - for DIM = 1 : 100; - for T = 1 : 1000 - if alpha / gauss(T, prob.DIM_means(DIM), DIM * beta * prob.DIM_vars(DIM)) >= 200 - DIM_costmatrix(T, DIM) = 200; - else - DIM_costmatrix(T, DIM) = alpha/ gauss(T, prob.DIM_means(DIM), DIM * beta * prob.DIM_vars(DIM)); - end - end - end - - DIM_costmatrix = DIM_costmatrix(15:end,:); - - for k = 1:100 - DIM_costmatrix(1:50, k) = DIM_costmatrix(1:50, k) + k * costm(1:50, k); - end - else - clear DIM DIM_costmatrix; - % - % DIM = load(fullfile(gRadar.path, param.layer_tracker.tracker.viterbi.DIM_matrix)); - DIM = load(fullfile(gRadar.path, param.layer_tracker.track.viterbi.DIM_matrix)); - DIM_costmatrix = DIM.Layer_tracking_2D_parameters; - DIM_costmatrix = DIM_costmatrix .* (200 ./ max(DIM_costmatrix(:))); - end - - labels = {}; - big_matrix = {}; - big_matrix.Data = []; - big_matrix.GPS_time = []; - big_matrix.Lat = []; - big_matrix.Lon = []; - good_frms = param.cmd.frms; - - try - Surface = opsLoadLayers(param,surf_layer_params); - catch ME - warning(ME.getReport); - keyboard - end - - %for frm = param.frm_nums - frm = param.layer_tracker.tracker.frm; - if param.layer_tracker.track.debug - fprintf('\nRunning frame %s_%03d.\n',param.day_seg,frm); - end - % -% try -% cur_matrix = mdata(frm); -% catch ME -% cur_matrix = mdata; -% end +transition_weight = track.viterbi.transition_weight; + +along_track_slope = zeros(1, size(data, 2) - 1); + +upper_bounds = track.min_bin; +lower_bounds = track.max_bin; +upper_bounds(gt(1, :)) = gt(2, :) - gt(3, :); +lower_bounds(gt(1, :)) = gt(2, :) + gt(3, :); + +upper_bounds = round(upper_bounds); +lower_bounds = round(lower_bounds); +lower_bounds(~track.ice_mask) = round(track.dem(~track.ice_mask)); +upper_bounds(~track.ice_mask) = round(track.dem(~track.ice_mask)); +upper_bounds(upper_bounds < 1 | ~isfinite(upper_bounds)) = 1; +upper_bounds(upper_bounds > size(data, 1)) = 1; +lower_bounds(lower_bounds > size(data, 1) | ~isfinite(lower_bounds)) = size(data, 1); +lower_bounds(lower_bounds < 1) = 1; + +% Set nonfinite values to a very low value. If this is not done and the +% bounds force the path through an area with nonfinite values, viterbi fails +min_data = min(data(:)); +max_data = max(data(:)); +data(~isfinite(data)) = min_data - 100*(max_data-min_data); - cur_matrix = data_struct; - - big_matrix.GPS_time = horzcat(big_matrix.GPS_time, cur_matrix.GPS_time); - big_matrix.Lat = horzcat(big_matrix.Lat, cur_matrix.Latitude); - big_matrix.Lon = horzcat(big_matrix.Lon, cur_matrix.Longitude); - big_matrix.Time = cur_matrix.Time; - - %% Image combine - if ~param.layer_tracker.track.viterbi.custom_combine - big_matrix.Data = horzcat(big_matrix.Data, cur_matrix.Data);%horzcat(big_matrix.Data, data_struct); % horzcat(big_matrix.Data, cur_matrix.Data); - else - try - mode = 'get_heights'; - % param.(mode).out_path = param.layer_tracker.track.name; - param.(mode).out_path = param.layer_tracker.cmds.layer_params(layer_idx).name; - param.load.frm = frm; - param.(mode).img_comb = param.combine.img_comb; - param.(mode).img_comb_mult = 1.15; - param.(mode).img_comb_bins = 50; - [Data, big_matrix.Time] = img_combine(param, mode, Surface); - big_matrix.Data = horzcat(big_matrix.Data, Data); - catch ME - if param.layer_tracker.track.debug - fprintf('\nProblem during custom image combining'); - end - big_matrix.Data = horzcat(big_matrix.Data, cur_matrix.Data);%horzcat(big_matrix.Data, data_struct); % horzcat(big_matrix.Data, cur_matrix.Data); - end - end - %end%frm for loop - - if param.layer_tracker.track.debug && param.layer_tracker.track.viterbi.custom_combine - fprintf('\nDone: image combine (%s)', datestr(now,'HH:MM:SS')); - end - - % Catch error with layer not having correctly loaded from OPS - if(isempty(Surface.gps_time) || any(isnan(Surface.gps_time)) || any(isnan(Surface.twtt))) - fprintf('\nProblem processing frame %s', param.day_seg); - return; - end - - if param.layer_tracker.track.debug - fprintf('\nDone: opsLoadLayers (%s)', datestr(now,'HH:MM:SS')); - end - - %% - - big_matrix.Surface = interp_finite(interp1(Surface.gps_time,Surface.twtt,big_matrix.GPS_time)); - - if param.layer_tracker.track.debug - fprintf('\nDone: opsAuthenticate, opsGetSegmentInfo, time-range bin interpolation (%s)', datestr(now,'HH:MM:SS')); - end - - big_matrix.Data = lp(big_matrix.Data); %(needed when not using data_struct but instead using mdata) - surf_bins = round(interp1(big_matrix.Time, 1:length(big_matrix.Time), big_matrix.Surface)); - - %% Top suppression - if param.layer_tracker.track.viterbi.top_sup - topbuffer = 5; - botbuffer = 15; - filtvalue = 50; - for rline = 1 : size(big_matrix.Data, 2) - column_chunk = big_matrix.Data(round(surf_bins(rline) - topbuffer) : ... - round(surf_bins(rline) + botbuffer), rline); - big_matrix.Data(round(surf_bins(rline) - topbuffer) : ... - round(surf_bins(rline) + botbuffer), rline) = imgaussfilt(column_chunk, filtvalue); - end - - if param.layer_tracker.track.debug - fprintf('\nDone: top suppression (%s)', datestr(now,'HH:MM:SS')); - end - end - - %% Multiple suppression - if param.layer_tracker.track.viterbi.mult_sup - topbuffer = 12; - botbuffer = 12; - filtvalue = 30; - - % Step one: Determine the bins associated with the surface multiple - surf_mult_twtt = 2*big_matrix.Surface; - surf_mult_bins = round(interp1(big_matrix.Time, 1:length(big_matrix.Time), surf_mult_twtt)); - - % Step two: Filter the whole image - filt_img = fir_dec(fir_dec(big_matrix.Data,ones(1,31),1).',ones(1,31),1).'; - - % Step three: replace the surface multiple bins with the filtered image - for rline = 1 : size(big_matrix.Data, 2) - try - big_matrix.Data(round(surf_mult_bins(rline) - topbuffer) : ... - round(surf_mult_bins(rline) + botbuffer), rline) = ... - filt_img(round(surf_mult_bins(rline) - topbuffer) : ... - round(surf_mult_bins(rline) + botbuffer), rline); - catch ME - continue; - end - end - end - - Nx = size(big_matrix.Data, 2); - slope = round(diff(big_matrix.Surface)); - viterbi_weight = ones([1 Nx]); - - %% Ice mask calculation - if param.layer_tracker.track.binary_icemask - mask = load(param.layer_tracker.track.ice_mask_mat_fn,'R','X','Y','proj'); - [fid,msg] = fopen(param.layer_tracker.track.icemask_fn,'r'); - if fid < 1 - fprintf('Could not open file %s\n', param.layer_tracker.track.ice_mask_bin_fn); - error(msg); - end - mask.maskmask = logical(fread(fid,[length(mask.Y),length(mask.X)],'uint8')); - fclose(fid); - else - [mask.maskmask,mask.R,~] = geotiffread(param.layer_tracker.track.icemask_fn); - mask.proj = geotiffinfo(param.layer_tracker.track.icemask_fn); - end - [mask.x, mask.y] = projfwd(mask.proj, big_matrix.Lat, big_matrix.Lon); - mask.X = mask.R(3,1) + mask.R(2,1)*(1:size(mask.maskmask,2)); - mask.Y = mask.R(3,2) + mask.R(1,2)*(1:size(mask.maskmask,1)); - [mask.X,mask.Y] = meshgrid(mask.X,mask.Y); - ice_mask.mask = round(interp2(mask.X, mask.Y, double(mask.maskmask), mask.x, mask.y)); - ice_mask.mask(isnan(ice_mask.mask)) = 1; - - if param.layer_tracker.track.debug - fprintf('\nDone: ice mask calculation (%s)', datestr(now,'HH:MM:SS')); - end - - %% Crossover loading - if param.layer_tracker.track.viterbi.crossoverload - opsAuthenticate(param,false); - layer_name = 'bottom'; - sys = ct_output_dir(param.radar_name); - ops_param = struct('properties',[]); - ops_param.properties.season = param.season_name; - ops_param.properties.segment = param.day_seg; - [~,ops_frames] = opsGetSegmentInfo(sys,ops_param); - - ops_param = struct('properties',[]); - ops_param.properties.location = param.post.ops.location; - ops_param.properties.season = param.season_name; - ops_param.properties.start_gps_time = ops_frames.properties.start_gps_time(1); - ops_param.properties.stop_gps_time = ops_frames.properties.stop_gps_time(end); - ops_param.properties.nativeGeom = true; - [~,ops_data] = opsGetPath(sys,ops_param); - - query = sprintf('SELECT rds_segments.id FROM rds_seasons,rds_segments where rds_seasons.name=''%s'' and rds_seasons.id=rds_segments.season_id and rds_segments.name=''%s''',param.season_name,param.day_seg); - query - [~,tables] = opsQuery(query); - segment_id = tables{1}; - - ops_param = struct('properties',[]); - ops_param.properties.location = param.post.ops.location; - ops_param.properties.lyr_name = layer_name; - ops_param.properties.frame = ops_frames.properties.frame; - ops_param.properties.segment_id = ones(size(ops_param.properties.frame)) ... - *double(segment_id); - - [~,data] = opsGetCrossovers(sys,ops_param); - - %% Load and align crossovers - rline = []; - rows = []; - cols = []; - gps_time = []; - season_name = {}; - for i = 1 : length(data.properties.source_point_path_id) - if ~isnan(data.properties.twtt(i)) - new_rline = find(ops_data.properties.id == data.properties.source_point_path_id(i)); - new_gps_time = ops_data.properties.gps_time(new_rline); - new_season_name = data.properties.season_name{i}; - if big_matrix.GPS_time(1) <= new_gps_time ... - && big_matrix.GPS_time(end) >= new_gps_time %... - %&& str2double(new_season_name(1:4)) >= 2006 - rline(end+1) = new_rline; - gps_time(end+1) = new_gps_time; - season_name{end+1} = new_season_name; - [~, cols(end+1)] = min(abs(big_matrix.GPS_time - ops_data.properties.gps_time(rline(end)))); - twtt = data.properties.twtt(i); - twtt = twtt + (data.properties.source_elev(i)-data.properties.cross_elev(i))/(c/2); - rows(end+1) = round(interp1(big_matrix.Time, 1:length(big_matrix.Time), twtt)); - end - end - end - gt = [cols(:).'; rows(:).']; - if param.layer_tracker.track.debug - fprintf('\nDone: opsGetCrossovers (%s)', datestr(now,'HH:MM:SS')); - end - else - gt = ''; - end - - %% Set variable echogram tracking parameters - bounds = [0 Nx]; - ice_mask.mask_dist = round(bwdist(ice_mask.mask == 0)); - transition_mu = -1 * ones(1, size(big_matrix.Data, 2)); - transition_sigma = 0.3759 * ones(1, size(big_matrix.Data, 2)); - - %% Detrending routine -% if param.layer_tracker.track.viterbi.detrending -% detect_data = big_matrix.Data; -% % Along track filtering -% detect_data = fir_dec(detect_data,ones(1,5)/5,1); -% % Estimate noise level -% noise_value = mean(mean(detect_data(end-80:end-60,:))); -% % Estimate trend -% trend = mean(detect_data,2); -% trend(trend num_rec_in_file + param.recs(2) = num_rec_in_file - param.recs(1); +end + +%% Read data + +% Seek to the first record to read in +fseek(fid,rec_size*param.recs(1),-1); + +% Preallocate memory +Nc = 4; +data = cell(Nc,1); +hdr = cell(Nc,1); +rec = zeros(Nc,1); +for chan = 1:Nc + data{chan} = zeros(nsamp,param.recs(2)); + hdr{chan}.offset = zeros(1,param.recs(2)); + hdr{chan}.nsamp = zeros(1,param.recs(2)); + hdr{chan}.choff = zeros(1,param.recs(2)); + hdr{chan}.tscount = zeros(1,param.recs(2)); + hdr{chan}.nchan = zeros(1,param.recs(2)); + hdr{chan}.vr0 = zeros(1,param.recs(2)); + hdr{chan}.vr1 = zeros(1,param.recs(2)); + hdr{chan}.ver = zeros(1,param.recs(2)); + hdr{chan}.resvd = zeros(1,param.recs(2)); + hdr{chan}.absix = zeros(1,param.recs(2)); + hdr{chan}.relix = zeros(1,param.recs(2)); + hdr{chan}.xinc = zeros(1,param.recs(2)); + hdr{chan}.rseq = zeros(1,param.recs(2)); + hdr{chan}.scount = zeros(1,param.recs(2)); + hdr{chan}.rtime = cell(1,param.recs(2)); + hdr{chan}.sequence_number = zeros(1,param.recs(2)); + hdr{chan}.ct_time = zeros(1,param.recs(2)); + hdr{chan}.ct_clk = zeros(1,param.recs(2)); +end + +% Read in each record +while any(rec < param.recs(2)) + + if ~param.bxds_en + % 68 byte CX header + % ----------------------------------------------------------------------- + % HEADER[0:47] + choff = 0; + hdr{choff+1}.marker = char(fread(fid,8,'char').'); + hdr{choff+1}.project = char(fread(fid,8,'char').'); + hdr{choff+1}.set = char(fread(fid,8,'char').'); + hdr{choff+1}.transect = char(fread(fid,8,'char').'); + hdr{choff+1}.stream_name = char(fread(fid,8,'char').'); + sequence_number = fread(fid,1,'uint32',0,'ieee-le'); + rec_length = fread(fid,1,'uint32',0,'ieee-le')+48; % Record length in bytes including the 68 byte CX header + % CT[48:67] + ct_clk_packed = fread(fid,1,'uint32',0,'ieee-le'); + year = 1000*mod(floor(ct_clk_packed/2^4),2^4) + 100*mod(ct_clk_packed,2^4) ... + + 10*mod(floor(ct_clk_packed/2^12),2^4) + mod(floor(ct_clk_packed/2^8),2^4); + month = 10*mod(floor(ct_clk_packed/2^20),2^4) + mod(floor(ct_clk_packed/2^16),2^4); + day= 10*mod(floor(ct_clk_packed/2^28),2^4) + mod(floor(ct_clk_packed/2^24),2^4); + ct_clk_packed = fread(fid,1,'uint32',0,'ieee-le'); + hour = 10*mod(floor(ct_clk_packed/2^4),2^4) + mod(ct_clk_packed,2^4); + min = 10*mod(floor(ct_clk_packed/2^12),2^4) + mod(floor(ct_clk_packed/2^8),2^4); + sec = 10*mod(floor(ct_clk_packed/2^20),2^4) + mod(floor(ct_clk_packed/2^16),2^4); + frac = 10*mod(floor(ct_clk_packed/2^28),2^4) + mod(floor(ct_clk_packed/2^24),2^4); + ct_clk = datenum([year,month,day,hour,min,sec]); + ct_time = fread(fid,1,'uint32',0,'ieee-le'); + % ct_clk is monotonically increasing by 511.76 from 197,224,397 + % ct_time is in seconds and is monotonically increasing by 5120 us + % Last 8 bytes of CT are reserved/zero + fseek(fid,8,0); + end + + % PAYLOAD[68:...] + % ----------------------------------------------------------------------- + if fread(fid,1,'uint16') ~= 3200 + error('Bad record'); + end + fseek(fid,3,0); + choff = fread(fid,1,'uint8'); + choff = bitand(choff,bin2dec('00011111')); + if rec(choff+1) >= param.recs(2) + fseek(fid,-6 + header_rec_size+data_rec_size,0); + continue + end + rec(choff+1) = rec(choff+1) + 1; + rec(choff+2) = rec(choff+2) + 1; + fseek(fid,-6,0); + + if ~param.bxds_en + hdr{choff+1}.sequence_number(rec(choff+1)) = sequence_number; + hdr{choff+1}.ct_clk(rec(choff+1)) = ct_clk; + hdr{choff+1}.ct_time(rec(choff+1)) = ct_time; + end + + % XDS header + % ----------------------------------------------------------------------- + % Bytes: 0-1 + hdr{choff+1}.nsamp(rec(choff+1)) = fread(fid,1,'uint16'); + % Bytes: 2 + hdr{choff+1}.nchan(rec(choff+1)) = fread(fid,1,'uint8'); + % Bytes: 3 + hdr{choff+1}.vr0(rec(choff+1)) = fread(fid,1,'uint8'); + % Bytes: 4 + hdr{choff+1}.vr1(rec(choff+1)) = fread(fid,1,'uint8'); + % Bytes: 5 + hdr{choff+1}.choff(rec(choff+1)) = fread(fid,1,'uint8'); + % Bytes: 6 + hdr{choff+1}.ver(rec(choff+1)) = fread(fid,1,'uint8'); + % Bytes: 7 + hdr{choff+1}.resvd(rec(choff+1)) = fread(fid,1,'uint8'); + % Bytes: 8 + hdr{choff+1}.absix(rec(choff+1)) = fread(fid,1,'double'); + % Bytes: 16 + hdr{choff+1}.relix(rec(choff+1)) = fread(fid,1,'double'); + % Bytes: 24 + hdr{choff+1}.xinc(rec(choff+1)) = fread(fid,1,'single'); + % Bytes: 28 + hdr{choff+1}.rseq(rec(choff+1)) = fread(fid,1,'uint32'); + % Bytes: 32 + hdr{choff+1}.scount(rec(choff+1)) = fread(fid,1,'uint16'); + % Bytes: 34 + hdr{choff+1}.tscount(rec(choff+1)) = fread(fid,1,'uint16'); + % Bytes: 36 + hdr{choff+1}.rtime{rec(choff+1)} = fread(fid,tscount,'double'); + % Odd stuff + fread(fid,17,'uint16'); + + hdr{choff+1}.offset(rec(choff+1)) = ftell(fid); + + data{choff+1}(:,rec(choff+1)) = fread(fid,hdr{choff+1}.nsamp(rec(choff+1)),'int16'); + data{choff+2}(:,rec(choff+2)) = fread(fid,hdr{choff+1}.nsamp(rec(choff+1)),'int16'); + + if 0 + figure(1); clf; + plot(data{choff+1}(:,rec(choff+1))); + hold on + plot(data{choff+1}(:,rec(choff+1))); + ylim([-1000 1000]); + keyboard + end +end + +fclose(fid); diff --git a/cresis-toolbox/utig/fname_info_utig.m b/cresis-toolbox/utig/fname_info_utig.m new file mode 100644 index 00000000..2f003b61 --- /dev/null +++ b/cresis-toolbox/utig/fname_info_utig.m @@ -0,0 +1,37 @@ +function fname = fname_info_utig(fn) +% fname = fname_info_utig(fn) +% +% Detects the type of utig filename and then parses it. +% +% fn: string containing utig radar file name +% +% fname: structure containing each of the fields of the UTIG filename with +% format RADARNAME_DATETIME-FILENUMBER.dat. For example: +% radar0_20230116-200145-0032.dat. +% +% .radar_name: RADARNAME field 'radar0' +% +% .datenum: YYYYMMDD-HHMMSS time stamp converted to Matlab date number. +% Common usage: "[year month day hour min sec] = datevec(fname.datenum)" +% +% .file_idx: FILENUMBER field 32 +% +% Example +% fn = 'data/UTIG/orig/xped/CXA1/acqn/MARFA/F13/radar0_20230116-200145-0032.dat'; +% fname = fname_info_utig(fn) +% +% Author: John Paden +% +% See also datenum + +[fn_dir,fn_name,fn_ext] = fileparts(fn); + +fname = []; + +[fname.radar_name fn_name] = strtok(fn_name,'_'); + +[date_str fn_name] = strtok(fn_name,'-'); +[time_str fn_name] = strtok(fn_name,'-'); +fname.datenum = datenum([date_str(2:end) time_str],'yyyymmddHHMMSS'); + +fname.file_idx = str2double(fn_name(2:end)); diff --git a/cresis-toolbox/utig/run_basic_load_utig.m b/cresis-toolbox/utig/run_basic_load_utig.m new file mode 100644 index 00000000..6dd64d2b --- /dev/null +++ b/cresis-toolbox/utig/run_basic_load_utig.m @@ -0,0 +1,72 @@ +fn = '/mnt/raid_ssdorange/UTIG/X53a_first_flight_clean/RADnh5/bxds.1000'; +[hdr,data] = basic_load_utig(fn,struct('bxds_en',true)); + +% 50 MHz +% 52.5-67.5 MHz +% 2.5-17.5 MHz +fs = 50e6; +Nt = 3200; +dt = 1/fs; +df = fs/Nt; +time = dt*(0:Nt-1).'; +freq = df*(-floor(Nt/2):floor((Nt-1)/2)); + +Nt_512 = 512; +df_512 = fs/Nt_512; +freq_512 = df_512*(-floor(Nt_512/2):floor((Nt_512-1)/2)); + +dt_x = 32*160e-6; +Nx = param.recs(2); +time_x = dt_x*(0:Nx-1); +BW_x = 1/dt_x; +df_x = BW_x/Nx; +freq_x = df_x*(-floor(Nx/2):floor((Nx-1)/2)); + +% figure(1); clf; +for chan = 1:4 + if 0 + h_fig = figure(chan); clf; + set(h_fig,'WindowStyle','docked'); + else + subplot(2,2,chan); + end + if 0 + imagesc(db(data{chan} - mean(data{chan},2))); + elseif 0 + % range-doppler + imagesc(db(fft(data{chan},[],2))); + elseif 0 + % frequency-space + imagesc([],freq_512/1e6,fftshift(db(fft(data{chan}(2500+(1:Nt_512),:))),1)); + elseif 1 + % frequency-space PSD + plot(freq_512/1e6, db(mean(abs(fft(data{chan}(2500+(1:Nt_512),:),[],1)).^2,2),'power')); + grid on; + xlabel('Frequency (MHz)') + ylabel('Relative power (dB)') + elseif 0 + % range-doppler noise/late-record + imagesc(fftshift(db(fft(data{chan}(2500+(1:Nt_512),:),[],2)),1)); + elseif 1 + % range-doppler noise/late-record PSD + plot(freq_x, fftshift(db(mean(abs(fft(data{chan}(2500+(1:Nt_512),:),[],2).^2),1),'power'),2)); + grid on; + xlabel('Spatial frequency (Hz)') + ylabel('Relative power (dB)') + elseif 1 + % f-k + imagesc(freq_x,freq_512/1e6,fftshift(db(fft2(data{chan}(2500+(1:Nt_512),:))))); + xlabel('Spatial frequency (Hz)') + ylabel('Frequency (MHz)') + end + if chan == 1 || chan == 3 + title(sprintf('Chan %d (low gain)',chan-1)) + else + title(sprintf('Chan %d (high gain)',chan-1)) + end + colormap(1-gray(256)) + caxis([0 150]) + hold on; +end +link_figures; +xlim([-25 25]); diff --git a/cresis-toolbox/utility/cat_structs.m b/cresis-toolbox/utility/cat_structs.m index 2ff1d418..9e6c9e2d 100644 --- a/cresis-toolbox/utility/cat_structs.m +++ b/cresis-toolbox/utility/cat_structs.m @@ -1,5 +1,5 @@ -function S_out = cat_structs(dim,S1,S2) -% S_out = cat_structs(dim,S1,S2) +function S_out = cat_structs(dim,S1,S2,cat_fields_flag) +% S_out = cat_structs(dim,S1,S2,cat_fields_flag) % % Concatenates two structures. Allows for dissimilar fields. Fields that do % not exist just get a [] put in them. @@ -40,6 +40,10 @@ % Input check +if ~exist('cat_fields_flag','var') + cat_fields_flag = false; +end + % Handle simple cases if isempty(S2) S_out = S1; @@ -68,4 +72,16 @@ S2(1).(missfields{idx}) = []; end -S_out = cat(dim,S1,S2); +if ~cat_fields_flag + S_out = cat(dim,S1,S2); +else + for idx = 1:length(S_out_fields) + try + S_out.(S_out_fields{idx}) = cat(dim,S1.(S_out_fields{idx}),S2.(S_out_fields{idx})); + catch + try + S_out.(S_out_fields{idx}) = S1.(S_out_fields{idx}); + end + end + end +end diff --git a/cresis-toolbox/utility/clip_vectors.m b/cresis-toolbox/utility/clip_vectors.m index 91164e81..7db2f440 100644 --- a/cresis-toolbox/utility/clip_vectors.m +++ b/cresis-toolbox/utility/clip_vectors.m @@ -27,8 +27,11 @@ old_ylims % Function handle hooks for customizing clip_vectors + fh_close_win fh_button_up + fh_button_motion fh_key_press + user_data % tool_list: structure array of tools % .event_key: string containing the matching event.Key, @@ -71,12 +74,21 @@ if ~isfield(param,'bad_val') param.bad_val = NaN; end + if ~isfield(param,'fh_close_win') + param.fh_close_win = []; + end + if ~isfield(param,'fh_button_motion') + param.fh_button_motion = []; + end if ~isfield(param,'fh_button_up') param.fh_button_up = []; end if ~isfield(param,'fh_key_press') param.fh_key_press = []; end + if ~isfield(param,'user_data') + param.user_data = []; + end if ~isfield(param,'tool_list') obj.tool_list = []; tool_idx = 0; @@ -106,8 +118,11 @@ % Set parameters obj.h_plots = h_plots; obj.bad_val = param.bad_val; + obj.fh_close_win = param.fh_close_win; obj.fh_button_up = param.fh_button_up; + obj.fh_button_motion = param.fh_button_motion; obj.fh_key_press = param.fh_key_press; + obj.user_data = param.user_data; obj.actions.getting_polygon = 0; % Get the parent axes and parent figure @@ -142,7 +157,10 @@ end % Set figure call back functions + set(obj.h_fig,'CloseRequestFcn',@obj.close_win); + set(obj.h_fig,'WindowButtonUpFcn',@obj.button_up); set(obj.h_fig,'WindowButtonUpFcn',@obj.button_up); + set(obj.h_fig,'WindowButtonMotionFcn',@obj.button_motion); set(obj.h_fig,'WindowButtonDownFcn',@obj.button_down); set(obj.h_fig,'WindowScrollWheelFcn',@obj.button_scroll); set(obj.h_fig,'WindowKeyPressFcn',@obj.key_press); @@ -163,6 +181,20 @@ function delete(obj) try delete(obj.sf.h_fig); end + try + delete(obj.h_fig); + end + end + + function close_win(obj,varargin) + try + if ~isempty(obj.fh_close_win) + obj.fh_close_win(obj); + end + end + try + delete(obj); + end end function button_down(obj,h_obj,event) @@ -176,6 +208,12 @@ function button_down(obj,h_obj,event) rbbox; end + function button_motion(obj,h_obj,event) + if ~isempty(obj.fh_button_motion) + obj.fh_button_motion(obj,h_obj,event); + end + end + function button_up(obj,h_obj,event) if obj.actions.getting_polygon return; @@ -245,7 +283,11 @@ function key_press(obj,src,event) switch event.Key case 'f1' + fprintf('============================================================\n'); + fprintf('clip_vectors.m help message\n'); + fprintf('============================================================\n'); fprintf('Key Short Cuts\n'); + fprintf('F1: print this help message\n'); fprintf('f: open the selection filter window\n'); fprintf('p: draw a polygon to select the working area\n'); fprintf('u: undo the last command\n'); @@ -266,6 +308,7 @@ function key_press(obj,src,event) fprintf('left-click: zoom in at point\n'); fprintf('right-click: zoom out at point\n'); fprintf('scroll: zoom in/out at point\n'); + fprintf('============================================================\n'); case 'z' if ctrl_pressed diff --git a/cresis-toolbox/utility/ct_ecef2lla.m b/cresis-toolbox/utility/ct_ecef2lla.m new file mode 100644 index 00000000..8210eee3 --- /dev/null +++ b/cresis-toolbox/utility/ct_ecef2lla.m @@ -0,0 +1,47 @@ +function [lat,lon,alt] = ecef2lla(x,y,z) +% ECEF2LLA - convert earth-centered earth-fixed (ECEF) +% cartesian coordinates to latitude, longitude, +% and altitude +% +% USAGE: +% [lat,lon,alt] = ecef2lla(x,y,z) +% +% lat = geodetic latitude (radians) +% lon = longitude (radians) +% alt = height above WGS84 ellipsoid (m) +% x = ECEF X-coordinate (m) +% y = ECEF Y-coordinate (m) +% z = ECEF Z-coordinate (m) +% +% Notes: (1) This function assumes the WGS84 model. +% (2) Latitude is customary geodetic (not geocentric). +% (3) Inputs may be scalars, vectors, or matrices of the same +% size and shape. Outputs will have that same size and shape. +% (4) Tested but no warranty; use at your own risk. +% (5) Michael Kleder, April 2006 +% +% Downloaded Nov 13, 2020 from: +% https://www.mathworks.com/matlabcentral/fileexchange/7941-convert-cartesian-ecef-coordinates-to-lat-lon-alt +% +% No license + + +% WGS84 ellipsoid constants: +a = 6378137; +e = 8.1819190842622e-2; +% calculations: +b = sqrt(a^2*(1-e^2)); +ep = sqrt((a^2-b^2)/b^2); +p = sqrt(x.^2+y.^2); +th = atan2(a*z,b*p); +lon = atan2(y,x); +lat = atan2((z+ep^2.*b.*sin(th).^3),(p-e^2.*a.*cos(th).^3)); +N = a./sqrt(1-e^2.*sin(lat).^2); +alt = p./cos(lat)-N; +% return lon in range [0,2*pi) +lon = mod(lon,2*pi); +% correct for numerical instability in altitude near exact poles: +% (after this correction, error is about 2 millimeters, which is about +% the same as the numerical precision of the overall function) +k=abs(x)<1 & abs(y)<1; +alt(k) = abs(z(k))-b; diff --git a/cresis-toolbox/utility/ct_lla2ecef.m b/cresis-toolbox/utility/ct_lla2ecef.m new file mode 100644 index 00000000..151daf53 --- /dev/null +++ b/cresis-toolbox/utility/ct_lla2ecef.m @@ -0,0 +1,40 @@ +function [x,y,z] = ct_lla2ecef(lat,lon,alt) +% LLA2ECEF - convert latitude, longitude, and altitude to +% earth-centered, earth-fixed (ECEF) cartesian +% +% USAGE: +% [x,y,z] = lla2ecef(lat,lon,alt) +% +% x = ECEF X-coordinate (m) +% y = ECEF Y-coordinate (m) +% z = ECEF Z-coordinate (m) +% lat = geodetic latitude (radians) +% lon = longitude (radians) +% alt = height above WGS84 ellipsoid (m) +% +% Notes: This function assumes the WGS84 model. +% Latitude is customary geodetic (not geocentric). +% +% Source: "Department of Defense World Geodetic System 1984" +% Page 4-4 +% National Imagery and Mapping Agency +% Last updated June, 2004 +% NIMA TR8350.2 +% +% Michael Kleder, July 2005 +% +% Downloaded Nov 13, 2020 from: +% https://www.mathworks.com/matlabcentral/fileexchange/7942-covert-lat-lon-alt-to-ecef-cartesian +% +% No license + +% WGS84 ellipsoid constants: +a = 6378137; +e = 8.1819190842622e-2; +% intermediate calculation +% (prime vertical radius of curvature) +N = a ./ sqrt(1 - e^2 .* sin(lat).^2); +% results: +x = (N+alt) .* cos(lat) .* cos(lon); +y = (N+alt) .* cos(lat) .* sin(lon); +z = ((1-e^2) .* N + alt) .* sin(lat); diff --git a/cresis-toolbox/utility/ct_rmfield.m b/cresis-toolbox/utility/ct_rmfield.m new file mode 100644 index 00000000..f2b0d5bb --- /dev/null +++ b/cresis-toolbox/utility/ct_rmfield.m @@ -0,0 +1,3 @@ +function s = ct_rmfield(s,rm_field_names) + +s = rmfield(s,intersect(fieldnames(s),rm_field_names)); diff --git a/cresis-toolbox/utility/ct_sgolayfilt.m b/cresis-toolbox/utility/ct_sgolayfilt.m new file mode 100644 index 00000000..493a6203 --- /dev/null +++ b/cresis-toolbox/utility/ct_sgolayfilt.m @@ -0,0 +1,20 @@ +function y=ct_sgolayfilt(X,K,F) +% y=ct_sgolayfilt(X,K,F) +% +% Wrapper function for sgolayfilt to handle arbitrary input data lengths. +% +% X is assumed to be a vector +% K is assumed to be a scalar +% F is assumed to be a scalar +% +% Y is assumed to match X is size + +if isempty(X) + y=[]; + return; +end + +F = min(2*floor((length(X)-1)/2)+1,2*floor((F-1)/2)+1); +K = min(F-1,K); + +y=sgolayfilt(X,K,F); diff --git a/cresis-toolbox/utility/detect_os.m b/cresis-toolbox/utility/detect_os.m new file mode 100644 index 00000000..efeb570d --- /dev/null +++ b/cresis-toolbox/utility/detect_os.m @@ -0,0 +1,173 @@ +function [OS, OSVersion] = detect_os +%DETECTOS Name and version number of the operating system. +% OS = DETECTOS returns the name of the operating system as one of the +% following character vectors: 'windows', 'macos' (which includes OS X), +% 'solaris', 'aix', or another Unix/Linux distro in all lowercase +% characters (such as 'ubuntu' or 'centos'). An error is thrown if the +% operating system cannot be determined. +% +% [~, OSVERSION] = DETECTOS returns the operating system version number +% as a numeric row vector. For example, version 6.1.7601 is reported as +% OSVERSION = [6, 1, 7601]. If the OS version cannot be determined, a +% warning is issued and the empty numeric array is returned. +% +%See also COMPUTER, ISMAC, ISPC, ISUNIX. +% Created 2016-01-05 by Jorg C. Woehl +% 2016-10-10 (JCW): Converted to standalone function, comments added (v1.0.1). +% 2018-04-20 (JCW): Used the recommended ?replace? instead of ?strrep?. +% 2021-04-22 (JCW): Version information added (v1.1). +% +% THIS CODE WAS DOWNLOADED FROM MATLAB CENTRAL +% +% detectOS +% version 1.1.0.1 (4.27 KB) by Jorg Woehl +% Name and version number of the operating system +% https://github.com/JorgWoehl/detectOS +% +% https://www.mathworks.com/matlabcentral/fileexchange/59695-detectos + + +if ismac + % Mac + % see https://support.apple.com/en-us/HT201260 for version numbers + OS = 'macos'; + [status, OSVersion] = system('sw_vers -productVersion'); + OSVersion = strtrim(OSVersion); + if (~(status == 0) || isempty(OSVersion)) + warning('detectOS:UnknownMacOSVersion',... + 'Unable to determine macOS/OS X version.'); + OSVersion = ''; + end +elseif ispc + % Windows + % see https://en.wikipedia.org/wiki/Ver_(command) for version numbers + OS = 'windows'; + [status, OSVersion] = system('ver'); + OSVersion = regexp(OSVersion, '\d[.\d]*', 'match'); + if (~(status == 0) || isempty(OSVersion)) + warning('detectOS:UnknownWindowsVersion',... + 'Unable to determine Windows version.'); + OSVersion = ''; + end +elseif isunix + % Unix/Linux + % inspired in part by + % http://linuxmafia.com/faq/Admin/release-files.html and + % http://unix.stackexchange.com/questions/92199/how-can-i-reliably-get-the-operating-systems-name/92218#92218 + [status, OS] = system('uname -s'); % results in 'SunOS', 'AIX', or 'Linux' + OS = strtrim(OS); + assert((status == 0), 'detectOS:UnknownUnixDistro',... + 'Unable to determine Unix distribution.'); + if strcmpi(OS, 'SunOS') + OS = 'solaris'; % newer name + [status, OSVersion] = system('uname -v'); % example: + OSVersion = regexp(OSVersion, '\d[.\d]*', 'match'); + if (~(status == 0) || isempty(OSVersion)) + warning('detectOS:UnknownSolarisVersion',... + 'Unable to determine Solaris version.'); + OSVersion = ''; + end + elseif strcmpi(OS, 'AIX') + OS = 'aix'; + [status, OSVersion] = system('oslevel'); % example: 6.1.0.0 + OSVersion = regexp(OSVersion, '\d[.\d]*', 'match'); + if (~(status == 0) || isempty(OSVersion)) + warning('detectOS:UnknownAIXVersion',... + 'Unable to determine AIX version.'); + OSVersion = ''; + end + elseif strcmpi(OS, 'Linux') + OS = ''; + OSVersion = ''; + % first check if /etc/os-release exists and read it + [status, result] = system('cat /etc/os-release'); + if (status == 0) + % add newline to beginning and end of output character vector (makes parsing easier) + result = sprintf('\n%s\n', result); + % determine OS + OS = regexpi(result, '(?<=\nID=).*?(?=\n)', 'match'); % ID=... (shortest match) + OS = lower(strtrim(replace(OS, '"', ''))); % remove quotes, leading/trailing spaces, and make lowercase + if ~isempty(OS) + % convert to character vector + OS = OS{1}; + end + % determine OS version + OSVersion = regexpi(result, '(?<=\nVERSION_ID=)"*\d[.\d]*"*(?=\n)', 'match'); % VERSION_ID=... (longest match) + OSVersion = replace(OSVersion, '"', ''); % remove quotes + else + % check for output from lsb_release (more standardized than /etc/lsb-release itself) + [status, result] = system('lsb_release -a'); + if (status == 0) + % add newline to beginning and end of output character vector (makes parsing easier) + result = sprintf('\n%s\n', result); + % determine OS + OS = regexpi(result, '(?<=\nDistributor ID:\t).*?(?=\n)', 'match'); % Distributor ID: ... (shortest match) + OS = lower(strtrim(OS)); % remove leading/trailing spaces, and convert to lowercase + if ~isempty(OS) + % convert to character vector + OS = OS{1}; + end + % determine OS version + OSVersion = regexpi(result, '(?<=\nRelease:\t)\d[.\d]*(?=\n)', 'match'); % Release: ... (longest match) + else + % extract information from /etc/*release or /etc/*version filename + [status, result] = system('ls -m /etc/*version'); % comma-delimited file listing + fileList = ''; + if (status == 0) + fileList = result; + end + [status, result] = system('ls -m /etc/*release'); % comma-delimited file listing + if (status == 0) + fileList = [fileList ', ' result]; + end + fileList = replace(fileList, ',', ' '); + % remove spaces and trailing newline + fileList = strtrim(fileList); + OSList = regexpi(fileList, '(?<=/etc/).*?(?=[-_][rv])', 'match'); % /etc/ ... -release/version or _release/version + fileList = strtrim(strsplit(fileList)); + % find the first entry that's different from 'os', 'lsb', 'system', '', or 'debian'/'redhat' (unless it's the only one) + ii = 1; + while (ii <= numel(OSList)) + if ~(strcmpi(OSList{ii}, 'os') || strcmpi(OSList{ii}, 'lsb') || strcmpi(OSList{ii}, 'system') || ... + isempty(OSList{ii}) || strcmpi(OSList{ii}, 'redhat') || strcmpi(OSList{ii}, 'debian')) + OS = OSList{ii}; + OSFile = fileList{ii}; + break; + elseif (strcmpi(OSList{ii}, 'redhat') || strcmpi(OSList{ii}, 'debian')) + OS = OSList{ii}; % assign temporarily, but keep going + OSFile = fileList{ii}; + end + ii = ii+1; + end + % determine OS version + if ~isempty(OSFile) + [status, OSVersion] = system(['cat ' OSFile]); + if (status == 0) + OSVersion = regexp(OSVersion, '\d[.\d]*', 'match'); + else + OSVersion = ''; + end + end + end + end + assert(~isempty(OS), 'detectOS:UnknownLinuxDistro',... + 'Unable to determine Linux distribution.'); + if isempty(OSVersion) + warning('detectOS:UnknownLinuxVersion',... + 'Unable to determine Linux version.'); + OSVersion = ''; + end + else + error('detectOS:UnknownUnixDistro',... + 'Unknown Unix distribution.') + end +else + error('detectOS:PlatformNotSupported',... + 'Platform not supported.'); +end +if iscell(OSVersion) + % convert to character vector + OSVersion = OSVersion{1}; +end +OSVersion = round(str2double(strsplit(OSVersion, '.'))); +end \ No newline at end of file diff --git a/cresis-toolbox/utility/extract_max_mean_from_data_files.m b/cresis-toolbox/utility/extract_max_mean_from_data_files.m index a89f5f72..c9a707dd 100644 --- a/cresis-toolbox/utility/extract_max_mean_from_data_files.m +++ b/cresis-toolbox/utility/extract_max_mean_from_data_files.m @@ -119,11 +119,7 @@ fprintf('%s: %s (%s)\n', dbstack_info(1).name, param.day_seg, datestr(now,'HH:MM:SS')); fprintf('=====================================================================\n'); - % Load frames file - load(ct_filename_support(param,'','frames')); - % Load records file - %records_fn = ct_filename_support(param,'','records'); - %records = load(records_fn); + frames = frames_load(param); if isempty(param.cmd.frms) param.cmd.frms = 1:length(frames.frame_idxs); diff --git a/cresis-toolbox/utility/get_filenames.m b/cresis-toolbox/utility/get_filenames.m index 6b9219b1..cd69336e 100644 --- a/cresis-toolbox/utility/get_filenames.m +++ b/cresis-toolbox/utility/get_filenames.m @@ -1,30 +1,47 @@ -function [filenames,status] = get_filenames(filepath,filename_start,filename_middle,filename_end,param) -%-------------------------% -% GetFilenames % -%-------------------------% -% Author: William Blake -% Input: -% filepath = path to files -% filename_start = beginning of filename -% filename_middle = middle identifier of filename -% filename_end = suffix of filename -% param = structure containing -% .recursive: boolean, default 0, recursive into directories -% .type: 'f' or 'd', default 'f', file type f=file, d=directory -% .exact: boolean, default 0, requires exact match -% .regexp: regular expression -% Legacy support: if param is a string, it can only be 'recursive' +function [fns,status] = get_filenames(filepath,fn_prefix,fn_midfix,fn_suffix,param) +% [fns,status] = get_filenames(filepath,fn_prefix,fn_midfix,fn_suffix,param) +% +% Gets a list of filenames that match the pattern specified by the input +% arguments. Supports regular expression, file/directory, and recursive +% file searches. +% +% Inputs: +% +% filepath: string containing path to search +% +% fn_prefix: beginning of filename must match this string +% +% fn_midfix: middle of filename must match this string +% +% fn_suffix: end of filename must match this string +% +% param: optional structure containing one or more of these fields: +% +% .recursive: Logical scalar. Default is false. If true, causes the search +% to recurse into subfolders. +% +% .type: String containing either 'f' or 'd'. Default is 'f'. Files +% matched according to f=file, d=directory. +% +% .exact: Logical scalar. Default is false. If true, requires that the +% string exactly match with no characters in between the prefix, midfix, +% and suffix. +% +% .regexp: Regular expression string. Default is empty. Field ignored if +% empty. +% % Output: % Returns a column cell vector of filenames that correspond to paths of % filenames that meet the pattern specified in the arguements % % Example: -% filenames = GetFilenames('\\emperor\d5\wblake','InSAR','','wf_01_tx_01_rx_01.mat'); -% filenames = GetFilenames('/d5/wblake','InSAR','','wf_01_tx_01_rx_01.mat'); +% fns = get_filenames('\\emperor\d5\wblake','InSAR','','wf_01_tx_01_rx_01.mat'); +% fns = get_filenames('/d5/wblake','InSAR','','wf_01_tx_01_rx_01.mat'); % % Get all files (including in subdirectories): -% filenames = GetFilenames('/d5/wblake','','','','recursive'); +% fns = get_filenames('/d5/wblake','','','',struct('recursive',true)); % +% Author: William Blake, John Paden if nargin < 4 || nargin > 5 error('Args:IncorrectFormat','Must have 4 or 5 arguments'); @@ -64,17 +81,17 @@ end end end -if ~ischar(filepath) || ~ischar(filename_start) || ~ischar(filename_middle) || ... - ~ischar(filename_end) +if ~ischar(filepath) || ~ischar(fn_prefix) || ~ischar(fn_midfix) || ... + ~ischar(fn_suffix) error('Args:IncorrectFormat','First four arguments must be strings'); end if param.exact - filename_exp = sprintf('%s%s%s', filename_start, filename_middle, ... - filename_end); + filename_exp = sprintf('%s%s%s', fn_prefix, fn_midfix, ... + fn_suffix); else - filename_exp = sprintf('%s*%s*%s', filename_start, filename_middle, ... - filename_end); + filename_exp = sprintf('%s*%s*%s', fn_prefix, fn_midfix, ... + fn_suffix); end if ispc @@ -109,7 +126,7 @@ end % Execute system command to get filenames [status,tmp_filenames] = system(sysCmd); - filenames = {}; + fns = {}; % Parse returned filenames if isempty(tmp_filenames) || status ~= 0 return; @@ -118,9 +135,9 @@ files = files{1}; for idx = 1:size(files,1) if ~param.recursive - filenames{end+1,1} = fullfile(filepath,files{idx}); + fns{end+1,1} = fullfile(filepath,files{idx}); else - filenames{end+1,1} = files{idx}; + fns{end+1,1} = files{idx}; end end else @@ -139,7 +156,7 @@ % Execute system command to get filenames [status,tmp_filenames] = system(sysCmd); - filenames = {}; + fns = {}; % Parse returned filenames if isempty(tmp_filenames) || status ~= 0 return; @@ -147,18 +164,15 @@ files = textscan(tmp_filenames,'%s','Delimiter','\n'); files = sort(files{1}); for idx = 1:size(files,1) - filenames{end+1,1} = files{idx}; + fns{end+1,1} = files{idx}; end - filenames = sort(filenames); + fns = sort(fns); end if ~isempty(param.regexp) - good_mask = logical(ones(size(filenames))); - for idx = 1:length(filenames) + good_mask = logical(ones(size(fns))); + for idx = 1:length(fns) good_mask(idx) = ~isempty(regexp(files{idx},param.regexp)); end - filenames = filenames(good_mask); + fns = fns(good_mask); end - -return; - diff --git a/cresis-toolbox/utility/get_filenames_lidar.m b/cresis-toolbox/utility/get_filenames_lidar.m index d25c86bf..d2654154 100644 --- a/cresis-toolbox/utility/get_filenames_lidar.m +++ b/cresis-toolbox/utility/get_filenames_lidar.m @@ -2,7 +2,9 @@ % lidar_fns = get_filenames_lidar(param,lidar_source,gps_time) % % Gets a cell array of absolute filename strings for LIDAR files. Works -% with AWI LIDAR and DTU LIDAR data. +% with AWI LIDAR and DTU LIDAR data. Also works with generic LAS files. For +% LAS files, see read_lidar_las.m to see how these files must be setup for +% this to work. % % Input: % param: structure controlling which LIDAR files are loaded @@ -54,7 +56,18 @@ lidar_fns(end+(1:length(new_fns)),1) = new_fns; end -else +elseif strcmpi(lidar_source,'las') + [year month day] = datevec(epoch_to_datenum(gps_time(1))); + day_of_year_start = datenum(year,month,day) - datenum(year,0,0); + [year month day] = datevec(epoch_to_datenum(gps_time(end))); + day_of_year_end = datenum(year,month,day) - datenum(year,0,0); + lidar_fns = {}; + for day_of_year = day_of_year_start:day_of_year_end + new_fns = get_filenames(in_base_path, datestr(datenum(year,0,day_of_year),'yyyymmdd'), '', '.las'); + lidar_fns(end+(1:length(new_fns)),1) = new_fns; + end + +else % strcmpi(lidar_source,'dtu') % dec.hour(UTC) latitude longitude elevation amplitude #points/swath GPS.h range [year month day] = datevec(epoch_to_datenum(gps_time(1))); day_of_year_start = datenum(year,month,day) - datenum(year,0,0); diff --git a/cresis-toolbox/utility/interp_finite.m b/cresis-toolbox/utility/interp_finite.m index 4a1ff13a..f29c57a3 100644 --- a/cresis-toolbox/utility/interp_finite.m +++ b/cresis-toolbox/utility/interp_finite.m @@ -1,51 +1,144 @@ -function vals = interp_finite(vals,default_val,interp_method,interp_vals_fh) -% vals = interp_finite(vals,default_val,interp_method,interp_vals_fh) +function vals = interp_finite(vals,default_val,interp_fh,interp_mask_fh,extrap_mode,dim) +% vals = interp_finite(vals,default_val,interp_fh,interp_mask_fh,extrap_mode,dim) % +% Inputs +% ========================================================================= % vals: a vector of numbers -% default_val: if all vals are ~isfinite, then the whole vector will -% be set to this value -% interp_method: interpolation method passed to interp1 (default is -% 'linear') -% interp_vals_fh: function handle that returns true for values that will -% not be interpolated. Default is @isfinite. Other common examples are: -% @isfinite % interpolate NaN, inf, and -inf +% +% default_val: scalar. If the mask created by interp_mask_fh returns no +% valid data, then the whole vector will be set to this value. If this +% value +% +% interp_fh: interpolation function handle that takes three arguments +% similar to Matlab's interp1. To support legacy interface, if this +% function is a string, then the default is interp1 with the method +% specified in the string (see interp1.m for supported methods). Otherwise +% the default is interp1 with 'linear' method. Note that edge samples that +% would require extrapolation are always interpolated with nearest neighbor +% in extrap_mode==0. Common examples: +% @interp1 % linear interpolation (default) +% 'linear' % linear interpolation with interp1 (default) +% 'nearest' % nearest neighbor interpolation with interp1 +% @(xi,yi,xq) interp1(xi,yi,xq,'spline') % spline interpolation +% @gps_interp1 % gps_interp1 interpolation for polar data +% +% interp_mask_fh: function handle that returns true for values that will be +% used to interpolate the values that return false. Default is @isfinite. +% Common examples are: +% @isfinite % interpolate NaN, inf, and -inf (default) % @(x) ~isnan(x) % interpolate only NaN % @(x) x~=inf % interpolate only inf -% % -% vals = all isfinite elements remain unchanged, all ~isfinite values are -% interpolated from the isfinite values using this scheme: +% extrap_mode: string containing 'nearest' (default) or 'interp' +% +% dim: dimension to operate on (if not specified, interp_finite works on +% the first non-singleton dimension) +% +% Outputs +% ========================================================================= +% vals: vector of the same size as vals. All elements which had a true mask +% remain unchanged, all elements that had a false mask are interpolated +% from the isfinite values using this scheme: % 1. values on the end are interpolated using nearest neighbor -% 2. values in the middle will be linearly interpolated +% 2. values in the middle will be interpolated with interp_fh % 3. if no isfinite values exist, the whole vector is set to default_val % % Author: John Paden -if ~exist('interp_method','var') || isempty(interp_method) - interp_method = 'linear'; + +if ~exist('dim','var') || isempty(dim) + [vals,nshifts] = shiftdim(vals); +else + perm = [dim:max(length(size(vals)),dim) 1:dim-1]; + vals = permute(vals,perm); end -if ~exist('interp_vals_fh','var') || isempty(interp_vals_fh) - interp_vals_fh = @isfinite; +siz = size(vals); +[~,ncols] = size(vals); + +if ~exist('interp_fh','var') || isempty(interp_fh) + interp_fh = @interp1; +elseif ischar(interp_fh) + interp_fh = @(x,y,z) interp1(x,y,z,interp_fh); end -good_mask = interp_vals_fh(vals); +if ~exist('interp_mask_fh','var') || isempty(interp_mask_fh) + interp_mask_fh = @isfinite; +end -%% For bad values at the beginning and end, use nearest neighbor interpolation -first_good = find(good_mask,1); -if isempty(first_good) - % The whole vector is ~isfinite, so set to default_val and return - vals(:) = default_val; - return; +if ~exist('extrap_mode','var') || isempty(extrap_mode) + extrap_mode = 0; +elseif strcmpi(extrap_mode,'interp') + extrap_mode = 1; +else + extrap_mode = 0; end -vals(1:first_good) = vals(first_good); -good_mask(1:first_good) = 1; -last_good = find(good_mask,1,'last'); -vals(last_good:end) = vals(last_good); -good_mask(last_good:end) = 1; +good_mask = interp_mask_fh(vals); + +input_complex_flag = ~isreal(vals); % See "Fix Matlab auto-complex/real check" + +if extrap_mode == 0 + %% For bad values at the beginning and end, use nearest neighbor interpolation + for col = 1:ncols + first_good = find(good_mask(:,col),2); + if isempty(first_good) + % The whole vector is ~isfinite, so set to default_val and return + if ~exist('default_val','var') || isempty(default_val) + error('No valid samples and no default_val was given.'); + end + vals(:,col) = default_val; + + elseif length(first_good) == 1 + vals(:,col) = vals(first_good,col); + + elseif ~extrap_mode + % Extrapolation uses nearest neighbor + first_good = first_good(1); + vals(1:first_good,col) = vals(first_good,col); + good_mask(1:first_good,col) = 1; + + last_good = find(good_mask(:,col),1,'last'); + vals(last_good:end,col) = vals(last_good,col); + good_mask(last_good:end,col) = 1; + + %% Use linear interpolation for everything in between + vals(~good_mask(:,col),col) = interp_fh(find(good_mask(:,col)),vals(good_mask(:,col),col),find(~good_mask(:,col))); + end -%% Use linear interpolation for everything in between -vals(~good_mask) = interp1(find(good_mask),vals(good_mask),find(~good_mask),interp_method); + %% Fix Matlab auto-complex/real check + % Because matlab tries to check if vals is real or complex after each + % assignment to vals and it does this by searching through the entire + % array, we force the first entry to be complex if we know vals is + % complex. We will fix the first entry at the end of this function. + if col == 1 && input_complex_flag + first_val = vals(1); + vals(1) = 1i; + end + end + +else + %% Interpolation and Extrapolation use interpolation function + for col = 1:ncols + vals(~good_mask(:,col),col) = interp_fh(find(good_mask(:,col)),vals(good_mask(:,col),col),find(~good_mask(:,col))); + + % See "Fix Matlab auto-complex/real check" + if col == 1 && input_complex_flag + first_val = vals(1); + vals(1) = 1i; + end + end + +end + +% See "Fix Matlab auto-complex/real check" +if input_complex_flag && ncols >= 1 + vals(1) = first_val; +end +if ~exist('dim','var') || isempty(dim) + vals = reshape(vals,[ones(1,nshifts) size(vals,1) siz(2:end)]); +else + vals = reshape(vals,[size(vals,1) siz(2:end)]); + vals = ipermute(vals,perm); end diff --git a/cresis-toolbox/utility/interpft_memeff.m b/cresis-toolbox/utility/interpft_memeff.m new file mode 100644 index 00000000..a8255253 --- /dev/null +++ b/cresis-toolbox/utility/interpft_memeff.m @@ -0,0 +1,50 @@ +function x = interpft_memeff(x,new_length,dim) +% x = interpft_memeff(x,new_length,dim) +% +% Simplified and memory efficient interpft.m, new_length > size(x,dim) + +if ~exist('dim','var') + dim = 1; +end + +% Get the current size +siz = size(x); + +% Operate on specified dimension (permute that dim to row-dim) +if dim ~= 1 + perm = [dim:max(length(size(x)),dim) 1:dim-1]; + x = permute(x,perm); +end + +% Get a few parameters about the data +[old_length,N] = size(x); +x_real = isreal(x); + +x = fft(x,[],1); + +% Determine where zero padding needs to be added +nyqst = ceil((old_length+1)/2); +% Zero pad in the middle of the frequency spectrum (this also turns the +% input x matrix into a 2D matrix) +x = [x(1:nyqst,:) ; zeros(new_length-old_length,N) ; x(nyqst+1:old_length,:)]; +% Handle split bin that occurs for even number +if rem(old_length,2) == 0 + x(nyqst,:) = x(nyqst,:)/2; + x(nyqst+new_length-old_length,:) = x(nyqst,:); +end + +x = ifft(x,[],1); + +% Handle potential rounding errors that lead to complex data +if x_real, x = real(x); end + +% Handle fft/ifft scaling +x = x * new_length / old_length; + +% Reshape matrix back to its original size before turning into 2D matrix +x = reshape(x,[size(x,1) siz(2:end)]); + +% Operate on specified dimension (permute back) +if dim ~= 1 + x = ipermute(x,perm); +end diff --git a/cresis-toolbox/utility/make_segment_list.m b/cresis-toolbox/utility/make_segment_list.m deleted file mode 100644 index c2039180..00000000 --- a/cresis-toolbox/utility/make_segment_list.m +++ /dev/null @@ -1,43 +0,0 @@ -function segs = make_segment_list(xls_fn,param) -% segs = make_segment_list(xls_fn,param) -% -% xls_fn = parameter spreadsheet (.xls) filename or directory (for multiple -% .csv files) -% param = structure controlling operation of the function -% .all = boolean; when true it will process all segments, ignoring the -% param.vectors.verification field -% segs = cell vector of segments -% -% Examples: -% segs = make_segment_list('/users/paden/scripts/matlab/kuband_param_2011_Greenland_P3_dravid2.xls'); -% segs = make_segment_list('cresis/scratch2/mdce/mcords/2011_Antarctica_DC8/CSARP_post/csv'); -% -% Author: John Paden -% Contributions by: Steve Foga - -if ~exist('param','var') || ~isfield('param','all') - param.all = false; -end - -if ~isdir(xls_fn) % If an individual param file is called into the function -params = read_param_xls(xls_fn); - -segs = {}; - - - for seg_idx = 1:length(params) - if param.all || isempty(strfind(lower(params(seg_idx).vectors.verification),'do not process')) - segs{end+1} = params(seg_idx).day_seg; - end - end - -else % Else a directory (where CSV files exist) is called into the function - files = get_filenames(xls_fn,'Data_','','.csv'); - segs = cell(length(files),1)'; - for file_idx = 1:length(files) - segs{file_idx} = files{file_idx}(end-14:end-4); - end - - -end -return; diff --git a/cresis-toolbox/utility/mat2str_generic.m b/cresis-toolbox/utility/mat2str_generic.m index f381e1c0..3f9e6f10 100644 --- a/cresis-toolbox/utility/mat2str_generic.m +++ b/cresis-toolbox/utility/mat2str_generic.m @@ -30,7 +30,7 @@ elseif ischar(M) str = mat2str(M); -elseif isstruct(M) +elseif isstruct(M) || isobject(M) str = ''; if length(M) > 1 str = [str '[']; @@ -124,6 +124,9 @@ str = mat2str(M); end end +else + str = '[]'; + warning('Setting to empty unsupported type: %s', class(M)); end end \ No newline at end of file diff --git a/cresis-toolbox/utility/max_filt1.m b/cresis-toolbox/utility/max_filt1.m index 6e01ad3c..12cfc0bb 100644 --- a/cresis-toolbox/utility/max_filt1.m +++ b/cresis-toolbox/utility/max_filt1.m @@ -14,7 +14,7 @@ y = zeros(size_x); if size(x,1) == 1 - y = max_filt1_1D(x.',n).'; + y = max_filt1_1D(x.',n); else for col = 1:prod(size_x(2:end)) y(:,col) = max_filt1_1D(x(:,col),n); diff --git a/cresis-toolbox/utility/mean_without_outliers.m b/cresis-toolbox/utility/mean_without_outliers.m index e124d4be..b80a55d9 100644 --- a/cresis-toolbox/utility/mean_without_outliers.m +++ b/cresis-toolbox/utility/mean_without_outliers.m @@ -1,5 +1,5 @@ -function m = mean_without_outliers(x, num_stddev, percent, dim) -% m = mean_without_outliers(x, num_stddev, percent, dim) +function [m,mask] = mean_without_outliers(x, num_stddev, percent, dim) +% [m,mask] = mean_without_outliers(x, num_stddev, percent, dim) % % Finds the mean of x without outliers. Similar to Matlab's trimmean % @@ -30,7 +30,7 @@ % m = mean_without_outliers(x, 1, 0.2, 2) % % x = [-20 1 1 1 1 2 2 2 1000 3 3 3;-20 1 1 1 1 2 2 2 1000 3 3 3].'; -% m = mean_without_outliers(x, 1, 0.2) +% [m,mask] = mean_without_outliers(x, 1, 0.2) % % Author: John Paden @@ -68,19 +68,22 @@ median_x = nanmedian(x,1); dist_median = abs(bsxfun(@minus,x,median_x)).^2; [~,dist_idxs] = sort(dist_median,1); - x = x(dist_idxs); - x(end-round(percent*size(x,1))+1 : end, :) = NaN; + for remaining_idxs = 1:numel(x)/size(x,1) + x(dist_idxs(:,remaining_idxs) > round((1-percent)*size(x,1)),remaining_idxs) = NaN; + end end % Remove standard deviation outliers median_x = nanmedian(x,1); x_zeromean = abs(bsxfun(@minus,x,median_x)).^2; -var_x = mean(x_zeromean,1); +var_x = nanmean(x_zeromean,1); x(bsxfun(@gt, x_zeromean, num_stddev^2 * var_x)) = NaN; m = nanmean(x,1); +mask = ~isnan(x); % Permute matrix back to original form m = permute(m, permute_idxs); +mask = permute(mask, permute_idxs); diff --git a/cresis-toolbox/utility/merge_structs.m b/cresis-toolbox/utility/merge_structs.m index ebbe7313..308fdba0 100644 --- a/cresis-toolbox/utility/merge_structs.m +++ b/cresis-toolbox/utility/merge_structs.m @@ -1,7 +1,9 @@ function S_out = merge_structs(S_in,S_over,truncate_flag) % S_out = merge_structs(S_in,S_over,truncate_flag) % -% Merges two structures. S_over takes precedence. +% Merges two structures. S_over takes precedence. Note that structure +% arrays that exist in S_in and S_over will take on the size defined in +% S_over. % % S_in: input struct % S_over: struct which will override any fields in S_in @@ -11,7 +13,38 @@ % % S_out = merged structure % -% Example: See bottom of file +% Example: +% +% S_in.A = 1; +% S_in.B = 2; +% S_in.C(1).D = 1; +% S_in.C(1).D2 = 4.2; +% S_in.C(2).D = 2; +% S_in.C(3).D = 3; +% S_over.A = 5; +% S_over.C(2).D = 22; +% S_over.C(2).E = 5.2; +% S_over.F = 7; +% S_out = merge_structs(S_in,S_over) +% +% S_out: +% struct with fields: +% A: 5 +% B: 2 +% C: [1×2 struct] % field C is only 1x2 because S_over.C is only 1x2 +% F: 7 +% +% S_out.C(1): +% struct with fields: +% D: [] +% D2: 4.200000000000000 +% E: [] +% +% S_out.C(2): +% struct with fields: +% D: 22 +% D2: [] +% E: 5.200000000000000 % % Authors: Huan Zhao, John Paden @@ -71,23 +104,3 @@ end end end - -return ; - -% ================================================================== -% ================================================================== -% Examples -% ================================================================== -% ================================================================== - -S_in.A = 1; -S_in.B = 2; -S_in.C(1).D = 4; -S_in.C(1).D2 = 4.2; -S_in.C(2).D = 3; -S_over.A = 5; -S_over.C.D = 42; -S_over.C.E = 6; -S_over.F = 7; -S_out = merge_structs(S_in,S_over) - diff --git a/cresis-toolbox/utility/physical_constants.m b/cresis-toolbox/utility/physical_constants.m index 5dc183d2..5d2bda70 100644 --- a/cresis-toolbox/utility/physical_constants.m +++ b/cresis-toolbox/utility/physical_constants.m @@ -3,7 +3,7 @@ % Adds desired or all physical constants to caller's workspace % Outputs desired constants to assigned arguments % -% Usage: +% Usage: % physical_constants; % Adds all constants to workspace. General use. % physical_constants('c'); @@ -17,7 +17,9 @@ % [light_vel, grav_const] = physical_constants('c','G'); % Assigns according to the input and ouput arguments % -% Authors: John Paden, Hara Madhav Talasila +% Authors: John Paden, Hara Madhav Talasila +% +% See Also: physical_constants.m, proj_load_standard.m, project_locations.m %% Physical Constants % ========================================================================= @@ -27,12 +29,14 @@ % Permeability of free-space (N*A^-2), http://en.wikipedia.org/wiki/Permeability_%28electromagnetism%29 u0 = 4e-7*pi; % Speed of light in free-space -c = 1/sqrt(e0*u0); +% c = 1/sqrt(e0*u0); +c = 2.997924580003452e+08; % Boltzmann's constant (J * K^-1 * Hz^-1), http://en.wikipedia.org/wiki/Boltzmann_constant BoltzmannConst = 1.380650524e-23; % Earth Radius (m) from WGS-84 ellipsoid (larger: equatorial radius, % smaller: polar radius) -earthRadius = 0.5*(6378137+6356752.31424518); +% earthRadius = 0.5*(6378137+6356752.31424518); +earthRadius = 6.367444657122590e+06; % Earth mass (kg), http://en.wikipedia.org/wiki/Mass_of_the_Earth earthMass = 5.9742e24; % Gravitational constant (N * m^2 * kg^-2), http://en.wikipedia.org/wiki/Gravity_constant @@ -41,12 +45,30 @@ % which does match the values above. GearthMassProd = 398600.5e9; -% WGS84 ellipsoid parameters [semimajor sqrt(e2)] +% WGS84 ellipsoid parameters [semimajor sqrt(e2)] for geodetic2ecef WGS84.semimajor = 6378137; -WGS84.semiminor = 6356752.314245; +WGS84.semiminor = 6356752.31424518; WGS84.flattening = 298.257223563; -WGS84.eccentricity = sqrt(0.00669437999013); +WGS84.eccentricity = 0.0818191908426215; WGS84.ellipsoid = [WGS84.semimajor WGS84.eccentricity]; +WGS84.spheroid = wgs84Ellipsoid('meter'); +% wgs84Ellipsoid = referenceEllipsoid with defining properties: +% Code: 7030 +% Name: 'WGS 84' +% LengthUnit: 'meter' +% SemimajorAxis: 6378137 +% SemiminorAxis: 6356752.31424518 +% InverseFlattening: 298.257223563 +% Eccentricity: 0.0818191908426215 + +% GRS80 ellipsoid parameters [semimajor sqrt(e2)] for geodetic2ecef +GRS80.semimajor = 6378137; +GRS80.flattening = 298.25722210; +% GRS80.semiminor = GRS80.semimajor - 1/GRS80.flattening * GRS80.semimajor; +GRS80.semiminor = 6.356752314140284e+06; +% GRS80.eccentricity = sqrt(1 - GRS80.semiminor^2/GRS80.semimajor^2); +GRS80.eccentricity = 0.081819191042952; +GRS80.ellipsoid = [GRS80.semimajor GRS80.eccentricity]; % Basic ice properties er_ice = 3.15; @@ -66,7 +88,9 @@ else % Get only required constants for idx = 1:length(varargin) loc = find(strcmpi(varargin{idx},local_var_list)); - if loc && nargout==nargin % assigns physical constant to out variable + if isempty(loc) + error('Invalid constant request: varargin{%d} = %s', idx, varargin{idx}); + elseif nargout==nargin % assigns physical constant to out variable varargout{idx} = eval(local_var_list{loc}); else % adds physical constant to caller's workspace assignin('caller',local_var_list{loc},eval(local_var_list{loc})); @@ -74,4 +98,4 @@ end end -end \ No newline at end of file +end diff --git a/cresis-toolbox/utility/standard_projections.m b/cresis-toolbox/utility/proj_load_standard.m similarity index 73% rename from cresis-toolbox/utility/standard_projections.m rename to cresis-toolbox/utility/proj_load_standard.m index 30ca500a..2dae257b 100644 --- a/cresis-toolbox/utility/standard_projections.m +++ b/cresis-toolbox/utility/proj_load_standard.m @@ -1,3 +1,13 @@ +% script proj_load_standard +% +% Script for loading the standard projections: +% arctic_proj +% antarctic_proj +% +% Author: John Paden +% +% See Also: physical_constants.m, proj_load_standard.m + % Arctic North Polar Stereographic NSIDC arctic_proj.Filename='P:\GIS_data\greenland\Landsat-7\Greenland_natural_90m.tif'; arctic_proj.FileModDate='22-Sep-2011 13:51:54'; @@ -31,8 +41,11 @@ arctic_proj.TiePoints.WorldPoints.X=reshape([-693482.9938],[1 1]); arctic_proj.TiePoints.WorldPoints.Y=reshape([-774973.5364],[1 1]); arctic_proj.PixelScale=reshape([90 90 1],[3 1]); -arctic_proj.SpatialRef = maprasterref('XLimWorld',reshape([-693482.9938 897717.0062],[1 2]),'YLimWorld',reshape([-3439603.5364 -774973.5364],[1 2]),... - 'RasterSize',reshape([29607 17680],[1 2]),'RasterInterpretation','cells','ColumnsStartFrom','north','RowsStartFrom','west'); +if ~isempty(which('maprasterref')) + % Check for the existance of mapping toolbox function maprasterref + arctic_proj.SpatialRef = maprasterref('XLimWorld',reshape([-693482.9938 897717.0062],[1 2]),'YLimWorld',reshape([-3439603.5364 -774973.5364],[1 2]),... + 'RasterSize',reshape([29607 17680],[1 2]),'RasterInterpretation','cells','ColumnsStartFrom','north','RowsStartFrom','west'); +end arctic_proj.RefMatrix=reshape([0 90 -693527.9938 -90 0 -774928.5364],[3 2]); arctic_proj.BoundingBox=reshape([-693482.99377 897717.00623 -3439603.5364 -774973.5364],[2 2]); arctic_proj.CornerCoords.X=reshape([-693482.9938 897717.0062 897717.0062 -693482.9938],[1 4]); @@ -125,8 +138,11 @@ antarctic_proj.TiePoints.WorldPoints.X=reshape([-2668274.9891],[1 1]); antarctic_proj.TiePoints.WorldPoints.Y=reshape([2362334.97],[1 1]); antarctic_proj.PixelScale=reshape([240 240 0],[3 1]); -antarctic_proj.SpatialRef = maprasterref('XLimWorld',reshape([-2668274.9891 2813805.0226],[1 2]),'YLimWorld',reshape([-2294625.04 2362334.97],[1 2]),... - 'RasterSize',reshape([19404 22842],[1 2]),'RasterInterpretation','cells','ColumnsStartFrom','north','RowsStartFrom','west'); +if ~isempty(which('maprasterref')) + % Check for the existance of mapping toolbox function maprasterref + antarctic_proj.SpatialRef = maprasterref('XLimWorld',reshape([-2668274.9891 2813805.0226],[1 2]),'YLimWorld',reshape([-2294625.04 2362334.97],[1 2]),... + 'RasterSize',reshape([19404 22842],[1 2]),'RasterInterpretation','cells','ColumnsStartFrom','north','RowsStartFrom','west'); +end antarctic_proj.RefMatrix=reshape([0 240.00000052 -2668394.9891 -240.00000052 0 2362454.97],[3 2]); antarctic_proj.BoundingBox=reshape([-2668274.9891 2813805.0226 -2294625.04 2362334.97],[2 2]); antarctic_proj.CornerCoords.X=reshape([-2668274.9891 2813805.0226 2813805.0226 -2668274.9891],[1 4]); @@ -174,4 +190,6 @@ antarctic_proj.GeoTIFFTags.GeoKeyDirectoryTag.ProjScaleAtNatOriginGeoKey=reshape([1],[1 1]); antarctic_proj.GeoTIFFTags.GeoKeyDirectoryTag.ProjStraightVertPoleLongGeoKey=reshape([0],[1 1]); antarctic_proj.GeoTIFFTags.GeoDoubleParamsTag=reshape([-71 0 1 0 0 298.25722356 6378137],[1 7]); -antarctic_proj.GeoTIFFTags.GeoAsciiParamsTag='PCS Name = Polar_Stereographic|GCS_WGS_1984|'; \ No newline at end of file +antarctic_proj.GeoTIFFTags.GeoAsciiParamsTag='PCS Name = Polar_Stereographic|GCS_WGS_1984|'; + +usa_ned_proj = struct('Filename','/cresis/snfs1/dataproducts/GIS_data/usa/DEM/NED/National_Elevation_Data_DEM_10m.tif','FileModDate','30-Mar-2020 14:42:44','FileSize',1676407294,'Format','tif','FormatVersion',[],'Height',23091,'Width',18148,'BitDepth',32,'ColorType','grayscale','ModelType','ModelTypeProjected','PCS','WGS 84 / World Mercator','Projection','World Mercator','MapSys','','Zone',[],'CTProjection','CT_Mercator','ProjParm',[0;0;0;0;1;0;0],'ProjParmId',{{'ProjNatOriginLatGeoKey';'ProjNatOriginLongGeoKey';'Unknown-0';'Unknown-0';'ProjScaleAtNatOriginGeoKey';'ProjFalseEastingGeoKey';'ProjFalseNorthingGeoKey'}},'GCS','WGS 84','Datum','World Geodetic System 1984','Ellipsoid','WGS 84','SemiMajor',6378137,'SemiMinor',6356752.314245,'PM','Greenwich','PMLongToGreenwich',0,'UOMLength','metre','UOMLengthInMeters',1,'UOMAngle','degree','UOMAngleInDegrees',1,'TiePoints',struct('ImagePoints',struct('Row',0.5,'Col',0.5),'WorldPoints',struct('X',-11634132.5968164,'Y',5529999.92422665)),'PixelScale',[9.55462853564696;9.55462853564696;1],'SpatialRef',struct('RasterInterpretation','postings','XIntrinsicLimits',[1 18148],'YIntrinsicLimits',[1 23091],'SampleSpacingInWorldX',9.55462853564696,'SampleSpacingInWorldY',9.55462853564696,'XWorldLimits',[-11634132.5968164 -11460744.75278],'YWorldLimits',[5309383.55133856 5529999.92422665],'RasterSize',[23091 18148],'ColumnsStartFrom','north','RowsStartFrom','west','RasterExtentInWorldX',173387.844036385,'RasterExtentInWorldY',220616.372888088,'TransformationType','rectilinear','CoordinateSystemType','planar'),'RefMatrix',[0 -9.55462853564696;9.55462853564696 0;-11634142.1514449 5530009.47885519],'BoundingBox',[-11634132.5968164 5309383.55133856;-11460744.75278 5529999.92422665],'CornerCoords',struct('X',[-11634132.5968164 -11460744.75278 -11460744.75278 -11634132.5968164],'Y',[5529999.92422665 5529999.92422665 5309383.55133856 5309383.55133856],'Row',[1 1 23091 23091],'Col',[1 18148 18148 1],'Lat',[44.6079919806532 44.6079919806532 43.1748845216065 43.1748845216065],'Lon',[-104.511191291933 -102.953621788149 -102.953621788149 -104.511191291933]),'GeoTIFFCodes',struct('Model',1,'PCS',3395,'GCS',4326,'UOMLength',9001,'UOMAngle',9122,'Datum',6326,'PM',8901,'Ellipsoid',7030,'ProjCode',19883,'Projection',9804,'CTProjection',7,'MapSys',32767,'ProjParmId',[3081;3080;0;0;3092;3082;3083]),'GeoTIFFTags',struct('ModelPixelScaleTag',[9.55462853564696 9.55462853564696 1],'ModelTiepointTag',[0 0 0 -11634132.5968164 5529999.92422665 0],'GeoKeyDirectoryTag',struct('GTModelTypeGeoKey',1,'GTRasterTypeGeoKey',2,'GeographicTypeGeoKey',4326,'GeogAngularUnitsGeoKey',9102,'GeogSemiMajorAxisGeoKey',6378137,'GeogSemiMinorAxisGeoKey',6356752.314245,'GeogInvFlatteningGeoKey',298.257223560493,'ProjectedCSTypeGeoKey',3395,'ProjCoordTransGeoKey',7,'ProjLinearUnitsGeoKey',9001,'ProjStdParallel1GeoKey',0,'ProjStdParallel2GeoKey',0,'ProjNatOriginLongGeoKey',0,'ProjNatOriginLatGeoKey',0,'ProjFalseEastingGeoKey',0,'ProjFalseNorthingGeoKey',0,'ProjFalseOriginLongGeoKey',0,'ProjFalseOriginLatGeoKey',0,'ProjFalseOriginEastingGeoKey',0,'ProjFalseOriginNorthingGeoKey',0,'ProjCenterLongGeoKey',0,'ProjCenterLatGeoKey',0,'ProjCenterEastingGeoKey',0,'ProjCenterNorthingGeoKey',0,'ProjScaleAtNatOriginGeoKey',1,'ProjAzimuthAngleGeoKey',0,'ProjStraightVertPoleLongGeoKey',0,'VerticalUnitsGeoKey',9001),'GeoDoubleParamsTag',[6378137 6356752.314245 298.257223560493 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0])); diff --git a/cresis-toolbox/utility/robust_rmdir.m b/cresis-toolbox/utility/robust_rmdir.m index e8d3fd33..44c5e031 100644 --- a/cresis-toolbox/utility/robust_rmdir.m +++ b/cresis-toolbox/utility/robust_rmdir.m @@ -13,7 +13,7 @@ function robust_rmdir(fn_dir) cluster_attempts = cluster_attempts + 1; pause(3); if cluster_attempts > 2 - warning('rmdir failed repeatedly. There is probably a permission or file system issue. Delete the directory later:\n %s', fn_dir); + warning('rmdir failed repeatedly. There is probably a permission or file system issue. The directory may need to be moved manually to prevent problems with the processing. Usually the directory can be deleted at some later time once the file system issues are resolved. The directory is:\n %s', fn_dir); status = 0; end end diff --git a/cresis-toolbox/utility/struct_truncate.m b/cresis-toolbox/utility/struct_truncate.m index 10df8bc5..03484c02 100644 --- a/cresis-toolbox/utility/struct_truncate.m +++ b/cresis-toolbox/utility/struct_truncate.m @@ -9,7 +9,7 @@ % % Author: Hara Madhav Talasila % -% See also: records_aux_files_read.m +% See also: rec_fields = fieldnames(records); diff --git a/cresis-toolbox/utility/tukeywin.m b/cresis-toolbox/utility/tukeywin.m index abf1d180..37cc8fff 100644 --- a/cresis-toolbox/utility/tukeywin.m +++ b/cresis-toolbox/utility/tukeywin.m @@ -4,7 +4,7 @@ % Replacement for Matlab's tukeywin to reduce dependence on signal % processing toolbox. -if nargin < 2 || isempty(R), +if nargin < 2 || isempty(R) R = 0.5; end diff --git a/cresis-toolbox/utility/tukeywin_cont.m b/cresis-toolbox/utility/tukeywin_cont.m index 0a072056..2b548e23 100644 --- a/cresis-toolbox/utility/tukeywin_cont.m +++ b/cresis-toolbox/utility/tukeywin_cont.m @@ -21,7 +21,6 @@ H = zeros(size(L)); H(0 <= L & L <= r) = 0.5*(1 + cos(pi*(L(0 <= L & L <= r)/r - 1))); -H(1-r <= L & L <= 1) = 0.5*(1 + cos(pi*(L(1-r <= L & L <= 1)/r - 2/r + 1))); H(1-r <= L & L <= 1) = 0.5*(1 + cos(pi*(L(1-r <= L & L <= 1)/r - 2/r + 1))); diff --git a/python/tapes/load_files_from_tape.py b/python/tapes/load_files_from_tape.py new file mode 100755 index 00000000..efbcf1fc --- /dev/null +++ b/python/tapes/load_files_from_tape.py @@ -0,0 +1,268 @@ +#!/usr/bin/python3 +""" +Automatically load tapes in the Qualstar Q40 and pull the files from the given list. + +TAPES_FILE should point to a file produced by run_get_raw_files.m and map files +to tapes. + +Author: Reece Mathews +""" +from subprocess import check_output +import subprocess +from collections import defaultdict, namedtuple +from contextlib import contextmanager +from pathlib import Path +import re +import os +import shutil +import tarfile +import time + +TAPES_FILE = '/root/utilities/tapes.txt' # Produced from run_get_raw_files.m +TAPE_LIB_DEV = "/dev/sg4" +TAPE_MOUNT_PATH = "/mnt/ltfs" + +# Determine with `lsscsi -g` +TAPE_DRIVE_DEVS = { + "0": "/dev/sg3", + "1": "/dev/sg5", +} +SKIP_EXISTING = True # Do not ask to overwrite existing files and just skip instead +FS_PATH_SUBS = { # Path subsitutions for destinations on the filesystem + "/N/dc2/projects/cresis/": "/cresis/snfs1/data/Accum_Data/", + "/N/dcwan/projects/cresis/": "/cresis/snfs1/data/Accum_Data/" +} + + +def removeprefix(string, prefix): + """Reimplementing the Python 3.9 method for removing a string prefix.""" + return string[len(prefix):] if string.startswith(prefix) else string + +def removesuffix(string, suffix): + """Reimplementing the Python 3.9 method for removing a string suffix.""" + return string[:-len(suffix)] if string.endswith(suffix) else string + + +def get_tape_num(tape): + """Retrieve the number from the tape label.""" + return int(removesuffix(removeprefix(tape, "OIB"), "L8")) + + +def parse_tapes_file(): + """Read the tapes file into a dict.""" + tape_mapping = defaultdict(list) + path_mapping = {} + all_files = set() + + with open(TAPES_FILE) as f: + season = None + for line in f: + if season is None: + season = line + continue + if line.startswith("Filelist: "): + continue + if line.startswith("tapes filename stored_filename"): + continue + if line.strip() == "": + season = None + continue + + parts = line.split() + tapes = parts[0].split(",") + original_path = parts[1] + tape_path = parts[2] + + path_mapping[tape_path] = original_path + + if tape_path in all_files: + input("Duplicate file " + tape_path) + all_files |= {tape_path} + + for tape in tapes: + tape_num = get_tape_num(tape) + + if tape_num % 2 == 0: # We have even tapes + tape_mapping[tape].append(tape_path) + break + else: + raise RuntimeError("No even tape for " + tape_path) + + if SKIP_EXISTING: + # Remove existing files from list + for tape, file_list in tape_mapping.items(): + files = [file for file in file_list if not Path(path_subs(path_mapping[file])).exists()] + tape_mapping[tape] = files + + return tape_mapping, path_mapping + + +def inventory(): + """Check the tape library's current inventory.""" + INV_RE_MATCH_INDICES = { + "slot_type": 0, + "slot_num": 1, + "mailslot": 2, + "full": 3, + "original_slot_num": 6, + "barcode": 7 + } + # Hope someone else doesn't end up working on this 👍 - reece + INV_RE_PATTERN = r"(Storage|Data Transfer) Element ([0-9]+)\s?(IMPORT\/EXPORT)?:(Full|Empty)( (\(Storage Element ([0-9]+) Loaded\))?:VolumeTag\s?=\s?(\w+))?" + InvSlot = namedtuple("InvSlot", "slot_type slot_num mailslot full original_slot_num barcode") + + inv_output = check_output(["mtx", "-f", TAPE_LIB_DEV, "status"]).decode() + inv_output = "\n".join(line.strip() for line in inv_output.split("\n")) + matches = re.findall(INV_RE_PATTERN, inv_output) + return [InvSlot(**{k: match[i] for k, i in INV_RE_MATCH_INDICES.items()}) for match in matches] + + +def load_tape(tape_barcode): + """Load a tape into a drive""" + inv = inventory() + + # Find tape + slot = None + for slot_ in inv: + if slot_.barcode == tape_barcode: + slot = slot_ + break + else: + raise RuntimeError("Tape not present: " + tape_barcode) + + if slot.slot_type == "Data Transfer": + # tape already in drive + print("Tape " + slot.barcode + " already loaded in drive " + slot.slot_num) + return slot.slot_num + + # Find open drive + drive_slot = None + for slot_ in inv: + if slot_.slot_type == "Data Transfer": + drive_slot = slot_ + if slot_.full == "Empty": + break + else: + # All drives full, unload last drive found + if drive_slot is None: + raise RuntimeError("No drives found") + + print("Unloading " + drive_slot.barcode + " from drive " + drive_slot.slot_num + " to slot " + drive_slot.original_slot_num) + check_output(["mtx", "-f", TAPE_LIB_DEV, "unload", drive_slot.original_slot_num, drive_slot.slot_num]) + + print("Loading " + slot.barcode + " from slot " + slot.slot_num + " to drive " + drive_slot.slot_num) + check_output(["mtx", "-f", TAPE_LIB_DEV, "load", slot.slot_num, drive_slot.slot_num]) + + return drive_slot.slot_num + + +def unmount_drive(): + """Unmount the TAPE_MOUNT_PATH.""" + if os.path.ismount(TAPE_MOUNT_PATH): + print("Unmounting", TAPE_MOUNT_PATH) + check_output(["umount", TAPE_MOUNT_PATH]) + + +@contextmanager +def mount_drive(drive_num): + """Mount the given drive with LTFS.""" + unmount_drive() + drive_dev = TAPE_DRIVE_DEVS[drive_num] + # TODO[reece]: Handle mount hanging forever + print("Mounting " + drive_dev) + subprocess.call(["ltfs", "-o", f"devname={drive_dev}", TAPE_MOUNT_PATH], + stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) + if not os.listdir(TAPE_MOUNT_PATH): + raise RuntimeError("Failed to mount " + drive_dev) + try: + yield + finally: + unmount_drive() + + +def path_subs(path): + """Perform each path substitution on the given path.""" + for sub in FS_PATH_SUBS: + path = path.replace(sub, FS_PATH_SUBS[sub]) + return path.replace('\\', '/') + + +def copy_file(file, path_mapping, attempts=1): + """Copy the given file to its original locations.""" + # Perform sanity checks on file and destination + # fs = filesystem (destination) paths as opposed to tape (source) paths + + file_path = Path(TAPE_MOUNT_PATH) / file.replace('\\', '/').lstrip('/') + file_exists = file_path.exists() + file_size = os.path.getsize(file_path) / 1024 ** 2 if file_exists else None + print("source (exists:)", file_exists, f"{file_size if file_size is not None else 0} MB", file_path) + + fs_file_path = Path(path_subs(path_mapping[file])) + fs_file_exists = fs_file_path.exists() + fs_file_size = os.path.getsize(fs_file_path) / 1024 ** 2 if fs_file_exists else None + print("-> destination (exists:)", fs_file_exists, f"{fs_file_size if fs_file_size is not None else 0} MB", fs_file_path) + + # Check if source exists on tape + if not file_exists: + if attempts > 0: + print("Could not find source file on tape, waiting one second and retrying.") + time.sleep(1) + return copy_file(file, path_mapping, attempts=attempts-1) + input("**Could not find source file on tape, perhaps try remounting. Press enter to skip.**") + + # Check if destination already exists on filesystem + if fs_file_exists: + if SKIP_EXISTING or input("**File already exists on file system**, skip (y) or halt (n)?") == "y": + return + else: + raise RuntimeError("File already exists on file system") + # Check if parent folder path exists + fs_parent_path = fs_file_path.parent + if not fs_parent_path.exists(): + os.makedirs(fs_parent_path, exist_ok=True) + + # Perform copy + shutil.copy2(file_path, fs_file_path) + + if fs_file_path.name.endswith("small_file_archive.tar"): + os.chdir(fs_parent_path) + with tarfile.open(fs_file_path) as tar: + tar.extractall() + if os.path.exists("delete_this_zero_file"): + os.remove("delete_this_zero_file") + + os.remove(fs_file_path) + + os.chdir("/") + + +def load_tapes(tape_mapping, path_mapping): + """Load each tape and retrieve the files back to their original location.""" + for tape in tape_mapping: + + if not tape_mapping[tape]: + # Skip empty file sets + print("Skipping tape with no files: " + tape) + continue + + try: + drive_num = load_tape(tape) + except RuntimeError: + print("Skipping missing tape: " + tape) + print("- Did not restore files:") + for file in tape_mapping[tape]: + print("* ", TAPE_MOUNT_PATH + file, "->", path_subs(path_mapping[file])) + continue + + files = tape_mapping[tape] + with mount_drive(drive_num): + for file in files: + copy_file(file, path_mapping) + + +if __name__ == "__main__": + if os.geteuid() != 0: + raise RuntimeError("Must be ran as root to access tape device") + + tape_mapping, path_mapping = parse_tapes_file() + load_tapes(tape_mapping, path_mapping)