QCEFF Table

.gitignore

@@ -189,6 +189,7 @@ stacktest
 obs_rwtest
 test_quad_irreg_interp
 test_quad_reg_interp
+test_table_read
 
 # Directories to NOT IGNORE ... same as executable names
 # as far as I know, these must be listed after the executables

assimilation_code/modules/assimilation/assim_tools_mod.f90

@@ -143,7 +143,6 @@ module assim_tools_mod
 !  special_localization_obs_types -> Special treatment for the specified observation types
 !  special_localization_cutoffs   -> Different cutoff value for each specified obs type
 !
-logical  :: use_algorithm_info_mod          = .true.
 integer  :: filter_kind                     = 1
 real(r8) :: cutoff                          = 0.2_r8
 logical  :: sort_obs_inc                    = .false.
@@ -203,8 +202,7 @@ module assim_tools_mod
 ! compared to previous versions of this namelist item.
 logical  :: distribute_mean  = .false.
 
-namelist / assim_tools_nml / use_algorithm_info_mod,                           &
-   filter_kind, cutoff, sort_obs_inc,                                      &
+namelist / assim_tools_nml / filter_kind, cutoff, sort_obs_inc,            &
    spread_restoration, sampling_error_correction,                          &
    adaptive_localization_threshold, adaptive_cutoff_floor,                 &
    print_every_nth_obs, rectangular_quadrature, gaussian_likelihood_tails, &
@@ -976,7 +974,8 @@ subroutine obs_increment(ens_in, ens_size, obs, obs_var, obs_kind, obs_inc, &
 !--------------------------begin algorithm_info control block-----------------
 ! More flexible abilities to control the observation space increments are 
 ! available with this code block. It gets information about the increment method
-! for the current observation is use_algorithm_info_mod is set to true in the namelist.
+! for the current observation.
+
 ! This is not an extensible mechanism for doing this as the number of 
 ! obs increments distributions and associated information goes up
 ! Implications for sorting increments and for spread restoration need to be examined
@@ -988,9 +987,8 @@ subroutine obs_increment(ens_in, ens_size, obs, obs_var, obs_kind, obs_inc, &
 bounded_below = .false.;      lower_bound = 0.0_r8
 bounded_above = .false.;      upper_bound = 0.0_r8
 
-if(use_algorithm_info_mod) &
-   call obs_inc_info(obs_kind, filter_kind, rectangular_quadrature, gaussian_likelihood_tails, &
-      sort_obs_inc, spread_restoration, bounded_below, bounded_above, lower_bound, upper_bound)
+call obs_inc_info(obs_kind, filter_kind, rectangular_quadrature, gaussian_likelihood_tails, &
+   sort_obs_inc, spread_restoration, bounded_below, bounded_above, lower_bound, upper_bound)
 
 ! Could add logic to check on sort being true when not needed.
 ! Could also add logic to limit the use of spread_restoration to EAKF. It will fail

assimilation_code/modules/assimilation/filter_mod.f90

@@ -90,7 +90,7 @@ module filter_mod
 
 use probit_transform_mod,  only : transform_to_probit, transform_from_probit
 
-use algorithm_info_mod, only : probit_dist_info
+use algorithm_info_mod, only : probit_dist_info, init_algorithm_info_mod, end_algorithm_info_mod
 
 use distribution_params_mod, only : distribution_params_type, NORMAL_DISTRIBUTION
 
@@ -166,7 +166,7 @@ module filter_mod
 !----------------------------------------------------------------
 ! Namelist input with default values
 !
-logical  :: use_algorithm_info_mod = .true.
+character(len = 129) :: qcf_table_filename = '' 
 integer  :: async = 0, ens_size = 20
 integer  :: tasks_per_model_advance = 1
 ! if init_time_days and seconds are negative initial time is 0, 0
@@ -261,7 +261,7 @@ module filter_mod
 
 
 namelist /filter_nml/ async,     &
-   use_algorithm_info_mod,       &
+   qcf_table_filename,           &
    adv_ens_command,              &
    ens_size,                     &
    tasks_per_model_advance,      &
@@ -361,13 +361,13 @@ subroutine filter_main()
 
 real(r8), allocatable   :: prior_qc_copy(:)
 
-call filter_initialize_modules_used() ! static_init_model called in here
-
 ! Read the namelist entry
 call find_namelist_in_file("input.nml", "filter_nml", iunit)
 read(iunit, nml = filter_nml, iostat = io)
 call check_namelist_read(iunit, io, "filter_nml")
 
+call filter_initialize_modules_used() ! static_init_model called in here
+
 ! Record the namelist values used for the run ...
 if (do_nml_file()) write(nmlfileunit, nml=filter_nml)
 if (do_nml_term()) write(     *     , nml=filter_nml)
@@ -1150,6 +1150,9 @@ subroutine filter_main()
 call end_assim_model()
 call trace_message('After  end_model call')
 
+! deallocate qcf_table_data structures
+call end_algorithm_info_mod()
+
 call trace_message('Before ensemble and obs memory cleanup')
 call end_ensemble_manager(state_ens_handle)
 
@@ -1272,6 +1275,10 @@ subroutine filter_initialize_modules_used()
 call static_init_assim_model()
 call state_vector_io_init()
 call initialize_qc()
+
+! Initialize algorothm_info_mod and read in QCF table data
+call init_algorithm_info_mod(qcf_table_filename)
+
 call trace_message('After filter_initialize_module_used call')
 
 end subroutine filter_initialize_modules_used
@@ -1599,16 +1606,9 @@ subroutine filter_ensemble_inflate(ens_handle, inflate_copy, inflate, ENS_MEAN_C
          call get_state_meta_data(ens_handle%my_vars(j), my_state_loc, my_state_kind)    
 
          ! Need to specify what kind of prior to use for each
-         ! Use default of untransformed if use_algorithm_info_mod is not true
-         if(use_algorithm_info_mod) then
-            call probit_dist_info(my_state_kind, .true., .true., dist_type, &
-               bounded_below, bounded_above, lower_bound, upper_bound)
-         else
-            ! Default is just a normal which does nothing
-            dist_type = NORMAL_DISTRIBUTION
-            bounded_below = .false. ;  bounded_above = .false.
-            lower_bound = 0.0_r8;      upper_bound = 0.0_r8
-         endif
+         call probit_dist_info(my_state_kind, .true., .true., dist_type, &
+            bounded_below, bounded_above, lower_bound, upper_bound)
+
          call transform_to_probit(grp_size, ens_handle%copies(grp_bot:grp_top, j), &
             dist_type, dist_params, probit_ens(1:grp_size), .false., &
                bounded_below, bounded_above, lower_bound, upper_bound)

assimilation_code/modules/assimilation/filter_mod.nml

@@ -1,4 +1,5 @@
 &filter_nml
+   qcf_table_filename           = ''
    use_algorithm_info_mod       = .true.,
    single_file_in               = .false.,
    input_state_files            = ''

assimilation_code/programs/qcf_table/qcf_table_template.yaml

@@ -0,0 +1,69 @@
+QCF table version: 1
+QTY_TEMPLATE:
+  obs_error_info:
+    bounded_below
+    bounded_above
+    lower_bound
+    upper_bound
+  probit_inflation:
+    dist_type
+    bounded_below
+    bounded_above
+    lower_bound
+    upper_bound
+  probit_state:
+    dist_type
+    bounded_below
+    bounded_above
+    lower_bound
+    upper_bound
+  probit_extended_state:
+    dist_type
+    bounded_below
+    bounded_above
+    lower_bound
+    upper_bound
+  obs_inc_info:
+    filter_kind
+    rectangular_quadrature
+    gaussian_likelihood_tails
+    sort_obs_inc
+    spread_restoration
+    bounded_below
+    bounded_above
+    lower_bound
+    upper_bound
+QTY_STATE_VARIABLE:
+  obs_error_info:
+    bounded_below: .false.
+    bounded_above: .false.
+    lower_bound: -888888.0
+    upper_bound: -888888.0
+  probit_inflation:
+    dist_type: BOUNDED_NORMAL_RH_DISTRIBUTION
+    bounded_below: .false.
+    bounded_above: .false.
+    lower_bound: -888888.0
+    upper_bound: -888888.0
+  probit_state:
+    dist_type: BOUNDED_NORMAL_RH_DISTRIBUTION
+    bounded_below: .false.
+    bounded_above: .false.
+    lower_bound: -888888.0
+    upper_bound: -888888.0
+  probit_extended_state:
+    dist_type: BOUNDED_NORMAL_RH_DISTRIBUTION
+    bounded_below: .false.
+    bounded_above: .false.
+    lower_bound: -888888.0
+    upper_bound: -888888.0
+  obs_inc_info:
+    filter_kind: BOUNDED_NORMAL_RHF
+    rectangular_quadrature: .false.
+    gaussian_likelihood_tails: .false.
+    sort_obs_inc: .false.
+    spread_restoration: .false.
+    bounded_below: .false.
+    bounded_above: .false.
+    lower_bound: -888888.0
+    upper_bound: -888888.0

assimilation_code/programs/qcf_table/yaml_to_table.py

@@ -0,0 +1,77 @@
+import yaml
+
+#Prompt user for name of input and output files
+input_yaml = input('Please enter the name of your input yaml file (filename must end in ".yaml") OR press enter/return to use the default filename "qcf_table.yaml"\n')
+output_txt = input('Please enter the name for the output file for the table (filename must end in ".txt") OR press enter/return to use the default filename "qcf_table.txt"\n')
+
+#Using deault names for input/output files if not specified
+if (input_yaml == ''):
+    input_yaml = 'qcf_table.yaml'
+
+if (output_txt == ''):
+    output_txt = 'qcf_table.txt'
+
+#Open and load yaml file
+with open(input_yaml) as file:
+    dict = yaml.safe_load(file)
+
+    column_headers = list(dict.keys())
+    column_data = list(dict.values())
+
+    obs_errror_info_header = dict['QTY_TEMPLATE']['obs_error_info']
+    probit_inflation_header = dict['QTY_TEMPLATE']['probit_inflation']
+    probit_state_header = dict['QTY_TEMPLATE']['probit_state']
+    probit_extended_state_header = dict['QTY_TEMPLATE']['probit_extended_state']
+    obs_inc_info_header = dict['QTY_TEMPLATE']['obs_inc_info']
+
+    f = open(output_txt, "w")
+
+#Write the table's headers to the output file
+    f.write(column_headers[0] + ": " + str(column_data[0]) + "\n")
+
+    f.write(column_headers[1] + ": ")
+    for name in obs_errror_info_header:
+        f.write(name)
+    f.write(" ")
+    for name in probit_inflation_header:
+        f.write(name)
+    f.write(" ")
+    for name in probit_state_header:
+        f.write(name)
+    f.write(" ")
+    for name in probit_extended_state_header:
+        f.write(name)
+    f.write(" ")
+    for name in obs_inc_info_header:
+        f.write(name)
+    f.write("\n")
+
+#Writing table data to the output file 
+    for i in range(2, len(column_headers)):
+        f.write(column_headers[i] + " ")
+
+        obs_error_info = dict[column_headers[i]]['obs_error_info'].items()
+        for key, value in obs_error_info:
+            f.write(str(value) + " ")
+
+        probit_inflation = dict[column_headers[i]]['probit_inflation'].items()
+        for key, value in probit_inflation:
+            f.write(str(value) + " ")
+
+        probit_state = dict[column_headers[i]]['probit_state'].items()
+        for key, value in probit_state:
+            f.write(str(value) + " ")
+
+        probit_extended_state = dict[column_headers[i]]['probit_extended_state'].items()
+        for key, value in probit_extended_state:
+            f.write(str(value) + " ")
+
+        obs_inc_info = dict[column_headers[i]]['obs_inc_info'].items()
+        for key, value in obs_inc_info:
+            f.write(str(value) + " ")
+
+        f.write("\n")
+
+    f.close
+
+print('QCF table produced in ' + output_txt)

developer_tests/qceff/test_table_read.f90

@@ -0,0 +1,24 @@
+! DART software - Copyright UCAR. This open source software is provided
+! by UCAR, "as is", without charge, subject to all terms of use at
+! http://www.image.ucar.edu/DAReS/DART/DART_download
+
+! qcf_table_filename expected as command line arguement
+program test_table_read
+
+use algorithm_info_mod, only : init_algorithm_info_mod, end_algorithm_info_mod
+use utilities_mod,      only : initialize_utilities
+
+implicit none
+
+character(len=129) :: qcf_table_filename
+
+call initialize_utilities('test_table_read')
+
+call get_command_argument(1,qcf_table_filename)
+
+call init_algorithm_info_mod(qcf_table_filename)
+
+call end_algorithm_info_mod()
+
+
+end program test_table_read

developer_tests/qceff/work/input.nml

@@ -0,0 +1,28 @@
+&utilities_nml
+   TERMLEVEL = 1,
+   module_details = .false.
+   logfilename = 'dart_log.out'  
+   /
+
+# pick a random set of inputs
+&preprocess_nml
+   overwrite_output        = .true.
+   input_obs_qty_mod_file  = '../../../assimilation_code/modules/observations/DEFAULT_obs_kind_mod.F90'
+   output_obs_qty_mod_file = '../../../assimilation_code/modules/observations/obs_kind_mod.f90'
+   input_obs_def_mod_file  = '../../../observations/forward_operators/DEFAULT_obs_def_mod.F90'
+   output_obs_def_mod_file = '../../../observations/forward_operators/obs_def_mod.f90'
+   obs_type_files          = '../../../observations/forward_operators/obs_def_reanalysis_bufr_mod.f90',
+                            '../../../observations/forward_operators/obs_def_radar_mod.f90',
+                            '../../../observations/forward_operators/obs_def_metar_mod.f90',
+                            '../../../observations/forward_operators/obs_def_dew_point_mod.f90',
+                            '../../../observations/forward_operators/obs_def_rel_humidity_mod.f90',
+                            '../../../observations/forward_operators/obs_def_altimeter_mod.f90',
+                            '../../../observations/forward_operators/obs_def_gps_mod.f90',
+                            '../../../observations/forward_operators/obs_def_vortex_mod.f90',
+                            '../../../observations/forward_operators/obs_def_gts_mod.f90',
+                            '../../../observations/forward_operators/obs_def_QuikSCAT_mod.f90'
+   quantity_files         = '../../../assimilation_code/modules/observations/default_quantities_mod.f90',
+   /
+
+&obs_kind_nml
+/

developer_tests/qceff/work/qcf_table.txt

@@ -0,0 +1,3 @@
+QCF table version: 1
+QTY_TEMPLATE: bounded_below bounded_above lower_bound upper_bound dist_type bounded_below bounded_above lower_bound upper_bound dist_type bounded_below bounded_above lower_bound upper_bound dist_type bounded_below bounded_above lower_bound upper_bound filter_kind rectangular_quadrature gaussian_likelihood_tails sort_obs_inc spread_restoration bounded_below bounded_above lower_bound upper_bound
+QTY_STATE_VARIABLE .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RHF .false. .false. .false. .false. .false. .false. -888888.0 -888888.0 

developer_tests/qceff/work/qcf_table_bad_qty.txt

@@ -0,0 +1,3 @@
+QCF table version: 1
+QTY_TEMPLATE: bounded_below bounded_above lower_bound upper_bound dist_type bounded_below bounded_above lower_bound upper_bound dist_type bounded_below bounded_above lower_bound upper_bound dist_type bounded_below bounded_above lower_bound upper_bound filter_kind rectangular_quadrature gaussian_likelihood_tails sort_obs_inc spread_restoration bounded_below bounded_above lower_bound upper_bound
+QTY_HAIRCUT .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RHF .false. .false. .false. .false. .false. .false. -888888.0 -888888.0 

developer_tests/qceff/work/qcf_table_broke.txt

@@ -0,0 +1,3 @@
+QCF table version: 1
+QTY_TEMPLATE: bounded_below bounded_above lower_bound upper_bound dist_type bounded_below bounded_above lower_bound upper_bound dist_type bounded_below bounded_above lower_bound upper_bound dist_type bounded_below bounded_above lower_bound upper_bound filter_kind rectangular_quadrature gaussian_likelihood_tails sort_obs_inc spread_restoration bounded_below bounded_above lower_bound upper_bound
+QTY_STATE_VARIABLE .false.  -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RHF .false. .false. .false. .false. .false. .false. -888888.0 -888888.0 

developer_tests/qceff/work/qcf_table_duplicates.txt

@@ -0,0 +1,6 @@
+QCF table version: 1
+QTY_TEMPLATE: bounded_below bounded_above lower_bound upper_bound dist_type bounded_below bounded_above lower_bound upper_bound dist_type bounded_below bounded_above lower_bound upper_bound dist_type bounded_below bounded_above lower_bound upper_bound filter_kind rectangular_quadrature gaussian_likelihood_tails sort_obs_inc spread_restoration bounded_below bounded_above lower_bound upper_bound
+QTY_VEGETATED_AREA_FRACTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RHF .false. .false. .false. .false. .false. .false. -888888.0 -888888.0 
+QTY_AQUIFER_WATER .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RHF .false. .false. .false. .false. .false. .false. -888888.0 -888888.0 
+QTY_SEAICE_SALINITY008 .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RHF .false. .false. .false. .false. .false. .false. -888888.0 -888888.0 
+QTY_VEGETATED_AREA_FRACTION .true. .false. 0 1000 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RHF .false. .false. .false. .false. .false. .false. -888888.0 -888888.0 

developer_tests/qceff/work/qcf_table_extra_columns.txt

@@ -0,0 +1,3 @@
+QCF table version: 1
+QTY_TEMPLATE: bounded_below bounded_above lower_bound upper_bound dist_type bounded_below bounded_above lower_bound upper_bound dist_type bounded_below bounded_above lower_bound upper_bound dist_type bounded_below bounded_above lower_bound upper_bound filter_kind rectangular_quadrature gaussian_likelihood_tails sort_obs_inc spread_restoration bounded_below bounded_above lower_bound upper_bound frog 
+QTY_STATE_VARIABLE .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RHF .false. .false. .false. .false. .false. .false. -888888.0 -888888.0 toad newt

developer_tests/qceff/work/qcf_table_lower_bound_only.txt

@@ -0,0 +1,6 @@
+QCF table version: 1
+QTY_TEMPLATE: bounded_below bounded_above lower_bound upper_bound dist_type bounded_below bounded_above lower_bound upper_bound dist_type bounded_below bounded_above lower_bound upper_bound dist_type bounded_below bounded_above lower_bound upper_bound filter_kind rectangular_quadrature gaussian_likelihood_tails sort_obs_inc spread_restoration bounded_below bounded_above lower_bound upper_bound
+QTY_STATE_VARIABLE .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RHF .false. .false. .false. .false. .false. .false. -888888.0 -888888.0 
+QTY_AQUIFER_WATER .true. .false. 0.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RHF .false. .false. .false. .false. .false. .false. -888888.0 -888888.0 
+QTY_SEAICE_SALINITY008 .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RHF .false. .false. .false. .false. .false. .false. -888888.0 -888888.0 
+QTY_VEGETATED_AREA_FRACTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RH_DISTRIBUTION .false. .false. -888888.0 -888888.0 BOUNDED_NORMAL_RHF .false. .false. .false. .false. .false. .false. -888888.0 -888888.0