Hydrofabric refactor to support latest hydrofabric data model

include/core/HY_Features.hpp

@@ -8,6 +8,7 @@
 #include <HY_HydroNexus.hpp>
 #include <network.hpp>
 #include <Formulation_Manager.hpp>
+#include <HY_Features_Ids.hpp>
 
 namespace hy_features {
 
@@ -85,6 +86,17 @@ namespace hy_features {
           return nullptr;
         }
 
+        /**
+         * @brief Construct a new HY_Features object from a Network and a set of formulations.
+         * 
+         * Constructs the HY_Catchment objects for each catchment feature in the network, and attaches tha formaulation
+         * associated with the catchment found in the Formulation_Manager.  Also constucts each nexus as a HY_PointHydroNexus.
+         * 
+         * @param network 
+         * @param formulations 
+        */
+        HY_Features( network::Network network, std::shared_ptr<Formulation_Manager> formulations, geojson::GeoJSON fabric);
+
         /**
          * @brief Get the HY_HydroNexus pointer identifed by @p id
          * 
@@ -105,15 +117,15 @@ namespace hy_features {
          * 
          * @return auto 
          */
-        inline auto catchments(){return network.filter("cat");}
+        inline auto catchments(){return network.filter(hy_features::identifiers::catchment);}
 
         /**
          * @brief An iterator of only the catchment feature ids from only the specified layer
          * 
          * @return auto 
          */
         inline auto catchments(long lyr) {
-            return network.filter("cat",lyr);
+            return network.filter(hy_features::identifiers::catchment,lyr);
         }
 
         /**
@@ -127,7 +139,7 @@ namespace hy_features {
          * 
          * @return auto 
          */
-        inline auto nexuses(){return network.filter("nex");}
+        inline auto nexuses(){return network.filter(hy_features::identifiers::nexus);}
 
         /**
          * @brief Get a vector of destination (downstream) nexus pointers.
@@ -186,8 +198,6 @@ namespace hy_features {
 
       private:
 
-        void init();
-
         /**
          * @brief Internal mapping of catchment id -> HY_Catchment pointer.
          * 
@@ -217,8 +227,6 @@ namespace hy_features {
         */
         std::set<long> hf_layers;
 
-        geojson::GeoJSON fabric;
-
     };
 }
 

include/core/HY_Features_Ids.hpp

@@ -0,0 +1,50 @@
+#ifndef HY_FEATURES_IDS_H
+#define HY_FEATURES_IDS_H
+#include <string>
+
+namespace hy_features {
+    namespace identifiers{
+      /**
+       * Namespace constants, define the string prefixes for various hydrofabric
+       * identifier types.
+      */
+      static const std::string seperator = "-";
+      static const std::string nexus = "nex";
+      static const std::string coastal = "cnx";
+      static const std::string terminal = "tnx";
+      static const std::string catchment = "cat";
+      static const std::string flowpath = "wb";
+      static const std::string inland = "inx";
+
+      /**
+       * Determine if a provided prefix/type is representative of a Nexus concept
+       * 
+       * @param type string to test
+      */
+      inline const bool isNexus(const std::string& type){
+        return (type == nexus || type == terminal || type == coastal) || type == inland;
+      }
+
+      /**
+       * Determine if a provided prefix/type is representative of a Catchment concept
+       * 
+       * @param type string to test
+      */
+      inline const bool isCatchment(const std::string& type){
+        //This is a next-gen specific identifier, hiding it here for now
+        static const std::string aggregate = "agg";
+        return (type == catchment || type == aggregate || type == flowpath);
+      }
+
+      /**
+       * Determine if a provided prefix/type is representative of a Flowpath concept
+       * 
+       * @param type string to test
+      */
+      inline const bool isFlowpath(const std::string& type){
+        return (type == flowpath);
+      }
+    }
+}
+
+#endif //HY_FEATURES_IDS_H

include/core/Layer.hpp

@@ -116,6 +116,9 @@ namespace ngen
                     //TODO in a DENDRITIC network, only one destination nexus per catchment
                     //If there is more than one, some form of catchment partitioning will be required.
                     //for now, only contribute to the first one in the list
+                    if(nexus == nullptr){
+                        throw std::runtime_error("Invalid (null) nexus instantiation downstream of "+id+". "+SOURCE_LOC);
+                    }
                     nexus->add_upstream_flow(response_m_h, id, output_time_index);
                     /*std::cerr << "Add water to nexus ID = " << nexus->get_id() << " from catchment ID = " << id << " value = "
                               << response << ", ID = " << id << ", time-index = " << output_time_index << std::endl; */

include/core/network.hpp

@@ -17,6 +17,8 @@
 #include <features/Features.hpp>
 #include <FeatureBuilder.hpp>
 
+#include "HY_Features_Ids.hpp"
+
 namespace network {
 
   const static int DEFAULT_LAYER_ID = 0;
@@ -208,13 +210,18 @@ namespace network {
           return get_sorted_index(order) | boost::adaptors::reversed
                         | boost::adaptors::transformed([this](int const& i) { return get_id(i); })
                         | boost::adaptors::filtered([type](std::string const& s) { 
-                          if(type == "nex"){
-                            return s.substr(0,3) == type || s.substr(0,3) == "tnx" || s.substr(0,4) == "tnex";
+                          //seperate the prefix from the numeric id
+                          std::string id_type = s.substr(0, s.find(hy_features::identifiers::seperator) );
+                          //Allow subtypes, e.g. inx, tnx, cnx, to be pass the filter for a generic nexus type
+                          if(type == hy_features::identifiers::nexus){
+                            return hy_features::identifiers::isNexus(id_type);
                           }
-                          if(type == "cat"){
-                            return s.substr(0,3) == type || s.substr(0,3) == "agg";
+                          //Allow subtypes, e.g. wb to be pass the filter for a generic catchment type
+                          if(type == hy_features::identifiers::catchment){
+                            return hy_features::identifiers::isCatchment(id_type);
                           }
-                          return s.substr(0,3) == type; 
+                          //any other subtype filter gets only exact matches
+                          return id_type == type; 
                         });
 
         }
@@ -244,15 +251,17 @@ namespace network {
           return get_sorted_index(order) | boost::adaptors::reversed
                         | boost::adaptors::transformed([this](int const& i) { return get_id(i); })
                         | boost::adaptors::filtered([this,type,target_layer](std::string const& s) { 
-                          if(type == "nex"){
-                            return (s.substr(0,3) == type || s.substr(0,3) == "tnx" || s.substr(0,4) == "tnex") && 
+                          //seperate the prefix from the numeric id
+                          std::string id_type = s.substr(0, s.find(hy_features::identifiers::seperator) );
+                          if(type == hy_features::identifiers::nexus){
+                            return hy_features::identifiers::isNexus(id_type) && 
                                    (this->layer_map.find(s) != this->layer_map.end() && this->layer_map[s] == target_layer);
                           }
-                          if(type == "cat"){
-                            return (s.substr(0,3) == type || s.substr(0,3) == "agg") && 
+                          if(type == hy_features::identifiers::catchment){
+                            return hy_features::identifiers::isCatchment(id_type) && 
                                    (this->layer_map.find(s) != this->layer_map.end() && this->layer_map[s] == target_layer);
                           }
-                          return (s.substr(0,3) == type) && 
+                          return (id_type == type) && 
                                   (this->layer_map.find(s) != this->layer_map.end() && (this->layer_map[s] == target_layer)); 
                         });
         }

include/core/nexus/HY_PointHydroNexusRemote.hpp

@@ -4,6 +4,7 @@
 #ifdef NGEN_MPI_ACTIVE
 
 #include <HY_PointHydroNexus.hpp>
+#include <HY_Features_Ids.hpp>
 #include <mpi.h>
 #include <vector>
 
@@ -39,7 +40,7 @@ class HY_PointHydroNexusRemote : public HY_PointHydroNexus
         void add_upstream_flow(double val, std::string catchment_id, time_step_t t);
 
         /** extract a numeric id from the catchment id for use as a mpi tag */
-        static long extract(std::string s) {  return std::stoi(s.substr(4)); }
+        static long extract(std::string s) {  return std::stoi( s.substr( s.find(hy_features::identifiers::seperator)+1 ) ); }
 
         const Catchments& get_local_contributing_catchments(){
             return local_contributers;

include/geojson/FeatureCollection.hpp

@@ -182,7 +182,12 @@ namespace geojson {
 
             void set_ids(const std::string& id_field_name = "id");
 
-            void update_ids();
+            /**
+             * Update the collection's feature id mapping
+             * 
+             * @param alt_id Optional, An alternative feature property to also index the feature by
+            */
+            void update_ids(const std::string& alt_id = {});
 
             int link_features_from_property(std::string* from_property = nullptr, std::string* to_property = nullptr);
 

include/geojson/features/FeatureBase.hpp

@@ -396,6 +396,15 @@ namespace geojson {
                 return id;
             }
 
+            std::string get_id(std::string alt_id) const {
+                try{
+                    return get_property(alt_id).as_string();
+                }
+                catch(...){
+                    return id;
+                }
+            }
+
             int get_number_of_destination_features() {
                 return destination.size();
             }

src/NGen.cpp

@@ -283,7 +283,10 @@ int main(int argc, char *argv[]) {
         nexus_collection->add_feature(feature);
         //std::cout<<"Catchment "<<feature->get_id()<<" -> Nexus "<<feature->get_property("toID").as_string()<<std::endl;
     }
-
+    //Update the feature ids for the combined collection, using the alternative property 'id'
+    //to map features to their primary id as well as the alternative property
+    nexus_collection->update_ids("id");
+    std::cout<<"Initializing formulations\n";
     std::shared_ptr<realization::Formulation_Manager> manager = std::make_shared<realization::Formulation_Manager>(REALIZATION_CONFIG_PATH);
     manager->read(catchment_collection, utils::getStdOut());
 
@@ -308,7 +311,7 @@ int main(int argc, char *argv[]) {
     }
     #endif //NGEN_MPI_ACTIVE
     #endif //NGEN_ROUTING_ACTIVE
-
+    std::cout<<"Building Feature Index\n";
     std::string link_key = "toid";
     nexus_collection->link_features_from_property(nullptr, &link_key);
     #ifdef NGEN_MPI_ACTIVE

src/core/HY_Features.cpp

@@ -3,67 +3,69 @@
 
 using namespace hy_features;
 
-HY_Features::HY_Features(geojson::GeoJSON linked_hydro_fabric, std::shared_ptr<Formulation_Manager> formulations)
-  :network(linked_hydro_fabric), formulations(formulations), fabric(linked_hydro_fabric)
+HY_Features::HY_Features( geojson::GeoJSON fabric, std::shared_ptr<Formulation_Manager> formulations)
+  :HY_Features(network::Network(fabric), formulations, fabric)
 {
-  init();
+  /* If you nest the constuctor calls like this, then the network, while valid in the next constructor,
+     is empty once the collection is fully instansiated...
+    HY_Features(network::Network(fabric));
+  */
 }
 
-HY_Features::HY_Features( geojson::GeoJSON catchments, std::string* link_key, std::shared_ptr<Formulation_Manager> formulations)
-    : network(catchments, link_key), formulations(formulations), fabric(catchments)
-{
-  init();
+HY_Features::HY_Features( geojson::GeoJSON catchments, std::string* link_key, std::shared_ptr<Formulation_Manager> formulations):
+    HY_Features(network::Network(catchments, link_key), formulations, catchments){  
 }
 
-void HY_Features::init()
+HY_Features::HY_Features(network::Network network, std::shared_ptr<Formulation_Manager> formulations, geojson::GeoJSON fabric)
+  :network(network), formulations(formulations)
 {
-  std::string feat_id;
-  std::string feat_type;
-  std::vector<std::string> origins, destinations;
+      std::string feat_id;
+      std::string feat_type;
+      std::vector<std::string> origins, destinations;
 
-  for(const auto& feat_idx : network)
-  {
-    feat_id = network.get_id(feat_idx);//feature->get_id();
-    feat_type = feat_id.substr(0, 3);
+      for(const auto& feat_idx : network){
+        feat_id = network.get_id(feat_idx);//feature->get_id();
+        feat_type = feat_id.substr(0, feat_id.find(hy_features::identifiers::seperator) );
 
-    destinations  = network.get_destination_ids(feat_id);
-    if(feat_type == "cat" || feat_type == "agg")
-    {
-      //Find and prepare formulation
-      auto formulation = formulations->get_formulation(feat_id);
-      formulation->set_output_stream(formulations->get_output_root() + feat_id + ".csv"); // MERGE bd63e5b
-      // TODO: add command line or config option to have this be omitted
-      //FIXME why isn't default param working here??? get_output_header_line() fails.
-      formulation->write_output("Time Step,""Time,"+formulation->get_output_header_line(",")+"\n");
-      //Find upstream nexus ids
-      origins = network.get_origination_ids(feat_id);
+        destinations  = network.get_destination_ids(feat_id);
+        if(hy_features::identifiers::isCatchment(feat_type))
+        {
+          //Find and prepare formulation
+          auto formulation = formulations->get_formulation(feat_id);
+          formulation->set_output_stream(formulations->get_output_root() + feat_id + ".csv");
+          // TODO: add command line or config option to have this be omitted
+          //FIXME why isn't default param working here??? get_output_header_line() fails.
+          formulation->write_output("Time Step,""Time,"+formulation->get_output_header_line(",")+"\n");
+          //Find upstream nexus ids
+          origins = network.get_origination_ids(feat_id);
 
-      // get the catchment layer from the hydro fabric
-      const auto& cat_json_node = fabric->get_feature(feat_id);
-      long lyr = cat_json_node->has_key("layer") ? cat_json_node->get_property("layer").as_natural_number() : 0;
+          // get the catchment layer from the hydro fabric
+          const auto& cat_json_node = fabric->get_feature(feat_id);
+          long lyr = cat_json_node->has_key("layer") ? cat_json_node->get_property("layer").as_natural_number() : 0;
 
-      // add this layer to the set of layers if needed
-      if (hf_layers.find(lyr) == hf_layers.end() )
-      {
-          hf_layers.insert(lyr);
-      }
+          // add this layer to the set of layers if needed
+          if (hf_layers.find(lyr) == hf_layers.end() )
+          {
+              hf_layers.insert(lyr);
+          }
 
-      //Create the HY_Catchment with the formulation realization
-      std::shared_ptr<HY_Catchment> c = std::make_shared<HY_Catchment>(
-          HY_Catchment(feat_id, origins, destinations, formulation, lyr)
-        );
+          //Create the HY_Catchment with the formulation realization
+          std::shared_ptr<HY_Catchment> c = std::make_shared<HY_Catchment>(
+              HY_Catchment(feat_id, origins, destinations, formulation, lyr)
+            );
 
-      _catchments.emplace(feat_id, c);
-    }
-    else if(feat_type == "nex" || feat_type == "tnx")
-    {
-        _nexuses.emplace(feat_id, std::make_unique<HY_PointHydroNexus>(
-                                      HY_PointHydroNexus(feat_id, destinations) ));
-    }
-    else
-    {
-      std::cerr<<"HY_Features::HY_Features unknown feature identifier type "<<feat_type<<" for feature id."<<feat_id
-                <<" Skipping feature"<<std::endl;
-    }
-  }
-}
+          _catchments.emplace(feat_id, c);
+        }
+        else if(hy_features::identifiers::isNexus(feat_type))
+        {
+            _nexuses.emplace(feat_id, std::make_unique<HY_PointHydroNexus>(
+                                          HY_PointHydroNexus(feat_id, destinations) ));
+        }
+        else
+        {
+          std::cerr<<"HY_Features::HY_Features unknown feature identifier type "<<feat_type<<" for feature id."<<feat_id
+                   <<" Skipping feature"<<std::endl;
+        }
+      }
+
+}

src/core/nexus/HY_PointHydroNexusRemote.cpp

@@ -4,6 +4,7 @@
 
 #ifdef NGEN_MPI_ACTIVE
 
+#include <HY_Features_Ids.hpp>
 #include <chrono>
 #include <thread>
 
@@ -196,7 +197,7 @@ void HY_PointHydroNexusRemote::add_upstream_flow(double val, std::string catchme
 
 		    // fill the message buffer
 		    stored_sends.back().buffer->time_step = t;
-		    stored_sends.back().buffer->catchment_id = std::stoi( id.substr(4) );
+		    stored_sends.back().buffer->catchment_id = std::stoi( id.substr( id.find(hy_features::identifiers::seperator)+1 ) );
 
 		    // get the correct amount of flow using the inherted function this means are local bookkeeping is accurate
 		    stored_sends.back().buffer->flow = HY_PointHydroNexus::get_downstream_flow(id, t, 100.0);;