From d495f3f820508070210d22e88a910f28dca212cd Mon Sep 17 00:00:00 2001 From: Brad Whitlock Date: Wed, 30 Aug 2023 11:27:30 -0700 Subject: [PATCH 1/3] Added field_prefix to the options for conduit::blueprint::mpi::mesh::partition_map_back. (#1160) --- CHANGELOG.md | 3 +++ .../conduit_blueprint_mesh_partition.cpp | 20 ++++++++++++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2374ba4f2..5ee0c1ff6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,9 @@ and this project aspires to adhere to [Semantic Versioning](https://semver.org/s #### Relay - Added ability to read N-dimensional hyperslabs from HDF5 leaf arrays into linear memory arrays. +#### Blueprint +- The `conduit::blueprint::mpi::mesh::partition_map_back()` function was enhanced so it accepts a "field_prefix" value in its options. The prefix is used when looking for the `global_vertex_ids` field, which could have been created with a prefix by the same option in the `conduit::blueprint::mpi::mesh::generate_partition_field()` function. + ### Fixed #### Blueprint diff --git a/src/libs/blueprint/conduit_blueprint_mesh_partition.cpp b/src/libs/blueprint/conduit_blueprint_mesh_partition.cpp index ec515cfb9..509aaf5a4 100644 --- a/src/libs/blueprint/conduit_blueprint_mesh_partition.cpp +++ b/src/libs/blueprint/conduit_blueprint_mesh_partition.cpp @@ -10003,6 +10003,12 @@ Partitioner::map_back_fields(const conduit::Node& repart_mesh, } } + // Get a field prefix. Some of the mapping fields used here may have + // used a field_prefix when they were generated. + std::string field_prefix; + if(options.has_child("field_prefix")) + field_prefix = options.fetch_existing("field_prefix").as_string(); + // map repart domid -> orig domids vector> map_tgt_domains(repart_doms.size()); // map repart domid -> orig domid -> original elem/vertex ids @@ -10044,16 +10050,19 @@ Partitioner::map_back_fields(const conduit::Node& repart_mesh, // first. if (has_vert_fields) { + std::string global_vertex_ids(field_prefix + "global_vertex_ids"); + // map of orig domid -> global vert ids map> orig_dom_gvids; for (const auto& dom_ent : gid_to_orig_dom) { index_t orig_idx = dom_ent.first; const Node& orig_dom = *dom_ent.second; + const Node& orig_dom_fields = orig_dom.fetch_existing("fields"); vector& orig_gvids = orig_dom_gvids[orig_idx]; - if (orig_dom["fields"].has_child("global_vertex_ids")) + if (orig_dom_fields.has_child(global_vertex_ids)) { - const index_t_accessor gvids = orig_dom["fields/global_vertex_ids/values"].value(); + const index_t_accessor gvids = orig_dom_fields[global_vertex_ids + "/values"].value(); orig_gvids.resize(gvids.number_of_elements()); for (index_t ivert = 0; ivert < gvids.number_of_elements(); ivert++) { @@ -10073,12 +10082,13 @@ Partitioner::map_back_fields(const conduit::Node& repart_mesh, for (index_t repart_idx = 0; repart_idx < static_cast(repart_doms.size()); repart_idx++) { - const conduit::Node& dom = *repart_doms[repart_idx]; + const Node& dom = *repart_doms[repart_idx]; + const Node& dom_fields = dom.fetch_existing("fields"); std::unordered_map gvid_to_repart_vid; - if (dom["fields"].has_child("global_vertex_ids")) + if (dom_fields.has_child(global_vertex_ids)) { - const Node& global_vid_node = dom["fields/global_vertex_ids/values"]; + const Node& global_vid_node = dom_fields[global_vertex_ids + "/values"]; const index_t_accessor gvids = global_vid_node.value(); for (index_t ivert = 0; ivert < gvids.number_of_elements(); ivert++) { From cf8eef37a4eef52d2f8edf641a2bc36f0a1cf59e Mon Sep 17 00:00:00 2001 From: Andrew Bauer Date: Thu, 31 Aug 2023 19:18:37 -0400 Subject: [PATCH 2/3] Fixing typo in nestset validation (#1165) Co-authored-by: Andrew Bauer --- src/libs/blueprint/conduit_blueprint_mesh.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libs/blueprint/conduit_blueprint_mesh.cpp b/src/libs/blueprint/conduit_blueprint_mesh.cpp index 276a0a7e7..cbabce8f3 100644 --- a/src/libs/blueprint/conduit_blueprint_mesh.cpp +++ b/src/libs/blueprint/conduit_blueprint_mesh.cpp @@ -847,7 +847,7 @@ verify_single_domain(const Node &n, chld, chld_info, "topology", "topologies"); } - log::validation(info["nestets"],nset_res); + log::validation(info["nestsets"],nset_res); res &= nset_res; } } From a6b0b179716eb804a0749cc20083c24c21ed682b Mon Sep 17 00:00:00 2001 From: Cyrus Harrison Date: Fri, 1 Sep 2023 09:00:30 -0700 Subject: [PATCH 3/3] add support for conduit_json_external protocol (#1162) * add to_json conduit_json_external proto gen * start parsing support for conduit_json_external * work out paths for conduit_json_external support and improve efficiency of json parsing for several cases * cleanup and refactor to_hex helper * accept conduit_json_external as parse case for gen + parse sym --- CHANGELOG.md | 9 + scripts/uberenv_configs/spack_configs | 2 +- src/libs/conduit/conduit_generator.cpp | 467 ++++++++++++++++++---- src/libs/conduit/conduit_node.cpp | 177 +++++--- src/libs/conduit/conduit_node.hpp | 23 ++ src/libs/conduit/conduit_utils.hpp | 22 +- src/libs/relay/conduit_relay_io_silo.cpp | 1 - src/tests/conduit/t_conduit_generator.cpp | 26 +- src/tests/conduit/t_conduit_json.cpp | 94 +++++ src/tests/conduit/t_conduit_utils.cpp | 13 + src/tests/relay/t_relay_io_silo.cpp | 43 +- 11 files changed, 721 insertions(+), 156 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ee0c1ff6..7575c9a1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,18 @@ and this project aspires to adhere to [Semantic Versioning](https://semver.org/s ### Added +#### General +- Added `conduit_json_external` protocol. Creates a json schema representation of a node that includes all addresses that the node is pointing to. Parsing this schema will create a node equivalent to `set_external()`. + #### Relay - Added ability to read N-dimensional hyperslabs from HDF5 leaf arrays into linear memory arrays. + +### Changed + +#### General +- Improved the efficiency of json parsing logic. + #### Blueprint - The `conduit::blueprint::mpi::mesh::partition_map_back()` function was enhanced so it accepts a "field_prefix" value in its options. The prefix is used when looking for the `global_vertex_ids` field, which could have been created with a prefix by the same option in the `conduit::blueprint::mpi::mesh::generate_partition_field()` function. diff --git a/scripts/uberenv_configs/spack_configs b/scripts/uberenv_configs/spack_configs index 979ab0c13..68987700f 160000 --- a/scripts/uberenv_configs/spack_configs +++ b/scripts/uberenv_configs/spack_configs @@ -1 +1 @@ -Subproject commit 979ab0c136083082e2fcb86dffdf933391bd2e08 +Subproject commit 68987700f98c632e525e6eeb07337100dd797f16 diff --git a/src/libs/conduit/conduit_generator.cpp b/src/libs/conduit/conduit_generator.cpp index deb392bc7..de9790e13 100644 --- a/src/libs/conduit/conduit_generator.cpp +++ b/src/libs/conduit/conduit_generator.cpp @@ -167,7 +167,8 @@ class Generator::Parser static void parse_inline_leaf(const conduit_rapidjson::Value &jvalue, Node &node); - + static void* parse_inline_address(const conduit_rapidjson::Value& jvalue); + static void parse_inline_value(const conduit_rapidjson::Value &jvalue, Node &node); @@ -178,13 +179,21 @@ class Generator::Parser static void walk_pure_json_schema(Node *node, Schema *schema, const conduit_rapidjson::Value &jvalue); - + + // if data pointer is provided, data is copied into dest node static void walk_json_schema(Node *node, Schema *schema, void *data, const conduit_rapidjson::Value &jvalue, index_t curr_offset); - + + // if data pointer is provided, data is set_external into dest node + static void walk_json_schema_external(Node *node, + Schema *schema, + void *data, + const conduit_rapidjson::Value &jvalue, + index_t curr_offset); + static void parse_base64(Node *node, const conduit_rapidjson::Value &jvalue); @@ -988,6 +997,28 @@ Generator::Parser::JSON::parse_inline_leaf(const conduit_rapidjson::Value &jvalu } } +//---------------------------------------------------------------------------// +void * +Generator::Parser::JSON::parse_inline_address(const conduit_rapidjson::Value &jvalue) +{ + void * res = nullptr; + if(jvalue.IsString()) + { + std::string sval(jvalue.GetString()); + res = utils::hex_string_to_value(sval); + } + // else if(jvalue.IsNumber()) + // { + // // TODO: FUTURE? ... + // } + else + { + CONDUIT_ERROR("JSON Generator error:\n" + << "inline address should be a string"); + } + return res; +} + //---------------------------------------------------------------------------// void Generator::Parser::JSON::parse_inline_value(const conduit_rapidjson::Value &jvalue, @@ -1343,28 +1374,46 @@ Generator::Parser::JSON::walk_json_schema(Node *node, else { // handle leaf node with explicit props - DataType dtype; - - parse_leaf_dtype(jvalue,curr_offset,dtype); - - if(data != NULL) + DataType src_dtype; + parse_leaf_dtype(jvalue,curr_offset,src_dtype); + + DataType des_dtype; + src_dtype.compact_to(des_dtype); + + // check for explciit address + if(jvalue.HasMember("address")) { - // node is already linked to the schema pointer - schema->set(dtype); - node->set_data_ptr(data); + void *data_ptr = parse_inline_address(jvalue["address"]); + node->set_external(src_dtype,data_ptr); } else { - // node is already linked to the schema pointer - // we need to dynamically alloc - node->set(dtype); // causes an init - } + if(data != NULL) + { + uint8 *src_data_ptr = ((uint8*)data) + src_dtype.offset(); + // node is already linked to the schema pointer + // we need to dynamically alloc, use compact dtype + node->set(des_dtype); // causes an init + // copy bytes from src data to node's memory + utils::conduit_memcpy_strided_elements(node->data_ptr(), // dest data + des_dtype.number_of_elements(), // num ele + des_dtype.element_bytes(), // ele bytes + des_dtype.stride(), // dest stride + src_data_ptr, // src data + src_dtype.stride()); // src stride + } + else + { + // node is already linked to the schema pointer + // we need to dynamically alloc, use compact dtype + node->set(des_dtype); // causes an init + } - // check for inline json values - if(jvalue.HasMember("value")) - { - - parse_inline_value(jvalue["value"],*node); + // check for inline json values + if(jvalue.HasMember("value")) + { + parse_inline_value(jvalue["value"],*node); + } } } } @@ -1447,7 +1496,6 @@ Generator::Parser::JSON::walk_json_schema(Node *node, { // node is already linked to the schema pointer node->set_data_ptr(data); - } else { @@ -1464,6 +1512,201 @@ Generator::Parser::JSON::walk_json_schema(Node *node, } } +//---------------------------------------------------------------------------// +void +Generator::Parser::JSON::walk_json_schema_external(Node *node, + Schema *schema, + void *data, + const conduit_rapidjson::Value &jvalue, + index_t curr_offset) +{ + // object cases + if(jvalue.IsObject()) + { + + if (jvalue.HasMember("dtype")) + { + // if dtype is an object, we have a "list_of" case + const conduit_rapidjson::Value &dt_value = jvalue["dtype"]; + if(dt_value.IsObject()) + { + index_t length =1; + if(jvalue.HasMember("length")) + { + if(jvalue["length"].IsNumber()) + { + length = jvalue["length"].GetInt(); + } + else if(jvalue["length"].IsObject() && + jvalue["length"].HasMember("reference")) + { + std::string ref_path = + jvalue["length"]["reference"].GetString(); + length = node->fetch(ref_path).to_index_t(); + } + else + { + CONDUIT_ERROR("JSON Parsing error:\n" + << "'length' must be a number " + << "or reference."); + } + + } + // we will create `length' # of objects of obj des by dt_value + + // TODO: we only need to parse this once, not leng # of times + // but this is the easiest way to start. + for(index_t i=0;i< length;i++) + { + schema->append(); + Schema *curr_schema = schema->child_ptr(i); + Node *curr_node = new Node(); + curr_node->set_schema_ptr(curr_schema); + curr_node->set_parent(node); + node->append_node_ptr(curr_node); + walk_json_schema_external(curr_node, + curr_schema, + data, + dt_value, + curr_offset); + // auto offset only makes sense when we have data + if(data != NULL) + curr_offset += curr_schema->total_strided_bytes(); + } + } + else + { + // handle leaf node with explicit props + DataType dtype; + + parse_leaf_dtype(jvalue,curr_offset,dtype); + + // check for explciit address + if(jvalue.HasMember("address")) + { + void *data_ptr = parse_inline_address(jvalue["address"]); + node->set_external(dtype,data_ptr); + } + else + { + + if(data != NULL) + { + // node is already linked to the schema pointer + schema->set(dtype); + node->set_data_ptr(data); + } + else + { + // node is already linked to the schema pointer + // we need to dynamically alloc + node->set(dtype); // causes an init + } + + // check for inline json values + if(jvalue.HasMember("value")) + { + parse_inline_value(jvalue["value"],*node); + } + } + } + } + else // object case + { + schema->set(DataType::object()); + // standard object case - loop over all entries + for (conduit_rapidjson::Value::ConstMemberIterator itr = + jvalue.MemberBegin(); + itr != jvalue.MemberEnd(); ++itr) + { + std::string entry_name(itr->name.GetString()); + + // json files may have duplicate object names + // we could provide some clear semantics, such as: + // always use first instance, or always use last instance. + // however duplicate object names are most likely a + // typo, so it's best to throw an error + // + // also its highly unlikely that the auto offset case + // could safely deal with offsets for the + // duplicate key case + if(schema->has_child(entry_name)) + { + CONDUIT_ERROR("JSON Generator error:\n" + << "Duplicate JSON object name: " + << utils::join_path(node->path(),entry_name)); + } + + Schema *curr_schema = &schema->add_child(entry_name); + + Node *curr_node = new Node(); + curr_node->set_schema_ptr(curr_schema); + curr_node->set_parent(node); + node->append_node_ptr(curr_node); + walk_json_schema_external(curr_node, + curr_schema, + data, + itr->value, + curr_offset); + + // auto offset only makes sense when we have data + if(data != NULL) + curr_offset += curr_schema->total_strided_bytes(); + } + + } + } + // List case + else if (jvalue.IsArray()) + { + schema->set(DataType::list()); + + for (conduit_rapidjson::SizeType i = 0; i < jvalue.Size(); i++) + { + schema->append(); + Schema *curr_schema = schema->child_ptr(i); + Node *curr_node = new Node(); + curr_node->set_schema_ptr(curr_schema); + curr_node->set_parent(node); + node->append_node_ptr(curr_node); + walk_json_schema_external(curr_node, + curr_schema, + data, + jvalue[i], + curr_offset); + // auto offset only makes sense when we have data + if(data != NULL) + curr_offset += curr_schema->total_strided_bytes(); + } + } + // Simplest case, handles "uint32", "float64", with extended type info + else if(jvalue.IsString()) + { + DataType dtype; + parse_leaf_dtype(jvalue,curr_offset,dtype); + schema->set(dtype); + + if(data != NULL) + { + // node is already linked to the schema pointer + node->set_data_ptr(data); + } + else + { + // node is already linked to the schema pointer + // we need to dynamically alloc + node->set(dtype); // causes an init + } + } + else + { + CONDUIT_ERROR("JSON Generator error:\n" + << "Invalid JSON type for parsing Node." + << " Expected: JSON Object, Array, or String"); + } +} + + //---------------------------------------------------------------------------// void Generator::Parser::JSON::parse_base64(Node *node, @@ -2199,72 +2442,154 @@ Generator::walk(Schema &schema) const void Generator::walk(Node &node) const { - /// TODO: This is an inefficient code path, need better solution? - Node n; - walk_external(n); - n.compact_to(node); + // try catch b/c: + // if something goes wrong we will clear the node and re-throw + // if exception is caught downstream + // we want node to be empty instead of partially inited + try + { + node.reset(); + // json, yaml, and conduit_base64_json don't leverage "data" + if(m_protocol == "json") + { + conduit_rapidjson::Document document; + std::string res = utils::json_sanitize(m_schema); + + if(document.Parse(res.c_str()).HasParseError()) + { + CONDUIT_JSON_PARSE_ERROR(res, document); + } + + Parser::JSON::walk_pure_json_schema(&node, + node.schema_ptr(), + document); + } + else if(m_protocol == "yaml") + { + // errors will flow up from this call + Parser::YAML::walk_pure_yaml_schema(&node, + node.schema_ptr(), + m_schema.c_str()); + } + else if( m_protocol == "conduit_base64_json") + { + conduit_rapidjson::Document document; + std::string res = utils::json_sanitize(m_schema); + + if(document.Parse(res.c_str()).HasParseError()) + { + CONDUIT_JSON_PARSE_ERROR(res, document); + } + + Parser::JSON::parse_base64(&node, + document); + } + + else if( m_protocol == "conduit_json" || m_protocol == "conduit_json_external") + { + // Note: conduit_json_external if case here for symmetry with gen / read options + // this case is fully handled by conduit_json logic + conduit_rapidjson::Document document; + std::string res = utils::json_sanitize(m_schema); + + if(document.Parse(res.c_str()).HasParseError()) + { + CONDUIT_JSON_PARSE_ERROR(res, document); + } + index_t curr_offset = 0; + + Parser::JSON::walk_json_schema(&node, + node.schema_ptr(), + m_data, + document, + curr_offset); + } + else + { + CONDUIT_ERROR("Generator unknown parsing protocol: " << m_protocol); + } + } + catch(const conduit::Error& e) + { + node.reset(); + throw e; + } } //---------------------------------------------------------------------------// void Generator::walk_external(Node &node) const { - node.reset(); - // if data is null, we can parse the schema via the other 'walk' method - if(m_protocol == "json") + // try catch b/c: + // if something goes wrong we will clear the node and re-throw + // if exception is caught downstream + // we want node to be empty instead of partially inited + try { - conduit_rapidjson::Document document; - std::string res = utils::json_sanitize(m_schema); - - if(document.Parse(res.c_str()).HasParseError()) + node.reset(); + // if data is null, we can parse the schema via other 'walk' methods + if(m_protocol == "json") { - CONDUIT_JSON_PARSE_ERROR(res, document); - } + conduit_rapidjson::Document document; + std::string res = utils::json_sanitize(m_schema); + + if(document.Parse(res.c_str()).HasParseError()) + { + CONDUIT_JSON_PARSE_ERROR(res, document); + } - Parser::JSON::walk_pure_json_schema(&node, - node.schema_ptr(), - document); - } - else if(m_protocol == "yaml") - { - // errors will flow up from this call - Parser::YAML::walk_pure_yaml_schema(&node, - node.schema_ptr(), - m_schema.c_str()); - } - else if( m_protocol == "conduit_base64_json") - { - conduit_rapidjson::Document document; - std::string res = utils::json_sanitize(m_schema); - - if(document.Parse(res.c_str()).HasParseError()) + Parser::JSON::walk_pure_json_schema(&node, + node.schema_ptr(), + document); + } + else if(m_protocol == "yaml") { - CONDUIT_JSON_PARSE_ERROR(res, document); + // errors will flow up from this call + Parser::YAML::walk_pure_yaml_schema(&node, + node.schema_ptr(), + m_schema.c_str()); } - - Parser::JSON::parse_base64(&node, - document); - } - else if( m_protocol == "conduit_json") - { - conduit_rapidjson::Document document; - std::string res = utils::json_sanitize(m_schema); - - if(document.Parse(res.c_str()).HasParseError()) + else if( m_protocol == "conduit_base64_json") { - CONDUIT_JSON_PARSE_ERROR(res, document); + conduit_rapidjson::Document document; + std::string res = utils::json_sanitize(m_schema); + + if(document.Parse(res.c_str()).HasParseError()) + { + CONDUIT_JSON_PARSE_ERROR(res, document); + } + + Parser::JSON::parse_base64(&node, + document); } - index_t curr_offset = 0; + else if( m_protocol == "conduit_json" || m_protocol == "conduit_json_external") + { + // Note: conduit_json_external if case here for symmetry with gen / read options + // this case is fully handled by conduit_json logic + conduit_rapidjson::Document document; + std::string res = utils::json_sanitize(m_schema); + + if(document.Parse(res.c_str()).HasParseError()) + { + CONDUIT_JSON_PARSE_ERROR(res, document); + } + index_t curr_offset = 0; - Parser::JSON::walk_json_schema(&node, - node.schema_ptr(), - m_data, - document, - curr_offset); + Parser::JSON::walk_json_schema_external(&node, + node.schema_ptr(), + m_data, + document, + curr_offset); + } + else + { + CONDUIT_ERROR("Generator unknown parsing protocol: " << m_protocol); + } } - else + catch(const conduit::Error& e) { - CONDUIT_ERROR("Generator unknown parsing protocol: " << m_protocol); + node.reset(); + throw e; } } diff --git a/src/libs/conduit/conduit_node.cpp b/src/libs/conduit/conduit_node.cpp index 000de3f18..9bf97ba8f 100644 --- a/src/libs/conduit/conduit_node.cpp +++ b/src/libs/conduit/conduit_node.cpp @@ -186,15 +186,14 @@ Node::Node(const Schema &schema, bool external) { init_defaults(); - std::string json_schema =schema.to_json(); - Generator g(json_schema,"conduit_json",data); + if(external) { - g.walk_external(*this); + set_external(schema,data); } else { - g.walk(*this); + set(schema,data); } } @@ -13206,6 +13205,10 @@ Node::to_json_stream(std::ostream &os, { return to_detailed_json(os,indent,depth,pad,eoe); } + else if(protocol == "conduit_json_external") + { + return to_detailed_json_external(os,indent,depth,pad,eoe); + } else if(protocol == "conduit_base64_json") { return to_base64_json(os,indent,depth,pad,eoe); @@ -13455,13 +13458,14 @@ Node::to_yaml_default() const //---------------------------------------------------------------------------// std::string Node::to_json_generic(bool detailed, + bool address, index_t indent, index_t depth, const std::string &pad, const std::string &eoe) const { std::ostringstream oss; - to_json_generic(oss,detailed,indent,depth,pad,eoe); + to_json_generic(oss,detailed,address,indent,depth,pad,eoe); return oss.str(); } @@ -13470,6 +13474,7 @@ Node::to_json_generic(bool detailed, void Node::to_json_generic(const std::string &stream_path, bool detailed, + bool address, index_t indent, index_t depth, const std::string &pad, @@ -13482,7 +13487,7 @@ Node::to_json_generic(const std::string &stream_path, CONDUIT_ERROR(" failed to open file: " << "\"" << stream_path << "\""); } - to_json_generic(ofs,detailed,indent,depth,pad,eoe); + to_json_generic(ofs,detailed,address,indent,depth,pad,eoe); ofs.close(); } @@ -13491,6 +13496,7 @@ Node::to_json_generic(const std::string &stream_path, void Node::to_json_generic(std::ostream &os, bool detailed, + bool address, index_t indent, index_t depth, const std::string &pad, @@ -13511,6 +13517,7 @@ Node::to_json_generic(std::ostream &os, os << "\""<< m_schema->object_order()[i] << "\": "; m_children[i]->to_json_generic(os, detailed, + address, indent, depth+1, pad, @@ -13534,6 +13541,7 @@ Node::to_json_generic(std::ostream &os, utils::indent(os,indent,depth+1,pad); m_children[i]->to_json_generic(os, detailed, + address, indent, depth+1, pad, @@ -13567,55 +13575,64 @@ Node::to_json_generic(std::ostream &os, os << dtype_content; os << "\"," << eoe; utils::indent(os,indent,depth+1,pad); - os << "\"value\": "; + if(!address) + { + os << "\"value\": "; + } } - switch(dtype().id()) + if(address) { - // ints - case DataType::INT8_ID: - as_int8_array().to_json_stream(os); - break; - case DataType::INT16_ID: - as_int16_array().to_json_stream(os); - break; - case DataType::INT32_ID: - as_int32_array().to_json_stream(os); - break; - case DataType::INT64_ID: - as_int64_array().to_json_stream(os); - break; - // uints - case DataType::UINT8_ID: - as_uint8_array().to_json_stream(os); - break; - case DataType::UINT16_ID: - as_uint16_array().to_json_stream(os); - break; - case DataType::UINT32_ID: - as_uint32_array().to_json_stream(os); - break; - case DataType::UINT64_ID: - as_uint64_array().to_json_stream(os); - break; - // floats - case DataType::FLOAT32_ID: - as_float32_array().to_json_stream(os); - break; - case DataType::FLOAT64_ID: - as_float64_array().to_json_stream(os); - break; - // char8_str - case DataType::CHAR8_STR_ID: - os << "\"" - << utils::escape_special_chars(as_string()) - << "\""; - break; - // empty - case DataType::EMPTY_ID: - os << "null"; - break; - + os << "\"address\": \"" << utils::to_hex_string(m_data) << "\""; + } + else + { + switch(dtype().id()) + { + // ints + case DataType::INT8_ID: + as_int8_array().to_json_stream(os); + break; + case DataType::INT16_ID: + as_int16_array().to_json_stream(os); + break; + case DataType::INT32_ID: + as_int32_array().to_json_stream(os); + break; + case DataType::INT64_ID: + as_int64_array().to_json_stream(os); + break; + // uints + case DataType::UINT8_ID: + as_uint8_array().to_json_stream(os); + break; + case DataType::UINT16_ID: + as_uint16_array().to_json_stream(os); + break; + case DataType::UINT32_ID: + as_uint32_array().to_json_stream(os); + break; + case DataType::UINT64_ID: + as_uint64_array().to_json_stream(os); + break; + // floats + case DataType::FLOAT32_ID: + as_float32_array().to_json_stream(os); + break; + case DataType::FLOAT64_ID: + as_float64_array().to_json_stream(os); + break; + // char8_str + case DataType::CHAR8_STR_ID: + os << "\"" + << utils::escape_special_chars(as_string()) + << "\""; + break; + // empty + case DataType::EMPTY_ID: + os << "null"; + break; + } } if(detailed) @@ -13637,7 +13654,7 @@ Node::to_pure_json(index_t indent, const std::string &pad, const std::string &eoe) const { - return to_json_generic(false,indent,depth,pad,eoe); + return to_json_generic(false,false,indent,depth,pad,eoe); } //---------------------------------------------------------------------------// @@ -13655,7 +13672,7 @@ Node::to_pure_json(const std::string &stream_path, CONDUIT_ERROR(" failed to open file: " << "\"" << stream_path << "\""); } - to_json_generic(ofs,false,indent,depth,pad,eoe); + to_json_generic(ofs,false,false,indent,depth,pad,eoe); ofs.close(); } @@ -13667,7 +13684,7 @@ Node::to_pure_json(std::ostream &os, const std::string &pad, const std::string &eoe) const { - to_json_generic(os,false,indent,depth,pad,eoe); + to_json_generic(os,false,false,indent,depth,pad,eoe); } //---------------------------------------------------------------------------// @@ -13677,7 +13694,7 @@ Node::to_detailed_json(index_t indent, const std::string &pad, const std::string &eoe) const { - return to_json_generic(true,indent,depth,pad,eoe); + return to_json_generic(true,false,indent,depth,pad,eoe); } //---------------------------------------------------------------------------// @@ -13695,7 +13712,7 @@ Node::to_detailed_json(const std::string &stream_path, CONDUIT_ERROR(" failed to open file: " << "\"" << stream_path << "\""); } - to_json_generic(ofs,true,indent,depth,pad,eoe); + to_json_generic(ofs,true,false,indent,depth,pad,eoe); ofs.close(); } @@ -13708,9 +13725,53 @@ Node::to_detailed_json(std::ostream &os, const std::string &pad, const std::string &eoe) const { - to_json_generic(os,true,indent,depth,pad,eoe); + to_json_generic(os,true,false,indent,depth,pad,eoe); +} + + +//---------------------------------------------------------------------------// +std::string +Node::to_detailed_json_external(index_t indent, + index_t depth, + const std::string &pad, + const std::string &eoe) const +{ + return to_json_generic(true,true,indent,depth,pad,eoe); } +//---------------------------------------------------------------------------// +void +Node::to_detailed_json_external(const std::string &stream_path, + index_t indent, + index_t depth, + const std::string &pad, + const std::string &eoe) const +{ + std::ofstream ofs; + ofs.open(stream_path.c_str()); + if(!ofs.is_open()) + { + CONDUIT_ERROR(" failed to open file: " + << "\"" << stream_path << "\""); + } + to_json_generic(ofs,true,true,indent,depth,pad,eoe); + ofs.close(); +} + + +//---------------------------------------------------------------------------// +void +Node::to_detailed_json_external(std::ostream &os, + index_t indent, + index_t depth, + const std::string &pad, + const std::string &eoe) const +{ + to_json_generic(os,true,true,indent,depth,pad,eoe); +} + + + //---------------------------------------------------------------------------// std::string Node::to_base64_json(index_t indent, diff --git a/src/libs/conduit/conduit_node.hpp b/src/libs/conduit/conduit_node.hpp index 6c1a79b28..5325af696 100644 --- a/src/libs/conduit/conduit_node.hpp +++ b/src/libs/conduit/conduit_node.hpp @@ -4576,12 +4576,14 @@ class CONDUIT_API Node // the generic to_json methods are used by the specialized cases //------------------------------------------------------------------------- std::string to_json_generic(bool detailed, + bool address, index_t indent=2, index_t depth=0, const std::string &pad=" ", const std::string &eoe="\n") const; void to_json_generic(const std::string &stream_path, + bool address, bool detailed, index_t indent=2, index_t depth=0, @@ -4590,6 +4592,7 @@ class CONDUIT_API Node void to_json_generic(std::ostream &os, bool detailed, + bool address, index_t indent=2, index_t depth=0, const std::string &pad=" ", @@ -4635,6 +4638,26 @@ class CONDUIT_API Node const std::string &pad=" ", const std::string &eoe="\n") const; + //------------------------------------------------------------------------- + // transforms the node to detailed json with address entry + //------------------------------------------------------------------------- + std::string to_detailed_json_external(index_t indent=2, + index_t depth=0, + const std::string &pad=" ", + const std::string &eoe="\n") const; + + void to_detailed_json_external(const std::string &stream_path, + index_t indent=2, + index_t depth=0, + const std::string &pad=" ", + const std::string &eoe="\n") const; + + void to_detailed_json_external(std::ostream &os, + index_t indent=2, + index_t depth=0, + const std::string &pad=" ", + const std::string &eoe="\n") const; + //------------------------------------------------------------------------- // transforms the node to json with data payload encoded using base64 //------------------------------------------------------------------------- diff --git a/src/libs/conduit/conduit_utils.hpp b/src/libs/conduit/conduit_utils.hpp index 453030d5a..d841c1e08 100644 --- a/src/libs/conduit/conduit_utils.hpp +++ b/src/libs/conduit/conduit_utils.hpp @@ -20,6 +20,7 @@ #include #include #include +#include //----------------------------------------------------------------------------- // -- conduit includes -- @@ -521,9 +522,9 @@ namespace utils template< typename T > std::string to_hex_string(T value) { - std::stringstream oss; - oss << std::hex << value; - return oss.str(); + std::stringstream oss; + oss << std::hex << value; + return oss.str(); } //----------------------------------------------------------------------------- @@ -565,7 +566,20 @@ namespace utils T res; std::istringstream iss(s); iss >> res; - return res; + return res; + } + + // declare then define to avoid icc warnings + template< typename T > + T hex_string_to_value(const std::string &s); + + template< typename T > + T hex_string_to_value(const std::string &s) + { + T res; + std::istringstream iss(s); + iss >> std::hex >> res; + return res; } diff --git a/src/libs/relay/conduit_relay_io_silo.cpp b/src/libs/relay/conduit_relay_io_silo.cpp index 4f888a661..d4da1228d 100644 --- a/src/libs/relay/conduit_relay_io_silo.cpp +++ b/src/libs/relay/conduit_relay_io_silo.cpp @@ -240,7 +240,6 @@ void silo_read(DBfile *dbfile, char *data = new char[data_len]; - DBReadVar(dbfile, src_json.c_str(), schema); DBReadVar(dbfile, src_data.c_str(), data); diff --git a/src/tests/conduit/t_conduit_generator.cpp b/src/tests/conduit/t_conduit_generator.cpp index 60923bdd7..9f7775e6b 100644 --- a/src/tests/conduit/t_conduit_generator.cpp +++ b/src/tests/conduit/t_conduit_generator.cpp @@ -451,8 +451,6 @@ TEST(conduit_generator, simple_gen_schema_yaml) } - - //----------------------------------------------------------------------------- TEST(conduit_generator, yaml_parsing_errors) { @@ -466,3 +464,27 @@ TEST(conduit_generator, yaml_parsing_errors) } +//----------------------------------------------------------------------------- +TEST(conduit_generator, json_external_gen) +{ + Node n; + n["a"] = 42; + n["b"] = 144; + n["c/d"] = 3.1415; + n.print(); + + std::string json_ext = n.to_json("conduit_json_external"); + std::cout << json_ext << std::endl; + + Node n2; + Generator g(json_ext,"conduit_json"); + g.walk(n2); + n2.print(); + + // n2 should be full external, refing data owned by n + EXPECT_EQ(n["a"].data_ptr(),n2["a"].data_ptr()); + EXPECT_EQ(n["b"].data_ptr(),n2["b"].data_ptr()); + EXPECT_EQ(n["c"].data_ptr(),n2["c"].data_ptr()); + +} + diff --git a/src/tests/conduit/t_conduit_json.cpp b/src/tests/conduit/t_conduit_json.cpp index e62eecea5..7444f088d 100644 --- a/src/tests/conduit/t_conduit_json.cpp +++ b/src/tests/conduit/t_conduit_json.cpp @@ -896,3 +896,97 @@ TEST(conduit_json, to_json_opts) } +//----------------------------------------------------------------------------- +TEST(conduit_json, to_json_external) +{ + + uint32 a_val = 10; + uint32 b_val = 20; + + Node n; + n["a"] = a_val; + n["b"] = b_val; + + EXPECT_EQ(n["a"].as_uint32(),a_val); + EXPECT_EQ(n["b"].as_uint32(),b_val); + + std::cout << n.to_json("conduit_json_external") << std::endl; +} + +//----------------------------------------------------------------------------- +TEST(conduit_json, to_json_external_2) +{ + + uint32 a_val = 10; + uint32 b_val = 20; + + Node n; + n["a"] = a_val; + n["path/b"] = b_val; + + EXPECT_EQ(n["a"].as_uint32(),a_val); + EXPECT_EQ(n["path/b"].as_uint32(),b_val); + + std::string json = n.to_json("conduit_json_external") ; + + Node n2, n3; + + n2.set_external(n); + n3.parse(json,"conduit_json"); + + Node info; + EXPECT_FALSE(n.diff(n2,info)); + EXPECT_FALSE(n.diff(n3,info)); + + EXPECT_FALSE(n2.diff(n3,info)); + + EXPECT_EQ(n["a"].data_ptr(),n2["a"].data_ptr()); + EXPECT_EQ(n["path/b"].data_ptr(),n2["path/b"].data_ptr()); + + EXPECT_EQ(n["a"].data_ptr(),n3["a"].data_ptr()); + EXPECT_EQ(n["path/b"].data_ptr(),n3["path/b"].data_ptr()); +} + +//----------------------------------------------------------------------------- +TEST(conduit_json, json_walk_with_pointer) +{ + uint32 a_val = 20; + uint32 b_val = 8; + uint32 c_val = 13; + + Node n; + n["a"] = a_val; + n["b"] = b_val; + n["c"] = c_val; + + EXPECT_EQ(n["a"].as_uint32(), a_val); + EXPECT_EQ(n["b"].as_uint32(), b_val); + EXPECT_EQ(n["c"].as_uint32(), c_val); + + Node n_compact; + n.compact_to(n_compact); + + n_compact.print(); + + std::string schema_json = n_compact.schema().to_json(); + std::cout << schema_json << std::endl; + Node n_res; + + Generator node_gen(schema_json, "conduit_json", n_compact.data_ptr()); + /// gen copy + node_gen.walk(n_res); + + std::cout << "round trip" << std::endl; + n_res.print(); + + std::cout << n_res.schema().to_json() << std::endl; + + EXPECT_EQ(n_res["a"].as_uint32(), a_val); + EXPECT_EQ(n_res["b"].as_uint32(), b_val); + EXPECT_EQ(n_res["c"].as_uint32(), c_val); +} + + + + + diff --git a/src/tests/conduit/t_conduit_utils.cpp b/src/tests/conduit/t_conduit_utils.cpp index fb4a60fab..c950c2dd1 100644 --- a/src/tests/conduit/t_conduit_utils.cpp +++ b/src/tests/conduit/t_conduit_utils.cpp @@ -420,6 +420,9 @@ TEST(conduit_utils, base64_enc_dec) // apply schema Node n_res(n.schema(),b64_decode_ptr,false); + n.print(); + n_res.print(); + // check we have the same values EXPECT_EQ(n_src["a"].as_int32(), n_res["a"].as_int32()); EXPECT_EQ(n_src["b"].as_int32(), n_res["b"].as_int32()); @@ -894,6 +897,16 @@ TEST(conduit_utils, value_fits) } +//----------------------------------------------------------------------------- +TEST(conduit_utils, to_and_from_hex_string) +{ + int64 val = 1024; + std::string hstring = utils::to_hex_string(val); + std::cout << hstring << std::endl; + int64 val_check = conduit::utils::hex_string_to_value(hstring); + EXPECT_EQ(val,val_check); +} + //----------------------------------------------------------------------------- TEST(conduit_utils, timer) { diff --git a/src/tests/relay/t_relay_io_silo.cpp b/src/tests/relay/t_relay_io_silo.cpp index 9f063569b..f33832339 100644 --- a/src/tests/relay/t_relay_io_silo.cpp +++ b/src/tests/relay/t_relay_io_silo.cpp @@ -36,11 +36,16 @@ TEST(conduit_relay_io_silo, conduit_silo_cold_storage) EXPECT_EQ(n["b"].as_uint32(), b_val); EXPECT_EQ(n["c"].as_uint32(), c_val); + n.print(); + io::silo_write(n,"tout_cold_storage_test.silo:myobj"); Node n_load; io::silo_read("tout_cold_storage_test.silo:myobj",n_load); + std::cout << "round trip" << std::endl; + n_load.print(); + EXPECT_EQ(n_load["a"].as_uint32(), a_val); EXPECT_EQ(n_load["b"].as_uint32(), b_val); EXPECT_EQ(n_load["c"].as_uint32(), c_val); @@ -94,7 +99,7 @@ TEST(conduit_relay_io_silo, load_mesh_geometry) std::vector dims_vec = {2, 3, /*2,*/ 2, /*2,*/ 2}; std::vector coordset_length_vec = {4, 8, /*36,*/ 1994, /*16,*/ 961}; std::vector topology_length_vec = {1, 1, /*33,*/ 1920, /*9,*/ 900}; - for (int i = 0; i < filename_vec.size(); ++i) + for (int i = 0; i < filename_vec.size(); ++i) { Node mesh, info; std::string path = utils::join_file_path("overlink", filename_vec.at(i)); @@ -334,9 +339,9 @@ TEST(conduit_relay_io_silo, round_trip_julia) } //----------------------------------------------------------------------------- -// +// // test read and write semantics -// +// //----------------------------------------------------------------------------- TEST(conduit_relay_io_silo, read_and_write_semantics) @@ -377,13 +382,13 @@ TEST(conduit_relay_io_silo, read_and_write_semantics) } } //----------------------------------------------------------------------------- -// +// // special case tests -// +// //----------------------------------------------------------------------------- // var is not defined on a domain -// +// // tests the silo "EMPTY" capability TEST(conduit_relay_io_silo, missing_domain_var) { @@ -427,11 +432,11 @@ TEST(conduit_relay_io_silo, missing_domain_var) //----------------------------------------------------------------------------- // mesh is not defined on a domain -// +// // This case is much less interesting. // data passes through the clean mesh filter which // deletes domains that are missing topos. -// They simply are not part of the mesh and so silo +// They simply are not part of the mesh and so silo // doesn't have to deal with it. TEST(conduit_relay_io_silo, missing_domain_mesh_trivial) { @@ -508,7 +513,7 @@ TEST(conduit_relay_io_silo, missing_domain_mesh) remove_path_if_exists(filename); io::silo::save_mesh(save_mesh, basename); - + opts["mesh_name"] = "mesh_topo2"; io::silo::load_mesh(filename, opts, load_mesh); opts["mesh_name"] = "mesh_topo"; @@ -636,9 +641,9 @@ TEST(conduit_relay_io_silo, unstructured_points) //----------------------------------------------------------------------------- -// +// // save and read option tests -// +// // save options: /// opts: @@ -650,7 +655,7 @@ TEST(conduit_relay_io_silo, unstructured_points) /// silo_type: "default", "pdb", "hdf5", "unknown" /// when the file we are writing to exists, "default" ==> "unknown" /// else, "default" ==> "hdf5" -/// note: these are additional silo_type options that we could add +/// note: these are additional silo_type options that we could add /// support for in the future: /// "hdf5_sec2", "hdf5_stdio", "hdf5_mpio", "hdf5_mpiposix", "taurus" /// @@ -728,8 +733,8 @@ TEST(conduit_relay_io_silo, round_trip_save_option_number_of_files) opts["file_style"] = "multi_file"; opts["number_of_files"] = number_of_files[i]; - const std::string basename = "silo_save_option_number_of_files_" + - std::to_string(number_of_files[i]) + + const std::string basename = "silo_save_option_number_of_files_" + + std::to_string(number_of_files[i]) + "_spiral"; const std::string filename = basename + ".cycle_000000.root"; @@ -879,7 +884,7 @@ TEST(conduit_relay_io_silo, round_trip_save_option_silo_type) // this is to pass the diff, as silo will add cycle in if it is not there save_mesh["state/cycle"] = (int64) 0; save_mesh["state/domain_id"] = 0; - + silo_name_changer("mesh", save_mesh); // the loaded mesh will be in the multidomain format @@ -969,7 +974,7 @@ TEST(conduit_relay_io_silo, read_silo) {"multidir_test_data", "multidir0000", ".root", "Mesh" }, }; - for (int i = 0; i < file_info.size(); i ++) + for (int i = 0; i < file_info.size(); i ++) { const std::string dirname = file_info[i][0]; const std::string basename = file_info[i][1]; @@ -1023,7 +1028,7 @@ TEST(conduit_relay_io_silo, read_fake_overlink) {"utpyr4", "OvlTop", ".silo", "MMESH"}, }; - for (int i = 0; i < file_info.size(); i ++) + for (int i = 0; i < file_info.size(); i ++) { const std::string dirname = file_info[i][0]; const std::string basename = file_info[i][1]; @@ -1082,7 +1087,7 @@ TEST(conduit_relay_io_silo, read_overlink_symlink_format) {".", "donordiv.s2_materials3", ".silo", "MMESH"}, }; - for (int i = 0; i < file_info.size(); i ++) + for (int i = 0; i < file_info.size(); i ++) { const std::string dirname = file_info[i][0]; const std::string basename = file_info[i][1]; @@ -1139,7 +1144,7 @@ TEST(conduit_relay_io_silo, read_overlink_directly) {"donordiv.s2_materials3", "OvlTop", ".silo", "MMESH"}, }; - for (int i = 0; i < file_info.size(); i ++) + for (int i = 0; i < file_info.size(); i ++) { const std::string dirname = file_info[i][0]; const std::string basename = file_info[i][1];