Added an option to split the output proto files by top level category

xml_converter/integration_tests/run_tests.py

@@ -21,6 +21,7 @@ def run_xml_converter(
     output_proto: Optional[List[str]] = None,
     split_output_proto: Optional[str] = None,
     allow_duplicates: Optional[bool] = None,
+    split_proto_by_category: Optional[bool] = None,
 ) -> Tuple[str, str, int]:
 
     # Build the command to execute the C++ program with the desired function and arguments
@@ -38,6 +39,8 @@ def run_xml_converter(
         cmd += ["--output-split-guildpoint-path"] + [split_output_proto]
     if allow_duplicates:
         cmd += ["--allow-duplicates"]
+    if split_proto_by_category:
+        cmd += ["--split-guildpoint-by-category"]
 
     # Run the C++ program and capture its output
     result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
@@ -177,7 +180,8 @@ def main() -> bool:
             input_proto=testcase.proto_input_paths,
             output_xml=[xml_output_dir_path],
             output_proto=[proto_output_dir_path],
-            allow_duplicates=testcase.allow_duplicates
+            allow_duplicates=testcase.allow_duplicates,
+            split_proto_by_category=testcase.split_proto_by_category
         )
 
         # Sanitize and denoise the lines

xml_converter/integration_tests/src/testcase_loader.py

@@ -22,6 +22,7 @@ class Testcase:
     expected_stderr: List[str]
     expected_returncode: int
     allow_duplicates: bool
+    split_proto_by_category: bool
 
 
 ################################################################################
@@ -60,6 +61,7 @@ def load_testcase(path: str) -> Optional[Testcase]:
     xml_input_paths: List[str] = []
     proto_input_paths: List[str] = []
     allow_duplicates: bool = False
+    split_proto_by_category: bool = False
     for pack_name, pack_type in data["input_paths"].items():
         if not isinstance(pack_name, str):
             print(f"Invalid pack name, expecting a string but got {pack_name}")
@@ -113,6 +115,13 @@ def load_testcase(path: str) -> Optional[Testcase]:
         else:
             allow_duplicates = data["allow_duplicates"]
 
+    if "split_proto_by_category" in data:
+        if not isinstance(data["split_proto_by_category"], bool):
+            print(f"Invalid Test, expecting bool value for 'split_proto_by_category' in {path}")
+            return None
+        else:
+            split_proto_by_category = data["split_proto_by_category"]
+
     return Testcase(
         name=os.path.basename(path),
         xml_input_paths=xml_input_paths,
@@ -122,7 +131,8 @@ def load_testcase(path: str) -> Optional[Testcase]:
         expected_stdout=to_lines(data["expected_stdout"]),
         expected_stderr=to_lines(data["expected_stderr"]),
         expected_returncode=data["expected_returncode"],
-        allow_duplicates=allow_duplicates
+        allow_duplicates=allow_duplicates,
+        split_proto_by_category=split_proto_by_category
     )
 
 

xml_converter/integration_tests/test_cases/proto_and_xml_input_no_duplicates/testcase.yaml

@@ -10,7 +10,7 @@ expected_stdout: |
     The following top level categories were found in more than one pack:
         "mycategory" in files:
             pack/xml_file.xml
-            pack2/markers.bin
+            pack2/markers.guildpoint
             pack3/xml_file.xml
 expected_stderr: |
 expected_returncode: 0

xml_converter/integration_tests/test_cases/xml_split_by_category/input/pack/xml_file.xml

@@ -0,0 +1,13 @@
+<OverlayData>
+    <MarkerCategory DisplayName="TopLevel1" Name="toplevel1" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" ></MarkerCategory>
+
+    <MarkerCategory DisplayName="TopLevel2" Name="toplevel2" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" ></MarkerCategory>
+
+    <MarkerCategory DisplayName="TopLevel3" Name="toplevel3" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" ></MarkerCategory>
+
+    <POIs>
+        <POI Type="toplevel1" />
+        <POI Type="toplevel2" />
+        <POI Type="toplevel3" />
+    </POIs>
+</OverlayData>

xml_converter/integration_tests/test_cases/xml_split_by_category/output_proto/toplevel1.guildpoint

@@ -0,0 +1,3 @@
+
+*
+	TopLevel1 2B\Ï)Cf¦RC{ÔWCBL]ÚBñÓ

xml_converter/integration_tests/test_cases/xml_split_by_category/output_proto/toplevel2.guildpoint

@@ -0,0 +1,3 @@
+
+*
+	TopLevel2 2B\Ï)Cf¦RC{ÔWCBL]ÛBñÓ

xml_converter/integration_tests/test_cases/xml_split_by_category/output_proto/toplevel3.guildpoint

@@ -0,0 +1,3 @@
+
+*
+	TopLevel3 2B\Ï)Cf¦RC{ÔWCBL]ÜBñÓ

xml_converter/integration_tests/test_cases/xml_split_by_category/output_xml/xml_file.xml

@@ -0,0 +1,16 @@
+<OverlayData>
+<MarkerCategory  DisplayName="TopLevel1" ID="CAdMXdpC8dM=" Name="toplevel1">
+</MarkerCategory>
+
+<MarkerCategory  DisplayName="TopLevel2" ID="CAdMXdtC8dM=" Name="toplevel2">
+</MarkerCategory>
+
+<MarkerCategory  DisplayName="TopLevel3" ID="CAdMXdxC8dM=" Name="toplevel3">
+</MarkerCategory>
+
+<POIs>
+<POI  Type="toplevel1" MapID="50" XPos="169.809998" YPos="210.649994" ZPos="215.830002"/>
+<POI  Type="toplevel2" MapID="50" XPos="169.809998" YPos="210.649994" ZPos="215.830002"/>
+<POI  Type="toplevel3" MapID="50" XPos="169.809998" YPos="210.649994" ZPos="215.830002"/>
+</POIs>
+</OverlayData>

xml_converter/integration_tests/test_cases/xml_split_by_category/testcase.yaml

@@ -0,0 +1,6 @@
+input_paths: 
+    "pack": "xml"
+expected_stdout: |
+expected_stderr: |
+expected_returncode: 0
+split_proto_by_category: true

xml_converter/src/packaging_protobin.cpp

@@ -211,7 +211,6 @@ void _write_protobuf_file(
 
 void write_protobuf_file(
     const string& marker_pack_root_directory,
-    const StringHierarchy& category_filter,
     const map<string, Category>* marker_categories,
     const vector<Parseable*>* parsed_pois) {
     std::map<string, std::vector<Parseable*>> category_to_pois;
@@ -235,6 +234,8 @@ void write_protobuf_file(
         }
     }
 
+    StringHierarchy category_filter;
+    category_filter.add_path({}, true);
     _write_protobuf_file(
         join_file_paths(state.marker_pack_root_directory, "markers.guildpoint"),
         category_filter,
@@ -243,10 +244,47 @@ void write_protobuf_file(
         &state);
 }
 
+// Write protobuf per top level category
+void write_protobuf_file_per_category(
+    const string& marker_pack_root_directory,
+    const map<string, Category>* marker_categories,
+    const vector<Parseable*>* parsed_pois) {
+    std::map<string, std::vector<Parseable*>> category_to_pois;
+    ProtoWriterState state;
+    state.marker_pack_root_directory = marker_pack_root_directory;
+
+    for (size_t i = 0; i < parsed_pois->size(); i++) {
+        Parseable* parsed_poi = (*parsed_pois)[i];
+        if (parsed_poi->classname() == "POI") {
+            Icon* icon = dynamic_cast<Icon*>(parsed_poi);
+            // TODO(331): This is the wrong place to lowercase() the category and is hiding some crimes elsewhere
+            category_to_pois[lowercase(icon->category.category)].push_back(parsed_poi);
+        }
+        else if (parsed_poi->classname() == "Trail") {
+            Trail* trail = dynamic_cast<Trail*>(parsed_poi);
+            // TODO(331): This is the wrong place to lowercase() the category and is hiding some crimes elsewhere
+            category_to_pois[lowercase(trail->category.category)].push_back(parsed_poi);
+        }
+        else {
+            std::cout << "Unknown type" << std::endl;
+        }
+    }
+
+    for (map<string, Category>::const_iterator it = marker_categories->begin(); it != marker_categories->end(); it++) {
+        StringHierarchy category_filter;
+        category_filter.add_path({it->first}, true);
+        _write_protobuf_file(
+            join_file_paths(state.marker_pack_root_directory, it->first + ".guildpoint"),
+            category_filter,
+            marker_categories,
+            category_to_pois,
+            &state);
+    }
+}
+
 // Write protobuf per map id
 void write_protobuf_file_per_map_id(
     const string& marker_pack_root_directory,
-    const StringHierarchy& category_filter,
     const map<string, Category>* marker_categories,
     const vector<Parseable*>* parsed_pois) {
     std::map<int, std::map<string, std::vector<Parseable*>>> mapid_to_category_to_pois;
@@ -268,6 +306,9 @@ void write_protobuf_file_per_map_id(
         }
     }
 
+    StringHierarchy category_filter;
+    category_filter.add_path({}, true);
+
     for (auto iterator = mapid_to_category_to_pois.begin(); iterator != mapid_to_category_to_pois.end(); iterator++) {
         string output_filepath = join_file_paths(state.marker_pack_root_directory, to_string(iterator->first) + ".guildpoint");
 

xml_converter/src/packaging_protobin.hpp

@@ -19,12 +19,15 @@ std::set<std::string> read_protobuf_file(
 
 void write_protobuf_file(
     const std::string& marker_pack_root_directory,
-    const StringHierarchy& category_filter,
+    const std::map<std::string, Category>* marker_categories,
+    const std::vector<Parseable*>* parsed_pois);
+
+void write_protobuf_file_per_category(
+    const std::string& marker_pack_root_directory,
     const std::map<std::string, Category>* marker_categories,
     const std::vector<Parseable*>* parsed_pois);
 
 void write_protobuf_file_per_map_id(
     const std::string& marker_pack_root_directory,
-    const StringHierarchy& category_filter,
     const std::map<std::string, Category>* marker_categories,
     const std::vector<Parseable*>* parsed_pois);

xml_converter/src/xml_converter.cpp

@@ -138,16 +138,20 @@ void write_taco_directory(
 void write_burrito_directory(
     string output_path,
     map<string, Category>* marker_categories,
-    vector<Parseable*>* parsed_pois) {
+    vector<Parseable*>* parsed_pois,
+    bool split_proto_by_category) {
     if (!filesystem::is_directory(output_path)) {
         if (!filesystem::create_directory(output_path)) {
             cout << "Error: " << output_path << "is not a valid directory path" << endl;
             return;
         }
     }
-    StringHierarchy category_filter;
-    category_filter.add_path({}, true);
-    write_protobuf_file(output_path, category_filter, marker_categories, parsed_pois);
+    if (split_proto_by_category) {
+        write_protobuf_file_per_category(output_path, marker_categories, parsed_pois);
+    }
+    else {
+        write_protobuf_file(output_path, marker_categories, parsed_pois);
+    }
 }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -166,6 +170,9 @@ void process_data(
     // allow for splitting out a single markerpack
     vector<string> output_taco_paths,
     vector<string> output_guildpoint_paths,
+    // This value will change how the guildpoint files are written so that
+    // each top level category is in its own file
+    bool split_proto_by_category,
 
     // This is a special output path used for burrito internal use that splits
     // the guildpoint protobins by map id.
@@ -245,15 +252,13 @@ void process_data(
 
     // Write all of the protobin guildpoint paths
     for (size_t i = 0; i < output_guildpoint_paths.size(); i++) {
-        write_burrito_directory(output_guildpoint_paths[i], &marker_categories, &parsed_pois);
+        write_burrito_directory(output_guildpoint_paths[i], &marker_categories, &parsed_pois, split_proto_by_category);
     }
 
     // Write the special map-split protbin guildpoint file
     begin = chrono::high_resolution_clock::now();
     if (output_split_guildpoint_dir != "") {
-        StringHierarchy category_filter;
-        category_filter.add_path({}, true);
-        write_protobuf_file_per_map_id(output_split_guildpoint_dir, category_filter, &marker_categories, &parsed_pois);
+        write_protobuf_file_per_map_id(output_split_guildpoint_dir, &marker_categories, &parsed_pois);
     }
     end = chrono::high_resolution_clock::now();
     dur = end - begin;
@@ -278,6 +283,7 @@ int main(int argc, char* argv[]) {
     vector<string> input_guildpoint_paths;
     vector<string> output_guildpoint_paths;
     bool allow_duplicates = false;
+    bool split_proto_by_category = false;
 
     // Typically "~/.local/share/godot/app_userdata/Burrito/protobins" for
     // converting from xml markerpacks to internal protobuf files.
@@ -308,6 +314,10 @@ int main(int argc, char* argv[]) {
             allow_duplicates = true;
             arg_target = nullptr;
         }
+        else if (!strcmp(argv[i], "--split-guildpoint-by-category")) {
+            split_proto_by_category = true;
+            arg_target = nullptr;
+        }
         else {
             if (arg_target != nullptr) {
                 arg_target->push_back(argv[i]);
@@ -335,7 +345,7 @@ int main(int argc, char* argv[]) {
         allow_duplicates,
         output_taco_paths,
         output_guildpoint_paths,
+        split_proto_by_category,
         output_split_guildpoint_dir);
-
     return 0;
 }