Reworking copying functions to only copy files for textures

xml_converter/src/attribute/image.cpp

@@ -1,14 +1,15 @@
 #include "image.hpp"
 
-#include <iosfwd>
+#include <filesystem>
 #include <string>
 #include <vector>
 
 #include "../rapid_helpers.hpp"
 #include "../rapidxml-1.13/rapidxml.hpp"
-#include "waypoint.pb.h"
+#include "../string_helper.hpp"
 
 using namespace std;
+namespace fs = std::filesystem;
 
 ////////////////////////////////////////////////////////////////////////////////
 // parse_image
@@ -21,8 +22,19 @@ void xml_attribute_to_image(
     XMLReaderState* state,
     Image* value,
     bool* is_set) {
-    value->filename = get_attribute_value(input);
-    value->original_filepath = state->xml_filedir + "/" + value->filename;
+    Image image;
+    image.filename = get_attribute_value(input);
+    image.original_filepath = join_file_paths(state->xml_filedir, image.filename);
+    if (fs::exists(fs::path(image.original_filepath))) {
+        for (const string& path : state->all_output_dirs) {
+            fs::path output_path = fs::path(path) / image.filename;
+            if (!fs::exists(output_path)) {
+                fs::create_directories(output_path.parent_path());
+                fs::copy_file(fs::path(image.original_filepath), output_path);
+            }
+        }
+    }
+    *value = image;
     *is_set = true;
 }
 
@@ -48,9 +60,19 @@ void proto_to_image(
     ProtoReaderState* state,
     Image* value,
     bool* is_set) {
-    // TODO: this is broken until we load the string index into the proto read state
     Image image;
-    // image.path = input.path();
+    image.filename = state->textures_index_to_texture_path[input];
+    image.original_filepath = state->proto_filedir + "/" + image.filename;
+    if (fs::exists(fs::path(image.original_filepath))) {
+        for (const string& path : state->all_output_dirs) {
+            fs::path output_path = fs::path(path) / image.filename;
+            if (!fs::exists(output_path)) {
+                fs::create_directories(output_path.parent_path());
+                fs::copy_file(fs::path(image.original_filepath), output_path);
+            }
+        }
+    }
+
     *value = image;
     *is_set = true;
 }

xml_converter/src/packaging_protobin.cpp

@@ -25,47 +25,51 @@ void parse_waypoint_categories(
     string full_category_name,
     ::waypoint::Category proto_category,
     map<string, Category>* marker_categories,
-    vector<Parseable*>* parsed_pois) {
+    vector<Parseable*>* parsed_pois,
+    ProtoReaderState* state) {
     full_category_name += proto_category.name();
     Category* this_category = &(*marker_categories)[full_category_name];
 
-    ProtoReaderState state;
-
-    this_category->parse_protobuf(proto_category, &state);
+    this_category->parse_protobuf(proto_category, state);
 
     for (int i = 0; i < proto_category.icon_size(); i++) {
         Icon* icon = new Icon();
-        icon->parse_protobuf(proto_category.icon(i), &state);
+        icon->parse_protobuf(proto_category.icon(i), state);
         // TODO: The field category in Icon is being deprciated
         // This overwrites any icon.category with its position in the heirarchy
         icon->category.category = full_category_name;
         parsed_pois->push_back(icon);
     }
     for (int i = 0; i < proto_category.trail_size(); i++) {
         Trail* trail = new Trail();
-        trail->parse_protobuf(proto_category.trail(i), &state);
+        trail->parse_protobuf(proto_category.trail(i), state);
         // TODO: The field category in Trail is being deprciated
         // This overwrites any trail.category with its position in the heirarchy
         trail->category.category = full_category_name;
         parsed_pois->push_back(trail);
     }
 
     for (int i = 0; i < proto_category.children_size(); i++) {
-        parse_waypoint_categories(full_category_name + ".", proto_category.children(i), &(this_category->children), parsed_pois);
+        parse_waypoint_categories(full_category_name + ".", proto_category.children(i), &(this_category->children), parsed_pois, state);
     }
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 //
 ////////////////////////////////////////////////////////////////////////////////
-void read_protobuf_file(string proto_filepath, map<string, Category>* marker_categories, vector<Parseable*>* parsed_pois) {
+void read_protobuf_file(string proto_filepath, map<string, Category>* marker_categories, vector<Parseable*>* parsed_pois, ProtoReaderState* state) {
     fstream infile;
     waypoint::Waypoint proto_message;
 
     infile.open(proto_filepath, ios::in | ios::binary);
     proto_message.ParseFromIstream(&infile);
+
+    for (int i = 0; i < proto_message.textures_size(); i++) {
+        state->textures_index_to_texture_path[i] = proto_message.textures(i).filepath();
+    }
+
     for (int i = 0; i < proto_message.category_size(); i++) {
-        parse_waypoint_categories("", proto_message.category(i), marker_categories, parsed_pois);
+        parse_waypoint_categories("", proto_message.category(i), marker_categories, parsed_pois, state);
     }
 }
 
@@ -251,7 +255,7 @@ void write_protobuf_file_per_map_id(
     }
 
     for (auto iterator = mapid_to_category_to_pois.begin(); iterator != mapid_to_category_to_pois.end(); iterator++) {
-        string output_filepath = join_file_paths(proto_directory, to_string(iterator->first) + ".data");
+        string output_filepath = join_file_paths(proto_directory, to_string(iterator->first) + ".bin");
 
         _write_protobuf_file(
             output_filepath,

xml_converter/src/packaging_protobin.hpp

@@ -13,7 +13,8 @@
 void read_protobuf_file(
     std::string proto_filepath,
     std::map<std::string, Category>* marker_categories,
-    std::vector<Parseable*>* parsed_pois);
+    std::vector<Parseable*>* parsed_pois,
+    ProtoReaderState* state);
 
 void write_protobuf_file(
     const std::string& proto_directory,

xml_converter/src/packaging_xml.cpp

@@ -13,19 +13,14 @@ using namespace std;
 ////////////////////////////////// SERIALIZE ///////////////////////////////////
 ////////////////////////////////////////////////////////////////////////////////
 
-void parse_marker_categories(rapidxml::xml_node<>* node, map<string, Category>* marker_categories, vector<XMLError*>* errors, string base_dir, int depth = 0) {
+void parse_marker_categories(rapidxml::xml_node<>* node, map<string, Category>* marker_categories, vector<XMLError*>* errors, XMLReaderState* state, int depth = 0) {
     if (get_node_name(node) == "MarkerCategory") {
         string name = lowercase(find_attribute_value(node, "name"));
 
-        XMLReaderState state = {
-            base_dir,
-            marker_categories,
-        };
-
         Category* this_category = &(*marker_categories)[name];
-        this_category->init_from_xml(node, errors, &state);
+        this_category->init_from_xml(node, errors, state);
         for (rapidxml::xml_node<>* child_node = node->first_node(); child_node; child_node = child_node->next_sibling()) {
-            parse_marker_categories(child_node, &(this_category->children), errors, base_dir, depth + 1);
+            parse_marker_categories(child_node, &(this_category->children), errors, state, depth + 1);
         }
     }
     else {
@@ -76,14 +71,9 @@ Category* get_category(rapidxml::xml_node<>* node, map<string, Category>* marker
 //
 // Parse the <POIs> xml block into an in-memory array of Markers.
 ////////////////////////////////////////////////////////////////////////////////
-vector<Parseable*> parse_pois(rapidxml::xml_node<>* root_node, map<string, Category>* marker_categories, vector<XMLError*>* errors, string base_dir) {
+vector<Parseable*> parse_pois(rapidxml::xml_node<>* root_node, map<string, Category>* marker_categories, vector<XMLError*>* errors, XMLReaderState* state) {
     vector<Parseable*> markers;
 
-    XMLReaderState state = {
-        base_dir,
-        marker_categories,
-    };
-
     for (rapidxml::xml_node<>* node = root_node->first_node(); node; node = node->next_sibling()) {
         if (get_node_name(node) == "POI") {
             Category* default_category = get_category(node, marker_categories, errors);
@@ -94,7 +84,7 @@ vector<Parseable*> parse_pois(rapidxml::xml_node<>* root_node, map<string, Categ
                 *icon = default_category->default_icon;
             }
 
-            icon->init_from_xml(node, errors, &state);
+            icon->init_from_xml(node, errors, state);
             markers.push_back(icon);
         }
         else if (get_node_name(node) == "Trail") {
@@ -106,7 +96,7 @@ vector<Parseable*> parse_pois(rapidxml::xml_node<>* root_node, map<string, Categ
                 *trail = default_category->default_trail;
             }
 
-            trail->init_from_xml(node, errors, &state);
+            trail->init_from_xml(node, errors, state);
             markers.push_back(trail);
         }
         else {
@@ -121,7 +111,7 @@ vector<Parseable*> parse_pois(rapidxml::xml_node<>* root_node, map<string, Categ
 //
 // A function which parses a single XML file into their corrisponding classes.
 ////////////////////////////////////////////////////////////////////////////////
-void parse_xml_file(string xml_filepath, map<string, Category>* marker_categories, vector<Parseable*>* parsed_pois) {
+void parse_xml_file(string xml_filepath, map<string, Category>* marker_categories, vector<Parseable*>* parsed_pois, XMLReaderState* state) {
     vector<XMLError*> errors;
     rapidxml::xml_document<> doc;
     rapidxml::xml_node<>* root_node;
@@ -130,7 +120,6 @@ void parse_xml_file(string xml_filepath, map<string, Category>* marker_categorie
     doc.parse<rapidxml::parse_non_destructive | rapidxml::parse_no_data_nodes>(xml_file.data(), xml_filepath.c_str());
 
     root_node = doc.first_node();
-    string base_dir = get_base_dir(xml_filepath);
     // Validate the Root Node
     if (get_node_name(root_node) != "OverlayData") {
         errors.push_back(new XMLNodeNameError("Root node should be of type OverlayData", root_node));
@@ -141,10 +130,10 @@ void parse_xml_file(string xml_filepath, map<string, Category>* marker_categorie
 
     for (rapidxml::xml_node<>* node = root_node->first_node(); node; node = node->next_sibling()) {
         if (get_node_name(node) == "MarkerCategory") {
-            parse_marker_categories(node, marker_categories, &errors, base_dir);
+            parse_marker_categories(node, marker_categories, &errors, state);
         }
         else if (get_node_name(node) == "POIs") {
-            vector<Parseable*> temp_vector = parse_pois(node, marker_categories, &errors, base_dir);
+            vector<Parseable*> temp_vector = parse_pois(node, marker_categories, &errors, state);
             move(temp_vector.begin(), temp_vector.end(), back_inserter(*parsed_pois));
         }
         else {

xml_converter/src/packaging_xml.hpp

@@ -13,7 +13,8 @@
 void parse_xml_file(
     std::string xml_filepath,
     std::map<std::string, Category>* marker_categories,
-    std::vector<Parseable*>* parsed_pois);
+    std::vector<Parseable*>* parsed_pois,
+    XMLReaderState* state);
 
 void write_xml_file(
     std::string xml_filepath,

xml_converter/src/state_structs/proto_reader_state.hpp

@@ -1,4 +1,12 @@
 #pragma once
 
+#include <map>
+#include <string>
+#include <vector>
+
 struct ProtoReaderState {
+    // A map from the index within "textures" to the texture path.
+    std::map<uint32_t, std::string> textures_index_to_texture_path;
+    std::string proto_filedir;
+    std::vector<std::string> all_output_dirs;
 };

xml_converter/src/state_structs/xml_reader_state.hpp

@@ -1,11 +1,11 @@
 #pragma once
 
-#include <map>
 #include <string>
+#include <vector>
 
 class Category;
 
 struct XMLReaderState {
     std::string xml_filedir;
-    std::map<std::string, Category>* marker_categories;
+    std::vector<std::string> all_output_dirs;
 };

xml_converter/src/xml_converter.cpp

@@ -77,7 +77,7 @@ void move_supplementary_files(string input_directory, string output_directory) {
                 }
                 move_supplementary_files(path, new_directory);
             }
-            else if (has_suffix(filename, ".trl") || has_suffix(filename, ".xml")) {
+            else if (has_suffix(filename, ".trl") || has_suffix(filename, ".xml") || has_suffix(filename, ".bin")) {
                 continue;
             }
             else {
@@ -90,18 +90,35 @@ void move_supplementary_files(string input_directory, string output_directory) {
     }
 }
 
-void read_taco_directory(string input_path, map<string, Category>* marker_categories, vector<Parseable*>* parsed_pois) {
+void read_taco_directory(string input_path, map<string, Category>* marker_categories, vector<Parseable*>* parsed_pois, XMLReaderState* state) {
     if (!filesystem::exists(input_path)) {
         cout << "Error: " << input_path << " is not an existing directory or file" << endl;
     }
     else if (filesystem::is_directory(input_path)) {
         vector<string> xml_files = get_files_by_suffix(input_path, ".xml");
         for (const string& path : xml_files) {
-            parse_xml_file(path, marker_categories, parsed_pois);
+            parse_xml_file(path, marker_categories, parsed_pois, state);
         }
     }
     else if (filesystem::is_regular_file(input_path)) {
-        parse_xml_file(input_path, marker_categories, parsed_pois);
+        state->xml_filedir = get_base_dir(input_path);
+        parse_xml_file(input_path, marker_categories, parsed_pois, state);
+    }
+}
+
+void read_burrito_directory(string input_path, map<string, Category>* marker_categories, vector<Parseable*>* parsed_pois, ProtoReaderState* state) {
+    if (!filesystem::exists(input_path)) {
+        cout << "Error: " << input_path << " is not an existing directory or file" << endl;
+    }
+    else if (filesystem::is_directory(input_path)) {
+        vector<string> burrito_files = get_files_by_suffix(input_path, ".bin");
+        for (const string& path : burrito_files) {
+            read_protobuf_file(path, marker_categories, parsed_pois, state);
+        }
+    }
+    else if (filesystem::is_regular_file(input_path)) {
+        state->proto_filedir = get_base_dir(input_path);
+        read_protobuf_file(input_path, marker_categories, parsed_pois, state);
     }
 }
 
@@ -137,20 +154,36 @@ void process_data(
     vector<Parseable*> parsed_pois;
     map<string, Category> marker_categories;
 
+    vector<string> all_output_paths;
+    if (output_split_waypoint_dir != "") {
+        all_output_paths.push_back(output_split_waypoint_dir);
+    }
+    if (output_taco_paths.size() != 0) {
+        for (size_t i = 0; i < output_taco_paths.size(); i++) {
+            all_output_paths.push_back(output_taco_paths[i]);
+        }
+    }
+    if (output_waypoint_paths.size() != 0) {
+        for (size_t i = 0; i < output_waypoint_paths.size(); i++) {
+            all_output_paths.push_back(output_waypoint_paths[i]);
+        }
+    }
+
     // Read in all the xml taco markerpacks
     auto begin = chrono::high_resolution_clock::now();
     for (size_t i = 0; i < input_taco_paths.size(); i++) {
         cout << "Loading taco pack " << input_taco_paths[i] << endl;
+
+        XMLReaderState state = {
+            input_taco_paths[i],
+            all_output_paths,
+        };
+
         read_taco_directory(
             input_taco_paths[i],
             &marker_categories,
-            &parsed_pois);
-
-        // TODO: This is wildly incorrect now because we might have a
-        //       different output directory then output_split_waypoint_dir
-        if (output_split_waypoint_dir != "") {
-            move_supplementary_files(input_taco_paths[i], output_split_waypoint_dir);
-        }
+            &parsed_pois,
+            &state);
     }
     auto end = chrono::high_resolution_clock::now();
     auto dur = end - begin;
@@ -160,10 +193,18 @@ void process_data(
     // Read in all the protobin waypoint markerpacks
     for (size_t i = 0; i < input_waypoint_paths.size(); i++) {
         cout << "Loading waypoint pack " << input_waypoint_paths[i] << endl;
-        read_protobuf_file(
+
+        ProtoReaderState state = {
+            {},
+            input_waypoint_paths[i],
+            all_output_paths,
+        };
+
+        read_burrito_directory(
             input_waypoint_paths[i],
             &marker_categories,
-            &parsed_pois);
+            &parsed_pois,
+            &state);
     }
 
     // Write all of the xml taco paths