Merge pull request #98 from AstroTechies/unreal_pak-no-buffer

unreal_pak: move buffering out of crate

unreal_mod_integrator/examples/integration.rs

@@ -1,7 +1,7 @@
 use std::collections::HashMap;
 use std::env;
 use std::fs::File;
-use std::io;
+use std::io::{self, BufReader};
 use std::path::PathBuf;
 
 use unreal_asset::engine_version::EngineVersion;
@@ -14,8 +14,8 @@ pub struct Config;
 fn handle_linked_actor_components(
     _data: &(),
     _integrated_pak: &mut PakMemory,
-    _game_paks: &mut Vec<PakReader<File>>,
-    _mod_paks: &mut Vec<PakReader<File>>,
+    _game_paks: &mut Vec<PakReader<BufReader<File>>>,
+    _mod_paks: &mut Vec<PakReader<BufReader<File>>>,
     actors: &Vec<serde_json::Value>,
 ) -> Result<(), io::Error> {
     println!("Example linked actors: {actors:?}");

unreal_mod_integrator/src/handlers/mod.rs

@@ -1,4 +1,5 @@
 use std::fs::File;
+use std::io::BufReader;
 
 use unreal_pak::{PakMemory, PakReader};
 
@@ -13,8 +14,8 @@ pub fn handle_persistent_actors(
     game_name: &'static str,
     map_paths: &[&str],
     integrated_pak: &mut PakMemory,
-    game_paks: &mut Vec<PakReader<File>>,
-    mod_paks: &mut Vec<PakReader<File>>,
+    game_paks: &mut Vec<PakReader<BufReader<File>>>,
+    mod_paks: &mut Vec<PakReader<BufReader<File>>>,
     persistent_actor_arrays: &Vec<serde_json::Value>,
 ) -> Result<(), Error> {
     #[cfg(feature = "ue4_23")]

unreal_mod_integrator/src/handlers/ue4_23/persistent_actors.rs

@@ -1,6 +1,6 @@
 use std::collections::HashMap;
 use std::fs::File;
-use std::io::{self, Cursor, ErrorKind};
+use std::io::{self, BufReader, Cursor, ErrorKind};
 use std::path::Path;
 
 use unreal_asset::engine_version::EngineVersion;
@@ -37,8 +37,8 @@ pub fn handle_persistent_actors(
     game_name: &'static str,
     map_paths: &[&str],
     integrated_pak: &mut PakMemory,
-    game_paks: &mut Vec<PakReader<File>>,
-    mod_paks: &mut Vec<PakReader<File>>,
+    game_paks: &mut Vec<PakReader<BufReader<File>>>,
+    mod_paks: &mut Vec<PakReader<BufReader<File>>>,
     persistent_actor_arrays: &Vec<serde_json::Value>,
 ) -> Result<(), Error> {
     let level_asset = Asset::new(

unreal_mod_integrator/src/helpers.rs

@@ -1,5 +1,5 @@
 use std::fs::File;
-use std::io::Cursor;
+use std::io::{BufReader, Cursor};
 use std::path::Path;
 
 use unreal_asset::{engine_version::EngineVersion, reader::ArchiveTrait, Asset};
@@ -9,8 +9,8 @@ use crate::{error::IntegrationError, Error};
 
 pub fn get_asset(
     integrated_pak: &mut PakMemory,
-    game_paks: &mut [PakReader<File>],
-    mod_paks: &mut [PakReader<File>],
+    game_paks: &mut [PakReader<BufReader<File>>],
+    mod_paks: &mut [PakReader<BufReader<File>>],
     name: &String,
     version: EngineVersion,
 ) -> Result<Asset<Cursor<Vec<u8>>>, Error> {
@@ -62,7 +62,7 @@ pub fn get_asset(
     )
 }
 
-pub fn find_asset(paks: &mut [PakReader<File>], name: &String) -> Option<usize> {
+pub fn find_asset(paks: &mut [PakReader<BufReader<File>>], name: &String) -> Option<usize> {
     for (i, pak) in paks.iter().enumerate() {
         if pak.contains_entry(name) {
             return Some(i);

unreal_mod_integrator/src/lib.rs

@@ -1,6 +1,6 @@
 use std::collections::HashMap;
 use std::fs::{self, File, OpenOptions};
-use std::io::{Cursor, Write};
+use std::io::{BufReader, BufWriter, Cursor, Write};
 use std::path::{Path, PathBuf};
 
 use error::IntegrationError;
@@ -149,16 +149,16 @@ pub trait DynamicMod<E: std::error::Error>: IntegratorModInfo {
     fn integrate(
         &self,
         integrated_pak: &mut PakMemory,
-        game_paks: &mut Vec<PakReader<File>>,
-        mod_paks: &mut Vec<PakReader<File>>,
+        game_paks: &mut Vec<PakReader<BufReader<File>>>,
+        mod_paks: &mut Vec<PakReader<BufReader<File>>>,
     ) -> Result<(), E>;
 }
 
 pub type HandlerFn<D, E> = dyn FnMut(
     &D,
     &mut PakMemory,
-    &mut Vec<PakReader<File>>,
-    &mut Vec<PakReader<File>>,
+    &mut Vec<PakReader<BufReader<File>>>,
+    &mut Vec<PakReader<BufReader<File>>>,
     &Vec<Value>,
 ) -> Result<(), E>;
 
@@ -411,8 +411,8 @@ pub fn integrate_mods<
     let mut read_mods = Vec::new();
     let mut optional_mods_data = HashMap::new();
 
-    for mod_file in &mod_files {
-        let mut pak = PakReader::new(mod_file);
+    for mod_file in mod_files {
+        let mut pak = PakReader::new(BufReader::new(mod_file));
         pak.load_index()?;
 
         let record = pak.read_entry(&String::from("metadata.json"))?;
@@ -488,8 +488,8 @@ pub fn integrate_mods<
         }
 
         let mut game_paks = Vec::new();
-        for game_file in &game_files {
-            let mut pak = PakReader::new(game_file);
+        for game_file in game_files {
+            let mut pak = PakReader::new(BufReader::new(game_file));
             pak.load_index()?;
             game_paks.push(pak);
         }
@@ -539,14 +539,14 @@ pub fn integrate_mods<
         }
 
         let path = Path::new(paks_path).join(INTEGRATOR_PAK_FILE_NAME);
-        let mut file = OpenOptions::new()
+        let file = OpenOptions::new()
             .create(true)
             .write(true)
             .truncate(true)
             .open(path)?;
 
-        generated_pak.write(&mut file)?;
-        file.sync_data()?;
+        let mut writer = BufWriter::new(file);
+        generated_pak.write(&mut writer)?;
     }
 
     Ok(())

unreal_pak/src/pakreader.rs

@@ -1,7 +1,7 @@
 //! PakFile data structure for reading large pak files
 
 use std::collections::BTreeMap;
-use std::io::{BufReader, Read, Seek};
+use std::io::{Read, Seek};
 
 use crate::compression::CompressionMethods;
 use crate::entry::read_entry;
@@ -12,31 +12,33 @@ use crate::pakversion::PakVersion;
 
 /// An Unreal pak file reader with it's data kept on disk and only read on demand.
 #[derive(Debug)]
-pub struct PakReader<'data, R>
+pub struct PakReader<R>
 where
-    &'data R: Read + Seek,
+    R: Read + Seek,
 {
     /// version of the pak file format this one is using
     pak_version: PakVersion,
     /// mount point (Unreal stuff)
     pub mount_point: String,
     compression: CompressionMethods,
     entries: BTreeMap<String, Header>,
-    reader: BufReader<&'data R>,
+    reader: R,
 }
 
-impl<'data, R> PakReader<'data, R>
+impl<R> PakReader<R>
 where
-    &'data R: Read + Seek,
+    R: Read + Seek,
 {
-    /// Creates a new `PakFile` configured to read files.
-    pub fn new(reader: &'data R) -> Self {
+    /// Creates a new `PakReader` that reads from the provided reader.
+    /// When using a reader that uses syscalls like a `File` it is recommended to wrap it in a
+    /// [`std::io::BufReader`] to avoid unnecessary syscalls.
+    pub fn new(reader: R) -> Self {
         Self {
             pak_version: PakVersion::Invalid,
             mount_point: "".to_owned(),
             compression: Default::default(),
             entries: BTreeMap::new(),
-            reader: BufReader::new(reader),
+            reader,
         }
     }
 
@@ -84,30 +86,36 @@ where
     }
 
     /// Iterate over the entries in the PakReader
-    pub fn iter<'a: 'data>(&'a mut self) -> PakReaderIter<'a, 'data, R> {
+    pub fn iter(&mut self) -> PakReaderIter<R> {
         PakReaderIter {
             reader: &mut self.reader,
             pak_version: self.pak_version,
             compression: self.compression,
             iter: self.entries.iter(),
         }
     }
+
+    /// Consumes the `PakReader`, returning the wrapped reader.
+    /// There are no guarantees for what state the reader might be in.
+    pub fn into_inner(self) -> R {
+        self.reader
+    }
 }
 
 /// An iterator over the entries of a PakReader
-pub struct PakReaderIter<'a, 'data, R>
+pub struct PakReaderIter<'a, R>
 where
-    &'data R: Read + Seek,
+    R: Read + Seek,
 {
-    reader: &'data mut BufReader<&'data R>,
+    reader: &'a mut R,
     pak_version: PakVersion,
     compression: CompressionMethods,
     iter: std::collections::btree_map::Iter<'a, String, Header>,
 }
 
-impl<'a, 'data, R> Iterator for PakReaderIter<'a, 'data, R>
+impl<'a, R> Iterator for PakReaderIter<'a, R>
 where
-    &'data R: Read + Seek,
+    R: Read + Seek,
 {
     type Item = (&'a String, Result<Vec<u8>, PakError>);
 
@@ -126,13 +134,13 @@ where
     }
 }
 
-impl<'a: 'data, 'data, R> IntoIterator for &'a mut PakReader<'data, R>
+impl<'a, R> IntoIterator for &'a mut PakReader<R>
 where
-    &'data R: Read + Seek,
+    R: Read + Seek,
 {
     type Item = (&'a String, Result<Vec<u8>, PakError>);
 
-    type IntoIter = PakReaderIter<'a, 'data, R>;
+    type IntoIter = PakReaderIter<'a, R>;
 
     fn into_iter(self) -> Self::IntoIter {
         self.iter()

unreal_pak/src/pakwriter.rs

@@ -1,7 +1,7 @@
 //! PakFile data structure for writing large pak files
 
 use std::collections::BTreeMap;
-use std::io::{BufWriter, Seek, Write};
+use std::io::{Seek, Write};
 
 use crate::compression::CompressionMethods;
 use crate::entry::write_entry;
@@ -14,9 +14,9 @@ use crate::pakversion::PakVersion;
 /// Good for working with very large files, but it has restrictions when it
 /// comes to writing files. For a more flexible alternative see \<insert doc link\> PakMemory
 #[derive(Debug)]
-pub struct PakWriter<'data, W>
+pub struct PakWriter<W>
 where
-    &'data W: Write + Seek,
+    W: Write + Seek,
 {
     /// version of the pak file format this one is using
     pub pak_version: PakVersion,
@@ -27,22 +27,24 @@ where
     /// the compression block size
     pub block_size: u32,
     entries: BTreeMap<String, Header>,
-    writer: BufWriter<&'data W>,
+    writer: W,
 }
 
-impl<'data, W> PakWriter<'data, W>
+impl<W> PakWriter<W>
 where
-    &'data W: Write + Seek,
+    W: Write + Seek,
 {
-    /// Creates a new `PakFile` configured to write files.
-    pub fn new(writer: &'data W, pak_version: PakVersion) -> Self {
+    /// Creates a new `PakWriter` that writes to the provided writer.
+    /// When using a writer that uses syscalls like a `File` it is recommended to wrap it in a
+    /// [`std::io::BufWriter`] to avoid unnecessary syscalls.
+    pub fn new(writer: W, pak_version: PakVersion) -> Self {
         Self {
             pak_version,
             mount_point: "../../../".to_owned(),
             compression: CompressionMethods::zlib(),
             block_size: 0x010000,
             entries: BTreeMap::new(),
-            writer: BufWriter::new(writer),
+            writer,
         }
     }
 

unreal_pak_cli/src/main.rs

@@ -1,5 +1,5 @@
 use std::fs::{File, OpenOptions};
-use std::io::Write;
+use std::io::{BufReader, BufWriter, Write};
 use std::path::{Path, PathBuf};
 use std::process::exit;
 use std::time::SystemTime;
@@ -62,24 +62,18 @@ fn main() {
     match args.commands {
         Commands::CheckHeader { pakfile } => {
             let file = open_file(Path::new(&pakfile));
-            let mut pak = PakReader::new(&file);
+            let mut pak = PakReader::new(file);
             check_header(&mut pak);
         }
         Commands::Check { pakfile } => {
             let file = open_file(Path::new(&pakfile));
-            let mut pak = PakReader::new(&file);
+            let mut pak = PakReader::new(file);
             check_header(&mut pak);
 
-            // TODO: get rid of this clone
-            let names = pak
-                .get_entry_names()
-                .into_iter()
-                .cloned()
-                .collect::<Vec<_>>();
-            for (i, file_name) in names.iter().enumerate() {
+            for (i, (file_name, data)) in pak.iter().enumerate() {
                 println!("Record {i}: {file_name:?}");
 
-                match pak.read_entry(file_name) {
+                match data {
                     Ok(_) => (),
                     Err(e) => {
                         eprintln!("Error reading record {i}: {file_name:?}! Error: {e}");
@@ -91,7 +85,7 @@ fn main() {
         Commands::Extract { pakfile, outdir } => {
             let path = Path::new(&pakfile);
             let file = open_file(path);
-            let mut pak = PakReader::new(&file);
+            let mut pak = PakReader::new(file);
             check_header(&mut pak);
 
             // temp values required to extend lifetimes outside of match scope
@@ -185,7 +179,10 @@ fn main() {
 
             let file = OpenOptions::new().append(true).open(&pakfile).unwrap();
 
-            let mut pak = PakWriter::new(&file, PakVersion::FnameBasedCompressionMethod);
+            let mut pak = PakWriter::new(
+                BufWriter::new(file),
+                PakVersion::FnameBasedCompressionMethod,
+            );
 
             // Get all files and write them to the .pak file
             let files = WalkDir::new(&indir)
@@ -242,17 +239,17 @@ fn main() {
     )
 }
 
-fn open_file(path: &Path) -> File {
+fn open_file(path: &Path) -> BufReader<File> {
     match OpenOptions::new().read(true).open(path) {
-        Ok(file) => file,
+        Ok(file) => BufReader::new(file),
         Err(err) => {
             eprintln!("Could not find/open file! Error: {err}");
             exit(1);
         }
     }
 }
 
-fn check_header(pak: &mut PakReader<File>) {
+fn check_header(pak: &mut PakReader<BufReader<File>>) {
     match pak.load_index() {
         Ok(_) => println!("Header is ok"),
         Err(err) => {