Cache downloaded assets locally if possible

Cargo.toml

@@ -23,6 +23,8 @@ futures-io = "0.3.29"
 glam = "0"
 bevy_screen_diagnostics = { git = "https://github.com/oli-obk/bevy_screen_diagnostics.git" }
 globe-rs = "0.1.8"
+directories = "5.0.1"
+async-fs = "2.1.0"
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
 surf = { version = "2.3.2", default-features = false, features = [

src/http_assets.rs

@@ -1,32 +1,41 @@
 // Taken from https://github.com/lizelive/bevy_http and modified.
 
+use async_fs::File;
 use bevy::{
     asset::{
         io::{
             AssetReader, AssetReaderError, AssetSource, AssetSourceId, PathStream, Reader,
             VecReader,
         },
-        AsyncReadExt,
+        AsyncReadExt, AsyncWriteExt,
     },
     prelude::*,
     utils::BoxedFuture,
 };
 use flate2::read::GzDecoder;
-use std::{io::Read, path::Path};
+use std::{
+    collections::HashSet,
+    io::Read,
+    path::{Path, PathBuf},
+    sync::{Arc, RwLock},
+};
 
 /// A custom asset reader implementation that wraps a given asset reader implementation
-pub struct HttpAssetReader {
+struct HttpAssetReader {
     #[cfg(target_arch = "wasm32")]
     base_url: String,
     #[cfg(not(target_arch = "wasm32"))]
     client: surf::Client,
     /// Whether to load tiles from this path
     tile: bool,
+    /// Used to ensure the same asset doesn't get its cache file written twice at the same time,
+    /// as that depends on the OS whether it succeeds (could result in broken cache files).
+    sync: Arc<RwLock<HashSet<PathBuf>>>,
 }
 
 impl HttpAssetReader {
     /// Creates a new `HttpAssetReader`. The path provided will be used to build URLs to query for assets.
-    pub fn new(base_url: &str, tile: bool) -> Self {
+    fn new(base_url: &str, tile: bool, sync: Arc<RwLock<HashSet<PathBuf>>>) -> Self {
         #[cfg(not(target_arch = "wasm32"))]
         {
             let base_url = surf::Url::parse(base_url).expect("invalid base url");
@@ -35,19 +44,21 @@ impl HttpAssetReader {
             let client = client.set_base_url(base_url);
 
             let client = client.try_into().expect("could not create http client");
-            Self { client, tile }
+            Self { client, tile, sync }
         }
         #[cfg(target_arch = "wasm32")]
         {
             Self {
                 base_url: base_url.into(),
                 tile,
+                sync,
             }
         }
     }
 
     #[cfg(target_arch = "wasm32")]
     async fn fetch_bytes<'a>(&self, path: &str) -> Result<Box<Reader<'a>>, AssetReaderError> {
+        info!("downloading {path}");
         use js_sys::Uint8Array;
         use wasm_bindgen::JsCast;
         use wasm_bindgen_futures::JsFuture;
@@ -95,6 +106,7 @@ impl HttpAssetReader {
 
     #[cfg(not(target_arch = "wasm32"))]
     async fn fetch_bytes<'a>(&self, path: &str) -> Result<Box<Reader<'a>>, AssetReaderError> {
+        info!("downloading {path}");
         let resp = self.client.get(path).await;
 
         trace!("fetched {resp:?} ... ");
@@ -134,11 +146,26 @@ impl AssetReader for HttpAssetReader {
         &'a self,
         path: &'a Path,
     ) -> BoxedFuture<'a, Result<Box<Reader<'a>>, AssetReaderError>> {
-        let path = path.display().to_string();
-        if self.tile {
-            let (x, rest) = path.split_once('_').unwrap();
-            let path = format!("lod1/15/{x}/{rest}.gz");
-            Box::pin(async move {
+        Box::pin(async move {
+            let cache_path = directories::ProjectDirs::from("org", "osmeta", "OSMeta")
+                .map(|dirs| dirs.cache_dir().join(path));
+            // Load from cache if the asset exists there.
+            if let Some(cache_path) = cache_path.clone() {
+                if cache_path.exists() {
+                    let file = File::open(&cache_path).await?;
+                    return Ok(Box::new(file) as Box<Reader>);
+                }
+            }
+            let path = path.display().to_string();
+
+            let mut bytes = vec![];
+            if self.tile {
+                // `tile://` urls are special for now, because we can't use `/` in the tile paths,
+                // as that will cause texture loading to be attempted in the subfolders instead of the root.
+                let (x, rest) = path.split_once('_').unwrap();
+                // The tile servers we're using have their files gzipped, so we download that and unzip it
+                // transparently and act as if there's a .glb file there.
+                let path = format!("lod1/15/{x}/{rest}.gz");
                 let mut bytes_compressed = Vec::new();
                 self.fetch_bytes(&path)
                     .await?
@@ -147,15 +174,22 @@ impl AssetReader for HttpAssetReader {
 
                 let mut decoder = GzDecoder::new(bytes_compressed.as_slice());
 
-                let mut bytes_uncompressed = Vec::new();
-
-                decoder.read_to_end(&mut bytes_uncompressed)?;
-
-                Ok(Box::new(VecReader::new(bytes_uncompressed)) as Box<Reader<'static>>)
-            })
-        } else {
-            Box::pin(async move { self.fetch_bytes(&path).await })
-        }
+                decoder.read_to_end(&mut bytes)?;
+            } else {
+                self.fetch_bytes(&path)
+                    .await?
+                    .read_to_end(&mut bytes)
+                    .await?;
+            };
+            if let Some(cache_path) = cache_path {
+                // Write asset to cache, but ensure only one HttpAssetReader writes at any given point in time
+                if self.sync.write().unwrap().insert(cache_path.clone()) {
+                    async_fs::create_dir_all(cache_path.parent().unwrap()).await?;
+                    File::create(&cache_path).await?.write_all(&bytes).await?;
+                }
+            }
+            Ok(Box::new(VecReader::new(bytes)) as Box<Reader<'static>>)
+        })
     }
 
     fn read_meta<'a>(
@@ -188,16 +222,20 @@ pub struct HttpAssetReaderPlugin {
 impl Plugin for HttpAssetReaderPlugin {
     fn build(&self, app: &mut App) {
         let base_url = self.base_url.clone();
+        let sync = Arc::new(RwLock::new(HashSet::new()));
+        let sync2 = sync.clone();
         app.register_asset_source(
             AssetSourceId::Default,
-            AssetSource::build()
-                .with_reader(move || Box::new(HttpAssetReader::new(&base_url, false))),
+            AssetSource::build().with_reader(move || {
+                Box::new(HttpAssetReader::new(&base_url, false, sync.clone()))
+            }),
         );
         let base_url = self.base_url.clone();
         app.register_asset_source(
             AssetSourceId::Name("tile".into()),
-            AssetSource::build()
-                .with_reader(move || Box::new(HttpAssetReader::new(&base_url, true))),
+            AssetSource::build().with_reader(move || {
+                Box::new(HttpAssetReader::new(&base_url, true, sync2.clone()))
+            }),
         );
     }
 }

src/tilemap.rs

@@ -92,7 +92,6 @@ impl TileMap {
         }
         // https://gltiles.osm2world.org/glb/lod1/15/17388/11332.glb#Scene0"
         let name: String = format!("tile://{}_{}.glb", pos.x, pos.y);
-        info!("Name: {}", name);
         // Start loading next tile
         self.loading = Some((pos, server.load(name))); // "models/17430_11371.glb#Scene0"
                                                        // Insert dummy tile while loading.