From 3504dc86a95baa655a3464d4f84257a02ea496e7 Mon Sep 17 00:00:00 2001 From: Oliver Scherer Date: Tue, 28 Nov 2023 20:47:09 +0100 Subject: [PATCH] Cache downloaded assets locally if possible --- Cargo.lock | 66 ++++++++++++++++++++++++++++++++++-- Cargo.toml | 2 ++ src/http_assets.rs | 84 +++++++++++++++++++++++++++++++++------------- src/tilemap.rs | 1 - 4 files changed, 127 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c8c2728..ab1bbf4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -341,6 +341,17 @@ dependencies = [ "futures-lite 1.13.0", ] +[[package]] +name = "async-fs" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd1f344136bad34df1f83a47f3fd7f2ab85d75cb8a940af4ccf6d482a84ea01b" +dependencies = [ + "async-lock 3.1.1", + "blocking", + "futures-lite 2.0.1", +] + [[package]] name = "async-global-executor" version = "2.4.0" @@ -602,7 +613,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccb2b67984088b23e223cfe9ec1befd89a110665a679acb06839bc4334ed37d6" dependencies = [ "async-broadcast", - "async-fs", + "async-fs 1.6.0", "async-lock 2.8.0", "bevy_app", "bevy_asset_macros", @@ -1854,6 +1865,27 @@ dependencies = [ "generic-array", ] +[[package]] +name = "directories" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a49173b84e034382284f27f1af4dcbbd231ffa358c0fe316541a7337f376a35" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.48.0", +] + [[package]] name = "discard" version = "1.0.4" @@ -2814,6 +2846,17 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "libredox" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" +dependencies = [ + "bitflags 2.4.1", + "libc", + "redox_syscall 0.4.1", +] + [[package]] name = "libredox" version = "0.0.2" @@ -3363,23 +3406,31 @@ dependencies = [ "mint", ] +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "orbclient" version = "0.3.47" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52f0d54bde9774d3a51dcf281a5def240c71996bc6ca05d2c847ec8b2b216166" dependencies = [ - "libredox", + "libredox 0.0.2", ] [[package]] name = "osmeta" version = "0.1.0" dependencies = [ + "async-fs 2.1.0", "bevy", "bevy_flycam", "bevy_oxr", "bevy_screen_diagnostics", + "directories", "flate2", "futures-core", "futures-io", @@ -3741,6 +3792,17 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_users" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4" +dependencies = [ + "getrandom 0.2.11", + "libredox 0.0.1", + "thiserror", +] + [[package]] name = "regex" version = "1.10.2" diff --git a/Cargo.toml b/Cargo.toml index 34f6fd3..bca2ba2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,8 @@ futures-io = "0.3.29" glam = "0" bevy_screen_diagnostics = { git = "https://github.com/oli-obk/bevy_screen_diagnostics.git" } globe-rs = "0.1.8" +directories = "5.0.1" +async-fs = "2.1.0" [target.'cfg(not(target_arch = "wasm32"))'.dependencies] surf = { version = "2.3.2", default-features = false, features = [ diff --git a/src/http_assets.rs b/src/http_assets.rs index f25788a..5b14f38 100644 --- a/src/http_assets.rs +++ b/src/http_assets.rs @@ -1,32 +1,41 @@ // Taken from https://github.com/lizelive/bevy_http and modified. +use async_fs::File; use bevy::{ asset::{ io::{ AssetReader, AssetReaderError, AssetSource, AssetSourceId, PathStream, Reader, VecReader, }, - AsyncReadExt, + AsyncReadExt, AsyncWriteExt, }, prelude::*, utils::BoxedFuture, }; use flate2::read::GzDecoder; -use std::{io::Read, path::Path}; +use std::{ + collections::HashSet, + io::Read, + path::{Path, PathBuf}, + sync::{Arc, RwLock}, +}; /// A custom asset reader implementation that wraps a given asset reader implementation -pub struct HttpAssetReader { +struct HttpAssetReader { #[cfg(target_arch = "wasm32")] base_url: String, #[cfg(not(target_arch = "wasm32"))] client: surf::Client, /// Whether to load tiles from this path tile: bool, + /// Used to ensure the same asset doesn't get its cache file written twice at the same time, + /// as that depends on the OS whether it succeeds (could result in broken cache files). + sync: Arc>>, } impl HttpAssetReader { /// Creates a new `HttpAssetReader`. The path provided will be used to build URLs to query for assets. - pub fn new(base_url: &str, tile: bool) -> Self { + fn new(base_url: &str, tile: bool, sync: Arc>>) -> Self { #[cfg(not(target_arch = "wasm32"))] { let base_url = surf::Url::parse(base_url).expect("invalid base url"); @@ -35,19 +44,21 @@ impl HttpAssetReader { let client = client.set_base_url(base_url); let client = client.try_into().expect("could not create http client"); - Self { client, tile } + Self { client, tile, sync } } #[cfg(target_arch = "wasm32")] { Self { base_url: base_url.into(), tile, + sync, } } } #[cfg(target_arch = "wasm32")] async fn fetch_bytes<'a>(&self, path: &str) -> Result>, AssetReaderError> { + info!("downloading {path}"); use js_sys::Uint8Array; use wasm_bindgen::JsCast; use wasm_bindgen_futures::JsFuture; @@ -95,6 +106,7 @@ impl HttpAssetReader { #[cfg(not(target_arch = "wasm32"))] async fn fetch_bytes<'a>(&self, path: &str) -> Result>, AssetReaderError> { + info!("downloading {path}"); let resp = self.client.get(path).await; trace!("fetched {resp:?} ... "); @@ -134,11 +146,26 @@ impl AssetReader for HttpAssetReader { &'a self, path: &'a Path, ) -> BoxedFuture<'a, Result>, AssetReaderError>> { - let path = path.display().to_string(); - if self.tile { - let (x, rest) = path.split_once('_').unwrap(); - let path = format!("lod1/15/{x}/{rest}.gz"); - Box::pin(async move { + Box::pin(async move { + let cache_path = directories::ProjectDirs::from("org", "osmeta", "OSMeta") + .map(|dirs| dirs.cache_dir().join(path)); + // Load from cache if the asset exists there. + if let Some(cache_path) = cache_path.clone() { + if cache_path.exists() { + let file = File::open(&cache_path).await?; + return Ok(Box::new(file) as Box); + } + } + let path = path.display().to_string(); + + let mut bytes = vec![]; + if self.tile { + // `tile://` urls are special for now, because we can't use `/` in the tile paths, + // as that will cause texture loading to be attempted in the subfolders instead of the root. + let (x, rest) = path.split_once('_').unwrap(); + // The tile servers we're using have their files gzipped, so we download that and unzip it + // transparently and act as if there's a .glb file there. + let path = format!("lod1/15/{x}/{rest}.gz"); let mut bytes_compressed = Vec::new(); self.fetch_bytes(&path) .await? @@ -147,15 +174,22 @@ impl AssetReader for HttpAssetReader { let mut decoder = GzDecoder::new(bytes_compressed.as_slice()); - let mut bytes_uncompressed = Vec::new(); - - decoder.read_to_end(&mut bytes_uncompressed)?; - - Ok(Box::new(VecReader::new(bytes_uncompressed)) as Box>) - }) - } else { - Box::pin(async move { self.fetch_bytes(&path).await }) - } + decoder.read_to_end(&mut bytes)?; + } else { + self.fetch_bytes(&path) + .await? + .read_to_end(&mut bytes) + .await?; + }; + if let Some(cache_path) = cache_path { + // Write asset to cache, but ensure only one HttpAssetReader writes at any given point in time + if self.sync.write().unwrap().insert(cache_path.clone()) { + async_fs::create_dir_all(cache_path.parent().unwrap()).await?; + File::create(&cache_path).await?.write_all(&bytes).await?; + } + } + Ok(Box::new(VecReader::new(bytes)) as Box>) + }) } fn read_meta<'a>( @@ -188,16 +222,20 @@ pub struct HttpAssetReaderPlugin { impl Plugin for HttpAssetReaderPlugin { fn build(&self, app: &mut App) { let base_url = self.base_url.clone(); + let sync = Arc::new(RwLock::new(HashSet::new())); + let sync2 = sync.clone(); app.register_asset_source( AssetSourceId::Default, - AssetSource::build() - .with_reader(move || Box::new(HttpAssetReader::new(&base_url, false))), + AssetSource::build().with_reader(move || { + Box::new(HttpAssetReader::new(&base_url, false, sync.clone())) + }), ); let base_url = self.base_url.clone(); app.register_asset_source( AssetSourceId::Name("tile".into()), - AssetSource::build() - .with_reader(move || Box::new(HttpAssetReader::new(&base_url, true))), + AssetSource::build().with_reader(move || { + Box::new(HttpAssetReader::new(&base_url, true, sync2.clone())) + }), ); } } diff --git a/src/tilemap.rs b/src/tilemap.rs index cf9e9a7..977c37b 100644 --- a/src/tilemap.rs +++ b/src/tilemap.rs @@ -92,7 +92,6 @@ impl TileMap { } // https://gltiles.osm2world.org/glb/lod1/15/17388/11332.glb#Scene0" let name: String = format!("tile://{}_{}.glb", pos.x, pos.y); - info!("Name: {}", name); // Start loading next tile self.loading = Some((pos, server.load(name))); // "models/17430_11371.glb#Scene0" // Insert dummy tile while loading.