From ef5a344b42c24487b471498352cb301cb39c6eab Mon Sep 17 00:00:00 2001 From: Ryo Yamashita Date: Tue, 13 Feb 2024 00:33:44 +0900 Subject: [PATCH 1/2] =?UTF-8?q?async=5Fzip=E3=82=92v0.0.16=E3=81=AB?= =?UTF-8?q?=E4=B8=8A=E3=81=92=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 98 ++++++++++++++++++------- Cargo.toml | 3 +- crates/test_util/Cargo.toml | 1 + crates/test_util/src/lib.rs | 17 ++--- crates/voicevox_core/src/voice_model.rs | 27 +++---- 5 files changed, 92 insertions(+), 54 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2baa09fc0..af3a7e682 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -219,19 +219,20 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.3.15" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942c7cd7ae39e91bde4820d74132e9862e62c2f386c3aa90ccf55949f5bad63a" +checksum = "a116f46a969224200a0a97f29cfd4c50e7534e4b4826bd23ea2c3c533039c82c" dependencies = [ "bzip2", + "deflate64", "flate2", "futures-core", + "futures-io", "memchr", "pin-project-lite", - "tokio", "xz2", - "zstd", - "zstd-safe", + "zstd 0.13.0", + "zstd-safe 7.0.0", ] [[package]] @@ -242,8 +243,8 @@ checksum = "871f9bb5e0a22eeb7e8cf16641feb87c9dc67032ccf8ff49e772eb9941d3a965" dependencies = [ "async-task", "concurrent-queue", - "fastrand", - "futures-lite", + "fastrand 1.8.0", + "futures-lite 1.12.0", "once_cell", "slab", ] @@ -259,7 +260,7 @@ dependencies = [ "async-io", "async-lock", "blocking", - "futures-lite", + "futures-lite 1.12.0", "once_cell", ] @@ -271,7 +272,7 @@ checksum = "83e21f3a490c72b3b0cf44962180e60045de2925d8dff97918f7ee43c8f637c7" dependencies = [ "autocfg", "concurrent-queue", - "futures-lite", + "futures-lite 1.12.0", "libc", "log", "once_cell", @@ -307,7 +308,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-io", - "futures-lite", + "futures-lite 1.12.0", "gloo-timers", "kv-log-macro", "log", @@ -338,17 +339,18 @@ dependencies = [ [[package]] name = "async_zip" -version = "0.0.11" +version = "0.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c50d29ab7e2f9e808cca1a69ea56a36f4ff216f54a41a23aae1fd4afc05cc020" +checksum = "527207465fb6dcafbf661b0d4a51d0d2306c9d0c2975423079a6caa807930daf" dependencies = [ "async-compression", "chrono", "crc32fast", - "log", + "futures-lite 2.2.0", "pin-project", "thiserror", "tokio", + "tokio-util", ] [[package]] @@ -479,8 +481,8 @@ dependencies = [ "async-channel", "async-task", "atomic-waker", - "fastrand", - "futures-lite", + "fastrand 1.8.0", + "futures-lite 1.12.0", "once_cell", ] @@ -1080,6 +1082,12 @@ dependencies = [ "syn 2.0.38", ] +[[package]] +name = "deflate64" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9576c1de19747eb6f5efb6a806c3e836512bbdb17bfedc984ccb0bcc953c8390" + [[package]] name = "derive-getters" version = "0.2.0" @@ -1362,6 +1370,12 @@ dependencies = [ "instant", ] +[[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + [[package]] name = "filetime" version = "0.2.17" @@ -1474,7 +1488,7 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48" dependencies = [ - "fastrand", + "fastrand 1.8.0", "futures-core", "futures-io", "memchr", @@ -1483,6 +1497,19 @@ dependencies = [ "waker-fn", ] +[[package]] +name = "futures-lite" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445ba825b27408685aaecefd65178908c36c6e96aaf6d8599419d46e624192ba" +dependencies = [ + "fastrand 2.0.1", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + [[package]] name = "futures-macro" version = "0.3.28" @@ -1750,7 +1777,7 @@ dependencies = [ "async-std", "base64 0.13.0", "cookie", - "futures-lite", + "futures-lite 1.12.0", "infer", "pin-project-lite", "rand 0.7.3", @@ -1971,7 +1998,7 @@ dependencies = [ "curl", "curl-sys", "flume", - "futures-lite", + "futures-lite 1.12.0", "http", "log", "once_cell", @@ -2603,9 +2630,9 @@ checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" [[package]] name = "parking" -version = "2.0.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" +checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" [[package]] name = "parking_lot" @@ -3820,7 +3847,7 @@ checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" dependencies = [ "autocfg", "cfg-if", - "fastrand", + "fastrand 1.8.0", "redox_syscall 0.3.5", "rustix 0.37.19", "windows-sys 0.48.0", @@ -3850,6 +3877,7 @@ dependencies = [ "async_zip", "flate2", "fs-err", + "futures-lite 2.2.0", "once_cell", "serde", "serde_json", @@ -4021,6 +4049,7 @@ checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" dependencies = [ "bytes 1.1.0", "futures-core", + "futures-io", "futures-sink", "pin-project-lite", "tokio", @@ -4919,7 +4948,7 @@ dependencies = [ "pbkdf2", "sha1 0.10.5", "time 0.3.15", - "zstd", + "zstd 0.11.2+zstd.1.5.2", ] [[package]] @@ -4928,7 +4957,16 @@ version = "0.11.2+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" dependencies = [ - "zstd-safe", + "zstd-safe 5.0.2+zstd.1.5.2", +] + +[[package]] +name = "zstd" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" +dependencies = [ + "zstd-safe 7.0.0", ] [[package]] @@ -4941,13 +4979,21 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "zstd-safe" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" +dependencies = [ + "zstd-sys", +] + [[package]] name = "zstd-sys" -version = "2.0.5+zstd.1.5.2" +version = "2.0.9+zstd.1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edc50ffce891ad571e9f9afe5039c4837bede781ac4bb13052ed7ae695518596" +checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" dependencies = [ "cc", - "libc", "pkg-config", ] diff --git a/Cargo.toml b/Cargo.toml index 97c39f93b..8500292a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ anstyle-query = "1.0.0" anyhow = "1.0.65" assert_cmd = "2.0.8" async-std = "1.12.0" -async_zip = "0.0.11" +async_zip = "=0.0.16" binstall-tar = "0.4.39" bytes = "1.1.0" camino = "1.1.6" @@ -33,6 +33,7 @@ fs-err = "2.9.0" futures = "0.3.26" futures-core = "0.3.25" futures-util = "0.3.25" +futures-lite = "2.2.0" heck = "0.4.1" humansize = "2.1.2" indexmap = "2.0.0" diff --git a/crates/test_util/Cargo.toml b/crates/test_util/Cargo.toml index 6394b321a..d9c69d26d 100644 --- a/crates/test_util/Cargo.toml +++ b/crates/test_util/Cargo.toml @@ -4,6 +4,7 @@ edition.workspace = true [dependencies] async_zip = { workspace = true, features = ["full"] } +futures-lite.workspace = true once_cell.workspace = true serde = { workspace = true, features = ["derive"] } serde_json.workspace = true diff --git a/crates/test_util/src/lib.rs b/crates/test_util/src/lib.rs index 05da2a33f..f5c9b9de4 100644 --- a/crates/test_util/src/lib.rs +++ b/crates/test_util/src/lib.rs @@ -1,6 +1,7 @@ mod typing; -use async_zip::{write::ZipFileWriter, Compression, ZipEntryBuilder}; +use async_zip::{base::write::ZipFileWriter, Compression, ZipEntryBuilder}; +use futures_lite::AsyncWriteExt as _; use once_cell::sync::Lazy; use std::{ collections::HashMap, @@ -8,7 +9,7 @@ use std::{ }; use tokio::{ fs::{self, File}, - io::{AsyncReadExt, AsyncWriteExt}, + io::AsyncReadExt, sync::Mutex, }; @@ -50,18 +51,12 @@ pub async fn convert_zip_vvm(dir: impl AsRef) -> PathBuf { fs::create_dir_all(out_file_path.parent().unwrap()) .await .unwrap(); - let mut out_file = File::create(&out_file_path).await.unwrap(); - let mut writer = ZipFileWriter::new(&mut out_file); + let out_file = File::create(&out_file_path).await.unwrap(); + let mut writer = ZipFileWriter::with_tokio(out_file); for entry in dir.read_dir().unwrap().flatten() { let entry_builder = ZipEntryBuilder::new( - entry - .path() - .file_name() - .unwrap() - .to_str() - .unwrap() - .to_string(), + entry.path().file_name().unwrap().to_str().unwrap().into(), Compression::Deflate, ); let mut entry_writer = writer.write_entry_stream(entry_builder).await.unwrap(); diff --git a/crates/voicevox_core/src/voice_model.rs b/crates/voicevox_core/src/voice_model.rs index fc8f4d20f..52945bc50 100644 --- a/crates/voicevox_core/src/voice_model.rs +++ b/crates/voicevox_core/src/voice_model.rs @@ -288,13 +288,13 @@ pub(crate) mod tokio { #[derive(new)] struct AsyncVvmEntryReader { - reader: async_zip::read::fs::ZipFileReader, + reader: async_zip::tokio::read::fs::ZipFileReader, entry_map: HashMap, } impl AsyncVvmEntryReader { async fn open(path: &Path) -> LoadModelResult { - let reader = async_zip::read::fs::ZipFileReader::new(path) + let reader = async_zip::tokio::read::fs::ZipFileReader::new(path) .await .map_err(|source| LoadModelError { path: path.to_owned(), @@ -305,17 +305,14 @@ pub(crate) mod tokio { .file() .entries() .iter() - .filter(|e| !e.entry().dir()) - .enumerate() - .map(|(i, e)| { - ( - e.entry().filename().to_string(), - AsyncVvmEntry { - index: i, - entry: e.entry().clone(), - }, - ) + .flat_map(|e| { + // 非UTF-8のファイルを利用することはないため、無視する + let filename = e.filename().as_str().ok()?; + (!e.dir().ok()?).then_some(())?; + Some((filename.to_owned(), (**e).clone())) }) + .enumerate() + .map(|(i, (filename, entry))| (filename, AsyncVvmEntry { index: i, entry })) .collect(); Ok(AsyncVvmEntryReader::new(reader, entry_map)) } @@ -336,11 +333,9 @@ pub(crate) mod tokio { .entry_map .get(filename) .ok_or_else(|| io::Error::from(io::ErrorKind::NotFound))?; - let mut manifest_reader = self.reader.entry(me.index).await?; + let mut manifest_reader = self.reader.reader_with_entry(me.index).await?; let mut buf = Vec::with_capacity(me.entry.uncompressed_size() as usize); - manifest_reader - .read_to_end_checked(&mut buf, &me.entry) - .await?; + manifest_reader.read_to_end_checked(&mut buf).await?; Ok::<_, anyhow::Error>(buf) } .await From 7b6d1b8a7dfc4a30dbdb022a998c44650611acf4 Mon Sep 17 00:00:00 2001 From: Ryo Yamashita Date: Thu, 15 Feb 2024 02:57:53 +0900 Subject: [PATCH 2/2] =?UTF-8?q?`async=5Fzip::base::read::mem`=E3=82=92?= =?UTF-8?q?=E4=BD=BF=E3=81=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 57 +------------------------ crates/test_util/Cargo.toml | 2 +- crates/test_util/src/lib.rs | 6 +-- crates/voicevox_core/Cargo.toml | 2 +- crates/voicevox_core/src/voice_model.rs | 32 ++++++++------ 5 files changed, 25 insertions(+), 74 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index af3a7e682..5cae922ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -223,16 +223,11 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a116f46a969224200a0a97f29cfd4c50e7534e4b4826bd23ea2c3c533039c82c" dependencies = [ - "bzip2", - "deflate64", "flate2", "futures-core", "futures-io", "memchr", "pin-project-lite", - "xz2", - "zstd 0.13.0", - "zstd-safe 7.0.0", ] [[package]] @@ -344,13 +339,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "527207465fb6dcafbf661b0d4a51d0d2306c9d0c2975423079a6caa807930daf" dependencies = [ "async-compression", - "chrono", "crc32fast", "futures-lite 2.2.0", "pin-project", "thiserror", - "tokio", - "tokio-util", ] [[package]] @@ -1082,12 +1074,6 @@ dependencies = [ "syn 2.0.38", ] -[[package]] -name = "deflate64" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9576c1de19747eb6f5efb6a806c3e836512bbdb17bfedc984ccb0bcc953c8390" - [[package]] name = "derive-getters" version = "0.2.0" @@ -2205,17 +2191,6 @@ dependencies = [ "value-bag", ] -[[package]] -name = "lzma-sys" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "matchers" version = "0.1.0" @@ -4049,7 +4024,6 @@ checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" dependencies = [ "bytes 1.1.0", "futures-core", - "futures-io", "futures-sink", "pin-project-lite", "tokio", @@ -4910,15 +4884,6 @@ dependencies = [ "fs-err", ] -[[package]] -name = "xz2" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" -dependencies = [ - "lzma-sys", -] - [[package]] name = "yansi" version = "0.5.1" @@ -4948,7 +4913,7 @@ dependencies = [ "pbkdf2", "sha1 0.10.5", "time 0.3.15", - "zstd 0.11.2+zstd.1.5.2", + "zstd", ] [[package]] @@ -4957,16 +4922,7 @@ version = "0.11.2+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" dependencies = [ - "zstd-safe 5.0.2+zstd.1.5.2", -] - -[[package]] -name = "zstd" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" -dependencies = [ - "zstd-safe 7.0.0", + "zstd-safe", ] [[package]] @@ -4979,15 +4935,6 @@ dependencies = [ "zstd-sys", ] -[[package]] -name = "zstd-safe" -version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" -dependencies = [ - "zstd-sys", -] - [[package]] name = "zstd-sys" version = "2.0.9+zstd.1.5.5" diff --git a/crates/test_util/Cargo.toml b/crates/test_util/Cargo.toml index d9c69d26d..bea6e4d4e 100644 --- a/crates/test_util/Cargo.toml +++ b/crates/test_util/Cargo.toml @@ -3,7 +3,7 @@ name = "test_util" edition.workspace = true [dependencies] -async_zip = { workspace = true, features = ["full"] } +async_zip = { workspace = true, features = ["deflate"] } futures-lite.workspace = true once_cell.workspace = true serde = { workspace = true, features = ["derive"] } diff --git a/crates/test_util/src/lib.rs b/crates/test_util/src/lib.rs index f5c9b9de4..7522c81f7 100644 --- a/crates/test_util/src/lib.rs +++ b/crates/test_util/src/lib.rs @@ -51,8 +51,7 @@ pub async fn convert_zip_vvm(dir: impl AsRef) -> PathBuf { fs::create_dir_all(out_file_path.parent().unwrap()) .await .unwrap(); - let out_file = File::create(&out_file_path).await.unwrap(); - let mut writer = ZipFileWriter::with_tokio(out_file); + let mut writer = ZipFileWriter::new(vec![]); for entry in dir.read_dir().unwrap().flatten() { let entry_builder = ZipEntryBuilder::new( @@ -66,7 +65,8 @@ pub async fn convert_zip_vvm(dir: impl AsRef) -> PathBuf { entry_writer.write_all(&buf).await.unwrap(); entry_writer.close().await.unwrap(); } - writer.close().await.unwrap(); + let zip = writer.close().await.unwrap(); + fs::write(&out_file_path, zip).await.unwrap(); } out_file_path } diff --git a/crates/voicevox_core/Cargo.toml b/crates/voicevox_core/Cargo.toml index 8b1ab92f5..36d501b06 100644 --- a/crates/voicevox_core/Cargo.toml +++ b/crates/voicevox_core/Cargo.toml @@ -10,7 +10,7 @@ directml = ["onnxruntime/directml"] [dependencies] anyhow.workspace = true -async_zip = { workspace = true, features = ["full"] } +async_zip = { workspace = true, features = ["deflate"] } camino.workspace = true derive-getters.workspace = true derive-new.workspace = true diff --git a/crates/voicevox_core/src/voice_model.rs b/crates/voicevox_core/src/voice_model.rs index 52945bc50..77e7bd784 100644 --- a/crates/voicevox_core/src/voice_model.rs +++ b/crates/voicevox_core/src/voice_model.rs @@ -287,20 +287,24 @@ pub(crate) mod tokio { } #[derive(new)] - struct AsyncVvmEntryReader { - reader: async_zip::tokio::read::fs::ZipFileReader, + struct AsyncVvmEntryReader<'a> { + path: &'a Path, + reader: async_zip::base::read::mem::ZipFileReader, entry_map: HashMap, } - impl AsyncVvmEntryReader { - async fn open(path: &Path) -> LoadModelResult { - let reader = async_zip::tokio::read::fs::ZipFileReader::new(path) - .await - .map_err(|source| LoadModelError { - path: path.to_owned(), - context: LoadModelErrorKind::OpenZipFile, - source: Some(source.into()), - })?; + impl<'a> AsyncVvmEntryReader<'a> { + async fn open(path: &'a Path) -> LoadModelResult { + let reader = async { + let file = fs_err::tokio::read(path).await?; + async_zip::base::read::mem::ZipFileReader::new(file).await + } + .await + .map_err(|source| LoadModelError { + path: path.to_owned(), + context: LoadModelErrorKind::OpenZipFile, + source: Some(source.into()), + })?; let entry_map: HashMap<_, _> = reader .file() .entries() @@ -314,12 +318,12 @@ pub(crate) mod tokio { .enumerate() .map(|(i, (filename, entry))| (filename, AsyncVvmEntry { index: i, entry })) .collect(); - Ok(AsyncVvmEntryReader::new(reader, entry_map)) + Ok(AsyncVvmEntryReader::new(path, reader, entry_map)) } async fn read_vvm_json(&self, filename: &str) -> LoadModelResult { let bytes = self.read_vvm_entry(filename).await?; serde_json::from_slice(&bytes).map_err(|source| LoadModelError { - path: self.reader.path().to_owned(), + path: self.path.to_owned(), context: LoadModelErrorKind::ReadZipEntry { filename: filename.to_owned(), }, @@ -340,7 +344,7 @@ pub(crate) mod tokio { } .await .map_err(|source| LoadModelError { - path: self.reader.path().to_owned(), + path: self.path.to_owned(), context: LoadModelErrorKind::ReadZipEntry { filename: filename.to_owned(), },