diff --git a/Cargo.lock b/Cargo.lock index f2d718e..2bcbc61 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,7 +23,7 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "const-random", "getrandom", "once_cell", @@ -31,13 +31,22 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "aho-corasick" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81ce3d38065e618af2d7b77e10c5ad9a069859b4be3c2250f674af3840d9c8a5" +dependencies = [ + "memchr 2.7.2", +] + [[package]] name = "aho-corasick" version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ - "memchr", + "memchr 2.7.2", ] [[package]] @@ -82,6 +91,15 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi 0.3.9", +] + [[package]] name = "anstream" version = "0.6.13" @@ -209,7 +227,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "num", ] @@ -259,9 +277,9 @@ dependencies = [ "chrono", "csv", "csv-core", - "lazy_static", + "lazy_static 1.4.0", "lexical-core", - "regex", + "regex 1.10.4", ] [[package]] @@ -337,7 +355,7 @@ dependencies = [ "arrow-data", "arrow-schema", "half", - "hashbrown 0.14.3", + "hashbrown 0.14.5", ] [[package]] @@ -374,12 +392,18 @@ dependencies = [ "arrow-data", "arrow-schema", "arrow-select", - "memchr", + "memchr 2.7.2", "num", - "regex", - "regex-syntax", + "regex 1.10.4", + "regex-syntax 0.8.3", ] +[[package]] +name = "ascii" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" + [[package]] name = "atoi" version = "2.0.0" @@ -403,7 +427,7 @@ checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ "hermit-abi 0.1.19", "libc", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -420,7 +444,7 @@ checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" dependencies = [ "addr2line", "cc", - "cfg-if", + "cfg-if 1.0.0", "libc", "miniz_oxide", "object", @@ -439,6 +463,25 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51" +[[package]] +name = "bindgen" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57253399c086f4f29e57ffd3b5cdbc23a806a00292619351aa4cfa39cb49d4ea" +dependencies = [ + "cexpr", + "cfg-if 0.1.10", + "clang-sys", + "clap 2.34.0", + "env_logger 0.4.3", + "lazy_static 0.2.11", + "log 0.3.9", + "peeking_take_while", + "quote 0.3.15", + "regex 0.2.11", + "which", +] + [[package]] name = "binrw" version = "0.13.3" @@ -459,7 +502,7 @@ dependencies = [ "either", "owo-colors", "proc-macro2", - "quote", + "quote 1.0.36", "syn 1.0.109", ] @@ -505,6 +548,16 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "build-probe-mpi" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3234fa6de2f6e0e338c7183ba09ae68c8f2bd6919d8763362597627362b7f8fe" +dependencies = [ + "pkg-config", + "shell-words", +] + [[package]] name = "bumpalo" version = "3.16.0" @@ -527,7 +580,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4da9a32f3fed317401fa3c862968128267c3106685286e15d5aaa3d7389c2f60" dependencies = [ "proc-macro2", - "quote", + "quote 1.0.36", "syn 2.0.60", ] @@ -558,9 +611,9 @@ dependencies = [ "clap 3.2.25", "heck", "indexmap 1.9.3", - "log", + "log 0.4.21", "proc-macro2", - "quote", + "quote 1.0.36", "serde", "serde_json", "syn 1.0.109", @@ -579,6 +632,21 @@ dependencies = [ "once_cell", ] +[[package]] +name = "cexpr" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42aac45e9567d97474a834efdee3081b3c942b2205be932092f53354ce503d6c" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + [[package]] name = "cfg-if" version = "1.0.0" @@ -596,7 +664,7 @@ dependencies = [ "js-sys", "num-traits", "wasm-bindgen", - "windows-targets 0.52.5", + "windows-targets", ] [[package]] @@ -648,6 +716,32 @@ dependencies = [ "half", ] +[[package]] +name = "clang-sys" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e414af9726e1d11660801e73ccc7fb81803fb5f49e5903a25b348b2b3b480d2e" +dependencies = [ + "glob 0.2.11", + "libc", + "libloading 0.4.3", +] + +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "ansi_term", + "atty", + "bitflags 1.3.2", + "strsim 0.8.0", + "textwrap 0.11.0", + "unicode-width", + "vec_map", +] + [[package]] name = "clap" version = "3.2.25" @@ -660,7 +754,7 @@ dependencies = [ "indexmap 1.9.3", "strsim 0.10.0", "termcolor", - "textwrap", + "textwrap 0.16.1", ] [[package]] @@ -758,7 +852,7 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -779,7 +873,7 @@ dependencies = [ "oorandom", "plotters", "rayon", - "regex", + "regex 1.10.4", "serde", "serde_derive", "serde_json", @@ -849,8 +943,8 @@ dependencies = [ "bitflags 2.5.0", "crossterm_winapi", "libc", - "parking_lot", - "winapi", + "parking_lot 0.12.2", + "winapi 0.3.9", ] [[package]] @@ -859,7 +953,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" dependencies = [ - "winapi", + "winapi 0.3.9", ] [[package]] @@ -896,7 +990,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" dependencies = [ - "memchr", + "memchr 2.7.2", ] [[package]] @@ -927,7 +1021,7 @@ version = "0.8.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -938,7 +1032,7 @@ checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" dependencies = [ "once_cell", "proc-macro2", - "quote", + "quote 1.0.36", "syn 2.0.60", ] @@ -948,8 +1042,18 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea" dependencies = [ - "log", - "regex", + "log 0.4.21", + "regex 1.10.4", +] + +[[package]] +name = "env_logger" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ddf21e73e016298f5cb37d6ef8e8da8e39f91f9ec8b0df44b7deb16a9f8cd5b" +dependencies = [ + "log 0.3.9", + "regex 0.2.11", ] [[package]] @@ -962,7 +1066,7 @@ dependencies = [ "anstyle", "env_filter", "humantime", - "log", + "log 0.4.21", ] [[package]] @@ -971,6 +1075,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "errno" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +dependencies = [ + "errno-dragonfly", + "libc", + "winapi 0.3.9", +] + [[package]] name = "errno" version = "0.3.8" @@ -981,6 +1096,16 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "ethnum" version = "1.5.0" @@ -1007,9 +1132,9 @@ checksum = "4f4cdac9e4065d7c48e30770f8665b8cef9a3a73a63a4056a33a5f395bc7cf75" [[package]] name = "fastrand" -version = "2.0.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" [[package]] name = "flatbuffers" @@ -1023,9 +1148,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.28" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" +checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" dependencies = [ "crc32fast", "miniz_oxide", @@ -1037,6 +1162,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee1b05cbd864bcaecbd3455d6d967862d446e4ebfc3c2e5e5b9841e53cba6673" +[[package]] +name = "gcc" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" + [[package]] name = "generic-array" version = "0.14.7" @@ -1053,7 +1184,7 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "js-sys", "libc", "wasi", @@ -1066,6 +1197,12 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +[[package]] +name = "glob" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb" + [[package]] name = "glob" version = "0.3.1" @@ -1078,7 +1215,7 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "crunchy", "num-traits", ] @@ -1091,15 +1228,75 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.14.3" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash", "allocator-api2", "rayon", ] +[[package]] +name = "hdf5" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdcd9b131fd67bb827b386d0dc63d3e74196a14616ef800acf87ca5fef741a10" +dependencies = [ + "bitflags 1.3.2", + "cfg-if 1.0.0", + "errno 0.2.8", + "hdf5-derive", + "hdf5-sys", + "hdf5-types", + "lazy_static 1.4.0", + "libc", + "lzf-sys", + "mpi-sys", + "ndarray", + "parking_lot 0.11.2", + "paste", +] + +[[package]] +name = "hdf5-derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5a77ac6a41e6880594d506118c0b8bc665ec959fe4636e0c84809756d224820" +dependencies = [ + "proc-macro2", + "quote 1.0.36", + "syn 1.0.109", +] + +[[package]] +name = "hdf5-sys" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4842d5980dc311a7c8933c7b45534fdae84df5ae7939a0ae8e449a56d4beb3d2" +dependencies = [ + "libc", + "libloading 0.7.4", + "mpi-sys", + "pkg-config", + "regex 1.10.4", + "serde", + "serde_derive", + "winreg", +] + +[[package]] +name = "hdf5-types" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b47268c0dfb499b1ffe5638b6e7694e7a87fe49fb92eca998a4346e5483e428f" +dependencies = [ + "ascii", + "cfg-if 1.0.0", + "hdf5-sys", + "libc", +] + [[package]] name = "heck" version = "0.4.1" @@ -1182,7 +1379,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", - "hashbrown 0.14.3", + "hashbrown 0.14.5", ] [[package]] @@ -1191,6 +1388,15 @@ version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "integer-encoding" version = "3.0.4" @@ -1256,6 +1462,22 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "lazy_static" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73" + [[package]] name = "lazy_static" version = "1.4.0" @@ -1328,9 +1550,30 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.153" +version = "0.2.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346" + +[[package]] +name = "libloading" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +checksum = "fd38073de8f7965d0c17d30546d4bb6da311ab428d1c7a3fc71dff7f9d4979b9" +dependencies = [ + "kernel32-sys", + "lazy_static 1.4.0", + "winapi 0.2.8", +] + +[[package]] +name = "libloading" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +dependencies = [ + "cfg-if 1.0.0", + "winapi 0.3.9", +] [[package]] name = "libm" @@ -1346,14 +1589,23 @@ checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "lock_api" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" dependencies = [ "autocfg", "scopeguard", ] +[[package]] +name = "log" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" +dependencies = [ + "log 0.4.21", +] + [[package]] name = "log" version = "0.4.21" @@ -1389,6 +1641,15 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "lzf-sys" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0798d023ce0905e2c77ed96de92aab929ff9db2036cbef4edfee0daf33582aec" +dependencies = [ + "cc", +] + [[package]] name = "matrixmultiply" version = "0.3.8" @@ -1405,13 +1666,13 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "digest", ] [[package]] name = "mdfr" -version = "0.6.1" +version = "0.6.2" dependencies = [ "anyhow", "arrow", @@ -1424,17 +1685,19 @@ dependencies = [ "criterion", "crossbeam-channel", "encoding_rs", - "env_logger", + "env_logger 0.11.3", "fasteval", - "glob", + "glob 0.3.1", "half", + "hdf5", "itertools 0.12.1", "libc", - "log", + "log 0.4.21", "md-5", + "ndarray", "num", "numpy", - "parking_lot", + "parking_lot 0.12.2", "parquet", "polars", "pyo3", @@ -1448,6 +1711,15 @@ dependencies = [ "yazi", ] +[[package]] +name = "memchr" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "148fab2e51b4f1cfc66da2a7c32981d1d3c083a803978268bb11fe4b86925e7a" +dependencies = [ + "libc", +] + [[package]] name = "memchr" version = "2.7.2" @@ -1481,6 +1753,17 @@ dependencies = [ "adler", ] +[[package]] +name = "mpi-sys" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d6059465fe3500b0acdef3107ce0fa4635f4672725a74be5353e5e1fc8c4745" +dependencies = [ + "bindgen", + "build-probe-mpi", + "gcc", +] + [[package]] name = "multiversion" version = "0.7.4" @@ -1498,7 +1781,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79a74ddee9e0c27d2578323c13905793e91622148f138ba29738f9dddb835e90" dependencies = [ "proc-macro2", - "quote", + "quote 1.0.36", "syn 1.0.109", "target-features", ] @@ -1516,6 +1799,15 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "nom" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05aec50c70fd288702bcd93284a8444607f3292dbdf2a30de5ea5dcdbe72287b" +dependencies = [ + "memchr 1.0.2", +] + [[package]] name = "now" version = "0.1.3" @@ -1531,7 +1823,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" dependencies = [ - "winapi", + "winapi 0.3.9", ] [[package]] @@ -1631,7 +1923,7 @@ version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ - "memchr", + "memchr 2.7.2", ] [[package]] @@ -1669,25 +1961,50 @@ checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" [[package]] name = "parking_lot" -version = "0.12.1" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + +[[package]] +name = "parking_lot" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb" dependencies = [ "lock_api", - "parking_lot_core", + "parking_lot_core 0.9.10", ] [[package]] name = "parking_lot_core" -version = "0.9.9" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi 0.3.9", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", - "redox_syscall", + "redox_syscall 0.5.1", "smallvec", - "windows-targets 0.48.5", + "windows-targets", ] [[package]] @@ -1710,7 +2027,7 @@ dependencies = [ "chrono", "flate2", "half", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "lz4_flex", "num", "num-bigint", @@ -1734,7 +2051,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c705f256449c60da65e11ff6626e0c16a0a0b96aaa348de61376b249bc340f41" dependencies = [ - "regex", + "regex 1.10.4", ] [[package]] @@ -1743,6 +2060,12 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + [[package]] name = "percent-encoding" version = "2.3.1" @@ -1868,7 +2191,7 @@ dependencies = [ "fast-float", "foreign_vec", "getrandom", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "itoa", "itoap", "lz4", @@ -1924,7 +2247,7 @@ dependencies = [ "chrono-tz", "comfy-table", "either", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "indexmap 2.2.6", "num-traits", "once_cell", @@ -1936,7 +2259,7 @@ dependencies = [ "rand", "rand_distr", "rayon", - "regex", + "regex 1.10.4", "serde_json", "smartstring", "thiserror", @@ -1951,7 +2274,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5224d5d05e6b8a6f78b75951ae1b5f82c8ab1979e11ffaf5fd41941e3d5b0757" dependencies = [ "polars-arrow-format", - "regex", + "regex 1.10.4", "simdutf8", "thiserror", ] @@ -1969,7 +2292,7 @@ dependencies = [ "fast-float", "home", "itoa", - "memchr", + "memchr 2.7.2", "memmap2", "num-traits", "once_cell", @@ -1980,7 +2303,7 @@ dependencies = [ "polars-time", "polars-utils", "rayon", - "regex", + "regex 1.10.4", "ryu", "simdutf8", "smartstring", @@ -1994,7 +2317,7 @@ checksum = "89b2632b1af668e2058d5f8f916d8fbde3cac63d03ae29a705f598e41dcfeb7f" dependencies = [ "ahash", "bitflags 2.5.0", - "glob", + "glob 0.3.1", "once_cell", "polars-arrow", "polars-core", @@ -2022,10 +2345,10 @@ dependencies = [ "chrono", "chrono-tz", "either", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "hex", "indexmap 2.2.6", - "memchr", + "memchr 2.7.2", "num-traits", "polars-arrow", "polars-compute", @@ -2033,7 +2356,7 @@ dependencies = [ "polars-error", "polars-utils", "rayon", - "regex", + "regex 1.10.4", "smartstring", "unicode-reverse", "version_check", @@ -2067,7 +2390,7 @@ dependencies = [ "crossbeam-channel", "crossbeam-queue", "enum_dispatch", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "num-traits", "polars-arrow", "polars-compute", @@ -2092,7 +2415,7 @@ dependencies = [ "ahash", "bytemuck", "chrono-tz", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "once_cell", "percent-encoding", "polars-arrow", @@ -2103,7 +2426,7 @@ dependencies = [ "polars-utils", "rayon", "recursive", - "regex", + "regex 1.10.4", "smartstring", "strum_macros 0.25.3", "version_check", @@ -2155,7 +2478,7 @@ dependencies = [ "polars-error", "polars-ops", "polars-utils", - "regex", + "regex 1.10.4", "smartstring", ] @@ -2167,7 +2490,7 @@ checksum = "c760b6c698cfe2fbbbd93d6cfb408db14ececfe1d92445dae2229ce1b5b21ae8" dependencies = [ "ahash", "bytemuck", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "indexmap 2.2.6", "num-traits", "once_cell", @@ -2217,12 +2540,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53bdbb96d49157e65d45cc287af5f32ffadd5f4761438b527b055fb0d4bb8233" dependencies = [ "anyhow", - "cfg-if", + "cfg-if 1.0.0", "indoc", "libc", "memoffset", "num-complex", - "parking_lot", + "parking_lot 0.12.2", "portable-atomic", "pyo3-build-config", "pyo3-ffi", @@ -2258,7 +2581,7 @@ checksum = "7305c720fa01b8055ec95e484a6eca7a83c841267f0dd5280f0c8b8551d2c158" dependencies = [ "proc-macro2", "pyo3-macros-backend", - "quote", + "quote 1.0.36", "syn 2.0.60", ] @@ -2271,10 +2594,16 @@ dependencies = [ "heck", "proc-macro2", "pyo3-build-config", - "quote", + "quote 1.0.36", "syn 2.0.60", ] +[[package]] +name = "quote" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" + [[package]] name = "quote" version = "1.0.36" @@ -2326,9 +2655,9 @@ dependencies = [ [[package]] name = "raw-cpuid" -version = "11.0.1" +version = "11.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d86a7c4638d42c44551f4791a20e687dbb4c3de1f33c43dd71e355cd429def1" +checksum = "e29830cbb1290e404f24c73af91c5d8d631ce7e128691e9477556b540cd01ecd" dependencies = [ "bitflags 2.5.0", ] @@ -2375,10 +2704,19 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ - "quote", + "quote 1.0.36", "syn 2.0.60", ] +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -2388,16 +2726,38 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" +dependencies = [ + "bitflags 2.5.0", +] + +[[package]] +name = "regex" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9329abc99e39129fcceabd24cf5d85b4671ef7c29c50e972bc5afe32438ec384" +dependencies = [ + "aho-corasick 0.6.10", + "memchr 2.7.2", + "regex-syntax 0.5.6", + "thread_local", + "utf8-ranges", +] + [[package]] name = "regex" version = "1.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" dependencies = [ - "aho-corasick", - "memchr", + "aho-corasick 1.1.3", + "memchr 2.7.2", "regex-automata", - "regex-syntax", + "regex-syntax 0.8.3", ] [[package]] @@ -2406,9 +2766,18 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", + "aho-corasick 1.1.3", + "memchr 2.7.2", + "regex-syntax 0.8.3", +] + +[[package]] +name = "regex-syntax" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d707a4fa2637f2dca2ef9fd02225ec7661fe01a53623c1e6515b6916511f7a7" +dependencies = [ + "ucd-util", ] [[package]] @@ -2451,7 +2820,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ "bitflags 2.5.0", - "errno", + "errno 0.3.8", "libc", "linux-raw-sys", "windows-sys", @@ -2498,21 +2867,21 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.198" +version = "1.0.199" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9846a40c979031340571da2545a4e5b7c4163bdae79b301d5f86d03979451fcc" +checksum = "0c9f6e76df036c77cd94996771fb40db98187f096dd0b9af39c6c6e452ba966a" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.198" +version = "1.0.199" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e88edab869b01783ba905e7d0153f9fc1a6505a96e4ad3018011eedb838566d9" +checksum = "11bd257a6541e141e42ca6d24ae26f7714887b47e89aa739099104c7e4d3b7fc" dependencies = [ "proc-macro2", - "quote", + "quote 1.0.36", "syn 2.0.60", ] @@ -2527,6 +2896,12 @@ dependencies = [ "serde", ] +[[package]] +name = "shell-words" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" + [[package]] name = "simdutf8" version = "0.1.4" @@ -2568,7 +2943,7 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "743b4dc2cbde11890ccb254a8fc9d537fa41b36da00de2a1c5e9848c9bc42bd7" dependencies = [ - "log", + "log 0.4.21", ] [[package]] @@ -2578,10 +2953,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c886bd4480155fd3ef527d45e9ac8dd7118a898a46530b7b94c3e21866259fce" dependencies = [ "cc", - "cfg-if", + "cfg-if 1.0.0", "libc", "psm", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -2611,6 +2986,12 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + [[package]] name = "strsim" version = "0.10.0" @@ -2637,7 +3018,7 @@ checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" dependencies = [ "heck", "proc-macro2", - "quote", + "quote 1.0.36", "rustversion", "syn 2.0.60", ] @@ -2650,7 +3031,7 @@ checksum = "c6cf59daf282c0a494ba14fd21610a0325f9f90ec9d1231dea26bcb1d696c946" dependencies = [ "heck", "proc-macro2", - "quote", + "quote 1.0.36", "rustversion", "syn 2.0.60", ] @@ -2662,7 +3043,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2", - "quote", + "quote 1.0.36", "unicode-ident", ] @@ -2673,7 +3054,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "909518bc7b1c9b779f1bbf07f2929d35af9f0f37e47c6e9ef7f9dddc1e1821f3" dependencies = [ "proc-macro2", - "quote", + "quote 1.0.36", "unicode-ident", ] @@ -2683,7 +3064,7 @@ version = "0.30.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87341a165d73787554941cd5ef55ad728011566fe714e987d1b976c15dbc3a83" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "core-foundation-sys", "libc", "ntapi", @@ -2709,7 +3090,7 @@ version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "fastrand", "rustix", "windows-sys", @@ -2730,7 +3111,7 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b319995299c65d522680decf80f2c108d85b861d81dfe340a10d16cee29d9e6" dependencies = [ - "env_logger", + "env_logger 0.11.3", "test-log-macros", ] @@ -2741,10 +3122,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8f546451eaa38373f549093fe9fd05e7d2bade739e2ddf834b9968621d60107" dependencies = [ "proc-macro2", - "quote", + "quote 1.0.36", "syn 2.0.60", ] +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + [[package]] name = "textwrap" version = "0.16.1" @@ -2767,10 +3157,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1cd413b5d558b4c5bf3680e324a6fa5014e7b7c067a51e69dbdf47eb7148b66" dependencies = [ "proc-macro2", - "quote", + "quote 1.0.36", "syn 2.0.60", ] +[[package]] +name = "thread_local" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" +dependencies = [ + "lazy_static 1.4.0", +] + [[package]] name = "thrift" version = "0.17.0" @@ -2826,7 +3225,7 @@ version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "static_assertions", ] @@ -2836,6 +3235,12 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +[[package]] +name = "ucd-util" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abd2fc5d32b590614af8b0a20d837f32eca055edd0bbead59a9cfe80858be003" + [[package]] name = "unicode-ident" version = "1.0.12" @@ -2859,9 +3264,9 @@ checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" [[package]] name = "unicode-width" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" +checksum = "68f5e5f3158ecfd4b8ff6fe086db7c8467a2dfdac97fe420f2b7c4aa97af66d6" [[package]] name = "unindent" @@ -2869,6 +3274,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" +[[package]] +name = "utf8-ranges" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" + [[package]] name = "utf8parse" version = "0.2.1" @@ -2884,6 +3295,12 @@ dependencies = [ "getrandom", ] +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + [[package]] name = "version_check" version = "0.9.4" @@ -2918,7 +3335,7 @@ version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "wasm-bindgen-macro", ] @@ -2929,10 +3346,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" dependencies = [ "bumpalo", - "log", + "log 0.4.21", "once_cell", "proc-macro2", - "quote", + "quote 1.0.36", "syn 2.0.60", "wasm-bindgen-shared", ] @@ -2943,7 +3360,7 @@ version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" dependencies = [ - "quote", + "quote 1.0.36", "wasm-bindgen-macro-support", ] @@ -2954,7 +3371,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", - "quote", + "quote 1.0.36", "syn 2.0.60", "wasm-bindgen-backend", "wasm-bindgen-shared", @@ -2976,17 +3393,32 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "which" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e84a603e7e0b1ce1aa1ee2b109c7be00155ce52df5081590d1ffb93f4f515cb2" +dependencies = [ + "libc", +] + [[package]] name = "whoami" version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" dependencies = [ - "redox_syscall", + "redox_syscall 0.4.1", "wasite", "web-sys", ] +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" + [[package]] name = "winapi" version = "0.3.9" @@ -2997,6 +3429,12 @@ dependencies = [ "winapi-x86_64-pc-windows-gnu", ] +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" + [[package]] name = "winapi-i686-pc-windows-gnu" version = "0.4.0" @@ -3025,7 +3463,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" dependencies = [ "windows-core", - "windows-targets 0.52.5", + "windows-targets", ] [[package]] @@ -3034,7 +3472,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.5", + "windows-targets", ] [[package]] @@ -3043,22 +3481,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.5", -] - -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", + "windows-targets", ] [[package]] @@ -3067,46 +3490,28 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" dependencies = [ - "windows_aarch64_gnullvm 0.52.5", - "windows_aarch64_msvc 0.52.5", - "windows_i686_gnu 0.52.5", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", "windows_i686_gnullvm", - "windows_i686_msvc 0.52.5", - "windows_x86_64_gnu 0.52.5", - "windows_x86_64_gnullvm 0.52.5", - "windows_x86_64_msvc 0.52.5", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - [[package]] name = "windows_aarch64_msvc" version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - [[package]] name = "windows_i686_gnu" version = "0.52.5" @@ -3119,36 +3524,18 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - [[package]] name = "windows_i686_msvc" version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - [[package]] name = "windows_x86_64_gnu" version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.5" @@ -3157,15 +3544,19 @@ checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" [[package]] name = "windows_x86_64_msvc" -version = "0.48.5" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" [[package]] -name = "windows_x86_64_msvc" -version = "0.52.5" +name = "winreg" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "serde", + "winapi 0.3.9", +] [[package]] name = "xxhash-rust" @@ -3195,7 +3586,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", - "quote", + "quote 1.0.36", "syn 2.0.60", ] diff --git a/Cargo.toml b/Cargo.toml index 589a61c..99294a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mdfr" -version = "0.6.1" +version = "0.6.2" description = "A package for reading and writing MDF files" authors = ["ratal "] edition = "2021" @@ -12,10 +12,13 @@ readme = "README.md" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] -default = ["numpy", "parquet", "polars"] +default = ["numpy", "parquet", "polars", "hdf5"] numpy = ["dep:numpy", "dep:pyo3"] polars = ["dep:polars", "dep:numpy", "dep:pyo3"] parquet = ["dep:parquet"] +hdf5 = ["dep:hdf5", "ndarray"] +ndarray = ["dep:ndarray"] +hdf5-mpio = ["hdf5/mpio"] [dependencies] clap = "4" # for input arguments @@ -54,6 +57,10 @@ polars = { version = "0.39", features = [ "fmt", ], optional = true } # for python dataframe parquet = { version = "51.0.0", optional = true } # to write parquet file +hdf5 = { version = "0.8", optional = true, features = [ + "lzf", +] } # to export into hdf5 file +ndarray = { version = "0.15", optional = true } # to convert arraw data into ndarray, needed for hdf5 [dependencies.pyo3] version = "0.20" diff --git a/README.md b/README.md index 8916dc7..8b4dc53 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,8 @@ obj.plot('channel_name') # Export to parquet: obj.export_to_parquet('file_name', compression_option) +# Export to hdf5: +obj.export_to_hdf5('file_name') # write to mdf4 file, compressed or not -obj.write('file_name', conpression_flag) +obj.write('file_name', conpression_algo) ``` diff --git a/src/c_api.rs b/src/c_api.rs index 55dcc11..87a8ab9 100644 --- a/src/c_api.rs +++ b/src/c_api.rs @@ -191,42 +191,82 @@ pub unsafe extern "C" fn get_channel_array( } } -// / export to Parquet file -// / Compression can be one of the following strings -// / "snappy", "gzip", "lzo", "brotli", "lz4", "lz4raw" -// / or null pointer if no compression wanted -// #[no_mangle] -// pub unsafe extern "C" fn export_to_parquet( -// mdf: *const Mdf, -// file_name: *const c_char, -// compression: *const c_char, -// ) { -// // # Safety -// // -// // It is the caller's guarantee to ensure `file_name`: -// // -// // - is not a null pointer -// // - points to valid, initialized data -// // - points to memory ending in a null byte -// // - won't be mutated for the duration of this function call -// let name = CStr::from_ptr(file_name) -// .to_str() -// .expect("Could not convert into utf8 the file name string"); -// let comp = if compression.is_null() { -// None -// } else { -// Some( -// CStr::from_ptr(compression) -// .to_str() -// .expect("Could not convert into utf8 the compression string"), -// ) -// }; -// if let Some(mdf) = mdf.as_ref() { -// match mdf.export_to_parquet(name, comp) { -// Ok(_) => {} -// Err(e) => panic!("{}", e), -// } -// } else { -// panic!("Null pointer given for Mdf Rust object") -// } -// } +// export to Parquet file +// Compression can be one of the following strings +// "snappy", "gzip", "lzo", "brotli", "lz4", "lz4raw" +// or null pointer if no compression wanted +#[no_mangle] +pub unsafe extern "C" fn export_to_parquet( + mdf: *const Mdf, + file_name: *const c_char, + compression: *const c_char, +) { + // # Safety + // + // It is the caller's guarantee to ensure `file_name`: + // + // - is not a null pointer + // - points to valid, initialized data + // - points to memory ending in a null byte + // - won't be mutated for the duration of this function call + let name = CStr::from_ptr(file_name) + .to_str() + .expect("Could not convert into utf8 the file name string"); + let comp = if compression.is_null() { + None + } else { + Some( + CStr::from_ptr(compression) + .to_str() + .expect("Could not convert into utf8 the compression string"), + ) + }; + if let Some(mdf) = mdf.as_ref() { + match mdf.export_to_parquet(name, comp) { + Ok(_) => {} + Err(e) => panic!("{}", e), + } + } else { + panic!("Null pointer given for Mdf Rust object") + } +} + +// export to hdf5 file +// Compression can be one of the following strings +// "deflate", "lzf" +// or null pointer if no compression wanted +#[no_mangle] +pub unsafe extern "C" fn export_to_hdf5( + mdf: *const Mdf, + file_name: *const c_char, + compression: *const c_char, +) { + // # Safety + // + // It is the caller's guarantee to ensure `file_name`: + // + // - is not a null pointer + // - points to valid, initialized data + // - points to memory ending in a null byte + // - won't be mutated for the duration of this function call + let name = CStr::from_ptr(file_name) + .to_str() + .expect("Could not convert into utf8 the file name string"); + let comp = if compression.is_null() { + None + } else { + Some( + CStr::from_ptr(compression) + .to_str() + .expect("Could not convert into utf8 the compression string"), + ) + }; + if let Some(mdf) = mdf.as_ref() { + match mdf.export_to_hdf5(name, comp) { + Ok(_) => {} + Err(e) => panic!("{}", e), + } + } else { + panic!("Null pointer given for Mdf Rust object") + } +} diff --git a/src/data_holder/channel_data.rs b/src/data_holder/channel_data.rs index ccaf8ea..c36cea1 100644 --- a/src/data_holder/channel_data.rs +++ b/src/data_holder/channel_data.rs @@ -587,6 +587,7 @@ impl ChannelData { } } } + /// returns the arrow DataType equivalent to the ChannelData pub fn arrow_data_type(&self) -> DataType { match self { ChannelData::Int8(_) => DataType::Int8, diff --git a/src/data_holder/complex_arrow.rs b/src/data_holder/complex_arrow.rs index 5df6840..8048aea 100644 --- a/src/data_holder/complex_arrow.rs +++ b/src/data_holder/complex_arrow.rs @@ -1,9 +1,13 @@ //! complex number stored in primitive builders, fixedsizearraybuilder being too restricted +#[cfg(feature = "ndarray")] +use anyhow::{Context, Error, Result}; use arrow::{ array::{ArrayBuilder, BooleanBufferBuilder, PrimitiveArray, PrimitiveBuilder}, buffer::{BooleanBuffer, MutableBuffer}, datatypes::{ArrowPrimitiveType, Float32Type, Float64Type}, }; +#[cfg(feature = "ndarray")] +use ndarray::{Array, Ix2}; /// Complex struct #[derive(Debug)] @@ -94,6 +98,26 @@ impl ComplexArrow { } } +#[cfg(feature = "ndarray")] +impl ComplexArrow { + /// to convert ComplexArrow into ndarray + pub fn to_ndarray(&self) -> Result, Error> { + let vector: Vec = self.values_builder.values_slice().to_vec(); + Array::from_shape_vec((self.len(), 2), vector) + .context("Failed reshaping f32 complex arrow into ndarray") + } +} + +#[cfg(feature = "ndarray")] +impl ComplexArrow { + /// to convert ComplexArrow into ndarray + pub fn to_ndarray(&self) -> Result, Error> { + let vector: Vec = self.values_builder.values_slice().to_vec(); + Array::from_shape_vec((vector.len() / 2, 2), vector) + .context("Failed reshaping f64 complex arrow into ndarray") + } +} + impl Default for ComplexArrow { fn default() -> Self { Self::new() diff --git a/src/data_holder/tensor_arrow.rs b/src/data_holder/tensor_arrow.rs index 6c3f1c9..b8ec42a 100644 --- a/src/data_holder/tensor_arrow.rs +++ b/src/data_holder/tensor_arrow.rs @@ -1,4 +1,6 @@ //! tensor arrow array adapted to mdf4 specificities (samples of tensors) +#[cfg(feature = "ndarray")] +use anyhow::{Context, Error, Result}; use arrow::{ array::{ArrayBuilder, BooleanBufferBuilder, PrimitiveArray, PrimitiveBuilder}, buffer::{BooleanBuffer, MutableBuffer}, @@ -7,6 +9,8 @@ use arrow::{ UInt16Type, UInt32Type, UInt64Type, UInt8Type, }, }; +#[cfg(feature = "ndarray")] +use ndarray::{Array, IxDyn}; /// Tensor with innner arrow primitive builder #[derive(Debug)] @@ -19,7 +23,7 @@ pub struct TensorArrow { len: usize, /// shape of tensor shape: Vec, - /// order of tesnor, row or column major + /// order of tensor, row or column major order: Order, } @@ -205,3 +209,32 @@ tensor_arrow_clone!(Int64Type); tensor_arrow_clone!(UInt64Type); tensor_arrow_clone!(Float32Type); tensor_arrow_clone!(Float64Type); + +#[macro_export] +macro_rules! tensor_arrow_to_ndarray { + ($arrow_type:ty, $rust_type:ty) => { + #[cfg(feature = "ndarray")] + impl TensorArrow<$arrow_type> { + /// to convert TensorArrow into ndarray + pub fn to_ndarray(&self) -> Result, Error> { + let vector: Vec<$rust_type> = + self.values_builder.values_slice().iter().copied().collect(); + let mut shape = self.shape().clone(); + shape.push(self.len()); + Array::from_shape_vec(IxDyn(&shape), vector) + .context("Failed reshaping tensor arrow into ndarray") + } + } + }; +} + +tensor_arrow_to_ndarray!(UInt8Type, u8); +tensor_arrow_to_ndarray!(Int8Type, i8); +tensor_arrow_to_ndarray!(Int16Type, i16); +tensor_arrow_to_ndarray!(UInt16Type, u16); +tensor_arrow_to_ndarray!(Int32Type, i32); +tensor_arrow_to_ndarray!(UInt32Type, u32); +tensor_arrow_to_ndarray!(Int64Type, i64); +tensor_arrow_to_ndarray!(UInt64Type, u64); +tensor_arrow_to_ndarray!(Float32Type, f32); +tensor_arrow_to_ndarray!(Float64Type, f64); diff --git a/src/export/hdf5.rs b/src/export/hdf5.rs new file mode 100644 index 0000000..627e695 --- /dev/null +++ b/src/export/hdf5.rs @@ -0,0 +1,549 @@ +//! Exporting mdf to hdf5 files. +use anyhow::{Context, Error, Result}; +use arrow::array::Array; +use hdf5::{ + file::File, + types::{VarLenArray, VarLenUnicode}, + Dataset, DatasetBuilder, Group, H5Type, +}; +use log::info; +use ndarray::{Array as NdArray, IxDyn}; + +use crate::mdfreader::Mdf; +use crate::{ + data_holder::channel_data::ChannelData, + mdfinfo::{ + mdfinfo3::{Cg3, Cn3, MdfInfo3}, + mdfinfo4::{Cg4, Cn4, Dg4, MdfInfo4}, + MdfInfo, + }, +}; +#[cfg(feature = "hdf5-mpio")] +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; + +/// writes mdf into hdf5 file +pub fn export_to_hdf5(mdf: &Mdf, file_name: &str, compression: Option<&str>) -> Result<(), Error> { + let mut file = File::create(file_name).context("failed creating hdf5 file")?; + let hdf5_compression = hdf5_compression_from_string(compression); + match &mdf.mdf_info { + MdfInfo::V4(mdfinfo4) => { + mdf4_metadata(&mut file, mdfinfo4).context("failed creating metadata for mdf4")?; + mdfinfo4.dg.iter().try_for_each( + |(_dg_block_position, dg): (&i64, &Dg4)| -> Result<(), Error> { + dg.cg.iter().try_for_each( + |(_rec_id, cg): (&u64, &Cg4)| -> Result<(), Error> { + mdf4_cg_to_hdf5(&mut file, mdfinfo4, cg, &hdf5_compression) + .context("failed converting Channel Group 4 to hdf5")?; + Ok(()) + }, + )?; + Ok(()) + }, + )?; + } + MdfInfo::V3(mdfinfo3) => { + mdf3_metadata(&mut file, mdfinfo3).context("failed creating metadata for mdf3")?; + for (_dg_block_position, dg) in mdfinfo3.dg.iter() { + for (_rec_id, cg) in dg.cg.iter() { + mdf3_cg_to_hdf5(&mut file, mdfinfo3, cg, &hdf5_compression) + .context("failed converting Channel Group 3 to hdf5")?; + } + } + } + } + Ok(()) +} + +/// writes a dataframe or channel group defined by a given channel into a hdf5 file +pub fn export_dataframe_to_hdf5( + mdf: &Mdf, + channel_name: &str, + file_name: &str, + compression: Option<&str>, +) -> Result<(), Error> { + let mut file = File::create(file_name).context("failed creating hdf5 file")?; + let hdf5_compression = hdf5_compression_from_string(compression); + match &mdf.mdf_info { + MdfInfo::V4(mdfinfo4) => { + mdf4_metadata(&mut file, mdfinfo4).context("failed creating metadata for mdf4")?; + if let Some((_master, dg_pos, (_cg_pos, rec_id), (_cn_pos, _rec_pos))) = + mdfinfo4.get_channel_id(channel_name) + { + if let Some(dg) = mdfinfo4.dg.get(dg_pos) { + if let Some(cg) = dg.cg.get(rec_id) { + mdf4_cg_to_hdf5(&mut file, mdfinfo4, cg, &hdf5_compression).context( + "failed converting Channel Group 4 to hdf5 containing channel", + )?; + } + } + } + } + MdfInfo::V3(mdfinfo3) => { + mdf3_metadata(&mut file, mdfinfo3).context("failed creating metadata for mdf3")?; + if let Some((_master, dg_pos, (_cg_pos, rec_id), _cn_pos)) = + mdfinfo3.get_channel_id(channel_name) + { + if let Some(dg) = mdfinfo3.dg.get(dg_pos) { + if let Some(cg) = dg.cg.get(rec_id) { + mdf3_cg_to_hdf5(&mut file, mdfinfo3, cg, &hdf5_compression).context( + "failed converting Channel Group 3 to hdf5 containing channel", + )?; + } + } + } + } + } + file.close().context("failed closing hdf5 file") +} + +/// create a hdf5 file for the given CG4 block +#[inline] +pub fn mdf4_cg_to_hdf5( + file: &mut File, + mdfinfo4: &MdfInfo4, + cg: &Cg4, + compression: &Hdf5Compression, +) -> Result<()> { + let master_channel = cg + .master_channel_name + .clone() + .unwrap_or(format!("no_master_channel_{}", cg.block_position)); + let group = file + .create_group(&master_channel) + .with_context(|| format!("failed creating group {:?}", master_channel))?; + #[cfg(feature = "hdf5-mpio")] + cg.cn + .par_iter() + .try_for_each(|(_rec_pos, cn): (&i32, &Cn4)| -> Result<(), Error> { + if !cn.data.is_empty() { + mdf4_cn_to_hdf5(mdfinfo4, cn, compression, &group) + .context("failed writing dataset")?; + } + Ok(()) + }) + .context("failed extracting data")?; + #[cfg(not(feature = "hdf5-mpio"))] + cg.cn + .iter() + .try_for_each(|(_rec_pos, cn): (&i32, &Cn4)| -> Result<(), Error> { + if !cn.data.is_empty() { + mdf4_cn_to_hdf5(mdfinfo4, cn, compression, &group) + .context("failed writing dataset")?; + } + Ok(()) + }) + .context("failed extracting data")?; + Ok(()) +} + +#[inline] +pub fn mdf4_cn_to_hdf5( + mdfinfo4: &MdfInfo4, + cn: &Cn4, + compression: &Hdf5Compression, + group: &Group, +) -> Result<(), Error> { + let builder = match compression { + Hdf5Compression::Deflate(level) => group.new_dataset_builder().shuffle().deflate(*level), + Hdf5Compression::Lzf => group.new_dataset_builder().shuffle().lzf(), + Hdf5Compression::Uncompressed => group.new_dataset_builder(), + }; + let dataset = convert_channel_data_into_ndarray(builder, &cn.data, &cn.unique_name) + .with_context(|| format!("failed writing channel {} dataset", cn.unique_name))?; + // writing channel unit if existing + if let Ok(Some(unit)) = mdfinfo4.sharable.get_tx(cn.block.cn_md_unit) { + if !unit.is_empty() { + create_str_attr(&dataset, "unit", &unit).with_context(|| { + format!( + "failed writing unit attribute for channel {}", + cn.unique_name + ) + })?; + } + } + // writing channel description if existing + if let Ok(Some(desc)) = mdfinfo4.sharable.get_tx(cn.block.cn_md_comment) { + if !desc.is_empty() { + create_str_attr(&dataset, "description", &desc).with_context(|| { + format!( + "failed writing description attribute for channel {}", + cn.unique_name + ) + })?; + } + }; + // sync type + create_scalar_attr(&dataset, "sync_type", &cn.block.cn_sync_type).with_context(|| { + format!( + "failed writing sync type attribute for channel {}", + cn.unique_name + ) + })?; + Ok(()) +} + +/// create a hdf5 file for the given CG3 block +#[inline] +pub fn mdf3_cg_to_hdf5( + file: &mut File, + mdfinfo3: &MdfInfo3, + cg: &Cg3, + compression: &Hdf5Compression, +) -> Result<()> { + let master_channel = cg + .master_channel_name + .clone() + .unwrap_or(format!("no_master_channel_{}", cg.block_position)); + let group = file + .create_group(&master_channel) + .with_context(|| format!("failed creating group {:?}", cg.master_channel_name))?; + #[cfg(feature = "hdf5-mpio")] + cg.cn + .par_iter() + .try_for_each(|(_rec_pos, cn): (&u32, &Cn3)| -> Result<(), Error> { + if !cn.data.is_empty() { + mdf3_cn_to_hdf5(mdfinfo3, cn, compression, &group) + .context("failed writing dataset")?; + } + Ok(()) + }) + .context("failed extracting data")?; + #[cfg(not(feature = "hdf5-mpio"))] + cg.cn + .iter() + .try_for_each(|(_rec_pos, cn): (&u32, &Cn3)| -> Result<(), Error> { + if !cn.data.is_empty() { + mdf3_cn_to_hdf5(mdfinfo3, cn, compression, &group) + .context("failed writing dataset")?; + } + Ok(()) + }) + .context("failed extracting data")?; + Ok(()) +} + +#[inline] +fn mdf3_cn_to_hdf5( + mdfinfo3: &MdfInfo3, + cn: &Cn3, + compression: &Hdf5Compression, + group: &Group, +) -> Result<(), Error> { + let builder = match compression { + Hdf5Compression::Deflate(level) => group.new_dataset_builder().shuffle().deflate(*level), + Hdf5Compression::Lzf => group.new_dataset_builder().shuffle().lzf(), + Hdf5Compression::Uncompressed => group.new_dataset_builder(), + }; + let dataset = convert_channel_data_into_ndarray(builder, &cn.data, &cn.unique_name) + .with_context(|| format!("failed writing channel {} dataset", cn.unique_name))?; + // writing channel unit if existing + if let Some(unit) = mdfinfo3._get_unit(&cn.block1.cn_cc_conversion) { + if !unit.is_empty() { + create_str_attr(&dataset, "unit", &unit).with_context(|| { + format!( + "failed writing unit attribute for channel {}", + cn.unique_name + ) + })?; + } + } + // writing channel description if existing + create_str_attr(&dataset, "description", &cn.description).with_context(|| { + format!( + "failed writing description attribute for channel {}", + cn.unique_name + ) + })?; + // sync type + create_scalar_attr(&dataset, "sync_type", &cn.block1.cn_type).with_context(|| { + format!( + "failed writing sync type attribute for channel {}", + cn.unique_name + ) + })?; + Ok(()) +} + +fn mdf4_metadata(file: &mut File, mdfinfo4: &MdfInfo4) -> Result<()> { + create_scalar_group_attr::( + file, + "start_time_ns", + &mdfinfo4.hd_block.hd_start_time_ns, + ) + .with_context(|| { + format!( + "failed writing attribute start_time_ns with value {}", + mdfinfo4.hd_block.hd_start_time_ns + ) + })?; + let comments = mdfinfo4 + .sharable + .get_hd_comments(mdfinfo4.hd_block.hd_md_comment); + comments + .iter() + .try_for_each(|(name, comment)| -> Result<(), Error> { + create_str_group_attr::(file, name, comment).with_context(|| { + format!("failed writing attribute {} with value {}", name, comment,) + })?; + Ok(()) + }) + .context("failed writing hd comments")?; + Ok(()) +} + +fn mdf3_metadata(file: &mut File, mdfinfo3: &MdfInfo3) -> Result<()> { + let time = mdfinfo3.hd_block.hd_start_time_ns.unwrap_or(0); + create_scalar_group_attr::(file, "start_time_ns", &time) + .with_context(|| format!("failed writing attribute start_time_ns with value {}", time))?; + create_str_group_attr::(file, "Author", &mdfinfo3.hd_block.hd_author).with_context( + || { + format!( + "failed writing attribute author {}", + mdfinfo3.hd_block.hd_author + ) + }, + )?; + create_str_group_attr::(file, "Project", &mdfinfo3.hd_block.hd_project).with_context( + || { + format!( + "failed writing attribute project {}", + mdfinfo3.hd_block.hd_project + ) + }, + )?; + create_str_group_attr::(file, "Subject", &mdfinfo3.hd_block.hd_subject).with_context( + || { + format!( + "failed writing attribute subject {}", + mdfinfo3.hd_block.hd_subject + ) + }, + )?; + create_str_group_attr::(file, "Organization", &mdfinfo3.hd_block.hd_organization) + .with_context(|| { + format!( + "failed writing attribute organization {}", + mdfinfo3.hd_block.hd_organization + ) + })?; + Ok(()) +} + +#[inline] +fn create_str_attr(location: &T, name: &str, value: &str) -> Result<()> +where + T: std::ops::Deref, +{ + let attr = location + .new_attr::() + .create(name) + .with_context(|| format!("failed creating attribute {}", name))?; + let value: VarLenUnicode = value.parse().unwrap_or("None".parse().unwrap()); + attr.write_scalar(&value) + .with_context(|| format!("failed writing attribute {} with value {}", name, value)) +} + +#[inline] +fn create_str_group_attr(location: &T, name: &str, value: &str) -> Result<()> +where + T: std::ops::Deref, +{ + let attr = location + .new_attr::() + .create(name) + .with_context(|| format!("failed creating attribute {}", name))?; + let value: VarLenUnicode = value.parse().unwrap_or("None".parse().unwrap()); + attr.write_scalar(&value) + .with_context(|| format!("failed writing attribute {} with value {}", name, value)) +} + +#[inline] +fn create_scalar_attr(location: &T, name: &str, value: &N) -> Result<()> +where + T: std::ops::Deref, + N: H5Type + std::fmt::Debug, +{ + let attr = location + .new_attr::() + .create(name) + .with_context(|| format!("failed creating attribute {}", name))?; + attr.write_scalar(value) + .with_context(|| format!("failed writing attribute {} with value {:?}", name, value)) +} + +#[inline] +fn create_scalar_group_attr(location: &T, name: &str, value: &N) -> Result<()> +where + T: std::ops::Deref, + N: H5Type + std::fmt::Debug, +{ + let attr = location + .new_attr::() + .create(name) + .with_context(|| format!("failed creating attribute {}", name))?; + attr.write_scalar(value) + .with_context(|| format!("failed writing attribute {} with value {:?}", name, value)) +} + +#[inline] +fn convert_channel_data_into_ndarray( + builder: DatasetBuilder, + data: &ChannelData, + name: &str, +) -> Result { + match data { + ChannelData::Int8(data) => Ok(builder.with_data(data.values_slice()).create(name)?), + ChannelData::UInt8(data) => Ok(builder.with_data(data.values_slice()).create(name)?), + ChannelData::Int16(data) => Ok(builder.with_data(data.values_slice()).create(name)?), + ChannelData::UInt16(data) => Ok(builder.with_data(data.values_slice()).create(name)?), + ChannelData::Int32(data) => Ok(builder.with_data(data.values_slice()).create(name)?), + ChannelData::UInt32(data) => Ok(builder.with_data(data.values_slice()).create(name)?), + ChannelData::Float32(data) => Ok(builder.with_data(data.values_slice()).create(name)?), + ChannelData::Int64(data) => Ok(builder.with_data(data.values_slice()).create(name)?), + ChannelData::UInt64(data) => Ok(builder.with_data(data.values_slice()).create(name)?), + ChannelData::Float64(data) => Ok(builder.with_data(data.values_slice()).create(name)?), + ChannelData::Complex32(data) => Ok(builder + .with_data( + &data + .to_ndarray() + .context("Failed converting channelData f32 complex into ndarray")?, + ) + .create(name)?), + ChannelData::Complex64(data) => Ok(builder + .with_data( + &data + .to_ndarray() + .context("Failed converting channelData f64 complex into ndarray")?, + ) + .create(name)?), + ChannelData::Utf8(data) => { + let string_vect: Vec = data + .finish_cloned() + .iter() + .map(|x| match x { + Some(x) => match x.parse() { + Ok(s) => s, + Err(e) => { + info!("failed parsing value {:?}, error {}", x, e); + "null".parse().unwrap() + } + }, + None => "null".parse().unwrap(), + }) + .collect(); + Ok(builder.with_data(&string_vect).create(name)?) + } + ChannelData::VariableSizeByteArray(data) => { + let bytes_vect: Vec> = data + .finish_cloned() + .iter() + .map(|x| match x { + Some(x) => VarLenArray::from_slice(x), + None => VarLenArray::from_slice(&[0]), + }) + .collect(); + Ok(builder.with_data(&bytes_vect).create(name)?) + } + ChannelData::FixedSizeByteArray(data) => { + let fixed_binary = data.finish_cloned(); + let value_length = fixed_binary.value_length(); + let vector = fixed_binary.value_data().to_vec(); + let shape = vec![fixed_binary.len(), value_length as usize]; + Ok(builder + .with_data( + &NdArray::from_shape_vec(IxDyn(&shape), vector) + .context("Failed reshaping byteArray arrow into ndarray")?, + ) + .create(name)?) + } + ChannelData::ArrayDInt8(data) => Ok(builder + .with_data( + &data + .to_ndarray() + .context("Failed converting channelData nd i8 into ndarray")?, + ) + .create(name)?), + ChannelData::ArrayDUInt8(data) => Ok(builder + .with_data( + &data + .to_ndarray() + .context("Failed converting channelData nd u8 into ndarray")?, + ) + .create(name)?), + ChannelData::ArrayDInt16(data) => Ok(builder + .with_data( + &data + .to_ndarray() + .context("Failed converting channelData nd i16 into ndarray")?, + ) + .create(name)?), + ChannelData::ArrayDUInt16(data) => Ok(builder + .with_data( + &data + .to_ndarray() + .context("Failed converting channelData nd u16 into ndarray")?, + ) + .create(name)?), + ChannelData::ArrayDInt32(data) => Ok(builder + .with_data( + &data + .to_ndarray() + .context("Failed converting channelData nd i32 into ndarray")?, + ) + .create(name)?), + ChannelData::ArrayDUInt32(data) => Ok(builder + .with_data( + &data + .to_ndarray() + .context("Failed converting channelData nd u32 into ndarray")?, + ) + .create(name)?), + ChannelData::ArrayDFloat32(data) => Ok(builder + .with_data( + &data + .to_ndarray() + .context("Failed converting channelData nd f32 into ndarray")?, + ) + .create(name)?), + ChannelData::ArrayDInt64(data) => Ok(builder + .with_data( + &data + .to_ndarray() + .context("Failed converting channelData nd i64 into ndarray")?, + ) + .create(name)?), + ChannelData::ArrayDUInt64(data) => Ok(builder + .with_data( + &data + .to_ndarray() + .context("Failed converting channelData nd u64 into ndarray")?, + ) + .create(name)?), + ChannelData::ArrayDFloat64(data) => Ok(builder + .with_data( + &data + .to_ndarray() + .context("Failed converting channelData nd f64 into ndarray")?, + ) + .create(name)?), + } +} + +/// converts a clap compression string into a Hdf5Compression enum +#[inline] +pub fn hdf5_compression_from_string(compression_option: Option<&str>) -> Hdf5Compression { + match compression_option { + Some(option) => match option { + "deflate" => Hdf5Compression::Deflate(8), + "lzf" => Hdf5Compression::Lzf, + _ => Hdf5Compression::Uncompressed, + }, + None => Hdf5Compression::Uncompressed, + } +} + +pub enum Hdf5Compression { + Deflate(u8), + Lzf, + Uncompressed, +} diff --git a/src/export/mod.rs b/src/export/mod.rs index 02588a1..82a6f76 100644 --- a/src/export/mod.rs +++ b/src/export/mod.rs @@ -5,3 +5,6 @@ pub mod numpy; pub mod parquet; #[cfg(feature = "polars")] pub mod polars; + +#[cfg(feature = "hdf5")] +pub mod hdf5; diff --git a/src/export/parquet.rs b/src/export/parquet.rs index 7bafb47..23bc964 100644 --- a/src/export/parquet.rs +++ b/src/export/parquet.rs @@ -80,7 +80,7 @@ pub fn export_to_parquet( Ok(()) } -/// writes mdf into parquet file +/// writes a dataframe or channel group defined by a given channel into a parquet file pub fn export_dataframe_to_parquet( mdf: &Mdf, channel_name: &str, @@ -230,8 +230,6 @@ pub fn parquet_compression_from_string(compression_option: Option<&str>) -> Comp } } -// create - /// Create parquet file name appending Channel Group's master channel /// Or if no master existing, add. /// Appending at the end of name the . parquet file extension diff --git a/src/main.rs b/src/main.rs index 02cc4ec..b980e58 100644 --- a/src/main.rs +++ b/src/main.rs @@ -65,6 +65,23 @@ fn main() -> Result<(), Error> { .value_name("ALGORITHM") .help("Compression algorithm for writing data in parquet file, valid values are snappy, gzip, lzo, lz4, zstd, brotli. Default is uncompressed"), ) + .arg( + Arg::new("export_to_hdf5") + .long("export_to_hdf5") + .short('h') + .required(false) + .num_args(1) + .value_name("FILE_NAME") + .help("Converts mdf into hdf5 file, file name to be given without extension"), + ) + .arg( + Arg::new("hdf5_compression") + .long("hdf5_compression") + .required(false) + .num_args(1) + .value_name("FILTER") + .help("Compression algorithm for writing data in hdf5 file, valid values are deflate and lzf. Default is uncompressed"), + ) .arg( Arg::new("info") .short('i') @@ -87,8 +104,9 @@ fn main() -> Result<(), Error> { let mdf4_file_name = matches.get_one::("write"); let parquet_file_name = matches.get_one::("export_to_parquet"); + let hdf5_file_name = matches.get_one::("export_to_hdf5"); - if mdf4_file_name.is_some() || parquet_file_name.is_some() { + if mdf4_file_name.is_some() || parquet_file_name.is_some() || hdf5_file_name.is_some() { mdf_file .load_all_channels_data_in_memory() .with_context(|| format!("failed reading channels data from file {}", file_name))?; @@ -105,6 +123,7 @@ fn main() -> Result<(), Error> { } } + #[cfg(feature = "parquet")] let parquet_compression = matches.get_one::("parquet_compression"); #[cfg(feature = "parquet")] if let Some(file_name) = parquet_file_name { @@ -117,5 +136,15 @@ fn main() -> Result<(), Error> { ); } + #[cfg(feature = "hdf5")] + let hdf5_compression = matches.get_one::("hdf5_compression"); + #[cfg(feature = "hdf5")] + if let Some(file_name) = hdf5_file_name { + mdf_file + .export_to_hdf5(file_name, hdf5_compression.map(|x| &**x)) + .with_context(|| format!("failed to export into hdf5 file {}", file_name))?; + info!("Wrote hdf5 file {}", file_name); + } + Ok(()) } diff --git a/src/mdfinfo/mdfinfo3.rs b/src/mdfinfo/mdfinfo3.rs index 317b6a9..9f965fc 100644 --- a/src/mdfinfo/mdfinfo3.rs +++ b/src/mdfinfo/mdfinfo3.rs @@ -390,7 +390,7 @@ pub struct Hd3 { /// subject or measurement object pub hd_subject: String, /// time stamp at which recording was started in nanosecond - hd_start_time_ns: Option, + pub hd_start_time_ns: Option, /// time stamp at which recording was started in nanosecond hd_time_offset: Option, /// time quality class @@ -399,7 +399,7 @@ pub struct Hd3 { hd_time_identifier: Option, } -/// HD3 block strucutre +/// HD3 block structure #[derive(Debug, PartialEq, Eq, Default, BinRead)] #[repr(C)] pub struct Hd3Block { @@ -796,7 +796,7 @@ fn parse_cg3_block( pub struct Cg3 { pub block: Cg3Block, pub cn: HashMap, // hashmap of channels - block_position: u32, + pub block_position: u32, pub master_channel_name: Option, pub channel_names: HashSet, pub record_length: u16, // record length including recordId diff --git a/src/mdfr.rs b/src/mdfr.rs index d9c8a3f..e4dc36b 100644 --- a/src/mdfr.rs +++ b/src/mdfr.rs @@ -357,6 +357,25 @@ df=polars.DataFrame(series) mdf.export_dataframe_to_parquet(channel_name, file_name, compression)?; Ok(()) } + /// export to hdf5 files + #[cfg(feature = "hdf5")] + pub fn export_to_hdf5(&self, file_name: &str, compression: Option<&str>) -> PyResult<()> { + let Mdfr(mdf) = self; + mdf.export_to_hdf5(file_name, compression)?; + Ok(()) + } + /// export dataframe to Parquet files + #[cfg(feature = "parquet")] + pub fn export_dataframe_to_hdf5( + &self, + channel_name: String, + file_name: &str, + compression: Option<&str>, + ) -> PyResult<()> { + let Mdfr(mdf) = self; + mdf.export_dataframe_to_hdf5(channel_name, file_name, compression)?; + Ok(()) + } /// get attachment blocks pub fn get_attachment_blocks(&mut self) -> Py { let Mdfr(mdf) = self; diff --git a/src/mdfreader.rs b/src/mdfreader.rs index 3e5bec8..9b550cb 100644 --- a/src/mdfreader.rs +++ b/src/mdfreader.rs @@ -30,6 +30,11 @@ use crate::export::parquet::export_dataframe_to_parquet; #[cfg(feature = "parquet")] use crate::export::parquet::export_to_parquet; +#[cfg(feature = "hdf5")] +use crate::export::hdf5::export_dataframe_to_hdf5; +#[cfg(feature = "hdf5")] +use crate::export::hdf5::export_to_hdf5; + use crate::data_holder::arrow_helpers::{ arrow_bit_count, arrow_byte_count, arrow_to_mdf_data_type, }; @@ -241,12 +246,12 @@ impl Mdf { Ok(()) } - /// export to Parquet files + /// export to Parquet files, one for each channel group (or dataframe) #[cfg(feature = "parquet")] pub fn export_to_parquet(&self, file_name: &str, compression: Option<&str>) -> Result<()> { export_to_parquet(self, file_name, compression) } - /// export to Parquet files + /// export a dataframe including a given channel to a Parquet file #[cfg(feature = "parquet")] pub fn export_dataframe_to_parquet( &self, @@ -256,6 +261,21 @@ impl Mdf { ) -> Result<()> { export_dataframe_to_parquet(self, &channel_name, file_name, compression) } + /// export a dataframe including a given channel to a hdf5 file + #[cfg(feature = "hdf5")] + pub fn export_dataframe_to_hdf5( + &self, + channel_name: String, + file_name: &str, + compression: Option<&str>, + ) -> Result<()> { + export_dataframe_to_hdf5(self, &channel_name, file_name, compression) + } + /// export all data to hdf5 file + #[cfg(feature = "hdf5")] + pub fn export_to_hdf5(&self, file_name: &str, compression: Option<&str>) -> Result<()> { + export_to_hdf5(self, file_name, compression) + } /// Writes mdf4 file pub fn write(&mut self, file_name: &str, compression: bool) -> Result { mdfwriter4(self, file_name, compression) diff --git a/src/tests.rs b/src/tests.rs index 86585a9..5d2c0ff 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -22,6 +22,7 @@ mod tests { static BASE_TEST_PATH: &str = "/home/ratal/workspace/mdfr/test_files"; static WRITING_MDF_FILE: &str = "/home/ratal/workspace/mdfr/test_files/test.mf4"; static WRITING_PARQUET_FILE: &str = "/home/ratal/workspace/mdfr/test_files/test_parquet"; + static WRITING_HDF5_FILE: &str = "/home/ratal/workspace/mdfr/test_files/test_hdf5.hdf5"; #[test] fn info_test() -> Result<()> { @@ -1195,4 +1196,32 @@ mod tests { } Ok(()) } + #[test] + fn export_to_hdf5() -> Result<()> { + // Export mdf4 to Parquet file + let file = format!( + "{}{}", + BASE_PATH_MDF4, &"Simple/PCV_iO_Gen3_LK1__3l_TDI.mf4" + ); + let extension = "*.hdf5"; + let mut mdf = Mdf::new(&file)?; + mdf.load_all_channels_data_in_memory()?; + mdf.export_to_hdf5(&WRITING_HDF5_FILE, Some(&"lzf")) + .expect("failed writing mdf4 hdf5 file"); + // Export mdf3 to Parquet file + let file = format!( + "{}{}", + BASE_PATH_MDF3, &"RJ_N16-12-363_BM-15C-0024_228_2_20170116094355_CAN.dat" + ); + let mut mdf = Mdf::new(&file)?; + mdf.load_all_channels_data_in_memory()?; + mdf.export_to_hdf5(&WRITING_HDF5_FILE, Some(&"deflate")) + .expect("failed writing mdf3 hdf5 file"); + // remove all generated hdf5 files + let pattern = format!("{}/{}", BASE_TEST_PATH, extension); + for path in glob(&pattern).unwrap().filter_map(Result::ok) { + fs::remove_file(path)?; + } + Ok(()) + } }