From dbfbd57a98fcf6834d86caaa403081d2f01b2124 Mon Sep 17 00:00:00 2001 From: nazeh Date: Tue, 28 May 2024 10:09:07 +0300 Subject: [PATCH 01/84] feat(pkarr): replace z32 with base32 --- Cargo.lock | 14 +++++++------- pkarr/Cargo.toml | 2 +- pkarr/src/error.rs | 4 ++-- pkarr/src/keys.rs | 16 ++++++++++++---- pkarr/src/signed_packet.rs | 2 +- 5 files changed, 23 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0c8ed1a..b57595d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -207,6 +207,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base32" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1ce0365f4d5fb6646220bb52fe547afd51796d90f914d4063cb0b032ebee088" + [[package]] name = "base64" version = "0.22.1" @@ -1302,6 +1308,7 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" name = "pkarr" version = "2.0.1" dependencies = [ + "base32", "bytes", "clap", "document-features", @@ -1324,7 +1331,6 @@ dependencies = [ "wasm-bindgen-futures", "wasm-bindgen-test", "web-sys", - "z32", ] [[package]] @@ -2532,12 +2538,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "z32" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edb37266251c28b03d08162174a91c3a092e3bd4f476f8205ee1c507b78b7bdc" - [[package]] name = "zeroize" version = "1.7.0" diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index 8a3d753..365cf2c 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -16,10 +16,10 @@ self_cell = "1.0.2" simple-dns = "0.6.1" thiserror = "1.0.49" tracing = "0.1.40" -z32 = "1.1.1" rand = { version = "0.8.5", optional = true } lru = { version = "0.12.3", default-features = false } flume = { version = "0.11.0", features = ["select", "eventual-fairness"], default-features = false } +base32 = "0.5.0" [target.'cfg(not(target_arch = "wasm32"))'.dependencies] # Dht client dependencies: diff --git a/pkarr/src/error.rs b/pkarr/src/error.rs index eb329d4..7c8e906 100644 --- a/pkarr/src/error.rs +++ b/pkarr/src/error.rs @@ -25,8 +25,8 @@ pub enum Error { #[error("Invalid Ed25519 signature")] InvalidEd25519Signature, - #[error(transparent)] - InvalidPublicKeyEncoding(#[from] z32::Z32Error), + #[error("Invalid PublicKey encoding")] + InvalidPublicKeyEncoding, // === Packets errors === #[error(transparent)] diff --git a/pkarr/src/keys.rs b/pkarr/src/keys.rs index deca50a..a67a8b3 100644 --- a/pkarr/src/keys.rs +++ b/pkarr/src/keys.rs @@ -53,12 +53,12 @@ impl Keypair { /// Ed25519 public key to verify a signature over dns [Packet](crate::SignedPacket)s. /// -/// It can formatted to and parsed from a [zbase32](z32) string. +/// It can formatted to and parsed from a z-base32 string. #[derive(Clone, Eq, PartialEq, Hash)] pub struct PublicKey(VerifyingKey); impl PublicKey { - /// Format the public key as [zbase32](z32) string. + /// Format the public key as z-base32 string. pub fn to_z32(&self) -> String { self.to_string() } @@ -179,7 +179,11 @@ impl TryFrom<&str> for PublicKey { } } - let bytes = z32::decode(s.as_bytes())?; + let bytes = if let Some(v) = base32::decode(base32::Alphabet::Z, s) { + Ok(v) + } else { + Err(Error::InvalidPublicKeyEncoding) + }?; let verifying_key = VerifyingKey::try_from(bytes.as_slice()) .map_err(|_| Error::InvalidPublicKeyLength(bytes.len()))?; @@ -198,7 +202,11 @@ impl TryFrom for PublicKey { impl Display for PublicKey { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!(f, "{}", z32::encode(self.0.as_bytes())) + write!( + f, + "{}", + base32::encode(base32::Alphabet::Z, self.0.as_bytes()) + ) } } diff --git a/pkarr/src/signed_packet.rs b/pkarr/src/signed_packet.rs index 7c9e3ef..19c67f4 100644 --- a/pkarr/src/signed_packet.rs +++ b/pkarr/src/signed_packet.rs @@ -131,7 +131,7 @@ impl SignedPacket { /// Creates a new [SignedPacket] from a [Keypair] and a DNS [Packet]. /// /// It will also normalize the names of the [ResourceRecord]s to be relative to the origin, - /// which would be the [zbase32](z32) encoded [PublicKey] of the [Keypair] used to sign the Packet. + /// which would be the z-base32 encoded [PublicKey] of the [Keypair] used to sign the Packet. /// /// # Errors /// - Returns [crate::Error::DnsError] if the packet is invalid or it failed to compress or encode it. From 0b9e7cbe0ddc968b1043b7f93127c13bbfeaf83d Mon Sep 17 00:00:00 2001 From: nazeh Date: Tue, 27 Aug 2024 10:54:21 +0300 Subject: [PATCH 02/84] feat(pkarr): add serde to serialize public keys --- Cargo.lock | 85 ++++++++++++++++++++++++++++++++++++++++++++--- pkarr/Cargo.toml | 7 +++- pkarr/src/keys.rs | 42 +++++++++++++++++++++++ 3 files changed, 129 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d5546d2..190317a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -108,6 +108,15 @@ dependencies = [ "syn", ] +[[package]] +name = "atomic-polyfill" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" +dependencies = [ + "critical-section", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -317,6 +326,12 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +[[package]] +name = "cobs" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" + [[package]] name = "colorchoice" version = "1.0.2" @@ -373,6 +388,12 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "critical-section" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f64009896348fc5af4222e9cf7d7d82a95a256c634ebcf61c53e4ea461422242" + [[package]] name = "crossbeam-queue" version = "0.3.11" @@ -528,6 +549,18 @@ dependencies = [ "zeroize", ] +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + [[package]] name = "equivalent" version = "1.0.1" @@ -740,12 +773,35 @@ dependencies = [ "tracing", ] +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + [[package]] name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +[[package]] +name = "heapless" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" +dependencies = [ + "atomic-polyfill", + "hash32", + "rustc_version", + "serde", + "spin", + "stable_deref_trait", +] + [[package]] name = "heck" version = "0.5.0" @@ -1263,8 +1319,10 @@ dependencies = [ "lru", "mainline", "mockito", + "postcard", "rand", "self_cell", + "serde", "simple-dns", "thiserror", "tracing", @@ -1319,6 +1377,19 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265" +[[package]] +name = "postcard" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f7f0a8d620d71c457dd1d47df76bb18960378da56af4527aaa10f515eee732e" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "heapless", + "serde", +] + [[package]] name = "ppv-lite86" version = "0.2.20" @@ -1583,9 +1654,9 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" -version = "1.0.207" +version = "1.0.209" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5665e14a49a4ea1b91029ba7d3bca9f299e1f7cfa194388ccc20f14743e784f2" +checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" dependencies = [ "serde_derive", ] @@ -1611,9 +1682,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.207" +version = "1.0.209" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6aea2634c86b0e8ef2cfdc0c340baede54ec27b1e46febd7f80dffb2aa44a00e" +checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" dependencies = [ "proc-macro2", "quote", @@ -1787,6 +1858,12 @@ dependencies = [ "der", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "strsim" version = "0.11.1" diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index 53c020f..9caf0aa 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -20,6 +20,7 @@ z32 = "1.1.1" rand = { version = "0.8.5", optional = true } lru = { version = "0.12.3", default-features = false } flume = { version = "0.11.0", features = ["select", "eventual-fairness"], default-features = false } +serde = { version = "1.0.209", features = ["derive"], optional = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] # Dht client dependencies: @@ -45,6 +46,7 @@ web-sys = { version = "0.3.69", features = [ [dev-dependencies] futures = "0.3.29" +postcard = { version = "1.0.10", features = ["alloc"] } [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] clap = { version = "4.4.8", features = ["derive"] } @@ -63,8 +65,11 @@ rand = ["dep:rand", "ed25519-dalek/rand_core"] async = ["flume/async"] ## Use [PkarrRelayClient] relay = ["dep:ureq"] +## Derive serde Serialize/Deserialize for PublicKey +serde = ["dep:serde"] + ## Use all features -full = ["dht", "async", "relay", "rand"] +full = ["dht", "async", "relay", "rand", "serde"] default = ["dht", "rand"] diff --git a/pkarr/src/keys.rs b/pkarr/src/keys.rs index 40a9627..1664db4 100644 --- a/pkarr/src/keys.rs +++ b/pkarr/src/keys.rs @@ -9,6 +9,9 @@ use std::{ hash::Hash, }; +#[cfg(feature = "serde")] +use serde::{Deserialize, Serialize}; + #[derive(Clone)] /// Ed25519 keypair to sign dns [Packet](crate::SignedPacket)s. pub struct Keypair(SigningKey); @@ -223,6 +226,29 @@ impl Debug for PublicKey { } } +#[cfg(feature = "serde")] +impl Serialize for PublicKey { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let bytes = self.to_bytes(); + bytes.serialize(serializer) + } +} + +#[cfg(feature = "serde")] +impl<'de> Deserialize<'de> for PublicKey { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let bytes: [u8; 32] = Deserialize::deserialize(deserializer)?; + + (&bytes).try_into().map_err(serde::de::Error::custom) + } +} + #[cfg(test)] mod tests { use super::*; @@ -391,4 +417,20 @@ mod tests { let public_key: PublicKey = str.try_into().unwrap(); assert_eq!(public_key.verifying_key().as_bytes(), &expected); } + + #[cfg(feature = "serde")] + #[test] + fn serde() { + let str = "yg4gxe7z1r7mr6orids9fh95y7gxhdsxjqi6nngsxxtakqaxr5no"; + let expected = [ + 1, 180, 103, 163, 183, 145, 58, 178, 122, 4, 168, 237, 242, 243, 251, 7, 76, 254, 14, + 207, 75, 171, 225, 8, 214, 123, 227, 133, 59, 15, 38, 197, + ]; + + let public_key: PublicKey = str.try_into().unwrap(); + + let bytes = postcard::to_allocvec(&public_key).unwrap(); + + assert_eq!(bytes, expected) + } } From a40ce2f639811a740326b0a1c4548c8ac67c8865 Mon Sep 17 00:00:00 2001 From: nazeh Date: Wed, 11 Sep 2024 12:07:22 +0300 Subject: [PATCH 03/84] test(pkarr): PublicKey::try_from(&str) convert TLD not any subdomain --- Cargo.lock | 21 +++++++++++++++++++++ pkarr/src/keys.rs | 16 ++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 4467d06..73bbba0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2562,6 +2562,27 @@ dependencies = [ "memchr", ] +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zeroize" version = "1.8.1" diff --git a/pkarr/src/keys.rs b/pkarr/src/keys.rs index e33e371..b2fd32f 100644 --- a/pkarr/src/keys.rs +++ b/pkarr/src/keys.rs @@ -138,6 +138,7 @@ impl TryFrom<&str> for PublicKey { /// - `https://foo.o4dksfbqk85ogzdb5osziw6befigbuxmuxkuxq8434q89uj56uyy.#hash` /// - `https://foo@bar.o4dksfbqk85ogzdb5osziw6befigbuxmuxkuxq8434q89uj56uyy.?q=v` /// - `https://foo@bar.o4dksfbqk85ogzdb5osziw6befigbuxmuxkuxq8434q89uj56uyy.:8888?q=v` + /// - `https://yg4gxe7z1r7mr6orids9fh95y7gxhdsxjqi6nngsxxtakqaxr5no.o4dksfbqk85ogzdb5osziw6befigbuxmuxkuxq8434q89uj56uyy` fn try_from(s: &str) -> Result { let mut s = s; @@ -441,4 +442,19 @@ mod tests { assert_eq!(bytes, expected) } + + #[test] + fn from_uri_multiple_pkarr() { + // Should only catch the TLD. + + let str = + "https://o4dksfbqk85ogzdb5osziw6befigbuxmuxkuxq8434q89uj56uyy.yg4gxe7z1r7mr6orids9fh95y7gxhdsxjqi6nngsxxtakqaxr5no"; + let expected = [ + 1, 180, 103, 163, 183, 145, 58, 178, 122, 4, 168, 237, 242, 243, 251, 7, 76, 254, 14, + 207, 75, 171, 225, 8, 214, 123, 227, 133, 59, 15, 38, 197, + ]; + + let public_key: PublicKey = str.try_into().unwrap(); + assert_eq!(public_key.verifying_key().as_bytes(), &expected); + } } From 8d68ad7091d1f4d7d3903e245a0227437c2dc7da Mon Sep 17 00:00:00 2001 From: nazeh Date: Fri, 13 Sep 2024 12:29:25 +0300 Subject: [PATCH 04/84] feat(pkarr): add PkarrClientBuilder::testnet method to simplify testing --- pkarr/src/client.rs | 58 ++++++++------------------------------- pkarr/src/client_async.rs | 32 +++------------------ 2 files changed, 16 insertions(+), 74 deletions(-) diff --git a/pkarr/src/client.rs b/pkarr/src/client.rs index c61d5e3..82768fc 100644 --- a/pkarr/src/client.rs +++ b/pkarr/src/client.rs @@ -7,7 +7,7 @@ use mainline::{ messages, QueryResponse, QueryResponseSpecific, ReceivedFrom, ReceivedMessage, Response, Rpc, }, - Id, MutableItem, + Id, MutableItem, Testnet, }; use std::{ collections::HashMap, @@ -124,6 +124,12 @@ impl PkarrClientBuilder { self } + /// Convienent methot to set the [DhtSettings::bootstrap] from [mainline::Testnet::bootstrap] + pub fn testnet(mut self, testnet: &Testnet) -> Self { + self.settings.dht.bootstrap = testnet.bootstrap.clone().into(); + self + } + pub fn build(self) -> Result { PkarrClient::new(self.settings) } @@ -468,15 +474,7 @@ mod tests { fn shutdown() { let testnet = Testnet::new(3); - let mut a = PkarrClient::builder() - .dht_settings(DhtSettings { - bootstrap: Some(testnet.bootstrap), - request_timeout: None, - server: None, - port: None, - }) - .build() - .unwrap(); + let mut a = PkarrClient::builder().testnet(&testnet).build().unwrap(); assert_ne!(a.local_addr(), None); @@ -489,15 +487,7 @@ mod tests { fn publish_resolve() { let testnet = Testnet::new(10); - let a = PkarrClient::builder() - .dht_settings(DhtSettings { - bootstrap: Some(testnet.bootstrap.clone()), - request_timeout: None, - server: None, - port: None, - }) - .build() - .unwrap(); + let a = PkarrClient::builder().testnet(&testnet).build().unwrap(); let keypair = Keypair::random(); @@ -513,15 +503,7 @@ mod tests { let _ = a.publish(&signed_packet); - let b = PkarrClient::builder() - .dht_settings(DhtSettings { - bootstrap: Some(testnet.bootstrap), - request_timeout: None, - server: None, - port: None, - }) - .build() - .unwrap(); + let b = PkarrClient::builder().testnet(&testnet).build().unwrap(); let resolved = b.resolve(&keypair.public_key()).unwrap().unwrap(); assert_eq!(resolved.as_bytes(), signed_packet.as_bytes()); @@ -535,15 +517,7 @@ mod tests { fn thread_safe() { let testnet = Testnet::new(10); - let a = PkarrClient::builder() - .dht_settings(DhtSettings { - bootstrap: Some(testnet.bootstrap.clone()), - request_timeout: None, - server: None, - port: None, - }) - .build() - .unwrap(); + let a = PkarrClient::builder().testnet(&testnet).build().unwrap(); let keypair = Keypair::random(); @@ -559,15 +533,7 @@ mod tests { let _ = a.publish(&signed_packet); - let b = PkarrClient::builder() - .dht_settings(DhtSettings { - bootstrap: Some(testnet.bootstrap), - request_timeout: None, - server: None, - port: None, - }) - .build() - .unwrap(); + let b = PkarrClient::builder().testnet(&testnet).build().unwrap(); thread::spawn(move || { let resolved = b.resolve(&keypair.public_key()).unwrap().unwrap(); diff --git a/pkarr/src/client_async.rs b/pkarr/src/client_async.rs index 0c49b1a..6b51c56 100644 --- a/pkarr/src/client_async.rs +++ b/pkarr/src/client_async.rs @@ -88,7 +88,7 @@ impl PkarrClientAsync { #[cfg(test)] mod tests { - use mainline::{dht::DhtSettings, Testnet}; + use mainline::Testnet; use super::*; use crate::{dns, Keypair, SignedPacket}; @@ -98,15 +98,7 @@ mod tests { async fn test() { let testnet = Testnet::new(3); - let mut a = PkarrClient::builder() - .dht_settings(DhtSettings { - bootstrap: Some(testnet.bootstrap), - request_timeout: None, - server: None, - port: None, - }) - .build() - .unwrap(); + let mut a = PkarrClient::builder().testnet(&testnet).build().unwrap(); assert_ne!(a.local_addr(), None); @@ -123,15 +115,7 @@ mod tests { async fn test() { let testnet = Testnet::new(10); - let a = PkarrClient::builder() - .dht_settings(DhtSettings { - bootstrap: Some(testnet.bootstrap.clone()), - request_timeout: None, - server: None, - port: None, - }) - .build() - .unwrap(); + let a = PkarrClient::builder().testnet(&testnet).build().unwrap(); let keypair = Keypair::random(); @@ -147,15 +131,7 @@ mod tests { let _ = a.publish(&signed_packet); - let b = PkarrClient::builder() - .dht_settings(DhtSettings { - bootstrap: Some(testnet.bootstrap), - request_timeout: None, - server: None, - port: None, - }) - .build() - .unwrap(); + let b = PkarrClient::builder().testnet(&testnet).build().unwrap(); let resolved = b.resolve(&keypair.public_key()).unwrap().unwrap(); assert_eq!(resolved.as_bytes(), signed_packet.as_bytes()); From 2d1abb4d28624c298a38996ee3f40914676cd5d1 Mon Sep 17 00:00:00 2001 From: nazeh Date: Fri, 13 Sep 2024 14:17:58 +0300 Subject: [PATCH 05/84] feat(pkarr): add resolve_endpoint and test basic direct resolution --- pkarr/src/error.rs | 3 + pkarr/src/lib.rs | 1 + pkarr/src/svcb.rs | 172 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 176 insertions(+) create mode 100644 pkarr/src/svcb.rs diff --git a/pkarr/src/error.rs b/pkarr/src/error.rs index 3b509ff..845af14 100644 --- a/pkarr/src/error.rs +++ b/pkarr/src/error.rs @@ -64,6 +64,9 @@ pub enum Error { /// Failed to publish because there is a more recent packet. NotMostRecent, + #[error("Could not resolve Endpoint for domain: {0}")] + ResolveEndpoint(String), + // === Relay errors === #[cfg(all(not(target_arch = "wasm32"), feature = "relay"))] #[error(transparent)] diff --git a/pkarr/src/lib.rs b/pkarr/src/lib.rs index 7fffe55..9e838e2 100644 --- a/pkarr/src/lib.rs +++ b/pkarr/src/lib.rs @@ -53,6 +53,7 @@ macro_rules! if_relay { if_dht! { mod cache; mod client; + mod svcb; if_async! { mod client_async; diff --git a/pkarr/src/svcb.rs b/pkarr/src/svcb.rs new file mode 100644 index 0000000..b4e39e6 --- /dev/null +++ b/pkarr/src/svcb.rs @@ -0,0 +1,172 @@ +use simple_dns::rdata::{RData, SVCB}; + +use crate::{ + error::{Error, Result}, + PkarrClient, PublicKey, SignedPacket, +}; + +const MAX_ENDPOINT_RESOLUTION_RECURSION: u8 = 3; + +impl PkarrClient { + pub fn resolve_endpoint(&self, target: &str) -> Result { + // TODO: cache the result of this function? + + let mut step = 0; + let mut svcb: Option = None; + + loop { + let current = svcb.clone().map_or(target.to_string(), |s| s.target); + if let Ok(tld) = PublicKey::try_from(current.clone()) { + if let Ok(Some(signed_packet)) = self.resolve(&tld) { + if step >= MAX_ENDPOINT_RESOLUTION_RECURSION { + break; + }; + step += 1; + + // Choose most prior SVCB record + svcb = getx(&signed_packet, ¤t); + + // Try wildcards + if svcb.is_none() { + let parts: Vec<&str> = current.split('.').collect(); + + for i in 1..parts.len() { + let xx = format!("*.{}", parts[i..].join(".")); + + svcb = getx(&signed_packet, &xx); + + if svcb.is_some() { + break; + } + } + } + + if step >= MAX_ENDPOINT_RESOLUTION_RECURSION { + break; + }; + } else { + break; + } + } else { + break; + } + } + + if let Some(svcb) = svcb { + if PublicKey::try_from(svcb.target.as_str()).is_err() { + return Ok(svcb.clone()); + } + } + + Err(Error::ResolveEndpoint(target.into())) + } +} + +#[derive(Debug, Clone)] +pub struct Endpoint { + target: String, +} + +fn getx(signed_packet: &SignedPacket, target: &str) -> Option { + signed_packet + .resource_records(target) + .fold(None, |prev: Option, answer| { + if let Some(svcb) = match &answer.rdata { + RData::SVCB(svcb) => Some(svcb), + RData::HTTPS(curr) => Some(&curr.0), + _ => None, + } { + let curr = svcb.clone(); + + if curr.priority == 0 { + return Some(curr); + } + if let Some(prev) = &prev { + if curr.priority >= prev.priority { + return Some(curr); + } + } else { + return Some(curr); + } + } + + prev + }) + .map(|s| Endpoint { + target: s.target.to_string(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + use crate::{ + dns::{self, rdata::RData}, + mainline::Testnet, + Keypair, + }; + + fn publish_packets(client: &PkarrClient, tree: Vec>) -> Vec { + let mut keypairs: Vec = Vec::with_capacity(tree.len()); + + for node in tree { + let mut packet = dns::Packet::new_reply(0); + + for record in node { + packet.answers.push(dns::ResourceRecord::new( + dns::Name::new(record.0).unwrap(), + dns::CLASS::IN, + 3600, + record.1, + )); + } + + let keypair = Keypair::random(); + + let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); + + keypairs.push(keypair); + + client.publish(&signed_packet).unwrap(); + } + + keypairs + } + + #[test] + fn resolve_direct_endpoint() { + let testnet = Testnet::new(3); + + let client = PkarrClient::builder().testnet(&testnet).build().unwrap(); + + let keypairs = publish_packets( + &client, + vec![vec![ + ( + "foo", + RData::HTTPS(SVCB::new(0, "https.example.com".try_into().unwrap()).into()), + ), + // // Make sure HTTPS only follows HTTPs + // ( + // "foo", + // RData::SVCB(SVCB::new(0, "protocol.example.com".try_into().unwrap())), + // ), + ( + "_foo", + RData::SVCB(SVCB::new(0, "protocol.example.com".try_into().unwrap())), + ), + ]], + ); + + let tld = keypairs.first().unwrap().public_key(); + + let endpoint = client.resolve_endpoint(&format!("foo.{tld}")).unwrap(); + + assert_eq!(endpoint.target, "https.example.com"); + + let endpoint = client.resolve_endpoint(&format!("_foo.{tld}")).unwrap(); + + assert_eq!(endpoint.target, "protocol.example.com"); + } +} From 7a4575f4c60689765ba0567aa27248df2b31b2d4 Mon Sep 17 00:00:00 2001 From: Nuh Date: Sun, 15 Sep 2024 00:47:11 +0300 Subject: [PATCH 06/84] Update svcb.rs --- pkarr/src/svcb.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkarr/src/svcb.rs b/pkarr/src/svcb.rs index b4e39e6..67142ba 100644 --- a/pkarr/src/svcb.rs +++ b/pkarr/src/svcb.rs @@ -64,7 +64,7 @@ impl PkarrClient { #[derive(Debug, Clone)] pub struct Endpoint { - target: String, + pub target: String, } fn getx(signed_packet: &SignedPacket, target: &str) -> Option { From 5a2c288a757ee87374d3b2a1e13829377f439058 Mon Sep 17 00:00:00 2001 From: nazeh Date: Wed, 18 Sep 2024 18:42:47 +0300 Subject: [PATCH 07/84] feat(pkarr): update unit tests to make sure resolve_endpoint follows correct SVCB type --- pkarr/src/{svcb.rs => endpoints.rs} | 49 ++++++++++++++++++++++------- pkarr/src/lib.rs | 2 +- 2 files changed, 39 insertions(+), 12 deletions(-) rename pkarr/src/{svcb.rs => endpoints.rs} (74%) diff --git a/pkarr/src/svcb.rs b/pkarr/src/endpoints.rs similarity index 74% rename from pkarr/src/svcb.rs rename to pkarr/src/endpoints.rs index 67142ba..771fe4b 100644 --- a/pkarr/src/svcb.rs +++ b/pkarr/src/endpoints.rs @@ -8,7 +8,13 @@ use crate::{ const MAX_ENDPOINT_RESOLUTION_RECURSION: u8 = 3; impl PkarrClient { - pub fn resolve_endpoint(&self, target: &str) -> Result { + /// Resolve a `qname` to an alternative [Endpoint] as defined in [RFC9460](https://www.rfc-editor.org/rfc/rfc9460#name-terminology). + /// + /// A `qname` is can be either a regular domain name for HTTPS endpoints, + /// or it could use Attrleaf naming pattern for cusotm protcol. For example: + /// `_foo.example.com` for `foo://example.com`. + pub fn resolve_endpoint<'a>(&self, qname: &str) -> Result { + let target = qname; // TODO: cache the result of this function? let mut step = 0; @@ -54,7 +60,9 @@ impl PkarrClient { if let Some(svcb) = svcb { if PublicKey::try_from(svcb.target.as_str()).is_err() { - return Ok(svcb.clone()); + return Ok(Endpoint { + target: svcb.target, + }); } } @@ -68,12 +76,26 @@ pub struct Endpoint { } fn getx(signed_packet: &SignedPacket, target: &str) -> Option { + let is_svcb = target.starts_with('_'); + signed_packet .resource_records(target) .fold(None, |prev: Option, answer| { if let Some(svcb) = match &answer.rdata { - RData::SVCB(svcb) => Some(svcb), - RData::HTTPS(curr) => Some(&curr.0), + RData::SVCB(svcb) => { + if is_svcb { + Some(svcb) + } else { + None + } + } + RData::HTTPS(curr) => { + if is_svcb { + None + } else { + Some(&curr.0) + } + } _ => None, } { let curr = svcb.clone(); @@ -147,11 +169,16 @@ mod tests { "foo", RData::HTTPS(SVCB::new(0, "https.example.com".try_into().unwrap()).into()), ), - // // Make sure HTTPS only follows HTTPs - // ( - // "foo", - // RData::SVCB(SVCB::new(0, "protocol.example.com".try_into().unwrap())), - // ), + // Make sure HTTPS only follows HTTPs + ( + "foo", + RData::SVCB(SVCB::new(0, "protocol.example.com".try_into().unwrap())), + ), + // Make sure SVCB only follows SVCB + ( + "foo", + RData::HTTPS(SVCB::new(0, "https.example.com".try_into().unwrap()).into()), + ), ( "_foo", RData::SVCB(SVCB::new(0, "protocol.example.com".try_into().unwrap())), @@ -161,12 +188,12 @@ mod tests { let tld = keypairs.first().unwrap().public_key(); + // Follow foo.tld HTTPS records let endpoint = client.resolve_endpoint(&format!("foo.{tld}")).unwrap(); - assert_eq!(endpoint.target, "https.example.com"); + // Follow _foo.tld SVCB records let endpoint = client.resolve_endpoint(&format!("_foo.{tld}")).unwrap(); - assert_eq!(endpoint.target, "protocol.example.com"); } } diff --git a/pkarr/src/lib.rs b/pkarr/src/lib.rs index 9e838e2..8110348 100644 --- a/pkarr/src/lib.rs +++ b/pkarr/src/lib.rs @@ -53,7 +53,7 @@ macro_rules! if_relay { if_dht! { mod cache; mod client; - mod svcb; + mod endpoints; if_async! { mod client_async; From 5b2f8e53735f63521b0d587167609a2cb9b8f2ab Mon Sep 17 00:00:00 2001 From: nazeh Date: Thu, 19 Sep 2024 13:38:41 +0300 Subject: [PATCH 08/84] remove endpoints method --- pkarr/src/endpoints.rs | 199 ----------------------------------------- pkarr/src/error.rs | 3 - pkarr/src/lib.rs | 1 - 3 files changed, 203 deletions(-) delete mode 100644 pkarr/src/endpoints.rs diff --git a/pkarr/src/endpoints.rs b/pkarr/src/endpoints.rs deleted file mode 100644 index 771fe4b..0000000 --- a/pkarr/src/endpoints.rs +++ /dev/null @@ -1,199 +0,0 @@ -use simple_dns::rdata::{RData, SVCB}; - -use crate::{ - error::{Error, Result}, - PkarrClient, PublicKey, SignedPacket, -}; - -const MAX_ENDPOINT_RESOLUTION_RECURSION: u8 = 3; - -impl PkarrClient { - /// Resolve a `qname` to an alternative [Endpoint] as defined in [RFC9460](https://www.rfc-editor.org/rfc/rfc9460#name-terminology). - /// - /// A `qname` is can be either a regular domain name for HTTPS endpoints, - /// or it could use Attrleaf naming pattern for cusotm protcol. For example: - /// `_foo.example.com` for `foo://example.com`. - pub fn resolve_endpoint<'a>(&self, qname: &str) -> Result { - let target = qname; - // TODO: cache the result of this function? - - let mut step = 0; - let mut svcb: Option = None; - - loop { - let current = svcb.clone().map_or(target.to_string(), |s| s.target); - if let Ok(tld) = PublicKey::try_from(current.clone()) { - if let Ok(Some(signed_packet)) = self.resolve(&tld) { - if step >= MAX_ENDPOINT_RESOLUTION_RECURSION { - break; - }; - step += 1; - - // Choose most prior SVCB record - svcb = getx(&signed_packet, ¤t); - - // Try wildcards - if svcb.is_none() { - let parts: Vec<&str> = current.split('.').collect(); - - for i in 1..parts.len() { - let xx = format!("*.{}", parts[i..].join(".")); - - svcb = getx(&signed_packet, &xx); - - if svcb.is_some() { - break; - } - } - } - - if step >= MAX_ENDPOINT_RESOLUTION_RECURSION { - break; - }; - } else { - break; - } - } else { - break; - } - } - - if let Some(svcb) = svcb { - if PublicKey::try_from(svcb.target.as_str()).is_err() { - return Ok(Endpoint { - target: svcb.target, - }); - } - } - - Err(Error::ResolveEndpoint(target.into())) - } -} - -#[derive(Debug, Clone)] -pub struct Endpoint { - pub target: String, -} - -fn getx(signed_packet: &SignedPacket, target: &str) -> Option { - let is_svcb = target.starts_with('_'); - - signed_packet - .resource_records(target) - .fold(None, |prev: Option, answer| { - if let Some(svcb) = match &answer.rdata { - RData::SVCB(svcb) => { - if is_svcb { - Some(svcb) - } else { - None - } - } - RData::HTTPS(curr) => { - if is_svcb { - None - } else { - Some(&curr.0) - } - } - _ => None, - } { - let curr = svcb.clone(); - - if curr.priority == 0 { - return Some(curr); - } - if let Some(prev) = &prev { - if curr.priority >= prev.priority { - return Some(curr); - } - } else { - return Some(curr); - } - } - - prev - }) - .map(|s| Endpoint { - target: s.target.to_string(), - }) -} - -#[cfg(test)] -mod tests { - use super::*; - - use crate::{ - dns::{self, rdata::RData}, - mainline::Testnet, - Keypair, - }; - - fn publish_packets(client: &PkarrClient, tree: Vec>) -> Vec { - let mut keypairs: Vec = Vec::with_capacity(tree.len()); - - for node in tree { - let mut packet = dns::Packet::new_reply(0); - - for record in node { - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new(record.0).unwrap(), - dns::CLASS::IN, - 3600, - record.1, - )); - } - - let keypair = Keypair::random(); - - let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); - - keypairs.push(keypair); - - client.publish(&signed_packet).unwrap(); - } - - keypairs - } - - #[test] - fn resolve_direct_endpoint() { - let testnet = Testnet::new(3); - - let client = PkarrClient::builder().testnet(&testnet).build().unwrap(); - - let keypairs = publish_packets( - &client, - vec![vec![ - ( - "foo", - RData::HTTPS(SVCB::new(0, "https.example.com".try_into().unwrap()).into()), - ), - // Make sure HTTPS only follows HTTPs - ( - "foo", - RData::SVCB(SVCB::new(0, "protocol.example.com".try_into().unwrap())), - ), - // Make sure SVCB only follows SVCB - ( - "foo", - RData::HTTPS(SVCB::new(0, "https.example.com".try_into().unwrap()).into()), - ), - ( - "_foo", - RData::SVCB(SVCB::new(0, "protocol.example.com".try_into().unwrap())), - ), - ]], - ); - - let tld = keypairs.first().unwrap().public_key(); - - // Follow foo.tld HTTPS records - let endpoint = client.resolve_endpoint(&format!("foo.{tld}")).unwrap(); - assert_eq!(endpoint.target, "https.example.com"); - - // Follow _foo.tld SVCB records - let endpoint = client.resolve_endpoint(&format!("_foo.{tld}")).unwrap(); - assert_eq!(endpoint.target, "protocol.example.com"); - } -} diff --git a/pkarr/src/error.rs b/pkarr/src/error.rs index 845af14..3b509ff 100644 --- a/pkarr/src/error.rs +++ b/pkarr/src/error.rs @@ -64,9 +64,6 @@ pub enum Error { /// Failed to publish because there is a more recent packet. NotMostRecent, - #[error("Could not resolve Endpoint for domain: {0}")] - ResolveEndpoint(String), - // === Relay errors === #[cfg(all(not(target_arch = "wasm32"), feature = "relay"))] #[error(transparent)] diff --git a/pkarr/src/lib.rs b/pkarr/src/lib.rs index 8110348..7fffe55 100644 --- a/pkarr/src/lib.rs +++ b/pkarr/src/lib.rs @@ -53,7 +53,6 @@ macro_rules! if_relay { if_dht! { mod cache; mod client; - mod endpoints; if_async! { mod client_async; From 5e27d40069027bce32c28b49c3237de7ff2b6018 Mon Sep 17 00:00:00 2001 From: nazeh Date: Sat, 21 Sep 2024 14:02:13 +0300 Subject: [PATCH 09/84] feat(pkarr): replace ureq with reqwest and remove blocking relay client --- Cargo.lock | 500 ++++++++++++++++++++++++++------ pkarr/Cargo.toml | 42 +-- pkarr/src/client.rs | 2 +- pkarr/src/client_async.rs | 58 ++-- pkarr/src/error.rs | 18 +- pkarr/src/lib.rs | 10 - pkarr/src/relay_client.rs | 275 +++++++++--------- pkarr/src/relay_client_async.rs | 151 ---------- pkarr/src/relay_client_web.rs | 231 --------------- pkarr/src/signed_packet.rs | 6 +- 10 files changed, 601 insertions(+), 692 deletions(-) delete mode 100644 pkarr/src/relay_client_async.rs delete mode 100644 pkarr/src/relay_client_web.rs diff --git a/Cargo.lock b/Cargo.lock index 73bbba0..300d358 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -355,20 +355,26 @@ dependencies = [ ] [[package]] -name = "console_error_panic_hook" -version = "0.1.7" +name = "const-oid" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" dependencies = [ - "cfg-if", - "wasm-bindgen", + "core-foundation-sys", + "libc", ] [[package]] -name = "const-oid" -version = "0.9.6" +name = "core-foundation-sys" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" @@ -567,12 +573,37 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" +[[package]] +name = "encoding_rs" +version = "0.8.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "errno" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "fastrand" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" + [[package]] name = "fiat-crypto" version = "0.2.9" @@ -597,6 +628,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -918,6 +964,41 @@ dependencies = [ "pin-project-lite", "smallvec", "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" +dependencies = [ + "futures-util", + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", ] [[package]] @@ -927,12 +1008,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" dependencies = [ "bytes", + "futures-channel", "futures-util", "http", "http-body", "hyper", "pin-project-lite", + "socket2", "tokio", + "tower", + "tower-service", + "tracing", ] [[package]] @@ -955,6 +1041,12 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "ipnet" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -998,6 +1090,12 @@ dependencies = [ "libc", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + [[package]] name = "litrs" version = "0.4.1" @@ -1084,16 +1182,6 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" -[[package]] -name = "minicov" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c71e683cd655513b99affab7d317deb690528255a0d5f717f1024093c12b169" -dependencies = [ - "cc", - "walkdir", -] - [[package]] name = "miniz_oxide" version = "0.7.4" @@ -1148,6 +1236,23 @@ dependencies = [ "getrandom", ] +[[package]] +name = "native-tls" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "no-std-compat" version = "0.4.1" @@ -1191,6 +1296,50 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "openssl" +version = "0.10.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "overload" version = "0.1.1" @@ -1321,24 +1470,20 @@ dependencies = [ "dyn-clone", "ed25519-dalek", "flume", - "futures", "js-sys", "lru", "mainline", "mockito", "postcard", "rand", + "reqwest", "self_cell", "serde", "simple-dns", "thiserror", + "tokio", "tracing", "tracing-subscriber", - "ureq", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-bindgen-test", - "web-sys", ] [[package]] @@ -1377,6 +1522,12 @@ dependencies = [ "spki", ] +[[package]] +name = "pkg-config" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" + [[package]] name = "portable-atomic" version = "1.7.0" @@ -1429,6 +1580,54 @@ dependencies = [ "winapi", ] +[[package]] +name = "quinn" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" +dependencies = [ + "bytes", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "quinn-proto" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" +dependencies = [ + "bytes", + "rand", + "ring", + "rustc-hash", + "rustls", + "slab", + "thiserror", + "tinyvec", + "tracing", +] + +[[package]] +name = "quinn-udp" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bffec3605b73c6f1754535084a85229fa8a30f86014e6c81aeec4abb68b0285" +dependencies = [ + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.52.0", +] + [[package]] name = "quote" version = "1.0.36" @@ -1541,6 +1740,54 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +[[package]] +name = "reqwest" +version = "0.12.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-tls", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pemfile", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper 1.0.1", + "system-configuration", + "tokio", + "tokio-native-tls", + "tokio-rustls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", + "windows-registry", +] + [[package]] name = "ring" version = "0.17.8" @@ -1562,6 +1809,12 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc-hash" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" + [[package]] name = "rustc_version" version = "0.4.0" @@ -1571,13 +1824,25 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "0.38.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + [[package]] name = "rustls" version = "0.23.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" dependencies = [ - "log", "once_cell", "ring", "rustls-pki-types", @@ -1626,26 +1891,43 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] -name = "same-file" -version = "1.0.6" +name = "schannel" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b" dependencies = [ - "winapi-util", + "windows-sys 0.59.0", ] -[[package]] -name = "scoped-tls" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" - [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "self_cell" version = "1.0.4" @@ -1904,6 +2186,9 @@ name = "sync_wrapper" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +dependencies = [ + "futures-core", +] [[package]] name = "synchronoise" @@ -1914,6 +2199,40 @@ dependencies = [ "crossbeam-queue", ] +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" +dependencies = [ + "cfg-if", + "fastrand", + "once_cell", + "rustix", + "windows-sys 0.59.0", +] + [[package]] name = "thiserror" version = "1.0.63" @@ -1961,9 +2280,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.39.2" +version = "1.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1" +checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" dependencies = [ "backtrace", "bytes", @@ -1988,6 +2307,16 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.0" @@ -2169,6 +2498,12 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "typenum" version = "1.17.0" @@ -2202,21 +2537,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" -[[package]] -name = "ureq" -version = "2.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b74fc6b57825be3373f7054754755f03ac3a8f5d70015ccad699ba2029956f4a" -dependencies = [ - "base64", - "log", - "once_cell", - "rustls", - "rustls-pki-types", - "url", - "webpki-roots", -] - [[package]] name = "url" version = "2.5.2" @@ -2240,6 +2560,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" @@ -2247,13 +2573,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] -name = "walkdir" -version = "2.5.0" +name = "want" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" dependencies = [ - "same-file", - "winapi-util", + "try-lock", ] [[package]] @@ -2329,32 +2654,6 @@ version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" -[[package]] -name = "wasm-bindgen-test" -version = "0.3.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68497a05fb21143a08a7d24fc81763384a3072ee43c44e86aad1744d6adef9d9" -dependencies = [ - "console_error_panic_hook", - "js-sys", - "minicov", - "scoped-tls", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-bindgen-test-macro", -] - -[[package]] -name = "wasm-bindgen-test-macro" -version = "0.3.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b8220be1fa9e4c889b30fd207d4906657e7e90b12e0e6b0c8b8d8709f5de021" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "web-sys" version = "0.3.70" @@ -2391,19 +2690,40 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] -name = "winapi-util" -version = "0.1.9" +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-registry" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" dependencies = [ - "windows-sys 0.59.0", + "windows-result", + "windows-strings", + "windows-targets 0.52.6", ] [[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" +name = "windows-result" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-strings" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +dependencies = [ + "windows-result", + "windows-targets 0.52.6", +] [[package]] name = "windows-sys" diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index 792af78..e3b9dce 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -11,67 +11,53 @@ keywords = ["mainline", "dht", "dns", "decentralized", "identity"] [dependencies] base32 = "0.5.0" bytes = "1.7.1" -document-features = "0.2.8" ed25519-dalek = "2.0.0" self_cell = "1.0.2" simple-dns = "0.6.1" thiserror = "1.0.49" tracing = "0.1.40" -rand = { version = "0.8.5", optional = true } lru = { version = "0.12.3", default-features = false } -flume = { version = "0.11.0", features = ["select", "eventual-fairness"], default-features = false } serde = { version = "1.0.209", features = ["derive"], optional = true } +reqwest = { version = "0.12.7", features = ["rustls-tls"], optional = true } +rand = { version = "0.8.5", optional = true } + +document-features = "0.2.8" [target.'cfg(not(target_arch = "wasm32"))'.dependencies] # Dht client dependencies: mainline = { version = "2.0.1", optional = true } dyn-clone = { version = "1.0.17", optional = true } - -# Relay client dependencies -ureq = { version = "2.10", default-features = false, features = ["tls"], optional = true } +flume = { version = "0.11.0", features = ["select", "eventual-fairness"], default-features = false , optional = true } +tokio = { version = "1.40.0", optional = true } [target.'cfg(target_arch = "wasm32")'.dependencies] -futures = "0.3.29" js-sys = "0.3.69" -wasm-bindgen = "0.2.92" -wasm-bindgen-futures = "0.4.42" -web-sys = { version = "0.3.69", features = [ - "console", - "Request", - "RequestInit", - "RequestMode", - "Response", - "Window", -] } [dev-dependencies] -futures = "0.3.29" postcard = { version = "1.0.10", features = ["alloc"] } +tokio = { version = "1.40.0", features = ["macros"] } [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] clap = { version = "4.4.8", features = ["derive"] } mockito = "1.4.0" tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } -[target.'cfg(target_arch = "wasm32")'.dev-dependencies] -wasm-bindgen-test = "0.3.42" - [features] -## Use [PkarrClient] -dht = ["dep:mainline", "dep:dyn-clone"] ## Use [Keypair::random] rand = ["dep:rand", "ed25519-dalek/rand_core"] -## Use async versions of [PkarrClient] and/or [PkarrRelayClient] -async = ["flume/async"] -## Use [PkarrRelayClient] -relay = ["dep:ureq"] ## Derive serde Serialize/Deserialize for PublicKey serde = ["dep:serde"] +## Use [PkarrClient] +dht = ["dep:mainline", "dep:dyn-clone"] +## Use async versions of [PkarrClient] +async = ["flume/async"] +## Use [PkarrRelayClient] +relay = ["dep:reqwest", "dep:tokio"] ## Use all features full = ["dht", "async", "relay", "rand", "serde"] -default = ["dht", "rand"] +default = ["full"] [package.metadata.docs.rs] all-features = true diff --git a/pkarr/src/client.rs b/pkarr/src/client.rs index 82768fc..d4f1f4c 100644 --- a/pkarr/src/client.rs +++ b/pkarr/src/client.rs @@ -292,7 +292,7 @@ impl PkarrClient { debug!(expires_in, "Have expired signed_packet in cache."); } else { - debug!("Cache mess"); + debug!("Cache miss"); } self.sender diff --git a/pkarr/src/client_async.rs b/pkarr/src/client_async.rs index 6b51c56..5586b79 100644 --- a/pkarr/src/client_async.rs +++ b/pkarr/src/client_async.rs @@ -95,52 +95,44 @@ mod tests { #[test] fn shutdown() { - async fn test() { - let testnet = Testnet::new(3); + let testnet = Testnet::new(3); - let mut a = PkarrClient::builder().testnet(&testnet).build().unwrap(); + let mut a = PkarrClient::builder().testnet(&testnet).build().unwrap(); - assert_ne!(a.local_addr(), None); + assert_ne!(a.local_addr(), None); - a.shutdown().unwrap(); + a.shutdown().unwrap(); - assert_eq!(a.local_addr(), None); - } - - futures::executor::block_on(test()); + assert_eq!(a.local_addr(), None); } - #[test] - fn publish_resolve() { - async fn test() { - let testnet = Testnet::new(10); + #[tokio::test] + async fn publish_resolve() { + let testnet = Testnet::new(10); - let a = PkarrClient::builder().testnet(&testnet).build().unwrap(); + let a = PkarrClient::builder().testnet(&testnet).build().unwrap(); - let keypair = Keypair::random(); + let keypair = Keypair::random(); - let mut packet = dns::Packet::new_reply(0); - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("foo").unwrap(), - dns::CLASS::IN, - 30, - dns::rdata::RData::TXT("bar".try_into().unwrap()), - )); + let mut packet = dns::Packet::new_reply(0); + packet.answers.push(dns::ResourceRecord::new( + dns::Name::new("foo").unwrap(), + dns::CLASS::IN, + 30, + dns::rdata::RData::TXT("bar".try_into().unwrap()), + )); - let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); + let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); - let _ = a.publish(&signed_packet); + let _ = a.publish(&signed_packet); - let b = PkarrClient::builder().testnet(&testnet).build().unwrap(); + let b = PkarrClient::builder().testnet(&testnet).build().unwrap(); - let resolved = b.resolve(&keypair.public_key()).unwrap().unwrap(); - assert_eq!(resolved.as_bytes(), signed_packet.as_bytes()); - - let from_cache = b.resolve(&keypair.public_key()).unwrap().unwrap(); - assert_eq!(from_cache.as_bytes(), signed_packet.as_bytes()); - assert_eq!(from_cache.last_seen(), resolved.last_seen()); - } + let resolved = b.resolve(&keypair.public_key()).unwrap().unwrap(); + assert_eq!(resolved.as_bytes(), signed_packet.as_bytes()); - futures::executor::block_on(test()); + let from_cache = b.resolve(&keypair.public_key()).unwrap().unwrap(); + assert_eq!(from_cache.as_bytes(), signed_packet.as_bytes()); + assert_eq!(from_cache.last_seen(), resolved.last_seen()); } } diff --git a/pkarr/src/error.rs b/pkarr/src/error.rs index 3b509ff..9f22f32 100644 --- a/pkarr/src/error.rs +++ b/pkarr/src/error.rs @@ -65,23 +65,13 @@ pub enum Error { NotMostRecent, // === Relay errors === - #[cfg(all(not(target_arch = "wasm32"), feature = "relay"))] + #[cfg(feature = "relay")] #[error(transparent)] - /// Transparent [ureq::Error] - RelayError(#[from] Box), + /// Transparent [reqwest::Error] + RelayError(#[from] reqwest::Error), - #[cfg(any(feature = "relay", target_arch = "wasm32"))] + #[cfg(feature = "relay")] #[error("Empty list of relays")] /// Empty list of relays EmptyListOfRelays, - - // === Wasm === - #[cfg(target_arch = "wasm32")] - #[error("Relay response with status: {0}, and message: {1}")] - /// A response was successfully received but had status code >= 400. - WasmRelayError(u16, String), - - #[cfg(target_arch = "wasm32")] - #[error("JS error")] - JsError(wasm_bindgen::JsValue), } diff --git a/pkarr/src/lib.rs b/pkarr/src/lib.rs index 7fffe55..09daa97 100644 --- a/pkarr/src/lib.rs +++ b/pkarr/src/lib.rs @@ -69,14 +69,4 @@ if_dht! { if_relay! { mod relay_client; pub use relay_client::{PkarrRelayClient, RelaySettings}; - - if_async! { - mod relay_client_async; - pub use relay_client_async::PkarrRelayClientAsync; - } } - -#[cfg(target_arch = "wasm32")] -mod relay_client_web; -#[cfg(target_arch = "wasm32")] -pub use relay_client_web::PkarrRelayClient; diff --git a/pkarr/src/relay_client.rs b/pkarr/src/relay_client.rs index e49193a..a5ac040 100644 --- a/pkarr/src/relay_client.rs +++ b/pkarr/src/relay_client.rs @@ -1,16 +1,14 @@ //! Pkarr client for publishing and resolving [SignedPacket]s over [relays](https://pkarr.org/relays). use std::{ - io::Read, num::NonZeroUsize, sync::{Arc, Mutex}, - thread, }; -use flume::Receiver; use lru::LruCache; +use reqwest::{Client, StatusCode}; +use tokio::task::JoinSet; use tracing::debug; -use ureq::Agent; use crate::{ Error, PublicKey, Result, SignedPacket, DEFAULT_CACHE_SIZE, DEFAULT_MAXIMUM_TTL, @@ -31,8 +29,8 @@ pub struct RelaySettings { /// /// Defaults to [DEFAULT_MAXIMUM_TTL] pub maximum_ttl: u32, - /// Custom [ureq::Agent] - pub http_client: Agent, + /// Custom [reqwest::Client] + pub http_client: Client, } impl Default for RelaySettings { @@ -42,7 +40,7 @@ impl Default for RelaySettings { cache_size: NonZeroUsize::new(DEFAULT_CACHE_SIZE).unwrap(), minimum_ttl: DEFAULT_MINIMUM_TTL, maximum_ttl: DEFAULT_MAXIMUM_TTL, - http_client: ureq::Agent::new(), + http_client: Client::new(), } } } @@ -50,7 +48,7 @@ impl Default for RelaySettings { #[derive(Debug, Clone)] /// Pkarr client for publishing and resolving [SignedPacket]s over [relays](https://pkarr.org/relays). pub struct PkarrRelayClient { - http_client: Agent, + http_client: Client, relays: Vec, cache: Arc>>, minimum_ttl: u32, @@ -91,14 +89,51 @@ impl PkarrRelayClient { /// - Returns a [Error::NotMostRecent] if the provided signed packet is older than most recent. /// - Returns a [Error::RelayError] from the last responding relay, if all relays /// responded with non-2xx status codes. - pub fn publish(&self, signed_packet: &SignedPacket) -> Result<()> { + pub async fn publish(&self, signed_packet: &SignedPacket) -> Result<()> { + let public_key = signed_packet.public_key(); + + // Let the compiler know we are dropping the cache before await + { + let mut cache = self.cache.lock().unwrap(); + + if let Some(current) = cache.get(&public_key) { + if current.timestamp() > signed_packet.timestamp() { + return Err(Error::NotMostRecent); + } + }; + + cache.put(public_key.to_owned(), signed_packet.clone()); + } + + let mut futures = JoinSet::new(); + + for relay in self.relays.clone() { + let url = format!("{relay}/{public_key}"); + let http_client = self.http_client.clone(); + let signed_packet = signed_packet.clone(); + + futures.spawn(async move { + http_client + .put(&url) + .body(signed_packet.to_relay_payload()) + .send() + .await + .map_err(|error| { + debug!(?url, ?error, "Error response"); + + error + }) + }); + } + let mut last_error = Error::EmptyListOfRelays; - while let Ok(response) = self.publish_inner(signed_packet)?.recv() { - match response { - Ok(_) => return Ok(()), - Err(error) => { - last_error = error; + while let Some(result) = futures.join_next().await { + match result { + Ok(Ok(_)) => return Ok(()), + Ok(Err(error)) => last_error = Error::RelayError(error), + Err(joinerror) => { + debug!(?joinerror); } } } @@ -114,144 +149,120 @@ impl PkarrRelayClient { /// /// - Returns [Error::RelayError] if the relay responded with a status >= 400 /// (except 404 in which case you should receive Ok(None)) or something wrong - /// with the transport, transparent from [ureq::Error]. + /// with the transport, transparent from [reqwest::Error]. /// - Returns [Error::IO] if something went wrong while reading the payload. - pub fn resolve(&self, public_key: &PublicKey) -> Result> { - if let Some(signed_packet) = self.resolve_inner(public_key).recv()?? { - self.cache - .lock() - .unwrap() - .put(public_key.clone(), signed_packet.clone()); - - return Ok(Some(signed_packet)); - }; + pub async fn resolve(&self, public_key: &PublicKey) -> Result> { + let cached_packet = { + let mut cache = self.cache.lock().unwrap(); - Ok(None) - } + let cached_packet = cache.get(public_key); - // === Private Methods === + if let Some(cached) = cached_packet { + let expires_in = cached.expires_in(self.minimum_ttl, self.maximum_ttl); - pub(crate) fn publish_inner( - &self, - signed_packet: &SignedPacket, - ) -> Result>> { - let public_key = signed_packet.public_key(); - let mut cache = self.cache.lock().unwrap(); + if expires_in > 0 { + debug!(expires_in, "Have fresh signed_packet in cache."); - if let Some(current) = cache.get(&public_key) { - if current.timestamp() > signed_packet.timestamp() { - return Err(Error::NotMostRecent); - } - }; + return Ok(Some(cached.clone())); + } + + debug!(expires_in, "Have expired signed_packet in cache."); + } else { + debug!("Cache miss"); + }; - cache.put(public_key.to_owned(), signed_packet.clone()); - drop(cache); + None + }; - let (sender, receiver) = flume::bounded::>(1); + let mut futures = JoinSet::new(); for relay in self.relays.clone() { - let url = format!("{relay}/{public_key}"); let http_client = self.http_client.clone(); - let sender = sender.clone(); - let signed_packet = signed_packet.clone(); + let public_key = public_key.clone(); + let cached = cached_packet.clone(); - thread::spawn(move || { - match http_client - .put(&url) - .send_bytes(&signed_packet.to_relay_payload()) - { - Ok(_) => { - let _ = sender.send(Ok(())); - } - Err(error) => { - debug!(?url, ?error, "Error response"); - let _ = sender.send(Err(Error::RelayError(Box::new(error)))); - } - } + futures.spawn(async move { + resolve_from_relay(relay, public_key, http_client, cached).await }); } - Ok(receiver) - } - - pub(crate) fn resolve_inner( - &self, - public_key: &PublicKey, - ) -> Receiver>> { - let mut cache = self.cache.lock().unwrap(); + let mut result: Result> = Ok(None); - let cached_packet = cache.get(public_key); + while let Some(task_result) = futures.join_next().await { + match task_result { + Ok(Ok(Some(signed_packet))) => { + let mut cache = self.cache.lock().unwrap(); - let (sender, receiver) = flume::bounded::>>(1); + cache.put(signed_packet.public_key(), signed_packet.clone()); - if let Some(cached) = cached_packet { - let expires_in = cached.expires_in(self.minimum_ttl, self.maximum_ttl); + return Ok(Some(signed_packet)); + } + Ok(Err(error)) => result = Err(error), + Ok(_) => {} + Err(joinerror) => { + debug!(?joinerror); + } + } + } - if expires_in > 0 { - debug!(expires_in, "Have fresh signed_packet in cache."); - let _ = sender.send(Ok(Some(cached.clone()))); + result + } +} - return receiver; +async fn resolve_from_relay( + relay: String, + public_key: PublicKey, + http_client: Client, + cached_packet: Option, +) -> Result> { + let url = format!("{relay}/{public_key}"); + + match http_client.get(&url).send().await { + Ok(mut response) => { + if response.status() == StatusCode::NOT_FOUND { + debug!(?url, "SignedPacket not found"); + return Ok(None); } - debug!(expires_in, "Have expired signed_packet in cache."); - } else { - debug!("Cache mess"); - }; + let mut total_size = 0; + let mut payload = Vec::new(); - for relay in self.relays.clone() { - let url = format!("{relay}/{public_key}"); - let http_client = self.http_client.clone(); - let public_key = public_key.clone(); - let cached_packet = cached_packet.cloned(); - let sender = sender.clone(); - - thread::spawn(move || match http_client.get(&url).call() { - Ok(response) => { - let mut reader = response.into_reader(); - let mut payload = vec![]; + while let Ok(Some(chunk)) = response.chunk().await { + total_size += chunk.len(); + payload.extend_from_slice(&chunk); + if total_size >= SignedPacket::MAX_BYTES { + break; + } + } - if let Err(err) = reader.read_to_end(&mut payload) { - let _ = sender.send(Err(err.into())); + match SignedPacket::from_relay_payload(&public_key, &payload.into()) { + Ok(signed_packet) => { + let new_packet = if let Some(ref cached) = cached_packet { + if signed_packet.more_recent_than(cached) { + debug!(?public_key, "Received more recent packet than in cache"); + Some(signed_packet) + } else { + None + } } else { - match SignedPacket::from_relay_payload(&public_key, &payload.into()) { - Ok(signed_packet) => { - let new_packet = if let Some(ref cached) = cached_packet { - if signed_packet.more_recent_than(cached) { - debug!( - ?public_key, - "Received more recent packet than in cache" - ); - Some(signed_packet) - } else { - None - } - } else { - debug!(?public_key, "Received new packet after cache miss"); - Some(signed_packet) - }; - - let _ = sender.send(Ok(new_packet)); - } - Err(error) => { - debug!(?url, ?error, "Invalid signed_packet"); - let _ = sender.send(Err(error)); - } - }; - } - } - Err(ureq::Error::Status(404, _)) => { - debug!(?url, "SignedPacket not found"); - let _ = sender.send(Ok(None)); + debug!(?public_key, "Received new packet after cache miss"); + Some(signed_packet) + }; + + Ok(new_packet) } Err(error) => { - debug!(?url, ?error, "Error response"); - let _ = sender.send(Err(Error::RelayError(Box::new(error)))); + debug!(?url, ?error, "Invalid signed_packet"); + + Err(error) } - }); + } } + Err(error) => { + debug!(?url, ?error, "Error response"); - receiver + Err(Error::RelayError(error)) + } } } @@ -260,8 +271,8 @@ mod tests { use super::*; use crate::{dns, Keypair, SignedPacket}; - #[test] - fn publish_resolve() { + #[tokio::test] + async fn publish_resolve() { let keypair = Keypair::random(); let mut packet = dns::Packet::new_reply(0); @@ -274,7 +285,7 @@ mod tests { let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); - let mut server = mockito::Server::new(); + let mut server = mockito::Server::new_async().await; let path = format!("/{}", signed_packet.public_key()); @@ -297,9 +308,9 @@ mod tests { let a = PkarrRelayClient::new(settings.clone()).unwrap(); let b = PkarrRelayClient::new(settings).unwrap(); - a.publish(&signed_packet).unwrap(); + a.publish(&signed_packet).await.unwrap(); - let resolved = b.resolve(&keypair.public_key()).unwrap().unwrap(); + let resolved = b.resolve(&keypair.public_key()).await.unwrap().unwrap(); assert_eq!(a.cache().lock().unwrap().len(), 1); assert_eq!(b.cache().lock().unwrap().len(), 1); @@ -307,11 +318,11 @@ mod tests { assert_eq!(resolved.as_bytes(), signed_packet.as_bytes()); } - #[test] - fn not_found() { + #[tokio::test] + async fn not_found() { let keypair = Keypair::random(); - let mut server = mockito::Server::new(); + let mut server = mockito::Server::new_async().await; let path = format!("/{}", keypair.public_key()); @@ -325,7 +336,7 @@ mod tests { let client = PkarrRelayClient::new(settings.clone()).unwrap(); - let resolved = client.resolve(&keypair.public_key()).unwrap(); + let resolved = client.resolve(&keypair.public_key()).await.unwrap(); assert!(resolved.is_none()); } diff --git a/pkarr/src/relay_client_async.rs b/pkarr/src/relay_client_async.rs deleted file mode 100644 index c9fd38a..0000000 --- a/pkarr/src/relay_client_async.rs +++ /dev/null @@ -1,151 +0,0 @@ -//! Pkarr client for publishing and resolving [SignedPacket]s over [relays](https://pkarr.org/relays). - -use std::sync::Mutex; - -use lru::LruCache; - -use crate::{Error, PkarrRelayClient, PublicKey, Result, SignedPacket}; - -pub struct PkarrRelayClientAsync(PkarrRelayClient); - -impl PkarrRelayClient { - pub fn as_async(self) -> PkarrRelayClientAsync { - PkarrRelayClientAsync(self) - } -} - -impl PkarrRelayClientAsync { - /// Returns a reference to the internal cache. - pub fn cache(&self) -> &Mutex> { - self.0.cache() - } - - /// Publishes a [SignedPacket] to this client's relays. - /// - /// Return the first successful completion, or the last failure. - /// - /// # Errors - /// - Returns a [Error::NotMostRecent] if the provided signed packet is older than most recent. - /// - Returns a [Error::RelayError] from the last responding relay, if all relays - /// responded with non-2xx status codes. - pub async fn publish(&self, signed_packet: &SignedPacket) -> Result<()> { - let mut last_error = Error::EmptyListOfRelays; - - while let Ok(response) = self.0.publish_inner(signed_packet)?.recv_async().await { - match response { - Ok(_) => return Ok(()), - Err(error) => { - last_error = error; - } - } - } - - Err(last_error) - } - - /// Resolve a [SignedPacket] from this client's relays. - /// - /// Return the first successful response, or the failure from the last responding relay. - /// - /// # Errors - /// - /// - Returns [Error::RelayError] if the relay responded with a status >= 400 - /// (except 404 in which case you should receive Ok(None)) or something wrong - /// with the transport, transparent from [ureq::Error]. - /// - Returns [Error::IO] if something went wrong while reading the payload. - pub async fn resolve(&self, public_key: &PublicKey) -> Result> { - if let Some(signed_packet) = self.0.resolve_inner(public_key).recv_async().await?? { - self.cache() - .lock() - .unwrap() - .put(public_key.clone(), signed_packet.clone()); - - return Ok(Some(signed_packet)); - }; - - Ok(None) - } -} - -#[cfg(test)] -mod tests { - use crate::{dns, Keypair, PkarrRelayClient, RelaySettings, SignedPacket}; - - #[test] - fn publish_resolve() { - async fn test() { - let keypair = Keypair::random(); - - let mut packet = dns::Packet::new_reply(0); - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("foo").unwrap(), - dns::CLASS::IN, - 30, - dns::rdata::RData::TXT("bar".try_into().unwrap()), - )); - - let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); - - let mut server = mockito::Server::new(); - - let path = format!("/{}", signed_packet.public_key()); - - server - .mock("PUT", path.as_str()) - .with_header("content-type", "text/plain") - .with_status(200) - .create(); - server - .mock("GET", path.as_str()) - .with_body(signed_packet.to_relay_payload()) - .create(); - - let relays: Vec = vec![server.url()]; - let settings = RelaySettings { - relays, - ..RelaySettings::default() - }; - - let a = PkarrRelayClient::new(settings.clone()).unwrap().as_async(); - let b = PkarrRelayClient::new(settings).unwrap().as_async(); - - a.publish(&signed_packet).await.unwrap(); - - let resolved = b.resolve(&keypair.public_key()).await.unwrap().unwrap(); - - assert_eq!(a.cache().lock().unwrap().len(), 1); - assert_eq!(b.cache().lock().unwrap().len(), 1); - - assert_eq!(resolved.as_bytes(), signed_packet.as_bytes()); - } - - futures::executor::block_on(test()); - } - - #[test] - fn not_found() { - async fn test() { - let keypair = Keypair::random(); - - let mut server = mockito::Server::new(); - - let path = format!("/{}", keypair.public_key()); - - server.mock("GET", path.as_str()).with_status(404).create(); - - let relays: Vec = vec![server.url()]; - let settings = RelaySettings { - relays, - ..RelaySettings::default() - }; - - let client = PkarrRelayClient::new(settings.clone()).unwrap().as_async(); - - let resolved = client.resolve(&keypair.public_key()).await.unwrap(); - - assert!(resolved.is_none()); - } - - futures::executor::block_on(test()); - } -} diff --git a/pkarr/src/relay_client_web.rs b/pkarr/src/relay_client_web.rs deleted file mode 100644 index ba1aab3..0000000 --- a/pkarr/src/relay_client_web.rs +++ /dev/null @@ -1,231 +0,0 @@ -use futures::future::select_ok; -use std::str; - -use wasm_bindgen::{JsCast, JsValue}; -use wasm_bindgen_futures::JsFuture; -use web_sys::RequestMode; - -use crate::{Error, PublicKey, Result, SignedPacket, DEFAULT_RELAYS}; - -use tracing::debug; - -#[derive(Debug, Clone)] -pub struct PkarrRelayClient { - relays: Vec, -} - -impl Default for PkarrRelayClient { - fn default() -> Self { - Self::new(DEFAULT_RELAYS.map(|s| s.into()).to_vec()).unwrap() - } -} - -impl PkarrRelayClient { - pub fn new(relays: Vec) -> Result { - if relays.is_empty() { - return Err(Error::EmptyListOfRelays); - } - - Ok(Self { relays }) - } - - /// Publishes a [SignedPacket] to this client's relays. - /// - /// Return the first successful completion, or the last failure. - /// - /// # Errors - /// - Returns [Error::WasmRelayError] For Error responses - /// - Returns [Error::JsError] If an error happened on JS side. - pub async fn publish(&self, signed_packet: &SignedPacket) -> Result<()> { - let futures = self.relays.iter().map(|relay| { - Box::pin(async move { - let url = format!("{relay}/{}", signed_packet.public_key()); - publish_inner(&url, signed_packet.to_relay_payload().to_vec()).await - }) - }); - - match select_ok(futures).await { - Ok((response, _)) => Ok(response), - Err(e) => Err(e), - } - } - - /// Resolve a [SignedPacket] from this client's relays. - /// - /// Return the first successful response, or the failure from the last responding relay. - /// - /// # Errors - /// - /// - Returns [Error::WasmRelayError] For Error responses - /// (except 404, these get converted to Ok(None). - /// - Returns [Error::JsError] If an error happened on JS side. - pub async fn resolve(&self, public_key: &PublicKey) -> Result> { - let futures = self.relays.iter().map(|relay| { - Box::pin(async move { - let url = format!("{relay}/{public_key}"); - - match resolve_inner(&url).await { - Ok(bytes) => { - match SignedPacket::from_relay_payload(public_key, &bytes.into()) { - Ok(signed_packet) => { - return Ok(Some(signed_packet)); - } - Err(error) => { - debug!(?url, ?error, "Invalid signed_packet"); - return Err(error); - } - } - } - Err(error) => { - debug!(?url, ?error, "Error response"); - return Err(error); - } - } - }) - }); - - match select_ok(futures).await { - Ok((response, _)) => Ok(response), - Err(e) => { - if let Error::WasmRelayError(404, _) = e { - return Ok(None); - } - - Err(e) - } - } - } -} - -async fn publish_inner(url: &String, bytes: Vec) -> Result<()> { - let response = fetch_base(url, "PUT", Some(bytes)).await?; - let bytes = response_body(&response).await?; - - if !response.ok() { - return Err(Error::WasmRelayError( - response.status(), - str::from_utf8(&bytes).ok().unwrap_or("").to_string(), - )); - } - - Ok(()) -} - -async fn resolve_inner(url: &String) -> Result> { - let response = fetch_base(url, "GET", None).await?; - let bytes = response_body(&response).await?; - - if !response.ok() { - return Err(Error::WasmRelayError( - response.status(), - str::from_utf8(&bytes).ok().unwrap_or("").to_string(), - )); - } - - Ok(bytes) -} - -async fn response_body(response: &web_sys::Response) -> Result> { - let array_buffer = JsFuture::from( - response - .array_buffer() - .map_err(|error| Error::JsError(error))?, - ) - .await - .map_err(|error| Error::JsError(error))?; - - let uint8_array = js_sys::Uint8Array::new(&array_buffer); - - Ok(uint8_array.to_vec()) -} - -async fn fetch_base( - url: &String, - method: &str, - body: Option>, -) -> Result { - let mut opts = web_sys::RequestInit::new(); - opts.method(method); - opts.mode(RequestMode::Cors); - - if let Some(body) = body { - let body_bytes: &[u8] = &body; - let body_array: js_sys::Uint8Array = body_bytes.into(); - let js_value: &JsValue = body_array.as_ref(); - opts.body(Some(js_value)); - } - - let js_request = web_sys::Request::new_with_str_and_init(url, &opts) - .map_err(|error| Error::JsError(error))?; - - let window = web_sys::window().unwrap(); - let response = JsFuture::from(window.fetch_with_request(&js_request)) - .await - .map_err(|error| Error::JsError(error))?; - - let response: web_sys::Response = response.dyn_into().map_err(|error| Error::JsError(error))?; - - Ok(response) -} - -#[cfg(test)] -mod tests { - use wasm_bindgen_test::wasm_bindgen_test; - - wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); - - use crate::{dns, Keypair, PkarrRelayClient, SignedPacket}; - - #[macro_export] - macro_rules! log { - ($($arg:expr),*) => { - web_sys::console::debug_1(&format!($($arg),*).into()); - }; - } - - const TEST_RELAY: &str = "http://localhost:6881"; - - #[wasm_bindgen_test] - async fn basic() { - let keypair = Keypair::random(); - - let mut packet = dns::Packet::new_reply(0); - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("foo").unwrap(), - dns::CLASS::IN, - 30, - dns::rdata::RData::TXT("bar".try_into().unwrap()), - )); - - let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); - - let client = - PkarrRelayClient::new(vec!["http://fail.non".to_string(), TEST_RELAY.to_string()]) - .unwrap(); - - client.publish(&signed_packet).await.unwrap(); - - let resolved = client - .resolve(&keypair.public_key()) - .await - .unwrap() - .unwrap(); - - log!("{:?}", resolved); - - assert_eq!(resolved.as_bytes(), signed_packet.as_bytes()); - } - - #[wasm_bindgen_test] - async fn not_found() { - let keypair = Keypair::random(); - - let client = PkarrRelayClient::new(vec![TEST_RELAY.to_string()]).unwrap(); - - let resolved = client.resolve(&keypair.public_key()).await.unwrap(); - - log!("{:?}", resolved); - - assert!(resolved.is_none()); - } -} diff --git a/pkarr/src/signed_packet.rs b/pkarr/src/signed_packet.rs index 842f84e..a54b3e5 100644 --- a/pkarr/src/signed_packet.rs +++ b/pkarr/src/signed_packet.rs @@ -65,6 +65,8 @@ pub struct SignedPacket { } impl SignedPacket { + pub const MAX_BYTES: usize = 1104; + /// Creates a [Self] from the serialized representation: /// `<32 bytes public_key><64 bytes signature><8 bytes big-endian timestamp in microseconds>` /// @@ -80,7 +82,7 @@ impl SignedPacket { /// /// # Errors /// - Returns [crate::Error::InvalidSignedPacketBytesLength] if `bytes.len()` is smaller than 104 bytes - /// - Returns [crate::Error::PacketTooLarge] if `bytes.len()` is bigger than 1104 bytes + /// - Returns [crate::Error::PacketTooLarge] if `bytes.len()` is bigger than [SignedPacket::MAX_BYTES] bytes /// - Returns [crate::Error::InvalidEd25519PublicKey] if the first 32 bytes are invalid `ed25519` public key /// - Returns [crate::Error::InvalidEd25519Signature] if the following 64 bytes are invalid `ed25519` signature /// - Returns [crate::Error::DnsError] if it failed to parse the DNS Packet after the first 104 bytes @@ -88,7 +90,7 @@ impl SignedPacket { if bytes.len() < 104 { return Err(Error::InvalidSignedPacketBytesLength(bytes.len())); } - if bytes.len() > 1104 { + if bytes.len() > SignedPacket::MAX_BYTES { return Err(Error::PacketTooLarge(bytes.len())); } let public_key = PublicKey::try_from(&bytes[..32])?; From 17a0b0532067ec9e0e0674689f07d6f5d209a72c Mon Sep 17 00:00:00 2001 From: nazeh Date: Sat, 21 Sep 2024 14:31:48 +0300 Subject: [PATCH 10/84] refactor(pkarr): move racing logic in relay client to separate methods --- pkarr/src/relay_client.rs | 231 ++++++++++++++++++++++---------------- 1 file changed, 134 insertions(+), 97 deletions(-) diff --git a/pkarr/src/relay_client.rs b/pkarr/src/relay_client.rs index a5ac040..700828c 100644 --- a/pkarr/src/relay_client.rs +++ b/pkarr/src/relay_client.rs @@ -6,7 +6,7 @@ use std::{ }; use lru::LruCache; -use reqwest::{Client, StatusCode}; +use reqwest::{Client, Response, StatusCode}; use tokio::task::JoinSet; use tracing::debug; @@ -105,25 +105,38 @@ impl PkarrRelayClient { cache.put(public_key.to_owned(), signed_packet.clone()); } + self.race_publish(signed_packet).await + } + + /// Resolve a [SignedPacket] from this client's relays. + /// + /// Return the first successful response, or the failure from the last responding relay. + /// + /// # Errors + /// + /// - Returns [Error::RelayError] if the relay responded with a status >= 400 + /// (except 404 in which case you should receive Ok(None)) or something wrong + /// with the transport, transparent from [reqwest::Error]. + /// - Returns [Error::IO] if something went wrong while reading the payload. + pub async fn resolve(&self, public_key: &PublicKey) -> Result> { + let cached_packet = match self.get_from_cache(public_key) { + None => None, + Some(signed_packet) => return Ok(Some(signed_packet)), + }; + + self.race_resolve(public_key, cached_packet).await + } + + // === Native Race implementation === + + async fn race_publish(&self, signed_packet: &SignedPacket) -> Result<()> { let mut futures = JoinSet::new(); for relay in self.relays.clone() { - let url = format!("{relay}/{public_key}"); - let http_client = self.http_client.clone(); let signed_packet = signed_packet.clone(); + let this = self.clone(); - futures.spawn(async move { - http_client - .put(&url) - .body(signed_packet.to_relay_payload()) - .send() - .await - .map_err(|error| { - debug!(?url, ?error, "Error response"); - - error - }) - }); + futures.spawn(async move { this.publish_to_relay(relay, signed_packet).await }); } let mut last_error = Error::EmptyListOfRelays; @@ -131,7 +144,7 @@ impl PkarrRelayClient { while let Some(result) = futures.join_next().await { match result { Ok(Ok(_)) => return Ok(()), - Ok(Err(error)) => last_error = Error::RelayError(error), + Ok(Err(error)) => last_error = error, Err(joinerror) => { debug!(?joinerror); } @@ -141,49 +154,19 @@ impl PkarrRelayClient { Err(last_error) } - /// Resolve a [SignedPacket] from this client's relays. - /// - /// Return the first successful response, or the failure from the last responding relay. - /// - /// # Errors - /// - /// - Returns [Error::RelayError] if the relay responded with a status >= 400 - /// (except 404 in which case you should receive Ok(None)) or something wrong - /// with the transport, transparent from [reqwest::Error]. - /// - Returns [Error::IO] if something went wrong while reading the payload. - pub async fn resolve(&self, public_key: &PublicKey) -> Result> { - let cached_packet = { - let mut cache = self.cache.lock().unwrap(); - - let cached_packet = cache.get(public_key); - - if let Some(cached) = cached_packet { - let expires_in = cached.expires_in(self.minimum_ttl, self.maximum_ttl); - - if expires_in > 0 { - debug!(expires_in, "Have fresh signed_packet in cache."); - - return Ok(Some(cached.clone())); - } - - debug!(expires_in, "Have expired signed_packet in cache."); - } else { - debug!("Cache miss"); - }; - - None - }; - + async fn race_resolve( + &self, + public_key: &PublicKey, + cached_packet: Option, + ) -> Result> { let mut futures = JoinSet::new(); for relay in self.relays.clone() { - let http_client = self.http_client.clone(); let public_key = public_key.clone(); let cached = cached_packet.clone(); + let this = self.clone(); - futures.spawn(async move { - resolve_from_relay(relay, public_key, http_client, cached).await - }); + futures.spawn(async move { this.resolve_from_relay(relay, public_key, cached).await }); } let mut result: Result> = Ok(None); @@ -207,62 +190,116 @@ impl PkarrRelayClient { result } -} -async fn resolve_from_relay( - relay: String, - public_key: PublicKey, - http_client: Client, - cached_packet: Option, -) -> Result> { - let url = format!("{relay}/{public_key}"); - - match http_client.get(&url).send().await { - Ok(mut response) => { - if response.status() == StatusCode::NOT_FOUND { - debug!(?url, "SignedPacket not found"); - return Ok(None); - } + // === Private Methods === + + async fn publish_to_relay( + &self, + relay: String, + signed_packet: SignedPacket, + ) -> Result { + let url = format!("{relay}/{}", signed_packet.public_key()); + + self.http_client + .put(&url) + .body(signed_packet.to_relay_payload()) + .send() + .await + .map_err(|error| { + debug!(?url, ?error, "Error response"); + + Error::RelayError(error) + }) + } - let mut total_size = 0; - let mut payload = Vec::new(); + fn get_from_cache(&self, public_key: &PublicKey) -> Option { + let mut cache = self.cache.lock().unwrap(); - while let Ok(Some(chunk)) = response.chunk().await { - total_size += chunk.len(); - payload.extend_from_slice(&chunk); - if total_size >= SignedPacket::MAX_BYTES { - break; - } + let cached_packet = cache.get(public_key); + + if let Some(cached) = cached_packet { + let expires_in = cached.expires_in(self.minimum_ttl, self.maximum_ttl); + + if expires_in > 0 { + debug!(expires_in, "Have fresh signed_packet in cache."); + + return Some(cached.clone()); } - match SignedPacket::from_relay_payload(&public_key, &payload.into()) { - Ok(signed_packet) => { - let new_packet = if let Some(ref cached) = cached_packet { - if signed_packet.more_recent_than(cached) { - debug!(?public_key, "Received more recent packet than in cache"); - Some(signed_packet) - } else { - None - } - } else { - debug!(?public_key, "Received new packet after cache miss"); - Some(signed_packet) - }; - - Ok(new_packet) + debug!(expires_in, "Have expired signed_packet in cache."); + } else { + debug!("Cache miss"); + }; + + None + } + + async fn resolve_from_relay( + &self, + relay: String, + public_key: PublicKey, + cached_packet: Option, + ) -> Result> { + let url = format!("{relay}/{public_key}"); + + match self.http_client.get(&url).send().await { + Ok(mut response) => { + if response.status() == StatusCode::NOT_FOUND { + debug!(?url, "SignedPacket not found"); + return Ok(None); } - Err(error) => { - debug!(?url, ?error, "Invalid signed_packet"); - Err(error) + let payload = read(&mut response).await; + + match SignedPacket::from_relay_payload(&public_key, &payload.into()) { + Ok(signed_packet) => Ok(choose_most_recent(signed_packet, cached_packet)), + Err(error) => { + debug!(?url, ?error, "Invalid signed_packet"); + + Err(error) + } } } + Err(error) => { + debug!(?url, ?error, "Error response"); + + Err(Error::RelayError(error)) + } + } + } +} + +async fn read(response: &mut Response) -> Vec { + let mut total_size = 0; + let mut payload = Vec::new(); + + while let Ok(Some(chunk)) = response.chunk().await { + total_size += chunk.len(); + payload.extend_from_slice(&chunk); + if total_size >= SignedPacket::MAX_BYTES { + break; } - Err(error) => { - debug!(?url, ?error, "Error response"); + } + payload +} - Err(Error::RelayError(error)) +fn choose_most_recent( + signed_packet: SignedPacket, + cached_packet: Option, +) -> Option { + if let Some(ref cached) = cached_packet { + if signed_packet.more_recent_than(cached) { + debug!( + public_key = ?signed_packet.public_key(), + "Received more recent packet than in cache" + ); + Some(signed_packet) + } else { + None } + } else { + debug!(public_key= ?signed_packet.public_key(), "Received new packet after cache miss"); + Some(signed_packet) } } From 8de0ee6bc5300b071ada20761ac69ea4a75ced09 Mon Sep 17 00:00:00 2001 From: nazeh Date: Sat, 21 Sep 2024 20:33:11 +0300 Subject: [PATCH 11/84] feat(pkarr): rename all clients to Client, and make async first class --- Cargo.lock | 2 +- pkarr/Cargo.toml | 18 +-- pkarr/examples/publish.rs | 9 +- pkarr/examples/resolve.rs | 15 +- pkarr/src/client_async.rs | 138 ---------------- pkarr/src/{client.rs => dht.rs} | 199 +++++++++++++++++++----- pkarr/src/error.rs | 2 +- pkarr/src/lib.rs | 38 +---- pkarr/src/{relay_client.rs => relay.rs} | 87 ++++++++--- server/Cargo.toml | 2 +- server/src/http_server.rs | 6 +- server/src/main.rs | 4 +- 12 files changed, 268 insertions(+), 252 deletions(-) delete mode 100644 pkarr/src/client_async.rs rename pkarr/src/{client.rs => dht.rs} (72%) rename pkarr/src/{relay_client.rs => relay.rs} (82%) diff --git a/Cargo.lock b/Cargo.lock index 300d358..a565f79 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1461,7 +1461,7 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkarr" -version = "2.2.0" +version = "3.0.0" dependencies = [ "base32", "bytes", diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index e3b9dce..8042dfb 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pkarr" -version = "2.2.0" +version = "3.0.0" authors = ["Nuh "] edition = "2021" description = "Public-Key Addressable Resource Records (Pkarr); publish and resolve DNS records over Mainline DHT" @@ -27,7 +27,7 @@ document-features = "0.2.8" # Dht client dependencies: mainline = { version = "2.0.1", optional = true } dyn-clone = { version = "1.0.17", optional = true } -flume = { version = "0.11.0", features = ["select", "eventual-fairness"], default-features = false , optional = true } +flume = { version = "0.11.0", features = ["select", "eventual-fairness", "async"], default-features = false , optional = true } tokio = { version = "1.40.0", optional = true } [target.'cfg(target_arch = "wasm32")'.dependencies] @@ -35,7 +35,7 @@ js-sys = "0.3.69" [dev-dependencies] postcard = { version = "1.0.10", features = ["alloc"] } -tokio = { version = "1.40.0", features = ["macros"] } +tokio = { version = "1.40.0", features = ["macros", "rt-multi-thread"] } [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] clap = { version = "4.4.8", features = ["derive"] } @@ -47,15 +47,15 @@ tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } rand = ["dep:rand", "ed25519-dalek/rand_core"] ## Derive serde Serialize/Deserialize for PublicKey serde = ["dep:serde"] -## Use [PkarrClient] -dht = ["dep:mainline", "dep:dyn-clone"] -## Use async versions of [PkarrClient] -async = ["flume/async"] -## Use [PkarrRelayClient] + +# Clients +## Use [dht::Client] +dht = ["dep:mainline", "dep:dyn-clone", "flume"] +## Use [relay::Client] relay = ["dep:reqwest", "dep:tokio"] ## Use all features -full = ["dht", "async", "relay", "rand", "serde"] +full = ["dht", "relay", "rand", "serde"] default = ["full"] diff --git a/pkarr/examples/publish.rs b/pkarr/examples/publish.rs index 99bc2c7..c32ba82 100644 --- a/pkarr/examples/publish.rs +++ b/pkarr/examples/publish.rs @@ -11,14 +11,15 @@ use tracing_subscriber; use std::time::Instant; -use pkarr::{dns, Keypair, PkarrClient, Result, SignedPacket}; +use pkarr::{dns, Client, Keypair, Result, SignedPacket}; -fn main() -> Result<()> { +#[tokio::main] +async fn main() -> Result<()> { tracing_subscriber::fmt() .with_max_level(Level::DEBUG) .init(); - let client = PkarrClient::builder().build().unwrap(); + let client = Client::builder().build()?; let keypair = Keypair::random(); @@ -36,7 +37,7 @@ fn main() -> Result<()> { println!("\nPublishing {} ...", keypair.public_key()); - match client.publish(&signed_packet) { + match client.publish(&signed_packet).await { Ok(()) => { println!( "\nSuccessfully published {} in {:?}", diff --git a/pkarr/examples/resolve.rs b/pkarr/examples/resolve.rs index 1a1f2bc..4902bee 100644 --- a/pkarr/examples/resolve.rs +++ b/pkarr/examples/resolve.rs @@ -11,7 +11,7 @@ use std::{ time::{Duration, Instant}, }; -use pkarr::{PkarrClient, PublicKey}; +use pkarr::{Client, PublicKey}; use clap::Parser; @@ -22,7 +22,8 @@ struct Cli { public_key: String, } -fn main() { +#[tokio::main] +async fn main() { tracing_subscriber::fmt() .with_max_level(Level::DEBUG) .with_env_filter("pkarr") @@ -36,23 +37,23 @@ fn main() { .try_into() .expect("Invalid zbase32 encoded key"); - let client = PkarrClient::builder().build().unwrap(); + let client = Client::builder().build().unwrap(); println!("Resolving Pkarr: {} ...", cli.public_key); println!("\n=== COLD LOOKUP ==="); - resolve(&client, &public_key); + resolve(&client, &public_key).await; // loop { sleep(Duration::from_secs(1)); println!("=== SUBSEQUENT LOOKUP ==="); - resolve(&client, &public_key) + resolve(&client, &public_key).await // } } -fn resolve(client: &PkarrClient, public_key: &PublicKey) { +async fn resolve(client: &Client, public_key: &PublicKey) { let start = Instant::now(); - match client.resolve(public_key) { + match client.resolve(public_key).await { Ok(Some(signed_packet)) => { println!( "\nResolved in {:?} milliseconds {}", diff --git a/pkarr/src/client_async.rs b/pkarr/src/client_async.rs deleted file mode 100644 index 5586b79..0000000 --- a/pkarr/src/client_async.rs +++ /dev/null @@ -1,138 +0,0 @@ -//! Async version of [PkarrClient] - -use std::net::SocketAddr; - -use super::{ - cache::PkarrCache, - client::{ActorMessage, PkarrClient}, -}; -use crate::{Error, PublicKey, Result, SignedPacket}; - -#[derive(Clone, Debug)] -/// Async version of [PkarrClient] -pub struct PkarrClientAsync(PkarrClient); - -impl PkarrClient { - /// Returns [PkarrClientAsync] - pub fn as_async(self) -> PkarrClientAsync { - PkarrClientAsync(self) - } -} - -impl PkarrClientAsync { - // === Getters === - - /// Returns the local address of the udp socket this node is listening on. - /// - /// Returns `None` if the node is shutdown - pub fn local_addr(&self) -> Option { - self.0.address - } - - /// Returns a reference to the internal cache. - pub fn cache(&self) -> &dyn PkarrCache { - self.0.cache.as_ref() - } - - // === Public Methods === - - /// Publishes a [SignedPacket] to the Dht. - /// - /// # Errors - /// - Returns a [Error::DhtIsShutdown] if [PkarrClient::shutdown] was called, or - /// the loop in the actor thread is stopped for any reason (like thread panic). - /// - Returns a [Error::PublishInflight] if the client is currently publishing the same public_key. - /// - Returns a [Error::NotMostRecent] if the provided signed packet is older than most recent. - /// - Returns a [Error::MainlineError] if the Dht received an unexpected error otherwise. - pub async fn publish(&self, signed_packet: &SignedPacket) -> Result<()> { - match self.0.publish_inner(signed_packet)?.recv_async().await { - Ok(Ok(_)) => Ok(()), - Ok(Err(error)) => match error { - mainline::Error::PutQueryIsInflight(_) => Err(Error::PublishInflight), - _ => Err(Error::MainlineError(error)), - }, - // Since we pass this sender to `Rpc::put`, the only reason the sender, - // would be dropped, is if `Rpc` is dropped, which should only happeng on shutdown. - Err(_) => Err(Error::DhtIsShutdown), - } - } - - /// Returns the first valid [SignedPacket] available from cache, or the Dht. - /// - /// If the Dht was called, in the background, it continues receiving responses - /// and updating the cache. - /// - /// # Errors - /// - Returns a [Error::DhtIsShutdown] if [PkarrClient::shutdown] was called, or - /// the loop in the actor thread is stopped for any reason (like thread panic). - pub async fn resolve(&self, public_key: &PublicKey) -> Result> { - Ok(self.0.resolve_inner(public_key)?.recv_async().await.ok()) - } - - /// Shutdown the actor thread loop. - pub async fn shutdown(&mut self) -> Result<()> { - let (sender, receiver) = flume::bounded(1); - - self.0 - .sender - .send(ActorMessage::Shutdown(sender)) - .map_err(|_| Error::DhtIsShutdown)?; - - receiver.recv_async().await?; - - self.0.address = None; - - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use mainline::Testnet; - - use super::*; - use crate::{dns, Keypair, SignedPacket}; - - #[test] - fn shutdown() { - let testnet = Testnet::new(3); - - let mut a = PkarrClient::builder().testnet(&testnet).build().unwrap(); - - assert_ne!(a.local_addr(), None); - - a.shutdown().unwrap(); - - assert_eq!(a.local_addr(), None); - } - - #[tokio::test] - async fn publish_resolve() { - let testnet = Testnet::new(10); - - let a = PkarrClient::builder().testnet(&testnet).build().unwrap(); - - let keypair = Keypair::random(); - - let mut packet = dns::Packet::new_reply(0); - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("foo").unwrap(), - dns::CLASS::IN, - 30, - dns::rdata::RData::TXT("bar".try_into().unwrap()), - )); - - let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); - - let _ = a.publish(&signed_packet); - - let b = PkarrClient::builder().testnet(&testnet).build().unwrap(); - - let resolved = b.resolve(&keypair.public_key()).unwrap().unwrap(); - assert_eq!(resolved.as_bytes(), signed_packet.as_bytes()); - - let from_cache = b.resolve(&keypair.public_key()).unwrap().unwrap(); - assert_eq!(from_cache.as_bytes(), signed_packet.as_bytes()); - assert_eq!(from_cache.last_seen(), resolved.last_seen()); - } -} diff --git a/pkarr/src/client.rs b/pkarr/src/dht.rs similarity index 72% rename from pkarr/src/client.rs rename to pkarr/src/dht.rs index d4f1f4c..d967523 100644 --- a/pkarr/src/client.rs +++ b/pkarr/src/dht.rs @@ -24,7 +24,7 @@ use crate::{ use crate::{Error, PublicKey, Result, SignedPacket}; #[derive(Debug)] -/// [PkarrClient]'s settings +/// [Client]'s settings pub struct Settings { pub dht: DhtSettings, /// A set of [resolver](https://pkarr.org/resolvers)s @@ -68,12 +68,12 @@ impl Default for Settings { } #[derive(Debug, Default)] -/// Builder for [PkarrClient] -pub struct PkarrClientBuilder { +/// Builder for [Client] +pub struct ClientBuilder { settings: Settings, } -impl PkarrClientBuilder { +impl ClientBuilder { /// Set custom set of [resolvers](Settings::resolvers). pub fn resolvers(mut self, resolvers: Option>) -> Self { self.settings.resolvers = resolvers.map(|resolvers| { @@ -130,14 +130,14 @@ impl PkarrClientBuilder { self } - pub fn build(self) -> Result { - PkarrClient::new(self.settings) + pub fn build(self) -> Result { + Client::new(self.settings) } } #[derive(Clone, Debug)] /// Pkarr client for publishing and resolving [SignedPacket]s over [mainline]. -pub struct PkarrClient { +pub struct Client { pub(crate) address: Option, pub(crate) sender: Sender, pub(crate) cache: Box, @@ -145,8 +145,8 @@ pub struct PkarrClient { pub(crate) maximum_ttl: u32, } -impl PkarrClient { - pub fn new(settings: Settings) -> Result { +impl Client { + pub fn new(settings: Settings) -> Result { let (sender, receiver) = flume::bounded(32); let rpc = Rpc::new(&settings.dht)?; @@ -159,7 +159,7 @@ impl PkarrClient { .unwrap_or(Box::new(InMemoryPkarrCache::new(settings.cache_size))); let cache_clone = cache.clone(); - let client = PkarrClient { + let client = Client { address: Some(local_addr), sender, cache, @@ -168,15 +168,15 @@ impl PkarrClient { }; thread::Builder::new() - .name("PkarrClient loop".to_string()) + .name("Client loop".to_string()) .spawn(move || run(rpc, cache_clone, settings, receiver))?; Ok(client) } - /// Returns a builder to edit settings before creating PkarrClient. - pub fn builder() -> PkarrClientBuilder { - PkarrClientBuilder::default() + /// Returns a builder to edit settings before creating Client. + pub fn builder() -> ClientBuilder { + ClientBuilder::default() } // === Getters === @@ -198,12 +198,64 @@ impl PkarrClient { /// Publishes a [SignedPacket] to the Dht. /// /// # Errors - /// - Returns a [Error::DhtIsShutdown] if [PkarrClient::shutdown] was called, or + /// - Returns a [Error::DhtIsShutdown] if [Client::shutdown] was called, or /// the loop in the actor thread is stopped for any reason (like thread panic). /// - Returns a [Error::PublishInflight] if the client is currently publishing the same public_key. /// - Returns a [Error::NotMostRecent] if the provided signed packet is older than most recent. /// - Returns a [Error::MainlineError] if the Dht received an unexpected error otherwise. - pub fn publish(&self, signed_packet: &SignedPacket) -> Result<()> { + pub async fn publish(&self, signed_packet: &SignedPacket) -> Result<()> { + match self.publish_inner(signed_packet)?.recv_async().await { + Ok(Ok(_)) => Ok(()), + Ok(Err(error)) => match error { + mainline::Error::PutQueryIsInflight(_) => Err(Error::PublishInflight), + _ => Err(Error::MainlineError(error)), + }, + // Since we pass this sender to `Rpc::put`, the only reason the sender, + // would be dropped, is if `Rpc` is dropped, which should only happeng on shutdown. + Err(_) => Err(Error::DhtIsShutdown), + } + } + + /// Returns a [SignedPacket] from cache if it is not expired, otherwise, + /// it will query the Dht, and return the first valid response, which may + /// or may not be expired itself. + /// + /// If the Dht was called, in the background, it continues receiving responses + /// and updating the cache with any more recent valid packets it receives. + /// + /// # Errors + /// - Returns a [Error::DhtIsShutdown] if [Client::shutdown] was called, or + /// the loop in the actor thread is stopped for any reason (like thread panic). + pub async fn resolve(&self, public_key: &PublicKey) -> Result> { + Ok(self.resolve_inner(public_key)?.recv_async().await.ok()) + } + + /// Shutdown the actor thread loop. + pub async fn shutdown(&mut self) -> Result<()> { + let (sender, receiver) = flume::bounded(1); + + self.sender + .send(ActorMessage::Shutdown(sender)) + .map_err(|_| Error::DhtIsShutdown)?; + + receiver.recv_async().await?; + + self.address = None; + + Ok(()) + } + + // === Sync === + + /// Publishes a [SignedPacket] to the Dht. + /// + /// # Errors + /// - Returns a [Error::DhtIsShutdown] if [Client::shutdown] was called, or + /// the loop in the actor thread is stopped for any reason (like thread panic). + /// - Returns a [Error::PublishInflight] if the client is currently publishing the same public_key. + /// - Returns a [Error::NotMostRecent] if the provided signed packet is older than most recent. + /// - Returns a [Error::MainlineError] if the Dht received an unexpected error otherwise. + pub fn publish_sync(&self, signed_packet: &SignedPacket) -> Result<()> { match self.publish_inner(signed_packet)?.recv() { Ok(Ok(_)) => Ok(()), Ok(Err(error)) => match error { @@ -224,14 +276,14 @@ impl PkarrClient { /// and updating the cache with any more recent valid packets it receives. /// /// # Errors - /// - Returns a [Error::DhtIsShutdown] if [PkarrClient::shutdown] was called, or + /// - Returns a [Error::DhtIsShutdown] if [Client::shutdown] was called, or /// the loop in the actor thread is stopped for any reason (like thread panic). - pub fn resolve(&self, public_key: &PublicKey) -> Result> { + pub fn resolve_sync(&self, public_key: &PublicKey) -> Result> { Ok(self.resolve_inner(public_key)?.recv().ok()) } /// Shutdown the actor thread loop. - pub fn shutdown(&mut self) -> Result<()> { + pub fn shutdown_sync(&mut self) -> Result<()> { let (sender, receiver) = flume::bounded(1); self.sender @@ -316,7 +368,7 @@ fn run( settings: Settings, receiver: Receiver, ) { - debug!(?settings, "Starting PkarrClient main loop.."); + debug!(?settings, "Starting Client main loop.."); let mut server = settings.dht.server; let mut senders: HashMap>> = HashMap::new(); @@ -454,7 +506,7 @@ fn run( } } - debug!("PkarrClient main terminated"); + debug!("Client main terminated"); } pub enum ActorMessage { @@ -471,23 +523,23 @@ mod tests { use crate::{dns, Keypair, SignedPacket}; #[test] - fn shutdown() { + fn shutdown_sync() { let testnet = Testnet::new(3); - let mut a = PkarrClient::builder().testnet(&testnet).build().unwrap(); + let mut a = Client::builder().testnet(&testnet).build().unwrap(); assert_ne!(a.local_addr(), None); - a.shutdown().unwrap(); + a.shutdown_sync().unwrap(); assert_eq!(a.local_addr(), None); } #[test] - fn publish_resolve() { + fn publish_resolve_sync() { let testnet = Testnet::new(10); - let a = PkarrClient::builder().testnet(&testnet).build().unwrap(); + let a = Client::builder().testnet(&testnet).build().unwrap(); let keypair = Keypair::random(); @@ -501,23 +553,23 @@ mod tests { let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); - let _ = a.publish(&signed_packet); + a.publish_sync(&signed_packet).unwrap(); - let b = PkarrClient::builder().testnet(&testnet).build().unwrap(); + let b = Client::builder().testnet(&testnet).build().unwrap(); - let resolved = b.resolve(&keypair.public_key()).unwrap().unwrap(); + let resolved = b.resolve_sync(&keypair.public_key()).unwrap().unwrap(); assert_eq!(resolved.as_bytes(), signed_packet.as_bytes()); - let from_cache = b.resolve(&keypair.public_key()).unwrap().unwrap(); + let from_cache = b.resolve_sync(&keypair.public_key()).unwrap().unwrap(); assert_eq!(from_cache.as_bytes(), signed_packet.as_bytes()); assert_eq!(from_cache.last_seen(), resolved.last_seen()); } #[test] - fn thread_safe() { + fn thread_safe_sync() { let testnet = Testnet::new(10); - let a = PkarrClient::builder().testnet(&testnet).build().unwrap(); + let a = Client::builder().testnet(&testnet).build().unwrap(); let keypair = Keypair::random(); @@ -531,19 +583,94 @@ mod tests { let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); - let _ = a.publish(&signed_packet); + a.publish_sync(&signed_packet).unwrap(); - let b = PkarrClient::builder().testnet(&testnet).build().unwrap(); + let b = Client::builder().testnet(&testnet).build().unwrap(); thread::spawn(move || { - let resolved = b.resolve(&keypair.public_key()).unwrap().unwrap(); + let resolved = b.resolve_sync(&keypair.public_key()).unwrap().unwrap(); assert_eq!(resolved.as_bytes(), signed_packet.as_bytes()); - let from_cache = b.resolve(&keypair.public_key()).unwrap().unwrap(); + let from_cache = b.resolve_sync(&keypair.public_key()).unwrap().unwrap(); assert_eq!(from_cache.as_bytes(), signed_packet.as_bytes()); assert_eq!(from_cache.last_seen(), resolved.last_seen()); }) .join() .unwrap(); } + + #[tokio::test] +async fn shutdown() { + let testnet = Testnet::new(3); + + let mut a = Client::builder().testnet(&testnet).build().unwrap(); + + assert_ne!(a.local_addr(), None); + + a.shutdown().await.unwrap(); + + assert_eq!(a.local_addr(), None); + } + + #[tokio::test] + async fn publish_resolve() { + let testnet = Testnet::new(10); + + let a = Client::builder().testnet(&testnet).build().unwrap(); + + let keypair = Keypair::random(); + + let mut packet = dns::Packet::new_reply(0); + packet.answers.push(dns::ResourceRecord::new( + dns::Name::new("foo").unwrap(), + dns::CLASS::IN, + 30, + dns::rdata::RData::TXT("bar".try_into().unwrap()), + )); + + let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); + + a.publish(&signed_packet).await.unwrap(); + + let b = Client::builder().testnet(&testnet).build().unwrap(); + + let resolved = b.resolve_sync(&keypair.public_key()).unwrap().unwrap(); + assert_eq!(resolved.as_bytes(), signed_packet.as_bytes()); + + let from_cache = b.resolve_sync(&keypair.public_key()).unwrap().unwrap(); + assert_eq!(from_cache.as_bytes(), signed_packet.as_bytes()); + assert_eq!(from_cache.last_seen(), resolved.last_seen()); + } + + #[tokio::test] + async fn thread_safe() { + let testnet = Testnet::new(10); + + let a = Client::builder().testnet(&testnet).build().unwrap(); + + let keypair = Keypair::random(); + + let mut packet = dns::Packet::new_reply(0); + packet.answers.push(dns::ResourceRecord::new( + dns::Name::new("foo").unwrap(), + dns::CLASS::IN, + 30, + dns::rdata::RData::TXT("bar".try_into().unwrap()), + )); + + let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); + + a.publish(&signed_packet).await.unwrap(); + + let b = Client::builder().testnet(&testnet).build().unwrap(); + + tokio::spawn( async move { + let resolved = b.resolve(&keypair.public_key()).await.unwrap().unwrap(); + assert_eq!(resolved.as_bytes(), signed_packet.as_bytes()); + + let from_cache = b.resolve(&keypair.public_key()).await.unwrap().unwrap(); + assert_eq!(from_cache.as_bytes(), signed_packet.as_bytes()); + assert_eq!(from_cache.last_seen(), resolved.last_seen()); + }).await.unwrap(); + } } diff --git a/pkarr/src/error.rs b/pkarr/src/error.rs index 9f22f32..63b052a 100644 --- a/pkarr/src/error.rs +++ b/pkarr/src/error.rs @@ -57,7 +57,7 @@ pub enum Error { DhtIsShutdown, #[error("Publish query is already inflight for the same public_key")] - /// [crate::PkarrClient::publish] is already inflight to the same public_key + /// [crate::dht::Client::publish] is already inflight to the same public_key PublishInflight, #[error("SignedPacket's timestamp is not the most recent")] diff --git a/pkarr/src/lib.rs b/pkarr/src/lib.rs index 09daa97..78375a6 100644 --- a/pkarr/src/lib.rs +++ b/pkarr/src/lib.rs @@ -26,47 +26,25 @@ pub const DEFAULT_MINIMUM_TTL: u32 = 300; pub const DEFAULT_MAXIMUM_TTL: u32 = 24 * 60 * 60; /// Default cache size: 1000 pub const DEFAULT_CACHE_SIZE: usize = 1000; - -pub const DEFAULT_RELAYS: [&str; 1] = ["https://relay.pkarr.org"]; - -pub const DEFAULT_RESOLVERS: [&str; 1] = ["resolver.pkarr.org:6881"]; +/// Default [relay](https://pkarr.org/relays)s +pub const DEFAULT_RELAYS: [&str; 2] = ["https://relay.pkarr.org", "https://pkarr.pubky.app"]; +/// Default [resolver](https://pkarr.org/resolvers)s +pub const DEFAULT_RESOLVERS: [&str; 2] = ["resolver.pkarr.org:6881", "pkarr.pubky.app:6881"]; // Rexports pub use bytes; pub use simple_dns as dns; -#[cfg(not(target_arch = "wasm32"))] -macro_rules! if_async { - ($($item:item)*) => {$( - #[cfg(all(not(target_arch = "wasm32"), feature = "async"))] - $item - )*} -} - -macro_rules! if_relay { - ($($item:item)*) => {$( - #[cfg(all(not(target_arch = "wasm32"), feature = "relay"))] - $item - )*} -} - if_dht! { mod cache; - mod client; - - if_async! { - mod client_async; - pub use client_async::PkarrClientAsync; - } + mod dht; - pub use client::{PkarrClientBuilder, PkarrClient, Settings}; + pub use dht::{ClientBuilder, Client, Settings}; pub use cache::{PkarrCache, PkarrCacheKey, InMemoryPkarrCache}; // Rexports pub use mainline; } -if_relay! { - mod relay_client; - pub use relay_client::{PkarrRelayClient, RelaySettings}; -} +#[cfg(feature = "relay")] +pub mod relay; diff --git a/pkarr/src/relay_client.rs b/pkarr/src/relay.rs similarity index 82% rename from pkarr/src/relay_client.rs rename to pkarr/src/relay.rs index 700828c..052b12e 100644 --- a/pkarr/src/relay_client.rs +++ b/pkarr/src/relay.rs @@ -6,18 +6,18 @@ use std::{ }; use lru::LruCache; -use reqwest::{Client, Response, StatusCode}; +use reqwest::{Response, StatusCode}; use tokio::task::JoinSet; use tracing::debug; use crate::{ - Error, PublicKey, Result, SignedPacket, DEFAULT_CACHE_SIZE, DEFAULT_MAXIMUM_TTL, + Error, PkarrCache, PublicKey, Result, SignedPacket, DEFAULT_CACHE_SIZE, DEFAULT_MAXIMUM_TTL, DEFAULT_MINIMUM_TTL, DEFAULT_RELAYS, }; #[derive(Debug, Clone)] -/// [PkarrRelayClient]'s settings -pub struct RelaySettings { +/// [Client]'s settings +pub struct Settings { pub relays: Vec, /// Defaults to [DEFAULT_CACHE_SIZE] pub cache_size: NonZeroUsize, @@ -30,39 +30,86 @@ pub struct RelaySettings { /// Defaults to [DEFAULT_MAXIMUM_TTL] pub maximum_ttl: u32, /// Custom [reqwest::Client] - pub http_client: Client, + pub http_client: reqwest::Client, + /// Custom [PkarrCache] implementation, defaults to [InMemoryPkarrCache] + pub cache: Option>, } -impl Default for RelaySettings { +impl Default for Settings { fn default() -> Self { Self { relays: DEFAULT_RELAYS.map(|s| s.into()).to_vec(), cache_size: NonZeroUsize::new(DEFAULT_CACHE_SIZE).unwrap(), minimum_ttl: DEFAULT_MINIMUM_TTL, maximum_ttl: DEFAULT_MAXIMUM_TTL, - http_client: Client::new(), + http_client: reqwest::Client::new(), + cache: None, } } } +#[derive(Debug, Default)] +/// Builder for [Client] +pub struct ClientBuilder { + settings: Settings, +} + +impl ClientBuilder { + /// Set the [Settings::cache_size]. + /// + /// Controls the capacity of [PkarrCache]. + pub fn cache_size(mut self, cache_size: NonZeroUsize) -> Self { + self.settings.cache_size = cache_size; + self + } + + /// Set the [Settings::minimum_ttl] value. + /// + /// Limits how soon a [SignedPacket] is considered expired. + pub fn minimum_ttl(mut self, ttl: u32) -> Self { + self.settings.minimum_ttl = ttl; + self.settings.maximum_ttl = self.settings.maximum_ttl.clamp(ttl, u32::MAX); + self + } + + /// Set the [Settings::maximum_ttl] value. + /// + /// Limits how long it takes before a [SignedPacket] is considered expired. + pub fn maximum_ttl(mut self, ttl: u32) -> Self { + self.settings.maximum_ttl = ttl; + self.settings.minimum_ttl = self.settings.minimum_ttl.clamp(0, ttl); + self + } + + /// Set a custom implementation of [PkarrCache]. + pub fn cache(mut self, cache: Box) -> Self { + self.settings.cache = Some(cache); + self + } + + pub fn build(self) -> Result { + Client::new(self.settings) + } +} + #[derive(Debug, Clone)] /// Pkarr client for publishing and resolving [SignedPacket]s over [relays](https://pkarr.org/relays). -pub struct PkarrRelayClient { - http_client: Client, +pub struct Client { + http_client: reqwest::Client, relays: Vec, cache: Arc>>, minimum_ttl: u32, maximum_ttl: u32, } -impl Default for PkarrRelayClient { +impl Default for Client { fn default() -> Self { - Self::new(RelaySettings::default()).unwrap() + Self::new(Settings::default()).unwrap() } } -impl PkarrRelayClient { - pub fn new(settings: RelaySettings) -> Result { +impl Client { + pub fn new(settings: Settings) -> Result { if settings.relays.is_empty() { return Err(Error::EmptyListOfRelays); } @@ -337,13 +384,13 @@ mod tests { .create(); let relays: Vec = vec![server.url()]; - let settings = RelaySettings { + let settings = Settings { relays, - ..RelaySettings::default() + ..Settings::default() }; - let a = PkarrRelayClient::new(settings.clone()).unwrap(); - let b = PkarrRelayClient::new(settings).unwrap(); + let a = Client::new(settings.clone()).unwrap(); + let b = Client::new(settings).unwrap(); a.publish(&signed_packet).await.unwrap(); @@ -366,12 +413,12 @@ mod tests { server.mock("GET", path.as_str()).with_status(404).create(); let relays: Vec = vec![server.url()]; - let settings = RelaySettings { + let settings = Settings { relays, - ..RelaySettings::default() + ..Settings::default() }; - let client = PkarrRelayClient::new(settings.clone()).unwrap(); + let client = Client::new(settings.clone()).unwrap(); let resolved = client.resolve(&keypair.public_key()).await.unwrap(); diff --git a/server/Cargo.toml b/server/Cargo.toml index a7753f4..0bb615d 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -26,4 +26,4 @@ serde = { version = "1.0.199", features = ["derive"] } toml = "0.8.12" clap = { version = "4.5.1", features = ["derive"] } dirs-next = "2.0.0" -pkarr = { version = "2.2.0", path = "../pkarr", features = ["async"] } +pkarr = { version = "3.0.0", path = "../pkarr" } diff --git a/server/src/http_server.rs b/server/src/http_server.rs index 2ddce2a..7079310 100644 --- a/server/src/http_server.rs +++ b/server/src/http_server.rs @@ -7,7 +7,7 @@ use tower_http::cors::{self, CorsLayer}; use tower_http::trace::TraceLayer; use tracing::{info, warn}; -use pkarr::PkarrClientAsync; +use pkarr::ClientAsync; use crate::rate_limiting::IpRateLimiter; @@ -18,7 +18,7 @@ pub struct HttpServer { impl HttpServer { /// Spawn the server pub async fn spawn( - client: PkarrClientAsync, + client: ClientAsync, port: u16, rate_limiter: IpRateLimiter, ) -> Result { @@ -96,5 +96,5 @@ pub fn create_app(state: AppState, rate_limiter: IpRateLimiter) -> Router { #[derive(Debug, Clone)] pub struct AppState { - pub client: PkarrClientAsync, + pub client: ClientAsync, } diff --git a/server/src/main.rs b/server/src/main.rs index 4099932..1530067 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -15,7 +15,7 @@ use std::path::PathBuf; use tracing::{debug, info}; use http_server::HttpServer; -use pkarr::{mainline::dht::DhtSettings, PkarrClient}; +use pkarr::{mainline::dht::DhtSettings, Client}; #[derive(Parser, Debug)] struct Cli { @@ -52,7 +52,7 @@ async fn main() -> Result<()> { let rate_limiter = rate_limiting::IpRateLimiter::new(config.rate_limiter()); - let client = PkarrClient::builder() + let client = Client::builder() .dht_settings(DhtSettings { port: Some(config.dht_port()), server: Some(Box::new(dht_server::DhtServer::new( From 848c96c0d290c65611da79c37e73665d632a2c5c Mon Sep 17 00:00:00 2001 From: nazeh Date: Sat, 21 Sep 2024 23:13:06 +0300 Subject: [PATCH 12/84] feat(pkarr): rename PkarrCache to Cache and use in relay client --- pkarr/Cargo.toml | 6 ++-- pkarr/src/cache.rs | 59 ++++++++++++++++++++------------ pkarr/src/dht.rs | 42 +++++++++++------------ pkarr/src/error.rs | 1 + pkarr/src/keys.rs | 14 ++++++++ pkarr/src/lib.rs | 10 +++--- pkarr/src/relay.rs | 68 ++++++++++++++++++------------------- server/src/cache.rs | 71 ++++++++++++++++++++------------------- server/src/dht_server.rs | 10 +++--- server/src/http_server.rs | 6 ++-- server/src/main.rs | 7 ++-- 11 files changed, 161 insertions(+), 133 deletions(-) diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index 8042dfb..c10ffab 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -16,19 +16,19 @@ self_cell = "1.0.2" simple-dns = "0.6.1" thiserror = "1.0.49" tracing = "0.1.40" +dyn-clone = "1.0.17" lru = { version = "0.12.3", default-features = false } serde = { version = "1.0.209", features = ["derive"], optional = true } reqwest = { version = "0.12.7", features = ["rustls-tls"], optional = true } -rand = { version = "0.8.5", optional = true } document-features = "0.2.8" [target.'cfg(not(target_arch = "wasm32"))'.dependencies] # Dht client dependencies: mainline = { version = "2.0.1", optional = true } -dyn-clone = { version = "1.0.17", optional = true } flume = { version = "0.11.0", features = ["select", "eventual-fairness", "async"], default-features = false , optional = true } tokio = { version = "1.40.0", optional = true } +rand = { version = "0.8.5", optional = true } [target.'cfg(target_arch = "wasm32")'.dependencies] js-sys = "0.3.69" @@ -50,7 +50,7 @@ serde = ["dep:serde"] # Clients ## Use [dht::Client] -dht = ["dep:mainline", "dep:dyn-clone", "flume"] +dht = ["dep:mainline", "flume"] ## Use [relay::Client] relay = ["dep:reqwest", "dep:tokio"] diff --git a/pkarr/src/cache.rs b/pkarr/src/cache.rs index 5179e2a..8a121ca 100644 --- a/pkarr/src/cache.rs +++ b/pkarr/src/cache.rs @@ -1,49 +1,64 @@ -//! Trait and inmemory implementation of [PkarrCache] +//! Trait and inmemory implementation of [Cache] use dyn_clone::DynClone; use lru::LruCache; -use mainline::Id; use std::fmt::Debug; use std::num::NonZeroUsize; use std::sync::{Arc, Mutex}; -use crate::SignedPacket; +use crate::{PublicKey, SignedPacket}; -/// The sha1 hash of the [crate::PublicKey] used as the key in [PkarrCache]. -pub type PkarrCacheKey = Id; +/// The sha1 hash of the [PublicKey] used as the key in [Cache]. +pub type CacheKey = [u8; 20]; -pub trait PkarrCache: Debug + Send + Sync + DynClone { +impl From<&PublicKey> for CacheKey { + /// Take the first 20 bytes from a [PublicKey] instead of hashing with sha1. + /// Useful when you don't want to import Sha1 hasher, or mainline crate. + /// + /// Risks evicting SignedPackets from the cache if there ar two packets whose + /// PublicKey share the first 20 bytes. + /// + /// Don't use both PublicKey and SignedPacket::target() to store packets, + /// otherwise you will end up duplicating and wasting cache space. + fn from(public_key: &PublicKey) -> CacheKey { + let cache_key: [u8; 20] = public_key.as_bytes()[0..20].try_into().unwrap(); + + cache_key + } +} + +pub trait Cache: Debug + Send + Sync + DynClone { fn len(&self) -> usize; fn is_empty(&self) -> bool { self.len() == 0 } /// Puts [SignedPacket] into cache. - fn put(&self, key: &PkarrCacheKey, signed_packet: &SignedPacket); + fn put(&self, key: &CacheKey, signed_packet: &SignedPacket); /// Reads [SignedPacket] from cache, while moving it to the head of the LRU list. - fn get(&self, key: &PkarrCacheKey) -> Option; + fn get(&self, key: &CacheKey) -> Option; /// Reads [SignedPacket] from cache, without changing the LRU list. /// /// Used for internal reads that are not initiated by the user directly, /// like comparing an received signed packet with existing one. /// - /// Useful to implement differently from [PkarrCache::get], if you are implementing + /// Useful to implement differently from [Cache::get], if you are implementing /// persistent cache where writes are slower than reads. /// - /// Otherwise it will just use [PkarrCache::get]. - fn get_read_only(&self, key: &PkarrCacheKey) -> Option { + /// Otherwise it will just use [Cache::get]. + fn get_read_only(&self, key: &CacheKey) -> Option { self.get(key) } } -dyn_clone::clone_trait_object!(PkarrCache); +dyn_clone::clone_trait_object!(Cache); -/// A thread safe wrapper around `LruCache` +/// A thread safe wrapper around [lru::LruCache] #[derive(Debug, Clone)] -pub struct InMemoryPkarrCache { - inner: Arc>>, +pub struct InMemoryCache { + inner: Arc>>, } -impl InMemoryPkarrCache { +impl InMemoryCache { /// Creats a new `LRU` cache that holds at most `cap` items. pub fn new(capacity: NonZeroUsize) -> Self { Self { @@ -52,7 +67,7 @@ impl InMemoryPkarrCache { } } -impl PkarrCache for InMemoryPkarrCache { +impl Cache for InMemoryCache { fn len(&self) -> usize { self.inner.lock().unwrap().len() } @@ -60,25 +75,25 @@ impl PkarrCache for InMemoryPkarrCache { /// Puts [SignedPacket], if a version of the packet already exists, /// and it has the same [SignedPacket::as_bytes], then only [SignedPacket::last_seen] will be /// updated, otherwise the input will be cloned. - fn put(&self, target: &Id, signed_packet: &SignedPacket) { + fn put(&self, key: &CacheKey, signed_packet: &SignedPacket) { let mut lock = self.inner.lock().unwrap(); - match lock.get_mut(target) { + match lock.get_mut(key) { Some(existing) => { if existing.as_bytes() == signed_packet.as_bytes() { // just refresh the last_seen existing.set_last_seen(signed_packet.last_seen()) } else { - lock.put(*target, signed_packet.clone()); + lock.put(*key, signed_packet.clone()); } } None => { - lock.put(*target, signed_packet.clone()); + lock.put(*key, signed_packet.clone()); } } } - fn get(&self, key: &PkarrCacheKey) -> Option { + fn get(&self, key: &CacheKey) -> Option { self.inner.lock().unwrap().get(key).cloned() } } diff --git a/pkarr/src/dht.rs b/pkarr/src/dht.rs index d967523..ff3f15b 100644 --- a/pkarr/src/dht.rs +++ b/pkarr/src/dht.rs @@ -18,7 +18,7 @@ use std::{ use tracing::{debug, trace}; use crate::{ - cache::{InMemoryPkarrCache, PkarrCache}, + cache::{InMemoryCache, Cache}, DEFAULT_CACHE_SIZE, DEFAULT_MAXIMUM_TTL, DEFAULT_MINIMUM_TTL, DEFAULT_RESOLVERS, }; use crate::{Error, PublicKey, Result, SignedPacket}; @@ -44,8 +44,8 @@ pub struct Settings { /// /// Defaults to [DEFAULT_MAXIMUM_TTL] pub maximum_ttl: u32, - /// Custom [PkarrCache] implementation, defaults to [InMemoryPkarrCache] - pub cache: Option>, + /// Custom [Cache] implementation, defaults to [InMemoryCache] + pub cache: Option>, } impl Default for Settings { @@ -88,7 +88,7 @@ impl ClientBuilder { /// Set the [Settings::cache_size]. /// - /// Controls the capacity of [PkarrCache]. + /// Controls the capacity of [Cache]. pub fn cache_size(mut self, cache_size: NonZeroUsize) -> Self { self.settings.cache_size = cache_size; self @@ -112,8 +112,8 @@ impl ClientBuilder { self } - /// Set a custom implementation of [PkarrCache]. - pub fn cache(mut self, cache: Box) -> Self { + /// Set a custom implementation of [Cache]. + pub fn cache(mut self, cache: Box) -> Self { self.settings.cache = Some(cache); self } @@ -138,11 +138,11 @@ impl ClientBuilder { #[derive(Clone, Debug)] /// Pkarr client for publishing and resolving [SignedPacket]s over [mainline]. pub struct Client { - pub(crate) address: Option, - pub(crate) sender: Sender, - pub(crate) cache: Box, - pub(crate) minimum_ttl: u32, - pub(crate) maximum_ttl: u32, + address: Option, + sender: Sender, + cache: Box, + minimum_ttl: u32, + maximum_ttl: u32, } impl Client { @@ -156,7 +156,7 @@ impl Client { let cache = settings .cache .clone() - .unwrap_or(Box::new(InMemoryPkarrCache::new(settings.cache_size))); + .unwrap_or(Box::new(InMemoryCache::new(settings.cache_size))); let cache_clone = cache.clone(); let client = Client { @@ -189,7 +189,7 @@ impl Client { } /// Returns a reference to the internal cache. - pub fn cache(&self) -> &dyn PkarrCache { + pub fn cache(&self) -> &dyn Cache { self.cache.as_ref() } @@ -305,13 +305,13 @@ impl Client { ) -> Result>> { let mutable_item: MutableItem = (signed_packet).into(); - if let Some(current) = self.cache.get(mutable_item.target()) { + if let Some(current) = self.cache.get(&mutable_item.target().bytes) { if current.timestamp() > signed_packet.timestamp() { return Err(Error::NotMostRecent); } }; - self.cache.put(mutable_item.target(), signed_packet); + self.cache.put(&mutable_item.target().bytes, signed_packet); let (sender, receiver) = flume::bounded::>(1); @@ -327,7 +327,7 @@ impl Client { let (sender, receiver) = flume::bounded::(1); - let cached_packet = self.cache.get(&target); + let cached_packet = self.cache.get(&target.bytes); if let Some(ref cached) = cached_packet { let expires_in = cached.expires_in(self.minimum_ttl, self.maximum_ttl); @@ -364,7 +364,7 @@ impl Client { fn run( mut rpc: Rpc, - cache: Box, + cache: Box, settings: Settings, receiver: Receiver, ) { @@ -444,7 +444,7 @@ fn run( } => { if let Ok(signed_packet) = &SignedPacket::try_from(mutable_item) { let new_packet = - if let Some(ref cached) = cache.get_read_only(target) { + if let Some(ref cached) = cache.get_read_only(&target.bytes) { if signed_packet.more_recent_than(cached) { debug!( ?target, @@ -460,7 +460,7 @@ fn run( }; if let Some(packet) = new_packet { - cache.put(target, packet); + cache.put(&target.bytes, packet); if let Some(set) = senders.get(target) { for sender in set { @@ -475,12 +475,12 @@ fn run( target, response: QueryResponseSpecific::Value(Response::NoMoreRecentValue(seq)), } => { - if let Some(mut cached) = cache.get_read_only(target) { + if let Some(mut cached) = cache.get_read_only(&target.bytes) { if (*seq as u64) == cached.timestamp() { trace!("Remote node has the a packet with same timestamp, refreshing cached packet."); cached.refresh(); - cache.put(target, &cached); + cache.put(&target.bytes, &cached); // Send the found sequence as a timestamp to the caller to decide what to do // with it. diff --git a/pkarr/src/error.rs b/pkarr/src/error.rs index 63b052a..9cb1fe2 100644 --- a/pkarr/src/error.rs +++ b/pkarr/src/error.rs @@ -48,6 +48,7 @@ pub enum Error { PacketTooLarge(usize), // === Flume errors === + #[cfg(not(target_arch = "wasm32"))] #[error(transparent)] /// Transparent [flume::RecvError] Receive(#[from] flume::RecvError), diff --git a/pkarr/src/keys.rs b/pkarr/src/keys.rs index b2fd32f..d86cd46 100644 --- a/pkarr/src/keys.rs +++ b/pkarr/src/keys.rs @@ -2,6 +2,7 @@ use crate::{Error, Result}; use ed25519_dalek::{SecretKey, Signature, Signer, SigningKey, Verifier, VerifyingKey}; +#[cfg(not(target_arch = "wasm32"))] #[cfg(feature = "rand")] use rand::rngs::OsRng; use std::{ @@ -17,6 +18,7 @@ use serde::{Deserialize, Serialize}; pub struct Keypair(SigningKey); impl Keypair { + #[cfg(not(target_arch = "wasm32"))] #[cfg(feature = "rand")] pub fn random() -> Keypair { let mut csprng = OsRng; @@ -98,6 +100,18 @@ impl PublicKey { } } +impl AsRef for Keypair { + fn as_ref(&self) -> &Keypair { + self + } +} + +impl AsRef for PublicKey { + fn as_ref(&self) -> &PublicKey { + self + } +} + impl TryFrom<&[u8]> for PublicKey { type Error = Error; diff --git a/pkarr/src/lib.rs b/pkarr/src/lib.rs index 78375a6..835fd9a 100644 --- a/pkarr/src/lib.rs +++ b/pkarr/src/lib.rs @@ -11,14 +11,16 @@ macro_rules! if_dht { } // Modules +mod cache; mod error; mod keys; mod signed_packet; // Common exports -pub use crate::error::{Error, Result}; -pub use crate::keys::{Keypair, PublicKey}; -pub use crate::signed_packet::{system_time, SignedPacket}; +pub use cache::{Cache, CacheKey, InMemoryCache}; +pub use error::{Error, Result}; +pub use keys::{Keypair, PublicKey}; +pub use signed_packet::{system_time, SignedPacket}; /// Default minimum TTL: 5 minutes pub const DEFAULT_MINIMUM_TTL: u32 = 300; @@ -36,11 +38,9 @@ pub use bytes; pub use simple_dns as dns; if_dht! { - mod cache; mod dht; pub use dht::{ClientBuilder, Client, Settings}; - pub use cache::{PkarrCache, PkarrCacheKey, InMemoryPkarrCache}; // Rexports pub use mainline; diff --git a/pkarr/src/relay.rs b/pkarr/src/relay.rs index 052b12e..18b51e3 100644 --- a/pkarr/src/relay.rs +++ b/pkarr/src/relay.rs @@ -1,18 +1,16 @@ //! Pkarr client for publishing and resolving [SignedPacket]s over [relays](https://pkarr.org/relays). -use std::{ - num::NonZeroUsize, - sync::{Arc, Mutex}, -}; +use std::num::NonZeroUsize; -use lru::LruCache; use reqwest::{Response, StatusCode}; -use tokio::task::JoinSet; use tracing::debug; +#[cfg(not(target_arch = "wasm32"))] +use tokio::task::JoinSet; + use crate::{ - Error, PkarrCache, PublicKey, Result, SignedPacket, DEFAULT_CACHE_SIZE, DEFAULT_MAXIMUM_TTL, - DEFAULT_MINIMUM_TTL, DEFAULT_RELAYS, + Cache, Error, InMemoryCache, PublicKey, Result, SignedPacket, DEFAULT_CACHE_SIZE, + DEFAULT_MAXIMUM_TTL, DEFAULT_MINIMUM_TTL, DEFAULT_RELAYS, }; #[derive(Debug, Clone)] @@ -31,8 +29,8 @@ pub struct Settings { pub maximum_ttl: u32, /// Custom [reqwest::Client] pub http_client: reqwest::Client, - /// Custom [PkarrCache] implementation, defaults to [InMemoryPkarrCache] - pub cache: Option>, + /// Custom [Cache] implementation, defaults to [InMemoryCache] + pub cache: Option>, } impl Default for Settings { @@ -57,7 +55,7 @@ pub struct ClientBuilder { impl ClientBuilder { /// Set the [Settings::cache_size]. /// - /// Controls the capacity of [PkarrCache]. + /// Controls the capacity of [Cache]. pub fn cache_size(mut self, cache_size: NonZeroUsize) -> Self { self.settings.cache_size = cache_size; self @@ -81,8 +79,8 @@ impl ClientBuilder { self } - /// Set a custom implementation of [PkarrCache]. - pub fn cache(mut self, cache: Box) -> Self { + /// Set a custom implementation of [Cache]. + pub fn cache(mut self, cache: Box) -> Self { self.settings.cache = Some(cache); self } @@ -97,7 +95,7 @@ impl ClientBuilder { pub struct Client { http_client: reqwest::Client, relays: Vec, - cache: Arc>>, + cache: Box, minimum_ttl: u32, maximum_ttl: u32, } @@ -114,17 +112,22 @@ impl Client { return Err(Error::EmptyListOfRelays); } + let cache = settings + .cache + .clone() + .unwrap_or(Box::new(InMemoryCache::new(settings.cache_size))); + Ok(Self { http_client: settings.http_client, relays: settings.relays, - cache: Arc::new(Mutex::new(LruCache::new(settings.cache_size))), + cache, minimum_ttl: settings.minimum_ttl, maximum_ttl: settings.maximum_ttl, }) } /// Returns a reference to the internal cache. - pub fn cache(&self) -> &Mutex> { + pub fn cache(&self) -> &dyn Cache { self.cache.as_ref() } @@ -139,18 +142,13 @@ impl Client { pub async fn publish(&self, signed_packet: &SignedPacket) -> Result<()> { let public_key = signed_packet.public_key(); - // Let the compiler know we are dropping the cache before await - { - let mut cache = self.cache.lock().unwrap(); - - if let Some(current) = cache.get(&public_key) { - if current.timestamp() > signed_packet.timestamp() { - return Err(Error::NotMostRecent); - } - }; + if let Some(current) = self.cache.get(&public_key.as_ref().into()) { + if current.timestamp() > signed_packet.timestamp() { + return Err(Error::NotMostRecent); + } + }; - cache.put(public_key.to_owned(), signed_packet.clone()); - } + self.cache.put(&public_key.as_ref().into(), signed_packet); self.race_publish(signed_packet).await } @@ -176,6 +174,7 @@ impl Client { // === Native Race implementation === + #[cfg(not(target_arch = "wasm32"))] async fn race_publish(&self, signed_packet: &SignedPacket) -> Result<()> { let mut futures = JoinSet::new(); @@ -201,6 +200,7 @@ impl Client { Err(last_error) } + #[cfg(not(target_arch = "wasm32"))] async fn race_resolve( &self, public_key: &PublicKey, @@ -221,9 +221,8 @@ impl Client { while let Some(task_result) = futures.join_next().await { match task_result { Ok(Ok(Some(signed_packet))) => { - let mut cache = self.cache.lock().unwrap(); - - cache.put(signed_packet.public_key(), signed_packet.clone()); + self.cache + .put(&signed_packet.public_key().as_ref().into(), &signed_packet); return Ok(Some(signed_packet)); } @@ -260,9 +259,7 @@ impl Client { } fn get_from_cache(&self, public_key: &PublicKey) -> Option { - let mut cache = self.cache.lock().unwrap(); - - let cached_packet = cache.get(public_key); + let cached_packet = self.cache.get(&public_key.as_ref().into()); if let Some(cached) = cached_packet { let expires_in = cached.expires_in(self.minimum_ttl, self.maximum_ttl); @@ -316,6 +313,7 @@ impl Client { } } +#[cfg(not(target_arch = "wasm32"))] async fn read(response: &mut Response) -> Vec { let mut total_size = 0; let mut payload = Vec::new(); @@ -396,8 +394,8 @@ mod tests { let resolved = b.resolve(&keypair.public_key()).await.unwrap().unwrap(); - assert_eq!(a.cache().lock().unwrap().len(), 1); - assert_eq!(b.cache().lock().unwrap().len(), 1); + assert_eq!(a.cache().len(), 1); + assert_eq!(b.cache().len(), 1); assert_eq!(resolved.as_bytes(), signed_packet.as_bytes()); } diff --git a/server/src/cache.rs b/server/src/cache.rs index 4aa92b2..d54d43b 100644 --- a/server/src/cache.rs +++ b/server/src/cache.rs @@ -1,6 +1,6 @@ use std::{borrow::Cow, path::Path, time::Duration}; -use pkarr::{system_time, PkarrCache, PkarrCacheKey, SignedPacket}; +use pkarr::{system_time, Cache, CacheKey, SignedPacket}; use byteorder::LittleEndian; use heed::{types::U64, BoxedError, BytesDecode, BytesEncode, Database, Env, EnvOpenOptions}; @@ -12,25 +12,26 @@ const PKARR_CACHE_TABLE_NAME_SIGNED_PACKET: &str = "pkarrcache:signed_packet"; const PKARR_CACHE_TABLE_NAME_KEY_TO_TIME: &str = "pkarrcache:key_to_time"; const PKARR_CACHE_TABLE_NAME_TIME_TO_KEY: &str = "pkarrcache:time_to_key"; -type PkarrCacheSignedPacketsTable = Database; -type PkarrCacheKeyToTimeTable = Database>; -type PkarrCacheTimeToKeyTable = Database, PkarrCacheKeyCodec>; +type CacheSignedPacketsTable = Database; +type CacheKeyToTimeTable = Database>; +type CacheTimeToKeyTable = Database, CacheKeyCodec>; -pub struct PkarrCacheKeyCodec; +pub struct CacheKeyCodec; -impl<'a> BytesEncode<'a> for PkarrCacheKeyCodec { - type EItem = PkarrCacheKey; +impl<'a> BytesEncode<'a> for CacheKeyCodec { + type EItem = CacheKey; fn bytes_encode(key: &Self::EItem) -> Result, BoxedError> { - Ok(Cow::Owned(key.bytes.to_vec())) + Ok(Cow::Owned(key.to_vec())) } } -impl<'a> BytesDecode<'a> for PkarrCacheKeyCodec { - type DItem = PkarrCacheKey; +impl<'a> BytesDecode<'a> for CacheKeyCodec { + type DItem = CacheKey; fn bytes_decode(bytes: &'a [u8]) -> Result { - Ok(PkarrCacheKey::from_bytes(bytes)?) + let key: [u8; 20] = bytes.try_into()?; + Ok(key) } } @@ -65,12 +66,12 @@ impl<'a> BytesDecode<'a> for SignedPacketCodec { } #[derive(Debug, Clone)] -pub struct HeedPkarrCache { +pub struct HeedCache { capacity: usize, env: Env, } -impl HeedPkarrCache { +impl HeedCache { pub fn new(env_path: &Path, capacity: usize) -> Result { let env = unsafe { EnvOpenOptions::new() @@ -80,11 +81,11 @@ impl HeedPkarrCache { }; let mut wtxn = env.write_txn()?; - let _: PkarrCacheSignedPacketsTable = + let _: CacheSignedPacketsTable = env.create_database(&mut wtxn, Some(PKARR_CACHE_TABLE_NAME_SIGNED_PACKET))?; - let _: PkarrCacheKeyToTimeTable = + let _: CacheKeyToTimeTable = env.create_database(&mut wtxn, Some(PKARR_CACHE_TABLE_NAME_KEY_TO_TIME))?; - let _: PkarrCacheTimeToKeyTable = + let _: CacheTimeToKeyTable = env.create_database(&mut wtxn, Some(PKARR_CACHE_TABLE_NAME_TIME_TO_KEY))?; wtxn.commit()?; @@ -103,7 +104,7 @@ impl HeedPkarrCache { pub fn internal_len(&self) -> Result { let rtxn = self.env.read_txn()?; - let db: PkarrCacheSignedPacketsTable = self + let db: CacheSignedPacketsTable = self .env .open_database(&rtxn, Some(PKARR_CACHE_TABLE_NAME_SIGNED_PACKET))? .unwrap(); @@ -111,24 +112,24 @@ impl HeedPkarrCache { Ok(db.len(&rtxn)? as usize) } - pub fn internal_put(&self, key: &PkarrCacheKey, signed_packet: &SignedPacket) -> Result<()> { + pub fn internal_put(&self, key: &CacheKey, signed_packet: &SignedPacket) -> Result<()> { if self.capacity == 0 { return Ok(()); } let mut wtxn = self.env.write_txn()?; - let packets: PkarrCacheSignedPacketsTable = self + let packets: CacheSignedPacketsTable = self .env .open_database(&wtxn, Some(PKARR_CACHE_TABLE_NAME_SIGNED_PACKET))? .unwrap(); - let key_to_time: PkarrCacheKeyToTimeTable = self + let key_to_time: CacheKeyToTimeTable = self .env .open_database(&wtxn, Some(PKARR_CACHE_TABLE_NAME_KEY_TO_TIME))? .unwrap(); - let time_to_key: PkarrCacheTimeToKeyTable = self + let time_to_key: CacheTimeToKeyTable = self .env .open_database(&wtxn, Some(PKARR_CACHE_TABLE_NAME_TIME_TO_KEY))? .unwrap(); @@ -165,19 +166,19 @@ impl HeedPkarrCache { Ok(()) } - pub fn internal_get(&self, key: &PkarrCacheKey) -> Result> { + pub fn internal_get(&self, key: &CacheKey) -> Result> { let mut wtxn = self.env.write_txn()?; - let packets: PkarrCacheSignedPacketsTable = self + let packets: CacheSignedPacketsTable = self .env .open_database(&wtxn, Some(PKARR_CACHE_TABLE_NAME_SIGNED_PACKET))? .unwrap(); - let key_to_time: PkarrCacheKeyToTimeTable = self + let key_to_time: CacheKeyToTimeTable = self .env .open_database(&wtxn, Some(PKARR_CACHE_TABLE_NAME_KEY_TO_TIME))? .unwrap(); - let time_to_key: PkarrCacheTimeToKeyTable = self + let time_to_key: CacheTimeToKeyTable = self .env .open_database(&wtxn, Some(PKARR_CACHE_TABLE_NAME_TIME_TO_KEY))? .unwrap(); @@ -202,10 +203,10 @@ impl HeedPkarrCache { Ok(None) } - pub fn internal_get_read_only(&self, key: &PkarrCacheKey) -> Result> { + pub fn internal_get_read_only(&self, key: &CacheKey) -> Result> { let rtxn = self.env.read_txn()?; - let packets: PkarrCacheSignedPacketsTable = self + let packets: CacheSignedPacketsTable = self .env .open_database(&rtxn, Some(PKARR_CACHE_TABLE_NAME_SIGNED_PACKET))? .unwrap(); @@ -220,39 +221,39 @@ impl HeedPkarrCache { } } -impl PkarrCache for HeedPkarrCache { +impl Cache for HeedCache { fn len(&self) -> usize { match self.internal_len() { Ok(result) => result, Err(error) => { - debug!(?error, "Error in HeedPkarrCache::len"); + debug!(?error, "Error in HeedCache::len"); 0 } } } - fn put(&self, key: &PkarrCacheKey, signed_packet: &SignedPacket) { + fn put(&self, key: &CacheKey, signed_packet: &SignedPacket) { if let Err(error) = self.internal_put(key, signed_packet) { - debug!(?error, "Error in HeedPkarrCache::put"); + debug!(?error, "Error in HeedCache::put"); }; } - fn get(&self, key: &PkarrCacheKey) -> Option { + fn get(&self, key: &CacheKey) -> Option { match self.internal_get(key) { Ok(result) => result, Err(error) => { - debug!(?error, "Error in HeedPkarrCache::get"); + debug!(?error, "Error in HeedCache::get"); None } } } - fn get_read_only(&self, key: &PkarrCacheKey) -> Option { + fn get_read_only(&self, key: &CacheKey) -> Option { match self.internal_get_read_only(key) { Ok(result) => result, Err(error) => { - debug!(?error, "Error in HeedPkarrCache::get"); + debug!(?error, "Error in HeedCache::get"); None } diff --git a/server/src/dht_server.rs b/server/src/dht_server.rs index 37832ea..8067ee0 100644 --- a/server/src/dht_server.rs +++ b/server/src/dht_server.rs @@ -16,18 +16,18 @@ use pkarr::{ server::Server, MutableItem, }, - PkarrCache, + Cache, }; use tracing::debug; -use crate::{cache::HeedPkarrCache, rate_limiting::IpRateLimiter}; +use crate::{cache::HeedCache, rate_limiting::IpRateLimiter}; /// DhtServer with Rate limiting pub struct DhtServer { inner: mainline::server::DhtServer, resolvers: Option>, - cache: Box, + cache: Box, minimum_ttl: u32, maximum_ttl: u32, rate_limiter: IpRateLimiter, @@ -41,7 +41,7 @@ impl Debug for DhtServer { impl DhtServer { pub fn new( - cache: Box, + cache: Box, resolvers: Option>, minimum_ttl: u32, maximum_ttl: u32, @@ -78,7 +78,7 @@ impl Server for DhtServer { .. } = request { - let should_query = if let Some(cached) = self.cache.get(target) { + let should_query = if let Some(cached) = self.cache.get(&target.bytes) { debug!( public_key = ?cached.public_key(), ?target, diff --git a/server/src/http_server.rs b/server/src/http_server.rs index 7079310..9ff19a9 100644 --- a/server/src/http_server.rs +++ b/server/src/http_server.rs @@ -7,7 +7,7 @@ use tower_http::cors::{self, CorsLayer}; use tower_http::trace::TraceLayer; use tracing::{info, warn}; -use pkarr::ClientAsync; +use pkarr::Client; use crate::rate_limiting::IpRateLimiter; @@ -18,7 +18,7 @@ pub struct HttpServer { impl HttpServer { /// Spawn the server pub async fn spawn( - client: ClientAsync, + client: Client, port: u16, rate_limiter: IpRateLimiter, ) -> Result { @@ -96,5 +96,5 @@ pub fn create_app(state: AppState, rate_limiter: IpRateLimiter) -> Router { #[derive(Debug, Clone)] pub struct AppState { - pub client: ClientAsync, + pub client: Client, } diff --git a/server/src/main.rs b/server/src/main.rs index 1530067..329069c 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -7,7 +7,7 @@ mod http_server; mod rate_limiting; use anyhow::Result; -use cache::HeedPkarrCache; +use cache::HeedCache; use clap::Parser; use config::Config; use std::fs; @@ -48,7 +48,7 @@ async fn main() -> Result<()> { let env_path = &config.cache_path()?; fs::create_dir_all(env_path)?; - let cache = Box::new(HeedPkarrCache::new(env_path, config.cache_size()).unwrap()); + let cache = Box::new(HeedCache::new(env_path, config.cache_size()).unwrap()); let rate_limiter = rate_limiting::IpRateLimiter::new(config.rate_limiter()); @@ -68,8 +68,7 @@ async fn main() -> Result<()> { .minimum_ttl(config.minimum_ttl()) .maximum_ttl(config.maximum_ttl()) .cache(cache) - .build()? - .as_async(); + .build()?; let udp_address = client.local_addr().unwrap(); From 3a607ae36b8c3fe0878c627309083b4dcb948102 Mon Sep 17 00:00:00 2001 From: nazeh Date: Sat, 21 Sep 2024 23:58:45 +0300 Subject: [PATCH 13/84] feat(pkarr): add wasm support for relay::Client --- Cargo.lock | 1 + pkarr/Cargo.toml | 1 + pkarr/src/relay.rs | 88 +++++++++++++++++++++++++++----------- pkarr/src/signed_packet.rs | 4 +- 4 files changed, 66 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a565f79..6feec7d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1470,6 +1470,7 @@ dependencies = [ "dyn-clone", "ed25519-dalek", "flume", + "futures", "js-sys", "lru", "mainline", diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index c10ffab..5c088e3 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -32,6 +32,7 @@ rand = { version = "0.8.5", optional = true } [target.'cfg(target_arch = "wasm32")'.dependencies] js-sys = "0.3.69" +futures = "0.3.30" [dev-dependencies] postcard = { version = "1.0.10", features = ["alloc"] } diff --git a/pkarr/src/relay.rs b/pkarr/src/relay.rs index 18b51e3..3f4b4ff 100644 --- a/pkarr/src/relay.rs +++ b/pkarr/src/relay.rs @@ -5,6 +5,9 @@ use std::num::NonZeroUsize; use reqwest::{Response, StatusCode}; use tracing::debug; +#[cfg(target_arch = "wasm32")] +use futures::future::select_ok; + #[cfg(not(target_arch = "wasm32"))] use tokio::task::JoinSet; @@ -182,7 +185,7 @@ impl Client { let signed_packet = signed_packet.clone(); let this = self.clone(); - futures.spawn(async move { this.publish_to_relay(relay, signed_packet).await }); + futures.spawn(async move { this.publish_to_relay(&relay, signed_packet).await }); } let mut last_error = Error::EmptyListOfRelays; @@ -213,7 +216,7 @@ impl Client { let cached = cached_packet.clone(); let this = self.clone(); - futures.spawn(async move { this.resolve_from_relay(relay, public_key, cached).await }); + futures.spawn(async move { this.resolve_from_relay(&relay, public_key, cached).await }); } let mut result: Result> = Ok(None); @@ -237,13 +240,56 @@ impl Client { result } - // === Private Methods === + // === Wasm === + + #[cfg(target_arch = "wasm32")] + async fn race_publish(&self, signed_packet: &SignedPacket) -> Result<()> { + let futures = self.relays.iter().map(|relay| { + let signed_packet = signed_packet.clone(); + let this = self.clone(); + + Box::pin(async move { this.publish_to_relay(relay, signed_packet).await }) + }); + + match select_ok(futures).await { + Ok((_, _)) => Ok(()), + Err(e) => Err(e), + } + } - async fn publish_to_relay( + #[cfg(target_arch = "wasm32")] + async fn race_resolve( &self, - relay: String, - signed_packet: SignedPacket, - ) -> Result { + public_key: &PublicKey, + cached_packet: Option, + ) -> Result> { + let futures = self.relays.iter().map(|relay| { + let public_key = public_key.clone(); + let cached = cached_packet.clone(); + let this = self.clone(); + + Box::pin(async move { this.resolve_from_relay(relay, public_key, cached).await }) + }); + + let mut result: Result> = Ok(None); + + match select_ok(futures).await { + Ok((Some(signed_packet), _)) => { + self.cache + .put(&signed_packet.public_key().as_ref().into(), &signed_packet); + + return Ok(Some(signed_packet)); + } + Err(error) => result = Err(error), + Ok(_) => {} + } + + result + } + + // === Private Methods === + + async fn publish_to_relay(&self, relay: &str, signed_packet: SignedPacket) -> Result { let url = format!("{relay}/{}", signed_packet.public_key()); self.http_client @@ -280,22 +326,27 @@ impl Client { async fn resolve_from_relay( &self, - relay: String, + relay: &str, public_key: PublicKey, cached_packet: Option, ) -> Result> { let url = format!("{relay}/{public_key}"); match self.http_client.get(&url).send().await { - Ok(mut response) => { + Ok(response) => { if response.status() == StatusCode::NOT_FOUND { debug!(?url, "SignedPacket not found"); return Ok(None); } + if response.content_length().unwrap_or_default() > SignedPacket::MAX_BYTES { + debug!(?url, "Response too large"); - let payload = read(&mut response).await; + return Ok(None); + } - match SignedPacket::from_relay_payload(&public_key, &payload.into()) { + let payload = response.bytes().await?; + + match SignedPacket::from_relay_payload(&public_key, &payload) { Ok(signed_packet) => Ok(choose_most_recent(signed_packet, cached_packet)), Err(error) => { debug!(?url, ?error, "Invalid signed_packet"); @@ -313,21 +364,6 @@ impl Client { } } -#[cfg(not(target_arch = "wasm32"))] -async fn read(response: &mut Response) -> Vec { - let mut total_size = 0; - let mut payload = Vec::new(); - - while let Ok(Some(chunk)) = response.chunk().await { - total_size += chunk.len(); - payload.extend_from_slice(&chunk); - if total_size >= SignedPacket::MAX_BYTES { - break; - } - } - payload -} - fn choose_most_recent( signed_packet: SignedPacket, cached_packet: Option, diff --git a/pkarr/src/signed_packet.rs b/pkarr/src/signed_packet.rs index a54b3e5..855d193 100644 --- a/pkarr/src/signed_packet.rs +++ b/pkarr/src/signed_packet.rs @@ -65,7 +65,7 @@ pub struct SignedPacket { } impl SignedPacket { - pub const MAX_BYTES: usize = 1104; + pub const MAX_BYTES: u64 = 1104; /// Creates a [Self] from the serialized representation: /// `<32 bytes public_key><64 bytes signature><8 bytes big-endian timestamp in microseconds>` @@ -90,7 +90,7 @@ impl SignedPacket { if bytes.len() < 104 { return Err(Error::InvalidSignedPacketBytesLength(bytes.len())); } - if bytes.len() > SignedPacket::MAX_BYTES { + if (bytes.len() as u64) > SignedPacket::MAX_BYTES { return Err(Error::PacketTooLarge(bytes.len())); } let public_key = PublicKey::try_from(&bytes[..32])?; From fddf29a638aeb33a6319dae144265396e14980b2 Mon Sep 17 00:00:00 2001 From: nazeh Date: Sun, 22 Sep 2024 11:05:31 +0300 Subject: [PATCH 14/84] feat(pkarr): export same structs from wasm as in native --- pkarr/Cargo.toml | 2 +- pkarr/src/lib.rs | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index 5c088e3..ffd8fce 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -58,7 +58,7 @@ relay = ["dep:reqwest", "dep:tokio"] ## Use all features full = ["dht", "relay", "rand", "serde"] -default = ["full"] +default = ["dht", "rand", "serde"] [package.metadata.docs.rs] all-features = true diff --git a/pkarr/src/lib.rs b/pkarr/src/lib.rs index 835fd9a..5764933 100644 --- a/pkarr/src/lib.rs +++ b/pkarr/src/lib.rs @@ -48,3 +48,8 @@ if_dht! { #[cfg(feature = "relay")] pub mod relay; + +#[cfg(target_arch = "wasm32")] +pub mod relay; +#[cfg(target_arch = "wasm32")] +pub use relay::{Client, ClientBuilder, Settings}; From 60fb3a10a6bd8cd6a1878b2ba7842b43266e5323 Mon Sep 17 00:00:00 2001 From: nazeh Date: Sun, 22 Sep 2024 11:32:07 +0300 Subject: [PATCH 15/84] fix(pkarr): wasm dependencies --- Cargo.lock | 1 + pkarr/Cargo.toml | 4 +++- pkarr/src/error.rs | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6feec7d..61cbd97 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1471,6 +1471,7 @@ dependencies = [ "ed25519-dalek", "flume", "futures", + "getrandom", "js-sys", "lru", "mainline", diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index ffd8fce..7c07414 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -19,7 +19,6 @@ tracing = "0.1.40" dyn-clone = "1.0.17" lru = { version = "0.12.3", default-features = false } serde = { version = "1.0.209", features = ["derive"], optional = true } -reqwest = { version = "0.12.7", features = ["rustls-tls"], optional = true } document-features = "0.2.8" @@ -29,10 +28,13 @@ mainline = { version = "2.0.1", optional = true } flume = { version = "0.11.0", features = ["select", "eventual-fairness", "async"], default-features = false , optional = true } tokio = { version = "1.40.0", optional = true } rand = { version = "0.8.5", optional = true } +reqwest = { version = "0.12.7", features = ["rustls-tls"], optional = true } [target.'cfg(target_arch = "wasm32")'.dependencies] js-sys = "0.3.69" futures = "0.3.30" +getrandom = { version = "0.2", features = ["js"] } +reqwest = "0.12.7" [dev-dependencies] postcard = { version = "1.0.10", features = ["alloc"] } diff --git a/pkarr/src/error.rs b/pkarr/src/error.rs index 9cb1fe2..7dd6815 100644 --- a/pkarr/src/error.rs +++ b/pkarr/src/error.rs @@ -66,12 +66,12 @@ pub enum Error { NotMostRecent, // === Relay errors === - #[cfg(feature = "relay")] + #[cfg(any(feature = "relay", target_arch = "wasm32"))] #[error(transparent)] /// Transparent [reqwest::Error] RelayError(#[from] reqwest::Error), - #[cfg(feature = "relay")] + #[cfg(any(feature = "relay", target_arch = "wasm32"))] #[error("Empty list of relays")] /// Empty list of relays EmptyListOfRelays, From d0be23324bcf706f5a359236d2945944abbb40f8 Mon Sep 17 00:00:00 2001 From: nazeh Date: Sun, 22 Sep 2024 11:42:33 +0300 Subject: [PATCH 16/84] fix(pkarr): add rand in wasm target --- pkarr/Cargo.toml | 2 +- pkarr/README.md | 7 +++++++ pkarr/src/keys.rs | 2 -- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index 7c07414..fb62494 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -19,6 +19,7 @@ tracing = "0.1.40" dyn-clone = "1.0.17" lru = { version = "0.12.3", default-features = false } serde = { version = "1.0.209", features = ["derive"], optional = true } +rand = { version = "0.8.5", optional = true } document-features = "0.2.8" @@ -27,7 +28,6 @@ document-features = "0.2.8" mainline = { version = "2.0.1", optional = true } flume = { version = "0.11.0", features = ["select", "eventual-fairness", "async"], default-features = false , optional = true } tokio = { version = "1.40.0", optional = true } -rand = { version = "0.8.5", optional = true } reqwest = { version = "0.12.7", features = ["rustls-tls"], optional = true } [target.'cfg(target_arch = "wasm32")'.dependencies] diff --git a/pkarr/README.md b/pkarr/README.md index 37ae8b3..e02292f 100644 --- a/pkarr/README.md +++ b/pkarr/README.md @@ -9,3 +9,10 @@ Publish and resolve DNS packets over Mainline DHT. ## Get started Check the [Examples](https://github.com/Nuhvi/pkarr/tree/main/pkarr/examples). + +## WebAssembly support + +This version of Pkarr assumes that you are running Wasm in a JavaScript environment, +and using the Relays clients, so you can't use it in Wasi for example, nor can you +use some Wasi bindings to use the DHT directly. If you really need Wasi support, please +open an issue on `https://git.pkarr.org`. diff --git a/pkarr/src/keys.rs b/pkarr/src/keys.rs index d86cd46..c1966a8 100644 --- a/pkarr/src/keys.rs +++ b/pkarr/src/keys.rs @@ -2,7 +2,6 @@ use crate::{Error, Result}; use ed25519_dalek::{SecretKey, Signature, Signer, SigningKey, Verifier, VerifyingKey}; -#[cfg(not(target_arch = "wasm32"))] #[cfg(feature = "rand")] use rand::rngs::OsRng; use std::{ @@ -18,7 +17,6 @@ use serde::{Deserialize, Serialize}; pub struct Keypair(SigningKey); impl Keypair { - #[cfg(not(target_arch = "wasm32"))] #[cfg(feature = "rand")] pub fn random() -> Keypair { let mut csprng = OsRng; From 445c0db448b9e70f2a63a5b91e6d84f1ef466aeb Mon Sep 17 00:00:00 2001 From: nazeh Date: Sun, 22 Sep 2024 18:45:42 +0300 Subject: [PATCH 17/84] feat(pkarr): add ClientBuilder::relays() --- pkarr/src/relay.rs | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/pkarr/src/relay.rs b/pkarr/src/relay.rs index 3f4b4ff..8f3ba1a 100644 --- a/pkarr/src/relay.rs +++ b/pkarr/src/relay.rs @@ -56,6 +56,12 @@ pub struct ClientBuilder { } impl ClientBuilder { + /// Set the relays to publish and resolve [SignedPacket]s to and from. + pub fn relays(mut self, relays: Vec) -> Self { + self.settings.relays = relays; + self + } + /// Set the [Settings::cache_size]. /// /// Controls the capacity of [Cache]. @@ -129,6 +135,11 @@ impl Client { }) } + /// Returns a builder to edit settings before creating Client. + pub fn builder() -> ClientBuilder { + ClientBuilder::default() + } + /// Returns a reference to the internal cache. pub fn cache(&self) -> &dyn Cache { self.cache.as_ref() @@ -417,14 +428,9 @@ mod tests { .with_body(signed_packet.to_relay_payload()) .create(); - let relays: Vec = vec![server.url()]; - let settings = Settings { - relays, - ..Settings::default() - }; - - let a = Client::new(settings.clone()).unwrap(); - let b = Client::new(settings).unwrap(); + let relays = vec![server.url()]; + let a = Client::builder().relays(relays.clone()).build().unwrap(); + let b = Client::builder().relays(relays).build().unwrap(); a.publish(&signed_packet).await.unwrap(); @@ -446,13 +452,8 @@ mod tests { server.mock("GET", path.as_str()).with_status(404).create(); - let relays: Vec = vec![server.url()]; - let settings = Settings { - relays, - ..Settings::default() - }; - - let client = Client::new(settings.clone()).unwrap(); + let relays = vec![server.url()]; + let client = Client::builder().relays(relays).build().unwrap(); let resolved = client.resolve(&keypair.public_key()).await.unwrap(); From a01a71c79372f5d4f46173a21376341c0994c19c Mon Sep 17 00:00:00 2001 From: nazeh Date: Tue, 24 Sep 2024 19:03:25 +0300 Subject: [PATCH 18/84] refactor(pkarr): move clients to client mod, and the rest to base --- pkarr/src/{ => base}/cache.rs | 0 pkarr/src/{ => base}/keys.rs | 0 pkarr/src/base/mod.rs | 6 +++ pkarr/src/{ => base}/signed_packet.rs | 72 ++++++++++----------------- pkarr/src/base/time.rs | 20 ++++++++ pkarr/src/{ => client}/dht.rs | 23 ++++----- pkarr/src/client/mod.rs | 11 ++++ pkarr/src/{ => client}/relay.rs | 0 pkarr/src/error.rs | 9 ++++ pkarr/src/lib.rs | 41 ++++----------- 10 files changed, 94 insertions(+), 88 deletions(-) rename pkarr/src/{ => base}/cache.rs (100%) rename pkarr/src/{ => base}/keys.rs (100%) create mode 100644 pkarr/src/base/mod.rs rename pkarr/src/{ => base}/signed_packet.rs (94%) create mode 100644 pkarr/src/base/time.rs rename pkarr/src/{ => client}/dht.rs (98%) create mode 100644 pkarr/src/client/mod.rs rename pkarr/src/{ => client}/relay.rs (100%) diff --git a/pkarr/src/cache.rs b/pkarr/src/base/cache.rs similarity index 100% rename from pkarr/src/cache.rs rename to pkarr/src/base/cache.rs diff --git a/pkarr/src/keys.rs b/pkarr/src/base/keys.rs similarity index 100% rename from pkarr/src/keys.rs rename to pkarr/src/base/keys.rs diff --git a/pkarr/src/base/mod.rs b/pkarr/src/base/mod.rs new file mode 100644 index 0000000..7588c5b --- /dev/null +++ b/pkarr/src/base/mod.rs @@ -0,0 +1,6 @@ +//! Basic typest, traits and utilities. + +pub mod cache; +pub mod keys; +pub mod signed_packet; +pub mod time; diff --git a/pkarr/src/signed_packet.rs b/pkarr/src/base/signed_packet.rs similarity index 94% rename from pkarr/src/signed_packet.rs rename to pkarr/src/base/signed_packet.rs index 855d193..ec75858 100644 --- a/pkarr/src/signed_packet.rs +++ b/pkarr/src/base/signed_packet.rs @@ -14,8 +14,7 @@ use std::{ net::{Ipv4Addr, Ipv6Addr}, }; -#[cfg(not(target_arch = "wasm32"))] -use std::time::SystemTime; +use crate::system_time; const DOT: char = '.'; @@ -340,55 +339,38 @@ fn signable(timestamp: u64, v: &Bytes) -> Bytes { signable.into() } -#[cfg(not(target_arch = "wasm32"))] -/// Return the number of microseconds since [SystemTime::UNIX_EPOCH] -pub fn system_time() -> u64 { - SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .expect("time drift") - .as_micros() as u64 -} +#[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] +use mainline::MutableItem; -#[cfg(target_arch = "wasm32")] -/// Return the number of microseconds since [SystemTime::UNIX_EPOCH] -pub fn system_time() -> u64 { - // Won't be an issue for more than 5000 years! - (js_sys::Date::now() as u64 ) - // Turn miliseconds to microseconds - * 1000 -} +#[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] +impl From<&SignedPacket> for MutableItem { + fn from(s: &SignedPacket) -> Self { + let seq: i64 = s.timestamp() as i64; + let packet = s.inner.borrow_owner().slice(104..); -if_dht! { - use mainline::MutableItem; - - impl From<&SignedPacket> for MutableItem { - fn from(s: &SignedPacket) -> Self { - let seq: i64 = s.timestamp() as i64; - let packet = s.inner.borrow_owner().slice(104..); - - Self::new_signed_unchecked( - s.public_key().to_bytes(), - s.signature().to_bytes(), - packet, - seq, - None, - ) - } + Self::new_signed_unchecked( + s.public_key().to_bytes(), + s.signature().to_bytes(), + packet, + seq, + None, + ) } +} - impl TryFrom<&MutableItem> for SignedPacket { - type Error = Error; +#[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] +impl TryFrom<&MutableItem> for SignedPacket { + type Error = Error; - fn try_from(i: &MutableItem) -> Result { - let public_key = PublicKey::try_from(i.key()).unwrap(); - let seq = *i.seq() as u64; - let signature: Signature = i.signature().into(); + fn try_from(i: &MutableItem) -> Result { + let public_key = PublicKey::try_from(i.key()).unwrap(); + let seq = *i.seq() as u64; + let signature: Signature = i.signature().into(); - Ok(Self { - inner: Inner::try_from_parts(&public_key, &signature, seq, i.value())?, - last_seen: system_time(), - }) - } + Ok(Self { + inner: Inner::try_from_parts(&public_key, &signature, seq, i.value())?, + last_seen: system_time(), + }) } } diff --git a/pkarr/src/base/time.rs b/pkarr/src/base/time.rs new file mode 100644 index 0000000..1fc8d3a --- /dev/null +++ b/pkarr/src/base/time.rs @@ -0,0 +1,20 @@ +#[cfg(not(target_arch = "wasm32"))] +use std::time::SystemTime; + +#[cfg(not(target_arch = "wasm32"))] +/// Return the number of microseconds since [SystemTime::UNIX_EPOCH] +pub fn system_time() -> u64 { + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .expect("time drift") + .as_micros() as u64 +} + +#[cfg(target_arch = "wasm32")] +/// Return the number of microseconds since [SystemTime::UNIX_EPOCH] +pub fn system_time() -> u64 { + // Won't be an issue for more than 5000 years! + (js_sys::Date::now() as u64 ) + // Turn miliseconds to microseconds + * 1000 +} diff --git a/pkarr/src/dht.rs b/pkarr/src/client/dht.rs similarity index 98% rename from pkarr/src/dht.rs rename to pkarr/src/client/dht.rs index ff3f15b..cc67a7d 100644 --- a/pkarr/src/dht.rs +++ b/pkarr/src/client/dht.rs @@ -18,8 +18,8 @@ use std::{ use tracing::{debug, trace}; use crate::{ - cache::{InMemoryCache, Cache}, - DEFAULT_CACHE_SIZE, DEFAULT_MAXIMUM_TTL, DEFAULT_MINIMUM_TTL, DEFAULT_RESOLVERS, + Cache, InMemoryCache, DEFAULT_CACHE_SIZE, DEFAULT_MAXIMUM_TTL, DEFAULT_MINIMUM_TTL, + DEFAULT_RESOLVERS, }; use crate::{Error, PublicKey, Result, SignedPacket}; @@ -362,12 +362,7 @@ impl Client { } } -fn run( - mut rpc: Rpc, - cache: Box, - settings: Settings, - receiver: Receiver, -) { +fn run(mut rpc: Rpc, cache: Box, settings: Settings, receiver: Receiver) { debug!(?settings, "Starting Client main loop.."); let mut server = settings.dht.server; @@ -600,7 +595,7 @@ mod tests { } #[tokio::test] -async fn shutdown() { + async fn shutdown() { let testnet = Testnet::new(3); let mut a = Client::builder().testnet(&testnet).build().unwrap(); @@ -613,7 +608,7 @@ async fn shutdown() { } #[tokio::test] - async fn publish_resolve() { + async fn publish_resolve() { let testnet = Testnet::new(10); let a = Client::builder().testnet(&testnet).build().unwrap(); @@ -643,7 +638,7 @@ async fn shutdown() { } #[tokio::test] - async fn thread_safe() { + async fn thread_safe() { let testnet = Testnet::new(10); let a = Client::builder().testnet(&testnet).build().unwrap(); @@ -664,13 +659,15 @@ async fn shutdown() { let b = Client::builder().testnet(&testnet).build().unwrap(); - tokio::spawn( async move { + tokio::spawn(async move { let resolved = b.resolve(&keypair.public_key()).await.unwrap().unwrap(); assert_eq!(resolved.as_bytes(), signed_packet.as_bytes()); let from_cache = b.resolve(&keypair.public_key()).await.unwrap().unwrap(); assert_eq!(from_cache.as_bytes(), signed_packet.as_bytes()); assert_eq!(from_cache.last_seen(), resolved.last_seen()); - }).await.unwrap(); + }) + .await + .unwrap(); } } diff --git a/pkarr/src/client/mod.rs b/pkarr/src/client/mod.rs new file mode 100644 index 0000000..037385e --- /dev/null +++ b/pkarr/src/client/mod.rs @@ -0,0 +1,11 @@ +//! Client implementation. + +#[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] +mod dht; +#[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] +pub use dht::{Client, ClientBuilder, Settings}; + +#[cfg(target_arch = "wasm32")] +mod relay; +#[cfg(target_arch = "wasm32")] +pub use relay::{Client, ClientBuilder, Settings}; diff --git a/pkarr/src/relay.rs b/pkarr/src/client/relay.rs similarity index 100% rename from pkarr/src/relay.rs rename to pkarr/src/client/relay.rs diff --git a/pkarr/src/error.rs b/pkarr/src/error.rs index 7dd6815..fda8f6b 100644 --- a/pkarr/src/error.rs +++ b/pkarr/src/error.rs @@ -6,6 +6,10 @@ pub type Result = core::result::Result; #[derive(thiserror::Error, Debug)] /// Pkarr crate error enum. pub enum Error { + #[error("{0}")] + // TODO: replace with proper errors. + Generic(String), + #[error(transparent)] /// Transparent [std::io::Error] IO(#[from] std::io::Error), @@ -75,4 +79,9 @@ pub enum Error { #[error("Empty list of relays")] /// Empty list of relays EmptyListOfRelays, + + // === endpoints === + #[error("Could not resolve enpdoint {0}")] + /// Could not resolve enpdoint + ResolveEndpoint(String), } diff --git a/pkarr/src/lib.rs b/pkarr/src/lib.rs index 5764933..0573bde 100644 --- a/pkarr/src/lib.rs +++ b/pkarr/src/lib.rs @@ -3,24 +3,17 @@ #![doc = document_features::document_features!()] //! -macro_rules! if_dht { - ($($item:item)*) => {$( - #[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] - $item - )*} -} - // Modules -mod cache; +mod base; +mod client; mod error; -mod keys; -mod signed_packet; -// Common exports -pub use cache::{Cache, CacheKey, InMemoryCache}; +// Exports +pub use base::cache::{Cache, CacheKey, InMemoryCache}; +pub use base::keys::{Keypair, PublicKey}; +pub use base::signed_packet::SignedPacket; +pub use base::time::system_time; pub use error::{Error, Result}; -pub use keys::{Keypair, PublicKey}; -pub use signed_packet::{system_time, SignedPacket}; /// Default minimum TTL: 5 minutes pub const DEFAULT_MINIMUM_TTL: u32 = 300; @@ -33,23 +26,11 @@ pub const DEFAULT_RELAYS: [&str; 2] = ["https://relay.pkarr.org", "https://pkarr /// Default [resolver](https://pkarr.org/resolvers)s pub const DEFAULT_RESOLVERS: [&str; 2] = ["resolver.pkarr.org:6881", "pkarr.pubky.app:6881"]; +pub use client::{Client, ClientBuilder, Settings}; + // Rexports pub use bytes; pub use simple_dns as dns; -if_dht! { - mod dht; - - pub use dht::{ClientBuilder, Client, Settings}; - - // Rexports - pub use mainline; -} - -#[cfg(feature = "relay")] -pub mod relay; - -#[cfg(target_arch = "wasm32")] -pub mod relay; -#[cfg(target_arch = "wasm32")] -pub use relay::{Client, ClientBuilder, Settings}; +#[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] +pub use mainline; From 5e40b2a97226be611c2160329c37c51e7bb8ddb4 Mon Sep 17 00:00:00 2001 From: nazeh Date: Tue, 24 Sep 2024 20:11:37 +0300 Subject: [PATCH 19/84] feat(pkarr): export pkarr::relay; --- pkarr/src/client/mod.rs | 3 +++ pkarr/src/lib.rs | 2 ++ 2 files changed, 5 insertions(+) diff --git a/pkarr/src/client/mod.rs b/pkarr/src/client/mod.rs index 037385e..79ed76e 100644 --- a/pkarr/src/client/mod.rs +++ b/pkarr/src/client/mod.rs @@ -9,3 +9,6 @@ pub use dht::{Client, ClientBuilder, Settings}; mod relay; #[cfg(target_arch = "wasm32")] pub use relay::{Client, ClientBuilder, Settings}; + +#[cfg(all(not(target_arch = "wasm32"), feature = "relay"))] +pub mod relay; diff --git a/pkarr/src/lib.rs b/pkarr/src/lib.rs index 0573bde..51f5c46 100644 --- a/pkarr/src/lib.rs +++ b/pkarr/src/lib.rs @@ -32,5 +32,7 @@ pub use client::{Client, ClientBuilder, Settings}; pub use bytes; pub use simple_dns as dns; +#[cfg(all(not(target_arch = "wasm32"), feature = "relay"))] +pub use client::relay; #[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] pub use mainline; From affe38ed60f621b868ded654139a9727ebb393e5 Mon Sep 17 00:00:00 2001 From: nazeh Date: Tue, 24 Sep 2024 20:21:21 +0300 Subject: [PATCH 20/84] feat(pkarr): extra::endpoint resolver --- Cargo.lock | 16 +- pkarr/Cargo.toml | 6 +- pkarr/src/extra/endpoints/endpoint.rs | 217 +++++++++++++++++++++++++ pkarr/src/extra/endpoints/mod.rs | 219 ++++++++++++++++++++++++++ pkarr/src/lib.rs | 1 + 5 files changed, 450 insertions(+), 9 deletions(-) create mode 100644 pkarr/src/extra/endpoints/endpoint.rs create mode 100644 pkarr/src/extra/endpoints/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 61cbd97..3ded479 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1841,9 +1841,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.12" +version = "0.23.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8" dependencies = [ "once_cell", "ring", @@ -1871,9 +1871,9 @@ checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" [[package]] name = "rustls-webpki" -version = "0.102.6" +version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" dependencies = [ "ring", "rustls-pki-types", @@ -2168,9 +2168,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.74" +version = "2.0.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fceb41e3d546d0bd83421d3409b1460cc7444cd389341a4c880fe7a042cb3d7" +checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" dependencies = [ "proc-macro2", "quote", @@ -2668,9 +2668,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.26.3" +version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd7c23921eeb1713a4e851530e9b9756e4fb0e89978582942612524cf09f01cd" +checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958" dependencies = [ "rustls-pki-types", ] diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index fb62494..03911f5 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -57,8 +57,12 @@ dht = ["dep:mainline", "flume"] ## Use [relay::Client] relay = ["dep:reqwest", "dep:tokio"] +# Extra +## +endpoints = [] + ## Use all features -full = ["dht", "relay", "rand", "serde"] +full = ["dht", "relay", "rand", "serde", "endpoints"] default = ["dht", "rand", "serde"] diff --git a/pkarr/src/extra/endpoints/endpoint.rs b/pkarr/src/extra/endpoints/endpoint.rs new file mode 100644 index 0000000..9cfb01b --- /dev/null +++ b/pkarr/src/extra/endpoints/endpoint.rs @@ -0,0 +1,217 @@ +use crate::{ + dns::{ + rdata::{RData, SVCB}, + ResourceRecord, + }, + SignedPacket, +}; +use std::net::{IpAddr, SocketAddr, ToSocketAddrs}; + +use crate::system_time; + +#[derive(Debug, Clone)] +pub struct Endpoint { + pub(crate) target: String, + // public_key: PublicKey, + pub(crate) port: u16, + pub(crate) addrs: Vec, +} + +impl Endpoint { + /// 1. Find the SVCB or HTTPS records with the lowest priority + /// 2. Choose a random one of the list of the above + /// 3. If the target is `.`, check A and AAAA records (https://www.rfc-editor.org/rfc/rfc9460#name-special-handling-of-in-targ) + pub(crate) fn find( + signed_packet: &SignedPacket, + target: &str, + is_svcb: bool, + ) -> Option { + let mut lowest_priority = u16::MAX; + let mut lowest_priority_index = 0; + let mut records = vec![]; + + for record in signed_packet.resource_records(target) { + if let Some(svcb) = get_svcb(record, is_svcb) { + match svcb.priority.cmp(&lowest_priority) { + std::cmp::Ordering::Equal => records.push(svcb), + std::cmp::Ordering::Less => { + lowest_priority_index = records.len(); + lowest_priority = svcb.priority; + records.push(svcb) + } + _ => {} + } + } + } + + // Good enough random selection + let now = system_time(); + let slice = &records[lowest_priority_index..]; + let index = if slice.is_empty() { + 0 + } else { + (now as usize) % slice.len() + }; + + slice.get(index).map(|s| { + let target = s.target.to_string(); + + let mut addrs: Vec = vec![]; + + if &target == "." { + for record in signed_packet.resource_records("@") { + match &record.rdata { + RData::A(ip) => addrs.push(IpAddr::V4(ip.address.into())), + RData::AAAA(ip) => addrs.push(IpAddr::V6(ip.address.into())), + _ => {} + } + } + } + + Endpoint { + target, + // public_key: signed_packet.public_key(), + port: u16::from_be_bytes( + s.get_param(SVCB::PORT) + .unwrap_or_default() + .try_into() + .unwrap_or([0, 0]), + ), + addrs, + } + }) + } + + pub fn to_socket_addrs(&self) -> std::io::Result> { + if self.target == "." { + let port = self.port; + return Ok(self + .addrs + .iter() + .map(|addr| SocketAddr::from((*addr, port))) + .collect::>() + .into_iter()); + } + + format!("{}:{}", self.target, self.port).to_socket_addrs() + } +} + +fn get_svcb<'a>(record: &'a ResourceRecord, is_svcb: bool) -> Option<&'a SVCB<'a>> { + match &record.rdata { + RData::SVCB(svcb) => { + if is_svcb { + Some(svcb) + } else { + None + } + } + + RData::HTTPS(curr) => { + if is_svcb { + None + } else { + Some(&curr.0) + } + } + _ => None, + } +} + +#[cfg(test)] +mod tests { + use std::net::{Ipv4Addr, Ipv6Addr}; + use std::str::FromStr; + + use super::*; + + use crate::{dns, Keypair}; + + #[tokio::test] + async fn endpoint_target() { + let mut packet = dns::Packet::new_reply(0); + packet.answers.push(dns::ResourceRecord::new( + dns::Name::new("foo").unwrap(), + dns::CLASS::IN, + 3600, + RData::HTTPS(SVCB::new(0, "https.example.com".try_into().unwrap()).into()), + )); + // Make sure HTTPS only follows HTTPs + packet.answers.push(dns::ResourceRecord::new( + dns::Name::new("foo").unwrap(), + dns::CLASS::IN, + 3600, + RData::SVCB(SVCB::new(0, "protocol.example.com".try_into().unwrap())), + )); + // Make sure SVCB only follows SVCB + packet.answers.push(dns::ResourceRecord::new( + dns::Name::new("foo").unwrap(), + dns::CLASS::IN, + 3600, + RData::HTTPS(SVCB::new(0, "https.example.com".try_into().unwrap()).into()), + )); + packet.answers.push(dns::ResourceRecord::new( + dns::Name::new("_foo").unwrap(), + dns::CLASS::IN, + 3600, + RData::SVCB(SVCB::new(0, "protocol.example.com".try_into().unwrap())), + )); + let keypair = Keypair::random(); + let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); + + let tld = keypair.public_key(); + + // Follow foo.tld HTTPS records + let endpoint = Endpoint::find(&signed_packet, &format!("foo.{tld}"), false).unwrap(); + assert_eq!(endpoint.target, "https.example.com"); + + // Follow _foo.tld SVCB records + let endpoint = Endpoint::find(&signed_packet, &format!("_foo.{tld}"), true).unwrap(); + assert_eq!(endpoint.target, "protocol.example.com"); + } + + #[test] + fn endpoint_to_socket_addrs() { + let mut packet = dns::Packet::new_reply(0); + packet.answers.push(dns::ResourceRecord::new( + dns::Name::new("@").unwrap(), + dns::CLASS::IN, + 3600, + RData::A(Ipv4Addr::from_str("209.151.148.15").unwrap().into()), + )); + packet.answers.push(dns::ResourceRecord::new( + dns::Name::new("@").unwrap(), + dns::CLASS::IN, + 3600, + RData::AAAA(Ipv6Addr::from_str("2a05:d014:275:6201::64").unwrap().into()), + )); + + let mut svcb = SVCB::new(1, ".".try_into().unwrap()); + svcb.set_port(6881); + + packet.answers.push(dns::ResourceRecord::new( + dns::Name::new("@").unwrap(), + dns::CLASS::IN, + 3600, + RData::HTTPS(svcb.into()), + )); + let keypair = Keypair::random(); + let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); + + // Follow foo.tld HTTPS records + let endpoint = Endpoint::find( + &signed_packet, + &signed_packet.public_key().to_string(), + false, + ) + .unwrap(); + + assert_eq!(endpoint.target, "."); + + let addrs = endpoint.to_socket_addrs().unwrap(); + assert_eq!( + addrs.map(|s| s.to_string()).collect::>(), + vec!["209.151.148.15:6881", "[2a05:d014:275:6201::64]:6881"] + ) + } +} diff --git a/pkarr/src/extra/endpoints/mod.rs b/pkarr/src/extra/endpoints/mod.rs new file mode 100644 index 0000000..44d428a --- /dev/null +++ b/pkarr/src/extra/endpoints/mod.rs @@ -0,0 +1,219 @@ +mod endpoint; + +use crate::{ + error::{Error, Result}, + Client, PublicKey, +}; + +use endpoint::Endpoint; + +const DEFAULT_MAX_CHAIN_LENGTH: u8 = 3; + +#[derive(Debug, Clone)] +pub struct EndpointResolver { + pkarr: Client, + max_chain_length: u8, +} + +impl EndpointResolver { + pub fn new(pkarr: Client, max_chain_length: u8) -> Self { + EndpointResolver { + pkarr, + max_chain_length, + } + } + + /// Resolve a `qname` to an alternative [Endpoint] as defined in [RFC9460](https://www.rfc-editor.org/rfc/rfc9460#name-terminology). + /// + /// A `qname` is can be either a regular domain name for HTTPS endpoints, + /// or it could use Attrleaf naming pattern for cusotm protcol. For example: + /// `_foo.example.com` for `foo://example.com`. + async fn resolve_endpoint(&self, qname: &str) -> Result { + let target = qname; + // TODO: cache the result of this function? + + let is_svcb = target.starts_with('_'); + + let mut step = 0; + let mut svcb: Option = None; + + loop { + let current = svcb.clone().map_or(target.to_string(), |s| s.target); + if let Ok(tld) = PublicKey::try_from(current.clone()) { + if let Ok(Some(signed_packet)) = self.pkarr.resolve(&tld).await { + if step >= self.max_chain_length { + break; + }; + step += 1; + + // Choose most prior SVCB record + svcb = Endpoint::find(&signed_packet, ¤t, is_svcb); + + // TODO: support wildcard? + } else { + break; + } + } else { + break; + } + } + + if let Some(svcb) = svcb { + if PublicKey::try_from(svcb.target.as_str()).is_err() { + return Ok(svcb); + } + } + + Err(Error::Generic(format!( + "Failed to find an endopint {}", + target + ))) + } +} + +impl From<&Client> for EndpointResolver { + fn from(pkarr: &Client) -> Self { + pkarr.clone().into() + } +} + +impl From for EndpointResolver { + /// Creates [EndpointResolver] from [Client] and default settings + fn from(pkarr: Client) -> Self { + Self::new(pkarr, DEFAULT_MAX_CHAIN_LENGTH) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::dns::rdata::{A, SVCB}; + use crate::dns::{self, rdata::RData}; + use crate::SignedPacket; + use crate::{mainline::Testnet, Keypair}; + + use std::future::Future; + use std::pin::Pin; + + fn generate_subtree( + client: Client, + depth: u8, + branching: u8, + domain: Option, + ) -> Pin>> { + Box::pin(async move { + let keypair = Keypair::random(); + + let mut packet = dns::Packet::new_reply(0); + + for _ in 0..branching { + let mut svcb = SVCB::new(0, ".".try_into().unwrap()); + + if depth == 0 { + svcb.priority = 1; + svcb.set_port((branching) as u16 * 1000); + + if let Some(target) = &domain { + let target: &'static str = Box::leak(target.clone().into_boxed_str()); + svcb.target = target.try_into().unwrap() + } + } else { + let target = + generate_subtree(client.clone(), depth - 1, branching, domain.clone()) + .await + .to_string(); + let target: &'static str = Box::leak(target.into_boxed_str()); + svcb.target = target.try_into().unwrap(); + }; + + packet.answers.push(dns::ResourceRecord::new( + dns::Name::new("@").unwrap(), + dns::CLASS::IN, + 3600, + RData::HTTPS(svcb.into()), + )); + } + + if depth == 0 { + packet.answers.push(dns::ResourceRecord::new( + dns::Name::new("@").unwrap(), + dns::CLASS::IN, + 3600, + RData::A(A { address: 10 }), + )); + } + + let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); + client.publish(&signed_packet).await.unwrap(); + + keypair.public_key() + }) + } + + fn generate( + client: Client, + depth: u8, + branching: u8, + domain: Option, + ) -> Pin>> { + generate_subtree(client, depth - 1, branching, domain) + } + + #[tokio::test] + async fn resolve_endpoints() { + let testnet = Testnet::new(3); + let pkarr = Client::builder().testnet(&testnet).build().unwrap(); + + let resolver: EndpointResolver = (&pkarr).into(); + let tld = generate(pkarr, 3, 3, Some("example.com".to_string())).await; + + let endpoint = resolver.resolve_endpoint(&tld.to_string()).await.unwrap(); + assert_eq!(endpoint.target, "example.com"); + } + + #[tokio::test] + async fn max_chain_exceeded() { + let testnet = Testnet::new(3); + let pkarr = Client::builder().testnet(&testnet).build().unwrap(); + + let resolver: EndpointResolver = (&pkarr).into(); + + let tld = generate(pkarr, 4, 3, Some("example.com".to_string())).await; + + let endpoint = resolver.resolve_endpoint(&tld.to_string()).await; + + assert!(endpoint.is_err()); + // TODO: test error correctly + + // assert_eq!( + // match endpoint { + // Err(error) => error.to_string(), + // _ => "".to_string(), + // }, + // Error::Generic(tld.to_string()) + // ) + } + + #[tokio::test] + async fn resolve_addresses() { + let testnet = Testnet::new(3); + let pkarr = Client::builder().testnet(&testnet).build().unwrap(); + + let resolver: EndpointResolver = (&pkarr).into(); + let tld = generate(pkarr, 3, 3, None).await; + + let endpoint = resolver.resolve_endpoint(&tld.to_string()).await.unwrap(); + assert_eq!(endpoint.target, "."); + assert_eq!(endpoint.port, 3000); + assert_eq!( + endpoint + .to_socket_addrs() + .unwrap() + .into_iter() + .map(|s| s.to_string()) + .collect::>(), + vec!["0.0.0.10:3000"] + ); + dbg!(&endpoint); + } +} diff --git a/pkarr/src/lib.rs b/pkarr/src/lib.rs index 51f5c46..6b5f7b3 100644 --- a/pkarr/src/lib.rs +++ b/pkarr/src/lib.rs @@ -7,6 +7,7 @@ mod base; mod client; mod error; +mod extra; // Exports pub use base::cache::{Cache, CacheKey, InMemoryCache}; From b8d159797cbf8e137f51898b56ba7b78e37c96a8 Mon Sep 17 00:00:00 2001 From: nazeh Date: Tue, 24 Sep 2024 20:28:10 +0300 Subject: [PATCH 21/84] wip(pkarr): add EndpointResolver --- pkarr/src/error.rs | 2 +- pkarr/src/extra/endpoints/endpoint.rs | 2 +- pkarr/src/extra/mod.rs | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 pkarr/src/extra/mod.rs diff --git a/pkarr/src/error.rs b/pkarr/src/error.rs index fda8f6b..1c2dbe2 100644 --- a/pkarr/src/error.rs +++ b/pkarr/src/error.rs @@ -62,7 +62,7 @@ pub enum Error { DhtIsShutdown, #[error("Publish query is already inflight for the same public_key")] - /// [crate::dht::Client::publish] is already inflight to the same public_key + /// [crate::Client::publish] is already inflight to the same public_key PublishInflight, #[error("SignedPacket's timestamp is not the most recent")] diff --git a/pkarr/src/extra/endpoints/endpoint.rs b/pkarr/src/extra/endpoints/endpoint.rs index 9cfb01b..738cc3c 100644 --- a/pkarr/src/extra/endpoints/endpoint.rs +++ b/pkarr/src/extra/endpoints/endpoint.rs @@ -20,7 +20,7 @@ pub struct Endpoint { impl Endpoint { /// 1. Find the SVCB or HTTPS records with the lowest priority /// 2. Choose a random one of the list of the above - /// 3. If the target is `.`, check A and AAAA records (https://www.rfc-editor.org/rfc/rfc9460#name-special-handling-of-in-targ) + /// 3. If the target is `.`, check A and AAAA records see [rfc9460](https://www.rfc-editor.org/rfc/rfc9460#name-special-handling-of-in-targ) pub(crate) fn find( signed_packet: &SignedPacket, target: &str, diff --git a/pkarr/src/extra/mod.rs b/pkarr/src/extra/mod.rs new file mode 100644 index 0000000..3aa2c45 --- /dev/null +++ b/pkarr/src/extra/mod.rs @@ -0,0 +1,2 @@ +#[cfg(feature = "endpoints")] +mod endpoints; From 1ee9ff85b89a3e03e09eb19fe6f5377a9076a8d4 Mon Sep 17 00:00:00 2001 From: nazeh Date: Wed, 25 Sep 2024 09:53:23 +0300 Subject: [PATCH 22/84] feat(pkarr): add monotonic Timestamp --- Cargo.lock | 1 + pkarr/Cargo.toml | 5 +- pkarr/src/base/mod.rs | 2 +- pkarr/src/base/signed_packet.rs | 36 +-- pkarr/src/base/time.rs | 20 -- pkarr/src/base/timestamp.rs | 317 ++++++++++++++++++++++++++ pkarr/src/extra/endpoints/endpoint.rs | 4 +- pkarr/src/lib.rs | 2 +- server/src/cache.rs | 18 +- 9 files changed, 354 insertions(+), 51 deletions(-) delete mode 100644 pkarr/src/base/time.rs create mode 100644 pkarr/src/base/timestamp.rs diff --git a/Cargo.lock b/Cargo.lock index 3ded479..bf46ecd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1476,6 +1476,7 @@ dependencies = [ "lru", "mainline", "mockito", + "once_cell", "postcard", "rand", "reqwest", diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index 03911f5..634d616 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -17,11 +17,14 @@ simple-dns = "0.6.1" thiserror = "1.0.49" tracing = "0.1.40" dyn-clone = "1.0.17" +once_cell = {version = "1.19.0", default-features = false } lru = { version = "0.12.3", default-features = false } +document-features = "0.2.8" + +# optional dependencies serde = { version = "1.0.209", features = ["derive"], optional = true } rand = { version = "0.8.5", optional = true } -document-features = "0.2.8" [target.'cfg(not(target_arch = "wasm32"))'.dependencies] # Dht client dependencies: diff --git a/pkarr/src/base/mod.rs b/pkarr/src/base/mod.rs index 7588c5b..ceb8198 100644 --- a/pkarr/src/base/mod.rs +++ b/pkarr/src/base/mod.rs @@ -3,4 +3,4 @@ pub mod cache; pub mod keys; pub mod signed_packet; -pub mod time; +pub mod timestamp; diff --git a/pkarr/src/base/signed_packet.rs b/pkarr/src/base/signed_packet.rs index ec75858..fdd091e 100644 --- a/pkarr/src/base/signed_packet.rs +++ b/pkarr/src/base/signed_packet.rs @@ -14,7 +14,7 @@ use std::{ net::{Ipv4Addr, Ipv6Addr}, }; -use crate::system_time; +use crate::Timestamp; const DOT: char = '.'; @@ -60,7 +60,7 @@ impl Inner { /// Signed DNS packet pub struct SignedPacket { inner: Inner, - last_seen: u64, + last_seen: Timestamp, } impl SignedPacket { @@ -102,16 +102,16 @@ impl SignedPacket { Ok(SignedPacket { inner: Inner::try_from_bytes(bytes)?, - last_seen: system_time(), + last_seen: Timestamp::now(), }) } /// Useful for cloning a [SignedPacket], or cerating one from a previously checked bytes, /// like ones stored on disk or in a database. - pub fn from_bytes_unchecked(bytes: &Bytes, last_seen: u64) -> SignedPacket { + pub fn from_bytes_unchecked(bytes: &Bytes, last_seen: impl Into) -> SignedPacket { SignedPacket { inner: Inner::try_from_bytes(bytes).unwrap(), - last_seen, + last_seen: last_seen.into(), } } @@ -172,7 +172,7 @@ impl SignedPacket { return Err(Error::PacketTooLarge(encoded_packet.len())); } - let timestamp = system_time(); + let timestamp = Timestamp::now().into(); let signature = keypair.sign(&signable(timestamp, &encoded_packet)); @@ -183,7 +183,7 @@ impl SignedPacket { timestamp, &encoded_packet, )?, - last_seen: system_time(), + last_seen: Timestamp::now(), }) } @@ -236,22 +236,22 @@ impl SignedPacket { } /// Unix last_seen time in microseconds - pub fn last_seen(&self) -> &u64 { + pub fn last_seen(&self) -> &Timestamp { &self.last_seen } // === Setters === /// Set the [Self::last_seen] property - pub fn set_last_seen(&mut self, last_seen: &u64) { - self.last_seen = *last_seen; + pub fn set_last_seen(&mut self, last_seen: &Timestamp) { + self.last_seen = last_seen.into(); } // === Public Methods === /// Set the [Self::last_seen] to the current system time pub fn refresh(&mut self) { - self.last_seen = system_time(); + self.last_seen = Timestamp::now(); } /// Return whether this [SignedPacket] is more recent than the given one. @@ -329,7 +329,7 @@ impl SignedPacket { /// Time since the [Self::last_seen] in seconds fn elapsed(&self) -> u32 { - ((system_time() - self.last_seen) / 1_000_000) as u32 + ((Timestamp::now().into_u64() - self.last_seen.into_u64()) / 1_000_000) as u32 } } @@ -369,7 +369,7 @@ impl TryFrom<&MutableItem> for SignedPacket { Ok(Self { inner: Inner::try_from_parts(&public_key, &signature, seq, i.value())?, - last_seen: system_time(), + last_seen: Timestamp::now(), }) } } @@ -384,7 +384,7 @@ impl AsRef<[u8]> for SignedPacket { impl Clone for SignedPacket { fn clone(&self) -> Self { - Self::from_bytes_unchecked(self.as_bytes(), self.last_seen) + Self::from_bytes_unchecked(self.as_bytes(), &self.last_seen) } } @@ -651,7 +651,7 @@ mod tests { let bytes = signed.as_bytes(); let from_bytes = SignedPacket::from_bytes(bytes).unwrap(); assert_eq!(signed.as_bytes(), from_bytes.as_bytes()); - let from_bytes2 = SignedPacket::from_bytes_unchecked(bytes, signed.last_seen); + let from_bytes2 = SignedPacket::from_bytes_unchecked(bytes, &signed.last_seen); assert_eq!(signed.as_bytes(), from_bytes2.as_bytes()); let public_key = keypair.public_key(); @@ -690,7 +690,7 @@ mod tests { let mut signed = SignedPacket::from_packet(&keypair, &packet).unwrap(); - signed.last_seen = system_time() - (20 * 1_000_000); + signed.last_seen -= 20 * 1_000_000_u64; assert!( signed.expires_in(30, u32::MAX) > 0, @@ -716,7 +716,7 @@ mod tests { let mut signed = SignedPacket::from_packet(&keypair, &packet).unwrap(); - signed.last_seen = system_time() - (2 * (DEFAULT_MAXIMUM_TTL as u64) * 1_000_000); + signed.last_seen -= 2 * (DEFAULT_MAXIMUM_TTL as u64) * 1_000_000; assert!( signed.expires_in(0, DEFAULT_MAXIMUM_TTL) == 0, @@ -748,7 +748,7 @@ mod tests { let mut signed = SignedPacket::from_packet(&keypair, &packet).unwrap(); - signed.last_seen = system_time() - (30 * 1_000_000); + signed.last_seen -= 30 * 1_000_000; assert_eq!(signed.fresh_resource_records("_foo").count(), 1); } diff --git a/pkarr/src/base/time.rs b/pkarr/src/base/time.rs deleted file mode 100644 index 1fc8d3a..0000000 --- a/pkarr/src/base/time.rs +++ /dev/null @@ -1,20 +0,0 @@ -#[cfg(not(target_arch = "wasm32"))] -use std::time::SystemTime; - -#[cfg(not(target_arch = "wasm32"))] -/// Return the number of microseconds since [SystemTime::UNIX_EPOCH] -pub fn system_time() -> u64 { - SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .expect("time drift") - .as_micros() as u64 -} - -#[cfg(target_arch = "wasm32")] -/// Return the number of microseconds since [SystemTime::UNIX_EPOCH] -pub fn system_time() -> u64 { - // Won't be an issue for more than 5000 years! - (js_sys::Date::now() as u64 ) - // Turn miliseconds to microseconds - * 1000 -} diff --git a/pkarr/src/base/timestamp.rs b/pkarr/src/base/timestamp.rs new file mode 100644 index 0000000..f0c937b --- /dev/null +++ b/pkarr/src/base/timestamp.rs @@ -0,0 +1,317 @@ +//! Strictly monotonic unix timestamp in microseconds + +#[cfg(feature = "serde")] +use serde::{Deserialize, Serialize}; +use std::fmt::Display; +use std::{ + ops::{Add, AddAssign, Sub, SubAssign}, + sync::Mutex, +}; + +use once_cell::sync::Lazy; +use rand::Rng; + +#[cfg(not(target_arch = "wasm32"))] +use std::time::SystemTime; + +/// ~4% chance of none of 10 clocks have matching id. +const CLOCK_MASK: u64 = (1 << 8) - 1; +const TIME_MASK: u64 = !0 >> 8; + +pub struct TimestampFactory { + clock_id: u64, + last_time: u64, +} + +impl TimestampFactory { + pub fn new() -> Self { + Self { + clock_id: rand::thread_rng().gen::() & CLOCK_MASK, + last_time: system_time() & TIME_MASK, + } + } + + pub fn now(&mut self) -> Timestamp { + // Ensure strict monotonicity. + self.last_time = (system_time() & TIME_MASK).max(self.last_time + CLOCK_MASK + 1); + + // Add clock_id to the end of the timestamp + Timestamp(self.last_time | self.clock_id) + } +} + +impl Default for TimestampFactory { + fn default() -> Self { + Self::new() + } +} + +static DEFAULT_FACTORY: Lazy> = + Lazy::new(|| Mutex::new(TimestampFactory::default())); + +/// STrictly monotonic timestamp since [SystemTime::UNIX_EPOCH] in microseconds as u64. +/// +/// The purpose of this timestamp is to unique per "user", not globally, +/// it achieves this by: +/// 1. Override the last byte with a random `clock_id`, reducing the probability +/// of two matching timestamps across multiple machines/threads. +/// 2. Gurantee that the remaining 3 bytes are ever increasing (strictly monotonic) within +/// the same thread regardless of the wall clock value +/// +/// This timestamp is also serialized as BE bytes to remain sortable. +/// If a `utf-8` encoding is necessary, it is encoded as [base32::Alphabet::Crockford] +/// to act as a sortable Id. +/// +/// U64 of microseconds is valid for the next 500 thousand years! +#[derive(Debug, Clone, PartialEq, PartialOrd, Hash, Eq, Ord)] +pub struct Timestamp(u64); + +impl Timestamp { + pub fn now() -> Self { + DEFAULT_FACTORY.lock().unwrap().now() + } + + /// Return big endian bytes + pub fn to_bytes(&self) -> [u8; 8] { + self.0.to_be_bytes() + } + + pub fn difference(&self, rhs: &Timestamp) -> i64 { + (self.0 as i64) - (rhs.0 as i64) + } + + pub fn into_u64(&self) -> u64 { + self.0 + } +} + +impl Default for Timestamp { + fn default() -> Self { + Timestamp::now() + } +} + +impl TryFrom<&[u8]> for Timestamp { + type Error = TimestampError; + + fn try_from(bytes: &[u8]) -> Result { + let bytes: [u8; 8] = bytes + .try_into() + .map_err(|_| TimestampError::InvalidBytesLength(bytes.len()))?; + + Ok(bytes.into()) + } +} + +impl From<&Timestamp> for [u8; 8] { + fn from(timestamp: &Timestamp) -> Self { + timestamp.0.to_be_bytes() + } +} + +impl From<[u8; 8]> for Timestamp { + fn from(bytes: [u8; 8]) -> Self { + Self(u64::from_be_bytes(bytes)) + } +} + +impl From for Timestamp { + fn from(inner: u64) -> Self { + Self(inner) + } +} + +impl From<&Timestamp> for Timestamp { + fn from(timestamp: &Timestamp) -> Self { + timestamp.clone() + } +} + +impl From for u64 { + fn from(value: Timestamp) -> Self { + value.into_u64() + } +} + +impl From<&Timestamp> for u64 { + fn from(value: &Timestamp) -> Self { + value.into_u64() + } +} + +// === U64 conversion === + +impl Add for &Timestamp { + type Output = Timestamp; + + fn add(self, rhs: u64) -> Self::Output { + Timestamp(self.0 + rhs) + } +} + +impl Sub for &Timestamp { + type Output = Timestamp; + + fn sub(self, rhs: u64) -> Self::Output { + Timestamp(self.0 - rhs) + } +} + +impl AddAssign for Timestamp { + fn add_assign(&mut self, other: u64) { + self.0 += other; + } +} + +impl SubAssign for Timestamp { + fn sub_assign(&mut self, other: u64) { + self.0 -= other; + } +} + +// === Serialization === + +#[cfg(feature = "serde")] +impl Serialize for Timestamp { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let bytes = self.to_bytes(); + bytes.serialize(serializer) + } +} + +#[cfg(feature = "serde")] +impl<'de> Deserialize<'de> for Timestamp { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let bytes: [u8; 8] = Deserialize::deserialize(deserializer)?; + Ok(Timestamp(u64::from_be_bytes(bytes))) + } +} + +// === String representation (sortable base32 encoding) === + +impl Display for Timestamp { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let bytes: [u8; 8] = self.into(); + f.write_str(&base32::encode(base32::Alphabet::Crockford, &bytes)) + } +} + +impl TryFrom for Timestamp { + type Error = TimestampError; + + fn try_from(value: String) -> Result { + match base32::decode(base32::Alphabet::Crockford, &value) { + Some(vec) => { + let bytes: [u8; 8] = vec + .try_into() + .map_err(|_| TimestampError::InvalidEncoding)?; + + Ok(bytes.into()) + } + None => Err(TimestampError::InvalidEncoding), + } + } +} + +#[cfg(not(target_arch = "wasm32"))] +/// Return the number of microseconds since [SystemTime::UNIX_EPOCH] +fn system_time() -> u64 { + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .expect("time drift") + .as_micros() as u64 +} + +#[cfg(target_arch = "wasm32")] +/// Return the number of microseconds since [SystemTime::UNIX_EPOCH] +pub fn system_time() -> u64 { + // Won't be an issue for more than 5000 years! + (js_sys::Date::now() as u64 ) + // Turn miliseconds to microseconds + * 1000 +} + +#[derive(thiserror::Error, Debug)] +pub enum TimestampError { + #[error("Invalid bytes length, Timestamp should be encoded as 8 bytes, got {0}")] + InvalidBytesLength(usize), + #[error("Invalid timestamp encoding")] + InvalidEncoding, +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + + use super::*; + + #[test] + fn strictly_monotonic() { + const COUNT: usize = 100; + + let mut set = HashSet::with_capacity(COUNT); + let mut vec = Vec::with_capacity(COUNT); + + for _ in 0..COUNT { + let timestamp = Timestamp::now(); + + set.insert(timestamp.clone()); + vec.push(timestamp); + } + + let mut ordered = vec.clone(); + ordered.sort(); + + assert_eq!(set.len(), COUNT, "unique"); + assert_eq!(ordered, vec, "ordered"); + } + + #[test] + fn strings() { + const COUNT: usize = 100; + + let mut set = HashSet::with_capacity(COUNT); + let mut vec = Vec::with_capacity(COUNT); + + for _ in 0..COUNT { + let string = Timestamp::now().to_string(); + + set.insert(string.clone()); + vec.push(string) + } + + let mut ordered = vec.clone(); + ordered.sort(); + + assert_eq!(set.len(), COUNT, "unique"); + assert_eq!(ordered, vec, "ordered"); + } + + #[test] + fn to_from_string() { + let timestamp = Timestamp::now(); + let string = timestamp.to_string(); + let decoded: Timestamp = string.try_into().unwrap(); + + assert_eq!(decoded, timestamp) + } + + #[test] + fn serde() { + let timestamp = Timestamp::now(); + + let serialized = postcard::to_allocvec(×tamp).unwrap(); + + assert_eq!(serialized, timestamp.to_bytes()); + + let deserialized: Timestamp = postcard::from_bytes(&serialized).unwrap(); + + assert_eq!(deserialized, timestamp); + } +} diff --git a/pkarr/src/extra/endpoints/endpoint.rs b/pkarr/src/extra/endpoints/endpoint.rs index 738cc3c..1461fe0 100644 --- a/pkarr/src/extra/endpoints/endpoint.rs +++ b/pkarr/src/extra/endpoints/endpoint.rs @@ -7,7 +7,7 @@ use crate::{ }; use std::net::{IpAddr, SocketAddr, ToSocketAddrs}; -use crate::system_time; +use crate::Timestamp; #[derive(Debug, Clone)] pub struct Endpoint { @@ -45,7 +45,7 @@ impl Endpoint { } // Good enough random selection - let now = system_time(); + let now = Timestamp::now().into_u64(); let slice = &records[lowest_priority_index..]; let index = if slice.is_empty() { 0 diff --git a/pkarr/src/lib.rs b/pkarr/src/lib.rs index 6b5f7b3..350fb2e 100644 --- a/pkarr/src/lib.rs +++ b/pkarr/src/lib.rs @@ -13,7 +13,7 @@ mod extra; pub use base::cache::{Cache, CacheKey, InMemoryCache}; pub use base::keys::{Keypair, PublicKey}; pub use base::signed_packet::SignedPacket; -pub use base::time::system_time; +pub use base::timestamp::Timestamp; pub use error::{Error, Result}; /// Default minimum TTL: 5 minutes diff --git a/server/src/cache.rs b/server/src/cache.rs index d54d43b..34c48f8 100644 --- a/server/src/cache.rs +++ b/server/src/cache.rs @@ -1,6 +1,6 @@ use std::{borrow::Cow, path::Path, time::Duration}; -use pkarr::{system_time, Cache, CacheKey, SignedPacket}; +use pkarr::{Cache, CacheKey, SignedPacket, Timestamp}; use byteorder::LittleEndian; use heed::{types::U64, BoxedError, BytesDecode, BytesEncode, Database, Env, EnvOpenOptions}; @@ -45,7 +45,9 @@ impl<'a> BytesEncode<'a> for SignedPacketCodec { let mut vec = Vec::with_capacity(bytes.len() + 8); - vec.extend(>::bytes_encode(signed_packet.last_seen())?.as_ref()); + vec.extend( + >::bytes_encode(&signed_packet.last_seen().into_u64())?.as_ref(), + ); vec.extend(bytes); Ok(Cow::Owned(vec)) @@ -154,10 +156,10 @@ impl HeedCache { time_to_key.delete(&mut wtxn, &old_time)?; } - let new_time = system_time(); + let new_time = Timestamp::now(); - time_to_key.put(&mut wtxn, &new_time, key)?; - key_to_time.put(&mut wtxn, key, &new_time)?; + time_to_key.put(&mut wtxn, &new_time.into_u64(), key)?; + key_to_time.put(&mut wtxn, key, &new_time.into_u64())?; packets.put(&mut wtxn, key, signed_packet)?; @@ -188,10 +190,10 @@ impl HeedCache { time_to_key.delete(&mut wtxn, &time)?; }; - let new_time = system_time(); + let new_time = Timestamp::now(); - time_to_key.put(&mut wtxn, &new_time, key)?; - key_to_time.put(&mut wtxn, key, &new_time)?; + time_to_key.put(&mut wtxn, &new_time.into_u64(), key)?; + key_to_time.put(&mut wtxn, key, &new_time.into_u64())?; wtxn.commit()?; From 5a28b2666491a838de41452b451e1a8fca23843f Mon Sep 17 00:00:00 2001 From: nazeh Date: Wed, 25 Sep 2024 12:27:06 +0300 Subject: [PATCH 23/84] feat(pkarr): remove rand feature --- pkarr/Cargo.toml | 4 +--- pkarr/src/base/keys.rs | 2 -- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index 634d616..2f5ccdd 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -11,7 +11,7 @@ keywords = ["mainline", "dht", "dns", "decentralized", "identity"] [dependencies] base32 = "0.5.0" bytes = "1.7.1" -ed25519-dalek = "2.0.0" +ed25519-dalek = { version = "2.0.0", features = ["rand_core"] } self_cell = "1.0.2" simple-dns = "0.6.1" thiserror = "1.0.49" @@ -49,8 +49,6 @@ mockito = "1.4.0" tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } [features] -## Use [Keypair::random] -rand = ["dep:rand", "ed25519-dalek/rand_core"] ## Derive serde Serialize/Deserialize for PublicKey serde = ["dep:serde"] diff --git a/pkarr/src/base/keys.rs b/pkarr/src/base/keys.rs index c1966a8..3a97d86 100644 --- a/pkarr/src/base/keys.rs +++ b/pkarr/src/base/keys.rs @@ -2,7 +2,6 @@ use crate::{Error, Result}; use ed25519_dalek::{SecretKey, Signature, Signer, SigningKey, Verifier, VerifyingKey}; -#[cfg(feature = "rand")] use rand::rngs::OsRng; use std::{ fmt::{self, Debug, Display, Formatter}, @@ -17,7 +16,6 @@ use serde::{Deserialize, Serialize}; pub struct Keypair(SigningKey); impl Keypair { - #[cfg(feature = "rand")] pub fn random() -> Keypair { let mut csprng = OsRng; let signing_key: SigningKey = SigningKey::generate(&mut csprng); From 945ab1ad8ee318ceb14bbe98817aeea0d3957989 Mon Sep 17 00:00:00 2001 From: nazeh Date: Wed, 25 Sep 2024 14:29:53 +0300 Subject: [PATCH 24/84] feat(pkarr): add resolve_endpoint method to both clients --- pkarr/Cargo.toml | 3 +- pkarr/src/extra/endpoints/endpoint.rs | 4 + pkarr/src/extra/endpoints/mod.rs | 105 +++++++++++--------------- 3 files changed, 50 insertions(+), 62 deletions(-) diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index 2f5ccdd..d9deb61 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -25,7 +25,6 @@ document-features = "0.2.8" serde = { version = "1.0.209", features = ["derive"], optional = true } rand = { version = "0.8.5", optional = true } - [target.'cfg(not(target_arch = "wasm32"))'.dependencies] # Dht client dependencies: mainline = { version = "2.0.1", optional = true } @@ -65,7 +64,7 @@ endpoints = [] ## Use all features full = ["dht", "relay", "rand", "serde", "endpoints"] -default = ["dht", "rand", "serde"] +default = ["full"] [package.metadata.docs.rs] all-features = true diff --git a/pkarr/src/extra/endpoints/endpoint.rs b/pkarr/src/extra/endpoints/endpoint.rs index 1461fe0..bbe9c16 100644 --- a/pkarr/src/extra/endpoints/endpoint.rs +++ b/pkarr/src/extra/endpoints/endpoint.rs @@ -10,6 +10,7 @@ use std::net::{IpAddr, SocketAddr, ToSocketAddrs}; use crate::Timestamp; #[derive(Debug, Clone)] +/// An alternative Endpoint for a `qname`, from either [RData::SVCB] or [RData::HTTPS] dns records pub struct Endpoint { pub(crate) target: String, // public_key: PublicKey, @@ -82,6 +83,9 @@ impl Endpoint { }) } + /// Return an iterator of [SocketAddr], either by resolving the [Endpoint::target] using normal DNS, + /// or, if the target is ".", return the [RData::A] or [RData::AAAA] records + /// from the endpoint's [SignedPacket], if available. pub fn to_socket_addrs(&self) -> std::io::Result> { if self.target == "." { let port = self.port; diff --git a/pkarr/src/extra/endpoints/mod.rs b/pkarr/src/extra/endpoints/mod.rs index 44d428a..712c273 100644 --- a/pkarr/src/extra/endpoints/mod.rs +++ b/pkarr/src/extra/endpoints/mod.rs @@ -1,33 +1,20 @@ +//! EndpointResolver trait for different clients + +// mod async_iter; mod endpoint; use crate::{ error::{Error, Result}, - Client, PublicKey, + Client, PublicKey, SignedPacket, }; use endpoint::Endpoint; const DEFAULT_MAX_CHAIN_LENGTH: u8 = 3; -#[derive(Debug, Clone)] -pub struct EndpointResolver { - pkarr: Client, - max_chain_length: u8, -} +pub(crate) trait EndpointResolver { + async fn resolve(&self, public_key: &PublicKey) -> Result>; -impl EndpointResolver { - pub fn new(pkarr: Client, max_chain_length: u8) -> Self { - EndpointResolver { - pkarr, - max_chain_length, - } - } - - /// Resolve a `qname` to an alternative [Endpoint] as defined in [RFC9460](https://www.rfc-editor.org/rfc/rfc9460#name-terminology). - /// - /// A `qname` is can be either a regular domain name for HTTPS endpoints, - /// or it could use Attrleaf naming pattern for cusotm protcol. For example: - /// `_foo.example.com` for `foo://example.com`. async fn resolve_endpoint(&self, qname: &str) -> Result { let target = qname; // TODO: cache the result of this function? @@ -40,8 +27,8 @@ impl EndpointResolver { loop { let current = svcb.clone().map_or(target.to_string(), |s| s.target); if let Ok(tld) = PublicKey::try_from(current.clone()) { - if let Ok(Some(signed_packet)) = self.pkarr.resolve(&tld).await { - if step >= self.max_chain_length { + if let Ok(Some(signed_packet)) = self.resolve(&tld).await { + if step >= DEFAULT_MAX_CHAIN_LENGTH { break; }; step += 1; @@ -71,16 +58,16 @@ impl EndpointResolver { } } -impl From<&Client> for EndpointResolver { - fn from(pkarr: &Client) -> Self { - pkarr.clone().into() +impl EndpointResolver for Client { + async fn resolve(&self, public_key: &PublicKey) -> Result> { + self.resolve(public_key).await } } -impl From for EndpointResolver { - /// Creates [EndpointResolver] from [Client] and default settings - fn from(pkarr: Client) -> Self { - Self::new(pkarr, DEFAULT_MAX_CHAIN_LENGTH) +#[cfg(all(not(target_arch = "wasm32"), feature = "relay"))] +impl EndpointResolver for crate::relay::Client { + async fn resolve(&self, public_key: &PublicKey) -> Result> { + self.resolve(public_key).await } } @@ -151,58 +138,56 @@ mod tests { } fn generate( - client: Client, + client: &Client, depth: u8, branching: u8, domain: Option, ) -> Pin>> { - generate_subtree(client, depth - 1, branching, domain) + generate_subtree(client.clone(), depth - 1, branching, domain) } #[tokio::test] async fn resolve_endpoints() { let testnet = Testnet::new(3); - let pkarr = Client::builder().testnet(&testnet).build().unwrap(); + let client = Client::builder().testnet(&testnet).build().unwrap(); - let resolver: EndpointResolver = (&pkarr).into(); - let tld = generate(pkarr, 3, 3, Some("example.com".to_string())).await; + let tld = generate(&client, 3, 3, Some("example.com".to_string())).await; - let endpoint = resolver.resolve_endpoint(&tld.to_string()).await.unwrap(); + let endpoint = client.resolve_endpoint(&tld.to_string()).await.unwrap(); assert_eq!(endpoint.target, "example.com"); } - #[tokio::test] - async fn max_chain_exceeded() { - let testnet = Testnet::new(3); - let pkarr = Client::builder().testnet(&testnet).build().unwrap(); - - let resolver: EndpointResolver = (&pkarr).into(); - - let tld = generate(pkarr, 4, 3, Some("example.com".to_string())).await; - - let endpoint = resolver.resolve_endpoint(&tld.to_string()).await; - - assert!(endpoint.is_err()); - // TODO: test error correctly - - // assert_eq!( - // match endpoint { - // Err(error) => error.to_string(), - // _ => "".to_string(), - // }, - // Error::Generic(tld.to_string()) - // ) - } + // #[tokio::test] + // async fn max_chain_exceeded() { + // let testnet = Testnet::new(3); + // let pkarr = Client::builder().testnet(&testnet).build().unwrap(); + // + // let resolver: EndpointResolver = (&pkarr).into(); + // + // let tld = generate(pkarr, 4, 3, Some("example.com".to_string())).await; + // + // let endpoint = resolver.resolve_endpoint(&tld.to_string()).await; + // + // assert!(endpoint.is_err()); + // // TODO: test error correctly + // + // // assert_eq!( + // // match endpoint { + // // Err(error) => error.to_string(), + // // _ => "".to_string(), + // // }, + // // Error::Generic(tld.to_string()) + // // ) + // } #[tokio::test] async fn resolve_addresses() { let testnet = Testnet::new(3); - let pkarr = Client::builder().testnet(&testnet).build().unwrap(); + let client = Client::builder().testnet(&testnet).build().unwrap(); - let resolver: EndpointResolver = (&pkarr).into(); - let tld = generate(pkarr, 3, 3, None).await; + let tld = generate(&client, 3, 3, None).await; - let endpoint = resolver.resolve_endpoint(&tld.to_string()).await.unwrap(); + let endpoint = client.resolve_endpoint(&tld.to_string()).await.unwrap(); assert_eq!(endpoint.target, "."); assert_eq!(endpoint.port, 3000); assert_eq!( From 65e09017dd116679a32e0270c41b01e4775cac88 Mon Sep 17 00:00:00 2001 From: nazeh Date: Wed, 25 Sep 2024 15:15:15 +0300 Subject: [PATCH 25/84] feat(pkarr): export endpoint resolver --- pkarr/src/extra/endpoints/mod.rs | 81 ++++++++++++++++++-------------- pkarr/src/extra/mod.rs | 2 +- pkarr/src/lib.rs | 3 ++ 3 files changed, 50 insertions(+), 36 deletions(-) diff --git a/pkarr/src/extra/endpoints/mod.rs b/pkarr/src/extra/endpoints/mod.rs index 712c273..23e211e 100644 --- a/pkarr/src/extra/endpoints/mod.rs +++ b/pkarr/src/extra/endpoints/mod.rs @@ -12,49 +12,60 @@ use endpoint::Endpoint; const DEFAULT_MAX_CHAIN_LENGTH: u8 = 3; -pub(crate) trait EndpointResolver { - async fn resolve(&self, public_key: &PublicKey) -> Result>; - - async fn resolve_endpoint(&self, qname: &str) -> Result { - let target = qname; - // TODO: cache the result of this function? - - let is_svcb = target.starts_with('_'); - - let mut step = 0; - let mut svcb: Option = None; - - loop { - let current = svcb.clone().map_or(target.to_string(), |s| s.target); - if let Ok(tld) = PublicKey::try_from(current.clone()) { - if let Ok(Some(signed_packet)) = self.resolve(&tld).await { - if step >= DEFAULT_MAX_CHAIN_LENGTH { +pub trait EndpointResolver { + fn resolve( + &self, + public_key: &PublicKey, + ) -> impl std::future::Future>> + Send; + + fn resolve_endpoint( + &self, + qname: &str, + ) -> impl std::future::Future> + Send + where + Self: std::marker::Sync, + { + async move { + let target = qname; + // TODO: cache the result of this function? + + let is_svcb = target.starts_with('_'); + + let mut step = 0; + let mut svcb: Option = None; + + loop { + let current = svcb.clone().map_or(target.to_string(), |s| s.target); + if let Ok(tld) = PublicKey::try_from(current.clone()) { + if let Ok(Some(signed_packet)) = self.resolve(&tld).await { + if step >= DEFAULT_MAX_CHAIN_LENGTH { + break; + }; + step += 1; + + // Choose most prior SVCB record + svcb = Endpoint::find(&signed_packet, ¤t, is_svcb); + + // TODO: support wildcard? + } else { break; - }; - step += 1; - - // Choose most prior SVCB record - svcb = Endpoint::find(&signed_packet, ¤t, is_svcb); - - // TODO: support wildcard? + } } else { break; } - } else { - break; } - } - if let Some(svcb) = svcb { - if PublicKey::try_from(svcb.target.as_str()).is_err() { - return Ok(svcb); + if let Some(svcb) = svcb { + if PublicKey::try_from(svcb.target.as_str()).is_err() { + return Ok(svcb); + } } - } - Err(Error::Generic(format!( - "Failed to find an endopint {}", - target - ))) + Err(Error::Generic(format!( + "Failed to find an endopint {}", + target + ))) + } } } diff --git a/pkarr/src/extra/mod.rs b/pkarr/src/extra/mod.rs index 3aa2c45..abf2208 100644 --- a/pkarr/src/extra/mod.rs +++ b/pkarr/src/extra/mod.rs @@ -1,2 +1,2 @@ #[cfg(feature = "endpoints")] -mod endpoints; +pub mod endpoints; diff --git a/pkarr/src/lib.rs b/pkarr/src/lib.rs index 350fb2e..c7d0058 100644 --- a/pkarr/src/lib.rs +++ b/pkarr/src/lib.rs @@ -37,3 +37,6 @@ pub use simple_dns as dns; pub use client::relay; #[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] pub use mainline; + +#[cfg(feature = "endpoints")] +pub use extra::endpoints::EndpointResolver; From 5b0be109fe476c3686d89533a9105fb55fc27fe5 Mon Sep 17 00:00:00 2001 From: nazeh Date: Wed, 25 Sep 2024 16:38:15 +0300 Subject: [PATCH 26/84] feat(pkarr): export endpoint struct --- Cargo.lock | 196 ++++++++++++++------------ pkarr/src/client/dht.rs | 8 +- pkarr/src/client/relay.rs | 6 +- pkarr/src/extra/endpoints/endpoint.rs | 1 + pkarr/src/extra/endpoints/mod.rs | 2 +- pkarr/src/lib.rs | 2 +- 6 files changed, 115 insertions(+), 100 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bf46ecd..28c9e70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,18 +4,18 @@ version = 3 [[package]] name = "addr2line" -version = "0.22.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375" dependencies = [ "gimli", ] [[package]] -name = "adler" -version = "1.0.2" +name = "adler2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" [[package]] name = "aho-corasick" @@ -77,9 +77,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.86" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" [[package]] name = "arc-swap" @@ -99,9 +99,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.81" +version = "0.1.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" +checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", @@ -131,9 +131,9 @@ checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "axum" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" +checksum = "8f43644eed690f5374f1af436ecd6aea01cd201f6fbdf0178adaf6907afb2cec" dependencies = [ "async-trait", "axum-core", @@ -157,7 +157,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper 1.0.1", "tokio", - "tower", + "tower 0.5.1", "tower-layer", "tower-service", "tracing", @@ -165,9 +165,9 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3" +checksum = "5e6b8ba012a258d63c9adfa28b9ddcf66149da6f986c5b5452e629d5ee64bf00" dependencies = [ "async-trait", "bytes", @@ -178,7 +178,7 @@ dependencies = [ "mime", "pin-project-lite", "rustversion", - "sync_wrapper 0.1.2", + "sync_wrapper 1.0.1", "tower-layer", "tower-service", "tracing", @@ -204,30 +204,30 @@ dependencies = [ "rustls-pki-types", "tokio", "tokio-rustls", - "tower", + "tower 0.4.13", "tower-service", ] [[package]] name = "backtrace" -version = "0.3.73" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ "addr2line", - "cc", "cfg-if", "libc", "miniz_oxide", "object", "rustc-demangle", + "windows-targets 0.52.6", ] [[package]] name = "base32" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1ce0365f4d5fb6646220bb52fe547afd51796d90f914d4063cb0b032ebee088" +checksum = "022dfe9eb35f19ebbcb51e0b40a5ab759f46ad60cadf7297e0bd085afb50e076" [[package]] name = "base64" @@ -273,15 +273,15 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" +checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" [[package]] name = "cc" -version = "1.1.11" +version = "1.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fb8dd288a69fc53a1996d7ecfbf4a20d59065bff137ce7e56bbd620de191189" +checksum = "07b1695e2c7e8fc85310cde85aeaab7e3097f593c91d209d3f9df76c928100f0" dependencies = [ "shlex", ] @@ -294,9 +294,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "4.5.15" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11d8838454fda655dafd3accb2b6e2bea645b9e4078abe84a22ceb947235c5cc" +checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3" dependencies = [ "clap_builder", "clap_derive", @@ -304,9 +304,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.15" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216aec2b177652e3846684cbfe25c9964d18ec45234f0f5da5157b207ed1aab6" +checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b" dependencies = [ "anstream", "anstyle", @@ -316,9 +316,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.13" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" dependencies = [ "heck", "proc-macro2", @@ -378,9 +378,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e852e6dc9a5bed1fae92dd2375037bf2b768725bf3be87811edee3249d09ad" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" dependencies = [ "libc", ] @@ -782,9 +782,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.29.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" [[package]] name = "governor" @@ -808,9 +808,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" +checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" dependencies = [ "atomic-waker", "bytes", @@ -862,9 +862,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "heed" -version = "0.20.4" +version = "0.20.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "620033c8c8edfd2f53e6f99a30565eb56a33b42c468e3ad80e21d85fb93bafb0" +checksum = "7d4f449bab7320c56003d37732a917e18798e2f1709d80263face2b4f9436ddb" dependencies = [ "bitflags", "byteorder", @@ -1003,9 +1003,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.7" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" +checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b" dependencies = [ "bytes", "futures-channel", @@ -1016,7 +1016,6 @@ dependencies = [ "pin-project-lite", "socket2", "tokio", - "tower", "tower-service", "tracing", ] @@ -1033,9 +1032,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93ead53efc7ea8ed3cfb0c79fc8023fbb782a5432b52830b6518941cebe6505c" +checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" dependencies = [ "equivalent", "hashbrown", @@ -1076,9 +1075,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.155" +version = "0.2.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" [[package]] name = "libredox" @@ -1104,9 +1103,9 @@ checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5" [[package]] name = "lmdb-master-sys" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1de7e761853c15ca72821d9f928d7bb123ef4c05377c4e7ab69fa1c742f91d24" +checksum = "472c3760e2a8d0f61f322fb36788021bb36d573c502b50fa3e2bcaac3ec326c9" dependencies = [ "cc", "doxygen-rs", @@ -1184,11 +1183,11 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "miniz_oxide" -version = "0.7.4" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" dependencies = [ - "adler", + "adler2", ] [[package]] @@ -1283,9 +1282,9 @@ dependencies = [ [[package]] name = "object" -version = "0.36.3" +version = "0.36.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b64972346851a39438c60b341ebc01bba47464ae329e55cf343eb93964efd9" +checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" dependencies = [ "memchr", ] @@ -1527,15 +1526,15 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" [[package]] name = "portable-atomic" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265" +checksum = "d30538d42559de6b034bc76fd6dd4c38961b1ee5c6c56e3808c50128fdbc22ce" [[package]] name = "postcard" @@ -1620,22 +1619,22 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bffec3605b73c6f1754535084a85229fa8a30f86014e6c81aeec4abb68b0285" +checksum = "4fe68c2e9e1a1234e218683dbdf9f9dfcb094113c5ac2b938dfcb9bab4c4140b" dependencies = [ "libc", "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "quote" -version = "1.0.36" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" dependencies = [ "proc-macro2", ] @@ -1681,18 +1680,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.3" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" +checksum = "62871f2d65009c0256aed1b9cfeeb8ac272833c404e13d53d400cd0dad7a2ac0" dependencies = [ "bitflags", ] [[package]] name = "redox_users" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ "getrandom", "libredox", @@ -1820,18 +1819,18 @@ checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" [[package]] name = "rustc_version" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ "semver", ] [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" dependencies = [ "bitflags", "errno", @@ -1945,9 +1944,9 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" -version = "1.0.209" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" dependencies = [ "serde_derive", ] @@ -1973,9 +1972,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.209" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", @@ -1984,9 +1983,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.125" +version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83c8e735a073ccf5be70aa8066aa984eaf2fa000db6c8d0100ae605b366d31ed" +checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ "itoa", "memchr", @@ -2238,18 +2237,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" +checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" +checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" dependencies = [ "proc-macro2", "quote", @@ -2333,9 +2332,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" +checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" dependencies = [ "bytes", "futures-core", @@ -2367,9 +2366,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.20" +version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ "indexmap", "serde", @@ -2388,6 +2387,21 @@ dependencies = [ "futures-util", "pin-project", "pin-project-lite", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper 0.1.2", "tokio", "tower-layer", "tower-service", @@ -2435,7 +2449,7 @@ dependencies = [ "http", "pin-project", "thiserror", - "tower", + "tower 0.4.13", "tracing", ] @@ -2521,15 +2535,15 @@ checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-normalization" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" dependencies = [ "tinyvec", ] @@ -2878,9 +2892,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.18" +version = "0.6.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f" +checksum = "c52ac009d615e79296318c1bcce2d422aaca15ad08515e344feeda07df67a587" dependencies = [ "memchr", ] diff --git a/pkarr/src/client/dht.rs b/pkarr/src/client/dht.rs index cc67a7d..f56d2a7 100644 --- a/pkarr/src/client/dht.rs +++ b/pkarr/src/client/dht.rs @@ -199,7 +199,7 @@ impl Client { /// /// # Errors /// - Returns a [Error::DhtIsShutdown] if [Client::shutdown] was called, or - /// the loop in the actor thread is stopped for any reason (like thread panic). + /// the loop in the actor thread is stopped for any reason (like thread panic). /// - Returns a [Error::PublishInflight] if the client is currently publishing the same public_key. /// - Returns a [Error::NotMostRecent] if the provided signed packet is older than most recent. /// - Returns a [Error::MainlineError] if the Dht received an unexpected error otherwise. @@ -225,7 +225,7 @@ impl Client { /// /// # Errors /// - Returns a [Error::DhtIsShutdown] if [Client::shutdown] was called, or - /// the loop in the actor thread is stopped for any reason (like thread panic). + /// the loop in the actor thread is stopped for any reason (like thread panic). pub async fn resolve(&self, public_key: &PublicKey) -> Result> { Ok(self.resolve_inner(public_key)?.recv_async().await.ok()) } @@ -251,7 +251,7 @@ impl Client { /// /// # Errors /// - Returns a [Error::DhtIsShutdown] if [Client::shutdown] was called, or - /// the loop in the actor thread is stopped for any reason (like thread panic). + /// the loop in the actor thread is stopped for any reason (like thread panic). /// - Returns a [Error::PublishInflight] if the client is currently publishing the same public_key. /// - Returns a [Error::NotMostRecent] if the provided signed packet is older than most recent. /// - Returns a [Error::MainlineError] if the Dht received an unexpected error otherwise. @@ -277,7 +277,7 @@ impl Client { /// /// # Errors /// - Returns a [Error::DhtIsShutdown] if [Client::shutdown] was called, or - /// the loop in the actor thread is stopped for any reason (like thread panic). + /// the loop in the actor thread is stopped for any reason (like thread panic). pub fn resolve_sync(&self, public_key: &PublicKey) -> Result> { Ok(self.resolve_inner(public_key)?.recv().ok()) } diff --git a/pkarr/src/client/relay.rs b/pkarr/src/client/relay.rs index 8f3ba1a..6b56a13 100644 --- a/pkarr/src/client/relay.rs +++ b/pkarr/src/client/relay.rs @@ -152,7 +152,7 @@ impl Client { /// # Errors /// - Returns a [Error::NotMostRecent] if the provided signed packet is older than most recent. /// - Returns a [Error::RelayError] from the last responding relay, if all relays - /// responded with non-2xx status codes. + /// responded with non-2xx status codes. pub async fn publish(&self, signed_packet: &SignedPacket) -> Result<()> { let public_key = signed_packet.public_key(); @@ -174,8 +174,8 @@ impl Client { /// # Errors /// /// - Returns [Error::RelayError] if the relay responded with a status >= 400 - /// (except 404 in which case you should receive Ok(None)) or something wrong - /// with the transport, transparent from [reqwest::Error]. + /// (except 404 in which case you should receive Ok(None)) or something wrong + /// with the transport, transparent from [reqwest::Error]. /// - Returns [Error::IO] if something went wrong while reading the payload. pub async fn resolve(&self, public_key: &PublicKey) -> Result> { let cached_packet = match self.get_from_cache(public_key) { diff --git a/pkarr/src/extra/endpoints/endpoint.rs b/pkarr/src/extra/endpoints/endpoint.rs index bbe9c16..5b0d123 100644 --- a/pkarr/src/extra/endpoints/endpoint.rs +++ b/pkarr/src/extra/endpoints/endpoint.rs @@ -83,6 +83,7 @@ impl Endpoint { }) } + #[cfg(not(target_arch = "wasm32"))] /// Return an iterator of [SocketAddr], either by resolving the [Endpoint::target] using normal DNS, /// or, if the target is ".", return the [RData::A] or [RData::AAAA] records /// from the endpoint's [SignedPacket], if available. diff --git a/pkarr/src/extra/endpoints/mod.rs b/pkarr/src/extra/endpoints/mod.rs index 23e211e..38ce070 100644 --- a/pkarr/src/extra/endpoints/mod.rs +++ b/pkarr/src/extra/endpoints/mod.rs @@ -8,7 +8,7 @@ use crate::{ Client, PublicKey, SignedPacket, }; -use endpoint::Endpoint; +pub use endpoint::Endpoint; const DEFAULT_MAX_CHAIN_LENGTH: u8 = 3; diff --git a/pkarr/src/lib.rs b/pkarr/src/lib.rs index c7d0058..9e4519c 100644 --- a/pkarr/src/lib.rs +++ b/pkarr/src/lib.rs @@ -39,4 +39,4 @@ pub use client::relay; pub use mainline; #[cfg(feature = "endpoints")] -pub use extra::endpoints::EndpointResolver; +pub use extra::endpoints::{Endpoint, EndpointResolver}; From 58d2f0b8609a7542c3a5f775562d23b0e0bcbb45 Mon Sep 17 00:00:00 2001 From: nazeh Date: Wed, 25 Sep 2024 22:38:27 +0300 Subject: [PATCH 27/84] feat(pkarr): add getters to endpoint --- pkarr/src/extra/endpoints/endpoint.rs | 56 ++++++++++++++++++--------- pkarr/src/extra/endpoints/mod.rs | 21 +++++----- 2 files changed, 48 insertions(+), 29 deletions(-) diff --git a/pkarr/src/extra/endpoints/endpoint.rs b/pkarr/src/extra/endpoints/endpoint.rs index 5b0d123..8a4c836 100644 --- a/pkarr/src/extra/endpoints/endpoint.rs +++ b/pkarr/src/extra/endpoints/endpoint.rs @@ -12,10 +12,10 @@ use crate::Timestamp; #[derive(Debug, Clone)] /// An alternative Endpoint for a `qname`, from either [RData::SVCB] or [RData::HTTPS] dns records pub struct Endpoint { - pub(crate) target: String, + target: String, // public_key: PublicKey, - pub(crate) port: u16, - pub(crate) addrs: Vec, + port: Option, + addrs: Vec, } impl Endpoint { @@ -72,33 +72,53 @@ impl Endpoint { Endpoint { target, // public_key: signed_packet.public_key(), - port: u16::from_be_bytes( - s.get_param(SVCB::PORT) - .unwrap_or_default() - .try_into() - .unwrap_or([0, 0]), - ), + port: s.get_param(SVCB::PORT).map(|bytes| { + let mut arr = [0_u8; 2]; + arr[0] = bytes[0]; + arr[1] = bytes[1]; + + u16::from_be_bytes(arr) + }), addrs, } }) } - #[cfg(not(target_arch = "wasm32"))] + /// Return the endpoint target, i.e the domain it points to + /// "." means this endpoint points to its own [Endpoint::public_key] + pub fn target(&self) -> &str { + &self.target + } + + pub fn port(&self) -> Option { + self.port + } + + // pub fn public_key(&self) -> { + // &self.target + // } + /// Return an iterator of [SocketAddr], either by resolving the [Endpoint::target] using normal DNS, /// or, if the target is ".", return the [RData::A] or [RData::AAAA] records /// from the endpoint's [SignedPacket], if available. - pub fn to_socket_addrs(&self) -> std::io::Result> { + pub fn to_socket_addrs(&self) -> Vec { if self.target == "." { - let port = self.port; - return Ok(self + let port = self.port.unwrap_or(0); + + return self .addrs .iter() .map(|addr| SocketAddr::from((*addr, port))) - .collect::>() - .into_iter()); + .collect::>(); } - format!("{}:{}", self.target, self.port).to_socket_addrs() + if cfg!(target_arch = "wasm32") { + vec![] + } else { + format!("{}:{}", self.target, self.port.unwrap_or(0)) + .to_socket_addrs() + .map_or(vec![], |v| v.collect::>()) + } } } @@ -213,9 +233,9 @@ mod tests { assert_eq!(endpoint.target, "."); - let addrs = endpoint.to_socket_addrs().unwrap(); + let addrs = endpoint.to_socket_addrs(); assert_eq!( - addrs.map(|s| s.to_string()).collect::>(), + addrs.into_iter().map(|s| s.to_string()).collect::>(), vec!["209.151.148.15:6881", "[2a05:d014:275:6201::64]:6881"] ) } diff --git a/pkarr/src/extra/endpoints/mod.rs b/pkarr/src/extra/endpoints/mod.rs index 38ce070..ccdf67d 100644 --- a/pkarr/src/extra/endpoints/mod.rs +++ b/pkarr/src/extra/endpoints/mod.rs @@ -16,12 +16,9 @@ pub trait EndpointResolver { fn resolve( &self, public_key: &PublicKey, - ) -> impl std::future::Future>> + Send; + ) -> impl std::future::Future>>; - fn resolve_endpoint( - &self, - qname: &str, - ) -> impl std::future::Future> + Send + fn resolve_endpoint(&self, qname: &str) -> impl std::future::Future> where Self: std::marker::Sync, { @@ -35,7 +32,9 @@ pub trait EndpointResolver { let mut svcb: Option = None; loop { - let current = svcb.clone().map_or(target.to_string(), |s| s.target); + let current = svcb + .clone() + .map_or(target.to_string(), |s| s.target().to_string()); if let Ok(tld) = PublicKey::try_from(current.clone()) { if let Ok(Some(signed_packet)) = self.resolve(&tld).await { if step >= DEFAULT_MAX_CHAIN_LENGTH { @@ -56,7 +55,7 @@ pub trait EndpointResolver { } if let Some(svcb) = svcb { - if PublicKey::try_from(svcb.target.as_str()).is_err() { + if PublicKey::try_from(svcb.target()).is_err() { return Ok(svcb); } } @@ -165,9 +164,10 @@ mod tests { let tld = generate(&client, 3, 3, Some("example.com".to_string())).await; let endpoint = client.resolve_endpoint(&tld.to_string()).await.unwrap(); - assert_eq!(endpoint.target, "example.com"); + assert_eq!(endpoint.target(), "example.com"); } + // TODO: Test max_chain_exceeded // #[tokio::test] // async fn max_chain_exceeded() { // let testnet = Testnet::new(3); @@ -199,12 +199,11 @@ mod tests { let tld = generate(&client, 3, 3, None).await; let endpoint = client.resolve_endpoint(&tld.to_string()).await.unwrap(); - assert_eq!(endpoint.target, "."); - assert_eq!(endpoint.port, 3000); + assert_eq!(endpoint.target(), "."); + assert_eq!(endpoint.port(), Some(3000)); assert_eq!( endpoint .to_socket_addrs() - .unwrap() .into_iter() .map(|s| s.to_string()) .collect::>(), From 434fa104f049d34231910fcdebb6d3eebed6846b Mon Sep 17 00:00:00 2001 From: nazeh Date: Fri, 27 Sep 2024 19:16:07 +0300 Subject: [PATCH 28/84] feat: update mainline to v3.0.0 --- Cargo.lock | 4 ++-- pkarr/Cargo.toml | 2 +- pkarr/src/client/dht.rs | 14 +++++++------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 28c9e70..c2142e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1136,9 +1136,9 @@ checksum = "37ee39891760e7d94734f6f63fedc29a2e4a152f836120753a72503f09fcf904" [[package]] name = "mainline" -version = "2.0.1" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b751ffb57303217bcae8f490eee6044a5b40eadf6ca05ff476cad37e7b7970d" +checksum = "83025da75c65e1e0050def10ee022586c7e4da8eea31380b9603eb7ed02e1383" dependencies = [ "bytes", "crc", diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index d9deb61..0f2e134 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -27,7 +27,7 @@ rand = { version = "0.8.5", optional = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] # Dht client dependencies: -mainline = { version = "2.0.1", optional = true } +mainline = { version = "3.0.0", optional = true } flume = { version = "0.11.0", features = ["select", "eventual-fairness", "async"], default-features = false , optional = true } tokio = { version = "1.40.0", optional = true } reqwest = { version = "0.12.7", features = ["rustls-tls"], optional = true } diff --git a/pkarr/src/client/dht.rs b/pkarr/src/client/dht.rs index f56d2a7..9dd31fd 100644 --- a/pkarr/src/client/dht.rs +++ b/pkarr/src/client/dht.rs @@ -151,7 +151,7 @@ impl Client { let rpc = Rpc::new(&settings.dht)?; - let local_addr = rpc.local_addr(); + let local_addr = rpc.local_addr()?; let cache = settings .cache @@ -519,7 +519,7 @@ mod tests { #[test] fn shutdown_sync() { - let testnet = Testnet::new(3); + let testnet = Testnet::new(3).unwrap(); let mut a = Client::builder().testnet(&testnet).build().unwrap(); @@ -532,7 +532,7 @@ mod tests { #[test] fn publish_resolve_sync() { - let testnet = Testnet::new(10); + let testnet = Testnet::new(10).unwrap(); let a = Client::builder().testnet(&testnet).build().unwrap(); @@ -562,7 +562,7 @@ mod tests { #[test] fn thread_safe_sync() { - let testnet = Testnet::new(10); + let testnet = Testnet::new(10).unwrap(); let a = Client::builder().testnet(&testnet).build().unwrap(); @@ -596,7 +596,7 @@ mod tests { #[tokio::test] async fn shutdown() { - let testnet = Testnet::new(3); + let testnet = Testnet::new(3).unwrap(); let mut a = Client::builder().testnet(&testnet).build().unwrap(); @@ -609,7 +609,7 @@ mod tests { #[tokio::test] async fn publish_resolve() { - let testnet = Testnet::new(10); + let testnet = Testnet::new(10).unwrap(); let a = Client::builder().testnet(&testnet).build().unwrap(); @@ -639,7 +639,7 @@ mod tests { #[tokio::test] async fn thread_safe() { - let testnet = Testnet::new(10); + let testnet = Testnet::new(10).unwrap(); let a = Client::builder().testnet(&testnet).build().unwrap(); From 0ff223f67a0cad2539645846da7aab32ef4e175c Mon Sep 17 00:00:00 2001 From: nazeh Date: Mon, 30 Sep 2024 11:54:05 +0300 Subject: [PATCH 29/84] feat(pkarr): derive PartialEq and Eq for Keypair --- pkarr/src/base/keys.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkarr/src/base/keys.rs b/pkarr/src/base/keys.rs index 3a97d86..d99f763 100644 --- a/pkarr/src/base/keys.rs +++ b/pkarr/src/base/keys.rs @@ -11,7 +11,7 @@ use std::{ #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; -#[derive(Clone)] +#[derive(Clone, PartialEq, Eq)] /// Ed25519 keypair to sign dns [Packet](crate::SignedPacket)s. pub struct Keypair(SigningKey); From 0622003a851dfc34c4c3b44daaf13c99880d300a Mon Sep 17 00:00:00 2001 From: nazeh Date: Mon, 30 Sep 2024 19:52:33 +0300 Subject: [PATCH 30/84] feat: make Client::shutdown and Client::shutdown_sync idempotent --- CHANGELOG.md | 22 ++++++++++++++++++++++ pkarr/Cargo.toml | 16 ++++++++++------ pkarr/src/base/timestamp.rs | 2 +- pkarr/src/client/dht.rs | 30 +++++++++--------------------- 4 files changed, 42 insertions(+), 28 deletions(-) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..ce3bd55 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,22 @@ +# Changelog + +All notable changes to pkarr client and server will be documented in this file. + +## [Unreleased] + +### Added + +- Add strict monotonic unix `Timestamp`. + +### Changed + +- replace `z32` with *(base32)*. +- `SignedPacket::last_seen` is a `Timestamp` instead of u64. +- make `rand` non-optional, and remove the feature flag. +- replace `ureq` with `reqwest` to work with HTTP/2 relays, and Wasm. +- update `mainline` to v3.0.0 +- `Client::shutdown` and `Client::shutdown_sync` are now idempotent and return `()`. + +### Removed + +- Remvoed `relay_client_web`, replaced with *(pkarr::relay::Client)*. diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index 0f2e134..1a8c9f8 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -20,14 +20,15 @@ dyn-clone = "1.0.17" once_cell = {version = "1.19.0", default-features = false } lru = { version = "0.12.3", default-features = false } document-features = "0.2.8" +rand = "0.8.5" # optional dependencies serde = { version = "1.0.209", features = ["derive"], optional = true } -rand = { version = "0.8.5", optional = true } +futures = { version = "0.3.30", optional = true, default-features = false } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] # Dht client dependencies: -mainline = { version = "3.0.0", optional = true } +mainline = { git = "https://github.com/pubky/mainline", branch = "dev", optional = true } flume = { version = "0.11.0", features = ["select", "eventual-fairness", "async"], default-features = false , optional = true } tokio = { version = "1.40.0", optional = true } reqwest = { version = "0.12.7", features = ["rustls-tls"], optional = true } @@ -58,13 +59,16 @@ dht = ["dep:mainline", "flume"] relay = ["dep:reqwest", "dep:tokio"] # Extra -## -endpoints = [] +## Use [Client::resolve_endpoints] and [Client::resolve_https_endpoints] +endpoints = ["dep:futures"] ## Use all features -full = ["dht", "relay", "rand", "serde", "endpoints"] +full = ["dht", "relay", "serde", "endpoints"] -default = ["full"] +default = ["dht", "relay", "serde"] [package.metadata.docs.rs] all-features = true + +# [lints.clippy] +# unwrap_used = "deny" diff --git a/pkarr/src/base/timestamp.rs b/pkarr/src/base/timestamp.rs index f0c937b..1fd6ed6 100644 --- a/pkarr/src/base/timestamp.rs +++ b/pkarr/src/base/timestamp.rs @@ -14,7 +14,7 @@ use rand::Rng; #[cfg(not(target_arch = "wasm32"))] use std::time::SystemTime; -/// ~4% chance of none of 10 clocks have matching id. +/// ~0.4% chance of none of 10 clocks have matching id. const CLOCK_MASK: u64 = (1 << 8) - 1; const TIME_MASK: u64 = !0 >> 8; diff --git a/pkarr/src/client/dht.rs b/pkarr/src/client/dht.rs index 9dd31fd..6a0404b 100644 --- a/pkarr/src/client/dht.rs +++ b/pkarr/src/client/dht.rs @@ -231,18 +231,13 @@ impl Client { } /// Shutdown the actor thread loop. - pub async fn shutdown(&mut self) -> Result<()> { + pub async fn shutdown(&mut self) { let (sender, receiver) = flume::bounded(1); - self.sender - .send(ActorMessage::Shutdown(sender)) - .map_err(|_| Error::DhtIsShutdown)?; - - receiver.recv_async().await?; + let _ = self.sender.send(ActorMessage::Shutdown(sender)); + let _ = receiver.recv_async().await; self.address = None; - - Ok(()) } // === Sync === @@ -283,18 +278,13 @@ impl Client { } /// Shutdown the actor thread loop. - pub fn shutdown_sync(&mut self) -> Result<()> { + pub fn shutdown_sync(&mut self) { let (sender, receiver) = flume::bounded(1); - self.sender - .send(ActorMessage::Shutdown(sender)) - .map_err(|_| Error::DhtIsShutdown)?; - - receiver.recv()?; + let _ = self.sender.send(ActorMessage::Shutdown(sender)); + let _ = receiver.recv(); self.address = None; - - Ok(()) } // === Private Methods === @@ -426,8 +416,6 @@ fn run(mut rpc: Rpc, cache: Box, settings: Settings, receiver: Receiv // === Receive and handle incoming messages === if let Some(ReceivedFrom { from, message }) = &report.received_from { - // match &report.received_from { - // Some(ReceivedFrom { from, message }) => match message { match message { // === Responses === ReceivedMessage::QueryResponse(response) => { @@ -501,7 +489,7 @@ fn run(mut rpc: Rpc, cache: Box, settings: Settings, receiver: Receiv } } - debug!("Client main terminated"); + debug!("Client main loop terminated"); } pub enum ActorMessage { @@ -525,7 +513,7 @@ mod tests { assert_ne!(a.local_addr(), None); - a.shutdown_sync().unwrap(); + a.shutdown_sync(); assert_eq!(a.local_addr(), None); } @@ -602,7 +590,7 @@ mod tests { assert_ne!(a.local_addr(), None); - a.shutdown().await.unwrap(); + a.shutdown().await; assert_eq!(a.local_addr(), None); } From 3be6677df638cd031145f872759db7918e49bc21 Mon Sep 17 00:00:00 2001 From: nazeh Date: Tue, 8 Oct 2024 10:41:19 +0300 Subject: [PATCH 31/84] docs: small change to the README.md --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index a0b6d14..1d7f470 100644 --- a/README.md +++ b/README.md @@ -7,17 +7,17 @@ The simplest possible streamlined integration between the Domain Name System and Where we are going, this [https://o4dksfbqk85ogzdb5osziw6befigbuxmuxkuxq8434q89uj56uyy](https://app.pkarr.org/?pk=o4dksfbqk85ogzdb5osziw6befigbuxmuxkuxq8434q89uj56uyy) resolves everywhere! ## TLDR -- To publish resource records for your key, sign a small encoded DNS packet (<= 1000 bytes) and publish it on the [Mainline DHT](https://en.wikipedia.org/wiki/Mainline_DHT) (through a relay if necessary). -- To resolve some key's resources, applications query the DHT directly, or through a relay, and verify the signature themselves. -- The DHT drops records after a few hours, so users, their friends, or service providers should periodically republish their records to the DHT. -- Clients and Pkarr servers cache records extensively using the `TTL` values in them to minimize DHT traffic as much as possible for improved scalability and reliability. -- Existing applications unaware of Pkarr can still resolve Pkarr TLDs, if the DNS server they query recognize Pkarr TLDs and use Mainline as a parallel root server to ICANN. +- To publish resource records for your key, sign a small encoded DNS packet (<= 1000 bytes) and publish it on the DHT (through a relay if necessary). +- To resolve some key's resources, applications query the DHT directly, or through a [relay](./design/relays.md), and verify the signature themselves. +- Clients and Pkarr servers cache records extensively and minimize DHT traffic as much as possible for improved scalability. +- The DHT drops records after a few hours, so users, their friends, or service providers should periodically republish their records to the DHT. Also Pkarr servers could republish records recently requested, to keep popular records alive too. +- Optional: Existing applications unaware of Pkarr can still function if the user added a Pkarr-aware DNS servers to their operating system DNS servers. ## DEMO Try the [web app demo](https://app.pkarr.org). -Or if you prefer Rust [Examples](./pkarr/examples) +Or if you prefer Rust [Examples](./pkarr/examples/README.md) ## TOC - [Architecture](#Architecture) From 724f1a85284a86943be0298e2fc6ecaba99d3819 Mon Sep 17 00:00:00 2001 From: nazeh Date: Tue, 8 Oct 2024 10:44:29 +0300 Subject: [PATCH 32/84] chore: clippy --- Cargo.lock | 188 ++++++++++++++++--------------- pkarr/Cargo.toml | 2 +- pkarr/clippy.toml | 1 + pkarr/src/extra/endpoints/mod.rs | 4 +- 4 files changed, 100 insertions(+), 95 deletions(-) create mode 100644 pkarr/clippy.toml diff --git a/Cargo.lock b/Cargo.lock index c2142e2..679851c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] @@ -125,15 +125,15 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "axum" -version = "0.7.6" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f43644eed690f5374f1af436ecd6aea01cd201f6fbdf0178adaf6907afb2cec" +checksum = "504e3947307ac8326a5437504c517c4b56716c9d98fac0028c2acc7ca47d70ae" dependencies = [ "async-trait", "axum-core", @@ -165,9 +165,9 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e6b8ba012a258d63c9adfa28b9ddcf66149da6f986c5b5452e629d5ee64bf00" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" dependencies = [ "async-trait", "bytes", @@ -279,9 +279,9 @@ checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" [[package]] name = "cc" -version = "1.1.21" +version = "1.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07b1695e2c7e8fc85310cde85aeaab7e3097f593c91d209d3f9df76c928100f0" +checksum = "2e80e3b6a3ab07840e1cae9b0666a63970dc28e8ed5ffbcdacbfc760c281bfc1" dependencies = [ "shlex", ] @@ -294,9 +294,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "4.5.18" +version = "4.5.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3" +checksum = "7be5744db7978a28d9df86a214130d106a89ce49644cbc4e3f0c22c3fba30615" dependencies = [ "clap_builder", "clap_derive", @@ -304,9 +304,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.18" +version = "4.5.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b" +checksum = "a5fbc17d3ef8278f55b282b2a2e75ae6f6c7d4bb70ed3d0382375104bfafdb4b" dependencies = [ "anstream", "anstyle", @@ -465,7 +465,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown", + "hashbrown 0.14.5", "lock_api", "once_cell", "parking_lot_core", @@ -664,9 +664,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -679,9 +679,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -689,15 +689,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -706,15 +706,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", @@ -723,15 +723,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-timer" @@ -741,9 +741,9 @@ checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", @@ -782,9 +782,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.31.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "governor" @@ -840,6 +840,12 @@ version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +[[package]] +name = "hashbrown" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" + [[package]] name = "heapless" version = "0.7.17" @@ -936,9 +942,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.9.4" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" +checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" [[package]] name = "httpdate" @@ -1032,19 +1038,19 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.15.0", ] [[package]] name = "ipnet" -version = "2.10.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" +checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" [[package]] name = "is_terminal_polyfill" @@ -1130,15 +1136,14 @@ checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "lru" -version = "0.12.4" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37ee39891760e7d94734f6f63fedc29a2e4a152f836120753a72503f09fcf904" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" [[package]] name = "mainline" version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83025da75c65e1e0050def10ee022586c7e4da8eea31380b9603eb7ed02e1383" +source = "git+https://github.com/pubky/mainline?branch=dev#47edb4617f3ce25883b889064de5c7d257848b8f" dependencies = [ "bytes", "crc", @@ -1282,18 +1287,18 @@ dependencies = [ [[package]] name = "object" -version = "0.36.4" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "openssl" @@ -1428,18 +1433,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +checksum = "baf123a161dde1e524adf36f90bc5d8d3462824a9c43553ad07a8183161189ec" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +checksum = "a4502d8515ca9f32f1fb543d987f63d95a14934883db45bdb48060b6b69257f8" dependencies = [ "proc-macro2", "quote", @@ -1532,9 +1537,9 @@ checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" [[package]] name = "portable-atomic" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d30538d42559de6b034bc76fd6dd4c38961b1ee5c6c56e3808c50128fdbc22ce" +checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" [[package]] name = "postcard" @@ -1560,9 +1565,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "b3e4daa0dcf6feba26f985457cdf104d4b4256fc5a09547140f3631bb076b19a" dependencies = [ "unicode-ident", ] @@ -1671,18 +1676,18 @@ dependencies = [ [[package]] name = "raw-cpuid" -version = "11.1.0" +version = "11.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb9ee317cfe3fbd54b36a511efc1edd42e216903c9cd575e686dd68a2ba90d8d" +checksum = "1ab240315c661615f2ee9f0f2cd32d5a7343a84d5ebcccb99d46e6637565e7b0" dependencies = [ "bitflags", ] [[package]] name = "redox_syscall" -version = "0.5.5" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62871f2d65009c0256aed1b9cfeeb8ac272833c404e13d53d400cd0dad7a2ac0" +checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" dependencies = [ "bitflags", ] @@ -1700,14 +1705,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.6" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.7", - "regex-syntax 0.8.4", + "regex-automata 0.4.8", + "regex-syntax 0.8.5", ] [[package]] @@ -1721,13 +1726,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", ] [[package]] @@ -1738,15 +1743,15 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" -version = "0.12.7" +version = "0.12.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" +checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b" dependencies = [ "base64", "bytes", @@ -1841,9 +1846,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.13" +version = "0.23.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8" +checksum = "415d9944693cb90382053259f89fbb077ea730ad7273047ec63b19bc9b160ba8" dependencies = [ "once_cell", "ring", @@ -1855,19 +1860,18 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.1.3" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" dependencies = [ - "base64", "rustls-pki-types", ] [[package]] name = "rustls-pki-types" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" +checksum = "0e696e35370c65c9c541198af4543ccd580cf17fc25d8e05c5a242b202488c55" [[package]] name = "rustls-webpki" @@ -1894,9 +1898,9 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "schannel" -version = "0.1.24" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b" +checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1" dependencies = [ "windows-sys 0.59.0", ] @@ -2005,9 +2009,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" dependencies = [ "serde", ] @@ -2168,9 +2172,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.77" +version = "2.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" +checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" dependencies = [ "proc-macro2", "quote", @@ -2224,9 +2228,9 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.12.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" +checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" dependencies = [ "cfg-if", "fastrand", @@ -2529,9 +2533,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicode-bidi" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" +checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" [[package]] name = "unicode-ident" @@ -2892,9 +2896,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.19" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c52ac009d615e79296318c1bcce2d422aaca15ad08515e344feeda07df67a587" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" dependencies = [ "memchr", ] diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index 1a8c9f8..5fff22f 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -71,4 +71,4 @@ default = ["dht", "relay", "serde"] all-features = true # [lints.clippy] -# unwrap_used = "deny" +unwrap_used = "deny" diff --git a/pkarr/clippy.toml b/pkarr/clippy.toml new file mode 100644 index 0000000..154626e --- /dev/null +++ b/pkarr/clippy.toml @@ -0,0 +1 @@ +allow-unwrap-in-tests = true diff --git a/pkarr/src/extra/endpoints/mod.rs b/pkarr/src/extra/endpoints/mod.rs index ccdf67d..80c8d4c 100644 --- a/pkarr/src/extra/endpoints/mod.rs +++ b/pkarr/src/extra/endpoints/mod.rs @@ -158,7 +158,7 @@ mod tests { #[tokio::test] async fn resolve_endpoints() { - let testnet = Testnet::new(3); + let testnet = Testnet::new(3).unwrap(); let client = Client::builder().testnet(&testnet).build().unwrap(); let tld = generate(&client, 3, 3, Some("example.com".to_string())).await; @@ -193,7 +193,7 @@ mod tests { #[tokio::test] async fn resolve_addresses() { - let testnet = Testnet::new(3); + let testnet = Testnet::new(3).unwrap(); let client = Client::builder().testnet(&testnet).build().unwrap(); let tld = generate(&client, 3, 3, None).await; From a4eca9dc332aa6a85e7451734f1c2c11e2ccb336 Mon Sep 17 00:00:00 2001 From: nazeh Date: Tue, 8 Oct 2024 11:31:04 +0300 Subject: [PATCH 33/84] feat: update to use new mainline::Id struct --- Cargo.lock | 2 +- pkarr/src/client/dht.rs | 55 +++++++++++++++++++++++++--------------- server/src/dht_server.rs | 2 +- 3 files changed, 37 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 679851c..9c27dc6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1143,7 +1143,7 @@ checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" [[package]] name = "mainline" version = "3.0.0" -source = "git+https://github.com/pubky/mainline?branch=dev#47edb4617f3ce25883b889064de5c7d257848b8f" +source = "git+https://github.com/pubky/mainline?branch=dev#51ebed1bb69e7ab53d5857f44a10453b97ef0723" dependencies = [ "bytes", "crc", diff --git a/pkarr/src/client/dht.rs b/pkarr/src/client/dht.rs index 6a0404b..43f03ac 100644 --- a/pkarr/src/client/dht.rs +++ b/pkarr/src/client/dht.rs @@ -295,13 +295,14 @@ impl Client { ) -> Result>> { let mutable_item: MutableItem = (signed_packet).into(); - if let Some(current) = self.cache.get(&mutable_item.target().bytes) { + if let Some(current) = self.cache.get(mutable_item.target().as_bytes()) { if current.timestamp() > signed_packet.timestamp() { return Err(Error::NotMostRecent); } }; - self.cache.put(&mutable_item.target().bytes, signed_packet); + self.cache + .put(mutable_item.target().as_bytes(), signed_packet); let (sender, receiver) = flume::bounded::>(1); @@ -317,7 +318,7 @@ impl Client { let (sender, receiver) = flume::bounded::(1); - let cached_packet = self.cache.get(&target.bytes); + let cached_packet = self.cache.get(target.as_bytes()); if let Some(ref cached) = cached_packet { let expires_in = cached.expires_in(self.minimum_ttl, self.maximum_ttl); @@ -426,24 +427,25 @@ fn run(mut rpc: Rpc, cache: Box, settings: Settings, receiver: Receiv response: QueryResponseSpecific::Value(Response::Mutable(mutable_item)), } => { if let Ok(signed_packet) = &SignedPacket::try_from(mutable_item) { - let new_packet = - if let Some(ref cached) = cache.get_read_only(&target.bytes) { - if signed_packet.more_recent_than(cached) { - debug!( - ?target, - "Received more recent packet than in cache" - ); - Some(signed_packet) - } else { - None - } - } else { - debug!(?target, "Received new packet after cache miss"); + let new_packet = if let Some(ref cached) = + cache.get_read_only(target.as_bytes()) + { + if signed_packet.more_recent_than(cached) { + debug!( + ?target, + "Received more recent packet than in cache" + ); Some(signed_packet) - }; + } else { + None + } + } else { + debug!(?target, "Received new packet after cache miss"); + Some(signed_packet) + }; if let Some(packet) = new_packet { - cache.put(&target.bytes, packet); + cache.put(target.as_bytes(), packet); if let Some(set) = senders.get(target) { for sender in set { @@ -458,12 +460,12 @@ fn run(mut rpc: Rpc, cache: Box, settings: Settings, receiver: Receiv target, response: QueryResponseSpecific::Value(Response::NoMoreRecentValue(seq)), } => { - if let Some(mut cached) = cache.get_read_only(&target.bytes) { + if let Some(mut cached) = cache.get_read_only(target.as_bytes()) { if (*seq as u64) == cached.timestamp() { trace!("Remote node has the a packet with same timestamp, refreshing cached packet."); cached.refresh(); - cache.put(&target.bytes, &cached); + cache.put(target.as_bytes(), &cached); // Send the found sequence as a timestamp to the caller to decide what to do // with it. @@ -658,4 +660,17 @@ mod tests { .await .unwrap(); } + + #[tokio::test] + async fn return_expired_packet_fallback() { + let client = Client::builder().build().unwrap(); + + let keypair = Keypair::random(); + let mut packet = dns::Packet::new_reply(0); + let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); + + client + .cache() + .put(keypair.public_key().into(), &signed_packet); + } } diff --git a/server/src/dht_server.rs b/server/src/dht_server.rs index 8067ee0..93bd5c6 100644 --- a/server/src/dht_server.rs +++ b/server/src/dht_server.rs @@ -78,7 +78,7 @@ impl Server for DhtServer { .. } = request { - let should_query = if let Some(cached) = self.cache.get(&target.bytes) { + let should_query = if let Some(cached) = self.cache.get(target.as_bytes()) { debug!( public_key = ?cached.public_key(), ?target, From 4ae72cc1d897ed939b9b5a194c49e0ec6eaa14bf Mon Sep 17 00:00:00 2001 From: nazeh Date: Tue, 8 Oct 2024 13:07:49 +0300 Subject: [PATCH 34/84] feat: return expired SignedPacket as fallback, closes #67 --- CHANGELOG.md | 3 ++ Cargo.lock | 1 + pkarr/Cargo.toml | 6 ++- pkarr/src/base/cache.rs | 40 ++++++++++++------- pkarr/src/base/signed_packet.rs | 5 ++- pkarr/src/base/timestamp.rs | 1 + pkarr/src/client/dht.rs | 33 ++++++++++++++-- pkarr/src/client/relay.rs | 69 +++++++++++++++++++++------------ pkarr/src/error.rs | 2 +- pkarr/src/lib.rs | 1 + 10 files changed, 115 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce3bd55..d893d10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ All notable changes to pkarr client and server will be documented in this file. ### Added - Add strict monotonic unix `Timestamp`. +- Impl `PartialEq, Eq` for `SignedPacket`. +- Impl `From` for `CacheKey`. ### Changed @@ -16,6 +18,7 @@ All notable changes to pkarr client and server will be documented in this file. - replace `ureq` with `reqwest` to work with HTTP/2 relays, and Wasm. - update `mainline` to v3.0.0 - `Client::shutdown` and `Client::shutdown_sync` are now idempotent and return `()`. +- `Client::resolve`, `Client::resolve_sync` and `relay::Client::resolve` return expired cached `SignedPacket` if nothing else was found from the network. ### Removed diff --git a/Cargo.lock b/Cargo.lock index 9c27dc6..12189d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1486,6 +1486,7 @@ dependencies = [ "reqwest", "self_cell", "serde", + "sha1_smol", "simple-dns", "thiserror", "tokio", diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index 5fff22f..4665708 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -25,6 +25,7 @@ rand = "0.8.5" # optional dependencies serde = { version = "1.0.209", features = ["derive"], optional = true } futures = { version = "0.3.30", optional = true, default-features = false } +sha1_smol = { version = "1.0.1", optional = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] # Dht client dependencies: @@ -38,6 +39,7 @@ js-sys = "0.3.69" futures = "0.3.30" getrandom = { version = "0.2", features = ["js"] } reqwest = "0.12.7" +sha1_smol = "1.0.1" [dev-dependencies] postcard = { version = "1.0.10", features = ["alloc"] } @@ -56,7 +58,7 @@ serde = ["dep:serde"] ## Use [dht::Client] dht = ["dep:mainline", "flume"] ## Use [relay::Client] -relay = ["dep:reqwest", "dep:tokio"] +relay = ["dep:reqwest", "dep:tokio", "dep:sha1_smol"] # Extra ## Use [Client::resolve_endpoints] and [Client::resolve_https_endpoints] @@ -65,7 +67,7 @@ endpoints = ["dep:futures"] ## Use all features full = ["dht", "relay", "serde", "endpoints"] -default = ["dht", "relay", "serde"] +default = ["dht", "serde"] [package.metadata.docs.rs] all-features = true diff --git a/pkarr/src/base/cache.rs b/pkarr/src/base/cache.rs index 8a121ca..d466576 100644 --- a/pkarr/src/base/cache.rs +++ b/pkarr/src/base/cache.rs @@ -6,24 +6,38 @@ use std::fmt::Debug; use std::num::NonZeroUsize; use std::sync::{Arc, Mutex}; -use crate::{PublicKey, SignedPacket}; +#[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] +use mainline::MutableItem; + +use crate::SignedPacket; /// The sha1 hash of the [PublicKey] used as the key in [Cache]. pub type CacheKey = [u8; 20]; -impl From<&PublicKey> for CacheKey { - /// Take the first 20 bytes from a [PublicKey] instead of hashing with sha1. - /// Useful when you don't want to import Sha1 hasher, or mainline crate. - /// - /// Risks evicting SignedPackets from the cache if there ar two packets whose - /// PublicKey share the first 20 bytes. - /// - /// Don't use both PublicKey and SignedPacket::target() to store packets, - /// otherwise you will end up duplicating and wasting cache space. - fn from(public_key: &PublicKey) -> CacheKey { - let cache_key: [u8; 20] = public_key.as_bytes()[0..20].try_into().unwrap(); +#[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] +impl From<&crate::PublicKey> for CacheKey { + fn from(public_key: &crate::PublicKey) -> CacheKey { + MutableItem::target_from_key(public_key.as_bytes(), &None).into() + } +} + +#[cfg(any(target_arch = "wasm32", all(not(feature = "dht"), feature = "relay")))] +impl From<&crate::PublicKey> for CacheKey { + fn from(public_key: &crate::PublicKey) -> CacheKey { + let mut encoded = vec![]; + + encoded.extend(public_key); + + let mut hasher = Sha1::new(); + hasher.update(&encoded); + hasher.digest().bytes() + } +} - cache_key +#[cfg(any(target_arch = "wasm32", feature = "dht", feature = "relay"))] +impl From for CacheKey { + fn from(value: crate::PublicKey) -> Self { + (&value).into() } } diff --git a/pkarr/src/base/signed_packet.rs b/pkarr/src/base/signed_packet.rs index fdd091e..99fbaa8 100644 --- a/pkarr/src/base/signed_packet.rs +++ b/pkarr/src/base/signed_packet.rs @@ -26,7 +26,7 @@ self_cell!( dependent: Packet, } - impl{Debug} + impl{Debug, PartialEq, Eq} ); impl Inner { @@ -56,7 +56,7 @@ impl Inner { } } -#[derive(Debug)] +#[derive(Debug, PartialEq, Eq)] /// Signed DNS packet pub struct SignedPacket { inner: Inner, @@ -569,6 +569,7 @@ mod tests { } } + #[cfg(feature = "dht")] #[test] fn to_mutable() { let keypair = Keypair::random(); diff --git a/pkarr/src/base/timestamp.rs b/pkarr/src/base/timestamp.rs index 1fd6ed6..ba9252c 100644 --- a/pkarr/src/base/timestamp.rs +++ b/pkarr/src/base/timestamp.rs @@ -302,6 +302,7 @@ mod tests { assert_eq!(decoded, timestamp) } + #[cfg(feature = "serde")] #[test] fn serde() { let timestamp = Timestamp::now(); diff --git a/pkarr/src/client/dht.rs b/pkarr/src/client/dht.rs index 43f03ac..55510b0 100644 --- a/pkarr/src/client/dht.rs +++ b/pkarr/src/client/dht.rs @@ -412,7 +412,15 @@ fn run(mut rpc: Rpc, cache: Box, settings: Settings, receiver: Receiv // === Drop senders to done queries === for id in &report.done_get_queries { - senders.remove(id); + if let Some(senders) = senders.remove(id) { + if let Some(cached) = cache.get(id.as_bytes()) { + debug!(public_key = ?cached.public_key(), "Returning expired cache as a fallback"); + // Send cached packets if available + for sender in senders { + let _ = sender.send(cached.clone()); + } + } + }; } // === Receive and handle incoming messages === @@ -502,6 +510,8 @@ pub enum ActorMessage { #[cfg(test)] mod tests { + use std::time::Duration; + use mainline::Testnet; use super::*; @@ -663,14 +673,29 @@ mod tests { #[tokio::test] async fn return_expired_packet_fallback() { - let client = Client::builder().build().unwrap(); + let testnet = Testnet::new(10).unwrap(); + + let client = Client::builder() + .testnet(&testnet) + .dht_settings(DhtSettings { + request_timeout: Duration::from_millis(10).into(), + ..Default::default() + }) + // Everything is expired + .maximum_ttl(0) + .build() + .unwrap(); let keypair = Keypair::random(); - let mut packet = dns::Packet::new_reply(0); + let packet = dns::Packet::new_reply(0); let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); client .cache() - .put(keypair.public_key().into(), &signed_packet); + .put(&keypair.public_key().into(), &signed_packet); + + let resolved = client.resolve(&keypair.public_key()).await.unwrap(); + + assert_eq!(resolved, Some(signed_packet)); } } diff --git a/pkarr/src/client/relay.rs b/pkarr/src/client/relay.rs index 6b56a13..ba12743 100644 --- a/pkarr/src/client/relay.rs +++ b/pkarr/src/client/relay.rs @@ -178,12 +178,24 @@ impl Client { /// with the transport, transparent from [reqwest::Error]. /// - Returns [Error::IO] if something went wrong while reading the payload. pub async fn resolve(&self, public_key: &PublicKey) -> Result> { - let cached_packet = match self.get_from_cache(public_key) { - None => None, - Some(signed_packet) => return Ok(Some(signed_packet)), + if let Some(cached_packet) = self.cache.get(&public_key.as_ref().into()) { + let expires_in = cached_packet.expires_in(self.minimum_ttl, self.maximum_ttl); + + if expires_in > 0 { + debug!(expires_in, "Have fresh signed_packet in cache."); + return Ok(Some(cached_packet)); + } + + debug!(expires_in, "Have expired signed_packet in cache."); + + return Ok(self + .race_resolve(public_key, Some(cached_packet.clone())) + .await? + .or(Some(cached_packet))); }; + debug!("Cache miss"); - self.race_resolve(public_key, cached_packet).await + self.race_resolve(public_key, None).await } // === Native Race implementation === @@ -315,26 +327,6 @@ impl Client { }) } - fn get_from_cache(&self, public_key: &PublicKey) -> Option { - let cached_packet = self.cache.get(&public_key.as_ref().into()); - - if let Some(cached) = cached_packet { - let expires_in = cached.expires_in(self.minimum_ttl, self.maximum_ttl); - - if expires_in > 0 { - debug!(expires_in, "Have fresh signed_packet in cache."); - - return Some(cached.clone()); - } - - debug!(expires_in, "Have expired signed_packet in cache."); - } else { - debug!("Cache miss"); - }; - - None - } - async fn resolve_from_relay( &self, relay: &str, @@ -459,4 +451,33 @@ mod tests { assert!(resolved.is_none()); } + + #[tokio::test] + async fn return_expired_packet_fallback() { + let keypair = Keypair::random(); + + let mut server = mockito::Server::new_async().await; + + let path = format!("/{}", keypair.public_key()); + + server.mock("GET", path.as_str()).with_status(404).create(); + + let relays = vec![server.url()]; + let client = Client::builder() + .relays(relays) + .maximum_ttl(0) + .build() + .unwrap(); + + let packet = dns::Packet::new_reply(0); + let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); + + client + .cache() + .put(&keypair.public_key().into(), &signed_packet); + + let resolved = client.resolve(&keypair.public_key()).await.unwrap(); + + assert_eq!(resolved, Some(signed_packet)); + } } diff --git a/pkarr/src/error.rs b/pkarr/src/error.rs index 1c2dbe2..6c3e65b 100644 --- a/pkarr/src/error.rs +++ b/pkarr/src/error.rs @@ -52,7 +52,7 @@ pub enum Error { PacketTooLarge(usize), // === Flume errors === - #[cfg(not(target_arch = "wasm32"))] + #[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] #[error(transparent)] /// Transparent [flume::RecvError] Receive(#[from] flume::RecvError), diff --git a/pkarr/src/lib.rs b/pkarr/src/lib.rs index 9e4519c..0592cd5 100644 --- a/pkarr/src/lib.rs +++ b/pkarr/src/lib.rs @@ -27,6 +27,7 @@ pub const DEFAULT_RELAYS: [&str; 2] = ["https://relay.pkarr.org", "https://pkarr /// Default [resolver](https://pkarr.org/resolvers)s pub const DEFAULT_RESOLVERS: [&str; 2] = ["resolver.pkarr.org:6881", "pkarr.pubky.app:6881"]; +#[cfg(any(target_arch = "wasm32", feature = "dht"))] pub use client::{Client, ClientBuilder, Settings}; // Rexports From 67ddf4c6e3d55db3122fcf7f77976e13894bb381 Mon Sep 17 00:00:00 2001 From: nazeh Date: Tue, 8 Oct 2024 13:26:58 +0300 Subject: [PATCH 35/84] feat(server): update pkarr client to move fallback out of server --- server/src/handlers.rs | 46 +++++++++++++++--------------------------- 1 file changed, 16 insertions(+), 30 deletions(-) diff --git a/server/src/handlers.rs b/server/src/handlers.rs index 3449a1d..69a3ba3 100644 --- a/server/src/handlers.rs +++ b/server/src/handlers.rs @@ -4,7 +4,6 @@ use axum::{extract::State, response::IntoResponse}; use bytes::Bytes; use http::{header, StatusCode}; -use pkarr::mainline::MutableItem; use tracing::error; use pkarr::{PublicKey, DEFAULT_MAXIMUM_TTL, DEFAULT_MINIMUM_TTL}; @@ -51,35 +50,22 @@ pub async fn get( let public_key = PublicKey::try_from(public_key.as_str()) .map_err(|error| Error::new(StatusCode::BAD_REQUEST, Some(error)))?; - let signed_packet = { - if let Some(signed_packet) = - state - .client - .resolve(&public_key) - .await - .map_err(|error| match error { - pkarr::Error::DhtIsShutdown => { - error!("Dht is shutdown"); - Error::with_status(StatusCode::INTERNAL_SERVER_ERROR) - } - error => { - error!(?error, "Unexpected error"); - Error::with_status(StatusCode::INTERNAL_SERVER_ERROR) - } - })? - { - Some(signed_packet) - } else { - // Respond with what we have, even if expired. - // TODO: move this fallback to the client itself, closing #67 - state - .client - .cache() - .get_read_only(&MutableItem::target_from_key(public_key.as_bytes(), &None)) - } - }; - - if let Some(signed_packet) = signed_packet { + if let Some(signed_packet) = state + .client + .resolve(&public_key) + .await + // TODO: remove this map + .map_err(|error| match error { + pkarr::Error::DhtIsShutdown => { + error!("Dht is shutdown"); + Error::with_status(StatusCode::INTERNAL_SERVER_ERROR) + } + error => { + error!(?error, "Unexpected error"); + Error::with_status(StatusCode::INTERNAL_SERVER_ERROR) + } + })? + { tracing::debug!(?public_key, "cache hit responding with packet!"); let body = signed_packet.to_relay_payload(); From b5d1cc5e8d325b5ee1bd63e419d3972443acef8a Mon Sep 17 00:00:00 2001 From: nazeh Date: Tue, 8 Oct 2024 18:15:19 +0300 Subject: [PATCH 36/84] feat(pkarr): add SignedPacket::serialize and SignedPacket::deserialize --- CHANGELOG.md | 2 + pkarr/src/base/signed_packet.rs | 139 +++++++++++++++++++++++++++----- 2 files changed, 121 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d893d10..3fbfd22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ All notable changes to pkarr client and server will be documented in this file. - Add strict monotonic unix `Timestamp`. - Impl `PartialEq, Eq` for `SignedPacket`. - Impl `From` for `CacheKey`. +- Add `SignedPacket::serialize` and `SignedPacket::deserialize`. +- Derive `serde::Serialize` and `serde::Deserialize` for `SignedPacket`. ### Changed diff --git a/pkarr/src/base/signed_packet.rs b/pkarr/src/base/signed_packet.rs index 99fbaa8..0bda291 100644 --- a/pkarr/src/base/signed_packet.rs +++ b/pkarr/src/base/signed_packet.rs @@ -14,6 +14,9 @@ use std::{ net::{Ipv4Addr, Ipv6Addr}, }; +#[cfg(feature = "serde")] +use serde::{Deserialize, Serialize}; + use crate::Timestamp; const DOT: char = '.'; @@ -49,7 +52,7 @@ impl Inner { })?) } - fn try_from_bytes(bytes: &Bytes) -> Result { + fn try_from_bytes(bytes: Bytes) -> Result { Ok(Inner::try_new(bytes.to_owned(), |bytes| { Packet::parse(&bytes[104..]) })?) @@ -101,7 +104,7 @@ impl SignedPacket { public_key.verify(&signable(timestamp, encoded_packet), &signature)?; Ok(SignedPacket { - inner: Inner::try_from_bytes(bytes)?, + inner: Inner::try_from_bytes(bytes.to_owned())?, last_seen: Timestamp::now(), }) } @@ -110,7 +113,7 @@ impl SignedPacket { /// like ones stored on disk or in a database. pub fn from_bytes_unchecked(bytes: &Bytes, last_seen: impl Into) -> SignedPacket { SignedPacket { - inner: Inner::try_from_bytes(bytes).unwrap(), + inner: Inner::try_from_bytes(bytes.to_owned()).unwrap(), last_seen: last_seen.into(), } } @@ -195,6 +198,37 @@ impl SignedPacket { self.inner.borrow_owner() } + /// Returns a serialized representation of this [SignedPacket] including + /// the [SignedPacket::last_seen] timestamp followed by the returned value from [SignedPacket::as_bytes]. + pub fn serialize(&self) -> Bytes { + let mut bytes = Vec::with_capacity(SignedPacket::MAX_BYTES as usize); + bytes.extend_from_slice(&self.last_seen.to_bytes()); + bytes.extend_from_slice(self.as_bytes()); + + bytes.into() + } + + /// Deserialize [SignedPacket] from a serialized version for persistent storage using + /// [SignedPacket::serialize]. + /// + /// If deserializing the [SignedPacket::last_seen] failed, or is far in the future, + /// it will be unwrapped to default, i.e the UNIX_EPOCH. + /// + /// That is useful for backwards compatibility if you + /// ever stored the [SignedPacket::last_seen] as Little Endian in previous versions. + pub fn deserialize(bytes: &[u8]) -> Result { + let mut last_seen = Timestamp::try_from(&bytes[0..8]).unwrap_or_default(); + + if last_seen > (&Timestamp::now() + 60_000_000) { + last_seen = Timestamp::from(0) + } + + Ok(SignedPacket { + inner: Inner::try_from_bytes(bytes[8..].to_owned().into())?, + last_seen, + }) + } + /// Returns a slice of the serialized [SignedPacket] omitting the leading public_key, /// to be sent as a request/response body to or from [relays](https://github.com/Nuhvi/pkarr/blob/main/design/relays.md). pub fn to_relay_payload(&self) -> Bytes { @@ -339,6 +373,29 @@ fn signable(timestamp: u64, v: &Bytes) -> Bytes { signable.into() } +fn normalize_name(origin: &str, name: String) -> String { + let name = if name.ends_with(DOT) { + name[..name.len() - 1].to_string() + } else { + name + }; + + let parts: Vec<&str> = name.split('.').collect(); + let last = *parts.last().unwrap_or(&""); + + if last == origin { + // Already normalized. + return name.to_string(); + } + + if last == "@" || last.is_empty() { + // Shorthand of origin + return origin.to_string(); + } + + format!("{}.{}", name, origin) +} + #[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] use mainline::MutableItem; @@ -429,27 +486,30 @@ impl Display for SignedPacket { } } -fn normalize_name(origin: &str, name: String) -> String { - let name = if name.ends_with(DOT) { - name[..name.len() - 1].to_string() - } else { - name - }; - - let parts: Vec<&str> = name.split('.').collect(); - let last = *parts.last().unwrap_or(&""); +// === Serialization === - if last == origin { - // Already normalized. - return name.to_string(); +#[cfg(feature = "serde")] +impl Serialize for SignedPacket { + /// Serialize a [SignedPacket] for persistent storage. + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + self.serialize().serialize(serializer) } +} - if last == "@" || last.is_empty() { - // Shorthand of origin - return origin.to_string(); - } +#[cfg(feature = "serde")] +impl<'de> Deserialize<'de> for SignedPacket { + /// Deserialize a [SignedPacket] from persistent storage. + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let bytes: Vec = Deserialize::deserialize(deserializer)?; - format!("{}.{}", name, origin) + SignedPacket::deserialize(&bytes).map_err(serde::de::Error::custom) + } } #[cfg(all(test, not(target_arch = "wasm32")))] @@ -823,4 +883,43 @@ mod tests { DEFAULT_MAXIMUM_TTL * 2 ); } + + #[cfg(feature = "serde")] + #[test] + fn serde() { + use postcard::{from_bytes, to_allocvec}; + + let keypair = Keypair::random(); + + let mut packet = Packet::new_reply(0); + packet.answers.push(ResourceRecord::new( + Name::new("_derp_region.iroh.").unwrap(), + simple_dns::CLASS::IN, + 30, + RData::A(A { + address: Ipv4Addr::new(1, 1, 1, 1).into(), + }), + )); + + let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); + + let serialized = to_allocvec(&signed_packet).unwrap(); + let deserialized: SignedPacket = from_bytes(&serialized).unwrap(); + + assert_eq!(deserialized, signed_packet); + + // Backwards compat + { + let mut bytes = vec![]; + + bytes.extend_from_slice(&[210, 1]); + bytes.extend_from_slice(&signed_packet.last_seen().into_u64().to_le_bytes()); + bytes.extend_from_slice(signed_packet.as_bytes()); + + let deserialized: SignedPacket = from_bytes(&bytes).unwrap(); + + assert_eq!(deserialized.as_bytes(), signed_packet.as_bytes()); + assert_eq!(deserialized.last_seen(), &Timestamp::from(0)); + } + } } From 4024391e1501403c567efdc2f5b635dd138b5b64 Mon Sep 17 00:00:00 2001 From: nazeh Date: Tue, 8 Oct 2024 19:33:08 +0300 Subject: [PATCH 37/84] feat(pkarr): move server's HeedCache to pkarr::LmdbCache --- Cargo.lock | 4 +- pkarr/Cargo.toml | 8 +- .../cache.rs => pkarr/src/extra/lmdb_cache.rs | 177 ++++++++++-------- pkarr/src/extra/mod.rs | 3 + pkarr/src/lib.rs | 3 + server/Cargo.toml | 4 +- server/src/dht_server.rs | 6 +- server/src/main.rs | 9 +- 8 files changed, 117 insertions(+), 97 deletions(-) rename server/src/cache.rs => pkarr/src/extra/lmdb_cache.rs (54%) diff --git a/Cargo.lock b/Cargo.lock index 12189d0..c4d5fee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1468,6 +1468,7 @@ name = "pkarr" version = "3.0.0" dependencies = [ "base32", + "byteorder", "bytes", "clap", "document-features", @@ -1476,6 +1477,7 @@ dependencies = [ "flume", "futures", "getrandom", + "heed", "js-sys", "lru", "mainline", @@ -1501,12 +1503,10 @@ dependencies = [ "anyhow", "axum", "axum-server", - "byteorder", "bytes", "clap", "dirs-next", "governor", - "heed", "http", "pkarr", "rustls", diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index 4665708..9fab202 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -26,6 +26,8 @@ rand = "0.8.5" serde = { version = "1.0.209", features = ["derive"], optional = true } futures = { version = "0.3.30", optional = true, default-features = false } sha1_smol = { version = "1.0.1", optional = true } +heed = { version = "0.20.0", default-features = false, optional = true } +byteorder = { version = "1.5.0", default-features = false, optional = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] # Dht client dependencies: @@ -63,11 +65,13 @@ relay = ["dep:reqwest", "dep:tokio", "dep:sha1_smol"] # Extra ## Use [Client::resolve_endpoints] and [Client::resolve_https_endpoints] endpoints = ["dep:futures"] +## Use [LmdbCache] +lmdb-cache = ["dep:heed", "dep:byteorder"] ## Use all features -full = ["dht", "relay", "serde", "endpoints"] +full = ["dht", "relay", "serde", "endpoints", "lmdb-cache"] -default = ["dht", "serde"] +default = ["dht"] [package.metadata.docs.rs] all-features = true diff --git a/server/src/cache.rs b/pkarr/src/extra/lmdb_cache.rs similarity index 54% rename from server/src/cache.rs rename to pkarr/src/extra/lmdb_cache.rs index b9162f2..7e13293 100644 --- a/server/src/cache.rs +++ b/pkarr/src/extra/lmdb_cache.rs @@ -1,20 +1,22 @@ -use std::{borrow::Cow, path::Path, time::Duration}; +use core::result::Result; +use std::{borrow::Cow, fs, path::Path, time::Duration}; -use pkarr::{Cache, CacheKey, SignedPacket, Timestamp}; +use crate::{Cache, CacheKey, SignedPacket, Timestamp}; use byteorder::LittleEndian; use heed::{types::U64, BoxedError, BytesDecode, BytesEncode, Database, Env, EnvOpenOptions}; -use anyhow::Result; use tracing::debug; -const PKARR_CACHE_TABLE_NAME_SIGNED_PACKET: &str = "pkarrcache:signed_packet"; -const PKARR_CACHE_TABLE_NAME_KEY_TO_TIME: &str = "pkarrcache:key_to_time"; -const PKARR_CACHE_TABLE_NAME_TIME_TO_KEY: &str = "pkarrcache:time_to_key"; +const MAX_MAP_SIZE: usize = 10995116277760; // 10 TB -type CacheSignedPacketsTable = Database; -type CacheKeyToTimeTable = Database>; -type CacheTimeToKeyTable = Database, CacheKeyCodec>; +const SIGNED_PACKET_TABLE: &str = "pkarrcache:signed_packet"; +const KEY_TO_TIME_TABLE: &str = "pkarrcache:key_to_time"; +const TIME_TO_KEY_TABLE: &str = "pkarrcache:time_to_key"; + +type SignedPacketsTable = Database; +type KeyToTimeTable = Database>; +type TimeToKeyTable = Database, CacheKeyCodec>; pub struct CacheKeyCodec; @@ -41,16 +43,7 @@ impl<'a> BytesEncode<'a> for SignedPacketCodec { type EItem = SignedPacket; fn bytes_encode(signed_packet: &Self::EItem) -> Result, BoxedError> { - let bytes = signed_packet.as_bytes(); - - let mut vec = Vec::with_capacity(bytes.len() + 8); - - vec.extend( - >::bytes_encode(&signed_packet.last_seen().into_u64())?.as_ref(), - ); - vec.extend(bytes); - - Ok(Cow::Owned(vec)) + Ok(Cow::Owned(signed_packet.serialize().to_vec())) } } @@ -58,25 +51,33 @@ impl<'a> BytesDecode<'a> for SignedPacketCodec { type DItem = SignedPacket; fn bytes_decode(bytes: &'a [u8]) -> Result { - let last_seen = >::bytes_decode(bytes)?; - - Ok(SignedPacket::from_bytes_unchecked( - &bytes[8..].to_vec().into(), - last_seen, - )) + Ok(SignedPacket::deserialize(bytes)?) } } #[derive(Debug, Clone)] -pub struct HeedCache { +pub struct LmdbCache { capacity: usize, env: Env, + signed_packets_table: SignedPacketsTable, + key_to_time_table: KeyToTimeTable, + time_to_key_table: TimeToKeyTable, } -impl HeedCache { - pub fn new(env_path: &Path, capacity: usize) -> Result { +impl LmdbCache { + /// Creates a new [LmdbCache] at the `env_path` and set the [heed::EnvOpenOptions::map_size] + /// to a multiple of the `capacity` by [SignedPacket::MAX_BYTES], aligned to 4096 bytes, and + /// a maximum of 10 TB. + pub fn new(env_path: &Path, capacity: usize) -> Result { // Page aligned but more than enough bytes for `capacity` many SignedPacket - let map_size = (((capacity * 1112) + 4095) / 4096) * 4096; + let map_size = capacity + .checked_mul(SignedPacket::MAX_BYTES as usize) + .and_then(|x| x.checked_add(4096)) + .and_then(|x| x.checked_div(4096)) + .and_then(|x| x.checked_mul(4096)) + .unwrap_or(MAX_MAP_SIZE); + + fs::create_dir_all(env_path)?; let env = unsafe { EnvOpenOptions::new() @@ -86,16 +87,23 @@ impl HeedCache { }; let mut wtxn = env.write_txn()?; - let _: CacheSignedPacketsTable = - env.create_database(&mut wtxn, Some(PKARR_CACHE_TABLE_NAME_SIGNED_PACKET))?; - let _: CacheKeyToTimeTable = - env.create_database(&mut wtxn, Some(PKARR_CACHE_TABLE_NAME_KEY_TO_TIME))?; - let _: CacheTimeToKeyTable = - env.create_database(&mut wtxn, Some(PKARR_CACHE_TABLE_NAME_TIME_TO_KEY))?; + + let signed_packets_table: SignedPacketsTable = + env.create_database(&mut wtxn, Some(SIGNED_PACKET_TABLE))?; + let key_to_time_table: KeyToTimeTable = + env.create_database(&mut wtxn, Some(KEY_TO_TIME_TABLE))?; + let time_to_key_table: TimeToKeyTable = + env.create_database(&mut wtxn, Some(TIME_TO_KEY_TABLE))?; wtxn.commit()?; - let instance = Self { capacity, env }; + let instance = Self { + capacity, + env, + signed_packets_table, + key_to_time_table, + time_to_key_table, + }; let clone = instance.clone(); std::thread::spawn(move || loop { @@ -106,38 +114,28 @@ impl HeedCache { Ok(instance) } - pub fn internal_len(&self) -> Result { + pub fn internal_len(&self) -> Result { let rtxn = self.env.read_txn()?; + let len = self.signed_packets_table.len(&rtxn)? as usize; + rtxn.commit()?; - let db: CacheSignedPacketsTable = self - .env - .open_database(&rtxn, Some(PKARR_CACHE_TABLE_NAME_SIGNED_PACKET))? - .unwrap(); - - Ok(db.len(&rtxn)? as usize) + Ok(len) } - pub fn internal_put(&self, key: &CacheKey, signed_packet: &SignedPacket) -> Result<()> { + pub fn internal_put( + &self, + key: &CacheKey, + signed_packet: &SignedPacket, + ) -> Result<(), heed::Error> { if self.capacity == 0 { return Ok(()); } let mut wtxn = self.env.write_txn()?; - let packets: CacheSignedPacketsTable = self - .env - .open_database(&wtxn, Some(PKARR_CACHE_TABLE_NAME_SIGNED_PACKET))? - .unwrap(); - - let key_to_time: CacheKeyToTimeTable = self - .env - .open_database(&wtxn, Some(PKARR_CACHE_TABLE_NAME_KEY_TO_TIME))? - .unwrap(); - - let time_to_key: CacheTimeToKeyTable = self - .env - .open_database(&wtxn, Some(PKARR_CACHE_TABLE_NAME_TIME_TO_KEY))? - .unwrap(); + let packets = self.signed_packets_table; + let key_to_time = self.key_to_time_table; + let time_to_key = self.time_to_key_table; let len = packets.len(&wtxn)? as usize; @@ -171,22 +169,12 @@ impl HeedCache { Ok(()) } - pub fn internal_get(&self, key: &CacheKey) -> Result> { + pub fn internal_get(&self, key: &CacheKey) -> Result, heed::Error> { let mut wtxn = self.env.write_txn()?; - let packets: CacheSignedPacketsTable = self - .env - .open_database(&wtxn, Some(PKARR_CACHE_TABLE_NAME_SIGNED_PACKET))? - .unwrap(); - - let key_to_time: CacheKeyToTimeTable = self - .env - .open_database(&wtxn, Some(PKARR_CACHE_TABLE_NAME_KEY_TO_TIME))? - .unwrap(); - let time_to_key: CacheTimeToKeyTable = self - .env - .open_database(&wtxn, Some(PKARR_CACHE_TABLE_NAME_TIME_TO_KEY))? - .unwrap(); + let packets = self.signed_packets_table; + let key_to_time = self.key_to_time_table; + let time_to_key = self.time_to_key_table; if let Some(signed_packet) = packets.get(&wtxn, key)? { if let Some(time) = key_to_time.get(&wtxn, key)? { @@ -208,12 +196,15 @@ impl HeedCache { Ok(None) } - pub fn internal_get_read_only(&self, key: &CacheKey) -> Result> { + pub fn internal_get_read_only( + &self, + key: &CacheKey, + ) -> Result, heed::Error> { let rtxn = self.env.read_txn()?; - let packets: CacheSignedPacketsTable = self + let packets: SignedPacketsTable = self .env - .open_database(&rtxn, Some(PKARR_CACHE_TABLE_NAME_SIGNED_PACKET))? + .open_database(&rtxn, Some(SIGNED_PACKET_TABLE))? .unwrap(); if let Some(signed_packet) = packets.get(&rtxn, key)? { @@ -226,12 +217,12 @@ impl HeedCache { } } -impl Cache for HeedCache { +impl Cache for LmdbCache { fn len(&self) -> usize { match self.internal_len() { Ok(result) => result, Err(error) => { - debug!(?error, "Error in HeedCache::len"); + debug!(?error, "Error in LmdbCache::len"); 0 } } @@ -239,7 +230,7 @@ impl Cache for HeedCache { fn put(&self, key: &CacheKey, signed_packet: &SignedPacket) { if let Err(error) = self.internal_put(key, signed_packet) { - debug!(?error, "Error in HeedCache::put"); + debug!(?error, "Error in LmdbCache::put"); }; } @@ -247,7 +238,7 @@ impl Cache for HeedCache { match self.internal_get(key) { Ok(result) => result, Err(error) => { - debug!(?error, "Error in HeedCache::get"); + debug!(?error, "Error in LmdbCache::get"); None } @@ -258,10 +249,36 @@ impl Cache for HeedCache { match self.internal_get_read_only(key) { Ok(result) => result, Err(error) => { - debug!(?error, "Error in HeedCache::get"); + debug!(?error, "Error in LmdbCache::get"); None } } } } + +#[derive(thiserror::Error, Debug)] +/// Pkarr crate error enum. +pub enum Error { + #[error(transparent)] + /// Transparent [heed::Error] + Lmdb(#[from] heed::Error), + + #[error(transparent)] + /// Transparent [std::io::Error] + IO(#[from] std::io::Error), +} + +#[cfg(test)] +mod tests { + use std::usize; + + use super::*; + + #[test] + fn max_map_size() { + let env_path = std::env::temp_dir().join(Timestamp::now().to_string()); + + LmdbCache::new(&env_path, usize::MAX).unwrap(); + } +} diff --git a/pkarr/src/extra/mod.rs b/pkarr/src/extra/mod.rs index abf2208..fe20e28 100644 --- a/pkarr/src/extra/mod.rs +++ b/pkarr/src/extra/mod.rs @@ -1,2 +1,5 @@ #[cfg(feature = "endpoints")] pub mod endpoints; + +#[cfg(feature = "lmdb-cache")] +pub mod lmdb_cache; diff --git a/pkarr/src/lib.rs b/pkarr/src/lib.rs index 0592cd5..fdd3ba3 100644 --- a/pkarr/src/lib.rs +++ b/pkarr/src/lib.rs @@ -41,3 +41,6 @@ pub use mainline; #[cfg(feature = "endpoints")] pub use extra::endpoints::{Endpoint, EndpointResolver}; + +#[cfg(feature = "lmdb-cache")] +pub use extra::lmdb_cache::LmdbCache; diff --git a/server/Cargo.toml b/server/Cargo.toml index 0bb615d..50ed9f5 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -20,10 +20,8 @@ thiserror = "1.0.49" bytes = "1.7.1" tower_governor = "0.4.2" governor = "0.6.3" -heed = { version = "0.20.0", default-features = false } -byteorder = "1.5.0" serde = { version = "1.0.199", features = ["derive"] } toml = "0.8.12" clap = { version = "4.5.1", features = ["derive"] } dirs-next = "2.0.0" -pkarr = { version = "3.0.0", path = "../pkarr" } +pkarr = { version = "3.0.0", path = "../pkarr", features = ["dht", "lmdb-cache"] } diff --git a/server/src/dht_server.rs b/server/src/dht_server.rs index 93bd5c6..a8a4b07 100644 --- a/server/src/dht_server.rs +++ b/server/src/dht_server.rs @@ -21,13 +21,13 @@ use pkarr::{ use tracing::debug; -use crate::{cache::HeedCache, rate_limiting::IpRateLimiter}; +use crate::rate_limiting::IpRateLimiter; /// DhtServer with Rate limiting pub struct DhtServer { inner: mainline::server::DhtServer, resolvers: Option>, - cache: Box, + cache: Box, minimum_ttl: u32, maximum_ttl: u32, rate_limiter: IpRateLimiter, @@ -41,7 +41,7 @@ impl Debug for DhtServer { impl DhtServer { pub fn new( - cache: Box, + cache: Box, resolvers: Option>, minimum_ttl: u32, maximum_ttl: u32, diff --git a/server/src/main.rs b/server/src/main.rs index 329069c..f164e77 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -1,4 +1,3 @@ -mod cache; mod config; mod dht_server; mod error; @@ -7,15 +6,13 @@ mod http_server; mod rate_limiting; use anyhow::Result; -use cache::HeedCache; use clap::Parser; use config::Config; -use std::fs; use std::path::PathBuf; use tracing::{debug, info}; use http_server::HttpServer; -use pkarr::{mainline::dht::DhtSettings, Client}; +use pkarr::{mainline::dht::DhtSettings, Client, LmdbCache}; #[derive(Parser, Debug)] struct Cli { @@ -46,9 +43,7 @@ async fn main() -> Result<()> { debug!(?config, "Pkarr server config"); - let env_path = &config.cache_path()?; - fs::create_dir_all(env_path)?; - let cache = Box::new(HeedCache::new(env_path, config.cache_size()).unwrap()); + let cache = Box::new(LmdbCache::new(&config.cache_path()?, config.cache_size())?); let rate_limiter = rate_limiting::IpRateLimiter::new(config.rate_limiter()); From eb0a425900fc1622b0de795c7df652c954802f98 Mon Sep 17 00:00:00 2001 From: nazeh Date: Wed, 9 Oct 2024 16:00:14 +0300 Subject: [PATCH 38/84] feat: update mainline and remove mainline::error::Error --- CHANGELOG.md | 1 + Cargo.lock | 2 +- pkarr/src/client/dht.rs | 16 ++++++++-------- pkarr/src/error.rs | 3 +-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fbfd22..1bcda55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ All notable changes to pkarr client and server will be documented in this file. - Impl `From` for `CacheKey`. - Add `SignedPacket::serialize` and `SignedPacket::deserialize`. - Derive `serde::Serialize` and `serde::Deserialize` for `SignedPacket`. +- Add `pkarr::LmdbCache` for persistent cache using lmdb. ### Changed diff --git a/Cargo.lock b/Cargo.lock index c4d5fee..ffdca8b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1143,7 +1143,7 @@ checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" [[package]] name = "mainline" version = "3.0.0" -source = "git+https://github.com/pubky/mainline?branch=dev#51ebed1bb69e7ab53d5857f44a10453b97ef0723" +source = "git+https://github.com/pubky/mainline?branch=dev#bb68155c9110a1db74be1e7b30c30456de2458f6" dependencies = [ "bytes", "crc", diff --git a/pkarr/src/client/dht.rs b/pkarr/src/client/dht.rs index 55510b0..ff749bf 100644 --- a/pkarr/src/client/dht.rs +++ b/pkarr/src/client/dht.rs @@ -7,7 +7,7 @@ use mainline::{ messages, QueryResponse, QueryResponseSpecific, ReceivedFrom, ReceivedMessage, Response, Rpc, }, - Id, MutableItem, Testnet, + Id, MutableItem, PutError, Testnet, }; use std::{ collections::HashMap, @@ -207,8 +207,8 @@ impl Client { match self.publish_inner(signed_packet)?.recv_async().await { Ok(Ok(_)) => Ok(()), Ok(Err(error)) => match error { - mainline::Error::PutQueryIsInflight(_) => Err(Error::PublishInflight), - _ => Err(Error::MainlineError(error)), + PutError::PutQueryIsInflight(_) => Err(Error::PublishInflight), + _ => Err(Error::PublishError(error)), }, // Since we pass this sender to `Rpc::put`, the only reason the sender, // would be dropped, is if `Rpc` is dropped, which should only happeng on shutdown. @@ -254,8 +254,8 @@ impl Client { match self.publish_inner(signed_packet)?.recv() { Ok(Ok(_)) => Ok(()), Ok(Err(error)) => match error { - mainline::Error::PutQueryIsInflight(_) => Err(Error::PublishInflight), - _ => Err(Error::MainlineError(error)), + PutError::PutQueryIsInflight(_) => Err(Error::PublishInflight), + _ => Err(Error::PublishError(error)), }, // Since we pass this sender to `Rpc::put`, the only reason the sender, // would be dropped, is if `Rpc` is dropped, which should only happeng on shutdown. @@ -292,7 +292,7 @@ impl Client { pub(crate) fn publish_inner( &self, signed_packet: &SignedPacket, - ) -> Result>> { + ) -> Result>> { let mutable_item: MutableItem = (signed_packet).into(); if let Some(current) = self.cache.get(mutable_item.target().as_bytes()) { @@ -304,7 +304,7 @@ impl Client { self.cache .put(mutable_item.target().as_bytes(), signed_packet); - let (sender, receiver) = flume::bounded::>(1); + let (sender, receiver) = flume::bounded::>(1); self.sender .send(ActorMessage::Publish(mutable_item, sender)) @@ -503,7 +503,7 @@ fn run(mut rpc: Rpc, cache: Box, settings: Settings, receiver: Receiv } pub enum ActorMessage { - Publish(MutableItem, Sender>), + Publish(MutableItem, Sender>), Resolve(Id, Sender, Option), Shutdown(Sender<()>), } diff --git a/pkarr/src/error.rs b/pkarr/src/error.rs index 6c3e65b..bc9a2eb 100644 --- a/pkarr/src/error.rs +++ b/pkarr/src/error.rs @@ -16,8 +16,7 @@ pub enum Error { #[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] #[error(transparent)] - /// Transparent [mainline::Error] - MainlineError(#[from] mainline::Error), + PublishError(#[from] mainline::PutError), // === Keys errors === #[error("Invalid PublicKey length, expected 32 bytes but got: {0}")] From 5d564cdbadbebade9f1504e15a8f39f6e9ba726c Mon Sep 17 00:00:00 2001 From: nazeh Date: Thu, 10 Oct 2024 15:51:38 +0300 Subject: [PATCH 39/84] feat(pkarr): closes #23 more granular error enums --- Cargo.lock | 2 +- pkarr/Cargo.toml | 6 +- pkarr/examples/publish.rs | 12 ++- pkarr/src/base/cache.rs | 2 +- pkarr/src/base/keys.rs | 60 +++++++++------ pkarr/src/base/signed_packet.rs | 92 +++++++++++++---------- pkarr/src/base/timestamp.rs | 1 + pkarr/src/client/dht.rs | 123 ++++++++++++++++++------------- pkarr/src/client/mod.rs | 4 +- pkarr/src/client/relay.rs | 99 +++++++++++++++++-------- pkarr/src/error.rs | 86 --------------------- pkarr/src/extra/endpoints/mod.rs | 58 +++++++++++---- pkarr/src/extra/lmdb_cache.rs | 1 - pkarr/src/lib.rs | 8 +- server/src/handlers.rs | 26 +++---- 15 files changed, 306 insertions(+), 274 deletions(-) delete mode 100644 pkarr/src/error.rs diff --git a/Cargo.lock b/Cargo.lock index ffdca8b..4c6fc1b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1143,7 +1143,7 @@ checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" [[package]] name = "mainline" version = "3.0.0" -source = "git+https://github.com/pubky/mainline?branch=dev#bb68155c9110a1db74be1e7b30c30456de2458f6" +source = "git+https://github.com/pubky/mainline?branch=dev#be9c071426fe89a37ab30b764f0e2b24d2f83993" dependencies = [ "bytes", "crc", diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index 9fab202..f7f26bc 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -57,13 +57,13 @@ tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } serde = ["dep:serde"] # Clients -## Use [dht::Client] +## Use [client::dht::Client] dht = ["dep:mainline", "flume"] -## Use [relay::Client] +## Use [client::relay::Client] relay = ["dep:reqwest", "dep:tokio", "dep:sha1_smol"] # Extra -## Use [Client::resolve_endpoints] and [Client::resolve_https_endpoints] +## Use [extra::endpoints::EndpointResolver] trait endpoints = ["dep:futures"] ## Use [LmdbCache] lmdb-cache = ["dep:heed", "dep:byteorder"] diff --git a/pkarr/examples/publish.rs b/pkarr/examples/publish.rs index c32ba82..8a37d6f 100644 --- a/pkarr/examples/publish.rs +++ b/pkarr/examples/publish.rs @@ -11,15 +11,15 @@ use tracing_subscriber; use std::time::Instant; -use pkarr::{dns, Client, Keypair, Result, SignedPacket}; +use pkarr::{dns, Client, Keypair, SignedPacket}; #[tokio::main] -async fn main() -> Result<()> { +async fn main() { tracing_subscriber::fmt() .with_max_level(Level::DEBUG) .init(); - let client = Client::builder().build()?; + let client = Client::builder().build().unwrap(); let keypair = Keypair::random(); @@ -28,10 +28,10 @@ async fn main() -> Result<()> { dns::Name::new("_foo").unwrap(), dns::CLASS::IN, 30, - dns::rdata::RData::TXT("bar".try_into()?), + dns::rdata::RData::TXT("bar".try_into().unwrap()), )); - let signed_packet = SignedPacket::from_packet(&keypair, &packet)?; + let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); let instant = Instant::now(); @@ -49,6 +49,4 @@ async fn main() -> Result<()> { println!("\nFailed to publish {} \n {}", keypair.public_key(), err); } }; - - Ok(()) } diff --git a/pkarr/src/base/cache.rs b/pkarr/src/base/cache.rs index d466576..039fbd3 100644 --- a/pkarr/src/base/cache.rs +++ b/pkarr/src/base/cache.rs @@ -11,7 +11,7 @@ use mainline::MutableItem; use crate::SignedPacket; -/// The sha1 hash of the [PublicKey] used as the key in [Cache]. +/// The sha1 hash of the [crate::PublicKey] used as the key in [Cache]. pub type CacheKey = [u8; 20]; #[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] diff --git a/pkarr/src/base/keys.rs b/pkarr/src/base/keys.rs index d99f763..8c96d31 100644 --- a/pkarr/src/base/keys.rs +++ b/pkarr/src/base/keys.rs @@ -1,7 +1,8 @@ //! Utility structs for Ed25519 keys. -use crate::{Error, Result}; -use ed25519_dalek::{SecretKey, Signature, Signer, SigningKey, Verifier, VerifyingKey}; +use ed25519_dalek::{ + SecretKey, Signature, SignatureError, Signer, SigningKey, Verifier, VerifyingKey, +}; use rand::rngs::OsRng; use std::{ fmt::{self, Debug, Display, Formatter}, @@ -31,11 +32,8 @@ impl Keypair { self.0.sign(message) } - pub fn verify(&self, message: &[u8], signature: &Signature) -> Result<()> { - self.0 - .verify(message, signature) - .map_err(|_| Error::InvalidEd25519Signature)?; - Ok(()) + pub fn verify(&self, message: &[u8], signature: &Signature) -> Result<(), SignatureError> { + self.0.verify(message, signature) } pub fn secret_key(&self) -> SecretKey { @@ -73,11 +71,8 @@ impl PublicKey { } /// Verify a signature over a message. - pub fn verify(&self, message: &[u8], signature: &Signature) -> Result<()> { - self.0 - .verify(message, signature) - .map_err(|_| Error::InvalidEd25519Signature)?; - Ok(()) + pub fn verify(&self, message: &[u8], signature: &Signature) -> Result<(), SignatureError> { + self.0.verify(message, signature) } /// Return a reference to the underlying [VerifyingKey] @@ -109,31 +104,33 @@ impl AsRef for PublicKey { } impl TryFrom<&[u8]> for PublicKey { - type Error = Error; + type Error = PublicKeyError; fn try_from(bytes: &[u8]) -> Result { let bytes_32: &[u8; 32] = bytes .try_into() - .map_err(|_| Error::InvalidPublicKeyLength(bytes.len()))?; + .map_err(|_| PublicKeyError::InvalidPublicKeyLength(bytes.len()))?; Ok(Self( - VerifyingKey::from_bytes(bytes_32).map_err(|_| Error::InvalidEd25519PublicKey)?, + VerifyingKey::from_bytes(bytes_32) + .map_err(|_| PublicKeyError::InvalidEd25519PublicKey)?, )) } } impl TryFrom<&[u8; 32]> for PublicKey { - type Error = Error; + type Error = PublicKeyError; fn try_from(public: &[u8; 32]) -> Result { Ok(Self( - VerifyingKey::from_bytes(public).map_err(|_| Error::InvalidEd25519PublicKey)?, + VerifyingKey::from_bytes(public) + .map_err(|_| PublicKeyError::InvalidEd25519PublicKey)?, )) } } impl TryFrom<&str> for PublicKey { - type Error = Error; + type Error = PublicKeyError; /// Convert the TLD in a `&str` to a [PublicKey]. /// @@ -149,7 +146,7 @@ impl TryFrom<&str> for PublicKey { /// - `https://foo@bar.o4dksfbqk85ogzdb5osziw6befigbuxmuxkuxq8434q89uj56uyy.?q=v` /// - `https://foo@bar.o4dksfbqk85ogzdb5osziw6befigbuxmuxkuxq8434q89uj56uyy.:8888?q=v` /// - `https://yg4gxe7z1r7mr6orids9fh95y7gxhdsxjqi6nngsxxtakqaxr5no.o4dksfbqk85ogzdb5osziw6befigbuxmuxkuxq8434q89uj56uyy` - fn try_from(s: &str) -> Result { + fn try_from(s: &str) -> Result { let mut s = s; if s.len() > 52 { @@ -199,20 +196,20 @@ impl TryFrom<&str> for PublicKey { let bytes = if let Some(v) = base32::decode(base32::Alphabet::Z, s) { Ok(v) } else { - Err(Error::InvalidPublicKeyEncoding) + Err(PublicKeyError::InvalidPublicKeyEncoding) }?; let verifying_key = VerifyingKey::try_from(bytes.as_slice()) - .map_err(|_| Error::InvalidPublicKeyLength(bytes.len()))?; + .map_err(|_| PublicKeyError::InvalidPublicKeyLength(bytes.len()))?; Ok(PublicKey(verifying_key)) } } impl TryFrom for PublicKey { - type Error = Error; + type Error = PublicKeyError; - fn try_from(s: String) -> Result { + fn try_from(s: String) -> Result { s.as_str().try_into() } } @@ -268,6 +265,23 @@ impl<'de> Deserialize<'de> for PublicKey { } } +#[derive(thiserror::Error, Debug)] +/// Errors while trying to create a [PublicKey] +pub enum PublicKeyError { + #[error("Invalid PublicKey length, expected 32 bytes but got: {0}")] + InvalidPublicKeyLength(usize), + + #[error("Invalid Ed25519 publickey; Cannot decompress Edwards point")] + InvalidEd25519PublicKey, + + #[error("Invalid PublicKey encoding")] + InvalidPublicKeyEncoding, + + #[error("DNS Packet is too large, expected max 1000 bytes but got: {0}")] + // DNS packet endocded and compressed is larger than 1000 bytes + PacketTooLarge(usize), +} + #[cfg(test)] mod tests { use super::*; diff --git a/pkarr/src/base/signed_packet.rs b/pkarr/src/base/signed_packet.rs index 0bda291..811b510 100644 --- a/pkarr/src/base/signed_packet.rs +++ b/pkarr/src/base/signed_packet.rs @@ -1,12 +1,12 @@ //! Signed DNS packet -use crate::{Error, Keypair, PublicKey, Result}; +use crate::{Keypair, PublicKey}; use bytes::{Bytes, BytesMut}; -use ed25519_dalek::Signature; +use ed25519_dalek::{Signature, SignatureError}; use self_cell::self_cell; use simple_dns::{ rdata::{RData, A, AAAA}, - Name, Packet, ResourceRecord, + Name, Packet, ResourceRecord, SimpleDnsError, }; use std::{ char, @@ -38,7 +38,7 @@ impl Inner { signature: &Signature, timestamp: u64, encoded_packet: &Bytes, - ) -> Result { + ) -> Result { // Create the inner bytes from timestamp> let mut bytes = BytesMut::with_capacity(encoded_packet.len() + 104); @@ -47,15 +47,11 @@ impl Inner { bytes.extend_from_slice(×tamp.to_be_bytes()); bytes.extend_from_slice(encoded_packet); - Ok(Self::try_new(bytes.into(), |bytes| { - Packet::parse(&bytes[104..]) - })?) + Self::try_new(bytes.into(), |bytes| Packet::parse(&bytes[104..])) } - fn try_from_bytes(bytes: Bytes) -> Result { - Ok(Inner::try_new(bytes.to_owned(), |bytes| { - Packet::parse(&bytes[104..]) - })?) + fn try_from_bytes(bytes: Bytes) -> Result { + Inner::try_new(bytes.to_owned(), |bytes| Packet::parse(&bytes[104..])) } } @@ -81,19 +77,14 @@ impl SignedPacket { /// You can skip all these validations by using [Self::from_bytes_unchecked] instead. /// /// You can use [Self::from_relay_payload] instead if you are receiving a response from an HTTP relay. - /// - /// # Errors - /// - Returns [crate::Error::InvalidSignedPacketBytesLength] if `bytes.len()` is smaller than 104 bytes - /// - Returns [crate::Error::PacketTooLarge] if `bytes.len()` is bigger than [SignedPacket::MAX_BYTES] bytes - /// - Returns [crate::Error::InvalidEd25519PublicKey] if the first 32 bytes are invalid `ed25519` public key - /// - Returns [crate::Error::InvalidEd25519Signature] if the following 64 bytes are invalid `ed25519` signature - /// - Returns [crate::Error::DnsError] if it failed to parse the DNS Packet after the first 104 bytes - pub fn from_bytes(bytes: &Bytes) -> Result { + pub fn from_bytes(bytes: &Bytes) -> Result { if bytes.len() < 104 { - return Err(Error::InvalidSignedPacketBytesLength(bytes.len())); + return Err(SignedPacketError::InvalidSignedPacketBytesLength( + bytes.len(), + )); } if (bytes.len() as u64) > SignedPacket::MAX_BYTES { - return Err(Error::PacketTooLarge(bytes.len())); + return Err(SignedPacketError::PacketTooLarge(bytes.len())); } let public_key = PublicKey::try_from(&bytes[..32])?; let signature = Signature::from_bytes(bytes[32..96].try_into().unwrap()); @@ -119,13 +110,10 @@ impl SignedPacket { } /// Creates a [SignedPacket] from a [PublicKey] and the [relays](https://github.com/Nuhvi/pkarr/blob/main/design/relays.md) payload. - /// - /// # Errors - /// - Returns [crate::Error::InvalidSignedPacketBytesLength] if `payload` is too small - /// - Returns [crate::Error::PacketTooLarge] if the payload is too large. - /// - Returns [crate::Error::InvalidEd25519Signature] if the signature in the payload is invalid - /// - Returns [crate::Error::DnsError] if it failed to parse the DNS Packet - pub fn from_relay_payload(public_key: &PublicKey, payload: &Bytes) -> Result { + pub fn from_relay_payload( + public_key: &PublicKey, + payload: &Bytes, + ) -> Result { let mut bytes = BytesMut::with_capacity(payload.len() + 32); bytes.extend_from_slice(public_key.as_bytes()); @@ -138,10 +126,10 @@ impl SignedPacket { /// /// It will also normalize the names of the [ResourceRecord]s to be relative to the origin, /// which would be the z-base32 encoded [PublicKey] of the [Keypair] used to sign the Packet. - /// - /// # Errors - /// - Returns [crate::Error::DnsError] if the packet is invalid or it failed to compress or encode it. - pub fn from_packet(keypair: &Keypair, packet: &Packet) -> Result { + pub fn from_packet( + keypair: &Keypair, + packet: &Packet, + ) -> Result { // Normalize names to the origin TLD let mut inner = Packet::new_reply(0); @@ -172,7 +160,7 @@ impl SignedPacket { let encoded_packet: Bytes = inner.build_bytes_vec_compressed()?.into(); if encoded_packet.len() > 1000 { - return Err(Error::PacketTooLarge(encoded_packet.len())); + return Err(SignedPacketError::PacketTooLarge(encoded_packet.len())); } let timestamp = Timestamp::now().into(); @@ -216,7 +204,7 @@ impl SignedPacket { /// /// That is useful for backwards compatibility if you /// ever stored the [SignedPacket::last_seen] as Little Endian in previous versions. - pub fn deserialize(bytes: &[u8]) -> Result { + pub fn deserialize(bytes: &[u8]) -> Result { let mut last_seen = Timestamp::try_from(&bytes[0..8]).unwrap_or_default(); if last_seen > (&Timestamp::now() + 60_000_000) { @@ -399,6 +387,8 @@ fn normalize_name(origin: &str, name: String) -> String { #[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] use mainline::MutableItem; +use super::keys::PublicKeyError; + #[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] impl From<&SignedPacket> for MutableItem { fn from(s: &SignedPacket) -> Self { @@ -417,10 +407,10 @@ impl From<&SignedPacket> for MutableItem { #[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] impl TryFrom<&MutableItem> for SignedPacket { - type Error = Error; + type Error = SignedPacketError; - fn try_from(i: &MutableItem) -> Result { - let public_key = PublicKey::try_from(i.key()).unwrap(); + fn try_from(i: &MutableItem) -> Result { + let public_key = PublicKey::try_from(i.key())?; let seq = *i.seq() as u64; let signature: Signature = i.signature().into(); @@ -512,6 +502,34 @@ impl<'de> Deserialize<'de> for SignedPacket { } } +#[derive(thiserror::Error, Debug)] +/// Errors trying to parse or create a [SignedPacket] +pub enum SignedPacketError { + #[error(transparent)] + SignatureError(#[from] SignatureError), + + #[error(transparent)] + PublicKeyError(#[from] PublicKeyError), + + #[error(transparent)] + /// Transparent [simple_dns::SimpleDnsError] + DnsError(#[from] simple_dns::SimpleDnsError), + + #[error("Invalid SignedPacket bytes length, expected at least 104 bytes but got: {0}")] + /// Serialized signed packets are `<32 bytes publickey><64 bytes signature><8 bytes + /// timestamp>`. + InvalidSignedPacketBytesLength(usize), + + #[error("Invalid relay payload size, expected at least 72 bytes but got: {0}")] + /// Relay api http-body should be `<64 bytes signature><8 bytes timestamp> + /// `. + InvalidRelayPayloadSize(usize), + + #[error("DNS Packet is too large, expected max 1000 bytes but got: {0}")] + // DNS packet endocded and compressed is larger than 1000 bytes + PacketTooLarge(usize), +} + #[cfg(all(test, not(target_arch = "wasm32")))] mod tests { use super::*; diff --git a/pkarr/src/base/timestamp.rs b/pkarr/src/base/timestamp.rs index ba9252c..11b3d66 100644 --- a/pkarr/src/base/timestamp.rs +++ b/pkarr/src/base/timestamp.rs @@ -238,6 +238,7 @@ pub fn system_time() -> u64 { } #[derive(thiserror::Error, Debug)] +/// Errors trying to decode a [Timestamp] pub enum TimestampError { #[error("Invalid bytes length, Timestamp should be encoded as 8 bytes, got {0}")] InvalidBytesLength(usize), diff --git a/pkarr/src/client/dht.rs b/pkarr/src/client/dht.rs index ff749bf..7becc3e 100644 --- a/pkarr/src/client/dht.rs +++ b/pkarr/src/client/dht.rs @@ -21,7 +21,7 @@ use crate::{ Cache, InMemoryCache, DEFAULT_CACHE_SIZE, DEFAULT_MAXIMUM_TTL, DEFAULT_MINIMUM_TTL, DEFAULT_RESOLVERS, }; -use crate::{Error, PublicKey, Result, SignedPacket}; +use crate::{PublicKey, SignedPacket}; #[derive(Debug)] /// [Client]'s settings @@ -130,7 +130,7 @@ impl ClientBuilder { self } - pub fn build(self) -> Result { + pub fn build(self) -> Result { Client::new(self.settings) } } @@ -146,7 +146,7 @@ pub struct Client { } impl Client { - pub fn new(settings: Settings) -> Result { + pub fn new(settings: Settings) -> Result { let (sender, receiver) = flume::bounded(32); let rpc = Rpc::new(&settings.dht)?; @@ -196,24 +196,17 @@ impl Client { // === Public Methods === /// Publishes a [SignedPacket] to the Dht. - /// - /// # Errors - /// - Returns a [Error::DhtIsShutdown] if [Client::shutdown] was called, or - /// the loop in the actor thread is stopped for any reason (like thread panic). - /// - Returns a [Error::PublishInflight] if the client is currently publishing the same public_key. - /// - Returns a [Error::NotMostRecent] if the provided signed packet is older than most recent. - /// - Returns a [Error::MainlineError] if the Dht received an unexpected error otherwise. - pub async fn publish(&self, signed_packet: &SignedPacket) -> Result<()> { - match self.publish_inner(signed_packet)?.recv_async().await { - Ok(Ok(_)) => Ok(()), - Ok(Err(error)) => match error { - PutError::PutQueryIsInflight(_) => Err(Error::PublishInflight), - _ => Err(Error::PublishError(error)), - }, - // Since we pass this sender to `Rpc::put`, the only reason the sender, - // would be dropped, is if `Rpc` is dropped, which should only happeng on shutdown. - Err(_) => Err(Error::DhtIsShutdown), - } + pub async fn publish(&self, signed_packet: &SignedPacket) -> Result<(), PublishError> { + self.publish_inner(signed_packet)? + .recv_async() + .await + .expect("Query was dropped before sending a response, please open an issue.") + .map_err(|error| match error { + PutError::PutQueryIsInflight(_) => PublishError::PublishInflight, + _ => PublishError::MainlinePutError(error), + })?; + + Ok(()) } /// Returns a [SignedPacket] from cache if it is not expired, otherwise, @@ -224,9 +217,12 @@ impl Client { /// and updating the cache with any more recent valid packets it receives. /// /// # Errors - /// - Returns a [Error::DhtIsShutdown] if [Client::shutdown] was called, or + /// - Returns a [ClientWasShutdown] if [Client::shutdown] was called, or /// the loop in the actor thread is stopped for any reason (like thread panic). - pub async fn resolve(&self, public_key: &PublicKey) -> Result> { + pub async fn resolve( + &self, + public_key: &PublicKey, + ) -> Result, ClientWasShutdown> { Ok(self.resolve_inner(public_key)?.recv_async().await.ok()) } @@ -243,24 +239,16 @@ impl Client { // === Sync === /// Publishes a [SignedPacket] to the Dht. - /// - /// # Errors - /// - Returns a [Error::DhtIsShutdown] if [Client::shutdown] was called, or - /// the loop in the actor thread is stopped for any reason (like thread panic). - /// - Returns a [Error::PublishInflight] if the client is currently publishing the same public_key. - /// - Returns a [Error::NotMostRecent] if the provided signed packet is older than most recent. - /// - Returns a [Error::MainlineError] if the Dht received an unexpected error otherwise. - pub fn publish_sync(&self, signed_packet: &SignedPacket) -> Result<()> { - match self.publish_inner(signed_packet)?.recv() { - Ok(Ok(_)) => Ok(()), - Ok(Err(error)) => match error { - PutError::PutQueryIsInflight(_) => Err(Error::PublishInflight), - _ => Err(Error::PublishError(error)), - }, - // Since we pass this sender to `Rpc::put`, the only reason the sender, - // would be dropped, is if `Rpc` is dropped, which should only happeng on shutdown. - Err(_) => Err(Error::DhtIsShutdown), - } + pub fn publish_sync(&self, signed_packet: &SignedPacket) -> Result<(), PublishError> { + self.publish_inner(signed_packet)? + .recv() + .expect("Query was dropped before sending a response, please open an issue.") + .map_err(|error| match error { + PutError::PutQueryIsInflight(_) => PublishError::PublishInflight, + _ => PublishError::MainlinePutError(error), + })?; + + Ok(()) } /// Returns a [SignedPacket] from cache if it is not expired, otherwise, @@ -271,9 +259,12 @@ impl Client { /// and updating the cache with any more recent valid packets it receives. /// /// # Errors - /// - Returns a [Error::DhtIsShutdown] if [Client::shutdown] was called, or + /// - Returns a [ClientWasShutdown] if [Client::shutdown] was called, or /// the loop in the actor thread is stopped for any reason (like thread panic). - pub fn resolve_sync(&self, public_key: &PublicKey) -> Result> { + pub fn resolve_sync( + &self, + public_key: &PublicKey, + ) -> Result, ClientWasShutdown> { Ok(self.resolve_inner(public_key)?.recv().ok()) } @@ -292,12 +283,12 @@ impl Client { pub(crate) fn publish_inner( &self, signed_packet: &SignedPacket, - ) -> Result>> { + ) -> Result>, PublishError> { let mutable_item: MutableItem = (signed_packet).into(); if let Some(current) = self.cache.get(mutable_item.target().as_bytes()) { if current.timestamp() > signed_packet.timestamp() { - return Err(Error::NotMostRecent); + return Err(PublishError::NotMostRecent); } }; @@ -308,12 +299,15 @@ impl Client { self.sender .send(ActorMessage::Publish(mutable_item, sender)) - .map_err(|_| Error::DhtIsShutdown)?; + .map_err(|_| PublishError::ClientWasShutdown)?; Ok(receiver) } - pub(crate) fn resolve_inner(&self, public_key: &PublicKey) -> Result> { + pub(crate) fn resolve_inner( + &self, + public_key: &PublicKey, + ) -> Result, ClientWasShutdown> { let target = MutableItem::target_from_key(public_key.as_bytes(), &None); let (sender, receiver) = flume::bounded::(1); @@ -326,9 +320,7 @@ impl Client { if expires_in > 0 { debug!(expires_in, "Have fresh signed_packet in cache."); - sender - .send(cached.clone()) - .map_err(|_| Error::DhtIsShutdown)?; + sender.send(cached.clone()).map_err(|_| ClientWasShutdown)?; return Ok(receiver); } @@ -347,12 +339,41 @@ impl Client { // any more recent versions. cached_packet.as_ref().map(|cached| cached.timestamp()), )) - .map_err(|_| Error::DhtIsShutdown)?; + .map_err(|_| ClientWasShutdown)?; Ok(receiver) } } +#[derive(Debug)] +pub struct ClientWasShutdown; + +impl std::error::Error for ClientWasShutdown {} + +impl std::fmt::Display for ClientWasShutdown { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Pkarr Client was shutdown") + } +} + +#[derive(thiserror::Error, Debug)] +/// Errors occuring during publishing a [SignedPacket] +pub enum PublishError { + #[error("Found a more recent SignedPacket in the client's cache")] + /// Found a more recent SignedPacket in the client's cache + NotMostRecent, + + #[error("Pkarr Client was shutdown")] + ClientWasShutdown, + + #[error("Publish query is already inflight for the same public_key")] + /// [crate::Client::publish] is already inflight to the same public_key + PublishInflight, + + #[error(transparent)] + MainlinePutError(#[from] PutError), +} + fn run(mut rpc: Rpc, cache: Box, settings: Settings, receiver: Receiver) { debug!(?settings, "Starting Client main loop.."); diff --git a/pkarr/src/client/mod.rs b/pkarr/src/client/mod.rs index 79ed76e..df2dd77 100644 --- a/pkarr/src/client/mod.rs +++ b/pkarr/src/client/mod.rs @@ -1,12 +1,12 @@ //! Client implementation. #[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] -mod dht; +pub mod dht; #[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] pub use dht::{Client, ClientBuilder, Settings}; #[cfg(target_arch = "wasm32")] -mod relay; +pub mod relay; #[cfg(target_arch = "wasm32")] pub use relay::{Client, ClientBuilder, Settings}; diff --git a/pkarr/src/client/relay.rs b/pkarr/src/client/relay.rs index ba12743..a848cdd 100644 --- a/pkarr/src/client/relay.rs +++ b/pkarr/src/client/relay.rs @@ -1,5 +1,6 @@ //! Pkarr client for publishing and resolving [SignedPacket]s over [relays](https://pkarr.org/relays). +use std::fmt::{self, Debug, Display, Formatter}; use std::num::NonZeroUsize; use reqwest::{Response, StatusCode}; @@ -12,8 +13,8 @@ use futures::future::select_ok; use tokio::task::JoinSet; use crate::{ - Cache, Error, InMemoryCache, PublicKey, Result, SignedPacket, DEFAULT_CACHE_SIZE, - DEFAULT_MAXIMUM_TTL, DEFAULT_MINIMUM_TTL, DEFAULT_RELAYS, + Cache, InMemoryCache, PublicKey, SignedPacket, DEFAULT_CACHE_SIZE, DEFAULT_MAXIMUM_TTL, + DEFAULT_MINIMUM_TTL, DEFAULT_RELAYS, }; #[derive(Debug, Clone)] @@ -94,7 +95,7 @@ impl ClientBuilder { self } - pub fn build(self) -> Result { + pub fn build(self) -> Result { Client::new(self.settings) } } @@ -116,9 +117,9 @@ impl Default for Client { } impl Client { - pub fn new(settings: Settings) -> Result { + pub fn new(settings: Settings) -> Result { if settings.relays.is_empty() { - return Err(Error::EmptyListOfRelays); + return Err(EmptyListOfRelays); } let cache = settings @@ -148,23 +149,18 @@ impl Client { /// Publishes a [SignedPacket] to this client's relays. /// /// Return the first successful completion, or the last failure. - /// - /// # Errors - /// - Returns a [Error::NotMostRecent] if the provided signed packet is older than most recent. - /// - Returns a [Error::RelayError] from the last responding relay, if all relays - /// responded with non-2xx status codes. - pub async fn publish(&self, signed_packet: &SignedPacket) -> Result<()> { + pub async fn publish(&self, signed_packet: &SignedPacket) -> Result<(), PublishToRelayError> { let public_key = signed_packet.public_key(); if let Some(current) = self.cache.get(&public_key.as_ref().into()) { if current.timestamp() > signed_packet.timestamp() { - return Err(Error::NotMostRecent); + return Err(PublishToRelayError::NotMostRecent); } }; self.cache.put(&public_key.as_ref().into(), signed_packet); - self.race_publish(signed_packet).await + Ok(self.race_publish(signed_packet).await?) } /// Resolve a [SignedPacket] from this client's relays. @@ -172,12 +168,15 @@ impl Client { /// Return the first successful response, or the failure from the last responding relay. /// /// # Errors - /// - /// - Returns [Error::RelayError] if the relay responded with a status >= 400 + /// - Returns [reqwest::Error] if all relays responded with a status >= 400 /// (except 404 in which case you should receive Ok(None)) or something wrong /// with the transport, transparent from [reqwest::Error]. - /// - Returns [Error::IO] if something went wrong while reading the payload. - pub async fn resolve(&self, public_key: &PublicKey) -> Result> { + /// + /// In that case, return the last error we got from the last responding relay. + pub async fn resolve( + &self, + public_key: &PublicKey, + ) -> Result, reqwest::Error> { if let Some(cached_packet) = self.cache.get(&public_key.as_ref().into()) { let expires_in = cached_packet.expires_in(self.minimum_ttl, self.maximum_ttl); @@ -201,7 +200,7 @@ impl Client { // === Native Race implementation === #[cfg(not(target_arch = "wasm32"))] - async fn race_publish(&self, signed_packet: &SignedPacket) -> Result<()> { + async fn race_publish(&self, signed_packet: &SignedPacket) -> Result<(), reqwest::Error> { let mut futures = JoinSet::new(); for relay in self.relays.clone() { @@ -211,19 +210,19 @@ impl Client { futures.spawn(async move { this.publish_to_relay(&relay, signed_packet).await }); } - let mut last_error = Error::EmptyListOfRelays; + let mut last_error = None; while let Some(result) = futures.join_next().await { match result { Ok(Ok(_)) => return Ok(()), - Ok(Err(error)) => last_error = error, + Ok(Err(error)) => last_error = Some(error), Err(joinerror) => { debug!(?joinerror); } } } - Err(last_error) + Err(last_error.expect("failed to receive any error responses!")) } #[cfg(not(target_arch = "wasm32"))] @@ -231,7 +230,7 @@ impl Client { &self, public_key: &PublicKey, cached_packet: Option, - ) -> Result> { + ) -> Result, reqwest::Error> { let mut futures = JoinSet::new(); for relay in self.relays.clone() { @@ -242,7 +241,7 @@ impl Client { futures.spawn(async move { this.resolve_from_relay(&relay, public_key, cached).await }); } - let mut result: Result> = Ok(None); + let mut result: Result, reqwest::Error> = Ok(None); while let Some(task_result) = futures.join_next().await { match task_result { @@ -266,7 +265,7 @@ impl Client { // === Wasm === #[cfg(target_arch = "wasm32")] - async fn race_publish(&self, signed_packet: &SignedPacket) -> Result<()> { + async fn race_publish(&self, signed_packet: &SignedPacket) -> Result<(), reqwest::Error> { let futures = self.relays.iter().map(|relay| { let signed_packet = signed_packet.clone(); let this = self.clone(); @@ -285,7 +284,7 @@ impl Client { &self, public_key: &PublicKey, cached_packet: Option, - ) -> Result> { + ) -> Result, reqwest::Error> { let futures = self.relays.iter().map(|relay| { let public_key = public_key.clone(); let cached = cached_packet.clone(); @@ -312,7 +311,11 @@ impl Client { // === Private Methods === - async fn publish_to_relay(&self, relay: &str, signed_packet: SignedPacket) -> Result { + async fn publish_to_relay( + &self, + relay: &str, + signed_packet: SignedPacket, + ) -> Result { let url = format!("{relay}/{}", signed_packet.public_key()); self.http_client @@ -323,7 +326,7 @@ impl Client { .map_err(|error| { debug!(?url, ?error, "Error response"); - Error::RelayError(error) + error }) } @@ -332,7 +335,7 @@ impl Client { relay: &str, public_key: PublicKey, cached_packet: Option, - ) -> Result> { + ) -> Result, reqwest::Error> { let url = format!("{relay}/{public_key}"); match self.http_client.get(&url).send().await { @@ -341,6 +344,9 @@ impl Client { debug!(?url, "SignedPacket not found"); return Ok(None); } + + let response = response.error_for_status()?; + if response.content_length().unwrap_or_default() > SignedPacket::MAX_BYTES { debug!(?url, "Response too large"); @@ -354,14 +360,14 @@ impl Client { Err(error) => { debug!(?url, ?error, "Invalid signed_packet"); - Err(error) + Ok(None) } } } Err(error) => { - debug!(?url, ?error, "Error response"); + debug!(?url, ?error, "Resolve Error response"); - Err(Error::RelayError(error)) + Err(error) } } } @@ -387,6 +393,37 @@ fn choose_most_recent( } } +#[derive(thiserror::Error, Debug)] +/// Errors during publishing a [SignedPacket] to a list of relays +pub enum PublishToRelayError { + #[error("SignedPacket's timestamp is not the most recent")] + /// Failed to publish because there is a more recent packet. + NotMostRecent, + + #[error(transparent)] + /// Transparent [reqwest::Error] + /// + /// All relays responded with non-2xx status code, + /// or something wrong with the transport, transparent from [reqwest::Error]. + /// + /// This was last error response. + RelayError(#[from] reqwest::Error), +} + +#[derive(Debug)] +pub struct EmptyListOfRelays; + +impl std::error::Error for EmptyListOfRelays {} + +impl Display for EmptyListOfRelays { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!( + f, + "Can not create a Pkarr relay Client with an empty list of relays" + ) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/pkarr/src/error.rs b/pkarr/src/error.rs deleted file mode 100644 index bc9a2eb..0000000 --- a/pkarr/src/error.rs +++ /dev/null @@ -1,86 +0,0 @@ -//! Main Crate Error - -// Alias Result to be the crate Result. -pub type Result = core::result::Result; - -#[derive(thiserror::Error, Debug)] -/// Pkarr crate error enum. -pub enum Error { - #[error("{0}")] - // TODO: replace with proper errors. - Generic(String), - - #[error(transparent)] - /// Transparent [std::io::Error] - IO(#[from] std::io::Error), - - #[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] - #[error(transparent)] - PublishError(#[from] mainline::PutError), - - // === Keys errors === - #[error("Invalid PublicKey length, expected 32 bytes but got: {0}")] - InvalidPublicKeyLength(usize), - - #[error("Invalid Ed25519 publickey; Cannot decompress Edwards point")] - InvalidEd25519PublicKey, - - #[error("Invalid Ed25519 signature")] - InvalidEd25519Signature, - - #[error("Invalid PublicKey encoding")] - InvalidPublicKeyEncoding, - - // === Packets errors === - #[error(transparent)] - /// Transparent [simple_dns::SimpleDnsError] - DnsError(#[from] simple_dns::SimpleDnsError), - - #[error("Invalid SignedPacket bytes length, expected at least 104 bytes but got: {0}")] - /// Serialized signed packets are `<32 bytes publickey><64 bytes signature><8 bytes - /// timestamp>`. - InvalidSignedPacketBytesLength(usize), - - #[error("Invalid relay payload size, expected at least 72 bytes but got: {0}")] - /// Relay api http-body should be `<64 bytes signature><8 bytes timestamp> - /// `. - InvalidRelayPayloadSize(usize), - - #[error("DNS Packet is too large, expected max 1000 bytes but got: {0}")] - // DNS packet endocded and compressed is larger than 1000 bytes - PacketTooLarge(usize), - - // === Flume errors === - #[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] - #[error(transparent)] - /// Transparent [flume::RecvError] - Receive(#[from] flume::RecvError), - - #[error("Dht is shutdown")] - /// The dht was shutdown. - DhtIsShutdown, - - #[error("Publish query is already inflight for the same public_key")] - /// [crate::Client::publish] is already inflight to the same public_key - PublishInflight, - - #[error("SignedPacket's timestamp is not the most recent")] - /// Failed to publish because there is a more recent packet. - NotMostRecent, - - // === Relay errors === - #[cfg(any(feature = "relay", target_arch = "wasm32"))] - #[error(transparent)] - /// Transparent [reqwest::Error] - RelayError(#[from] reqwest::Error), - - #[cfg(any(feature = "relay", target_arch = "wasm32"))] - #[error("Empty list of relays")] - /// Empty list of relays - EmptyListOfRelays, - - // === endpoints === - #[error("Could not resolve enpdoint {0}")] - /// Could not resolve enpdoint - ResolveEndpoint(String), -} diff --git a/pkarr/src/extra/endpoints/mod.rs b/pkarr/src/extra/endpoints/mod.rs index 80c8d4c..96a80d5 100644 --- a/pkarr/src/extra/endpoints/mod.rs +++ b/pkarr/src/extra/endpoints/mod.rs @@ -3,10 +3,7 @@ // mod async_iter; mod endpoint; -use crate::{ - error::{Error, Result}, - Client, PublicKey, SignedPacket, -}; +use crate::{Client, PublicKey, SignedPacket}; pub use endpoint::Endpoint; @@ -16,9 +13,12 @@ pub trait EndpointResolver { fn resolve( &self, public_key: &PublicKey, - ) -> impl std::future::Future>>; + ) -> impl std::future::Future, ResolveError>>; - fn resolve_endpoint(&self, qname: &str) -> impl std::future::Future> + fn resolve_endpoint( + &self, + qname: &str, + ) -> impl std::future::Future> where Self: std::marker::Sync, { @@ -60,24 +60,54 @@ pub trait EndpointResolver { } } - Err(Error::Generic(format!( - "Failed to find an endopint {}", - target - ))) + Err(FailedToResolveEndpoint) } } } impl EndpointResolver for Client { - async fn resolve(&self, public_key: &PublicKey) -> Result> { - self.resolve(public_key).await + async fn resolve(&self, public_key: &PublicKey) -> Result, ResolveError> { + self.resolve(public_key).await.map_err(|error| match error { + crate::client::dht::ClientWasShutdown => ResolveError::ClientWasShutdown, + }) } } #[cfg(all(not(target_arch = "wasm32"), feature = "relay"))] impl EndpointResolver for crate::relay::Client { - async fn resolve(&self, public_key: &PublicKey) -> Result> { - self.resolve(public_key).await + async fn resolve(&self, public_key: &PublicKey) -> Result, ResolveError> { + self.resolve(public_key) + .await + .map_err(ResolveError::Reqwest) + } +} + +#[derive(Debug)] +struct FailedToResolveEndpoint; + +impl std::error::Error for FailedToResolveEndpoint {} + +impl std::fmt::Display for FailedToResolveEndpoint { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "The Pkarr Client was shutdown") + } +} + +#[derive(thiserror::Error, Debug)] +/// Resolve Error from a client +pub enum ResolveError { + ClientWasShutdown, + #[cfg(any(target_arch = "wasm32", feature = "relay"))] + Reqwest(reqwest::Error), +} + +impl std::fmt::Display for ResolveError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Resolve endpoint error from the client::resolve {:?}", + self + ) } } diff --git a/pkarr/src/extra/lmdb_cache.rs b/pkarr/src/extra/lmdb_cache.rs index 7e13293..3c88818 100644 --- a/pkarr/src/extra/lmdb_cache.rs +++ b/pkarr/src/extra/lmdb_cache.rs @@ -1,4 +1,3 @@ -use core::result::Result; use std::{borrow::Cow, fs, path::Path, time::Duration}; use crate::{Cache, CacheKey, SignedPacket, Timestamp}; diff --git a/pkarr/src/lib.rs b/pkarr/src/lib.rs index fdd3ba3..12ba7b7 100644 --- a/pkarr/src/lib.rs +++ b/pkarr/src/lib.rs @@ -6,7 +6,6 @@ // Modules mod base; mod client; -mod error; mod extra; // Exports @@ -14,7 +13,6 @@ pub use base::cache::{Cache, CacheKey, InMemoryCache}; pub use base::keys::{Keypair, PublicKey}; pub use base::signed_packet::SignedPacket; pub use base::timestamp::Timestamp; -pub use error::{Error, Result}; /// Default minimum TTL: 5 minutes pub const DEFAULT_MINIMUM_TTL: u32 = 300; @@ -30,6 +28,12 @@ pub const DEFAULT_RESOLVERS: [&str; 2] = ["resolver.pkarr.org:6881", "pkarr.pubk #[cfg(any(target_arch = "wasm32", feature = "dht"))] pub use client::{Client, ClientBuilder, Settings}; +// Errors +#[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] +pub use client::dht::{ClientWasShutdown, PublishError}; +#[cfg(any(target_arch = "wasm32", feature = "relay"))] +pub use client::relay::{EmptyListOfRelays, PublishToRelayError}; + // Rexports pub use bytes; pub use simple_dns as dns; diff --git a/server/src/handlers.rs b/server/src/handlers.rs index 69a3ba3..cc45ab6 100644 --- a/server/src/handlers.rs +++ b/server/src/handlers.rs @@ -28,15 +28,17 @@ pub async fn put( .publish(&signed_packet) .await .map_err(|error| match error { - pkarr::Error::PublishInflight => Error::new(StatusCode::TOO_MANY_REQUESTS, Some(error)), - pkarr::Error::NotMostRecent => Error::new(StatusCode::CONFLICT, Some(error)), - pkarr::Error::DhtIsShutdown => { - error!("Dht is shutdown"); - Error::with_status(StatusCode::INTERNAL_SERVER_ERROR) + pkarr::PublishError::PublishInflight => { + Error::new(StatusCode::TOO_MANY_REQUESTS, Some(error)) + } + pkarr::PublishError::NotMostRecent => Error::new(StatusCode::CONFLICT, Some(error)), + pkarr::PublishError::ClientWasShutdown => { + error!("Pkarr client was shutdown"); + Error::new(StatusCode::INTERNAL_SERVER_ERROR, Some(error)) } error => { error!(?error, "Unexpected error"); - Error::with_status(StatusCode::INTERNAL_SERVER_ERROR) + Error::new(StatusCode::INTERNAL_SERVER_ERROR, Some(error)) } })?; @@ -55,15 +57,9 @@ pub async fn get( .resolve(&public_key) .await // TODO: remove this map - .map_err(|error| match error { - pkarr::Error::DhtIsShutdown => { - error!("Dht is shutdown"); - Error::with_status(StatusCode::INTERNAL_SERVER_ERROR) - } - error => { - error!(?error, "Unexpected error"); - Error::with_status(StatusCode::INTERNAL_SERVER_ERROR) - } + .map_err(|error| { + error!("Pkarr Client was shutdown"); + Error::new(StatusCode::INTERNAL_SERVER_ERROR, Some(error)) })? { tracing::debug!(?public_key, "cache hit responding with packet!"); From fe4f2811c40e571644dc3ae00222545bb0b37ffe Mon Sep 17 00:00:00 2001 From: nazeh Date: Sun, 13 Oct 2024 14:18:30 +0300 Subject: [PATCH 40/84] feat(pkarr): use libc to get true page size --- Cargo.lock | 1 + pkarr/Cargo.toml | 26 ++++++++++++++++++-------- pkarr/src/extra/lmdb_cache.rs | 27 ++++++++++++++++++--------- 3 files changed, 37 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4c6fc1b..4a212e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1479,6 +1479,7 @@ dependencies = [ "getrandom", "heed", "js-sys", + "libc", "lru", "mainline", "mockito", diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index f7f26bc..a810ee7 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -22,20 +22,30 @@ lru = { version = "0.12.3", default-features = false } document-features = "0.2.8" rand = "0.8.5" -# optional dependencies +# feat: relay dependencies +tokio = { version = "1.40.0", optional = true } +## inherited from feat:dht as well. +sha1_smol = { version = "1.0.1", optional = true } + +# feat: serde dependencies serde = { version = "1.0.209", features = ["derive"], optional = true } + +# feat: endpoints dependencies futures = { version = "0.3.30", optional = true, default-features = false } -sha1_smol = { version = "1.0.1", optional = true } -heed = { version = "0.20.0", default-features = false, optional = true } -byteorder = { version = "1.5.0", default-features = false, optional = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] -# Dht client dependencies: +# feat: dht dependencies mainline = { git = "https://github.com/pubky/mainline", branch = "dev", optional = true } flume = { version = "0.11.0", features = ["select", "eventual-fairness", "async"], default-features = false , optional = true } -tokio = { version = "1.40.0", optional = true } + +# feat: relay dependencies reqwest = { version = "0.12.7", features = ["rustls-tls"], optional = true } +# feat: lmdb-cache defendencies +heed = { version = "0.20.0", default-features = false, optional = true } +byteorder = { version = "1.5.0", default-features = false, optional = true } +libc = { version = "0.2.159", optional = true } + [target.'cfg(target_arch = "wasm32")'.dependencies] js-sys = "0.3.69" futures = "0.3.30" @@ -58,7 +68,7 @@ serde = ["dep:serde"] # Clients ## Use [client::dht::Client] -dht = ["dep:mainline", "flume"] +dht = ["dep:mainline", "dep:flume"] ## Use [client::relay::Client] relay = ["dep:reqwest", "dep:tokio", "dep:sha1_smol"] @@ -66,7 +76,7 @@ relay = ["dep:reqwest", "dep:tokio", "dep:sha1_smol"] ## Use [extra::endpoints::EndpointResolver] trait endpoints = ["dep:futures"] ## Use [LmdbCache] -lmdb-cache = ["dep:heed", "dep:byteorder"] +lmdb-cache = ["dep:heed", "dep:byteorder", "dep:libc"] ## Use all features full = ["dht", "relay", "serde", "endpoints", "lmdb-cache"] diff --git a/pkarr/src/extra/lmdb_cache.rs b/pkarr/src/extra/lmdb_cache.rs index 3c88818..a032e65 100644 --- a/pkarr/src/extra/lmdb_cache.rs +++ b/pkarr/src/extra/lmdb_cache.rs @@ -1,13 +1,19 @@ use std::{borrow::Cow, fs, path::Path, time::Duration}; -use crate::{Cache, CacheKey, SignedPacket, Timestamp}; +use crate::{ + base::cache::{Cache, CacheKey}, + base::timestamp::Timestamp, + SignedPacket, +}; use byteorder::LittleEndian; use heed::{types::U64, BoxedError, BytesDecode, BytesEncode, Database, Env, EnvOpenOptions}; +use libc::{sysconf, _SC_PAGESIZE}; use tracing::debug; const MAX_MAP_SIZE: usize = 10995116277760; // 10 TB +const MIN_MAP_SIZE: usize = 10 * 1024 * 1024; // 10 mb const SIGNED_PACKET_TABLE: &str = "pkarrcache:signed_packet"; const KEY_TO_TIME_TABLE: &str = "pkarrcache:key_to_time"; @@ -68,13 +74,16 @@ impl LmdbCache { /// to a multiple of the `capacity` by [SignedPacket::MAX_BYTES], aligned to 4096 bytes, and /// a maximum of 10 TB. pub fn new(env_path: &Path, capacity: usize) -> Result { + let page_size = unsafe { sysconf(_SC_PAGESIZE) as usize }; + // Page aligned but more than enough bytes for `capacity` many SignedPacket let map_size = capacity .checked_mul(SignedPacket::MAX_BYTES as usize) - .and_then(|x| x.checked_add(4096)) - .and_then(|x| x.checked_div(4096)) - .and_then(|x| x.checked_mul(4096)) - .unwrap_or(MAX_MAP_SIZE); + .and_then(|x| x.checked_add(page_size)) + .and_then(|x| x.checked_div(page_size)) + .and_then(|x| x.checked_mul(page_size)) + .unwrap_or(MAX_MAP_SIZE) + .max(MIN_MAP_SIZE); fs::create_dir_all(env_path)?; @@ -158,8 +167,8 @@ impl LmdbCache { let new_time = Timestamp::now(); - time_to_key.put(&mut wtxn, &new_time.into_u64(), key)?; - key_to_time.put(&mut wtxn, key, &new_time.into_u64())?; + time_to_key.put(&mut wtxn, &new_time.as_u64(), key)?; + key_to_time.put(&mut wtxn, key, &new_time.as_u64())?; packets.put(&mut wtxn, key, signed_packet)?; @@ -182,8 +191,8 @@ impl LmdbCache { let new_time = Timestamp::now(); - time_to_key.put(&mut wtxn, &new_time.into_u64(), key)?; - key_to_time.put(&mut wtxn, key, &new_time.into_u64())?; + time_to_key.put(&mut wtxn, &new_time.as_u64(), key)?; + key_to_time.put(&mut wtxn, key, &new_time.as_u64())?; wtxn.commit()?; From 694df5b71b82eeb5e0c23cc84cbcaa6dc115c642 Mon Sep 17 00:00:00 2001 From: nazeh Date: Sun, 13 Oct 2024 14:19:33 +0300 Subject: [PATCH 41/84] chore(pkarr): update exports --- pkarr/src/base/signed_packet.rs | 6 +++--- pkarr/src/base/timestamp.rs | 6 +++--- pkarr/src/client/mod.rs | 4 ++-- pkarr/src/extra/endpoints/endpoint.rs | 4 ++-- pkarr/src/extra/endpoints/mod.rs | 2 +- pkarr/src/extra/mod.rs | 2 +- pkarr/src/lib.rs | 25 +++++++++---------------- server/src/config.rs | 8 ++++---- server/src/dht_server.rs | 5 +++-- server/src/error.rs | 6 ++++++ server/src/handlers.rs | 19 ++++++------------- server/src/main.rs | 2 +- 12 files changed, 41 insertions(+), 48 deletions(-) diff --git a/pkarr/src/base/signed_packet.rs b/pkarr/src/base/signed_packet.rs index 811b510..c60d59c 100644 --- a/pkarr/src/base/signed_packet.rs +++ b/pkarr/src/base/signed_packet.rs @@ -17,7 +17,7 @@ use std::{ #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; -use crate::Timestamp; +use super::timestamp::Timestamp; const DOT: char = '.'; @@ -351,7 +351,7 @@ impl SignedPacket { /// Time since the [Self::last_seen] in seconds fn elapsed(&self) -> u32 { - ((Timestamp::now().into_u64() - self.last_seen.into_u64()) / 1_000_000) as u32 + ((Timestamp::now().as_u64() - self.last_seen.as_u64()) / 1_000_000) as u32 } } @@ -931,7 +931,7 @@ mod tests { let mut bytes = vec![]; bytes.extend_from_slice(&[210, 1]); - bytes.extend_from_slice(&signed_packet.last_seen().into_u64().to_le_bytes()); + bytes.extend_from_slice(&signed_packet.last_seen().as_u64().to_le_bytes()); bytes.extend_from_slice(signed_packet.as_bytes()); let deserialized: SignedPacket = from_bytes(&bytes).unwrap(); diff --git a/pkarr/src/base/timestamp.rs b/pkarr/src/base/timestamp.rs index 11b3d66..52909b1 100644 --- a/pkarr/src/base/timestamp.rs +++ b/pkarr/src/base/timestamp.rs @@ -80,7 +80,7 @@ impl Timestamp { (self.0 as i64) - (rhs.0 as i64) } - pub fn into_u64(&self) -> u64 { + pub fn as_u64(&self) -> u64 { self.0 } } @@ -129,13 +129,13 @@ impl From<&Timestamp> for Timestamp { impl From for u64 { fn from(value: Timestamp) -> Self { - value.into_u64() + value.as_u64() } } impl From<&Timestamp> for u64 { fn from(value: &Timestamp) -> Self { - value.into_u64() + value.as_u64() } } diff --git a/pkarr/src/client/mod.rs b/pkarr/src/client/mod.rs index df2dd77..4dd13f5 100644 --- a/pkarr/src/client/mod.rs +++ b/pkarr/src/client/mod.rs @@ -3,12 +3,12 @@ #[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] pub mod dht; #[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] -pub use dht::{Client, ClientBuilder, Settings}; +pub use dht::Client; #[cfg(target_arch = "wasm32")] pub mod relay; #[cfg(target_arch = "wasm32")] -pub use relay::{Client, ClientBuilder, Settings}; +pub use relay::Client; #[cfg(all(not(target_arch = "wasm32"), feature = "relay"))] pub mod relay; diff --git a/pkarr/src/extra/endpoints/endpoint.rs b/pkarr/src/extra/endpoints/endpoint.rs index 8a4c836..85593ab 100644 --- a/pkarr/src/extra/endpoints/endpoint.rs +++ b/pkarr/src/extra/endpoints/endpoint.rs @@ -7,7 +7,7 @@ use crate::{ }; use std::net::{IpAddr, SocketAddr, ToSocketAddrs}; -use crate::Timestamp; +use super::super::super::base::timestamp::Timestamp; #[derive(Debug, Clone)] /// An alternative Endpoint for a `qname`, from either [RData::SVCB] or [RData::HTTPS] dns records @@ -46,7 +46,7 @@ impl Endpoint { } // Good enough random selection - let now = Timestamp::now().into_u64(); + let now = Timestamp::now().as_u64(); let slice = &records[lowest_priority_index..]; let index = if slice.is_empty() { 0 diff --git a/pkarr/src/extra/endpoints/mod.rs b/pkarr/src/extra/endpoints/mod.rs index 96a80d5..8b6660d 100644 --- a/pkarr/src/extra/endpoints/mod.rs +++ b/pkarr/src/extra/endpoints/mod.rs @@ -74,7 +74,7 @@ impl EndpointResolver for Client { } #[cfg(all(not(target_arch = "wasm32"), feature = "relay"))] -impl EndpointResolver for crate::relay::Client { +impl EndpointResolver for crate::client::relay::Client { async fn resolve(&self, public_key: &PublicKey) -> Result, ResolveError> { self.resolve(public_key) .await diff --git a/pkarr/src/extra/mod.rs b/pkarr/src/extra/mod.rs index fe20e28..585f435 100644 --- a/pkarr/src/extra/mod.rs +++ b/pkarr/src/extra/mod.rs @@ -1,5 +1,5 @@ #[cfg(feature = "endpoints")] pub mod endpoints; -#[cfg(feature = "lmdb-cache")] +#[cfg(all(not(target_arch = "wasm32"), feature = "lmdb-cache"))] pub mod lmdb_cache; diff --git a/pkarr/src/lib.rs b/pkarr/src/lib.rs index 12ba7b7..d71b362 100644 --- a/pkarr/src/lib.rs +++ b/pkarr/src/lib.rs @@ -5,14 +5,13 @@ // Modules mod base; -mod client; -mod extra; +pub mod client; +pub mod extra; // Exports pub use base::cache::{Cache, CacheKey, InMemoryCache}; pub use base::keys::{Keypair, PublicKey}; pub use base::signed_packet::SignedPacket; -pub use base::timestamp::Timestamp; /// Default minimum TTL: 5 minutes pub const DEFAULT_MINIMUM_TTL: u32 = 300; @@ -26,25 +25,19 @@ pub const DEFAULT_RELAYS: [&str; 2] = ["https://relay.pkarr.org", "https://pkarr pub const DEFAULT_RESOLVERS: [&str; 2] = ["resolver.pkarr.org:6881", "pkarr.pubky.app:6881"]; #[cfg(any(target_arch = "wasm32", feature = "dht"))] -pub use client::{Client, ClientBuilder, Settings}; - -// Errors -#[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] -pub use client::dht::{ClientWasShutdown, PublishError}; -#[cfg(any(target_arch = "wasm32", feature = "relay"))] -pub use client::relay::{EmptyListOfRelays, PublishToRelayError}; +pub use client::Client; // Rexports pub use bytes; pub use simple_dns as dns; -#[cfg(all(not(target_arch = "wasm32"), feature = "relay"))] -pub use client::relay; #[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] pub use mainline; -#[cfg(feature = "endpoints")] -pub use extra::endpoints::{Endpoint, EndpointResolver}; +pub mod errors { + #[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] + pub use super::client::dht::{ClientWasShutdown, PublishError}; -#[cfg(feature = "lmdb-cache")] -pub use extra::lmdb_cache::LmdbCache; + #[cfg(any(target_arch = "wasm32", feature = "relay"))] + pub use super::client::relay::{EmptyListOfRelays, PublishToRelayError}; +} diff --git a/server/src/config.rs b/server/src/config.rs index c151368..8b8a5cf 100644 --- a/server/src/config.rs +++ b/server/src/config.rs @@ -27,17 +27,17 @@ pub struct Config { /// /// Defaults to a directory in the OS data directory cache_path: Option, - /// See [pkarr::Settings::cache_size] + /// See [pkarr::client::dht::Settings::cache_size] cache_size: Option, /// Resolvers /// /// Other servers to query in parallel with the Dht queries /// - /// See [pkarr::Settings::resolvers] + /// See [pkarr::client::dht::Settings::resolvers] resolvers: Option>, - /// See [pkarr::Settings::minimum_ttl] + /// See [pkarr::client::dht::Settings::minimum_ttl] minimum_ttl: Option, - /// See [pkarr::Settings::maximum_ttl] + /// See [pkarr::client::dht::Settings::maximum_ttl] maximum_ttl: Option, rate_limiter: RateLimiterConfig, } diff --git a/server/src/dht_server.rs b/server/src/dht_server.rs index a8a4b07..53c81dd 100644 --- a/server/src/dht_server.rs +++ b/server/src/dht_server.rs @@ -4,6 +4,7 @@ use std::{ }; use pkarr::{ + extra::lmdb_cache::LmdbCache, mainline::{ self, rpc::{ @@ -27,7 +28,7 @@ use crate::rate_limiting::IpRateLimiter; pub struct DhtServer { inner: mainline::server::DhtServer, resolvers: Option>, - cache: Box, + cache: Box, minimum_ttl: u32, maximum_ttl: u32, rate_limiter: IpRateLimiter, @@ -41,7 +42,7 @@ impl Debug for DhtServer { impl DhtServer { pub fn new( - cache: Box, + cache: Box, resolvers: Option>, minimum_ttl: u32, maximum_ttl: u32, diff --git a/server/src/error.rs b/server/src/error.rs index cbe837d..92a36d2 100644 --- a/server/src/error.rs +++ b/server/src/error.rs @@ -68,3 +68,9 @@ impl From for Error { Self::new(StatusCode::INTERNAL_SERVER_ERROR, Some(value)) } } + +impl From for Error { + fn from(value: pkarr::errors::ClientWasShutdown) -> Self { + Self::new(StatusCode::INTERNAL_SERVER_ERROR, Some(value)) + } +} diff --git a/server/src/handlers.rs b/server/src/handlers.rs index cc45ab6..6f506e5 100644 --- a/server/src/handlers.rs +++ b/server/src/handlers.rs @@ -28,11 +28,13 @@ pub async fn put( .publish(&signed_packet) .await .map_err(|error| match error { - pkarr::PublishError::PublishInflight => { + pkarr::errors::PublishError::PublishInflight => { Error::new(StatusCode::TOO_MANY_REQUESTS, Some(error)) } - pkarr::PublishError::NotMostRecent => Error::new(StatusCode::CONFLICT, Some(error)), - pkarr::PublishError::ClientWasShutdown => { + pkarr::errors::PublishError::NotMostRecent => { + Error::new(StatusCode::CONFLICT, Some(error)) + } + pkarr::errors::PublishError::ClientWasShutdown => { error!("Pkarr client was shutdown"); Error::new(StatusCode::INTERNAL_SERVER_ERROR, Some(error)) } @@ -52,16 +54,7 @@ pub async fn get( let public_key = PublicKey::try_from(public_key.as_str()) .map_err(|error| Error::new(StatusCode::BAD_REQUEST, Some(error)))?; - if let Some(signed_packet) = state - .client - .resolve(&public_key) - .await - // TODO: remove this map - .map_err(|error| { - error!("Pkarr Client was shutdown"); - Error::new(StatusCode::INTERNAL_SERVER_ERROR, Some(error)) - })? - { + if let Some(signed_packet) = state.client.resolve(&public_key).await? { tracing::debug!(?public_key, "cache hit responding with packet!"); let body = signed_packet.to_relay_payload(); diff --git a/server/src/main.rs b/server/src/main.rs index f164e77..340ed1b 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -12,7 +12,7 @@ use std::path::PathBuf; use tracing::{debug, info}; use http_server::HttpServer; -use pkarr::{mainline::dht::DhtSettings, Client, LmdbCache}; +use pkarr::{extra::lmdb_cache::LmdbCache, mainline::dht::DhtSettings, Client}; #[derive(Parser, Debug)] struct Cli { From c93133629da627dd269028bba42e477de82b8314 Mon Sep 17 00:00:00 2001 From: nazeh Date: Sun, 13 Oct 2024 18:14:05 +0300 Subject: [PATCH 42/84] feat(pkarr): update mainline --- Cargo.lock | 5 +++-- pkarr/Cargo.toml | 8 ++++---- pkarr/src/client/dht.rs | 16 ++++++++-------- pkarr/src/extra/lmdb_cache.rs | 7 +++++-- pkarr/src/lib.rs | 2 ++ server/src/dht_server.rs | 4 ++-- server/src/main.rs | 6 +++--- 7 files changed, 27 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4a212e5..070369d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1142,11 +1142,12 @@ checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" [[package]] name = "mainline" -version = "3.0.0" -source = "git+https://github.com/pubky/mainline?branch=dev#be9c071426fe89a37ab30b764f0e2b24d2f83993" +version = "3.0.1" +source = "git+https://github.com/pubky/mainline?branch=dev#f80d3b9f2d9a293e7631b9468952fdff9b72fe44" dependencies = [ "bytes", "crc", + "document-features", "ed25519-dalek", "flume", "lru", diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index a810ee7..313e1e5 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -24,7 +24,7 @@ rand = "0.8.5" # feat: relay dependencies tokio = { version = "1.40.0", optional = true } -## inherited from feat:dht as well. +# inherited from feat:dht as well. sha1_smol = { version = "1.0.1", optional = true } # feat: serde dependencies @@ -63,15 +63,15 @@ mockito = "1.4.0" tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } [features] -## Derive serde Serialize/Deserialize for PublicKey -serde = ["dep:serde"] - # Clients ## Use [client::dht::Client] dht = ["dep:mainline", "dep:flume"] ## Use [client::relay::Client] relay = ["dep:reqwest", "dep:tokio", "dep:sha1_smol"] +## Derive serde Serialize/Deserialize for PublicKey +serde = ["dep:serde"] + # Extra ## Use [extra::endpoints::EndpointResolver] trait endpoints = ["dep:futures"] diff --git a/pkarr/src/client/dht.rs b/pkarr/src/client/dht.rs index 7becc3e..c95f856 100644 --- a/pkarr/src/client/dht.rs +++ b/pkarr/src/client/dht.rs @@ -2,12 +2,12 @@ use flume::{Receiver, Sender}; use mainline::{ - dht::DhtSettings, + errors::PutError, rpc::{ messages, QueryResponse, QueryResponseSpecific, ReceivedFrom, ReceivedMessage, Response, Rpc, }, - Id, MutableItem, PutError, Testnet, + Id, MutableItem, Testnet, }; use std::{ collections::HashMap, @@ -26,7 +26,7 @@ use crate::{PublicKey, SignedPacket}; #[derive(Debug)] /// [Client]'s settings pub struct Settings { - pub dht: DhtSettings, + pub dht: mainline::Settings, /// A set of [resolver](https://pkarr.org/resolvers)s /// to be queried alongside the Dht routing table, to /// lower the latency on cold starts, and help if the @@ -51,7 +51,7 @@ pub struct Settings { impl Default for Settings { fn default() -> Self { Self { - dht: DhtSettings::default(), + dht: mainline::Settings::default(), cache_size: NonZeroUsize::new(DEFAULT_CACHE_SIZE).unwrap(), resolvers: Some( DEFAULT_RESOLVERS @@ -118,13 +118,13 @@ impl ClientBuilder { self } - /// Set [DhtSettings] - pub fn dht_settings(mut self, settings: DhtSettings) -> Self { + /// Set [Settings::dht] + pub fn dht_settings(mut self, settings: mainline::Settings) -> Self { self.settings.dht = settings; self } - /// Convienent methot to set the [DhtSettings::bootstrap] from [mainline::Testnet::bootstrap] + /// Convienent methot to set the [mainline::Settings::bootstrap] from [mainline::Testnet::bootstrap] pub fn testnet(mut self, testnet: &Testnet) -> Self { self.settings.dht.bootstrap = testnet.bootstrap.clone().into(); self @@ -698,7 +698,7 @@ mod tests { let client = Client::builder() .testnet(&testnet) - .dht_settings(DhtSettings { + .dht_settings(mainline::Settings { request_timeout: Duration::from_millis(10).into(), ..Default::default() }) diff --git a/pkarr/src/extra/lmdb_cache.rs b/pkarr/src/extra/lmdb_cache.rs index a032e65..7938377 100644 --- a/pkarr/src/extra/lmdb_cache.rs +++ b/pkarr/src/extra/lmdb_cache.rs @@ -1,3 +1,5 @@ +//! Persistent [crate::base::cache::Cache] implementation using LMDB's bindings [heed] + use std::{borrow::Cow, fs, path::Path, time::Duration}; use crate::{ @@ -61,6 +63,7 @@ impl<'a> BytesDecode<'a> for SignedPacketCodec { } #[derive(Debug, Clone)] +/// Persistent [crate::base::cache::Cache] implementation using LMDB's bindings [heed] pub struct LmdbCache { capacity: usize, env: Env, @@ -71,8 +74,8 @@ pub struct LmdbCache { impl LmdbCache { /// Creates a new [LmdbCache] at the `env_path` and set the [heed::EnvOpenOptions::map_size] - /// to a multiple of the `capacity` by [SignedPacket::MAX_BYTES], aligned to 4096 bytes, and - /// a maximum of 10 TB. + /// to a multiple of the `capacity` by [SignedPacket::MAX_BYTES], aligned to system's page size, + /// a maximum of 10 TB, and a minimum of 10 MB. pub fn new(env_path: &Path, capacity: usize) -> Result { let page_size = unsafe { sysconf(_SC_PAGESIZE) as usize }; diff --git a/pkarr/src/lib.rs b/pkarr/src/lib.rs index d71b362..14917ff 100644 --- a/pkarr/src/lib.rs +++ b/pkarr/src/lib.rs @@ -35,6 +35,8 @@ pub use simple_dns as dns; pub use mainline; pub mod errors { + //! Exported errors + #[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] pub use super::client::dht::{ClientWasShutdown, PublishError}; diff --git a/server/src/dht_server.rs b/server/src/dht_server.rs index 53c81dd..a4bb6f4 100644 --- a/server/src/dht_server.rs +++ b/server/src/dht_server.rs @@ -26,7 +26,7 @@ use crate::rate_limiting::IpRateLimiter; /// DhtServer with Rate limiting pub struct DhtServer { - inner: mainline::server::DhtServer, + inner: mainline::server::DefaultServer, resolvers: Option>, cache: Box, minimum_ttl: u32, @@ -50,7 +50,7 @@ impl DhtServer { ) -> Self { Self { // Default DhtServer used to stay a good citizen servicing the Dht. - inner: mainline::server::DhtServer::default(), + inner: mainline::server::DefaultServer::default(), cache, resolvers: resolvers.map(|resolvers| { resolvers diff --git a/server/src/main.rs b/server/src/main.rs index 340ed1b..fdf36dd 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -12,7 +12,7 @@ use std::path::PathBuf; use tracing::{debug, info}; use http_server::HttpServer; -use pkarr::{extra::lmdb_cache::LmdbCache, mainline::dht::DhtSettings, Client}; +use pkarr::{extra::lmdb_cache::LmdbCache, mainline, Client}; #[derive(Parser, Debug)] struct Cli { @@ -48,7 +48,7 @@ async fn main() -> Result<()> { let rate_limiter = rate_limiting::IpRateLimiter::new(config.rate_limiter()); let client = Client::builder() - .dht_settings(DhtSettings { + .dht_settings(mainline::Settings { port: Some(config.dht_port()), server: Some(Box::new(dht_server::DhtServer::new( cache.clone(), @@ -57,7 +57,7 @@ async fn main() -> Result<()> { config.maximum_ttl(), rate_limiter.clone(), ))), - ..DhtSettings::default() + ..mainline::Settings::default() }) .resolvers(config.resolvers()) .minimum_ttl(config.minimum_ttl()) From 63123bd958c8f03295b40b024d6d207caa70186f Mon Sep 17 00:00:00 2001 From: nazeh Date: Fri, 18 Oct 2024 13:00:52 +0300 Subject: [PATCH 43/84] feat(pkarr): closes #89 return cached packets _before_ querying remote nodes in background --- pkarr/Cargo.toml | 2 +- pkarr/src/base/signed_packet.rs | 6 +++ pkarr/src/client/dht.rs | 61 ++++++++++++++---------- pkarr/src/client/relay.rs | 50 ++++++++++++++------ server/src/dht_server.rs | 84 ++++++++++++++------------------- 5 files changed, 114 insertions(+), 89 deletions(-) diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index 313e1e5..607c88b 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -23,7 +23,7 @@ document-features = "0.2.8" rand = "0.8.5" # feat: relay dependencies -tokio = { version = "1.40.0", optional = true } +tokio = { version = "1.40.0", optional = true, default-features = false } # inherited from feat:dht as well. sha1_smol = { version = "1.0.1", optional = true } diff --git a/pkarr/src/base/signed_packet.rs b/pkarr/src/base/signed_packet.rs index c60d59c..d7830c8 100644 --- a/pkarr/src/base/signed_packet.rs +++ b/pkarr/src/base/signed_packet.rs @@ -347,6 +347,12 @@ impl SignedPacket { .map_or(min, |v| v.clamp(min, max)) } + /// Returns whether or not this packet is considered expired according to + /// a given `min` and `max` TTLs + pub fn is_expired(&self, min: u32, max: u32) -> bool { + self.expires_in(min, max) == 0 + } + // === Private Methods === /// Time since the [Self::last_seen] in seconds diff --git a/pkarr/src/client/dht.rs b/pkarr/src/client/dht.rs index c95f856..3e4e0d0 100644 --- a/pkarr/src/client/dht.rs +++ b/pkarr/src/client/dht.rs @@ -310,38 +310,47 @@ impl Client { ) -> Result, ClientWasShutdown> { let target = MutableItem::target_from_key(public_key.as_bytes(), &None); - let (sender, receiver) = flume::bounded::(1); - let cached_packet = self.cache.get(target.as_bytes()); - if let Some(ref cached) = cached_packet { - let expires_in = cached.expires_in(self.minimum_ttl, self.maximum_ttl); - - if expires_in > 0 { - debug!(expires_in, "Have fresh signed_packet in cache."); - - sender.send(cached.clone()).map_err(|_| ClientWasShutdown)?; + let (tx, rx) = flume::bounded::(1); + + let as_ref = cached_packet.as_ref(); + + // Should query? + if as_ref + .as_ref() + .map(|c| c.is_expired(self.minimum_ttl, self.maximum_ttl)) + .unwrap_or(true) + { + debug!( + ?public_key, + "querying the DHT to hydrate our cache for later." + ); + + self.sender + .send(ActorMessage::Resolve( + target, + tx.clone(), + // Sending the `timestamp` of the known cache, help save some bandwith, + // since remote nodes will not send the encoded packet if they don't know + // any more recent versions. + // most_recent_known_timestamp, + as_ref.map(|cached| cached.timestamp()), + )) + .map_err(|_| ClientWasShutdown)?; + } - return Ok(receiver); - } + if let Some(cached_packet) = cached_packet { + debug!( + public_key = ?cached_packet.public_key(), + "responding with cached packet even if expired" + ); - debug!(expires_in, "Have expired signed_packet in cache."); - } else { - debug!("Cache miss"); + // If the receiver was dropped.. no harm. + let _ = tx.send(cached_packet); } - self.sender - .send(ActorMessage::Resolve( - target, - sender, - // Sending the `timestamp` of the known cache, help save some bandwith, - // since remote nodes will not send the encoded packet if they don't know - // any more recent versions. - cached_packet.as_ref().map(|cached| cached.timestamp()), - )) - .map_err(|_| ClientWasShutdown)?; - - Ok(receiver) + Ok(rx) } } diff --git a/pkarr/src/client/relay.rs b/pkarr/src/client/relay.rs index a848cdd..ec65c71 100644 --- a/pkarr/src/client/relay.rs +++ b/pkarr/src/client/relay.rs @@ -177,24 +177,46 @@ impl Client { &self, public_key: &PublicKey, ) -> Result, reqwest::Error> { - if let Some(cached_packet) = self.cache.get(&public_key.as_ref().into()) { - let expires_in = cached_packet.expires_in(self.minimum_ttl, self.maximum_ttl); + let cached_packet = self.cache.get(&(public_key.into())); - if expires_in > 0 { - debug!(expires_in, "Have fresh signed_packet in cache."); - return Ok(Some(cached_packet)); - } + let (tx, rx) = flume::bounded::, reqwest::Error>>(1); - debug!(expires_in, "Have expired signed_packet in cache."); + let as_ref = cached_packet.as_ref(); - return Ok(self - .race_resolve(public_key, Some(cached_packet.clone())) - .await? - .or(Some(cached_packet))); - }; - debug!("Cache miss"); + // Should query? + if as_ref + .as_ref() + .map(|c| c.is_expired(self.minimum_ttl, self.maximum_ttl)) + .unwrap_or(true) + { + debug!( + ?public_key, + "querying relays to hydrate our cache for later." + ); + + let pubky = public_key.clone(); + let tx = tx.clone(); + let this = self.clone(); + + tokio::task::spawn(async move { + // If the receiver was dropped.. no harm. + let _ = tx.send(this.race_resolve(&pubky, None).await); + }); + } + + if let Some(cached_packet) = cached_packet { + debug!( + public_key = ?cached_packet.public_key(), + "responding with cached packet even if expired" + ); - self.race_resolve(public_key, None).await + // If the receiver was dropped.. no harm. + let _ = tx.send(Ok(Some(cached_packet))); + } + + rx.recv_async() + .await + .expect("at least one sender should send before being dropped") } // === Native Race implementation === diff --git a/server/src/dht_server.rs b/server/src/dht_server.rs index a4bb6f4..6547eb5 100644 --- a/server/src/dht_server.rs +++ b/server/src/dht_server.rs @@ -79,57 +79,18 @@ impl Server for DhtServer { .. } = request { - let should_query = if let Some(cached) = self.cache.get(target.as_bytes()) { - debug!( - public_key = ?cached.public_key(), - ?target, - "cache hit responding with packet!" - ); + let cached_packet = self.cache.get(target.as_bytes()); - // Respond with what we have, even if expired. - let mutable_item = MutableItem::from(&cached); + let as_ref = cached_packet.as_ref(); - rpc.response( - from, - transaction_id, - ResponseSpecific::GetMutable(GetMutableResponseArguments { - responder_id: *rpc.id(), - // Token doesn't matter much, as we are most likely _not_ the - // closest nodes, so we shouldn't expect an PUT requests based on - // this response. - token: vec![0, 0, 0, 0], - nodes: None, - v: mutable_item.value().to_vec(), - k: mutable_item.key().to_vec(), - seq: *mutable_item.seq(), - sig: mutable_item.signature().to_vec(), - }), - ); - - // If expired, we try to hydrate the packet from the DHT. - let expires_in = cached.expires_in(self.minimum_ttl, self.maximum_ttl); - let expired = expires_in == 0; + // Should query? + if as_ref + .as_ref() + .map(|c| c.is_expired(self.minimum_ttl, self.maximum_ttl)) + .unwrap_or(true) + { + debug!(?target, "querying the DHT to hydrate our cache for later."); - if expired { - debug!( - public_key = ?cached.public_key(), - ?target, - ?expires_in, - "cache expired, querying the DHT to hydrate our cache for later." - ); - }; - - expired - } else { - debug!( - ?target, - "cache miss, querying the DHT to hydrate our cache for later." - ); - true - }; - - // Either cache miss or expired cached packet - if should_query { // Rate limit nodes that are making too many request forcing us to making too // many queries, either by querying the same non-existent key, or many unique keys. if self.rate_limiter.is_limited(&from.ip()) { @@ -147,6 +108,33 @@ impl Server for DhtServer { ); }; } + + // Respond with what we have, even if expired. + if let Some(cached_packet) = cached_packet { + debug!( + public_key = ?cached_packet.public_key(), + "responding with cached packet even if expired" + ); + + let mutable_item = MutableItem::from(&cached_packet); + + rpc.response( + from, + transaction_id, + ResponseSpecific::GetMutable(GetMutableResponseArguments { + responder_id: *rpc.id(), + // Token doesn't matter much, as we are most likely _not_ the + // closest nodes, so we shouldn't expect a PUT requests based on + // this response. + token: vec![0, 0, 0, 0], + nodes: None, + v: mutable_item.value().to_vec(), + k: mutable_item.key().to_vec(), + seq: *mutable_item.seq(), + sig: mutable_item.signature().to_vec(), + }), + ); + } }; // Do normal Dht request handling (peers, mutable, immutable, and routing). From f7124cc1f9d9f7798a2d927f9d733b0333206388 Mon Sep 17 00:00:00 2001 From: nazeh Date: Fri, 18 Oct 2024 14:31:19 +0300 Subject: [PATCH 44/84] feat(pkarr): use pubky_timestamp --- CHANGELOG.md | 4 +- Cargo.lock | 14 ++ pkarr/Cargo.toml | 7 +- pkarr/src/base/mod.rs | 1 - pkarr/src/base/signed_packet.rs | 4 +- pkarr/src/base/timestamp.rs | 319 -------------------------- pkarr/src/extra/endpoints/endpoint.rs | 2 +- pkarr/src/extra/lmdb_cache.rs | 13 +- 8 files changed, 30 insertions(+), 334 deletions(-) delete mode 100644 pkarr/src/base/timestamp.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 1bcda55..a891c12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ All notable changes to pkarr client and server will be documented in this file. ### Added -- Add strict monotonic unix `Timestamp`. +- Use `pubky_timestamp::Timestamp` - Impl `PartialEq, Eq` for `SignedPacket`. - Impl `From` for `CacheKey`. - Add `SignedPacket::serialize` and `SignedPacket::deserialize`. @@ -21,7 +21,7 @@ All notable changes to pkarr client and server will be documented in this file. - replace `ureq` with `reqwest` to work with HTTP/2 relays, and Wasm. - update `mainline` to v3.0.0 - `Client::shutdown` and `Client::shutdown_sync` are now idempotent and return `()`. -- `Client::resolve`, `Client::resolve_sync` and `relay::Client::resolve` return expired cached `SignedPacket` if nothing else was found from the network. +- `Client::resolve`, `Client::resolve_sync` and `relay::Client::resolve` return expired cached `SignedPacket` _before_ making query to the network (Relays/Resolvers/Dht). ### Removed diff --git a/Cargo.lock b/Cargo.lock index 070369d..171db2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1486,6 +1486,7 @@ dependencies = [ "mockito", "once_cell", "postcard", + "pubky-timestamp", "rand", "reqwest", "self_cell", @@ -1575,6 +1576,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "pubky-timestamp" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "084b6e5bfcc186781b71257d636b660f20e94bb588c3ba52393fd9faf7a7bfda" +dependencies = [ + "document-features", + "getrandom", + "js-sys", + "once_cell", + "serde", +] + [[package]] name = "quanta" version = "0.12.3" diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index 607c88b..d199f4a 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -17,10 +17,11 @@ simple-dns = "0.6.1" thiserror = "1.0.49" tracing = "0.1.40" dyn-clone = "1.0.17" -once_cell = {version = "1.19.0", default-features = false } -lru = { version = "0.12.3", default-features = false } document-features = "0.2.8" rand = "0.8.5" +once_cell = {version = "1.19.0", default-features = false } +lru = { version = "0.12.3", default-features = false } +pubky-timestamp = { version = "0.2.0", default-features = false } # feat: relay dependencies tokio = { version = "1.40.0", optional = true, default-features = false } @@ -70,7 +71,7 @@ dht = ["dep:mainline", "dep:flume"] relay = ["dep:reqwest", "dep:tokio", "dep:sha1_smol"] ## Derive serde Serialize/Deserialize for PublicKey -serde = ["dep:serde"] +serde = ["dep:serde", "pubky-timestamp/serde"] # Extra ## Use [extra::endpoints::EndpointResolver] trait diff --git a/pkarr/src/base/mod.rs b/pkarr/src/base/mod.rs index ceb8198..ad1894c 100644 --- a/pkarr/src/base/mod.rs +++ b/pkarr/src/base/mod.rs @@ -3,4 +3,3 @@ pub mod cache; pub mod keys; pub mod signed_packet; -pub mod timestamp; diff --git a/pkarr/src/base/signed_packet.rs b/pkarr/src/base/signed_packet.rs index d7830c8..b7dee58 100644 --- a/pkarr/src/base/signed_packet.rs +++ b/pkarr/src/base/signed_packet.rs @@ -17,7 +17,7 @@ use std::{ #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; -use super::timestamp::Timestamp; +use pubky_timestamp::Timestamp; const DOT: char = '.'; @@ -207,7 +207,7 @@ impl SignedPacket { pub fn deserialize(bytes: &[u8]) -> Result { let mut last_seen = Timestamp::try_from(&bytes[0..8]).unwrap_or_default(); - if last_seen > (&Timestamp::now() + 60_000_000) { + if last_seen > (Timestamp::now() + 60_000_000) { last_seen = Timestamp::from(0) } diff --git a/pkarr/src/base/timestamp.rs b/pkarr/src/base/timestamp.rs deleted file mode 100644 index 52909b1..0000000 --- a/pkarr/src/base/timestamp.rs +++ /dev/null @@ -1,319 +0,0 @@ -//! Strictly monotonic unix timestamp in microseconds - -#[cfg(feature = "serde")] -use serde::{Deserialize, Serialize}; -use std::fmt::Display; -use std::{ - ops::{Add, AddAssign, Sub, SubAssign}, - sync::Mutex, -}; - -use once_cell::sync::Lazy; -use rand::Rng; - -#[cfg(not(target_arch = "wasm32"))] -use std::time::SystemTime; - -/// ~0.4% chance of none of 10 clocks have matching id. -const CLOCK_MASK: u64 = (1 << 8) - 1; -const TIME_MASK: u64 = !0 >> 8; - -pub struct TimestampFactory { - clock_id: u64, - last_time: u64, -} - -impl TimestampFactory { - pub fn new() -> Self { - Self { - clock_id: rand::thread_rng().gen::() & CLOCK_MASK, - last_time: system_time() & TIME_MASK, - } - } - - pub fn now(&mut self) -> Timestamp { - // Ensure strict monotonicity. - self.last_time = (system_time() & TIME_MASK).max(self.last_time + CLOCK_MASK + 1); - - // Add clock_id to the end of the timestamp - Timestamp(self.last_time | self.clock_id) - } -} - -impl Default for TimestampFactory { - fn default() -> Self { - Self::new() - } -} - -static DEFAULT_FACTORY: Lazy> = - Lazy::new(|| Mutex::new(TimestampFactory::default())); - -/// STrictly monotonic timestamp since [SystemTime::UNIX_EPOCH] in microseconds as u64. -/// -/// The purpose of this timestamp is to unique per "user", not globally, -/// it achieves this by: -/// 1. Override the last byte with a random `clock_id`, reducing the probability -/// of two matching timestamps across multiple machines/threads. -/// 2. Gurantee that the remaining 3 bytes are ever increasing (strictly monotonic) within -/// the same thread regardless of the wall clock value -/// -/// This timestamp is also serialized as BE bytes to remain sortable. -/// If a `utf-8` encoding is necessary, it is encoded as [base32::Alphabet::Crockford] -/// to act as a sortable Id. -/// -/// U64 of microseconds is valid for the next 500 thousand years! -#[derive(Debug, Clone, PartialEq, PartialOrd, Hash, Eq, Ord)] -pub struct Timestamp(u64); - -impl Timestamp { - pub fn now() -> Self { - DEFAULT_FACTORY.lock().unwrap().now() - } - - /// Return big endian bytes - pub fn to_bytes(&self) -> [u8; 8] { - self.0.to_be_bytes() - } - - pub fn difference(&self, rhs: &Timestamp) -> i64 { - (self.0 as i64) - (rhs.0 as i64) - } - - pub fn as_u64(&self) -> u64 { - self.0 - } -} - -impl Default for Timestamp { - fn default() -> Self { - Timestamp::now() - } -} - -impl TryFrom<&[u8]> for Timestamp { - type Error = TimestampError; - - fn try_from(bytes: &[u8]) -> Result { - let bytes: [u8; 8] = bytes - .try_into() - .map_err(|_| TimestampError::InvalidBytesLength(bytes.len()))?; - - Ok(bytes.into()) - } -} - -impl From<&Timestamp> for [u8; 8] { - fn from(timestamp: &Timestamp) -> Self { - timestamp.0.to_be_bytes() - } -} - -impl From<[u8; 8]> for Timestamp { - fn from(bytes: [u8; 8]) -> Self { - Self(u64::from_be_bytes(bytes)) - } -} - -impl From for Timestamp { - fn from(inner: u64) -> Self { - Self(inner) - } -} - -impl From<&Timestamp> for Timestamp { - fn from(timestamp: &Timestamp) -> Self { - timestamp.clone() - } -} - -impl From for u64 { - fn from(value: Timestamp) -> Self { - value.as_u64() - } -} - -impl From<&Timestamp> for u64 { - fn from(value: &Timestamp) -> Self { - value.as_u64() - } -} - -// === U64 conversion === - -impl Add for &Timestamp { - type Output = Timestamp; - - fn add(self, rhs: u64) -> Self::Output { - Timestamp(self.0 + rhs) - } -} - -impl Sub for &Timestamp { - type Output = Timestamp; - - fn sub(self, rhs: u64) -> Self::Output { - Timestamp(self.0 - rhs) - } -} - -impl AddAssign for Timestamp { - fn add_assign(&mut self, other: u64) { - self.0 += other; - } -} - -impl SubAssign for Timestamp { - fn sub_assign(&mut self, other: u64) { - self.0 -= other; - } -} - -// === Serialization === - -#[cfg(feature = "serde")] -impl Serialize for Timestamp { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - let bytes = self.to_bytes(); - bytes.serialize(serializer) - } -} - -#[cfg(feature = "serde")] -impl<'de> Deserialize<'de> for Timestamp { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let bytes: [u8; 8] = Deserialize::deserialize(deserializer)?; - Ok(Timestamp(u64::from_be_bytes(bytes))) - } -} - -// === String representation (sortable base32 encoding) === - -impl Display for Timestamp { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let bytes: [u8; 8] = self.into(); - f.write_str(&base32::encode(base32::Alphabet::Crockford, &bytes)) - } -} - -impl TryFrom for Timestamp { - type Error = TimestampError; - - fn try_from(value: String) -> Result { - match base32::decode(base32::Alphabet::Crockford, &value) { - Some(vec) => { - let bytes: [u8; 8] = vec - .try_into() - .map_err(|_| TimestampError::InvalidEncoding)?; - - Ok(bytes.into()) - } - None => Err(TimestampError::InvalidEncoding), - } - } -} - -#[cfg(not(target_arch = "wasm32"))] -/// Return the number of microseconds since [SystemTime::UNIX_EPOCH] -fn system_time() -> u64 { - SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .expect("time drift") - .as_micros() as u64 -} - -#[cfg(target_arch = "wasm32")] -/// Return the number of microseconds since [SystemTime::UNIX_EPOCH] -pub fn system_time() -> u64 { - // Won't be an issue for more than 5000 years! - (js_sys::Date::now() as u64 ) - // Turn miliseconds to microseconds - * 1000 -} - -#[derive(thiserror::Error, Debug)] -/// Errors trying to decode a [Timestamp] -pub enum TimestampError { - #[error("Invalid bytes length, Timestamp should be encoded as 8 bytes, got {0}")] - InvalidBytesLength(usize), - #[error("Invalid timestamp encoding")] - InvalidEncoding, -} - -#[cfg(test)] -mod tests { - use std::collections::HashSet; - - use super::*; - - #[test] - fn strictly_monotonic() { - const COUNT: usize = 100; - - let mut set = HashSet::with_capacity(COUNT); - let mut vec = Vec::with_capacity(COUNT); - - for _ in 0..COUNT { - let timestamp = Timestamp::now(); - - set.insert(timestamp.clone()); - vec.push(timestamp); - } - - let mut ordered = vec.clone(); - ordered.sort(); - - assert_eq!(set.len(), COUNT, "unique"); - assert_eq!(ordered, vec, "ordered"); - } - - #[test] - fn strings() { - const COUNT: usize = 100; - - let mut set = HashSet::with_capacity(COUNT); - let mut vec = Vec::with_capacity(COUNT); - - for _ in 0..COUNT { - let string = Timestamp::now().to_string(); - - set.insert(string.clone()); - vec.push(string) - } - - let mut ordered = vec.clone(); - ordered.sort(); - - assert_eq!(set.len(), COUNT, "unique"); - assert_eq!(ordered, vec, "ordered"); - } - - #[test] - fn to_from_string() { - let timestamp = Timestamp::now(); - let string = timestamp.to_string(); - let decoded: Timestamp = string.try_into().unwrap(); - - assert_eq!(decoded, timestamp) - } - - #[cfg(feature = "serde")] - #[test] - fn serde() { - let timestamp = Timestamp::now(); - - let serialized = postcard::to_allocvec(×tamp).unwrap(); - - assert_eq!(serialized, timestamp.to_bytes()); - - let deserialized: Timestamp = postcard::from_bytes(&serialized).unwrap(); - - assert_eq!(deserialized, timestamp); - } -} diff --git a/pkarr/src/extra/endpoints/endpoint.rs b/pkarr/src/extra/endpoints/endpoint.rs index 85593ab..a37dfb0 100644 --- a/pkarr/src/extra/endpoints/endpoint.rs +++ b/pkarr/src/extra/endpoints/endpoint.rs @@ -7,7 +7,7 @@ use crate::{ }; use std::net::{IpAddr, SocketAddr, ToSocketAddrs}; -use super::super::super::base::timestamp::Timestamp; +use pubky_timestamp::Timestamp; #[derive(Debug, Clone)] /// An alternative Endpoint for a `qname`, from either [RData::SVCB] or [RData::HTTPS] dns records diff --git a/pkarr/src/extra/lmdb_cache.rs b/pkarr/src/extra/lmdb_cache.rs index 7938377..6939514 100644 --- a/pkarr/src/extra/lmdb_cache.rs +++ b/pkarr/src/extra/lmdb_cache.rs @@ -2,18 +2,19 @@ use std::{borrow::Cow, fs, path::Path, time::Duration}; -use crate::{ - base::cache::{Cache, CacheKey}, - base::timestamp::Timestamp, - SignedPacket, -}; - use byteorder::LittleEndian; use heed::{types::U64, BoxedError, BytesDecode, BytesEncode, Database, Env, EnvOpenOptions}; use libc::{sysconf, _SC_PAGESIZE}; use tracing::debug; +use pubky_timestamp::Timestamp; + +use crate::{ + base::cache::{Cache, CacheKey}, + SignedPacket, +}; + const MAX_MAP_SIZE: usize = 10995116277760; // 10 TB const MIN_MAP_SIZE: usize = 10 * 1024 * 1024; // 10 mb From f4d88cfd08cd54b8c92ae8b347f4f80caac7239c Mon Sep 17 00:00:00 2001 From: nazeh Date: Fri, 18 Oct 2024 15:19:19 +0300 Subject: [PATCH 45/84] feat(server): pkarr relay should handle if-modified-since headers closes #90 --- Cargo.lock | 3 +++ pkarr/src/base/signed_packet.rs | 10 ++++----- pkarr/src/client/dht.rs | 4 ++-- server/Cargo.toml | 2 ++ server/src/handlers.rs | 38 +++++++++++++++++++++++++++++---- 5 files changed, 46 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 171db2b..855df1e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1511,7 +1511,9 @@ dependencies = [ "dirs-next", "governor", "http", + "httpdate", "pkarr", + "pubky-timestamp", "rustls", "serde", "thiserror", @@ -1584,6 +1586,7 @@ checksum = "084b6e5bfcc186781b71257d636b660f20e94bb588c3ba52393fd9faf7a7bfda" dependencies = [ "document-features", "getrandom", + "httpdate", "js-sys", "once_cell", "serde", diff --git a/pkarr/src/base/signed_packet.rs b/pkarr/src/base/signed_packet.rs index b7dee58..29a5064 100644 --- a/pkarr/src/base/signed_packet.rs +++ b/pkarr/src/base/signed_packet.rs @@ -240,11 +240,11 @@ impl SignedPacket { /// and it is trusted as a way to order which packets where authored after which, /// but it shouldn't be used for caching for example, instead, use [Self::last_seen] /// which is set when you create a new packet. - pub fn timestamp(&self) -> u64 { + pub fn timestamp(&self) -> Timestamp { let bytes = self.inner.borrow_owner(); let slice: [u8; 8] = bytes[96..104].try_into().unwrap(); - u64::from_be_bytes(slice) + u64::from_be_bytes(slice).into() } /// Returns the DNS [Packet] compressed and encoded. @@ -398,7 +398,7 @@ use super::keys::PublicKeyError; #[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] impl From<&SignedPacket> for MutableItem { fn from(s: &SignedPacket) -> Self { - let seq: i64 = s.timestamp() as i64; + let seq = s.timestamp().as_u64() as i64; let packet = s.inner.borrow_owner().slice(104..); Self::new_signed_unchecked( @@ -437,7 +437,7 @@ impl AsRef<[u8]> for SignedPacket { impl Clone for SignedPacket { fn clone(&self) -> Self { - Self::from_bytes_unchecked(self.as_bytes(), &self.last_seen) + Self::from_bytes_unchecked(self.as_bytes(), self.last_seen) } } @@ -670,7 +670,7 @@ mod tests { let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); let item: MutableItem = (&signed_packet).into(); - let seq: i64 = signed_packet.timestamp() as i64; + let seq = signed_packet.timestamp().as_u64() as i64; let expected = MutableItem::new( keypair.secret_key().into(), diff --git a/pkarr/src/client/dht.rs b/pkarr/src/client/dht.rs index 3e4e0d0..c5a5bab 100644 --- a/pkarr/src/client/dht.rs +++ b/pkarr/src/client/dht.rs @@ -335,7 +335,7 @@ impl Client { // since remote nodes will not send the encoded packet if they don't know // any more recent versions. // most_recent_known_timestamp, - as_ref.map(|cached| cached.timestamp()), + as_ref.map(|cached| cached.timestamp().as_u64()), )) .map_err(|_| ClientWasShutdown)?; } @@ -499,7 +499,7 @@ fn run(mut rpc: Rpc, cache: Box, settings: Settings, receiver: Receiv response: QueryResponseSpecific::Value(Response::NoMoreRecentValue(seq)), } => { if let Some(mut cached) = cache.get_read_only(target.as_bytes()) { - if (*seq as u64) == cached.timestamp() { + if (*seq as u64) == cached.timestamp().as_u64() { trace!("Remote node has the a packet with same timestamp, refreshing cached packet."); cached.refresh(); diff --git a/server/Cargo.toml b/server/Cargo.toml index 50ed9f5..c1b45ac 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -25,3 +25,5 @@ toml = "0.8.12" clap = { version = "4.5.1", features = ["derive"] } dirs-next = "2.0.0" pkarr = { version = "3.0.0", path = "../pkarr", features = ["dht", "lmdb-cache"] } +httpdate = "1.0.3" +pubky-timestamp = { version = "0.2.0", features = ["httpdate"] } diff --git a/server/src/handlers.rs b/server/src/handlers.rs index 6f506e5..139f4ed 100644 --- a/server/src/handlers.rs +++ b/server/src/handlers.rs @@ -1,9 +1,12 @@ +use std::str::FromStr; + use axum::extract::Path; use axum::http::HeaderMap; use axum::{extract::State, response::IntoResponse}; use bytes::Bytes; use http::{header, StatusCode}; +use httpdate::HttpDate; use tracing::error; use pkarr::{PublicKey, DEFAULT_MAXIMUM_TTL, DEFAULT_MINIMUM_TTL}; @@ -50,16 +53,30 @@ pub async fn put( pub async fn get( State(state): State, Path(public_key): Path, + headers: HeaderMap, ) -> Result { let public_key = PublicKey::try_from(public_key.as_str()) .map_err(|error| Error::new(StatusCode::BAD_REQUEST, Some(error)))?; + let mut response = HeaderMap::new().into_response(); + if let Some(signed_packet) = state.client.resolve(&public_key).await? { tracing::debug!(?public_key, "cache hit responding with packet!"); - let body = signed_packet.to_relay_payload(); + // Handle IF_MODIFIED_SINCE + if let Some(condition_http_date) = headers + .get(header::IF_MODIFIED_SINCE) + .and_then(|h| h.to_str().ok()) + .and_then(|s| HttpDate::from_str(s).ok()) + { + let entry_http_date: HttpDate = signed_packet.timestamp().into(); - let ttl = signed_packet.ttl(DEFAULT_MINIMUM_TTL, DEFAULT_MAXIMUM_TTL); + if condition_http_date >= entry_http_date { + *response.status_mut() = StatusCode::NOT_MODIFIED; + } + } else { + *response.body_mut() = signed_packet.to_relay_payload().into(); + }; let mut header_map = HeaderMap::new(); @@ -69,10 +86,23 @@ pub async fn get( ); header_map.insert( header::CACHE_CONTROL, - format!("public, max-age={}", ttl).try_into().unwrap(), + format!( + "public, max-age={}", + signed_packet.ttl(DEFAULT_MINIMUM_TTL, DEFAULT_MAXIMUM_TTL) + ) + .try_into() + .unwrap(), + ); + header_map.insert( + header::LAST_MODIFIED, + signed_packet + .timestamp() + .format_http_date() + .try_into() + .expect("expect last-modified to be a valid HeaderValue"), ); - Ok((header_map, body)) + Ok(response) } else { Err(Error::with_status(StatusCode::NOT_FOUND)) } From 1ac48fae980e2c7f6a859b8c821142c00ed0e760 Mon Sep 17 00:00:00 2001 From: nazeh Date: Fri, 18 Oct 2024 15:28:34 +0300 Subject: [PATCH 46/84] fix(server): fix GET endpoint's response header --- server/src/handlers.rs | 44 +++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/server/src/handlers.rs b/server/src/handlers.rs index 139f4ed..c7e5080 100644 --- a/server/src/handlers.rs +++ b/server/src/handlers.rs @@ -53,38 +53,21 @@ pub async fn put( pub async fn get( State(state): State, Path(public_key): Path, - headers: HeaderMap, + request_headers: HeaderMap, ) -> Result { let public_key = PublicKey::try_from(public_key.as_str()) .map_err(|error| Error::new(StatusCode::BAD_REQUEST, Some(error)))?; - let mut response = HeaderMap::new().into_response(); - if let Some(signed_packet) = state.client.resolve(&public_key).await? { tracing::debug!(?public_key, "cache hit responding with packet!"); - // Handle IF_MODIFIED_SINCE - if let Some(condition_http_date) = headers - .get(header::IF_MODIFIED_SINCE) - .and_then(|h| h.to_str().ok()) - .and_then(|s| HttpDate::from_str(s).ok()) - { - let entry_http_date: HttpDate = signed_packet.timestamp().into(); - - if condition_http_date >= entry_http_date { - *response.status_mut() = StatusCode::NOT_MODIFIED; - } - } else { - *response.body_mut() = signed_packet.to_relay_payload().into(); - }; - - let mut header_map = HeaderMap::new(); + let mut response_headers = HeaderMap::new(); - header_map.insert( + response_headers.insert( header::CONTENT_TYPE, "application/pkarr.org/relays#payload".try_into().unwrap(), ); - header_map.insert( + response_headers.insert( header::CACHE_CONTROL, format!( "public, max-age={}", @@ -93,7 +76,7 @@ pub async fn get( .try_into() .unwrap(), ); - header_map.insert( + response_headers.insert( header::LAST_MODIFIED, signed_packet .timestamp() @@ -102,6 +85,23 @@ pub async fn get( .expect("expect last-modified to be a valid HeaderValue"), ); + let mut response = response_headers.into_response(); + + // Handle IF_MODIFIED_SINCE + if let Some(condition_http_date) = request_headers + .get(header::IF_MODIFIED_SINCE) + .and_then(|h| h.to_str().ok()) + .and_then(|s| HttpDate::from_str(s).ok()) + { + let entry_http_date: HttpDate = signed_packet.timestamp().into(); + + if condition_http_date >= entry_http_date { + *response.status_mut() = StatusCode::NOT_MODIFIED; + } + } else { + *response.body_mut() = signed_packet.to_relay_payload().into(); + }; + Ok(response) } else { Err(Error::with_status(StatusCode::NOT_FOUND)) From e835d13c2bc7022820c2c84774afa34269f64a3b Mon Sep 17 00:00:00 2001 From: nazeh Date: Fri, 18 Oct 2024 19:18:23 +0300 Subject: [PATCH 47/84] feat(pkarr): send if-modified-since headers in relay client --- design/relays.md | 3 +++ pkarr/src/client/relay.rs | 18 ++++++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/design/relays.md b/design/relays.md index 143f5a2..d4d5235 100644 --- a/design/relays.md +++ b/design/relays.md @@ -38,6 +38,7 @@ On receiving a PUT request, the relay server should: ``` GET /:z-base32-encoded-key HTTP/2 +If-Modified-Since: Fri, 18 Oct 2024 13:24:21 GMT ``` #### Response @@ -48,11 +49,13 @@ Access-Control-Allow-Origin: * Access-Control-Allow-Methods: GET, PUT, OPTIONS Content-Type: application/pkarr.org/relays#payload Cache-Control: public, max-age=300 +Last-Modified: Fri, 18 Oct 2024 13:24:21 GMT ``` `Cache-Control` header would help browsers reduce their reliance on the relay, the `max-age` should be set to be the minimum `ttl` in the resource records in the packet or some minimum ttl chosen by the relay. +`If-Modified-Since` can be sent by the client to avoid downloading packets they already have, when the relay responds with `304 Not Modified`. Body is described at [Payload](#Payload) encoding section. diff --git a/pkarr/src/client/relay.rs b/pkarr/src/client/relay.rs index ec65c71..2ede6ef 100644 --- a/pkarr/src/client/relay.rs +++ b/pkarr/src/client/relay.rs @@ -3,7 +3,8 @@ use std::fmt::{self, Debug, Display, Formatter}; use std::num::NonZeroUsize; -use reqwest::{Response, StatusCode}; +use reqwest::header::HeaderValue; +use reqwest::{header, Response, StatusCode}; use tracing::debug; #[cfg(target_arch = "wasm32")] @@ -360,7 +361,20 @@ impl Client { ) -> Result, reqwest::Error> { let url = format!("{relay}/{public_key}"); - match self.http_client.get(&url).send().await { + let mut request = self.http_client.get(&url); + + if let Some(httpdate) = cached_packet + .as_ref() + .map(|c| c.timestamp().format_http_date()) + { + request = request.header( + header::IF_MODIFIED_SINCE, + HeaderValue::from_str(httpdate.as_str()) + .expect("httpdate to be valid header value"), + ); + } + + match request.send().await { Ok(response) => { if response.status() == StatusCode::NOT_FOUND { debug!(?url, "SignedPacket not found"); From 134beaed4ad41b78578896e303d8053c8802e748 Mon Sep 17 00:00:00 2001 From: nazeh Date: Fri, 18 Oct 2024 20:13:37 +0300 Subject: [PATCH 48/84] examples(pkarr): enable using relay client --- pkarr/examples/README.md | 13 +++++++++++++ pkarr/examples/publish.rs | 8 +++++++- pkarr/examples/resolve.rs | 9 +++++++-- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/pkarr/examples/README.md b/pkarr/examples/README.md index 77f2a57..9483bd0 100644 --- a/pkarr/examples/README.md +++ b/pkarr/examples/README.md @@ -6,8 +6,21 @@ cargo run --example publish ``` +or to use a Relay client: + +```sh +cargo run --features relay --example publish +``` + ## Resolve ```sh cargo run --example resolve ``` + +or to use a Relay client: + +```sh +cargo run --features relay --example resolve +``` + diff --git a/pkarr/examples/publish.rs b/pkarr/examples/publish.rs index 8a37d6f..38d5312 100644 --- a/pkarr/examples/publish.rs +++ b/pkarr/examples/publish.rs @@ -11,12 +11,18 @@ use tracing_subscriber; use std::time::Instant; -use pkarr::{dns, Client, Keypair, SignedPacket}; +use pkarr::{dns, Keypair, SignedPacket}; + +#[cfg(feature = "relay")] +use pkarr::client::relay::Client; +#[cfg(not(feature = "relay"))] +use pkarr::Client; #[tokio::main] async fn main() { tracing_subscriber::fmt() .with_max_level(Level::DEBUG) + .with_env_filter("pkarr") .init(); let client = Client::builder().build().unwrap(); diff --git a/pkarr/examples/resolve.rs b/pkarr/examples/resolve.rs index 4902bee..05f28b5 100644 --- a/pkarr/examples/resolve.rs +++ b/pkarr/examples/resolve.rs @@ -11,10 +11,15 @@ use std::{ time::{Duration, Instant}, }; -use pkarr::{Client, PublicKey}; - use clap::Parser; +use pkarr::PublicKey; + +#[cfg(feature = "relay")] +use pkarr::client::relay::Client; +#[cfg(not(feature = "relay"))] +use pkarr::Client; + #[derive(Parser)] #[command(author, version, about, long_about = None)] struct Cli { From 959e786f9c7d46dfc2b082b6d09ad7479f5402f1 Mon Sep 17 00:00:00 2001 From: nazeh Date: Fri, 1 Nov 2024 14:23:26 +0300 Subject: [PATCH 49/84] feat: add pkarr.pubky.org as a default Resolver/Relay --- CHANGELOG.md | 1 + pkarr/src/lib.rs | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a891c12..aedc078 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ All notable changes to pkarr client and server will be documented in this file. - Add `SignedPacket::serialize` and `SignedPacket::deserialize`. - Derive `serde::Serialize` and `serde::Deserialize` for `SignedPacket`. - Add `pkarr::LmdbCache` for persistent cache using lmdb. +- Add `pkarr.pubky.org` as an extra default Relay and Resolver. ### Changed diff --git a/pkarr/src/lib.rs b/pkarr/src/lib.rs index 14917ff..ed77111 100644 --- a/pkarr/src/lib.rs +++ b/pkarr/src/lib.rs @@ -20,9 +20,9 @@ pub const DEFAULT_MAXIMUM_TTL: u32 = 24 * 60 * 60; /// Default cache size: 1000 pub const DEFAULT_CACHE_SIZE: usize = 1000; /// Default [relay](https://pkarr.org/relays)s -pub const DEFAULT_RELAYS: [&str; 2] = ["https://relay.pkarr.org", "https://pkarr.pubky.app"]; +pub const DEFAULT_RELAYS: [&str; 2] = ["https://relay.pkarr.org", "https://pkarr.pubky.org"]; /// Default [resolver](https://pkarr.org/resolvers)s -pub const DEFAULT_RESOLVERS: [&str; 2] = ["resolver.pkarr.org:6881", "pkarr.pubky.app:6881"]; +pub const DEFAULT_RESOLVERS: [&str; 2] = ["resolver.pkarr.org:6881", "pkarr.pubky.org:6881"]; #[cfg(any(target_arch = "wasm32", feature = "dht"))] pub use client::Client; From 7abf493f4990c13422f23405d6a9ee88f7afd166 Mon Sep 17 00:00:00 2001 From: nazeh Date: Fri, 1 Nov 2024 21:39:34 +0300 Subject: [PATCH 50/84] feat(pkarr): update mainline and convert Settings into builders --- CHANGELOG.md | 1 + Cargo.lock | 2 +- pkarr/src/client/dht.rs | 62 ++++++++++++++++++--------------------- pkarr/src/client/mod.rs | 4 +-- pkarr/src/client/relay.rs | 40 +++++++++++-------------- pkarr/src/lib.rs | 1 + server/src/main.rs | 22 +++++++------- 7 files changed, 61 insertions(+), 71 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aedc078..64138d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ All notable changes to pkarr client and server will be documented in this file. - update `mainline` to v3.0.0 - `Client::shutdown` and `Client::shutdown_sync` are now idempotent and return `()`. - `Client::resolve`, `Client::resolve_sync` and `relay::Client::resolve` return expired cached `SignedPacket` _before_ making query to the network (Relays/Resolvers/Dht). +- Export `Settings` as client builder. ### Removed diff --git a/Cargo.lock b/Cargo.lock index 855df1e..b0b5f7a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1143,7 +1143,7 @@ checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" [[package]] name = "mainline" version = "3.0.1" -source = "git+https://github.com/pubky/mainline?branch=dev#f80d3b9f2d9a293e7631b9468952fdff9b72fe44" +source = "git+https://github.com/pubky/mainline?branch=dev#6f9448bcdcd527ab5fec859194efa0eee4f3cf58" dependencies = [ "bytes", "crc", diff --git a/pkarr/src/client/dht.rs b/pkarr/src/client/dht.rs index c5a5bab..9e32b3f 100644 --- a/pkarr/src/client/dht.rs +++ b/pkarr/src/client/dht.rs @@ -26,32 +26,32 @@ use crate::{PublicKey, SignedPacket}; #[derive(Debug)] /// [Client]'s settings pub struct Settings { - pub dht: mainline::Settings, + pub(crate) dht_settings: mainline::Settings, /// A set of [resolver](https://pkarr.org/resolvers)s /// to be queried alongside the Dht routing table, to /// lower the latency on cold starts, and help if the /// Dht is missing values not't republished often enough. /// /// Defaults to [DEFAULT_RESOLVERS] - pub resolvers: Option>, + pub(crate) resolvers: Option>, /// Defaults to [DEFAULT_CACHE_SIZE] - pub cache_size: NonZeroUsize, + pub(crate) cache_size: NonZeroUsize, /// Used in the `min` parameter in [SignedPacket::expires_in]. /// /// Defaults to [DEFAULT_MINIMUM_TTL] - pub minimum_ttl: u32, + pub(crate) minimum_ttl: u32, /// Used in the `max` parameter in [SignedPacket::expires_in]. /// /// Defaults to [DEFAULT_MAXIMUM_TTL] - pub maximum_ttl: u32, + pub(crate) maximum_ttl: u32, /// Custom [Cache] implementation, defaults to [InMemoryCache] - pub cache: Option>, + pub(crate) cache: Option>, } impl Default for Settings { fn default() -> Self { Self { - dht: mainline::Settings::default(), + dht_settings: mainline::Dht::builder(), cache_size: NonZeroUsize::new(DEFAULT_CACHE_SIZE).unwrap(), resolvers: Some( DEFAULT_RESOLVERS @@ -67,16 +67,10 @@ impl Default for Settings { } } -#[derive(Debug, Default)] -/// Builder for [Client] -pub struct ClientBuilder { - settings: Settings, -} - -impl ClientBuilder { +impl Settings { /// Set custom set of [resolvers](Settings::resolvers). pub fn resolvers(mut self, resolvers: Option>) -> Self { - self.settings.resolvers = resolvers.map(|resolvers| { + self.resolvers = resolvers.map(|resolvers| { resolvers .iter() .flat_map(|resolver| resolver.to_socket_addrs()) @@ -90,7 +84,7 @@ impl ClientBuilder { /// /// Controls the capacity of [Cache]. pub fn cache_size(mut self, cache_size: NonZeroUsize) -> Self { - self.settings.cache_size = cache_size; + self.cache_size = cache_size; self } @@ -98,8 +92,8 @@ impl ClientBuilder { /// /// Limits how soon a [SignedPacket] is considered expired. pub fn minimum_ttl(mut self, ttl: u32) -> Self { - self.settings.minimum_ttl = ttl; - self.settings.maximum_ttl = self.settings.maximum_ttl.clamp(ttl, u32::MAX); + self.minimum_ttl = ttl; + self.maximum_ttl = self.maximum_ttl.clamp(ttl, u32::MAX); self } @@ -107,31 +101,32 @@ impl ClientBuilder { /// /// Limits how long it takes before a [SignedPacket] is considered expired. pub fn maximum_ttl(mut self, ttl: u32) -> Self { - self.settings.maximum_ttl = ttl; - self.settings.minimum_ttl = self.settings.minimum_ttl.clamp(0, ttl); + self.maximum_ttl = ttl; + self.minimum_ttl = self.minimum_ttl.clamp(0, ttl); self } /// Set a custom implementation of [Cache]. pub fn cache(mut self, cache: Box) -> Self { - self.settings.cache = Some(cache); + self.cache = Some(cache); self } /// Set [Settings::dht] pub fn dht_settings(mut self, settings: mainline::Settings) -> Self { - self.settings.dht = settings; + self.dht_settings = settings; self } /// Convienent methot to set the [mainline::Settings::bootstrap] from [mainline::Testnet::bootstrap] pub fn testnet(mut self, testnet: &Testnet) -> Self { - self.settings.dht.bootstrap = testnet.bootstrap.clone().into(); + self.dht_settings = self.dht_settings.bootstrap(&testnet.bootstrap); + self } pub fn build(self) -> Result { - Client::new(self.settings) + Client::new(self) } } @@ -149,7 +144,7 @@ impl Client { pub fn new(settings: Settings) -> Result { let (sender, receiver) = flume::bounded(32); - let rpc = Rpc::new(&settings.dht)?; + let rpc = settings.dht_settings.build_rpc()?; let local_addr = rpc.local_addr()?; @@ -167,6 +162,8 @@ impl Client { maximum_ttl: settings.maximum_ttl, }; + debug!(?settings, "Starting Client main loop.."); + thread::Builder::new() .name("Client loop".to_string()) .spawn(move || run(rpc, cache_clone, settings, receiver))?; @@ -175,8 +172,8 @@ impl Client { } /// Returns a builder to edit settings before creating Client. - pub fn builder() -> ClientBuilder { - ClientBuilder::default() + pub fn builder() -> Settings { + Settings::default() } // === Getters === @@ -384,9 +381,7 @@ pub enum PublishError { } fn run(mut rpc: Rpc, cache: Box, settings: Settings, receiver: Receiver) { - debug!(?settings, "Starting Client main loop.."); - - let mut server = settings.dht.server; + let mut server = settings.dht_settings.into_server(); let mut senders: HashMap>> = HashMap::new(); loop { @@ -707,10 +702,9 @@ mod tests { let client = Client::builder() .testnet(&testnet) - .dht_settings(mainline::Settings { - request_timeout: Duration::from_millis(10).into(), - ..Default::default() - }) + .dht_settings( + mainline::Settings::default().request_timeout(Duration::from_millis(10).into()), + ) // Everything is expired .maximum_ttl(0) .build() diff --git a/pkarr/src/client/mod.rs b/pkarr/src/client/mod.rs index 4dd13f5..d2cddeb 100644 --- a/pkarr/src/client/mod.rs +++ b/pkarr/src/client/mod.rs @@ -3,12 +3,12 @@ #[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] pub mod dht; #[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] -pub use dht::Client; +pub use dht::{Client, Settings}; #[cfg(target_arch = "wasm32")] pub mod relay; #[cfg(target_arch = "wasm32")] -pub use relay::Client; +pub use relay::{Client, Settings}; #[cfg(all(not(target_arch = "wasm32"), feature = "relay"))] pub mod relay; diff --git a/pkarr/src/client/relay.rs b/pkarr/src/client/relay.rs index 2ede6ef..aa2e27f 100644 --- a/pkarr/src/client/relay.rs +++ b/pkarr/src/client/relay.rs @@ -21,21 +21,21 @@ use crate::{ #[derive(Debug, Clone)] /// [Client]'s settings pub struct Settings { - pub relays: Vec, + pub(crate) relays: Vec, /// Defaults to [DEFAULT_CACHE_SIZE] - pub cache_size: NonZeroUsize, + pub(crate) cache_size: NonZeroUsize, /// Used in the `min` parameter in [SignedPacket::expires_in]. /// /// Defaults to [DEFAULT_MINIMUM_TTL] - pub minimum_ttl: u32, + pub(crate) minimum_ttl: u32, /// Used in the `max` parametere in [SignedPacket::expires_in]. /// /// Defaults to [DEFAULT_MAXIMUM_TTL] - pub maximum_ttl: u32, + pub(crate) maximum_ttl: u32, /// Custom [reqwest::Client] - pub http_client: reqwest::Client, + pub(crate) http_client: reqwest::Client, /// Custom [Cache] implementation, defaults to [InMemoryCache] - pub cache: Option>, + pub(crate) cache: Option>, } impl Default for Settings { @@ -51,16 +51,10 @@ impl Default for Settings { } } -#[derive(Debug, Default)] -/// Builder for [Client] -pub struct ClientBuilder { - settings: Settings, -} - -impl ClientBuilder { +impl Settings { /// Set the relays to publish and resolve [SignedPacket]s to and from. pub fn relays(mut self, relays: Vec) -> Self { - self.settings.relays = relays; + self.relays = relays; self } @@ -68,7 +62,7 @@ impl ClientBuilder { /// /// Controls the capacity of [Cache]. pub fn cache_size(mut self, cache_size: NonZeroUsize) -> Self { - self.settings.cache_size = cache_size; + self.cache_size = cache_size; self } @@ -76,8 +70,8 @@ impl ClientBuilder { /// /// Limits how soon a [SignedPacket] is considered expired. pub fn minimum_ttl(mut self, ttl: u32) -> Self { - self.settings.minimum_ttl = ttl; - self.settings.maximum_ttl = self.settings.maximum_ttl.clamp(ttl, u32::MAX); + self.minimum_ttl = ttl; + self.maximum_ttl = self.maximum_ttl.clamp(ttl, u32::MAX); self } @@ -85,19 +79,19 @@ impl ClientBuilder { /// /// Limits how long it takes before a [SignedPacket] is considered expired. pub fn maximum_ttl(mut self, ttl: u32) -> Self { - self.settings.maximum_ttl = ttl; - self.settings.minimum_ttl = self.settings.minimum_ttl.clamp(0, ttl); + self.maximum_ttl = ttl; + self.minimum_ttl = self.minimum_ttl.clamp(0, ttl); self } /// Set a custom implementation of [Cache]. pub fn cache(mut self, cache: Box) -> Self { - self.settings.cache = Some(cache); + self.cache = Some(cache); self } pub fn build(self) -> Result { - Client::new(self.settings) + Client::new(self) } } @@ -138,8 +132,8 @@ impl Client { } /// Returns a builder to edit settings before creating Client. - pub fn builder() -> ClientBuilder { - ClientBuilder::default() + pub fn builder() -> Settings { + Settings::default() } /// Returns a reference to the internal cache. diff --git a/pkarr/src/lib.rs b/pkarr/src/lib.rs index ed77111..3c61f7f 100644 --- a/pkarr/src/lib.rs +++ b/pkarr/src/lib.rs @@ -26,6 +26,7 @@ pub const DEFAULT_RESOLVERS: [&str; 2] = ["resolver.pkarr.org:6881", "pkarr.pubk #[cfg(any(target_arch = "wasm32", feature = "dht"))] pub use client::Client; +pub use client::Settings; // Rexports pub use bytes; diff --git a/server/src/main.rs b/server/src/main.rs index fdf36dd..62ca227 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -48,17 +48,17 @@ async fn main() -> Result<()> { let rate_limiter = rate_limiting::IpRateLimiter::new(config.rate_limiter()); let client = Client::builder() - .dht_settings(mainline::Settings { - port: Some(config.dht_port()), - server: Some(Box::new(dht_server::DhtServer::new( - cache.clone(), - config.resolvers(), - config.minimum_ttl(), - config.maximum_ttl(), - rate_limiter.clone(), - ))), - ..mainline::Settings::default() - }) + .dht_settings( + mainline::Settings::default() + .port(config.dht_port()) + .custom_server(Box::new(dht_server::DhtServer::new( + cache.clone(), + config.resolvers(), + config.minimum_ttl(), + config.maximum_ttl(), + rate_limiter.clone(), + ))), + ) .resolvers(config.resolvers()) .minimum_ttl(config.minimum_ttl()) .maximum_ttl(config.maximum_ttl()) From 6d00d1552c5c7690187602bd54aa813175ffc247 Mon Sep 17 00:00:00 2001 From: nazeh Date: Fri, 1 Nov 2024 22:27:08 +0300 Subject: [PATCH 51/84] feat(pkarr): export SignedPacketError and PublicKeyError --- pkarr/src/client/mod.rs | 4 ++-- pkarr/src/lib.rs | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pkarr/src/client/mod.rs b/pkarr/src/client/mod.rs index d2cddeb..6eed8ff 100644 --- a/pkarr/src/client/mod.rs +++ b/pkarr/src/client/mod.rs @@ -1,12 +1,12 @@ //! Client implementation. #[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] -pub mod dht; +pub(crate) mod dht; #[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] pub use dht::{Client, Settings}; #[cfg(target_arch = "wasm32")] -pub mod relay; +pub(crate) mod relay; #[cfg(target_arch = "wasm32")] pub use relay::{Client, Settings}; diff --git a/pkarr/src/lib.rs b/pkarr/src/lib.rs index 3c61f7f..0c05711 100644 --- a/pkarr/src/lib.rs +++ b/pkarr/src/lib.rs @@ -43,4 +43,7 @@ pub mod errors { #[cfg(any(target_arch = "wasm32", feature = "relay"))] pub use super::client::relay::{EmptyListOfRelays, PublishToRelayError}; + + pub use super::base::keys::PublicKeyError; + pub use super::base::signed_packet::SignedPacketError; } From ab2fb43c49185c3054f988fd5c548d774ce4f37e Mon Sep 17 00:00:00 2001 From: nazeh Date: Mon, 11 Nov 2024 15:47:39 +0300 Subject: [PATCH 52/84] feat: update mainline --- Cargo.lock | 2 +- pkarr/Cargo.toml | 4 +- pkarr/src/client/dht.rs | 107 +++++++++++++++++++++++++++++----------- server/src/main.rs | 3 +- 4 files changed, 83 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b0b5f7a..28f8bb6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1143,7 +1143,7 @@ checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" [[package]] name = "mainline" version = "3.0.1" -source = "git+https://github.com/pubky/mainline?branch=dev#6f9448bcdcd527ab5fec859194efa0eee4f3cf58" +source = "git+https://github.com/pubky/mainline#dc656d8c72c0a525e8575798eba987863bf37df2" dependencies = [ "bytes", "crc", diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index d199f4a..c0c0b6b 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -36,7 +36,7 @@ futures = { version = "0.3.30", optional = true, default-features = false } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] # feat: dht dependencies -mainline = { git = "https://github.com/pubky/mainline", branch = "dev", optional = true } +mainline = { git = "https://github.com/pubky/mainline", optional = true } flume = { version = "0.11.0", features = ["select", "eventual-fairness", "async"], default-features = false , optional = true } # feat: relay dependencies @@ -71,7 +71,7 @@ dht = ["dep:mainline", "dep:flume"] relay = ["dep:reqwest", "dep:tokio", "dep:sha1_smol"] ## Derive serde Serialize/Deserialize for PublicKey -serde = ["dep:serde", "pubky-timestamp/serde"] +serde = ["dep:serde", "pubky-timestamp/serde", "pubky-timestamp/httpdate"] # Extra ## Use [extra::endpoints::EndpointResolver] trait diff --git a/pkarr/src/client/dht.rs b/pkarr/src/client/dht.rs index 9e32b3f..e64cff5 100644 --- a/pkarr/src/client/dht.rs +++ b/pkarr/src/client/dht.rs @@ -133,7 +133,6 @@ impl Settings { #[derive(Clone, Debug)] /// Pkarr client for publishing and resolving [SignedPacket]s over [mainline]. pub struct Client { - address: Option, sender: Sender, cache: Box, minimum_ttl: u32, @@ -144,10 +143,6 @@ impl Client { pub fn new(settings: Settings) -> Result { let (sender, receiver) = flume::bounded(32); - let rpc = settings.dht_settings.build_rpc()?; - - let local_addr = rpc.local_addr()?; - let cache = settings .cache .clone() @@ -155,7 +150,6 @@ impl Client { let cache_clone = cache.clone(); let client = Client { - address: Some(local_addr), sender, cache, minimum_ttl: settings.minimum_ttl, @@ -165,8 +159,17 @@ impl Client { debug!(?settings, "Starting Client main loop.."); thread::Builder::new() - .name("Client loop".to_string()) - .spawn(move || run(rpc, cache_clone, settings, receiver))?; + .name("Pkarr Dht actor thread".to_string()) + .spawn(move || run(cache_clone, settings, receiver))?; + + let (tx, rx) = flume::bounded(1); + + client + .sender + .send(ActorMessage::Check(tx)) + .expect("actor thread unexpectedly shutdown"); + + rx.recv().expect("infallible")?; Ok(client) } @@ -178,11 +181,15 @@ impl Client { // === Getters === - /// Returns the local address of the udp socket this node is listening on. - /// - /// Returns `None` if the node is shutdown - pub fn local_addr(&self) -> Option { - self.address + /// Returns [Info] about the running session from the actor thread. + pub fn info(&self) -> Result { + let (tx, rx) = flume::bounded(1); + + self.sender + .send(ActorMessage::Info(tx)) + .map_err(|_| ClientWasShutdown)?; + + rx.recv().map_err(|_| ClientWasShutdown) } /// Returns a reference to the internal cache. @@ -229,8 +236,6 @@ impl Client { let _ = self.sender.send(ActorMessage::Shutdown(sender)); let _ = receiver.recv_async().await; - - self.address = None; } // === Sync === @@ -266,13 +271,11 @@ impl Client { } /// Shutdown the actor thread loop. - pub fn shutdown_sync(&mut self) { + pub fn shutdown_sync(&self) { let (sender, receiver) = flume::bounded(1); let _ = self.sender.send(ActorMessage::Shutdown(sender)); let _ = receiver.recv(); - - self.address = None; } // === Private Methods === @@ -380,8 +383,27 @@ pub enum PublishError { MainlinePutError(#[from] PutError), } -fn run(mut rpc: Rpc, cache: Box, settings: Settings, receiver: Receiver) { - let mut server = settings.dht_settings.into_server(); +fn run(cache: Box, settings: Settings, receiver: Receiver) { + match settings.dht_settings.build_rpc() { + Ok(mut rpc) => { + let mut server = settings.dht_settings.into_server(); + actor_thread(&mut rpc, &mut server, cache, receiver, settings.resolvers) + } + Err(err) => { + if let Ok(ActorMessage::Check(sender)) = receiver.try_recv() { + let _ = sender.send(Err(err)); + } + } + } +} + +fn actor_thread( + rpc: &mut Rpc, + server: &mut Option>, + cache: Box, + receiver: Receiver, + resolvers: Option>, +) { let mut senders: HashMap>> = HashMap::new(); loop { @@ -426,7 +448,15 @@ fn run(mut rpc: Rpc, cache: Box, settings: Settings, receiver: Receiv }, ); - rpc.get(target, request, None, settings.resolvers.clone()) + rpc.get(target, request, None, resolvers.clone()) + } + ActorMessage::Info(sender) => { + let local_addr = rpc.local_addr(); + + let _ = sender.send(Info { local_addr }); + } + ActorMessage::Check(sender) => { + let _ = sender.send(Ok(())); } } } @@ -517,7 +547,7 @@ fn run(mut rpc: Rpc, cache: Box, settings: Settings, receiver: Receiv // === Requests === ReceivedMessage::Request((transaction_id, request)) => { if let Some(server) = server.as_mut() { - server.handle_request(&mut rpc, *from, *transaction_id, request); + server.handle_request(rpc, *from, *transaction_id, request); } } }; @@ -527,10 +557,25 @@ fn run(mut rpc: Rpc, cache: Box, settings: Settings, receiver: Receiv debug!("Client main loop terminated"); } -pub enum ActorMessage { +enum ActorMessage { Publish(MutableItem, Sender>), Resolve(Id, Sender, Option), Shutdown(Sender<()>), + Info(Sender), + Check(Sender>), +} + +pub struct Info { + local_addr: Result, +} + +impl Info { + /// Local UDP Ipv4 socket address that this node is listening on. + pub fn local_addr(&self) -> Result<&SocketAddr, std::io::Error> { + self.local_addr + .as_ref() + .map_err(|e| std::io::Error::new(e.kind(), e.to_string())) + } } #[cfg(test)] @@ -546,13 +591,17 @@ mod tests { fn shutdown_sync() { let testnet = Testnet::new(3).unwrap(); - let mut a = Client::builder().testnet(&testnet).build().unwrap(); + let client = Client::builder().testnet(&testnet).build().unwrap(); + + let local_addr = client.info().unwrap().local_addr().unwrap().to_string(); + + println!("{}", local_addr); - assert_ne!(a.local_addr(), None); + assert!(client.info().unwrap().local_addr().is_ok()); - a.shutdown_sync(); + client.shutdown_sync(); - assert_eq!(a.local_addr(), None); + assert!(client.info().is_err()); } #[test] @@ -625,11 +674,11 @@ mod tests { let mut a = Client::builder().testnet(&testnet).build().unwrap(); - assert_ne!(a.local_addr(), None); + assert!(a.info().unwrap().local_addr().is_ok()); a.shutdown().await; - assert_eq!(a.local_addr(), None); + assert!(a.info().is_err()); } #[tokio::test] diff --git a/server/src/main.rs b/server/src/main.rs index 62ca227..4800470 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -65,7 +65,8 @@ async fn main() -> Result<()> { .cache(cache) .build()?; - let udp_address = client.local_addr().unwrap(); + let info = client.info()?; + let udp_address = info.local_addr()?; info!("Running as a resolver on UDP socket {udp_address}"); From 829aeef79a55514fd1fa5d98bde8b88739a0eb0a Mon Sep 17 00:00:00 2001 From: nazeh Date: Tue, 12 Nov 2024 18:40:16 +0300 Subject: [PATCH 53/84] feat(pkarr): add futures-lite and genawaiter --- Cargo.lock | 29 +++++++++++++++- pkarr/Cargo.toml | 7 ++-- pkarr/src/extra/endpoints/mod.rs | 59 ++++++++++++++++++-------------- 3 files changed, 66 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 28f8bb6..dbf08d8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -710,6 +710,16 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" +[[package]] +name = "futures-lite" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef40d21ae2c515b51041df9ed313ed21e572df340ea58a922a0aefe7e8891a1" +dependencies = [ + "futures-core", + "pin-project-lite", +] + [[package]] name = "futures-macro" version = "0.3.31" @@ -757,6 +767,22 @@ dependencies = [ "slab", ] +[[package]] +name = "genawaiter" +version = "0.99.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c86bd0361bcbde39b13475e6e36cb24c329964aa2611be285289d1e4b751c1a0" +dependencies = [ + "futures-core", + "genawaiter-macro", +] + +[[package]] +name = "genawaiter-macro" +version = "0.99.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b32dfe1fdfc0bbde1f22a5da25355514b5e450c33a6af6770884c8750aedfbc" + [[package]] name = "generic-array" version = "0.14.7" @@ -1476,7 +1502,8 @@ dependencies = [ "dyn-clone", "ed25519-dalek", "flume", - "futures", + "futures-lite", + "genawaiter", "getrandom", "heed", "js-sys", diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index c0c0b6b..257c4ff 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -32,7 +32,8 @@ sha1_smol = { version = "1.0.1", optional = true } serde = { version = "1.0.209", features = ["derive"], optional = true } # feat: endpoints dependencies -futures = { version = "0.3.30", optional = true, default-features = false } +futures-lite = { version = "2.5.0", default-features = false, optional = true } +genawaiter = { version = "0.99.1", default-features = false, features = ["futures03"] } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] # feat: dht dependencies @@ -49,7 +50,7 @@ libc = { version = "0.2.159", optional = true } [target.'cfg(target_arch = "wasm32")'.dependencies] js-sys = "0.3.69" -futures = "0.3.30" +futures-lite = { version = "2.5.0", default-features = false } getrandom = { version = "0.2", features = ["js"] } reqwest = "0.12.7" sha1_smol = "1.0.1" @@ -75,7 +76,7 @@ serde = ["dep:serde", "pubky-timestamp/serde", "pubky-timestamp/httpdate"] # Extra ## Use [extra::endpoints::EndpointResolver] trait -endpoints = ["dep:futures"] +endpoints = ["dep:futures-lite"] ## Use [LmdbCache] lmdb-cache = ["dep:heed", "dep:byteorder", "dep:libc"] diff --git a/pkarr/src/extra/endpoints/mod.rs b/pkarr/src/extra/endpoints/mod.rs index 8b6660d..68d08e7 100644 --- a/pkarr/src/extra/endpoints/mod.rs +++ b/pkarr/src/extra/endpoints/mod.rs @@ -1,8 +1,10 @@ //! EndpointResolver trait for different clients -// mod async_iter; mod endpoint; +use futures_lite::Stream; +use genawaiter::sync::Gen; + use crate::{Client, PublicKey, SignedPacket}; pub use endpoint::Endpoint; @@ -15,14 +17,8 @@ pub trait EndpointResolver { public_key: &PublicKey, ) -> impl std::future::Future, ResolveError>>; - fn resolve_endpoint( - &self, - qname: &str, - ) -> impl std::future::Future> - where - Self: std::marker::Sync, - { - async move { + fn resolve_endpoint(&self, qname: &str) -> impl Stream { + Gen::new(|co| async move { let target = qname; // TODO: cache the result of this function? @@ -56,12 +52,12 @@ pub trait EndpointResolver { if let Some(svcb) = svcb { if PublicKey::try_from(svcb.target()).is_err() { - return Ok(svcb); + co.yield_(svcb).await } } - Err(FailedToResolveEndpoint) - } + // co.yield_(None).await + }) } } @@ -82,17 +78,6 @@ impl EndpointResolver for crate::client::relay::Client { } } -#[derive(Debug)] -struct FailedToResolveEndpoint; - -impl std::error::Error for FailedToResolveEndpoint {} - -impl std::fmt::Display for FailedToResolveEndpoint { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "The Pkarr Client was shutdown") - } -} - #[derive(thiserror::Error, Debug)] /// Resolve Error from a client pub enum ResolveError { @@ -122,6 +107,8 @@ mod tests { use std::future::Future; use std::pin::Pin; + use futures_lite::StreamExt; + fn generate_subtree( client: Client, depth: u8, @@ -193,10 +180,27 @@ mod tests { let tld = generate(&client, 3, 3, Some("example.com".to_string())).await; - let endpoint = client.resolve_endpoint(&tld.to_string()).await.unwrap(); + let endpoint = client + .resolve_endpoint(&tld.to_string()) + .next() + .await + .unwrap(); + assert_eq!(endpoint.target(), "example.com"); } + #[tokio::test] + async fn empty() { + let testnet = Testnet::new(3).unwrap(); + let client = Client::builder().testnet(&testnet).build().unwrap(); + + let pubky = Keypair::random().public_key(); + + let endpoint = client.resolve_endpoint(&pubky.to_string()).next().await; + + assert!(endpoint.is_none()); + } + // TODO: Test max_chain_exceeded // #[tokio::test] // async fn max_chain_exceeded() { @@ -228,7 +232,12 @@ mod tests { let tld = generate(&client, 3, 3, None).await; - let endpoint = client.resolve_endpoint(&tld.to_string()).await.unwrap(); + let endpoint = client + .resolve_endpoint(&tld.to_string()) + .next() + .await + .unwrap(); + assert_eq!(endpoint.target(), "."); assert_eq!(endpoint.port(), Some(3000)); assert_eq!( From 1db37d09180b62904dedda266dab88138cfd022c Mon Sep 17 00:00:00 2001 From: nazeh Date: Wed, 13 Nov 2024 16:46:28 +0300 Subject: [PATCH 54/84] feat(pkarr): add EndpointResolver::resolve_endpoints and resolve_endpoint --- pkarr/src/extra/endpoints/mod.rs | 89 ++++++++++++++++++-------------- 1 file changed, 50 insertions(+), 39 deletions(-) diff --git a/pkarr/src/extra/endpoints/mod.rs b/pkarr/src/extra/endpoints/mod.rs index 68d08e7..ea6590a 100644 --- a/pkarr/src/extra/endpoints/mod.rs +++ b/pkarr/src/extra/endpoints/mod.rs @@ -2,7 +2,7 @@ mod endpoint; -use futures_lite::Stream; +use futures_lite::{pin, Stream, StreamExt}; use genawaiter::sync::Gen; use crate::{Client, PublicKey, SignedPacket}; @@ -12,12 +12,14 @@ pub use endpoint::Endpoint; const DEFAULT_MAX_CHAIN_LENGTH: u8 = 3; pub trait EndpointResolver { + /// A wrapper around the specific Pkarr client's resolve method. fn resolve( &self, public_key: &PublicKey, ) -> impl std::future::Future, ResolveError>>; - fn resolve_endpoint(&self, qname: &str) -> impl Stream { + /// Returns an async stream of [Endpoint]s + fn resolve_endpoints(&self, qname: &str) -> impl Stream { Gen::new(|co| async move { let target = qname; // TODO: cache the result of this function? @@ -59,6 +61,23 @@ pub trait EndpointResolver { // co.yield_(None).await }) } + + /// Returns the first [Endpoint] in the Async stream from [EndpointResolver::resolve_endpoints] + fn resolve_endpoint( + &self, + qname: &str, + ) -> impl std::future::Future> { + async { + let stream = self.resolve_endpoints(qname); + + pin!(stream); + + match stream.next().await { + Some(endpoint) => Ok(endpoint), + None => Err(FailedToResolveEndpoint), + } + } + } } impl EndpointResolver for Client { @@ -96,6 +115,20 @@ impl std::fmt::Display for ResolveError { } } +#[derive(Debug)] +struct FailedToResolveEndpoint; + +impl std::error::Error for FailedToResolveEndpoint {} + +impl std::fmt::Display for FailedToResolveEndpoint { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Could not resolve clear net endpoint for the Pkarr domain" + ) + } +} + #[cfg(test)] mod tests { use super::*; @@ -107,8 +140,6 @@ mod tests { use std::future::Future; use std::pin::Pin; - use futures_lite::StreamExt; - fn generate_subtree( client: Client, depth: u8, @@ -180,11 +211,7 @@ mod tests { let tld = generate(&client, 3, 3, Some("example.com".to_string())).await; - let endpoint = client - .resolve_endpoint(&tld.to_string()) - .next() - .await - .unwrap(); + let endpoint = client.resolve_endpoint(&tld.to_string()).await.unwrap(); assert_eq!(endpoint.target(), "example.com"); } @@ -196,34 +223,22 @@ mod tests { let pubky = Keypair::random().public_key(); - let endpoint = client.resolve_endpoint(&pubky.to_string()).next().await; + let endpoint = client.resolve_endpoint(&pubky.to_string()).await; - assert!(endpoint.is_none()); + assert!(endpoint.is_err()); } - // TODO: Test max_chain_exceeded - // #[tokio::test] - // async fn max_chain_exceeded() { - // let testnet = Testnet::new(3); - // let pkarr = Client::builder().testnet(&testnet).build().unwrap(); - // - // let resolver: EndpointResolver = (&pkarr).into(); - // - // let tld = generate(pkarr, 4, 3, Some("example.com".to_string())).await; - // - // let endpoint = resolver.resolve_endpoint(&tld.to_string()).await; - // - // assert!(endpoint.is_err()); - // // TODO: test error correctly - // - // // assert_eq!( - // // match endpoint { - // // Err(error) => error.to_string(), - // // _ => "".to_string(), - // // }, - // // Error::Generic(tld.to_string()) - // // ) - // } + #[tokio::test] + async fn max_chain_exceeded() { + let testnet = Testnet::new(3).unwrap(); + let client = Client::builder().testnet(&testnet).build().unwrap(); + + let tld = generate(&client, 4, 3, Some("example.com".to_string())).await; + + let endpoint = client.resolve_endpoint(&tld.to_string()).await; + + assert!(endpoint.is_err()); + } #[tokio::test] async fn resolve_addresses() { @@ -232,11 +247,7 @@ mod tests { let tld = generate(&client, 3, 3, None).await; - let endpoint = client - .resolve_endpoint(&tld.to_string()) - .next() - .await - .unwrap(); + let endpoint = client.resolve_endpoint(&tld.to_string()).await.unwrap(); assert_eq!(endpoint.target(), "."); assert_eq!(endpoint.port(), Some(3000)); From d4aae0710d51eaf433f638d2d267aba38939599c Mon Sep 17 00:00:00 2001 From: nazeh Date: Wed, 13 Nov 2024 19:58:47 +0300 Subject: [PATCH 55/84] feat(pkarr): add Reqwest resolve implementation --- Cargo.lock | 4 +- pkarr/Cargo.toml | 6 +- pkarr/examples/README.md | 15 +++++ pkarr/examples/http-get.rs | 39 ++++++++++++ pkarr/examples/http-serve.rs | 86 +++++++++++++++++++++++++++ pkarr/src/extra/endpoints/endpoint.rs | 4 -- pkarr/src/extra/endpoints/mod.rs | 2 +- pkarr/src/extra/mod.rs | 3 + pkarr/src/extra/reqwest.rs | 65 ++++++++++++++++++++ 9 files changed, 217 insertions(+), 7 deletions(-) create mode 100644 pkarr/examples/http-get.rs create mode 100644 pkarr/examples/http-serve.rs create mode 100644 pkarr/src/extra/reqwest.rs diff --git a/Cargo.lock b/Cargo.lock index dbf08d8..585c864 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1169,7 +1169,7 @@ checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" [[package]] name = "mainline" version = "3.0.1" -source = "git+https://github.com/pubky/mainline#dc656d8c72c0a525e8575798eba987863bf37df2" +source = "git+https://github.com/pubky/mainline#ce935827349341d2130589486a40e6411c8c790e" dependencies = [ "bytes", "crc", @@ -1494,6 +1494,8 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" name = "pkarr" version = "3.0.0" dependencies = [ + "axum", + "axum-server", "base32", "byteorder", "bytes", diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index 257c4ff..399eb76 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -56,6 +56,8 @@ reqwest = "0.12.7" sha1_smol = "1.0.1" [dev-dependencies] +axum = "0.7.7" +axum-server = "0.7.1" postcard = { version = "1.0.10", features = ["alloc"] } tokio = { version = "1.40.0", features = ["macros", "rt-multi-thread"] } @@ -77,11 +79,13 @@ serde = ["dep:serde", "pubky-timestamp/serde", "pubky-timestamp/httpdate"] # Extra ## Use [extra::endpoints::EndpointResolver] trait endpoints = ["dep:futures-lite"] +## Implement reqwest::dns::Resolve trait for Pkarr clients +reqwest-resolve = ["dep:reqwest", "endpoints"] ## Use [LmdbCache] lmdb-cache = ["dep:heed", "dep:byteorder", "dep:libc"] ## Use all features -full = ["dht", "relay", "serde", "endpoints", "lmdb-cache"] +full = ["dht", "relay", "serde", "endpoints", "lmdb-cache", "reqwest-resolve"] default = ["dht"] diff --git a/pkarr/examples/README.md b/pkarr/examples/README.md index 9483bd0..766443f 100644 --- a/pkarr/examples/README.md +++ b/pkarr/examples/README.md @@ -24,3 +24,18 @@ or to use a Relay client: cargo run --features relay --example resolve ``` +## HTTP + +Run an HTTP server listening on a Pkarr key + +```sh +cargo run --features endpoints --example http-serve +``` + +An HTTP url will be printend with the Pkarr key as the TLD, paste in another terminal window: + +```sh +cargo run --features reqwest-resolve --example http-get +``` + +And you should see a `Hello, World!` response. diff --git a/pkarr/examples/http-get.rs b/pkarr/examples/http-get.rs new file mode 100644 index 0000000..528b03a --- /dev/null +++ b/pkarr/examples/http-get.rs @@ -0,0 +1,39 @@ +//! Make an HTTP request over to a Pkarr address using Reqwest + +use std::sync::Arc; + +use reqwest::Method; +use tracing::Level; +use tracing_subscriber; + +use clap::Parser; + +use pkarr::Client; + +#[derive(Parser)] +#[command(author, version, about, long_about = None)] +struct Cli { + /// Url to GET from + url: String, +} + +#[tokio::main] +async fn main() { + tracing_subscriber::fmt().with_max_level(Level::INFO).init(); + + let cli = Cli::parse(); + let url = cli.url; + + let client = Client::builder().build().unwrap(); + + let reqwest = reqwest::Client::builder() + .dns_resolver(Arc::new(client)) + .build() + .unwrap(); + + let response = reqwest.request(Method::GET, &url).send().await.unwrap(); + + let body = response.text().await.unwrap(); + + println!("Resolved {url}\n{body}"); +} diff --git a/pkarr/examples/http-serve.rs b/pkarr/examples/http-serve.rs new file mode 100644 index 0000000..a0434fe --- /dev/null +++ b/pkarr/examples/http-serve.rs @@ -0,0 +1,86 @@ +//! Run an HTTP server listening on a Pkarr domain +//! +//! This server will _not_ be accessible from other networks +//! unless the provided IP is public and the port number is forwarded. + +use tracing::Level; +use tracing_subscriber; + +use axum::{response::Html, routing::get, Router}; +use axum_server::bind; +use std::net::{SocketAddr, ToSocketAddrs}; + +use clap::Parser; + +use pkarr::{ + dns::{rdata::SVCB, Packet}, + Client, Keypair, SignedPacket, +}; + +#[derive(Parser)] +#[command(author, version, about, long_about = None)] +struct Cli { + /// IP address to listen on + ip: String, + /// Port number to listen no + port: u16, +} + +#[tokio::main] +async fn main() { + tracing_subscriber::fmt().with_max_level(Level::INFO).init(); + + let cli = Cli::parse(); + + let app = Router::new().route("/", get(handler)); + + let addr = format!("{}:{}", cli.ip, cli.port) + .to_socket_addrs() + .unwrap() + .next() + .unwrap(); + + publish_server_pkarr(&addr).await; + + bind(addr).serve(app.into_make_service()).await.unwrap(); +} + +// Simple handler that responds with "Hello, World!" +async fn handler() -> Html<&'static str> { + Html("Hello, World!") +} + +async fn publish_server_pkarr(socket_addr: &SocketAddr) { + let client = Client::builder().build().unwrap(); + + let keypair = Keypair::random(); + + let mut packet = Packet::new_reply(1); + + let mut svcb = SVCB::new(0, ".".try_into().unwrap()); + + svcb.set_port(socket_addr.port()); + + packet.answers.push(pkarr::dns::ResourceRecord::new( + "@".try_into().unwrap(), + pkarr::dns::CLASS::IN, + 60 * 60, + pkarr::dns::rdata::RData::HTTPS(svcb.into()), + )); + + packet.answers.push(pkarr::dns::ResourceRecord::new( + "@".try_into().unwrap(), + pkarr::dns::CLASS::IN, + 60 * 60, + match socket_addr.ip() { + std::net::IpAddr::V4(ip) => pkarr::dns::rdata::RData::A(ip.into()), + std::net::IpAddr::V6(ip) => pkarr::dns::rdata::RData::AAAA(ip.into()), + }, + )); + + let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); + + println!("Server running on http://{}", keypair.public_key()); + + client.publish(&signed_packet).await.unwrap(); +} diff --git a/pkarr/src/extra/endpoints/endpoint.rs b/pkarr/src/extra/endpoints/endpoint.rs index a37dfb0..254cee8 100644 --- a/pkarr/src/extra/endpoints/endpoint.rs +++ b/pkarr/src/extra/endpoints/endpoint.rs @@ -94,10 +94,6 @@ impl Endpoint { self.port } - // pub fn public_key(&self) -> { - // &self.target - // } - /// Return an iterator of [SocketAddr], either by resolving the [Endpoint::target] using normal DNS, /// or, if the target is ".", return the [RData::A] or [RData::AAAA] records /// from the endpoint's [SignedPacket], if available. diff --git a/pkarr/src/extra/endpoints/mod.rs b/pkarr/src/extra/endpoints/mod.rs index ea6590a..3b486fe 100644 --- a/pkarr/src/extra/endpoints/mod.rs +++ b/pkarr/src/extra/endpoints/mod.rs @@ -116,7 +116,7 @@ impl std::fmt::Display for ResolveError { } #[derive(Debug)] -struct FailedToResolveEndpoint; +pub struct FailedToResolveEndpoint; impl std::error::Error for FailedToResolveEndpoint {} diff --git a/pkarr/src/extra/mod.rs b/pkarr/src/extra/mod.rs index 585f435..4915391 100644 --- a/pkarr/src/extra/mod.rs +++ b/pkarr/src/extra/mod.rs @@ -1,5 +1,8 @@ #[cfg(feature = "endpoints")] pub mod endpoints; +#[cfg(feature = "reqwest-resolve")] +pub mod reqwest; + #[cfg(all(not(target_arch = "wasm32"), feature = "lmdb-cache"))] pub mod lmdb_cache; diff --git a/pkarr/src/extra/reqwest.rs b/pkarr/src/extra/reqwest.rs new file mode 100644 index 0000000..35ab320 --- /dev/null +++ b/pkarr/src/extra/reqwest.rs @@ -0,0 +1,65 @@ +use reqwest::dns::{Addrs, Resolve}; + +use crate::{Client, PublicKey}; + +use super::endpoints::EndpointResolver; + +use std::net::ToSocketAddrs; + +impl Resolve for Client { + fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving { + let client = self.clone(); + Box::pin(resolve(client, name)) + } +} + +#[cfg(all(not(target_arch = "wasm32"), feature = "relay"))] +impl Resolve for crate::client::relay::Client { + fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving { + let client = self.clone(); + Box::pin(resolve(client, name)) + } +} + +async fn resolve( + client: impl EndpointResolver, + name: reqwest::dns::Name, +) -> Result> { + let name = name.as_str(); + + if PublicKey::try_from(name).is_ok() { + let endpoint = client.resolve_endpoint(name).await?; + + let addrs: Addrs = Box::new(endpoint.to_socket_addrs().into_iter()); + + return Ok(addrs); + }; + + Ok(Box::new(format!("{name}:0").to_socket_addrs().unwrap())) +} + +#[cfg(test)] +mod tests { + use axum::{response::Html, routing::get, Router}; + use rand::Rng; + use std::net::SocketAddr; + + #[tokio::test] + async fn http() { + // Define the route and handler + let app = Router::new().route("/", get(handler)); + + let port: u16 = rand::thread_rng().gen(); + + let addr = SocketAddr::from(([127, 0, 0, 1], port)); + + axum_server::bind(addr) + .serve(app.into_make_service()) + .await + .unwrap(); + + async fn handler() -> Html<&'static str> { + Html("Hello, World!") + } + } +} From f8fa37c59443ae68fe160c18b79f2830da65dcf5 Mon Sep 17 00:00:00 2001 From: nazeh Date: Thu, 14 Nov 2024 14:38:30 +0300 Subject: [PATCH 56/84] fix(pkarr): wasm bugs --- Cargo.lock | 583 +++++++++++++++++++++++-------- pkarr/Cargo.toml | 11 +- pkarr/src/base/cache.rs | 4 +- pkarr/src/client/relay.rs | 8 +- pkarr/src/extra/endpoints/mod.rs | 7 +- pkarr/src/lib.rs | 3 +- 6 files changed, 456 insertions(+), 160 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 28f8bb6..ced41a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,9 +28,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.15" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" dependencies = [ "anstyle", "anstyle-parse", @@ -43,43 +43,43 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.8" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anstyle-parse" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.4" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" dependencies = [ "anstyle", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "anyhow" -version = "1.0.89" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" +checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" [[package]] name = "arc-swap" @@ -273,15 +273,15 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.7.2" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" +checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" [[package]] name = "cc" -version = "1.1.28" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e80e3b6a3ab07840e1cae9b0666a63970dc28e8ed5ffbcdacbfc760c281bfc1" +checksum = "fd9de9f2205d5ef3fd67e685b0df337994ddd4495e2a28d185500d0e1edfea47" dependencies = [ "shlex", ] @@ -292,11 +292,17 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "clap" -version = "4.5.19" +version = "4.5.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7be5744db7978a28d9df86a214130d106a89ce49644cbc4e3f0c22c3fba30615" +checksum = "fb3b4b9e5a7c7514dfa52869339ee98b3156b0bfb4e8a77c4ff4babb64b1604f" dependencies = [ "clap_builder", "clap_derive", @@ -304,9 +310,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.19" +version = "4.5.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5fbc17d3ef8278f55b282b2a2e75ae6f6c7d4bb70ed3d0382375104bfafdb4b" +checksum = "b17a95aa67cc7b5ebd32aa5370189aa0d79069ef1c64ce893bd30fb24bff20ec" dependencies = [ "anstream", "anstyle", @@ -328,9 +334,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +checksum = "afb84c814227b90d6895e01398aee0d8033c00e7466aca416fb6a8e0eb19d8a7" [[package]] name = "cobs" @@ -340,9 +346,9 @@ checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" [[package]] name = "colorchoice" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] name = "colored" @@ -378,9 +384,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +checksum = "0ca741a962e1b0bff6d724a1a0958b686406e853bb14061f218562e1896f95e6" dependencies = [ "libc", ] @@ -402,9 +408,9 @@ checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "critical-section" -version = "1.1.3" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f64009896348fc5af4222e9cf7d7d82a95a256c634ebcf61c53e4ea461422242" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" [[package]] name = "crossbeam-queue" @@ -512,6 +518,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "document-features" version = "0.2.10" @@ -575,9 +592,9 @@ checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" [[package]] name = "encoding_rs" -version = "0.8.34" +version = "0.8.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" dependencies = [ "cfg-if", ] @@ -600,9 +617,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" +checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" [[package]] name = "fiat-crypto" @@ -612,9 +629,9 @@ checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" [[package]] name = "flume" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" dependencies = [ "futures-core", "futures-sink", @@ -659,7 +676,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8835f84f38484cc86f110a805655697908257fb9a7af005234060891557198e9" dependencies = [ "nonempty", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -842,9 +859,9 @@ checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" [[package]] name = "hashbrown" -version = "0.15.0" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" [[package]] name = "heapless" @@ -954,9 +971,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" +checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a" dependencies = [ "bytes", "futures-channel", @@ -1009,9 +1026,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b" +checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" dependencies = [ "bytes", "futures-channel", @@ -1026,14 +1043,143 @@ dependencies = [ "tracing", ] +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "idna" -version = "0.5.0" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "icu_normalizer", + "icu_properties", ] [[package]] @@ -1043,7 +1189,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.15.0", + "hashbrown 0.15.1", ] [[package]] @@ -1066,9 +1212,9 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "js-sys" -version = "0.3.70" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" dependencies = [ "wasm-bindgen", ] @@ -1081,9 +1227,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.159" +version = "0.2.162" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" +checksum = "18d287de67fe55fd7e1581fe933d965a5a9477b38e949cfa9f8574ef01506398" [[package]] name = "libredox" @@ -1101,6 +1247,12 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "litemap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" + [[package]] name = "litrs" version = "0.4.1" @@ -1143,7 +1295,7 @@ checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" [[package]] name = "mainline" version = "3.0.1" -source = "git+https://github.com/pubky/mainline#dc656d8c72c0a525e8575798eba987863bf37df2" +source = "git+https://github.com/pubky/mainline#ce935827349341d2130589486a40e6411c8c790e" dependencies = [ "bytes", "crc", @@ -1156,7 +1308,7 @@ dependencies = [ "serde_bencode", "serde_bytes", "sha1_smol", - "thiserror", + "thiserror 1.0.69", "tracing", ] @@ -1210,9 +1362,9 @@ dependencies = [ [[package]] name = "mockito" -version = "1.5.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09b34bd91b9e5c5b06338d392463e1318d683cf82ec3d3af4014609be6e2108d" +checksum = "652cd6d169a36eaf9d1e6bce1a221130439a966d7f27858af66a33a66e9c4ee2" dependencies = [ "assert-json-diff", "bytes", @@ -1303,9 +1455,9 @@ checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "openssl" -version = "0.10.66" +version = "0.10.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1" +checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" dependencies = [ "bitflags", "cfg-if", @@ -1335,9 +1487,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.103" +version = "0.9.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6" +checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" dependencies = [ "cc", "libc", @@ -1434,18 +1586,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.6" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf123a161dde1e524adf36f90bc5d8d3462824a9c43553ad07a8183161189ec" +checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.6" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4502d8515ca9f32f1fb543d987f63d95a14934883db45bdb48060b6b69257f8" +checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", @@ -1454,9 +1606,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" +checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" [[package]] name = "pin-utils" @@ -1493,10 +1645,11 @@ dependencies = [ "serde", "sha1_smol", "simple-dns", - "thiserror", + "thiserror 1.0.69", "tokio", "tracing", "tracing-subscriber", + "wasm-bindgen-futures", ] [[package]] @@ -1516,7 +1669,7 @@ dependencies = [ "pubky-timestamp", "rustls", "serde", - "thiserror", + "thiserror 1.0.69", "tokio", "toml", "tower-http", @@ -1571,9 +1724,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.87" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3e4daa0dcf6feba26f985457cdf104d4b4256fc5a09547140f3631bb076b19a" +checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" dependencies = [ "unicode-ident", ] @@ -1609,9 +1762,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" +checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef" dependencies = [ "bytes", "pin-project-lite", @@ -1620,34 +1773,38 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror", + "thiserror 2.0.3", "tokio", "tracing", ] [[package]] name = "quinn-proto" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" +checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" dependencies = [ "bytes", + "getrandom", "rand", "ring", "rustc-hash", "rustls", + "rustls-pki-types", "slab", - "thiserror", + "thiserror 2.0.3", "tinyvec", "tracing", + "web-time", ] [[package]] name = "quinn-udp" -version = "0.5.5" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fe68c2e9e1a1234e218683dbdf9f9dfcb094113c5ac2b938dfcb9bab4c4140b" +checksum = "7d5a626c6807713b15cac82a6acaccd6043c9a5408c24baae07611fec3f243da" dependencies = [ + "cfg_aliases", "libc", "once_cell", "socket2", @@ -1720,18 +1877,18 @@ checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ "getrandom", "libredox", - "thiserror", + "thiserror 1.0.69", ] [[package]] name = "regex" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.8", + "regex-automata 0.4.9", "regex-syntax 0.8.5", ] @@ -1746,9 +1903,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", @@ -1769,9 +1926,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" -version = "0.12.8" +version = "0.12.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b" +checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" dependencies = [ "base64", "bytes", @@ -1853,9 +2010,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.37" +version = "0.38.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" +checksum = "99e4ea3e1cdc4b559b8e5650f9c8e5998e3e5c1343b4eaf034565f32318d63c0" dependencies = [ "bitflags", "errno", @@ -1866,9 +2023,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.14" +version = "0.23.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "415d9944693cb90382053259f89fbb077ea730ad7273047ec63b19bc9b160ba8" +checksum = "eee87ff5d9b36712a58574e12e9f0ea80f915a5b0ac518d322b24a465617925e" dependencies = [ "once_cell", "ring", @@ -1889,9 +2046,12 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e696e35370c65c9c541198af4543ccd580cf17fc25d8e05c5a242b202488c55" +checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +dependencies = [ + "web-time", +] [[package]] name = "rustls-webpki" @@ -1906,9 +2066,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" +checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" [[package]] name = "ryu" @@ -1946,9 +2106,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.12.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" +checksum = "fa39c7303dc58b5543c94d22c1766b0d31f2ee58306363ea622b10bbc075eaa2" dependencies = [ "core-foundation-sys", "libc", @@ -1968,9 +2128,9 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" -version = "1.0.210" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" dependencies = [ "serde_derive", ] @@ -1996,9 +2156,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.210" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" dependencies = [ "proc-macro2", "quote", @@ -2007,9 +2167,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.128" +version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" dependencies = [ "itoa", "memchr", @@ -2192,9 +2352,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.79" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", @@ -2225,6 +2385,17 @@ dependencies = [ "crossbeam-queue", ] +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "system-configuration" version = "0.6.1" @@ -2248,9 +2419,9 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.13.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" +checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" dependencies = [ "cfg-if", "fastrand", @@ -2261,18 +2432,38 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.64" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c006c85c7651b3cf2ada4584faa36773bd07bac24acfb39f3c431b36d7e667aa" +dependencies = [ + "thiserror-impl 2.0.3", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] name = "thiserror-impl" -version = "1.0.64" +version = "2.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" +checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568" dependencies = [ "proc-macro2", "quote", @@ -2289,6 +2480,16 @@ dependencies = [ "once_cell", ] +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinyvec" version = "1.8.0" @@ -2306,9 +2507,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.40.0" +version = "1.41.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" +checksum = "22cfb5bee7a6a52939ca9224d6ac897bb669134078daa8735560897f69de4d33" dependencies = [ "backtrace", "bytes", @@ -2463,17 +2664,17 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tower_governor" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "313fa625fea5790ed56360a30ea980e41229cf482b4835801a67ef1922bf63b9" +checksum = "aea939ea6cfa7c4880f3e7422616624f97a567c16df67b53b11f0d03917a8e46" dependencies = [ "axum", "forwarded-header-value", "governor", "http", "pin-project", - "thiserror", - "tower 0.4.13", + "thiserror 1.0.69", + "tower 0.5.1", "tracing", ] @@ -2551,27 +2752,12 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" -[[package]] -name = "unicode-bidi" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" - [[package]] name = "unicode-ident" version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" -[[package]] -name = "unicode-normalization" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" -dependencies = [ - "tinyvec", -] - [[package]] name = "untrusted" version = "0.9.0" @@ -2580,15 +2766,27 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.2" +version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" dependencies = [ "form_urlencoded", "idna", "percent-encoding", ] +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -2630,9 +2828,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" dependencies = [ "cfg-if", "once_cell", @@ -2641,9 +2839,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" dependencies = [ "bumpalo", "log", @@ -2656,9 +2854,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.43" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed" +checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b" dependencies = [ "cfg-if", "js-sys", @@ -2668,9 +2866,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2678,9 +2876,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", @@ -2691,15 +2889,25 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" +checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" [[package]] name = "web-sys" -version = "0.3.70" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" +checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" dependencies = [ "js-sys", "wasm-bindgen", @@ -2923,6 +3131,42 @@ dependencies = [ "memchr", ] +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + +[[package]] +name = "yoke" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.7.35" @@ -2944,8 +3188,51 @@ dependencies = [ "syn", ] +[[package]] +name = "zerofrom" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index c0c0b6b..cf34148 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -21,7 +21,7 @@ document-features = "0.2.8" rand = "0.8.5" once_cell = {version = "1.19.0", default-features = false } lru = { version = "0.12.3", default-features = false } -pubky-timestamp = { version = "0.2.0", default-features = false } +flume = { version = "0.11.0", features = ["select", "eventual-fairness", "async"], default-features = false , optional = true } # feat: relay dependencies tokio = { version = "1.40.0", optional = true, default-features = false } @@ -35,9 +35,10 @@ serde = { version = "1.0.209", features = ["derive"], optional = true } futures = { version = "0.3.30", optional = true, default-features = false } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] +pubky-timestamp = { version = "0.2.0", default-features = false } + # feat: dht dependencies mainline = { git = "https://github.com/pubky/mainline", optional = true } -flume = { version = "0.11.0", features = ["select", "eventual-fairness", "async"], default-features = false , optional = true } # feat: relay dependencies reqwest = { version = "0.12.7", features = ["rustls-tls"], optional = true } @@ -48,11 +49,13 @@ byteorder = { version = "1.5.0", default-features = false, optional = true } libc = { version = "0.2.159", optional = true } [target.'cfg(target_arch = "wasm32")'.dependencies] +pubky-timestamp = { version = "0.2.0", default-features = false, features = ["httpdate"] } js-sys = "0.3.69" futures = "0.3.30" getrandom = { version = "0.2", features = ["js"] } reqwest = "0.12.7" sha1_smol = "1.0.1" +wasm-bindgen-futures = "0.4.45" [dev-dependencies] postcard = { version = "1.0.10", features = ["alloc"] } @@ -68,7 +71,7 @@ tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } ## Use [client::dht::Client] dht = ["dep:mainline", "dep:flume"] ## Use [client::relay::Client] -relay = ["dep:reqwest", "dep:tokio", "dep:sha1_smol"] +relay = ["dep:reqwest", "dep:tokio", "dep:sha1_smol", "dep:flume"] ## Derive serde Serialize/Deserialize for PublicKey serde = ["dep:serde", "pubky-timestamp/serde", "pubky-timestamp/httpdate"] @@ -82,7 +85,7 @@ lmdb-cache = ["dep:heed", "dep:byteorder", "dep:libc"] ## Use all features full = ["dht", "relay", "serde", "endpoints", "lmdb-cache"] -default = ["dht"] +default = ["full"] [package.metadata.docs.rs] all-features = true diff --git a/pkarr/src/base/cache.rs b/pkarr/src/base/cache.rs index 039fbd3..2a14a67 100644 --- a/pkarr/src/base/cache.rs +++ b/pkarr/src/base/cache.rs @@ -26,9 +26,9 @@ impl From<&crate::PublicKey> for CacheKey { fn from(public_key: &crate::PublicKey) -> CacheKey { let mut encoded = vec![]; - encoded.extend(public_key); + encoded.extend(public_key.as_bytes()); - let mut hasher = Sha1::new(); + let mut hasher = sha1_smol::Sha1::new(); hasher.update(&encoded); hasher.digest().bytes() } diff --git a/pkarr/src/client/relay.rs b/pkarr/src/client/relay.rs index aa2e27f..529cb5a 100644 --- a/pkarr/src/client/relay.rs +++ b/pkarr/src/client/relay.rs @@ -193,10 +193,16 @@ impl Client { let tx = tx.clone(); let this = self.clone(); + #[cfg(not(target_arch = "wasm32"))] tokio::task::spawn(async move { // If the receiver was dropped.. no harm. let _ = tx.send(this.race_resolve(&pubky, None).await); }); + #[cfg(target_arch = "wasm32")] + wasm_bindgen_futures::spawn_local(async move { + // If the receiver was dropped.. no harm. + let _ = tx.send(this.race_resolve(&pubky, None).await); + }); } if let Some(cached_packet) = cached_packet { @@ -310,7 +316,7 @@ impl Client { Box::pin(async move { this.resolve_from_relay(relay, public_key, cached).await }) }); - let mut result: Result> = Ok(None); + let mut result: Result, reqwest::Error> = Ok(None); match select_ok(futures).await { Ok((Some(signed_packet), _)) => { diff --git a/pkarr/src/extra/endpoints/mod.rs b/pkarr/src/extra/endpoints/mod.rs index 8b6660d..0a092a1 100644 --- a/pkarr/src/extra/endpoints/mod.rs +++ b/pkarr/src/extra/endpoints/mod.rs @@ -3,7 +3,7 @@ // mod async_iter; mod endpoint; -use crate::{Client, PublicKey, SignedPacket}; +use crate::{PublicKey, SignedPacket}; pub use endpoint::Endpoint; @@ -65,7 +65,8 @@ pub trait EndpointResolver { } } -impl EndpointResolver for Client { +#[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] +impl EndpointResolver for crate::client::dht::Client { async fn resolve(&self, public_key: &PublicKey) -> Result, ResolveError> { self.resolve(public_key).await.map_err(|error| match error { crate::client::dht::ClientWasShutdown => ResolveError::ClientWasShutdown, @@ -117,7 +118,7 @@ mod tests { use crate::dns::rdata::{A, SVCB}; use crate::dns::{self, rdata::RData}; use crate::SignedPacket; - use crate::{mainline::Testnet, Keypair}; + use crate::{mainline::Testnet, Client, Keypair}; use std::future::Future; use std::pin::Pin; diff --git a/pkarr/src/lib.rs b/pkarr/src/lib.rs index 0c05711..95be5c7 100644 --- a/pkarr/src/lib.rs +++ b/pkarr/src/lib.rs @@ -25,8 +25,7 @@ pub const DEFAULT_RELAYS: [&str; 2] = ["https://relay.pkarr.org", "https://pkarr pub const DEFAULT_RESOLVERS: [&str; 2] = ["resolver.pkarr.org:6881", "pkarr.pubky.org:6881"]; #[cfg(any(target_arch = "wasm32", feature = "dht"))] -pub use client::Client; -pub use client::Settings; +pub use client::{Client, Settings}; // Rexports pub use bytes; From 3c704a577dac9749b544a9c6f69ca2f64c89b3dd Mon Sep 17 00:00:00 2001 From: nazeh Date: Thu, 14 Nov 2024 17:04:55 +0300 Subject: [PATCH 57/84] test(pkarr): remove bad test --- pkarr/src/extra/reqwest.rs | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/pkarr/src/extra/reqwest.rs b/pkarr/src/extra/reqwest.rs index 35ab320..0229fb7 100644 --- a/pkarr/src/extra/reqwest.rs +++ b/pkarr/src/extra/reqwest.rs @@ -37,29 +37,3 @@ async fn resolve( Ok(Box::new(format!("{name}:0").to_socket_addrs().unwrap())) } - -#[cfg(test)] -mod tests { - use axum::{response::Html, routing::get, Router}; - use rand::Rng; - use std::net::SocketAddr; - - #[tokio::test] - async fn http() { - // Define the route and handler - let app = Router::new().route("/", get(handler)); - - let port: u16 = rand::thread_rng().gen(); - - let addr = SocketAddr::from(([127, 0, 0, 1], port)); - - axum_server::bind(addr) - .serve(app.into_make_service()) - .await - .unwrap(); - - async fn handler() -> Html<&'static str> { - Html("Hello, World!") - } - } -} From 54732ea14bb652e137547dda1ed295ebf1d7aa57 Mon Sep 17 00:00:00 2001 From: nazeh Date: Thu, 14 Nov 2024 17:20:47 +0300 Subject: [PATCH 58/84] fix(pkarr): wasm after merging https endpoints --- Cargo.lock | 1 + pkarr/Cargo.toml | 2 +- pkarr/src/extra/mod.rs | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9d217b5..c867316 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1656,6 +1656,7 @@ dependencies = [ "dyn-clone", "ed25519-dalek", "flume", + "futures", "futures-lite", "genawaiter", "getrandom", diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index 81be40e..80c8245 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -52,7 +52,7 @@ libc = { version = "0.2.159", optional = true } [target.'cfg(target_arch = "wasm32")'.dependencies] pubky-timestamp = { version = "0.2.0", default-features = false, features = ["httpdate"] } js-sys = "0.3.69" -futures-lite = { version = "2.5.0", default-features = false } +futures = "0.3.30" getrandom = { version = "0.2", features = ["js"] } reqwest = "0.12.7" sha1_smol = "1.0.1" diff --git a/pkarr/src/extra/mod.rs b/pkarr/src/extra/mod.rs index 4915391..58373a8 100644 --- a/pkarr/src/extra/mod.rs +++ b/pkarr/src/extra/mod.rs @@ -1,7 +1,7 @@ #[cfg(feature = "endpoints")] pub mod endpoints; -#[cfg(feature = "reqwest-resolve")] +#[cfg(all(not(target_arch = "wasm32"), feature = "reqwest-resolve"))] pub mod reqwest; #[cfg(all(not(target_arch = "wasm32"), feature = "lmdb-cache"))] From eb134b5d04885b85137c869e4f71be3c88470e6d Mon Sep 17 00:00:00 2001 From: nazeh Date: Fri, 15 Nov 2024 15:25:30 +0300 Subject: [PATCH 59/84] feat(pkarr): endpoints resolver uses HTTPS records by default --- pkarr/src/extra/endpoints/mod.rs | 133 ++++---------------------- pkarr/src/extra/endpoints/resolver.rs | 114 ++++++++++++++++++++++ pkarr/src/extra/reqwest.rs | 6 +- 3 files changed, 135 insertions(+), 118 deletions(-) create mode 100644 pkarr/src/extra/endpoints/resolver.rs diff --git a/pkarr/src/extra/endpoints/mod.rs b/pkarr/src/extra/endpoints/mod.rs index bdc5a8e..d48b709 100644 --- a/pkarr/src/extra/endpoints/mod.rs +++ b/pkarr/src/extra/endpoints/mod.rs @@ -1,87 +1,16 @@ -//! EndpointResolver trait for different clients +//! implementation of EndpointResolver trait for different clients mod endpoint; - -use futures_lite::{pin, Stream, StreamExt}; -use genawaiter::sync::Gen; - -use crate::{PublicKey, SignedPacket}; +mod resolver; pub use endpoint::Endpoint; +pub use resolver::EndpointsResolver; +use resolver::ResolveError; -const DEFAULT_MAX_CHAIN_LENGTH: u8 = 3; - -pub trait EndpointResolver { - /// A wrapper around the specific Pkarr client's resolve method. - fn resolve( - &self, - public_key: &PublicKey, - ) -> impl std::future::Future, ResolveError>>; - - /// Returns an async stream of [Endpoint]s - fn resolve_endpoints(&self, qname: &str) -> impl Stream { - Gen::new(|co| async move { - let target = qname; - // TODO: cache the result of this function? - - let is_svcb = target.starts_with('_'); - - let mut step = 0; - let mut svcb: Option = None; - - loop { - let current = svcb - .clone() - .map_or(target.to_string(), |s| s.target().to_string()); - if let Ok(tld) = PublicKey::try_from(current.clone()) { - if let Ok(Some(signed_packet)) = self.resolve(&tld).await { - if step >= DEFAULT_MAX_CHAIN_LENGTH { - break; - }; - step += 1; - - // Choose most prior SVCB record - svcb = Endpoint::find(&signed_packet, ¤t, is_svcb); - - // TODO: support wildcard? - } else { - break; - } - } else { - break; - } - } - - if let Some(svcb) = svcb { - if PublicKey::try_from(svcb.target()).is_err() { - co.yield_(svcb).await - } - } - - // co.yield_(None).await - }) - } - - /// Returns the first [Endpoint] in the Async stream from [EndpointResolver::resolve_endpoints] - fn resolve_endpoint( - &self, - qname: &str, - ) -> impl std::future::Future> { - async { - let stream = self.resolve_endpoints(qname); - - pin!(stream); - - match stream.next().await { - Some(endpoint) => Ok(endpoint), - None => Err(FailedToResolveEndpoint), - } - } - } -} +use crate::{PublicKey, SignedPacket}; #[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] -impl EndpointResolver for crate::client::dht::Client { +impl EndpointsResolver for crate::client::dht::Client { async fn resolve(&self, public_key: &PublicKey) -> Result, ResolveError> { self.resolve(public_key).await.map_err(|error| match error { crate::client::dht::ClientWasShutdown => ResolveError::ClientWasShutdown, @@ -90,7 +19,7 @@ impl EndpointResolver for crate::client::dht::Client { } #[cfg(all(not(target_arch = "wasm32"), feature = "relay"))] -impl EndpointResolver for crate::client::relay::Client { +impl EndpointsResolver for crate::client::relay::Client { async fn resolve(&self, public_key: &PublicKey) -> Result, ResolveError> { self.resolve(public_key) .await @@ -98,45 +27,13 @@ impl EndpointResolver for crate::client::relay::Client { } } -#[derive(thiserror::Error, Debug)] -/// Resolve Error from a client -pub enum ResolveError { - ClientWasShutdown, - #[cfg(any(target_arch = "wasm32", feature = "relay"))] - Reqwest(reqwest::Error), -} - -impl std::fmt::Display for ResolveError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "Resolve endpoint error from the client::resolve {:?}", - self - ) - } -} - -#[derive(Debug)] -pub struct FailedToResolveEndpoint; - -impl std::error::Error for FailedToResolveEndpoint {} - -impl std::fmt::Display for FailedToResolveEndpoint { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "Could not resolve clear net endpoint for the Pkarr domain" - ) - } -} - #[cfg(test)] mod tests { use super::*; use crate::dns::rdata::{A, SVCB}; use crate::dns::{self, rdata::RData}; - use crate::SignedPacket; use crate::{mainline::Testnet, Client, Keypair}; + use crate::{PublicKey, SignedPacket}; use std::future::Future; use std::pin::Pin; @@ -212,7 +109,10 @@ mod tests { let tld = generate(&client, 3, 3, Some("example.com".to_string())).await; - let endpoint = client.resolve_endpoint(&tld.to_string()).await.unwrap(); + let endpoint = client + .resolve_https_endpoint(&tld.to_string()) + .await + .unwrap(); assert_eq!(endpoint.target(), "example.com"); } @@ -224,7 +124,7 @@ mod tests { let pubky = Keypair::random().public_key(); - let endpoint = client.resolve_endpoint(&pubky.to_string()).await; + let endpoint = client.resolve_https_endpoint(&pubky.to_string()).await; assert!(endpoint.is_err()); } @@ -236,7 +136,7 @@ mod tests { let tld = generate(&client, 4, 3, Some("example.com".to_string())).await; - let endpoint = client.resolve_endpoint(&tld.to_string()).await; + let endpoint = client.resolve_https_endpoint(&tld.to_string()).await; assert!(endpoint.is_err()); } @@ -248,7 +148,10 @@ mod tests { let tld = generate(&client, 3, 3, None).await; - let endpoint = client.resolve_endpoint(&tld.to_string()).await.unwrap(); + let endpoint = client + .resolve_https_endpoint(&tld.to_string()) + .await + .unwrap(); assert_eq!(endpoint.target(), "."); assert_eq!(endpoint.port(), Some(3000)); diff --git a/pkarr/src/extra/endpoints/resolver.rs b/pkarr/src/extra/endpoints/resolver.rs new file mode 100644 index 0000000..603225c --- /dev/null +++ b/pkarr/src/extra/endpoints/resolver.rs @@ -0,0 +1,114 @@ +//! EndpointResolver trait + +use futures_lite::{pin, Stream, StreamExt}; +use genawaiter::sync::Gen; + +use crate::{PublicKey, SignedPacket}; + +use super::Endpoint; + +const DEFAULT_MAX_CHAIN_LENGTH: u8 = 3; + +pub trait EndpointsResolver { + /// Returns an async stream of HTTPS [Endpoint]s + fn resolve_https_endpoints(&self, qname: &str) -> impl Stream { + self.resolve_endpoints(qname, false) + } + + /// Returns an async stream of either [HTTPS] or [SVCB] [Endpoint]s + fn resolve_endpoints(&self, qname: &str, is_svcb: bool) -> impl Stream { + Gen::new(|co| async move { + let target = qname; + // TODO: cache the result of this function? + + let mut step = 0; + let mut svcb: Option = None; + + loop { + let current = svcb + .clone() + .map_or(target.to_string(), |s| s.target().to_string()); + if let Ok(tld) = PublicKey::try_from(current.clone()) { + if let Ok(Some(signed_packet)) = self.resolve(&tld).await { + if step >= DEFAULT_MAX_CHAIN_LENGTH { + break; + }; + step += 1; + + // Choose most prior SVCB record + svcb = Endpoint::find(&signed_packet, ¤t, is_svcb); + + // TODO: support wildcard? + } else { + break; + } + } else { + break; + } + } + + if let Some(svcb) = svcb { + if PublicKey::try_from(svcb.target()).is_err() { + co.yield_(svcb).await + } + } + + // co.yield_(None).await + }) + } + + /// Helper method that returns the first `HTTPS` [Endpoint] in the Async stream from [EndpointResolver::resolve_endpoints] + fn resolve_https_endpoint( + &self, + qname: &str, + ) -> impl std::future::Future> { + async move { + let stream = self.resolve_https_endpoints(qname); + + pin!(stream); + + match stream.next().await { + Some(endpoint) => Ok(endpoint), + None => Err(FailedToResolveEndpoint), + } + } + } + + /// A wrapper around the specific Pkarr client's resolve method. + fn resolve( + &self, + public_key: &PublicKey, + ) -> impl std::future::Future, ResolveError>>; +} + +#[derive(thiserror::Error, Debug)] +/// Resolve Error from a client +pub enum ResolveError { + ClientWasShutdown, + #[cfg(any(target_arch = "wasm32", feature = "relay"))] + Reqwest(reqwest::Error), +} + +impl std::fmt::Display for ResolveError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Resolve endpoint error from the client::resolve {:?}", + self + ) + } +} + +#[derive(Debug)] +pub struct FailedToResolveEndpoint; + +impl std::error::Error for FailedToResolveEndpoint {} + +impl std::fmt::Display for FailedToResolveEndpoint { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Could not resolve clear net endpoint for the Pkarr domain" + ) + } +} diff --git a/pkarr/src/extra/reqwest.rs b/pkarr/src/extra/reqwest.rs index 0229fb7..5caed52 100644 --- a/pkarr/src/extra/reqwest.rs +++ b/pkarr/src/extra/reqwest.rs @@ -2,7 +2,7 @@ use reqwest::dns::{Addrs, Resolve}; use crate::{Client, PublicKey}; -use super::endpoints::EndpointResolver; +use super::endpoints::EndpointsResolver; use std::net::ToSocketAddrs; @@ -22,13 +22,13 @@ impl Resolve for crate::client::relay::Client { } async fn resolve( - client: impl EndpointResolver, + client: impl EndpointsResolver, name: reqwest::dns::Name, ) -> Result> { let name = name.as_str(); if PublicKey::try_from(name).is_ok() { - let endpoint = client.resolve_endpoint(name).await?; + let endpoint = client.resolve_https_endpoint(name).await?; let addrs: Addrs = Box::new(endpoint.to_socket_addrs().into_iter()); From 56428b4c27c639486e54e6ed541ff4ef8f5ce563 Mon Sep 17 00:00:00 2001 From: nazeh Date: Sun, 17 Nov 2024 17:57:16 +0300 Subject: [PATCH 60/84] feat(pkarr): update endpoints and test direct resolution --- pkarr/src/extra/endpoints/endpoint.rs | 78 ++++++++++++++-------- pkarr/src/extra/endpoints/mod.rs | 95 ++++++++++++++++++++------- pkarr/src/extra/endpoints/resolver.rs | 23 ++++--- 3 files changed, 133 insertions(+), 63 deletions(-) diff --git a/pkarr/src/extra/endpoints/endpoint.rs b/pkarr/src/extra/endpoints/endpoint.rs index 254cee8..2d3aa71 100644 --- a/pkarr/src/extra/endpoints/endpoint.rs +++ b/pkarr/src/extra/endpoints/endpoint.rs @@ -3,9 +3,12 @@ use crate::{ rdata::{RData, SVCB}, ResourceRecord, }, - SignedPacket, + PublicKey, SignedPacket, +}; +use std::{ + collections::HashSet, + net::{IpAddr, SocketAddr, ToSocketAddrs}, }; -use std::net::{IpAddr, SocketAddr, ToSocketAddrs}; use pubky_timestamp::Timestamp; @@ -13,8 +16,9 @@ use pubky_timestamp::Timestamp; /// An alternative Endpoint for a `qname`, from either [RData::SVCB] or [RData::HTTPS] dns records pub struct Endpoint { target: String, - // public_key: PublicKey, - port: Option, + public_key: PublicKey, + port: u16, + /// SocketAddrs from the [SignedPacket] addrs: Vec, } @@ -57,13 +61,28 @@ impl Endpoint { slice.get(index).map(|s| { let target = s.target.to_string(); - let mut addrs: Vec = vec![]; + let mut addrs = HashSet::new(); + + let port = s + .get_param(SVCB::PORT) + .map(|bytes| { + let mut arr = [0_u8; 2]; + arr[0] = bytes[0]; + arr[1] = bytes[1]; + + u16::from_be_bytes(arr) + }) + .unwrap_or_default(); if &target == "." { for record in signed_packet.resource_records("@") { match &record.rdata { - RData::A(ip) => addrs.push(IpAddr::V4(ip.address.into())), - RData::AAAA(ip) => addrs.push(IpAddr::V6(ip.address.into())), + RData::A(ip) => { + addrs.insert(IpAddr::V4(ip.address.into())); + } + RData::AAAA(ip) => { + addrs.insert(IpAddr::V6(ip.address.into())); + } _ => {} } } @@ -71,35 +90,34 @@ impl Endpoint { Endpoint { target, - // public_key: signed_packet.public_key(), - port: s.get_param(SVCB::PORT).map(|bytes| { - let mut arr = [0_u8; 2]; - arr[0] = bytes[0]; - arr[1] = bytes[1]; - - u16::from_be_bytes(arr) - }), - addrs, + port, + public_key: signed_packet.public_key(), + addrs: addrs.into_iter().collect(), } }) } - /// Return the endpoint target, i.e the domain it points to - /// "." means this endpoint points to its own [Endpoint::public_key] - pub fn target(&self) -> &str { + /// Returns the [SVCB] record's `target` value. + /// + /// Useful in web browsers where we can't use [Self::to_socket_addrs] + pub fn domain(&self) -> &str { &self.target } - pub fn port(&self) -> Option { - self.port + /// Return the [PublicKey] of the [SignedPacket] this endpoint was found at. + /// + /// This is useful as the [PublicKey] of the endpoint (server), and could be + /// used for TLS. + pub fn public_key(&self) -> &PublicKey { + &self.public_key } - /// Return an iterator of [SocketAddr], either by resolving the [Endpoint::target] using normal DNS, + /// Return an iterator of [SocketAddr], either by resolving the [Endpoint::domain] using normal DNS, /// or, if the target is ".", return the [RData::A] or [RData::AAAA] records /// from the endpoint's [SignedPacket], if available. pub fn to_socket_addrs(&self) -> Vec { if self.target == "." { - let port = self.port.unwrap_or(0); + let port = self.port; return self .addrs @@ -111,7 +129,7 @@ impl Endpoint { if cfg!(target_arch = "wasm32") { vec![] } else { - format!("{}:{}", self.target, self.port.unwrap_or(0)) + format!("{}:{}", self.target, self.port) .to_socket_addrs() .map_or(vec![], |v| v.collect::>()) } @@ -149,7 +167,7 @@ mod tests { use crate::{dns, Keypair}; #[tokio::test] - async fn endpoint_target() { + async fn endpoint_domain() { let mut packet = dns::Packet::new_reply(0); packet.answers.push(dns::ResourceRecord::new( dns::Name::new("foo").unwrap(), @@ -184,11 +202,11 @@ mod tests { // Follow foo.tld HTTPS records let endpoint = Endpoint::find(&signed_packet, &format!("foo.{tld}"), false).unwrap(); - assert_eq!(endpoint.target, "https.example.com"); + assert_eq!(endpoint.domain(), "https.example.com"); // Follow _foo.tld SVCB records let endpoint = Endpoint::find(&signed_packet, &format!("_foo.{tld}"), true).unwrap(); - assert_eq!(endpoint.target, "protocol.example.com"); + assert_eq!(endpoint.domain(), "protocol.example.com"); } #[test] @@ -227,9 +245,11 @@ mod tests { ) .unwrap(); - assert_eq!(endpoint.target, "."); + assert_eq!(endpoint.domain(), "."); + + let mut addrs = endpoint.to_socket_addrs(); + addrs.sort(); - let addrs = endpoint.to_socket_addrs(); assert_eq!( addrs.into_iter().map(|s| s.to_string()).collect::>(), vec!["209.151.148.15:6881", "[2a05:d014:275:6201::64]:6881"] diff --git a/pkarr/src/extra/endpoints/mod.rs b/pkarr/src/extra/endpoints/mod.rs index d48b709..ff7801e 100644 --- a/pkarr/src/extra/endpoints/mod.rs +++ b/pkarr/src/extra/endpoints/mod.rs @@ -29,6 +29,8 @@ impl EndpointsResolver for crate::client::relay::Client { #[cfg(test)] mod tests { + use simple_dns::rdata::AAAA; + use super::*; use crate::dns::rdata::{A, SVCB}; use crate::dns::{self, rdata::RData}; @@ -36,13 +38,19 @@ mod tests { use crate::{PublicKey, SignedPacket}; use std::future::Future; + use std::net::IpAddr; use std::pin::Pin; + use std::str::FromStr; + + // TODO: test SVCB too. fn generate_subtree( client: Client, depth: u8, branching: u8, domain: Option, + ips: Vec, + port: Option, ) -> Pin>> { Box::pin(async move { let keypair = Keypair::random(); @@ -54,17 +62,42 @@ mod tests { if depth == 0 { svcb.priority = 1; - svcb.set_port((branching) as u16 * 1000); + + if let Some(port) = port { + svcb.set_port(port); + } if let Some(target) = &domain { let target: &'static str = Box::leak(target.clone().into_boxed_str()); svcb.target = target.try_into().unwrap() } + + for ip in ips.clone() { + packet.answers.push(dns::ResourceRecord::new( + dns::Name::new("@").unwrap(), + dns::CLASS::IN, + 3600, + match ip { + IpAddr::V4(address) => RData::A(A { + address: address.into(), + }), + IpAddr::V6(address) => RData::AAAA(AAAA { + address: address.into(), + }), + }, + )); + } } else { - let target = - generate_subtree(client.clone(), depth - 1, branching, domain.clone()) - .await - .to_string(); + let target = generate_subtree( + client.clone(), + depth - 1, + branching, + domain.clone(), + ips.clone(), + port, + ) + .await + .to_string(); let target: &'static str = Box::leak(target.into_boxed_str()); svcb.target = target.try_into().unwrap(); }; @@ -77,15 +110,6 @@ mod tests { )); } - if depth == 0 { - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("@").unwrap(), - dns::CLASS::IN, - 3600, - RData::A(A { address: 10 }), - )); - } - let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); client.publish(&signed_packet).await.unwrap(); @@ -93,13 +117,34 @@ mod tests { }) } + /// depth of (3): A -> B -> C + /// branch of (2): A -> B0, A -> B1 + /// domain, ips, and port are all at the end (C, or B1) fn generate( client: &Client, depth: u8, branching: u8, domain: Option, + ips: Vec, + port: Option, ) -> Pin>> { - generate_subtree(client.clone(), depth - 1, branching, domain) + generate_subtree(client.clone(), depth - 1, branching, domain, ips, port) + } + + #[tokio::test] + async fn direct_endpoint_resolution() { + let testnet = Testnet::new(3).unwrap(); + let client = Client::builder().testnet(&testnet).build().unwrap(); + + let tld = generate(&client, 1, 1, Some("example.com".to_string()), vec![], None).await; + + let endpoint = client + .resolve_https_endpoint(&tld.to_string()) + .await + .unwrap(); + + assert_eq!(endpoint.domain(), "example.com"); + assert_eq!(endpoint.public_key(), &tld); } #[tokio::test] @@ -107,14 +152,14 @@ mod tests { let testnet = Testnet::new(3).unwrap(); let client = Client::builder().testnet(&testnet).build().unwrap(); - let tld = generate(&client, 3, 3, Some("example.com".to_string())).await; + let tld = generate(&client, 3, 3, Some("example.com".to_string()), vec![], None).await; let endpoint = client .resolve_https_endpoint(&tld.to_string()) .await .unwrap(); - assert_eq!(endpoint.target(), "example.com"); + assert_eq!(endpoint.domain(), "example.com"); } #[tokio::test] @@ -134,7 +179,7 @@ mod tests { let testnet = Testnet::new(3).unwrap(); let client = Client::builder().testnet(&testnet).build().unwrap(); - let tld = generate(&client, 4, 3, Some("example.com".to_string())).await; + let tld = generate(&client, 4, 3, Some("example.com".to_string()), vec![], None).await; let endpoint = client.resolve_https_endpoint(&tld.to_string()).await; @@ -146,15 +191,22 @@ mod tests { let testnet = Testnet::new(3).unwrap(); let client = Client::builder().testnet(&testnet).build().unwrap(); - let tld = generate(&client, 3, 3, None).await; + let tld = generate( + &client, + 3, + 3, + None, + vec![IpAddr::from_str("0.0.0.10").unwrap()], + Some(3000), + ) + .await; let endpoint = client .resolve_https_endpoint(&tld.to_string()) .await .unwrap(); - assert_eq!(endpoint.target(), "."); - assert_eq!(endpoint.port(), Some(3000)); + assert_eq!(endpoint.domain(), "."); assert_eq!( endpoint .to_socket_addrs() @@ -163,6 +215,5 @@ mod tests { .collect::>(), vec!["0.0.0.10:3000"] ); - dbg!(&endpoint); } } diff --git a/pkarr/src/extra/endpoints/resolver.rs b/pkarr/src/extra/endpoints/resolver.rs index 603225c..df9a637 100644 --- a/pkarr/src/extra/endpoints/resolver.rs +++ b/pkarr/src/extra/endpoints/resolver.rs @@ -18,17 +18,15 @@ pub trait EndpointsResolver { /// Returns an async stream of either [HTTPS] or [SVCB] [Endpoint]s fn resolve_endpoints(&self, qname: &str, is_svcb: bool) -> impl Stream { Gen::new(|co| async move { - let target = qname; // TODO: cache the result of this function? let mut step = 0; - let mut svcb: Option = None; + let mut endpoint: Option = None; loop { - let current = svcb - .clone() - .map_or(target.to_string(), |s| s.target().to_string()); - if let Ok(tld) = PublicKey::try_from(current.clone()) { + let current = endpoint.as_ref().map(|s| s.domain()).unwrap_or(qname); + + if let Ok(tld) = PublicKey::try_from(current) { if let Ok(Some(signed_packet)) = self.resolve(&tld).await { if step >= DEFAULT_MAX_CHAIN_LENGTH { break; @@ -36,7 +34,10 @@ pub trait EndpointsResolver { step += 1; // Choose most prior SVCB record - svcb = Endpoint::find(&signed_packet, ¤t, is_svcb); + // TODO: test dns load balancing + // TODO: test failover + + endpoint = Endpoint::find(&signed_packet, current, is_svcb); // TODO: support wildcard? } else { @@ -47,13 +48,11 @@ pub trait EndpointsResolver { } } - if let Some(svcb) = svcb { - if PublicKey::try_from(svcb.target()).is_err() { - co.yield_(svcb).await + if let Some(endpoint) = endpoint { + if PublicKey::try_from(endpoint.domain()).is_err() { + co.yield_(endpoint).await } } - - // co.yield_(None).await }) } From 4fb394c22508adf47647d3c9a026d99c9076ae0b Mon Sep 17 00:00:00 2001 From: nazeh Date: Sun, 17 Nov 2024 20:56:06 +0300 Subject: [PATCH 61/84] feat(pkarr): update endpoints to resolve as DFS --- pkarr/src/extra/endpoints/endpoint.rs | 137 +++++++++++++------------- pkarr/src/extra/endpoints/mod.rs | 2 +- pkarr/src/extra/endpoints/resolver.rs | 51 +++++----- 3 files changed, 93 insertions(+), 97 deletions(-) diff --git a/pkarr/src/extra/endpoints/endpoint.rs b/pkarr/src/extra/endpoints/endpoint.rs index 2d3aa71..c52ec01 100644 --- a/pkarr/src/extra/endpoints/endpoint.rs +++ b/pkarr/src/extra/endpoints/endpoint.rs @@ -10,7 +10,7 @@ use std::{ net::{IpAddr, SocketAddr, ToSocketAddrs}, }; -use pubky_timestamp::Timestamp; +use rand::{seq::SliceRandom, thread_rng}; #[derive(Debug, Clone)] /// An alternative Endpoint for a `qname`, from either [RData::SVCB] or [RData::HTTPS] dns records @@ -23,78 +23,72 @@ pub struct Endpoint { } impl Endpoint { - /// 1. Find the SVCB or HTTPS records with the lowest priority - /// 2. Choose a random one of the list of the above - /// 3. If the target is `.`, check A and AAAA records see [rfc9460](https://www.rfc-editor.org/rfc/rfc9460#name-special-handling-of-in-targ) - pub(crate) fn find( + /// Returns a stack of endpoints from a SignedPacket + /// + /// 1. Find the SVCB or HTTPS records + /// 2. Sort them by priority (reverse) + /// 3. Shuffle records within each priority + /// 3. If the target is `.`, keep track of A and AAAA records see [rfc9460](https://www.rfc-editor.org/rfc/rfc9460#name-special-handling-of-in-targ) + pub(crate) fn parse( signed_packet: &SignedPacket, target: &str, is_svcb: bool, - ) -> Option { - let mut lowest_priority = u16::MAX; - let mut lowest_priority_index = 0; - let mut records = vec![]; - - for record in signed_packet.resource_records(target) { - if let Some(svcb) = get_svcb(record, is_svcb) { - match svcb.priority.cmp(&lowest_priority) { - std::cmp::Ordering::Equal => records.push(svcb), - std::cmp::Ordering::Less => { - lowest_priority_index = records.len(); - lowest_priority = svcb.priority; - records.push(svcb) - } - _ => {} + ) -> Vec { + let mut records = signed_packet + .resource_records(target) + .filter_map(|record| get_svcb(record, is_svcb)) + .collect::>(); + + // TODO: support wildcard? + + // Shuffle the vector first + let mut rng = thread_rng(); + records.shuffle(&mut rng); + // Sort by priority + records.sort_by(|a, b| b.priority.cmp(&a.priority)); + + let mut addrs = HashSet::new(); + for record in signed_packet.resource_records("@") { + match &record.rdata { + RData::A(ip) => { + addrs.insert(IpAddr::V4(ip.address.into())); } + RData::AAAA(ip) => { + addrs.insert(IpAddr::V6(ip.address.into())); + } + _ => {} } } - - // Good enough random selection - let now = Timestamp::now().as_u64(); - let slice = &records[lowest_priority_index..]; - let index = if slice.is_empty() { - 0 - } else { - (now as usize) % slice.len() - }; - - slice.get(index).map(|s| { - let target = s.target.to_string(); - - let mut addrs = HashSet::new(); - - let port = s - .get_param(SVCB::PORT) - .map(|bytes| { - let mut arr = [0_u8; 2]; - arr[0] = bytes[0]; - arr[1] = bytes[1]; - - u16::from_be_bytes(arr) - }) - .unwrap_or_default(); - - if &target == "." { - for record in signed_packet.resource_records("@") { - match &record.rdata { - RData::A(ip) => { - addrs.insert(IpAddr::V4(ip.address.into())); - } - RData::AAAA(ip) => { - addrs.insert(IpAddr::V6(ip.address.into())); - } - _ => {} - } + let addrs = addrs.into_iter().collect::>(); + + records + .into_iter() + .map(|s| { + let target = s.target.to_string(); + + let port = s + .get_param(SVCB::PORT) + .map(|bytes| { + let mut arr = [0_u8; 2]; + arr[0] = bytes[0]; + arr[1] = bytes[1]; + + u16::from_be_bytes(arr) + }) + .unwrap_or_default(); + + Endpoint { + target, + port, + public_key: signed_packet.public_key(), + addrs: if s.target.to_string() == "." { + addrs.clone() + } else { + Vec::with_capacity(0) + }, } - } - - Endpoint { - target, - port, - public_key: signed_packet.public_key(), - addrs: addrs.into_iter().collect(), - } - }) + }) + .collect::>() } /// Returns the [SVCB] record's `target` value. @@ -201,11 +195,15 @@ mod tests { let tld = keypair.public_key(); // Follow foo.tld HTTPS records - let endpoint = Endpoint::find(&signed_packet, &format!("foo.{tld}"), false).unwrap(); + let endpoint = Endpoint::parse(&signed_packet, &format!("foo.{tld}"), false) + .pop() + .unwrap(); assert_eq!(endpoint.domain(), "https.example.com"); // Follow _foo.tld SVCB records - let endpoint = Endpoint::find(&signed_packet, &format!("_foo.{tld}"), true).unwrap(); + let endpoint = Endpoint::parse(&signed_packet, &format!("_foo.{tld}"), true) + .pop() + .unwrap(); assert_eq!(endpoint.domain(), "protocol.example.com"); } @@ -238,11 +236,12 @@ mod tests { let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); // Follow foo.tld HTTPS records - let endpoint = Endpoint::find( + let endpoint = Endpoint::parse( &signed_packet, &signed_packet.public_key().to_string(), false, ) + .pop() .unwrap(); assert_eq!(endpoint.domain(), "."); diff --git a/pkarr/src/extra/endpoints/mod.rs b/pkarr/src/extra/endpoints/mod.rs index ff7801e..6df3f45 100644 --- a/pkarr/src/extra/endpoints/mod.rs +++ b/pkarr/src/extra/endpoints/mod.rs @@ -18,7 +18,7 @@ impl EndpointsResolver for crate::client::dht::Client { } } -#[cfg(all(not(target_arch = "wasm32"), feature = "relay"))] +#[cfg(any(target_arch = "wasm32", feature = "relay"))] impl EndpointsResolver for crate::client::relay::Client { async fn resolve(&self, public_key: &PublicKey) -> Result, ResolveError> { self.resolve(public_key) diff --git a/pkarr/src/extra/endpoints/resolver.rs b/pkarr/src/extra/endpoints/resolver.rs index df9a637..25ea9c2 100644 --- a/pkarr/src/extra/endpoints/resolver.rs +++ b/pkarr/src/extra/endpoints/resolver.rs @@ -19,38 +19,35 @@ pub trait EndpointsResolver { fn resolve_endpoints(&self, qname: &str, is_svcb: bool) -> impl Stream { Gen::new(|co| async move { // TODO: cache the result of this function? + // TODO: test load balancing + // TODO: test failover + // TODO: custom max_chain_length let mut step = 0; - let mut endpoint: Option = None; - - loop { - let current = endpoint.as_ref().map(|s| s.domain()).unwrap_or(qname); - - if let Ok(tld) = PublicKey::try_from(current) { - if let Ok(Some(signed_packet)) = self.resolve(&tld).await { - if step >= DEFAULT_MAX_CHAIN_LENGTH { - break; - }; - step += 1; - - // Choose most prior SVCB record - // TODO: test dns load balancing - // TODO: test failover - - endpoint = Endpoint::find(&signed_packet, current, is_svcb); - - // TODO: support wildcard? - } else { - break; - } - } else { - break; + let mut stack: Vec = Vec::new(); + + // Initialize the stack with endpoints from the starting domain. + if let Ok(tld) = PublicKey::try_from(qname) { + if let Ok(Some(signed_packet)) = self.resolve(&tld).await { + step += 1; + stack.extend(Endpoint::parse(&signed_packet, qname, is_svcb)); } } - if let Some(endpoint) = endpoint { - if PublicKey::try_from(endpoint.domain()).is_err() { - co.yield_(endpoint).await + while let Some(next) = stack.pop() { + let current = next.domain(); + + // Attempt to resolve the domain as a public key. + match PublicKey::try_from(current) { + Ok(tld) => match self.resolve(&tld).await { + Ok(Some(signed_packet)) if step < DEFAULT_MAX_CHAIN_LENGTH => { + step += 1; + stack.extend(Endpoint::parse(&signed_packet, current, is_svcb)); + } + _ => break, // Stop on resolution failure or chain length exceeded. + }, + // Yield if the domain is not pointing to another Pkarr TLD domain. + Err(_) => co.yield_(next).await, } } }) From ce1d6eb72c7de3efb05b00ff8eec33629272a05d Mon Sep 17 00:00:00 2001 From: nazeh Date: Mon, 18 Nov 2024 18:19:55 +0300 Subject: [PATCH 62/84] feat(pkarr): add ClientBuilder::from(pkarr::Client) with 'reqwest-builder' feat --- Cargo.lock | 6 ++- pkarr/Cargo.toml | 18 +++++--- pkarr/examples/http-get.rs | 20 ++++----- pkarr/examples/http-serve.rs | 15 ++++--- pkarr/examples/publish.rs | 12 +++--- pkarr/examples/resolve.rs | 8 ++-- pkarr/src/extra/mod.rs | 23 +++++++++++ pkarr/src/extra/tls.rs | 79 ++++++++++++++++++++++++++++++++++++ 8 files changed, 149 insertions(+), 32 deletions(-) create mode 100644 pkarr/src/extra/tls.rs diff --git a/Cargo.lock b/Cargo.lock index c867316..9c91e3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1646,6 +1646,7 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" name = "pkarr" version = "3.0.0" dependencies = [ + "anyhow", "axum", "axum-server", "base32", @@ -1671,6 +1672,7 @@ dependencies = [ "pubky-timestamp", "rand", "reqwest", + "rustls", "self_cell", "serde", "sha1_smol", @@ -2053,9 +2055,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.16" +version = "0.23.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eee87ff5d9b36712a58574e12e9f0ea80f915a5b0ac518d322b24a465617925e" +checksum = "7f1a745511c54ba6d4465e8d5dfbd81b45791756de28d4981af70d6dca128f1e" dependencies = [ "once_cell", "ring", diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index 80c8245..33e8fce 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -35,6 +35,9 @@ serde = { version = "1.0.209", features = ["derive"], optional = true } futures-lite = { version = "2.5.0", default-features = false, optional = true } genawaiter = { version = "0.99.1", default-features = false, features = ["futures03"] } +# feat: tls +rustls = { version = "0.23", default-features = false, features = ["ring"], optional = true } + [target.'cfg(not(target_arch = "wasm32"))'.dependencies] pubky-timestamp = { version = "0.2.0", default-features = false } @@ -59,6 +62,7 @@ sha1_smol = "1.0.1" wasm-bindgen-futures = "0.4.45" [dev-dependencies] +anyhow = "1.0.93" axum = "0.7.7" axum-server = "0.7.1" postcard = { version = "1.0.10", features = ["alloc"] } @@ -80,15 +84,19 @@ relay = ["dep:reqwest", "dep:tokio", "dep:sha1_smol", "dep:flume"] serde = ["dep:serde", "pubky-timestamp/serde", "pubky-timestamp/httpdate"] # Extra -## Use [extra::endpoints::EndpointResolver] trait -endpoints = ["dep:futures-lite"] -## Implement reqwest::dns::Resolve trait for Pkarr clients -reqwest-resolve = ["dep:reqwest", "endpoints"] ## Use [LmdbCache] lmdb-cache = ["dep:heed", "dep:byteorder", "dep:libc"] +## Use [extra::endpoints::EndpointResolver] trait implementation for [Client] and [client::relay::Client] +endpoints = ["dep:futures-lite"] +## Use [reqwest::dns::Resolve] trait implementation for [Client] and [client::relay::Client] +reqwest-resolve = ["dep:reqwest", "endpoints"] +## Implement +tls = ["rustls"] +## Create a [reqwest::ClientBuilder] from [Client] or [client::relay::Client] +reqwest-builder = ["tls", "reqwest-resolve"] ## Use all features -full = ["dht", "relay", "serde", "endpoints", "lmdb-cache", "reqwest-resolve"] +full = ["dht", "relay", "serde", "endpoints", "lmdb-cache", "reqwest-resolve", "tls", "reqwest-builder"] default = ["full"] diff --git a/pkarr/examples/http-get.rs b/pkarr/examples/http-get.rs index 528b03a..9f97da3 100644 --- a/pkarr/examples/http-get.rs +++ b/pkarr/examples/http-get.rs @@ -1,7 +1,5 @@ //! Make an HTTP request over to a Pkarr address using Reqwest -use std::sync::Arc; - use reqwest::Method; use tracing::Level; use tracing_subscriber; @@ -18,22 +16,22 @@ struct Cli { } #[tokio::main] -async fn main() { +async fn main() -> anyhow::Result<()> { tracing_subscriber::fmt().with_max_level(Level::INFO).init(); let cli = Cli::parse(); let url = cli.url; - let client = Client::builder().build().unwrap(); + let client = Client::builder().build()?; + + let reqwest = reqwest::ClientBuilder::from(client).build()?; - let reqwest = reqwest::Client::builder() - .dns_resolver(Arc::new(client)) - .build() - .unwrap(); + println!("GET {url}.."); + let response = reqwest.request(Method::GET, &url).send().await?; - let response = reqwest.request(Method::GET, &url).send().await.unwrap(); + let body = response.text().await?; - let body = response.text().await.unwrap(); + println!("{body}"); - println!("Resolved {url}\n{body}"); + Ok(()) } diff --git a/pkarr/examples/http-serve.rs b/pkarr/examples/http-serve.rs index a0434fe..77b3614 100644 --- a/pkarr/examples/http-serve.rs +++ b/pkarr/examples/http-serve.rs @@ -27,7 +27,7 @@ struct Cli { } #[tokio::main] -async fn main() { +async fn main() -> anyhow::Result<()> { tracing_subscriber::fmt().with_max_level(Level::INFO).init(); let cli = Cli::parse(); @@ -35,14 +35,17 @@ async fn main() { let app = Router::new().route("/", get(handler)); let addr = format!("{}:{}", cli.ip, cli.port) - .to_socket_addrs() - .unwrap() + .to_socket_addrs()? .next() - .unwrap(); + .ok_or(anyhow::anyhow!( + "Could not convert IP and port to socket addresses" + ))?; publish_server_pkarr(&addr).await; - bind(addr).serve(app.into_make_service()).await.unwrap(); + bind(addr).serve(app.into_make_service()).await?; + + Ok(()) } // Simple handler that responds with "Hello, World!" @@ -80,7 +83,7 @@ async fn publish_server_pkarr(socket_addr: &SocketAddr) { let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); - println!("Server running on http://{}", keypair.public_key()); + println!("Server running on https://{}", keypair.public_key()); client.publish(&signed_packet).await.unwrap(); } diff --git a/pkarr/examples/publish.rs b/pkarr/examples/publish.rs index 38d5312..05e9ee2 100644 --- a/pkarr/examples/publish.rs +++ b/pkarr/examples/publish.rs @@ -19,25 +19,25 @@ use pkarr::client::relay::Client; use pkarr::Client; #[tokio::main] -async fn main() { +async fn main() -> anyhow::Result<()> { tracing_subscriber::fmt() .with_max_level(Level::DEBUG) .with_env_filter("pkarr") .init(); - let client = Client::builder().build().unwrap(); + let client = Client::builder().build()?; let keypair = Keypair::random(); let mut packet = dns::Packet::new_reply(0); packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("_foo").unwrap(), + dns::Name::new("_foo")?, dns::CLASS::IN, 30, - dns::rdata::RData::TXT("bar".try_into().unwrap()), + dns::rdata::RData::TXT("bar".try_into()?), )); - let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); + let signed_packet = SignedPacket::from_packet(&keypair, &packet)?; let instant = Instant::now(); @@ -55,4 +55,6 @@ async fn main() { println!("\nFailed to publish {} \n {}", keypair.public_key(), err); } }; + + Ok(()) } diff --git a/pkarr/examples/resolve.rs b/pkarr/examples/resolve.rs index 05f28b5..9a04972 100644 --- a/pkarr/examples/resolve.rs +++ b/pkarr/examples/resolve.rs @@ -28,7 +28,7 @@ struct Cli { } #[tokio::main] -async fn main() { +async fn main() -> anyhow::Result<()> { tracing_subscriber::fmt() .with_max_level(Level::DEBUG) .with_env_filter("pkarr") @@ -42,7 +42,7 @@ async fn main() { .try_into() .expect("Invalid zbase32 encoded key"); - let client = Client::builder().build().unwrap(); + let client = Client::builder().build()?; println!("Resolving Pkarr: {} ...", cli.public_key); println!("\n=== COLD LOOKUP ==="); @@ -51,8 +51,10 @@ async fn main() { // loop { sleep(Duration::from_secs(1)); println!("=== SUBSEQUENT LOOKUP ==="); - resolve(&client, &public_key).await + resolve(&client, &public_key).await; // } + + Ok(()) } async fn resolve(client: &Client, public_key: &PublicKey) { diff --git a/pkarr/src/extra/mod.rs b/pkarr/src/extra/mod.rs index 58373a8..39f6b7e 100644 --- a/pkarr/src/extra/mod.rs +++ b/pkarr/src/extra/mod.rs @@ -1,8 +1,31 @@ +use std::sync::Arc; + #[cfg(feature = "endpoints")] pub mod endpoints; #[cfg(all(not(target_arch = "wasm32"), feature = "reqwest-resolve"))] pub mod reqwest; +#[cfg(all(not(target_arch = "wasm32"), feature = "tls"))] +pub mod tls; + #[cfg(all(not(target_arch = "wasm32"), feature = "lmdb-cache"))] pub mod lmdb_cache; + +#[cfg(all(not(target_arch = "wasm32"), feature = "reqwest-builder"))] +impl From for ::reqwest::ClientBuilder { + fn from(client: crate::Client) -> Self { + ::reqwest::ClientBuilder::new() + .dns_resolver(Arc::new(client.clone())) + .use_preconfigured_tls(rustls::ClientConfig::from(client)) + } +} + +#[cfg(all(not(target_arch = "wasm32"), feature = "reqwest-builder"))] +impl From for ::reqwest::ClientBuilder { + fn from(client: crate::client::relay::Client) -> Self { + ::reqwest::ClientBuilder::new() + .dns_resolver(Arc::new(client.clone())) + .use_preconfigured_tls(rustls::ClientConfig::from(client)) + } +} diff --git a/pkarr/src/extra/tls.rs b/pkarr/src/extra/tls.rs new file mode 100644 index 0000000..9a41569 --- /dev/null +++ b/pkarr/src/extra/tls.rs @@ -0,0 +1,79 @@ +use std::sync::Arc; + +use rustls::client::danger::ServerCertVerifier; + +use crate::Client; + +#[derive(Debug)] +struct CertVerifier; + +impl ServerCertVerifier for CertVerifier { + fn verify_server_cert( + &self, + _end_entity: &rustls::pki_types::CertificateDer<'_>, + _intermediates: &[rustls::pki_types::CertificateDer<'_>], + _server_name: &rustls::pki_types::ServerName<'_>, + _ocsp_response: &[u8], + _now: rustls::pki_types::UnixTime, + ) -> Result { + Ok(rustls::client::danger::ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + _message: &[u8], + _cert: &rustls::pki_types::CertificateDer<'_>, + _dss: &rustls::DigitallySignedStruct, + ) -> Result { + Ok(rustls::client::danger::HandshakeSignatureValid::assertion()) + } + + fn verify_tls13_signature( + &self, + _message: &[u8], + _cert: &rustls::pki_types::CertificateDer<'_>, + _dss: &rustls::DigitallySignedStruct, + ) -> Result { + Ok(rustls::client::danger::HandshakeSignatureValid::assertion()) + } + + fn supported_verify_schemes(&self) -> Vec { + vec![] + } +} + +impl From for CertVerifier { + fn from(client: Client) -> Self { + CertVerifier + } +} + +#[cfg(all(not(target_arch = "wasm32"), feature = "relay"))] +impl From for CertVerifier { + fn from(client: crate::client::relay::Client) -> Self { + CertVerifier + } +} + +impl From for rustls::ClientConfig { + fn from(client: Client) -> Self { + let verifier: CertVerifier = client.into(); + + rustls::ClientConfig::builder() + .dangerous() + .with_custom_certificate_verifier(Arc::new(verifier)) + .with_no_client_auth() + } +} + +#[cfg(all(not(target_arch = "wasm32"), feature = "relay"))] +impl From for rustls::ClientConfig { + fn from(client: crate::client::relay::Client) -> Self { + let verifier: CertVerifier = client.into(); + + rustls::ClientConfig::builder() + .dangerous() + .with_custom_certificate_verifier(Arc::new(verifier)) + .with_no_client_auth() + } +} From 883691da019d878e5e308abacbbf6f1530302d64 Mon Sep 17 00:00:00 2001 From: nazeh Date: Wed, 20 Nov 2024 09:46:28 +0300 Subject: [PATCH 63/84] feat(pkarr): add PublicKey::to_public_key_der() --- pkarr/Cargo.toml | 2 +- pkarr/src/base/keys.rs | 28 +++++++++++++++++++++++++++- pkarr/src/extra/tls.rs | 37 +++++++++++++++++++++++++++++++------ 3 files changed, 59 insertions(+), 8 deletions(-) diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index 33e8fce..fc9a1a7 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -91,7 +91,7 @@ endpoints = ["dep:futures-lite"] ## Use [reqwest::dns::Resolve] trait implementation for [Client] and [client::relay::Client] reqwest-resolve = ["dep:reqwest", "endpoints"] ## Implement -tls = ["rustls"] +tls = ["rustls", "ed25519-dalek/pkcs8"] ## Create a [reqwest::ClientBuilder] from [Client] or [client::relay::Client] reqwest-builder = ["tls", "reqwest-resolve"] diff --git a/pkarr/src/base/keys.rs b/pkarr/src/base/keys.rs index 8c96d31..de284ef 100644 --- a/pkarr/src/base/keys.rs +++ b/pkarr/src/base/keys.rs @@ -1,5 +1,7 @@ //! Utility structs for Ed25519 keys. +#[cfg(feature = "tls")] +use ed25519_dalek::pkcs8::{Document, EncodePublicKey}; use ed25519_dalek::{ SecretKey, Signature, SignatureError, Signer, SigningKey, Verifier, VerifyingKey, }; @@ -57,7 +59,7 @@ impl Keypair { /// /// It can formatted to and parsed from a z-base32 string. #[derive(Clone, Eq, PartialEq, Hash)] -pub struct PublicKey(VerifyingKey); +pub struct PublicKey(pub(crate) VerifyingKey); impl PublicKey { /// Format the public key as z-base32 string. @@ -89,6 +91,11 @@ impl PublicKey { pub fn as_bytes(&self) -> &[u8; 32] { self.0.as_bytes() } + + #[cfg(feature = "tls")] + pub fn to_public_key_der(&self) -> Document { + self.0.to_public_key_der().expect("to_public_key_der") + } } impl AsRef for Keypair { @@ -481,4 +488,23 @@ mod tests { let public_key: PublicKey = str.try_into().unwrap(); assert_eq!(public_key.verifying_key().as_bytes(), &expected); } + + #[test] + fn pkcs8() { + let str = "yg4gxe7z1r7mr6orids9fh95y7gxhdsxjqi6nngsxxtakqaxr5no"; + let public_key: PublicKey = str.try_into().unwrap(); + + let der = public_key.to_public_key_der(); + + assert_eq!( + der.as_bytes(), + [ + // Algorithm and other stuff. + 48, 42, 48, 5, 6, 3, 43, 101, 112, 3, 33, 0, // + // Key + 1, 180, 103, 163, 183, 145, 58, 178, 122, 4, 168, 237, 242, 243, 251, 7, 76, 254, + 14, 207, 75, 171, 225, 8, 214, 123, 227, 133, 59, 15, 38, 197, + ] + ) + } } diff --git a/pkarr/src/extra/tls.rs b/pkarr/src/extra/tls.rs index 9a41569..59f2f18 100644 --- a/pkarr/src/extra/tls.rs +++ b/pkarr/src/extra/tls.rs @@ -1,8 +1,12 @@ use std::sync::Arc; -use rustls::client::danger::ServerCertVerifier; +use rustls::{ + client::danger::{ServerCertVerified, ServerCertVerifier}, + pki_types::SubjectPublicKeyInfoDer, + CertificateError, +}; -use crate::Client; +use crate::{Client, PublicKey}; #[derive(Debug)] struct CertVerifier; @@ -10,13 +14,29 @@ struct CertVerifier; impl ServerCertVerifier for CertVerifier { fn verify_server_cert( &self, - _end_entity: &rustls::pki_types::CertificateDer<'_>, - _intermediates: &[rustls::pki_types::CertificateDer<'_>], - _server_name: &rustls::pki_types::ServerName<'_>, + endpoint_certificate: &rustls::pki_types::CertificateDer<'_>, + intermediates: &[rustls::pki_types::CertificateDer<'_>], + host_name: &rustls::pki_types::ServerName<'_>, _ocsp_response: &[u8], _now: rustls::pki_types::UnixTime, ) -> Result { - Ok(rustls::client::danger::ServerCertVerified::assertion()) + if !intermediates.is_empty() { + return Err(rustls::Error::InvalidCertificate( + CertificateError::UnknownIssuer, + )); + } + // if self.trusted_spki.is_empty() { + // return Ok(ServerCertVerified::assertion()); + // } + let end_entity_as_spki = SubjectPublicKeyInfoDer::from(endpoint_certificate.as_ref()); + + // TODO: confirm that this end_entity is valid for this server_name. + match true { + true => Ok(ServerCertVerified::assertion()), + false => Err(rustls::Error::InvalidCertificate( + CertificateError::UnknownIssuer, + )), + } } fn verify_tls12_signature( @@ -40,6 +60,11 @@ impl ServerCertVerifier for CertVerifier { fn supported_verify_schemes(&self) -> Vec { vec![] } + + fn requires_raw_public_keys(&self) -> bool { + // TODO: can we support x.509 certificates as well? + true + } } impl From for CertVerifier { From ac4e5d7ca1f2149ab2bc88fda9d9a092b8305743 Mon Sep 17 00:00:00 2001 From: nazeh Date: Wed, 20 Nov 2024 14:03:28 +0300 Subject: [PATCH 64/84] feat(pkarr): add support for webpki for reqwest builder --- Cargo.lock | 217 ++++++++++++++++++++++++++++++++++++++++- pkarr/Cargo.toml | 12 ++- pkarr/src/base/keys.rs | 52 +++++++++- pkarr/src/extra/mod.rs | 90 ++++++++++++++++- pkarr/src/extra/tls.rs | 91 +++++++++++------ 5 files changed, 417 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9c91e3a..d211168 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -129,6 +129,33 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "aws-lc-rs" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe7c2840b66236045acd2607d5866e274380afd87ef99d6226e961e2cb47df45" +dependencies = [ + "aws-lc-sys", + "mirai-annotations", + "paste", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad3a619a9de81e1d7de1f1186dcba4506ed661a0e483d84410fdef0ee87b2f96" +dependencies = [ + "bindgen", + "cc", + "cmake", + "dunce", + "fs_extra", + "libc", + "paste", +] + [[package]] name = "axum" version = "0.7.7" @@ -241,6 +268,29 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn", + "which", +] + [[package]] name = "bitflags" version = "2.6.0" @@ -283,9 +333,20 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd9de9f2205d5ef3fd67e685b0df337994ddd4495e2a28d185500d0e1edfea47" dependencies = [ + "jobserver", + "libc", "shlex", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -298,6 +359,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "4.5.21" @@ -338,6 +410,15 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "afb84c814227b90d6895e01398aee0d8033c00e7466aca416fb6a8e0eb19d8a7" +[[package]] +name = "cmake" +version = "0.1.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb1e43aa7fd152b1f968787f7dbcdeb306d1867ff373c69955211876c053f91a" +dependencies = [ + "cc", +] + [[package]] name = "cobs" version = "0.2.3" @@ -547,6 +628,12 @@ dependencies = [ "phf", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn-clone" version = "1.0.17" @@ -578,6 +665,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + [[package]] name = "embedded-io" version = "0.4.0" @@ -679,6 +772,12 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.31" @@ -829,6 +928,12 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "governor" version = "0.6.3" @@ -949,6 +1054,15 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "http" version = "1.1.0" @@ -997,9 +1111,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a" +checksum = "97818827ef4f364230e16705d4706e2897df2bb60617d6ca15d598025a3c481f" dependencies = [ "bytes", "futures-channel", @@ -1230,12 +1344,30 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "jobserver" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.72" @@ -1251,12 +1383,28 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" version = "0.2.162" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18d287de67fe55fd7e1581fe933d965a5a9477b38e949cfa9f8574ef01506398" +[[package]] +name = "libloading" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" +dependencies = [ + "cfg-if", + "windows-targets 0.48.5", +] + [[package]] name = "libredox" version = "0.1.3" @@ -1365,6 +1513,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.0" @@ -1386,6 +1540,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "mirai-annotations" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1" + [[package]] name = "mockito" version = "1.6.1" @@ -1442,6 +1602,16 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c" +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nonempty" version = "0.7.0" @@ -1562,6 +1732,12 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "percent-encoding" version = "2.3.1" @@ -1679,9 +1855,11 @@ dependencies = [ "simple-dns", "thiserror 1.0.69", "tokio", + "tokio-rustls", "tracing", "tracing-subscriber", "wasm-bindgen-futures", + "webpki-roots", ] [[package]] @@ -1754,6 +1932,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prettyplease" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" +dependencies = [ + "proc-macro2", + "syn", +] + [[package]] name = "proc-macro2" version = "1.0.89" @@ -1802,7 +1990,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash", + "rustc-hash 2.0.0", "rustls", "socket2", "thiserror 2.0.3", @@ -1820,7 +2008,7 @@ dependencies = [ "getrandom", "rand", "ring", - "rustc-hash", + "rustc-hash 2.0.0", "rustls", "rustls-pki-types", "slab", @@ -2025,6 +2213,12 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-hash" version = "2.0.0" @@ -2059,6 +2253,8 @@ version = "0.23.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f1a745511c54ba6d4465e8d5dfbd81b45791756de28d4981af70d6dca128f1e" dependencies = [ + "aws-lc-rs", + "log", "once_cell", "ring", "rustls-pki-types", @@ -2091,6 +2287,7 @@ version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -2954,6 +3151,18 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index fc9a1a7..3565533 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -35,8 +35,8 @@ serde = { version = "1.0.209", features = ["derive"], optional = true } futures-lite = { version = "2.5.0", default-features = false, optional = true } genawaiter = { version = "0.99.1", default-features = false, features = ["futures03"] } -# feat: tls -rustls = { version = "0.23", default-features = false, features = ["ring"], optional = true } +# feat: reqwest-builder +webpki-roots = { version = "0.26.6", optional = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] pubky-timestamp = { version = "0.2.0", default-features = false } @@ -47,6 +47,9 @@ mainline = { git = "https://github.com/pubky/mainline", optional = true } # feat: relay dependencies reqwest = { version = "0.12.7", features = ["rustls-tls"], optional = true } +# feat: tls +rustls = { version = "0.23", default-features = false, features = ["ring"], optional = true } + # feat: lmdb-cache defendencies heed = { version = "0.20.0", default-features = false, optional = true } byteorder = { version = "1.5.0", default-features = false, optional = true } @@ -67,6 +70,7 @@ axum = "0.7.7" axum-server = "0.7.1" postcard = { version = "1.0.10", features = ["alloc"] } tokio = { version = "1.40.0", features = ["macros", "rt-multi-thread"] } +tokio-rustls = "0.26.0" [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] clap = { version = "4.4.8", features = ["derive"] } @@ -90,10 +94,10 @@ lmdb-cache = ["dep:heed", "dep:byteorder", "dep:libc"] endpoints = ["dep:futures-lite"] ## Use [reqwest::dns::Resolve] trait implementation for [Client] and [client::relay::Client] reqwest-resolve = ["dep:reqwest", "endpoints"] -## Implement +## Use [rustls::ClientConfig] from [Client] and [client::relay::Client] for e2ee transport to Pkarr endpoints tls = ["rustls", "ed25519-dalek/pkcs8"] ## Create a [reqwest::ClientBuilder] from [Client] or [client::relay::Client] -reqwest-builder = ["tls", "reqwest-resolve"] +reqwest-builder = ["tls", "reqwest-resolve", "webpki-roots"] ## Use all features full = ["dht", "relay", "serde", "endpoints", "lmdb-cache", "reqwest-resolve", "tls", "reqwest-builder"] diff --git a/pkarr/src/base/keys.rs b/pkarr/src/base/keys.rs index de284ef..b4304d8 100644 --- a/pkarr/src/base/keys.rs +++ b/pkarr/src/base/keys.rs @@ -1,11 +1,13 @@ //! Utility structs for Ed25519 keys. -#[cfg(feature = "tls")] -use ed25519_dalek::pkcs8::{Document, EncodePublicKey}; +#[cfg(all(not(target_arch = "wasm32"), feature = "tls"))] +use ed25519_dalek::pkcs8::{Document, EncodePrivateKey, EncodePublicKey}; use ed25519_dalek::{ SecretKey, Signature, SignatureError, Signer, SigningKey, Verifier, VerifyingKey, }; use rand::rngs::OsRng; +#[cfg(all(not(target_arch = "wasm32"), feature = "tls"))] +use rustls::{crypto::ring::sign::any_eddsa_type, pki_types::CertificateDer, sign::CertifiedKey}; use std::{ fmt::{self, Debug, Display, Formatter}, hash::Hash, @@ -53,6 +55,28 @@ impl Keypair { pub fn to_uri_string(&self) -> String { self.public_key().to_uri_string() } + + #[cfg(all(not(target_arch = "wasm32"), feature = "tls"))] + /// Return a Raw PublicKey certified key according to [RFC 7250](https://tools.ietf.org/html/rfc7250) + /// useful to use with [rustls::ConfigBuilder::with_cert_resolver] and [rustls::server::AlwaysResolvesServerRawPublicKeys] + pub fn to_rpk_certified_key(&self) -> CertifiedKey { + let client_private_key = any_eddsa_type( + &self + .0 + .to_pkcs8_der() + .expect("Keypair::to_rpk_certificate: convert secret key to pkcs8 der") + .as_bytes() + .into(), + ) + .expect("Keypair::to_rpk_certificate: convert KeyPair to rustls SigningKey"); + + let client_public_key = client_private_key + .public_key() + .expect("Keypair::to_rpk_certificate: load SPKI"); + let client_public_key_as_cert = CertificateDer::from(client_public_key.to_vec()); + + CertifiedKey::new(vec![client_public_key_as_cert], client_private_key) + } } /// Ed25519 public key to verify a signature over dns [Packet](crate::SignedPacket)s. @@ -92,7 +116,7 @@ impl PublicKey { self.0.as_bytes() } - #[cfg(feature = "tls")] + #[cfg(all(not(target_arch = "wasm32"), feature = "tls"))] pub fn to_public_key_der(&self) -> Document { self.0.to_public_key_der().expect("to_public_key_der") } @@ -489,6 +513,7 @@ mod tests { assert_eq!(public_key.verifying_key().as_bytes(), &expected); } + #[cfg(all(not(target_arch = "wasm32"), feature = "tls"))] #[test] fn pkcs8() { let str = "yg4gxe7z1r7mr6orids9fh95y7gxhdsxjqi6nngsxxtakqaxr5no"; @@ -507,4 +532,25 @@ mod tests { ] ) } + + #[cfg(all(not(target_arch = "wasm32"), feature = "tls"))] + #[test] + fn certificate() { + use rustls::SignatureAlgorithm; + + let keypair = Keypair::from_secret_key(&[0; 32]); + + let certified_key = keypair.to_rpk_certified_key(); + + assert_eq!(certified_key.key.algorithm(), SignatureAlgorithm::ED25519); + + assert_eq!( + certified_key.end_entity_cert().unwrap().as_ref(), + [ + 48, 42, 48, 5, 6, 3, 43, 101, 112, 3, 33, 0, 59, 106, 39, 188, 206, 182, 164, 45, + 98, 163, 168, 208, 42, 111, 13, 115, 101, 50, 21, 119, 29, 226, 67, 166, 58, 192, + 72, 161, 139, 89, 218, 41, + ] + ) + } } diff --git a/pkarr/src/extra/mod.rs b/pkarr/src/extra/mod.rs index 39f6b7e..04e485e 100644 --- a/pkarr/src/extra/mod.rs +++ b/pkarr/src/extra/mod.rs @@ -1,5 +1,3 @@ -use std::sync::Arc; - #[cfg(feature = "endpoints")] pub mod endpoints; @@ -16,7 +14,7 @@ pub mod lmdb_cache; impl From for ::reqwest::ClientBuilder { fn from(client: crate::Client) -> Self { ::reqwest::ClientBuilder::new() - .dns_resolver(Arc::new(client.clone())) + .dns_resolver(std::sync::Arc::new(client.clone())) .use_preconfigured_tls(rustls::ClientConfig::from(client)) } } @@ -25,7 +23,91 @@ impl From for ::reqwest::ClientBuilder { impl From for ::reqwest::ClientBuilder { fn from(client: crate::client::relay::Client) -> Self { ::reqwest::ClientBuilder::new() - .dns_resolver(Arc::new(client.clone())) + .dns_resolver(std::sync::Arc::new(client.clone())) .use_preconfigured_tls(rustls::ClientConfig::from(client)) } } + +#[cfg(test)] +mod tests { + use rustls::{server::AlwaysResolvesServerRawPublicKeys, ServerConfig}; + use std::sync::Arc; + + use tokio::io::AsyncWriteExt; + use tokio::net::TcpListener; + use tokio_rustls::{rustls, TlsAcceptor}; + + use crate::{Client, Keypair}; + + #[tokio::test] + async fn reqwest_webpki() { + rustls::crypto::ring::default_provider() + .install_default() + .expect("Failed to install rustls crypto provider"); + + // Make sure request can still make request to https://example.com + let pkarr_client = Client::builder().build().unwrap(); + let reqwest = reqwest::ClientBuilder::from(pkarr_client).build().unwrap(); + + let response = reqwest.get("https://example.com").send().await.unwrap(); + + assert_eq!(response.status(), reqwest::StatusCode::OK); + } + + // #[tokio::test] + async fn reqwest_pkarr_domain() { + rustls::crypto::ring::default_provider() + .install_default() + .expect("Failed to install rustls crypto provider"); + + // Server setup + let server_keypair = Keypair::random(); + let cert_resolver = + AlwaysResolvesServerRawPublicKeys::new(server_keypair.to_rpk_certified_key().into()); + + let server_config = ServerConfig::builder() + .with_no_client_auth() + .with_cert_resolver(Arc::new(cert_resolver)); + let acceptor = TlsAcceptor::from(Arc::new(server_config)); + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); // Bind to any available port + let addr = listener.local_addr().unwrap(); + + tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + let acceptor = acceptor.clone(); + + tokio::spawn(async move { + let mut stream = acceptor.accept(stream).await.unwrap(); + + stream + .write_all( + &b"HTTP/1.0 200 ok\r\n\ + Connection: close\r\n\ + Content-length: 12\r\n\ + \r\n\ + Hello world!"[..], + ) + .await + .unwrap(); + + stream.shutdown().await.unwrap(); + }); + } + }); + + // Client setup + let pkarr_client = Client::builder().build().unwrap(); + let reqwest = reqwest::ClientBuilder::from(pkarr_client).build().unwrap(); + + // Make a request + let response = reqwest + .get(format!("https://{}", addr)) + .send() + .await + .unwrap(); + + assert_eq!(response.status(), reqwest::StatusCode::OK); + assert_eq!(response.text().await.unwrap(), "Hello world!"); + } +} diff --git a/pkarr/src/extra/tls.rs b/pkarr/src/extra/tls.rs index 59f2f18..abed3f9 100644 --- a/pkarr/src/extra/tls.rs +++ b/pkarr/src/extra/tls.rs @@ -1,35 +1,53 @@ -use std::sync::Arc; +use std::{fmt::Debug, sync::Arc}; use rustls::{ - client::danger::{ServerCertVerified, ServerCertVerifier}, + client::{ + danger::{ServerCertVerified, ServerCertVerifier}, + ServerCertVerifierBuilder, WebPkiServerVerifier, + }, pki_types::SubjectPublicKeyInfoDer, - CertificateError, + CertificateError, SignatureScheme, }; use crate::{Client, PublicKey}; +use crate::extra::endpoints::EndpointsResolver; + #[derive(Debug)] -struct CertVerifier; +struct CertVerifier { + pkarr_client: T, + webpki: Arc, +} -impl ServerCertVerifier for CertVerifier { +impl ServerCertVerifier for CertVerifier { + /// Verify Pkarr public keys fn verify_server_cert( &self, endpoint_certificate: &rustls::pki_types::CertificateDer<'_>, intermediates: &[rustls::pki_types::CertificateDer<'_>], host_name: &rustls::pki_types::ServerName<'_>, - _ocsp_response: &[u8], - _now: rustls::pki_types::UnixTime, + ocsp_response: &[u8], + now: rustls::pki_types::UnixTime, ) -> Result { + if let Err(_) = PublicKey::try_from(host_name.to_str().as_ref()) { + return self.webpki.verify_server_cert( + endpoint_certificate, + intermediates, + host_name, + ocsp_response, + now, + ); + } + if !intermediates.is_empty() { return Err(rustls::Error::InvalidCertificate( CertificateError::UnknownIssuer, )); } - // if self.trusted_spki.is_empty() { - // return Ok(ServerCertVerified::assertion()); - // } let end_entity_as_spki = SubjectPublicKeyInfoDer::from(endpoint_certificate.as_ref()); + dbg!(host_name, end_entity_as_spki); + // TODO: confirm that this end_entity is valid for this server_name. match true { true => Ok(ServerCertVerified::assertion()), @@ -39,50 +57,63 @@ impl ServerCertVerifier for CertVerifier { } } + /// Same as [WebPkiServerVerifier::verify_tls12_signature] fn verify_tls12_signature( &self, - _message: &[u8], - _cert: &rustls::pki_types::CertificateDer<'_>, - _dss: &rustls::DigitallySignedStruct, + message: &[u8], + cert: &rustls::pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, ) -> Result { - Ok(rustls::client::danger::HandshakeSignatureValid::assertion()) + self.webpki.verify_tls12_signature(message, cert, dss) } + /// Same as [WebPkiServerVerifier::verify_tls13_signature] fn verify_tls13_signature( &self, - _message: &[u8], - _cert: &rustls::pki_types::CertificateDer<'_>, - _dss: &rustls::DigitallySignedStruct, + message: &[u8], + cert: &rustls::pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, ) -> Result { - Ok(rustls::client::danger::HandshakeSignatureValid::assertion()) + self.webpki.verify_tls13_signature(message, cert, dss) } + /// Same as [WebPkiServerVerifier::supported_verify_schemes] fn supported_verify_schemes(&self) -> Vec { - vec![] + self.webpki.supported_verify_schemes() } +} + +impl CertVerifier { + pub(crate) fn new(pkarr_client: T) -> Self { + let mut root_cert_store = rustls::RootCertStore::empty(); + root_cert_store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); + let webpki = WebPkiServerVerifier::builder(root_cert_store.into()) + .build() + .expect("WebPkiServerVerifier build"); - fn requires_raw_public_keys(&self) -> bool { - // TODO: can we support x.509 certificates as well? - true + CertVerifier { + pkarr_client, + webpki, + } } } -impl From for CertVerifier { - fn from(client: Client) -> Self { - CertVerifier +impl From for CertVerifier { + fn from(pkarr_client: Client) -> Self { + CertVerifier::new(pkarr_client) } } #[cfg(all(not(target_arch = "wasm32"), feature = "relay"))] -impl From for CertVerifier { - fn from(client: crate::client::relay::Client) -> Self { - CertVerifier +impl From for CertVerifier { + fn from(pkarr_client: crate::client::relay::Client) -> Self { + CertVerifier::new(pkarr_client) } } impl From for rustls::ClientConfig { fn from(client: Client) -> Self { - let verifier: CertVerifier = client.into(); + let verifier: CertVerifier = client.into(); rustls::ClientConfig::builder() .dangerous() @@ -94,7 +125,7 @@ impl From for rustls::ClientConfig { #[cfg(all(not(target_arch = "wasm32"), feature = "relay"))] impl From for rustls::ClientConfig { fn from(client: crate::client::relay::Client) -> Self { - let verifier: CertVerifier = client.into(); + let verifier: CertVerifier = client.into(); rustls::ClientConfig::builder() .dangerous() From da5055800c348b6d504d55456db1120536d2fd45 Mon Sep 17 00:00:00 2001 From: nazeh Date: Wed, 20 Nov 2024 14:38:50 +0300 Subject: [PATCH 65/84] feat(pkarr): use ring crypto provider in rustls config --- pkarr/src/extra/mod.rs | 8 -------- pkarr/src/extra/tls.rs | 33 +++++++++++++++++++++------------ 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/pkarr/src/extra/mod.rs b/pkarr/src/extra/mod.rs index 04e485e..2cd5d8c 100644 --- a/pkarr/src/extra/mod.rs +++ b/pkarr/src/extra/mod.rs @@ -41,10 +41,6 @@ mod tests { #[tokio::test] async fn reqwest_webpki() { - rustls::crypto::ring::default_provider() - .install_default() - .expect("Failed to install rustls crypto provider"); - // Make sure request can still make request to https://example.com let pkarr_client = Client::builder().build().unwrap(); let reqwest = reqwest::ClientBuilder::from(pkarr_client).build().unwrap(); @@ -56,10 +52,6 @@ mod tests { // #[tokio::test] async fn reqwest_pkarr_domain() { - rustls::crypto::ring::default_provider() - .install_default() - .expect("Failed to install rustls crypto provider"); - // Server setup let server_keypair = Keypair::random(); let cert_resolver = diff --git a/pkarr/src/extra/tls.rs b/pkarr/src/extra/tls.rs index abed3f9..ba424af 100644 --- a/pkarr/src/extra/tls.rs +++ b/pkarr/src/extra/tls.rs @@ -2,11 +2,11 @@ use std::{fmt::Debug, sync::Arc}; use rustls::{ client::{ - danger::{ServerCertVerified, ServerCertVerifier}, - ServerCertVerifierBuilder, WebPkiServerVerifier, + danger::{DangerousClientConfigBuilder, ServerCertVerified, ServerCertVerifier}, + WebPkiServerVerifier, }, pki_types::SubjectPublicKeyInfoDer, - CertificateError, SignatureScheme, + CertificateError, }; use crate::{Client, PublicKey}; @@ -85,11 +85,15 @@ impl ServerCertVerifier for CertVeri impl CertVerifier { pub(crate) fn new(pkarr_client: T) -> Self { - let mut root_cert_store = rustls::RootCertStore::empty(); - root_cert_store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); - let webpki = WebPkiServerVerifier::builder(root_cert_store.into()) - .build() - .expect("WebPkiServerVerifier build"); + let webpki = WebPkiServerVerifier::builder_with_provider( + rustls::RootCertStore { + roots: webpki_roots::TLS_SERVER_ROOTS.to_vec(), + } + .into(), + rustls::crypto::ring::default_provider().into(), + ) + .build() + .expect("WebPkiServerVerifier build"); CertVerifier { pkarr_client, @@ -115,8 +119,7 @@ impl From for rustls::ClientConfig { fn from(client: Client) -> Self { let verifier: CertVerifier = client.into(); - rustls::ClientConfig::builder() - .dangerous() + create_client_config_with_ring() .with_custom_certificate_verifier(Arc::new(verifier)) .with_no_client_auth() } @@ -127,9 +130,15 @@ impl From for rustls::ClientConfig { fn from(client: crate::client::relay::Client) -> Self { let verifier: CertVerifier = client.into(); - rustls::ClientConfig::builder() - .dangerous() + create_client_config_with_ring() .with_custom_certificate_verifier(Arc::new(verifier)) .with_no_client_auth() } } + +fn create_client_config_with_ring() -> DangerousClientConfigBuilder { + rustls::ClientConfig::builder_with_provider(rustls::crypto::ring::default_provider().into()) + .with_safe_default_protocol_versions() + .expect("version supported by ring") + .dangerous() +} From 98c0e4a49332bac7c5370faf4a9a42f63cf8361c Mon Sep 17 00:00:00 2001 From: nazeh Date: Wed, 20 Nov 2024 20:24:07 +0300 Subject: [PATCH 66/84] feat(pkarr): verify RPK tls13 and tls12 --- Cargo.lock | 1 + pkarr/Cargo.toml | 5 +- pkarr/src/extra/mod.rs | 132 ++++++++++++++++++++++++++++------------- pkarr/src/extra/tls.rs | 56 +++++++++++------ 4 files changed, 133 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d211168..cc9dc49 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1849,6 +1849,7 @@ dependencies = [ "rand", "reqwest", "rustls", + "rustls-webpki", "self_cell", "serde", "sha1_smol", diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index 3565533..a9ccd7c 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -49,6 +49,7 @@ reqwest = { version = "0.12.7", features = ["rustls-tls"], optional = true } # feat: tls rustls = { version = "0.23", default-features = false, features = ["ring"], optional = true } +webpki = { package = "rustls-webpki", version = "0.102", optional = true } # feat: lmdb-cache defendencies heed = { version = "0.20.0", default-features = false, optional = true } @@ -67,7 +68,7 @@ wasm-bindgen-futures = "0.4.45" [dev-dependencies] anyhow = "1.0.93" axum = "0.7.7" -axum-server = "0.7.1" +axum-server = { version = "0.7.1", features = ["tls-rustls-no-provider"] } postcard = { version = "1.0.10", features = ["alloc"] } tokio = { version = "1.40.0", features = ["macros", "rt-multi-thread"] } tokio-rustls = "0.26.0" @@ -95,7 +96,7 @@ endpoints = ["dep:futures-lite"] ## Use [reqwest::dns::Resolve] trait implementation for [Client] and [client::relay::Client] reqwest-resolve = ["dep:reqwest", "endpoints"] ## Use [rustls::ClientConfig] from [Client] and [client::relay::Client] for e2ee transport to Pkarr endpoints -tls = ["rustls", "ed25519-dalek/pkcs8"] +tls = ["rustls", "ed25519-dalek/pkcs8", "dep:webpki"] ## Create a [reqwest::ClientBuilder] from [Client] or [client::relay::Client] reqwest-builder = ["tls", "reqwest-resolve", "webpki-roots"] diff --git a/pkarr/src/extra/mod.rs b/pkarr/src/extra/mod.rs index 2cd5d8c..e2a1a06 100644 --- a/pkarr/src/extra/mod.rs +++ b/pkarr/src/extra/mod.rs @@ -30,17 +30,32 @@ impl From for ::reqwest::ClientBuilder { #[cfg(test)] mod tests { + use axum_server::{ + tls_rustls::{RustlsAcceptor, RustlsConfig}, + Server, + }; + use mainline::Testnet; use rustls::{server::AlwaysResolvesServerRawPublicKeys, ServerConfig}; + use std::net::TcpListener; use std::sync::Arc; - use tokio::io::AsyncWriteExt; - use tokio::net::TcpListener; - use tokio_rustls::{rustls, TlsAcceptor}; + use tokio_rustls::rustls; - use crate::{Client, Keypair}; + use axum::{routing::get, Router}; + use std::net::SocketAddr; - #[tokio::test] + use crate::{ + dns::{rdata::SVCB, Packet}, + Client, Keypair, SignedPacket, + }; + + // #[tokio::test] async fn reqwest_webpki() { + // TODO: get this working + tracing_subscriber::fmt() + .with_env_filter("rustls=trace") + .init(); + // Make sure request can still make request to https://example.com let pkarr_client = Client::builder().build().unwrap(); let reqwest = reqwest::ClientBuilder::from(pkarr_client).build().unwrap(); @@ -50,56 +65,89 @@ mod tests { assert_eq!(response.status(), reqwest::StatusCode::OK); } - // #[tokio::test] - async fn reqwest_pkarr_domain() { - // Server setup - let server_keypair = Keypair::random(); + async fn publish_server_pkarr(client: &Client, keypair: &Keypair, socket_addr: &SocketAddr) { + let mut packet = Packet::new_reply(1); + + let mut svcb = SVCB::new(0, ".".try_into().unwrap()); + + svcb.set_port(socket_addr.port()); + + packet.answers.push(crate::dns::ResourceRecord::new( + "@".try_into().unwrap(), + crate::dns::CLASS::IN, + 60 * 60, + crate::dns::rdata::RData::HTTPS(svcb.into()), + )); + + packet.answers.push(crate::dns::ResourceRecord::new( + "@".try_into().unwrap(), + crate::dns::CLASS::IN, + 60 * 60, + match socket_addr.ip() { + std::net::IpAddr::V4(ip) => crate::dns::rdata::RData::A(ip.into()), + std::net::IpAddr::V6(ip) => crate::dns::rdata::RData::AAAA(ip.into()), + }, + )); + + let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); + + client.publish(&signed_packet).await.unwrap(); + } + + fn server_config(keypair: &Keypair) -> ServerConfig { let cert_resolver = - AlwaysResolvesServerRawPublicKeys::new(server_keypair.to_rpk_certified_key().into()); + AlwaysResolvesServerRawPublicKeys::new(keypair.to_rpk_certified_key().into()); - let server_config = ServerConfig::builder() + ServerConfig::builder_with_provider(rustls::crypto::ring::default_provider().into()) + .with_safe_default_protocol_versions() + .expect("version supported by ring") .with_no_client_auth() - .with_cert_resolver(Arc::new(cert_resolver)); - let acceptor = TlsAcceptor::from(Arc::new(server_config)); - - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); // Bind to any available port - let addr = listener.local_addr().unwrap(); - - tokio::spawn(async move { - while let Ok((stream, _)) = listener.accept().await { - let acceptor = acceptor.clone(); - - tokio::spawn(async move { - let mut stream = acceptor.accept(stream).await.unwrap(); - - stream - .write_all( - &b"HTTP/1.0 200 ok\r\n\ - Connection: close\r\n\ - Content-length: 12\r\n\ - \r\n\ - Hello world!"[..], - ) - .await - .unwrap(); - - stream.shutdown().await.unwrap(); - }); - } - }); + .with_cert_resolver(Arc::new(cert_resolver)) + } + + async fn axum_server(testnet: &Testnet, keypair: &Keypair) -> Server { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); // Bind to any available port + let address = listener.local_addr().unwrap(); + + let client = Client::builder().testnet(testnet).build().unwrap(); + publish_server_pkarr(&client, &keypair, &address).await; + + let server_config = server_config(&keypair); + + println!("Server running on https://{}", keypair.public_key()); + + let server = axum_server::from_tcp_rustls( + listener, + RustlsConfig::from_config(Arc::new(server_config)), + ); + + server + } + + #[tokio::test] + async fn reqwest_pkarr_domain() { + let testnet = Testnet::new(3).unwrap(); + + let keypair = Keypair::random(); + + let server = axum_server(&testnet, &keypair).await; + + // Run a server on Pkarr + let app = Router::new().route("/", get(|| async { "Hello, world!" })); + tokio::spawn(server.serve(app.into_make_service())); // Client setup - let pkarr_client = Client::builder().build().unwrap(); + let pkarr_client = Client::builder().testnet(&testnet).build().unwrap(); let reqwest = reqwest::ClientBuilder::from(pkarr_client).build().unwrap(); // Make a request let response = reqwest - .get(format!("https://{}", addr)) + .get(format!("https://{}", keypair.public_key())) .send() .await .unwrap(); assert_eq!(response.status(), reqwest::StatusCode::OK); - assert_eq!(response.text().await.unwrap(), "Hello world!"); + assert_eq!(response.text().await.unwrap(), "Hello, world!"); } } diff --git a/pkarr/src/extra/tls.rs b/pkarr/src/extra/tls.rs index ba424af..c173856 100644 --- a/pkarr/src/extra/tls.rs +++ b/pkarr/src/extra/tls.rs @@ -5,8 +5,9 @@ use rustls::{ danger::{DangerousClientConfigBuilder, ServerCertVerified, ServerCertVerifier}, WebPkiServerVerifier, }, + crypto::{verify_tls13_signature_with_raw_key, WebPkiSupportedAlgorithms}, pki_types::SubjectPublicKeyInfoDer, - CertificateError, + SignatureScheme, }; use crate::{Client, PublicKey}; @@ -19,6 +20,11 @@ struct CertVerifier { webpki: Arc, } +static SUPPORTED_ALGORITHMS: WebPkiSupportedAlgorithms = WebPkiSupportedAlgorithms { + all: &[webpki::ring::ED25519], + mapping: &[(SignatureScheme::ED25519, &[webpki::ring::ED25519])], +}; + impl ServerCertVerifier for CertVerifier { /// Verify Pkarr public keys fn verify_server_cert( @@ -29,7 +35,7 @@ impl ServerCertVerifier for CertVeri ocsp_response: &[u8], now: rustls::pki_types::UnixTime, ) -> Result { - if let Err(_) = PublicKey::try_from(host_name.to_str().as_ref()) { + if PublicKey::try_from(host_name.to_str().as_ref()).is_err() { return self.webpki.verify_server_cert( endpoint_certificate, intermediates, @@ -38,23 +44,18 @@ impl ServerCertVerifier for CertVeri now, ); } - - if !intermediates.is_empty() { - return Err(rustls::Error::InvalidCertificate( - CertificateError::UnknownIssuer, - )); - } let end_entity_as_spki = SubjectPublicKeyInfoDer::from(endpoint_certificate.as_ref()); + // TODO: confirm that this end_entity is valid for this server_name. dbg!(host_name, end_entity_as_spki); - // TODO: confirm that this end_entity is valid for this server_name. - match true { - true => Ok(ServerCertVerified::assertion()), - false => Err(rustls::Error::InvalidCertificate( - CertificateError::UnknownIssuer, - )), - } + Ok(ServerCertVerified::assertion()) + // match true { + // true => Ok(ServerCertVerified::assertion()), + // false => Err(rustls::Error::InvalidCertificate( + // CertificateError::UnknownIssuer, + // )), + // } } /// Same as [WebPkiServerVerifier::verify_tls12_signature] @@ -64,7 +65,15 @@ impl ServerCertVerifier for CertVeri cert: &rustls::pki_types::CertificateDer<'_>, dss: &rustls::DigitallySignedStruct, ) -> Result { - self.webpki.verify_tls12_signature(message, cert, dss) + // TODO: fallback to webpki + // self.webpki.verify_tls12_signature(message, cert, dss) + + verify_tls13_signature_with_raw_key( + message, + &SubjectPublicKeyInfoDer::from(cert.as_ref()), + dss, + &SUPPORTED_ALGORITHMS, + ) } /// Same as [WebPkiServerVerifier::verify_tls13_signature] @@ -74,13 +83,26 @@ impl ServerCertVerifier for CertVeri cert: &rustls::pki_types::CertificateDer<'_>, dss: &rustls::DigitallySignedStruct, ) -> Result { - self.webpki.verify_tls13_signature(message, cert, dss) + // TODO: fallback to webpki + // self.webpki.verify_tls13_signature(message, cert, dss) + + verify_tls13_signature_with_raw_key( + message, + &SubjectPublicKeyInfoDer::from(cert.as_ref()), + dss, + &SUPPORTED_ALGORITHMS, + ) } /// Same as [WebPkiServerVerifier::supported_verify_schemes] fn supported_verify_schemes(&self) -> Vec { self.webpki.supported_verify_schemes() } + + fn requires_raw_public_keys(&self) -> bool { + // TODO: can we change this to false and still work for pkarr domains? + true + } } impl CertVerifier { From b61be24cdba19c545fa49f486e8dd3ae42119e88 Mon Sep 17 00:00:00 2001 From: nazeh Date: Wed, 20 Nov 2024 21:40:36 +0300 Subject: [PATCH 67/84] feat(pkarr): rustls doesn't support x.509 and RPK, use separate Reqwest --- pkarr/examples/http-get.rs | 11 ++++++--- pkarr/src/extra/mod.rs | 16 ------------- pkarr/src/extra/tls.rs | 48 +++++++------------------------------- 3 files changed, 16 insertions(+), 59 deletions(-) diff --git a/pkarr/examples/http-get.rs b/pkarr/examples/http-get.rs index 9f97da3..b806d86 100644 --- a/pkarr/examples/http-get.rs +++ b/pkarr/examples/http-get.rs @@ -6,7 +6,7 @@ use tracing_subscriber; use clap::Parser; -use pkarr::Client; +use pkarr::{Client, PublicKey}; #[derive(Parser)] #[command(author, version, about, long_about = None)] @@ -22,9 +22,14 @@ async fn main() -> anyhow::Result<()> { let cli = Cli::parse(); let url = cli.url; - let client = Client::builder().build()?; + let reqwest = if PublicKey::try_from(url.as_str()).is_err() { + // If it is not a Pkarr domain, use normal Reqwest + reqwest::Client::new() + } else { + let client = Client::builder().build()?; - let reqwest = reqwest::ClientBuilder::from(client).build()?; + reqwest::ClientBuilder::from(client).build()? + }; println!("GET {url}.."); let response = reqwest.request(Method::GET, &url).send().await?; diff --git a/pkarr/src/extra/mod.rs b/pkarr/src/extra/mod.rs index e2a1a06..9cbac44 100644 --- a/pkarr/src/extra/mod.rs +++ b/pkarr/src/extra/mod.rs @@ -49,22 +49,6 @@ mod tests { Client, Keypair, SignedPacket, }; - // #[tokio::test] - async fn reqwest_webpki() { - // TODO: get this working - tracing_subscriber::fmt() - .with_env_filter("rustls=trace") - .init(); - - // Make sure request can still make request to https://example.com - let pkarr_client = Client::builder().build().unwrap(); - let reqwest = reqwest::ClientBuilder::from(pkarr_client).build().unwrap(); - - let response = reqwest.get("https://example.com").send().await.unwrap(); - - assert_eq!(response.status(), reqwest::StatusCode::OK); - } - async fn publish_server_pkarr(client: &Client, keypair: &Keypair, socket_addr: &SocketAddr) { let mut packet = Packet::new_reply(1); diff --git a/pkarr/src/extra/tls.rs b/pkarr/src/extra/tls.rs index c173856..0316ffc 100644 --- a/pkarr/src/extra/tls.rs +++ b/pkarr/src/extra/tls.rs @@ -1,10 +1,7 @@ use std::{fmt::Debug, sync::Arc}; use rustls::{ - client::{ - danger::{DangerousClientConfigBuilder, ServerCertVerified, ServerCertVerifier}, - WebPkiServerVerifier, - }, + client::danger::{DangerousClientConfigBuilder, ServerCertVerified, ServerCertVerifier}, crypto::{verify_tls13_signature_with_raw_key, WebPkiSupportedAlgorithms}, pki_types::SubjectPublicKeyInfoDer, SignatureScheme, @@ -15,10 +12,7 @@ use crate::{Client, PublicKey}; use crate::extra::endpoints::EndpointsResolver; #[derive(Debug)] -struct CertVerifier { - pkarr_client: T, - webpki: Arc, -} +struct CertVerifier(T); static SUPPORTED_ALGORITHMS: WebPkiSupportedAlgorithms = WebPkiSupportedAlgorithms { all: &[webpki::ring::ED25519], @@ -30,19 +24,13 @@ impl ServerCertVerifier for CertVeri fn verify_server_cert( &self, endpoint_certificate: &rustls::pki_types::CertificateDer<'_>, - intermediates: &[rustls::pki_types::CertificateDer<'_>], + _intermediates: &[rustls::pki_types::CertificateDer<'_>], host_name: &rustls::pki_types::ServerName<'_>, - ocsp_response: &[u8], - now: rustls::pki_types::UnixTime, + _ocsp_response: &[u8], + _now: rustls::pki_types::UnixTime, ) -> Result { if PublicKey::try_from(host_name.to_str().as_ref()).is_err() { - return self.webpki.verify_server_cert( - endpoint_certificate, - intermediates, - host_name, - ocsp_response, - now, - ); + // TODO: appropriate error. } let end_entity_as_spki = SubjectPublicKeyInfoDer::from(endpoint_certificate.as_ref()); @@ -65,9 +53,6 @@ impl ServerCertVerifier for CertVeri cert: &rustls::pki_types::CertificateDer<'_>, dss: &rustls::DigitallySignedStruct, ) -> Result { - // TODO: fallback to webpki - // self.webpki.verify_tls12_signature(message, cert, dss) - verify_tls13_signature_with_raw_key( message, &SubjectPublicKeyInfoDer::from(cert.as_ref()), @@ -83,9 +68,6 @@ impl ServerCertVerifier for CertVeri cert: &rustls::pki_types::CertificateDer<'_>, dss: &rustls::DigitallySignedStruct, ) -> Result { - // TODO: fallback to webpki - // self.webpki.verify_tls13_signature(message, cert, dss) - verify_tls13_signature_with_raw_key( message, &SubjectPublicKeyInfoDer::from(cert.as_ref()), @@ -96,31 +78,17 @@ impl ServerCertVerifier for CertVeri /// Same as [WebPkiServerVerifier::supported_verify_schemes] fn supported_verify_schemes(&self) -> Vec { - self.webpki.supported_verify_schemes() + vec![SignatureScheme::ED25519] } fn requires_raw_public_keys(&self) -> bool { - // TODO: can we change this to false and still work for pkarr domains? true } } impl CertVerifier { pub(crate) fn new(pkarr_client: T) -> Self { - let webpki = WebPkiServerVerifier::builder_with_provider( - rustls::RootCertStore { - roots: webpki_roots::TLS_SERVER_ROOTS.to_vec(), - } - .into(), - rustls::crypto::ring::default_provider().into(), - ) - .build() - .expect("WebPkiServerVerifier build"); - - CertVerifier { - pkarr_client, - webpki, - } + CertVerifier(pkarr_client) } } From e9cd179825b5a197307ead210d16fbe483d31cd7 Mon Sep 17 00:00:00 2001 From: nazeh Date: Wed, 20 Nov 2024 22:30:30 +0300 Subject: [PATCH 68/84] feat(pkarr): add Keypair::to_rpk_rustls_server_config() --- pkarr/examples/http-serve.rs | 59 ++++++++++++------------------ pkarr/src/base/keys.rs | 36 +++++++++++++++++- pkarr/src/extra/mod.rs | 71 +++++++++++++++--------------------- pkarr/src/extra/tls.rs | 12 +++++- 4 files changed, 96 insertions(+), 82 deletions(-) diff --git a/pkarr/examples/http-serve.rs b/pkarr/examples/http-serve.rs index 77b3614..9831012 100644 --- a/pkarr/examples/http-serve.rs +++ b/pkarr/examples/http-serve.rs @@ -6,61 +6,50 @@ use tracing::Level; use tracing_subscriber; -use axum::{response::Html, routing::get, Router}; -use axum_server::bind; -use std::net::{SocketAddr, ToSocketAddrs}; +use axum::{routing::get, Router}; +use axum_server::tls_rustls::RustlsConfig; -use clap::Parser; +use std::net::{SocketAddr, TcpListener}; +use std::sync::Arc; use pkarr::{ dns::{rdata::SVCB, Packet}, Client, Keypair, SignedPacket, }; -#[derive(Parser)] -#[command(author, version, about, long_about = None)] -struct Cli { - /// IP address to listen on - ip: String, - /// Port number to listen no - port: u16, -} - #[tokio::main] async fn main() -> anyhow::Result<()> { tracing_subscriber::fmt().with_max_level(Level::INFO).init(); - let cli = Cli::parse(); + let keypair = Keypair::random(); - let app = Router::new().route("/", get(handler)); + // Run a server on Pkarr + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); // Bind to any available port + let address = listener.local_addr()?; + println!("Server listening on {address}"); - let addr = format!("{}:{}", cli.ip, cli.port) - .to_socket_addrs()? - .next() - .ok_or(anyhow::anyhow!( - "Could not convert IP and port to socket addresses" - ))?; + let client = Client::builder().build()?; + // You should republish this every time the socket address change + // and once an hour otherwise. + publish_server_pkarr(&client, &keypair, &address).await; - publish_server_pkarr(&addr).await; + println!("Server running on https://{}", keypair.public_key()); - bind(addr).serve(app.into_make_service()).await?; + let server = axum_server::from_tcp_rustls( + listener, + RustlsConfig::from_config(Arc::new(keypair.to_rpk_rustls_server_config())), + ); - Ok(()) -} + let app = Router::new().route("/", get(|| async { "Hello, world!" })); + server.serve(app.into_make_service()).await?; -// Simple handler that responds with "Hello, World!" -async fn handler() -> Html<&'static str> { - Html("Hello, World!") + Ok(()) } -async fn publish_server_pkarr(socket_addr: &SocketAddr) { - let client = Client::builder().build().unwrap(); - - let keypair = Keypair::random(); - +async fn publish_server_pkarr(client: &Client, keypair: &Keypair, socket_addr: &SocketAddr) { let mut packet = Packet::new_reply(1); - let mut svcb = SVCB::new(0, ".".try_into().unwrap()); + let mut svcb = SVCB::new(0, ".".try_into().expect("infallible")); svcb.set_port(socket_addr.port()); @@ -83,7 +72,5 @@ async fn publish_server_pkarr(socket_addr: &SocketAddr) { let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); - println!("Server running on https://{}", keypair.public_key()); - client.publish(&signed_packet).await.unwrap(); } diff --git a/pkarr/src/base/keys.rs b/pkarr/src/base/keys.rs index b4304d8..6e95b3a 100644 --- a/pkarr/src/base/keys.rs +++ b/pkarr/src/base/keys.rs @@ -7,7 +7,10 @@ use ed25519_dalek::{ }; use rand::rngs::OsRng; #[cfg(all(not(target_arch = "wasm32"), feature = "tls"))] -use rustls::{crypto::ring::sign::any_eddsa_type, pki_types::CertificateDer, sign::CertifiedKey}; +use rustls::{ + crypto::ring::sign::any_eddsa_type, pki_types::CertificateDer, + server::AlwaysResolvesServerRawPublicKeys, sign::CertifiedKey, ServerConfig, +}; use std::{ fmt::{self, Debug, Display, Formatter}, hash::Hash, @@ -57,7 +60,7 @@ impl Keypair { } #[cfg(all(not(target_arch = "wasm32"), feature = "tls"))] - /// Return a Raw PublicKey certified key according to [RFC 7250](https://tools.ietf.org/html/rfc7250) + /// Return a RawPublicKey certified key according to [RFC 7250](https://tools.ietf.org/html/rfc7250) /// useful to use with [rustls::ConfigBuilder::with_cert_resolver] and [rustls::server::AlwaysResolvesServerRawPublicKeys] pub fn to_rpk_certified_key(&self) -> CertifiedKey { let client_private_key = any_eddsa_type( @@ -77,6 +80,35 @@ impl Keypair { CertifiedKey::new(vec![client_public_key_as_cert], client_private_key) } + + #[cfg(all(not(target_arch = "wasm32"), feature = "tls"))] + /// Create a [rustls::ServerConfig] using this keypair as a RawPublicKey certificate according to [RFC 7250](https://tools.ietf.org/html/rfc7250) + pub fn to_rpk_rustls_server_config(&self) -> ServerConfig { + let cert_resolver = + AlwaysResolvesServerRawPublicKeys::new(self.to_rpk_certified_key().into()); + + ServerConfig::builder_with_provider(rustls::crypto::ring::default_provider().into()) + .with_safe_default_protocol_versions() + .expect("version supported by ring") + .with_no_client_auth() + .with_cert_resolver(std::sync::Arc::new(cert_resolver)) + } +} + +#[cfg(all(not(target_arch = "wasm32"), feature = "tls"))] +impl From for ServerConfig { + /// calls [Keypair::to_rpk_rustls_server_config] + fn from(keypair: Keypair) -> Self { + keypair.to_rpk_rustls_server_config() + } +} + +#[cfg(all(not(target_arch = "wasm32"), feature = "tls"))] +impl From<&Keypair> for ServerConfig { + /// calls [Keypair::to_rpk_rustls_server_config] + fn from(keypair: &Keypair) -> Self { + keypair.to_rpk_rustls_server_config() + } } /// Ed25519 public key to verify a signature over dns [Packet](crate::SignedPacket)s. diff --git a/pkarr/src/extra/mod.rs b/pkarr/src/extra/mod.rs index 9cbac44..5556736 100644 --- a/pkarr/src/extra/mod.rs +++ b/pkarr/src/extra/mod.rs @@ -12,6 +12,11 @@ pub mod lmdb_cache; #[cfg(all(not(target_arch = "wasm32"), feature = "reqwest-builder"))] impl From for ::reqwest::ClientBuilder { + /// Create a [reqwest::ClientBuilder][::reqwest::ClientBuilder] from this Pkarr client, + /// using it as a [dns_resolver][::reqwest::ClientBuilder::dns_resolver], + /// and a [preconfigured_tls][::reqwest::ClientBuilder::use_preconfigured_tls] client + /// config that uses [rustls::crypto::ring::default_provider()] and follows the + /// [tls for pkarr domains](https://pkarr.org/tls) spec. fn from(client: crate::Client) -> Self { ::reqwest::ClientBuilder::new() .dns_resolver(std::sync::Arc::new(client.clone())) @@ -21,6 +26,11 @@ impl From for ::reqwest::ClientBuilder { #[cfg(all(not(target_arch = "wasm32"), feature = "reqwest-builder"))] impl From for ::reqwest::ClientBuilder { + /// Create a [reqwest::ClientBuilder][::reqwest::ClientBuilder] from this Pkarr client, + /// using it as a [dns_resolver][::reqwest::ClientBuilder::dns_resolver], + /// and a [preconfigured_tls][::reqwest::ClientBuilder::use_preconfigured_tls] client + /// config that uses [rustls::crypto::ring::default_provider()] and follows the + /// [tls for pkarr domains](https://pkarr.org/tls) spec. fn from(client: crate::client::relay::Client) -> Self { ::reqwest::ClientBuilder::new() .dns_resolver(std::sync::Arc::new(client.clone())) @@ -30,19 +40,13 @@ impl From for ::reqwest::ClientBuilder { #[cfg(test)] mod tests { - use axum_server::{ - tls_rustls::{RustlsAcceptor, RustlsConfig}, - Server, - }; use mainline::Testnet; - use rustls::{server::AlwaysResolvesServerRawPublicKeys, ServerConfig}; + use std::net::SocketAddr; use std::net::TcpListener; use std::sync::Arc; - use tokio_rustls::rustls; - use axum::{routing::get, Router}; - use std::net::SocketAddr; + use axum_server::tls_rustls::RustlsConfig; use crate::{ dns::{rdata::SVCB, Packet}, @@ -78,47 +82,30 @@ mod tests { client.publish(&signed_packet).await.unwrap(); } - fn server_config(keypair: &Keypair) -> ServerConfig { - let cert_resolver = - AlwaysResolvesServerRawPublicKeys::new(keypair.to_rpk_certified_key().into()); - - ServerConfig::builder_with_provider(rustls::crypto::ring::default_provider().into()) - .with_safe_default_protocol_versions() - .expect("version supported by ring") - .with_no_client_auth() - .with_cert_resolver(Arc::new(cert_resolver)) - } - - async fn axum_server(testnet: &Testnet, keypair: &Keypair) -> Server { - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); // Bind to any available port - let address = listener.local_addr().unwrap(); - - let client = Client::builder().testnet(testnet).build().unwrap(); - publish_server_pkarr(&client, &keypair, &address).await; - - let server_config = server_config(&keypair); - - println!("Server running on https://{}", keypair.public_key()); - - let server = axum_server::from_tcp_rustls( - listener, - RustlsConfig::from_config(Arc::new(server_config)), - ); - - server - } - #[tokio::test] async fn reqwest_pkarr_domain() { let testnet = Testnet::new(3).unwrap(); let keypair = Keypair::random(); - let server = axum_server(&testnet, &keypair).await; + { + // Run a server on Pkarr + let app = Router::new().route("/", get(|| async { "Hello, world!" })); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); // Bind to any available port + let address = listener.local_addr().unwrap(); + + let client = Client::builder().testnet(&testnet).build().unwrap(); + publish_server_pkarr(&client, &keypair, &address).await; + + println!("Server running on https://{}", keypair.public_key()); + + let server = axum_server::from_tcp_rustls( + listener, + RustlsConfig::from_config(Arc::new((&keypair).into())), + ); - // Run a server on Pkarr - let app = Router::new().route("/", get(|| async { "Hello, world!" })); - tokio::spawn(server.serve(app.into_make_service())); + tokio::spawn(server.serve(app.into_make_service())); + } // Client setup let pkarr_client = Client::builder().testnet(&testnet).build().unwrap(); diff --git a/pkarr/src/extra/tls.rs b/pkarr/src/extra/tls.rs index 0316ffc..a38f315 100644 --- a/pkarr/src/extra/tls.rs +++ b/pkarr/src/extra/tls.rs @@ -12,7 +12,7 @@ use crate::{Client, PublicKey}; use crate::extra::endpoints::EndpointsResolver; #[derive(Debug)] -struct CertVerifier(T); +pub struct CertVerifier(T); static SUPPORTED_ALGORITHMS: WebPkiSupportedAlgorithms = WebPkiSupportedAlgorithms { all: &[webpki::ring::ED25519], @@ -35,7 +35,7 @@ impl ServerCertVerifier for CertVeri let end_entity_as_spki = SubjectPublicKeyInfoDer::from(endpoint_certificate.as_ref()); // TODO: confirm that this end_entity is valid for this server_name. - dbg!(host_name, end_entity_as_spki); + // dbg!(host_name, end_entity_as_spki); Ok(ServerCertVerified::assertion()) // match true { @@ -106,6 +106,10 @@ impl From for CertVerifier for rustls::ClientConfig { + /// Creates a [rustls::ClientConfig] that uses [rustls::crypto::ring::default_provider()] + /// and no client auth and follows the [tls for pkarr domains](https://pkarr.org/tls) spec. + /// + /// If you want more control, create a [CertVerifier] from this [Client] to use as a [custom certificate verifier][DangerousClientConfigBuilder::with_custom_certificate_verifier]. fn from(client: Client) -> Self { let verifier: CertVerifier = client.into(); @@ -117,6 +121,10 @@ impl From for rustls::ClientConfig { #[cfg(all(not(target_arch = "wasm32"), feature = "relay"))] impl From for rustls::ClientConfig { + /// Creates a [rustls::ClientConfig] that uses [rustls::crypto::ring::default_provider()] + /// and no client auth and follows the [tls for pkarr domains](https://pkarr.org/tls) spec. + /// + /// If you want more control, create a [CertVerifier] from this [Client] to use as a [custom certificate verifier][DangerousClientConfigBuilder::with_custom_certificate_verifier]. fn from(client: crate::client::relay::Client) -> Self { let verifier: CertVerifier = client.into(); From 74e11eafe736079bacea576e13b85baa0f07f152 Mon Sep 17 00:00:00 2001 From: nazeh Date: Thu, 21 Nov 2024 13:50:56 +0300 Subject: [PATCH 69/84] feat(pkarr): implement verify_server_cert correctly --- Cargo.lock | 11 +++- pkarr/Cargo.toml | 8 +-- pkarr/src/extra/endpoints/resolver.rs | 72 +++++++++++++++++---------- pkarr/src/extra/tls.rs | 54 +++++++++++++------- 4 files changed, 98 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cc9dc49..62c7722 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -832,7 +832,10 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cef40d21ae2c515b51041df9ed313ed21e572df340ea58a922a0aefe7e8891a1" dependencies = [ + "fastrand", "futures-core", + "futures-io", + "parking", "pin-project-lite", ] @@ -1402,7 +1405,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" dependencies = [ "cfg-if", - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] @@ -1709,6 +1712,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.12.3" diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index a9ccd7c..7c0f199 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -32,8 +32,8 @@ sha1_smol = { version = "1.0.1", optional = true } serde = { version = "1.0.209", features = ["derive"], optional = true } # feat: endpoints dependencies -futures-lite = { version = "2.5.0", default-features = false, optional = true } -genawaiter = { version = "0.99.1", default-features = false, features = ["futures03"] } +futures-lite = { version = "2.5.0", default-features = false, features= ["std"], optional = true } +genawaiter = { version = "0.99.1", default-features = false, features = ["futures03"], optional = true } # feat: reqwest-builder webpki-roots = { version = "0.26.6", optional = true } @@ -92,13 +92,13 @@ serde = ["dep:serde", "pubky-timestamp/serde", "pubky-timestamp/httpdate"] ## Use [LmdbCache] lmdb-cache = ["dep:heed", "dep:byteorder", "dep:libc"] ## Use [extra::endpoints::EndpointResolver] trait implementation for [Client] and [client::relay::Client] -endpoints = ["dep:futures-lite"] +endpoints = ["dep:futures-lite", "dep:genawaiter"] ## Use [reqwest::dns::Resolve] trait implementation for [Client] and [client::relay::Client] reqwest-resolve = ["dep:reqwest", "endpoints"] ## Use [rustls::ClientConfig] from [Client] and [client::relay::Client] for e2ee transport to Pkarr endpoints tls = ["rustls", "ed25519-dalek/pkcs8", "dep:webpki"] ## Create a [reqwest::ClientBuilder] from [Client] or [client::relay::Client] -reqwest-builder = ["tls", "reqwest-resolve", "webpki-roots"] +reqwest-builder = ["tls", "reqwest-resolve"] ## Use all features full = ["dht", "relay", "serde", "endpoints", "lmdb-cache", "reqwest-resolve", "tls", "reqwest-builder"] diff --git a/pkarr/src/extra/endpoints/resolver.rs b/pkarr/src/extra/endpoints/resolver.rs index 25ea9c2..8b16e91 100644 --- a/pkarr/src/extra/endpoints/resolver.rs +++ b/pkarr/src/extra/endpoints/resolver.rs @@ -10,12 +10,57 @@ use super::Endpoint; const DEFAULT_MAX_CHAIN_LENGTH: u8 = 3; pub trait EndpointsResolver { - /// Returns an async stream of HTTPS [Endpoint]s + /// Returns an async stream of [HTTPS][crate::dns::rdata::RData::HTTPS] [Endpoint]s fn resolve_https_endpoints(&self, qname: &str) -> impl Stream { self.resolve_endpoints(qname, false) } - /// Returns an async stream of either [HTTPS] or [SVCB] [Endpoint]s + /// Returns an async stream of [SVCB][crate::dns::rdata::RData::SVCB] [Endpoint]s + fn resolve_svcb_endpoints(&self, qname: &str) -> impl Stream { + self.resolve_endpoints(qname, true) + } + + /// Helper method that returns the first [HTTPS][crate::dns::rdata::RData::HTTPS] [Endpoint] in the Async stream from [EndpointsResolver::resolve_https_endpoints] + fn resolve_https_endpoint( + &self, + qname: &str, + ) -> impl std::future::Future> { + async move { + let stream = self.resolve_https_endpoints(qname); + + pin!(stream); + + match stream.next().await { + Some(endpoint) => Ok(endpoint), + None => Err(FailedToResolveEndpoint), + } + } + } + + /// Helper method that returns the first [SVCB][crate::dns::rdata::RData::SVCB] [Endpoint] in the Async stream from [EndpointsResolver::resolve_svcb_endpoints] + fn resolve_svcb_endpoint( + &self, + qname: &str, + ) -> impl std::future::Future> { + async move { + let stream = self.resolve_https_endpoints(qname); + + pin!(stream); + + match stream.next().await { + Some(endpoint) => Ok(endpoint), + None => Err(FailedToResolveEndpoint), + } + } + } + + /// A wrapper around the specific Pkarr client's resolve method. + fn resolve( + &self, + public_key: &PublicKey, + ) -> impl std::future::Future, ResolveError>>; + + /// Returns an async stream of either [HTTPS][crate::dns::rdata::RData::HTTPS] or [SVCB][crate::dns::rdata::RData::SVCB] [Endpoint]s fn resolve_endpoints(&self, qname: &str, is_svcb: bool) -> impl Stream { Gen::new(|co| async move { // TODO: cache the result of this function? @@ -52,29 +97,6 @@ pub trait EndpointsResolver { } }) } - - /// Helper method that returns the first `HTTPS` [Endpoint] in the Async stream from [EndpointResolver::resolve_endpoints] - fn resolve_https_endpoint( - &self, - qname: &str, - ) -> impl std::future::Future> { - async move { - let stream = self.resolve_https_endpoints(qname); - - pin!(stream); - - match stream.next().await { - Some(endpoint) => Ok(endpoint), - None => Err(FailedToResolveEndpoint), - } - } - } - - /// A wrapper around the specific Pkarr client's resolve method. - fn resolve( - &self, - public_key: &PublicKey, - ) -> impl std::future::Future, ResolveError>>; } #[derive(thiserror::Error, Debug)] diff --git a/pkarr/src/extra/tls.rs b/pkarr/src/extra/tls.rs index a38f315..7602277 100644 --- a/pkarr/src/extra/tls.rs +++ b/pkarr/src/extra/tls.rs @@ -1,49 +1,69 @@ use std::{fmt::Debug, sync::Arc}; +use futures_lite::{pin, stream::block_on}; use rustls::{ client::danger::{DangerousClientConfigBuilder, ServerCertVerified, ServerCertVerifier}, crypto::{verify_tls13_signature_with_raw_key, WebPkiSupportedAlgorithms}, pki_types::SubjectPublicKeyInfoDer, - SignatureScheme, + CertificateError, SignatureScheme, }; -use crate::{Client, PublicKey}; +use crate::Client; use crate::extra::endpoints::EndpointsResolver; #[derive(Debug)] -pub struct CertVerifier(T); +pub struct CertVerifier(T); static SUPPORTED_ALGORITHMS: WebPkiSupportedAlgorithms = WebPkiSupportedAlgorithms { all: &[webpki::ring::ED25519], mapping: &[(SignatureScheme::ED25519, &[webpki::ring::ED25519])], }; -impl ServerCertVerifier for CertVerifier { +impl ServerCertVerifier for CertVerifier { /// Verify Pkarr public keys fn verify_server_cert( &self, endpoint_certificate: &rustls::pki_types::CertificateDer<'_>, - _intermediates: &[rustls::pki_types::CertificateDer<'_>], + intermediates: &[rustls::pki_types::CertificateDer<'_>], host_name: &rustls::pki_types::ServerName<'_>, _ocsp_response: &[u8], _now: rustls::pki_types::UnixTime, ) -> Result { - if PublicKey::try_from(host_name.to_str().as_ref()).is_err() { - // TODO: appropriate error. + if !intermediates.is_empty() { + return Err(rustls::Error::InvalidCertificate( + CertificateError::UnknownIssuer, + )); } + let end_entity_as_spki = SubjectPublicKeyInfoDer::from(endpoint_certificate.as_ref()); + let expected_spki = end_entity_as_spki.as_ref(); + + let qname = host_name.to_str(); + + // Resolve HTTPS endpoints and hope that the cached SignedPackets didn't chance + // since the last time we resolved endpoints to establish the connection in the + // first place. + let stream = self.0.resolve_https_endpoints(&qname); + pin!(stream); + for endpoint in block_on(stream) { + if endpoint.public_key().to_public_key_der().as_bytes() == expected_spki { + return Ok(ServerCertVerified::assertion()); + } + } - // TODO: confirm that this end_entity is valid for this server_name. - // dbg!(host_name, end_entity_as_spki); + // Repeat for SVCB endpoints + let stream = self.0.resolve_svcb_endpoints(&qname); + pin!(stream); + for endpoint in block_on(stream) { + if endpoint.public_key().to_public_key_der().as_bytes() == expected_spki { + return Ok(ServerCertVerified::assertion()); + } + } - Ok(ServerCertVerified::assertion()) - // match true { - // true => Ok(ServerCertVerified::assertion()), - // false => Err(rustls::Error::InvalidCertificate( - // CertificateError::UnknownIssuer, - // )), - // } + Err(rustls::Error::InvalidCertificate( + CertificateError::UnknownIssuer, + )) } /// Same as [WebPkiServerVerifier::verify_tls12_signature] @@ -86,7 +106,7 @@ impl ServerCertVerifier for CertVeri } } -impl CertVerifier { +impl CertVerifier { pub(crate) fn new(pkarr_client: T) -> Self { CertVerifier(pkarr_client) } From d4d012617323acf3b95f181b8257eb41913d6f60 Mon Sep 17 00:00:00 2001 From: nazeh Date: Thu, 21 Nov 2024 14:42:47 +0300 Subject: [PATCH 70/84] docs: update docs and changelog --- CHANGELOG.md | 4 ++++ pkarr/Cargo.toml | 4 ++-- pkarr/examples/README.md | 4 ++-- pkarr/src/client/dht.rs | 2 +- pkarr/src/extra/tls.rs | 7 ++++--- pkarr/src/lib.rs | 2 ++ server/src/config.rs | 8 ++++---- 7 files changed, 19 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64138d2..5ba4cf3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,10 @@ All notable changes to pkarr client and server will be documented in this file. - Derive `serde::Serialize` and `serde::Deserialize` for `SignedPacket`. - Add `pkarr::LmdbCache` for persistent cache using lmdb. - Add `pkarr.pubky.org` as an extra default Relay and Resolver. +- Add feature `endpoints` to resolve `HTTPS` and `SVCB` endpoints over Pkarr +- Add feature `reqwest-resolve` to create a custom `reqwest::dns::Resolve` implementation from `Client` and `relay::client::Client` +- Add feature `tls` to create `rustls::ClientConfig` from `Client` and `relay::client::Client` and create `rustls::ServerCongif` from `KeyPair`. +- Add feature `reqwest-builder` to create a `reqwest::ClientBuilder` from `Client` and `relay::client::Client` using custom dns resolver and preconfigured rustls client config. ### Changed diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index 7c0f199..bc67fd1 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -89,9 +89,9 @@ relay = ["dep:reqwest", "dep:tokio", "dep:sha1_smol", "dep:flume"] serde = ["dep:serde", "pubky-timestamp/serde", "pubky-timestamp/httpdate"] # Extra -## Use [LmdbCache] +## Use [crate::extra::lmdb-cache::LmdbCache] lmdb-cache = ["dep:heed", "dep:byteorder", "dep:libc"] -## Use [extra::endpoints::EndpointResolver] trait implementation for [Client] and [client::relay::Client] +## Use [extra::endpoints::EndpointsResolver] trait implementation for [Client] and [client::relay::Client] endpoints = ["dep:futures-lite", "dep:genawaiter"] ## Use [reqwest::dns::Resolve] trait implementation for [Client] and [client::relay::Client] reqwest-resolve = ["dep:reqwest", "endpoints"] diff --git a/pkarr/examples/README.md b/pkarr/examples/README.md index 766443f..d76a062 100644 --- a/pkarr/examples/README.md +++ b/pkarr/examples/README.md @@ -29,10 +29,10 @@ cargo run --features relay --example resolve +cargo run --features endpoints --example http-serve ``` -An HTTP url will be printend with the Pkarr key as the TLD, paste in another terminal window: +An HTTPS url will be printend with the Pkarr key as the TLD, paste in another terminal window: ```sh cargo run --features reqwest-resolve --example http-get diff --git a/pkarr/src/client/dht.rs b/pkarr/src/client/dht.rs index e64cff5..c0d3a0a 100644 --- a/pkarr/src/client/dht.rs +++ b/pkarr/src/client/dht.rs @@ -112,7 +112,7 @@ impl Settings { self } - /// Set [Settings::dht] + /// Set [Settings::dht_settings] pub fn dht_settings(mut self, settings: mainline::Settings) -> Self { self.dht_settings = settings; self diff --git a/pkarr/src/extra/tls.rs b/pkarr/src/extra/tls.rs index 7602277..bcf0bbc 100644 --- a/pkarr/src/extra/tls.rs +++ b/pkarr/src/extra/tls.rs @@ -66,7 +66,8 @@ impl ServerCertVerifier for )) } - /// Same as [WebPkiServerVerifier::verify_tls12_signature] + /// Verify a message signature using a raw public key and the first TLS 1.3 compatible + /// supported scheme. fn verify_tls12_signature( &self, message: &[u8], @@ -81,7 +82,8 @@ impl ServerCertVerifier for ) } - /// Same as [WebPkiServerVerifier::verify_tls13_signature] + /// Verify a message signature using a raw public key and the first TLS 1.3 compatible + /// supported scheme. fn verify_tls13_signature( &self, message: &[u8], @@ -96,7 +98,6 @@ impl ServerCertVerifier for ) } - /// Same as [WebPkiServerVerifier::supported_verify_schemes] fn supported_verify_schemes(&self) -> Vec { vec![SignatureScheme::ED25519] } diff --git a/pkarr/src/lib.rs b/pkarr/src/lib.rs index 95be5c7..87f69dd 100644 --- a/pkarr/src/lib.rs +++ b/pkarr/src/lib.rs @@ -24,6 +24,8 @@ pub const DEFAULT_RELAYS: [&str; 2] = ["https://relay.pkarr.org", "https://pkarr /// Default [resolver](https://pkarr.org/resolvers)s pub const DEFAULT_RESOLVERS: [&str; 2] = ["resolver.pkarr.org:6881", "pkarr.pubky.org:6881"]; +#[cfg(all(not(target_arch = "wasm32"), feature = "dht"))] +pub use client::dht::Info; #[cfg(any(target_arch = "wasm32", feature = "dht"))] pub use client::{Client, Settings}; diff --git a/server/src/config.rs b/server/src/config.rs index 8b8a5cf..410dbee 100644 --- a/server/src/config.rs +++ b/server/src/config.rs @@ -27,17 +27,17 @@ pub struct Config { /// /// Defaults to a directory in the OS data directory cache_path: Option, - /// See [pkarr::client::dht::Settings::cache_size] + /// See [pkarr::client::Settings::cache_size] cache_size: Option, /// Resolvers /// /// Other servers to query in parallel with the Dht queries /// - /// See [pkarr::client::dht::Settings::resolvers] + /// See [pkarr::client::Settings::resolvers] resolvers: Option>, - /// See [pkarr::client::dht::Settings::minimum_ttl] + /// See [pkarr::Settings::minimum_ttl] minimum_ttl: Option, - /// See [pkarr::client::dht::Settings::maximum_ttl] + /// See [pkarr::Settings::maximum_ttl] maximum_ttl: Option, rate_limiter: RateLimiterConfig, } From 12d41a8715d4c5bcb84843b176a823354eac4648 Mon Sep 17 00:00:00 2001 From: nazeh Date: Thu, 21 Nov 2024 15:55:26 +0300 Subject: [PATCH 71/84] example(pkarr): update http-serve example --- pkarr/examples/README.md | 4 ++-- pkarr/examples/http-serve.rs | 35 +++++++++++++++++++++++++++-------- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/pkarr/examples/README.md b/pkarr/examples/README.md index d76a062..d56f4a2 100644 --- a/pkarr/examples/README.md +++ b/pkarr/examples/README.md @@ -29,10 +29,10 @@ cargo run --features relay --example resolve ``` -An HTTPS url will be printend with the Pkarr key as the TLD, paste in another terminal window: +An HTTPs url will be printend with the Pkarr key as the TLD, paste in another terminal window: ```sh cargo run --features reqwest-resolve --example http-get diff --git a/pkarr/examples/http-serve.rs b/pkarr/examples/http-serve.rs index 9831012..17946f5 100644 --- a/pkarr/examples/http-serve.rs +++ b/pkarr/examples/http-serve.rs @@ -9,34 +9,53 @@ use tracing_subscriber; use axum::{routing::get, Router}; use axum_server::tls_rustls::RustlsConfig; -use std::net::{SocketAddr, TcpListener}; +use std::net::{SocketAddr, ToSocketAddrs}; use std::sync::Arc; +use clap::Parser; + use pkarr::{ dns::{rdata::SVCB, Packet}, Client, Keypair, SignedPacket, }; +#[derive(Parser)] +#[command(author, version, about, long_about = None)] +struct Cli { + /// IP address to listen on + ip: String, + /// Port number to listen no + port: u16, +} + #[tokio::main] async fn main() -> anyhow::Result<()> { tracing_subscriber::fmt().with_max_level(Level::INFO).init(); + let cli = Cli::parse(); + + let addr = format!("{}:{}", cli.ip, cli.port) + .to_socket_addrs()? + .next() + .ok_or(anyhow::anyhow!( + "Could not convert IP and port to socket addresses" + ))?; + let keypair = Keypair::random(); + let client = Client::builder().build()?; + // Run a server on Pkarr - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); // Bind to any available port - let address = listener.local_addr()?; - println!("Server listening on {address}"); + println!("Server listening on {addr}"); - let client = Client::builder().build()?; // You should republish this every time the socket address change // and once an hour otherwise. - publish_server_pkarr(&client, &keypair, &address).await; + publish_server_pkarr(&client, &keypair, &addr).await; println!("Server running on https://{}", keypair.public_key()); - let server = axum_server::from_tcp_rustls( - listener, + let server = axum_server::bind_rustls( + addr, RustlsConfig::from_config(Arc::new(keypair.to_rpk_rustls_server_config())), ); From d4555c7d6ff2fd3bfcf4c4528b396edf3abfc780 Mon Sep 17 00:00:00 2001 From: nazeh Date: Tue, 26 Nov 2024 09:05:28 +0300 Subject: [PATCH 72/84] feat(pkarr): add tracing to endpoint resolver --- pkarr/src/extra/endpoints/resolver.rs | 19 +++++++++++++------ pkarr/src/extra/reqwest.rs | 6 ++++-- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/pkarr/src/extra/endpoints/resolver.rs b/pkarr/src/extra/endpoints/resolver.rs index 8b16e91..b076aa4 100644 --- a/pkarr/src/extra/endpoints/resolver.rs +++ b/pkarr/src/extra/endpoints/resolver.rs @@ -32,7 +32,10 @@ pub trait EndpointsResolver { match stream.next().await { Some(endpoint) => Ok(endpoint), - None => Err(FailedToResolveEndpoint), + None => { + tracing::debug!(?qname, "failed to resolve endpoint"); + Err(FailedToResolveEndpoint) + } } } } @@ -68,13 +71,13 @@ pub trait EndpointsResolver { // TODO: test failover // TODO: custom max_chain_length - let mut step = 0; + let mut depth = 0; let mut stack: Vec = Vec::new(); // Initialize the stack with endpoints from the starting domain. if let Ok(tld) = PublicKey::try_from(qname) { if let Ok(Some(signed_packet)) = self.resolve(&tld).await { - step += 1; + depth += 1; stack.extend(Endpoint::parse(&signed_packet, qname, is_svcb)); } } @@ -85,9 +88,13 @@ pub trait EndpointsResolver { // Attempt to resolve the domain as a public key. match PublicKey::try_from(current) { Ok(tld) => match self.resolve(&tld).await { - Ok(Some(signed_packet)) if step < DEFAULT_MAX_CHAIN_LENGTH => { - step += 1; - stack.extend(Endpoint::parse(&signed_packet, current, is_svcb)); + Ok(Some(signed_packet)) if depth < DEFAULT_MAX_CHAIN_LENGTH => { + depth += 1; + let endpoints = Endpoint::parse(&signed_packet, current, is_svcb); + + tracing::trace!(?qname, ?depth, ?endpoints, "resolved endpoints"); + + stack.extend(endpoints); } _ => break, // Stop on resolution failure or chain length exceeded. }, diff --git a/pkarr/src/extra/reqwest.rs b/pkarr/src/extra/reqwest.rs index 5caed52..f96f978 100644 --- a/pkarr/src/extra/reqwest.rs +++ b/pkarr/src/extra/reqwest.rs @@ -30,9 +30,11 @@ async fn resolve( if PublicKey::try_from(name).is_ok() { let endpoint = client.resolve_https_endpoint(name).await?; - let addrs: Addrs = Box::new(endpoint.to_socket_addrs().into_iter()); + let addrs = endpoint.to_socket_addrs().into_iter(); - return Ok(addrs); + tracing::debug!(?name, ?endpoint, ?addrs, "Resolved an endpoint"); + + return Ok(Box::new(addrs.into_iter())); }; Ok(Box::new(format!("{name}:0").to_socket_addrs().unwrap())) From 6d6e35eddfdf48ab03bb3a35d085983d8cf16b06 Mon Sep 17 00:00:00 2001 From: nazeh Date: Tue, 26 Nov 2024 09:28:05 +0300 Subject: [PATCH 73/84] feat(pkarr): instrument TLS server certificate verifier --- pkarr/src/extra/tls.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkarr/src/extra/tls.rs b/pkarr/src/extra/tls.rs index bcf0bbc..3ed28be 100644 --- a/pkarr/src/extra/tls.rs +++ b/pkarr/src/extra/tls.rs @@ -7,6 +7,7 @@ use rustls::{ pki_types::SubjectPublicKeyInfoDer, CertificateError, SignatureScheme, }; +use tracing::{instrument, Level}; use crate::Client; @@ -21,6 +22,7 @@ static SUPPORTED_ALGORITHMS: WebPkiSupportedAlgorithms = WebPkiSupportedAlgorith }; impl ServerCertVerifier for CertVerifier { + #[instrument(ret(level = Level::DEBUG), err(level = Level::DEBUG))] /// Verify Pkarr public keys fn verify_server_cert( &self, @@ -66,6 +68,7 @@ impl ServerCertVerifier for )) } + #[instrument(ret(level = Level::DEBUG), err(level = Level::DEBUG))] /// Verify a message signature using a raw public key and the first TLS 1.3 compatible /// supported scheme. fn verify_tls12_signature( @@ -82,6 +85,7 @@ impl ServerCertVerifier for ) } + #[instrument(ret(level = Level::DEBUG), err(level = Level::DEBUG))] /// Verify a message signature using a raw public key and the first TLS 1.3 compatible /// supported scheme. fn verify_tls13_signature( From 45e905d273982697e2be19ed738b390fc98a2195 Mon Sep 17 00:00:00 2001 From: nazeh Date: Tue, 26 Nov 2024 12:03:36 +0300 Subject: [PATCH 74/84] feat(pkarr): change tracing::debug! to tracing::trace --- pkarr/src/extra/endpoints/resolver.rs | 2 +- pkarr/src/extra/reqwest.rs | 2 +- pkarr/src/extra/tls.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkarr/src/extra/endpoints/resolver.rs b/pkarr/src/extra/endpoints/resolver.rs index b076aa4..d0e2d76 100644 --- a/pkarr/src/extra/endpoints/resolver.rs +++ b/pkarr/src/extra/endpoints/resolver.rs @@ -33,7 +33,7 @@ pub trait EndpointsResolver { match stream.next().await { Some(endpoint) => Ok(endpoint), None => { - tracing::debug!(?qname, "failed to resolve endpoint"); + tracing::trace!(?qname, "failed to resolve endpoint"); Err(FailedToResolveEndpoint) } } diff --git a/pkarr/src/extra/reqwest.rs b/pkarr/src/extra/reqwest.rs index f96f978..575df79 100644 --- a/pkarr/src/extra/reqwest.rs +++ b/pkarr/src/extra/reqwest.rs @@ -32,7 +32,7 @@ async fn resolve( let addrs = endpoint.to_socket_addrs().into_iter(); - tracing::debug!(?name, ?endpoint, ?addrs, "Resolved an endpoint"); + tracing::trace!(?name, ?endpoint, ?addrs, "Resolved an endpoint"); return Ok(Box::new(addrs.into_iter())); }; diff --git a/pkarr/src/extra/tls.rs b/pkarr/src/extra/tls.rs index 3ed28be..3300134 100644 --- a/pkarr/src/extra/tls.rs +++ b/pkarr/src/extra/tls.rs @@ -22,7 +22,7 @@ static SUPPORTED_ALGORITHMS: WebPkiSupportedAlgorithms = WebPkiSupportedAlgorith }; impl ServerCertVerifier for CertVerifier { - #[instrument(ret(level = Level::DEBUG), err(level = Level::DEBUG))] + #[instrument(ret(level = Level::TRACE), err(level = Level::TRACE))] /// Verify Pkarr public keys fn verify_server_cert( &self, From e5592c52c62202c4efe7d81bc1ac07e451114aba Mon Sep 17 00:00:00 2001 From: nazeh Date: Wed, 27 Nov 2024 14:15:30 +0300 Subject: [PATCH 75/84] feat(pkarr): add Endpoint::port() getter --- pkarr/src/extra/endpoints/endpoint.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkarr/src/extra/endpoints/endpoint.rs b/pkarr/src/extra/endpoints/endpoint.rs index c52ec01..35fd764 100644 --- a/pkarr/src/extra/endpoints/endpoint.rs +++ b/pkarr/src/extra/endpoints/endpoint.rs @@ -98,6 +98,10 @@ impl Endpoint { &self.target } + pub fn port(&self) -> u16 { + self.port + } + /// Return the [PublicKey] of the [SignedPacket] this endpoint was found at. /// /// This is useful as the [PublicKey] of the endpoint (server), and could be From 164ab86f60330940fef6aba51fef3c27250cc5ba Mon Sep 17 00:00:00 2001 From: nazeh Date: Fri, 29 Nov 2024 20:55:48 +0300 Subject: [PATCH 76/84] chore: update mainline --- Cargo.lock | 11 ++++++----- pkarr/Cargo.toml | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 62c7722..f57db45 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -323,9 +323,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" +checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" [[package]] name = "cc" @@ -1471,8 +1471,9 @@ checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" [[package]] name = "mainline" -version = "3.0.1" -source = "git+https://github.com/pubky/mainline#ce935827349341d2130589486a40e6411c8c790e" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e18c8b0210572062a02c4de8c448865f4ca89824c4ac7da64a0c2669ea2c405" dependencies = [ "bytes", "crc", @@ -1485,7 +1486,7 @@ dependencies = [ "serde_bencode", "serde_bytes", "sha1_smol", - "thiserror 1.0.69", + "thiserror 2.0.3", "tracing", ] diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index bc67fd1..b840a6c 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -42,7 +42,7 @@ webpki-roots = { version = "0.26.6", optional = true } pubky-timestamp = { version = "0.2.0", default-features = false } # feat: dht dependencies -mainline = { git = "https://github.com/pubky/mainline", optional = true } +mainline = { version = "4.1.0", optional = true } # feat: relay dependencies reqwest = { version = "0.12.7", features = ["rustls-tls"], optional = true } From 05201609befb32d09c02d8772bc71a978071d474 Mon Sep 17 00:00:00 2001 From: nazeh Date: Fri, 29 Nov 2024 21:03:51 +0300 Subject: [PATCH 77/84] chore: add category to Cargo.toml --- pkarr/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index b840a6c..547538c 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -7,6 +7,7 @@ description = "Public-Key Addressable Resource Records (Pkarr); publish and reso license = "MIT" repository = "https://git.pkarr.org" keywords = ["mainline", "dht", "dns", "decentralized", "identity"] +categories = ["network-programming"] [dependencies] base32 = "0.5.0" From eb21d455af4c7ec80df0bbfcd25803539ab4951d Mon Sep 17 00:00:00 2001 From: nazeh Date: Sat, 30 Nov 2024 11:42:33 +0300 Subject: [PATCH 78/84] chore(pkarr): upgrade dependencies --- Cargo.lock | 161 ++++++++++++++++---------------- pkarr/Cargo.toml | 54 +++++------ pkarr/src/base/signed_packet.rs | 2 +- 3 files changed, 109 insertions(+), 108 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f57db45..59b16e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -158,9 +158,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.7.7" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "504e3947307ac8326a5437504c517c4b56716c9d98fac0028c2acc7ca47d70ae" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" dependencies = [ "async-trait", "axum-core", @@ -182,7 +182,7 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", - "sync_wrapper 1.0.1", + "sync_wrapper 1.0.2", "tokio", "tower 0.5.1", "tower-layer", @@ -205,7 +205,7 @@ dependencies = [ "mime", "pin-project-lite", "rustversion", - "sync_wrapper 1.0.1", + "sync_wrapper 1.0.2", "tower-layer", "tower-service", "tracing", @@ -329,9 +329,9 @@ checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" [[package]] name = "cc" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd9de9f2205d5ef3fd67e685b0df337994ddd4495e2a28d185500d0e1edfea47" +checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc" dependencies = [ "jobserver", "libc", @@ -465,9 +465,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca741a962e1b0bff6d724a1a0958b686406e853bb14061f218562e1896f95e6" +checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3" dependencies = [ "libc", ] @@ -700,12 +700,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -959,9 +959,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" +checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e" dependencies = [ "atomic-waker", "bytes", @@ -993,9 +993,9 @@ checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" [[package]] name = "hashbrown" -version = "0.15.1" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" [[package]] name = "heapless" @@ -1332,7 +1332,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.15.1", + "hashbrown 0.15.2", ] [[package]] @@ -1358,9 +1358,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.11" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" [[package]] name = "jobserver" @@ -1373,9 +1373,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.72" +version = "0.3.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" +checksum = "fb15147158e79fd8b8afd0252522769c4f48725460b37338544d8379d94fc8f9" dependencies = [ "wasm-bindgen", ] @@ -1394,9 +1394,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.162" +version = "0.2.167" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18d287de67fe55fd7e1581fe933d965a5a9477b38e949cfa9f8574ef01506398" +checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc" [[package]] name = "libloading" @@ -1426,9 +1426,9 @@ checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "litemap" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" +checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" [[package]] name = "litrs" @@ -1864,7 +1864,7 @@ dependencies = [ "serde", "sha1_smol", "simple-dns", - "thiserror 1.0.69", + "thiserror 2.0.3", "tokio", "tokio-rustls", "tracing", @@ -1923,9 +1923,9 @@ checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" [[package]] name = "postcard" -version = "1.0.10" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f7f0a8d620d71c457dd1d47df76bb18960378da56af4527aaa10f515eee732e" +checksum = "170a2601f67cc9dba8edd8c4870b15f71a6a2dc196daec8c83f72b59dff628a8" dependencies = [ "cobs", "embedded-io 0.4.0", @@ -1955,9 +1955,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.89" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" dependencies = [ "unicode-ident", ] @@ -2189,7 +2189,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "sync_wrapper 1.0.1", + "sync_wrapper 1.0.2", "system-configuration", "tokio", "tokio-native-tls", @@ -2247,9 +2247,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.40" +version = "0.38.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99e4ea3e1cdc4b559b8e5650f9c8e5998e3e5c1343b4eaf034565f32318d63c0" +checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6" dependencies = [ "bitflags", "errno", @@ -2260,9 +2260,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.17" +version = "0.23.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f1a745511c54ba6d4465e8d5dfbd81b45791756de28d4981af70d6dca128f1e" +checksum = "934b404430bb06b3fae2cba809eb45a1ab1aecd64491213d7c3301b88393f8d1" dependencies = [ "aws-lc-rs", "log", @@ -2407,9 +2407,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.132" +version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" +checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" dependencies = [ "itoa", "memchr", @@ -2506,9 +2506,9 @@ checksum = "1de1d4f81173b03af4c0cbed3c898f6bff5b870e4a7f5d6f4057d62a7a4b686e" [[package]] name = "simple-dns" -version = "0.6.2" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01607fe2e61894468c6dc0b26103abb073fb08b79a3d9e4b6d76a1a341549958" +checksum = "92bc7a4124e416342635ff7090cf6883a1835a91a7d2aa01c54c458a650781be" dependencies = [ "bitflags", ] @@ -2536,9 +2536,9 @@ checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "socket2" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" dependencies = [ "libc", "windows-sys 0.52.0", @@ -2592,9 +2592,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.87" +version = "2.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" +checksum = "44d46482f1c1c87acd84dea20c1bf5ebff4c757009ed6bf19cfd36fb10e92c4e" dependencies = [ "proc-macro2", "quote", @@ -2609,9 +2609,9 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] name = "sync_wrapper" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" dependencies = [ "futures-core", ] @@ -2920,9 +2920,9 @@ dependencies = [ [[package]] name = "tracing" -version = "0.1.40" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ "log", "pin-project-lite", @@ -2932,9 +2932,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", @@ -2943,9 +2943,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", "valuable", @@ -2964,9 +2964,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" dependencies = [ "matchers", "nu-ansi-term", @@ -2994,9 +2994,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicode-ident" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" [[package]] name = "untrusted" @@ -3006,9 +3006,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.3" +version = "2.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" dependencies = [ "form_urlencoded", "idna", @@ -3068,9 +3068,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.95" +version = "0.2.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" +checksum = "21d3b25c3ea1126a2ad5f4f9068483c2af1e64168f847abe863a526b8dbfe00b" dependencies = [ "cfg-if", "once_cell", @@ -3079,9 +3079,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.95" +version = "0.2.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" +checksum = "52857d4c32e496dc6537646b5b117081e71fd2ff06de792e3577a150627db283" dependencies = [ "bumpalo", "log", @@ -3094,21 +3094,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.45" +version = "0.4.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b" +checksum = "951fe82312ed48443ac78b66fa43eded9999f738f6022e67aead7b708659e49a" dependencies = [ "cfg-if", "js-sys", + "once_cell", "wasm-bindgen", "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.95" +version = "0.2.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" +checksum = "920b0ffe069571ebbfc9ddc0b36ba305ef65577c94b06262ed793716a1afd981" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3116,9 +3117,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.95" +version = "0.2.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" +checksum = "bf59002391099644be3524e23b781fa43d2be0c5aa0719a18c0731b9d195cab6" dependencies = [ "proc-macro2", "quote", @@ -3129,15 +3130,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.95" +version = "0.2.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" +checksum = "e5047c5392700766601942795a436d7d2599af60dcc3cc1248c9120bfb0827b0" [[package]] name = "web-sys" -version = "0.3.72" +version = "0.3.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" +checksum = "476364ff87d0ae6bfb661053a9104ab312542658c3d8f963b7ace80b6f9b26b9" dependencies = [ "js-sys", "wasm-bindgen", @@ -3155,9 +3156,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.26.6" +version = "0.26.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958" +checksum = "5d642ff16b7e79272ae451b7322067cdc17cadf68c23264be9d94a32319efe7e" dependencies = [ "rustls-pki-types", ] @@ -3397,9 +3398,9 @@ checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" [[package]] name = "yoke" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" dependencies = [ "serde", "stable_deref_trait", @@ -3409,9 +3410,9 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", @@ -3442,18 +3443,18 @@ dependencies = [ [[package]] name = "zerofrom" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index 547538c..d2fb788 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -10,34 +10,34 @@ keywords = ["mainline", "dht", "dns", "decentralized", "identity"] categories = ["network-programming"] [dependencies] -base32 = "0.5.0" -bytes = "1.7.1" -ed25519-dalek = { version = "2.0.0", features = ["rand_core"] } -self_cell = "1.0.2" -simple-dns = "0.6.1" -thiserror = "1.0.49" -tracing = "0.1.40" +base32 = "0.5.1" +bytes = "1.9.0" +ed25519-dalek = { version = "2.1.1", features = ["rand_core"] } +self_cell = "1.0.4" +simple-dns = "0.8.0" +thiserror = "2.0.3" +tracing = "0.1.41" dyn-clone = "1.0.17" -document-features = "0.2.8" +document-features = "0.2.10" rand = "0.8.5" -once_cell = {version = "1.19.0", default-features = false } -lru = { version = "0.12.3", default-features = false } -flume = { version = "0.11.0", features = ["select", "eventual-fairness", "async"], default-features = false , optional = true } +once_cell = {version = "1.20.2", default-features = false } +lru = { version = "0.12.5", default-features = false } +flume = { version = "0.11.1", features = ["select", "eventual-fairness", "async"], default-features = false , optional = true } # feat: relay dependencies -tokio = { version = "1.40.0", optional = true, default-features = false } +tokio = { version = "1.41.1", optional = true, default-features = false } # inherited from feat:dht as well. sha1_smol = { version = "1.0.1", optional = true } # feat: serde dependencies -serde = { version = "1.0.209", features = ["derive"], optional = true } +serde = { version = "1.0.215", features = ["derive"], optional = true } # feat: endpoints dependencies futures-lite = { version = "2.5.0", default-features = false, features= ["std"], optional = true } genawaiter = { version = "0.99.1", default-features = false, features = ["futures03"], optional = true } # feat: reqwest-builder -webpki-roots = { version = "0.26.6", optional = true } +webpki-roots = { version = "0.26.7", optional = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] pubky-timestamp = { version = "0.2.0", default-features = false } @@ -46,38 +46,38 @@ pubky-timestamp = { version = "0.2.0", default-features = false } mainline = { version = "4.1.0", optional = true } # feat: relay dependencies -reqwest = { version = "0.12.7", features = ["rustls-tls"], optional = true } +reqwest = { version = "0.12.9", features = ["rustls-tls"], optional = true } # feat: tls rustls = { version = "0.23", default-features = false, features = ["ring"], optional = true } webpki = { package = "rustls-webpki", version = "0.102", optional = true } # feat: lmdb-cache defendencies -heed = { version = "0.20.0", default-features = false, optional = true } +heed = { version = "0.20.5", default-features = false, optional = true } byteorder = { version = "1.5.0", default-features = false, optional = true } -libc = { version = "0.2.159", optional = true } +libc = { version = "0.2.167", optional = true } [target.'cfg(target_arch = "wasm32")'.dependencies] pubky-timestamp = { version = "0.2.0", default-features = false, features = ["httpdate"] } -js-sys = "0.3.69" -futures = "0.3.30" +js-sys = "0.3.73" +futures = "0.3.31" getrandom = { version = "0.2", features = ["js"] } -reqwest = "0.12.7" +reqwest = "0.12.9" sha1_smol = "1.0.1" -wasm-bindgen-futures = "0.4.45" +wasm-bindgen-futures = "0.4.46" [dev-dependencies] anyhow = "1.0.93" -axum = "0.7.7" +axum = "0.7.9" axum-server = { version = "0.7.1", features = ["tls-rustls-no-provider"] } -postcard = { version = "1.0.10", features = ["alloc"] } -tokio = { version = "1.40.0", features = ["macros", "rt-multi-thread"] } +postcard = { version = "1.1.1", features = ["alloc"] } +tokio = { version = "1.41.1", features = ["macros", "rt-multi-thread"] } tokio-rustls = "0.26.0" [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] -clap = { version = "4.4.8", features = ["derive"] } -mockito = "1.4.0" -tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } +clap = { version = "4.5.21", features = ["derive"] } +mockito = "1.6.1" +tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } [features] # Clients diff --git a/pkarr/src/base/signed_packet.rs b/pkarr/src/base/signed_packet.rs index 29a5064..1a53d29 100644 --- a/pkarr/src/base/signed_packet.rs +++ b/pkarr/src/base/signed_packet.rs @@ -635,7 +635,7 @@ mod tests { let mut packet = Packet::new_reply(0); packet.answers.push(target.clone()); packet.answers.push(ResourceRecord::new( - Name::new("something else").unwrap(), + Name::new("something-else").unwrap(), simple_dns::CLASS::IN, 30, RData::A(A { From 5237e8797ecaa5c8268e72717858ab13607cacb4 Mon Sep 17 00:00:00 2001 From: nazeh Date: Sat, 30 Nov 2024 15:26:33 +0300 Subject: [PATCH 79/84] feat(pkarr): add SignedPacket::builder() --- CHANGELOG.md | 3 + pkarr/examples/http-serve.rs | 31 +- pkarr/examples/publish.rs | 14 +- pkarr/src/base/signed_packet.rs | 521 ++++++++++++++------------ pkarr/src/client/dht.rs | 58 +-- pkarr/src/client/relay.rs | 18 +- pkarr/src/extra/endpoints/endpoint.rs | 93 ++--- pkarr/src/extra/endpoints/mod.rs | 30 +- pkarr/src/extra/mod.rs | 31 +- 9 files changed, 366 insertions(+), 433 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ba4cf3..caff1a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to pkarr client and server will be documented in this file. ### Added +- Add `SignedPacket::builder()` and convenient methods to create `A`, `AAAA`, `CNAME`, `TXT`, `SVCB`, and `HTTPS` records. - Use `pubky_timestamp::Timestamp` - Impl `PartialEq, Eq` for `SignedPacket`. - Impl `From` for `CacheKey`. @@ -32,3 +33,5 @@ All notable changes to pkarr client and server will be documented in this file. ### Removed - Remvoed `relay_client_web`, replaced with *(pkarr::relay::Client)*. +- Removed `SignedPacket::from_packet`. +- Removed `SignedPacket::packet` getter. diff --git a/pkarr/examples/http-serve.rs b/pkarr/examples/http-serve.rs index 17946f5..a055a44 100644 --- a/pkarr/examples/http-serve.rs +++ b/pkarr/examples/http-serve.rs @@ -14,10 +14,7 @@ use std::sync::Arc; use clap::Parser; -use pkarr::{ - dns::{rdata::SVCB, Packet}, - Client, Keypair, SignedPacket, -}; +use pkarr::{dns::rdata::SVCB, Client, Keypair, SignedPacket}; #[derive(Parser)] #[command(author, version, about, long_about = None)] @@ -66,30 +63,14 @@ async fn main() -> anyhow::Result<()> { } async fn publish_server_pkarr(client: &Client, keypair: &Keypair, socket_addr: &SocketAddr) { - let mut packet = Packet::new_reply(1); - let mut svcb = SVCB::new(0, ".".try_into().expect("infallible")); - svcb.set_port(socket_addr.port()); - packet.answers.push(pkarr::dns::ResourceRecord::new( - "@".try_into().unwrap(), - pkarr::dns::CLASS::IN, - 60 * 60, - pkarr::dns::rdata::RData::HTTPS(svcb.into()), - )); - - packet.answers.push(pkarr::dns::ResourceRecord::new( - "@".try_into().unwrap(), - pkarr::dns::CLASS::IN, - 60 * 60, - match socket_addr.ip() { - std::net::IpAddr::V4(ip) => pkarr::dns::rdata::RData::A(ip.into()), - std::net::IpAddr::V6(ip) => pkarr::dns::rdata::RData::AAAA(ip.into()), - }, - )); - - let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); + let signed_packet = SignedPacket::builder() + .https("@".try_into().unwrap(), svcb, 60 * 60) + .address("@".try_into().unwrap(), socket_addr.ip(), 60 * 60) + .sign(&keypair) + .unwrap(); client.publish(&signed_packet).await.unwrap(); } diff --git a/pkarr/examples/publish.rs b/pkarr/examples/publish.rs index 05e9ee2..edc4b9c 100644 --- a/pkarr/examples/publish.rs +++ b/pkarr/examples/publish.rs @@ -11,7 +11,7 @@ use tracing_subscriber; use std::time::Instant; -use pkarr::{dns, Keypair, SignedPacket}; +use pkarr::{Keypair, SignedPacket}; #[cfg(feature = "relay")] use pkarr::client::relay::Client; @@ -29,15 +29,9 @@ async fn main() -> anyhow::Result<()> { let keypair = Keypair::random(); - let mut packet = dns::Packet::new_reply(0); - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("_foo")?, - dns::CLASS::IN, - 30, - dns::rdata::RData::TXT("bar".try_into()?), - )); - - let signed_packet = SignedPacket::from_packet(&keypair, &packet)?; + let signed_packet = SignedPacket::builder() + .txt("_foo".try_into().unwrap(), "bar".try_into().unwrap(), 30) + .sign(&keypair)?; let instant = Instant::now(); diff --git a/pkarr/src/base/signed_packet.rs b/pkarr/src/base/signed_packet.rs index 1a53d29..0cdf2d9 100644 --- a/pkarr/src/base/signed_packet.rs +++ b/pkarr/src/base/signed_packet.rs @@ -5,13 +5,13 @@ use bytes::{Bytes, BytesMut}; use ed25519_dalek::{Signature, SignatureError}; use self_cell::self_cell; use simple_dns::{ - rdata::{RData, A, AAAA}, - Name, Packet, ResourceRecord, SimpleDnsError, + rdata::{RData, A, AAAA, HTTPS, SVCB, TXT}, + Name, Packet, ResourceRecord, SimpleDnsError, CLASS, }; use std::{ char, fmt::{self, Display, Formatter}, - net::{Ipv4Addr, Ipv6Addr}, + net::{IpAddr, Ipv4Addr, Ipv6Addr}, }; #[cfg(feature = "serde")] @@ -19,6 +19,83 @@ use serde::{Deserialize, Serialize}; use pubky_timestamp::Timestamp; +#[derive(Debug, Default)] +pub struct SignedPacketBuilder(Vec>); + +impl SignedPacketBuilder { + /// Insert a [ResourceRecord] + pub fn record(mut self, record: ResourceRecord<'_>) -> Self { + self.0.push(record.into_owned()); + self + } + + /// Insert any type of [RData] + pub fn rdata(self, name: Name<'_>, rdata: RData, ttl: u32) -> Self { + self.record(ResourceRecord::new(name.to_owned(), CLASS::IN, ttl, rdata)) + } + + /// Insert an `A` record. + pub fn a(self, name: Name<'_>, address: Ipv4Addr, ttl: u32) -> Self { + self.rdata( + name, + RData::A(A { + address: address.into(), + }), + ttl, + ) + } + + /// Insert an `AAAA` record. + pub fn aaaa(self, name: Name<'_>, address: Ipv6Addr, ttl: u32) -> Self { + self.rdata( + name, + RData::AAAA(AAAA { + address: address.into(), + }), + ttl, + ) + } + + /// Insert an `A` or `AAAA` record. + pub fn address(self, name: Name<'_>, address: IpAddr, ttl: u32) -> Self { + match address { + IpAddr::V4(addr) => self.a(name, addr, ttl), + IpAddr::V6(addr) => self.aaaa(name, addr, ttl), + } + } + + /// Insert a `CNAME` record. + pub fn cname(self, name: Name<'_>, cname: Name<'_>, ttl: u32) -> Self { + self.rdata(name, RData::CNAME(cname.into()), ttl) + } + + /// Insert a `TXT` record. + pub fn txt(self, name: Name<'_>, text: TXT<'_>, ttl: u32) -> Self { + self.rdata(name, RData::TXT(text), ttl) + } + + /// Insert an `HTTPS` record + pub fn https(self, name: Name<'_>, svcb: SVCB, ttl: u32) -> Self { + self.rdata(name, RData::HTTPS(HTTPS(svcb)), ttl) + } + + /// Insert an `SVCB record + pub fn svcb(self, name: Name<'_>, svcb: SVCB, ttl: u32) -> Self { + self.rdata(name, RData::SVCB(svcb), ttl) + } + + /// Alias to [Self::sign] + pub fn build(self, keypair: &Keypair) -> Result { + self.sign(keypair) + } + + /// Create a [Packet] from the [ResourceRecord]s inserted so far and sign + /// it with the given [Keypair]. + pub fn sign(self, keypair: &Keypair) -> Result { + SignedPacket::from_answers(keypair, &self.0) + } +} + const DOT: char = '.'; self_cell!( @@ -65,48 +142,8 @@ pub struct SignedPacket { impl SignedPacket { pub const MAX_BYTES: u64 = 1104; - /// Creates a [Self] from the serialized representation: - /// `<32 bytes public_key><64 bytes signature><8 bytes big-endian timestamp in microseconds>` - /// - /// Performs the following validations: - /// - Bytes minimum length - /// - Validates the PublicKey - /// - Verifies the Signature - /// - Validates the DNS packet encoding - /// - /// You can skip all these validations by using [Self::from_bytes_unchecked] instead. - /// - /// You can use [Self::from_relay_payload] instead if you are receiving a response from an HTTP relay. - pub fn from_bytes(bytes: &Bytes) -> Result { - if bytes.len() < 104 { - return Err(SignedPacketError::InvalidSignedPacketBytesLength( - bytes.len(), - )); - } - if (bytes.len() as u64) > SignedPacket::MAX_BYTES { - return Err(SignedPacketError::PacketTooLarge(bytes.len())); - } - let public_key = PublicKey::try_from(&bytes[..32])?; - let signature = Signature::from_bytes(bytes[32..96].try_into().unwrap()); - let timestamp = u64::from_be_bytes(bytes[96..104].try_into().unwrap()); - - let encoded_packet = &bytes.slice(104..); - - public_key.verify(&signable(timestamp, encoded_packet), &signature)?; - - Ok(SignedPacket { - inner: Inner::try_from_bytes(bytes.to_owned())?, - last_seen: Timestamp::now(), - }) - } - - /// Useful for cloning a [SignedPacket], or cerating one from a previously checked bytes, - /// like ones stored on disk or in a database. - pub fn from_bytes_unchecked(bytes: &Bytes, last_seen: impl Into) -> SignedPacket { - SignedPacket { - inner: Inner::try_from_bytes(bytes.to_owned()).unwrap(), - last_seen: last_seen.into(), - } + pub fn builder() -> SignedPacketBuilder { + SignedPacketBuilder::default() } /// Creates a [SignedPacket] from a [PublicKey] and the [relays](https://github.com/Nuhvi/pkarr/blob/main/design/relays.md) payload. @@ -122,42 +159,36 @@ impl SignedPacket { SignedPacket::from_bytes(&bytes.into()) } - /// Creates a new [SignedPacket] from a [Keypair] and a DNS [Packet]. + /// Creates a new [SignedPacket] from a [Keypair] and [ResourceRecord]s as the `answers` + /// section of a DNS [Packet]. /// /// It will also normalize the names of the [ResourceRecord]s to be relative to the origin, /// which would be the z-base32 encoded [PublicKey] of the [Keypair] used to sign the Packet. - pub fn from_packet( + fn from_answers( keypair: &Keypair, - packet: &Packet, + answers: &[ResourceRecord<'_>], ) -> Result { - // Normalize names to the origin TLD - let mut inner = Packet::new_reply(0); + let mut packet = Packet::new_reply(0); let origin = keypair.public_key().to_z32(); - let normalized_names: Vec = packet - .answers + // Normalize names to the origin TLD + let normalized_names: Vec = answers .iter() .map(|answer| normalize_name(&origin, answer.name.to_string())) .collect(); - packet - .answers - .iter() - .enumerate() - .for_each(|(index, answer)| { - let new_new_name = Name::new_unchecked(&normalized_names[index]); - - inner.answers.push(ResourceRecord::new( - new_new_name.clone(), - answer.class, - answer.ttl, - answer.rdata.clone(), - )) - }); + answers.iter().enumerate().for_each(|(index, answer)| { + packet.answers.push(ResourceRecord::new( + Name::new_unchecked(&normalized_names[index]).to_owned(), + answer.class, + answer.ttl, + answer.rdata.clone(), + )) + }); // Encode the packet as `v` and verify its length - let encoded_packet: Bytes = inner.build_bytes_vec_compressed()?.into(); + let encoded_packet: Bytes = packet.build_bytes_vec_compressed()?.into(); if encoded_packet.len() > 1000 { return Err(SignedPacketError::PacketTooLarge(encoded_packet.len())); @@ -253,7 +284,7 @@ impl SignedPacket { } /// Return the DNS [Packet]. - pub fn packet(&self) -> &Packet { + pub(crate) fn packet(&self) -> &Packet { self.inner.borrow_dependent() } @@ -332,7 +363,7 @@ impl SignedPacket { } } - /// Returns the smallest `ttl` in the [Self::packet] resource records, + /// Returns the smallest `ttl` in the resource records, /// calmped with `min` and `max`. /// /// # Panics @@ -359,6 +390,50 @@ impl SignedPacket { fn elapsed(&self) -> u32 { ((Timestamp::now().as_u64() - self.last_seen.as_u64()) / 1_000_000) as u32 } + + /// Creates a [Self] from the serialized representation: + /// `<32 bytes public_key><64 bytes signature><8 bytes big-endian timestamp in microseconds>` + /// + /// Performs the following validations: + /// - Bytes minimum length + /// - Validates the PublicKey + /// - Verifies the Signature + /// - Validates the DNS packet encoding + /// + /// You can skip all these validations by using [Self::from_bytes_unchecked] instead. + /// + /// You can use [Self::from_relay_payload] instead if you are receiving a response from an HTTP relay. + fn from_bytes(bytes: &Bytes) -> Result { + if bytes.len() < 104 { + return Err(SignedPacketError::InvalidSignedPacketBytesLength( + bytes.len(), + )); + } + if (bytes.len() as u64) > SignedPacket::MAX_BYTES { + return Err(SignedPacketError::PacketTooLarge(bytes.len())); + } + let public_key = PublicKey::try_from(&bytes[..32])?; + let signature = Signature::from_bytes(bytes[32..96].try_into().unwrap()); + let timestamp = u64::from_be_bytes(bytes[96..104].try_into().unwrap()); + + let encoded_packet = &bytes.slice(104..); + + public_key.verify(&signable(timestamp, encoded_packet), &signature)?; + + Ok(SignedPacket { + inner: Inner::try_from_bytes(bytes.to_owned())?, + last_seen: Timestamp::now(), + }) + } + + /// Useful for cloning a [SignedPacket], or cerating one from a previously checked bytes, + /// like ones stored on disk or in a database. + fn from_bytes_unchecked(bytes: &Bytes, last_seen: impl Into) -> SignedPacket { + SignedPacket { + inner: Inner::try_from_bytes(bytes.to_owned()).unwrap(), + last_seen: last_seen.into(), + } + } } fn signable(timestamp: u64, v: &Bytes) -> Bytes { @@ -538,8 +613,9 @@ pub enum SignedPacketError { #[cfg(all(test, not(target_arch = "wasm32")))] mod tests { + use simple_dns::rdata::CNAME; + use super::*; - use crate::dns; use crate::{DEFAULT_MAXIMUM_TTL, DEFAULT_MINIMUM_TTL}; @@ -569,17 +645,14 @@ mod tests { fn sign_verify() { let keypair = Keypair::random(); - let mut packet = Packet::new_reply(0); - packet.answers.push(ResourceRecord::new( - Name::new("_derp_region.iroh.").unwrap(), - simple_dns::CLASS::IN, - 30, - RData::A(A { - address: Ipv4Addr::new(1, 1, 1, 1).into(), - }), - )); - - let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); + let signed_packet = SignedPacket::builder() + .address( + "_derp_region.iroh.".try_into().unwrap(), + "1.1.1.1".parse().unwrap(), + 30, + ) + .sign(&keypair) + .unwrap(); assert!(SignedPacket::from_relay_payload( &signed_packet.public_key(), @@ -602,19 +675,17 @@ mod tests { fn from_too_large_packet() { let keypair = Keypair::random(); - let mut packet = Packet::new_reply(0); + let mut builder = SignedPacket::builder(); + for _ in 0..100 { - packet.answers.push(ResourceRecord::new( - Name::new("_derp_region.iroh.").unwrap(), - simple_dns::CLASS::IN, + builder = builder.address( + "_derp_region.iroh.".try_into().unwrap(), + "1.1.1.1".parse().unwrap(), 30, - RData::A(A { - address: Ipv4Addr::new(1, 1, 1, 1).into(), - }), - )); + ); } - let error = SignedPacket::from_packet(&keypair, &packet); + let error = builder.sign(&keypair); assert!(error.is_err()); } @@ -632,18 +703,15 @@ mod tests { }), ); - let mut packet = Packet::new_reply(0); - packet.answers.push(target.clone()); - packet.answers.push(ResourceRecord::new( - Name::new("something-else").unwrap(), - simple_dns::CLASS::IN, - 30, - RData::A(A { - address: Ipv4Addr::new(1, 1, 1, 1).into(), - }), - )); - - let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); + let signed_packet = SignedPacket::builder() + .record(target.clone()) + .address( + "something-else".try_into().unwrap(), + "1.1.1.1".parse().unwrap(), + 30, + ) + .sign(&keypair) + .unwrap(); let iter = signed_packet.resource_records("_derp_region.iroh"); assert_eq!(iter.count(), 1); @@ -658,17 +726,15 @@ mod tests { fn to_mutable() { let keypair = Keypair::random(); - let mut packet = Packet::new_reply(0); - packet.answers.push(ResourceRecord::new( - Name::new("_derp_region.iroh.").unwrap(), - simple_dns::CLASS::IN, - 30, - RData::A(A { - address: Ipv4Addr::new(1, 1, 1, 1).into(), - }), - )); + let signed_packet = SignedPacket::builder() + .address( + "_derp_region.iroh.".try_into().unwrap(), + "1.1.1.1".parse().unwrap(), + 30, + ) + .sign(&keypair) + .unwrap(); - let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); let item: MutableItem = (&signed_packet).into(); let seq = signed_packet.timestamp().as_u64() as i64; @@ -690,73 +756,54 @@ mod tests { fn compressed_names() { let keypair = Keypair::random(); - let name = "foobar"; - let dup = name; - - let mut packet = Packet::new_reply(0); - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("@").unwrap(), - dns::CLASS::IN, - 30, - dns::rdata::RData::CNAME(dns::Name::new(name).unwrap().into()), - )); - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("@").unwrap(), - dns::CLASS::IN, - 30, - dns::rdata::RData::CNAME(dns::Name::new(dup).unwrap().into()), - )); - - let signed = SignedPacket::from_packet(&keypair, &packet).unwrap(); + let signed_packet = SignedPacket::builder() + .cname("@".try_into().unwrap(), "foobar".try_into().unwrap(), 30) + .cname("@".try_into().unwrap(), "foobar".try_into().unwrap(), 30) + .sign(&keypair) + .unwrap(); assert_eq!( - signed + signed_packet .resource_records("@") .map(|r| r.rdata.clone()) .collect::>(), - packet - .answers - .iter() - .map(|r| r.rdata.clone()) - .collect::>() + vec![ + RData::CNAME(CNAME("foobar".try_into().unwrap())), + RData::CNAME(CNAME("foobar".try_into().unwrap())) + ] ) } #[test] fn to_bytes_from_bytes() { let keypair = Keypair::random(); - let mut packet = Packet::new_reply(0); - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("_foo").unwrap(), - dns::CLASS::IN, - 30, - RData::TXT("hello".try_into().unwrap()), - )); - let signed = SignedPacket::from_packet(&keypair, &packet).unwrap(); - let bytes = signed.as_bytes(); + + let signed_packet = SignedPacket::builder() + .txt("_foo".try_into().unwrap(), "hello".try_into().unwrap(), 30) + .sign(&keypair) + .unwrap(); + + let bytes = signed_packet.as_bytes(); let from_bytes = SignedPacket::from_bytes(bytes).unwrap(); - assert_eq!(signed.as_bytes(), from_bytes.as_bytes()); - let from_bytes2 = SignedPacket::from_bytes_unchecked(bytes, &signed.last_seen); - assert_eq!(signed.as_bytes(), from_bytes2.as_bytes()); + assert_eq!(signed_packet.as_bytes(), from_bytes.as_bytes()); + let from_bytes2 = SignedPacket::from_bytes_unchecked(bytes, &signed_packet.last_seen); + assert_eq!(signed_packet.as_bytes(), from_bytes2.as_bytes()); let public_key = keypair.public_key(); - let payload = signed.to_relay_payload(); + let payload = signed_packet.to_relay_payload(); let from_relay_payload = SignedPacket::from_relay_payload(&public_key, &payload).unwrap(); - assert_eq!(signed.as_bytes(), from_relay_payload.as_bytes()); + assert_eq!(signed_packet.as_bytes(), from_relay_payload.as_bytes()); } #[test] fn clone() { let keypair = Keypair::random(); - let mut packet = Packet::new_reply(0); - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("_foo").unwrap(), - dns::CLASS::IN, - 30, - RData::TXT("hello".try_into().unwrap()), - )); - let signed = SignedPacket::from_packet(&keypair, &packet).unwrap(); + let signed = SignedPacket::builder() + .txt("_foo".try_into().unwrap(), "hello".try_into().unwrap(), 30) + .sign(&keypair) + .unwrap(); + let cloned = signed.clone(); assert_eq!(cloned.as_bytes(), signed.as_bytes()); @@ -765,25 +812,21 @@ mod tests { #[test] fn expires_in_minimum_ttl() { let keypair = Keypair::random(); - let mut packet = Packet::new_reply(0); - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("_foo").unwrap(), - dns::CLASS::IN, - 10, - RData::TXT("hello".try_into().unwrap()), - )); - let mut signed = SignedPacket::from_packet(&keypair, &packet).unwrap(); + let mut signed_packet = SignedPacket::builder() + .txt("_foo".try_into().unwrap(), "hello".try_into().unwrap(), 10) + .sign(&keypair) + .unwrap(); - signed.last_seen -= 20 * 1_000_000_u64; + signed_packet.last_seen -= 20 * 1_000_000_u64; assert!( - signed.expires_in(30, u32::MAX) > 0, + signed_packet.expires_in(30, u32::MAX) > 0, "input minimum_ttl is 30 so ttl = 30" ); assert!( - signed.expires_in(0, u32::MAX) == 0, + signed_packet.expires_in(0, u32::MAX) == 0, "input minimum_ttl is 0 so ttl = 10 (smallest in resource records)" ); } @@ -791,25 +834,25 @@ mod tests { #[test] fn expires_in_maximum_ttl() { let keypair = Keypair::random(); - let mut packet = Packet::new_reply(0); - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("_foo").unwrap(), - dns::CLASS::IN, - 3 * DEFAULT_MAXIMUM_TTL, - RData::TXT("hello".try_into().unwrap()), - )); - let mut signed = SignedPacket::from_packet(&keypair, &packet).unwrap(); + let mut signed_packet = SignedPacket::builder() + .txt( + "_foo".try_into().unwrap(), + "hello".try_into().unwrap(), + 3 * DEFAULT_MAXIMUM_TTL, + ) + .sign(&keypair) + .unwrap(); - signed.last_seen -= 2 * (DEFAULT_MAXIMUM_TTL as u64) * 1_000_000; + signed_packet.last_seen -= 2 * (DEFAULT_MAXIMUM_TTL as u64) * 1_000_000; assert!( - signed.expires_in(0, DEFAULT_MAXIMUM_TTL) == 0, + signed_packet.expires_in(0, DEFAULT_MAXIMUM_TTL) == 0, "input maximum_ttl is the dfeault 86400 so maximum ttl = 86400" ); assert!( - signed.expires_in(0, 7 * DEFAULT_MAXIMUM_TTL) > 0, + signed_packet.expires_in(0, 7 * DEFAULT_MAXIMUM_TTL) > 0, "input maximum_ttl is 7 * 86400 so ttl = 3 * 86400 (smallest in resource records)" ); } @@ -817,93 +860,84 @@ mod tests { #[test] fn fresh_resource_records() { let keypair = Keypair::random(); - let mut packet = Packet::new_reply(0); - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("_foo").unwrap(), - dns::CLASS::IN, - 30, - RData::TXT("hello".try_into().unwrap()), - )); - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("_foo").unwrap(), - dns::CLASS::IN, - 60, - RData::TXT("world".try_into().unwrap()), - )); - let mut signed = SignedPacket::from_packet(&keypair, &packet).unwrap(); + let mut signed_packet = SignedPacket::builder() + .txt("_foo".try_into().unwrap(), "hello".try_into().unwrap(), 30) + .txt("_foo".try_into().unwrap(), "world".try_into().unwrap(), 60) + .sign(&keypair) + .unwrap(); - signed.last_seen -= 30 * 1_000_000; + signed_packet.last_seen -= 30 * 1_000_000; - assert_eq!(signed.fresh_resource_records("_foo").count(), 1); + assert_eq!(signed_packet.fresh_resource_records("_foo").count(), 1); } #[test] fn ttl_empty() { let keypair = Keypair::random(); - let packet = Packet::new_reply(0); - let signed = SignedPacket::from_packet(&keypair, &packet).unwrap(); + let signed_packet = SignedPacket::builder().sign(&keypair).unwrap(); - assert_eq!(signed.ttl(DEFAULT_MINIMUM_TTL, DEFAULT_MAXIMUM_TTL), 300); + assert_eq!( + signed_packet.ttl(DEFAULT_MINIMUM_TTL, DEFAULT_MAXIMUM_TTL), + 300 + ); } #[test] fn ttl_with_records_less_than_minimum() { let keypair = Keypair::random(); - let mut packet = Packet::new_reply(0); - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("_foo").unwrap(), - dns::CLASS::IN, - DEFAULT_MINIMUM_TTL / 2, - RData::TXT("hello".try_into().unwrap()), - )); - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("_foo").unwrap(), - dns::CLASS::IN, - DEFAULT_MINIMUM_TTL / 4, - RData::TXT("world".try_into().unwrap()), - )); - - let signed = SignedPacket::from_packet(&keypair, &packet).unwrap(); + let signed_packet = SignedPacket::builder() + .txt( + "_foo".try_into().unwrap(), + "hello".try_into().unwrap(), + DEFAULT_MINIMUM_TTL / 2, + ) + .txt( + "_foo".try_into().unwrap(), + "world".try_into().unwrap(), + DEFAULT_MINIMUM_TTL / 4, + ) + .sign(&keypair) + .unwrap(); assert_eq!( - signed.ttl(DEFAULT_MINIMUM_TTL, DEFAULT_MAXIMUM_TTL), + signed_packet.ttl(DEFAULT_MINIMUM_TTL, DEFAULT_MAXIMUM_TTL), DEFAULT_MINIMUM_TTL ); - assert_eq!(signed.ttl(0, DEFAULT_MAXIMUM_TTL), DEFAULT_MINIMUM_TTL / 4); + assert_eq!( + signed_packet.ttl(0, DEFAULT_MAXIMUM_TTL), + DEFAULT_MINIMUM_TTL / 4 + ); } #[test] fn ttl_with_records_more_than_maximum() { let keypair = Keypair::random(); - let mut packet = Packet::new_reply(0); - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("_foo").unwrap(), - dns::CLASS::IN, - DEFAULT_MAXIMUM_TTL * 2, - RData::TXT("world".try_into().unwrap()), - )); - - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("_foo").unwrap(), - dns::CLASS::IN, - DEFAULT_MAXIMUM_TTL * 4, - RData::TXT("world".try_into().unwrap()), - )); - - let signed = SignedPacket::from_packet(&keypair, &packet).unwrap(); + let signed_packet = SignedPacket::builder() + .txt( + "_foo".try_into().unwrap(), + "hello".try_into().unwrap(), + DEFAULT_MAXIMUM_TTL * 2, + ) + .txt( + "_foo".try_into().unwrap(), + "world".try_into().unwrap(), + DEFAULT_MAXIMUM_TTL * 4, + ) + .sign(&keypair) + .unwrap(); assert_eq!( - signed.ttl(DEFAULT_MINIMUM_TTL, DEFAULT_MAXIMUM_TTL), + signed_packet.ttl(DEFAULT_MINIMUM_TTL, DEFAULT_MAXIMUM_TTL), DEFAULT_MAXIMUM_TTL ); assert_eq!( - signed.ttl(0, DEFAULT_MAXIMUM_TTL * 8), + signed_packet.ttl(0, DEFAULT_MAXIMUM_TTL * 8), DEFAULT_MAXIMUM_TTL * 2 ); } @@ -915,17 +949,14 @@ mod tests { let keypair = Keypair::random(); - let mut packet = Packet::new_reply(0); - packet.answers.push(ResourceRecord::new( - Name::new("_derp_region.iroh.").unwrap(), - simple_dns::CLASS::IN, - 30, - RData::A(A { - address: Ipv4Addr::new(1, 1, 1, 1).into(), - }), - )); - - let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); + let signed_packet = SignedPacket::builder() + .address( + "_derp_region.iroh.".try_into().unwrap(), + "1.1.1.1".parse().unwrap(), + 30, + ) + .sign(&keypair) + .unwrap(); let serialized = to_allocvec(&signed_packet).unwrap(); let deserialized: SignedPacket = from_bytes(&serialized).unwrap(); diff --git a/pkarr/src/client/dht.rs b/pkarr/src/client/dht.rs index c0d3a0a..78a25db 100644 --- a/pkarr/src/client/dht.rs +++ b/pkarr/src/client/dht.rs @@ -585,7 +585,7 @@ mod tests { use mainline::Testnet; use super::*; - use crate::{dns, Keypair, SignedPacket}; + use crate::{Keypair, SignedPacket}; #[test] fn shutdown_sync() { @@ -612,15 +612,10 @@ mod tests { let keypair = Keypair::random(); - let mut packet = dns::Packet::new_reply(0); - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("foo").unwrap(), - dns::CLASS::IN, - 30, - dns::rdata::RData::TXT("bar".try_into().unwrap()), - )); - - let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); + let signed_packet = SignedPacket::builder() + .txt("foo".try_into().unwrap(), "bar".try_into().unwrap(), 30) + .sign(&keypair) + .unwrap(); a.publish_sync(&signed_packet).unwrap(); @@ -642,15 +637,10 @@ mod tests { let keypair = Keypair::random(); - let mut packet = dns::Packet::new_reply(0); - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("foo").unwrap(), - dns::CLASS::IN, - 30, - dns::rdata::RData::TXT("bar".try_into().unwrap()), - )); - - let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); + let signed_packet = SignedPacket::builder() + .txt("foo".try_into().unwrap(), "bar".try_into().unwrap(), 30) + .sign(&keypair) + .unwrap(); a.publish_sync(&signed_packet).unwrap(); @@ -689,15 +679,10 @@ mod tests { let keypair = Keypair::random(); - let mut packet = dns::Packet::new_reply(0); - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("foo").unwrap(), - dns::CLASS::IN, - 30, - dns::rdata::RData::TXT("bar".try_into().unwrap()), - )); - - let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); + let signed_packet = SignedPacket::builder() + .txt("foo".try_into().unwrap(), "bar".try_into().unwrap(), 30) + .sign(&keypair) + .unwrap(); a.publish(&signed_packet).await.unwrap(); @@ -719,15 +704,10 @@ mod tests { let keypair = Keypair::random(); - let mut packet = dns::Packet::new_reply(0); - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("foo").unwrap(), - dns::CLASS::IN, - 30, - dns::rdata::RData::TXT("bar".try_into().unwrap()), - )); - - let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); + let signed_packet = SignedPacket::builder() + .txt("foo".try_into().unwrap(), "bar".try_into().unwrap(), 30) + .sign(&keypair) + .unwrap(); a.publish(&signed_packet).await.unwrap(); @@ -760,8 +740,8 @@ mod tests { .unwrap(); let keypair = Keypair::random(); - let packet = dns::Packet::new_reply(0); - let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); + + let signed_packet = SignedPacket::builder().sign(&keypair).unwrap(); client .cache() diff --git a/pkarr/src/client/relay.rs b/pkarr/src/client/relay.rs index 529cb5a..955b2dc 100644 --- a/pkarr/src/client/relay.rs +++ b/pkarr/src/client/relay.rs @@ -463,21 +463,16 @@ impl Display for EmptyListOfRelays { #[cfg(test)] mod tests { use super::*; - use crate::{dns, Keypair, SignedPacket}; + use crate::{Keypair, SignedPacket}; #[tokio::test] async fn publish_resolve() { let keypair = Keypair::random(); - let mut packet = dns::Packet::new_reply(0); - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("foo").unwrap(), - dns::CLASS::IN, - 30, - dns::rdata::RData::TXT("bar".try_into().unwrap()), - )); - - let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); + let signed_packet = SignedPacket::builder() + .txt("foo".try_into().unwrap(), "bar".try_into().unwrap(), 30) + .sign(&keypair) + .unwrap(); let mut server = mockito::Server::new_async().await; @@ -542,8 +537,7 @@ mod tests { .build() .unwrap(); - let packet = dns::Packet::new_reply(0); - let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); + let signed_packet = SignedPacket::builder().sign(&keypair).unwrap(); client .cache() diff --git a/pkarr/src/extra/endpoints/endpoint.rs b/pkarr/src/extra/endpoints/endpoint.rs index 35fd764..1c08b6f 100644 --- a/pkarr/src/extra/endpoints/endpoint.rs +++ b/pkarr/src/extra/endpoints/endpoint.rs @@ -32,6 +32,7 @@ impl Endpoint { pub(crate) fn parse( signed_packet: &SignedPacket, target: &str, + // TODO: change is_svcb to a better name is_svcb: bool, ) -> Vec { let mut records = signed_packet @@ -157,44 +158,37 @@ fn get_svcb<'a>(record: &'a ResourceRecord, is_svcb: bool) -> Option<&'a SVCB<'a #[cfg(test)] mod tests { - use std::net::{Ipv4Addr, Ipv6Addr}; - use std::str::FromStr; - use super::*; - use crate::{dns, Keypair}; + use crate::Keypair; #[tokio::test] async fn endpoint_domain() { - let mut packet = dns::Packet::new_reply(0); - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("foo").unwrap(), - dns::CLASS::IN, - 3600, - RData::HTTPS(SVCB::new(0, "https.example.com".try_into().unwrap()).into()), - )); - // Make sure HTTPS only follows HTTPs - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("foo").unwrap(), - dns::CLASS::IN, - 3600, - RData::SVCB(SVCB::new(0, "protocol.example.com".try_into().unwrap())), - )); - // Make sure SVCB only follows SVCB - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("foo").unwrap(), - dns::CLASS::IN, - 3600, - RData::HTTPS(SVCB::new(0, "https.example.com".try_into().unwrap()).into()), - )); - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("_foo").unwrap(), - dns::CLASS::IN, - 3600, - RData::SVCB(SVCB::new(0, "protocol.example.com".try_into().unwrap())), - )); let keypair = Keypair::random(); - let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); + let signed_packet = SignedPacket::builder() + .https( + "foo".try_into().unwrap(), + SVCB::new(0, "https.example.com".try_into().unwrap()), + 3600, + ) + .svcb( + "foo".try_into().unwrap(), + SVCB::new(0, "protocol.example.com".try_into().unwrap()), + 3600, + ) + // Make sure SVCB only follows SVCB + .https( + "foo".try_into().unwrap(), + SVCB::new(0, "https.example.com".try_into().unwrap()), + 3600, + ) + .svcb( + "_foo".try_into().unwrap(), + SVCB::new(0, "protocol.example.com".try_into().unwrap()), + 3600, + ) + .sign(&keypair) + .unwrap(); let tld = keypair.public_key(); @@ -213,31 +207,24 @@ mod tests { #[test] fn endpoint_to_socket_addrs() { - let mut packet = dns::Packet::new_reply(0); - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("@").unwrap(), - dns::CLASS::IN, - 3600, - RData::A(Ipv4Addr::from_str("209.151.148.15").unwrap().into()), - )); - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("@").unwrap(), - dns::CLASS::IN, - 3600, - RData::AAAA(Ipv6Addr::from_str("2a05:d014:275:6201::64").unwrap().into()), - )); - let mut svcb = SVCB::new(1, ".".try_into().unwrap()); svcb.set_port(6881); - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("@").unwrap(), - dns::CLASS::IN, - 3600, - RData::HTTPS(svcb.into()), - )); let keypair = Keypair::random(); - let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); + let signed_packet = SignedPacket::builder() + .address( + "@".try_into().unwrap(), + "209.151.148.15".parse().unwrap(), + 3600, + ) + .address( + "@".try_into().unwrap(), + "2a05:d014:275:6201::64".parse().unwrap(), + 3600, + ) + .https("@".try_into().unwrap(), svcb, 3600) + .sign(&keypair) + .unwrap(); // Follow foo.tld HTTPS records let endpoint = Endpoint::parse( diff --git a/pkarr/src/extra/endpoints/mod.rs b/pkarr/src/extra/endpoints/mod.rs index 6df3f45..e79256d 100644 --- a/pkarr/src/extra/endpoints/mod.rs +++ b/pkarr/src/extra/endpoints/mod.rs @@ -29,11 +29,9 @@ impl EndpointsResolver for crate::client::relay::Client { #[cfg(test)] mod tests { - use simple_dns::rdata::AAAA; use super::*; - use crate::dns::rdata::{A, SVCB}; - use crate::dns::{self, rdata::RData}; + use crate::dns::rdata::SVCB; use crate::{mainline::Testnet, Client, Keypair}; use crate::{PublicKey, SignedPacket}; @@ -55,7 +53,7 @@ mod tests { Box::pin(async move { let keypair = Keypair::random(); - let mut packet = dns::Packet::new_reply(0); + let mut builder = SignedPacket::builder(); for _ in 0..branching { let mut svcb = SVCB::new(0, ".".try_into().unwrap()); @@ -73,19 +71,7 @@ mod tests { } for ip in ips.clone() { - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("@").unwrap(), - dns::CLASS::IN, - 3600, - match ip { - IpAddr::V4(address) => RData::A(A { - address: address.into(), - }), - IpAddr::V6(address) => RData::AAAA(AAAA { - address: address.into(), - }), - }, - )); + builder = builder.address("@".try_into().unwrap(), ip, 3600); } } else { let target = generate_subtree( @@ -102,15 +88,11 @@ mod tests { svcb.target = target.try_into().unwrap(); }; - packet.answers.push(dns::ResourceRecord::new( - dns::Name::new("@").unwrap(), - dns::CLASS::IN, - 3600, - RData::HTTPS(svcb.into()), - )); + builder = builder.https("@".try_into().unwrap(), svcb, 3600); } - let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); + let signed_packet = builder.sign(&keypair).unwrap(); + client.publish(&signed_packet).await.unwrap(); keypair.public_key() diff --git a/pkarr/src/extra/mod.rs b/pkarr/src/extra/mod.rs index 5556736..ff96045 100644 --- a/pkarr/src/extra/mod.rs +++ b/pkarr/src/extra/mod.rs @@ -48,36 +48,17 @@ mod tests { use axum::{routing::get, Router}; use axum_server::tls_rustls::RustlsConfig; - use crate::{ - dns::{rdata::SVCB, Packet}, - Client, Keypair, SignedPacket, - }; + use crate::{dns::rdata::SVCB, Client, Keypair, SignedPacket}; async fn publish_server_pkarr(client: &Client, keypair: &Keypair, socket_addr: &SocketAddr) { - let mut packet = Packet::new_reply(1); - let mut svcb = SVCB::new(0, ".".try_into().unwrap()); - svcb.set_port(socket_addr.port()); - packet.answers.push(crate::dns::ResourceRecord::new( - "@".try_into().unwrap(), - crate::dns::CLASS::IN, - 60 * 60, - crate::dns::rdata::RData::HTTPS(svcb.into()), - )); - - packet.answers.push(crate::dns::ResourceRecord::new( - "@".try_into().unwrap(), - crate::dns::CLASS::IN, - 60 * 60, - match socket_addr.ip() { - std::net::IpAddr::V4(ip) => crate::dns::rdata::RData::A(ip.into()), - std::net::IpAddr::V6(ip) => crate::dns::rdata::RData::AAAA(ip.into()), - }, - )); - - let signed_packet = SignedPacket::from_packet(&keypair, &packet).unwrap(); + let signed_packet = SignedPacket::builder() + .https("@".try_into().unwrap(), svcb, 60 * 60) + .address("@".try_into().unwrap(), socket_addr.ip(), 60 * 60) + .sign(&keypair) + .unwrap(); client.publish(&signed_packet).await.unwrap(); } From fef623bfc990e7c9d456a555fe9185e07a574615 Mon Sep 17 00:00:00 2001 From: nazeh Date: Sat, 30 Nov 2024 20:35:17 +0300 Subject: [PATCH 80/84] feat(pkarr): remove openssl dependency --- Cargo.lock | 211 ----------------------------------------------- pkarr/Cargo.toml | 4 +- 2 files changed, 2 insertions(+), 213 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 59b16e3..2fc7497 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -447,22 +447,6 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" -[[package]] -name = "core-foundation" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - [[package]] name = "cpufeatures" version = "0.2.16" @@ -683,15 +667,6 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" -[[package]] -name = "encoding_rs" -version = "0.8.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" -dependencies = [ - "cfg-if", -] - [[package]] name = "equivalent" version = "1.0.1" @@ -738,21 +713,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1151,22 +1111,6 @@ dependencies = [ "webpki-roots", ] -[[package]] -name = "hyper-tls" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" -dependencies = [ - "bytes", - "http-body-util", - "hyper", - "hyper-util", - "native-tls", - "tokio", - "tokio-native-tls", - "tower-service", -] - [[package]] name = "hyper-util" version = "0.1.10" @@ -1583,23 +1527,6 @@ dependencies = [ "getrandom", ] -[[package]] -name = "native-tls" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" -dependencies = [ - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "no-std-compat" version = "0.4.1" @@ -1653,50 +1580,6 @@ version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" -[[package]] -name = "openssl" -version = "0.10.68" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" -dependencies = [ - "bitflags", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "openssl-probe" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" - -[[package]] -name = "openssl-sys" -version = "0.9.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "overload" version = "0.1.1" @@ -1909,12 +1792,6 @@ dependencies = [ "spki", ] -[[package]] -name = "pkg-config" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" - [[package]] name = "portable-atomic" version = "1.9.0" @@ -2163,22 +2040,18 @@ checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" dependencies = [ "base64", "bytes", - "encoding_rs", "futures-core", "futures-util", - "h2", "http", "http-body", "http-body-util", "hyper", "hyper-rustls", - "hyper-tls", "hyper-util", "ipnet", "js-sys", "log", "mime", - "native-tls", "once_cell", "percent-encoding", "pin-project-lite", @@ -2190,9 +2063,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "sync_wrapper 1.0.2", - "system-configuration", "tokio", - "tokio-native-tls", "tokio-rustls", "tower-service", "url", @@ -2316,44 +2187,12 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" -[[package]] -name = "schannel" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1" -dependencies = [ - "windows-sys 0.59.0", -] - [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "security-framework" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" -dependencies = [ - "bitflags", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa39c7303dc58b5543c94d22c1766b0d31f2ee58306363ea622b10bbc075eaa2" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "self_cell" version = "1.0.4" @@ -2636,40 +2475,6 @@ dependencies = [ "syn", ] -[[package]] -name = "system-configuration" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" -dependencies = [ - "bitflags", - "core-foundation", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "tempfile" -version = "3.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" -dependencies = [ - "cfg-if", - "fastrand", - "once_cell", - "rustix", - "windows-sys 0.59.0", -] - [[package]] name = "thiserror" version = "1.0.69" @@ -2774,16 +2579,6 @@ dependencies = [ "syn", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.26.0" @@ -3039,12 +2834,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - [[package]] name = "version_check" version = "0.9.5" diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index d2fb788..6460d5e 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -46,7 +46,7 @@ pubky-timestamp = { version = "0.2.0", default-features = false } mainline = { version = "4.1.0", optional = true } # feat: relay dependencies -reqwest = { version = "0.12.9", features = ["rustls-tls"], optional = true } +reqwest = { version = "0.12.9", default-features = false, features = ["rustls-tls"], optional = true } # feat: tls rustls = { version = "0.23", default-features = false, features = ["ring"], optional = true } @@ -62,7 +62,7 @@ pubky-timestamp = { version = "0.2.0", default-features = false, features = ["ht js-sys = "0.3.73" futures = "0.3.31" getrandom = { version = "0.2", features = ["js"] } -reqwest = "0.12.9" +reqwest = { version = "0.12.9", default-features = false } sha1_smol = "1.0.1" wasm-bindgen-futures = "0.4.46" From 8a364dca392cc75cf64a77532b91a89f690b4e81 Mon Sep 17 00:00:00 2001 From: nazeh Date: Sun, 1 Dec 2024 11:07:43 +0300 Subject: [PATCH 81/84] feat: update dependencies and fixes accordingly --- CHANGELOG.md | 2 + Cargo.lock | 44 ++++++------- pkarr/Cargo.toml | 6 +- pkarr/examples/http-serve.rs | 4 +- pkarr/src/base/signed_packet.rs | 93 +++++++++++++++++++++++++-- pkarr/src/extra/endpoints/endpoint.rs | 24 ++++--- pkarr/src/extra/endpoints/mod.rs | 4 +- pkarr/src/extra/mod.rs | 4 +- server/Cargo.toml | 26 ++++---- 9 files changed, 149 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index caff1a4..bdf4daa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ All notable changes to pkarr client and server will be documented in this file. ### Added - Add `SignedPacket::builder()` and convenient methods to create `A`, `AAAA`, `CNAME`, `TXT`, `SVCB`, and `HTTPS` records. +- Add `SignedPacket::all_resource_records()` to access all resource records without accessing the dns packet. - Use `pubky_timestamp::Timestamp` - Impl `PartialEq, Eq` for `SignedPacket`. - Impl `From` for `CacheKey`. @@ -29,6 +30,7 @@ All notable changes to pkarr client and server will be documented in this file. - `Client::shutdown` and `Client::shutdown_sync` are now idempotent and return `()`. - `Client::resolve`, `Client::resolve_sync` and `relay::Client::resolve` return expired cached `SignedPacket` _before_ making query to the network (Relays/Resolvers/Dht). - Export `Settings` as client builder. +- Update `simple-dns` so you can't use `Name::new("@")`, instead you should use `Name::new(".")`, `SignedPacket::resource_records("@")` still works. ### Removed diff --git a/Cargo.lock b/Cargo.lock index 2fc7497..d33260e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1317,10 +1317,11 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.73" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb15147158e79fd8b8afd0252522769c4f48725460b37338544d8379d94fc8f9" +checksum = "a865e038f7f6ed956f788f0d7d60c541fff74c7bd74272c5d4cf15c63743e705" dependencies = [ + "once_cell", "wasm-bindgen", ] @@ -1773,7 +1774,7 @@ dependencies = [ "pubky-timestamp", "rustls", "serde", - "thiserror 1.0.69", + "thiserror 2.0.3", "tokio", "toml", "tower-http", @@ -2345,9 +2346,9 @@ checksum = "1de1d4f81173b03af4c0cbed3c898f6bff5b870e4a7f5d6f4057d62a7a4b686e" [[package]] name = "simple-dns" -version = "0.8.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92bc7a4124e416342635ff7090cf6883a1835a91a7d2aa01c54c458a650781be" +checksum = "b8f1740a36513fc97c5309eb1b8e8f108b0e95899c66c23fd7259625d4fdb686" dependencies = [ "bitflags", ] @@ -2670,15 +2671,14 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.5.2" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" +checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697" dependencies = [ "bitflags", "bytes", "http", "http-body", - "http-body-util", "pin-project-lite", "tower-layer", "tower-service", @@ -2857,9 +2857,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.96" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21d3b25c3ea1126a2ad5f4f9068483c2af1e64168f847abe863a526b8dbfe00b" +checksum = "d15e63b4482863c109d70a7b8706c1e364eb6ea449b201a76c5b89cedcec2d5c" dependencies = [ "cfg-if", "once_cell", @@ -2868,9 +2868,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.96" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52857d4c32e496dc6537646b5b117081e71fd2ff06de792e3577a150627db283" +checksum = "8d36ef12e3aaca16ddd3f67922bc63e48e953f126de60bd33ccc0101ef9998cd" dependencies = [ "bumpalo", "log", @@ -2883,9 +2883,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.46" +version = "0.4.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "951fe82312ed48443ac78b66fa43eded9999f738f6022e67aead7b708659e49a" +checksum = "9dfaf8f50e5f293737ee323940c7d8b08a66a95a419223d9f41610ca08b0833d" dependencies = [ "cfg-if", "js-sys", @@ -2896,9 +2896,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.96" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "920b0ffe069571ebbfc9ddc0b36ba305ef65577c94b06262ed793716a1afd981" +checksum = "705440e08b42d3e4b36de7d66c944be628d579796b8090bfa3471478a2260051" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2906,9 +2906,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.96" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf59002391099644be3524e23b781fa43d2be0c5aa0719a18c0731b9d195cab6" +checksum = "98c9ae5a76e46f4deecd0f0255cc223cfa18dc9b261213b8aa0c7b36f61b3f1d" dependencies = [ "proc-macro2", "quote", @@ -2919,15 +2919,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.96" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5047c5392700766601942795a436d7d2599af60dcc3cc1248c9120bfb0827b0" +checksum = "6ee99da9c5ba11bd675621338ef6fa52296b76b83305e9b6e5c77d4c286d6d49" [[package]] name = "web-sys" -version = "0.3.73" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "476364ff87d0ae6bfb661053a9104ab312542658c3d8f963b7ace80b6f9b26b9" +checksum = "a98bc3c33f0fe7e59ad7cd041b89034fa82a7c2d4365ca538dda6cdaf513863c" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index 6460d5e..4669295 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -14,7 +14,7 @@ base32 = "0.5.1" bytes = "1.9.0" ed25519-dalek = { version = "2.1.1", features = ["rand_core"] } self_cell = "1.0.4" -simple-dns = "0.8.0" +simple-dns = "0.9.1" thiserror = "2.0.3" tracing = "0.1.41" dyn-clone = "1.0.17" @@ -59,12 +59,12 @@ libc = { version = "0.2.167", optional = true } [target.'cfg(target_arch = "wasm32")'.dependencies] pubky-timestamp = { version = "0.2.0", default-features = false, features = ["httpdate"] } -js-sys = "0.3.73" +js-sys = "0.3.74" futures = "0.3.31" getrandom = { version = "0.2", features = ["js"] } reqwest = { version = "0.12.9", default-features = false } sha1_smol = "1.0.1" -wasm-bindgen-futures = "0.4.46" +wasm-bindgen-futures = "0.4.47" [dev-dependencies] anyhow = "1.0.93" diff --git a/pkarr/examples/http-serve.rs b/pkarr/examples/http-serve.rs index a055a44..facb8b1 100644 --- a/pkarr/examples/http-serve.rs +++ b/pkarr/examples/http-serve.rs @@ -67,8 +67,8 @@ async fn publish_server_pkarr(client: &Client, keypair: &Keypair, socket_addr: & svcb.set_port(socket_addr.port()); let signed_packet = SignedPacket::builder() - .https("@".try_into().unwrap(), svcb, 60 * 60) - .address("@".try_into().unwrap(), socket_addr.ip(), 60 * 60) + .https(".".try_into().unwrap(), svcb, 60 * 60) + .address(".".try_into().unwrap(), socket_addr.ip(), 60 * 60) .sign(&keypair) .unwrap(); diff --git a/pkarr/src/base/signed_packet.rs b/pkarr/src/base/signed_packet.rs index 0cdf2d9..c61029b 100644 --- a/pkarr/src/base/signed_packet.rs +++ b/pkarr/src/base/signed_packet.rs @@ -30,11 +30,17 @@ impl SignedPacketBuilder { } /// Insert any type of [RData] + /// + /// You can set the name to `.` to point ot the Apex + /// (the public key, of the keypair used in [Self::sign]) pub fn rdata(self, name: Name<'_>, rdata: RData, ttl: u32) -> Self { self.record(ResourceRecord::new(name.to_owned(), CLASS::IN, ttl, rdata)) } /// Insert an `A` record. + /// + /// You can set the name to `.` to point ot the Apex + /// (the public key, of the keypair used in [Self::sign]) pub fn a(self, name: Name<'_>, address: Ipv4Addr, ttl: u32) -> Self { self.rdata( name, @@ -46,6 +52,9 @@ impl SignedPacketBuilder { } /// Insert an `AAAA` record. + /// + /// You can set the name to `.` to point ot the Apex + /// (the public key, of the keypair used in [Self::sign]) pub fn aaaa(self, name: Name<'_>, address: Ipv6Addr, ttl: u32) -> Self { self.rdata( name, @@ -57,6 +66,9 @@ impl SignedPacketBuilder { } /// Insert an `A` or `AAAA` record. + /// + /// You can set the name to `.` to point ot the Apex + /// (the public key, of the keypair used in [Self::sign]) pub fn address(self, name: Name<'_>, address: IpAddr, ttl: u32) -> Self { match address { IpAddr::V4(addr) => self.a(name, addr, ttl), @@ -65,21 +77,33 @@ impl SignedPacketBuilder { } /// Insert a `CNAME` record. + /// + /// You can set the name to `.` to point ot the Apex + /// (the public key, of the keypair used in [Self::sign]) pub fn cname(self, name: Name<'_>, cname: Name<'_>, ttl: u32) -> Self { self.rdata(name, RData::CNAME(cname.into()), ttl) } /// Insert a `TXT` record. + /// + /// You can set the name to `.` to point ot the Apex + /// (the public key, of the keypair used in [Self::sign]) pub fn txt(self, name: Name<'_>, text: TXT<'_>, ttl: u32) -> Self { self.rdata(name, RData::TXT(text), ttl) } /// Insert an `HTTPS` record + /// + /// You can set the name to `.` to point ot the Apex + /// (the public key, of the keypair used in [Self::sign]) pub fn https(self, name: Name<'_>, svcb: SVCB, ttl: u32) -> Self { self.rdata(name, RData::HTTPS(HTTPS(svcb)), ttl) } /// Insert an `SVCB record + /// + /// You can set the name to `.` to point ot the Apex + /// (the public key, of the keypair used in [Self::sign]) pub fn svcb(self, name: Name<'_>, svcb: SVCB, ttl: u32) -> Self { self.rdata(name, RData::SVCB(svcb), ttl) } @@ -91,6 +115,8 @@ impl SignedPacketBuilder { /// Create a [Packet] from the [ResourceRecord]s inserted so far and sign /// it with the given [Keypair]. + /// + /// Read more about how names will be normalized in [SignedPacket::from_answers]. pub fn sign(self, keypair: &Keypair) -> Result { SignedPacket::from_answers(keypair, &self.0) } @@ -142,6 +168,54 @@ pub struct SignedPacket { impl SignedPacket { pub const MAX_BYTES: u64 = 1104; + /// Create a [SignedPacket] using a builder. + /// + /// ``` + /// use pkarr::{SignedPacket, Keypair}; + /// + /// let keypair = Keypair::random(); + /// + /// let signed_packet = SignedPacket::builder() + /// // A record + /// .address( + /// "_derp_region.iroh.".try_into().unwrap(), + /// "1.1.1.1".parse().unwrap(), + /// 30, + /// ) + /// // AAAA record + /// .address( + /// "_derp_region.iroh.".try_into().unwrap(), + /// "3002:0bd6:0000:0000:0000:ee00:0033:6778".parse().unwrap(), + /// 30, + /// ) + /// // CNAME record + /// .cname( + /// "subdomain.".try_into().unwrap(), + /// "example.com".try_into().unwrap(), + /// 30 + /// ) + /// // TXT record + /// .txt( + /// "_proto".try_into().unwrap(), + /// "foo=bar".try_into().unwrap(), + /// 30 + /// ) + /// // HTTPS record + /// .https( + /// // You can make a record at the Apex (at the same TLD as your public key) + /// ".".try_into().unwrap(), + /// SVCB::new(0, "https.example.com".try_into().unwrap()), + /// 3600, + /// ) + /// // SVCB record + /// .https( + /// ".".try_into().unwrap(), + /// SVCB::new(0, "https.example.com".try_into().unwrap()), + /// 3600, + /// ) + /// .sign(&keypair) + /// .unwrap(); + /// ``` pub fn builder() -> SignedPacketBuilder { SignedPacketBuilder::default() } @@ -164,6 +238,8 @@ impl SignedPacket { /// /// It will also normalize the names of the [ResourceRecord]s to be relative to the origin, /// which would be the z-base32 encoded [PublicKey] of the [Keypair] used to sign the Packet. + /// + /// If any name is empty or just a `.`, it will be normalized to the public key of the keypair. fn from_answers( keypair: &Keypair, answers: &[ResourceRecord<'_>], @@ -330,12 +406,12 @@ impl SignedPacket { /// Return and iterator over the [ResourceRecord]s in the Answers section of the DNS [Packet] /// that matches the given name. The name will be normalized to the origin TLD of this packet. + /// + /// You can use `@` to filter the resource records at the Apex (the public key). pub fn resource_records(&self, name: &str) -> impl Iterator { let origin = self.public_key().to_z32(); let normalized_name = normalize_name(&origin, name.to_string()); - self.packet() - .answers - .iter() + self.all_resource_records() .filter(move |rr| rr.name == Name::new(&normalized_name).unwrap()) } @@ -345,11 +421,16 @@ impl SignedPacket { let origin = self.public_key().to_z32(); let normalized_name = normalize_name(&origin, name.to_string()); - self.packet().answers.iter().filter(move |rr| { + self.all_resource_records().filter(move |rr| { rr.name == Name::new(&normalized_name).unwrap() && rr.ttl > self.elapsed() }) } + /// Returns all resource records in this packet + pub fn all_resource_records(&self) -> impl Iterator { + self.packet().answers.iter() + } + /// calculates the remaining seconds by comparing the [Self::ttl] (clamped by `min` and `max`) /// to the [Self::last_seen]. /// @@ -757,8 +838,8 @@ mod tests { let keypair = Keypair::random(); let signed_packet = SignedPacket::builder() - .cname("@".try_into().unwrap(), "foobar".try_into().unwrap(), 30) - .cname("@".try_into().unwrap(), "foobar".try_into().unwrap(), 30) + .cname(".".try_into().unwrap(), "foobar".try_into().unwrap(), 30) + .cname(".".try_into().unwrap(), "foobar".try_into().unwrap(), 30) .sign(&keypair) .unwrap(); diff --git a/pkarr/src/extra/endpoints/endpoint.rs b/pkarr/src/extra/endpoints/endpoint.rs index 1c08b6f..3e72cf5 100644 --- a/pkarr/src/extra/endpoints/endpoint.rs +++ b/pkarr/src/extra/endpoints/endpoint.rs @@ -67,6 +67,12 @@ impl Endpoint { .map(|s| { let target = s.target.to_string(); + let target = if target == "." || target.is_empty() { + ".".to_string() + } else { + target + }; + let port = s .get_param(SVCB::PORT) .map(|bytes| { @@ -78,15 +84,17 @@ impl Endpoint { }) .unwrap_or_default(); + let addrs = if &target == "." { + addrs.clone() + } else { + Vec::with_capacity(0) + }; + Endpoint { target, port, public_key: signed_packet.public_key(), - addrs: if s.target.to_string() == "." { - addrs.clone() - } else { - Vec::with_capacity(0) - }, + addrs, } }) .collect::>() @@ -213,16 +221,16 @@ mod tests { let keypair = Keypair::random(); let signed_packet = SignedPacket::builder() .address( - "@".try_into().unwrap(), + ".".try_into().unwrap(), "209.151.148.15".parse().unwrap(), 3600, ) .address( - "@".try_into().unwrap(), + ".".try_into().unwrap(), "2a05:d014:275:6201::64".parse().unwrap(), 3600, ) - .https("@".try_into().unwrap(), svcb, 3600) + .https(".".try_into().unwrap(), svcb, 3600) .sign(&keypair) .unwrap(); diff --git a/pkarr/src/extra/endpoints/mod.rs b/pkarr/src/extra/endpoints/mod.rs index e79256d..c63a785 100644 --- a/pkarr/src/extra/endpoints/mod.rs +++ b/pkarr/src/extra/endpoints/mod.rs @@ -71,7 +71,7 @@ mod tests { } for ip in ips.clone() { - builder = builder.address("@".try_into().unwrap(), ip, 3600); + builder = builder.address(".".try_into().unwrap(), ip, 3600); } } else { let target = generate_subtree( @@ -88,7 +88,7 @@ mod tests { svcb.target = target.try_into().unwrap(); }; - builder = builder.https("@".try_into().unwrap(), svcb, 3600); + builder = builder.https(".".try_into().unwrap(), svcb, 3600); } let signed_packet = builder.sign(&keypair).unwrap(); diff --git a/pkarr/src/extra/mod.rs b/pkarr/src/extra/mod.rs index ff96045..3ec4dff 100644 --- a/pkarr/src/extra/mod.rs +++ b/pkarr/src/extra/mod.rs @@ -55,8 +55,8 @@ mod tests { svcb.set_port(socket_addr.port()); let signed_packet = SignedPacket::builder() - .https("@".try_into().unwrap(), svcb, 60 * 60) - .address("@".try_into().unwrap(), socket_addr.ip(), 60 * 60) + .https(".".try_into().unwrap(), svcb, 60 * 60) + .address(".".try_into().unwrap(), socket_addr.ip(), 60 * 60) .sign(&keypair) .unwrap(); diff --git a/server/Cargo.toml b/server/Cargo.toml index c1b45ac..9da6dd4 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -7,22 +7,22 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -anyhow = "1.0.82" -axum = "0.7.5" -tokio = { version = "1.37.0", features = ["full"] } -tower-http = { version = "0.5.2", features = ["cors", "trace"] } -tracing = "0.1.40" -tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } -axum-server = { version = "0.7.0", features = ["tls-rustls-no-provider"] } +anyhow = "1.0.93" +axum = "0.7.9" +tokio = { version = "1.41.1", features = ["full"] } +tower-http = { version = "0.6.2", features = ["cors", "trace"] } +tracing = "0.1.41" +tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } +axum-server = { version = "0.7.1", features = ["tls-rustls-no-provider"] } rustls = { version = "0.23", default-features = false, features = ["ring"] } http = "1.1.0" -thiserror = "1.0.49" -bytes = "1.7.1" -tower_governor = "0.4.2" +thiserror = "2.0.3" +bytes = "1.9.0" +tower_governor = "0.4.3" governor = "0.6.3" -serde = { version = "1.0.199", features = ["derive"] } -toml = "0.8.12" -clap = { version = "4.5.1", features = ["derive"] } +serde = { version = "1.0.215", features = ["derive"] } +toml = "0.8.19" +clap = { version = "4.5.21", features = ["derive"] } dirs-next = "2.0.0" pkarr = { version = "3.0.0", path = "../pkarr", features = ["dht", "lmdb-cache"] } httpdate = "1.0.3" From caa3cb5800b7b38e0115fb5aa73b5c4e344eeec0 Mon Sep 17 00:00:00 2001 From: nazeh Date: Sun, 1 Dec 2024 11:30:58 +0300 Subject: [PATCH 82/84] feat(pkarr): deny unwrap --- pkarr/Cargo.toml | 2 +- pkarr/src/base/cache.rs | 6 +++--- pkarr/src/base/signed_packet.rs | 30 ++++++++++++++++++++---------- pkarr/src/client/dht.rs | 3 ++- pkarr/src/client/relay.rs | 5 +++-- pkarr/src/extra/lmdb_cache.rs | 7 +------ pkarr/src/extra/reqwest.rs | 6 +++++- 7 files changed, 35 insertions(+), 24 deletions(-) diff --git a/pkarr/Cargo.toml b/pkarr/Cargo.toml index 4669295..083db85 100644 --- a/pkarr/Cargo.toml +++ b/pkarr/Cargo.toml @@ -109,5 +109,5 @@ default = ["full"] [package.metadata.docs.rs] all-features = true -# [lints.clippy] +[lints.clippy] unwrap_used = "deny" diff --git a/pkarr/src/base/cache.rs b/pkarr/src/base/cache.rs index 2a14a67..0b7b4f2 100644 --- a/pkarr/src/base/cache.rs +++ b/pkarr/src/base/cache.rs @@ -83,14 +83,14 @@ impl InMemoryCache { impl Cache for InMemoryCache { fn len(&self) -> usize { - self.inner.lock().unwrap().len() + self.inner.lock().expect("mutex lock").len() } /// Puts [SignedPacket], if a version of the packet already exists, /// and it has the same [SignedPacket::as_bytes], then only [SignedPacket::last_seen] will be /// updated, otherwise the input will be cloned. fn put(&self, key: &CacheKey, signed_packet: &SignedPacket) { - let mut lock = self.inner.lock().unwrap(); + let mut lock = self.inner.lock().expect("mutex lock"); match lock.get_mut(key) { Some(existing) => { @@ -108,6 +108,6 @@ impl Cache for InMemoryCache { } fn get(&self, key: &CacheKey) -> Option { - self.inner.lock().unwrap().get(key).cloned() + self.inner.lock().expect("mutex lock").get(key).cloned() } } diff --git a/pkarr/src/base/signed_packet.rs b/pkarr/src/base/signed_packet.rs index c61029b..fe86e05 100644 --- a/pkarr/src/base/signed_packet.rs +++ b/pkarr/src/base/signed_packet.rs @@ -332,13 +332,13 @@ impl SignedPacket { /// Returns the [PublicKey] of the signer of this [SignedPacket] pub fn public_key(&self) -> PublicKey { - PublicKey::try_from(&self.inner.borrow_owner()[0..32]).unwrap() + PublicKey::try_from(&self.inner.borrow_owner()[0..32]).expect("SignedPacket::public_key()") } /// Returns the [Signature] of the the bencoded sequence number concatenated with the /// encoded and compressed packet, as defined in [BEP_0044](https://www.bittorrent.org/beps/bep_0044.html) pub fn signature(&self) -> Signature { - Signature::try_from(&self.inner.borrow_owner()[32..96]).unwrap() + Signature::try_from(&self.inner.borrow_owner()[32..96]).expect("SignedPacket::signature()") } /// Returns the timestamp in microseconds since the [UNIX_EPOCH](std::time::UNIX_EPOCH). @@ -349,7 +349,9 @@ impl SignedPacket { /// which is set when you create a new packet. pub fn timestamp(&self) -> Timestamp { let bytes = self.inner.borrow_owner(); - let slice: [u8; 8] = bytes[96..104].try_into().unwrap(); + let slice: [u8; 8] = bytes[96..104] + .try_into() + .expect("SignedPacket::timestamp()"); u64::from_be_bytes(slice).into() } @@ -412,7 +414,7 @@ impl SignedPacket { let origin = self.public_key().to_z32(); let normalized_name = normalize_name(&origin, name.to_string()); self.all_resource_records() - .filter(move |rr| rr.name == Name::new(&normalized_name).unwrap()) + .filter(move |rr| rr.name.to_string() == normalized_name) } /// Similar to [resource_records](SignedPacket::resource_records), but filters out @@ -421,9 +423,8 @@ impl SignedPacket { let origin = self.public_key().to_z32(); let normalized_name = normalize_name(&origin, name.to_string()); - self.all_resource_records().filter(move |rr| { - rr.name == Name::new(&normalized_name).unwrap() && rr.ttl > self.elapsed() - }) + self.all_resource_records() + .filter(move |rr| rr.name.to_string() == normalized_name && rr.ttl > self.elapsed()) } /// Returns all resource records in this packet @@ -494,8 +495,16 @@ impl SignedPacket { return Err(SignedPacketError::PacketTooLarge(bytes.len())); } let public_key = PublicKey::try_from(&bytes[..32])?; - let signature = Signature::from_bytes(bytes[32..96].try_into().unwrap()); - let timestamp = u64::from_be_bytes(bytes[96..104].try_into().unwrap()); + let signature = Signature::from_bytes( + bytes[32..96] + .try_into() + .expect("SignedPacket::from_bytes(); Signature from 64 bytes"), + ); + let timestamp = u64::from_be_bytes( + bytes[96..104] + .try_into() + .expect("SignedPacket::from_bytes(); Timestamp from 8 bytes"), + ); let encoded_packet = &bytes.slice(104..); @@ -511,7 +520,8 @@ impl SignedPacket { /// like ones stored on disk or in a database. fn from_bytes_unchecked(bytes: &Bytes, last_seen: impl Into) -> SignedPacket { SignedPacket { - inner: Inner::try_from_bytes(bytes.to_owned()).unwrap(), + inner: Inner::try_from_bytes(bytes.to_owned()) + .expect("called SignedPacket::from_bytes_unchecked on invalid bytes"), last_seen: last_seen.into(), } } diff --git a/pkarr/src/client/dht.rs b/pkarr/src/client/dht.rs index 78a25db..6c59561 100644 --- a/pkarr/src/client/dht.rs +++ b/pkarr/src/client/dht.rs @@ -52,7 +52,8 @@ impl Default for Settings { fn default() -> Self { Self { dht_settings: mainline::Dht::builder(), - cache_size: NonZeroUsize::new(DEFAULT_CACHE_SIZE).unwrap(), + cache_size: NonZeroUsize::new(DEFAULT_CACHE_SIZE) + .expect("NonZeroUsize from DEFAULT_CACHE_SIZE"), resolvers: Some( DEFAULT_RESOLVERS .iter() diff --git a/pkarr/src/client/relay.rs b/pkarr/src/client/relay.rs index 955b2dc..9c3ac6e 100644 --- a/pkarr/src/client/relay.rs +++ b/pkarr/src/client/relay.rs @@ -42,7 +42,8 @@ impl Default for Settings { fn default() -> Self { Self { relays: DEFAULT_RELAYS.map(|s| s.into()).to_vec(), - cache_size: NonZeroUsize::new(DEFAULT_CACHE_SIZE).unwrap(), + cache_size: NonZeroUsize::new(DEFAULT_CACHE_SIZE) + .expect("NonZeroUsize from DEFAULT_CACHE_SIZE"), minimum_ttl: DEFAULT_MINIMUM_TTL, maximum_ttl: DEFAULT_MAXIMUM_TTL, http_client: reqwest::Client::new(), @@ -107,7 +108,7 @@ pub struct Client { impl Default for Client { fn default() -> Self { - Self::new(Settings::default()).unwrap() + Self::new(Settings::default()).expect("Pkarr Relay client default") } } diff --git a/pkarr/src/extra/lmdb_cache.rs b/pkarr/src/extra/lmdb_cache.rs index 6939514..da8c41d 100644 --- a/pkarr/src/extra/lmdb_cache.rs +++ b/pkarr/src/extra/lmdb_cache.rs @@ -214,12 +214,7 @@ impl LmdbCache { ) -> Result, heed::Error> { let rtxn = self.env.read_txn()?; - let packets: SignedPacketsTable = self - .env - .open_database(&rtxn, Some(SIGNED_PACKET_TABLE))? - .unwrap(); - - if let Some(signed_packet) = packets.get(&rtxn, key)? { + if let Some(signed_packet) = self.signed_packets_table.get(&rtxn, key)? { return Ok(Some(signed_packet)); } diff --git a/pkarr/src/extra/reqwest.rs b/pkarr/src/extra/reqwest.rs index 575df79..9fb3583 100644 --- a/pkarr/src/extra/reqwest.rs +++ b/pkarr/src/extra/reqwest.rs @@ -37,5 +37,9 @@ async fn resolve( return Ok(Box::new(addrs.into_iter())); }; - Ok(Box::new(format!("{name}:0").to_socket_addrs().unwrap())) + Ok(Box::new( + format!("{name}:0") + .to_socket_addrs() + .expect("formatting a name and port to socket address"), + )) } From 9d62e695a2254e578c37332e795c8be6fe15d850 Mon Sep 17 00:00:00 2001 From: nazeh Date: Sun, 1 Dec 2024 17:43:13 +0300 Subject: [PATCH 83/84] feat(pkarr): add a batch to LmdbCache to close #100 and fix lru bug --- pkarr/src/extra/lmdb_cache.rs | 193 +++++++++++++++++++++++++++++----- 1 file changed, 166 insertions(+), 27 deletions(-) diff --git a/pkarr/src/extra/lmdb_cache.rs b/pkarr/src/extra/lmdb_cache.rs index da8c41d..3d1cdc0 100644 --- a/pkarr/src/extra/lmdb_cache.rs +++ b/pkarr/src/extra/lmdb_cache.rs @@ -1,9 +1,17 @@ //! Persistent [crate::base::cache::Cache] implementation using LMDB's bindings [heed] -use std::{borrow::Cow, fs, path::Path, time::Duration}; +use std::{ + borrow::Cow, + fs, + path::Path, + sync::{Arc, RwLock}, + time::Duration, +}; use byteorder::LittleEndian; -use heed::{types::U64, BoxedError, BytesDecode, BytesEncode, Database, Env, EnvOpenOptions}; +use heed::{ + types::U64, BoxedError, BytesDecode, BytesEncode, Database, Env, EnvOpenOptions, RwTxn, +}; use libc::{sysconf, _SC_PAGESIZE}; use tracing::debug; @@ -71,6 +79,7 @@ pub struct LmdbCache { signed_packets_table: SignedPacketsTable, key_to_time_table: KeyToTimeTable, time_to_key_table: TimeToKeyTable, + batch: Arc>>, } impl LmdbCache { @@ -115,6 +124,7 @@ impl LmdbCache { signed_packets_table, key_to_time_table, time_to_key_table, + batch: Arc::new(RwLock::new(vec![])), }; let clone = instance.clone(); @@ -149,12 +159,15 @@ impl LmdbCache { let key_to_time = self.key_to_time_table; let time_to_key = self.time_to_key_table; + let batch = self.batch.read().expect("LmdbCache::batch.read()"); + update_lru(&mut wtxn, packets, key_to_time, time_to_key, &batch)?; + let len = packets.len(&wtxn)? as usize; if len >= self.capacity { debug!(?len, ?self.capacity, "Reached cache capacity, deleting extra item."); - let mut iter = time_to_key.rev_iter(&wtxn)?; + let mut iter = time_to_key.iter(&wtxn)?; if let Some((time, key)) = iter.next().transpose()? { drop(iter); @@ -182,30 +195,12 @@ impl LmdbCache { } pub fn internal_get(&self, key: &CacheKey) -> Result, heed::Error> { - let mut wtxn = self.env.write_txn()?; - - let packets = self.signed_packets_table; - let key_to_time = self.key_to_time_table; - let time_to_key = self.time_to_key_table; - - if let Some(signed_packet) = packets.get(&wtxn, key)? { - if let Some(time) = key_to_time.get(&wtxn, key)? { - time_to_key.delete(&mut wtxn, &time)?; - }; - - let new_time = Timestamp::now(); + self.batch + .write() + .expect("LmdbCache::batch.write()") + .push(*key); - time_to_key.put(&mut wtxn, &new_time.as_u64(), key)?; - key_to_time.put(&mut wtxn, key, &new_time.as_u64())?; - - wtxn.commit()?; - - return Ok(Some(signed_packet)); - } - - wtxn.commit()?; - - Ok(None) + self.internal_get_read_only(key) } pub fn internal_get_read_only( @@ -224,6 +219,29 @@ impl LmdbCache { } } +fn update_lru( + wtxn: &mut RwTxn, + packets: SignedPacketsTable, + key_to_time: KeyToTimeTable, + time_to_key: TimeToKeyTable, + to_update: &[CacheKey], +) -> Result<(), heed::Error> { + for key in to_update { + if packets.get(wtxn, key)?.is_some() { + if let Some(time) = key_to_time.get(wtxn, key)? { + time_to_key.delete(wtxn, &time)?; + }; + + let new_time = Timestamp::now(); + + time_to_key.put(wtxn, &new_time.as_u64(), key)?; + key_to_time.put(wtxn, key, &new_time.as_u64())?; + } + } + + Ok(()) +} + impl Cache for LmdbCache { fn len(&self) -> usize { match self.internal_len() { @@ -278,7 +296,7 @@ pub enum Error { #[cfg(test)] mod tests { - use std::usize; + use crate::Keypair; use super::*; @@ -288,4 +306,125 @@ mod tests { LmdbCache::new(&env_path, usize::MAX).unwrap(); } + + #[test] + fn lru_capacity() { + let env_path = std::env::temp_dir().join(Timestamp::now().to_string()); + + let cache = LmdbCache::new(&env_path, 2).unwrap(); + + let mut keys = vec![]; + + for i in 0..2 { + let signed_packet = SignedPacket::builder() + .txt("foo".try_into().unwrap(), "bar".try_into().unwrap(), i) + .sign(&Keypair::random()) + .unwrap(); + + let key = CacheKey::from(signed_packet.public_key()); + cache.put(&key, &signed_packet); + + keys.push((key, signed_packet)); + } + + assert_eq!( + cache.get_read_only(&keys.first().unwrap().0).unwrap(), + keys.first().unwrap().1, + "first key saved" + ); + assert_eq!( + cache.get_read_only(&keys.last().unwrap().0).unwrap(), + keys.last().unwrap().1, + "second key saved" + ); + + assert_eq!(cache.len(), 2); + + // Put another one, effectively deleting the oldest. + let signed_packet = SignedPacket::builder() + .txt("foo".try_into().unwrap(), "bar".try_into().unwrap(), 3) + .sign(&Keypair::random()) + .unwrap(); + let key = CacheKey::from(signed_packet.public_key()); + cache.put(&key, &signed_packet); + + assert_eq!(cache.len(), 2); + + assert!( + cache.get_read_only(&keys.first().unwrap().0).is_none(), + "oldest key dropped" + ); + assert_eq!( + cache.get_read_only(&keys.last().unwrap().0).unwrap(), + keys.last().unwrap().1, + "more recent key survived" + ); + assert_eq!( + cache.get_read_only(&key).unwrap(), + signed_packet, + "most recent key survived" + ) + } + + #[test] + fn lru_capacity_refresh_oldest() { + let env_path = std::env::temp_dir().join(Timestamp::now().to_string()); + + let cache = LmdbCache::new(&env_path, 2).unwrap(); + + let mut keys = vec![]; + + for i in 0..2 { + let signed_packet = SignedPacket::builder() + .txt("foo".try_into().unwrap(), "bar".try_into().unwrap(), i) + .sign(&Keypair::random()) + .unwrap(); + + let key = CacheKey::from(signed_packet.public_key()); + cache.put(&key, &signed_packet); + + keys.push((key, signed_packet)); + } + + assert_eq!( + cache.get_read_only(&keys.first().unwrap().0).unwrap(), + keys.first().unwrap().1, + "first key saved" + ); + assert_eq!( + cache.get_read_only(&keys.last().unwrap().0).unwrap(), + keys.last().unwrap().1, + "second key saved" + ); + + // refresh the oldest + cache.get(&keys.first().unwrap().0).unwrap(); + + assert_eq!(cache.len(), 2); + + // Put another one, effectively deleting the oldest. + let signed_packet = SignedPacket::builder() + .txt("foo".try_into().unwrap(), "bar".try_into().unwrap(), 3) + .sign(&Keypair::random()) + .unwrap(); + let key = CacheKey::from(signed_packet.public_key()); + cache.put(&key, &signed_packet); + + assert_eq!(cache.len(), 2); + + assert!( + cache.get_read_only(&keys.last().unwrap().0).is_none(), + "oldest key dropped" + ); + assert_eq!( + cache.get_read_only(&keys.first().unwrap().0).unwrap(), + keys.first().unwrap().1, + "refreshed key survived" + ); + assert_eq!( + cache.get_read_only(&key).unwrap(), + signed_packet, + "most recent key survived" + ) + } } From 7b3d13472639699a68416629e126de069836cc33 Mon Sep 17 00:00:00 2001 From: nazeh Date: Sun, 1 Dec 2024 17:56:13 +0300 Subject: [PATCH 84/84] docs(pkarr): fix pkarr doctest --- pkarr/src/base/signed_packet.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkarr/src/base/signed_packet.rs b/pkarr/src/base/signed_packet.rs index fe86e05..1581289 100644 --- a/pkarr/src/base/signed_packet.rs +++ b/pkarr/src/base/signed_packet.rs @@ -171,7 +171,7 @@ impl SignedPacket { /// Create a [SignedPacket] using a builder. /// /// ``` - /// use pkarr::{SignedPacket, Keypair}; + /// use pkarr::{SignedPacket, Keypair, dns::rdata::SVCB}; /// /// let keypair = Keypair::random(); ///