From 90a309452812ffb365919594453934a419f5de57 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Wed, 27 Nov 2024 23:20:09 +0800 Subject: [PATCH] chore: remove libipld usage from bitswap module --- Cargo.lock | 6 +- Cargo.toml | 26 ++++-- interop-tests/Cargo.toml | 18 ++--- interop-tests/src/tests/bitswap_go_compat.rs | 12 ++- src/chain/store/chain_store.rs | 4 +- src/db/car/many.rs | 6 +- src/db/memory.rs | 4 +- src/db/parity_db.rs | 6 +- src/libp2p_bitswap/internals/prefix.rs | 8 +- src/libp2p_bitswap/mod.rs | 2 +- src/libp2p_bitswap/request_manager.rs | 2 +- src/libp2p_bitswap/store.rs | 83 +++++++++++++++++--- src/libp2p_bitswap/tests/request_manager.rs | 24 +++--- 13 files changed, 132 insertions(+), 69 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c85c26c633b0..8f5e983fca91 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3299,10 +3299,10 @@ name = "forest-interop-tests" version = "0.1.0" dependencies = [ "anyhow", + "cid", "flume 0.11.1", "forest-filecoin", "futures", - "libipld", "libp2p", "libp2p-swarm-test", "rust2go", @@ -8003,9 +8003,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.18" +version = "0.23.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c9cc1d47e243d655ace55ed38201c19ae02c148ae56412ab8750e8f0166ab7f" +checksum = "934b404430bb06b3fae2cba809eb45a1ab1aecd64491213d7c3301b88393f8d1" dependencies = [ "log", "once_cell", diff --git a/Cargo.toml b/Cargo.toml index 1947bd1479a1..1fb420c25e99 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,10 +8,20 @@ license = "MIT OR Apache-2.0" description = "Rust Filecoin implementation." exclude = [".config", ".github", ".maintain", "documentation", "scripts", "interop-tests", "go.work*"] +[workspace.dependencies] +anyhow = "1" +cid = { version = "0.10", default-features = false, features = ["std"] } +flume = "0.11" +futures = "0.3" +libp2p = { version = "0.54", default-features = false } +libp2p-swarm-test = "0.4" +rust2go = "0.3" +tokio = "1" + [dependencies] ahash = "0.8" anes = "0.2" -anyhow = "1.0" +anyhow = { workspace = true } argon2 = "0.5" async-compression = { version = "0.4", features = ["tokio", "zstd"] } async-fs = "2" @@ -33,7 +43,7 @@ cbor4ii = { version = "0.2", default-features = false, features = ["use_alloc", cfg-if = "1" cfg-vis = "0.3" chrono = { version = "0.4", default-features = false, features = ["clock"] } -cid = { version = "0.10", default-features = false, features = ["std"] } +cid = { workspace = true } clap = { version = "4", features = ["derive"] } colored = "2.0" crypto_secretbox = "0.1" @@ -60,9 +70,9 @@ fil_actor_reward_state = { version = "17.0.3" } fil_actor_system_state = { version = "17.0.3" } fil_actor_verifreg_state = { version = "17.0.3" } fil_actors_shared = { version = "17.0.3", features = ["json"] } -flume = "0.11" +flume = { workspace = true } fs_extra = "1" -futures = "0.3" +futures = { workspace = true } fvm2 = { package = "fvm", version = "~2.9", default-features = false } fvm3 = { package = "fvm", default-features = false, version = "~3.11", features = ["arb"] } fvm4 = { package = "fvm", default-features = false, version = "~4.3.3", features = ["arb", "verify-signature"] } @@ -93,7 +103,7 @@ lazy-regex = "3" libipld = { version = "0.16", default-features = false, features = ["dag-cbor", "dag-json", "derive", "serde-codec"] } libipld-core = { version = "0.16", features = ['arb', 'serde-codec'] } libipld-macro = "0.16" -libp2p = { version = "0.54", default-features = false, features = [ +libp2p = { workspace = true, features = [ 'autonat', 'gossipsub', 'kad', @@ -155,7 +165,7 @@ reqwest = { version = "0.12", default-features = false, features = [ rlimit = "0.10" rlp = "0.6" rs-car-ipfs = "0.3" -rust2go = { version = "0.3" } +rust2go = { workspace = true } sailfish = "0.9" schemars = { version = "0.8", features = ["chrono", "uuid1"] } scopeguard = "1" @@ -222,7 +232,7 @@ fvm_shared4 = { package = "fvm_shared", version = "~4.3.3", features = ["arb", " glob = "0.3" http-range-header = "0.4" insta = { version = "1", features = ["yaml"] } -libp2p-swarm-test = "0.4" +libp2p-swarm-test = { workspace = true } num-bigint = { version = "0.4", features = ['quickcheck'] } petgraph = "0.6" predicates = "3" @@ -236,7 +246,7 @@ syn = { version = "2", default-features = false, features = ["full", "parsing", tokio-test = "0.4" [build-dependencies] -rust2go = { version = "0.3", features = ["build"] } +rust2go = { workspace = true, features = ["build"] } # This needs to be set as default. Otherwise, a regular build or test will produce # gargantuan artifacts (around 70G for all tests). For a debugging session, you can diff --git a/interop-tests/Cargo.toml b/interop-tests/Cargo.toml index 5aaa06da4946..8184740f8587 100644 --- a/interop-tests/Cargo.toml +++ b/interop-tests/Cargo.toml @@ -11,12 +11,12 @@ publish = false [dependencies] [dev-dependencies] -anyhow = "1" -flume = "0.11" +anyhow = { workspace = true } +cid = { workspace = true } +flume = { workspace = true } forest-filecoin = { path = "../", default-features = false, features = ["interop-tests-private", "no-f3-sidecar"] } -futures = "0.3" -libipld = { version = "0.16", default-features = false } -libp2p = { version = "0.54", default-features = false, features = [ +futures = { workspace = true } +libp2p = { workspace = true, features = [ 'kad', 'identify', 'noise', @@ -30,9 +30,9 @@ libp2p = { version = "0.54", default-features = false, features = [ 'ed25519', 'secp256k1', ] } -libp2p-swarm-test = "0.4" -rust2go = { version = "0.3" } -tokio = { version = "1", features = ['full'] } +libp2p-swarm-test = { workspace = true } +rust2go = { workspace = true } +tokio = { workspace = true, features = ['full'] } [build-dependencies] -rust2go = { version = "0.3", features = ["build"] } +rust2go = { workspace = true, features = ["build"] } diff --git a/interop-tests/src/tests/bitswap_go_compat.rs b/interop-tests/src/tests/bitswap_go_compat.rs index 95b617182fad..7cbdb8de2c9f 100644 --- a/interop-tests/src/tests/bitswap_go_compat.rs +++ b/interop-tests/src/tests/bitswap_go_compat.rs @@ -2,20 +2,18 @@ // SPDX-License-Identifier: Apache-2.0, MIT use super::go_ffi::*; - -use std::time::Duration; - -use forest_filecoin::interop_tests_private::libp2p_bitswap::{ - BitswapBehaviour, BitswapBehaviourEvent, BitswapMessage, BitswapRequest, BitswapResponse, -}; -use libipld::{ +use cid::{ multihash::{self, MultihashDigest as _}, Cid, }; +use forest_filecoin::interop_tests_private::libp2p_bitswap::{ + BitswapBehaviour, BitswapBehaviourEvent, BitswapMessage, BitswapRequest, BitswapResponse, +}; use libp2p::{ futures::StreamExt as _, noise, request_response, swarm::SwarmEvent, tcp, yamux, Multiaddr, Swarm, SwarmBuilder, }; +use std::time::Duration; const TIMEOUT: Duration = Duration::from_secs(60); const LISTEN_ADDR: &str = "/ip4/127.0.0.1/tcp/0"; diff --git a/src/chain/store/chain_store.rs b/src/chain/store/chain_store.rs index d66f48535bb5..610b78411459 100644 --- a/src/chain/store/chain_store.rs +++ b/src/chain/store/chain_store.rs @@ -100,9 +100,9 @@ impl BitswapStoreReadWrite for ChainStore where DB: BitswapStoreReadWrite, { - type Params = ::Params; + type Hashes = ::Hashes; - fn insert(&self, block: &libipld::Block) -> anyhow::Result<()> { + fn insert(&self, block: &crate::libp2p_bitswap::Block64) -> anyhow::Result<()> { self.db.insert(block) } } diff --git a/src/db/car/many.rs b/src/db/car/many.rs index 9820fed4c580..73b70c48c68e 100644 --- a/src/db/car/many.rs +++ b/src/db/car/many.rs @@ -186,10 +186,10 @@ impl BitswapStoreRead for ManyCar BitswapStoreReadWrite for ManyCar { - type Params = libipld::DefaultParams; + type Hashes = cid::multihash::Code; - fn insert(&self, block: &libipld::Block) -> anyhow::Result<()> { - Blockstore::put_keyed(self, block.cid(), block.data()) + fn insert(&self, block: &crate::libp2p_bitswap::Block64) -> anyhow::Result<()> { + self.put_keyed(block.cid(), block.data()) } } diff --git a/src/db/memory.rs b/src/db/memory.rs index 62194952ea45..90a36a056dcf 100644 --- a/src/db/memory.rs +++ b/src/db/memory.rs @@ -150,9 +150,9 @@ impl BitswapStoreRead for MemoryDB { } impl BitswapStoreReadWrite for MemoryDB { - type Params = libipld::DefaultParams; + type Hashes = cid::multihash::Code; - fn insert(&self, block: &libipld::Block) -> anyhow::Result<()> { + fn insert(&self, block: &crate::libp2p_bitswap::Block64) -> anyhow::Result<()> { self.put_keyed(block.cid(), block.data()) } } diff --git a/src/db/parity_db.rs b/src/db/parity_db.rs index 1482c0e99697..3252a52c87fa 100644 --- a/src/db/parity_db.rs +++ b/src/db/parity_db.rs @@ -291,11 +291,9 @@ impl BitswapStoreRead for ParityDb { } impl BitswapStoreReadWrite for ParityDb { - /// `fvm_ipld_encoding::DAG_CBOR(0x71)` is covered by - /// [`libipld::DefaultParams`] under feature `dag-cbor` - type Params = libipld::DefaultParams; + type Hashes = cid::multihash::Code; - fn insert(&self, block: &libipld::Block) -> anyhow::Result<()> { + fn insert(&self, block: &crate::libp2p_bitswap::Block64) -> anyhow::Result<()> { self.put_keyed(block.cid(), block.data()) } } diff --git a/src/libp2p_bitswap/internals/prefix.rs b/src/libp2p_bitswap/internals/prefix.rs index 91ea49e5f424..5f16f9351c6d 100644 --- a/src/libp2p_bitswap/internals/prefix.rs +++ b/src/libp2p_bitswap/internals/prefix.rs @@ -1,16 +1,14 @@ // Copyright 2019-2024 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT -use std::convert::TryFrom; - -use libipld::cid::{ +use crate::libp2p_bitswap::*; +use cid::{ multihash::{Code, MultihashDigest}, Version, }; +use std::convert::TryFrom; use unsigned_varint::{decode as varint_decode, encode as varint_encode}; -use crate::libp2p_bitswap::*; - /// Prefix represents all metadata of a CID, without the actual content. #[derive(PartialEq, Eq, Clone, Debug)] pub struct Prefix { diff --git a/src/libp2p_bitswap/mod.rs b/src/libp2p_bitswap/mod.rs index 364f6ad24e6e..493468208e77 100644 --- a/src/libp2p_bitswap/mod.rs +++ b/src/libp2p_bitswap/mod.rs @@ -23,7 +23,7 @@ use std::io::Result as IOResult; -use libipld::{cid::Cid, prelude::*}; +use cid::Cid; use tracing::*; mod bitswap_pb; diff --git a/src/libp2p_bitswap/request_manager.rs b/src/libp2p_bitswap/request_manager.rs index 2c9675d9f245..c612524d212c 100644 --- a/src/libp2p_bitswap/request_manager.rs +++ b/src/libp2p_bitswap/request_manager.rs @@ -11,9 +11,9 @@ use std::{ use crate::cid_collections::CidHashMap; use ahash::{HashSet, HashSetExt}; +use cid::Cid; use flume::TryRecvError; use futures::StreamExt; -use libipld::{Block, Cid}; use libp2p::PeerId; use parking_lot::RwLock; diff --git a/src/libp2p_bitswap/store.rs b/src/libp2p_bitswap/store.rs index 6e54bb658b59..047f6a4c9990 100644 --- a/src/libp2p_bitswap/store.rs +++ b/src/libp2p_bitswap/store.rs @@ -1,11 +1,10 @@ // Copyright 2019-2024 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT -use std::sync::Arc; - -use libipld::Block; - -use crate::libp2p_bitswap::*; +use super::*; +use cid::multihash::MultihashDigest; +use std::ops::Deref; +use std::{marker::PhantomData, sync::Arc}; /// Trait implemented by a block store for reading. pub trait BitswapStoreRead { @@ -18,11 +17,11 @@ pub trait BitswapStoreRead { /// Trait implemented by a block store for reading and writing. pub trait BitswapStoreReadWrite: BitswapStoreRead + Send + Sync + 'static { - /// The store parameters. - type Params: StoreParams; + /// The hashes parameters. + type Hashes: cid::multihash::MultihashDigest<64>; /// A block response needs to insert the block into the store. - fn insert(&self, block: &Block) -> anyhow::Result<()>; + fn insert(&self, block: &Block64) -> anyhow::Result<()>; } impl BitswapStoreRead for Arc { @@ -36,11 +35,71 @@ impl BitswapStoreRead for Arc { } impl BitswapStoreReadWrite for Arc { - /// `fvm_ipld_encoding::DAG_CBOR(0x71)` is covered by - /// [`libipld::DefaultParams`] under feature `dag-cbor` - type Params = ::Params; + type Hashes = ::Hashes; - fn insert(&self, block: &libipld::Block) -> anyhow::Result<()> { + fn insert(&self, block: &Block64) -> anyhow::Result<()> { BitswapStoreReadWrite::insert(self.as_ref(), block) } } + +pub type Block64 = Block; + +/// Block +#[derive(Clone, Debug)] +pub struct Block { + /// Content identifier. + cid: Cid, + /// Binary data. + data: Vec, + _pd: PhantomData, +} + +impl Deref for Block { + type Target = Cid; + + fn deref(&self) -> &Self::Target { + &self.cid + } +} + +impl PartialEq for Block { + fn eq(&self, other: &Self) -> bool { + self.cid == other.cid + } +} + +impl Eq for Block {} + +impl, const S: usize> Block { + /// Creates a new block. Returns an error if the hash doesn't match + /// the data. + pub fn new(cid: Cid, data: Vec) -> anyhow::Result { + Self::verify_cid(&cid, &data)?; + Ok(Self { + cid, + data, + _pd: Default::default(), + }) + } + + /// Returns the [`Cid`]. + pub fn cid(&self) -> &Cid { + &self.cid + } + + /// Returns the payload. + pub fn data(&self) -> &[u8] { + &self.data + } + + fn verify_cid(cid: &Cid, payload: &[u8]) -> anyhow::Result<()> { + let code = cid.hash().code(); + let mh = H::try_from(code) + .map_err(|_| anyhow::anyhow!("unsupported multihash code {code}"))? + .digest(payload); + if mh.digest() != cid.hash().digest() { + anyhow::bail!("invalid multihash digest"); + } + Ok(()) + } +} diff --git a/src/libp2p_bitswap/tests/request_manager.rs b/src/libp2p_bitswap/tests/request_manager.rs index c418c47405aa..cf241f406f0a 100644 --- a/src/libp2p_bitswap/tests/request_manager.rs +++ b/src/libp2p_bitswap/tests/request_manager.rs @@ -3,19 +3,18 @@ #[cfg(test)] mod tests { - use std::{sync::Arc, time::Duration}; - use crate::libp2p_bitswap::*; use ahash::HashMap; - use futures::StreamExt; - use libipld::{ - multihash::{self, MultihashDigest}, - Block, Cid, + use cid::{ + multihash::{Code, MultihashDigest as _}, + Cid, }; + use futures::StreamExt; use libp2p::{multiaddr::Protocol, swarm::SwarmEvent, Multiaddr, PeerId, Swarm}; use libp2p_swarm_test::SwarmExt as _; use parking_lot::RwLock; use rand::{rngs::OsRng, Rng}; + use std::{sync::Arc, time::Duration}; use tokio::{select, task::JoinSet}; const TIMEOUT: Duration = Duration::from_secs(5); @@ -142,11 +141,12 @@ mod tests { Ok(()) } - fn new_random_block() -> anyhow::Result> { + fn new_random_block( + ) -> anyhow::Result::Hashes, 64>> { // 100KB let mut data = vec![0; 100 * 1024]; OsRng.fill(&mut data[..]); - let cid = Cid::new_v0(multihash::Code::Sha2_256.digest(data.as_slice()))?; + let cid = Cid::new_v0(Code::Sha2_256.digest(data.as_slice()))?; Block::new(cid, data) } @@ -156,19 +156,19 @@ mod tests { type TestStore = Arc; impl BitswapStoreRead for TestStoreInner { - fn contains(&self, cid: &libipld::Cid) -> anyhow::Result { + fn contains(&self, cid: &Cid) -> anyhow::Result { Ok(self.0.read().contains_key(&cid.to_bytes())) } - fn get(&self, cid: &libipld::Cid) -> anyhow::Result>> { + fn get(&self, cid: &Cid) -> anyhow::Result>> { Ok(self.0.read().get(&cid.to_bytes()).cloned()) } } impl BitswapStoreReadWrite for TestStoreInner { - type Params = libipld::DefaultParams; + type Hashes = cid::multihash::Code; - fn insert(&self, block: &libipld::Block) -> anyhow::Result<()> { + fn insert(&self, block: &Block) -> anyhow::Result<()> { self.0 .write() .insert(block.cid().to_bytes(), block.data().to_vec());