From 9b0fcffbe19f3ba7df08ee660f81fe7a6d8bbb2e Mon Sep 17 00:00:00 2001 From: David Craven Date: Sun, 15 Nov 2020 16:10:47 +0100 Subject: [PATCH 1/8] Fix possible leakage of closure after block removal. --- db/src/blocks.rs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/db/src/blocks.rs b/db/src/blocks.rs index c8dfe1c..83c4a5f 100644 --- a/db/src/blocks.rs +++ b/db/src/blocks.rs @@ -285,7 +285,15 @@ where pub async fn alias(&self, alias: &[u8], cid: Option<&Cid>) -> Result<()> { let closure = if let Some(cid) = cid { - self.blocks.closure(cid)? + if let Some(id) = self.blocks.lookup_id(cid)? { + if let Some(closure) = self.closure.get(id)? { + Ids::from(closure) + } else { + self.blocks.closure(cid)? + } + } else { + self.blocks.closure(cid)? + } } else { Default::default() }; @@ -314,19 +322,10 @@ where log::debug!("unpinned {}", id); } - let rm_closure = if let Some(id) = prev_id.as_ref() { - !filter.contains(id) - } else { - false - }; - let res = (&self.alias, &self.closure) .transaction(|(talias, tclosure)| { - if let Some(id) = prev_id.as_ref() { + if prev_id.is_some() { talias.remove(alias)?; - if rm_closure { - tclosure.remove(id)?; - } } if let Some(id) = id.as_ref() { talias.insert(alias, id)?; @@ -385,6 +384,7 @@ where let id = res?; if !filter.contains(&id) { self.blocks.remove(&id)?; + self.closure.remove(&id)?; nevict -= 1; } } From 97f1646d4bdcd0a8e3f1b91a224bdfb1e5fe53cf Mon Sep 17 00:00:00 2001 From: David Craven Date: Sun, 15 Nov 2020 21:41:13 +0100 Subject: [PATCH 2/8] Fix bug in aliasing and add default ipfs. --- Cargo.lock | 85 ++++++++++++++++++++++++++++++++++-------------- Cargo.toml | 7 ++-- db/src/blocks.rs | 21 ++++++------ src/lib.rs | 39 ++++++++++++++-------- 4 files changed, 100 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ea314cd..1710c9c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -143,9 +143,9 @@ dependencies = [ [[package]] name = "async-global-executor" -version = "1.3.0" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fefeb39da249f4c33af940b779a56723ce45809ef5c54dad84bb538d4ffb6d9e" +checksum = "73079b49cd26b8fd5a15f68fc7707fc78698dc2a3d61430f2a7a9430230dfa04" dependencies = [ "async-executor", "async-io", @@ -183,16 +183,16 @@ dependencies = [ [[package]] name = "async-std" -version = "1.6.5" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9fa76751505e8df1c7a77762f60486f60c71bbd9b8557f4da6ad47d083732ed" +checksum = "a7e82538bc65a25dbdff70e4c5439d52f068048ab97cdea0acd73f131594caa1" dependencies = [ "async-attributes", "async-global-executor", "async-io", "async-mutex", "blocking", - "crossbeam-utils", + "crossbeam-utils 0.8.0", "futures-channel", "futures-core", "futures-io", @@ -311,7 +311,7 @@ dependencies = [ "arrayref", "arrayvec", "cc", - "cfg-if", + "cfg-if 0.1.10", "constant_time_eq", "crypto-mac 0.8.0", "digest 0.9.0", @@ -440,6 +440,12 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + [[package]] name = "chacha20" version = "0.4.3" @@ -490,6 +496,12 @@ dependencies = [ "cache-padded", ] +[[package]] +name = "const_fn" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c478836e029dcef17fb47c89023448c64f781a046e0300e257ad8225ae59afab" + [[package]] name = "constant_time_eq" version = "0.1.5" @@ -508,7 +520,7 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", ] [[package]] @@ -518,8 +530,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" dependencies = [ "autocfg", - "cfg-if", - "crossbeam-utils", + "cfg-if 0.1.10", + "crossbeam-utils 0.7.2", "lazy_static", "maybe-uninit", "memoffset", @@ -533,7 +545,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" dependencies = [ "autocfg", - "cfg-if", + "cfg-if 0.1.10", + "lazy_static", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec91540d98355f690a86367e566ecad2e9e579f230230eb7c21398372be73ea5" +dependencies = [ + "autocfg", + "cfg-if 1.0.0", + "const_fn", "lazy_static", ] @@ -935,7 +959,7 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "libc", "wasi", ] @@ -1179,6 +1203,7 @@ dependencies = [ "cached", "fnv", "libipld-cbor", + "libipld-cbor-derive", "libipld-core", "libipld-macro", "log", @@ -1196,6 +1221,18 @@ dependencies = [ "thiserror", ] +[[package]] +name = "libipld-cbor-derive" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37dc95be4e7215b886f739566b131b8b48c75ba1991a64c39dc4833ee982d886" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "libipld-core" version = "0.7.0" @@ -1543,7 +1580,7 @@ version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", ] [[package]] @@ -1673,7 +1710,7 @@ version = "0.2.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ba7c918ac76704fb42afcbbb43891e72731f3dcca3bef2a19786297baf14af7" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "libc", "winapi 0.3.9", ] @@ -1763,7 +1800,7 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "cloudabi 0.0.3", "libc", "redox_syscall", @@ -1777,7 +1814,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "cloudabi 0.1.0", "instant", "libc", @@ -1840,7 +1877,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0720e0b9ea9d52451cf29d3413ba8a9303f8815d9d9653ef70e03ff73e65566" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "libc", "log", "wepoll-sys-stjepang", @@ -1862,7 +1899,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9a50142b55ab3ed0e9f68dfb3709f1d90d29da24e91033f28b96330643107dc" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "universal-hash", ] @@ -2188,7 +2225,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "170a36ea86c864a3f16dd2687712dd6646f7019f301e57537c7f4dc9f5916770" dependencies = [ "block-buffer 0.9.0", - "cfg-if", + "cfg-if 0.1.10", "cpuid-bool", "digest 0.9.0", "opaque-debug 0.3.0", @@ -2213,7 +2250,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2933378ddfeda7ea26f48c555bdad8bb446bf8a3d17832dc83e380d444cfb8c1" dependencies = [ "block-buffer 0.9.0", - "cfg-if", + "cfg-if 0.1.10", "cpuid-bool", "digest 0.9.0", "opaque-debug 0.3.0", @@ -2251,7 +2288,7 @@ checksum = "f72c064e63fbca3138ad07f3588c58093f1684f3a99f60dcfa6d46b87e60fde7" dependencies = [ "crc32fast", "crossbeam-epoch", - "crossbeam-utils", + "crossbeam-utils 0.7.2", "fs2", "fxhash", "libc", @@ -2289,7 +2326,7 @@ version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03088793f677dce356f3ccc2edb1b314ad191ab702a5de3faf49304f7e104918" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "libc", "redox_syscall", "winapi 0.3.9", @@ -2379,7 +2416,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "libc", "rand 0.7.3", "redox_syscall", @@ -2624,7 +2661,7 @@ version = "0.2.67" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0563a9a4b071746dd5aedbc3a28c6fe9be4586fb3fbadb67c400d4f53c6b16c" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "wasm-bindgen-macro", ] @@ -2649,7 +2686,7 @@ version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95f8d235a77f880bcef268d379810ea6c0af2eacfa90b1ad5af731776e0c4699" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "js-sys", "wasm-bindgen", "web-sys", diff --git a/Cargo.toml b/Cargo.toml index 655508f..0b4b412 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ repository = "https://github.com/ipfs-rust/ipfs-embed" [features] default = ["db", "net"] -db = ["ipfs-embed-db"] +db = ["ipfs-embed-db", "sled"] net = ["ipfs-embed-net"] [dependencies] @@ -25,15 +25,16 @@ ipfs-embed-db = { version = "0.8.0", path = "db", optional = true } ipfs-embed-net = { version = "0.8.0", path = "net", optional = true } libipld = { version = "0.7.1", default-features = false } log = "0.4.11" +sled = { version = "0.34.4", optional = true } [dev-dependencies] +async-std = { version = "1.7.0", features = ["attributes"] } #async-log = "2.0.0" #criterion = "0.3.3" env_logger = "0.7.1" #ipld-collections = "0.2.0" -libipld = { version = "0.7.1", default-features = false, features = ["dag-cbor"] } +libipld = { version = "0.7.1", default-features = false, features = ["dag-cbor", "derive"] } #model = "0.1.2" -sled = "0.34.4" #tempdir = "0.3.7" [profile.release] diff --git a/db/src/blocks.rs b/db/src/blocks.rs index 83c4a5f..43b2647 100644 --- a/db/src/blocks.rs +++ b/db/src/blocks.rs @@ -102,8 +102,7 @@ where Ok(ids) } - pub fn closure(&self, cid: &Cid) -> Result { - let id = self.lookup_id(cid)?.ok_or_else(|| BlockNotFound(*cid))?; + pub fn closure(&self, id: Id) -> Result { let mut refs = FnvHashSet::default(); let mut todo = Vec::new(); todo.push(id); @@ -284,20 +283,20 @@ where } pub async fn alias(&self, alias: &[u8], cid: Option<&Cid>) -> Result<()> { - let closure = if let Some(cid) = cid { - if let Some(id) = self.blocks.lookup_id(cid)? { - if let Some(closure) = self.closure.get(id)? { - Ids::from(closure) - } else { - self.blocks.closure(cid)? - } + let id = if let Some(cid) = cid { + self.blocks.lookup_id(cid)? + } else { + None + }; + let closure = if let Some(id) = id.as_ref() { + if let Some(closure) = self.closure.get(id)? { + Ids::from(closure) } else { - self.blocks.closure(cid)? + self.blocks.closure(id.clone())? } } else { Default::default() }; - let id = closure.iter().next(); log::debug!("alias {:?} {:?}", alias, id.as_ref()); let prev_id = self.alias.get(alias)?.map(Id::from); diff --git a/src/lib.rs b/src/lib.rs index 6743c73..ad1036f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,20 +3,8 @@ //! ``` //! # fn main() -> Result<(), Box> { //! use ipfs_embed::Ipfs; -//! use ipfs_embed::core::BitswapStorage; -//! use ipfs_embed::db::StorageService; -//! use ipfs_embed::net::{NetworkConfig, NetworkService}; -//! use libipld::DefaultParams; -//! use std::sync::Arc; -//! use std::time::Duration; -//! let sled_config = sled::Config::new().temporary(true); -//! let cache_size = 10; -//! let sweep_interval = Duration::from_millis(10000); -//! let net_config = NetworkConfig::new(); -//! let storage = Arc::new(StorageService::open(&sled_config, cache_size, sweep_interval).unwrap()); -//! let bitswap_storage = BitswapStorage::new(storage.clone()); -//! let network = Arc::new(NetworkService::new(net_config, bitswap_storage).unwrap()); -//! let ipfs = Ipfs::::new(storage, network); +//! let cache_size = 100; +//! let _ipfs = Ipfs::default(None, cache_size)?; //! # Ok(()) } //! ``` use async_std::task; @@ -102,6 +90,29 @@ where } } +#[cfg(all(feature = "db", feature = "net"))] +use libipld::store::DefaultParams; +#[cfg(all(feature = "db", feature = "net"))] +pub type DefaultIpfs = + Ipfs, net::NetworkService>; +#[cfg(all(feature = "db", feature = "net"))] +impl DefaultIpfs { + /// If no path is provided a temporary db will be created. + pub fn default(path: Option, cache_size: usize) -> Result { + let sled_config = if let Some(path) = path { + sled::Config::new().path(path) + } else { + sled::Config::new().temporary(true) + }; + let sweep_interval = std::time::Duration::from_millis(10000); + let net_config = net::NetworkConfig::new(); + let storage = Arc::new(db::StorageService::open(&sled_config, cache_size, sweep_interval)?); + let bitswap_storage = core::BitswapStorage::new(storage.clone()); + let network = Arc::new(net::NetworkService::new(net_config, bitswap_storage)?); + Ok(Self::new(storage, network)) + } +} + #[async_trait] impl Store for Ipfs where From dbeb93eaaeca9fc6606e0e8939df5ffa2e29c5da Mon Sep 17 00:00:00 2001 From: David Craven Date: Sun, 15 Nov 2020 23:17:43 +0100 Subject: [PATCH 3/8] Support custom syncer. --- src/lib.rs | 43 +++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index ad1036f..2c92a61 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -88,6 +88,32 @@ where pub async fn pinned(&self, cid: &Cid) -> Result> { self.storage.pinned(cid).await } + + pub async fn alias_with_syncer + Send + Sync>( + &self, alias: T, + cid: Option<&Cid>, + syncer: Option>, + ) -> Result<()> { + if let Some(cid) = cid { + let (tx, rx) = oneshot::channel(); + self.tx + .clone() + .send(( + Query { + cid: *cid, + ty: QueryType::Sync, + syncer: syncer.unwrap_or_else(|| { + Arc::new(BitswapStorage::new(self.storage.clone())) + }), + }, + tx, + )) + .await?; + rx.await??; + } + self.storage.alias(alias.as_ref(), cid).await?; + Ok(()) + } } #[cfg(all(feature = "db", feature = "net"))] @@ -154,22 +180,7 @@ where } async fn alias + Send + Sync>(&self, alias: T, cid: Option<&Cid>) -> Result<()> { - if let Some(cid) = cid { - let (tx, rx) = oneshot::channel(); - self.tx - .clone() - .send(( - Query { - cid: *cid, - ty: QueryType::Sync, - }, - tx, - )) - .await?; - rx.await??; - } - self.storage.alias(alias.as_ref(), cid).await?; - Ok(()) + self.alias_with_syncer(alias, cid, None).await } async fn resolve + Send + Sync>(&self, alias: T) -> Result> { From 77989b078e2a25d067ee7de38d2f22ad97e665ef Mon Sep 17 00:00:00 2001 From: David Craven Date: Mon, 16 Nov 2020 12:40:14 +0100 Subject: [PATCH 4/8] Perf improvements. --- Cargo.lock | 2 -- Cargo.toml | 3 +++ db/src/blocks.rs | 8 ++------ net/src/config.rs | 4 ++-- net/src/lib.rs | 6 ++++-- src/lib.rs | 27 ++++++++++++++++----------- 6 files changed, 27 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1710c9c..6534f44 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1289,8 +1289,6 @@ dependencies = [ [[package]] name = "libp2p-bitswap" version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac3c3eece30baa63e9162e8c799b45d43043d6f8f7ecccaa53de9a9c86ba2f7" dependencies = [ "async-trait", "fnv", diff --git a/Cargo.toml b/Cargo.toml index 0b4b412..2906f61 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,3 +43,6 @@ debug = true [[bench]] name = "list" harness = false + +[patch.crates-io] +libp2p-bitswap = { git = "https://github.com/ipfs-rust/libp2p-bitswap" } diff --git a/db/src/blocks.rs b/db/src/blocks.rs index 43b2647..a96a561 100644 --- a/db/src/blocks.rs +++ b/db/src/blocks.rs @@ -137,11 +137,11 @@ where Ok(()) }) .map_err(map_tx_error)?; - log::debug!("hit {}", id); + //log::trace!("hit {}", id); return Ok(Some(data.to_vec())); } } - log::debug!("miss {}", cid.to_string()); + //log::trace!("miss {}", cid.to_string()); Ok(None) } @@ -314,11 +314,9 @@ where } for id in closure.iter() { filter.add(&id).unwrap(); - log::debug!("pinned {}", id); } for id in prev_closure.iter() { filter.delete(&id); - log::debug!("unpinned {}", id); } let res = (&self.alias, &self.closure) @@ -337,11 +335,9 @@ where if res.is_err() { for id in prev_closure.iter() { filter.add(&id).unwrap(); - log::debug!("pinned {}", id); } for id in closure.iter() { filter.delete(&id); - log::debug!("unpinned {}", id); } } drop(filter); diff --git a/net/src/config.rs b/net/src/config.rs index fa72e54..6650bd3 100644 --- a/net/src/config.rs +++ b/net/src/config.rs @@ -44,8 +44,8 @@ impl NetworkConfig { node_name: names::Generator::with_naming(names::Name::Numbered) .next() .unwrap(), - bitswap_request_timeout: Duration::from_secs(10), - bitswap_connection_keepalive: Duration::from_secs(10), + bitswap_request_timeout: Duration::from_secs(1), + bitswap_connection_keepalive: Duration::from_secs(1), bitswap_receive_limit: NonZeroU16::new(20).expect("20 > 0"), } } diff --git a/net/src/lib.rs b/net/src/lib.rs index 533d147..73a51bd 100644 --- a/net/src/lib.rs +++ b/net/src/lib.rs @@ -10,7 +10,7 @@ use libp2p::core::transport::Transport; use libp2p::dns::DnsConfig; use libp2p::mplex::MplexConfig; use libp2p::noise::{Keypair, NoiseConfig, X25519Spec}; -use libp2p::swarm::Swarm; +use libp2p::swarm::{Swarm, SwarmBuilder}; use libp2p::tcp::TcpConfig; //use libp2p::yamux::Config as YamuxConfig; use std::marker::PhantomData; @@ -47,7 +47,9 @@ impl NetworkService

{ let peer_id = config.peer_id(); let behaviour = NetworkBackendBehaviour::

::new(config.clone(), store)?; - let mut swarm = Swarm::new(transport, behaviour, peer_id.clone()); + let mut swarm = SwarmBuilder::new(transport, behaviour, peer_id.clone()) + .executor(Box::new(|fut| { async_std::task::spawn(fut); })) + .build(); for addr in config.listen_addresses { Swarm::listen_on(&mut swarm, addr)?; } diff --git a/src/lib.rs b/src/lib.rs index 2c92a61..66a75ff 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,7 +15,7 @@ use futures::future::Future; use futures::sink::SinkExt; use futures::stream::Stream; use ipfs_embed_core::{ - BitswapStorage, Block, Cid, Multiaddr, Network, NetworkEvent, PeerId, Query, QueryResult, + BitswapStorage, BitswapSync, Block, Cid, Multiaddr, Network, NetworkEvent, PeerId, Query, QueryResult, QueryType, Result, Storage, StorageEvent, StoreParams, }; use libipld::codec::Decode; @@ -37,7 +37,7 @@ pub struct Ipfs { _marker: PhantomData

, storage: Arc, network: Arc, - tx: mpsc::Sender<(Query, oneshot::Sender)>, + tx: mpsc::Sender<(Query, Option>, oneshot::Sender)>, } impl Clone for Ipfs { @@ -89,10 +89,14 @@ where self.storage.pinned(cid).await } + pub fn bitswap_storage(&self) -> BitswapStorage { + BitswapStorage::new(self.storage.clone()) + } + pub async fn alias_with_syncer + Send + Sync>( &self, alias: T, cid: Option<&Cid>, - syncer: Option>, + syncer: Option>, ) -> Result<()> { if let Some(cid) = cid { let (tx, rx) = oneshot::channel(); @@ -102,10 +106,8 @@ where Query { cid: *cid, ty: QueryType::Sync, - syncer: syncer.unwrap_or_else(|| { - Arc::new(BitswapStorage::new(self.storage.clone())) - }), }, + syncer, tx, )) .await?; @@ -162,6 +164,7 @@ where cid: *cid, ty: QueryType::Get, }, + None, tx, )) .await?; @@ -195,7 +198,7 @@ struct IpfsTask, N: Network

> { network: Arc, network_events: N::Subscription, queries: FnvHashMap>>, - rx: mpsc::Receiver<(Query, oneshot::Sender)>, + rx: mpsc::Receiver<(Query, Option>, oneshot::Sender)>, } impl IpfsTask @@ -208,7 +211,7 @@ where pub fn new( storage: Arc, network: Arc, - rx: mpsc::Receiver<(Query, oneshot::Sender)>, + rx: mpsc::Receiver<(Query, Option>, oneshot::Sender)>, ) -> Self { let storage_events = storage.subscribe(); let network_events = network.subscribe(); @@ -235,7 +238,7 @@ where fn poll(mut self: Pin<&mut Self>, ctx: &mut Context) -> Poll { loop { - let (query, tx) = match Pin::new(&mut self.rx).poll_next(ctx) { + let (query, syncer, tx) = match Pin::new(&mut self.rx).poll_next(ctx) { Poll::Ready(Some(query)) => query, Poll::Ready(None) => return Poll::Ready(()), Poll::Pending => break, @@ -244,8 +247,10 @@ where match query.ty { QueryType::Get => self.network.get(query.cid), QueryType::Sync => { - let syncer = BitswapStorage::new(self.storage.clone()); - self.network.sync(query.cid, Arc::new(syncer)) + let syncer = syncer.unwrap_or_else(|| { + Arc::new(BitswapStorage::new(self.storage.clone())) + }); + self.network.sync(query.cid, syncer) } } } From 7a11cea2f941486ec506b31e88988f541d2072cb Mon Sep 17 00:00:00 2001 From: David Craven Date: Mon, 16 Nov 2020 12:40:23 +0100 Subject: [PATCH 5/8] Chain example. --- examples/chain.rs | 240 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 240 insertions(+) create mode 100644 examples/chain.rs diff --git a/examples/chain.rs b/examples/chain.rs new file mode 100644 index 0000000..fbd6f1b --- /dev/null +++ b/examples/chain.rs @@ -0,0 +1,240 @@ +use ipfs_embed::DefaultIpfs; +use ipfs_embed::core::{BitswapStorage, BitswapStore, BitswapSync, Result, Storage}; +use libipld::{alias, DagCbor, Cid}; +use libipld::cbor::DagCborCodec; +use libipld::multihash::Code; +use libipld::store::{Store, StoreParams}; +use std::convert::TryFrom; +use std::path::Path; +use std::sync::Arc; + +const ROOT: &str = alias!(root); + +#[derive(Debug, Default, DagCbor)] +pub struct Block { + prev: Option, + id: u32, + loopback: Option, + payload: Vec, +} + +fn loopback(block: u32) -> Option { + let x = block.trailing_zeros(); + if x > 1 && block > 0 { + Some(block - (1 << (x - 1))) + } else { + None + } +} + +pub struct ChainSyncer> { + index: sled::Db, + storage: BitswapStorage, +} + +impl> ChainSyncer { + pub fn new(index: sled::Db, storage: BitswapStorage) -> Arc { + Arc::new(Self { index, storage }) + } + + fn lookup_cid(&self, id: u32) -> Result> { + if let Some(cid) = self.index.get(id.to_be_bytes())? { + Ok(Some(Cid::try_from(&cid[..]).unwrap())) + } else { + Ok(None) + } + } + + fn loopback_cid(&self, id: u32) -> Result> { + if let Some(id) = loopback(id) { + Ok(self.lookup_cid(id)?) + } else { + Ok(None) + } + } +} + +impl> BitswapSync for ChainSyncer +where + S::Codecs: Into, +{ + fn references(&self, cid: &Cid) -> Box> { + if let Some(data) = self.storage.get(cid) { + let ipld_block = libipld::Block::::new_unchecked(*cid, data); + if let Ok(block) = ipld_block.decode::() { + let loopback = self.loopback_cid(block.id); + let block_loopback = block.loopback; + let valid_loopback = match (loopback, block_loopback) { + (Ok(Some(cid1)), Some(cid2)) => cid1 == cid2, + (Ok(None), _) => true, + (Ok(Some(_)), None) => false, + (Err(err), _) => { + log::error!("{:?}", err); + true + } + }; + if valid_loopback { + match (block.prev, block.loopback) { + (Some(cid1), Some(cid2)) => { + let cid1 = std::iter::once(cid1); + let cid2 = std::iter::once(cid2); + return Box::new(cid1.chain(cid2)); + } + (Some(cid), None) => return Box::new(std::iter::once(cid)), + _ => {} + } + } else { + log::error!("rejecting block due to invalid loopback"); + } + } + } + Box::new(std::iter::empty()) + } + + fn contains(&self, cid: &Cid) -> bool { + self.storage.contains(cid) + } +} + +pub struct BlockChain { + index: sled::Db, + ipfs: DefaultIpfs, + root_cid: Option, + root_id: u32, +} + +impl BlockChain { + pub async fn open>(path: P, cache_size: usize) -> Result { + let index = sled::open(path.as_ref().join("index"))?; + let ipfs = DefaultIpfs::default(Some(path.as_ref().join("blocks")), cache_size)?; + let root_cid = ipfs.resolve(ROOT).await?; + let mut chain = Self { + index, + ipfs, + root_cid, + root_id: 0, + }; + if root_cid.is_none() { + // insert the genesis block + chain.push(vec![], false).await?; + } + chain.root_id = chain.get_by_cid(chain.root_cid.as_ref().unwrap()).await?.id; + Ok(chain) + } + + pub fn lookup_cid(&self, id: u32) -> Result> { + if let Some(cid) = self.index.get(id.to_be_bytes())? { + Ok(Some(Cid::try_from(&cid[..]).unwrap())) + } else { + Ok(None) + } + } + + pub async fn get_by_cid(&self, cid: &Cid) -> Result { + let block = self.ipfs.get(&cid).await?; + let block = block.decode::()?; + Ok(block) + } + + pub async fn get_by_id(&self, id: u32) -> Result> { + if let Some(cid) = self.lookup_cid(id)? { + Ok(Some(self.get_by_cid(&cid).await?)) + } else { + Ok(None) + } + } + + pub fn loopback_cid(&self, id: u32) -> Result> { + if let Some(id) = loopback(id) { + Ok(Some(self.lookup_cid(id)?.unwrap())) + } else { + Ok(None) + } + } + + fn index_block(&self, id: u32, cid: &Cid) -> Result<()> { + self.index.insert(id.to_be_bytes(), cid.to_bytes())?; + Ok(()) + } + + pub async fn push(&mut self, payload: Vec, import: bool) -> Result { + let id = if self.root_cid.is_none() { + 0 + } else { + self.root_id + 1 + }; + let loopback = self.loopback_cid(id)?; + let block = Block { + prev: self.root_cid.clone(), + id, + loopback, + payload, + }; + let ipld_block = libipld::Block::encode(DagCborCodec, Code::Blake3_256, &block)?; + self.ipfs.insert(&ipld_block).await?; + self.index_block(id, ipld_block.cid())?; + if !import { + self.ipfs.alias(ROOT, Some(ipld_block.cid())).await?; + } + self.root_id = id; + self.root_cid = Some(*ipld_block.cid()); + Ok(*ipld_block.cid()) + } + + pub async fn sync(&mut self, root: &Cid) -> Result<()> { + let syncer = ChainSyncer::new(self.index.clone(), self.ipfs.bitswap_storage()); + self.ipfs.alias_with_syncer(ROOT, Some(root), Some(syncer)).await?; + + let mut cid = *root; + let mut block = self.get_by_cid(root).await?; + let prev_root_id = self.root_id; + + self.root_id = block.id; + self.root_cid = Some(cid); + + for _ in prev_root_id..self.root_id { + self.index_block(block.id, &cid)?; + cid = block.prev.unwrap(); + block = self.get_by_cid(&cid).await?; + } + Ok(()) + } + + pub fn root(&self) -> &Cid { + self.root_cid.as_ref().unwrap() + } +} + +#[async_std::main] +async fn main() -> Result<(), Box> { + //env_logger::init(); + let mut local1 = BlockChain::open("/tmp/local1", 1000).await?; + let mut local2 = BlockChain::open("/tmp/local2", 1000).await?; + + for i in 1..10 { + local1.push(vec![i as u8], true).await?; + } + + let root = *local1.root(); + local1.sync(&root).await?; + + println!("starting sync"); + let start = std::time::Instant::now(); + local2.sync(&root).await?; + let end = std::time::Instant::now(); + println!("time to sync {}ms", end.duration_since(start).as_millis()); + + /*let root = local1.push(b"hello world".to_vec()).await?; + local2.sync(&root).await?; + let block = local2.get_by_id(1).await?; + assert_eq!(block.unwrap().payload, b"hello world".to_vec()); + + let root = local2.push(b"another block".to_vec()).await?; + local1.sync(&root).await?; + let block = local1.get_by_id(1).await?; + assert_eq!(block.unwrap().payload, b"hello world".to_vec()); + let block = local1.get_by_id(2).await?; + assert_eq!(block.unwrap().payload, b"another block".to_vec());*/ + + Ok(()) +} From d534bc956e01c7e173963bc99a5f718da15923dd Mon Sep 17 00:00:00 2001 From: David Craven Date: Mon, 16 Nov 2020 12:45:53 +0100 Subject: [PATCH 6/8] Update lock. --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index 6534f44..a9b5a57 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1289,6 +1289,7 @@ dependencies = [ [[package]] name = "libp2p-bitswap" version = "0.8.0" +source = "git+https://github.com/ipfs-rust/libp2p-bitswap#2c5db79969438ee21bc7d8e2aac3f0ac6f8ddf92" dependencies = [ "async-trait", "fnv", From f9023ef417580880c1843cc71671ad47a304b3e0 Mon Sep 17 00:00:00 2001 From: David Craven Date: Wed, 18 Nov 2020 20:26:09 +0100 Subject: [PATCH 7/8] More stuff. --- Cargo.lock | 351 ++++++++++++++++++++++------------------------ Cargo.toml | 4 +- core/Cargo.toml | 2 +- examples/chain.rs | 14 +- net/Cargo.toml | 2 +- 5 files changed, 182 insertions(+), 191 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a9b5a57..c62cb2e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -228,9 +228,12 @@ dependencies = [ [[package]] name = "atomic" -version = "0.4.6" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64f46ca51dca4837f1520754d1c8c36636356b81553d928dc9c177025369a06e" +checksum = "c3410529e8288c463bedb5930f82833bc0c90e5d2fe639a56582a4d09220b281" +dependencies = [ + "autocfg", +] [[package]] name = "atomic-waker" @@ -246,7 +249,7 @@ checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ "hermit-abi", "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -379,9 +382,9 @@ dependencies = [ [[package]] name = "bs58" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "476e9cd489f9e121e02ffa6014a8ef220ecb15c05ed23fc34cca13925dc283fb" +checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" [[package]] name = "bumpalo" @@ -407,12 +410,6 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" -[[package]] -name = "c_linked_list" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4964518bd3b4a8190e832886cdc0da9794f12e8e6c1613a9e90ff331c4c8724b" - [[package]] name = "cache-padded" version = "1.1.1" @@ -469,15 +466,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "cloudabi" -version = "0.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" -dependencies = [ - "bitflags", -] - [[package]] name = "cloudabi" version = "0.1.0" @@ -691,9 +679,9 @@ dependencies = [ [[package]] name = "ed25519-dalek" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d2e93f837d749c16d118e7ddf7a4dfd0ac8f452cf51e46e9348824e5ef6851" +checksum = "c762bae6dcaf24c4c84667b8579785430908723d5c889f469d76a41d59cc7a9d" dependencies = [ "curve25519-dalek 3.0.0", "ed25519", @@ -759,7 +747,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" dependencies = [ "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -878,7 +866,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr", - "pin-project", + "pin-project 0.4.23", "pin-utils", "proc-macro-hack", "proc-macro-nested", @@ -894,7 +882,7 @@ dependencies = [ "bytes", "futures", "memchr", - "pin-project", + "pin-project 0.4.23", ] [[package]] @@ -906,12 +894,6 @@ dependencies = [ "byteorder", ] -[[package]] -name = "gcc" -version = "0.3.55" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" - [[package]] name = "generic-array" version = "0.12.3" @@ -931,28 +913,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "get_if_addrs" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abddb55a898d32925f3148bd281174a68eeb68bbfd9a5938a57b18f506ee4ef7" -dependencies = [ - "c_linked_list", - "get_if_addrs-sys", - "libc", - "winapi 0.2.8", -] - -[[package]] -name = "get_if_addrs-sys" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d04f9fb746cf36b191c00f3ede8bde9c8e64f9f4b05ae2694a9ccf5e3f5ab48" -dependencies = [ - "gcc", - "libc", -] - [[package]] name = "getrandom" version = "0.1.14" @@ -1055,6 +1015,27 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "if-addrs" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28538916eb3f3976311f5dfbe67b5362d0add1293d0a9cad17debf86f8e3aa48" +dependencies = [ + "if-addrs-sys", + "libc", + "winapi", +] + +[[package]] +name = "if-addrs-sys" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de74b9dd780476e837e5eb5ab7c88b49ed304126e412030a0adba99c8efe79ea" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "indexmap" version = "1.5.2" @@ -1257,9 +1238,9 @@ dependencies = [ [[package]] name = "libp2p" -version = "0.28.1" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "571f5a4604c1a40d75651da141dfde29ad15329f537a779528803297d2220274" +checksum = "e3c2b4c99f8798be90746fc226acf95d3e6cff0655883634cc30dab1f64f438b" dependencies = [ "atomic", "bytes", @@ -1277,11 +1258,12 @@ dependencies = [ "libp2p-request-response", "libp2p-swarm", "libp2p-tcp", + "libp2p-wasm-ext", "libp2p-yamux", "multihash", "parity-multiaddr", - "parking_lot 0.10.2", - "pin-project", + "parking_lot", + "pin-project 1.0.1", "smallvec", "wasm-timer", ] @@ -1289,7 +1271,6 @@ dependencies = [ [[package]] name = "libp2p-bitswap" version = "0.8.0" -source = "git+https://github.com/ipfs-rust/libp2p-bitswap#2c5db79969438ee21bc7d8e2aac3f0ac6f8ddf92" dependencies = [ "async-trait", "fnv", @@ -1298,14 +1279,14 @@ dependencies = [ "libp2p", "log", "thiserror", - "unsigned-varint 0.5.1", + "unsigned-varint", ] [[package]] name = "libp2p-core" -version = "0.22.1" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52f13ba8c7df0768af2eb391696d562c7de88cc3a35122531aaa6a7d77754d25" +checksum = "1b8186060d6bd415e4e928e6cb44c4fe7e7a7dd53437bd936ce7e5f421e45a51" dependencies = [ "asn1_der", "bs58", @@ -1320,17 +1301,17 @@ dependencies = [ "multihash", "multistream-select", "parity-multiaddr", - "parking_lot 0.10.2", - "pin-project", + "parking_lot", + "pin-project 1.0.1", "prost", "prost-build", "rand 0.7.3", "ring", "rw-stream-sink", - "sha2 0.8.2", + "sha2 0.9.1", "smallvec", "thiserror", - "unsigned-varint 0.4.0", + "unsigned-varint", "void", "zeroize", ] @@ -1347,9 +1328,9 @@ dependencies = [ [[package]] name = "libp2p-dns" -version = "0.22.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cf319822e08dd65c8e060d2354e9f952895bbc433f5706c75ed010c152aee5e" +checksum = "0baeff71fb5cb1fe1604f74a712a44b66a8c5900f4022411a1d550f09d6bb776" dependencies = [ "futures", "libp2p-core", @@ -1358,9 +1339,9 @@ dependencies = [ [[package]] name = "libp2p-identify" -version = "0.22.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56396ee63aa9164eacf40c2c5d2bda8c4133c2f57e1b0425d51d3a4e362583b1" +checksum = "e074124669840484de564901d47f2d0892e73f6d8ee7c37e9c2644af1b217bf4" dependencies = [ "futures", "libp2p-core", @@ -1374,9 +1355,9 @@ dependencies = [ [[package]] name = "libp2p-kad" -version = "0.23.0" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7fa9047f8b8f544278a35c2d9d45d3b2c1785f2d86d4e1629d6edf97be3955" +checksum = "78a2653b2e3254a3bbeb66bfc3f0dca7d6cba6aa2a96791db114003dec1b5394" dependencies = [ "arrayvec", "bytes", @@ -1391,19 +1372,19 @@ dependencies = [ "prost", "prost-build", "rand 0.7.3", - "sha2 0.8.2", + "sha2 0.9.1", "smallvec", "uint", - "unsigned-varint 0.4.0", + "unsigned-varint", "void", "wasm-timer", ] [[package]] name = "libp2p-mdns" -version = "0.22.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3173b5a6b2f690c29ae07798d85b9441a131ac76ddae9015ef22905b623d0c69" +checksum = "786b068098794322239f8f04df88a52daeb7863b2e77501c4d85d32e0a8f2d26" dependencies = [ "async-std", "data-encoding", @@ -1423,28 +1404,30 @@ dependencies = [ [[package]] name = "libp2p-mplex" -version = "0.22.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a73a799cc8410b36e40b8f4c4b6babbcb9efd3727111bf517876e4acfa612d3" +checksum = "ed764eab613a8fb6b7dcf6c796f55a06fef2270e528329903e25cd3311b99663" dependencies = [ "bytes", - "fnv", "futures", "futures_codec", "libp2p-core", "log", - "parking_lot 0.10.2", - "unsigned-varint 0.4.0", + "nohash-hasher", + "parking_lot", + "rand 0.7.3", + "smallvec", + "unsigned-varint", ] [[package]] name = "libp2p-noise" -version = "0.24.0" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ef6c490042f549fb1025f2892dfe6083d97a77558f450c1feebe748ca9eb15a" +checksum = "fb441fb015ec16690099c5d910fcba271d357763b3dcb784db7b27bbb0b68372" dependencies = [ "bytes", - "curve25519-dalek 2.1.0", + "curve25519-dalek 3.0.0", "futures", "lazy_static", "libp2p-core", @@ -1452,18 +1435,18 @@ dependencies = [ "prost", "prost-build", "rand 0.7.3", - "sha2 0.8.2", + "sha2 0.9.1", "snow", "static_assertions", - "x25519-dalek", + "x25519-dalek 1.1.0", "zeroize", ] [[package]] name = "libp2p-ping" -version = "0.22.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad063c21dfcea4518ac9e8bd4119d33a5b26c41e674f602f41f05617a368a5c8" +checksum = "82e5c50936cfdbe96a514e8992f304fa44cd3a681b6f779505f1ae62b3474705" dependencies = [ "futures", "libp2p-core", @@ -1476,9 +1459,9 @@ dependencies = [ [[package]] name = "libp2p-request-response" -version = "0.3.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c0c9e8a4cd69d97e9646c54313d007512f411aba8c5226cfcda16df6a6e84a3" +checksum = "2dd9a1e0e6563dec1c9e702f7e68bdaa43da62a84536aa06372d3fed3e25d4ca" dependencies = [ "async-trait", "bytes", @@ -1490,15 +1473,15 @@ dependencies = [ "minicbor", "rand 0.7.3", "smallvec", - "unsigned-varint 0.5.1", + "unsigned-varint", "wasm-timer", ] [[package]] name = "libp2p-swarm" -version = "0.22.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7193e444210132237b81b755ec7fe53f1c4bd2f53cf719729b94c0c72eb6eaa1" +checksum = "565f0e06674b4033c978471e4083d5aaa8e03cef0719a0ec0905aaeaad39a919" dependencies = [ "either", "futures", @@ -1512,29 +1495,43 @@ dependencies = [ [[package]] name = "libp2p-tcp" -version = "0.22.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44f42ec130d7a37a7e47bf4398026b7ad9185c08ed26972e2720f8b94112796f" +checksum = "33f3dce259c0d3127af5167f45c275b6c047320efdd0e40fde947482487af0a3" dependencies = [ "async-std", "futures", "futures-timer", - "get_if_addrs", + "if-addrs", "ipnet", "libp2p-core", "log", "socket2", ] +[[package]] +name = "libp2p-wasm-ext" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c703816f4170477a375b49c56d349e535ce68388f81ba1d9a3c8e2517effa82" +dependencies = [ + "futures", + "js-sys", + "libp2p-core", + "parity-send-wrapper", + "wasm-bindgen", + "wasm-bindgen-futures", +] + [[package]] name = "libp2p-yamux" -version = "0.25.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "781d9b9f043dcdabc40640807125368596b849fd4d96cdca2dcf052fdf6f33fd" +checksum = "1a0798cbb58535162c40858493d09af06eac42a26e4966e58de0df701f559348" dependencies = [ "futures", "libp2p-core", - "parking_lot 0.11.0", + "parking_lot", "thiserror", "yamux", ] @@ -1555,15 +1552,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "lock_api" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" -dependencies = [ - "scopeguard", -] - [[package]] name = "lock_api" version = "0.4.1" @@ -1620,18 +1608,18 @@ dependencies = [ [[package]] name = "minicbor" -version = "0.5.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fc03ad6f8f548db7194a5ff5a6f96342ecae4e3ef67d2bf18bacc0e245cd041" +checksum = "0164190d1771b1458c3742075b057ed55d25cd9dfb930aade99315a1eb1fe12d" dependencies = [ "minicbor-derive", ] [[package]] name = "minicbor-derive" -version = "0.4.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c214bf3d90099b52f3e4b328ae0fe34837fd0fab683ad1e10fceb4629106df48" +checksum = "2e071b3159835ee91df62dbdbfdd7ec366b7ea77c838f43aff4acda6b61bcfb9" dependencies = [ "proc-macro2", "quote", @@ -1661,7 +1649,7 @@ dependencies = [ "sha-1", "sha2 0.9.1", "sha3", - "unsigned-varint 0.5.1", + "unsigned-varint", ] [[package]] @@ -1672,16 +1660,16 @@ checksum = "1255076139a83bb467426e7f8d0134968a8118844faa755985e077cf31850333" [[package]] name = "multistream-select" -version = "0.8.2" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9157e87afbc2ef0d84cc0345423d715f445edde00141c93721c162de35a05e5" +checksum = "93faf2e41f9ee62fb01680ed48f3cc26652352327aa2e59869070358f6b7dd75" dependencies = [ "bytes", "futures", "log", - "pin-project", + "pin-project 1.0.1", "smallvec", - "unsigned-varint 0.4.0", + "unsigned-varint", ] [[package]] @@ -1700,7 +1688,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e847c76b390f44529c2071ef06d0b52fbb4bdb04cc8987a5cfa63954c000abca" dependencies = [ "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1711,7 +1699,7 @@ checksum = "2ba7c918ac76704fb42afcbbb43891e72731f3dcca3bef2a19786297baf14af7" dependencies = [ "cfg-if 0.1.10", "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1750,9 +1738,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "parity-multiaddr" -version = "0.9.2" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2165a93382a93de55868dcbfa11e4a8f99676a9164eee6a2b4a9479ad319c257" +checksum = "60d477bda9666bc37e5ac9e7e7ee3684f745ec33e6e86a5b48640e0407acda26" dependencies = [ "arrayref", "bs58", @@ -1762,25 +1750,21 @@ dependencies = [ "percent-encoding", "serde", "static_assertions", - "unsigned-varint 0.4.0", + "unsigned-varint", "url", ] [[package]] -name = "parking" -version = "2.0.0" +name = "parity-send-wrapper" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" +checksum = "aa9777aa91b8ad9dd5aaa04a9b6bcb02c7f1deb952fca5a66034d5e63afc5c6f" [[package]] -name = "parking_lot" -version = "0.10.2" +name = "parking" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e" -dependencies = [ - "lock_api 0.3.4", - "parking_lot_core 0.7.2", -] +checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" [[package]] name = "parking_lot" @@ -1789,22 +1773,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4893845fa2ca272e647da5d0e46660a314ead9c2fdd9a883aabc32e481a8733" dependencies = [ "instant", - "lock_api 0.4.1", - "parking_lot_core 0.8.0", -] - -[[package]] -name = "parking_lot_core" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3" -dependencies = [ - "cfg-if 0.1.10", - "cloudabi 0.0.3", - "libc", - "redox_syscall", - "smallvec", - "winapi 0.3.9", + "lock_api", + "parking_lot_core", ] [[package]] @@ -1814,12 +1784,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b" dependencies = [ "cfg-if 0.1.10", - "cloudabi 0.1.0", + "cloudabi", "instant", "libc", "redox_syscall", "smallvec", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1844,7 +1814,16 @@ version = "0.4.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca4433fff2ae79342e497d9f8ee990d174071408f28f726d6d83af93e58e48aa" dependencies = [ - "pin-project-internal", + "pin-project-internal 0.4.23", +] + +[[package]] +name = "pin-project" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee41d838744f60d959d7074e3afb6b35c7456d0f61cad38a24e35e6553f73841" +dependencies = [ + "pin-project-internal 1.0.1", ] [[package]] @@ -1858,6 +1837,17 @@ dependencies = [ "syn", ] +[[package]] +name = "pin-project-internal" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81a4ffa594b66bff340084d4081df649a7dc049ac8d7fc458d8e628bfbbb2f86" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.1.7" @@ -1880,7 +1870,7 @@ dependencies = [ "libc", "log", "wepoll-sys-stjepang", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -2048,7 +2038,7 @@ dependencies = [ "libc", "rand_core 0.3.1", "rdrand", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -2146,7 +2136,7 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -2161,7 +2151,7 @@ dependencies = [ "spin", "untrusted", "web-sys", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -2186,7 +2176,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4da5fcb054c46f5a5dff833b129285a93d3f0179531735e6c866e8cc307d2020" dependencies = [ "futures", - "pin-project", + "pin-project 0.4.23", "static_assertions", ] @@ -2292,7 +2282,7 @@ dependencies = [ "fxhash", "libc", "log", - "parking_lot 0.11.0", + "parking_lot", ] [[package]] @@ -2316,7 +2306,7 @@ dependencies = [ "rustc_version", "sha2 0.9.1", "subtle 2.2.3", - "x25519-dalek", + "x25519-dalek 0.6.0", ] [[package]] @@ -2328,7 +2318,7 @@ dependencies = [ "cfg-if 0.1.10", "libc", "redox_syscall", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -2378,9 +2368,9 @@ checksum = "502d53007c02d7605a05df1c1a73ee436952781653da5d0bf57ad608f66932c1" [[package]] name = "syn" -version = "1.0.42" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c51d92969d209b54a98397e1b91c8ae82d8c87a7bb87df0b29aa2ad81454228" +checksum = "cc371affeffc477f42a221a1e4297aedcea33d47d19b61455588bd9d8f6b19ac" dependencies = [ "proc-macro2", "quote", @@ -2420,7 +2410,7 @@ dependencies = [ "rand 0.7.3", "redox_syscall", "remove_dir_all", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -2469,7 +2459,7 @@ checksum = "add6ab1597c883a38fc0ade108679461b297ab696d417151b85996047a605c81" dependencies = [ "multibase", "tiny-multihash", - "unsigned-varint 0.5.1", + "unsigned-varint", ] [[package]] @@ -2497,7 +2487,7 @@ dependencies = [ "sha3", "strobe-rs", "tiny-multihash-derive", - "unsigned-varint 0.5.1", + "unsigned-varint", ] [[package]] @@ -2587,24 +2577,16 @@ dependencies = [ "subtle 2.2.3", ] -[[package]] -name = "unsigned-varint" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "669d776983b692a906c881fcd0cfb34271a48e197e4d6cb8df32b05bfc3d3fa5" -dependencies = [ - "bytes", - "futures_codec", -] - [[package]] name = "unsigned-varint" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7fdeedbf205afadfe39ae559b75c3240f24e257d0ca27e85f85cb82aa19ac35" dependencies = [ + "bytes", "futures-io", "futures-util", + "futures_codec", ] [[package]] @@ -2728,7 +2710,7 @@ checksum = "be0ecb0db480561e9a7642b5d3e4187c128914e58aa84330b9493e3eb68c5e7f" dependencies = [ "futures", "js-sys", - "parking_lot 0.11.0", + "parking_lot", "pin-utils", "wasm-bindgen", "wasm-bindgen-futures", @@ -2763,12 +2745,6 @@ dependencies = [ "libc", ] -[[package]] -name = "winapi" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" - [[package]] name = "winapi" version = "0.3.9" @@ -2791,7 +2767,7 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -2811,6 +2787,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "x25519-dalek" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc614d95359fd7afc321b66d2107ede58b246b844cf5d8a0adcca413e439f088" +dependencies = [ + "curve25519-dalek 3.0.0", + "rand_core 0.5.1", + "zeroize", +] + [[package]] name = "yamux" version = "0.8.0" @@ -2820,7 +2807,7 @@ dependencies = [ "futures", "log", "nohash-hasher", - "parking_lot 0.11.0", + "parking_lot", "rand 0.7.3", "static_assertions", ] diff --git a/Cargo.toml b/Cargo.toml index 2906f61..d841219 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ db = ["ipfs-embed-db", "sled"] net = ["ipfs-embed-net"] [dependencies] -async-std = "1.6.5" +async-std = "1.7.0" async-trait = "0.1.41" fnv = "1.0.7" futures = "0.3.5" @@ -45,4 +45,4 @@ name = "list" harness = false [patch.crates-io] -libp2p-bitswap = { git = "https://github.com/ipfs-rust/libp2p-bitswap" } +libp2p-bitswap = { path = "../libp2p-bitswap" } #git = "https://github.com/ipfs-rust/libp2p-bitswap" } diff --git a/core/Cargo.toml b/core/Cargo.toml index 268595a..d0ae09c 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -12,6 +12,6 @@ anyhow = "1.0.32" async-trait = "0.1.41" futures = "0.3.5" libipld = { version = "0.7.1", default-features = false } -libp2p-core = "0.22.1" +libp2p-core = "0.24.0" libp2p-bitswap = "0.8.0" log = "0.4.11" diff --git a/examples/chain.rs b/examples/chain.rs index fbd6f1b..ca264c5 100644 --- a/examples/chain.rs +++ b/examples/chain.rs @@ -8,6 +8,7 @@ use std::convert::TryFrom; use std::path::Path; use std::sync::Arc; +const TMP_ROOT: &str = alias!(tmp_root); const ROOT: &str = alias!(root); #[derive(Debug, Default, DagCbor)] @@ -183,20 +184,23 @@ impl BlockChain { pub async fn sync(&mut self, root: &Cid) -> Result<()> { let syncer = ChainSyncer::new(self.index.clone(), self.ipfs.bitswap_storage()); - self.ipfs.alias_with_syncer(ROOT, Some(root), Some(syncer)).await?; + self.ipfs.alias_with_syncer(TMP_ROOT, Some(root), Some(syncer)).await?; let mut cid = *root; let mut block = self.get_by_cid(root).await?; let prev_root_id = self.root_id; + let new_root_id = block.id; - self.root_id = block.id; - self.root_cid = Some(cid); - - for _ in prev_root_id..self.root_id { + for _ in prev_root_id..new_root_id { self.index_block(block.id, &cid)?; cid = block.prev.unwrap(); block = self.get_by_cid(&cid).await?; } + + self.ipfs.alias(ROOT, Some(root), Some(syncer)).await?; + self.root_id = block.id; + self.root_cid = Some(cid); + Ok(()) } diff --git a/net/Cargo.toml b/net/Cargo.toml index 0590a48..56d6a01 100644 --- a/net/Cargo.toml +++ b/net/Cargo.toml @@ -19,7 +19,7 @@ names = "0.11.0" thiserror = "1.0.20" [dependencies.libp2p] -version = "0.28.1" +version = "0.30.1" default-features = false features = [ "dns", From 58652071a4018f2c2aaec3f4ae7953ec0c3a1826 Mon Sep 17 00:00:00 2001 From: David Craven Date: Wed, 18 Nov 2020 20:26:20 +0100 Subject: [PATCH 8/8] wasm --- net/Cargo.toml | 1 + net/src/behaviour.rs | 14 +++++++------- net/src/lib.rs | 25 ++++++++++++++++--------- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/net/Cargo.toml b/net/Cargo.toml index 56d6a01..76e744f 100644 --- a/net/Cargo.toml +++ b/net/Cargo.toml @@ -29,4 +29,5 @@ features = [ "ping", # "quic", "mplex", "noise", "tcp-async-std", "yamux", + "wasm-ext-websocket", ] diff --git a/net/src/behaviour.rs b/net/src/behaviour.rs index 2e4a4b1..17ec8a1 100644 --- a/net/src/behaviour.rs +++ b/net/src/behaviour.rs @@ -10,7 +10,7 @@ use libp2p::kad::{ BootstrapError, BootstrapOk, GetProvidersOk, Kademlia, KademliaEvent, QueryResult as KademliaQueryResult, }; -use libp2p::mdns::{Mdns, MdnsEvent}; +//use libp2p::mdns::{Mdns, MdnsEvent}; use libp2p::multiaddr::Protocol; use libp2p::ping::{Ping, PingEvent}; use libp2p::swarm::toggle::Toggle; @@ -33,7 +33,7 @@ pub struct NetworkBackendBehaviour { allow_non_globals_in_dht: bool, kad: Kademlia, - mdns: Toggle, + //mdns: Toggle, ping: Toggle, identify: Identify, bitswap: Bitswap

, @@ -46,7 +46,7 @@ pub struct NetworkBackendBehaviour { events: VecDeque, } -impl NetworkBehaviourEventProcess for NetworkBackendBehaviour

{ +/*impl NetworkBehaviourEventProcess for NetworkBackendBehaviour

{ fn inject_event(&mut self, event: MdnsEvent) { match event { MdnsEvent::Discovered(list) => { @@ -65,7 +65,7 @@ impl NetworkBehaviourEventProcess for NetworkBackendB } } } -} +}*/ impl NetworkBehaviourEventProcess for NetworkBackendBehaviour

{ fn inject_event(&mut self, event: KademliaEvent) { @@ -178,12 +178,12 @@ impl NetworkBackendBehaviour

{ pub fn new>(config: NetworkConfig, store: S) -> Result { let peer_id = config.peer_id(); - let mdns = if config.enable_mdns { + /*let mdns = if config.enable_mdns { Some(Mdns::new()?) } else { None } - .into(); + .into();*/ let kad_store = MemoryStore::new(peer_id.clone()); let mut kad = Kademlia::new(peer_id.clone(), kad_store); @@ -213,7 +213,7 @@ impl NetworkBackendBehaviour

{ Ok(Self { peer_id, allow_non_globals_in_dht: config.allow_non_globals_in_dht, - mdns, + //mdns, kad, ping, identify, diff --git a/net/src/lib.rs b/net/src/lib.rs index 73a51bd..eea18db 100644 --- a/net/src/lib.rs +++ b/net/src/lib.rs @@ -7,12 +7,13 @@ use ipfs_embed_core::{ }; use libp2p::core::transport::upgrade::Version; use libp2p::core::transport::Transport; -use libp2p::dns::DnsConfig; +//use libp2p::dns::DnsConfig; use libp2p::mplex::MplexConfig; use libp2p::noise::{Keypair, NoiseConfig, X25519Spec}; use libp2p::swarm::{Swarm, SwarmBuilder}; -use libp2p::tcp::TcpConfig; +//use libp2p::tcp::TcpConfig; //use libp2p::yamux::Config as YamuxConfig; +use libp2p::wasm_ext::{ExtTransport, ffi::websocket_transport}; use std::marker::PhantomData; use std::pin::Pin; use std::sync::Arc; @@ -36,20 +37,26 @@ impl NetworkService

{ let dh_key = Keypair::::new() .into_authentic(&config.node_key) .unwrap(); - let transport = DnsConfig::new( + /*let transport = DnsConfig::new( TcpConfig::new() .nodelay(true) .upgrade(Version::V1) .authenticate(NoiseConfig::xx(dh_key).into_authenticated()) .multiplex(MplexConfig::new()) .timeout(Duration::from_secs(5)), - )?; + )?;*/ + let transport = ExtTransport::new(websocket_transport()) + .upgrade(Version::V1) + .authenticate(NoiseConfig::xx(dh_key).into_authenticated()) + .multiplex(MplexConfig::new()) + .timeout(Duration::from_secs(5)) + .boxed(); let peer_id = config.peer_id(); let behaviour = NetworkBackendBehaviour::

::new(config.clone(), store)?; - let mut swarm = SwarmBuilder::new(transport, behaviour, peer_id.clone()) - .executor(Box::new(|fut| { async_std::task::spawn(fut); })) - .build(); + let mut swarm = Swarm::new(transport, behaviour, peer_id.clone()); + // .executor(Box::new(|fut| { async_std::task::spawn(fut); })) + // .build(); for addr in config.listen_addresses { Swarm::listen_on(&mut swarm, addr)?; } @@ -58,11 +65,11 @@ impl NetworkService

{ } let (tx, rx) = mpsc::unbounded(); - task::spawn(NetworkWorker { + /*task::spawn(NetworkWorker { swarm, rx, subscriptions: Default::default(), - }); + });*/ Ok(Self { _marker: PhantomData,