From 6a66f62f69e07f66aebbbe041467901a50479471 Mon Sep 17 00:00:00 2001 From: User65k <15049544+User65k@users.noreply.github.com> Date: Sat, 13 Apr 2024 13:52:43 +0200 Subject: [PATCH 1/3] h2 fix (#9) * h2 fix --- Cargo.toml | 8 ++- src/hyper/connector.rs | 110 +++++++++++++++++++++++++++++------------ src/hyper/mod.rs | 59 +++++++++++++++------- src/lib.rs | 45 +++++++++++++---- src/request.rs | 41 +++++++-------- src/response.rs | 6 ++- src/tcp/mod.rs | 56 +++++++++++++-------- 7 files changed, 217 insertions(+), 108 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 983aa42..ea18c83 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "generic-async-http-client" -version = "0.5.0" +version = "0.5.1" authors = ["User65k <15049544+User65k@users.noreply.github.com>"] edition = "2021" @@ -59,4 +59,8 @@ default = [] [dev-dependencies] async-std = "1.9.0" tokio = {version = "1.6.1", features=["rt", "net"]} -serde = {version = "1.0", features=["derive"]} \ No newline at end of file +serde = {version = "1.0", features=["derive"]} + +[package.metadata.docs.rs] +features = ["proxies"] +rustdoc-args = ["--cfg", "docsrs"] diff --git a/src/hyper/connector.rs b/src/hyper/connector.rs index 1111ac3..5ff1c03 100644 --- a/src/hyper/connector.rs +++ b/src/hyper/connector.rs @@ -1,20 +1,22 @@ +use crate::tcp::Stream; use hyper::{ - header::{HeaderValue, HOST}, http::uri::{Scheme, Uri} + client::conn::{http1, http2}, + header::{HeaderValue, HOST}, + http::uri::{Scheme, Uri}, }; -use crate::tcp::Stream; async fn connect_to_uri(dst: &Uri) -> Result { let tls = match dst.scheme_str() { Some("https") => true, Some("http") => false, - _ => { - return Err(super::Error::Scheme) - } + _ => return Err(super::Error::Scheme), }; let host = match dst.host() { Some(s) => s, None => { - return Err(hyper::http::uri::Authority::try_from("]").unwrap_err().into()); + return Err(hyper::http::uri::Authority::try_from("]") + .unwrap_err() + .into()); } }; let port = match dst.port() { @@ -27,16 +29,15 @@ async fn connect_to_uri(dst: &Uri) -> Result { } } }; - Stream::connect(host, port, tls).await.map_err(|e|e.into()) + Stream::connect(host, port, tls).await.map_err(|e| e.into()) } -#[derive(Debug, Clone, Default)] +#[derive(Debug, Default)] pub enum HyperClient { #[default] - New,/* - H1(hyper::client::conn::http1::SendRequest), - TlsH1(), - TlsH2(),*/ + New, + H1(http1::SendRequest), + H2(http2::SendRequest), } fn origin_form(uri: &mut Uri) { @@ -54,29 +55,74 @@ fn origin_form(uri: &mut Uri) { *uri = path } +#[derive(Clone)] +struct TokioExecutor; +impl hyper::rt::Executor for TokioExecutor +where + F: std::future::Future + Send + 'static, + F::Output: Send + 'static, +{ + fn execute(&self, future: F) { + tokio::spawn(future); + } +} + impl HyperClient { - pub async fn request(&mut self, mut req: super::Request) -> Result, super::Error> { - let io = connect_to_uri(req.uri()).await?; - let (mut sender, conn) = hyper::client::conn::http1::handshake(io).await?; - tokio::task::spawn(async move { - if let Err(err) = conn.await { - println!("Connection failed: {:?}", err); + pub async fn request( + &mut self, + mut req: super::Request, + ) -> Result, super::Error> { + match self { + HyperClient::New => { + let io = connect_to_uri(req.uri()).await?; + match io.get_proto() { + hyper::Version::HTTP_2 => { + let (sender, conn) = + hyper::client::conn::http2::handshake(TokioExecutor, io).await?; + tokio::task::spawn(async move { + if let Err(err) = conn.await { + println!("Connection failed: {:?}", err); + } + }); + let _ = std::mem::replace(self, HyperClient::H2(sender)); + } + hyper::Version::HTTP_11 => { + let (sender, conn) = hyper::client::conn::http1::handshake(io).await?; + tokio::task::spawn(async move { + if let Err(err) = conn.await { + println!("Connection failed: {:?}", err); + } + }); + let _ = std::mem::replace(self, HyperClient::H1(sender)); + } + _ => unreachable!(), + }; } - }); - let uri = req.uri().clone(); - req.headers_mut().entry(HOST).or_insert_with(|| { - let hostname = uri.host().expect("authority implies host"); - if let Some(port) = uri.port() { - let s = format!("{}:{}", hostname, port); - HeaderValue::from_str(&s) - } else { - HeaderValue::from_str(hostname) - } - .expect("uri host is valid header value") - }); + HyperClient::H1(_) => {} + HyperClient::H2(_) => {} + } - origin_form(req.uri_mut()); - sender.send_request(req).await.map_err(|e|e.into()) + match self { + HyperClient::New => unreachable!(), + HyperClient::H1(sender) => { + let uri = req.uri().authority().cloned().expect("authority implies host"); + req.headers_mut().entry(HOST).or_insert_with(|| { + let hostname = uri.host(); + if let Some(port) = uri.port() { + let s = format!("{}:{}", hostname, port); + HeaderValue::from_str(&s) + } else { + HeaderValue::from_str(hostname) + } + .expect("uri host is valid header value") + }); + + origin_form(req.uri_mut()); + + sender.send_request(req).await.map_err(|e| e.into()) + }, + HyperClient::H2(sender) => sender.send_request(req).await.map_err(|e| e.into()), + } } } diff --git a/src/hyper/mod.rs b/src/hyper/mod.rs index 807c3b9..ccb4272 100644 --- a/src/hyper/mod.rs +++ b/src/hyper/mod.rs @@ -1,18 +1,24 @@ -use std::{convert::{Infallible, TryFrom}, str::FromStr}; +use std::{ + convert::{Infallible, TryFrom}, + str::FromStr, +}; use serde::Serialize; pub use hyper::{ - header::{HeaderName, HeaderValue}, body::Incoming, + header::{HeaderName, HeaderValue}, }; use hyper::{ - body::{Body as BodyTrait, Bytes, Frame, SizeHint}, header::{InvalidHeaderName, InvalidHeaderValue, CONTENT_TYPE}, http::{ + body::{Body as BodyTrait, Bytes, Frame, SizeHint}, + header::{InvalidHeaderName, InvalidHeaderValue, CONTENT_TYPE}, + http::{ method::{InvalidMethod, Method}, request::Builder, uri::{Builder as UriBuilder, InvalidUri, PathAndQuery, Uri}, Error as HTTPError, - }, Error as HyperError, Request, Response + }, + Error as HyperError, Request, Response, }; use std::mem::take; @@ -27,7 +33,7 @@ pub(crate) fn get_client() -> HyperClient { pub struct Req { req: Builder, body: Body, - pub(crate) client: Option + pub(crate) client: Option, } pub struct Resp { resp: Response, @@ -39,23 +45,22 @@ impl Into> for crate::Response { } } -impl TryFrom<(M,U)> for crate::Request +impl TryFrom<(M, U)> for crate::Request where Method: TryFrom, >::Error: Into, Uri: TryFrom, >::Error: Into, - { - +{ type Error = Infallible; - fn try_from(value: (M,U)) -> Result { + fn try_from(value: (M, U)) -> Result { let req = Builder::new().method(value.0).uri(value.1); Ok(crate::Request(Req { req, body: Body::empty(), - client: None + client: None, })) } } @@ -88,7 +93,7 @@ impl Req { Req { req, body: Body::empty(), - client: None + client: None, } } pub async fn send_request(mut self) -> Result { @@ -96,7 +101,7 @@ impl Req { let resp = if let Some(mut client) = self.client.take() { client.request(req).await? - }else{ + } else { get_client().request(req).await? }; Ok(Resp { resp }) @@ -165,9 +170,22 @@ impl Resp { pub async fn bytes(&mut self) -> Result, Error> { let mut b = aggregate(self.resp.body_mut()).await?; let capacity = b.remaining(); - //TODO uninit - let mut v = vec![0;capacity]; - b.copy_to_slice(&mut v); + let mut v = Vec::with_capacity(capacity); + let ptr = v.spare_capacity_mut().as_mut_ptr(); + let mut off = 0; + while off < capacity { + let cnt; + unsafe { + let src = b.chunk(); + cnt = src.len(); + std::ptr::copy_nonoverlapping(src.as_ptr(), ptr.add(off).cast(), cnt); + off += cnt; + } + b.advance(cnt); + } + unsafe { + v.set_len(capacity); + } Ok(v) } pub async fn string(&mut self) -> Result { @@ -215,7 +233,10 @@ struct Framed<'a>(&'a mut Incoming); impl<'a> futures::Future for Framed<'a> { type Output = Option, hyper::Error>>; - fn poll(mut self: std::pin::Pin<&mut Self>, ctx: &mut std::task::Context<'_>) -> std::task::Poll { + fn poll( + mut self: std::pin::Pin<&mut Self>, + ctx: &mut std::task::Context<'_>, + ) -> std::task::Poll { std::pin::Pin::new(&mut self.0).poll_frame(ctx) } } @@ -241,7 +262,7 @@ pub enum Error { InvalidHeaderName(InvalidHeaderName), InvalidUri(InvalidUri), Urlencoded(serde_urlencoded::ser::Error), - Io(std::io::Error) + Io(std::io::Error), } impl std::error::Error for Error {} use std::fmt; @@ -361,7 +382,7 @@ impl hyper::body::Body for Body { ) -> std::task::Poll, Self::Error>>> { if self.0.is_empty() { std::task::Poll::Ready(None) - }else{ + } else { let v: Vec = std::mem::take(self.0.as_mut()); std::task::Poll::Ready(Some(Ok(Frame::data(v.into())))) } @@ -369,4 +390,4 @@ impl hyper::body::Body for Body { fn size_hint(&self) -> SizeHint { SizeHint::with_exact(self.0.len() as u64) } -} \ No newline at end of file +} diff --git a/src/lib.rs b/src/lib.rs index d9089b6..662b3a3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,14 +15,21 @@ You need to specify via features what crates are used to the actual work. Without anything specified you will end up with *No HTTP backend was selected*. +If performing more than one HTTP Request you should favor the use of [`Session`] over [`Request`]. */ +#![cfg_attr(docsrs, feature(doc_cfg))] + #[cfg(not(any(feature = "use_hyper", feature = "use_async_h1")))] #[path = "dummy/mod.rs"] mod imp; #[cfg(any(feature = "use_hyper", feature = "use_async_h1"))] mod tcp; -#[cfg(all(any(feature = "use_hyper", feature = "use_async_h1"), feature = "proxies"))] +#[cfg(all( + any(feature = "use_hyper", feature = "use_async_h1"), + feature = "proxies" +))] +#[cfg_attr(docsrs, doc(cfg(feature = "proxies")))] pub use tcp::proxy; #[cfg(feature = "use_async_h1")] @@ -46,7 +53,6 @@ pub use body::Body; pub use header::{HeaderName, HeaderValue}; pub use imp::Error; - #[cfg(test)] mod tests { #[cfg(feature = "use_async_h1")] @@ -88,7 +94,10 @@ mod tests { async { jh.await.expect("spawn failed") } } - pub(crate) async fn assert_stream(stream: &mut TcpStream, should_be: &[u8]) -> std::io::Result<()> { + pub(crate) async fn assert_stream( + stream: &mut TcpStream, + should_be: &[u8], + ) -> std::io::Result<()> { let l = should_be.len(); let mut req: Vec = vec![0; l]; let _r = stream.read(req.as_mut_slice()).await?; @@ -109,7 +118,11 @@ mod tests { let mut output = Vec::with_capacity(1); assert_stream( &mut stream, - format!("GET / HTTP/1.1\r\nhost: {}:{}\r\ncontent-length: 0\r\n\r\n",host,port).as_bytes(), + format!( + "GET / HTTP/1.1\r\nhost: {}:{}\r\ncontent-length: 0\r\n\r\n", + host, port + ) + .as_bytes(), ) .await?; @@ -121,8 +134,8 @@ mod tests { } block_on(async { let (listener, port, host) = listen_somewhere().await?; - let uri = format!("http://{}:{}",host,port); - let t = spawn(server(listener,host,port)); + let uri = format!("http://{}:{}", host, port); + let t = spawn(server(listener, host, port)); let r = Request::get(&uri); let mut aw = r.exec().await?; @@ -140,11 +153,23 @@ mod tests { //let mut output = Vec::with_capacity(2); #[cfg(feature = "use_async_h1")] - assert_stream(&mut stream, format!("PUT / HTTP/1.1\r\nhost: {}:{}\r\ncontent-length: 0\r\ncookies: jo\r\n\r\n",host,port).as_bytes()).await?; + assert_stream( + &mut stream, + format!( + "PUT / HTTP/1.1\r\nhost: {}:{}\r\ncontent-length: 0\r\ncookies: jo\r\n\r\n", + host, port + ) + .as_bytes(), + ) + .await?; #[cfg(feature = "use_hyper")] assert_stream( &mut stream, - format!("PUT / HTTP/1.1\r\ncookies: jo\r\nhost: {}:{}\r\ncontent-length: 0\r\n\r\n",host,port).as_bytes(), + format!( + "PUT / HTTP/1.1\r\ncookies: jo\r\nhost: {}:{}\r\ncontent-length: 0\r\n\r\n", + host, port + ) + .as_bytes(), ) .await?; @@ -157,8 +182,8 @@ mod tests { } block_on(async { let (listener, port, host) = listen_somewhere().await?; - let uri = format!("http://{}:{}",host,port); - let server = spawn(server(listener,host,port)); + let uri = format!("http://{}:{}", host, port); + let server = spawn(server(listener, host, port)); let r = Request::new("PUT", &uri)?; let r = r.set_header("Cookies", "jo")?; let resp = r.exec().await; diff --git a/src/request.rs b/src/request.rs index a0757ec..12f6b1c 100644 --- a/src/request.rs +++ b/src/request.rs @@ -1,5 +1,5 @@ +use crate::{imp, Body, Error, HeaderName, HeaderValue, Response}; use serde::Serialize; -use crate::{Error, Body, HeaderName, HeaderValue, Response, imp}; use std::convert::TryInto; /// Builds a HTTP request, poll it to query @@ -11,6 +11,8 @@ use std::convert::TryInto; /// # Ok(()) /// # } /// ``` +/// +/// Depending on the chosen implementation, `Request` implements `TryFrom<(TryInto, TryInto)>`. pub struct Request(pub(crate) imp::Req); impl Request { //auth @@ -104,16 +106,13 @@ impl Request { /// # req.exec().await /// # } /// ``` - pub fn set_header( - mut self, - name: N, - value: V, - ) -> Result - where - N: TryInto, - V: TryInto, - Error: From, - Error: From, { + pub fn set_header(mut self, name: N, value: V) -> Result + where + N: TryInto, + V: TryInto, + Error: From, + Error: From, + { let val: HeaderValue = value.try_into()?; let name: HeaderName = name.try_into()?; self.0.set_header(name.into(), val.into())?; @@ -122,16 +121,13 @@ impl Request { } /// Add a single header to the request /// If the map did have this key present, the new value is pushed to the end of the list of values - pub fn add_header( - mut self, - name: N, - value: V, - ) -> Result - where - N: TryInto, - V: TryInto, - Error: From, - Error: From, { + pub fn add_header(mut self, name: N, value: V) -> Result + where + N: TryInto, + V: TryInto, + Error: From, + Error: From, + { let val: HeaderValue = value.try_into()?; let name: HeaderName = name.try_into()?; self.0.add_header(name.into(), val.into())?; @@ -147,6 +143,7 @@ impl Request { /// Send the request to the webserver pub async fn exec(self) -> Result { let r = self.0.send_request().await.map(Response)?; + //https://crates.io/crates/hreq if r.status_code() > 299 && r.status_code() < 399 { if let Some(loc) = r.header("Location").and_then(|l| l.try_into().ok()) { @@ -193,4 +190,4 @@ impl Future for Request2{ }, } } -}*/ \ No newline at end of file +}*/ diff --git a/src/response.rs b/src/response.rs index d939d1a..b1a8389 100644 --- a/src/response.rs +++ b/src/response.rs @@ -1,10 +1,12 @@ +use crate::{imp, Error, HeaderName, HeaderValue}; use serde::de::DeserializeOwned; -use crate::{Error, HeaderName, HeaderValue, imp}; use std::convert::TryInto; /// The response of a webserver. /// Headers and Status are available from the start, /// the body must be polled/awaited again +/// +/// Depending on the chosen implementation, `Response` implements `Into` or `Into`. pub struct Response(pub(crate) imp::Resp); impl Response { /// Return the status code @@ -53,4 +55,4 @@ impl std::fmt::Debug for Response { let h: Vec<(&HeaderName, &HeaderValue)> = self.headers().collect(); write!(f, "HTTP {} Header: {:?}", self.status_code(), h) } -} \ No newline at end of file +} diff --git a/src/tcp/mod.rs b/src/tcp/mod.rs index 03c21ca..2641a73 100644 --- a/src/tcp/mod.rs +++ b/src/tcp/mod.rs @@ -23,25 +23,25 @@ use http_types::Url as Uri; #[cfg(all(feature = "use_hyper", feature = "proxies"))] use hyper::http::uri::Uri; #[cfg(feature = "use_hyper")] +use hyper::rt::{Read, ReadBufCursor, Write}; +#[cfg(feature = "use_hyper")] use tokio::{ io::{AsyncRead as _, AsyncWrite as _}, net::TcpStream, }; -#[cfg(feature = "use_hyper")] -use hyper::rt::{Read, Write, ReadBufCursor}; -#[cfg(any(feature = "async_native_tls",feature = "hyper_native_tls"))] +#[cfg(any(feature = "async_native_tls", feature = "hyper_native_tls"))] use async_native_tls::{TlsConnector, TlsStream}; #[cfg(all(feature = "rustls", feature = "use_async_h1"))] use futures_rustls::{ client::TlsStream, - rustls::{ClientConfig, RootCertStore, pki_types::ServerName}, + rustls::{pki_types::ServerName, ClientConfig, RootCertStore}, TlsConnector, }; #[cfg(all(feature = "rustls", feature = "use_hyper"))] use tokio_rustls::{ client::TlsStream, - rustls::{ClientConfig, RootCertStore, pki_types::ServerName}, + rustls::{pki_types::ServerName, ClientConfig, RootCertStore}, TlsConnector, }; #[cfg(feature = "rustls")] @@ -62,8 +62,8 @@ enum State { #[cfg(feature = "proxies")] pub mod proxy { - use async_trait::async_trait; use super::*; + use async_trait::async_trait; /// Sets the global proxy to a `&'static Proxy`. pub fn set_proxy(proxy: &'static dyn Proxy) { @@ -93,8 +93,12 @@ pub mod proxy { pub struct NoProxy; #[async_trait] impl Proxy for NoProxy { - async fn connect_w_proxy(&self, host: &str, port: u16, _tls: bool) -> io::Result - { + async fn connect_w_proxy( + &self, + host: &str, + port: u16, + _tls: bool, + ) -> io::Result { TcpStream::connect((host, port)).await } } @@ -177,7 +181,8 @@ pub mod proxy { match scheme { Some("http") => connect_via_http_prx(host, port, phost, pport).await, Some(socks5) if socks5 == "socks5" || socks5 == "socks5h" => { - connect_via_socks_prx(host, port, phost, pport, socks5 == "socks5h").await + connect_via_socks_prx(host, port, phost, pport, socks5 == "socks5h") + .await } _ => { return Err(io::Error::new( @@ -193,7 +198,9 @@ pub mod proxy { #[cfg(test)] mod tests { - use crate::tests::{assert_stream, TcpListener, spawn, block_on, listen_somewhere, WriteExt}; + use crate::tests::{ + assert_stream, block_on, listen_somewhere, spawn, TcpListener, WriteExt, + }; #[test] fn prx_from_env() { async fn server(listener: TcpListener) -> std::io::Result { @@ -205,10 +212,11 @@ pub mod proxy { ) .await?; stream.write_all(b"HTTP/1.1 200 Connected\r\n\r\n").await?; - + assert_stream( &mut stream, - format!("GET /bla HTTP/1.1\r\nhost: whatever\r\ncontent-length: 0\r\n\r\n").as_bytes(), + format!("GET /bla HTTP/1.1\r\nhost: whatever\r\ncontent-length: 0\r\n\r\n") + .as_bytes(), ) .await?; stream @@ -225,7 +233,7 @@ pub mod proxy { let r = crate::Request::get("http://whatever/bla"); let mut aw = r.exec().await?; - + assert_eq!(aw.status_code(), 200, "wrong status"); assert_eq!(aw.text().await?, "abc", "wrong text"); assert!(t.await?, "not cool"); @@ -241,7 +249,7 @@ pub mod proxy { feature = "hyper_native_tls", feature = "async_native_tls" ))] -fn get_tls_connector() -> io::Result{ +fn get_tls_connector() -> io::Result { #[cfg(feature = "rustls")] { let mut root_store = RootCertStore::empty(); @@ -257,7 +265,7 @@ fn get_tls_connector() -> io::Result{ Ok(TlsConnector::from(Arc::new(config))) } - #[cfg(any(feature = "async_native_tls",feature = "hyper_native_tls"))] + #[cfg(any(feature = "async_native_tls", feature = "hyper_native_tls"))] return Ok(TlsConnector::new()); } @@ -270,12 +278,16 @@ impl Stream { log::trace!("connected to {}:{}", host, port); if tls { - #[cfg(any(feature = "hyper_native_tls", feature = "async_native_tls", feature = "rustls"))] + #[cfg(any( + feature = "hyper_native_tls", + feature = "async_native_tls", + feature = "rustls" + ))] { #[cfg(feature = "rustls")] - let host = ServerName::try_from(host).map_err(|_e| { - io::Error::new(io::ErrorKind::InvalidInput, "Invalid DNS name") - })?.to_owned(); + let host = ServerName::try_from(host) + .map_err(|_e| io::Error::new(io::ErrorKind::InvalidInput, "Invalid DNS name"))? + .to_owned(); let tlsc = get_tls_connector()?; let tls = tlsc.connect(host, tcp).await; @@ -289,7 +301,9 @@ impl Stream { Err(e) => { log::error!("TLS Handshake: {}", e); #[cfg(feature = "rustls")] - {Err(e)} + { + Err(e) + } #[cfg(any(feature = "hyper_native_tls", feature = "async_native_tls"))] Err(io::Error::new(io::ErrorKind::InvalidInput, e)) } @@ -314,7 +328,7 @@ impl Stream { #[cfg(feature = "use_hyper")] impl Stream { - fn get_proto(&self) -> hyper::Version { + pub fn get_proto(&self) -> hyper::Version { #[cfg(feature = "rustls")] if let State::Tls(ref t) = self.state { let (_, s) = t.get_ref(); From dc2a3ed5913faac8acefcb0e0b882cc936cbc7ee Mon Sep 17 00:00:00 2001 From: giangndm <45644921+giangndm@users.noreply.github.com> Date: Fri, 17 Jan 2025 22:01:14 +0700 Subject: [PATCH 2/3] feat: make http2 optional (#10) * feat: make http2 optional * fixed io-util missing --------- Co-authored-by: User65k <15049544+User65k@users.noreply.github.com> --- Cargo.toml | 7 ++++--- src/hyper/connector.rs | 22 +++++++++++++++++----- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ea18c83..5f2b3e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,12 +43,13 @@ cookie_store = { version = "0.21.0", optional = true } async-trait = { version = "0.1", optional = true } [features] -use_hyper = ["tokio/net", "tokio/rt", "hyper/http1", "hyper/http2", "hyper/client", "serde_qs", "serde_urlencoded","serde_json"] +use_hyper = ["tokio/net", "tokio/rt", "hyper/http1", "hyper/client", "serde_qs", "serde_urlencoded","serde_json"] use_async_h1 = ["async-std", "async-h1", "http-types"] use_web_sys = ["web-sys", "wasm-bindgen", "wasm-bindgen-futures", "js-sys"] +http2 = ["hyper/http2"] cookies = ["cookie_store"] -proxies = ["async-trait"] +proxies = ["async-trait", "tokio/io-util"] rustls = ["futures-rustls", "tokio-rustls", "webpki-roots"] async_native_tls = ["use_async_h1","async-native-tls/runtime-async-std"] @@ -58,7 +59,7 @@ default = [] [dev-dependencies] async-std = "1.9.0" -tokio = {version = "1.6.1", features=["rt", "net"]} +tokio = {version = "1.6.1", features=["rt", "net", "io-util"]} serde = {version = "1.0", features=["derive"]} [package.metadata.docs.rs] diff --git a/src/hyper/connector.rs b/src/hyper/connector.rs index 5ff1c03..970c430 100644 --- a/src/hyper/connector.rs +++ b/src/hyper/connector.rs @@ -1,6 +1,8 @@ use crate::tcp::Stream; +#[cfg(feature = "http2")] +use hyper::client::conn::http2; use hyper::{ - client::conn::{http1, http2}, + client::conn::http1, header::{HeaderValue, HOST}, http::uri::{Scheme, Uri}, }; @@ -37,6 +39,7 @@ pub enum HyperClient { #[default] New, H1(http1::SendRequest), + #[cfg(feature = "http2")] H2(http2::SendRequest), } @@ -55,8 +58,11 @@ fn origin_form(uri: &mut Uri) { *uri = path } +#[cfg(feature = "http2")] #[derive(Clone)] struct TokioExecutor; + +#[cfg(feature = "http2")] impl hyper::rt::Executor for TokioExecutor where F: std::future::Future + Send + 'static, @@ -76,6 +82,7 @@ impl HyperClient { HyperClient::New => { let io = connect_to_uri(req.uri()).await?; match io.get_proto() { + #[cfg(feature = "http2")] hyper::Version::HTTP_2 => { let (sender, conn) = hyper::client::conn::http2::handshake(TokioExecutor, io).await?; @@ -99,14 +106,18 @@ impl HyperClient { }; } HyperClient::H1(_) => {} + #[cfg(feature = "http2")] HyperClient::H2(_) => {} } - match self { HyperClient::New => unreachable!(), HyperClient::H1(sender) => { - let uri = req.uri().authority().cloned().expect("authority implies host"); + let uri = req + .uri() + .authority() + .cloned() + .expect("authority implies host"); req.headers_mut().entry(HOST).or_insert_with(|| { let hostname = uri.host(); if let Some(port) = uri.port() { @@ -116,12 +127,13 @@ impl HyperClient { HeaderValue::from_str(hostname) } .expect("uri host is valid header value") - }); + }); origin_form(req.uri_mut()); sender.send_request(req).await.map_err(|e| e.into()) - }, + } + #[cfg(feature = "http2")] HyperClient::H2(sender) => sender.send_request(req).await.map_err(|e| e.into()), } } From 732915cb1901989c11d5ba3c43449ec48860b689 Mon Sep 17 00:00:00 2001 From: User65k <15049544+User65k@users.noreply.github.com> Date: Fri, 17 Jan 2025 22:14:49 +0100 Subject: [PATCH 3/3] rel plz, fmt, updates (#11) * h2 fix * only fiddle with url on h1 * rel plz, readme, fmt --- .github/workflows/release-plz.yml | 27 ++++++++ Cargo.toml | 33 +++++----- README.md | 3 +- src/hyper/connector.rs | 52 ++++++++------- src/hyper/mod.rs | 12 ++-- src/lib.rs | 43 +++++++------ src/tcp/http.rs | 22 ++----- src/tcp/socks5.rs | 102 ++++++++++-------------------- 8 files changed, 140 insertions(+), 154 deletions(-) create mode 100644 .github/workflows/release-plz.yml diff --git a/.github/workflows/release-plz.yml b/.github/workflows/release-plz.yml new file mode 100644 index 0000000..faa4328 --- /dev/null +++ b/.github/workflows/release-plz.yml @@ -0,0 +1,27 @@ +name: Release-plz + +permissions: + pull-requests: write + contents: write + +on: + push: + branches: + - main + +jobs: + release-plz: + name: Release-plz + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + - name: Run release-plz + uses: MarcoIeni/release-plz-action@v0.5 + env: + GITHUB_TOKEN: ${{ secrets.RELEASE_PLZ_TOKEN }} + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} diff --git a/Cargo.toml b/Cargo.toml index 5f2b3e9..ae037b4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "generic-async-http-client" -version = "0.5.1" +version = "0.6.0" authors = ["User65k <15049544+User65k@users.noreply.github.com>"] edition = "2021" @@ -11,20 +11,20 @@ keywords = ["http", "request", "client", "async"] readme = "README.md" [dependencies] -async-std = {version="1.9.0",optional=true} +async-std = {version="1",optional=true} async-h1 = {version="2.3",optional=true} http-types = {version="2.11",optional=true} -hyper = { version = "1.2", optional=true } -serde_qs = { version ="0.12", optional=true } +hyper = { version = "1.5", optional=true } +serde_qs = { version ="0.13", optional=true } serde_urlencoded = { version ="0.7", optional=true } serde_json = {version="1.0",optional=true} -tokio = {version = "1.6", optional=true} +tokio = {version = "1", optional=true} -web-sys = {version = "0.3.4", features = ['Headers', 'Request', 'RequestInit', 'RequestMode', 'Response', 'Window'],optional=true} -wasm-bindgen = { version = "0.2.74", features = ["serde-serialize"], optional=true} -js-sys = {version = "0.3.51",optional=true} -wasm-bindgen-futures = {version = "0.4.24",optional=true} +web-sys = {version = "0.3", features = ['Headers', 'Request', 'RequestInit', 'RequestMode', 'Response', 'Window'],optional=true} +wasm-bindgen = { version = "0.2", features = ["serde-serialize"], optional=true} +js-sys = {version = "0.3",optional=true} +wasm-bindgen-futures = {version = "0.4",optional=true} futures = "0.3" @@ -32,14 +32,14 @@ log = "0.4" serde = "1.0" #pin-project = "1.0" -futures-rustls = {version="0.25.0",optional=true} -tokio-rustls = { version = "0.25.0", optional = true } -webpki-roots = {version="0.26.0",optional=true} +futures-rustls = {version="0.26", optional = true, default-features = false, features = ["tls12"]} +tokio-rustls = { version = "0.26", optional = true, default-features = false, features = ["tls12"]} +webpki-roots = {version="0.26", optional = true} #rustls-native-certs async-native-tls = { version = "0.5", default-features = false, optional = true } -cookie_store = { version = "0.21.0", optional = true } +cookie_store = { version = "0.21", optional = true } async-trait = { version = "0.1", optional = true } [features] @@ -51,15 +51,16 @@ http2 = ["hyper/http2"] cookies = ["cookie_store"] proxies = ["async-trait", "tokio/io-util"] -rustls = ["futures-rustls", "tokio-rustls", "webpki-roots"] +rustls = ["futures-rustls/ring", "tokio-rustls/ring", "webpki-roots"] +rustls_aws_lc_rs = ["futures-rustls/aws_lc_rs", "tokio-rustls/aws_lc_rs", "webpki-roots"] async_native_tls = ["use_async_h1","async-native-tls/runtime-async-std"] hyper_native_tls = ["use_hyper","async-native-tls/runtime-tokio"] default = [] [dev-dependencies] -async-std = "1.9.0" -tokio = {version = "1.6.1", features=["rt", "net", "io-util"]} +async-std = "1" +tokio = {version = "1", features=["rt", "net", "io-util"]} serde = {version = "1.0", features=["derive"]} [package.metadata.docs.rs] diff --git a/README.md b/README.md index d4ce3ec..eeb0c18 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ You need to specify via features what crates are used to do the actual work. |proxies|Add support for Socks5 and HTTP proxy| |hyper_native_tls|Use [hyper](https://crates.io/crates/hyper) for HTTP and do HTTPS via [native_tls](https://crates.io/crates/native_tls) ![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/User65k/generic-async-http-client/test_hyper_nativetls.yml)| |async_native_tls|Use [async_h1](https://crates.io/crates/async_h1) for HTTP and do HTTPS via [native_tls](https://crates.io/crates/native_tls) ![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/User65k/generic-async-http-client/test_async_std_nativetls.yml)| +|http2|Use http2 if available (only works with use_hyper)| Without anything specified you will end up with *No HTTP backend was selected*. If you use this crate for a library, please [reexport](https://doc.rust-lang.org/cargo/reference/features.html#dependency-features) the appropriate features. @@ -45,7 +46,7 @@ A crate I found did what I needed but used async-h1 and async-std. While that wo So I wrote this. You can specify which backend to use. In the Webserver case, using tokio which is already a dependency VS async-std did lead to 81 less crates and a 350kB smaller binary. Using (and [async-acme](https://crates.io/crates/async-acme)): -``` +```toml [profile.release] lto = "fat" codegen-units = 1 diff --git a/src/hyper/connector.rs b/src/hyper/connector.rs index 970c430..de18531 100644 --- a/src/hyper/connector.rs +++ b/src/hyper/connector.rs @@ -43,25 +43,9 @@ pub enum HyperClient { H2(http2::SendRequest), } -fn origin_form(uri: &mut Uri) { - let path = match uri.path_and_query() { - Some(path) if path.as_str() != "/" => { - let mut parts = hyper::http::uri::Parts::default(); - parts.path_and_query = Some(path.clone()); - Uri::from_parts(parts).expect("path is valid uri") - } - _none_or_just_slash => { - debug_assert!(Uri::default() == "/"); - Uri::default() - } - }; - *uri = path -} - -#[cfg(feature = "http2")] #[derive(Clone)] +#[cfg(feature = "http2")] struct TokioExecutor; - #[cfg(feature = "http2")] impl hyper::rt::Executor for TokioExecutor where @@ -110,17 +94,18 @@ impl HyperClient { HyperClient::H2(_) => {} } + match self { HyperClient::New => unreachable!(), HyperClient::H1(sender) => { - let uri = req - .uri() - .authority() - .cloned() - .expect("authority implies host"); - req.headers_mut().entry(HOST).or_insert_with(|| { - let hostname = uri.host(); - if let Some(port) = uri.port() { + + let (mut parts, body) = req.into_parts(); + let mut up = parts.uri.into_parts(); + + let auth = up.authority.take().expect("authority implies host"); + parts.headers.entry(HOST).or_insert_with(|| { + let hostname = auth.host(); + if let Some(port) = auth.port() { let s = format!("{}:{}", hostname, port); HeaderValue::from_str(&s) } else { @@ -129,10 +114,23 @@ impl HyperClient { .expect("uri host is valid header value") }); - origin_form(req.uri_mut()); + //origin_form + parts.uri = match up.path_and_query { + Some(path) if path.as_str() != "/" => { + let mut parts = hyper::http::uri::Parts::default(); + parts.path_and_query = Some(path); + Uri::from_parts(parts).expect("path is valid uri") + } + _none_or_just_slash => { + debug_assert!(Uri::default() == "/"); + Uri::default() + } + }; + + let req = hyper::Request::from_parts(parts, body); sender.send_request(req).await.map_err(|e| e.into()) - } + }, #[cfg(feature = "http2")] HyperClient::H2(sender) => sender.send_request(req).await.map_err(|e| e.into()), } diff --git a/src/hyper/mod.rs b/src/hyper/mod.rs index ccb4272..2422ded 100644 --- a/src/hyper/mod.rs +++ b/src/hyper/mod.rs @@ -39,9 +39,9 @@ pub struct Resp { resp: Response, } -impl Into> for crate::Response { - fn into(self) -> Response { - self.0.resp +impl From for Response { + fn from(val: crate::Response) -> Self { + val.0.resp } } @@ -121,8 +121,12 @@ impl Req { self.body = query.into(); Ok(()) } + #[inline] pub fn query(&mut self, query: &T) -> Result<(), Error> { - let query = serde_qs::to_string(&query)?; + // codegen trampoline: https://github.com/rust-lang/rust/issues/77960 + self._query(serde_qs::to_string(&query)?) + } + fn _query(&mut self, query: String) -> Result<(), Error> { let old = self.req.uri_ref().expect("no uri"); let mut p_and_p = String::with_capacity(old.path().len() + query.len() + 1); diff --git a/src/lib.rs b/src/lib.rs index 662b3a3..59ea8f3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,22 +1,23 @@ -/*! A generic async HTTP request create. - -It is meant to be a thin wrapper around various HTTP clients -and handles TLS, serialisation and parsing. - -The main goal is to allow binaries (that pull in some libraries that make use of a HTTP client) -to specify what implementation should be used. - -And if there is a Proxy. If not specified auto detection is performed by looking at HTTP_PROXY. - -You need to specify via features what crates are used to the actual work. - -- `use_hyper` (and tokio) -- `use_async_h1` (and async-std) - -Without anything specified you will end up with *No HTTP backend was selected*. - -If performing more than one HTTP Request you should favor the use of [`Session`] over [`Request`]. -*/ +#![doc = include_str!("../README.md")] +/*! + * # Example + * ``` + * # use generic_async_http_client::{Request, Response, Error}; + * # use serde::Serialize; + * #[derive(Serialize)] + * struct ContactForm { + * email: String, + * text: String, + * } + * async fn post_form(form: &ContactForm) -> Result<(), Error> { + * let req = Request::post("http://example.com/").form(form)?; + * assert_eq!(req.exec().await?.text().await?, "ok"); + * Ok(()) + * } + * ``` + * + * If performing more than one HTTP Request you should favor the use of [`Session`] over [`Request`]. + */ #![cfg_attr(docsrs, feature(doc_cfg))] #[cfg(not(any(feature = "use_hyper", feature = "use_async_h1")))] @@ -53,7 +54,9 @@ pub use body::Body; pub use header::{HeaderName, HeaderValue}; pub use imp::Error; -#[cfg(test)] +#[cfg(all(test, + any(feature = "use_hyper", feature = "use_async_h1") +))] mod tests { #[cfg(feature = "use_async_h1")] pub(crate) use async_std::{ diff --git a/src/tcp/http.rs b/src/tcp/http.rs index ef5bf45..258ca29 100644 --- a/src/tcp/http.rs +++ b/src/tcp/http.rs @@ -45,16 +45,13 @@ pub async fn connect_via_http_prx( read = &buffer[..r]; } } - Err(io::Error::new( - io::ErrorKind::InvalidData, - host.to_string(), - )) + Err(io::Error::new(io::ErrorKind::InvalidData, host.to_string())) } #[cfg(test)] mod tests { use super::*; - use crate::tests::{assert_stream, TcpListener, spawn, block_on, listen_somewhere}; + use crate::tests::{assert_stream, block_on, listen_somewhere, spawn, TcpListener}; #[test] fn http_proxy() { async fn server(listener: TcpListener) -> std::io::Result { @@ -66,11 +63,7 @@ mod tests { ) .await?; stream.write_all(b"HTTP/1.1 200 Connected\r\n\r\n").await?; - assert_stream( - &mut stream, - b"n0ice", - ) - .await?; + assert_stream(&mut stream, b"n0ice").await?; Ok(true) } @@ -78,12 +71,7 @@ mod tests { let (listener, pport, phost) = listen_somewhere().await?; let t = spawn(server(listener)); - let mut stream = connect_via_http_prx( - "host", - 1234, - &phost, - pport, - ).await?; + let mut stream = connect_via_http_prx("host", 1234, &phost, pport).await?; stream.write_all(b"n0ice").await?; assert!(t.await?, "not cool"); @@ -91,4 +79,4 @@ mod tests { }) .unwrap(); } -} \ No newline at end of file +} diff --git a/src/tcp/socks5.rs b/src/tcp/socks5.rs index 62cc112..99b219c 100644 --- a/src/tcp/socks5.rs +++ b/src/tcp/socks5.rs @@ -51,10 +51,7 @@ pub async fn connect_via_socks_prx( let a = (host, port) .to_socket_addrs()? .next() - .ok_or_else(|| io::Error::new( - io::ErrorKind::NotFound, - "Could not resolve the host", - ))?; + .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "Could not resolve the host"))?; match a.ip() { IpAddr::V4(ip) => { buf.push(1); @@ -68,16 +65,13 @@ pub async fn connect_via_socks_prx( } buf.extend_from_slice(&port.to_be_bytes()); let mut socket = TcpStream::connect((phost, pport)).await?; - socket.write_all(b"\x05\x01\0").await?;//client auth methods: [no auth] + socket.write_all(b"\x05\x01\0").await?; //client auth methods: [no auth] let mut auth = [0, 0]; socket.read_exact(&mut auth).await?; - if auth[0]==5 && auth[1]==0 { + if auth[0] == 5 && auth[1] == 0 { //proxy wants no auth - }else{ - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "socks error", - )); + } else { + return Err(io::Error::new(io::ErrorKind::InvalidData, "socks error")); } socket.write_all(&buf).await?; let bytes_read = socket.read(&mut buf[..5]).await?; @@ -91,14 +85,14 @@ pub async fn connect_via_socks_prx( match socks_header_len.checked_sub(bytes_read) { Some(missing_bytes) if missing_bytes > 0 => { buf.resize(socks_header_len, 0); - socket.read_exact(&mut buf[bytes_read..socks_header_len]).await?; + socket + .read_exact(&mut buf[bytes_read..socks_header_len]) + .await?; } - Some(_) => {} //0 - None => { //already read to much - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "socks error", - )) + Some(_) => {} //0 + None => { + //already read to much + return Err(io::Error::new(io::ErrorKind::InvalidData, "socks error")); } } Ok(socket) @@ -110,11 +104,10 @@ pub async fn connect_via_socks_prx( } } - #[cfg(test)] mod tests { use super::*; - use crate::tests::{assert_stream, TcpListener, spawn, block_on, listen_somewhere}; + use crate::tests::{assert_stream, block_on, listen_somewhere, spawn, TcpListener}; #[test] fn socks5h() { @@ -123,21 +116,17 @@ mod tests { assert_stream( &mut stream, - b"\x05\x01\0",//client auth methods: [no auth] + b"\x05\x01\0", //client auth methods: [no auth] ) .await?; - stream.write_all(b"\x05\0").await?;//proxy wants no auth + stream.write_all(b"\x05\0").await?; //proxy wants no auth assert_stream( &mut stream, b"\x05\x01\0\x03\x04host\x12\x34", //version connect reserved dns len host port ) .await?; stream.write_all(b"\x05\0\0\x03\x04host\x12\x34").await?; //version ok reserved dns len host port - assert_stream( - &mut stream, - b"n0ice", - ) - .await?; + assert_stream(&mut stream, b"n0ice").await?; stream.write_all(b"indeed").await?; //version ok reserved dns len host port Ok(true) @@ -146,19 +135,9 @@ mod tests { let (listener, pport, phost) = listen_somewhere().await?; let t = spawn(server(listener)); - let mut stream = connect_via_socks_prx( - "host", - 0x1234, - &phost, - pport, - true, - ).await?; + let mut stream = connect_via_socks_prx("host", 0x1234, &phost, pport, true).await?; stream.write_all(b"n0ice").await?; - assert_stream( - &mut stream, - b"indeed", - ) - .await?; + assert_stream(&mut stream, b"indeed").await?; assert!(t.await?, "not cool"); Ok(()) @@ -172,21 +151,19 @@ mod tests { assert_stream( &mut stream, - b"\x05\x01\0",//client auth methods: [no auth] + b"\x05\x01\0", //client auth methods: [no auth] ) .await?; - stream.write_all(b"\x05\0").await?;//proxy wants no auth + stream.write_all(b"\x05\0").await?; //proxy wants no auth assert_stream( &mut stream, b"\x05\x01\0\x01\x7f\0\0\x01\x12\x34", //version connect reserved dns len host port ) .await?; - stream.write_all(b"\x05\0\0\x01\x7f\0\0\x01\x12\x34").await?; //version ok reserved dns len host port - assert_stream( - &mut stream, - b"n0ice", - ) - .await?; + stream + .write_all(b"\x05\0\0\x01\x7f\0\0\x01\x12\x34") + .await?; //version ok reserved dns len host port + assert_stream(&mut stream, b"n0ice").await?; Ok(true) } @@ -194,13 +171,8 @@ mod tests { let (listener, pport, phost) = listen_somewhere().await?; let t = spawn(server(listener)); - let mut stream = connect_via_socks_prx( - "127.0.0.1", - 0x1234, - &phost, - pport, - true, - ).await?; + let mut stream = + connect_via_socks_prx("127.0.0.1", 0x1234, &phost, pport, true).await?; stream.write_all(b"n0ice").await?; assert!(t.await?, "not cool"); @@ -215,21 +187,19 @@ mod tests { assert_stream( &mut stream, - b"\x05\x01\0",//client auth methods: [no auth] + b"\x05\x01\0", //client auth methods: [no auth] ) .await?; - stream.write_all(b"\x05\0").await?;//proxy wants no auth + stream.write_all(b"\x05\0").await?; //proxy wants no auth assert_stream( &mut stream, b"\x05\x01\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\x12\x34", //version connect reserved dns len host port ) .await?; - stream.write_all(b"\x05\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\x12\x34").await?; //version ok reserved dns len host port - assert_stream( - &mut stream, - b"n0ice", - ) - .await?; + stream + .write_all(b"\x05\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\x12\x34") + .await?; //version ok reserved dns len host port + assert_stream(&mut stream, b"n0ice").await?; Ok(true) } @@ -237,13 +207,7 @@ mod tests { let (listener, pport, phost) = listen_somewhere().await?; let t = spawn(server(listener)); - let mut stream = connect_via_socks_prx( - "::1", - 0x1234, - &phost, - pport, - true, - ).await?; + let mut stream = connect_via_socks_prx("::1", 0x1234, &phost, pport, true).await?; stream.write_all(b"n0ice").await?; assert!(t.await?, "not cool");