netutils/http: follow redirects, http AutoConnector

userspace/lib/hclient/src/connection/auto.rs

@@ -0,0 +1,96 @@
+use std::{io, net::TcpStream};
+
+use crate::{
+    connection::{tls::TlsConnection, TlsConnector},
+    HttpConnection, HttpConnector, TcpConnector,
+};
+
+pub struct AutoConnector {
+    tcp: TcpConnector,
+    tls: TlsConnector,
+}
+
+pub enum AutoConnection {
+    Tls(TlsConnection),
+    Tcp(TcpStream),
+}
+
+impl AutoConnector {
+    pub fn new(tcp: TcpConnector, tls: TlsConnector) -> Self {
+        Self { tcp, tls }
+    }
+
+    pub fn insecure() -> Self {
+        let tls = TlsConnector::insecure();
+        let tcp = TcpConnector;
+        Self { tcp, tls }
+    }
+}
+
+impl HttpConnector for AutoConnector {
+    type Error = io::Error;
+    type Connection = AutoConnection;
+
+    fn connect(
+        &mut self,
+        remote: &std::net::SocketAddr,
+        scheme: &str,
+        server_name: &str,
+        timeout: Option<std::time::Duration>,
+        options: super::HttpConnectionOptions,
+    ) -> Result<Self::Connection, Self::Error> {
+        if scheme == "https" {
+            self.tls
+                .connect(remote, scheme, server_name, timeout, options)
+                .map(AutoConnection::Tls)
+        } else {
+            self.tcp
+                .connect(remote, scheme, server_name, timeout, options)
+                .map(AutoConnection::Tcp)
+        }
+    }
+
+    fn supports_scheme(&self, scheme: &str) -> bool {
+        scheme == "https" || scheme == "http"
+    }
+
+    fn default_port(&self, scheme: &str) -> u16 {
+        if scheme == "https" {
+            443
+        } else {
+            80
+        }
+    }
+}
+
+impl HttpConnection for AutoConnection {
+    type Error = io::Error;
+
+    fn send(&mut self, buffer: &[u8]) -> Result<usize, Self::Error> {
+        match self {
+            Self::Tls(c) => c.send(buffer),
+            Self::Tcp(c) => c.send(buffer),
+        }
+    }
+
+    fn send_all(&mut self, buffer: &[u8]) -> Result<(), Self::Error> {
+        match self {
+            Self::Tls(c) => c.send_all(buffer),
+            Self::Tcp(c) => c.send_all(buffer),
+        }
+    }
+
+    fn recv(&mut self, buffer: &mut [u8]) -> Result<usize, Self::Error> {
+        match self {
+            Self::Tls(c) => c.recv(buffer),
+            Self::Tcp(c) => c.recv(buffer),
+        }
+    }
+
+    fn recv_exact(&mut self, buffer: &mut [u8]) -> Result<(), Self::Error> {
+        match self {
+            Self::Tls(c) => c.recv_exact(buffer),
+            Self::Tcp(c) => c.recv_exact(buffer),
+        }
+    }
+}

userspace/lib/hclient/src/connection/mod.rs

@@ -1,10 +1,16 @@
 use std::{net::SocketAddr, time::Duration};
 
 pub use tcp::TcpConnector;
+
+#[cfg(feature = "https")]
+pub use auto::AutoConnector;
 #[cfg(feature = "https")]
 pub use tls::TlsConnector;
 
 pub mod tcp;
+
+#[cfg(feature = "https")]
+pub mod auto;
 #[cfg(feature = "https")]
 pub mod tls;
 
@@ -21,8 +27,6 @@ pub trait HttpConnector {
     type Connection: HttpConnection<Error = Self::Error>;
     type Error: std::error::Error + Send + 'static;
 
-    const DEFAULT_PORT: u16;
-
     fn connect(
         &mut self,
         remote: &SocketAddr,
@@ -33,6 +37,7 @@ pub trait HttpConnector {
     ) -> Result<Self::Connection, Self::Error>;
 
     fn supports_scheme(&self, scheme: &str) -> bool;
+    fn default_port(&self, scheme: &str) -> u16;
 }
 
 #[derive(Clone)]

userspace/lib/hclient/src/connection/tcp.rs

@@ -12,8 +12,6 @@ impl HttpConnector for TcpConnector {
     type Connection = TcpStream;
     type Error = io::Error;
 
-    const DEFAULT_PORT: u16 = 80;
-
     fn connect(
         &mut self,
         remote: &SocketAddr,
@@ -34,6 +32,10 @@ impl HttpConnector for TcpConnector {
     fn supports_scheme(&self, scheme: &str) -> bool {
         scheme.eq_ignore_ascii_case("http")
     }
+
+    fn default_port(&self, _scheme: &str) -> u16 {
+        80
+    }
 }
 
 impl HttpConnection for TcpStream {

userspace/lib/hclient/src/connection/tls.rs

@@ -29,8 +29,6 @@ impl HttpConnector for TlsConnector {
     type Connection = TlsConnection;
     type Error = io::Error;
 
-    const DEFAULT_PORT: u16 = 443;
-
     fn connect(
         &mut self,
         remote: &SocketAddr,
@@ -59,6 +57,10 @@ impl HttpConnector for TlsConnector {
     fn supports_scheme(&self, scheme: &str) -> bool {
         scheme == "https"
     }
+
+    fn default_port(&self, _scheme: &str) -> u16 {
+        443
+    }
 }
 
 impl HttpConnection for TlsConnection {

userspace/lib/hclient/src/lib.rs

@@ -66,7 +66,9 @@ impl<C: HttpConnector> HttpClient<C> {
         }
 
         let host = url.host().ok_or(HttpError::MalformedUrl)?;
-        let port = url.port_u16().unwrap_or(C::DEFAULT_PORT);
+        let port = url
+            .port_u16()
+            .unwrap_or(self.connector.default_port(scheme));
 
         let request = if let Some(path) = url.path_and_query() {
             request.uri(path.as_str())
@@ -127,6 +129,18 @@ impl Default for HttpClient<TcpConnector> {
 }
 
 impl<C: HttpConnector, U: TryInto<Uri>> HttpRequestBuilder<'_, C, U> {
+    pub fn header<V: TryInto<HeaderValue>>(
+        mut self,
+        name: HeaderName,
+        value: V,
+    ) -> Result<Self, HttpError<C::Error>>
+    where
+        http::Error: From<V::Error>,
+    {
+        self.builder = self.builder.header(name, value);
+        Ok(self)
+    }
+
     pub fn call<R: HttpBody>(self) -> Result<HttpResponse<C::Connection, R>, HttpError<C::Error>> {
         self.client.send(self.url, self.builder, ())
     }

userspace/netutils/src/http.rs

@@ -3,11 +3,17 @@ use std::{
     io::{self, stdout, Stdout, Write},
     path::{Path, PathBuf},
     process::ExitCode,
+    str::FromStr,
 };
 
 use clap::{Parser, Subcommand};
-use hclient::{connection::TlsConnector, HttpClient, HttpError};
-use http::{Method, Uri};
+use hclient::{
+    connection::AutoConnector, HttpBody, HttpClient, HttpConnector, HttpError, HttpResponse,
+};
+use http::{
+    header::{self, ToStrError},
+    Method, Uri,
+};
 
 #[derive(Debug, thiserror::Error)]
 enum Error {
@@ -15,12 +21,22 @@ enum Error {
     IoError(#[from] io::Error),
     #[error("HTTP error: {0}")]
     HttpError(#[from] HttpError<io::Error>),
+    #[error("Malformed response: {0}")]
+    MalformedResponse(&'static str),
+    #[error("Too many redirects")]
+    TooManyRedirects,
+    #[error("Invalid header '{0}' value: {1}")]
+    InvalidHeaderValue(header::HeaderName, ToStrError),
+    #[error("Invalid URL ('{0}'): {1}")]
+    InvalidUrl(String, http::uri::InvalidUri),
 }
 
 #[derive(Debug, Parser)]
 struct Arguments {
     #[clap(short, long)]
     output: Option<PathBuf>,
+    #[clap(short, long)]
+    follow: bool,
     #[clap(subcommand)]
     method: RequestMethod,
 }
@@ -64,51 +80,71 @@ impl Write for Output {
     }
 }
 
-fn do_request_tls(url: Uri, mut output: Output) -> Result<(), Error> {
-    let connector = TlsConnector::insecure();
-    let mut client = HttpClient::new_default(connector);
+fn request_call<C: HttpConnector, B: HttpBody>(
+    client: &mut HttpClient<C>,
+    method: Method,
+    mut url: Uri,
+) -> Result<HttpResponse<C::Connection, B>, Error>
+where
+    Error: From<HttpError<C::Error>>,
+{
+    let mut redirect_count = 0;
+    let max_redirects = 5;
+    while redirect_count < max_redirects {
+        log::info!("Try URL: {url:?}");
+        let response = client
+            .request(method.clone(), url)
+            .header(header::CONNECTION, "close")?
+            .call()?;
 
-    let mut buffer = [0; 4096];
-    let mut response = client.request(Method::GET, url).call().unwrap();
-    loop {
-        let len = response.read(&mut buffer)?;
-        if len == 0 {
-            break;
+        if response.status().is_redirection() {
+            let location =
+                response
+                    .headers()
+                    .get(header::LOCATION)
+                    .ok_or(Error::MalformedResponse(
+                        "No \"Location\" header in a redirect response",
+                    ))?;
+
+            let redirect_url = location
+                .to_str()
+                .map_err(|e| Error::InvalidHeaderValue(header::LOCATION, e))?;
+            let redirect_url = Uri::from_str(redirect_url)
+                .map_err(|e| Error::InvalidUrl(redirect_url.into(), e))?;
+
+            log::info!("Redirect to {redirect_url:?}");
+
+            url = redirect_url;
+
+            redirect_count += 1;
+        } else {
+            return Ok(response);
         }
-        output.write_all(&buffer[..len])?;
     }
 
-    Ok(())
-}
-
-fn do_request_tcp(url: Uri, mut output: Output) -> Result<(), Error> {
-    let mut client = HttpClient::default();
-
-    let mut buffer = [0; 4096];
-    let mut response = client.request(Method::GET, url).call().unwrap();
-    loop {
-        let len = response.read(&mut buffer)?;
-        if len == 0 {
-            break;
-        }
-        output.write_all(&buffer[..len])?;
-    }
-
-    Ok(())
+    Err(Error::TooManyRedirects)
 }
 
 fn get(url: Uri, output: Option<PathBuf>) -> Result<(), Error> {
-    let output = Output::open(output)?;
-    let use_https = url.scheme_str().map_or(false, |scheme| scheme == "https");
+    let mut output = Output::open(output)?;
+    let connector = AutoConnector::insecure();
+    let mut client = HttpClient::new_default(connector);
+    let mut buffer = [0; 4096];
 
-    if use_https {
-        do_request_tls(url, output)
-    } else {
-        do_request_tcp(url, output)
+    let mut response = request_call(&mut client, Method::GET, url)?;
+
+    loop {
+        let len = response.read(&mut buffer)?;
+        if len == 0 {
+            break;
+        }
+        output.write_all(&buffer[..len])?;
     }
+    Ok(())
 }
 
 fn main() -> ExitCode {
+    logsink::setup_logging(false);
     let args = Arguments::parse();
 
     let result = match args.method {