fix: refactor config

apps/recorder/src/fetch/bytes.rs

@@ -1,24 +1,11 @@
 use bytes::Bytes;
 use reqwest::IntoUrl;
 
-use super::{core::DEFAULT_HTTP_CLIENT_USER_AGENT, HttpClient};
+use super::HttpClient;
 
-pub async fn download_bytes<T: IntoUrl>(url: T) -> eyre::Result<Bytes> {
-    let request_client = reqwest::Client::builder()
-        .user_agent(DEFAULT_HTTP_CLIENT_USER_AGENT)
-        .build()?;
-    let bytes = request_client.get(url).send().await?.bytes().await?;
+pub async fn fetch_bytes<T: IntoUrl>(client: Option<&HttpClient>, url: T) -> eyre::Result<Bytes> {
+    let client = client.unwrap_or_default();
+
+    let bytes = client.get(url).send().await?.bytes().await?;
     Ok(bytes)
 }
-
-pub async fn download_bytes_with_client<T: IntoUrl>(
-    client: Option<&HttpClient>,
-    url: T,
-) -> eyre::Result<Bytes> {
-    if let Some(client) = client {
-        let bytes = client.get(url).send().await?.bytes().await?;
-        Ok(bytes)
-    } else {
-        download_bytes(url).await
-    }
-}

apps/recorder/src/fetch/client.rs

@@ -2,6 +2,7 @@ use std::{ops::Deref, time::Duration};
 
 use axum::http::Extensions;
 use leaky_bucket::RateLimiter;
+use once_cell::sync::OnceCell;
 use reqwest::{ClientBuilder, Request, Response};
 use reqwest_middleware::{
     ClientBuilder as ClientWithMiddlewareBuilder, ClientWithMiddleware, Next,
@@ -11,7 +12,7 @@ use reqwest_tracing::TracingMiddleware;
 use serde::{Deserialize, Serialize};
 use serde_with::serde_as;
 
-use super::DEFAULT_HTTP_CLIENT_USER_AGENT;
+use crate::fetch::DEFAULT_HTTP_CLIENT_USER_AGENT;
 
 #[serde_as]
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
@@ -27,6 +28,7 @@ pub struct HttpClientConfig {
 
 pub struct HttpClient {
     client: ClientWithMiddleware,
+    pub config: HttpClientConfig,
 }
 
 impl Deref for HttpClient {
@@ -55,42 +57,73 @@ impl reqwest_middleware::Middleware for RateLimiterMiddleware {
 }
 
 impl HttpClient {
-    pub fn new(config: Option<HttpClientConfig>) -> reqwest::Result<Self> {
-        let mut config = config.unwrap_or_default();
-        let retry_policy = ExponentialBackoff::builder()
-            .build_with_max_retries(config.exponential_backoff_max_retries.take().unwrap_or(3));
-        let rate_limiter = RateLimiter::builder()
-            .max(config.leaky_bucket_max_tokens.take().unwrap_or(3) as usize)
-            .initial(
-                config
-                    .leaky_bucket_initial_tokens
-                    .take()
-                    .unwrap_or_default() as usize,
-            )
-            .refill(config.leaky_bucket_refill_tokens.take().unwrap_or(1) as usize)
-            .interval(
-                config
-                    .leaky_bucket_refill_interval
-                    .take()
-                    .unwrap_or_else(|| Duration::from_millis(500)),
-            )
-            .build();
+    pub fn from_config(config: HttpClientConfig) -> reqwest::Result<Self> {
+        let reqwest_client_builder = ClientBuilder::new().user_agent(
+            config
+                .user_agent
+                .as_deref()
+                .unwrap_or(DEFAULT_HTTP_CLIENT_USER_AGENT),
+        );
 
-        let client = ClientBuilder::new()
-            .user_agent(
-                config
-                    .user_agent
-                    .take()
-                    .unwrap_or_else(|| DEFAULT_HTTP_CLIENT_USER_AGENT.to_owned()),
-            )
-            .build()?;
+        let reqwest_client = reqwest_client_builder.build()?;
+
+        let mut reqwest_with_middleware_builder =
+            ClientWithMiddlewareBuilder::new(reqwest_client).with(TracingMiddleware::default());
+
+        if let Some(ref x) = config.exponential_backoff_max_retries {
+            let retry_policy = ExponentialBackoff::builder().build_with_max_retries(*x);
+
+            reqwest_with_middleware_builder = reqwest_with_middleware_builder
+                .with(RetryTransientMiddleware::new_with_policy(retry_policy));
+        }
+
+        if let (None, None, None, None) = (
+            config.leaky_bucket_initial_tokens.as_ref(),
+            config.leaky_bucket_refill_tokens.as_ref(),
+            config.leaky_bucket_refill_interval.as_ref(),
+            config.leaky_bucket_max_tokens.as_ref(),
+        ) {
+        } else {
+            let mut rate_limiter_builder = RateLimiter::builder();
+
+            if let Some(ref x) = config.leaky_bucket_max_tokens {
+                rate_limiter_builder.max(*x as usize);
+            }
+            if let Some(ref x) = config.leaky_bucket_initial_tokens {
+                rate_limiter_builder.initial(*x as usize);
+            }
+            if let Some(ref x) = config.leaky_bucket_refill_tokens {
+                rate_limiter_builder.refill(*x as usize);
+            }
+            if let Some(ref x) = config.leaky_bucket_refill_interval {
+                rate_limiter_builder.interval(*x);
+            }
+
+            let rate_limiter = rate_limiter_builder.build();
+
+            reqwest_with_middleware_builder =
+                reqwest_with_middleware_builder.with(RateLimiterMiddleware { rate_limiter });
+        }
+
+        let reqwest_with_middleware = reqwest_with_middleware_builder.build();
 
         Ok(Self {
-            client: ClientWithMiddlewareBuilder::new(client)
-                .with(TracingMiddleware::default())
-                .with(RateLimiterMiddleware { rate_limiter })
-                .with(RetryTransientMiddleware::new_with_policy(retry_policy))
-                .build(),
+            client: reqwest_with_middleware,
+            config,
         })
     }
 }
+
+static DEFAULT_HTTP_CLIENT: OnceCell<HttpClient> = OnceCell::new();
+
+impl Default for HttpClient {
+    fn default() -> Self {
+        HttpClient::from_config(Default::default()).expect("Failed to create default HttpClient")
+    }
+}
+
+impl Default for &HttpClient {
+    fn default() -> Self {
+        DEFAULT_HTTP_CLIENT.get_or_init(HttpClient::default)
+    }
+}

apps/recorder/src/fetch/html.rs

@@ -1,23 +1,10 @@
 use reqwest::IntoUrl;
 
-use super::{core::DEFAULT_HTTP_CLIENT_USER_AGENT, HttpClient};
+use super::HttpClient;
+
+pub async fn fetch_html<T: IntoUrl>(client: Option<&HttpClient>, url: T) -> eyre::Result<String> {
+    let client = client.unwrap_or_default();
+    let content = client.get(url).send().await?.text().await?;
 
-pub async fn download_html<U: IntoUrl>(url: U) -> eyre::Result<String> {
-    let request_client = reqwest::Client::builder()
-        .user_agent(DEFAULT_HTTP_CLIENT_USER_AGENT)
-        .build()?;
-    let content = request_client.get(url).send().await?.text().await?;
     Ok(content)
 }
-
-pub async fn download_html_with_client<T: IntoUrl>(
-    client: Option<&HttpClient>,
-    url: T,
-) -> eyre::Result<String> {
-    if let Some(client) = client {
-        let content = client.get(url).send().await?.text().await?;
-        Ok(content)
-    } else {
-        download_html(url).await
-    }
-}

apps/recorder/src/fetch/image.rs

@@ -1,18 +1,8 @@
 use bytes::Bytes;
 use reqwest::IntoUrl;
 
-use super::{
-    bytes::{download_bytes, download_bytes_with_client},
-    HttpClient,
-};
+use super::{bytes::fetch_bytes, HttpClient};
 
-pub async fn download_image<U: IntoUrl>(url: U) -> eyre::Result<Bytes> {
-    download_bytes(url).await
-}
-
-pub async fn download_image_with_client<T: IntoUrl>(
-    client: Option<&HttpClient>,
-    url: T,
-) -> eyre::Result<Bytes> {
-    download_bytes_with_client(client, url).await
+pub async fn fetch_image<T: IntoUrl>(client: Option<&HttpClient>, url: T) -> eyre::Result<Bytes> {
+    fetch_bytes(client, url).await
 }

apps/recorder/src/fetch/mod.rs

@@ -6,6 +6,7 @@ pub mod image;
 
 pub use core::DEFAULT_HTTP_CLIENT_USER_AGENT;
 
-pub use bytes::download_bytes;
+pub use bytes::fetch_bytes;
 pub use client::{HttpClient, HttpClientConfig};
-pub use image::download_image;
+pub use html::fetch_html;
+pub use image::fetch_image;