feat: add basic webui

apps/recorder/src/fetch/bytes.rs

@@ -0,0 +1,24 @@
+use bytes::Bytes;
+use reqwest::IntoUrl;
+
+use super::{core::DEFAULT_HTTP_CLIENT_USER_AGENT, HttpClient};
+
+pub async fn download_bytes<T: IntoUrl>(url: T) -> eyre::Result<Bytes> {
+    let request_client = reqwest::Client::builder()
+        .user_agent(DEFAULT_HTTP_CLIENT_USER_AGENT)
+        .build()?;
+    let bytes = request_client.get(url).send().await?.bytes().await?;
+    Ok(bytes)
+}
+
+pub async fn download_bytes_with_client<T: IntoUrl>(
+    client: Option<&HttpClient>,
+    url: T,
+) -> eyre::Result<Bytes> {
+    if let Some(client) = client {
+        let bytes = client.get(url).send().await?.bytes().await?;
+        Ok(bytes)
+    } else {
+        download_bytes(url).await
+    }
+}

apps/recorder/src/fetch/client.rs

@@ -0,0 +1,96 @@
+use std::{ops::Deref, time::Duration};
+
+use axum::http::Extensions;
+use leaky_bucket::RateLimiter;
+use reqwest::{ClientBuilder, Request, Response};
+use reqwest_middleware::{
+    ClientBuilder as ClientWithMiddlewareBuilder, ClientWithMiddleware, Next,
+};
+use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware};
+use reqwest_tracing::TracingMiddleware;
+use serde::{Deserialize, Serialize};
+use serde_with::serde_as;
+
+use super::DEFAULT_HTTP_CLIENT_USER_AGENT;
+
+#[serde_as]
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
+pub struct HttpClientConfig {
+    pub exponential_backoff_max_retries: Option<u32>,
+    pub leaky_bucket_max_tokens: Option<u32>,
+    pub leaky_bucket_initial_tokens: Option<u32>,
+    pub leaky_bucket_refill_tokens: Option<u32>,
+    #[serde_as(as = "Option<serde_with::DurationMilliSeconds>")]
+    pub leaky_bucket_refill_interval: Option<Duration>,
+    pub user_agent: Option<String>,
+}
+
+pub struct HttpClient {
+    client: ClientWithMiddleware,
+}
+
+impl Deref for HttpClient {
+    type Target = ClientWithMiddleware;
+
+    fn deref(&self) -> &Self::Target {
+        &self.client
+    }
+}
+
+pub struct RateLimiterMiddleware {
+    rate_limiter: RateLimiter,
+}
+
+#[async_trait::async_trait]
+impl reqwest_middleware::Middleware for RateLimiterMiddleware {
+    async fn handle(
+        &self,
+        req: Request,
+        extensions: &'_ mut Extensions,
+        next: Next<'_>,
+    ) -> reqwest_middleware::Result<Response> {
+        self.rate_limiter.acquire_one().await;
+        next.run(req, extensions).await
+    }
+}
+
+impl HttpClient {
+    pub fn new(config: Option<HttpClientConfig>) -> reqwest::Result<Self> {
+        let mut config = config.unwrap_or_default();
+        let retry_policy = ExponentialBackoff::builder()
+            .build_with_max_retries(config.exponential_backoff_max_retries.take().unwrap_or(3));
+        let rate_limiter = RateLimiter::builder()
+            .max(config.leaky_bucket_max_tokens.take().unwrap_or(3) as usize)
+            .initial(
+                config
+                    .leaky_bucket_initial_tokens
+                    .take()
+                    .unwrap_or_default() as usize,
+            )
+            .refill(config.leaky_bucket_refill_tokens.take().unwrap_or(1) as usize)
+            .interval(
+                config
+                    .leaky_bucket_refill_interval
+                    .take()
+                    .unwrap_or_else(|| Duration::from_millis(500)),
+            )
+            .build();
+
+        let client = ClientBuilder::new()
+            .user_agent(
+                config
+                    .user_agent
+                    .take()
+                    .unwrap_or_else(|| DEFAULT_HTTP_CLIENT_USER_AGENT.to_owned()),
+            )
+            .build()?;
+
+        Ok(Self {
+            client: ClientWithMiddlewareBuilder::new(client)
+                .with(TracingMiddleware::default())
+                .with(RateLimiterMiddleware { rate_limiter })
+                .with(RetryTransientMiddleware::new_with_policy(retry_policy))
+                .build(),
+        })
+    }
+}

apps/recorder/src/fetch/core.rs

@@ -0,0 +1 @@
+pub const DEFAULT_HTTP_CLIENT_USER_AGENT: &str = "Wget/1.13.4 (linux-gnu)";

apps/recorder/src/fetch/html.rs

@@ -0,0 +1,23 @@
+use reqwest::IntoUrl;
+
+use super::{core::DEFAULT_HTTP_CLIENT_USER_AGENT, HttpClient};
+
+pub async fn download_html<U: IntoUrl>(url: U) -> eyre::Result<String> {
+    let request_client = reqwest::Client::builder()
+        .user_agent(DEFAULT_HTTP_CLIENT_USER_AGENT)
+        .build()?;
+    let content = request_client.get(url).send().await?.text().await?;
+    Ok(content)
+}
+
+pub async fn download_html_with_client<T: IntoUrl>(
+    client: Option<&HttpClient>,
+    url: T,
+) -> eyre::Result<String> {
+    if let Some(client) = client {
+        let content = client.get(url).send().await?.text().await?;
+        Ok(content)
+    } else {
+        download_html(url).await
+    }
+}

apps/recorder/src/fetch/image.rs

@@ -0,0 +1,18 @@
+use bytes::Bytes;
+use reqwest::IntoUrl;
+
+use super::{
+    bytes::{download_bytes, download_bytes_with_client},
+    HttpClient,
+};
+
+pub async fn download_image<U: IntoUrl>(url: U) -> eyre::Result<Bytes> {
+    download_bytes(url).await
+}
+
+pub async fn download_image_with_client<T: IntoUrl>(
+    client: Option<&HttpClient>,
+    url: T,
+) -> eyre::Result<Bytes> {
+    download_bytes_with_client(client, url).await
+}

apps/recorder/src/fetch/mod.rs

@@ -0,0 +1,11 @@
+pub mod bytes;
+pub mod client;
+pub mod core;
+pub mod html;
+pub mod image;
+
+pub use core::DEFAULT_HTTP_CLIENT_USER_AGENT;
+
+pub use bytes::download_bytes;
+pub use client::{HttpClient, HttpClientConfig};
+pub use image::download_image;