fix: add sync subscription webui and check credential web ui

packages/fetch/Cargo.toml

@@ -16,6 +16,7 @@ axum-extra = { workspace = true }
 async-trait = { workspace = true }
 moka = { workspace = true }
 reqwest = { workspace = true }
+tracing = { workspace = true }
 leaky-bucket = "1.1"
 http-cache-reqwest = { version = "0.15", features = [
     "manager-cacache",
@@ -32,5 +33,6 @@ http-cache = { version = "0.20", features = [
     "manager-moka",
 ], default-features = false }
 reqwest_cookie_store = { version = "0.8.0", features = ["serde"] }
+http-serde = "2.1.1"
 
 util = { workspace = true }

packages/fetch/src/client/core.rs

@@ -1,7 +1,7 @@
 use std::{fmt::Debug, ops::Deref, sync::Arc, time::Duration};
 
 use async_trait::async_trait;
-use axum::http::{self, Extensions};
+use axum::http::Extensions;
 use http_cache_reqwest::{
     Cache, CacheManager, CacheMode, HttpCache, HttpCacheOptions, MokaManager,
 };
@@ -15,10 +15,9 @@ use reqwest_retry::{RetryTransientMiddleware, policies::ExponentialBackoff};
 use reqwest_tracing::TracingMiddleware;
 use serde::{Deserialize, Serialize};
 use serde_with::serde_as;
-use snafu::Snafu;
 use util::OptDynErr;
 
-use crate::get_random_ua;
+use crate::{HttpClientError, client::proxy::HttpClientProxyConfig, get_random_ua};
 
 pub struct RateLimiterMiddleware {
     rate_limiter: RateLimiter,
@@ -56,7 +55,7 @@ impl Default for HttpClientCachePresetConfig {
 }
 
 #[serde_as]
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
 pub struct HttpClientConfig {
     pub exponential_backoff_max_retries: Option<u32>,
     pub leaky_bucket_max_tokens: Option<u32>,
@@ -67,6 +66,7 @@ pub struct HttpClientConfig {
     pub user_agent: Option<String>,
     pub cache_backend: Option<HttpClientCacheBackendConfig>,
     pub cache_preset: Option<HttpClientCachePresetConfig>,
+    pub proxy: Option<HttpClientProxyConfig>,
 }
 
 pub(crate) struct CacheBackend(Box<dyn CacheManager>);
@@ -102,20 +102,6 @@ impl CacheManager for CacheBackend {
     }
 }
 
-#[derive(Debug, Snafu)]
-pub enum HttpClientError {
-    #[snafu(transparent)]
-    ReqwestError { source: reqwest::Error },
-    #[snafu(transparent)]
-    ReqwestMiddlewareError { source: reqwest_middleware::Error },
-    #[snafu(transparent)]
-    HttpError { source: http::Error },
-    #[snafu(display("Failed to parse cookies: {}", source))]
-    ParseCookiesError { source: serde_json::Error },
-    #[snafu(display("Failed to save cookies, message: {}, source: {:?}", message, source))]
-    SaveCookiesError { message: String, source: OptDynErr },
-}
-
 pub trait HttpClientTrait: Deref<Target = ClientWithMiddleware> + Debug {}
 
 pub struct HttpClientFork {
@@ -179,13 +165,29 @@ impl Deref for HttpClient {
 impl HttpClient {
     pub fn from_config(config: HttpClientConfig) -> Result<Self, HttpClientError> {
         let mut middleware_stack: Vec<Arc<dyn Middleware>> = vec![];
-        let reqwest_client_builder = ClientBuilder::new().user_agent(
+        let mut reqwest_client_builder = ClientBuilder::new().user_agent(
             config
                 .user_agent
                 .as_deref()
                 .unwrap_or_else(|| get_random_ua()),
         );
 
+        if let Some(proxy) = config.proxy.as_ref() {
+            let accept_invalid_certs = proxy
+                .accept_invalid_certs
+                .as_ref()
+                .map(|b| b.as_bool())
+                .unwrap_or_default();
+            let proxy = proxy.clone().into_proxy()?;
+            if let Some(proxy) = proxy {
+                reqwest_client_builder = reqwest_client_builder.proxy(proxy);
+                if accept_invalid_certs {
+                    reqwest_client_builder =
+                        reqwest_client_builder.danger_accept_invalid_certs(true);
+                }
+            }
+        }
+
         #[cfg(not(target_arch = "wasm32"))]
         let reqwest_client_builder =
             reqwest_client_builder.redirect(reqwest::redirect::Policy::none());
@@ -294,13 +296,29 @@ impl HttpClient {
     }
 
     pub fn fork(&self) -> HttpClientFork {
-        let reqwest_client_builder = ClientBuilder::new().user_agent(
+        let mut reqwest_client_builder = ClientBuilder::new().user_agent(
             self.config
                 .user_agent
                 .as_deref()
                 .unwrap_or_else(|| get_random_ua()),
         );
 
+        if let Some(proxy) = self.config.proxy.as_ref() {
+            let accept_invalid_certs = proxy
+                .accept_invalid_certs
+                .as_ref()
+                .map(|b| b.as_bool())
+                .unwrap_or_default();
+            let proxy = proxy.clone().into_proxy().unwrap_or_default();
+            if let Some(proxy) = proxy {
+                reqwest_client_builder = reqwest_client_builder.proxy(proxy);
+                if accept_invalid_certs {
+                    reqwest_client_builder =
+                        reqwest_client_builder.danger_accept_invalid_certs(true);
+                }
+            }
+        }
+
         #[cfg(not(target_arch = "wasm32"))]
         let reqwest_client_builder =
             reqwest_client_builder.redirect(reqwest::redirect::Policy::none());

packages/fetch/src/client/error.rs

@@ -0,0 +1,21 @@
+use axum::http;
+use snafu::Snafu;
+use util::OptDynErr;
+
+#[derive(Debug, Snafu)]
+pub enum HttpClientError {
+    #[snafu(transparent)]
+    ReqwestError { source: reqwest::Error },
+    #[snafu(transparent)]
+    ReqwestMiddlewareError { source: reqwest_middleware::Error },
+    #[snafu(transparent)]
+    HttpError { source: http::Error },
+    #[snafu(display("Failed to parse cookies: {}", source))]
+    ParseCookiesError { source: serde_json::Error },
+    #[snafu(display("Failed to save cookies, message: {}, source: {:?}", message, source))]
+    SaveCookiesError { message: String, source: OptDynErr },
+    #[snafu(display("Failed to parse fetch client proxy: {source}"))]
+    ProxyParseError { source: reqwest::Error },
+    #[snafu(display("Failed to parse fetch client proxy auth header"))]
+    ProxyAuthHeaderParseError,
+}

packages/fetch/src/client/mod.rs

@@ -1,6 +1,11 @@
-pub mod core;
+mod core;
+mod error;
+mod proxy;
 
 pub use core::{
     HttpClient, HttpClientCacheBackendConfig, HttpClientCachePresetConfig, HttpClientConfig,
-    HttpClientError, HttpClientTrait,
+    HttpClientTrait,
 };
+
+pub use error::HttpClientError;
+pub use proxy::HttpClientProxyConfig;

packages/fetch/src/client/proxy.rs

@@ -0,0 +1,56 @@
+use axum::http::{HeaderMap, HeaderValue};
+use reqwest::{NoProxy, Proxy};
+use serde::{Deserialize, Serialize};
+use serde_with::{NoneAsEmptyString, serde_as};
+use util::BooleanLike;
+
+use crate::HttpClientError;
+
+#[serde_as]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct HttpClientProxyConfig {
+    #[serde_as(as = "NoneAsEmptyString")]
+    pub server: Option<String>,
+    #[serde_as(as = "NoneAsEmptyString")]
+    pub auth_header: Option<String>,
+    #[serde(with = "http_serde::option::header_map")]
+    pub headers: Option<HeaderMap>,
+    #[serde_as(as = "NoneAsEmptyString")]
+    pub no_proxy: Option<String>,
+    pub accept_invalid_certs: Option<BooleanLike>,
+}
+
+impl HttpClientProxyConfig {
+    pub fn into_proxy(self) -> Result<Option<Proxy>, HttpClientError> {
+        if let Some(server) = self.server {
+            let mut proxy = Proxy::all(server)
+                .map_err(|err| HttpClientError::ProxyParseError { source: err })?;
+
+            if let Some(auth_header) = self.auth_header {
+                let auth_header = HeaderValue::from_str(&auth_header)
+                    .map_err(|_| HttpClientError::ProxyAuthHeaderParseError)?;
+                proxy = proxy.custom_http_auth(auth_header);
+            }
+
+            if let Some(headers) = self.headers {
+                proxy = proxy.headers(headers);
+            }
+
+            if let Some(no_proxy) = self.no_proxy {
+                proxy = proxy.no_proxy(NoProxy::from_string(&no_proxy));
+            }
+
+            Ok(Some(proxy))
+        } else {
+            Ok(None)
+        }
+    }
+}
+
+impl TryFrom<HttpClientProxyConfig> for Option<Proxy> {
+    type Error = HttpClientError;
+
+    fn try_from(value: HttpClientProxyConfig) -> Result<Option<Proxy>, Self::Error> {
+        value.into_proxy()
+    }
+}