dumtruck/konobangu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add rss feeds and episode enclosure

Browse Source
This commit is contained in:
master
2025-06-22 01:04:23 +08:00
parent 16429a44b4
commit f055011b86
44 changed files with 1466 additions and 480 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
apps/recorder/src/extract/bittorrent/extract.rs
View File

@@ -1,3 +1,4 @@
use chrono::{DateTime, Utc};
use fancy_regex::Regex as FancyRegex;
use lazy_static::lazy_static;
use quirks_path::Path;
@@ -33,6 +34,14 @@ lazy_static! {
Regex::new(r"([Ss]|Season )(\d{1,3})").unwrap();
}
#[derive(Clone, Debug)]
pub struct EpisodeEnclosureMeta {
pub magnet_link: Option<String>,
pub torrent_link: Option<String>,
pub pub_date: Option<DateTime<Utc>>,
pub content_length: Option<u64>,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
pub struct TorrentEpisodeMediaMeta {
pub fansub: Option<String>,

25
apps/recorder/src/extract/http.rs
View File

@@ -1,7 +1,12 @@
use axum::http::{HeaderName, HeaderValue, Uri, header, request::Parts};
use axum::{
extract::FromRequestParts,
http::{HeaderName, HeaderValue, Uri, header, request::Parts},
};
use itertools::Itertools;
use url::Url;
use crate::errors::RecorderError;
/// Fields from a "Forwarded" header per [RFC7239 sec 4](https://www.rfc-editor.org/rfc/rfc7239#section-4)
#[derive(Debug, Clone)]
pub struct ForwardedHeader {
@@ -101,9 +106,13 @@ pub struct ForwardedRelatedInfo {
pub origin: Option<String>,
}
impl ForwardedRelatedInfo {
pub fn from_request_parts(request_parts: &Parts) -> ForwardedRelatedInfo {
let headers = &request_parts.headers;
impl<T> FromRequestParts<T> for ForwardedRelatedInfo {
type Rejection = RecorderError;
fn from_request_parts(
parts: &mut Parts,
_state: &T,
) -> impl Future<Output = Result<Self, Self::Rejection>> + Send {
let headers = &parts.headers;
let forwarded = headers
.get(header::FORWARDED)
.and_then(|s| ForwardedHeader::try_from(s.clone()).ok());
@@ -132,17 +141,19 @@ impl ForwardedRelatedInfo {
.get(header::ORIGIN)
.and_then(|s| s.to_str().map(String::from).ok());
ForwardedRelatedInfo {
futures::future::ready(Ok(ForwardedRelatedInfo {
host,
x_forwarded_for,
x_forwarded_host,
x_forwarded_proto,
forwarded,
uri: request_parts.uri.clone(),
uri: parts.uri.clone(),
origin,
}
}))
}
}
impl ForwardedRelatedInfo {
pub fn resolved_protocol(&self) -> Option<&str> {
self.forwarded
.as_ref()

39
apps/recorder/src/extract/mikan/subscription.rs
View File

@@ -20,12 +20,15 @@ use super::scrape_mikan_bangumi_meta_stream_from_season_flow_url;
use crate::{
app::AppContextTrait,
errors::{RecorderError, RecorderResult},
extract::mikan::{
MikanBangumiHash, MikanBangumiMeta, MikanEpisodeHash, MikanEpisodeMeta,
MikanRssEpisodeItem, MikanSeasonFlowUrlMeta, MikanSeasonStr,
MikanSubscriberSubscriptionRssUrlMeta, build_mikan_bangumi_subscription_rss_url,
build_mikan_season_flow_url, build_mikan_subscriber_subscription_rss_url,
scrape_mikan_episode_meta_from_episode_homepage_url,
extract::{
bittorrent::EpisodeEnclosureMeta,
mikan::{
MikanBangumiHash, MikanBangumiMeta, MikanEpisodeHash, MikanEpisodeMeta,
MikanRssEpisodeItem, MikanSeasonFlowUrlMeta, MikanSeasonStr,
MikanSubscriberSubscriptionRssUrlMeta, build_mikan_bangumi_subscription_rss_url,
build_mikan_season_flow_url, build_mikan_subscriber_subscription_rss_url,
scrape_mikan_episode_meta_from_episode_homepage_url,
},
},
models::{
bangumi, episodes, subscription_bangumi, subscription_episode,
@@ -54,7 +57,7 @@ async fn sync_mikan_feeds_from_rss_item_list(
.map(|(episode_id, hash, bangumi_id)| (hash.mikan_episode_id, (episode_id, bangumi_id)))
.collect::<HashMap<_, _>>();
let mut new_episode_meta_list: Vec<MikanEpisodeMeta> = vec![];
let mut new_episode_meta_list: Vec<(MikanEpisodeMeta, EpisodeEnclosureMeta)> = vec![];
let mikan_client = ctx.mikan();
for to_insert_rss_item in rss_item_list.into_iter().filter(|rss_item| {
@@ -65,7 +68,8 @@ async fn sync_mikan_feeds_from_rss_item_list(
to_insert_rss_item.build_homepage_url(mikan_base_url.clone()),
)
.await?;
new_episode_meta_list.push(episode_meta);
let episode_enclosure_meta = EpisodeEnclosureMeta::from(to_insert_rss_item);
new_episode_meta_list.push((episode_meta, episode_enclosure_meta));
}
(new_episode_meta_list, existed_episode_hash2id_map)
@@ -92,22 +96,22 @@ async fn sync_mikan_feeds_from_rss_item_list(
let new_episode_meta_list_group_by_bangumi_hash: HashMap<
MikanBangumiHash,
Vec<MikanEpisodeMeta>,
Vec<(MikanEpisodeMeta, EpisodeEnclosureMeta)>,
> = {
let mut m = hashmap! {};
for episode_meta in new_episode_meta_list {
for (episode_meta, episode_enclosure_meta) in new_episode_meta_list {
let bangumi_hash = episode_meta.bangumi_hash();
m.entry(bangumi_hash)
.or_insert_with(Vec::new)
.push(episode_meta);
.push((episode_meta, episode_enclosure_meta));
}
m
};
for (group_bangumi_hash, group_episode_meta_list) in new_episode_meta_list_group_by_bangumi_hash
{
let first_episode_meta = group_episode_meta_list.first().unwrap();
let (first_episode_meta, _) = group_episode_meta_list.first().unwrap();
let group_bangumi_model = bangumi::Model::get_or_insert_from_mikan(
ctx,
group_bangumi_hash,
@@ -126,9 +130,12 @@ async fn sync_mikan_feeds_from_rss_item_list(
},
)
.await?;
let group_episode_creation_list = group_episode_meta_list
.into_iter()
.map(|episode_meta| (&group_bangumi_model, episode_meta));
let group_episode_creation_list =
group_episode_meta_list
.into_iter()
.map(|(episode_meta, episode_enclosure_meta)| {
(&group_bangumi_model, episode_meta, episode_enclosure_meta)
});
episodes::Model::add_mikan_episodes_for_subscription(
ctx,
@@ -273,7 +280,7 @@ impl MikanSubscriberSubscription {
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, InputObject, SimpleObject)]
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct MikanSeasonSubscription {
pub subscription_id: i32,
pub year: i32,

30
apps/recorder/src/extract/mikan/web.rs
View File

@@ -2,7 +2,7 @@ use std::{borrow::Cow, fmt, str::FromStr, sync::Arc};
use async_stream::try_stream;
use bytes::Bytes;
use chrono::DateTime;
use chrono::{DateTime, Utc};
use downloader::bittorrent::defs::BITTORRENT_MIME_TYPE;
use fetch::{html::fetch_html, image::fetch_image};
use futures::{Stream, TryStreamExt, pin_mut};
@@ -17,6 +17,7 @@ use crate::{
app::AppContextTrait,
errors::app_error::{RecorderError, RecorderResult},
extract::{
bittorrent::EpisodeEnclosureMeta,
html::{extract_background_image_src_from_style_attr, extract_inner_text_from_element_ref},
media::extract_image_src_from_str,
mikan::{
@@ -39,11 +40,12 @@ use crate::{
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct MikanRssEpisodeItem {
pub title: String,
pub url: Url,
pub torrent_link: Url,
pub content_length: Option<u64>,
pub mime: String,
pub pub_date: Option<i64>,
pub pub_date: Option<DateTime<Utc>>,
pub mikan_episode_id: String,
pub magnet_link: Option<String>,
}
impl MikanRssEpisodeItem {
@@ -95,18 +97,30 @@ impl TryFrom<rss::Item> for MikanRssEpisodeItem {
Ok(MikanRssEpisodeItem {
title,
url: enclosure_url,
torrent_link: enclosure_url,
content_length: enclosure.length.parse().ok(),
mime: mime_type,
pub_date: item
.pub_date
.and_then(|s| DateTime::parse_from_rfc2822(&s).ok())
.map(|s| s.timestamp_millis()),
pub_date: item.pub_date.and_then(|s| {
DateTime::parse_from_rfc2822(&s)
.ok()
.map(|s| s.with_timezone(&Utc))
}),
mikan_episode_id,
magnet_link: None,
})
}
}
impl From<MikanRssEpisodeItem> for EpisodeEnclosureMeta {
fn from(item: MikanRssEpisodeItem) -> Self {
Self {
magnet_link: item.magnet_link,
torrent_link: Some(item.torrent_link.to_string()),
pub_date: item.pub_date,
content_length: item.content_length,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct MikanSubscriberSubscriptionRssUrlMeta {
pub mikan_subscription_token: String,