feat: add testing-torrents

.github/workflows/testing-torrents-container.yaml

@@ -0,0 +1,28 @@
+name: Build and Push Testing Torrents Container
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build-container:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Log in to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and push Docker image
+              uses: docker/build-push-action@v5
+              with:
+                context: 'packages/testing-torrents'
+                file: './Dockerfile'
+                push: true
+                tags: 'ghcr.io/${{ env.ORG }}/konobangu-testing-torrents:latest'
+                cache-from: type=gha
+                cache-to: type=gha,mode=max

.vscode/extensions.json

@@ -5,6 +5,7 @@
     "unifiedjs.vscode-mdx",
     "mikestead.dotenv",
     "christian-kohler.npm-intellisense",
-    "skellock.just"
+    "skellock.just",
+    "charliermarsh.ruff"
   ]
 }

.vscode/settings.json

@@ -27,7 +27,9 @@
   },
   "emmet.showExpandedAbbreviation": "never",
   "prettier.enable": false,
-  "tailwindCSS.experimental.configFile": "./packages/tailwind-config/config.ts",
   "typescript.tsdk": "node_modules/typescript/lib",
-  "rust-analyzer.cargo.features": ["testcontainers"]
+  "rust-analyzer.cargo.features": ["testcontainers"],
+  "[python]": {
+    "editor.defaultFormatter": "charliermarsh.ruff" 
+  }
 }

apps/recorder/Cargo.toml

@@ -65,7 +65,7 @@ once_cell = "1.20.2"
 reqwest-middleware = "0.4.0"
 reqwest-retry = "0.7.0"
 reqwest-tracing = "0.5.5"
-scraper = "0.22.0"
+scraper = "0.23"
 leaky-bucket = "1.1.2"
 serde_with = "3"
 jwt-authorizer = "0.15.0"
@@ -87,8 +87,8 @@ color-eyre = "0.6"
 log = "0.4.22"
 anyhow = "1.0.95"
 bollard = { version = "0.18", optional = true }
-async-graphql = { version = "7.0.15", features = [] }
+async-graphql = { version = "7", features = [] }
-async-graphql-axum = "7.0.15"
+async-graphql-axum = "7"
 fastrand = "2.3.0"
 seaography = { version = "1.1" }
 quirks_path = "0.1.1"
@@ -129,13 +129,11 @@ tracing-appender = "0.2.3"
 clap = "4.5.31"
 futures-util = "0.3.31"
 ipnetwork = "0.21.1"
-kanal = "0.1.0-pre8"
-append-only-vec = "0.1.7"
-typed-builder = "0.20.0"
 ctor = "0.4.0"
+librqbit = "8.0.0"
 
 [dev-dependencies]
 serial_test = "3"
 insta = { version = "1", features = ["redactions", "yaml", "filters"] }
 mockito = "1.6.1"
-rstest = "0.24.0"
+rstest = "0.25"

apps/recorder/src/download/mod.rs

@@ -1,11 +1,12 @@
 pub mod core;
 pub mod error;
 pub mod qbit;
-mod utils;
+pub mod rqbit;
+pub mod utils;
 
 pub use core::{
-    Torrent, TorrentContent, TorrentDownloader, TorrentFilter, TorrentSource, BITTORRENT_MIME_TYPE,
+    BITTORRENT_MIME_TYPE, MAGNET_SCHEMA, Torrent, TorrentContent, TorrentDownloader, TorrentFilter,
-    MAGNET_SCHEMA,
+    TorrentSource,
 };
 
 pub use error::TorrentDownloadError;

apps/recorder/src/download/rqbit/mod.rs

@@ -0,0 +1 @@
+use librqbit::TorrentMetadata;

apps/recorder/src/extract/mikan/rss_extract.rs

@@ -8,13 +8,13 @@ use tracing::instrument;
 use url::Url;
 
 use crate::{
+    download::core::BITTORRENT_MIME_TYPE,
     errors::{RError, RResult},
     extract::mikan::{
         MikanClient,
         web_extract::{MikanEpisodeHomepage, extract_mikan_episode_id_from_homepage},
     },
     fetch::bytes::fetch_bytes,
-    sync::core::BITTORRENT_MIME_TYPE,
 };
 
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
@@ -339,11 +339,11 @@ mod tests {
     use url::Url;
 
     use crate::{
+        download::core::BITTORRENT_MIME_TYPE,
         extract::mikan::{
             MikanBangumiAggregationRssChannel, MikanBangumiRssChannel, MikanRssChannel,
             extract_mikan_rss_channel_from_rss_link,
         },
-        sync::core::BITTORRENT_MIME_TYPE,
         test_utils::mikan::build_testing_mikan_client,
     };
 

apps/recorder/src/lib.rs

@@ -12,6 +12,7 @@ pub mod app;
 pub mod auth;
 pub mod cache;
 pub mod database;
+pub mod download;
 pub mod errors;
 pub mod extract;
 pub mod fetch;
@@ -20,7 +21,6 @@ pub mod logger;
 pub mod migrations;
 pub mod models;
 pub mod storage;
-pub mod sync;
 pub mod tasks;
 #[cfg(test)]
 pub mod test_utils;

packages/testing-torrents/Dockerfile

@@ -0,0 +1,14 @@
+FROM node:23-slim AS nodebt
+
+FROM nodebt AS deps
+
+RUN mkdir -p /app/workspace
+WORKDIR /app
+COPY package.json /app/
+RUN npm install
+
+FROM deps AS app
+
+COPY main.ts /app/
+
+CMD [ "npm", "start" ]

packages/testing-torrents/README.md

@@ -0,0 +1,20 @@
+# Konobangu Testing Torrents Container
+
+## Build
+
+```bash
+docker buildx build --platform linux/amd64 --tag konobangu-testing-torrents:latest --load .
+```
+
+## Run
+
+```bash
+docker run --network_mode=host --name konobangu-testing-torrents konobangu-testing-torrents:latest
+```
+
+## Publish
+
+```bash
+docker tag konobangu-testing-torrents:latest ghcr.io/dumtruck/konobangu-testing-torrents:latest
+docker push ghcr.io/dumtruck/konobangu-testing-torrents:latest
+```

packages/testing-torrents/docker-compose.yaml

@@ -0,0 +1,5 @@
+services:
+  konobangu-testing-torrents:
+    build: .
+    network_mode: host
+    container_name: konobangu-testing-torrents

packages/testing-torrents/main.ts

@@ -0,0 +1,181 @@
+import Fastify from 'fastify';
+import fastifyStatic from '@fastify/static';
+import { join } from 'node:path';
+
+import WebTorrent, { Torrent } from 'webtorrent';
+import createTorrent from 'create-torrent'
+// @ts-ignore
+import TrackerServer from 'bittorrent-tracker/server'
+import fs, { existsSync, mkdirSync, writeFileSync } from 'node:fs';
+
+// Configuration
+const API_PORT = 6080;
+const TRACKER_PORT = 6081;
+// Get local IP address for broader accessibility
+const LOCAL_IP = '127.0.0.1';
+const TRACKER_URL = `http://${LOCAL_IP}:${TRACKER_PORT}/announce`;
+const API_BASE_URL = `http://${LOCAL_IP}:${API_PORT}/api/static/`;
+
+// Initialize Fastify instance
+const app = Fastify({ logger: true });
+
+// Mount static file service, mapping ./workspace directory to /api/static route
+app.register(fastifyStatic, {
+    root: join(process.cwd(), 'workspace'),
+    prefix: '/api/static',
+});
+
+const tracker = new TrackerServer({
+    udp: true, // enable udp server? [default=true]
+    http: true, // enable http server? [default=true]
+    ws: true, // enable websocket server? [default=true]
+    stats: true, // enable web-based statistics? [default=true]
+    trustProxy: true, // enable trusting x-forwarded-for header for remote IP [default=false]
+});
+
+
+// Define request and response type definitions
+interface FileItem {
+  path: string;
+  size: number;
+}
+
+interface RequestSchema {
+  id: string;
+  fileList: FileItem[];
+}
+
+interface ResponseSchema {
+  torrentUrl: string;
+  magnetUrl: string;
+}
+
+// Start local Tracker
+async function startTracker(): Promise<void> {
+    return new Promise<void>((resolve, reject) => {
+        tracker.listen(TRACKER_PORT, "localhost", () => {
+            console.log(`Tracker listening on port ${TRACKER_PORT}`);
+            resolve();
+        });
+        tracker.on('error', (err: any) => {
+            console.error(`Tracker error: ${err}`)
+            reject(`Tracker error: ${err}`);
+        });
+        tracker.on('warning', (warn: any) => console.warn(`Tracker warning: ${warn}`));
+        // Log tracked torrents
+        tracker.on('update', (addr: any, params: any) => {
+            console.log(`Tracker update: ${params.info_hash} from ${addr}`);
+        });
+        tracker.on('stop', function () {
+            reject(`Tracker stopped`);
+        })
+        
+    });
+}
+
+// Tracker and WebTorrent client
+const webTorrent = new WebTorrent({});
+
+// Generate mock file
+async function generateMockFile(filePath: string, size: number) {
+  const dir = join(filePath, '..');
+  if (!existsSync(dir)) {
+    mkdirSync(dir, { recursive: true })
+  };
+
+  fs.writeFileSync(filePath, 'w');
+  fs.truncateSync(filePath, size);
+}
+
+// Generate torrent file
+function generateTorrent(folderPath: string, torrentPath: string): Promise<void> {
+    return new Promise((resolve, reject) => {
+      createTorrent(
+        folderPath,
+        {
+          announceList: [[TRACKER_URL]], // Specify tracker URL
+          private: false,
+          createdBy: 'WebTorrent',
+          comment: 'Generated by WebTorrent server',
+          urlList: [API_BASE_URL]
+        },
+        (err, torrent) => {
+          if (err) {
+            reject(new Error(`Failed to create torrent: ${err}`));
+            return;
+          }
+          writeFileSync(torrentPath, torrent);
+          if (!existsSync(torrentPath)) {
+            reject(new Error(`Torrent file ${torrentPath} was not created`));
+            return;
+          }
+          console.log(`Generated torrent with tracker: ${TRACKER_URL}`);
+          resolve();
+        }
+      );
+    });
+  }
+
+// Add torrent and seed
+async function seedTorrent(torrentPath: string): Promise<Torrent> {
+    return new Promise((resolve) => {
+        const torrent = webTorrent.seed(torrentPath, {
+            announce: [TRACKER_URL],
+        }, (t) => {
+            resolve(t);
+        });
+        torrent.on('error', (err) => console.error(`Torrent error: ${err}`));
+        torrent.on('wire', (wire) => console.log(`Connected to peer: ${wire.peerId}`));
+        torrent.on('done', () => console.log(`Torrent ${torrent.infoHash} fully seeded`));
+    })
+}
+
+// Handle POST request to /api/torrents/mock
+app.post<{ Body: RequestSchema }>('/api/torrents/mock', async (req, _reply) => {
+  const { id, fileList } = req.body;
+
+  const idFolder = join('./workspace', id);
+  if (!existsSync(idFolder)) {
+    mkdirSync(idFolder, { recursive: true })
+  };
+
+  for (const fileItem of fileList) {
+    const filePath = join(idFolder, fileItem.path);
+    await generateMockFile(filePath, fileItem.size);
+  }
+
+  const torrentPath = join('./workspace', `${id}.torrent`);
+  await generateTorrent(idFolder, torrentPath);
+
+  const torrent = await seedTorrent(torrentPath);
+  const magnetUrl = `magnet:?xt=urn:btih:${torrent.infoHash}&tr=${TRACKER_URL}`;
+
+  return {
+    torrentUrl: `${API_BASE_URL}${id}.torrent`,
+    magnetUrl,
+  } as ResponseSchema;
+});
+
+// Main program entry
+async function main() {
+  try {
+    await startTracker();
+    await app.listen({ port: API_PORT, host: '0.0.0.0' });
+    console.log(`Fastify running on http://0.0.0.0:${API_PORT}`);
+  } catch (err) {
+    console.error('Startup error:', err);
+    webTorrent.destroy();
+    tracker.close();
+    process.exit(1);
+  }
+}
+
+main();
+
+// Graceful shutdown
+process.on('SIGINT', () => {
+    console.log('Shutting down...');
+    tracker.close();
+    webTorrent.destroy();
+    process.exit(0);
+});

packages/testing-torrents/package.json

@@ -0,0 +1,22 @@
+{
+    "name": "@konobangu/testing-torrents",
+    "version": "0.0.1",
+    "description": "Kono bangumi? Testing Torrents",
+    "main": "main.ts",
+    "type": "commonjs",
+    "scripts": {
+        "start": "tsx main.ts"
+    },
+    "dependencies": {
+        "@fastify/static": "^8.1.1",
+        "bittorrent-tracker": "^11.2.1",
+        "create-torrent": "^6.1.0",
+        "fastify": "^5.2.2",
+        "tsx": "^4.19.2",
+        "webtorrent": "^2.5.19"
+    },
+    "devDependencies": {
+        "@types/create-torrent": "^5.0.2",
+        "@types/webtorrent": "^0.110.0"
+    }
+}

packages/testing/.gitignore

@@ -0,0 +1,2 @@
+resources/*
+!resources/.gitkeep