feat: basic support rss

.cargo/config.toml

@@ -1,34 +1,5 @@
 [alias]
-recorder = "run -p recorder --bin recorder_cli -- --environment development"
 recorder-playground = "run -p recorder  --example playground -- --environment development"
 
 [build]
-rustflags = ["-Zthreads=8"]
-
-[target.x86_64-unknown-linux-gnu]
-linker = "clang"
-rustflags = ["-Zthreads=8", "-Clink-arg=-fuse-ld=lld", "-Zshare-generics=y"]
-
-[target.x86_64-pc-windows-msvc]
-linker = "rust-lld.exe"
-rustflags = ["-Zthreads=8", "-Zshare-generics=n"]
-
-# NOTE: you must install [Mach-O LLD Port](https://lld.llvm.org/MachO/index.html) on mac. you can easily do this by installing llvm which includes lld with the "brew" package manager:
-# `brew install llvm`
-#[target.x86_64-apple-darwin]
-#rustflags = [
-#    "-Zthreads=8",
-#    "-C",
-#    "link-arg=-fuse-ld=/usr/local/opt/llvm/bin/ld64.lld",
-#    "-Zshare-generics=y",
-#]
-
-# NOTE: you must install [Mach-O LLD Port](https://lld.llvm.org/MachO/index.html) on mac. you can easily do this by installing llvm which includes lld with the "brew" package manager:
-# `brew install llvm`
-#[target.aarch64-apple-darwin]
-#rustflags = [
-#    "-Zthreads=8",
-#    "-C",
-#    "link-arg=-fuse-ld=/opt/homebrew/opt/llvm/bin/ld64.lld",
-#    "-Zshare-generics=y",
-#]
+rustflags = ["-Zthreads=8", "-Zshare-generics=y"]

.github.bk/workflows/ci.yaml

@@ -1,107 +0,0 @@
-name: CI
-on:
-  push:
-    branches:
-      - master
-      - main
-  pull_request:
-
-env:
-  RUST_TOOLCHAIN: stable
-  TOOLCHAIN_PROFILE: minimal
-
-jobs:
-  rustfmt:
-    name: Check Style
-    runs-on: ubuntu-latest
-
-    permissions:
-      contents: read
-
-    steps:
-      - name: Checkout the code
-        uses: actions/checkout@v4
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: ${{ env.TOOLCHAIN_PROFILE }}
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
-          override: true
-          components: rustfmt
-      - name: Run cargo fmt
-        uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all -- --check
-
-  clippy:
-    name: Run Clippy
-    runs-on: ubuntu-latest
-
-    permissions:
-      contents: read
-
-    steps:
-      - name: Checkout the code
-        uses: actions/checkout@v4
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: ${{ env.TOOLCHAIN_PROFILE }}
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
-          override: true
-      - name: Setup Rust cache
-        uses: Swatinem/rust-cache@v2
-      - name: Run cargo clippy
-        uses: actions-rs/cargo@v1
-        with:
-          command: clippy
-          args: --all-features -- -D warnings -W clippy::pedantic -W clippy::nursery -W rust-2018-idioms
-
-  test:
-    name: Run Tests
-    runs-on: ubuntu-latest
-
-    permissions:
-      contents: read
-
-    services:
-      redis:
-        image: redis
-        options: >-
-          --health-cmd "redis-cli ping"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - "6379:6379"
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_DB: postgress_test
-          POSTGRES_USER: postgress
-          POSTGRES_PASSWORD: postgress
-        ports:
-          - "5432:5432"
-        # Set health checks to wait until postgres has started
-        options: --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
-    steps:
-      - name: Checkout the code
-        uses: actions/checkout@v4
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: ${{ env.TOOLCHAIN_PROFILE }}
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
-          override: true
-      - name: Setup Rust cache
-        uses: Swatinem/rust-cache@v2
-      - name: Run cargo test
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --all-features --all
-        env:
-          REDIS_URL: redis://localhost:${{job.services.redis.ports[6379]}}
-          DATABASE_URL: postgres://postgress:postgress@localhost:5432/postgress_test

.github/workflows/testing-torrents-container.yaml

@@ -0,0 +1,36 @@
+name: Testing Torrents Container
+
+on:
+  workflow_dispatch:
+
+env:
+    REGISTRY: ghcr.io
+    ORG: dumtruck
+    PROJECT: konobangu
+
+jobs:
+  build-container:
+    runs-on: ubuntu-latest
+    permissions:
+        contents: read
+        packages: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Log in to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: 'packages/testing-torrents'
+          file: 'packages/testing-torrents/Dockerfile'
+          push: true
+          tags: '${{ env.REGISTRY }}/${{ env.ORG }}/${{ env.PROJECT }}-testing-torrents:latest'
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.gitignore

@@ -158,11 +158,8 @@ web_modules/
 .yarn-integrity
 
 # Local env files
-.env
 .env.local
-.env.development.local
-.env.test.local
-.env.production.local
+.env.*.local
 
 # parcel-bundler cache (https://parceljs.org/)
 .cache
@@ -219,6 +216,7 @@ index.d.ts.map
 # Added by cargo
 
 /target
+/ide-target
 !/examples/.gitkeep
 /.env
 /.env.bk

.vscode/extensions.json

@@ -5,6 +5,7 @@
     "unifiedjs.vscode-mdx",
     "mikestead.dotenv",
     "christian-kohler.npm-intellisense",
-    "skellock.just"
+    "skellock.just",
+    "zerotaskx.rust-extension-pack"
   ]
 }

.vscode/settings.json

@@ -1,6 +1,5 @@
 {
     "npm.packageManager": "pnpm",
-  "rust-analyzer.showUnlinkedFileNotification": false,
     "[javascript]": {
         "editor.defaultFormatter": "vscode.typescript-language-features",
         "editor.formatOnSave": true
@@ -27,7 +26,27 @@
     },
     "emmet.showExpandedAbbreviation": "never",
     "prettier.enable": false,
-  "tailwindCSS.experimental.configFile": "./packages/tailwind-config/config.ts",
     "typescript.tsdk": "node_modules/typescript/lib",
-  "rust-analyzer.cargo.features": ["testcontainers"]
+    "rust-analyzer.showUnlinkedFileNotification": false,
+    "sqltools.connections": [
+        {
+            "previewLimit": 50,
+            "server": "localhost",
+            "port": 5432,
+            "driver": "PostgreSQL",
+            "name": "konobangu-dev",
+            "database": "konobangu",
+            "username": "konobangu"
+        }
+    ],
+    "rust-analyzer.cargo.features": "all",
+    "rust-analyzer.testExplorer": true
+    // https://github.com/rust-lang/rust/issues/141540
+    // "rust-analyzer.cargo.targetDir": "target/rust-analyzer",
+    // "rust-analyzer.check.extraEnv": {
+    //     "CARGO_TARGET_DIR": "target/rust-analyzer"
+    // },
+    // "rust-analyzer.cargo.extraEnv": {
+    //     "CARGO_TARGET_DIR": "target/analyzer"
+    // }
 }

.vscode/tasks.json

@@ -0,0 +1,112 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "dev-all",
+            "dependsOn": [
+                "dev-webui",
+                "dev-recorder",
+                "dev-proxy",
+                "dev-codegen-wait",
+                "dev-deps",
+            ],
+            "dependsOrder": "parallel",
+            "group": {
+                "kind": "build",
+                "isDefault": false,
+            },
+            "presentation": {
+                "group": "new-group",
+                "echo": true,
+                "reveal": "always",
+                "panel": "shared",
+                "clear": false
+            }
+        },
+        {
+            "label": "dev-webui",
+            "type": "shell",
+            "command": "just",
+            "args": [
+                "dev-webui"
+            ],
+            "isBackground": true,
+            "problemMatcher": [],
+            "presentation": {
+                "panel": "dedicated",
+                "reveal": "always",
+                "focus": true,
+                "showReuseMessage": true,
+                "clear": true,
+            }
+        },
+        {
+            "label": "dev-deps",
+            "type": "shell",
+            "command": "just",
+            "args": [
+                "dev-deps"
+            ],
+            "isBackground": true,
+            "problemMatcher": [],
+            "presentation": {
+                "panel": "dedicated",
+                "reveal": "never",
+                "focus": false,
+                "showReuseMessage": true,
+                "clear": true,
+            }
+        },
+        {
+            "label": "dev-codegen-wait",
+            "type": "shell",
+            "command": "just",
+            "args": [
+                "dev-codegen-wait"
+            ],
+            "isBackground": true,
+            "problemMatcher": [],
+            "presentation": {
+                "panel": "dedicated",
+                "reveal": "never",
+                "focus": false,
+                "showReuseMessage": true,
+                "clear": true,
+            }
+        },
+        {
+            "label": "dev-recorder",
+            "type": "shell",
+            "command": "just",
+            "args": [
+                "dev-recorder"
+            ],
+            "isBackground": true,
+            "problemMatcher": [],
+            "presentation": {
+                "panel": "dedicated",
+                "reveal": "never",
+                "focus": false,
+                "showReuseMessage": true,
+                "clear": true,
+            }
+        },
+        {
+            "label": "dev-proxy",
+            "type": "shell",
+            "command": "just",
+            "args": [
+                "dev-proxy",
+            ],
+            "isBackground": true,
+            "problemMatcher": [],
+            "presentation": {
+                "panel": "dedicated",
+                "reveal": "never",
+                "focus": false,
+                "showReuseMessage": true,
+                "clear": true,
+            }
+        }
+    ]
+}

Cargo.toml

@@ -1,9 +1,90 @@
+# cargo-features = ["codegen-backend"]
+
 [workspace]
-members = ["apps/recorder"]
+members = [
+    "packages/testing-torrents",
+    "packages/util",
+    "packages/util-derive",
+    "packages/fetch",
+    "packages/downloader",
+    "apps/recorder",
+    "apps/proxy",
+]
 resolver = "2"
 
-[patch.crates-io]
-jwt-authorizer = { git = "https://github.com/blablacio/jwt-authorizer.git", rev = "e956774" }
+[profile.dev]
+debug = 0
+# https://github.com/rust-lang/rust/issues/141540
+incremental = false
+# [simd not supported by cranelift](https://github.com/rust-lang/rustc_codegen_cranelift/issues/171)
+# codegen-backend = "cranelift"
 
-# [patch."https://github.com/lonelyhentxi/qbit.git"]
-# qbit-rs = { path = "./patches/qbit-rs" }
+[workspace.dependencies]
+testing-torrents = { path = "./packages/testing-torrents" }
+util = { path = "./packages/util" }
+util-derive = { path = "./packages/util-derive" }
+fetch = { path = "./packages/fetch" }
+downloader = { path = "./packages/downloader" }
+recorder = { path = "./apps/recorder" }
+
+reqwest = { version = "0.12.20", features = [
+    "charset",
+    "http2",
+    "json",
+    "macos-system-configuration",
+    "cookies",
+] }
+moka = "0.12"
+futures = "0.3"
+quirks_path = "0.1"
+snafu = { version = "0.8", features = ["futures"] }
+testcontainers = { version = "0.24" }
+testcontainers-modules = { version = "0.12.1" }
+testcontainers-ext = { version = "0.1.0", features = ["tracing"] }
+serde = { version = "1", features = ["derive"] }
+tokio = { version = "1.45.1", features = [
+    "macros",
+    "fs",
+    "rt-multi-thread",
+    "signal",
+] }
+serde_json = "1"
+async-trait = "0.1"
+tracing = "0.1"
+url = "2.5.2"
+anyhow = "1"
+itertools = "0.14"
+chrono = "0.4"
+bytes = "1"
+serde_with = "3"
+regex = "1.11"
+lazy_static = "1.5"
+axum = { version = "0.8.3", features = ["macros"] }
+tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
+axum-extra = { version = "0.10", features = ["typed-header"] }
+mockito = { version = "1.6.1" }
+convert_case = "0.8"
+color-eyre = "0.6.5"
+inquire = "0.7.5"
+image = "0.25.6"
+uuid = { version = "1.6.0", features = ["v4"] }
+maplit = "1.0.2"
+once_cell = "1.20.2"
+rand = "0.9.1"
+rust_decimal = "1.37.2"
+base64 = "0.22.1"
+nom = "8.0.0"
+percent-encoding = "2.3.1"
+num-traits = "0.2.19"
+http = "1.2.0"
+async-stream = "0.3.6"
+serde_variant = "0.1.3"
+tracing-appender = "0.2.3"
+clap = "4.5.40"
+ipnetwork = "0.21.1"
+typed-builder = "0.21.0"
+nanoid = "0.4.0"
+webp = "0.3.0"
+
+[patch.crates-io]
+seaography = { git = "https://github.com/dumtruck/seaography.git", rev = "a787c3a" }

apps/app/.env.development

@@ -1,29 +0,0 @@
-# Server
-AUTH_TYPE="basic" # 
-
-BASIC_USER="konobangu"
-BASIC_PASSWORD="konobangu"
-
-OIDC_PROVIDER_ENDPOINT="https://some-oidc-auth.com/oidc/.well-known/openid-configuration"
-OIDC_CLIENT_ID=""
-OIDC_CLIENT_SECRET=""
-OIDC_ISSUER="https://some-oidc-auth.com/oidc"
-OIDC_AUDIENCE="https://konobangu.com/api"
-OIDC_ICON_URL=""
-OIDC_EXTRA_SCOPE_REGEX=""
-OIDC_EXTRA_CLAIM_KEY=""
-OIDC_EXTRA_CLAIM_VALUE=""
-
-DATABASE_URL="postgres://konobangu:konobangu@127.0.0.1:5432/konobangu"
-BETTERSTACK_API_KEY=""
-BETTERSTACK_URL=""
-FLAGS_SECRET=""
-ARCJET_KEY=""
-SVIX_TOKEN=""
-LIVEBLOCKS_SECRET=""
-
-# Client
-NEXT_PUBLIC_APP_URL="http://localhost:5000"
-NEXT_PUBLIC_WEB_URL="http://localhost:5001"
-NEXT_PUBLIC_DOCS_URL="http://localhost:5004"
-NEXT_PUBLIC_VERCEL_PROJECT_PRODUCTION_URL="https://konobangu.com"

apps/app/.env.example

@@ -1,29 +0,0 @@
-# AUTH
-AUTH_TYPE="basic"
-
-NEXT_PUBLIC_OIDC_PROVIDER_ENDPOINT="https://some-oidc-auth.com/oidc/.well-known/openid-configuration"
-NEXT_PUBLIC_OIDC_CLIENT_ID=""
-NEXT_PUBLIC_OIDC_CLIENT_SECRET=""
-NEXT_PUBLIC_OIDC_ICON_URL=""
-OIDC_ISSUER="https://some-oidc-auth.com/oidc"
-OIDC_AUDIENCE="https://konobangu.com/api"
-OIDC_EXTRA_SCOPES="" # 如 "read:konobangu,write:konobangu"
-OIDC_EXTRA_CLAIM_KEY=""
-OIDC_EXTRA_CLAIM_VALUE=""
-
-# DATABASE
-DATABASE_URL="postgres://konobangu:konobangu@127.0.0.1:5432/konobangu"
-
-# SERVER MISC
-BETTERSTACK_API_KEY=""
-BETTERSTACK_URL=""
-FLAGS_SECRET=""
-ARCJET_KEY=""
-SVIX_TOKEN=""
-LIVEBLOCKS_SECRET=""
-
-# WEBUI
-NEXT_PUBLIC_APP_URL="http://localhost:5000"
-NEXT_PUBLIC_WEB_URL="http://localhost:5001"
-NEXT_PUBLIC_DOCS_URL="http://localhost:5004"
-NEXT_PUBLIC_VERCEL_PROJECT_PRODUCTION_URL="https://konobangu.com"

apps/app/.gitignore

@@ -1,45 +0,0 @@
-# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
-
-# dependencies
-/node_modules
-/.pnp
-.pnp.js
-
-# testing
-/coverage
-
-# next.js
-/.next/
-/out/
-
-# production
-/build
-
-# misc
-.DS_Store
-*.pem
-
-# debug
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-.pnpm-debug.log*
-
-# local env files
-.env*.local
-
-# vercel
-.vercel
-
-# typescript
-*.tsbuildinfo
-next-env.d.ts
-
-# prisma
-.env
-
-# react.email
-.react-email
-
-# Sentry
-.sentryclirc

apps/app/__tests__/sign-in.test.tsx

@@ -1,13 +0,0 @@
-import { render, screen } from '@testing-library/react';
-import { expect, test } from 'vitest';
-import Page from '../app/(unauthenticated)/sign-in/[[...sign-in]]/page';
-
-test('Sign In Page', () => {
-  render(<Page />);
-  expect(
-    screen.getByRole('heading', {
-      level: 1,
-      name: 'Welcome back',
-    })
-  ).toBeDefined();
-});

apps/app/__tests__/sign-up.test.tsx

@@ -1,13 +0,0 @@
-import { render, screen } from '@testing-library/react';
-import { expect, test } from 'vitest';
-import Page from '../app/(unauthenticated)/sign-up/[[...sign-up]]/page';
-
-test('Sign Up Page', () => {
-  render(<Page />);
-  expect(
-    screen.getByRole('heading', {
-      level: 1,
-      name: 'Create an account',
-    })
-  ).toBeDefined();
-});

apps/app/app/(authenticated)/components/avatar-stack.tsx

@@ -1,59 +0,0 @@
-'use client';
-
-import { useOthers, useSelf } from '@konobangu/collaboration/hooks';
-import {
-  Avatar,
-  AvatarFallback,
-  AvatarImage,
-} from '@konobangu/design-system/components/ui/avatar';
-import {
-  Tooltip,
-  TooltipContent,
-  TooltipTrigger,
-} from '@konobangu/design-system/components/ui/tooltip';
-import { tailwind } from '@konobangu/tailwind-config';
-
-type PresenceAvatarProps = {
-  info?: Liveblocks['UserMeta']['info'];
-};
-
-const PresenceAvatar = ({ info }: PresenceAvatarProps) => (
-  <Tooltip delayDuration={0}>
-    <TooltipTrigger>
-      <Avatar className="h-7 w-7 bg-secondary ring-1 ring-background">
-        <AvatarImage src={info?.avatar} alt={info?.name} />
-        <AvatarFallback className="text-xs">
-          {info?.name?.slice(0, 2)}
-        </AvatarFallback>
-      </Avatar>
-    </TooltipTrigger>
-    <TooltipContent collisionPadding={4}>
-      <p>{info?.name ?? 'Unknown'}</p>
-    </TooltipContent>
-  </Tooltip>
-);
-
-export const AvatarStack = () => {
-  const others = useOthers();
-  const self = useSelf();
-  const hasMoreUsers = others.length > 3;
-
-  return (
-    <div className="-space-x-1 flex items-center px-4">
-      {others.slice(0, 3).map(({ connectionId, info }) => (
-        <PresenceAvatar key={connectionId} info={info} />
-      ))}
-
-      {hasMoreUsers && (
-        <PresenceAvatar
-          info={{
-            name: `+${others.length - 3}`,
-            color: tailwind.theme.colors.gray[500],
-          }}
-        />
-      )}
-
-      {self && <PresenceAvatar info={self.info} />}
-    </div>
-  );
-};

apps/app/app/(authenticated)/components/collaboration-provider.tsx

@@ -1,48 +0,0 @@
-'use client';
-
-import { getUsers } from '@/app/actions/users/get';
-import { searchUsers } from '@/app/actions/users/search';
-import { Room } from '@konobangu/collaboration/room';
-import type { ReactNode } from 'react';
-
-export const CollaborationProvider = ({
-  orgId,
-  children,
-}: {
-  orgId: string;
-  children: ReactNode;
-}) => {
-  const resolveUsers = async ({ userIds }: { userIds: string[] }) => {
-    const response = await getUsers(userIds);
-
-    if ('error' in response) {
-      throw new Error('Problem resolving users');
-    }
-
-    return response.data;
-  };
-
-  const resolveMentionSuggestions = async ({ text }: { text: string }) => {
-    const response = await searchUsers(text);
-
-    if ('error' in response) {
-      throw new Error('Problem resolving mention suggestions');
-    }
-
-    return response.data;
-  };
-
-  return (
-    <Room
-      id={`${orgId}:presence`}
-      authEndpoint="/api/collaboration/auth"
-      fallback={
-        <div className="px-3 text-muted-foreground text-xs">Loading...</div>
-      }
-      resolveUsers={resolveUsers}
-      resolveMentionSuggestions={resolveMentionSuggestions}
-    >
-      {children}
-    </Room>
-  );
-};

apps/app/app/(authenticated)/components/cursors.tsx

@@ -1,106 +0,0 @@
-'use client';
-
-import { useMyPresence, useOthers } from '@konobangu/collaboration/hooks';
-import { useEffect } from 'react';
-
-const Cursor = ({
-  name,
-  color,
-  x,
-  y,
-}: {
-  name: string | undefined;
-  color: string;
-  x: number;
-  y: number;
-}) => (
-  <div
-    className="pointer-events-none absolute top-0 left-0 z-[999] select-none transition-transform duration-100"
-    style={{
-      transform: `translateX(${x}px) translateY(${y}px)`,
-    }}
-  >
-    <svg
-      className="absolute top-0 left-0"
-      width="24"
-      height="36"
-      viewBox="0 0 24 36"
-      fill="none"
-      xmlns="http://www.w3.org/2000/svg"
-    >
-      <title>Cursor</title>
-      <path
-        d="M5.65376 12.3673H5.46026L5.31717 12.4976L0.500002 16.8829L0.500002 1.19841L11.7841 12.3673H5.65376Z"
-        fill={color}
-      />
-    </svg>
-    <div
-      className="absolute top-4 left-1.5 whitespace-nowrap rounded-full px-2 py-0.5 text-white text-xs"
-      style={{
-        backgroundColor: color,
-      }}
-    >
-      {name}
-    </div>
-  </div>
-);
-
-export const Cursors = () => {
-  /**
-   * useMyPresence returns the presence of the current user and a function to update it.
-   * updateMyPresence is different than the setState function returned by the useState hook from React.
-   * You don't need to pass the full presence object to update it.
-   * See https://liveblocks.io/docs/api-reference/liveblocks-react#useMyPresence for more information
-   */
-  const [_cursor, updateMyPresence] = useMyPresence();
-
-  /**
-   * Return all the other users in the room and their presence (a cursor position in this case)
-   */
-  const others = useOthers();
-
-  useEffect(() => {
-    const onPointerMove = (event: PointerEvent) => {
-      // Update the user cursor position on every pointer move
-      updateMyPresence({
-        cursor: {
-          x: Math.round(event.clientX),
-          y: Math.round(event.clientY),
-        },
-      });
-    };
-
-    const onPointerLeave = () => {
-      // When the pointer goes out, set cursor to null
-      updateMyPresence({
-        cursor: null,
-      });
-    };
-
-    document.body.addEventListener('pointermove', onPointerMove);
-    document.body.addEventListener('pointerleave', onPointerLeave);
-
-    return () => {
-      document.body.removeEventListener('pointermove', onPointerMove);
-      document.body.removeEventListener('pointerleave', onPointerLeave);
-    };
-  }, [updateMyPresence]);
-
-  return others.map(({ connectionId, presence, info }) => {
-    if (!presence.cursor) {
-      return null;
-    }
-
-    return (
-      <Cursor
-        key={`cursor-${connectionId}`}
-        // connectionId is an integer that is incremented at every new connections
-        // Assigning a color with a modulo makes sure that a specific user has the same colors on every clients
-        color={info.color}
-        x={presence.cursor.x}
-        y={presence.cursor.y}
-        name={info?.name}
-      />
-    );
-  });
-};

apps/app/app/(authenticated)/components/header.tsx

@@ -1,43 +0,0 @@
-import {
-  Breadcrumb,
-  BreadcrumbItem,
-  BreadcrumbLink,
-  BreadcrumbList,
-  BreadcrumbPage,
-  BreadcrumbSeparator,
-} from '@konobangu/design-system/components/ui/breadcrumb';
-import { Separator } from '@konobangu/design-system/components/ui/separator';
-import { SidebarTrigger } from '@konobangu/design-system/components/ui/sidebar';
-import { Fragment, type ReactNode } from 'react';
-
-type HeaderProps = {
-  pages: string[];
-  page: string;
-  children?: ReactNode;
-};
-
-export const Header = ({ pages, page, children }: HeaderProps) => (
-  <header className="flex h-16 shrink-0 items-center justify-between gap-2">
-    <div className="flex items-center gap-2 px-4">
-      <SidebarTrigger className="-ml-1" />
-      <Separator orientation="vertical" className="mr-2 h-4" />
-      <Breadcrumb>
-        <BreadcrumbList>
-          {pages.map((page, index) => (
-            <Fragment key={page}>
-              {index > 0 && <BreadcrumbSeparator className="hidden md:block" />}
-              <BreadcrumbItem className="hidden md:block">
-                <BreadcrumbLink href="#">{page}</BreadcrumbLink>
-              </BreadcrumbItem>
-            </Fragment>
-          ))}
-          <BreadcrumbSeparator className="hidden md:block" />
-          <BreadcrumbItem>
-            <BreadcrumbPage>{page}</BreadcrumbPage>
-          </BreadcrumbItem>
-        </BreadcrumbList>
-      </Breadcrumb>
-    </div>
-    {children}
-  </header>
-);

apps/app/app/(authenticated)/components/posthog-identifier.tsx

@@ -1,44 +0,0 @@
-'use client';
-
-import { analytics } from '@konobangu/analytics/client';
-import { useSession } from '@konobangu/auth/client';
-import { usePathname, useSearchParams } from 'next/navigation';
-import { useEffect, useRef } from 'react';
-
-export const PostHogIdentifier = () => {
-  const session = useSession();
-  const user = session?.data?.user;
-  const identified = useRef(false);
-  const pathname = usePathname();
-  const searchParams = useSearchParams();
-
-  useEffect(() => {
-    // Track pageviews
-    if (pathname && analytics) {
-      let url = window.origin + pathname;
-      if (searchParams.toString()) {
-        url = `${url}?${searchParams.toString()}`;
-      }
-      analytics.capture('$pageview', {
-        $current_url: url,
-      });
-    }
-  }, [pathname, searchParams]);
-
-  useEffect(() => {
-    if (!user || identified.current) {
-      return;
-    }
-
-    analytics.identify(user.id, {
-      email: user.email,
-      name: user.name,
-      createdAt: user.createdAt,
-      avatar: user.image,
-    });
-
-    identified.current = true;
-  }, [user]);
-
-  return null;
-};

apps/app/app/(authenticated)/components/sidebar.tsx

@@ -1,342 +0,0 @@
-'use client';
-
-// import { OrganizationSwitcher, UserButton } from '@konobangu/auth/client';
-import { ModeToggle } from '@konobangu/design-system/components/mode-toggle';
-import {
-  Collapsible,
-  CollapsibleContent,
-  CollapsibleTrigger,
-} from '@konobangu/design-system/components/ui/collapsible';
-import {
-  DropdownMenu,
-  DropdownMenuContent,
-  DropdownMenuItem,
-  DropdownMenuSeparator,
-  DropdownMenuTrigger,
-} from '@konobangu/design-system/components/ui/dropdown-menu';
-import {
-  Sidebar,
-  SidebarContent,
-  SidebarFooter,
-  SidebarGroup,
-  SidebarGroupContent,
-  SidebarGroupLabel,
-  SidebarHeader,
-  SidebarInset,
-  SidebarMenu,
-  SidebarMenuAction,
-  SidebarMenuButton,
-  SidebarMenuItem,
-  SidebarMenuSub,
-  SidebarMenuSubButton,
-  SidebarMenuSubItem,
-  useSidebar,
-} from '@konobangu/design-system/components/ui/sidebar';
-import { cn } from '@konobangu/design-system/lib/utils';
-import {
-  AnchorIcon,
-  BookOpenIcon,
-  BotIcon,
-  ChevronRightIcon,
-  FolderIcon,
-  FrameIcon,
-  LifeBuoyIcon,
-  MapIcon,
-  MoreHorizontalIcon,
-  PieChartIcon,
-  SendIcon,
-  Settings2Icon,
-  ShareIcon,
-  SquareTerminalIcon,
-  Trash2Icon,
-} from 'lucide-react';
-import type { ReactNode } from 'react';
-
-type GlobalSidebarProperties = {
-  readonly children: ReactNode;
-};
-
-const data = {
-  user: {
-    name: 'shadcn',
-    email: 'm@example.com',
-    avatar: '/avatars/shadcn.jpg',
-  },
-  navMain: [
-    {
-      title: 'Playground',
-      url: '#',
-      icon: SquareTerminalIcon,
-      isActive: true,
-      items: [
-        {
-          title: 'History',
-          url: '#',
-        },
-        {
-          title: 'Starred',
-          url: '#',
-        },
-        {
-          title: 'Settings',
-          url: '#',
-        },
-      ],
-    },
-    {
-      title: 'Models',
-      url: '#',
-      icon: BotIcon,
-      items: [
-        {
-          title: 'Genesis',
-          url: '#',
-        },
-        {
-          title: 'Explorer',
-          url: '#',
-        },
-        {
-          title: 'Quantum',
-          url: '#',
-        },
-      ],
-    },
-    {
-      title: 'Documentation',
-      url: '#',
-      icon: BookOpenIcon,
-      items: [
-        {
-          title: 'Introduction',
-          url: '#',
-        },
-        {
-          title: 'Get Started',
-          url: '#',
-        },
-        {
-          title: 'Tutorials',
-          url: '#',
-        },
-        {
-          title: 'Changelog',
-          url: '#',
-        },
-      ],
-    },
-    {
-      title: 'Settings',
-      url: '#',
-      icon: Settings2Icon,
-      items: [
-        {
-          title: 'General',
-          url: '#',
-        },
-        {
-          title: 'Team',
-          url: '#',
-        },
-        {
-          title: 'Billing',
-          url: '#',
-        },
-        {
-          title: 'Limits',
-          url: '#',
-        },
-      ],
-    },
-  ],
-  navSecondary: [
-    {
-      title: 'Webhooks',
-      url: '/webhooks',
-      icon: AnchorIcon,
-    },
-    {
-      title: 'Support',
-      url: '#',
-      icon: LifeBuoyIcon,
-    },
-    {
-      title: 'Feedback',
-      url: '#',
-      icon: SendIcon,
-    },
-  ],
-  projects: [
-    {
-      name: 'Design Engineering',
-      url: '#',
-      icon: FrameIcon,
-    },
-    {
-      name: 'Sales & Marketing',
-      url: '#',
-      icon: PieChartIcon,
-    },
-    {
-      name: 'Travel',
-      url: '#',
-      icon: MapIcon,
-    },
-  ],
-};
-
-export const GlobalSidebar = ({ children }: GlobalSidebarProperties) => {
-  const sidebar = useSidebar();
-
-  return (
-    <>
-      <Sidebar variant="inset">
-        <SidebarHeader>
-          <SidebarMenu>
-            <SidebarMenuItem>
-              <div
-                className={cn(
-                  'h-[36px] overflow-hidden transition-all [&>div]:w-full',
-                  sidebar.open ? '' : '-mx-1'
-                )}
-              >
-                {/* <OrganizationSwitcher
-                  hidePersonal
-                  afterSelectOrganizationUrl="/"
-                /> */}
-              </div>
-            </SidebarMenuItem>
-          </SidebarMenu>
-        </SidebarHeader>
-        <SidebarContent>
-          <SidebarGroup>
-            <SidebarGroupLabel>Platform</SidebarGroupLabel>
-            <SidebarMenu>
-              {data.navMain.map((item) => (
-                <Collapsible
-                  key={item.title}
-                  asChild
-                  defaultOpen={item.isActive}
-                >
-                  <SidebarMenuItem>
-                    <SidebarMenuButton asChild tooltip={item.title}>
-                      <a href={item.url}>
-                        <item.icon />
-                        <span>{item.title}</span>
-                      </a>
-                    </SidebarMenuButton>
-                    {item.items?.length ? (
-                      <>
-                        <CollapsibleTrigger asChild>
-                          <SidebarMenuAction className="data-[state=open]:rotate-90">
-                            <ChevronRightIcon />
-                            <span className="sr-only">Toggle</span>
-                          </SidebarMenuAction>
-                        </CollapsibleTrigger>
-                        <CollapsibleContent>
-                          <SidebarMenuSub>
-                            {item.items?.map((subItem) => (
-                              <SidebarMenuSubItem key={subItem.title}>
-                                <SidebarMenuSubButton asChild>
-                                  <a href={subItem.url}>
-                                    <span>{subItem.title}</span>
-                                  </a>
-                                </SidebarMenuSubButton>
-                              </SidebarMenuSubItem>
-                            ))}
-                          </SidebarMenuSub>
-                        </CollapsibleContent>
-                      </>
-                    ) : null}
-                  </SidebarMenuItem>
-                </Collapsible>
-              ))}
-            </SidebarMenu>
-          </SidebarGroup>
-          <SidebarGroup className="group-data-[collapsible=icon]:hidden">
-            <SidebarGroupLabel>Projects</SidebarGroupLabel>
-            <SidebarMenu>
-              {data.projects.map((item) => (
-                <SidebarMenuItem key={item.name}>
-                  <SidebarMenuButton asChild>
-                    <a href={item.url}>
-                      <item.icon />
-                      <span>{item.name}</span>
-                    </a>
-                  </SidebarMenuButton>
-                  <DropdownMenu>
-                    <DropdownMenuTrigger asChild>
-                      <SidebarMenuAction showOnHover>
-                        <MoreHorizontalIcon />
-                        <span className="sr-only">More</span>
-                      </SidebarMenuAction>
-                    </DropdownMenuTrigger>
-                    <DropdownMenuContent
-                      className="w-48"
-                      side="bottom"
-                      align="end"
-                    >
-                      <DropdownMenuItem>
-                        <FolderIcon className="text-muted-foreground" />
-                        <span>View Project</span>
-                      </DropdownMenuItem>
-                      <DropdownMenuItem>
-                        <ShareIcon className="text-muted-foreground" />
-                        <span>Share Project</span>
-                      </DropdownMenuItem>
-                      <DropdownMenuSeparator />
-                      <DropdownMenuItem>
-                        <Trash2Icon className="text-muted-foreground" />
-                        <span>Delete Project</span>
-                      </DropdownMenuItem>
-                    </DropdownMenuContent>
-                  </DropdownMenu>
-                </SidebarMenuItem>
-              ))}
-              <SidebarMenuItem>
-                <SidebarMenuButton>
-                  <MoreHorizontalIcon />
-                  <span>More</span>
-                </SidebarMenuButton>
-              </SidebarMenuItem>
-            </SidebarMenu>
-          </SidebarGroup>
-          <SidebarGroup className="mt-auto">
-            <SidebarGroupContent>
-              <SidebarMenu>
-                {data.navSecondary.map((item) => (
-                  <SidebarMenuItem key={item.title}>
-                    <SidebarMenuButton asChild>
-                      <a href={item.url}>
-                        <item.icon />
-                        <span>{item.title}</span>
-                      </a>
-                    </SidebarMenuButton>
-                  </SidebarMenuItem>
-                ))}
-              </SidebarMenu>
-            </SidebarGroupContent>
-          </SidebarGroup>
-        </SidebarContent>
-        <SidebarFooter>
-          <SidebarMenu>
-            <SidebarMenuItem className="flex items-center gap-2">
-              {/* <UserButton
-                showName
-                appearance={{
-                  elements: {
-                    rootBox: 'flex overflow-hidden w-full',
-                    userButtonBox: 'flex-row-reverse',
-                    userButtonOuterIdentifier: 'truncate pl-0',
-                  },
-                }}
-              /> */}
-              <ModeToggle />
-            </SidebarMenuItem>
-          </SidebarMenu>
-        </SidebarFooter>
-      </Sidebar>
-      <SidebarInset>{children}</SidebarInset>
-    </>
-  );
-};

apps/app/app/(authenticated)/layout.tsx

@@ -1,42 +0,0 @@
-import { getSessionFromHeaders } from '@konobangu/auth/server';
-import { SidebarProvider } from '@konobangu/design-system/components/ui/sidebar';
-import { env } from '@konobangu/env';
-import { showBetaFeature } from '@konobangu/feature-flags';
-import { secure } from '@konobangu/security';
-import { redirect } from 'next/navigation';
-import type { ReactNode } from 'react';
-import { PostHogIdentifier } from './components/posthog-identifier';
-import { GlobalSidebar } from './components/sidebar';
-
-type AppLayoutProperties = {
-  readonly children: ReactNode;
-};
-
-const AppLayout = async ({ children }: AppLayoutProperties) => {
-  if (env.ARCJET_KEY) {
-    await secure(['CATEGORY:PREVIEW']);
-  }
-
-  const { user } = await getSessionFromHeaders();
-
-  if (!user) {
-    return redirect('/sign-in'); // from next/navigation
-  }
-  const betaFeature = await showBetaFeature();
-
-  return (
-    <SidebarProvider>
-      <GlobalSidebar>
-        {betaFeature && (
-          <div className="m-4 rounded-full bg-success p-1.5 text-center text-sm text-success-foreground">
-            Beta feature now available
-          </div>
-        )}
-        {children}
-      </GlobalSidebar>
-      <PostHogIdentifier />
-    </SidebarProvider>
-  );
-};
-
-export default AppLayout;

apps/app/app/(authenticated)/page.tsx

@@ -1,57 +0,0 @@
-import { getSessionFromHeaders } from '@konobangu/auth/server';
-import { database } from '@konobangu/database';
-import { env } from '@konobangu/env';
-import type { Metadata } from 'next';
-import dynamic from 'next/dynamic';
-import { notFound } from 'next/navigation';
-import { AvatarStack } from './components/avatar-stack';
-import { Cursors } from './components/cursors';
-import { Header } from './components/header';
-
-const title = 'Acme Inc';
-const description = 'My application.';
-
-const CollaborationProvider = dynamic(() =>
-  import('./components/collaboration-provider').then(
-    (mod) => mod.CollaborationProvider
-  )
-);
-
-export const metadata: Metadata = {
-  title,
-  description,
-};
-
-const App = async () => {
-  const pages = await database.selectFrom('page').selectAll().execute();
-  const { orgId } = await getSessionFromHeaders();
-
-  if (!orgId) {
-    notFound();
-  }
-
-  return (
-    <>
-      <Header pages={['Building Your Application']} page="Data Fetching">
-        {env.LIVEBLOCKS_SECRET && (
-          <CollaborationProvider orgId={orgId}>
-            <AvatarStack />
-            <Cursors />
-          </CollaborationProvider>
-        )}
-      </Header>
-      <div className="flex flex-1 flex-col gap-4 p-4 pt-0">
-        <div className="grid auto-rows-min gap-4 md:grid-cols-3">
-          {pages.map((page) => (
-            <div key={page.id} className="aspect-video rounded-xl bg-muted/50">
-              {page.name}
-            </div>
-          ))}
-        </div>
-        <div className="min-h-[100vh] flex-1 rounded-xl bg-muted/50 md:min-h-min" />
-      </div>
-    </>
-  );
-};
-
-export default App;

apps/app/app/(authenticated)/webhooks/page.tsx

@@ -1,29 +0,0 @@
-import { webhooks } from '@konobangu/webhooks';
-import { notFound } from 'next/navigation';
-
-export const metadata = {
-  title: 'Webhooks',
-  description: 'Send webhooks to your users.',
-};
-
-const WebhooksPage = async () => {
-  const response = await webhooks.getAppPortal();
-
-  if (!response?.url) {
-    notFound();
-  }
-
-  return (
-    <div className="h-full w-full overflow-hidden">
-      <iframe
-        title="Webhooks"
-        src={response.url}
-        className="h-full w-full border-none"
-        allow="clipboard-write"
-        loading="lazy"
-      />
-    </div>
-  );
-};
-
-export default WebhooksPage;

apps/app/app/(unauthenticated)/layout.tsx

@@ -1,58 +0,0 @@
-import { ModeToggle } from '@konobangu/design-system/components/mode-toggle';
-import { env } from '@konobangu/env';
-import { CommandIcon } from 'lucide-react';
-import Link from 'next/link';
-import type { ReactNode } from 'react';
-
-type AuthLayoutProps = {
-  readonly children: ReactNode;
-};
-
-const AuthLayout = ({ children }: AuthLayoutProps) => (
-  <div className="container relative grid h-dvh flex-col items-center justify-center lg:max-w-none lg:grid-cols-2 lg:px-0">
-    <div className="relative hidden h-full flex-col bg-muted p-10 text-white lg:flex dark:border-r">
-      <div className="absolute inset-0 bg-zinc-900" />
-      <div className="relative z-20 flex items-center font-medium text-lg">
-        <CommandIcon className="mr-2 h-6 w-6" />
-        Acme Inc
-      </div>
-      <div className="absolute top-4 right-4">
-        <ModeToggle />
-      </div>
-      <div className="relative z-20 mt-auto">
-        <blockquote className="space-y-2">
-          <p className="text-lg">
-            &ldquo;This library has saved me countless hours of work and helped
-            me deliver stunning designs to my clients faster than ever
-            before.&rdquo;
-          </p>
-          <footer className="text-sm">Sofia Davis</footer>
-        </blockquote>
-      </div>
-    </div>
-    <div className="lg:p-8">
-      <div className="mx-auto flex w-full max-w-[400px] flex-col justify-center space-y-6">
-        {children}
-        <p className="px-8 text-center text-muted-foreground text-sm">
-          By clicking continue, you agree to our{' '}
-          <Link
-            href={new URL('/legal/terms', env.NEXT_PUBLIC_WEB_URL).toString()}
-            className="underline underline-offset-4 hover:text-primary"
-          >
-            Terms of Service
-          </Link>{' '}
-          and{' '}
-          <Link
-            href={new URL('/legal/privacy', env.NEXT_PUBLIC_WEB_URL).toString()}
-            className="underline underline-offset-4 hover:text-primary"
-          >
-            Privacy Policy
-          </Link>
-          .
-        </p>
-      </div>
-    </div>
-  </div>
-);
-
-export default AuthLayout;

apps/app/app/(unauthenticated)/sign-in/[[...sign-in]]/page.tsx

@@ -1,23 +0,0 @@
-import { createMetadata } from '@konobangu/seo/metadata';
-import type { Metadata } from 'next';
-import dynamic from 'next/dynamic';
-
-const title = 'Welcome back';
-const description = 'Enter your details to sign in.';
-const SignIn = dynamic(() =>
-  import('@konobangu/auth/components/sign-in').then((mod) => mod.SignIn)
-);
-
-export const metadata: Metadata = createMetadata({ title, description });
-
-const SignInPage = () => (
-  <>
-    <div className="flex flex-col space-y-2 text-center">
-      <h1 className="font-semibold text-2xl tracking-tight">{title}</h1>
-      <p className="text-muted-foreground text-sm">{description}</p>
-    </div>
-    <SignIn />
-  </>
-);
-
-export default SignInPage;

apps/app/app/(unauthenticated)/sign-up/[[...sign-up]]/page.tsx

@@ -1,23 +0,0 @@
-import { createMetadata } from '@konobangu/seo/metadata';
-import type { Metadata } from 'next';
-import dynamic from 'next/dynamic';
-
-const title = 'Create an account';
-const description = 'Enter your details to get started.';
-const SignUp = dynamic(() =>
-  import('@konobangu/auth/components/sign-up').then((mod) => mod.SignUp)
-);
-
-export const metadata: Metadata = createMetadata({ title, description });
-
-const SignUpPage = () => (
-  <>
-    <div className="flex flex-col space-y-2 text-center">
-      <h1 className="font-semibold text-2xl tracking-tight">{title}</h1>
-      <p className="text-muted-foreground text-sm">{description}</p>
-    </div>
-    <SignUp />
-  </>
-);
-
-export default SignUpPage;

apps/app/app/.well-known/vercel/flags/route.ts

@@ -1,3 +0,0 @@
-import { getFlags } from '@konobangu/feature-flags/access';
-
-export const GET = getFlags;

apps/app/app/actions/users/get.ts

@@ -1,63 +0,0 @@
-'use server';
-
-import {
-  getFullOrganizationFromSession,
-  getSessionFromHeaders,
-} from '@konobangu/auth/server';
-import { tailwind } from '@konobangu/tailwind-config';
-
-const colors = [
-  tailwind.theme.colors.red[500],
-  tailwind.theme.colors.orange[500],
-  tailwind.theme.colors.amber[500],
-  tailwind.theme.colors.yellow[500],
-  tailwind.theme.colors.lime[500],
-  tailwind.theme.colors.green[500],
-  tailwind.theme.colors.emerald[500],
-  tailwind.theme.colors.teal[500],
-  tailwind.theme.colors.cyan[500],
-  tailwind.theme.colors.sky[500],
-  tailwind.theme.colors.blue[500],
-  tailwind.theme.colors.indigo[500],
-  tailwind.theme.colors.violet[500],
-  tailwind.theme.colors.purple[500],
-  tailwind.theme.colors.fuchsia[500],
-  tailwind.theme.colors.pink[500],
-  tailwind.theme.colors.rose[500],
-];
-
-export const getUsers = async (
-  userIds: string[]
-): Promise<
-  | {
-      data: Liveblocks['UserMeta']['info'][];
-    }
-  | {
-      error: unknown;
-    }
-> => {
-  try {
-    const session = await getSessionFromHeaders();
-    const { orgId } = session;
-
-    if (!orgId) {
-      throw new Error('Not logged in');
-    }
-
-    const { fullOrganization } = await getFullOrganizationFromSession(session);
-
-    const members = fullOrganization?.members || [];
-
-    const data: Liveblocks['UserMeta']['info'][] = members
-      .filter((user) => user?.userId && userIds.includes(user?.userId))
-      .map((user) => ({
-        name: user.user.name ?? user.user.email ?? 'Unknown user',
-        picture: user.user.image,
-        color: colors[Math.floor(Math.random() * colors.length)],
-      }));
-
-    return { data };
-  } catch (error) {
-    return { error };
-  }
-};

apps/app/app/actions/users/search.ts

@@ -1,50 +0,0 @@
-'use server';
-
-import {
-  getFullOrganizationFromSession,
-  getSessionFromHeaders,
-} from '@konobangu/auth/server';
-import Fuse from 'fuse.js';
-
-export const searchUsers = async (
-  query: string
-): Promise<
-  | {
-      data: string[];
-    }
-  | {
-      error: unknown;
-    }
-> => {
-  try {
-    const session = await getSessionFromHeaders();
-    const { orgId } = session;
-
-    if (!orgId) {
-      throw new Error('Not logged in');
-    }
-
-    const { fullOrganization } = await getFullOrganizationFromSession(session);
-
-    const members = fullOrganization?.members || [];
-
-    const users = members.map((user) => ({
-      id: user.id,
-      name: user.user.name ?? user.user.email ?? 'Unknown user',
-      imageUrl: user.user.image,
-    }));
-
-    const fuse = new Fuse(users, {
-      keys: ['name'],
-      minMatchCharLength: 1,
-      threshold: 0.3,
-    });
-
-    const results = fuse.search(query);
-    const data = results.map((result) => result.item.id);
-
-    return { data };
-  } catch (error) {
-    return { error };
-  }
-};

apps/app/app/api/collaboration/auth/route.ts

@@ -1,42 +0,0 @@
-import { getSessionFromHeaders } from '@konobangu/auth/server';
-import { authenticate } from '@konobangu/collaboration/auth';
-import { tailwind } from '@konobangu/tailwind-config';
-
-const COLORS = [
-  tailwind.theme.colors.red[500],
-  tailwind.theme.colors.orange[500],
-  tailwind.theme.colors.amber[500],
-  tailwind.theme.colors.yellow[500],
-  tailwind.theme.colors.lime[500],
-  tailwind.theme.colors.green[500],
-  tailwind.theme.colors.emerald[500],
-  tailwind.theme.colors.teal[500],
-  tailwind.theme.colors.cyan[500],
-  tailwind.theme.colors.sky[500],
-  tailwind.theme.colors.blue[500],
-  tailwind.theme.colors.indigo[500],
-  tailwind.theme.colors.violet[500],
-  tailwind.theme.colors.purple[500],
-  tailwind.theme.colors.fuchsia[500],
-  tailwind.theme.colors.pink[500],
-  tailwind.theme.colors.rose[500],
-];
-
-export const POST = async () => {
-  const session = await getSessionFromHeaders();
-  const { orgId, user } = session;
-
-  if (!user || !orgId) {
-    return new Response('Unauthorized', { status: 401 });
-  }
-
-  return authenticate({
-    userId: user.id,
-    orgId,
-    userInfo: {
-      name: user.name ?? user.email ?? undefined,
-      avatar: user.image ?? undefined,
-      color: COLORS[Math.floor(Math.random() * COLORS.length)],
-    },
-  });
-};

apps/app/app/cron/keep-alive/route.ts

@@ -1,17 +0,0 @@
-import { database } from '@konobangu/database';
-
-export const POST = async () => {
-  const newPage = await database
-    .insertInto('page')
-    .values([
-      {
-        name: 'cron-temp',
-      },
-    ])
-    .returning('id')
-    .executeTakeFirstOrThrow();
-
-  await database.deleteFrom('page').where('id', '=', newPage.id);
-
-  return new Response('OK', { status: 200 });
-};

apps/app/app/global-error.tsx

@@ -1,29 +0,0 @@
-'use client';
-
-import { Button } from '@konobangu/design-system/components/ui/button';
-import { fonts } from '@konobangu/design-system/lib/fonts';
-import { captureException } from '@sentry/nextjs';
-import type NextError from 'next/error';
-import { useEffect } from 'react';
-
-type GlobalErrorProperties = {
-  readonly error: NextError & { digest?: string };
-  readonly reset: () => void;
-};
-
-const GlobalError = ({ error, reset }: GlobalErrorProperties) => {
-  useEffect(() => {
-    captureException(error);
-  }, [error]);
-
-  return (
-    <html lang="en" className={fonts}>
-      <body>
-        <h1>Oops, something went wrong</h1>
-        <Button onClick={() => reset()}>Try again</Button>
-      </body>
-    </html>
-  );
-};
-
-export default GlobalError;

apps/app/app/health/route.ts

@@ -1,3 +0,0 @@
-export const runtime = 'edge';
-
-export const GET = (): Response => new Response('OK', { status: 200 });

apps/app/app/layout.tsx

@@ -1,18 +0,0 @@
-import '@konobangu/design-system/styles/globals.css';
-import { DesignSystemProvider } from '@konobangu/design-system';
-import { fonts } from '@konobangu/design-system/lib/fonts';
-import type { ReactNode } from 'react';
-
-type RootLayoutProperties = {
-  readonly children: ReactNode;
-};
-
-const RootLayout = ({ children }: RootLayoutProperties) => (
-  <html lang="en" className={fonts} suppressHydrationWarning>
-    <body>
-      <DesignSystemProvider>{children}</DesignSystemProvider>
-    </body>
-  </html>
-);
-
-export default RootLayout;

apps/app/instrumentation.ts

@@ -1,3 +0,0 @@
-import { initializeSentry } from '@konobangu/next-config/instrumentation';
-
-export const register = initializeSentry();

apps/app/liveblocks.config.ts

@@ -1 +0,0 @@
-export * from '@konobangu/collaboration/config';

apps/app/middleware.ts

@@ -1,22 +0,0 @@
-import { authMiddleware } from '@konobangu/auth/middleware';
-import {
-  noseconeConfig,
-  noseconeMiddleware,
-} from '@konobangu/security/middleware';
-import type { NextRequest } from 'next/server';
-
-const securityHeaders = noseconeMiddleware(noseconeConfig);
-
-export async function middleware(_request: NextRequest) {
-  const response = await securityHeaders();
-  return authMiddleware(response as any);
-}
-
-export const config = {
-  matcher: [
-    // Skip Next.js internals and all static files, unless found in search params
-    '/((?!_next|[^?]*\\.(?:html?|css|js(?!on)|jpe?g|webp|png|gif|svg|ttf|woff2?|ico|csv|docx?|xlsx?|zip|webmanifest)).*)',
-    // Always run for API routes
-    '/(api|trpc)(.*)',
-  ],
-};

apps/app/next.config.ts

@@ -1,15 +0,0 @@
-import { env } from '@konobangu/env';
-import { config, withAnalyzer, withSentry } from '@konobangu/next-config';
-import type { NextConfig } from 'next';
-
-let nextConfig: NextConfig = { ...config };
-
-if (env.VERCEL) {
-  nextConfig = withSentry(nextConfig);
-}
-
-if (env.ANALYZE === 'true') {
-  nextConfig = withAnalyzer(nextConfig);
-}
-
-export default nextConfig;

apps/app/package.json

@@ -1,51 +0,0 @@
-{
-  "name": "app",
-  "private": true,
-  "scripts": {
-    "dev": "next dev -p 5000 --turbopack",
-    "build": "next build",
-    "start": "next start",
-    "analyze": "ANALYZE=true pnpm build",
-    "test": "vitest run",
-    "clean": "git clean -xdf .cache .turbo dist node_modules",
-    "typecheck": "tsc --noEmit --emitDeclarationOnly false"
-  },
-  "dependencies": {
-    "@konobangu/analytics": "workspace:*",
-    "@konobangu/auth": "workspace:*",
-    "@konobangu/collaboration": "workspace:*",
-    "@konobangu/database": "workspace:*",
-    "@konobangu/design-system": "workspace:*",
-    "@konobangu/env": "workspace:*",
-    "@konobangu/feature-flags": "workspace:*",
-    "@konobangu/migrate": "workspace:*",
-    "@konobangu/next-config": "workspace:*",
-    "@konobangu/security": "workspace:*",
-    "@konobangu/seo": "workspace:*",
-    "@konobangu/tailwind-config": "workspace:*",
-    "@konobangu/webhooks": "workspace:*",
-    "@prisma/client": "6.0.1",
-    "@sentry/nextjs": "^8.48.0",
-    "fuse.js": "^7.0.0",
-    "import-in-the-middle": "^1.12.0",
-    "lucide-react": "^0.468.0",
-    "next": "^15.1.4",
-    "next-themes": "^0.4.4",
-    "react": "^19.0.0",
-    "react-dom": "^19.0.0",
-    "require-in-the-middle": "^7.4.0"
-  },
-  "devDependencies": {
-    "@konobangu/testing": "workspace:*",
-    "@konobangu/typescript-config": "workspace:*",
-    "@testing-library/dom": "^10.4.0",
-    "@testing-library/react": "^16.1.0",
-    "@types/node": "22.10.1",
-    "@types/react": "19.0.1",
-    "@types/react-dom": "19.0.2",
-    "jsdom": "^25.0.1",
-    "tailwindcss": "^3.4.17",
-    "typescript": "^5.7.3",
-    "vitest": "^2.1.8"
-  }
-}

apps/app/postcss.config.mjs

@@ -1 +0,0 @@
-export { default } from '@konobangu/design-system/postcss.config.mjs';

apps/app/sentry.client.config.ts

@@ -1,34 +0,0 @@
-/*
- * This file configures the initialization of Sentry on the client.
- * The config you add here will be used whenever a users loads a page in their browser.
- * https://docs.sentry.io/platforms/javascript/guides/nextjs/
- */
-
-import { init, replayIntegration } from '@sentry/nextjs';
-
-init({
-  dsn: process.env.NEXT_PUBLIC_SENTRY_DSN,
-
-  // Adjust this value in production, or use tracesSampler for greater control
-  tracesSampleRate: 1,
-
-  // Setting this option to true will print useful information to the console while you're setting up Sentry.
-  debug: false,
-
-  replaysOnErrorSampleRate: 1,
-
-  /*
-   * This sets the sample rate to be 10%. You may want this to be 100% while
-   * in development and sample at a lower rate in production
-   */
-  replaysSessionSampleRate: 0.1,
-
-  // You can remove this option if you're not planning to use the Sentry Session Replay feature:
-  integrations: [
-    replayIntegration({
-      // Additional Replay configuration goes in here, for example:
-      maskAllText: true,
-      blockAllMedia: true,
-    }),
-  ],
-});

apps/app/tailwind.config.ts

@@ -1 +0,0 @@
-export { config as default } from '@konobangu/tailwind-config/config';

apps/app/tsconfig.json

@@ -1,17 +0,0 @@
-{
-  "extends": "@konobangu/typescript-config/nextjs.json",
-  "compilerOptions": {
-    "baseUrl": ".",
-    "paths": {
-      "@/*": ["./*"],
-      "@konobangu/*": ["../../packages/*"]
-    }
-  },
-  "include": [
-    "next-env.d.ts",
-    "next.config.ts",
-    "**/*.ts",
-    "**/*.tsx",
-    ".next/types/**/*.ts"
-  ]
-}

apps/app/vercel.json

@@ -1,8 +0,0 @@
-{
-  "crons": [
-    {
-      "path": "/cron/keep-alive",
-      "schedule": "0 1 * * *"
-    }
-  ]
-}

apps/app/vitest.config.ts

@@ -1 +0,0 @@
-export { default } from '@konobangu/testing';

apps/docs/package.json

@@ -4,8 +4,5 @@
   "scripts": {
     "dev": "npx --yes mintlify dev --port 5004",
     "lint": "npx --yes mintlify broken-links"
-  },
-  "devDependencies": {
-    "typescript": "^5.7.3"
   }
 }

apps/email-playground/package.json

@@ -6,19 +6,16 @@
     "build": "email build",
     "dev": "email dev --port 5003",
     "export": "email export",
-    "clean": "git clean -xdf .cache .turbo dist node_modules",
+    "clean": "git clean -xdf .cache dist node_modules",
     "typecheck": "tsc --noEmit --emitDeclarationOnly false"
   },
   "dependencies": {
-    "@konobangu/email": "workspace:*",
-    "@react-email/components": "0.0.31",
+    "@react-email/components": "^0.0.42",
     "react": "^19.0.0",
-    "react-email": "3.0.4"
+    "react-email": "^4.0.16",
+    "@konobangu/email": "workspace:*"
   },
   "devDependencies": {
-    "@konobangu/typescript-config": "workspace:*",
-    "@types/node": "22.10.1",
-    "@types/react": "19.0.1",
-    "typescript": "^5.7.3"
+    "@types/react": "19.0.1"
   }
 }

apps/email-playground/tsconfig.json

@@ -1,5 +1,13 @@
 {
-  "extends": "@konobangu/typescript-config/nextjs.json",
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "composite": true,
+    "jsx": "react-jsx",
+    "jsxImportSource": "react",
+    "module": "ESNext",
+    "moduleResolution": "bundler"
+  },
+  "references": [{ "path": "../../packages/email" }],
   "include": ["**/*.ts", "**/*.tsx"],
   "exclude": ["node_modules"]
 }

apps/proxy/.whistle/rules/files/0.konobangu

@@ -5,8 +5,7 @@
 }
 ```
 
-^https://konobangu.com/api/playground*** reqHeaders://{x-forwarded.json} http://127.0.0.1:5002/api/playground$1
-^wss://konobangu.com/api/playground*** reqHeaders://{x-forwarded.json} ws://127.0.0.1:5002/api/playground$1
-^https://konobangu.com/api*** reqHeaders://{x-forwarded.json} http://127.0.0.1:5001/api$1 excludeFilter://^^https://konobangu.com/api/playground***
-^https://konobangu.com*** reqHeaders://{x-forwarded.json} http://127.0.0.1:5000$1 excludeFilter://^https://konobangu.com/api***
-
+#^https://konobangu.com/api*** statusCode://500
+^https://konobangu.com/api*** reqHeaders://{x-forwarded.json} http://127.0.0.1:5001/api$1 
+^https://konobangu.com/*** reqHeaders://{x-forwarded.json} http://127.0.0.1:5000/$1 excludeFilter://^https://konobangu.com/api***
+^wss://konobangu.com/*** reqHeaders://{x-forwarded.json} ws://127.0.0.1:5000/$1 excludeFilter://^wss://konobangu.com/api

apps/proxy/.whistle/rules/files/1.mikan_doppel

@@ -0,0 +1 @@
+^https://mikanani.me/*** http://127.0.0.1:5005/$1 excludeFilter://^**/***.svg excludeFilter://^**/***.css excludeFilter://^**/***.js

apps/proxy/.whistle/rules/properties

@@ -1 +1 @@
-{"filesOrder":["konobangu"],"selectedList":["konobangu"],"disabledDefalutRules":true}
+{"filesOrder":["konobangu","mikan_doppel"],"selectedList":["konobangu","mikan_doppel"],"disabledDefalutRules":true,"defalutRules":""}

apps/proxy/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "proxy"
+version = "0.1.0"
+edition = "2024"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[lib]
+name = "proxy"
+path = "src/lib.rs"
+
+[[bin]]
+name = "mikan_doppel"
+path = "src/bin/mikan_doppel.rs"
+
+[dependencies]
+recorder = { workspace = true }
+tokio = { workspace = true }
+tracing-subscriber = { workspace = true }
+tracing = { workspace = true }

apps/proxy/package.json

@@ -3,13 +3,13 @@
   "version": "0.1.0",
   "private": true,
   "scripts": {
-    "start": "cross-env WHISTLE_MODE=\"prod|capture|keepXFF|x-forwarded-host|x-forwarded-proto\" whistle run -p 8899 -t 30000 -D .",
-    "dev": "pnpm run start"
+    "whistle": "cross-env WHISTLE_MODE=\"prod|capture|keepXFF|x-forwarded-host|x-forwarded-proto\" whistle run -p 8899 -t 30000 -D .",
+    "mikan_doppel": "cargo run -p proxy --bin mikan_doppel",
+    "dev": "npm-run-all -p mikan_doppel whistle"
   },
   "keywords": [],
   "license": "MIT",
   "devDependencies": {
-    "cross-env": "^7.0.3",
-    "whistle": "^2.9.93"
+    "whistle": "^2.9.99"
   }
 }

apps/proxy/src/bin/mikan_doppel.rs

@@ -0,0 +1,22 @@
+use std::time::Duration;
+
+use recorder::{errors::RecorderResult, test_utils::mikan::MikanMockServer};
+use tracing::Level;
+
+#[allow(unused_variables)]
+#[tokio::main]
+async fn main() -> RecorderResult<()> {
+    tracing_subscriber::fmt()
+        .with_max_level(Level::DEBUG)
+        .init();
+
+    let mut mikan_server = MikanMockServer::new_with_port(5005).await.unwrap();
+
+    let resources_mock = mikan_server.mock_resources_with_doppel();
+
+    let login_mock = mikan_server.mock_get_login_page();
+
+    loop {
+        tokio::time::sleep(Duration::from_secs(1)).await;
+    }
+}

apps/recorder/.devcontainer/Dockerfile

@@ -1,8 +0,0 @@
-FROM mcr.microsoft.com/vscode/devcontainers/rust:0-1
-
-RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-     && apt-get -y install --no-install-recommends postgresql-client \
-     && cargo install sea-orm-cli cargo-insta \
-     && chown -R vscode /usr/local/cargo
-
-COPY .env /.env

apps/recorder/.devcontainer/devcontainer.json

@@ -1,7 +0,0 @@
-{
-  "name": "Konobangu Recorder",
-  "dockerComposeFile": "docker-compose.yml",
-  "service": "app",
-  "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
-  "forwardPorts": [5001]
-}

apps/recorder/.devcontainer/docker-compose.yml

@@ -1,40 +0,0 @@
-version: "3"
-
-services:
-  app:
-    build:
-      context: .
-      dockerfile: Dockerfile
-    command: sleep infinity
-    networks:
-      - db
-      - redis
-    volumes:
-      - ../..:/workspaces:cached
-    env_file:
-      - .env
-  db:
-    image: postgres:15.3-alpine
-    restart: unless-stopped
-    ports:
-      - 5432:5432
-    networks:
-      - db
-    volumes:
-      - postgres-data:/var/lib/postgresql/data
-    env_file:
-      - .env
-  redis:
-    image: redis:latest
-    restart: unless-stopped
-    ports:
-      - 6379:6379
-    networks:
-      - redis
-
-volumes:
-  postgres-data:
-
-networks:
-  db:
-  redis:

apps/recorder/.env

@@ -0,0 +1,17 @@
+HOST="konobangu.com"
+DATABASE_URL = "postgres://konobangu:konobangu@localhost:5432/konobangu"
+STORAGE_DATA_DIR = "./data"
+AUTH_TYPE = "basic" # or oidc
+BASIC_USER = "konobangu"
+BASIC_PASSWORD = "konobangu"
+# OIDC_ISSUER="https://auth.logto.io/oidc"
+# OIDC_AUDIENCE = "https://konobangu.com/api"
+# OIDC_CLIENT_ID = "client_id"
+# OIDC_CLIENT_SECRET = "client_secret" # optional
+# OIDC_EXTRA_SCOPES = "read:konobangu write:konobangu"
+# OIDC_EXTRA_CLAIM_KEY = ""
+# OIDC_EXTRA_CLAIM_VALUE = ""
+# MIKAN_PROXY = ""
+# MIKAN_PROXY_AUTH_HEADER = ""
+# MIKAN_NO_PROXY = ""
+# MIKAN_PROXY_ACCEPT_INVALID_CERTS = "true"

apps/recorder/.env.dev

@@ -0,0 +1,17 @@
+HOST="konobangu.com"
+DATABASE_URL = "postgres://konobangu:konobangu@localhost:5432/konobangu"
+STORAGE_DATA_DIR = "./data"
+AUTH_TYPE = "basic" # or oidc
+BASIC_USER = "konobangu"
+BASIC_PASSWORD = "konobangu"
+# OIDC_ISSUER="https://auth.logto.io/oidc"
+# OIDC_AUDIENCE = "https://konobangu.com/api"
+# OIDC_CLIENT_ID = "client_id"
+# OIDC_CLIENT_SECRET = "client_secret" # optional
+# OIDC_EXTRA_SCOPES = "read:konobangu write:konobangu"
+# OIDC_EXTRA_CLAIM_KEY = ""
+# OIDC_EXTRA_CLAIM_VALUE = ""
+MIKAN_PROXY = "http://127.0.0.1:8899"
+# MIKAN_PROXY_AUTH_HEADER = ""
+# MIKAN_NO_PROXY = ""
+MIKAN_PROXY_ACCEPT_INVALID_CERTS = true

apps/recorder/.gitignore

@@ -25,3 +25,7 @@ Cargo.lock
 # Dist
 node_modules
 dist/
+temp/*
+!temp/.gitkeep
+tests/resources/mikan/classic_episodes/*/*
+!tests/resources/mikan/classic_episodes/parquet/tiny.parquet

apps/recorder/Cargo.toml

@@ -2,8 +2,20 @@
 name = "recorder"
 version = "0.1.0"
 edition = "2024"
-
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[features]
+default = ["jxl"]
+playground = ["dep:inquire", "dep:color-eyre", "dep:polars"]
+testcontainers = [
+    "dep:testcontainers",
+    "dep:testcontainers-modules",
+    "dep:testcontainers-ext",
+    "downloader/testcontainers",
+    "testcontainers-modules/postgres",
+]
+jxl = ["dep:jpegxl-rs", "dep:jpegxl-sys"]
+
 [lib]
 name = "recorder"
 path = "src/lib.rs"
@@ -13,89 +25,108 @@ name = "recorder_cli"
 path = "src/bin/main.rs"
 required-features = []
 
-[features]
-default = []
-testcontainers = [
-    "dep:testcontainers",
-    "dep:testcontainers-modules",
-    "dep:bollard",
-]
+[[example]]
+name = "mikan_collect_classic_eps"
+path = "examples/mikan_collect_classic_eps.rs"
+required-features = ["playground"]
+
+[[example]]
+name = "mikan_doppel_season_subscription"
+path = "examples/mikan_doppel_season_subscription.rs"
+required-features = ["playground"]
+
+[[example]]
+name = "mikan_doppel_subscriber_subscription"
+path = "examples/mikan_doppel_subscriber_subscription.rs"
+required-features = ["playground"]
+
+[[example]]
+name = "playground"
+path = "examples/playground.rs"
+required-features = ["playground"]
 
 [dependencies]
-serde = { version = "1", features = ["derive"] }
-serde_json = "1"
-tokio = { version = "1.42", features = ["macros", "fs", "rt-multi-thread"] }
-async-trait = "0.1.83"
-tracing = "0.1"
-chrono = "0.4"
+downloader = { workspace = true }
+util = { workspace = true }
+util-derive = { workspace = true }
+fetch = { workspace = true }
+
+serde = { workspace = true }
+tokio = { workspace = true }
+serde_json = { workspace = true }
+async-trait = { workspace = true }
+testcontainers = { workspace = true, optional = true }
+testcontainers-modules = { workspace = true, optional = true }
+testcontainers-ext = { workspace = true, optional = true, features = [
+    "tracing",
+] }
+tracing = { workspace = true }
+axum = { workspace = true }
+axum-extra = { workspace = true }
+snafu = { workspace = true }
+itertools = { workspace = true }
+url = { workspace = true }
+regex = { workspace = true }
+lazy_static = { workspace = true }
+quirks_path = { workspace = true }
+futures = { workspace = true }
+bytes = { workspace = true }
+serde_with = { workspace = true }
+moka = { workspace = true }
+chrono = { workspace = true }
+tracing-subscriber = { workspace = true }
+mockito = { workspace = true }
+color-eyre = { workspace = true, optional = true }
+inquire = { workspace = true, optional = true }
+convert_case = { workspace = true }
+image = { workspace = true }
+uuid = { workspace = true }
+maplit = { workspace = true }
+once_cell = { workspace = true }
+rand = { workspace = true }
+rust_decimal = { workspace = true }
+base64 = { workspace = true }
+nom = { workspace = true }
+percent-encoding = { workspace = true }
+num-traits = { workspace = true }
+http = { workspace = true }
+async-stream = { workspace = true }
+serde_variant = { workspace = true }
+tracing-appender = { workspace = true }
+clap = { workspace = true }
+ipnetwork = { workspace = true }
+typed-builder = { workspace = true }
+nanoid = { workspace = true }
+webp = { workspace = true }
+
 sea-orm = { version = "1.1", features = [
     "sqlx-sqlite",
     "sqlx-postgres",
-    "runtime-tokio-rustls",
+    "runtime-tokio",
     "macros",
     "debug-print",
 ] }
 figment = { version = "0.10", features = ["toml", "json", "env", "yaml"] }
-
-axum = "0.8"
-uuid = { version = "1.6.0", features = ["v4"] }
-tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
-sea-orm-migration = { version = "1.1", features = ["runtime-tokio-rustls"] }
-reqwest = { version = "0.12", default-features = false, features = [
-    "charset",
-    "http2",
-    "json",
-    "macos-system-configuration",
-    "rustls-tls",
-    "cookies",
-] }
-thiserror = "2"
+sea-orm-migration = { version = "1.1", features = ["runtime-tokio"] }
 rss = "2"
-bytes = "1.9"
-itertools = "0.14"
-url = "2.5"
 fancy-regex = "0.14"
-regex = "1.11"
-lazy_static = "1.5"
-maplit = "1.0.2"
-lightningcss = "1.0.0-alpha.61"
+lightningcss = "1.0.0-alpha.66"
 html-escape = "0.2.13"
-opendal = { version = "0.51.0", features = ["default", "services-fs"] }
-zune-image = "0.4.15"
-once_cell = "1.20.2"
-reqwest-middleware = "0.4.0"
-reqwest-retry = "0.7.0"
-reqwest-tracing = "0.5.5"
-scraper = "0.22.0"
-leaky-bucket = "1.1.2"
-serde_with = "3"
-jwt-authorizer = "0.15.0"
-futures = "0.3.31"
-librqbit-core = "4"
-qbit-rs = { git = "https://github.com/lonelyhentxi/qbit.git", rev = "72d53138ebe", features = [
-    "default",
-    "builder",
+opendal = { version = "0.53", features = ["default", "services-fs"] }
+scraper = "0.23.1"
+async-graphql = { version = "7", features = ["dynamic-schema"] }
+async-graphql-axum = "7"
+seaography = { version = "1.1", features = [
+    "with-json",
+    "with-chrono",
+    "with-time",
+    "with-uuid",
+    "with-decimal",
+    "with-bigdecimal",
+    "with-postgres-array",
+    "with-json-as-scalar",
 ] }
-testcontainers = { version = "0.23.3", features = [
-    "default",
-    "properties-config",
-    "watchdog",
-    "http_wait",
-    "reusable-containers",
-], optional = true }
-testcontainers-modules = { version = "0.11.4", optional = true }
-color-eyre = "0.6"
-log = "0.4.22"
-anyhow = "1.0.95"
-bollard = { version = "0.18", optional = true }
-async-graphql = { version = "7.0.15", features = [] }
-async-graphql-axum = "7.0.15"
-fastrand = "2.3.0"
-seaography = { version = "1.1" }
-quirks_path = "0.1.1"
-base64 = "0.22.1"
 tower = "0.5.2"
-axum-extra = "0.10"
 tower-http = { version = "0.6", features = [
     "trace",
     "catch-panic",
@@ -106,34 +137,33 @@ tower-http = { version = "0.6", features = [
     "set-header",
     "compression-full",
 ] }
-serde_yaml = "0.9.34"
 tera = "1.20.0"
-openidconnect = { version = "4", features = ["rustls-tls"] }
-http-cache-reqwest = { version = "0.15", features = [
-    "manager-cacache",
-    "manager-moka",
-] }
-moka = "0.12.10"
-http-cache = { version = "0.20.0", features = [
-    "cacache-tokio",
-    "manager-cacache",
-    "manager-moka",
-], default-features = false }
-http-cache-semantics = "2.1.0"
-dotenv = "0.15.0"
-nom = "8.0.0"
-secrecy = { version = "0.10.3", features = ["serde"] }
-http = "1.2.0"
-cookie = "0.18.1"
-async-stream = "0.3.6"
-serde_variant = "0.1.3"
-tracing-appender = "0.2.3"
-clap = "4.5.31"
-futures-util = "0.3.31"
-ipnetwork = "0.21.1"
+openidconnect = { version = "4" }
+dotenvy = "0.15.7"
+jpegxl-rs = { version = "0.11.2", optional = true }
+jpegxl-sys = { version = "0.11.2", optional = true }
+
+apalis = { version = "0.7", features = ["limit", "tracing", "catch-panic"] }
+apalis-sql = { version = "0.7", features = ["postgres"] }
+cocoon = { version = "0.4.3", features = ["getrandom", "thiserror"] }
+reqwest_cookie_store = "0.8.0"
+jwtk = "0.4.0"
+mime_guess = "2.0.5"
+icu_properties = "2.0.1"
+icu = "2.0.0"
+tracing-tree = "0.4.0"
+num_cpus = "1.17.0"
+headers-accept = "0.1.4"
+polars = { version = "0.49.1", features = [
+    "parquet",
+    "lazy",
+    "diagonal_concat",
+], optional = true }
 
 [dev-dependencies]
+inquire = { workspace = true }
+color-eyre = { workspace = true }
 serial_test = "3"
-insta = { version = "1", features = ["redactions", "yaml", "filters"] }
-mockito = "1.6.1"
-rstest = "0.24.0"
+insta = { version = "1", features = ["redactions", "toml", "filters"] }
+rstest = "0.25"
+ctor = "0.4.0"

apps/recorder/examples/mikan_collect_classic_eps.rs

@@ -0,0 +1,584 @@
+use std::collections::HashSet;
+
+use chrono::{DateTime, Duration, FixedOffset, NaiveDate, NaiveTime, TimeZone, Utc};
+use fetch::{HttpClientConfig, fetch_html};
+use itertools::Itertools;
+use lazy_static::lazy_static;
+use nom::{
+    IResult, Parser,
+    branch::alt,
+    bytes::complete::{tag, take, take_till1},
+    character::complete::space1,
+    combinator::map,
+};
+use recorder::{
+    errors::{RecorderError, RecorderResult},
+    extract::{
+        html::extract_inner_text_from_element_ref,
+        mikan::{MikanClient, MikanConfig, MikanEpisodeHash, MikanFansubHash},
+    },
+};
+use regex::Regex;
+use scraper::{ElementRef, Html, Selector};
+use snafu::FromString;
+use url::Url;
+
+lazy_static! {
+    static ref TEST_FOLDER: std::path::PathBuf =
+        if cfg!(any(test, debug_assertions, feature = "playground")) {
+            std::path::PathBuf::from(format!(
+                "{}/tests/resources/mikan/classic_episodes",
+                env!("CARGO_MANIFEST_DIR")
+            ))
+        } else {
+            std::path::PathBuf::from("tests/resources/mikan/classic_episodes")
+        };
+}
+
+lazy_static! {
+    static ref TOTAL_PAGE_REGEX: Regex =
+        Regex::new(r#"\$\(\'\.classic-view-pagination2\'\)\.bootpag\(\{\s*total:\s*(\d+)"#)
+            .unwrap();
+}
+
+pub struct MikanClassicEpisodeTableRow {
+    pub id: i32,
+    pub publish_at: DateTime<Utc>,
+    pub mikan_fansub_id: Option<String>,
+    pub fansub_name: Option<String>,
+    pub mikan_episode_id: String,
+    pub original_name: String,
+    pub magnet_link: Option<String>,
+    pub file_size: Option<String>,
+    pub torrent_link: Option<String>,
+}
+
+impl MikanClassicEpisodeTableRow {
+    fn timezone() -> FixedOffset {
+        FixedOffset::east_opt(8 * 3600).unwrap()
+    }
+
+    fn fixed_date_parser(input: &str) -> IResult<&str, NaiveDate> {
+        alt((
+            map(tag("今天"), move |_| {
+                Utc::now().with_timezone(&Self::timezone()).date_naive()
+            }),
+            map(tag("昨天"), move |_| {
+                Utc::now().with_timezone(&Self::timezone()).date_naive() - Duration::days(1)
+            }),
+        ))
+        .parse(input)
+    }
+
+    fn formatted_date_parser(input: &str) -> IResult<&str, NaiveDate> {
+        let (remain, date_str) = take_till1(|c: char| c.is_whitespace()).parse(input)?;
+        let date = NaiveDate::parse_from_str(date_str, "%Y/%m/%d").map_err(|_| {
+            nom::Err::Error(nom::error::Error::new(input, nom::error::ErrorKind::Verify))
+        })?;
+        Ok((remain, date))
+    }
+
+    fn date_parser(input: &str) -> IResult<&str, NaiveDate> {
+        alt((Self::fixed_date_parser, Self::formatted_date_parser)).parse(input)
+    }
+
+    fn time_parser(input: &str) -> IResult<&str, NaiveTime> {
+        let (remain, time_str) = take(5usize).parse(input)?;
+        let time = NaiveTime::parse_from_str(time_str, "%H:%M").map_err(|_| {
+            nom::Err::Error(nom::error::Error::new(input, nom::error::ErrorKind::Verify))
+        })?;
+        Ok((remain, time))
+    }
+
+    fn extract_publish_at(text: &str) -> Option<DateTime<Utc>> {
+        let (_, (date, _, time)) = (Self::date_parser, space1, Self::time_parser)
+            .parse(text)
+            .ok()?;
+        let local_dt = Self::timezone()
+            .from_local_datetime(&date.and_time(time))
+            .single()?;
+        Some(local_dt.with_timezone(&Utc))
+    }
+
+    pub fn from_element_ref(
+        row: ElementRef<'_>,
+        rev_id: i32,
+        idx: i32,
+        mikan_base_url: &Url,
+    ) -> RecorderResult<Self> {
+        let publish_at_selector = &Selector::parse("td:nth-of-type(1)").unwrap();
+        let fansub_selector = &Selector::parse("td:nth-of-type(2) > a").unwrap();
+        let original_name_selector =
+            &Selector::parse("td:nth-of-type(3) > a:nth-of-type(1)").unwrap();
+        let magnet_link_selector =
+            &Selector::parse("td:nth-of-type(3) > a:nth-of-type(2)").unwrap();
+        let file_size_selector = &Selector::parse("td:nth-of-type(4)").unwrap();
+        let torrent_link_selector = &Selector::parse("td:nth-of-type(5) > a").unwrap();
+
+        let publish_at = row
+            .select(publish_at_selector)
+            .next()
+            .map(extract_inner_text_from_element_ref)
+            .and_then(|e| Self::extract_publish_at(&e));
+
+        let (mikan_fansub_hash, fansub_name) = row
+            .select(fansub_selector)
+            .next()
+            .and_then(|e| {
+                e.attr("href")
+                    .and_then(|s| mikan_base_url.join(s).ok())
+                    .and_then(|u| MikanFansubHash::from_homepage_url(&u))
+                    .map(|h| (h, extract_inner_text_from_element_ref(e)))
+            })
+            .unzip();
+
+        let (mikan_episode_hash, original_name) = row
+            .select(original_name_selector)
+            .next()
+            .and_then(|el| {
+                el.attr("href")
+                    .and_then(|s| mikan_base_url.join(s).ok())
+                    .and_then(|u| MikanEpisodeHash::from_homepage_url(&u))
+                    .map(|h| (h, extract_inner_text_from_element_ref(el)))
+            })
+            .unzip();
+
+        let magnet_link = row
+            .select(magnet_link_selector)
+            .next()
+            .and_then(|el| el.attr("data-clipboard-text"));
+
+        let file_size = row
+            .select(file_size_selector)
+            .next()
+            .map(extract_inner_text_from_element_ref);
+
+        let torrent_link = row
+            .select(torrent_link_selector)
+            .next()
+            .and_then(|el| el.attr("href"));
+
+        if let (Some(mikan_episode_hash), Some(original_name), Some(publish_at)) = (
+            mikan_episode_hash.as_ref(),
+            original_name.as_ref(),
+            publish_at.as_ref(),
+        ) {
+            Ok(Self {
+                id: rev_id * 1000 + idx,
+                publish_at: *publish_at,
+                mikan_fansub_id: mikan_fansub_hash.map(|h| h.mikan_fansub_id.clone()),
+                fansub_name,
+                mikan_episode_id: mikan_episode_hash.mikan_episode_id.clone(),
+                original_name: original_name.clone(),
+                magnet_link: magnet_link.map(|s| s.to_string()),
+                file_size: file_size.map(|s| s.to_string()),
+                torrent_link: torrent_link.map(|s| s.to_string()),
+            })
+        } else {
+            let mut missing_fields = vec![];
+            if mikan_episode_hash.is_none() {
+                missing_fields.push("mikan_episode_id");
+            }
+            if original_name.is_none() {
+                missing_fields.push("original_name");
+            }
+            if publish_at.is_none() {
+                missing_fields.push("publish_at");
+            }
+            Err(RecorderError::without_source(format!(
+                "Failed to parse episode table row, missing fields: {missing_fields:?}, row \
+                 index: {idx}"
+            )))
+        }
+    }
+}
+
+pub struct MikanClassicEpisodeTablePage {
+    pub page: i32,
+    pub total: i32,
+    pub html: String,
+    pub rows: Vec<MikanClassicEpisodeTableRow>,
+}
+
+impl MikanClassicEpisodeTablePage {
+    pub fn from_html(
+        html: String,
+        mikan_base_url: &Url,
+        page: i32,
+        updated_info: Option<(i32, i32)>,
+    ) -> RecorderResult<Self> {
+        let tr_selector = &Selector::parse("tbody tr").unwrap();
+        let doc = Html::parse_document(&html);
+        if let Some(mut total) = TOTAL_PAGE_REGEX
+            .captures(&html)
+            .and_then(|c| c.get(1))
+            .and_then(|s| s.as_str().parse::<i32>().ok())
+        {
+            if let Some((_, update_total)) = updated_info {
+                total = update_total;
+            }
+
+            let rev_id = total - page;
+            let rows = doc
+                .select(tr_selector)
+                .rev()
+                .enumerate()
+                .map(|(idx, tr)| {
+                    MikanClassicEpisodeTableRow::from_element_ref(
+                        tr,
+                        rev_id,
+                        idx as i32,
+                        mikan_base_url,
+                    )
+                })
+                .collect::<RecorderResult<Vec<_>>>()?;
+            Ok(Self {
+                page,
+                total,
+                html,
+                rows,
+            })
+        } else {
+            Err(RecorderError::without_source(
+                "Failed to parse pagination meta and rows".into(),
+            ))
+        }
+    }
+
+    pub fn save_to_files(&self) -> RecorderResult<()> {
+        use polars::prelude::*;
+
+        let rev_id = self.total - self.page;
+        let parquet_path = TEST_FOLDER.join(format!("parquet/rev_{rev_id}.parquet"));
+        let csv_path = TEST_FOLDER.join(format!("csv/rev_{rev_id}.csv"));
+        let html_path = TEST_FOLDER.join(format!("html/rev_{rev_id}.html"));
+
+        std::fs::write(html_path, self.html.clone())?;
+
+        let mut id_vec = Vec::new();
+        let mut publish_at_vec = Vec::new();
+        let mut mikan_fansub_id_vec = Vec::new();
+        let mut fansub_name_vec = Vec::new();
+        let mut mikan_episode_id_vec = Vec::new();
+        let mut original_name_vec = Vec::new();
+        let mut magnet_link_vec = Vec::new();
+        let mut file_size_vec = Vec::new();
+        let mut torrent_link_vec = Vec::new();
+
+        for row in &self.rows {
+            id_vec.push(row.id);
+            publish_at_vec.push(row.publish_at.to_rfc3339());
+            mikan_fansub_id_vec.push(row.mikan_fansub_id.clone());
+            fansub_name_vec.push(row.fansub_name.clone());
+            mikan_episode_id_vec.push(row.mikan_episode_id.clone());
+            original_name_vec.push(row.original_name.clone());
+            magnet_link_vec.push(row.magnet_link.clone());
+            file_size_vec.push(row.file_size.clone());
+            torrent_link_vec.push(row.torrent_link.clone());
+        }
+
+        let df = df! [
+            "id" => id_vec,
+            "publish_at_timestamp" => publish_at_vec,
+            "mikan_fansub_id" => mikan_fansub_id_vec,
+            "fansub_name" => fansub_name_vec,
+            "mikan_episode_id" => mikan_episode_id_vec,
+            "original_name" => original_name_vec,
+            "magnet_link" => magnet_link_vec,
+            "file_size" => file_size_vec,
+            "torrent_link" => torrent_link_vec,
+        ]
+        .map_err(|e| {
+            let message = format!("Failed to create DataFrame: {e}");
+            RecorderError::with_source(Box::new(e), message)
+        })?;
+
+        let mut parquet_file = std::fs::File::create(&parquet_path)?;
+
+        ParquetWriter::new(&mut parquet_file)
+            .finish(&mut df.clone())
+            .map_err(|e| {
+                let message = format!("Failed to write parquet file: {e}");
+                RecorderError::with_source(Box::new(e), message)
+            })?;
+
+        let mut csv_file = std::fs::File::create(&csv_path)?;
+
+        CsvWriter::new(&mut csv_file)
+            .include_header(true)
+            .with_quote_style(QuoteStyle::Always)
+            .finish(&mut df.clone())
+            .map_err(|e| {
+                let message = format!("Failed to write csv file: {e}");
+                RecorderError::with_source(Box::new(e), message)
+            })?;
+
+        println!(
+            "[{}/{}] Saved {} rows to rev_{}.{{parquet,html,csv}}",
+            self.page,
+            self.total,
+            self.rows.len(),
+            rev_id
+        );
+
+        Ok(())
+    }
+
+    pub fn waiting_rev_ids(total: i32) -> RecorderResult<Vec<i32>> {
+        let dir = TEST_FOLDER.join("csv");
+
+        let files = std::fs::read_dir(dir)?;
+
+        let rev_ids = files
+            .filter_map(|f| f.ok())
+            .filter_map(|f| {
+                f.path().file_stem().and_then(|s| {
+                    s.to_str().and_then(|s| {
+                        if s.starts_with("rev_") {
+                            s.replace("rev_", "").parse::<i32>().ok()
+                        } else {
+                            None
+                        }
+                    })
+                })
+            })
+            .collect::<HashSet<_>>();
+
+        Ok((0..total)
+            .filter(|rev_id| !rev_ids.contains(rev_id))
+            .collect::<Vec<_>>())
+    }
+}
+
+async fn scrape_mikan_classic_episode_table_page(
+    mikan_client: &MikanClient,
+    page: i32,
+    updated_info: Option<(i32, i32)>,
+) -> RecorderResult<MikanClassicEpisodeTablePage> {
+    let mikan_base_url = mikan_client.base_url();
+    let url = mikan_base_url.join(&format!("/Home/Classic/{page}"))?;
+
+    if let Some((rev_id, update_total)) = updated_info.as_ref() {
+        let html_path = TEST_FOLDER.join(format!("html/rev_{rev_id}.html"));
+        if html_path.exists() {
+            let html = std::fs::read_to_string(&html_path)?;
+            println!("[{page}/{update_total}] html exists, skipping fetch");
+            return MikanClassicEpisodeTablePage::from_html(
+                html,
+                mikan_base_url,
+                page,
+                updated_info,
+            );
+        }
+    }
+
+    let total = if let Some((_, update_total)) = updated_info.as_ref() {
+        update_total.to_string()
+    } else {
+        "Unknown".to_string()
+    };
+
+    println!("[{page}/{total}] fetching html...");
+
+    let html = fetch_html(mikan_client, url).await?;
+
+    println!("[{page}/{total}] fetched html done");
+
+    std::fs::write(TEST_FOLDER.join("html/temp.html"), html.clone())?;
+
+    MikanClassicEpisodeTablePage::from_html(html, mikan_base_url, page, updated_info)
+}
+
+async fn scrape_mikan_classic_episode_table_page_from_rev_id(
+    mikan_client: &MikanClient,
+    total: i32,
+    rev_idx: i32,
+) -> RecorderResult<MikanClassicEpisodeTablePage> {
+    let page = total - rev_idx;
+
+    scrape_mikan_classic_episode_table_page(mikan_client, page, Some((rev_idx, total))).await
+}
+
+async fn merge_mikan_classic_episodes_and_strip_columns() -> RecorderResult<()> {
+    use polars::prelude::*;
+
+    let dir = TEST_FOLDER.join("parquet");
+    let files = std::fs::read_dir(dir)?;
+
+    let parquet_paths = files
+        .filter_map(|f| f.ok())
+        .filter_map(|f| {
+            let path = f.path();
+            if let Some(ext) = path.extension()
+                && ext == "parquet"
+                && path
+                    .file_stem()
+                    .is_some_and(|f| f.to_string_lossy().starts_with("rev_"))
+            {
+                Some(path)
+            } else {
+                None
+            }
+        })
+        .collect::<Vec<_>>();
+
+    if parquet_paths.is_empty() {
+        return Err(RecorderError::without_source(
+            "No parquet files found to merge".into(),
+        ));
+    }
+
+    println!("Found {} parquet files to merge", parquet_paths.len());
+
+    // 读取并合并所有 parquet 文件
+    let mut all_dfs = Vec::new();
+    for path in &parquet_paths {
+        println!("Reading {path:?}");
+        let file = std::fs::File::open(path)?;
+        let df = ParquetReader::new(file).finish().map_err(|e| {
+            let message = format!("Failed to read parquet file {path:?}: {e}");
+            RecorderError::with_source(Box::new(e), message)
+        })?;
+        all_dfs.push(df);
+    }
+
+    let lazy_frames: Vec<LazyFrame> = all_dfs.into_iter().map(|df| df.lazy()).collect();
+
+    let merged_df = concat_lf_diagonal(&lazy_frames, UnionArgs::default())
+        .map_err(|e| {
+            let message = format!("Failed to concat DataFrames: {e}");
+            RecorderError::with_source(Box::new(e), message)
+        })?
+        .sort(
+            ["publish_at_timestamp"],
+            SortMultipleOptions::default().with_order_descending(true),
+        )
+        .unique(
+            Some(vec![
+                "mikan_fansub_id".to_string(),
+                "mikan_episode_id".to_string(),
+            ]),
+            UniqueKeepStrategy::First,
+        )
+        .collect()
+        .map_err(|e| {
+            let message = format!("Failed to collect lazy DataFrame: {e}");
+            RecorderError::with_source(Box::new(e), message)
+        })?;
+
+    fn select_columns_and_write(
+        merged_df: DataFrame,
+        name: &str,
+        columns: &[&str],
+    ) -> RecorderResult<()> {
+        let result_df = merged_df
+            .lazy()
+            .sort(["publish_at_timestamp"], SortMultipleOptions::default())
+            .select(columns.iter().map(|c| col(*c)).collect_vec())
+            .collect()
+            .map_err(|e| {
+                let message = format!("Failed to sort and select columns: {e}");
+                RecorderError::with_source(Box::new(e), message)
+            })?;
+
+        let output_path = TEST_FOLDER.join(format!("parquet/{name}.parquet"));
+        let mut output_file = std::fs::File::create(&output_path)?;
+
+        ParquetWriter::new(&mut output_file)
+            .set_parallel(true)
+            .with_compression(ParquetCompression::Zstd(Some(
+                ZstdLevel::try_new(22).unwrap(),
+            )))
+            .finish(&mut result_df.clone())
+            .map_err(|e| {
+                let message = format!("Failed to write merged parquet file: {e}");
+                RecorderError::with_source(Box::new(e), message)
+            })?;
+
+        println!("Merged {} rows into {output_path:?}", result_df.height());
+        Ok(())
+    }
+
+    select_columns_and_write(merged_df.clone(), "tiny", &["fansub_name", "original_name"])?;
+    // select_columns_and_write(
+    //     merged_df.clone(),
+    //     "lite",
+    //     &[
+    //         "mikan_fansub_id",
+    //         "fansub_name",
+    //         "mikan_episode_id",
+    //         "original_name",
+    //     ],
+    // )?;
+    // select_columns_and_write(
+    //     merged_df,
+    //     "full",
+    //     &[
+    //         "id",
+    //         "publish_at_timestamp",
+    //         "mikan_fansub_id",
+    //         "fansub_name",
+    //         "mikan_episode_id",
+    //         "original_name",
+    //         "magnet_link",
+    //         "file_size",
+    //         "torrent_link",
+    //     ],
+    // )?;
+
+    Ok(())
+}
+
+#[tokio::main]
+async fn main() -> RecorderResult<()> {
+    std::fs::create_dir_all(TEST_FOLDER.join("html"))?;
+    std::fs::create_dir_all(TEST_FOLDER.join("parquet"))?;
+    std::fs::create_dir_all(TEST_FOLDER.join("csv"))?;
+
+    let mikan_scrape_client = MikanClient::from_config(MikanConfig {
+        http_client: HttpClientConfig {
+            exponential_backoff_max_retries: Some(3),
+            leaky_bucket_max_tokens: Some(2),
+            leaky_bucket_initial_tokens: Some(1),
+            leaky_bucket_refill_tokens: Some(1),
+            leaky_bucket_refill_interval: Some(std::time::Duration::from_millis(1000)),
+            user_agent: Some(
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) \
+                 Chrome/136.0.0.0 Safari/537.36 Edg/136.0.0.0"
+                    .to_string(),
+            ),
+            ..Default::default()
+        },
+        base_url: Url::parse("https://mikanani.me")?,
+    })
+    .await?;
+
+    let first_page_and_pagination_info =
+        scrape_mikan_classic_episode_table_page(&mikan_scrape_client, 1, None).await?;
+
+    let total_page = first_page_and_pagination_info.total;
+
+    first_page_and_pagination_info.save_to_files()?;
+
+    let next_rev_ids = MikanClassicEpisodeTablePage::waiting_rev_ids(total_page)?;
+
+    for todo_rev_id in next_rev_ids {
+        let page = scrape_mikan_classic_episode_table_page_from_rev_id(
+            &mikan_scrape_client,
+            total_page,
+            todo_rev_id,
+        )
+        .await?;
+
+        page.save_to_files()?;
+    }
+
+    // 合并所有 parquet 文件
+    println!("\nMerging all parquet files...");
+
+    merge_mikan_classic_episodes_and_strip_columns().await?;
+
+    println!("Merge completed!");
+
+    Ok(())
+}

apps/recorder/examples/mikan_doppel_season_subscription.rs

@@ -0,0 +1,249 @@
+use std::time::Duration;
+
+use color_eyre::{Result, eyre::OptionExt};
+use fetch::{FetchError, HttpClientConfig, fetch_bytes, fetch_html, fetch_image, reqwest};
+use inquire::{Password, Text, validator::Validation};
+use recorder::{
+    crypto::UserPassCredential,
+    extract::mikan::{
+        MikanClient, MikanConfig, MikanRssEpisodeItem, build_mikan_bangumi_expand_subscribed_url,
+        extract_mikan_bangumi_index_meta_list_from_season_flow_fragment,
+        extract_mikan_bangumi_meta_from_expand_subscribed_fragment,
+    },
+    test_utils::mikan::{MikanDoppelMeta, MikanDoppelPath},
+};
+use scraper::Html;
+use tokio::fs;
+use url::Url;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    tracing_subscriber::fmt()
+        .with_max_level(tracing::Level::INFO)
+        .init();
+    std::env::set_current_dir(std::path::Path::new("apps/recorder"))?;
+
+    let mikan_scrape_client = MikanClient::from_config(MikanConfig {
+        http_client: HttpClientConfig {
+            exponential_backoff_max_retries: Some(3),
+            leaky_bucket_max_tokens: Some(2),
+            leaky_bucket_initial_tokens: Some(0),
+            leaky_bucket_refill_tokens: Some(1),
+            leaky_bucket_refill_interval: Some(Duration::from_millis(1000)),
+            user_agent: Some(
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) \
+                 Chrome/136.0.0.0 Safari/537.36 Edg/136.0.0.0"
+                    .to_string(),
+            ),
+            ..Default::default()
+        },
+        base_url: Url::parse("https://mikanani.me")?,
+    })
+    .await?;
+
+    let username_validator = |input: &str| {
+        if input.trim().is_empty() {
+            Ok(Validation::Invalid("Username cannot be empty".into()))
+        } else {
+            Ok(Validation::Valid)
+        }
+    };
+    let password_validator = |input: &str| {
+        if input.trim().is_empty() {
+            Ok(Validation::Invalid("Password cannot be empty".into()))
+        } else {
+            Ok(Validation::Valid)
+        }
+    };
+    let username = Text::new("Please enter your mikan username:")
+        .with_validator(username_validator)
+        .prompt()?;
+    let password = Password::new("Please enter your mikan password:")
+        .without_confirmation()
+        .with_display_mode(inquire::PasswordDisplayMode::Masked)
+        .with_validator(password_validator)
+        .prompt()?;
+
+    let mikan_scrape_client = mikan_scrape_client
+        .fork_with_userpass_credential(UserPassCredential {
+            username,
+            password,
+            user_agent: None,
+            cookies: None,
+        })
+        .await?;
+
+    tracing::info!("Checking if logged in...");
+    if !mikan_scrape_client.has_login().await? {
+        tracing::info!("Logging in to mikan...");
+        mikan_scrape_client.login().await?;
+        tracing::info!("Logged in to mikan");
+    }
+
+    let mikan_base_url = mikan_scrape_client.base_url().clone();
+    tracing::info!("Scraping season subscription...");
+    let season_subscription =
+        fs::read("tests/resources/mikan/BangumiCoverFlow-2025-spring.html").await?;
+    let html = Html::parse_fragment(String::from_utf8(season_subscription)?.as_str());
+    let bangumi_index_list =
+        extract_mikan_bangumi_index_meta_list_from_season_flow_fragment(&html, &mikan_base_url);
+
+    for bangumi_index in bangumi_index_list {
+        let bangumi_meta = {
+            let bangumi_expand_subscribed_url = build_mikan_bangumi_expand_subscribed_url(
+                mikan_base_url.clone(),
+                bangumi_index.mikan_bangumi_id.as_ref(),
+            );
+            let bangumi_expand_subscribed_doppel_path =
+                MikanDoppelPath::new(bangumi_expand_subscribed_url.clone());
+            tracing::info!(
+                bangumi_title = bangumi_index.bangumi_title,
+                "Scraping bangumi expand subscribed..."
+            );
+            let bangumi_expand_subscribed_data =
+                if !bangumi_expand_subscribed_doppel_path.exists_any() {
+                    let bangumi_expand_subscribed_data =
+                        fetch_html(&mikan_scrape_client, bangumi_expand_subscribed_url).await?;
+                    bangumi_expand_subscribed_doppel_path.write(&bangumi_expand_subscribed_data)?;
+                    tracing::info!(
+                        bangumi_title = bangumi_index.bangumi_title,
+                        "Bangumi expand subscribed saved"
+                    );
+                    bangumi_expand_subscribed_data
+                } else {
+                    tracing::info!(
+                        bangumi_title = bangumi_index.bangumi_title,
+                        "Bangumi expand subscribed already exists"
+                    );
+                    String::from_utf8(bangumi_expand_subscribed_doppel_path.read()?)?
+                };
+
+            let html = Html::parse_fragment(&bangumi_expand_subscribed_data);
+            extract_mikan_bangumi_meta_from_expand_subscribed_fragment(
+                &html,
+                bangumi_index.clone(),
+                mikan_base_url.clone(),
+            )
+            .ok_or_eyre(format!(
+                "Failed to extract bangumi meta from expand subscribed fragment: {:?}",
+                bangumi_index.bangumi_title
+            ))
+        }?;
+        {
+            if let Some(poster_url) = bangumi_meta.origin_poster_src.as_ref() {
+                let poster_doppel_path = MikanDoppelPath::new(poster_url.clone());
+                tracing::info!(
+                    title = bangumi_meta.bangumi_title,
+                    "Scraping bangumi poster..."
+                );
+                if !poster_doppel_path.exists_any() {
+                    let poster_data = fetch_image(&mikan_scrape_client, poster_url.clone()).await?;
+                    poster_doppel_path.write(&poster_data)?;
+                    tracing::info!(title = bangumi_meta.bangumi_title, "Bangumi poster saved");
+                } else {
+                    tracing::info!(
+                        title = bangumi_meta.bangumi_title,
+                        "Bangumi poster already exists"
+                    );
+                }
+            }
+        }
+        {
+            let bangumi_homepage_url = bangumi_meta
+                .bangumi_hash()
+                .build_homepage_url(mikan_base_url.clone());
+            let bangumi_homepage_doppel_path = MikanDoppelPath::new(bangumi_homepage_url.clone());
+            tracing::info!(
+                title = bangumi_meta.bangumi_title,
+                "Scraping bangumi homepage..."
+            );
+            if !bangumi_homepage_doppel_path.exists_any() {
+                let bangumi_homepage_data =
+                    fetch_html(&mikan_scrape_client, bangumi_homepage_url).await?;
+                bangumi_homepage_doppel_path.write(&bangumi_homepage_data)?;
+                tracing::info!(title = bangumi_meta.bangumi_title, "Bangumi homepage saved");
+            } else {
+                tracing::info!(
+                    title = bangumi_meta.bangumi_title,
+                    "Bangumi homepage already exists"
+                );
+            }
+        }
+        let rss_items = {
+            let bangumi_rss_url = bangumi_meta
+                .bangumi_hash()
+                .build_rss_url(mikan_base_url.clone());
+            let bangumi_rss_doppel_path = MikanDoppelPath::new(bangumi_rss_url.clone());
+            tracing::info!(
+                title = bangumi_meta.bangumi_title,
+                "Scraping bangumi rss..."
+            );
+            let bangumi_rss_data = if !bangumi_rss_doppel_path.exists_any() {
+                let bangumi_rss_data = fetch_html(&mikan_scrape_client, bangumi_rss_url).await?;
+                bangumi_rss_doppel_path.write(&bangumi_rss_data)?;
+                tracing::info!(title = bangumi_meta.bangumi_title, "Bangumi rss saved");
+                bangumi_rss_data
+            } else {
+                tracing::info!(
+                    title = bangumi_meta.bangumi_title,
+                    "Bangumi rss already exists"
+                );
+                String::from_utf8(bangumi_rss_doppel_path.read()?)?
+            };
+            let rss_items = rss::Channel::read_from(bangumi_rss_data.as_bytes())?.items;
+            rss_items
+                .into_iter()
+                .map(MikanRssEpisodeItem::try_from)
+                .collect::<Result<Vec<_>, _>>()
+        }?;
+        for rss_item in rss_items {
+            {
+                let episode_homepage_url = rss_item.build_homepage_url(mikan_base_url.clone());
+                let episode_homepage_doppel_path =
+                    MikanDoppelPath::new(episode_homepage_url.clone());
+                tracing::info!(title = rss_item.title, "Scraping episode...");
+                if !episode_homepage_doppel_path.exists_any() {
+                    let episode_homepage_data =
+                        fetch_html(&mikan_scrape_client, episode_homepage_url).await?;
+                    episode_homepage_doppel_path.write(&episode_homepage_data)?;
+                    tracing::info!(title = rss_item.title, "Episode saved");
+                } else {
+                    tracing::info!(title = rss_item.title, "Episode already exists");
+                };
+            }
+            {
+                let episode_torrent_url = rss_item.torrent_link;
+                let episode_torrent_doppel_path = MikanDoppelPath::new(episode_torrent_url.clone());
+                tracing::info!(title = rss_item.title, "Scraping episode torrent...");
+                if !episode_torrent_doppel_path.exists_any() {
+                    match fetch_bytes(&mikan_scrape_client, episode_torrent_url).await {
+                        Ok(episode_torrent_data) => {
+                            episode_torrent_doppel_path.write(&episode_torrent_data)?;
+                            tracing::info!(title = rss_item.title, "Episode torrent saved");
+                        }
+                        Err(e) => {
+                            if let FetchError::ReqwestError { source } = &e
+                                && source
+                                    .status()
+                                    .is_some_and(|status| status == reqwest::StatusCode::NOT_FOUND)
+                            {
+                                tracing::warn!(
+                                    title = rss_item.title,
+                                    "Episode torrent not found, maybe deleted since new version"
+                                );
+                                episode_torrent_doppel_path
+                                    .write_meta(MikanDoppelMeta { status: 404 })?;
+                            } else {
+                                Err(e)?;
+                            }
+                        }
+                    }
+                } else {
+                    tracing::info!(title = rss_item.title, "Episode torrent already exists");
+                }
+            }
+        }
+    }
+    tracing::info!("Scraping season subscription done");
+    Ok(())
+}

apps/recorder/examples/mikan_doppel_subscriber_subscription.rs

@@ -0,0 +1,215 @@
+use std::time::Duration;
+
+use fetch::{FetchError, HttpClientConfig, fetch_bytes, fetch_html, fetch_image, reqwest};
+use recorder::{
+    errors::RecorderResult,
+    extract::mikan::{
+        MikanClient, MikanConfig, MikanRssEpisodeItem,
+        extract_mikan_episode_meta_from_episode_homepage_html,
+    },
+    test_utils::mikan::{MikanDoppelMeta, MikanDoppelPath},
+};
+use scraper::Html;
+use tokio::fs;
+use url::Url;
+
+#[tokio::main]
+async fn main() -> RecorderResult<()> {
+    tracing_subscriber::fmt()
+        .with_max_level(tracing::Level::INFO)
+        .init();
+    std::env::set_current_dir(std::path::Path::new("apps/recorder"))?;
+
+    let mikan_scrape_client = MikanClient::from_config(MikanConfig {
+        http_client: HttpClientConfig {
+            exponential_backoff_max_retries: Some(3),
+            leaky_bucket_max_tokens: Some(2),
+            leaky_bucket_initial_tokens: Some(0),
+            leaky_bucket_refill_tokens: Some(1),
+            leaky_bucket_refill_interval: Some(Duration::from_millis(500)),
+            user_agent: Some(
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) \
+                 Chrome/136.0.0.0 Safari/537.36 Edg/136.0.0.0"
+                    .to_string(),
+            ),
+            ..Default::default()
+        },
+        base_url: Url::parse("https://mikanani.me")?,
+    })
+    .await?;
+
+    let mikan_base_url = mikan_scrape_client.base_url().clone();
+    tracing::info!("Scraping subscriber subscription...");
+    let subscriber_subscription =
+        fs::read("tests/resources/mikan/doppel/RSS/MyBangumi-token%3Dtest.html").await?;
+    let channel = rss::Channel::read_from(&subscriber_subscription[..])?;
+    let rss_items: Vec<MikanRssEpisodeItem> = channel
+        .items
+        .into_iter()
+        .map(MikanRssEpisodeItem::try_from)
+        .collect::<Result<Vec<_>, _>>()?;
+    for rss_item in rss_items {
+        let episode_homepage_meta = {
+            tracing::info!(title = rss_item.title, "Scraping episode homepage...");
+            let episode_homepage_url = rss_item.build_homepage_url(mikan_base_url.clone());
+            let episode_homepage_doppel_path = MikanDoppelPath::new(episode_homepage_url.clone());
+            let episode_homepage_data = if !episode_homepage_doppel_path.exists_any() {
+                let episode_homepage_data =
+                    fetch_html(&mikan_scrape_client, episode_homepage_url.clone()).await?;
+                episode_homepage_doppel_path.write(&episode_homepage_data)?;
+                tracing::info!(title = rss_item.title, "Episode homepage saved");
+                episode_homepage_data
+            } else {
+                tracing::info!(title = rss_item.title, "Episode homepage already exists");
+                String::from_utf8(episode_homepage_doppel_path.read()?)?
+            };
+            let html = Html::parse_document(&episode_homepage_data);
+            extract_mikan_episode_meta_from_episode_homepage_html(
+                &html,
+                mikan_base_url.clone(),
+                episode_homepage_url,
+            )
+        }?;
+
+        {
+            let episode_torrent_url = rss_item.torrent_link;
+            let episode_torrent_doppel_path = MikanDoppelPath::new(episode_torrent_url.clone());
+            tracing::info!(title = rss_item.title, "Scraping episode torrent...");
+            if !episode_torrent_doppel_path.exists_any() {
+                match fetch_bytes(&mikan_scrape_client, episode_torrent_url).await {
+                    Ok(episode_torrent_data) => {
+                        episode_torrent_doppel_path.write(&episode_torrent_data)?;
+                        tracing::info!(title = rss_item.title, "Episode torrent saved");
+                    }
+                    Err(e) => {
+                        if let FetchError::ReqwestError { source } = &e
+                            && source
+                                .status()
+                                .is_some_and(|status| status == reqwest::StatusCode::NOT_FOUND)
+                        {
+                            tracing::warn!(
+                                title = rss_item.title,
+                                "Episode torrent not found, maybe deleted since new version"
+                            );
+                            episode_torrent_doppel_path
+                                .write_meta(MikanDoppelMeta { status: 404 })?;
+                        } else {
+                            Err(e)?;
+                        }
+                    }
+                }
+
+                tracing::info!(title = rss_item.title, "Episode torrent saved");
+            } else {
+                tracing::info!(title = rss_item.title, "Episode torrent already exists");
+            }
+        }
+        {
+            if let Some(episode_poster_url) = episode_homepage_meta.origin_poster_src.as_ref() {
+                let episode_poster_doppel_path = MikanDoppelPath::new(episode_poster_url.clone());
+                tracing::info!(title = rss_item.title, "Scraping episode poster...");
+                if !episode_poster_doppel_path.exists_any() {
+                    let episode_poster_data =
+                        fetch_image(&mikan_scrape_client, episode_poster_url.clone()).await?;
+                    episode_poster_doppel_path.write(&episode_poster_data)?;
+                    tracing::info!(title = rss_item.title, "Episode poster saved");
+                } else {
+                    tracing::info!(title = rss_item.title, "Episode poster already exists");
+                }
+            }
+        }
+
+        {
+            let bangumi_homepage_url = episode_homepage_meta
+                .bangumi_hash()
+                .build_homepage_url(mikan_base_url.clone());
+            let bangumi_homepage_doppel_path = MikanDoppelPath::new(bangumi_homepage_url.clone());
+            tracing::info!(title = rss_item.title, "Scraping bangumi homepage...");
+            if !bangumi_homepage_doppel_path.exists_any() {
+                let bangumi_homepage_data =
+                    fetch_html(&mikan_scrape_client, bangumi_homepage_url).await?;
+                bangumi_homepage_doppel_path.write(&bangumi_homepage_data)?;
+                tracing::info!(title = rss_item.title, "Bangumi homepage saved");
+            } else {
+                tracing::info!(title = rss_item.title, "Bangumi homepage already exists");
+            };
+        }
+        {
+            let bangumi_rss_url = episode_homepage_meta
+                .bangumi_hash()
+                .build_rss_url(mikan_base_url.clone());
+            let bangumi_rss_doppel_path = MikanDoppelPath::new(bangumi_rss_url.clone());
+            tracing::info!(title = rss_item.title, "Scraping bangumi rss...");
+            let bangumi_rss_data = if !bangumi_rss_doppel_path.exists_any() {
+                let bangumi_rss_data = fetch_html(&mikan_scrape_client, bangumi_rss_url).await?;
+                bangumi_rss_doppel_path.write(&bangumi_rss_data)?;
+                tracing::info!(title = rss_item.title, "Bangumi rss saved");
+                bangumi_rss_data
+            } else {
+                tracing::info!(title = rss_item.title, "Bangumi rss already exists");
+                String::from_utf8(bangumi_rss_doppel_path.read()?)?
+            };
+
+            let channel = rss::Channel::read_from(bangumi_rss_data.as_bytes())?;
+            let rss_items: Vec<MikanRssEpisodeItem> = channel
+                .items
+                .into_iter()
+                .map(MikanRssEpisodeItem::try_from)
+                .collect::<Result<Vec<_>, _>>()?;
+            for rss_item in rss_items {
+                {
+                    tracing::info!(title = rss_item.title, "Scraping episode homepage...");
+                    let episode_homepage_url = rss_item.build_homepage_url(mikan_base_url.clone());
+                    let episode_homepage_doppel_path =
+                        MikanDoppelPath::new(episode_homepage_url.clone());
+                    if !episode_homepage_doppel_path.exists_any() {
+                        let episode_homepage_data =
+                            fetch_html(&mikan_scrape_client, episode_homepage_url.clone()).await?;
+                        episode_homepage_doppel_path.write(&episode_homepage_data)?;
+                        tracing::info!(title = rss_item.title, "Episode homepage saved");
+                    } else {
+                        tracing::info!(title = rss_item.title, "Episode homepage already exists");
+                    };
+                };
+
+                {
+                    let episode_torrent_url = rss_item.torrent_link;
+                    let episode_torrent_doppel_path =
+                        MikanDoppelPath::new(episode_torrent_url.clone());
+                    tracing::info!(title = rss_item.title, "Scraping episode torrent...");
+                    if !episode_torrent_doppel_path.exists_any() {
+                        match fetch_bytes(&mikan_scrape_client, episode_torrent_url).await {
+                            Ok(episode_torrent_data) => {
+                                episode_torrent_doppel_path.write(&episode_torrent_data)?;
+                                tracing::info!(title = rss_item.title, "Episode torrent saved");
+                            }
+                            Err(e) => {
+                                if let FetchError::ReqwestError { source } = &e
+                                    && source.status().is_some_and(|status| {
+                                        status == reqwest::StatusCode::NOT_FOUND
+                                    })
+                                {
+                                    tracing::warn!(
+                                        title = rss_item.title,
+                                        "Episode torrent not found, maybe deleted since new \
+                                         version"
+                                    );
+                                    episode_torrent_doppel_path
+                                        .write_meta(MikanDoppelMeta { status: 404 })?;
+                                } else {
+                                    Err(e)?;
+                                }
+                            }
+                        }
+
+                        tracing::info!(title = rss_item.title, "Episode torrent saved");
+                    } else {
+                        tracing::info!(title = rss_item.title, "Episode torrent already exists");
+                    }
+                }
+            }
+        }
+    }
+    tracing::info!("Scraping subscriber subscription done");
+    Ok(())
+}

apps/recorder/examples/playground.rs

@@ -1,68 +1,33 @@
-// #![allow(unused_imports)]
-// use color_eyre::eyre::Context;
-// use itertools::Itertools;
-// use loco_rs::{
-//     app::Hooks,
-//     boot::{BootResult, StartMode},
-//     environment::Environment,
-//     prelude::AppContext as LocoContext,
-// };
-// use recorder::{
-//     app::{App1, AppContext},
-//     errors::RResult,
-//     migrations::Migrator,
-//     models::{
-//         subscribers::SEED_SUBSCRIBER,
-//         subscriptions::{self, SubscriptionCreateFromRssDto},
-//     },
-// };
-// use sea_orm::ColumnTrait;
-// use sea_orm_migration::MigratorTrait;
+#![feature(duration_constructors_lite)]
+use std::{sync::Arc, time::Duration};
 
-// async fn pull_mikan_bangumi_rss(ctx: &AppContext) -> RResult<()> {
-//     let rss_link = "https://mikanani.me/RSS/Bangumi?bangumiId=3416&subgroupid=370";
+use apalis_sql::postgres::PostgresStorage;
+use recorder::{
+    app::AppContextTrait,
+    errors::RecorderResult,
+    test_utils::{
+        app::TestingAppContext,
+        database::{TestingDatabaseServiceConfig, build_testing_database_service},
+    },
+};
 
-//     // let rss_link =
-//     //     "https://mikanani.me/RSS/MyBangumi?token=FE9tccsML2nBPUUqpCuJW2uJZydAXCntHJ7RpD9LDP8%3d";
-//     let subscription = if let Some(subscription) =
-// subscriptions::Entity::find()
-//         .filter(subscriptions::Column::SourceUrl.eq(String::from(rss_link)))
-//         .one(&ctx.db)
-//         .await?
-//     {
-//         subscription
-//     } else {
-//         subscriptions::Model::add_subscription(
-//             ctx,
-//
-// subscriptions::SubscriptionCreateDto::Mikan(SubscriptionCreateFromRssDto {
-//                 rss_link: rss_link.to_string(),
-//                 display_name: String::from("Mikan Project - 我的番组"),
-//                 enabled: Some(true),
-//             }),
-//             1,
-//         )
-//         .await?
-//     };
+#[tokio::main]
+async fn main() -> RecorderResult<()> {
+    let app_ctx = {
+        let db_service = build_testing_database_service(TestingDatabaseServiceConfig {
+            auto_migrate: false,
+        })
+        .await?;
+        Arc::new(TestingAppContext::builder().db(db_service).build())
+    };
 
-//     subscription.pull_subscription(ctx).await?;
+    let db = app_ctx.db();
 
-//     Ok(())
-// }
+    PostgresStorage::setup(db.get_postgres_connection_pool()).await?;
 
-// async fn init() -> RResult<LocoContext> {
-//     let ctx = loco_rs::cli::playground::<App1>().await?;
-//     let BootResult {
-//         app_context: ctx, ..
-//     } = loco_rs::boot::run_app::<App1>(&StartMode::ServerOnly, ctx).await?;
-//     Migrator::up(&ctx.db, None).await?;
-//     Ok(ctx)
-// }
+    dbg!(db.get_postgres_connection_pool().connect_options());
 
-// #[tokio::main]
-// async fn main() -> color_eyre::eyre::Result<()> {
-//     pull_mikan_bangumi_rss(&ctx).await?;
+    tokio::time::sleep(Duration::from_hours(1)).await;
 
-//     Ok(())
-// }
-fn main() {}
+    Ok(())
+}

apps/recorder/package.json

@@ -1,37 +0,0 @@
-{
-  "name": "recorder",
-  "version": "1.0.0",
-  "type": "module",
-  "scripts": {
-    "dev": "rsbuild dev",
-    "build": "rsbuild build",
-    "preview": "rsbuild preview"
-  },
-  "dependencies": {
-    "@abraham/reflection": "^0.12.0",
-    "@graphiql/react": "^0.28.2",
-    "@graphiql/toolkit": "^0.11.1",
-    "@konobangu/design-system": "workspace:*",
-    "@konobangu/tailwind-config": "workspace:*",
-    "@outposts/injection-js": "^2.5.1",
-    "@tanstack/react-router": "^1.95.6",
-    "@tanstack/router-devtools": "^1.95.6",
-    "graphiql": "^3.8.3",
-    "graphql-ws": "^5.16.2",
-    "observable-hooks": "^4.2.4",
-    "oidc-client-rx": "0.1.0-alpha.6",
-    "react": "^19.0.0",
-    "react-dom": "^19.0.0",
-    "rxjs": "^7.8.1"
-  },
-  "devDependencies": {
-    "@konobangu/typescript-config": "workspace:*",
-    "@rsbuild/core": "1.1.3",
-    "@rsbuild/plugin-react": "^1.1.1",
-    "@tanstack/router-plugin": "^1.95.6",
-    "@types/react": "^19.0.7",
-    "@types/react-dom": "^19.0.3",
-    "tailwindcss": "^3.4.17",
-    "typescript": "^5.7.3"
-  }
-}

apps/recorder/recorder.config.toml

@@ -0,0 +1,114 @@
+# Application logging configuration
+[logger]
+# Enable or disable logging.
+enable = true
+# Enable pretty backtrace (sets RUST_BACKTRACE=1)
+pretty_backtrace = true
+# Log level, options: trace, debug, info, warn or error.
+level = "debug"
+# Define the logging format. options: compact, pretty or Json
+format = "compact"
+# By default the logger has filtering only logs that came from your code or logs that came from `loco` framework. to see all third party libraries
+# Uncomment the line below to override to see all third party libraries you can enable this config and override the logger filters.
+# override_filter: trace
+
+# Web server configuration
+[server]
+# Port on which the server will listen. the server binding is 0.0.0.0:{PORT}
+port = 5001
+binding = "0.0.0.0"
+# The UI hostname or IP address that mailers will point to.
+host = '{{ get_env(name="HOST", default="localhost") }}'
+# Out of the box middleware configuration. to disable middleware you can changed the `enable` field to `false` of comment the middleware block
+
+# Enable Etag cache header middleware
+[server.middlewares.etag]
+enable = true
+
+# Generating a unique request ID and enhancing logging with additional information such as the start and completion of request processing, latency, status code, and other request details.
+[server.middlewares.request_id]
+enable = true
+
+[server.middlewares.logger]
+enable = true
+
+# when your code is panicked, the request still returns 500 status code.
+[server.middlewares.catch_panic]
+enable = true
+
+# Timeout for incoming requests middleware. requests that take more time from the configuration will cute and 408 status code will returned.
+[server.middlewares.timeout_request]
+enable = false
+# Duration time in milliseconds.
+timeout = 5000
+
+# Set the value of the [`Access-Control-Allow-Origin`][mdn] header
+# allow_origins:
+#   - https://konobangu.com
+# Set the value of the [`Access-Control-Allow-Headers`][mdn] header
+# allow_headers:
+# - Content-Type
+# Set the value of the [`Access-Control-Allow-Methods`][mdn] header
+# allow_methods:
+#   - POST
+# Set the value of the [`Access-Control-Max-Age`][mdn] header in seconds
+# max_age: 3600
+[server.middlewares.cors]
+enable = true
+
+[server.middlewares.compression]
+enable = true
+
+# Database Configuration
+[database]
+# Database connection URI
+uri = '{{ get_env(name="DATABASE_URL", default="postgres://konobangu:konobangu@localhost:5432/konobangu") }}'
+# When enabled, the sql query will be logged.
+enable_logging = true
+# Set the timeout duration when acquiring a connection.
+connect_timeout = 500
+# Set the idle duration before closing a connection.
+idle_timeout = 500
+# Minimum number of connections for a pool.
+min_connections = 1
+# Maximum number of connections for a pool.
+max_connections = 10
+# Run migration up when application loaded
+auto_migrate = true
+
+[storage]
+data_dir = '{{ get_env(name="STORAGE_DATA_DIR", default="./data") }}'
+
+[mikan]
+base_url = "https://mikanani.me/"
+
+[mikan.http_client]
+exponential_backoff_max_retries = 3
+leaky_bucket_max_tokens = 2
+leaky_bucket_initial_tokens = 1
+leaky_bucket_refill_tokens = 1
+leaky_bucket_refill_interval = 500
+
+
+[mikan.http_client.proxy]
+server = '{{ get_env(name="MIKAN_PROXY", default = "") }}'
+auth_header = '{{ get_env(name="MIKAN_PROXY_AUTH_HEADER", default = "") }}'
+no_proxy = '{{ get_env(name="MIKAN_NO_PROXY", default = "") }}'
+accept_invalid_certs = '{{ get_env(name="MIKAN_PROXY_ACCEPT_INVALID_CERTS", default = "false") }}'
+
+
+[auth]
+auth_type = '{{ get_env(name="AUTH_TYPE", default = "basic") }}'
+basic_user = '{{ get_env(name="BASIC_USER", default = "konobangu") }}'
+basic_password = '{{ get_env(name="BASIC_PASSWORD", default = "konobangu") }}'
+oidc_issuer = '{{ get_env(name="OIDC_ISSUER", default = "") }}'
+oidc_audience = '{{ get_env(name="OIDC_AUDIENCE", default = "") }}'
+oidc_client_id = '{{ get_env(name="OIDC_CLIENT_ID", default = "") }}'
+oidc_client_secret = '{{ get_env(name="OIDC_CLIENT_SECRET", default = "") }}'
+oidc_extra_scopes = '{{ get_env(name="OIDC_EXTRA_SCOPES", default = "") }}'
+oidc_extra_claim_key = '{{ get_env(name="OIDC_EXTRA_CLAIM_KEY", default = "") }}'
+oidc_extra_claim_value = '{{ get_env(name="OIDC_EXTRA_CLAIM_VALUE", default = "") }}'
+
+[graphql]
+# depth_limit = inf
+# complexity_limit = inf

apps/recorder/recorder.config.yaml

@@ -1,101 +0,0 @@
-# Loco configuration file documentation
-
-# Application logging configuration
-logger:
-    # Enable or disable logging.
-    enable: true
-    # Enable pretty backtrace (sets RUST_BACKTRACE=1)
-    pretty_backtrace: true
-    # Log level, options: trace, debug, info, warn or error.
-    level: debug
-    # Define the logging format. options: compact, pretty or Json
-    format: compact
-    # By default the logger has filtering only logs that came from your code or logs that came from `loco` framework. to see all third party libraries
-    # Uncomment the line below to override to see all third party libraries you can enable this config and override the logger filters.
-    # override_filter: trace
-
-# Web server configuration
-server:
-    # Port on which the server will listen. the server binding is 0.0.0.0:{PORT}
-    port: 5001
-    binding: "0.0.0.0"
-    # The UI hostname or IP address that mailers will point to.
-    host: '{{ get_env(name="HOST", default="localhost") }}'
-    # Out of the box middleware configuration. to disable middleware you can changed the `enable` field to `false` of comment the middleware block
-    middlewares:
-        # Enable Etag cache header middleware
-        etag:
-            enable: true
-        # Generating a unique request ID and enhancing logging with additional information such as the start and completion of request processing, latency, status code, and other request details.
-        logger:
-            # Enable/Disable the middleware.
-            enable: true
-        # when your code is panicked, the request still returns 500 status code.
-        catch_panic:
-            # Enable/Disable the middleware.
-            enable: true
-        # Timeout for incoming requests middleware. requests that take more time from the configuration will cute and 408 status code will returned.
-        timeout_request:
-            # Enable/Disable the middleware.
-            enable: false
-            # Duration time in milliseconds.
-            timeout: 5000
-
-        cors:
-            enable: true
-            # Set the value of the [`Access-Control-Allow-Origin`][mdn] header
-            # allow_origins:
-            #   - https://loco.rs
-            # Set the value of the [`Access-Control-Allow-Headers`][mdn] header
-            # allow_headers:
-            # - Content-Type
-            # Set the value of the [`Access-Control-Allow-Methods`][mdn] header
-            # allow_methods:
-            #   - POST
-            # Set the value of the [`Access-Control-Max-Age`][mdn] header in seconds
-            # max_age: 3600
-
-# Database Configuration
-database:
-    # Database connection URI
-    uri: '{{ get_env(name="DATABASE_URL", default="postgres://konobangu:konobangu@localhost:5432/konobangu") }}'
-    # When enabled, the sql query will be logged.
-    enable_logging: true
-    # Set the timeout duration when acquiring a connection.
-    connect_timeout: 500
-    # Set the idle duration before closing a connection.
-    idle_timeout: 500
-    # Minimum number of connections for a pool.
-    min_connections: 1
-    # Maximum number of connections for a pool.
-    max_connections: 1
-    # Run migration up when application loaded
-    auto_migrate: true
-
-storage:
-    data_dir: '{{ get_env(name="STORAGE_DATA_DIR", default="./data") }}'
-
-mikan:
-    base_url: "https://mikanani.me/"
-    http_client:
-        exponential_backoff_max_retries: 3
-        leaky_bucket_max_tokens: 2
-        leaky_bucket_initial_tokens: 0
-        leaky_bucket_refill_tokens: 1
-        leaky_bucket_refill_interval: 500
-
-auth:
-    auth_type: '{{ get_env(name="AUTH_TYPE", default = "basic") }}'
-    basic_user: '{{ get_env(name="BASIC_USER", default = "konobangu") }}'
-    basic_password: '{{ get_env(name="BASIC_PASSWORD", default = "konobangu") }}'
-    oidc_issuer: '{{ get_env(name="OIDC_ISSUER", default = "") }}'
-    oidc_audience: '{{ get_env(name="OIDC_AUDIENCE", default = "") }}'
-    oidc_client_id: '{{ get_env(name="OIDC_CLIENT_ID", default = "") }}'
-    oidc_client_secret: '{{ get_env(name="OIDC_CLIENT_SECRET", default = "") }}'
-    oidc_extra_scopes: '{{ get_env(name="OIDC_EXTRA_SCOPES", default = "") }}'
-    oidc_extra_claim_key: '{{ get_env(name="OIDC_EXTRA_CLAIM_KEY", default = "") }}'
-    oidc_extra_claim_value: '{{ get_env(name="OIDC_EXTRA_CLAIM_VALUE", default = "") }}'
-
-graphql:
-    depth_limit: null
-    complexity_limit: null

apps/recorder/src/app/builder.rs

@@ -1,9 +1,7 @@
-use std::sync::Arc;
-
 use clap::{Parser, command};
 
 use super::{AppContext, core::App, env::Environment};
-use crate::{app::config::AppConfig, errors::RResult};
+use crate::{app::config::AppConfig, errors::RecorderResult};
 
 #[derive(Parser, Debug)]
 #[command(version, about, long_about = None)]
@@ -23,17 +21,21 @@ pub struct MainCliArgs {
     /// Explicit environment
     #[arg(short, long)]
     environment: Option<Environment>,
+
+    #[arg(long)]
+    graceful_shutdown: Option<bool>,
 }
 
 pub struct AppBuilder {
     dotenv_file: Option<String>,
     config_file: Option<String>,
     working_dir: String,
-    enviornment: Environment,
+    environment: Environment,
+    pub graceful_shutdown: bool,
 }
 
 impl AppBuilder {
-    pub async fn from_main_cli(environment: Option<Environment>) -> RResult<Self> {
+    pub async fn from_main_cli(environment: Option<Environment>) -> RecorderResult<Self> {
         let args = MainCliArgs::parse();
 
         let environment = environment.unwrap_or_else(|| {
@@ -63,29 +65,19 @@ impl AppBuilder {
         builder = builder
             .config_file(args.config_file)
             .dotenv_file(args.dotenv_file)
-            .environment(environment);
+            .environment(environment)
+            .graceful_shutdown(args.graceful_shutdown.unwrap_or(true));
 
         Ok(builder)
     }
 
-    pub async fn build(self) -> RResult<App> {
-        AppConfig::load_dotenv(
-            &self.enviornment,
-            &self.working_dir,
-            self.dotenv_file.as_deref(),
-        )
-        .await?;
+    pub async fn build(self) -> RecorderResult<App> {
+        self.load_env().await?;
 
-        let config = AppConfig::load_config(
-            &self.enviornment,
-            &self.working_dir,
-            self.config_file.as_deref(),
-        )
-        .await?;
+        let config = self.load_config().await?;
 
-        let app_context = Arc::new(
-            AppContext::new(self.enviornment.clone(), config, self.working_dir.clone()).await?,
-        );
+        let app_context =
+            AppContext::new(self.environment.clone(), config, self.working_dir.clone()).await?;
 
         Ok(App {
             context: app_context,
@@ -93,6 +85,26 @@ impl AppBuilder {
         })
     }
 
+    pub async fn load_env(&self) -> RecorderResult<()> {
+        AppConfig::load_dotenv(
+            &self.environment,
+            &self.working_dir,
+            self.dotenv_file.as_deref(),
+        )
+        .await?;
+        Ok(())
+    }
+
+    pub async fn load_config(&self) -> RecorderResult<AppConfig> {
+        let config = AppConfig::load_config(
+            &self.environment,
+            &self.working_dir,
+            self.config_file.as_deref(),
+        )
+        .await?;
+        Ok(config)
+    }
+
     pub fn working_dir(self, working_dir: String) -> Self {
         let mut ret = self;
         ret.working_dir = working_dir;
@@ -101,7 +113,7 @@ impl AppBuilder {
 
     pub fn environment(self, environment: Environment) -> Self {
         let mut ret = self;
-        ret.enviornment = environment;
+        ret.environment = environment;
         ret
     }
 
@@ -111,6 +123,12 @@ impl AppBuilder {
         ret
     }
 
+    pub fn graceful_shutdown(self, graceful_shutdown: bool) -> Self {
+        let mut ret = self;
+        ret.graceful_shutdown = graceful_shutdown;
+        ret
+    }
+
     pub fn dotenv_file(self, dotenv_file: Option<String>) -> Self {
         let mut ret = self;
         ret.dotenv_file = dotenv_file;
@@ -130,10 +148,11 @@ impl AppBuilder {
 impl Default for AppBuilder {
     fn default() -> Self {
         Self {
-            enviornment: Environment::Production,
+            environment: Environment::Production,
             dotenv_file: None,
             config_file: None,
             working_dir: String::from("."),
+            graceful_shutdown: true,
         }
     }
 }

apps/recorder/src/app/config/default_mixin.toml

@@ -11,8 +11,21 @@ leaky_bucket_initial_tokens = 0
 leaky_bucket_refill_tokens = 1
 leaky_bucket_refill_interval = 500
 
+
+[mikan.http_client.proxy]
+
+[mikan.http_client.proxy.headers]
+
 [graphql]
 depth_limit = inf
 complexity_limit = inf
 
 [cache]
+
+[crypto]
+
+[task]
+
+[message]
+
+[media]

apps/recorder/src/app/config/mod.rs

@@ -9,9 +9,10 @@ use serde::{Deserialize, Serialize};
 
 use super::env::Environment;
 use crate::{
-    auth::AuthConfig, cache::CacheConfig, database::DatabaseConfig, errors::RResult,
-    extract::mikan::MikanConfig, graphql::GraphQLConfig, logger::LoggerConfig,
-    storage::StorageConfig, web::WebServerConfig,
+    auth::AuthConfig, cache::CacheConfig, crypto::CryptoConfig, database::DatabaseConfig,
+    errors::RecorderResult, extract::mikan::MikanConfig, graphql::GraphQLConfig,
+    logger::LoggerConfig, media::MediaConfig, message::MessageConfig, storage::StorageConfig,
+    task::TaskConfig, web::WebServerConfig,
 };
 
 const DEFAULT_CONFIG_MIXIN: &str = include_str!("./default_mixin.toml");
@@ -24,9 +25,13 @@ pub struct AppConfig {
     pub auth: AuthConfig,
     pub storage: StorageConfig,
     pub mikan: MikanConfig,
+    pub crypto: CryptoConfig,
     pub graphql: GraphQLConfig,
+    pub media: MediaConfig,
     pub logger: LoggerConfig,
     pub database: DatabaseConfig,
+    pub task: TaskConfig,
+    pub message: MessageConfig,
 }
 
 impl AppConfig {
@@ -64,7 +69,7 @@ impl AppConfig {
         fig: Figment,
         filepath: impl AsRef<Path>,
         ext: &str,
-    ) -> RResult<Figment> {
+    ) -> RecorderResult<Figment> {
         let content = fs::read_to_string(filepath)?;
 
         let rendered = tera::Tera::one_off(
@@ -85,7 +90,7 @@ impl AppConfig {
         environment: &Environment,
         working_dir: &str,
         dotenv_file: Option<&str>,
-    ) -> RResult<()> {
+    ) -> RecorderResult<()> {
         let try_dotenv_file_or_dirs = if dotenv_file.is_some() {
             vec![dotenv_file]
         } else {
@@ -106,12 +111,12 @@ impl AppConfig {
                     for f in try_filenames.iter() {
                         let p = try_dotenv_file_or_dir_path.join(f);
                         if p.exists() && p.is_file() {
-                            dotenv::from_path(p)?;
+                            dotenvy::from_path(p)?;
                             break;
                         }
                     }
                 } else if try_dotenv_file_or_dir_path.is_file() {
-                    dotenv::from_path(try_dotenv_file_or_dir_path)?;
+                    dotenvy::from_path(try_dotenv_file_or_dir_path)?;
                     break;
                 }
             }
@@ -124,7 +129,7 @@ impl AppConfig {
         environment: &Environment,
         working_dir: &str,
         config_file: Option<&str>,
-    ) -> RResult<AppConfig> {
+    ) -> RecorderResult<AppConfig> {
         let try_config_file_or_dirs = if config_file.is_some() {
             vec![config_file]
         } else {
@@ -140,7 +145,7 @@ impl AppConfig {
             .flat_map(|ps| {
                 allowed_extensions
                     .iter()
-                    .map(move |ext| (format!("{}{}{}", convention_prefix, ps, ext), ext))
+                    .map(move |ext| (format!("{convention_prefix}{ps}{ext}"), ext))
             })
             .collect_vec();
 

apps/recorder/src/app/context.rs

@@ -1,21 +1,47 @@
+use std::{fmt::Debug, sync::Arc};
+
+use tokio::sync::OnceCell;
+
 use super::{Environment, config::AppConfig};
 use crate::{
-    auth::AuthService, cache::CacheService, database::DatabaseService, errors::RResult,
-    extract::mikan::MikanClient, graphql::GraphQLService, logger::LoggerService,
-    storage::StorageService,
+    auth::AuthService, cache::CacheService, crypto::CryptoService, database::DatabaseService,
+    errors::RecorderResult, extract::mikan::MikanClient, graphql::GraphQLService,
+    logger::LoggerService, media::MediaService, message::MessageService, storage::StorageService,
+    task::TaskService,
 };
 
+pub trait AppContextTrait: Send + Sync + Debug {
+    fn logger(&self) -> &LoggerService;
+    fn db(&self) -> &DatabaseService;
+    fn config(&self) -> &AppConfig;
+    fn cache(&self) -> &CacheService;
+    fn mikan(&self) -> &MikanClient;
+    fn auth(&self) -> &AuthService;
+    fn graphql(&self) -> &GraphQLService;
+    fn storage(&self) -> &StorageService;
+    fn working_dir(&self) -> &String;
+    fn environment(&self) -> &Environment;
+    fn crypto(&self) -> &CryptoService;
+    fn task(&self) -> &TaskService;
+    fn message(&self) -> &MessageService;
+    fn media(&self) -> &MediaService;
+}
+
 pub struct AppContext {
-    pub logger: LoggerService,
-    pub db: DatabaseService,
-    pub config: AppConfig,
-    pub cache: CacheService,
-    pub mikan: MikanClient,
-    pub auth: AuthService,
-    pub graphql: GraphQLService,
-    pub storage: StorageService,
-    pub working_dir: String,
-    pub environment: Environment,
+    logger: LoggerService,
+    db: DatabaseService,
+    config: AppConfig,
+    cache: CacheService,
+    mikan: MikanClient,
+    auth: AuthService,
+    storage: StorageService,
+    crypto: CryptoService,
+    working_dir: String,
+    environment: Environment,
+    message: MessageService,
+    media: MediaService,
+    task: OnceCell<TaskService>,
+    graphql: OnceCell<GraphQLService>,
 }
 
 impl AppContext {
@@ -23,18 +49,20 @@ impl AppContext {
         environment: Environment,
         config: AppConfig,
         working_dir: impl ToString,
-    ) -> RResult<Self> {
+    ) -> RecorderResult<Arc<Self>> {
         let config_cloned = config.clone();
 
         let logger = LoggerService::from_config(config.logger).await?;
         let cache = CacheService::from_config(config.cache).await?;
         let db = DatabaseService::from_config(config.database).await?;
         let storage = StorageService::from_config(config.storage).await?;
+        let message = MessageService::from_config(config.message).await?;
         let auth = AuthService::from_conf(config.auth).await?;
         let mikan = MikanClient::from_config(config.mikan).await?;
-        let graphql = GraphQLService::from_config_and_database(config.graphql, db.clone()).await?;
+        let crypto = CryptoService::from_config(config.crypto).await?;
+        let media = MediaService::from_config(config.media).await?;
 
-        Ok(AppContext {
+        let ctx = Arc::new(AppContext {
             config: config_cloned,
             environment,
             logger,
@@ -44,7 +72,76 @@ impl AppContext {
             storage,
             mikan,
             working_dir: working_dir.to_string(),
-            graphql,
+            crypto,
+            message,
+            media,
+            task: OnceCell::new(),
+            graphql: OnceCell::new(),
+        });
+
+        ctx.task
+            .get_or_try_init(async || {
+                TaskService::from_config_and_ctx(config.task, ctx.clone()).await
             })
+            .await?;
+
+        ctx.graphql
+            .get_or_try_init(async || {
+                GraphQLService::from_config_and_ctx(config.graphql, ctx.clone()).await
+            })
+            .await?;
+
+        Ok(ctx)
+    }
+}
+
+impl Debug for AppContext {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "AppContext")
+    }
+}
+
+impl AppContextTrait for AppContext {
+    fn logger(&self) -> &LoggerService {
+        &self.logger
+    }
+    fn db(&self) -> &DatabaseService {
+        &self.db
+    }
+    fn config(&self) -> &AppConfig {
+        &self.config
+    }
+    fn cache(&self) -> &CacheService {
+        &self.cache
+    }
+    fn mikan(&self) -> &MikanClient {
+        &self.mikan
+    }
+    fn auth(&self) -> &AuthService {
+        &self.auth
+    }
+    fn graphql(&self) -> &GraphQLService {
+        self.graphql.get().expect("graphql should be set")
+    }
+    fn storage(&self) -> &StorageService {
+        &self.storage
+    }
+    fn working_dir(&self) -> &String {
+        &self.working_dir
+    }
+    fn environment(&self) -> &Environment {
+        &self.environment
+    }
+    fn crypto(&self) -> &CryptoService {
+        &self.crypto
+    }
+    fn task(&self) -> &TaskService {
+        self.task.get().expect("task should be set")
+    }
+    fn message(&self) -> &MessageService {
+        &self.message
+    }
+    fn media(&self) -> &MediaService {
+        &self.media
     }
 }

apps/recorder/src/app/core.rs

@@ -1,20 +1,22 @@
 use std::{net::SocketAddr, sync::Arc};
 
 use axum::Router;
-use futures::try_join;
-use tokio::signal;
+use tokio::{net::TcpSocket, signal};
+use tracing::instrument;
 
-use super::{builder::AppBuilder, context::AppContext};
+use super::{builder::AppBuilder, context::AppContextTrait};
 use crate::{
-    errors::RResult,
+    errors::{RecorderError, RecorderResult},
     web::{
         controller::{self, core::ControllerTrait},
         middleware::default_middleware_stack,
     },
 };
 
+pub const PROJECT_NAME: &str = "konobangu";
+
 pub struct App {
-    pub context: Arc<AppContext>,
+    pub context: Arc<dyn AppContextTrait>,
     pub builder: AppBuilder,
 }
 
@@ -23,42 +25,104 @@ impl App {
         AppBuilder::default()
     }
 
-    pub async fn serve(&self) -> RResult<()> {
+    #[instrument(err, skip(self))]
+    pub async fn serve(&self) -> RecorderResult<()> {
         let context = &self.context;
-        let config = &context.config;
-        let listener = tokio::net::TcpListener::bind(&format!(
-            "{}:{}",
-            config.server.binding, config.server.port
-        ))
-        .await?;
+        let config = context.config();
 
-        let mut router = Router::<Arc<AppContext>>::new();
+        let listener = {
+            let addr: SocketAddr =
+                format!("{}:{}", config.server.binding, config.server.port).parse()?;
 
-        let (graphqlc, oidcc) = try_join!(
+            let socket = if addr.is_ipv4() {
+                TcpSocket::new_v4()
+            } else {
+                TcpSocket::new_v6()
+            }?;
+
+            socket.set_reuseaddr(true)?;
+
+            #[cfg(all(unix, not(target_os = "solaris")))]
+            if let Err(e) = socket.set_reuseport(true) {
+                tracing::warn!("Failed to set SO_REUSEPORT: {}", e);
+            }
+
+            socket.bind(addr)?;
+            socket.listen(1024)
+        }?;
+
+        let mut router = Router::<Arc<dyn AppContextTrait>>::new();
+
+        let (graphql_c, oidc_c, metadata_c, static_c, feeds_c) = futures::try_join!(
             controller::graphql::create(context.clone()),
             controller::oidc::create(context.clone()),
+            controller::metadata::create(context.clone()),
+            controller::r#static::create(context.clone()),
+            controller::feeds::create(context.clone()),
         )?;
 
-        for c in [graphqlc, oidcc] {
+        for c in [graphql_c, oidc_c, metadata_c, static_c, feeds_c] {
             router = c.apply_to(router);
         }
 
         let middlewares = default_middleware_stack(context.clone());
         for mid in middlewares {
+            if mid.is_enabled() {
                 router = mid.apply(router)?;
                 tracing::info!(name = mid.name(), "+middleware");
             }
+        }
 
         let router = router
             .with_state(context.clone())
             .into_make_service_with_connect_info::<SocketAddr>();
 
-        axum::serve(listener, router)
+        let task = context.task();
+
+        let graceful_shutdown = self.builder.graceful_shutdown;
+
+        tokio::try_join!(
+            async {
+                let axum_serve = axum::serve(listener, router);
+
+                if graceful_shutdown {
+                    axum_serve
                         .with_graceful_shutdown(async move {
                             Self::shutdown_signal().await;
-                tracing::info!("shutting down...");
+                            tracing::info!("axum shutting down...");
                         })
                         .await?;
+                } else {
+                    axum_serve.await?;
+                }
+
+                Ok::<(), RecorderError>(())
+            },
+            async {
+                {
+                    let monitor = task.setup_monitor().await?;
+                    if graceful_shutdown {
+                        monitor
+                            .run_with_signal(async move {
+                                Self::shutdown_signal().await;
+                                tracing::info!("apalis shutting down...");
+                                Ok(())
+                            })
+                            .await?;
+                    } else {
+                        monitor.run().await?;
+                    }
+                }
+
+                Ok::<(), RecorderError>(())
+            },
+            async {
+                let listener = task.setup_listener().await?;
+                listener.listen().await?;
+
+                Ok::<(), RecorderError>(())
+            }
+        )?;
 
         Ok(())
     }
@@ -78,12 +142,25 @@ impl App {
                 .await;
         };
 
+        #[cfg(all(unix, debug_assertions))]
+        let quit = async {
+            signal::unix::signal(signal::unix::SignalKind::quit())
+                .expect("Failed to install SIGQUIT handler")
+                .recv()
+                .await;
+            println!("Received SIGQUIT");
+        };
+
         #[cfg(not(unix))]
         let terminate = std::future::pending::<()>();
 
+        #[cfg(not(all(unix, debug_assertions)))]
+        let quit = std::future::pending::<()>();
+
         tokio::select! {
             () = ctrl_c => {},
             () = terminate => {},
+            () = quit => {},
         }
     }
 }

apps/recorder/src/app/mod.rs

@@ -4,9 +4,9 @@ pub mod context;
 pub mod core;
 pub mod env;
 
-pub use core::App;
+pub use core::{App, PROJECT_NAME};
 
 pub use builder::AppBuilder;
 pub use config::AppConfig;
-pub use context::AppContext;
+pub use context::{AppContext, AppContextTrait};
 pub use env::Environment;

apps/recorder/src/auth/basic.rs

@@ -1,7 +1,7 @@
 use async_trait::async_trait;
 use axum::http::{HeaderValue, request::Parts};
 use base64::{self, Engine};
-use reqwest::header::AUTHORIZATION;
+use http::header::AUTHORIZATION;
 
 use super::{
     config::BasicAuthConfig,
@@ -9,7 +9,7 @@ use super::{
     service::{AuthServiceTrait, AuthUserInfo},
 };
 use crate::{
-    app::AppContext,
+    app::{AppContextTrait, PROJECT_NAME},
     models::{auth::AuthType, subscribers::SEED_SUBSCRIBER},
 };
 
@@ -64,15 +64,14 @@ pub struct BasicAuthService {
 impl AuthServiceTrait for BasicAuthService {
     async fn extract_user_info(
         &self,
-        ctx: &AppContext,
+        ctx: &dyn AppContextTrait,
         request: &mut Parts,
     ) -> Result<AuthUserInfo, AuthError> {
         if let Ok(AuthBasic {
             user: found_user,
             password: found_password,
         }) = AuthBasic::decode_request_parts(request)
-        {
-            if self.config.user == found_user
+            && self.config.user == found_user
             && self.config.password == found_password.unwrap_or_default()
         {
             let subscriber_auth = crate::models::auth::Model::find_by_pid(ctx, SEED_SUBSCRIBER)
@@ -83,12 +82,11 @@ impl AuthServiceTrait for BasicAuthService {
                 auth_type: AuthType::Basic,
             });
         }
-        }
         Err(AuthError::BasicInvalidCredentials)
     }
 
     fn www_authenticate_header_value(&self) -> Option<HeaderValue> {
-        Some(HeaderValue::from_static(r#"Basic realm="konobangu""#))
+        Some(HeaderValue::from_str(format!("Basic realm=\"{PROJECT_NAME}\"").as_str()).unwrap())
     }
 
     fn auth_type(&self) -> AuthType {

apps/recorder/src/auth/config.rs

@@ -1,6 +1,8 @@
-use jwt_authorizer::OneOrArray;
+use std::collections::HashMap;
+
+use jwtk::OneOrMany;
 use serde::{Deserialize, Serialize};
-use serde_with::{NoneAsEmptyString, serde_as};
+use serde_with::serde_as;
 
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub struct BasicAuthConfig {
@@ -22,13 +24,9 @@ pub struct OidcAuthConfig {
     #[serde(rename = "oidc_client_secret")]
     pub client_secret: String,
     #[serde(rename = "oidc_extra_scopes")]
-    pub extra_scopes: Option<OneOrArray<String>>,
-    #[serde_as(as = "NoneAsEmptyString")]
-    #[serde(rename = "oidc_extra_claim_key")]
-    pub extra_claim_key: Option<String>,
-    #[serde(rename = "oidc_extra_claim_value")]
-    #[serde_as(as = "NoneAsEmptyString")]
-    pub extra_claim_value: Option<String>,
+    pub extra_scopes: Option<OneOrMany<String>>,
+    #[serde(rename = "oidc_extra_claims")]
+    pub extra_claims: Option<HashMap<String, Option<String>>>,
 }
 
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]

apps/recorder/src/auth/errors.rs

@@ -1,82 +1,95 @@
-use std::fmt;
-
 use async_graphql::dynamic::ResolverContext;
 use axum::{
     Json,
     http::StatusCode,
     response::{IntoResponse, Response},
 };
+use fetch::HttpClientError;
 use openidconnect::{
     ConfigurationError, RequestTokenError, SignatureVerificationError, SigningError,
     StandardErrorResponse, core::CoreErrorResponseType,
 };
 use serde::{Deserialize, Serialize};
-use thiserror::Error;
+use snafu::prelude::*;
 
-use crate::{fetch::HttpClientError, models::auth::AuthType};
+use crate::models::auth::AuthType;
 
-#[derive(Debug, Error)]
+#[derive(Debug, Snafu)]
+#[snafu(visibility(pub(crate)))]
 pub enum AuthError {
-    #[error("Not support auth method")]
+    #[snafu(display("Permission denied"))]
+    PermissionError,
+    #[snafu(display("Not support auth method"))]
     NotSupportAuthMethod {
         supported: Vec<AuthType>,
         current: AuthType,
     },
-    #[error("Failed to find auth record")]
+    #[snafu(display("Failed to find auth record"))]
     FindAuthRecordError,
-    #[error("Invalid credentials")]
+    #[snafu(display("Invalid credentials"))]
     BasicInvalidCredentials,
-    #[error(transparent)]
-    OidcInitError(#[from] jwt_authorizer::error::InitError),
-    #[error("Invalid oidc provider meta client error: {0}")]
-    OidcProviderHttpClientError(HttpClientError),
-    #[error(transparent)]
-    OidcProviderMetaError(#[from] openidconnect::DiscoveryError<HttpClientError>),
-    #[error("Invalid oidc provider URL: {0}")]
-    OidcProviderUrlError(url::ParseError),
-    #[error("Invalid oidc redirect URI: {0}")]
-    OidcRequestRedirectUriError(url::ParseError),
-    #[error("Oidc request session not found or expired")]
+    #[snafu(display("Invalid oidc provider meta client error: {source}"))]
+    OidcProviderHttpClientError { source: HttpClientError },
+    #[snafu(transparent)]
+    OidcProviderMetaError {
+        source: openidconnect::DiscoveryError<HttpClientError>,
+    },
+    #[snafu(display("Invalid oidc provider URL: {source}"))]
+    OidcProviderUrlError { source: url::ParseError },
+    #[snafu(display("Invalid oidc redirect URI: {source}"))]
+    OidcRequestRedirectUriError {
+        #[snafu(source)]
+        source: url::ParseError,
+    },
+    #[snafu(display("Oidc request session not found or expired"))]
     OidcCallbackRecordNotFoundOrExpiredError,
-    #[error("Invalid oidc request callback nonce")]
+    #[snafu(display("Invalid oidc request callback nonce"))]
     OidcInvalidNonceError,
-    #[error("Invalid oidc request callback state")]
+    #[snafu(display("Invalid oidc request callback state"))]
     OidcInvalidStateError,
-    #[error("Invalid oidc request callback code")]
+    #[snafu(display("Invalid oidc request callback code"))]
     OidcInvalidCodeError,
-    #[error(transparent)]
-    OidcCallbackTokenConfigrationError(#[from] ConfigurationError),
-    #[error(transparent)]
-    OidcRequestTokenError(
-        #[from] RequestTokenError<HttpClientError, StandardErrorResponse<CoreErrorResponseType>>,
-    ),
-    #[error("Invalid oidc id token")]
+    #[snafu(transparent)]
+    OidcCallbackTokenConfigurationError { source: ConfigurationError },
+    #[snafu(transparent)]
+    OidcRequestTokenError {
+        source: RequestTokenError<HttpClientError, StandardErrorResponse<CoreErrorResponseType>>,
+    },
+    #[snafu(display("Invalid oidc id token"))]
     OidcInvalidIdTokenError,
-    #[error("Invalid oidc access token")]
+    #[snafu(display("Invalid oidc access token"))]
     OidcInvalidAccessTokenError,
-    #[error(transparent)]
-    OidcSignatureVerificationError(#[from] SignatureVerificationError),
-    #[error(transparent)]
-    OidcSigningError(#[from] SigningError),
-    #[error(transparent)]
-    OidcJwtAuthError(#[from] jwt_authorizer::AuthError),
-    #[error("Extra scopes {expected} do not match found scopes {found}")]
+    #[snafu(transparent)]
+    OidcSignatureVerificationError { source: SignatureVerificationError },
+    #[snafu(transparent)]
+    OidcSigningError { source: SigningError },
+    #[snafu(display("Missing Bearer token"))]
+    OidcMissingBearerToken,
+    #[snafu(transparent)]
+    OidcJwtkError { source: jwtk::Error },
+    #[snafu(display("Extra scopes {expected} do not match found scopes {found}"))]
     OidcExtraScopesMatchError { expected: String, found: String },
-    #[error("Extra claim {key} does not match expected value {expected}, found {found}")]
+    #[snafu(display("Extra claim {key} does not match expected value {expected}, found {found}"))]
     OidcExtraClaimMatchError {
         key: String,
         expected: String,
         found: String,
     },
-    #[error("Extra claim {0} missing")]
-    OidcExtraClaimMissingError(String),
-    #[error("Audience {0} missing")]
-    OidcAudMissingError(String),
-    #[error("Subject missing")]
+    #[snafu(display("Extra claim {claim} missing"))]
+    OidcExtraClaimMissingError { claim: String },
+    #[snafu(display("Audience {aud} missing"))]
+    OidcAudMissingError { aud: String },
+    #[snafu(display("Subject missing"))]
     OidcSubMissingError,
-    #[error(fmt = display_graphql_permission_error)]
-    GraphQLPermissionError {
-        inner_error: async_graphql::Error,
+    #[snafu(display(
+        "GraphQL permission denied since {context_path}{}{field}{}{column}: {}",
+        (if field.is_empty() { "" } else { "." }),
+        (if column.is_empty() { "" } else { "." }),
+        source.message
+    ))]
+    GraphqlDynamicPermissionError {
+        #[snafu(source(false))]
+        source: Box<async_graphql::Error>,
         field: String,
         column: String,
         context_path: String,
@@ -84,14 +97,14 @@ pub enum AuthError {
 }
 
 impl AuthError {
-    pub fn from_graphql_subscribe_id_guard(
-        inner_error: async_graphql::Error,
+    pub fn from_graphql_dynamic_subscribe_id_guard(
+        source: async_graphql::Error,
         context: &ResolverContext,
         field_name: &str,
         column_name: &str,
     ) -> AuthError {
-        AuthError::GraphQLPermissionError {
-            inner_error,
+        AuthError::GraphqlDynamicPermissionError {
+            source: Box::new(source),
             field: field_name.to_string(),
             column: column_name.to_string(),
             context_path: context
@@ -103,39 +116,27 @@ impl AuthError {
     }
 }
 
-fn display_graphql_permission_error(
-    inner_error: &async_graphql::Error,
-    field: &String,
-    column: &String,
-    context_path: &String,
-    formatter: &mut fmt::Formatter<'_>,
-) -> fmt::Result {
-    write!(
-        formatter,
-        "GraphQL permission denied since {context_path}{}{field}{}{column}: {}",
-        (if field.is_empty() { "" } else { "." }),
-        (if column.is_empty() { "" } else { "." }),
-        inner_error.message
-    )
-}
-
 #[derive(Clone, Debug, Serialize, Deserialize)]
-pub struct AuthErrorBody {
-    pub error_code: i32,
-    pub error_msg: String,
+pub struct AuthErrorResponse {
+    pub success: bool,
+    pub message: String,
 }
 
-impl From<AuthError> for AuthErrorBody {
+impl From<AuthError> for AuthErrorResponse {
     fn from(value: AuthError) -> Self {
-        AuthErrorBody {
-            error_code: StatusCode::UNAUTHORIZED.as_u16() as i32,
-            error_msg: value.to_string(),
+        AuthErrorResponse {
+            success: false,
+            message: value.to_string(),
         }
     }
 }
 
 impl IntoResponse for AuthError {
     fn into_response(self) -> Response {
-        (StatusCode::UNAUTHORIZED, Json(AuthErrorBody::from(self))).into_response()
+        (
+            StatusCode::UNAUTHORIZED,
+            Json(AuthErrorResponse::from(self)),
+        )
+            .into_response()
     }
 }

apps/recorder/src/auth/event.ts

@@ -1,41 +0,0 @@
-import type { Observable } from '@graphiql/toolkit';
-import { InjectionToken, inject } from '@outposts/injection-js';
-import {
-  type AuthFeature,
-  EventTypes,
-  PublicEventsService,
-} from 'oidc-client-rx';
-import { filter, shareReplay } from 'rxjs';
-
-export type CheckAuthResultEventType =
-  | { type: EventTypes.CheckingAuthFinished }
-  | {
-      type: EventTypes.CheckingAuthFinishedWithError;
-      value: string;
-    };
-export const CHECK_AUTH_RESULT_EVENT = new InjectionToken<
-  Observable<CheckAuthResultEventType>
->('CHECK_AUTH_RESULT_EVENT');
-
-export function withCheckAuthResultEvent(): AuthFeature {
-  return {
-    ɵproviders: [
-      {
-        provide: CHECK_AUTH_RESULT_EVENT,
-        useFactory: () => {
-          const publishEventService = inject(PublicEventsService);
-
-          return publishEventService.registerForEvents().pipe(
-            filter(
-              (e) =>
-                e.type === EventTypes.CheckingAuthFinishedWithError ||
-                e.type === EventTypes.CheckingAuthFinished
-            ),
-            shareReplay(1)
-          );
-        },
-        deps: [PublicEventsService],
-      },
-    ],
-  };
-}

apps/recorder/src/auth/guard.ts

@@ -1,19 +0,0 @@
-import { runInInjectionContext } from '@outposts/injection-js';
-import { autoLoginPartialRoutesGuard } from 'oidc-client-rx';
-import { firstValueFrom } from 'rxjs';
-import type { RouterContext } from '../controllers/__root';
-
-export const beforeLoadGuard = async ({
-  context,
-}: { context: RouterContext }) => {
-  if (!context.isAuthenticated) {
-    const guard$ = runInInjectionContext(context.injector, () =>
-      autoLoginPartialRoutesGuard()
-    );
-
-    const isAuthenticated = await firstValueFrom(guard$);
-    if (!isAuthenticated) {
-      throw !isAuthenticated;
-    }
-  }
-};

apps/recorder/src/auth/hooks.ts

@@ -1,52 +0,0 @@
-import { useObservableEagerState, useObservableState } from 'observable-hooks';
-import {
-  InjectorContextVoidInjector,
-  useOidcClient,
-} from 'oidc-client-rx/adapters/react';
-import { useMemo } from 'react';
-import { NEVER, type Observable, of } from 'rxjs';
-import { isBasicAuth } from './config';
-import {
-  CHECK_AUTH_RESULT_EVENT,
-  type CheckAuthResultEventType,
-} from './event';
-
-const BASIC_AUTH_IS_AUTHENTICATED$ = of({
-  isAuthenticated: true,
-  allConfigsAuthenticated: [],
-});
-
-const BASIC_AUTH_USER_DATA$ = of({
-  userData: {},
-  allUserData: [],
-});
-
-export function useAuth() {
-  const { oidcSecurityService, injector } = isBasicAuth
-    ? { oidcSecurityService: undefined, injector: InjectorContextVoidInjector }
-    : // biome-ignore lint/correctness/useHookAtTopLevel: <explanation>
-      useOidcClient();
-
-  const { isAuthenticated } = useObservableEagerState(
-    oidcSecurityService?.isAuthenticated$ ?? BASIC_AUTH_IS_AUTHENTICATED$
-  );
-
-  const { userData } = useObservableEagerState(
-    oidcSecurityService?.userData$ ?? BASIC_AUTH_USER_DATA$
-  );
-
-  const checkAuthResultEvent = useObservableState(
-    useMemo(
-      () => (isBasicAuth ? NEVER : injector.get(CHECK_AUTH_RESULT_EVENT)),
-      [injector]
-    ) as Observable<CheckAuthResultEventType>
-  );
-
-  return {
-    oidcSecurityService,
-    isAuthenticated,
-    userData,
-    injector,
-    checkAuthResultEvent,
-  };
-}

apps/recorder/src/auth/middleware.rs

@@ -7,18 +7,21 @@ use axum::{
     response::{IntoResponse, Response},
 };
 
-use crate::{app::AppContext, auth::AuthServiceTrait};
+use crate::{app::AppContextTrait, auth::AuthServiceTrait};
 
-pub async fn header_www_authenticate_middleware(
-    State(ctx): State<Arc<AppContext>>,
+pub async fn auth_middleware(
+    State(ctx): State<Arc<dyn AppContextTrait>>,
     request: Request,
     next: Next,
 ) -> Response {
-    let auth_service = &ctx.auth;
+    let auth_service = ctx.auth();
 
     let (mut parts, body) = request.into_parts();
 
-    let mut response = match auth_service.extract_user_info(&ctx, &mut parts).await {
+    let mut response = match auth_service
+        .extract_user_info(ctx.as_ref() as &dyn AppContextTrait, &mut parts)
+        .await
+    {
         Ok(auth_user_info) => {
             let mut request = Request::from_parts(parts, body);
             request.extensions_mut().insert(auth_user_info);

apps/recorder/src/auth/mod.rs

@@ -7,5 +7,5 @@ pub mod service;
 
 pub use config::{AuthConfig, BasicAuthConfig, OidcAuthConfig};
 pub use errors::AuthError;
-pub use middleware::header_www_authenticate_middleware;
+pub use middleware::auth_middleware;
 pub use service::{AuthService, AuthServiceTrait, AuthUserInfo};

apps/recorder/src/auth/oidc.rs

@@ -1,12 +1,20 @@
 use std::{
     collections::{HashMap, HashSet},
+    future::Future,
+    ops::Deref,
+    pin::Pin,
     sync::Arc,
 };
 
 use async_trait::async_trait;
-use axum::http::{HeaderValue, request::Parts};
+use axum::{
+    http,
+    http::{HeaderValue, request::Parts},
+};
+use fetch::{HttpClient, client::HttpClientError};
+use http::header::AUTHORIZATION;
 use itertools::Itertools;
-use jwt_authorizer::{NumericDate, OneOrArray, authorizer::Authorizer};
+use jwtk::jwk::RemoteJwksVerifier;
 use moka::future::Cache;
 use openidconnect::{
     AccessTokenHash, AuthorizationCode, ClientId, ClientSecret, CsrfToken, IssuerUrl, Nonce,
@@ -16,32 +24,64 @@ use openidconnect::{
 use sea_orm::DbErr;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
+use snafu::ResultExt;
 use url::Url;
 
 use super::{
     config::OidcAuthConfig,
-    errors::AuthError,
+    errors::{AuthError, OidcProviderUrlSnafu, OidcRequestRedirectUriSnafu},
     service::{AuthServiceTrait, AuthUserInfo},
 };
-use crate::{app::AppContext, errors::RError, fetch::HttpClient, models::auth::AuthType};
+use crate::{
+    app::{AppContextTrait, PROJECT_NAME},
+    errors::RecorderError,
+    models::auth::AuthType,
+};
+
+pub struct OidcHttpClient(pub Arc<HttpClient>);
+
+impl Deref for OidcHttpClient {
+    type Target = HttpClient;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl<'c> openidconnect::AsyncHttpClient<'c> for OidcHttpClient {
+    type Error = HttpClientError;
+
+    #[cfg(target_arch = "wasm32")]
+    type Future =
+        Pin<Box<dyn Future<Output = Result<openidconnect::HttpResponse, Self::Error>> + 'c>>;
+    #[cfg(not(target_arch = "wasm32"))]
+    type Future =
+        Pin<Box<dyn Future<Output = Result<openidconnect::HttpResponse, Self::Error>> + Send + 'c>>;
+
+    fn call(&'c self, request: openidconnect::HttpRequest) -> Self::Future {
+        Box::pin(async move {
+            let response = self.execute(request.try_into()?).await?;
+
+            let mut builder = http::Response::builder().status(response.status());
+
+            #[cfg(not(target_arch = "wasm32"))]
+            {
+                builder = builder.version(response.version());
+            }
+
+            for (name, value) in response.headers().iter() {
+                builder = builder.header(name, value);
+            }
+
+            builder
+                .body(response.bytes().await?.to_vec())
+                .map_err(HttpClientError::from)
+        })
+    }
+}
 
 #[derive(Deserialize, Serialize, Clone, Debug)]
 pub struct OidcAuthClaims {
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub iss: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub sub: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub aud: Option<OneOrArray<String>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub exp: Option<NumericDate>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub nbf: Option<NumericDate>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub iat: Option<NumericDate>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub jti: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub scope: Option<String>,
     #[serde(flatten)]
     pub custom: HashMap<String, Value>,
@@ -51,40 +91,6 @@ impl OidcAuthClaims {
     pub fn scopes(&self) -> std::str::Split<'_, char> {
         self.scope.as_deref().unwrap_or_default().split(',')
     }
-
-    pub fn get_claim(&self, key: &str) -> Option<String> {
-        match key {
-            "iss" => self.iss.clone(),
-            "sub" => self.sub.clone(),
-            "aud" => self.aud.as_ref().map(|s| s.iter().join(",")),
-            "exp" => self.exp.clone().map(|s| s.0.to_string()),
-            "nbf" => self.nbf.clone().map(|s| s.0.to_string()),
-            "iat" => self.iat.clone().map(|s| s.0.to_string()),
-            "jti" => self.jti.clone(),
-            "scope" => self.scope.clone(),
-            key => self.custom.get(key).map(|s| s.to_string()),
-        }
-    }
-
-    pub fn has_claim(&self, key: &str) -> bool {
-        match key {
-            "iss" => self.iss.is_some(),
-            "sub" => self.sub.is_some(),
-            "aud" => self.aud.is_some(),
-            "exp" => self.exp.is_some(),
-            "nbf" => self.nbf.is_some(),
-            "iat" => self.iat.is_some(),
-            "jti" => self.jti.is_some(),
-            "scope" => self.scope.is_some(),
-            key => self.custom.contains_key(key),
-        }
-    }
-
-    pub fn contains_audience(&self, aud: &str) -> bool {
-        self.aud
-            .as_ref()
-            .is_some_and(|arr| arr.iter().any(|s| s == aud))
-    }
 }
 
 #[derive(Debug, Clone, Serialize)]
@@ -114,8 +120,8 @@ pub struct OidcAuthCallbackPayload {
 
 pub struct OidcAuthService {
     pub config: OidcAuthConfig,
-    pub api_authorizer: Authorizer<OidcAuthClaims>,
-    pub oidc_provider_client: HttpClient,
+    pub jwk_verifier: RemoteJwksVerifier,
+    pub oidc_provider_client: Arc<HttpClient>,
     pub oidc_request_cache: Cache<String, OidcAuthRequest>,
 }
 
@@ -124,14 +130,15 @@ impl OidcAuthService {
         &self,
         redirect_uri: &str,
     ) -> Result<OidcAuthRequest, AuthError> {
+        let oidc_provider_client = OidcHttpClient(self.oidc_provider_client.clone());
         let provider_metadata = CoreProviderMetadata::discover_async(
-            IssuerUrl::new(self.config.issuer.clone()).map_err(AuthError::OidcProviderUrlError)?,
-            &self.oidc_provider_client,
+            IssuerUrl::new(self.config.issuer.clone()).context(OidcProviderUrlSnafu)?,
+            &oidc_provider_client,
         )
         .await?;
 
-        let redirect_uri = RedirectUrl::new(redirect_uri.to_string())
-            .map_err(AuthError::OidcRequestRedirectUriError)?;
+        let redirect_uri =
+            RedirectUrl::new(redirect_uri.to_string()).context(OidcRequestRedirectUriSnafu)?;
 
         let oidc_client = CoreClient::from_provider_metadata(
             provider_metadata,
@@ -200,6 +207,7 @@ impl OidcAuthService {
         &self,
         query: OidcAuthCallbackQuery,
     ) -> Result<OidcAuthCallbackPayload, AuthError> {
+        let oidc_http_client = OidcHttpClient(self.oidc_provider_client.clone());
         let csrf_token = query.state.ok_or(AuthError::OidcInvalidStateError)?;
 
         let code = query.code.ok_or(AuthError::OidcInvalidCodeError)?;
@@ -207,8 +215,8 @@ impl OidcAuthService {
         let request_cache = self.load_authorization_request(&csrf_token).await?;
 
         let provider_metadata = CoreProviderMetadata::discover_async(
-            IssuerUrl::new(self.config.issuer.clone()).map_err(AuthError::OidcProviderUrlError)?,
-            &self.oidc_provider_client,
+            IssuerUrl::new(self.config.issuer.clone()).context(OidcProviderUrlSnafu)?,
+            &oidc_http_client,
         )
         .await?;
 
@@ -224,7 +232,7 @@ impl OidcAuthService {
         let token_response = oidc_client
             .exchange_code(AuthorizationCode::new(code))?
             .set_pkce_verifier(pkce_verifier)
-            .request_async(&HttpClient::default())
+            .request_async(&oidc_http_client)
             .await?;
 
         let id_token = token_response
@@ -245,11 +253,11 @@ impl OidcAuthService {
             id_token.signing_key(id_token_verifier)?,
         )?;
 
-        if let Some(expected_access_token_hash) = claims.access_token_hash() {
-            if actual_access_token_hash != *expected_access_token_hash {
+        if let Some(expected_access_token_hash) = claims.access_token_hash()
+            && actual_access_token_hash != *expected_access_token_hash
+        {
             return Err(AuthError::OidcInvalidAccessTokenError);
         }
-        }
 
         Ok(OidcAuthCallbackPayload {
             access_token: access_token.secret().to_string(),
@@ -261,57 +269,84 @@ impl OidcAuthService {
 impl AuthServiceTrait for OidcAuthService {
     async fn extract_user_info(
         &self,
-        ctx: &AppContext,
+        ctx: &dyn AppContextTrait,
         request: &mut Parts,
     ) -> Result<AuthUserInfo, AuthError> {
         let config = &self.config;
-        let token = self.api_authorizer.extract_token(&request.headers).ok_or(
-            AuthError::OidcJwtAuthError(jwt_authorizer::AuthError::MissingToken()),
-        )?;
+        let token = request
+            .headers
+            .get(AUTHORIZATION)
+            .and_then(|authorization| {
+                authorization
+                    .to_str()
+                    .ok()
+                    .and_then(|s| s.strip_prefix("Bearer "))
+            })
+            .ok_or(AuthError::OidcMissingBearerToken)?;
 
-        let token_data = self.api_authorizer.check_auth(&token).await?;
-        let claims = token_data.claims;
+        let token_data = self.jwk_verifier.verify::<OidcAuthClaims>(token).await?;
+        let claims = token_data.claims();
         let sub = if let Some(sub) = claims.sub.as_deref() {
             sub
         } else {
             return Err(AuthError::OidcSubMissingError);
         };
-        if !claims.contains_audience(&config.audience) {
-            return Err(AuthError::OidcAudMissingError(config.audience.clone()));
+        if !claims.aud.iter().any(|aud| aud == &config.audience) {
+            return Err(AuthError::OidcAudMissingError {
+                aud: config.audience.clone(),
+            });
         }
+        let extra_claims = &claims.extra;
         if let Some(expected_scopes) = config.extra_scopes.as_ref() {
-            let found_scopes = claims.scopes().collect::<HashSet<_>>();
+            let found_scopes = extra_claims.scopes().collect::<HashSet<_>>();
             if !expected_scopes
                 .iter()
                 .all(|es| found_scopes.contains(es as &str))
             {
                 return Err(AuthError::OidcExtraScopesMatchError {
                     expected: expected_scopes.iter().join(","),
-                    found: claims.scope.unwrap_or_default(),
+                    found: extra_claims
+                        .scope
+                        .as_deref()
+                        .unwrap_or_default()
+                        .to_string(),
                 });
             }
         }
-        if let Some(key) = config.extra_claim_key.as_ref() {
-            if !claims.has_claim(key) {
-                return Err(AuthError::OidcExtraClaimMissingError(key.clone()));
-            }
-            if let Some(value) = config.extra_claim_value.as_ref() {
-                if claims.get_claim(key).is_none_or(|v| &v != value) {
+        if let Some(expected_extra_claims) = config.extra_claims.as_ref() {
+            for (expected_key, expected_value) in expected_extra_claims.iter() {
+                match (extra_claims.custom.get(expected_key), expected_value) {
+                    (found_value, Some(expected_value)) => {
+                        if let Some(Value::String(found_value)) = found_value
+                            && expected_value == found_value
+                        {
+                        } else {
                             return Err(AuthError::OidcExtraClaimMatchError {
-                        expected: value.clone(),
-                        found: claims.get_claim(key).unwrap_or_default().to_string(),
-                        key: key.clone(),
+                                expected: expected_value.clone(),
+                                found: found_value.map(|v| v.to_string()).unwrap_or_default(),
+                                key: expected_key.clone(),
                             });
                         }
                     }
+                    (None, None) => {
+                        return Err(AuthError::OidcExtraClaimMissingError {
+                            claim: expected_key.clone(),
+                        });
+                    }
+                    _ => {}
+                }
+            }
         }
         let subscriber_auth = match crate::models::auth::Model::find_by_pid(ctx, sub).await {
-            Err(RError::DbError(DbErr::RecordNotFound(..))) => {
-                crate::models::auth::Model::create_from_oidc(ctx, sub.to_string()).await
-            }
+            Err(RecorderError::DbError {
+                source: DbErr::RecordNotFound(..),
+            }) => crate::models::auth::Model::create_from_oidc(ctx, sub.to_string()).await,
             r => r,
         }
-        .map_err(|_| AuthError::FindAuthRecordError)?;
+        .map_err(|e| {
+            tracing::error!("Error finding auth record: {:?}", e);
+            AuthError::FindAuthRecordError
+        })?;
 
         Ok(AuthUserInfo {
             subscriber_auth,
@@ -320,7 +355,7 @@ impl AuthServiceTrait for OidcAuthService {
     }
 
     fn www_authenticate_header_value(&self) -> Option<HeaderValue> {
-        Some(HeaderValue::from_static(r#"Bearer realm="konobangu""#))
+        Some(HeaderValue::from_str(format!("Bearer realm=\"{PROJECT_NAME}\"").as_str()).unwrap())
     }
 
     fn auth_type(&self) -> AuthType {

apps/recorder/src/auth/service.rs

@@ -1,29 +1,24 @@
-use std::time::Duration;
+use std::{sync::Arc, time::Duration};
 
 use async_trait::async_trait;
-use axum::{
-    extract::FromRequestParts,
-    http::request::Parts,
-    response::{IntoResponse as _, Response},
+use axum::http::request::Parts;
+use fetch::{
+    HttpClient, HttpClientConfig,
+    client::{HttpClientCacheBackendConfig, HttpClientCachePresetConfig},
 };
-use jwt_authorizer::{JwtAuthorizer, Validation};
+use http::header::HeaderValue;
+use jwtk::jwk::RemoteJwksVerifier;
 use moka::future::Cache;
-use reqwest::header::HeaderValue;
+use openidconnect::{IssuerUrl, core::CoreProviderMetadata};
+use snafu::prelude::*;
 
 use super::{
     AuthConfig,
     basic::BasicAuthService,
-    errors::AuthError,
-    oidc::{OidcAuthClaims, OidcAuthService},
-};
-use crate::{
-    app::AppContext,
-    fetch::{
-        HttpClient, HttpClientConfig,
-        client::{HttpClientCacheBackendConfig, HttpClientCachePresetConfig},
-    },
-    models::auth::AuthType,
+    errors::{AuthError, OidcProviderHttpClientSnafu, OidcProviderUrlSnafu},
+    oidc::{OidcAuthService, OidcHttpClient},
 };
+use crate::{app::AppContextTrait, models::auth::AuthType};
 
 #[derive(Clone, Debug)]
 pub struct AuthUserInfo {
@@ -31,27 +26,11 @@ pub struct AuthUserInfo {
     pub auth_type: AuthType,
 }
 
-impl FromRequestParts<AppContext> for AuthUserInfo {
-    type Rejection = Response;
-
-    async fn from_request_parts(
-        parts: &mut Parts,
-        state: &AppContext,
-    ) -> Result<Self, Self::Rejection> {
-        let auth_service = &state.auth;
-
-        auth_service
-            .extract_user_info(state, parts)
-            .await
-            .map_err(|err| err.into_response())
-    }
-}
-
 #[async_trait]
 pub trait AuthServiceTrait {
     async fn extract_user_info(
         &self,
-        ctx: &AppContext,
+        ctx: &dyn AppContextTrait,
         request: &mut Parts,
     ) -> Result<AuthUserInfo, AuthError>;
     fn www_authenticate_header_value(&self) -> Option<HeaderValue>;
@@ -59,41 +38,47 @@ pub trait AuthServiceTrait {
 }
 
 pub enum AuthService {
-    Basic(BasicAuthService),
-    Oidc(OidcAuthService),
+    Basic(Box<BasicAuthService>),
+    Oidc(Box<OidcAuthService>),
 }
 
 impl AuthService {
     pub async fn from_conf(config: AuthConfig) -> Result<Self, AuthError> {
         let result = match config {
-            AuthConfig::Basic(config) => AuthService::Basic(BasicAuthService { config }),
+            AuthConfig::Basic(config) => AuthService::Basic(Box::new(BasicAuthService { config })),
             AuthConfig::Oidc(config) => {
-                let validation = Validation::new()
-                    .iss(&[&config.issuer])
-                    .aud(&[&config.audience]);
-
-                let oidc_provider_client = HttpClient::from_config(HttpClientConfig {
+                let oidc_provider_client = Arc::new(
+                    HttpClient::from_config(HttpClientConfig {
                         exponential_backoff_max_retries: Some(3),
                         cache_backend: Some(HttpClientCacheBackendConfig::Moka { cache_size: 1 }),
                         cache_preset: Some(HttpClientCachePresetConfig::RFC7234),
                         ..Default::default()
                     })
-                .map_err(AuthError::OidcProviderHttpClientError)?;
+                    .context(OidcProviderHttpClientSnafu)?,
+                );
 
-                let api_authorizer = JwtAuthorizer::<OidcAuthClaims>::from_oidc(&config.issuer)
-                    .validation(validation)
-                    .build()
-                    .await?;
+                let provider_metadata = {
+                    let client = OidcHttpClient(oidc_provider_client.clone());
+                    let issuer_url =
+                        IssuerUrl::new(config.issuer.clone()).context(OidcProviderUrlSnafu)?;
+                    CoreProviderMetadata::discover_async(issuer_url, &client).await
+                }?;
 
-                AuthService::Oidc(OidcAuthService {
+                let jwk_verifier = RemoteJwksVerifier::new(
+                    provider_metadata.jwks_uri().to_string().clone(),
+                    None,
+                    Duration::from_secs(300),
+                );
+
+                AuthService::Oidc(Box::new(OidcAuthService {
                     config,
-                    api_authorizer,
+                    jwk_verifier,
                     oidc_provider_client,
                     oidc_request_cache: Cache::builder()
                         .time_to_live(Duration::from_mins(5))
                         .name("oidc_request_cache")
                         .build(),
-                })
+                }))
             }
         };
         Ok(result)
@@ -102,9 +87,10 @@ impl AuthService {
 
 #[async_trait]
 impl AuthServiceTrait for AuthService {
+    #[tracing::instrument(skip(self, ctx, request))]
     async fn extract_user_info(
         &self,
-        ctx: &AppContext,
+        ctx: &dyn AppContextTrait,
         request: &mut Parts,
     ) -> Result<AuthUserInfo, AuthError> {
         match self {

apps/recorder/src/bin/main.rs

@@ -1,10 +1,7 @@
-use color_eyre::{self, eyre};
-use recorder::app::AppBuilder;
+use recorder::{app::AppBuilder, errors::RecorderResult};
 
 #[tokio::main]
-async fn main() -> eyre::Result<()> {
-    color_eyre::install()?;
-
+async fn main() -> RecorderResult<()> {
     let builder = AppBuilder::from_main_cli(None).await?;
 
     let app = builder.build().await?;

apps/recorder/src/bin/migrate_down.rs

@@ -0,0 +1,16 @@
+use recorder::{app::AppBuilder, database::DatabaseService, errors::RecorderResult};
+
+#[tokio::main]
+async fn main() -> RecorderResult<()> {
+    let builder = AppBuilder::from_main_cli(None).await?;
+
+    builder.load_env().await?;
+    let mut database_config = builder.load_config().await?.database;
+    database_config.auto_migrate = false;
+
+    let database_service = DatabaseService::from_config(database_config).await?;
+
+    database_service.migrate_down().await?;
+
+    Ok(())
+}

apps/recorder/src/cache/service.rs

@@ -1,10 +1,10 @@
 use super::CacheConfig;
-use crate::errors::RResult;
+use crate::errors::RecorderResult;
 
 pub struct CacheService {}
 
 impl CacheService {
-    pub async fn from_config(_config: CacheConfig) -> RResult<Self> {
+    pub async fn from_config(_config: CacheConfig) -> RecorderResult<Self> {
         Ok(Self {})
     }
 }

apps/recorder/src/crypto/config.rs

@@ -0,0 +1,4 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct CryptoConfig {}

apps/recorder/src/crypto/error.rs

@@ -0,0 +1,20 @@
+use async_graphql::Error as AsyncGraphQLError;
+use seaography::SeaographyError;
+
+#[derive(Debug, snafu::Snafu)]
+pub enum CryptoError {
+    #[snafu(transparent)]
+    Base64DecodeError { source: base64::DecodeError },
+    #[snafu(display("CocoonError: {source:?}"), context(false))]
+    CocoonError { source: cocoon::Error },
+    #[snafu(transparent)]
+    FromUtf8Error { source: std::string::FromUtf8Error },
+    #[snafu(transparent)]
+    SerdeJsonError { source: serde_json::Error },
+}
+
+impl From<CryptoError> for SeaographyError {
+    fn from(error: CryptoError) -> Self {
+        SeaographyError::AsyncGraphQLError(AsyncGraphQLError::new(error.to_string()))
+    }
+}

apps/recorder/src/crypto/mod.rs

@@ -0,0 +1,9 @@
+pub mod config;
+pub mod error;
+pub mod service;
+pub mod userpass;
+
+pub use config::CryptoConfig;
+pub use error::CryptoError;
+pub use service::CryptoService;
+pub use userpass::UserPassCredential;

apps/recorder/src/crypto/service.rs

@@ -0,0 +1,62 @@
+use base64::prelude::{BASE64_URL_SAFE, *};
+use cocoon::Cocoon;
+use rand::Rng;
+use serde::{Deserialize, Serialize};
+
+use super::CryptoConfig;
+use crate::crypto::error::CryptoError;
+
+pub struct CryptoService {
+    #[allow(dead_code)]
+    config: CryptoConfig,
+}
+
+impl CryptoService {
+    pub async fn from_config(config: CryptoConfig) -> Result<Self, CryptoError> {
+        Ok(Self { config })
+    }
+
+    pub fn encrypt_string(&self, data: String) -> Result<String, CryptoError> {
+        let key = rand::rng().random::<[u8; 32]>();
+        let mut cocoon = Cocoon::new(&key);
+
+        let mut data = data.into_bytes();
+
+        let detached_prefix = cocoon.encrypt(&mut data)?;
+
+        let mut combined = Vec::with_capacity(key.len() + detached_prefix.len() + data.len());
+        combined.extend_from_slice(&key);
+        combined.extend_from_slice(&detached_prefix);
+        combined.extend_from_slice(&data);
+
+        Ok(BASE64_URL_SAFE.encode(combined))
+    }
+
+    pub fn decrypt_string(&self, data: &str) -> Result<String, CryptoError> {
+        let decoded = BASE64_URL_SAFE.decode(data)?;
+
+        let (key, remain) = decoded.split_at(32);
+        let (detached_prefix, data) = remain.split_at(60);
+        let mut data = data.to_vec();
+        let cocoon = Cocoon::new(key);
+
+        cocoon.decrypt(&mut data, detached_prefix)?;
+
+        String::from_utf8(data).map_err(CryptoError::from)
+    }
+
+    pub fn encrypt_serialize<T: Serialize>(&self, credentials: &T) -> Result<String, CryptoError> {
+        let json = serde_json::to_string(credentials)?;
+
+        self.encrypt_string(json)
+    }
+
+    pub fn decrypt_deserialize<T: for<'de> Deserialize<'de>>(
+        &self,
+        encrypted: &str,
+    ) -> Result<T, CryptoError> {
+        let data = self.decrypt_string(encrypted)?;
+
+        serde_json::from_str(&data).map_err(CryptoError::from)
+    }
+}

apps/recorder/src/crypto/userpass.rs

@@ -0,0 +1,19 @@
+use std::fmt::Debug;
+
+pub struct UserPassCredential {
+    pub username: String,
+    pub password: String,
+    pub user_agent: Option<String>,
+    pub cookies: Option<String>,
+}
+
+impl Debug for UserPassCredential {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("UserPassCredential")
+            .field("username", &"[Secret]")
+            .field("password", &"[Secret]")
+            .field("cookies", &"[Secret]")
+            .field("user_agent", &self.user_agent)
+            .finish()
+    }
+}

apps/recorder/src/database/service.rs

@@ -1,20 +1,26 @@
 use std::{ops::Deref, time::Duration};
 
+use apalis_sql::postgres::PostgresStorage;
 use sea_orm::{
-    ConnectOptions, ConnectionTrait, Database, DatabaseBackend, DatabaseConnection, DbBackend,
-    DbErr, ExecResult, QueryResult, Statement,
+    ConnectOptions, ConnectionTrait, Database, DatabaseConnection, DbBackend, DbErr, ExecResult,
+    QueryResult, Statement,
 };
 use sea_orm_migration::MigratorTrait;
 
 use super::DatabaseConfig;
-use crate::{errors::RResult, migrations::Migrator};
+use crate::{errors::RecorderResult, migrations::Migrator};
 
 pub struct DatabaseService {
+    pub config: DatabaseConfig,
     connection: DatabaseConnection,
+    #[cfg(feature = "testcontainers")]
+    pub container:
+        Option<testcontainers::ContainerAsync<testcontainers_modules::postgres::Postgres>>,
 }
 
 impl DatabaseService {
-    pub async fn from_config(config: DatabaseConfig) -> RResult<Self> {
+    pub async fn from_config(config: DatabaseConfig) -> RecorderResult<Self> {
+        let db_config = config.clone();
         let mut opt = ConnectOptions::new(&config.uri);
         opt.max_connections(config.max_connections)
             .min_connections(config.min_connections)
@@ -28,26 +34,52 @@ impl DatabaseService {
 
         let db = Database::connect(opt).await?;
 
-        if db.get_database_backend() == DatabaseBackend::Sqlite {
-            db.execute(Statement::from_string(
-                DatabaseBackend::Sqlite,
-                "
-                PRAGMA foreign_keys = ON;
-                PRAGMA journal_mode = WAL;
-                PRAGMA synchronous = NORMAL;
-                PRAGMA mmap_size = 134217728;
-                PRAGMA journal_size_limit = 67108864;
-                PRAGMA cache_size = 2000;
-                ",
-            ))
-            .await?;
-        }
+        // only support postgres for now
+        // if db.get_database_backend() == DatabaseBackend::Sqlite {
+        //     db.execute(Statement::from_string(
+        //         DatabaseBackend::Sqlite,
+        //         "
+        //         PRAGMA foreign_keys = ON;
+        //         PRAGMA journal_mode = WAL;
+        //         PRAGMA synchronous = NORMAL;
+        //         PRAGMA mmap_size = 134217728;
+        //         PRAGMA journal_size_limit = 67108864;
+        //         PRAGMA cache_size = 2000;
+        //         ",
+        //     ))
+        //     .await?;
+        // }
+
+        let me = Self {
+            connection: db,
+            #[cfg(feature = "testcontainers")]
+            container: None,
+            config: db_config,
+        };
 
         if config.auto_migrate {
-            Migrator::up(&db, None).await?;
+            me.migrate_up().await?;
         }
 
-        Ok(Self { connection: db })
+        Ok(me)
+    }
+
+    pub async fn migrate_up(&self) -> RecorderResult<()> {
+        {
+            let pool = &self.get_postgres_connection_pool();
+            PostgresStorage::setup(pool).await?;
+        }
+        Migrator::up(&self.connection, None).await?;
+        Ok(())
+    }
+
+    pub async fn migrate_down(&self) -> RecorderResult<()> {
+        Migrator::down(&self.connection, None).await?;
+        {
+            self.execute_unprepared(r#"DROP SCHEMA IF EXISTS apalis CASCADE"#)
+                .await?;
+        }
+        Ok(())
     }
 }
 

apps/recorder/src/errors/app_error.rs

@@ -0,0 +1,297 @@
+use std::borrow::Cow;
+
+use axum::{
+    Json,
+    response::{IntoResponse, Response},
+};
+use fetch::{FetchError, HttpClientError, reqwest, reqwest_middleware};
+use http::{HeaderMap, StatusCode};
+use snafu::Snafu;
+
+use crate::{
+    auth::AuthError,
+    crypto::CryptoError,
+    downloader::DownloaderError,
+    errors::{OptDynErr, response::StandardErrorResponse},
+};
+
+#[derive(Snafu, Debug)]
+#[snafu(visibility(pub(crate)))]
+pub enum RecorderError {
+    #[snafu(display(
+        "HTTP {status} {reason}, source = {source:?}",
+        status = status,
+        reason = status.canonical_reason().unwrap_or("Unknown")
+    ))]
+    HttpResponseError {
+        status: StatusCode,
+        headers: Option<HeaderMap>,
+        #[snafu(source(from(Box<dyn std::error::Error + Send + Sync>, OptDynErr::some)))]
+        source: OptDynErr,
+    },
+    #[snafu(transparent)]
+    ImageError { source: image::ImageError },
+    #[cfg(feature = "jxl")]
+    #[snafu(transparent)]
+    JxlEncodeError { source: jpegxl_rs::EncodeError },
+    #[snafu(transparent, context(false))]
+    HttpError { source: http::Error },
+    #[snafu(transparent, context(false))]
+    FancyRegexError {
+        #[snafu(source(from(fancy_regex::Error, Box::new)))]
+        source: Box<fancy_regex::Error>,
+    },
+    #[snafu(transparent)]
+    NetAddrParseError { source: std::net::AddrParseError },
+    #[snafu(transparent)]
+    RegexError { source: regex::Error },
+    #[snafu(display("Invalid method"))]
+    InvalidMethodError,
+    #[snafu(display("Invalid header value"))]
+    InvalidHeaderValueError,
+    #[snafu(display("Invalid header name"))]
+    InvalidHeaderNameError,
+    #[snafu(display("Missing origin (protocol or host) in headers and forwarded info"))]
+    MissingOriginError,
+    #[snafu(transparent)]
+    TracingAppenderInitError {
+        source: tracing_appender::rolling::InitError,
+    },
+    #[snafu(transparent)]
+    GraphQLSchemaError {
+        source: async_graphql::dynamic::SchemaError,
+    },
+    #[snafu(transparent)]
+    AuthError { source: AuthError },
+    #[snafu(transparent)]
+    DownloadError { source: DownloaderError },
+    #[snafu(transparent)]
+    RSSError { source: rss::Error },
+    #[snafu(transparent)]
+    DotEnvError { source: dotenvy::Error },
+    #[snafu(transparent)]
+    TeraError { source: tera::Error },
+    #[snafu(transparent)]
+    IOError { source: std::io::Error },
+    #[snafu(transparent)]
+    DbError { source: sea_orm::DbErr },
+    #[snafu(transparent)]
+    DbSqlxError { source: sea_orm::SqlxError },
+    #[snafu(transparent, context(false))]
+    FigmentError {
+        #[snafu(source(from(figment::Error, Box::new)))]
+        source: Box<figment::Error>,
+    },
+    #[snafu(transparent)]
+    SerdeJsonError { source: serde_json::Error },
+    #[snafu(transparent)]
+    ParseUrlError { source: url::ParseError },
+    #[snafu(display("{source}"), context(false))]
+    OpenDALError {
+        #[snafu(source(from(opendal::Error, Box::new)))]
+        source: Box<opendal::Error>,
+    },
+    #[snafu(transparent)]
+    HttpClientError { source: HttpClientError },
+    #[cfg(feature = "testcontainers")]
+    #[snafu(transparent)]
+    TestcontainersError {
+        source: testcontainers::TestcontainersError,
+    },
+    #[snafu(display("Extract {desc} with mime error, expected {expected}, but got {found}"))]
+    MimeError {
+        desc: String,
+        expected: String,
+        found: String,
+    },
+    #[snafu(display("Invalid or unknown format in extracting mikan rss"))]
+    MikanRssInvalidFormatError,
+    #[snafu(display("Invalid field {field} in extracting mikan rss"))]
+    MikanRssInvalidFieldError {
+        field: Cow<'static, str>,
+        #[snafu(source(from(Box<dyn std::error::Error + Send + Sync>, OptDynErr::some)))]
+        source: OptDynErr,
+    },
+    #[snafu(display("Missing field {field} in extracting mikan meta"))]
+    MikanMetaMissingFieldError {
+        field: Cow<'static, str>,
+        #[snafu(source(from(Box<dyn std::error::Error + Send + Sync>, OptDynErr::some)))]
+        source: OptDynErr,
+    },
+    #[snafu(display("Model Entity {entity} not found or not belong to subscriber"))]
+    ModelEntityNotFound { entity: Cow<'static, str> },
+    #[snafu(transparent)]
+    FetchError { source: FetchError },
+    #[snafu(display("Credential3rdError: {message}, source = {source}"))]
+    Credential3rdError {
+        message: String,
+        #[snafu(source(from(Box<dyn std::error::Error + Send + Sync>, OptDynErr::some)))]
+        source: OptDynErr,
+    },
+    #[snafu(transparent)]
+    CryptoError { source: CryptoError },
+    #[snafu(transparent)]
+    StringFromUtf8Error { source: std::string::FromUtf8Error },
+    #[snafu(display("{message}"))]
+    Whatever {
+        message: String,
+        #[snafu(source(from(Box<dyn std::error::Error + Send + Sync>, OptDynErr::some)))]
+        source: OptDynErr,
+    },
+    #[snafu(display("Invalid task id: {message}"))]
+    InvalidTaskId { message: String },
+}
+
+impl RecorderError {
+    pub fn from_status(status: StatusCode) -> Self {
+        Self::HttpResponseError {
+            status,
+            headers: None,
+            source: None.into(),
+        }
+    }
+
+    pub fn from_status_and_headers(status: StatusCode, headers: HeaderMap) -> Self {
+        Self::HttpResponseError {
+            status,
+            headers: Some(headers),
+            source: None.into(),
+        }
+    }
+
+    pub fn from_mikan_meta_missing_field(field: Cow<'static, str>) -> Self {
+        Self::MikanMetaMissingFieldError {
+            field,
+            source: None.into(),
+        }
+    }
+
+    pub fn from_mikan_rss_invalid_field(field: Cow<'static, str>) -> Self {
+        Self::MikanRssInvalidFieldError {
+            field,
+            source: None.into(),
+        }
+    }
+
+    pub fn from_mikan_rss_invalid_field_and_source(
+        field: Cow<'static, str>,
+        source: impl std::error::Error + Send + Sync + 'static,
+    ) -> Self {
+        Self::MikanRssInvalidFieldError {
+            field,
+            source: OptDynErr::some_boxed(source),
+        }
+    }
+
+    pub fn from_db_record_not_found<T: ToString>(detail: T) -> Self {
+        Self::DbError {
+            source: sea_orm::DbErr::RecordNotFound(detail.to_string()),
+        }
+    }
+}
+
+impl snafu::FromString for RecorderError {
+    type Source = Box<dyn std::error::Error + Send + Sync>;
+
+    fn without_source(message: String) -> Self {
+        Self::Whatever {
+            message,
+            source: OptDynErr::none(),
+        }
+    }
+
+    fn with_source(source: Self::Source, message: String) -> Self {
+        Self::Whatever {
+            message,
+            source: OptDynErr::some(source),
+        }
+    }
+}
+
+impl From<StatusCode> for RecorderError {
+    fn from(status: StatusCode) -> Self {
+        Self::HttpResponseError {
+            status,
+            headers: None,
+            source: None.into(),
+        }
+    }
+}
+
+impl From<(StatusCode, HeaderMap)> for RecorderError {
+    fn from((status, headers): (StatusCode, HeaderMap)) -> Self {
+        Self::HttpResponseError {
+            status,
+            headers: Some(headers),
+            source: None.into(),
+        }
+    }
+}
+
+impl IntoResponse for RecorderError {
+    fn into_response(self) -> Response {
+        match self {
+            Self::AuthError { source: auth_error } => auth_error.into_response(),
+            Self::HttpResponseError {
+                status,
+                headers,
+                source,
+            } => {
+                let message = source
+                    .into_inner()
+                    .map(|s| s.to_string())
+                    .unwrap_or_else(|| {
+                        String::from(status.canonical_reason().unwrap_or("Unknown"))
+                    });
+                (
+                    status,
+                    headers,
+                    Json::<StandardErrorResponse>(StandardErrorResponse::from(message)),
+                )
+                    .into_response()
+            }
+            Self::ModelEntityNotFound { entity } => (
+                StatusCode::NOT_FOUND,
+                Json::<StandardErrorResponse>(StandardErrorResponse::from(entity.to_string())),
+            )
+                .into_response(),
+            err => (
+                StatusCode::INTERNAL_SERVER_ERROR,
+                Json::<StandardErrorResponse>(StandardErrorResponse::from(err.to_string())),
+            )
+                .into_response(),
+        }
+    }
+}
+
+impl From<reqwest::Error> for RecorderError {
+    fn from(error: reqwest::Error) -> Self {
+        FetchError::from(error).into()
+    }
+}
+
+impl From<reqwest_middleware::Error> for RecorderError {
+    fn from(error: reqwest_middleware::Error) -> Self {
+        FetchError::from(error).into()
+    }
+}
+
+impl From<http::header::InvalidHeaderValue> for RecorderError {
+    fn from(_error: http::header::InvalidHeaderValue) -> Self {
+        Self::InvalidHeaderValueError
+    }
+}
+
+impl From<http::header::InvalidHeaderName> for RecorderError {
+    fn from(_error: http::header::InvalidHeaderName) -> Self {
+        Self::InvalidHeaderNameError
+    }
+}
+
+impl From<http::method::InvalidMethod> for RecorderError {
+    fn from(_error: http::method::InvalidMethod) -> Self {
+        Self::InvalidMethodError
+    }
+}
+
+pub type RecorderResult<T> = Result<T, RecorderError>;

apps/recorder/src/errors/mod.rs

@@ -1,116 +1,6 @@
-use std::{borrow::Cow, error::Error as StdError};
+pub mod app_error;
+pub mod response;
 
-use axum::response::{IntoResponse, Response};
-use http::StatusCode;
-use thiserror::Error as ThisError;
-
-use crate::{auth::AuthError, fetch::HttpClientError};
-
-#[derive(ThisError, Debug)]
-pub enum RError {
-    #[error(transparent)]
-    InvalidMethodError(#[from] http::method::InvalidMethod),
-    #[error(transparent)]
-    InvalidHeaderNameError(#[from] http::header::InvalidHeaderName),
-    #[error(transparent)]
-    TracingAppenderInitError(#[from] tracing_appender::rolling::InitError),
-    #[error(transparent)]
-    GraphQLSchemaError(#[from] async_graphql::dynamic::SchemaError),
-    #[error(transparent)]
-    AuthError(#[from] AuthError),
-    #[error(transparent)]
-    RSSError(#[from] rss::Error),
-    #[error(transparent)]
-    DotEnvError(#[from] dotenv::Error),
-    #[error(transparent)]
-    TeraError(#[from] tera::Error),
-    #[error(transparent)]
-    IOError(#[from] std::io::Error),
-    #[error(transparent)]
-    DbError(#[from] sea_orm::DbErr),
-    #[error(transparent)]
-    CookieParseError(#[from] cookie::ParseError),
-    #[error(transparent)]
-    FigmentError(#[from] figment::Error),
-    #[error(transparent)]
-    SerdeJsonError(#[from] serde_json::Error),
-    #[error(transparent)]
-    ReqwestMiddlewareError(#[from] reqwest_middleware::Error),
-    #[error(transparent)]
-    ReqwestError(#[from] reqwest::Error),
-    #[error(transparent)]
-    ParseUrlError(#[from] url::ParseError),
-    #[error(transparent)]
-    OpenDALError(#[from] opendal::Error),
-    #[error(transparent)]
-    InvalidHeaderValueError(#[from] http::header::InvalidHeaderValue),
-    #[error(transparent)]
-    HttpClientError(#[from] HttpClientError),
-    #[error("Extract {desc} with mime error, expected {expected}, but got {found}")]
-    MimeError {
-        desc: String,
-        expected: String,
-        found: String,
-    },
-    #[error("Invalid or unknown format in extracting mikan rss")]
-    MikanRssInvalidFormatError,
-    #[error("Invalid field {field} in extracting mikan rss")]
-    MikanRssInvalidFieldError {
-        field: Cow<'static, str>,
-        #[source]
-        source: Option<Box<dyn StdError + Send + Sync>>,
-    },
-    #[error("Missing field {field} in extracting mikan meta")]
-    MikanMetaMissingFieldError {
-        field: Cow<'static, str>,
-        #[source]
-        source: Option<Box<dyn StdError + Send + Sync>>,
-    },
-    #[error("Model Entity {entity} not found")]
-    ModelEntityNotFound { entity: Cow<'static, str> },
-    #[error("{0}")]
-    CustomMessageStr(&'static str),
-    #[error("{0}")]
-    CustomMessageString(String),
-}
-
-impl RError {
-    pub fn from_mikan_meta_missing_field(field: Cow<'static, str>) -> Self {
-        Self::MikanMetaMissingFieldError {
-            field,
-            source: None,
-        }
-    }
-
-    pub fn from_mikan_rss_invalid_field(field: Cow<'static, str>) -> Self {
-        Self::MikanRssInvalidFieldError {
-            field,
-            source: None,
-        }
-    }
-
-    pub fn from_mikan_rss_invalid_field_and_source(
-        field: Cow<'static, str>,
-        source: Box<dyn StdError + Send + Sync>,
-    ) -> Self {
-        Self::MikanRssInvalidFieldError {
-            field,
-            source: Some(source),
-        }
-    }
-
-    pub fn from_db_record_not_found<T: ToString>(detail: T) -> Self {
-        Self::DbError(sea_orm::DbErr::RecordNotFound(detail.to_string()))
-    }
-}
-
-impl IntoResponse for RError {
-    fn into_response(self) -> Response {
-        match self {
-            Self::AuthError(auth_error) => auth_error.into_response(),
-            err => (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(),
-        }
-    }
-}
-
-pub type RResult<T> = Result<T, RError>;
+pub use app_error::{RecorderError, RecorderResult};
+pub use response::StandardErrorResponse;
+pub use util::errors::OptDynErr;