diff --git a/.gitignore b/.gitignore index 4e55447..910beec 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ **/node_modules -**/target \ No newline at end of file +**/target +**/dist \ No newline at end of file diff --git a/apps/mock/nest-cli.json b/apps/mock/nest-cli.json new file mode 100644 index 0000000..9b63989 --- /dev/null +++ b/apps/mock/nest-cli.json @@ -0,0 +1,8 @@ +{ + "$schema": "https://json.schemastore.org/nest-cli", + "collection": "@nestjs/schematics", + "sourceRoot": "src", + "compilerOptions": { + "deleteOutDir": true + } +} \ No newline at end of file diff --git a/apps/mock/package.json b/apps/mock/package.json new file mode 100644 index 0000000..e5393f8 --- /dev/null +++ b/apps/mock/package.json @@ -0,0 +1,25 @@ +{ + "name": "mock", + "version": "0.1.0", + "private": true, + "scripts": { + "start": "nest start --watch -b swc", + "dev": "pnpm run start" + }, + "keywords": [], + "license": "MIT", + "dependencies": { + "@nestjs/common": "^11.0.11", + "@nestjs/config": "^4.0.1", + "@nestjs/core": "^11.0.11", + "@nestjs/platform-express": "^11.0.11", + "@nestjs/serve-static": "^5.0.3", + "reflect-metadata": "^0.2.2" + }, + "devDependencies": { + "@nestjs/cli": "^11.0.0", + "@nestjs/schematics": "^11.0.0", + "@swc/cli": "^0.6.0", + "@swc/core": "^1.11.8" + } +} \ No newline at end of file diff --git a/apps/mock/public/video-sample/audiosample.webm b/apps/mock/public/video-sample/audiosample.webm new file mode 100644 index 0000000..e49dc5b Binary files /dev/null and b/apps/mock/public/video-sample/audiosample.webm differ diff --git a/apps/mock/public/video-sample/test.webm b/apps/mock/public/video-sample/test.webm new file mode 100644 index 0000000..6c2138d Binary files /dev/null and b/apps/mock/public/video-sample/test.webm differ diff --git a/apps/mock/public/video-sample/video-webm-codecs-avc1-42E01E.webm b/apps/mock/public/video-sample/video-webm-codecs-avc1-42E01E.webm new file mode 100644 index 0000000..d1e58b0 Binary files /dev/null and b/apps/mock/public/video-sample/video-webm-codecs-avc1-42E01E.webm differ diff --git a/apps/mock/public/video-sample/video-webm-codecs-vp8.webm b/apps/mock/public/video-sample/video-webm-codecs-vp8.webm new file mode 100644 index 0000000..ec3576e Binary files /dev/null and b/apps/mock/public/video-sample/video-webm-codecs-vp8.webm differ diff --git a/apps/mock/src/app.module.ts b/apps/mock/src/app.module.ts new file mode 100644 index 0000000..150a10d --- /dev/null +++ b/apps/mock/src/app.module.ts @@ -0,0 +1,19 @@ +import { Module } from '@nestjs/common'; +import { ServeStaticModule } from '@nestjs/serve-static'; +import path from 'node:path'; + +@Module({ + imports: [ + ServeStaticModule.forRoot({ + rootPath: path.join(__dirname, '..', 'public'), + serveRoot: '/api/static', + serveStaticOptions: { + cacheControl: true, + maxAge: '1d', + }, + }) + ], + controllers: [], + providers: [], +}) +export class AppModule { } diff --git a/apps/mock/src/main.ts b/apps/mock/src/main.ts new file mode 100644 index 0000000..62525e5 --- /dev/null +++ b/apps/mock/src/main.ts @@ -0,0 +1,8 @@ +import { NestFactory } from '@nestjs/core'; +import { AppModule } from './app.module'; + +async function bootstrap() { + const app = await NestFactory.create(AppModule); + await app.listen(process.env.port ?? 5001); +} +bootstrap(); diff --git a/apps/mock/tsconfig.json b/apps/mock/tsconfig.json new file mode 100644 index 0000000..4f82a26 --- /dev/null +++ b/apps/mock/tsconfig.json @@ -0,0 +1,28 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "composite": true, + "module": "CommonJS", + "moduleResolution": "node", + "declaration": true, + "emitDeclarationOnly": false, + "emitDecoratorMetadata": true, + "experimentalDecorators": true, + "allowImportingTsExtensions": false, + "outDir": "./dist", + "rootDir": ".", + "baseUrl": ".", + "lib": [ + "ES2024" + ] + }, + "include": [ + "src/**/*" + ], + "exclude": [ + "node_modules", + "dist", + "test", + "**/*spec.ts" + ] +} \ No newline at end of file diff --git a/apps/playground/.gitignore b/apps/playground/.gitignore index 38d7344..57cba4c 100644 --- a/apps/playground/.gitignore +++ b/apps/playground/.gitignore @@ -10,4 +10,4 @@ dist/ # IDE .vscode/* !.vscode/extensions.json -.idea +.idea \ No newline at end of file diff --git a/apps/playground/package.json b/apps/playground/package.json index c6c3ed6..730bed2 100644 --- a/apps/playground/package.json +++ b/apps/playground/package.json @@ -9,8 +9,8 @@ "preview": "rsbuild preview" }, "dependencies": { - "lit": "^3.2.1", - "rxjs": "^7.8.2" + "konoebml": "0.1.0-rc.6", + "lit": "^3.2.1" }, "devDependencies": { "@rsbuild/core": "^1.2.14", diff --git a/apps/playground/rsbuild.config.ts b/apps/playground/rsbuild.config.ts index 8285d12..9ad1a96 100644 --- a/apps/playground/rsbuild.config.ts +++ b/apps/playground/rsbuild.config.ts @@ -1,7 +1,9 @@ import { defineConfig } from '@rsbuild/core'; export default defineConfig({ + html: { + title: 'Konoplayer Playground', template: './src/index.html', }, source: { @@ -9,4 +11,8 @@ export default defineConfig({ version: 'legacy', }, }, + server: { + host: '0.0.0.0', + port: 5000, + }, }); diff --git a/apps/playground/src/index.css b/apps/playground/src/index.css index 85e7e2b..a2e219a 100644 --- a/apps/playground/src/index.css +++ b/apps/playground/src/index.css @@ -1,6 +1,4 @@ body { margin: 0; - color: #fff; font-family: Inter, Avenir, Helvetica, Arial, sans-serif; - background-image: linear-gradient(to bottom, #020917, #101725); } diff --git a/apps/playground/src/index.html b/apps/playground/src/index.html index 9c001c2..819ba76 100644 --- a/apps/playground/src/index.html +++ b/apps/playground/src/index.html @@ -1,5 +1,8 @@ + + - + + \ No newline at end of file diff --git a/apps/playground/src/index.ts b/apps/playground/src/index.ts index ec31c9e..76aefcf 100644 --- a/apps/playground/src/index.ts +++ b/apps/playground/src/index.ts @@ -1,4 +1,4 @@ import './index.css'; -import { MyElement } from './my-element'; +import { VideoPipelineDemo } from './video-pipeline-demo'; -customElements.define('my-element', MyElement); +customElements.define('video-pipeline-demo', VideoPipelineDemo); diff --git a/apps/playground/src/media/mkv/index.ts b/apps/playground/src/media/mkv/index.ts new file mode 100644 index 0000000..e69de29 diff --git a/apps/playground/src/media/mkv/model.ts b/apps/playground/src/media/mkv/model.ts new file mode 100644 index 0000000..0fb0b46 --- /dev/null +++ b/apps/playground/src/media/mkv/model.ts @@ -0,0 +1,218 @@ +import { + type EbmlTagType, + EbmlTagIdEnum, + EbmlTagPosition, + type EbmlCuePointTagType, + type EbmlTracksTagType, + type EbmlInfoTagType, + type EbmlCuesTagType, + type EbmlSeekHeadTagType, + type EbmlSegmentTagType, + type EbmlClusterTagType, +} from 'konoebml'; +import { isTagEnd } from './util'; +import { isEqual } from 'lodash-es'; + +export const SEEK_ID_KAX_INFO = new Uint8Array([0x15, 0x49, 0xa9, 0x66]); +export const SEEK_ID_KAX_TRACKS = new Uint8Array([0x16, 0x54, 0xae, 0x6b]); +export const SEEK_ID_KAX_CUES = new Uint8Array([0x1c, 0x53, 0xbb, 0x6b]); + +export class EbmlSegment { + startNode: EbmlSegmentTagType; + seekHeadNode?: EbmlSeekHeadTagType; + seekEntries: EbmlSeekEntry[]; + tracksNode?: EbmlTracksTagType; + infoNode?: EbmlInfoTagType; + cuesNode?: EbmlCuesTagType; + metaBuffer: EbmlTagType[] = []; + metaOffsets: Map = new Map(); + + constructor(startNode: EbmlSegmentTagType) { + this.startNode = startNode; + this.seekEntries = []; + this.metaBuffer = []; + } + + get dataOffset() { + return this.startNode.startOffset + this.startNode.headerLength; + } + + private addSeekHead(node: EbmlSeekHeadTagType) { + this.seekHeadNode = node; + this.seekEntries = this.seekHeadNode.children + .filter(isTagEnd) + .filter((c) => c.id === EbmlTagIdEnum.Seek) + .map((c) => { + const seekId = c.children.find( + (item) => item.id === EbmlTagIdEnum.SeekID + )?.data; + const seekPosition = c.children.find( + (item) => item.id === EbmlTagIdEnum.SeekPosition + )?.data as number; + if (seekId && seekPosition) { + return { + seekId, + seekPosition, + }; + } + return null; + }) + .filter((c): c is EbmlSeekEntry => !!c); + } + + findLocalNodeBySeekId(seekId: Uint8Array): EbmlTagType | undefined { + return this.findLocalNodeBySeekPosition( + this.seekEntries.find((c) => isEqual(c.seekId, seekId))?.seekPosition + ); + } + + findLocalNodeBySeekPosition( + seekPosition: number | undefined + ): EbmlTagType | undefined { + return Number.isSafeInteger(seekPosition) + ? this.metaOffsets.get(seekPosition as number) + : undefined; + } + + markMetaEnd() { + this.infoNode = this.findLocalNodeBySeekId( + SEEK_ID_KAX_INFO + ) as EbmlInfoTagType; + this.tracksNode = this.findLocalNodeBySeekId( + SEEK_ID_KAX_TRACKS + ) as EbmlTracksTagType; + this.cuesNode = this.findLocalNodeBySeekId( + SEEK_ID_KAX_CUES + ) as EbmlCuesTagType; + } + + scanMeta(node: EbmlTagType): boolean { + if ( + node.id === EbmlTagIdEnum.SeekHead && + node.position === EbmlTagPosition.End + ) { + this.addSeekHead(node); + } + this.metaBuffer.push(node); + this.metaOffsets.set(node.startOffset - this.dataOffset, node); + return true; + } +} + +export interface EbmlSeekEntry { + seekId: Uint8Array; + seekPosition: number; +} + +export class EbmlHead { + head: EbmlTagType; + + constructor(head: EbmlTagType) { + this.head = head; + } +} + +export class EbmlCluster { + cluster: EbmlClusterTagType; + _timestamp: number; + + constructor(cluster: EbmlClusterTagType) { + this.cluster = cluster; + this._timestamp = cluster.children.find( + (c) => c.id === EbmlTagIdEnum.Timecode + )?.data as number; + } + + get timestamp(): number { + return this._timestamp; + } +} + +export class EbmlCue { + node: EbmlCuePointTagType; + _timestamp: number; + trackPositions: { track: number; position: number }[]; + + get timestamp(): number { + return this._timestamp; + } + + get position(): number { + return Math.max(...this.trackPositions.map((t) => t.position)); + } + + constructor(node: EbmlCuePointTagType) { + this.node = node; + this._timestamp = node.children.find((c) => c.id === EbmlTagIdEnum.CueTime) + ?.data as number; + this.trackPositions = node.children + .map((t) => { + if ( + t.id === EbmlTagIdEnum.CueTrackPositions && + t.position === EbmlTagPosition.End + ) { + const track = t.children.find((t) => t.id === EbmlTagIdEnum.CueTrack) + ?.data as number; + const position = t.children.find( + (t) => t.id === EbmlTagIdEnum.CueClusterPosition + )?.data as number; + + return track! >= 0 && position! >= 0 ? { track, position } : null; + } + return null; + }) + .filter((a): a is { track: number; position: number } => !!a); + } +} + +export class EbmlCues { + node: EbmlCuesTagType; + cues: EbmlCue[]; + + constructor(node: EbmlCuesTagType) { + this.node = node; + this.cues = node.children + .filter(isTagEnd) + .filter((c) => c.id === EbmlTagIdEnum.CuePoint) + .map((c) => new EbmlCue(c)); + } + + findClosestCue(seekTime: number): EbmlCue | null { + const cues = this.cues; + if (!cues || cues.length === 0) { + return null; + } + + let left = 0; + let right = cues.length - 1; + + if (seekTime <= cues[0].timestamp) { + return cues[0]; + } + + if (seekTime >= cues[right].timestamp) { + return cues[right]; + } + + while (left <= right) { + const mid = Math.floor((left + right) / 2); + + if (cues[mid].timestamp === seekTime) { + return cues[mid]; + } + + if (cues[mid].timestamp < seekTime) { + left = mid + 1; + } else { + right = mid - 1; + } + } + + const before = cues[right]; + const after = cues[left]; + return Math.abs(before.timestamp - seekTime) < + Math.abs(after.timestamp - seekTime) + ? before + : after; + } +} diff --git a/apps/playground/src/media/mkv/util.ts b/apps/playground/src/media/mkv/util.ts new file mode 100644 index 0000000..aebf5d6 --- /dev/null +++ b/apps/playground/src/media/mkv/util.ts @@ -0,0 +1,5 @@ +import { EbmlTagPosition, type EbmlTagType } from 'konoebml'; + +export function isTagEnd(tag: EbmlTagType): boolean { + return tag.position === EbmlTagPosition.End; +} diff --git a/apps/playground/src/media/shared/fetch.ts b/apps/playground/src/media/shared/fetch.ts new file mode 100644 index 0000000..43dc653 --- /dev/null +++ b/apps/playground/src/media/shared/fetch.ts @@ -0,0 +1,59 @@ +export interface RangedVideoStream { + controller: AbortController; + response: Response; + stream: ReadableStream; + totalSize?: number; +} + +export async function createRangedVideoStream( + url: string, + byteStart = 0, + byteEnd?: number +) { + const controller = new AbortController(); + const signal = controller.signal; + const headers = new Headers(); + headers.append( + 'Range', + typeof byteEnd === 'number' + ? `bytes=${byteStart}-${byteEnd}` + : `bytes=${byteStart}-` + ); + + const response = await fetch(url, { signal, headers }); + + if (!response.ok) { + throw new Error('fetch video stream failed'); + } + + const acceptRanges = response.headers.get('Accept-Ranges'); + + if (acceptRanges !== 'bytes') { + throw new Error('video server does not support byte ranges'); + } + + const body = response.body; + + if (!(body instanceof ReadableStream)) { + throw new Error('can not get readable stream from response.body'); + } + + const contentRange = response.headers.get('Content-Range'); + + // + // Content-Range Header Syntax: + // Content-Range: -/ + // Content-Range: -/* + // Content-Range: */ + // + const totalSize = contentRange + ? Number.parseInt(contentRange.split('/')[1], 10) + : undefined; + + return { + controller, + response, + stream: body, + totalSize, + }; +} diff --git a/apps/playground/src/media/shared/index.ts b/apps/playground/src/media/shared/index.ts new file mode 100644 index 0000000..29469df --- /dev/null +++ b/apps/playground/src/media/shared/index.ts @@ -0,0 +1 @@ +export { createRangedVideoStream, type RangedVideoStream } from './fetch'; diff --git a/apps/playground/src/my-element.ts b/apps/playground/src/my-element.ts deleted file mode 100644 index 4fd10a7..0000000 --- a/apps/playground/src/my-element.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { html, css, LitElement } from 'lit'; - -export class MyElement extends LitElement { - static styles = css` - .content { - display: flex; - min-height: 100vh; - line-height: 1.1; - text-align: center; - flex-direction: column; - justify-content: center; - } - - .content h1 { - font-size: 3.6rem; - font-weight: 700; - } - - .content p { - font-size: 1.2rem; - font-weight: 400; - opacity: 0.5; - } - `; - - render() { - return html` -
-

Rsbuild with Lit

-

Start building amazing things with Rsbuild.

-
- `; - } -} diff --git a/apps/playground/src/utils/types.ts b/apps/playground/src/utils/types.ts new file mode 100644 index 0000000..e69de29 diff --git a/apps/playground/src/video-pipeline-demo.ts b/apps/playground/src/video-pipeline-demo.ts index f933260..fbd47d8 100644 --- a/apps/playground/src/video-pipeline-demo.ts +++ b/apps/playground/src/video-pipeline-demo.ts @@ -1,15 +1,285 @@ import { html, css, LitElement } from 'lit'; +import { property } from 'lit/decorators.js'; +import { + EbmlStreamDecoder, + EbmlTagIdEnum, + EbmlTagPosition, + type EbmlTagType, +} from 'konoebml'; +import { + EMPTY, + filter, + from, + isEmpty, + map, + merge, + mergeMap, + Observable, + of, + reduce, + share, + Subject, + type Subscription, + switchMap, + take, + takeUntil, + withLatestFrom, +} from 'rxjs'; +import { createRangedVideoStream } from './media/shared'; +import { + EbmlCluster, + EbmlCues, + EbmlSegment, + SEEK_ID_KAX_CUES, +} from './media/mkv/model'; +import { isTagEnd } from './media/mkv/util'; + +export function createRangedEbmlStream( + url: string, + byteStart = 0, + byteEnd?: number +): Observable<{ + ebml$: Observable; + totalSize?: number; + response: Response; + stream: ReadableStream; + controller: AbortController; +}> { + const stream$ = from(createRangedVideoStream(url, byteStart, byteEnd)); + + return stream$.pipe( + mergeMap(({ controller, stream, totalSize, response }) => { + const ebml$ = new Observable((subscriber) => { + stream + .pipeThrough( + new EbmlStreamDecoder({ + streamStartOffset: byteStart, + collectChild: (child) => child.id !== EbmlTagIdEnum.Cluster, + }) + ) + .pipeTo( + new WritableStream({ + write: (tag) => { + subscriber.next(tag); + }, + close: () => { + subscriber.complete(); + }, + abort: (err: any) => { + subscriber.error(err); + }, + }) + ); + + return () => { + controller.abort(); + }; + }).pipe( + share({ + connector: () => new Subject(), + resetOnComplete: false, + resetOnError: false, + resetOnRefCountZero: false, + }) + ); + + return of({ + ebml$, + totalSize, + response, + stream, + controller, + }); + }) + ); +} export class VideoPipelineDemo extends LitElement { - static styles = css` - `; + @property() + src!: string; + + subscripton?: Subscription; + + static styles = css``; + + async prepareVideoPipeline() { + if (!this.src) { + return; + } + + const ebmlRequest$ = createRangedEbmlStream(this.src, 0); + + const ebmlInit$ = ebmlRequest$.pipe( + map(({ totalSize, ebml$, response, controller }) => { + const head = 1; + console.debug( + `stream of video "${this.src}" created, total size is ${totalSize ?? 'unknown'}` + ); + + const segmentStart$ = ebml$.pipe( + filter((s) => s.position === EbmlTagPosition.Start), + filter((tag) => tag.id === EbmlTagIdEnum.Segment) + ); + + const segmentEnd$ = ebml$.pipe( + filter( + (tag) => + tag.id === EbmlTagIdEnum.Segment && + tag.position === EbmlTagPosition.End + ) + ); + + const segments$ = segmentStart$.pipe( + map((startTag) => { + const segment = new EbmlSegment(startTag); + const tag$ = ebml$.pipe(takeUntil(segmentEnd$)); + const cluster$ = tag$.pipe( + filter(isTagEnd), + filter((tag) => tag.id === EbmlTagIdEnum.Cluster), + map((tag) => new EbmlCluster(tag)) + ); + const meta$ = tag$.pipe(takeUntil(cluster$)); + + const withMeta$ = meta$.pipe( + reduce((segment, meta) => { + segment.scanMeta(meta); + return segment; + }, segment), + map((segment) => { + segment.markMetaEnd(); + return segment; + }) + ); + + const withRemoteCues$ = withMeta$.pipe( + map((s) => + s.cuesNode + ? Number.NaN + : s.dataOffset + + (s.seekEntries.find((e) => e.seekId === SEEK_ID_KAX_CUES) + ?.seekPosition ?? Number.NaN) + ), + filter((cuesStartOffset) => cuesStartOffset >= 0), + switchMap((cuesStartOffset) => + createRangedEbmlStream(this.src, cuesStartOffset).pipe( + switchMap((req) => req.ebml$) + ) + ), + filter(isTagEnd), + filter((tag) => tag?.id === EbmlTagIdEnum.Cues), + take(1), + withLatestFrom(withMeta$), + map(([cues, withMeta]) => { + withMeta.cuesNode = cues; + return withMeta; + }) + ); + + const withLocalCues$ = withMeta$.pipe(filter((s) => !!s.cuesNode)); + + const withCues$ = merge(withRemoteCues$, withLocalCues$); + + const withoutCues$ = withCues$.pipe( + isEmpty(), + switchMap((empty) => (empty ? withMeta$ : EMPTY)) + ); + + const seekWithoutCues = ( + cluster$: Observable, + seekTime: number + ): Observable => { + if (seekTime === 0) { + return cluster$; + } + return cluster$.pipe(filter((c) => c.timestamp >= seekTime)); + }; + + const seekWithCues = ( + cues: EbmlCues, + cluster$: Observable, + seekTime: number + ): Observable => { + if (seekTime === 0) { + return cluster$; + } + + const cuePoint = cues.findClosestCue(seekTime); + + if (!cuePoint) { + return seekWithoutCues(cluster$, seekTime); + } + + return createRangedEbmlStream( + this.src, + cuePoint.position + segment.dataOffset + ).pipe( + switchMap((req) => req.ebml$), + filter(isTagEnd), + filter((tag) => tag.id === EbmlTagIdEnum.Cluster), + map((c) => new EbmlCluster(c)) + ); + }; + + const seek = (seekTime: number): Observable => { + return merge( + withCues$.pipe( + switchMap((s) => + seekWithCues(new EbmlCues(s.cuesNode!), cluster$, seekTime) + ) + ), + withoutCues$.pipe( + switchMap((_) => seekWithoutCues(cluster$, seekTime)) + ) + ); + }; + + return { + startTag, + head, + segment, + tag$, + meta$, + cluster$, + withMeta$, + withCues$, + withoutCues$, + seekWithCues, + seekWithoutCues, + seek, + }; + }) + ); + + return { + segments$, + head, + totalSize, + ebml$, + controller, + response, + }; + }) + ); + + this.subscripton = ebmlInit$ + .pipe( + switchMap(({ segments$ }) => segments$), + take(1), + switchMap(({ seek }) => seek(2000)) + ) + .subscribe(console.log); + } + + connectedCallback(): void { + super.connectedCallback(); + this.prepareVideoPipeline(); + } + + disconnectedCallback(): void { + super.disconnectedCallback(); + } render() { - return html` -
-

Rsbuild with Lit

-

Start building amazing things with Rsbuild.

-
- `; + return html`