fix: fix frame seq and compression = 3

apps/proxy/.whistle/rules/files/0.konoplayer

@@ -5,7 +5,7 @@
 }
 ```
 
-# ^https://konoplayer.com/api/static/*** resSpeed://10240
+^https://konoplayer.com/api/static/*** resSpeed://10240
 ^https://konoplayer.com/api*** reqHeaders://{x-forwarded.json} http://127.0.0.1:5001/api$1
 ^https://konoplayer.com/*** reqHeaders://{x-forwarded.json} http://127.0.0.1:5000/$1 excludeFilter://^https://konoplayer.com/api weinre://test
 ^wss://konoplayer.com/*** reqHeaders://{x-forwarded.json} ws://127.0.0.1:5000/$1 excludeFilter://^wss://konoplayer.com/api

packages/core/src/errors.ts

@@ -23,3 +23,9 @@ export class ParseCodecErrors extends Error {
     super('failed to parse codecs');
   }
 }
+
+export class UnimplementedError extends Error {
+  constructor(detail: string) {
+    super(`unimplemented: ${detail}`);
+  }
+}

packages/matroska/src/model/index.ts

@@ -6,10 +6,16 @@ import {
   shareReplay,
   map,
   combineLatest,
-  of, type Observable, delayWhen, pipe, finalize, tap, throwIfEmpty,
+  of,
+  type Observable,
+  delayWhen,
+  throwIfEmpty,
 } from 'rxjs';
 import { isTagIdPos } from '../util';
-import {createRangedEbmlStream, type CreateRangedEbmlStreamOptions} from './resource';
+import {
+  createRangedEbmlStream,
+  type CreateRangedEbmlStreamOptions,
+} from './resource';
 import { type MatroskaSegmentModel, createMatroskaSegment } from './segment';
 
 export type CreateMatroskaOptions = Omit<
@@ -24,7 +30,9 @@ export interface MatroskaModel {
   segment: MatroskaSegmentModel;
 }
 
-export function createMatroska(options: CreateMatroskaOptions): Observable<MatroskaModel> {
+export function createMatroska(
+  options: CreateMatroskaOptions
+): Observable<MatroskaModel> {
   const metadataRequest$ = createRangedEbmlStream({
     ...options,
     byteStart: 0,
@@ -32,21 +40,20 @@ export function createMatroska(options: CreateMatroskaOptions): Observable<Matro
 
   return metadataRequest$.pipe(
     switchMap(({ totalSize, ebml$, response }) => {
-
       /**
        * while [matroska v4](https://www.matroska.org/technical/elements.html) doc tell that there is only one segment in a file
        * some mkv generated by strange tools will emit several
        */
       const segment$ = ebml$.pipe(
         filter(isTagIdPos(EbmlTagIdEnum.Segment, EbmlTagPosition.Start)),
-        map((startTag) => createMatroskaSegment({
+        map((startTag) =>
+          createMatroskaSegment({
             startTag,
             matroskaOptions: options,
             ebml$,
-         })),
+          })
-        delayWhen(
-          ({ loadedMetadata$ }) => loadedMetadata$
         ),
+        delayWhen(({ loadedMetadata$ }) => loadedMetadata$),
         take(1),
         shareReplay(1)
       );
@@ -55,7 +62,7 @@ export function createMatroska(options: CreateMatroskaOptions): Observable<Matro
         filter(isTagIdPos(EbmlTagIdEnum.EBML, EbmlTagPosition.End)),
         take(1),
         shareReplay(1),
-        throwIfEmpty(() => new Error("failed to find head tag"))
+        throwIfEmpty(() => new Error('failed to find head tag'))
       );
 
       return combineLatest({

packages/matroska/src/model/segment.ts

@@ -24,6 +24,7 @@ import {
   finalize,
   delayWhen,
   from,
+  combineLatest,
 } from 'rxjs';
 import type { CreateMatroskaOptions } from '.';
 import { type ClusterType, TrackTypeRestrictionEnum } from '../schema';
@@ -113,7 +114,7 @@ export function createMatroskaSegment({
     filter(({ canComplete }) => canComplete),
     map(({ segment }) => segment),
     take(1),
-    shareReplay(1),
+    shareReplay(1)
   );
 
   const loadedRemoteCues$ = loadedMetadata$.pipe(
@@ -304,22 +305,33 @@ export function createMatroskaSegment({
       map(({ decoder, frame$ }) => {
         const clusterSystem = segment.cluster;
         const infoSystem = segment.info;
+        const trackSystem = segment.track;
         const timestampScale = Number(infoSystem.info.TimestampScale) / 1000;
 
+        const frameProcessing = trackSystem.buildFrameEncodingProcessor(
+          track.trackEntry
+        );
+
         const decodeSubscription = cluster$.subscribe((cluster) => {
           for (const block of clusterSystem.enumerateBlocks(
             cluster,
             track.trackEntry
           )) {
-            const blockTime = (Number(cluster.Timestamp) + block.relTime) * timestampScale;
+            const blockTime =
+              (Number(cluster.Timestamp) + block.relTime) * timestampScale;
             const blockDuration =
-              frames.length > 1 ? track.predictBlockDuration(blockTime) * timestampScale : 0;
+              frames.length > 1
+                ? track.predictBlockDuration(blockTime) * timestampScale
+                : 0;
             const perFrameDuration =
               frames.length > 1 && blockDuration
                 ? blockDuration / block.frames.length
                 : 0;
 
-            for (const frame of block.frames) {
+            for (let frame of block.frames) {
+              if (frameProcessing) {
+                frame = frameProcessing(frame);
+              }
               const chunk = new EncodedVideoChunk({
                 type: block.keyframe ? 'key' : 'delta',
                 data: frame,
@@ -334,13 +346,12 @@ export function createMatroskaSegment({
         return {
           track,
           decoder,
-          frame$: frame$
+          frame$: frame$.pipe(
-            .pipe(
             finalize(() => {
               decodeSubscription.unsubscribe();
             })
-            )
+          ),
-        }
+        };
       })
     );
   };
@@ -353,14 +364,20 @@ export function createMatroskaSegment({
       map(({ decoder, frame$ }) => {
         const clusterSystem = segment.cluster;
         const infoSystem = segment.info;
+        const trackSystem = segment.track;
         const timestampScale = Number(infoSystem.info.TimestampScale) / 1000;
 
+        const frameProcessing = trackSystem.buildFrameEncodingProcessor(
+          track.trackEntry
+        );
+
         const decodeSubscription = cluster$.subscribe((cluster) => {
           for (const block of clusterSystem.enumerateBlocks(
             cluster,
             track.trackEntry
           )) {
-            const blockTime = (Number(cluster.Timestamp) + block.relTime) * timestampScale;
+            const blockTime =
+              (Number(cluster.Timestamp) + block.relTime) * timestampScale;
             const blockDuration =
               frames.length > 1 ? track.predictBlockDuration(blockTime) : 0;
             const perFrameDuration =
@@ -369,7 +386,10 @@ export function createMatroskaSegment({
                 : 0;
 
             let i = 0;
-            for (const frame of block.frames) {
+            for (let frame of block.frames) {
+              if (frameProcessing) {
+                frame = frameProcessing(frame);
+              }
               const chunk = new EncodedAudioChunk({
                 type: block.keyframe ? 'key' : 'delta',
                 data: frame,
@@ -387,7 +407,8 @@ export function createMatroskaSegment({
           decoder,
           frame$: frame$.pipe(finalize(() => decodeSubscription.unsubscribe())),
         };
-    }));
+      })
+    );
   };
 
   const defaultVideoTrack$ = loadedMetadata$.pipe(
@@ -422,6 +443,6 @@ export function createMatroskaSegment({
     videoTrackDecoder,
     audioTrackDecoder,
     defaultVideoTrack$,
-    defaultAudioTrack$
+    defaultAudioTrack$,
   };
 }

packages/matroska/src/systems/cluster.ts

@@ -7,7 +7,7 @@ import {
   type TrackEntryType,
 } from '../schema';
 import { type SegmentComponent } from './segment';
-import {SegmentComponentSystemTrait} from "./segment-component";
+import { SegmentComponentSystemTrait } from './segment-component';
 
 export abstract class BlockViewTrait {
   abstract get keyframe(): boolean;
@@ -82,6 +82,21 @@ export class ClusterSystem extends SegmentComponentSystemTrait<
     cluster: ClusterType,
     track: TrackEntryType
   ): Generator<BlockViewTrait> {
+    if (cluster.BlockGroup && cluster.SimpleBlock) {
+      const blocks = [];
+      for (const block of cluster.BlockGroup) {
+        if (block.Block.track === track.TrackNumber) {
+          blocks.push(new BlockGroupView(block));
+        }
+      }
+      for (const block of cluster.SimpleBlock) {
+        if (block.track === track.TrackNumber) {
+          blocks.push(new SimpleBlockView(block));
+        }
+      }
+      blocks.sort((a, b) => a.relTime - b.relTime);
+      yield* blocks;
+    } else {
       if (cluster.SimpleBlock) {
         for (const block of cluster.SimpleBlock) {
           if (block.track === track.TrackNumber) {
@@ -98,3 +113,4 @@ export class ClusterSystem extends SegmentComponentSystemTrait<
       }
     }
   }
+}

packages/matroska/src/systems/track.ts

@@ -1,5 +1,6 @@
 import {
   ParseCodecErrors,
+  UnimplementedError,
   UnsupportedCodecError,
 } from '@konoplayer/core/errors';
 import {
@@ -15,13 +16,14 @@ import {
   type VideoDecoderConfigExt,
 } from '../codecs';
 import {
+  ContentCompAlgoRestrictionEnum,
+  ContentEncodingTypeRestrictionEnum,
   TrackEntrySchema,
   type TrackEntryType,
   TrackTypeRestrictionEnum,
 } from '../schema';
 import type { SegmentComponent } from './segment';
-import {SegmentComponentSystemTrait} from "./segment-component";
+import { SegmentComponentSystemTrait } from './segment-component';
-import {pick} from "lodash-es";
 
 export interface GetTrackEntryOptions {
   priority?: (v: SegmentComponent<TrackEntryType>) => number;
@@ -226,4 +228,49 @@ export class TrackSystem extends SegmentComponentSystemTrait<
     }
     return true;
   }
+
+  buildFrameEncodingProcessor(
+    track: TrackEntryType
+  ): undefined | ((source: Uint8Array) => Uint8Array) {
+    let encodings = track.ContentEncodings?.ContentEncoding;
+    if (!encodings?.length) {
+      return undefined;
+    }
+    encodings = encodings.toSorted(
+      (a, b) => Number(b.ContentEncodingOrder) - Number(a.ContentEncodingOrder)
+    );
+    const processors: Array<(source: Uint8Array) => Uint8Array> = [];
+    for (const encoing of encodings) {
+      if (
+        encoing.ContentEncodingType ===
+        ContentEncodingTypeRestrictionEnum.COMPRESSION
+      ) {
+        const compression = encoing.ContentCompression;
+        const algo = compression?.ContentCompAlgo;
+        if (algo === ContentCompAlgoRestrictionEnum.HEADER_STRIPPING) {
+          const settings = compression?.ContentCompSettings;
+          if (settings?.length) {
+            processors.push((source: Uint8Array) => {
+              const dest = new Uint8Array(source.length + settings.length);
+              dest.set(source);
+              dest.set(settings, source.length);
+              return dest;
+            });
+          }
+        } else {
+          // TODO: dynamic import packages to support more compression algos
+          throw new UnimplementedError(
+            `compression algo ${ContentCompAlgoRestrictionEnum[algo as ContentCompAlgoRestrictionEnum]}`
+          );
+        }
+      }
+    }
+    return function processor(source: Uint8Array): Uint8Array<ArrayBufferLike> {
+      let dest = source;
+      for (const processor of processors) {
+        dest = processor(dest);
+      }
+      return dest;
+    };
+  }
 }