Migrate downloadWithProgress to util

Dockerfile

@@ -1,13 +1,13 @@
 # syntax=docker/dockerfile-upstream:master-labs
 
 # Base emsdk image with environment variables.
-FROM emscripten/emsdk:3.1.18 AS emsdk-base
+FROM emscripten/emsdk:3.1.29 AS emsdk-base
 ARG EXTRA_CFLAGS
 ARG EXTRA_LDFLAGS
 ARG FFMPEG_ST
 ARG FFMPEG_MT
 ENV INSTALL_DIR=/opt
-ENV FFMPEG_VERSION=n5.1
+ENV FFMPEG_VERSION=n5.1.2
 ENV CFLAGS="-I$INSTALL_DIR/include $CFLAGS $EXTRA_CFLAGS"
 ENV CXXFLAGS="$CFLAGS"
 ENV LDFLAGS="-L$INSTALL_DIR/lib $LDFLAGS $CFLAGS $EXTRA_LDFLAGS"

apps/website/docs/intro.md

@@ -1,43 +1 @@
 # Introduction
-
-Let's discover **Docusaurus in less than 5 minutes**.
-
-## Getting Started
-
-Get started by **creating a new site**.
-
-Or **try Docusaurus immediately** with **[docusaurus.new](https://docusaurus.new)**.
-
-### What you'll need
-
-- [Node.js](https://nodejs.org/en/download/) version 16.14 or above:
-  - When installing Node.js, you are recommended to check all checkboxes related to dependencies.
-
-## Generate a new site
-
-Generate a new Docusaurus site using the **classic template**.
-
-The classic template will automatically be added to your project after you run the command:
-
-```bash
-npm init docusaurus@latest my-website classic
-```
-
-You can type this command into Command Prompt, Powershell, Terminal, or any other integrated terminal of your code editor.
-
-The command also installs all necessary dependencies you need to run Docusaurus.
-
-## Start your site
-
-Run the development server:
-
-```bash
-cd my-website
-npm run start
-```
-
-The `cd` command changes the directory you're working with. In order to work with your newly created Docusaurus site, you'll need to navigate the terminal there.
-
-The `npm run start` command builds your website locally and serves it through a development server, ready for you to view at http://localhost:3000/.
-
-Open `docs/intro.md` (this page) and edit some lines: the site **reloads automatically** and displays your changes.

package.json

@@ -8,11 +8,11 @@
     "pretest": "lerna run build --scope='@ffmpeg/*'",
     "test": "server-test test:browser:server http://localhost:3000 test:all",
     "test:all": "npm-run-all test:*:*:*",
-    "test:browser": "mocha-headless-chrome -a allow-file-access-from-files -a enable-features=SharedArrayBuffer",
+    "test:browser": "mocha-headless-chrome -a enable-features=SharedArrayBuffer",
-    "test:browser:core:mt": "npm run test:browser -- -f tests/ffmpeg-core-mt.test.html",
+    "test:browser:core:mt": "npm run test:browser -- -f http://localhost:3000/tests/ffmpeg-core-mt.test.html",
-    "test:browser:core:st": "npm run test:browser -- -f tests/ffmpeg-core-st.test.html",
+    "test:browser:core:st": "npm run test:browser -- -f http://localhost:3000/tests/ffmpeg-core-st.test.html",
-    "test:browser:ffmpeg:mt": "npm run test:browser -- -f tests/ffmpeg-mt.test.html",
+    "test:browser:ffmpeg:mt": "npm run test:browser -- -f http://localhost:3000/tests/ffmpeg-mt.test.html",
-    "test:browser:ffmpeg:st": "npm run test:browser -- -f tests/ffmpeg-st.test.html",
+    "test:browser:ffmpeg:st": "npm run test:browser -- -f http://localhost:3000/tests/ffmpeg-st.test.html",
     "test:browser:server": "http-server -c-1 --cors -p 3000 .",
     "test:node": "mocha --exit --bail -t 60000",
     "test:node:core:mt": "npm run test:node -- --require tests/test-helper-mt.js tests/ffmpeg-core.test.js",

packages/ffmpeg/src/classes.ts

@@ -3,7 +3,6 @@ import { FFMessageType } from "./const";
 import {
   CallbackData,
   Callbacks,
-  DownloadProgressEvent,
   FSNode,
   FFMessageEventCallback,
   FFMessageLoadConfig,
@@ -18,22 +17,6 @@ import { getMessageID } from "./utils";
 import { ERROR_TERMINATED, ERROR_NOT_LOADED } from "./errors";
 
 export declare interface FFmpeg {
-  /**
-   * Listen to download progress events from `ffmpeg.load()`.
-   *
-   * @example
-   * ```ts
-   * ffmpeg.on(FFmpeg.DOWNLOAD, ({ url, total, received, delta, done }) => {
-   *   // ...
-   * })
-   * ```
-   *
-   * @category Event
-   */
-  on(
-    event: typeof FFmpeg.DOWNLOAD,
-    listener: (data: DownloadProgressEvent) => void
-  ): this;
   /**
    * Listen to log events from `ffmpeg.exec()`.
    *
@@ -81,7 +64,6 @@ export declare interface FFmpeg {
  * ```
  */
 export class FFmpeg extends EventEmitter {
-  /** @event */ static readonly DOWNLOAD = "download" as const;
   /** @event */ static readonly LOG = "log" as const;
   /** @event */ static readonly PROGRESS = "progress" as const;
 
@@ -89,7 +71,6 @@ export class FFmpeg extends EventEmitter {
   /**
    * #resolves and #rejects tracks Promise resolves and rejects to
    * be called when we receive message from web worker.
-   *
    */
   #resolves: Callbacks = {};
   #rejects: Callbacks = {};
@@ -123,9 +104,6 @@ export class FFmpeg extends EventEmitter {
           case FFMessageType.DELETE_DIR:
             this.#resolves[id](data);
             break;
-          case FFMessageType.DOWNLOAD:
-            this.emit(FFmpeg.DOWNLOAD, data as DownloadProgressEvent);
-            break;
           case FFMessageType.LOG:
             this.emit(FFmpeg.LOG, data as LogEvent);
             break;

packages/ffmpeg/src/const.ts

@@ -1,4 +1,3 @@
-export const HeaderContentLength = "Content-Length";
 export const MIME_TYPE_JAVASCRIPT = "text/javascript";
 export const MIME_TYPE_WASM = "application/wasm";
 

packages/ffmpeg/src/errors.ts

@@ -1,11 +1,5 @@
-export const ERROR_RESPONSE_BODY_READER = new Error(
-  "failed to get response body reader"
-);
 export const ERROR_UNKNOWN_MESSAGE_TYPE = new Error("unknown message type");
 export const ERROR_NOT_LOADED = new Error(
   "ffmpeg is not loaded, call `await ffmpeg.load()` first"
 );
-export const ERROR_INCOMPLETED_DOWNLOAD = new Error(
-  "failed to complete download"
-);
 export const ERROR_TERMINATED = new Error("called FFmpeg.terminate()");

packages/ffmpeg/src/types.ts

@@ -112,14 +112,6 @@ export interface FFMessageEvent extends MessageEvent {
   data: FFMessage;
 }
 
-export interface DownloadProgressEvent {
-  url: string | URL;
-  total: number;
-  received: number;
-  delta: number;
-  done: boolean;
-}
-
 export interface LogEvent {
   type: string;
   message: string;
@@ -144,7 +136,6 @@ export type CallbackData =
   | FileData
   | ExitCode
   | ErrorMessage
-  | DownloadProgressEvent
   | LogEvent
   | Progress
   | IsFirst
@@ -164,5 +155,3 @@ export interface FFMessageEventCallback {
     data: CallbackData;
   };
 }
-
-export type ProgressCallback = (event: DownloadProgressEvent) => void;

packages/ffmpeg/src/utils.ts

@@ -1,10 +1,3 @@
-import {
-  ERROR_RESPONSE_BODY_READER,
-  ERROR_INCOMPLETED_DOWNLOAD,
-} from "./errors";
-import { HeaderContentLength } from "./const";
-import { ProgressCallback } from "./types";
-
 /**
  * Generate an unique message ID.
  */
@@ -12,79 +5,3 @@ export const getMessageID = (() => {
   let messageID = 0;
   return () => messageID++;
 })();
-
-/**
- * Download content of a URL with progress.
- *
- * Progress only works when Content-Length is provided by the server.
- *
- */
-export const downloadWithProgress = async (
-  url: string | URL,
-  cb: ProgressCallback
-): Promise<ArrayBuffer> => {
-  const resp = await fetch(url);
-  let buf;
-
-  try {
-    // Set total to -1 to indicate that there is not Content-Type Header.
-    const total = parseInt(resp.headers.get(HeaderContentLength) || "-1");
-
-    const reader = resp.body?.getReader();
-    if (!reader) throw ERROR_RESPONSE_BODY_READER;
-
-    const chunks = [];
-    let received = 0;
-    for (;;) {
-      const { done, value } = await reader.read();
-      const delta = value ? value.length : 0;
-
-      if (done) {
-        if (total != -1 && total !== received) throw ERROR_INCOMPLETED_DOWNLOAD;
-        cb({ url, total, received, delta, done });
-        break;
-      }
-
-      chunks.push(value);
-      received += delta;
-      cb({ url, total, received, delta, done });
-    }
-
-    const data = new Uint8Array(received);
-    let position = 0;
-    for (const chunk of chunks) {
-      data.set(chunk, position);
-      position += chunk.length;
-    }
-
-    buf = data.buffer;
-  } catch (e) {
-    console.log(`failed to send download progress event: `, e);
-    // Fetch arrayBuffer directly when it is not possible to get progress.
-    buf = await resp.arrayBuffer();
-    cb({
-      url,
-      total: buf.byteLength,
-      received: buf.byteLength,
-      delta: 0,
-      done: true,
-    });
-  }
-
-  return buf;
-};
-
-/**
- * Convert an URL to an Blob URL to avoid issues like CORS.
- */
-export const toBlobURL = async (
-  url: string,
-  /** mime type like `text/javascript` and `application/wasm` */
-  mimeType: string,
-  cb: ProgressCallback
-): Promise<string> =>
-  URL.createObjectURL(
-    new Blob([await downloadWithProgress(url, cb)], {
-      type: mimeType,
-    })
-  );

packages/ffmpeg/src/worker.ts

@@ -21,13 +21,7 @@ import type {
   FSNode,
   FileData,
 } from "./types";
-import { toBlobURL } from "./utils";
+import { CORE_URL, FFMessageType } from "./const";
-import {
-  CORE_URL,
-  FFMessageType,
-  MIME_TYPE_JAVASCRIPT,
-  MIME_TYPE_WASM,
-} from "./const";
 import { ERROR_UNKNOWN_MESSAGE_TYPE, ERROR_NOT_LOADED } from "./errors";
 
 declare global {
@@ -42,30 +36,14 @@ const load = async ({
   coreURL: _coreURL = CORE_URL,
   wasmURL: _wasmURL,
   workerURL: _workerURL,
-  blob = true,
-  thread = false,
 }: FFMessageLoadConfig): Promise<IsFirst> => {
   const first = !ffmpeg;
-  let coreURL = _coreURL;
+  const coreURL = _coreURL;
-  let wasmURL = _wasmURL ? _wasmURL : _coreURL.replace(/.js$/g, ".wasm");
+  const wasmURL = _wasmURL ? _wasmURL : _coreURL.replace(/.js$/g, ".wasm");
-  let workerURL = _workerURL
+  const workerURL = _workerURL
     ? _workerURL
     : _coreURL.replace(/.js$/g, ".worker.js");
 
-  if (blob) {
-    coreURL = await toBlobURL(coreURL, MIME_TYPE_JAVASCRIPT, (data) =>
-      self.postMessage({ type: FFMessageType.DOWNLOAD, data })
-    );
-    wasmURL = await toBlobURL(wasmURL, MIME_TYPE_WASM, (data) =>
-      self.postMessage({ type: FFMessageType.DOWNLOAD, data })
-    );
-    if (thread) {
-      workerURL = await toBlobURL(workerURL, MIME_TYPE_JAVASCRIPT, (data) =>
-        self.postMessage({ type: FFMessageType.DOWNLOAD, data })
-      );
-    }
-  }
-
   importScripts(coreURL);
   ffmpeg = await (self as WorkerGlobalScope).createFFmpegCore({
     // Fix `Overload resolution failed.` when using multi-threaded ffmpeg-core.

packages/util/src/const.ts

@@ -0,0 +1 @@
+export const HeaderContentLength = "Content-Length";

packages/util/src/errors.ts

@@ -0,0 +1,6 @@
+export const ERROR_RESPONSE_BODY_READER = new Error(
+  "failed to get response body reader"
+);
+export const ERROR_INCOMPLETED_DOWNLOAD = new Error(
+  "failed to complete download"
+);

packages/util/src/index.ts

@@ -1,3 +1,10 @@
+import {
+  ERROR_RESPONSE_BODY_READER,
+  ERROR_INCOMPLETED_DOWNLOAD,
+} from "./errors";
+import { HeaderContentLength } from "./const";
+import { ProgressCallback } from "./types";
+
 export const readFromBlobOrFile = (blob: Blob | File): Promise<Uint8Array> =>
   new Promise((resolve, reject) => {
     const fileReader = new FileReader();
@@ -105,3 +112,79 @@ export const toBlobURL = async (
   const blob = new Blob([buf], { type: mimeType });
   return URL.createObjectURL(blob);
 };
+
+/**
+ * Download content of a URL with progress.
+ *
+ * Progress only works when Content-Length is provided by the server.
+ *
+ */
+export const downloadWithProgress = async (
+  url: string | URL,
+  cb: ProgressCallback
+): Promise<ArrayBuffer> => {
+  const resp = await fetch(url);
+  let buf;
+
+  try {
+    // Set total to -1 to indicate that there is not Content-Type Header.
+    const total = parseInt(resp.headers.get(HeaderContentLength) || "-1");
+
+    const reader = resp.body?.getReader();
+    if (!reader) throw ERROR_RESPONSE_BODY_READER;
+
+    const chunks = [];
+    let received = 0;
+    for (;;) {
+      const { done, value } = await reader.read();
+      const delta = value ? value.length : 0;
+
+      if (done) {
+        if (total != -1 && total !== received) throw ERROR_INCOMPLETED_DOWNLOAD;
+        cb({ url, total, received, delta, done });
+        break;
+      }
+
+      chunks.push(value);
+      received += delta;
+      cb({ url, total, received, delta, done });
+    }
+
+    const data = new Uint8Array(received);
+    let position = 0;
+    for (const chunk of chunks) {
+      data.set(chunk, position);
+      position += chunk.length;
+    }
+
+    buf = data.buffer;
+  } catch (e) {
+    console.log(`failed to send download progress event: `, e);
+    // Fetch arrayBuffer directly when it is not possible to get progress.
+    buf = await resp.arrayBuffer();
+    cb({
+      url,
+      total: buf.byteLength,
+      received: buf.byteLength,
+      delta: 0,
+      done: true,
+    });
+  }
+
+  return buf;
+};
+
+/**
+ * Convert an URL to an Blob URL to avoid issues like CORS.
+ */
+export const toBlobURLWithProgress = async (
+  url: string,
+  /** mime type like `text/javascript` and `application/wasm` */
+  mimeType: string,
+  cb: ProgressCallback
+): Promise<string> =>
+  URL.createObjectURL(
+    new Blob([await downloadWithProgress(url, cb)], {
+      type: mimeType,
+    })
+  );

packages/util/src/types.ts

@@ -0,0 +1,9 @@
+export interface DownloadProgressEvent {
+  url: string | URL;
+  total: number;
+  received: number;
+  delta: number;
+  done: boolean;
+}
+
+export type ProgressCallback = (event: DownloadProgressEvent) => void;

tests/ffmpeg.test.js

@@ -23,33 +23,6 @@ describe(genName("FFmpeg.load()"), function () {
   //   await ffmpeg.load();
   //   expect(ffmpeg).to.be.ok;
   // });
-
-  it("should work when blob is false", async function () {
-    // FIXME: failed to load ffmpeg-core.worker.js when blob is false.
-    if (FFMPEG_TYPE === "mt") this.skip();
-
-    const ffmpeg = new FFmpeg();
-    await ffmpeg.load({
-      coreURL: CORE_URL,
-      blob: false,
-    });
-    expect(ffmpeg).to.be.ok;
-    ffmpeg.terminate();
-  });
-
-  it("should receive download progress events", async () => {
-    const ffmpeg = new FFmpeg();
-    let done = false;
-    ffmpeg.on(FFmpeg.DOWNLOAD, ({ done: _done }) => {
-      done = _done;
-    });
-    await ffmpeg.load({
-      coreURL: CORE_URL,
-      thread: FFMPEG_TYPE === "mt",
-    });
-    expect(done).to.be.true;
-    ffmpeg.terminate();
-  });
 });
 
 describe(