dumtruck/konowebcodecs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Migrate downloadWithProgress to util

Browse Source
This commit is contained in:
Jerome Wu 2023-01-09 22:47:39 +08:00
parent 36c666aecf
commit ec183935fe
14 changed files with 110 additions and 225 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Dockerfile
View File

@ -1,13 +1,13 @@
# syntax=docker/dockerfile-upstream:master-labs
# Base emsdk image with environment variables.
FROM emscripten/emsdk:3.1.18 AS emsdk-base
FROM emscripten/emsdk:3.1.29 AS emsdk-base
ARG EXTRA_CFLAGS
ARG EXTRA_LDFLAGS
ARG FFMPEG_ST
ARG FFMPEG_MT
ENV INSTALL_DIR=/opt
ENV FFMPEG_VERSION=n5.1
ENV FFMPEG_VERSION=n5.1.2
ENV CFLAGS="-I$INSTALL_DIR/include $CFLAGS $EXTRA_CFLAGS"
ENV CXXFLAGS="$CFLAGS"
ENV LDFLAGS="-L$INSTALL_DIR/lib $LDFLAGS $CFLAGS $EXTRA_LDFLAGS"

42
apps/website/docs/intro.md
View File

@ -1,43 +1 @@
# Introduction
Let's discover **Docusaurus in less than 5 minutes**.
## Getting Started
Get started by **creating a new site**.
Or **try Docusaurus immediately** with **[docusaurus.new](https://docusaurus.new)**.
### What you'll need
- [Node.js](https://nodejs.org/en/download/) version 16.14 or above:
- When installing Node.js, you are recommended to check all checkboxes related to dependencies.
## Generate a new site
Generate a new Docusaurus site using the **classic template**.
The classic template will automatically be added to your project after you run the command:
```bash
npm init docusaurus@latest my-website classic
```
You can type this command into Command Prompt, Powershell, Terminal, or any other integrated terminal of your code editor.
The command also installs all necessary dependencies you need to run Docusaurus.
## Start your site
Run the development server:
```bash
cd my-website
npm run start
```
The `cd` command changes the directory you're working with. In order to work with your newly created Docusaurus site, you'll need to navigate the terminal there.
The `npm run start` command builds your website locally and serves it through a development server, ready for you to view at http://localhost:3000/.
Open `docs/intro.md` (this page) and edit some lines: the site **reloads automatically** and displays your changes.

10
package.json
View File

@ -8,11 +8,11 @@
"pretest": "lerna run build --scope='@ffmpeg/*'",
"test": "server-test test:browser:server http://localhost:3000 test:all",
"test:all": "npm-run-all test:*:*:*",
"test:browser": "mocha-headless-chrome -a allow-file-access-from-files -a enable-features=SharedArrayBuffer",
"test:browser:core:mt": "npm run test:browser -- -f tests/ffmpeg-core-mt.test.html",
"test:browser:core:st": "npm run test:browser -- -f tests/ffmpeg-core-st.test.html",
"test:browser:ffmpeg:mt": "npm run test:browser -- -f tests/ffmpeg-mt.test.html",
"test:browser:ffmpeg:st": "npm run test:browser -- -f tests/ffmpeg-st.test.html",
"test:browser": "mocha-headless-chrome -a enable-features=SharedArrayBuffer",
"test:browser:core:mt": "npm run test:browser -- -f http://localhost:3000/tests/ffmpeg-core-mt.test.html",
"test:browser:core:st": "npm run test:browser -- -f http://localhost:3000/tests/ffmpeg-core-st.test.html",
"test:browser:ffmpeg:mt": "npm run test:browser -- -f http://localhost:3000/tests/ffmpeg-mt.test.html",
"test:browser:ffmpeg:st": "npm run test:browser -- -f http://localhost:3000/tests/ffmpeg-st.test.html",
"test:browser:server": "http-server -c-1 --cors -p 3000 .",
"test:node": "mocha --exit --bail -t 60000",
"test:node:core:mt": "npm run test:node -- --require tests/test-helper-mt.js tests/ffmpeg-core.test.js",

22
packages/ffmpeg/src/classes.ts
View File

@ -3,7 +3,6 @@ import { FFMessageType } from "./const";
import {
CallbackData,
Callbacks,
DownloadProgressEvent,
FSNode,
FFMessageEventCallback,
FFMessageLoadConfig,
@ -18,22 +17,6 @@ import { getMessageID } from "./utils";
import { ERROR_TERMINATED, ERROR_NOT_LOADED } from "./errors";
export declare interface FFmpeg {
/**
* Listen to download progress events from `ffmpeg.load()`.
*
* @example
* ```ts
* ffmpeg.on(FFmpeg.DOWNLOAD, ({ url, total, received, delta, done }) => {
* // ...
* })
* ```
*
* @category Event
*/
on(
event: typeof FFmpeg.DOWNLOAD,
listener: (data: DownloadProgressEvent) => void
): this;
/**
* Listen to log events from `ffmpeg.exec()`.
*
@ -81,7 +64,6 @@ export declare interface FFmpeg {
* ```
*/
export class FFmpeg extends EventEmitter {
/** @event */ static readonly DOWNLOAD = "download" as const;
/** @event */ static readonly LOG = "log" as const;
/** @event */ static readonly PROGRESS = "progress" as const;
@ -89,7 +71,6 @@ export class FFmpeg extends EventEmitter {
/**
* #resolves and #rejects tracks Promise resolves and rejects to
* be called when we receive message from web worker.
*
*/
#resolves: Callbacks = {};
#rejects: Callbacks = {};
@ -123,9 +104,6 @@ export class FFmpeg extends EventEmitter {
case FFMessageType.DELETE_DIR:
this.#resolves[id](data);
break;
case FFMessageType.DOWNLOAD:
this.emit(FFmpeg.DOWNLOAD, data as DownloadProgressEvent);
break;
case FFMessageType.LOG:
this.emit(FFmpeg.LOG, data as LogEvent);
break;

1
packages/ffmpeg/src/const.ts
View File

@ -1,4 +1,3 @@
export const HeaderContentLength = "Content-Length";
export const MIME_TYPE_JAVASCRIPT = "text/javascript";
export const MIME_TYPE_WASM = "application/wasm";

6
packages/ffmpeg/src/errors.ts
View File

@ -1,11 +1,5 @@
export const ERROR_RESPONSE_BODY_READER = new Error(
"failed to get response body reader"
);
export const ERROR_UNKNOWN_MESSAGE_TYPE = new Error("unknown message type");
export const ERROR_NOT_LOADED = new Error(
"ffmpeg is not loaded, call `await ffmpeg.load()` first"
);
export const ERROR_INCOMPLETED_DOWNLOAD = new Error(
"failed to complete download"
);
export const ERROR_TERMINATED = new Error("called FFmpeg.terminate()");

11
packages/ffmpeg/src/types.ts
View File

@ -112,14 +112,6 @@ export interface FFMessageEvent extends MessageEvent {
data: FFMessage;
}
export interface DownloadProgressEvent {
url: string | URL;
total: number;
received: number;
delta: number;
done: boolean;
}
export interface LogEvent {
type: string;
message: string;
@ -144,7 +136,6 @@ export type CallbackData =
| FileData
| ExitCode
| ErrorMessage
| DownloadProgressEvent
| LogEvent
| Progress
| IsFirst
@ -164,5 +155,3 @@ export interface FFMessageEventCallback {
data: CallbackData;
};
}
export type ProgressCallback = (event: DownloadProgressEvent) => void;

83
packages/ffmpeg/src/utils.ts
View File

@ -1,10 +1,3 @@
import {
ERROR_RESPONSE_BODY_READER,
ERROR_INCOMPLETED_DOWNLOAD,
} from "./errors";
import { HeaderContentLength } from "./const";
import { ProgressCallback } from "./types";
/**
* Generate an unique message ID.
*/
@ -12,79 +5,3 @@ export const getMessageID = (() => {
let messageID = 0;
return () => messageID++;
})();
/**
* Download content of a URL with progress.
*
* Progress only works when Content-Length is provided by the server.
*
*/
export const downloadWithProgress = async (
url: string | URL,
cb: ProgressCallback
): Promise<ArrayBuffer> => {
const resp = await fetch(url);
let buf;
try {
// Set total to -1 to indicate that there is not Content-Type Header.
const total = parseInt(resp.headers.get(HeaderContentLength) || "-1");
const reader = resp.body?.getReader();
if (!reader) throw ERROR_RESPONSE_BODY_READER;
const chunks = [];
let received = 0;
for (;;) {
const { done, value } = await reader.read();
const delta = value ? value.length : 0;
if (done) {
if (total != -1 && total !== received) throw ERROR_INCOMPLETED_DOWNLOAD;
cb({ url, total, received, delta, done });
break;
}
chunks.push(value);
received += delta;
cb({ url, total, received, delta, done });
}
const data = new Uint8Array(received);
let position = 0;
for (const chunk of chunks) {
data.set(chunk, position);
position += chunk.length;
}
buf = data.buffer;
} catch (e) {
console.log(`failed to send download progress event: `, e);
// Fetch arrayBuffer directly when it is not possible to get progress.
buf = await resp.arrayBuffer();
cb({
url,
total: buf.byteLength,
received: buf.byteLength,
delta: 0,
done: true,
});
}
return buf;
};
/**
* Convert an URL to an Blob URL to avoid issues like CORS.
*/
export const toBlobURL = async (
url: string,
/** mime type like `text/javascript` and `application/wasm` */
mimeType: string,
cb: ProgressCallback
): Promise<string> =>
URL.createObjectURL(
new Blob([await downloadWithProgress(url, cb)], {
type: mimeType,
})
);

30
packages/ffmpeg/src/worker.ts
View File

@ -21,13 +21,7 @@ import type {
FSNode,
FileData,
} from "./types";
import { toBlobURL } from "./utils";
import {
CORE_URL,
FFMessageType,
MIME_TYPE_JAVASCRIPT,
MIME_TYPE_WASM,
} from "./const";
import { CORE_URL, FFMessageType } from "./const";
import { ERROR_UNKNOWN_MESSAGE_TYPE, ERROR_NOT_LOADED } from "./errors";
declare global {
@ -42,30 +36,14 @@ const load = async ({
coreURL: _coreURL = CORE_URL,
wasmURL: _wasmURL,
workerURL: _workerURL,
blob = true,
thread = false,
}: FFMessageLoadConfig): Promise<IsFirst> => {
const first = !ffmpeg;
let coreURL = _coreURL;
let wasmURL = _wasmURL ? _wasmURL : _coreURL.replace(/.js$/g, ".wasm");
let workerURL = _workerURL
const coreURL = _coreURL;
const wasmURL = _wasmURL ? _wasmURL : _coreURL.replace(/.js$/g, ".wasm");
const workerURL = _workerURL
? _workerURL
: _coreURL.replace(/.js$/g, ".worker.js");
if (blob) {
coreURL = await toBlobURL(coreURL, MIME_TYPE_JAVASCRIPT, (data) =>
self.postMessage({ type: FFMessageType.DOWNLOAD, data })
);
wasmURL = await toBlobURL(wasmURL, MIME_TYPE_WASM, (data) =>
self.postMessage({ type: FFMessageType.DOWNLOAD, data })
);
if (thread) {
workerURL = await toBlobURL(workerURL, MIME_TYPE_JAVASCRIPT, (data) =>
self.postMessage({ type: FFMessageType.DOWNLOAD, data })
);
}
}
importScripts(coreURL);
ffmpeg = await (self as WorkerGlobalScope).createFFmpegCore({
// Fix `Overload resolution failed.` when using multi-threaded ffmpeg-core.

1
packages/util/src/const.ts Normal file
View File

@ -0,0 +1 @@
export const HeaderContentLength = "Content-Length";

6
packages/util/src/errors.ts Normal file
View File

@ -0,0 +1,6 @@
export const ERROR_RESPONSE_BODY_READER = new Error(
"failed to get response body reader"
);
export const ERROR_INCOMPLETED_DOWNLOAD = new Error(
"failed to complete download"
);

83
packages/util/src/index.ts
View File

@ -1,3 +1,10 @@
import {
ERROR_RESPONSE_BODY_READER,
ERROR_INCOMPLETED_DOWNLOAD,
} from "./errors";
import { HeaderContentLength } from "./const";
import { ProgressCallback } from "./types";
export const readFromBlobOrFile = (blob: Blob | File): Promise<Uint8Array> =>
new Promise((resolve, reject) => {
const fileReader = new FileReader();
@ -105,3 +112,79 @@ export const toBlobURL = async (
const blob = new Blob([buf], { type: mimeType });
return URL.createObjectURL(blob);
};
/**
* Download content of a URL with progress.
*
* Progress only works when Content-Length is provided by the server.
*
*/
export const downloadWithProgress = async (
url: string | URL,
cb: ProgressCallback
): Promise<ArrayBuffer> => {
const resp = await fetch(url);
let buf;
try {
// Set total to -1 to indicate that there is not Content-Type Header.
const total = parseInt(resp.headers.get(HeaderContentLength) || "-1");
const reader = resp.body?.getReader();
if (!reader) throw ERROR_RESPONSE_BODY_READER;
const chunks = [];
let received = 0;
for (;;) {
const { done, value } = await reader.read();
const delta = value ? value.length : 0;
if (done) {
if (total != -1 && total !== received) throw ERROR_INCOMPLETED_DOWNLOAD;
cb({ url, total, received, delta, done });
break;
}
chunks.push(value);
received += delta;
cb({ url, total, received, delta, done });
}
const data = new Uint8Array(received);
let position = 0;
for (const chunk of chunks) {
data.set(chunk, position);
position += chunk.length;
}
buf = data.buffer;
} catch (e) {
console.log(`failed to send download progress event: `, e);
// Fetch arrayBuffer directly when it is not possible to get progress.
buf = await resp.arrayBuffer();
cb({
url,
total: buf.byteLength,
received: buf.byteLength,
delta: 0,
done: true,
});
}
return buf;
};
/**
* Convert an URL to an Blob URL to avoid issues like CORS.
*/
export const toBlobURLWithProgress = async (
url: string,
/** mime type like `text/javascript` and `application/wasm` */
mimeType: string,
cb: ProgressCallback
): Promise<string> =>
URL.createObjectURL(
new Blob([await downloadWithProgress(url, cb)], {
type: mimeType,
})
);

9
packages/util/src/types.ts Normal file
View File

@ -0,0 +1,9 @@
export interface DownloadProgressEvent {
url: string | URL;
total: number;
received: number;
delta: number;
done: boolean;
}
export type ProgressCallback = (event: DownloadProgressEvent) => void;

27
tests/ffmpeg.test.js
View File

@ -23,33 +23,6 @@ describe(genName("FFmpeg.load()"), function () {
// await ffmpeg.load();
// expect(ffmpeg).to.be.ok;
// });
it("should work when blob is false", async function () {
// FIXME: failed to load ffmpeg-core.worker.js when blob is false.
if (FFMPEG_TYPE === "mt") this.skip();
const ffmpeg = new FFmpeg();
await ffmpeg.load({
coreURL: CORE_URL,
blob: false,
});
expect(ffmpeg).to.be.ok;
ffmpeg.terminate();
});
it("should receive download progress events", async () => {
const ffmpeg = new FFmpeg();
let done = false;
ffmpeg.on(FFmpeg.DOWNLOAD, ({ done: _done }) => {
done = _done;
});
await ffmpeg.load({
coreURL: CORE_URL,
thread: FFMPEG_TYPE === "mt",
});
expect(done).to.be.true;
ffmpeg.terminate();
});
});
describe(