From 501f971395eb7428dc65dd3f9ac69685a867367e Mon Sep 17 00:00:00 2001 From: Philipp Hancke Date: Mon, 27 Feb 2023 12:39:43 +0100 Subject: [PATCH 1/2] add video_replay sample which allows replaying IVF files generated by video_replay in the browser. This is useful for debugging issues like https://bugs.chromium.org/p/chromium/issues/detail?id=1418596 where the native video_replay is using software decoder which do not show the issue. --- index.html | 7 +- .../video-replay/index.html | 66 ++++++++++ .../video-replay/js/main.js | 121 ++++++++++++++++++ 3 files changed, 191 insertions(+), 3 deletions(-) create mode 100644 src/content/insertable-streams/video-replay/index.html create mode 100644 src/content/insertable-streams/video-replay/js/main.js diff --git a/index.html b/index.html index 04bf0154d4..9a7fcb0f97 100644 --- a/index.html +++ b/index.html @@ -100,7 +100,7 @@

Screensharing with getDisplayMedia
  • Control camera pan, tilt, and zoom
    • - +
  • Control exposure

    • Devices:

    @@ -210,8 +210,9 @@

    Insertable Streams:

  • Video processing using MediaStream Insertable Streams
    • (Experimental)
  • Audio processing using MediaStream Insertable Streams
    • (Experimental)
  • Video cropping using MediaStream Insertable Streams in a Worker
    • (Experimental) -
  • Integrations with WebGPU for custom video rendering:
    • (Experimental) - +
  • Integrations with WebGPU for custom video rendering
    • (Experimental) +
  • Play a IVF file generated by video_replay in the browser
    • (Experimental) + diff --git a/src/content/insertable-streams/video-replay/index.html b/src/content/insertable-streams/video-replay/index.html new file mode 100644 index 0000000000..64179c335a --- /dev/null +++ b/src/content/insertable-streams/video-replay/index.html @@ -0,0 +1,66 @@ + + + + + + + + + + + + + + + + + Insertable Streams - video_replay in the browser using WebCodecs + + + + + + + + +
    +

    WebRTC samples + video_replay for Chrome

    + +

    + This sample shows how how load an IVF file generated by libWebRTC's + + video_replay tool in the browser using WebCodecs and MediaStreamTrackGenerator. + This is useful since the browser uses different decoders for video than the native libWebRTC ones. +

    + + + +
    + + +
    +
    +
    + +

    + Note: This sample is using an experimental API that has not yet been standardized. As + of 2023-02-27, this API is available in the latest version of Chrome based browsers. +

    + View source on GitHub + +
    + + + + + + + diff --git a/src/content/insertable-streams/video-replay/js/main.js b/src/content/insertable-streams/video-replay/js/main.js new file mode 100644 index 0000000000..9184ed07e3 --- /dev/null +++ b/src/content/insertable-streams/video-replay/js/main.js @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2023 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. + */ + +'use strict'; + +/* global MediaStreamTrackGenerator, EncodedVideoChunk */ +if (typeof MediaStreamTrackGenerator === 'undefined') { + alert( + 'Your browser does not support the experimental MediaStreamTrack API ' + + 'for Insertable Streams of Media. See the note at the bottom of the ' + + 'page.'); +} + +// Reader for the IVF file format as described by +// https://wiki.multimedia.cx/index.php/Duck_IVF +class IVF { + constructor(file) { + this.blob = file; + this.offset = 0; + } + + async readHeader() { + if (this.offset !== 0) { + console.error('readHeader called not at start of file.'); + return; + } + this.offset = 32; + + const header = await this.blob.slice(0, 32).arrayBuffer(); + const view = new DataView(header); + const decoder = new TextDecoder('ascii'); + return { + codec: decoder.decode(header.slice(8, 12)), + width: view.getUint16(12, true), + height: view.getUint16(14, true), + fpsDenominator: view.getUint32(16, true), + fpsNumerator: view.getUint32(20, true), + }; + } + + async readFrame() { + if (this.offset == this.blob.size) { + return; // done. + } else if (this.offset === 0) { + console.error('readFrame called without reading header.'); + return; + } + const header = await this.blob.slice(this.offset, this.offset + 12).arrayBuffer(); + const view = new DataView(header); + const frameLength = view.getUint32(0, true); + const timestamp = view.getBigUint64(4, true); + const currentOffset = this.offset; + this.offset += 12 + frameLength; + return { + timestamp, + data: new Uint8Array(await this.blob.slice(currentOffset + 12, currentOffset + 12 + frameLength).arrayBuffer()), + }; + } +} + +// Translate between IVF fourcc codec names and WebCodec named. +const IVF2WebCodecs = { + VP80: 'vp8', + VP90: 'vp09.00.10.08', + H264: 'avc1.42E01F', +}; + +const input = document.getElementById('input'); +const localVideo = document.getElementById('localVideo'); +const metadata = document.getElementById('metadata'); + +input.onchange = async (event) => { + event.target.disabled = true; + const file = event.target.files[0]; + const ivf = new IVF(file); + const generator = new MediaStreamTrackGenerator('video'); + const writer = generator.writable.getWriter(); + localVideo.srcObject = new MediaStream([generator]); + + const header = await ivf.readHeader(); + if (header) { + metadata.innerText = 'File metadata: ' + JSON.stringify(header, null, ' '); + } else { + metadata.innerText = 'Failed to load IVF file.'; + return; + } + + const decoder = new VideoDecoder({ + output: async (frame) => { + await writer.write(frame); + frame.close(); + const nextFrame = await ivf.readFrame(); + if (nextFrame) { + decoder.decode(new EncodedVideoChunk({ + timestamp: Number(nextFrame.timestamp - firstFrame.timestamp) * 1000, + type: 'delta', + data: nextFrame.data, + })); + } else { + decoder.flush(); + } + }, + error: e => console.error(e.message, e), + }); + decoder.configure({ + codec: IVF2WebCodecs[header.codec], + codedWidth: header.width, + codedHeight: header.height, + }); + const firstFrame = await ivf.readFrame(); + decoder.decode(new EncodedVideoChunk({ + timestamp: 0, + type: 'key', + data: firstFrame.data, + })); +}; From ee14ac8acd95e06ba6919a4adb93f94d7c5664f7 Mon Sep 17 00:00:00 2001 From: Philipp Hancke Date: Wed, 14 Jun 2023 14:03:21 +0200 Subject: [PATCH 2/2] add some logging --- src/content/insertable-streams/video-replay/js/main.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/content/insertable-streams/video-replay/js/main.js b/src/content/insertable-streams/video-replay/js/main.js index 9184ed07e3..0e98191071 100644 --- a/src/content/insertable-streams/video-replay/js/main.js +++ b/src/content/insertable-streams/video-replay/js/main.js @@ -68,12 +68,12 @@ const IVF2WebCodecs = { VP80: 'vp8', VP90: 'vp09.00.10.08', H264: 'avc1.42E01F', + AV01: 'av01.0.08M.08.0.110.09', // AV1 Main Profile, level 4.0, Main tier, 8-bit content, non-monochrome, with 4:2:0 chroma subsampling }; const input = document.getElementById('input'); const localVideo = document.getElementById('localVideo'); const metadata = document.getElementById('metadata'); - input.onchange = async (event) => { event.target.disabled = true; const file = event.target.files[0]; @@ -96,6 +96,7 @@ input.onchange = async (event) => { frame.close(); const nextFrame = await ivf.readFrame(); if (nextFrame) { + decoder.decode(new EncodedVideoChunk({ timestamp: Number(nextFrame.timestamp - firstFrame.timestamp) * 1000, type: 'delta', @@ -107,6 +108,8 @@ input.onchange = async (event) => { }, error: e => console.error(e.message, e), }); + VideoDecoder.isConfigSupported({codec: IVF2WebCodecs[header.codec], codedWidth: header.width, codedHeight: header.height}) + .then(config => console.log(config)) decoder.configure({ codec: IVF2WebCodecs[header.codec], codedWidth: header.width,