Merge pull request #1607 from webrtc/guidou/audio-demo-worker

Move audio processing to worker in breakout box demo

Author: guidou

.eslintrc.js

@@ -35,6 +35,9 @@ module.exports = {
     // From Streams specification
     "TransformStream": true,
     // From WebCodec specification
+    "AudioData": true,
+    "AudioEncoder": true,
+    "AudioDecoder": true,
     "VideoFrame": true,
     "VideoEncoder": true,
     "VideoDecoder": true,

src/content/insertable-streams/audio-processing/index.html

@@ -48,6 +48,7 @@ <h1><a href="//webrtc.github.io/samples/" title="WebRTC samples homepage">WebRTC
         <button type="button" id="stopButton" disabled>Stop</button>
     <div>
     <p class="warning">Warning: if you're not using headphones, pressing Start will cause feedback.</p>
+    <div id="errorMsg" class="warning"></div>
 
     <p>View the console to see logging. The <code>audio</code>, <code>processor</code>,
         <code>generator</code>, <code>transformer</code>, <code>stream</code> and

src/content/insertable-streams/audio-processing/js/main.js

@@ -44,19 +44,14 @@ let stopButton;
 // Transformation chain elements
 let processor;
 let generator;
-let transformer;
 
 // Stream from getUserMedia
 let stream;
 // Output from the transform
 let processedStream;
 
-// Adjust this value to increase/decrease the amount of filtering.
-// eslint-disable-next-line prefer-const
-let cutoff = 100;
-
-// An AbortController used to stop the transform.
-let abortController;
+// Worker for processing
+let worker;
 
 // Initialize on page load.
 async function init() {
@@ -73,85 +68,35 @@ const constraints = window.constraints = {
   video: false
 };
 
-// Returns a low-pass transform function for use with TransformStream.
-function lowPassFilter() {
-  const format = 'f32-planar';
-  let lastValuePerChannel = undefined;
-  return (data, controller) => {
-    const rc = 1.0 / (cutoff * 2 * Math.PI);
-    const dt = 1.0 / data.sampleRate;
-    const alpha = dt / (rc + dt);
-    const nChannels = data.numberOfChannels;
-    if (!lastValuePerChannel) {
-      console.log(`Audio stream has ${nChannels} channels.`);
-      lastValuePerChannel = Array(nChannels).fill(0);
-    }
-    const buffer = new Float32Array(data.numberOfFrames * nChannels);
-    for (let c = 0; c < nChannels; c++) {
-      const offset = data.numberOfFrames * c;
-      const samples = buffer.subarray(offset, offset + data.numberOfFrames);
-      data.copyTo(samples, {planeIndex: c, format});
-      let lastValue = lastValuePerChannel[c];
-
-      // Apply low-pass filter to samples.
-      for (let i = 0; i < samples.length; ++i) {
-        lastValue = lastValue + alpha * (samples[i] - lastValue);
-        samples[i] = lastValue;
-      }
-
-      lastValuePerChannel[c] = lastValue;
-    }
-    controller.enqueue(new AudioData({
-      format,
-      sampleRate: data.sampleRate,
-      numberOfFrames: data.numberOfFrames,
-      numberOfChannels: nChannels,
-      timestamp: data.timestamp,
-      data: buffer
-    }));
-  };
-}
-
 async function start() {
   startButton.disabled = true;
   try {
     stream = await navigator.mediaDevices.getUserMedia(constraints);
+    const audioTracks = stream.getAudioTracks();
+    console.log('Using audio device: ' + audioTracks[0].label);
+    stream.oninactive = () => {
+      console.log('Stream ended');
+    };
+
+    processor = new MediaStreamTrackProcessor(audioTracks[0]);
+    generator = new MediaStreamTrackGenerator('audio');
+    const source = processor.readable;
+    const sink = generator.writable;
+    worker = new Worker('js/worker.js');
+    worker.postMessage({source: source, sink: sink}, [source, sink]);
+
+    processedStream = new MediaStream();
+    processedStream.addTrack(generator);
+    audio.srcObject = processedStream;
+    stopButton.disabled = false;
+    await audio.play();
   } catch (error) {
     const errorMessage =
           'navigator.MediaDevices.getUserMedia error: ' + error.message + ' ' +
           error.name;
     document.getElementById('errorMsg').innerText = errorMessage;
     console.log(errorMessage);
   }
-  const audioTracks = stream.getAudioTracks();
-  console.log('Using audio device: ' + audioTracks[0].label);
-  stream.oninactive = () => {
-    console.log('Stream ended');
-  };
-
-  processor = new MediaStreamTrackProcessor(audioTracks[0]);
-  generator = new MediaStreamTrackGenerator('audio');
-  const source = processor.readable;
-  const sink = generator.writable;
-  transformer = new TransformStream({transform: lowPassFilter()});
-  abortController = new AbortController();
-  const signal = abortController.signal;
-  const promise = source.pipeThrough(transformer, {signal}).pipeTo(sink);
-  promise.catch((e) => {
-    if (signal.aborted) {
-      console.log('Shutting down streams after abort.');
-    } else {
-      console.error('Error from stream transform:', e);
-    }
-    source.cancel(e);
-    sink.abort(e);
-  });
-
-  processedStream = new MediaStream();
-  processedStream.addTrack(generator);
-  audio.srcObject = processedStream;
-  stopButton.disabled = false;
-  await audio.play();
 }
 
 async function stop() {
@@ -161,8 +106,7 @@ async function stop() {
   stream.getTracks().forEach(track => {
     track.stop();
   });
-  abortController.abort();
-  abortController = null;
+  worker.postMessage({command: 'abort'});
   startButton.disabled = false;
 }
 

src/content/insertable-streams/audio-processing/js/worker.js

@@ -0,0 +1,75 @@
+/*
+ *  Copyright (c) 2023 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree.
+ */
+
+// Adjust this value to increase/decrease the amount of filtering.
+// eslint-disable-next-line prefer-const
+let cutoff = 100;
+
+// Returns a low-pass transform function for use with TransformStream.
+function lowPassFilter() {
+  const format = 'f32-planar';
+  let lastValuePerChannel = undefined;
+  return (data, controller) => {
+    const rc = 1.0 / (cutoff * 2 * Math.PI);
+    const dt = 1.0 / data.sampleRate;
+    const alpha = dt / (rc + dt);
+    const nChannels = data.numberOfChannels;
+    if (!lastValuePerChannel) {
+      console.log(`Audio stream has ${nChannels} channels.`);
+      lastValuePerChannel = Array(nChannels).fill(0);
+    }
+    const buffer = new Float32Array(data.numberOfFrames * nChannels);
+    for (let c = 0; c < nChannels; c++) {
+      const offset = data.numberOfFrames * c;
+      const samples = buffer.subarray(offset, offset + data.numberOfFrames);
+      data.copyTo(samples, {planeIndex: c, format});
+      let lastValue = lastValuePerChannel[c];
+
+      // Apply low-pass filter to samples.
+      for (let i = 0; i < samples.length; ++i) {
+        lastValue = lastValue + alpha * (samples[i] - lastValue);
+        samples[i] = lastValue;
+      }
+
+      lastValuePerChannel[c] = lastValue;
+    }
+    controller.enqueue(new AudioData({
+      format,
+      sampleRate: data.sampleRate,
+      numberOfFrames: data.numberOfFrames,
+      numberOfChannels: nChannels,
+      timestamp: data.timestamp,
+      data: buffer
+    }));
+  };
+}
+
+let abortController;
+
+onmessage = async (event) => {
+  if (event.data.command == 'abort') {
+    abortController.abort();
+    abortController = null;
+  } else {
+    const source = event.data.source;
+    const sink = event.data.sink;
+    const transformer = new TransformStream({transform: lowPassFilter()});
+    abortController = new AbortController();
+    const signal = abortController.signal;
+    const promise = source.pipeThrough(transformer, {signal}).pipeTo(sink);
+    promise.catch((e) => {
+      if (signal.aborted) {
+        console.log('Shutting down streams after abort.');
+      } else {
+        console.error('Error from stream transform:', e);
+      }
+      source.cancel(e);
+      sink.abort(e);
+    });
+  }
+};