Enable AV1 stream forwarding (#1085)

Author: starwarfan

source/agent/conference/conference.js

@@ -60,7 +60,7 @@ const {
    *    }
    *
    *    object(VideoFormat):: {
-   *      codec: 'h264' | 'vp8' | 'h265' | 'vp9',
+   *      codec: 'h264' | 'vp8' | 'h265' | 'vp9' | 'av1x',
    *      profile: 'constrained-baseline' | 'baseline' | 'main' | 'high' | undefined
    *    }
    *

source/agent/video/videoCapability.js

@@ -20,9 +20,10 @@ const isHWAccAppliable = () => {
 module.exports.detected = (requireHWAcc) => {
   /*FIXME: should be double checked whether hardware acceleration is actually running*/
   var useHW = false;
+  // TODO: support av1x.
   var codecs = {
-    decode: ['vp8', 'vp9', 'h264', 'h265'],
-    encode: ['vp8', 'vp9']
+    decode: ['vp8', 'vp9', 'h264', 'h265', 'av1x'],
+    encode: ['vp8', 'vp9', 'av1x']
   };
 
   if (requireHWAcc && isHWAccAppliable()) {

source/core/common/rtputils.h

@@ -69,6 +69,7 @@ struct redheader {
 #define VP9_90000_PT        101 // VP9 Video Codec
 #define H264_90000_PT       127 // H264 Video Codec
 #define H265_90000_PT       121 // H265 Video Codec
+#define AV1_90000_PT        35  // AV1X Video Codec
 #define RED_90000_PT        116 // REDundancy (RFC 2198)
 #define ULP_90000_PT        117 // ULP/FEC
 #define ISAC_16000_PT       103 // ISAC Audio Codec

source/core/owt_base/MediaFramePipeline.h

@@ -22,6 +22,7 @@ enum FrameFormat {
     FRAME_FORMAT_VP9,
     FRAME_FORMAT_H264,
     FRAME_FORMAT_H265,
+    FRAME_FORMAT_AV1,
 
     FRAME_FORMAT_MSDK       = 300,
 
@@ -104,6 +105,8 @@ inline FrameFormat getFormat(const std::string& codec) {
         return owt_base::FRAME_FORMAT_VP9;
     } else if (codec == "h265") {
         return owt_base::FRAME_FORMAT_H265;
+    } else if (codec == "av1x") {
+        return owt_base::FRAME_FORMAT_AV1;
     } else if (codec == "pcm_48000_2" || codec == "pcm_raw") {
         return owt_base::FRAME_FORMAT_PCM_48000_2;
     } else if (codec == "pcmu") {
@@ -152,6 +155,8 @@ inline const char *getFormatStr(const FrameFormat &format) {
             return "H264";
         case FRAME_FORMAT_H265:
             return "H265";
+        case FRAME_FORMAT_AV1:
+            return "AV1X";
         case FRAME_FORMAT_PCM_48000_2:
             return "PCM_48000_2";
         case FRAME_FORMAT_PCMU:
@@ -205,7 +210,8 @@ inline bool isVideoFrame(const Frame& frame) {
           || frame.format == FRAME_FORMAT_VP8
           || frame.format == FRAME_FORMAT_VP9
           || frame.format == FRAME_FORMAT_H264
-          || frame.format == FRAME_FORMAT_H265;
+          || frame.format == FRAME_FORMAT_H265
+          || frame.format == FRAME_FORMAT_AV1;
 }
 
 inline bool isDataFrame(const Frame& frame) {

source/core/rtc_adapter/VideoReceiveAdapter.cc

@@ -77,6 +77,9 @@ int32_t VideoReceiveAdapterImpl::AdapterDecoder::Decode(const webrtc::EncodedIma
     case webrtc::VideoCodecType::kVideoCodecH265:
         format = FRAME_FORMAT_H265;
         break;
+    case webrtc::VideoCodecType::kVideoCodecAV1:
+        format = FRAME_FORMAT_AV1;
+        break;
     default:
         RTC_LOG(LS_WARNING) << "Unknown FORMAT";
         return 0;
@@ -242,6 +245,12 @@ void VideoReceiveAdapterImpl::CreateReceiveVideo()
                 webrtc::CodecTypeToPayloadString(webrtc::VideoCodecType::kVideoCodecH265));
             RTC_DLOG(LS_INFO) << "Config add decoder:" << decoder.ToString();
             video_recv_config.decoders.push_back(decoder);
+            // Add AV1 decoder
+            decoder.payload_type = AV1_90000_PT;
+            decoder.video_format = webrtc::SdpVideoFormat(
+                webrtc::CodecTypeToPayloadString(webrtc::VideoCodecType::kVideoCodecAV1));
+            RTC_DLOG(LS_INFO) << "Config add decoder:" << decoder.ToString();
+            video_recv_config.decoders.push_back(decoder);
 
             RTC_DLOG(LS_INFO) << "VideoReceiveStream::Config " << video_recv_config.ToString();
             m_videoRecvStream = call()->CreateVideoReceiveStream(std::move(video_recv_config));
@@ -267,7 +276,9 @@ std::vector<webrtc::SdpVideoFormat> VideoReceiveAdapterImpl::GetSupportedFormats
         webrtc::SdpVideoFormat(
             webrtc::CodecTypeToPayloadString(webrtc::VideoCodecType::kVideoCodecH264)),
         webrtc::SdpVideoFormat(
-            webrtc::CodecTypeToPayloadString(webrtc::VideoCodecType::kVideoCodecH265))
+            webrtc::CodecTypeToPayloadString(webrtc::VideoCodecType::kVideoCodecH265)),
+        webrtc::SdpVideoFormat(
+            webrtc::CodecTypeToPayloadString(webrtc::VideoCodecType::kVideoCodecAV1))
     };
 }
 

source/core/rtc_adapter/VideoSendAdapter.cc

@@ -397,6 +397,18 @@ void VideoSendAdapterImpl::onFrame(const Frame& frame)
                 m_rtpRtcp->ExpectedRetransmissionTimeMs(),
                 0);
         }
+    } else if (frame.format == FRAME_FORMAT_AV1) {
+        h.codec = webrtc::VideoCodecType::kVideoCodecAV1;
+        boost::shared_lock<boost::shared_mutex> lock(m_rtpRtcpMutex);
+        m_senderVideo->SendVideo(
+            AV1_90000_PT,
+            webrtc::kVideoCodecAV1,
+            timeStamp,
+            timeStamp,
+            rtc::ArrayView<const uint8_t>(frame.payload, frame.length),
+            h,
+            m_rtpRtcp->ExpectedRetransmissionTimeMs(),
+            0);
     }
 }
 

source/management_console/public/js/App.js

@@ -604,6 +604,7 @@ class RoomModal extends React.Component {
       'h265': {codec: 'h265'},
       'vp8': {codec: 'vp8'},
       'vp9': {codec: 'vp9'},
+      'av1x': {codec: 'av1x'},
       // audio
       'opus': {codec: 'opus', sampleRate: 48000, channelNum: 2},
       'isac-16000': {codec: 'isac', sampleRate: 16000},
@@ -702,7 +703,7 @@ class RoomModal extends React.Component {
       return mediaCheckBox(name, 'mediaIn.audio');
     });
 
-    const videoNames = ['h264', 'h265', 'vp8', 'vp9'];
+    const videoNames = ['h264', 'h265', 'vp8', 'vp9', 'av1x'];
     const videoCheckBoxes = videoNames.map((name) => {
       return mediaCheckBox(name, 'mediaIn.video');
     });
@@ -748,7 +749,7 @@ class RoomModal extends React.Component {
       'h264-baseline',
       'h264-main',
       'h264-high',
-      'h265', 'vp8', 'vp9'
+      'h265', 'vp8', 'vp9', 'av1x',
     ];
     const videoCheckBoxes = videoNames.map((name) => {
       return mediaCheckBox(name, 'mediaOut.video.format');