Skip to content

Commit 10fa54a

Browse files
tkmn0tkmn0
committed
feat: add custom video frame capturer
1 parent 2882e1a commit 10fa54a

File tree

5 files changed

+140
-11
lines changed

5 files changed

+140
-11
lines changed

‎SimpleWebRTC.xcodeproj/project.pbxproj‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,6 @@
88

99
/* Begin PBXBuildFile section */
1010
28D5CAE0E83A76C0FFBC9536 /* Pods_SimpleWebRTC.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 9BFABE961561EF9B2240A098 /* Pods_SimpleWebRTC.framework */; };
11-
37D42462220C447500F16598 /* WebRTC.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 37D42461220C447500F16598 /* WebRTC.framework */; };
12-
37D42463220C447500F16598 /* WebRTC.framework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 37D42461220C447500F16598 /* WebRTC.framework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; };
1311
AF15C0B221E422D9005B4ED6 /* UIViewExtention.swift in Sources */ = {isa = PBXBuildFile; fileRef = AF15C0B121E422D9005B4ED6 /* UIViewExtention.swift */; };
1412
AF15C0B521E4F1F7005B4ED6 /* SignalingMessage.swift in Sources */ = {isa = PBXBuildFile; fileRef = AF15C0B421E4F1F7005B4ED6 /* SignalingMessage.swift */; };
1513
AF15C0B721E54EB1005B4ED6 /* SignalingCandidate.swift in Sources */ = {isa = PBXBuildFile; fileRef = AF15C0B621E54EB1005B4ED6 /* SignalingCandidate.swift */; };
@@ -22,6 +20,10 @@
2220
AF711AC521DFAD6E005CA4FA /* SimpleWebRTCUITests.swift in Sources */ = {isa = PBXBuildFile; fileRef = AF711AC421DFAD6E005CA4FA /* SimpleWebRTCUITests.swift */; };
2321
AF711ADA21E1454E005CA4FA /* WebRTCClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = AF711AD921E1454E005CA4FA /* WebRTCClient.swift */; };
2422
AF711ADD21E14DDC005CA4FA /* ScreenSizeUtil.swift in Sources */ = {isa = PBXBuildFile; fileRef = AF711ADC21E14DDC005CA4FA /* ScreenSizeUtil.swift */; };
23+
AF90A8DD220CEB0E0083687C /* WebRTC.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = AF90A8DC220CEB0E0083687C /* WebRTC.framework */; };
24+
AF90A8DE220CEB0E0083687C /* WebRTC.framework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = AF90A8DC220CEB0E0083687C /* WebRTC.framework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; };
25+
AF90A8E0220CF07A0083687C /* RTCCustomFrameCapture.swift in Sources */ = {isa = PBXBuildFile; fileRef = AF90A8DF220CF07A0083687C /* RTCCustomFrameCapture.swift */; };
26+
AF90A8E3220D00D60083687C /* CameraSession.swift in Sources */ = {isa = PBXBuildFile; fileRef = AF90A8E2220D00D60083687C /* CameraSession.swift */; };
2527
AFAF58EA21EA0AD300609BB7 /* UIColorExtention.swift in Sources */ = {isa = PBXBuildFile; fileRef = AFAF58E921EA0AD300609BB7 /* UIColorExtention.swift */; };
2628
AFB7E9B121E56D8B00A0E27A /* sample.mp4 in Resources */ = {isa = PBXBuildFile; fileRef = AFF1031B21E569B60011C1FC /* sample.mp4 */; };
2729
AFBCD53121E8F52B00694C14 /* like_filled.png in Resources */ = {isa = PBXBuildFile; fileRef = AFBCD53021E8F52B00694C14 /* like_filled.png */; };
@@ -55,7 +57,7 @@
5557
dstPath = "";
5658
dstSubfolderSpec = 10;
5759
files = (
58-
37D42463220C447500F16598 /* WebRTC.framework in Embed Frameworks */,
60+
AF90A8DE220CEB0E0083687C /* WebRTC.framework in Embed Frameworks */,
5961
);
6062
name = "Embed Frameworks";
6163
runOnlyForDeploymentPostprocessing = 0;
@@ -67,7 +69,6 @@
6769
05D95DE48BD4CC42D39620AF /* Pods-SimpleWebRTCTests.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-SimpleWebRTCTests.debug.xcconfig"; path = "Pods/Target Support Files/Pods-SimpleWebRTCTests/Pods-SimpleWebRTCTests.debug.xcconfig"; sourceTree = "<group>"; };
6870
2D2A5F0229AE703BFD44F9F1 /* Pods-SimpleWebRTCUITests.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-SimpleWebRTCUITests.release.xcconfig"; path = "Pods/Target Support Files/Pods-SimpleWebRTCUITests/Pods-SimpleWebRTCUITests.release.xcconfig"; sourceTree = "<group>"; };
6971
31148F7848EBC2EEA7F21F0F /* Pods_SimpleWebRTCTests.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_SimpleWebRTCTests.framework; sourceTree = BUILT_PRODUCTS_DIR; };
70-
37D42461220C447500F16598 /* WebRTC.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; path = WebRTC.framework; sourceTree = "<group>"; };
7172
53C8300D672FBC2658413CC6 /* Pods-SimpleWebRTC.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-SimpleWebRTC.release.xcconfig"; path = "Pods/Target Support Files/Pods-SimpleWebRTC/Pods-SimpleWebRTC.release.xcconfig"; sourceTree = "<group>"; };
7273
8D9A48F6D592CC1505389F06 /* Pods-SimpleWebRTCTests.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-SimpleWebRTCTests.release.xcconfig"; path = "Pods/Target Support Files/Pods-SimpleWebRTCTests/Pods-SimpleWebRTCTests.release.xcconfig"; sourceTree = "<group>"; };
7374
9BFABE961561EF9B2240A098 /* Pods_SimpleWebRTC.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_SimpleWebRTC.framework; sourceTree = BUILT_PRODUCTS_DIR; };
@@ -90,6 +91,9 @@
9091
AF711AC621DFAD6E005CA4FA /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
9192
AF711AD921E1454E005CA4FA /* WebRTCClient.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WebRTCClient.swift; sourceTree = "<group>"; };
9293
AF711ADC21E14DDC005CA4FA /* ScreenSizeUtil.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ScreenSizeUtil.swift; sourceTree = "<group>"; };
94+
AF90A8DC220CEB0E0083687C /* WebRTC.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; path = WebRTC.framework; sourceTree = "<group>"; };
95+
AF90A8DF220CF07A0083687C /* RTCCustomFrameCapture.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RTCCustomFrameCapture.swift; sourceTree = "<group>"; };
96+
AF90A8E2220D00D60083687C /* CameraSession.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CameraSession.swift; sourceTree = "<group>"; };
9397
AFAF58E921EA0AD300609BB7 /* UIColorExtention.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UIColorExtention.swift; sourceTree = "<group>"; };
9498
AFBCD53021E8F52B00694C14 /* like_filled.png */ = {isa = PBXFileReference; lastKnownFileType = image.png; path = like_filled.png; sourceTree = "<group>"; };
9599
AFBCD53321E8F55900694C14 /* UIDeviceExtention.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UIDeviceExtention.swift; sourceTree = "<group>"; };
@@ -104,8 +108,8 @@
104108
isa = PBXFrameworksBuildPhase;
105109
buildActionMask = 2147483647;
106110
files = (
107-
37D42462220C447500F16598 /* WebRTC.framework in Frameworks */,
108111
28D5CAE0E83A76C0FFBC9536 /* Pods_SimpleWebRTC.framework in Frameworks */,
112+
AF90A8DD220CEB0E0083687C /* WebRTC.framework in Frameworks */,
109113
);
110114
runOnlyForDeploymentPostprocessing = 0;
111115
};
@@ -163,7 +167,7 @@
163167
AF711A9821DFAD6C005CA4FA = {
164168
isa = PBXGroup;
165169
children = (
166-
37D42461220C447500F16598 /* WebRTC.framework */,
170+
AF90A8DC220CEB0E0083687C /* WebRTC.framework */,
167171
AF711AA321DFAD6C005CA4FA /* SimpleWebRTC */,
168172
AF711AB821DFAD6E005CA4FA /* SimpleWebRTCTests */,
169173
AF711AC321DFAD6E005CA4FA /* SimpleWebRTCUITests */,
@@ -186,6 +190,7 @@
186190
AF711AA321DFAD6C005CA4FA /* SimpleWebRTC */ = {
187191
isa = PBXGroup;
188192
children = (
193+
AF90A8E1220D00500083687C /* CameraSession */,
189194
AFF1031921E569990011C1FC /* Assets */,
190195
AF15C0B321E4F1B9005B4ED6 /* Data */,
191196
AF15C0AE21E41A58005B4ED6 /* Extentions */,
@@ -231,6 +236,7 @@
231236
isa = PBXGroup;
232237
children = (
233238
AF711AD921E1454E005CA4FA /* WebRTCClient.swift */,
239+
AF90A8DF220CF07A0083687C /* RTCCustomFrameCapture.swift */,
234240
);
235241
path = WebRTC;
236242
sourceTree = "<group>";
@@ -243,6 +249,14 @@
243249
path = Utility;
244250
sourceTree = "<group>";
245251
};
252+
AF90A8E1220D00500083687C /* CameraSession */ = {
253+
isa = PBXGroup;
254+
children = (
255+
AF90A8E2220D00D60083687C /* CameraSession.swift */,
256+
);
257+
path = CameraSession;
258+
sourceTree = "<group>";
259+
};
246260
AFBCD52F21E8F51600694C14 /* Image */ = {
247261
isa = PBXGroup;
248262
children = (
@@ -499,11 +513,13 @@
499513
files = (
500514
AF15C0B721E54EB1005B4ED6 /* SignalingCandidate.swift in Sources */,
501515
AF711AA721DFAD6C005CA4FA /* ViewController.swift in Sources */,
516+
AF90A8E0220CF07A0083687C /* RTCCustomFrameCapture.swift in Sources */,
502517
AF15C0B221E422D9005B4ED6 /* UIViewExtention.swift in Sources */,
503518
AF711ADD21E14DDC005CA4FA /* ScreenSizeUtil.swift in Sources */,
504519
AF711AA521DFAD6C005CA4FA /* AppDelegate.swift in Sources */,
505520
AF15C0B521E4F1F7005B4ED6 /* SignalingMessage.swift in Sources */,
506521
AFBCD53421E8F55900694C14 /* UIDeviceExtention.swift in Sources */,
522+
AF90A8E3220D00D60083687C /* CameraSession.swift in Sources */,
507523
AF711ADA21E1454E005CA4FA /* WebRTCClient.swift in Sources */,
508524
AFAF58EA21EA0AD300609BB7 /* UIColorExtention.swift in Sources */,
509525
);

‎SimpleWebRTC/CameraSession/CameraSession.swift‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
//
2+
// CameraSession.swift
3+
// SimpleWebRTC
4+
//
5+
// Created by tkmngch on 2019/02/08.
6+
// Copyright © 2019 tkmngch. All rights reserved.
7+
//
8+
9+
import Foundation
10+
import AVFoundation
11+
12+
@objc protocol CameraSessionDelegate {
13+
func didOutput(_ sampleBuffer: CMSampleBuffer)
14+
}
15+
class CameraSession: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate {
16+
17+
private var session: AVCaptureSession?
18+
private var output: AVCaptureVideoDataOutput?
19+
private var device: AVCaptureDevice?
20+
weak var delegate: CameraSessionDelegate?
21+
22+
override init() {
23+
super.init()
24+
}
25+
26+
func setupSession(){
27+
self.session = AVCaptureSession()
28+
self.device = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .front)
29+
guard let input = try? AVCaptureDeviceInput(device: device!) else {
30+
print("Caught exception!")
31+
return
32+
}
33+
34+
self.session?.addInput(input)
35+
36+
self.output = AVCaptureVideoDataOutput()
37+
let queue: DispatchQueue = DispatchQueue(label: "videodata", attributes: .concurrent)
38+
self.output?.setSampleBufferDelegate(self, queue: queue)
39+
self.output?.alwaysDiscardsLateVideoFrames = false
40+
self.output?.videoSettings = [kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_420YpCbCr8BiPlanarFullRange] as [String : Any]
41+
self.session?.addOutput(self.output!)
42+
43+
self.session?.sessionPreset = AVCaptureSession.Preset.inputPriority
44+
self.session?.usesApplicationAudioSession = false
45+
46+
self.session?.startRunning()
47+
}
48+
49+
func captureOutput(_ output: AVCaptureOutput, didDrop sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
50+
51+
}
52+
53+
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
54+
self.delegate?.didOutput(sampleBuffer)
55+
}
56+
}

‎SimpleWebRTC/ViewController/ViewController.swift‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,14 @@ import Starscream
1111
import WebRTC
1212
import UIKit
1313

14-
class ViewController: UIViewController, WebSocketDelegate, WebRTCClientDelegate {
14+
class ViewController: UIViewController, WebSocketDelegate, WebRTCClientDelegate, CameraSessionDelegate {
15+
16+
func didOutput(_ sampleBuffer: CMSampleBuffer) {
17+
if self.useCustomCapturer {
18+
self.webRTCClient.captureCurrentFrame(sampleBuffer: sampleBuffer)
19+
}
20+
}
21+
1522

1623
enum messageType {
1724
case greet
@@ -30,9 +37,13 @@ class ViewController: UIViewController, WebSocketDelegate, WebRTCClientDelegate
3037
var webRTCClient: WebRTCClient!
3138
var socket: WebSocket!
3239
var tryToConnectWebSocket: Timer!
40+
var cameraSession: CameraSession?
41+
42+
// You can create video source from CMSampleBuffer :)
43+
var useCustomCapturer: Bool = false
3344

3445
// Constants
35-
let ipAddress: String = "192.168.1.36"
46+
let ipAddress: String = "192.168.11.4"
3647
let wsStatusMessageBase = "WebSocket: "
3748
let webRTCStatusMesasgeBase = "WebRTC: "
3849
let likeStr: String = "Like"
@@ -49,7 +60,14 @@ class ViewController: UIViewController, WebSocketDelegate, WebRTCClientDelegate
4960

5061
webRTCClient = WebRTCClient()
5162
webRTCClient.delegate = self
52-
webRTCClient.setup(videoTrack: true, audioTrack: true, dataChannel: true)
63+
webRTCClient.setup(videoTrack: true, audioTrack: true, dataChannel: true, customFrameCapturer: useCustomCapturer)
64+
65+
if useCustomCapturer {
66+
print("--- use custom capturer ---")
67+
self.cameraSession = CameraSession()
68+
self.cameraSession?.delegate = self
69+
self.cameraSession?.setupSession()
70+
}
5371

5472
socket = WebSocket(url: URL(string: "ws://" + ipAddress + ":8080/")!)
5573
socket.delegate = self

‎SimpleWebRTC/WebRTC/RTCCustomFrameCapture.swift‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
//
2+
// RTCDataCapture.swift
3+
// SimpleWebRTC
4+
//
5+
// Created by tkmngch on 2019/02/08.
6+
// Copyright © 2019 tkmngch. All rights reserved.
7+
//
8+
9+
import Foundation
10+
import WebRTC
11+
12+
class RTCCustomFrameCapturer: RTCVideoCapturer {
13+
14+
let kNanosecondsPerSecond: Float64 = 1000000000
15+
16+
override init(delegate: RTCVideoCapturerDelegate) {
17+
super.init(delegate: delegate)
18+
}
19+
20+
public func capture(_ sampleBuffer: CMSampleBuffer){
21+
let _pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer)
22+
if let pixelBuffer = _pixelBuffer {
23+
let rtcPixelBuffer = RTCCVPixelBuffer(pixelBuffer: pixelBuffer)
24+
let timeStampNs = CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer)) * kNanosecondsPerSecond
25+
let rtcVideoFrame = RTCVideoFrame(buffer: rtcPixelBuffer, rotation: RTCVideoRotation._90, timeStampNs: Int64(timeStampNs))
26+
self.delegate?.capturer(self, didCapture: rtcVideoFrame)
27+
}
28+
}
29+
}

‎SimpleWebRTC/WebRTC/WebRTCClient.swift‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ class WebRTCClient: NSObject, RTCPeerConnectionDelegate, RTCVideoViewDelegate, R
3333
private var remoteStream: RTCMediaStream?
3434
private var dataChannel: RTCDataChannel?
3535
private var channels: (video: Bool, audio: Bool, datachannel: Bool) = (false, false, false)
36+
private var customFrameCapturer: Bool = false
3637

3738
var delegate: WebRTCClientDelegate?
3839
public private(set) var isConnected: Bool = false
@@ -57,11 +58,12 @@ class WebRTCClient: NSObject, RTCPeerConnectionDelegate, RTCVideoViewDelegate, R
5758
}
5859

5960
// MARK: - Public functions
60-
func setup(videoTrack: Bool, audioTrack: Bool, dataChannel: Bool){
61+
func setup(videoTrack: Bool, audioTrack: Bool, dataChannel: Bool, customFrameCapturer: Bool){
6162
print("set up")
6263
self.channels.video = videoTrack
6364
self.channels.audio = audioTrack
6465
self.channels.datachannel = dataChannel
66+
self.customFrameCapturer = customFrameCapturer
6567

6668
let videoEncoderFactory = RTCDefaultVideoEncoderFactory()
6769
let videoDecoderFactory = RTCDefaultVideoDecoderFactory()
@@ -183,6 +185,12 @@ class WebRTCClient: NSObject, RTCPeerConnectionDelegate, RTCVideoViewDelegate, R
183185
}
184186
}
185187

188+
func captureCurrentFrame(sampleBuffer: CMSampleBuffer){
189+
if let capturer = self.videoCapturer as? RTCCustomFrameCapturer {
190+
capturer.capture(sampleBuffer)
191+
}
192+
}
193+
186194
// MARK: - Private functions
187195
// MARK: - Setup
188196
private func setupPeerConnection() -> RTCPeerConnection{
@@ -227,7 +235,9 @@ class WebRTCClient: NSObject, RTCPeerConnectionDelegate, RTCVideoViewDelegate, R
227235
private func createVideoTrack() -> RTCVideoTrack {
228236
let videoSource = self.peerConnectionFactory.videoSource()
229237

230-
if TARGET_OS_SIMULATOR != 0 {
238+
if self.customFrameCapturer {
239+
self.videoCapturer = RTCCustomFrameCapturer(delegate: videoSource)
240+
}else if TARGET_OS_SIMULATOR != 0 {
231241
print("now runnnig on simulator...")
232242
self.videoCapturer = RTCFileVideoCapturer(delegate: videoSource)
233243
}

0 commit comments

Comments
 (0)