Screen Sharing using Twilio in iOS - swift

Question

I am using Twilio iOS framework to connect in the group room.
On the click on connect room button below is the code which I used
let recorder = RPScreenRecorder.shared()
recorder.isMicrophoneEnabled = false
recorder.isCameraEnabled = false
// The source produces either downscaled buffers with smoother motion, or an HD screen recording.
videoSource = ReplayKitVideoSource(isScreencast: true, telecineOptions: ReplayKitVideoSource.TelecineOptions.disabled)
screenTrack = LocalVideoTrack(source: videoSource!,
enabled: true,
name: "Screen")
recorder.startCapture(handler: { (sampleBuffer, type, error) in
if error != nil {
print("Capture error: ", error as Any)
return
}
switch type {
case RPSampleBufferType.video:
self.videoSource?.processFrame(sampleBuffer: sampleBuffer)
break
case RPSampleBufferType.audioApp:
break
case RPSampleBufferType.audioMic:
// We use `TVIDefaultAudioDevice` to capture and playback audio for conferencing.
break
}
}) { (error) in
if error != nil {
print("Screen capture error: ", error as Any)
} else {
print("Screen capture started.")
}
}
if (accessToken == "TWILIO_ACCESS_TOKEN") {
do {
accessToken = try TokenUtils.fetchToken(url: tokenUrl)
} catch {
let message = "Failed to fetch access token"
logMessage(messageText: message)
return
}
}
// Prepare local media which we will share with Room Participants.
self.prepareLocalMedia()
// Preparing the connect options with the access token that we fetched (or hardcoded).
let connectOptions = ConnectOptions(token: accessToken) { (builder) in
// Use the local media that we prepared earlier.
builder.audioTracks = self.localAudioTrack != nil ? [self.localAudioTrack!] : [LocalAudioTrack]()
builder.videoTracks = self.localVideoTrack != nil ? [self.localVideoTrack!, self.screenTrack!] : [LocalVideoTrack]()
// Use the preferred audio codec
if let preferredAudioCodec = Settings.shared.audioCodec {
builder.preferredAudioCodecs = [preferredAudioCodec]
}
// Use the preferred video codec
if let preferredVideoCodec = Settings.shared.videoCodec {
builder.preferredVideoCodecs = [preferredVideoCodec]
}
// Use the preferred encoding parameters
if let encodingParameters = Settings.shared.getEncodingParameters() {
builder.encodingParameters = encodingParameters
}
// Use the preferred signaling region
if let signalingRegion = Settings.shared.signalingRegion {
builder.region = signalingRegion
}
builder.roomName = self.roomTextField.text
}
// Connect to the Room using the options we provided.
room = TwilioVideoSDK.connect(options: connectOptions, delegate: self)
logMessage(messageText: "Attempting to connect to room \(String(describing: self.roomTextField.text))")
When I connected in the group with remote participant I want to share the screen with remote participant.
To implement this feature I have referred the “ReplayKitExample” with in-app capture method. But not able to do that.
Remote participant not able to see the screen share content.
Nothing is happening related to screen share with this, and looking for inputs on implementing it.
I want to share the screen to remote participant.

Answer 1

Its happening because you are trying to send "cameraSource" and "videoSource" both data at the same time you have to unsubscribe the "cameraSource" before sending "viseoSource".
Heres my code you can refer:
//MARK: - Screen Sharing via replaykit
extension CallRoomViewController: RPScreenRecorderDelegate {
func broadCastButtonTapped(){
guard screenRecorder.isAvailable else {
print("Not able to Broadcast")
return
}
print("Can Broadcast")
if self.videoSource != nil {
self.stopConference()
} else {
self.startConference()
}
}
func publishVideoTrack(){
if let participant = self.room?.localParticipant,
let videoTrack = self.localVideoTrack {
participant.publishVideoTrack(videoTrack)
}
}
func unpublishVideoTrack(){
if let participant = self.room?.localParticipant,
let videoTrack = self.localVideoTrack {
participant.unpublishVideoTrack(videoTrack)
}
}
func stopConference() {
self.unpublishVideoTrack()
self.localVideoTrack = nil
self.videoSource = nil
self.localVideoTrack = LocalVideoTrack(source: cameraSource!, enabled: true, name: "Camera")
screenRecorder.stopCapture{ (captureError) in
if let error = captureError {
print("Screen capture stop error: ", error as Any)
} else {
print("Screen capture stopped.")
self.publishVideoTrack()
}
}
}
func startConference() {
self.unpublishVideoTrack()
self.localVideoTrack = nil
// We are only using ReplayKit to capture the screen.
// Use a LocalAudioTrack to capture the microphone for sharing audio in the room.
screenRecorder.isMicrophoneEnabled = false
// Use a LocalVideoTrack with a CameraSource to capture the camera for sharing camera video in the room.
screenRecorder.isCameraEnabled = false
// The source produces either downscaled buffers with smoother motion, or an HD screen recording.
self.videoSource = ReplayKitVideoSource(isScreencast: true,
telecineOptions: ReplayKitVideoSource.TelecineOptions.p60to24or25or30)
self.localVideoTrack = LocalVideoTrack(source: videoSource!,
enabled: true,
name: "Screen")
let videoCodec = Settings.shared.videoCodec ?? Vp8Codec()!
let (_, outputFormat) = ReplayKitVideoSource.getParametersForUseCase(codec: videoCodec,
isScreencast: true,
telecineOptions:ReplayKitVideoSource.TelecineOptions.p60to24or25or30)
self.videoSource?.requestOutputFormat(outputFormat)
screenRecorder.startCapture(handler: { (sampleBuffer, type, error) in
if error != nil {
print("Capture error: ", error as Any)
return
}
switch type {
case RPSampleBufferType.video:
self.videoSource?.processFrame(sampleBuffer: sampleBuffer)
break
case RPSampleBufferType.audioApp:
break
case RPSampleBufferType.audioMic:
// We use `TVIDefaultAudioDevice` to capture and playback audio for conferencing.
break
default:
print(error ?? "screenRecorder error")
}
}) { (error) in
if error != nil {
print("Screen capture error: ", error as Any)
} else {
print("Screen capture started.")
self.publishVideoTrack()
}
}
}
}
And you can connect room from viewDidLoad()
func connectToChatRoom(){
// Configure access token either from server or manually.
// If the default wasn't changed, try fetching from server.
accessToken = self.callRoomDetail.charRoomAccessToken
guard accessToken != "TWILIO_ACCESS_TOKEN" else {
let message = "Failed to fetch access token"
print( message)
return
}
// Prepare local media which we will share with Room Participants.
self.prepareLocalMedia()
// Preparing the connect options with the access token that we fetched (or hardcoded).
let connectOptions = ConnectOptions(token: accessToken) { (builder) in
// The name of the Room where the Client will attempt to connect to. Please note that if you pass an empty
// Room `name`, the Client will create one for you. You can get the name or sid from any connected Room.
builder.roomName = self.callRoomDetail.chatRoomName
// Use the local media that we prepared earlier.
if let audioTrack = self.localAudioTrack {
builder.audioTracks = [ audioTrack ]
}
if let videoTrack = self.localVideoTrack {
builder.videoTracks = [ videoTrack ]
}
// Use the preferred audio codec
if let preferredAudioCodec = Settings.shared.audioCodec {
builder.preferredAudioCodecs = [preferredAudioCodec]
}
// Use the preferred video codec
if let preferredVideoCodec = Settings.shared.videoCodec {
builder.preferredVideoCodecs = [preferredVideoCodec]
}
// Use the preferred encoding parameters
let videoCodec = Settings.shared.videoCodec ?? Vp8Codec()!
let (encodingParams, _) = ReplayKitVideoSource.getParametersForUseCase(codec: videoCodec,
isScreencast: true,
telecineOptions:ReplayKitVideoSource.TelecineOptions.p60to24or25or30)
builder.encodingParameters = encodingParams
// Use the preferred signaling region
if let signalingRegion = Settings.shared.signalingRegion {
builder.region = signalingRegion
}
builder.isAutomaticSubscriptionEnabled = true
builder.isNetworkQualityEnabled = true
builder.networkQualityConfiguration = NetworkQualityConfiguration(localVerbosity: .minimal,
remoteVerbosity: .minimal)
}
// Connect to the Room using the options we provided.
room = TwilioVideoSDK.connect(options: connectOptions, delegate: self)
print( "Attempting to connect to room \(self.callRoomDetail.chatRoomName ?? ""))")
self.showRoomUI(inRoom: true)
}
You can get ReplayKitVideoSource file and other files from twilio repository https://github.com/twilio/video-quickstart-ios/tree/master/ReplayKitExample

Answer 2

I work at Twilio and I can confirm that you should be able to publish video tracks for both camera and screen at the same time without issue.
It is difficult to identify why this is not working for you without a completely functional example app.
However I have tested this using one of our reference apps and confirmed it is working. More details are here: https://github.com/twilio/video-quickstart-ios/issues/650#issuecomment-1178232542
Hopefully this is a useful example for how to publish both camera and screen video at the same time.