How to play a stream from GStreamer in Unity through TCP - unity3d

Question

I'm working on a project which require streaming video with GStreamer, and then I need to display the video in Unity.
I am now trying a method:
I use the following GStreamer command to put webcam view onto tcp
gst-launch-1.0 v4l2src ! videoconvert ! video/x-raw,width=640,height=480,framerate=20/1 ! jpegenc ! multipartmux ! tcpserversink host=127.0.0.1 port=5000
which can be viewed by GStreamer with:
gst-launch-1.0 tcpclientsrc port=5000 host=127.0.0.1 ! multipartdemux ! jpegdec ! autovideosink
In Unity, I use the following script to get the streaming bytes and decode them
public class _TextureReceiver : MonoBehaviour {
public int port = 5000;
public string IP = "127.0.0.1";
TcpClient client;
[HideInInspector]
public Texture2D texture;
private bool stop = false;
[Header("Must be the same in sender and receiver")]
public int messageByteLength = 24;
// Use this for initialization
void Start() {
Application.runInBackground = true;
client = new TcpClient();
//Connect to server from another Thread
Loom.RunAsync(() => {
// if on desktop
// client.Connect(IPAddress.Loopback, port);
client.Connect(IPAddress.Parse(IP), port);
imageReceiver();
});
}
void imageReceiver() {
//While loop in another Thread is fine so we don't block main Unity Thread
Loom.RunAsync(() => {
while (!stop) {
//Read Image Count
int imageSize = readImageByteSize(messageByteLength);
//Read Image Bytes and Display it
readFrameByteArray(imageSize);
//readFrameByteArray(12288000);
}
});
}
//Converts the byte array to the data size and returns the result
int frameByteArrayToByteLength(byte[] frameBytesLength) {
/*if (BitConverter.IsLittleEndian)
{
Array.Reverse(frameBytesLength);
UnityEngine.Debug.Log("- BitConverter.IsLittleEndian");
}*/
var sb = new StringBuilder("frameBytesLength[] { ");
foreach (var b in frameBytesLength)
{
sb.Append(b + ", ");
}
sb.Append("}");
UnityEngine.Debug.Log(sb.ToString());
//int byteLength = BitConverter.ToInt32(frameBytesLength, 0);
int byteLength = frameBytesLength[0] + frameBytesLength[1] * 256;
return byteLength;
}
private int readImageByteSize(int size) {
UnityEngine.Debug.Log("- image byte size: " + size);
bool disconnected = false;
NetworkStream serverStream = client.GetStream();
UnityEngine.Debug.Log("- serverStream: " + serverStream);
byte[] imageBytesCount = new byte[size];
var total = 0;
do {
var read = serverStream.Read(imageBytesCount, total, size - total);
UnityEngine.Debug.LogFormat("Client recieved "+read+" bytes");
if (read == 0)
{
disconnected = true;
break;
}
total += read;
UnityEngine.Debug.Log("- image byte read: " + read);
UnityEngine.Debug.Log("- image byte total: " + total);
} while (total != size);
UnityEngine.Debug.Log("- break While");
int byteLength;
if (disconnected) {
UnityEngine.Debug.Log("disconnected");
byteLength = -1;
} else {
byteLength = frameByteArrayToByteLength(imageBytesCount);
}
return byteLength;
}
private void readFrameByteArray(int size) {
bool disconnected = false;
UnityEngine.Debug.Log("- image size: " + size);
NetworkStream serverStream = client.GetStream();
byte[] imageBytes = new byte[size];
var total = 0;
//do{
var read = serverStream.Read(imageBytes, total, size - total);
if (read == 0)
{
disconnected = true;
//break;
}
total += read;
UnityEngine.Debug.Log("- read: " + read);
UnityEngine.Debug.Log("- total: " + total);
//} while (total != size);
byte[] imageBytes2 = new byte[read];
imageBytes.CopyTo(imageBytes2, 0);
UnityEngine.Debug.Log("break while");
var sb = new StringBuilder("imageBytes[] { ");
foreach (var b in imageBytes2)
{
sb.Append(b + ", ");
}
sb.Append("}");
UnityEngine.Debug.Log(sb.ToString());
bool readyToReadAgain = false;
//Display Image
if (!disconnected) {
//Display Image on the main Thread
Loom.QueueOnMainThread(() => {
loadReceivedImage(imageBytes2);
readyToReadAgain = true;
});
}
//Wait until old Image is displayed
while (!readyToReadAgain) {
System.Threading.Thread.Sleep(1);
}
}
void loadReceivedImage(byte[] receivedImageBytes) {
if(texture) texture.LoadImage(receivedImageBytes);
}
public void SetTargetTexture (Texture2D t) {
texture = t;
}
void OnApplicationQuit() {
stop = true;
if (client != null) {
client.Close();
}
}
}
However, It's not working.
I think it is because the encoding method and decoding method does not match, but I don't know how to interpretate GStreamer's byte sequence.
At int imageSize = readImageByteSize(messageByteLength);,
the imageSize should be computed with an incoming byte that look something like this:
frameBytesLength[] { 196, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }
But instead it look like this:
frameBytesLength[] { 255, 216, 255, 224, 0, 16, 74, 70, 73, 70, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 255, 219, 0, 67, }
causing the size is too large that it goes negative, and thus can not continue the decoding process.
That's what I've ot so far.
Here is the reference of Unity code:
https://github.com/BarakChamo/uTextureSendReceive
p.s. I'm using Linux, so I cannot use mray GStreamer Unity Plugin since it does not support Linux.
Thank you for your time.

Answer 1

I over come this problem in a way that is not very elegant...
I just assign a fixed max-byte-size to every frame that is received.
It's not the best way, but that's working for me.