Processing heterogenous information as it arrives from a Socket - sockets

Question

A server sends heterogeneous information as stream of bytes. Strings, for instance, are sent with 4 bytes indicating the length and then the characters. This means that my client app must read 4 bytes as an int (which implies that the 4 bytes are available) and then read as many bytes as indicated (the strings are latin1-encoded).
So far I tried two methods: read synchronously with a RawSocket and read the full data dump asynchronously with Socket.listen and process it later. The first method blocks the application, the second is wasteful as it requires to store all the data before processing it.
What I could do for asynchronously read N bytes from a Socket, process them, then read M bytes, process them, etc?

Answer 1

This sounds like you need a ring buffer/byte queue where you can append more data when it arrives, then consume as much as needed from the head when it's available.
There are different ways to implement one, depending on how much you want to avoid copying the bytes. The simplest would be a growing backing buffer with copying. The second approach would be keeping the original lists and combining them when you read only.
Here's a sample implementation:
// Copyright 2021 Google LLC.
// SPDX-License-Identifier: BSD-3-Clause
import "dart:typed_data" show Uint8List;
import "dart:collection" show Queue;
/// A cyclic buffer of bytes.
///
/// Bytes can be appended to the end of the buffer using [append]
/// and consumed from the start of the buffer by [read].
class ByteQueue {
Uint8List _buffer;
int _start = 0;
int _length = 0;
/// Creates a buffer with zero bytes.
///
/// If [initialCapacity] is provided, the buffer will start out
/// with that many bytes of initial capacity. It will not need to
/// grow until that capacity is exhausted.
ByteQueue({int initialCapacity = 256}) : _buffer = Uint8List(initialCapacity);
int get _end {
var end = _start + _length;
if (end > _buffer.length) end -= _buffer.length;
return end;
}
/// Number of bytes currently in the buffer.
///
/// This is the maximal number that can be read by [read] and [peek].
int get length => _length;
int operator [](int index) {
RangeError.checkValidIndex(index, _length);
var i = _start + index;
if (i > _buffer.length) i -= _buffer.length;
return _buffer[i];
}
/// Writes circular buffers into other circular buffers.
///
/// If [end] \< [start], the range of source wraps around at the end of the list.
/// If [offset] + ([end] - [start]) is greater than `target.length`, then
/// the write wraps around past the end of the list.
static void _write(
Uint8List source, int start, int end, Uint8List target, int offset) {
int length = end - start;
if (length >= 0) {
if (offset + length <= target.length) {
target.setRange(offset, offset + length, source, start);
} else {
var firstPart = target.length - offset;
target.setRange(offset, target.length, source, start);
target.setRange(0, length - firstPart, source, start + firstPart);
}
} else {
var firstPart = source.length - start;
_write(source, start, source.length, target, offset);
_write(source, 0, end, target, offset + firstPart);
}
}
static int _limit(int value, int limit) =>
value < limit ? value : value - limit;
/// Copies the next [count] bytes of the buffer into [target].
///
/// The bytes are *not* removed from the buffer, and can be read again.
/// The bytes are written starting at [offset] in [target].
void peek(int count, Uint8List target, [int offset = 0]) {
RangeError.checkValueInInterval(count, 0, _length, "count");
RangeError.checkValueInInterval(offset, 0, target.length, "offset");
if (target.length < count + offset) {
throw ArgumentError.value(
target, "target", "Must have room for $count elements");
}
var end = _limit(_start + count, _buffer.length);
_write(_buffer, _start, end, target, 0);
}
/// Returns the first byte of the buffer.
///
/// The buffer is not modified.
int peekByte() {
if (_length == 0) throw StateError("No element");
return _buffer[_start];
}
/// Consumes a single byte from the head of the buffer.
int readByte() {
if (_length == 0) throw StateError("No element");
var byte = _buffer[_start];
_start = _limit(_start + 1, _buffer.length);
_length -= 1;
return byte;
}
/// Consumes the next [count] bytes of the buffer and moves them into [target].
///
/// The bytes are removed from the head of the buffer.
/// The bytes are written starting at [offset] in [target].
void read(int count, Uint8List target, [int offset = 0]) {
RangeError.checkValueInInterval(count, 0, _length, "count");
RangeError.checkValueInInterval(offset, 0, target.length, "offset");
if (target.length < count + offset) {
throw ArgumentError.value(
target, "target", "Must have room for $count elements");
}
var end = _limit(_start + count, _buffer.length);
_write(_buffer, _start, end, target, 0);
_start = _limit(_start + count, _buffer.length);
_length -= count;
}
/// Removes the first [count] bytes from the buffer.
///
/// Can be useful after a [peek] has turned out to be the bytes
/// that you need, or if you know that the following bytes are
/// not useful.
void remove(int count) {
RangeError.checkValueInInterval(count, 0, _length, "count");
_start = _limit(_start + count, _buffer.length);
;
}
/// Appends [bytes] to the end of the buffer.
void append(Uint8List bytes) {
var newLength = _length + bytes.length;
if (newLength > _buffer.length) {
_grow(newLength);
}
_write(bytes, 0, bytes.length, _buffer, _end);
_length = newLength;
}
void _grow(int newLength) {
var capacity = _buffer.length;
do {
capacity *= 2;
} while (capacity < newLength);
var newBuffer = Uint8List(capacity);
_write(_buffer, _start, _end, newBuffer, 0);
_buffer = newBuffer;
_start = 0;
}
}
// Or another one with the same interface,
// but which doesn't copy bytes into the buffer, only out of it.
class ByteQueue2 {
final Queue<Uint8List> _source = Queue();
int _length = 0;
// The offset into the first element of _source that we haven't consumed.
int _start = 0;
int get length => _length;
void append(Uint8List bytes) {
_source.add(bytes);
_length += bytes.length;
}
int peekByte() {
if (_length == 0) throw StateError("No element");
return _source.first[_start];
}
int readByte() {
if (_length == 0) throw StateError("No element");
var first = _source.first;
var byte = first[_start];
_start += 1;
if (_start >= first.length) {
_source.removeFirst();
_start = 0;
}
_length -= 1;
return byte;
}
void peek(int count, Uint8List target, [int offset = 0]) {
RangeError.checkValueInInterval(count, 0, _length, "count");
RangeError.checkValueInInterval(offset, 0, target.length, "offset");
if (offset + count > target.length) {
throw ArgumentError.value(target, "target",
"Must have length >= ${offset + count}, was: ${target.length}");
}
var start = _start;
for (var source in _source) {
if (count == 0) return;
var length = source.length - start;
if (count <= length) {
target.setRange(offset, offset + count, source, start);
return;
}
target.setRange(offset, offset + length, source, start);
start = 0;
offset += length;
}
}
void read(int count, Uint8List target, [int offset = 0]) {
RangeError.checkValueInInterval(count, 0, _length, "count");
RangeError.checkValueInInterval(offset, 0, target.length, "offset");
if (offset + count > target.length) {
throw ArgumentError.value(target, "target",
"Must have length >= ${offset + count}, was: ${target.length}");
}
var start = _start;
while (count > 0) {
var source = _source.first;
var length = source.length - start;
if (count < length) {
target.setRange(offset, offset + count, source, start);
_start = start + count;
_length -= count;
return;
}
target.setRange(offset, offset + length, source, start);
offset += length;
count -= length;
_length -= length;
start = _start = 0;
_source.removeFirst();
}
}
}
(No priomises).