Running an asyncoio socket server and client on the same process for tests - sockets

I wrote a socket server and client package with asyncio that happens to be a single module with two classes Server and Client. The intention is to have the code check if the specified port is in use on the same device, and if so, it will choose the use the Client class instead of the Server class. I am doing this to enable local testing between client and server for a larger goal.
I wrote my unit test with pytest and I can get it to pass if I run the server in its own process first, then run the client in another process.
However, I want to test both server and client together in the same process using pytest. The test never completes this way, however.
Am I running into an issue with asyncio here? Or is it an issue with pytest? Or is it something I am doing wrong in my code?
WebSockets.websocket
import asyncio
class Server:
def __init__(self, host, port):
self.host = host
self.port = port
self.server = None
self.connections = {}
async def start(self):
self.server = server = await asyncio.start_server(self.handle_client, self.host, self.port)
addr = server.sockets[0].getsockname()
print(f'Serving on {addr}')
async with server:
await server.serve_forever()
async def handle_client(self, reader, writer):
addr = writer.get_extra_info('peername')
print(f'New connection from {addr}')
self.connections[addr] = writer
try:
while not reader.at_eof():
data = await reader.read(100)
message = data.decode()
if message:
print(f'Received {message!r} from {addr}')
await self.send(message, addr)
finally:
del self.connections[addr]
writer.close()
async def send(self, message, addr):
writer = self.connections.get(addr)
if not writer:
return
writer.write(message.encode())
await writer.drain()
async def recv(self, addr):
reader, _ = await asyncio.open_connection(addr[0], addr[1])
data = await reader.read(100)
return data.decode()
async def stop(self):
await self.server.close()
class Client:
def __init__(self, host, port):
self.host = host
self.port = port
self.reader = None
self.writer = None
async def connect(self):
self.reader, self.writer = await asyncio.open_connection(self.host, self.port)
async def send(self, message):
self.writer.write(message.encode())
async def recv(self):
data = await self.reader.read(100)
return data.decode()
async def close(self):
self.writer.close()
Test
import socket
import asyncio
from WebSockets.websocket import Server, Client
import pytest
def check_port(address, port):
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((address, port))
s.close()
return "server"
except OSError:
return "client"
async def serve():
server = Server('localhost', 8910)
await server.start()
async def client_connect():
client = Client('localhost', 8910)
await client.connect()
await client.send("Hello world!")
message = await client.recv()
if message:
print(f"Received {message!r} from server")
assert message == "Hello world!"
await client.close()
#pytest.mark.asyncio
async def test_sockets():
if check_port("localhost", 8910) == "server":
await serve()
# for reference on how to use in non pytest module
# loop = asyncio.new_event_loop()
# loop.run_until_complete(serve())
# loop.close()
else:
await client_connect()
# for reference on how to use in non pytest module
# loop = asyncio.new_event_loop()
# loop.run_until_complete(client_connect())
# loop.close()

Related

Update on sqlalchemy +asyncpg , returns another operation in progress on pytest

I just updated my app with sqlalchemy to create an async connections, no problem, but when writing tests, I can't do the update as opposed to create:
affected function:
#function
#classmethod
async def update_instance(
cls,
instance: InstanceInputOnUpdate
) -> Union['EdaNCEInstance', EdaNCEInstanceOperationError]:
async with get_session() as conn:
result = await conn.execute(
select(Instance).where(Instance.id==eda_nce_instance.id)
)
instance_to_update = result.scalars().unique().first()
if instance_to_update is not None:
instance_to_update.name = instance.name
instance_to_update.host = instance.host
await conn.commit()
return instance_to_update
else:
return InstanceOperationError(
result= False,
message= f"Can't find the instance ID '{instance.id}'"
)
test:
#pytest.mark.asyncio
async def test_04_update_instance():
async with AsyncClient(app=app, base_url="http://test") as c:
res = await Instance.update_eda_nce_instance(
instance=InstanceInputOnUpdate(
id=DUMMY_ID,
name="test_server1",
host="123.123.123.123",
)
)
assert isinstance(res, Instance) == True
assert res.host == "124.123.144.144"
return res
errors:
#...
E sqlalchemy.dialects.postgresql.asyncpg.AsyncAdapt_asyncpg_dbapi.InterfaceError: <class 'asyncpg.exceptions._base.InterfaceError'>: cannot perform operation: another operation is in progress
venv/lib/python3.9/site-packages/sqlalchemy/dialects/postgresql/asyncpg.py:682: InterfaceError
my db.py
SQLALCHEMY_DATABASE_URL = settings.get_settings().postgresql_conn_url
engine = create_async_engine(SQLALCHEMY_DATABASE_URL, echo=False, future=True)
async_session = sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)
Base = declarative_base()
async def init_db():
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
#asynccontextmanager
async def get_session() -> AsyncGenerator[AsyncSession, None]:
async with async_session() as session:
async with session.begin():
try:
yield session
finally:
await session.close()
In creation I have no problem, and it doesn't change much from the update
#classmethod
async def create_instance(
cls,
instance: InstanceInputOnCreate
) -> Union['Instance', InstanceOperationError]:
async with get_session() as conn:
new_instance = Instance(
name=instance.name,
host=instance.host
)
conn.add(new_instance)
try:
await conn.commit()
return new_instance
except Exception:
conn.rollback()
return InstanceOperationError(
result=False,
message=f"Instance '{instance.name}' already exists"
)
I didn't create mocks, because CI/CDs are set up to create a test database where Alembic can be started first for migrations, then the database is ready to test on it
sqlalchemy==1.4.46
fastapi-utils==0.2.1
aiosqlite==0.18.0
asyncpg==0.27.0
sqlalchemy-utils==0.39.0
pytest==7.2.1
pytest-asyncio==0.20.3
pytest-mock==3.10.0
pydantic~=1.10.4
aiokafka~=0.8.0
requests~=2.28.1
fastapi==0.70.1
If there are typo in the code, it's not a problem I had to change the names to create this question.

python-kafka asyncio is blocking

I am trying to create a websocket server that receives input from user, and at the same time read from kafka and continuously send data to user. I am having trouble getting both to run concurrently.
async def sender(websocket):
consumer = KafkaConsumer(...) # read from kafka
for message in consumer:
msg = json.loads(message.value.decode())
await websocket.send(msg)
async def handle_client(websocket):
asyncio.create_task(sender(websocket))
while True:
print("Waiting for client input")
try:
msg = await websocket.recv()
# do something with msg
async def main():
async with websockets.serve(handle_client, "localhost", 8765):
await asyncio.Future() # run forever
if __name__ == "__main__":
asyncio.run(main())
They seem to be blocking. I think the issuer is because sender() blocks it, it never sleeps?

asyncpg: How to construct SET strings with parameters

What is the correct way to pass in parameters in a SET query?
this will return asyncpg.exceptions.PostgresSyntaxError: syntax error at or near "$1"
import asyncio
import asyncpg
async def main():
conn = await asyncpg.connect(user="xx", password="yy", host="127.0.0.1", database="my_db")
# works
await conn.execute("select $1", "1")
identity = "arn:aws:sts::123456:assumed-role/thing_1"
# fails
await conn.execute("set session iam.identity = $1", identity)
asyncio.run(main())
asyncpg has a function for this:
await conn.execute("select set_config('iam.identity', $1, false)", identity)

SSLWantReadError when using async sock_recv with an SSL connection

I'm implementing a producer-consumer program in a server implementation using sockets and asyncio. The problem is the async function sock_recv() does not seem to be working properly when used with a socket wrapped in an ssl connection. Following is the working code.
Server side
import asyncio
import random
import socket
import ssl
SERVER_ADDRESS = (HOST, PORT) = "127.0.0.1", 8881
async def producer(queue, client_connection, event_loop):
while True:
print("Waiting for sock_recv")
await event_loop.sock_recv(client_connection, 4096)
r = random.randint(1,101)
print("Produced: %d" % r)
await queue.put(r)
await asyncio.sleep(0)
async def consumer(queue):
while True:
print("Wating for queue.get()")
r = await queue.get()
await asyncio.sleep(2)
print("Consumed: %d" % r)
async def main():
listen_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
listen_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
listen_socket.bind(SERVER_ADDRESS)
listen_socket.listen(5)
ssl_context = ssl.create_default_context(purpose=ssl.Purpose.CLIENT_AUTH)
ssl_context.load_cert_chain(certfile="certificate.pem", keyfile="key.pem")
client_connection, client_address = listen_socket.accept()
# client_connection = ssl_context.wrap_socket(
# client_connection, server_side=True
# )
client_connection.setblocking(False)
queue = asyncio.Queue()
t1 = asyncio.create_task(producer(queue, client_connection, asyncio.get_event_loop()))
t2 = asyncio.create_task(consumer(queue))
await asyncio.wait([t1, t2])
event_loop = asyncio.get_event_loop()
asyncio.run(main())
Client side
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(("127.0.0.1", 8881))
s.sendall(b"Hello")
Output
Waiting for sock_recv
Waiting for queue.get()
Produced: 49
Waiting for sock_recv
Consumed: 49
Waiting for queue.get()
Here's the problem, When I uncomment the following part
# client_connection = ssl_context.wrap_socket(
# client_connection, server_side=True
# )
It blocks on the sock_recv() function.
With the uncommented code, I get the following output:
Output
Waiting for sock_recv
Waiting for queue.get()
Client Code
import socket
import ssl
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock = ssl.wrap_socket(s)
sock.connect(("127.0.0.1", 8881))
sock.sendall(b"Hello")
Finally, when I shutdown the server with ctrl-c. I get the following output
^CTask exception was never retrieved
future: <Task finished coro=<producer() done, defined at asyncio_test.py:8> exception=SSLWantReadError(2, 'The operation did not complete (read) (_ssl.c:2488)')>
Traceback (most recent call last):
File "asyncio_test.py", line 11, in producer
await event_loop.sock_recv(client_connection, 4096)
File "/home/coverfox/.pyenv/versions/3.7.3/lib/python3.7/asyncio/selector_events.py", line 352, in sock_recv
return await fut
File "/home/coverfox/.pyenv/versions/3.7.3/lib/python3.7/asyncio/selector_events.py", line 366, in _sock_recv
data = sock.recv(n)
File "/home/coverfox/.pyenv/versions/3.7.3/lib/python3.7/ssl.py", line 1037, in recv
return self.read(buflen)
File "/home/coverfox/.pyenv/versions/3.7.3/lib/python3.7/ssl.py", line 913, in read
return self._sslobj.read(len)
ssl.SSLWantReadError: The operation did not complete (read) (_ssl.c:2488)
Traceback (most recent call last):
File "asyncio_test.py", line 42, in <module>
asyncio.run(main())
File "/home/coverfox/.pyenv/versions/3.7.3/lib/python3.7/asyncio/runners.py", line 43, in run
return loop.run_until_complete(main)
File "/home/coverfox/.pyenv/versions/3.7.3/lib/python3.7/asyncio/base_events.py", line 571, in run_until_complete
self.run_forever()
File "/home/coverfox/.pyenv/versions/3.7.3/lib/python3.7/asyncio/base_events.py", line 539, in run_forever
self._run_once()
File "/home/coverfox/.pyenv/versions/3.7.3/lib/python3.7/asyncio/base_events.py", line 1739, in _run_once
event_list = self._selector.select(timeout)
File "/home/coverfox/.pyenv/versions/3.7.3/lib/python3.7/selectors.py", line 468, in select
fd_event_list = self._selector.poll(timeout, max_ev)
KeyboardInterrupt
Edit:
I just found out that it works if I pass do_handshake_on_connect=False in the wrap_socket() function in the client code, but then ssl won't work.
So, it turns out that the async function sock_recv() does not support SSLSocket, since SSL needs to write to user-space buffer as described in this SO answer.
The way to work around this issue is to use the transports or streams in asyncio. Here, is the working version of the code in the above question.
import asyncio
import random
import socket
import ssl
SERVER_ADDRESS = (HOST, PORT) = "127.0.0.1", 8881
async def producer(reader, writer, queue):
while True:
print("Waiting for sock_recv")
await reader.read(16)
r = random.randint(1,101)
print("Produced: %d" % r)
await queue.put(r)
await asyncio.sleep(0)
async def consumer(queue):
while True:
print("Wating for queue.get()")
r = await queue.get()
await asyncio.sleep(2)
print("Consumed: %d" % r)
async def set_up_producer_consumer(reader, writer):
queue = asyncio.Queue()
t1 = asyncio.create_task(producer(reader, writer, queue))
t2 = asyncio.create_task(consumer(queue))
await asyncio.wait([t1, t2])
async def main():
ssl_context = ssl.create_default_context(purpose=ssl.Purpose.CLIENT_AUTH)
ssl_context.load_cert_chain(certfile="certificate.pem", keyfile="key.pem")
server = await asyncio.start_server(set_up_producer_consumer, HOST, PORT, family=socket.AF_INET, ssl=ssl_context, reuse_address=True)
await server.wait_closed()
event_loop = asyncio.get_event_loop()
asyncio.run(main())

Python code after sockets connection executed only once

What are the intentions of this program:
I want to send some commands from a client to a server using sockets, the server then send these command to an Arduino using serial. And another thing that I want the server to do in the future is that periodically sends other commands to the Arduino without getting any input from the client, so the sockets needs to be non-blocking or there needs to be another way to run the code separately from the sockets code.
The problem is that the part that is supposed to send the command to the Arduino only runs once.
What I have come up with after playing with the debugger in Pycharm, is that the problem is that the following line blocks after a connection has been established, and thus not allowing the rest of the code to be run.
conn, addr = s.accept()
Is this correct, or is there something else wrong?
I have tried to set the socket to non-blocking but when I do this I get an error.
"BlockingIOError: [WinError 10035] A non-blocking socket operation could not be completed immediately"
I have some basic knowledge of C/C++ and C# and am new to Python.
server.py
import socket
import serial
import sys
from _thread import *
import threading
import queue
# command that the client sends are "ON" and "OFF"
class serialConnect:
comPort =' '
baudrate = 115200
myserial = serial.Serial('COM5', baudrate)
def serialstart(self):
# self.comPort = input('Comport: ')
try:
self.myserial.open()
except IOError:
print('Port is already open!')
def serialRead(self):
data = self.myserial.read(16)
data.decode('UTF-8')
return data
def serialWrite(self, data):
data += '\n' #the arduino needs a \n after each command.
databytes = data.encode('UTF-8')
self.myserial.write(databytes)
print('send data: ', databytes)
def threaded_client(conn, dataqueue):
data = {bytes}
conn.send(str.encode('welcome, type your info \n'))
while True:
data = conn.recv(2048)
if not data:
break
reply = 'server output: ' + data.decode('UTF-8')
dataqueue.put(data.decode('UTF-8'))
print("Items in queue: ",dataqueue.qsize())
#conn.sendall(str.encode(reply))
print("Recieved data in threaded_client: ", data.decode('UTF-8') + '\n')
conn.close()
def Main():
ser = serialConnect()
host = ''
port = 5555
dataRecieved = 'hello'
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(2)
s.setblocking(1) #when set to non-blocking error occurs : "BlockingIOError: [WinError 10035] A non-blocking socket operation could not be completed immediately"
workQueue = queue.Queue(10)
try:
s.bind((host,port))
except socket.error as e:
print(str(e))
s.listen(5)
print('waiting for a connection')
while True:
try:
conn, addr = s.accept() #once connection is established it blocks?
print('connected to: ' + addr[0] + ':' + str())
t = threading.Thread(target=threaded_client, args=(conn, workQueue))
t.daemon = True
t.start()
except:
e = sys.exc_info()
print('Error:', e)
# This section of code is only run once, doesn't matter if put inside try block or not. :(
dataRecieved = workQueue.get()
print('The recieved data: ', dataRecieved)
ser.serialstart()
ser.serialWrite(dataRecieved)
if __name__ == '__main__':
Main()
client.py
import socket
def Main():
host = '127.0.0.1'
port = 5555
message = "<,R,G,B,>"
mySocket = socket.socket()
mySocket.connect((host, port))
while message != 'q':
message = input(" -> ")
mySocket.send(message.encode())
mySocket.close()
if __name__ == '__main__':
Main()
Arduino Code
String inputString = ""; // a string to hold incoming data
boolean stringComplete = false; // whether the string is complete
int LEDpin = 10;
// the setup function runs once when you press reset or power the board
void setup() {
// initialize digital pin 13 as an output.
pinMode(10, OUTPUT);
Serial.begin(19200);
}
// the loop function runs over and over again forever
void loop() {
serialEvent();
if(stringComplete){
Serial.println(inputString);
if(inputString == "ON\n"){
digitalWrite(LEDpin, HIGH); // turn the LED on (HIGH is the voltage level)
}
if(inputString == "OFF\n"){
digitalWrite(LEDpin, LOW); // turn the LED off by making the voltage LOW
}
inputString = "";
stringComplete = false;
}
}
void serialEvent()
{
while (Serial.available()) {
// get the new byte:
char inChar = (char)Serial.read();
// add it to the inputString:
inputString += inChar;
// if the incoming character is a newline, set a flag
// so the main loop can do something about it:
if (inChar == '\n') {
stringComplete = true;
}
}
}
Refactored server code for anyone that is interested in it.
I am not sure if this is up to standard, but it is working.
import serial
import socket
import queue
import sys
import threading
class serialConnect:
comPort = 'COM5'
baudrate = 115200
myserial = serial.Serial(comPort, baudrate)
def serial_run(self):
# self.comPort = input('Comport: ')
try:
if not self.myserial.isOpen():
self.myserial.open()
else:
print('Port is already open!')
except IOError as e:
print('Error: ', e)
def serial_read(self):
data = self.myserial.read(16)
data.decode('UTF-8')
return data
def serial_write(self, data):
data += '\n' #the arduino needs a \n after each command.
databytes = data.encode('UTF-8')
self.myserial.write(databytes)
print('send data: ', databytes)
class socketServer:
host = ''
port = 5555
soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
soc.setblocking(1)
data_queue = queue.Queue(1)
def __init__(self):
try:
self.soc.bind((self.host, self.port))
except:
print('Bind error: ', sys.exc_info())
self.soc.listen(5)
def socket_accept_thread(self):
while True:
try:
print('Waiting for a new connection')
conn, addr = self.soc.accept()
client_thread = threading.Thread(target=self.threaded_client, args=(conn, self.data_queue))
client_thread.daemon = True
client_thread.start()
except:
print('Accept thread Error: ', sys.exc_info())
def threaded_client(self, conn, data_queue):
# conn.send(str.encode('welcome, type your info \n'))
try:
while True:
data = conn.recv(2048)
if not data:
break
# reply = 'server output: ' + data.decode('UTF-8')
data_queue.put(data.decode('UTF-8'))
print("Items in queue: ", data_queue.qsize())
# conn.sendall(str.encode(reply))
print("Received data in threaded_client: ", data.decode('UTF-8'))
except:
print("Error: ", sys.exc_info())
conn.close()
def get_data(self):
data = self.data_queue.get()
return data
def Main():
server = socketServer()
arduino_conn = serialConnect()
accept_thread = threading.Thread(target=server.socket_accept_thread)
data_received = 'Nothing received'
while True:
if not accept_thread.is_alive():
accept_thread.daemon = True
accept_thread.start()
arduino_conn.serial_run()
data_received = server.get_data()
arduino_conn.serial_write(data_received)
if __name__ == '__main__':
Main()