SSLWantReadError when using async sock_recv with an SSL connection - sockets

I'm implementing a producer-consumer program in a server implementation using sockets and asyncio. The problem is the async function sock_recv() does not seem to be working properly when used with a socket wrapped in an ssl connection. Following is the working code.
Server side
import asyncio
import random
import socket
import ssl
SERVER_ADDRESS = (HOST, PORT) = "127.0.0.1", 8881
async def producer(queue, client_connection, event_loop):
while True:
print("Waiting for sock_recv")
await event_loop.sock_recv(client_connection, 4096)
r = random.randint(1,101)
print("Produced: %d" % r)
await queue.put(r)
await asyncio.sleep(0)
async def consumer(queue):
while True:
print("Wating for queue.get()")
r = await queue.get()
await asyncio.sleep(2)
print("Consumed: %d" % r)
async def main():
listen_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
listen_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
listen_socket.bind(SERVER_ADDRESS)
listen_socket.listen(5)
ssl_context = ssl.create_default_context(purpose=ssl.Purpose.CLIENT_AUTH)
ssl_context.load_cert_chain(certfile="certificate.pem", keyfile="key.pem")
client_connection, client_address = listen_socket.accept()
# client_connection = ssl_context.wrap_socket(
# client_connection, server_side=True
# )
client_connection.setblocking(False)
queue = asyncio.Queue()
t1 = asyncio.create_task(producer(queue, client_connection, asyncio.get_event_loop()))
t2 = asyncio.create_task(consumer(queue))
await asyncio.wait([t1, t2])
event_loop = asyncio.get_event_loop()
asyncio.run(main())
Client side
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(("127.0.0.1", 8881))
s.sendall(b"Hello")
Output
Waiting for sock_recv
Waiting for queue.get()
Produced: 49
Waiting for sock_recv
Consumed: 49
Waiting for queue.get()
Here's the problem, When I uncomment the following part
# client_connection = ssl_context.wrap_socket(
# client_connection, server_side=True
# )
It blocks on the sock_recv() function.
With the uncommented code, I get the following output:
Output
Waiting for sock_recv
Waiting for queue.get()
Client Code
import socket
import ssl
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock = ssl.wrap_socket(s)
sock.connect(("127.0.0.1", 8881))
sock.sendall(b"Hello")
Finally, when I shutdown the server with ctrl-c. I get the following output
^CTask exception was never retrieved
future: <Task finished coro=<producer() done, defined at asyncio_test.py:8> exception=SSLWantReadError(2, 'The operation did not complete (read) (_ssl.c:2488)')>
Traceback (most recent call last):
File "asyncio_test.py", line 11, in producer
await event_loop.sock_recv(client_connection, 4096)
File "/home/coverfox/.pyenv/versions/3.7.3/lib/python3.7/asyncio/selector_events.py", line 352, in sock_recv
return await fut
File "/home/coverfox/.pyenv/versions/3.7.3/lib/python3.7/asyncio/selector_events.py", line 366, in _sock_recv
data = sock.recv(n)
File "/home/coverfox/.pyenv/versions/3.7.3/lib/python3.7/ssl.py", line 1037, in recv
return self.read(buflen)
File "/home/coverfox/.pyenv/versions/3.7.3/lib/python3.7/ssl.py", line 913, in read
return self._sslobj.read(len)
ssl.SSLWantReadError: The operation did not complete (read) (_ssl.c:2488)
Traceback (most recent call last):
File "asyncio_test.py", line 42, in <module>
asyncio.run(main())
File "/home/coverfox/.pyenv/versions/3.7.3/lib/python3.7/asyncio/runners.py", line 43, in run
return loop.run_until_complete(main)
File "/home/coverfox/.pyenv/versions/3.7.3/lib/python3.7/asyncio/base_events.py", line 571, in run_until_complete
self.run_forever()
File "/home/coverfox/.pyenv/versions/3.7.3/lib/python3.7/asyncio/base_events.py", line 539, in run_forever
self._run_once()
File "/home/coverfox/.pyenv/versions/3.7.3/lib/python3.7/asyncio/base_events.py", line 1739, in _run_once
event_list = self._selector.select(timeout)
File "/home/coverfox/.pyenv/versions/3.7.3/lib/python3.7/selectors.py", line 468, in select
fd_event_list = self._selector.poll(timeout, max_ev)
KeyboardInterrupt
Edit:
I just found out that it works if I pass do_handshake_on_connect=False in the wrap_socket() function in the client code, but then ssl won't work.

So, it turns out that the async function sock_recv() does not support SSLSocket, since SSL needs to write to user-space buffer as described in this SO answer.
The way to work around this issue is to use the transports or streams in asyncio. Here, is the working version of the code in the above question.
import asyncio
import random
import socket
import ssl
SERVER_ADDRESS = (HOST, PORT) = "127.0.0.1", 8881
async def producer(reader, writer, queue):
while True:
print("Waiting for sock_recv")
await reader.read(16)
r = random.randint(1,101)
print("Produced: %d" % r)
await queue.put(r)
await asyncio.sleep(0)
async def consumer(queue):
while True:
print("Wating for queue.get()")
r = await queue.get()
await asyncio.sleep(2)
print("Consumed: %d" % r)
async def set_up_producer_consumer(reader, writer):
queue = asyncio.Queue()
t1 = asyncio.create_task(producer(reader, writer, queue))
t2 = asyncio.create_task(consumer(queue))
await asyncio.wait([t1, t2])
async def main():
ssl_context = ssl.create_default_context(purpose=ssl.Purpose.CLIENT_AUTH)
ssl_context.load_cert_chain(certfile="certificate.pem", keyfile="key.pem")
server = await asyncio.start_server(set_up_producer_consumer, HOST, PORT, family=socket.AF_INET, ssl=ssl_context, reuse_address=True)
await server.wait_closed()
event_loop = asyncio.get_event_loop()
asyncio.run(main())

Related

python-kafka asyncio is blocking

I am trying to create a websocket server that receives input from user, and at the same time read from kafka and continuously send data to user. I am having trouble getting both to run concurrently.
async def sender(websocket):
consumer = KafkaConsumer(...) # read from kafka
for message in consumer:
msg = json.loads(message.value.decode())
await websocket.send(msg)
async def handle_client(websocket):
asyncio.create_task(sender(websocket))
while True:
print("Waiting for client input")
try:
msg = await websocket.recv()
# do something with msg
async def main():
async with websockets.serve(handle_client, "localhost", 8765):
await asyncio.Future() # run forever
if __name__ == "__main__":
asyncio.run(main())
They seem to be blocking. I think the issuer is because sender() blocks it, it never sleeps?

Running an asyncoio socket server and client on the same process for tests

I wrote a socket server and client package with asyncio that happens to be a single module with two classes Server and Client. The intention is to have the code check if the specified port is in use on the same device, and if so, it will choose the use the Client class instead of the Server class. I am doing this to enable local testing between client and server for a larger goal.
I wrote my unit test with pytest and I can get it to pass if I run the server in its own process first, then run the client in another process.
However, I want to test both server and client together in the same process using pytest. The test never completes this way, however.
Am I running into an issue with asyncio here? Or is it an issue with pytest? Or is it something I am doing wrong in my code?
WebSockets.websocket
import asyncio
class Server:
def __init__(self, host, port):
self.host = host
self.port = port
self.server = None
self.connections = {}
async def start(self):
self.server = server = await asyncio.start_server(self.handle_client, self.host, self.port)
addr = server.sockets[0].getsockname()
print(f'Serving on {addr}')
async with server:
await server.serve_forever()
async def handle_client(self, reader, writer):
addr = writer.get_extra_info('peername')
print(f'New connection from {addr}')
self.connections[addr] = writer
try:
while not reader.at_eof():
data = await reader.read(100)
message = data.decode()
if message:
print(f'Received {message!r} from {addr}')
await self.send(message, addr)
finally:
del self.connections[addr]
writer.close()
async def send(self, message, addr):
writer = self.connections.get(addr)
if not writer:
return
writer.write(message.encode())
await writer.drain()
async def recv(self, addr):
reader, _ = await asyncio.open_connection(addr[0], addr[1])
data = await reader.read(100)
return data.decode()
async def stop(self):
await self.server.close()
class Client:
def __init__(self, host, port):
self.host = host
self.port = port
self.reader = None
self.writer = None
async def connect(self):
self.reader, self.writer = await asyncio.open_connection(self.host, self.port)
async def send(self, message):
self.writer.write(message.encode())
async def recv(self):
data = await self.reader.read(100)
return data.decode()
async def close(self):
self.writer.close()
Test
import socket
import asyncio
from WebSockets.websocket import Server, Client
import pytest
def check_port(address, port):
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((address, port))
s.close()
return "server"
except OSError:
return "client"
async def serve():
server = Server('localhost', 8910)
await server.start()
async def client_connect():
client = Client('localhost', 8910)
await client.connect()
await client.send("Hello world!")
message = await client.recv()
if message:
print(f"Received {message!r} from server")
assert message == "Hello world!"
await client.close()
#pytest.mark.asyncio
async def test_sockets():
if check_port("localhost", 8910) == "server":
await serve()
# for reference on how to use in non pytest module
# loop = asyncio.new_event_loop()
# loop.run_until_complete(serve())
# loop.close()
else:
await client_connect()
# for reference on how to use in non pytest module
# loop = asyncio.new_event_loop()
# loop.run_until_complete(client_connect())
# loop.close()

NotImplementedError asynchronous

I'm trying to use fastapi connect to Postgresql with async,but I got a NotimplementError,
It's seems the coderecord = await objects.get(test5, orderId=result['orderId'])
cause this problem.
but I don't know how to fixed it
there is some solution in network,but it did'n work
import platform
import asyncio
if platform.system() == "Windows":
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
*code
import peewee_async
import peewee_asyncext
from fastapi import FastAPI
from playhouse.postgres_ext import *
db = peewee_asyncext.PooledPostgresqlExtDatabase(
database = 'postgres',
host = '127.17.0.2',
port = '5432',
user = 'postgres',
password = 1234,
register_hstore = False,
max_connections = 20,
connect_timeout = 3
)
objects = peewee_async.Manager(database =db)
db.set_allow_sync = False
class test5(Model):
orderId = FixedCharField(primary_key = True)
transactionId = FixedCharField()
class Meta:
database = db
table_name = 'test'
app = FastAPI()
#app.post("/test")
async def test():
result = {
"orderId":"test",
"transactionId":"123"
}
try:
record = await objects.get(test5, orderId=result['orderId'])
except Exception as e:
if str(e) == "":
await objects.execute(test5.insert(result))
return result
*request
import requests,json
data={}
url='http://127.0.0.1:8000/test'
r=requests.post(url,json.dumps(data))
print(r.text)
*error
Future exception was never retrieved
future: <Future finished exception=NotImplementedError()>
Traceback (most recent call last):
File "D:\Python\lib\site-packages\peewee_async.py", line 852, in connect_async
await conn.connect()
File "D:\Python\lib\site-packages\peewee_async.py", line 1014, in connect
self.pool = await aiopg.create_pool(
File "D:\Python\lib\site-packages\aiopg\pool.py", line 300, in from_pool_fill
await self._fill_free_pool(False)
File "D:\Python\lib\site-packages\aiopg\pool.py", line 336, in _fill_free_pool
conn = await connect(
File "D:\Python\lib\site-packages\aiopg\connection.py", line 65, in connect
connection = Connection(
File "D:\Python\lib\site-packages\aiopg\connection.py", line 772, in __init__
self._loop.add_reader(
File "D:\Python\lib\asyncio\events.py", line 504, in add_reader
raise NotImplementedError
NotImplementedError
Future exception was never retrieved
future: <Future finished exception=NotImplementedError()>
Traceback (most recent call last):
File "C:\Users\user\Desktop\test\others\.\test5.py", line 39, in test
record = await objects.get(test5, orderId=result['orderId'])
File "D:\Python\lib\site-packages\peewee_async.py", line 166, in get
await self.connect()
File "D:\Python\lib\site-packages\peewee_async.py", line 302, in connect
await self.database.connect_async(loop=self.loop, timeout=self._timeout)
File "D:\Python\lib\site-packages\peewee_async.py", line 852, in connect_async
await conn.connect()
File "D:\Python\lib\site-packages\peewee_async.py", line 1014, in connect
self.pool = await aiopg.create_pool(
File "D:\Python\lib\site-packages\aiopg\pool.py", line 300, in from_pool_fill
await self._fill_free_pool(False)
File "D:\Python\lib\site-packages\aiopg\pool.py", line 336, in _fill_free_pool
conn = await connect(
File "D:\Python\lib\site-packages\aiopg\connection.py", line 65, in connect
connection = Connection(
File "D:\Python\lib\site-packages\aiopg\connection.py", line 772, in __init__
self._loop.add_reader(
File "D:\Python\lib\asyncio\events.py", line 504, in add_reader
raise NotImplementedError
NotImplementedError
*Windows version info:
Python 3.9.10 (tags/v3.9.10:f2f3f53, Jan 17 2022, 15:14:21) [MSC v.1929 64 bit (AMD64)] on
win32
Windows 10 Pro, version 20H2, OS build 19042.1526

Python code after sockets connection executed only once

What are the intentions of this program:
I want to send some commands from a client to a server using sockets, the server then send these command to an Arduino using serial. And another thing that I want the server to do in the future is that periodically sends other commands to the Arduino without getting any input from the client, so the sockets needs to be non-blocking or there needs to be another way to run the code separately from the sockets code.
The problem is that the part that is supposed to send the command to the Arduino only runs once.
What I have come up with after playing with the debugger in Pycharm, is that the problem is that the following line blocks after a connection has been established, and thus not allowing the rest of the code to be run.
conn, addr = s.accept()
Is this correct, or is there something else wrong?
I have tried to set the socket to non-blocking but when I do this I get an error.
"BlockingIOError: [WinError 10035] A non-blocking socket operation could not be completed immediately"
I have some basic knowledge of C/C++ and C# and am new to Python.
server.py
import socket
import serial
import sys
from _thread import *
import threading
import queue
# command that the client sends are "ON" and "OFF"
class serialConnect:
comPort =' '
baudrate = 115200
myserial = serial.Serial('COM5', baudrate)
def serialstart(self):
# self.comPort = input('Comport: ')
try:
self.myserial.open()
except IOError:
print('Port is already open!')
def serialRead(self):
data = self.myserial.read(16)
data.decode('UTF-8')
return data
def serialWrite(self, data):
data += '\n' #the arduino needs a \n after each command.
databytes = data.encode('UTF-8')
self.myserial.write(databytes)
print('send data: ', databytes)
def threaded_client(conn, dataqueue):
data = {bytes}
conn.send(str.encode('welcome, type your info \n'))
while True:
data = conn.recv(2048)
if not data:
break
reply = 'server output: ' + data.decode('UTF-8')
dataqueue.put(data.decode('UTF-8'))
print("Items in queue: ",dataqueue.qsize())
#conn.sendall(str.encode(reply))
print("Recieved data in threaded_client: ", data.decode('UTF-8') + '\n')
conn.close()
def Main():
ser = serialConnect()
host = ''
port = 5555
dataRecieved = 'hello'
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(2)
s.setblocking(1) #when set to non-blocking error occurs : "BlockingIOError: [WinError 10035] A non-blocking socket operation could not be completed immediately"
workQueue = queue.Queue(10)
try:
s.bind((host,port))
except socket.error as e:
print(str(e))
s.listen(5)
print('waiting for a connection')
while True:
try:
conn, addr = s.accept() #once connection is established it blocks?
print('connected to: ' + addr[0] + ':' + str())
t = threading.Thread(target=threaded_client, args=(conn, workQueue))
t.daemon = True
t.start()
except:
e = sys.exc_info()
print('Error:', e)
# This section of code is only run once, doesn't matter if put inside try block or not. :(
dataRecieved = workQueue.get()
print('The recieved data: ', dataRecieved)
ser.serialstart()
ser.serialWrite(dataRecieved)
if __name__ == '__main__':
Main()
client.py
import socket
def Main():
host = '127.0.0.1'
port = 5555
message = "<,R,G,B,>"
mySocket = socket.socket()
mySocket.connect((host, port))
while message != 'q':
message = input(" -> ")
mySocket.send(message.encode())
mySocket.close()
if __name__ == '__main__':
Main()
Arduino Code
String inputString = ""; // a string to hold incoming data
boolean stringComplete = false; // whether the string is complete
int LEDpin = 10;
// the setup function runs once when you press reset or power the board
void setup() {
// initialize digital pin 13 as an output.
pinMode(10, OUTPUT);
Serial.begin(19200);
}
// the loop function runs over and over again forever
void loop() {
serialEvent();
if(stringComplete){
Serial.println(inputString);
if(inputString == "ON\n"){
digitalWrite(LEDpin, HIGH); // turn the LED on (HIGH is the voltage level)
}
if(inputString == "OFF\n"){
digitalWrite(LEDpin, LOW); // turn the LED off by making the voltage LOW
}
inputString = "";
stringComplete = false;
}
}
void serialEvent()
{
while (Serial.available()) {
// get the new byte:
char inChar = (char)Serial.read();
// add it to the inputString:
inputString += inChar;
// if the incoming character is a newline, set a flag
// so the main loop can do something about it:
if (inChar == '\n') {
stringComplete = true;
}
}
}
Refactored server code for anyone that is interested in it.
I am not sure if this is up to standard, but it is working.
import serial
import socket
import queue
import sys
import threading
class serialConnect:
comPort = 'COM5'
baudrate = 115200
myserial = serial.Serial(comPort, baudrate)
def serial_run(self):
# self.comPort = input('Comport: ')
try:
if not self.myserial.isOpen():
self.myserial.open()
else:
print('Port is already open!')
except IOError as e:
print('Error: ', e)
def serial_read(self):
data = self.myserial.read(16)
data.decode('UTF-8')
return data
def serial_write(self, data):
data += '\n' #the arduino needs a \n after each command.
databytes = data.encode('UTF-8')
self.myserial.write(databytes)
print('send data: ', databytes)
class socketServer:
host = ''
port = 5555
soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
soc.setblocking(1)
data_queue = queue.Queue(1)
def __init__(self):
try:
self.soc.bind((self.host, self.port))
except:
print('Bind error: ', sys.exc_info())
self.soc.listen(5)
def socket_accept_thread(self):
while True:
try:
print('Waiting for a new connection')
conn, addr = self.soc.accept()
client_thread = threading.Thread(target=self.threaded_client, args=(conn, self.data_queue))
client_thread.daemon = True
client_thread.start()
except:
print('Accept thread Error: ', sys.exc_info())
def threaded_client(self, conn, data_queue):
# conn.send(str.encode('welcome, type your info \n'))
try:
while True:
data = conn.recv(2048)
if not data:
break
# reply = 'server output: ' + data.decode('UTF-8')
data_queue.put(data.decode('UTF-8'))
print("Items in queue: ", data_queue.qsize())
# conn.sendall(str.encode(reply))
print("Received data in threaded_client: ", data.decode('UTF-8'))
except:
print("Error: ", sys.exc_info())
conn.close()
def get_data(self):
data = self.data_queue.get()
return data
def Main():
server = socketServer()
arduino_conn = serialConnect()
accept_thread = threading.Thread(target=server.socket_accept_thread)
data_received = 'Nothing received'
while True:
if not accept_thread.is_alive():
accept_thread.daemon = True
accept_thread.start()
arduino_conn.serial_run()
data_received = server.get_data()
arduino_conn.serial_write(data_received)
if __name__ == '__main__':
Main()

Ignore/reconnect for closed connection in pool

I have some problem with pool.acquire() and closed connection in pool.
Setting on PG Server set connection timeout to 120 second. When i use pool.acquire() it is raise error because connection is closed:
Sleep 150
Traceback (most recent call last):
File "test.py", line 21, in <module>
loop.run_until_complete(test_select())
File "/usr/local/Cellar/python3/3.5.1/Frameworks/Python.framework/Versions/3.5/lib/python3.5/asyncio/base_events.py", line 337, in run_until_complete
return future.result()
File "/usr/local/Cellar/python3/3.5.1/Frameworks/Python.framework/Versions/3.5/lib/python3.5/asyncio/futures.py", line 274, in result
raise self._exception
File "/usr/local/Cellar/python3/3.5.1/Frameworks/Python.framework/Versions/3.5/lib/python3.5/asyncio/tasks.py", line 239, in _step
result = coro.send(None)
File "test.py", line 10, in test_select
async with pool.acquire() as conn:
File "/usr/local/lib/python3.5/site-packages/aiopg/utils.py", line 116, in __aenter__
self._conn = yield from self._coro
File "/usr/local/lib/python3.5/site-packages/aiopg/pool.py", line 170, in _acquire
assert not conn.closed, conn
AssertionError: <aiopg.connection.Connection object at 0x106f7df98>
and code
import asyncio
import aiopg
dsn = 'dbname=dbname user=user password=password host=127.0.0.1'
async def test_select():
async with aiopg.create_pool(dsn) as pool:
print("Sleep 150")
await asyncio.sleep(150)
async with pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute("SELECT 1")
ret = []
async for row in cur:
ret.append(row)
assert ret == [(1,)]
print("ALL DONE")
loop = asyncio.get_event_loop()
loop.run_until_complete(test_select())
How to solve this problem.
Thanks!