Ignore/reconnect for closed connection in pool - postgresql

I have some problem with pool.acquire() and closed connection in pool.
Setting on PG Server set connection timeout to 120 second. When i use pool.acquire() it is raise error because connection is closed:
Sleep 150
Traceback (most recent call last):
File "test.py", line 21, in <module>
loop.run_until_complete(test_select())
File "/usr/local/Cellar/python3/3.5.1/Frameworks/Python.framework/Versions/3.5/lib/python3.5/asyncio/base_events.py", line 337, in run_until_complete
return future.result()
File "/usr/local/Cellar/python3/3.5.1/Frameworks/Python.framework/Versions/3.5/lib/python3.5/asyncio/futures.py", line 274, in result
raise self._exception
File "/usr/local/Cellar/python3/3.5.1/Frameworks/Python.framework/Versions/3.5/lib/python3.5/asyncio/tasks.py", line 239, in _step
result = coro.send(None)
File "test.py", line 10, in test_select
async with pool.acquire() as conn:
File "/usr/local/lib/python3.5/site-packages/aiopg/utils.py", line 116, in __aenter__
self._conn = yield from self._coro
File "/usr/local/lib/python3.5/site-packages/aiopg/pool.py", line 170, in _acquire
assert not conn.closed, conn
AssertionError: <aiopg.connection.Connection object at 0x106f7df98>
and code
import asyncio
import aiopg
dsn = 'dbname=dbname user=user password=password host=127.0.0.1'
async def test_select():
async with aiopg.create_pool(dsn) as pool:
print("Sleep 150")
await asyncio.sleep(150)
async with pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute("SELECT 1")
ret = []
async for row in cur:
ret.append(row)
assert ret == [(1,)]
print("ALL DONE")
loop = asyncio.get_event_loop()
loop.run_until_complete(test_select())
How to solve this problem.
Thanks!

Related

Prisma python client - Postgres can't establish connection

i'm unsuccessfully trying to connect my python project to a postgres DB using Prisma
I'm following this https://prisma-client-py.readthedocs.io/en/stable/
# schema.prisma
generator db {
provider = "prisma-client-py"
interface = "asyncio"
}
datasource db {
provider = "postgresql"
url = "postgresql://imb123:pass3876#localhost:5432/ibm123?schema=public"
}
model Post {
id String #id #default(cuid())
created_at DateTime #default(now())
updated_at DateTime #updatedAt
title String
published Boolean
desc String?
}
main.py
import asyncio
from prisma import Prisma
async def main() -> None:
db = Prisma()
await db.connect()
post = await db.post.create(
{
'title': 'Hello from prisma!',
'desc': 'Prisma is a database toolkit and makes databases easy.',
'published': True,
}
)
print(f'created post: {post.json(indent=2, sort_keys=True)}')
found = await db.post.find_unique(where={'id': post.id})
assert found is not None
print(f'found post: {found.json(indent=2, sort_keys=True)}')
await db.disconnect()
if __name__ == '__main__':
asyncio.run(main())
db push and prisma generate finished successfully meaning the connect from CLI is ok.
table_catalog | table_schema | table_name | table_type | self_referencing_column_name | reference_generation | user_defined_type_catalog | user_defined_type_schema | user_defined_type_name | is_insertable_into | is_typed | commit_action
---------------+--------------------+---------------------------------------+------------+------------------------------+----------------------+---------------------------+--------------------------+------------------------+--------------------+----------+---------------
ibm123 | public | Post | BASE TABLE | | | | | | YES | NO |
The error raised from await db.connect()
/Users/ibm123/PycharmProjects/pythonProject/venv/bin/python /Users/ibm123/PycharmProjects/pythonProject/main.py
Traceback (most recent call last):
File "/Users/ibm123/PycharmProjects/pythonProject/venv/lib/python3.9/site-packages/anyio/_core/_sockets.py", line 164, in try_connect
stream = await asynclib.connect_tcp(remote_host, remote_port, local_address)
File "/Users/ibm123/PycharmProjects/pythonProject/venv/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 1691, in connect_tcp
await get_running_loop().create_connection(
File "/usr/local/Cellar/python#3.9/3.9.13_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/base_events.py", line 1065, in create_connection
raise exceptions[0]
File "/usr/local/Cellar/python#3.9/3.9.13_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/base_events.py", line 1050, in create_connection
sock = await self._connect_sock(
File "/usr/local/Cellar/python#3.9/3.9.13_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/base_events.py", line 961, in _connect_sock
await self.sock_connect(sock, address)
File "/usr/local/Cellar/python#3.9/3.9.13_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/selector_events.py", line 500, in sock_connect
return await fut
File "/usr/local/Cellar/python#3.9/3.9.13_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/selector_events.py", line 535, in _sock_connect_cb
raise OSError(err, f'Connect call failed {address}')
ConnectionRefusedError: [Errno 61] Connect call failed ('::1', 59712, 0, 0)
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/ibm123/PycharmProjects/pythonProject/venv/lib/python3.9/site-packages/httpcore/_exceptions.py", line 8, in map_exceptions
yield
File "/Users/ibm123/PycharmProjects/pythonProject/venv/lib/python3.9/site-packages/httpcore/backends/asyncio.py", line 109, in connect_tcp
stream: anyio.abc.ByteStream = await anyio.connect_tcp(
File "/Users/ibm123/PycharmProjects/pythonProject/venv/lib/python3.9/site-packages/anyio/_core/_sockets.py", line 222, in connect_tcp
raise OSError("All connection attempts failed") from cause
OSError: All connection attempts failed
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/ibm123/PycharmProjects/pythonProject/venv/lib/python3.9/site-packages/httpx/_transports/default.py", line 60, in map_httpcore_exceptions
yield
File "/Users/ibm123/PycharmProjects/pythonProject/venv/lib/python3.9/site-packages/httpx/_transports/default.py", line 353, in handle_async_request
resp = await self._pool.handle_async_request(req)
File "/Users/ibm123/PycharmProjects/pythonProject/venv/lib/python3.9/site-packages/httpcore/_async/connection_pool.py", line 253, in handle_async_request
raise exc
File "/Users/ibm123/PycharmProjects/pythonProject/venv/lib/python3.9/site-packages/httpcore/_async/connection_pool.py", line 237, in handle_async_request
response = await connection.handle_async_request(request)
File "/Users/ibm123/PycharmProjects/pythonProject/venv/lib/python3.9/site-packages/httpcore/_async/connection.py", line 86, in handle_async_request
raise exc
File "/Users/ibm123/PycharmProjects/pythonProject/venv/lib/python3.9/site-packages/httpcore/_async/connection.py", line 63, in handle_async_request
stream = await self._connect(request)
File "/Users/ibm123/PycharmProjects/pythonProject/venv/lib/python3.9/site-packages/httpcore/_async/connection.py", line 111, in _connect
stream = await self._network_backend.connect_tcp(**kwargs)
File "/Users/ibm123/PycharmProjects/pythonProject/venv/lib/python3.9/site-packages/httpcore/backends/auto.py", line 29, in connect_tcp
return await self._backend.connect_tcp(
File "/Users/ibm123/PycharmProjects/pythonProject/venv/lib/python3.9/site-packages/httpcore/backends/asyncio.py", line 109, in connect_tcp
stream: anyio.abc.ByteStream = await anyio.connect_tcp(
File "/usr/local/Cellar/python#3.9/3.9.13_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/contextlib.py", line 137, in __exit__
self.gen.throw(typ, value, traceback)
File "/Users/ibm123/PycharmProjects/pythonProject/venv/lib/python3.9/site-packages/httpcore/_exceptions.py", line 12, in map_exceptions
raise to_exc(exc)
httpcore.ConnectError: All connection attempts failed
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/ibm123/PycharmProjects/pythonProject/venv/lib/python3.9/site-packages/prisma/engine/query.py", line 161, in spawn
data = await self.request('GET', '/status')
File "/Users/ibm123/PycharmProjects/pythonProject/venv/lib/python3.9/site-packages/prisma/engine/http.py", line 96, in request
resp = await self.session.request(method, url, **kwargs)
File "/Users/ibm123/PycharmProjects/pythonProject/venv/lib/python3.9/site-packages/prisma/_async_http.py", line 28, in request
return Response(await self.session.request(method, url, **kwargs))
File "/Users/ibm123/PycharmProjects/pythonProject/venv/lib/python3.9/site-packages/httpx/_client.py", line 1527, in request
return await self.send(request, auth=auth, follow_redirects=follow_redirects)
File "/Users/ibm123/PycharmProjects/pythonProject/venv/lib/python3.9/site-packages/httpx/_client.py", line 1614, in send
response = await self._send_handling_auth(
File "/Users/ibm123/PycharmProjects/pythonProject/venv/lib/python3.9/site-packages/httpx/_client.py", line 1642, in _send_handling_auth
response = await self._send_handling_redirects(
File "/Users/ibm123/PycharmProjects/pythonProject/venv/lib/python3.9/site-packages/httpx/_client.py", line 1679, in _send_handling_redirects
response = await self._send_single_request(request)
File "/Users/ibm123/PycharmProjects/pythonProject/venv/lib/python3.9/site-packages/httpx/_client.py", line 1716, in _send_single_request
response = await transport.handle_async_request(request)
File "/Users/ibm123/PycharmProjects/pythonProject/venv/lib/python3.9/site-packages/httpx/_transports/default.py", line 353, in handle_async_request
resp = await self._pool.handle_async_request(req)
File "/usr/local/Cellar/python#3.9/3.9.13_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/contextlib.py", line 137, in __exit__
self.gen.throw(typ, value, traceback)
File "/Users/ibm123/PycharmProjects/pythonProject/venv/lib/python3.9/site-packages/httpx/_transports/default.py", line 77, in map_httpcore_exceptions
raise mapped_exc(message) from exc
httpx.ConnectError: All connection attempts failed
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/ibm123/PycharmProjects/pythonProject/main.py", line 25, in <module>
asyncio.run(main())
File "/usr/local/Cellar/python#3.9/3.9.13_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/local/Cellar/python#3.9/3.9.13_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/base_events.py", line 647, in run_until_complete
return future.result()
File "/Users/ibm123/PycharmProjects/pythonProject/main.py", line 6, in main
await db.connect()
File "/Users/ibm123/PycharmProjects/pythonProject/venv/lib/python3.9/site-packages/prisma/client.py", line 238, in connect
await self.__engine.connect(
File "/Users/ibm123/PycharmProjects/pythonProject/venv/lib/python3.9/site-packages/prisma/engine/query.py", line 109, in connect
await self.spawn(file, timeout=timeout, datasources=datasources)
File "/Users/ibm123/PycharmProjects/pythonProject/venv/lib/python3.9/site-packages/prisma/engine/query.py", line 180, in spawn
raise errors.EngineConnectionError(
prisma.engine.errors.EngineConnectionError: Could not connect to the query engine

NotImplementedError asynchronous

I'm trying to use fastapi connect to Postgresql with async,but I got a NotimplementError,
It's seems the coderecord = await objects.get(test5, orderId=result['orderId'])
cause this problem.
but I don't know how to fixed it
there is some solution in network,but it did'n work
import platform
import asyncio
if platform.system() == "Windows":
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
*code
import peewee_async
import peewee_asyncext
from fastapi import FastAPI
from playhouse.postgres_ext import *
db = peewee_asyncext.PooledPostgresqlExtDatabase(
database = 'postgres',
host = '127.17.0.2',
port = '5432',
user = 'postgres',
password = 1234,
register_hstore = False,
max_connections = 20,
connect_timeout = 3
)
objects = peewee_async.Manager(database =db)
db.set_allow_sync = False
class test5(Model):
orderId = FixedCharField(primary_key = True)
transactionId = FixedCharField()
class Meta:
database = db
table_name = 'test'
app = FastAPI()
#app.post("/test")
async def test():
result = {
"orderId":"test",
"transactionId":"123"
}
try:
record = await objects.get(test5, orderId=result['orderId'])
except Exception as e:
if str(e) == "":
await objects.execute(test5.insert(result))
return result
*request
import requests,json
data={}
url='http://127.0.0.1:8000/test'
r=requests.post(url,json.dumps(data))
print(r.text)
*error
Future exception was never retrieved
future: <Future finished exception=NotImplementedError()>
Traceback (most recent call last):
File "D:\Python\lib\site-packages\peewee_async.py", line 852, in connect_async
await conn.connect()
File "D:\Python\lib\site-packages\peewee_async.py", line 1014, in connect
self.pool = await aiopg.create_pool(
File "D:\Python\lib\site-packages\aiopg\pool.py", line 300, in from_pool_fill
await self._fill_free_pool(False)
File "D:\Python\lib\site-packages\aiopg\pool.py", line 336, in _fill_free_pool
conn = await connect(
File "D:\Python\lib\site-packages\aiopg\connection.py", line 65, in connect
connection = Connection(
File "D:\Python\lib\site-packages\aiopg\connection.py", line 772, in __init__
self._loop.add_reader(
File "D:\Python\lib\asyncio\events.py", line 504, in add_reader
raise NotImplementedError
NotImplementedError
Future exception was never retrieved
future: <Future finished exception=NotImplementedError()>
Traceback (most recent call last):
File "C:\Users\user\Desktop\test\others\.\test5.py", line 39, in test
record = await objects.get(test5, orderId=result['orderId'])
File "D:\Python\lib\site-packages\peewee_async.py", line 166, in get
await self.connect()
File "D:\Python\lib\site-packages\peewee_async.py", line 302, in connect
await self.database.connect_async(loop=self.loop, timeout=self._timeout)
File "D:\Python\lib\site-packages\peewee_async.py", line 852, in connect_async
await conn.connect()
File "D:\Python\lib\site-packages\peewee_async.py", line 1014, in connect
self.pool = await aiopg.create_pool(
File "D:\Python\lib\site-packages\aiopg\pool.py", line 300, in from_pool_fill
await self._fill_free_pool(False)
File "D:\Python\lib\site-packages\aiopg\pool.py", line 336, in _fill_free_pool
conn = await connect(
File "D:\Python\lib\site-packages\aiopg\connection.py", line 65, in connect
connection = Connection(
File "D:\Python\lib\site-packages\aiopg\connection.py", line 772, in __init__
self._loop.add_reader(
File "D:\Python\lib\asyncio\events.py", line 504, in add_reader
raise NotImplementedError
NotImplementedError
*Windows version info:
Python 3.9.10 (tags/v3.9.10:f2f3f53, Jan 17 2022, 15:14:21) [MSC v.1929 64 bit (AMD64)] on
win32
Windows 10 Pro, version 20H2, OS build 19042.1526

Use signal handle to shut down flask-socketio server

I am using macOS Mojave and Python 3.7. In my work, I want to shut down flask-socketio server completely from the signal handler, which is defined as SignalHandler in my code. I noticed that, without the signal handler, the server can be shut down completely with Ctrl-C. However, the signal handler is necessary for my work. I searched online and couldn't find the solution to shut the server down in my case. For your information, I found solutions to shut the server down by "call flask-socketio stop from HTTP or SocketIO handler function", or "shut down a gevent (pywsgi) server gracefully".
The example code is given as below:
import os
import threading
import signal
import requests
from flask import Flask, send_from_directory
from flask_socketio import SocketIO, Namespace
import eventlet
class WebsiteCreator(threading.Thread):
def __init__(self):
super().__init__()
def run(self):
app = Flask(__name__, template_folder="templates",
static_folder="templates/static")
app.config['SECRET_KEY'] = 'Secret!'
socketio = SocketIO(app, engineio_logger=True, logger=True)
# Create a URL route in our application for "/"
#app.route('/')
def test_page():
"""
This function loads the homepage
"""
return send_from_directory(
os.path.join(app.root_path, 'templates'),
"index1.html"
)
#app.route('/stop', methods=['POST'])
def shutdown_server():
"""
This function stops the flask-socketio server
"""
print("Received request to shut down the server.")
socketio.stop() #something wrong here, but don't know how to solve
return "The server has been shut down."
class MyCustomNamespace(Namespace):
def on_connect(self):
print("Client just connected")
def on_disconnect(self):
print("Client just left")
def on_messages(self, data):
print(f"\nReceived data from client: \n {data}\n")
return data
socketio.on_namespace(MyCustomNamespace('/channel_A'))
try:
eventlet.wsgi.server(
eventlet.wrap_ssl(eventlet.listen(("localhost", 8080)),
certfile='server.crt',
keyfile='server.key',
server_side=True), app)
except Exception as e:
print(f"Website is not established due to:\n{e}")
# Terminate code from shell
class SignalHandler(object):
def __init__(self):
pass
def __call__(self, signum, frame):
print("Shutting down the website.")
# Begin 'something' here to shut down the server...
shutdown_server = requests.post("https://localhost:8080/stop", data=None)
print(f"Shut down the server feedback: {shutdown_server}")
# 'Something' ends here
print("The website has been shut down.")
if __name__ == '__main__':
WebsiteCreator().start()
# If the following part is not included, the server can be shut down using Ctrl-C
handler = SignalHandler()
signal.signal(signal.SIGINT, handler)
In the code, I am running the flask-socketio server in a thread. I want to shut the server down by doing some actions in the SignalHandler.
Yet, when I exited the system with Ctrl-C, some exceptions raised:
^C
Shutting down the website.
(23066) accepted ('127.0.0.1', 49720)
Received request to shut down the server.
127.0.0.1 - - [22/Nov/2019 13:08:18] "POST /stop HTTP/1.1" 200 0 0.000365
wsgi exiting
Exception ignored in: <module 'threading' from '/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/threading.py'>
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/threading.py", line 1308, in _shutdown
lock.acquire()
File "web_app.py", line 74, in __call__
shutdown_server = requests.post("https://localhost:8080/stop", data=None)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/requests/api.py", line 116, in post
return request('post', url, data=data, json=json, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/requests/api.py", line 60, in request
return session.request(method=method, url=url, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/requests/sessions.py", line 533, in request
resp = self.send(prep, **send_kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/requests/sessions.py", line 646, in send
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/eventlet/hubs/kqueue.py", line 105, in wait
readers.get(fileno, hub.noop).cb(fileno)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/eventlet/greenthread.py", line 221, in main
result = function(*args, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/eventlet/wsgi.py", line 818, in process_request
proto.__init__(conn_state, self)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/eventlet/wsgi.py", line 357, in __init__
self.handle()
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/eventlet/wsgi.py", line 390, in handle
self.handle_one_request()
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/eventlet/wsgi.py", line 419, in handle_one_request
self.raw_requestline = self._read_request_line()
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/eventlet/wsgi.py", line 402, in _read_request_line
return self.rfile.readline(self.server.url_length_limit)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/socket.py", line 589, in readinto
return self._sock.recv_into(b)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/eventlet/green/ssl.py", line 241, in recv_into
return self._base_recv(nbytes, flags, into=True, buffer_=buffer)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/eventlet/green/ssl.py", line 256, in _base_recv
read = self.read(nbytes, buffer_)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/eventlet/green/ssl.py", line 176, in read
super(GreenSSLSocket, self).read, *args, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/eventlet/green/ssl.py", line 150, in _call_trampolining
return func(*a, **kw)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/ssl.py", line 926, in read
raise ValueError("Read on closed or unwrapped SSL socket.")
r = adapter.send(request, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/requests/adapters.py", line 498, in send
ValueError: Read on closed or unwrapped SSL socket.
Removing descriptor: 8
142f38bdaaf34c7e8883e99a766fe310: Unexpected error "Read on closed or unwrapped SSL socket.", closing connection
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
Please give me some ideas! Thanks in advance.
This is what I use in my projects, works great using gevent. Make sure you start your server AFTER you init your signal. The signal should be identical, just minus the gevent part.
def shutdown():
print('Shutting down ...')
server.stop(timeout=60)
exit(signal.SIGTERM)
gevent.signal(signal.SIGTERM, shutdown)
gevent.signal(signal.SIGINT, shutdown) #CTRL C
server.serve_forever()

SSLWantReadError when using async sock_recv with an SSL connection

I'm implementing a producer-consumer program in a server implementation using sockets and asyncio. The problem is the async function sock_recv() does not seem to be working properly when used with a socket wrapped in an ssl connection. Following is the working code.
Server side
import asyncio
import random
import socket
import ssl
SERVER_ADDRESS = (HOST, PORT) = "127.0.0.1", 8881
async def producer(queue, client_connection, event_loop):
while True:
print("Waiting for sock_recv")
await event_loop.sock_recv(client_connection, 4096)
r = random.randint(1,101)
print("Produced: %d" % r)
await queue.put(r)
await asyncio.sleep(0)
async def consumer(queue):
while True:
print("Wating for queue.get()")
r = await queue.get()
await asyncio.sleep(2)
print("Consumed: %d" % r)
async def main():
listen_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
listen_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
listen_socket.bind(SERVER_ADDRESS)
listen_socket.listen(5)
ssl_context = ssl.create_default_context(purpose=ssl.Purpose.CLIENT_AUTH)
ssl_context.load_cert_chain(certfile="certificate.pem", keyfile="key.pem")
client_connection, client_address = listen_socket.accept()
# client_connection = ssl_context.wrap_socket(
# client_connection, server_side=True
# )
client_connection.setblocking(False)
queue = asyncio.Queue()
t1 = asyncio.create_task(producer(queue, client_connection, asyncio.get_event_loop()))
t2 = asyncio.create_task(consumer(queue))
await asyncio.wait([t1, t2])
event_loop = asyncio.get_event_loop()
asyncio.run(main())
Client side
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(("127.0.0.1", 8881))
s.sendall(b"Hello")
Output
Waiting for sock_recv
Waiting for queue.get()
Produced: 49
Waiting for sock_recv
Consumed: 49
Waiting for queue.get()
Here's the problem, When I uncomment the following part
# client_connection = ssl_context.wrap_socket(
# client_connection, server_side=True
# )
It blocks on the sock_recv() function.
With the uncommented code, I get the following output:
Output
Waiting for sock_recv
Waiting for queue.get()
Client Code
import socket
import ssl
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock = ssl.wrap_socket(s)
sock.connect(("127.0.0.1", 8881))
sock.sendall(b"Hello")
Finally, when I shutdown the server with ctrl-c. I get the following output
^CTask exception was never retrieved
future: <Task finished coro=<producer() done, defined at asyncio_test.py:8> exception=SSLWantReadError(2, 'The operation did not complete (read) (_ssl.c:2488)')>
Traceback (most recent call last):
File "asyncio_test.py", line 11, in producer
await event_loop.sock_recv(client_connection, 4096)
File "/home/coverfox/.pyenv/versions/3.7.3/lib/python3.7/asyncio/selector_events.py", line 352, in sock_recv
return await fut
File "/home/coverfox/.pyenv/versions/3.7.3/lib/python3.7/asyncio/selector_events.py", line 366, in _sock_recv
data = sock.recv(n)
File "/home/coverfox/.pyenv/versions/3.7.3/lib/python3.7/ssl.py", line 1037, in recv
return self.read(buflen)
File "/home/coverfox/.pyenv/versions/3.7.3/lib/python3.7/ssl.py", line 913, in read
return self._sslobj.read(len)
ssl.SSLWantReadError: The operation did not complete (read) (_ssl.c:2488)
Traceback (most recent call last):
File "asyncio_test.py", line 42, in <module>
asyncio.run(main())
File "/home/coverfox/.pyenv/versions/3.7.3/lib/python3.7/asyncio/runners.py", line 43, in run
return loop.run_until_complete(main)
File "/home/coverfox/.pyenv/versions/3.7.3/lib/python3.7/asyncio/base_events.py", line 571, in run_until_complete
self.run_forever()
File "/home/coverfox/.pyenv/versions/3.7.3/lib/python3.7/asyncio/base_events.py", line 539, in run_forever
self._run_once()
File "/home/coverfox/.pyenv/versions/3.7.3/lib/python3.7/asyncio/base_events.py", line 1739, in _run_once
event_list = self._selector.select(timeout)
File "/home/coverfox/.pyenv/versions/3.7.3/lib/python3.7/selectors.py", line 468, in select
fd_event_list = self._selector.poll(timeout, max_ev)
KeyboardInterrupt
Edit:
I just found out that it works if I pass do_handshake_on_connect=False in the wrap_socket() function in the client code, but then ssl won't work.
So, it turns out that the async function sock_recv() does not support SSLSocket, since SSL needs to write to user-space buffer as described in this SO answer.
The way to work around this issue is to use the transports or streams in asyncio. Here, is the working version of the code in the above question.
import asyncio
import random
import socket
import ssl
SERVER_ADDRESS = (HOST, PORT) = "127.0.0.1", 8881
async def producer(reader, writer, queue):
while True:
print("Waiting for sock_recv")
await reader.read(16)
r = random.randint(1,101)
print("Produced: %d" % r)
await queue.put(r)
await asyncio.sleep(0)
async def consumer(queue):
while True:
print("Wating for queue.get()")
r = await queue.get()
await asyncio.sleep(2)
print("Consumed: %d" % r)
async def set_up_producer_consumer(reader, writer):
queue = asyncio.Queue()
t1 = asyncio.create_task(producer(reader, writer, queue))
t2 = asyncio.create_task(consumer(queue))
await asyncio.wait([t1, t2])
async def main():
ssl_context = ssl.create_default_context(purpose=ssl.Purpose.CLIENT_AUTH)
ssl_context.load_cert_chain(certfile="certificate.pem", keyfile="key.pem")
server = await asyncio.start_server(set_up_producer_consumer, HOST, PORT, family=socket.AF_INET, ssl=ssl_context, reuse_address=True)
await server.wait_closed()
event_loop = asyncio.get_event_loop()
asyncio.run(main())

Watch long term process in Ipython notebook

For now, after reconnect ipython notebook is not able to print any output from running kernel. I tried following work-around:
import sys, time
with open('foo.log','w') as sys.stdout:
for i in range(5):
print i
time.sleep(1)
everything is nice, I can see the process with tail -f foo.log, but at the end whole thing crashes and I get following error:
ERROR:tornado.general:Uncaught exception, closing connection.
Traceback (most recent call last):
File "/usr/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py", line 407, in _run_callback
callback(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py", line 260, in dispatcher
return self.dispatch_shell(stream, msg)
File "/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py", line 218, in dispatch_shell
sys.stdout.flush()
ValueError: I/O operation on closed file
ERROR:tornado.general:Uncaught exception, closing connection.
Traceback (most recent call last):
File "/usr/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py", line 433, in _handle_events
self._handle_recv()
File "/usr/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py", line 465, in _handle_recv
self._run_callback(callback, msg)
File "/usr/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py", line 407, in _run_callback
callback(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py", line 260, in dispatcher
return self.dispatch_shell(stream, msg)
File "/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py", line 218, in dispatch_shell
sys.stdout.flush()
ValueError: I/O operation on closed file
ERROR:tornado.application:Exception in callback None
Traceback (most recent call last):
File "/usr/local/lib/python2.7/dist-packages/tornado/ioloop.py", line 883, in start
handler_func(fd_obj, events)
File "/usr/local/lib/python2.7/dist-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/usr/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py", line 433, in _handle_events
self._handle_recv()
File "/usr/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py", line 465, in _handle_recv
self._run_callback(callback, msg)
File "/usr/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py", line 407, in _run_callback
callback(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py", line 260, in dispatcher
return self.dispatch_shell(stream, msg)
File "/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py", line 218, in dispatch_shell
sys.stdout.flush()
ValueError: I/O operation on closed file
I have finally managed to log messages from jupyter into file and flush memory immediately:
class InstantLogger():
def __init__(self, f):
self.f = open(f,'w')
def __getattr__(self,name):
return object.__getattribute__(self.f, name)
def write(self, x):
self.f.write(x)
self.f.flush()
def flush(self):
self.f.flush()
def close(self):
self.f.close()
than I can use it as:
old_stdout = sys.stdout
sys.stdout = InstantLogger('file.log')
print ....
sys.stdout.close()
sys.stdout = old_stdout