Update on sqlalchemy +asyncpg , returns another operation in progress on pytest - postgresql

I just updated my app with sqlalchemy to create an async connections, no problem, but when writing tests, I can't do the update as opposed to create:
affected function:
#function
#classmethod
async def update_instance(
cls,
instance: InstanceInputOnUpdate
) -> Union['EdaNCEInstance', EdaNCEInstanceOperationError]:
async with get_session() as conn:
result = await conn.execute(
select(Instance).where(Instance.id==eda_nce_instance.id)
)
instance_to_update = result.scalars().unique().first()
if instance_to_update is not None:
instance_to_update.name = instance.name
instance_to_update.host = instance.host
await conn.commit()
return instance_to_update
else:
return InstanceOperationError(
result= False,
message= f"Can't find the instance ID '{instance.id}'"
)
test:
#pytest.mark.asyncio
async def test_04_update_instance():
async with AsyncClient(app=app, base_url="http://test") as c:
res = await Instance.update_eda_nce_instance(
instance=InstanceInputOnUpdate(
id=DUMMY_ID,
name="test_server1",
host="123.123.123.123",
)
)
assert isinstance(res, Instance) == True
assert res.host == "124.123.144.144"
return res
errors:
#...
E sqlalchemy.dialects.postgresql.asyncpg.AsyncAdapt_asyncpg_dbapi.InterfaceError: <class 'asyncpg.exceptions._base.InterfaceError'>: cannot perform operation: another operation is in progress
venv/lib/python3.9/site-packages/sqlalchemy/dialects/postgresql/asyncpg.py:682: InterfaceError
my db.py
SQLALCHEMY_DATABASE_URL = settings.get_settings().postgresql_conn_url
engine = create_async_engine(SQLALCHEMY_DATABASE_URL, echo=False, future=True)
async_session = sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)
Base = declarative_base()
async def init_db():
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
#asynccontextmanager
async def get_session() -> AsyncGenerator[AsyncSession, None]:
async with async_session() as session:
async with session.begin():
try:
yield session
finally:
await session.close()
In creation I have no problem, and it doesn't change much from the update
#classmethod
async def create_instance(
cls,
instance: InstanceInputOnCreate
) -> Union['Instance', InstanceOperationError]:
async with get_session() as conn:
new_instance = Instance(
name=instance.name,
host=instance.host
)
conn.add(new_instance)
try:
await conn.commit()
return new_instance
except Exception:
conn.rollback()
return InstanceOperationError(
result=False,
message=f"Instance '{instance.name}' already exists"
)
I didn't create mocks, because CI/CDs are set up to create a test database where Alembic can be started first for migrations, then the database is ready to test on it
sqlalchemy==1.4.46
fastapi-utils==0.2.1
aiosqlite==0.18.0
asyncpg==0.27.0
sqlalchemy-utils==0.39.0
pytest==7.2.1
pytest-asyncio==0.20.3
pytest-mock==3.10.0
pydantic~=1.10.4
aiokafka~=0.8.0
requests~=2.28.1
fastapi==0.70.1
If there are typo in the code, it's not a problem I had to change the names to create this question.

Related

Running an asyncoio socket server and client on the same process for tests

I wrote a socket server and client package with asyncio that happens to be a single module with two classes Server and Client. The intention is to have the code check if the specified port is in use on the same device, and if so, it will choose the use the Client class instead of the Server class. I am doing this to enable local testing between client and server for a larger goal.
I wrote my unit test with pytest and I can get it to pass if I run the server in its own process first, then run the client in another process.
However, I want to test both server and client together in the same process using pytest. The test never completes this way, however.
Am I running into an issue with asyncio here? Or is it an issue with pytest? Or is it something I am doing wrong in my code?
WebSockets.websocket
import asyncio
class Server:
def __init__(self, host, port):
self.host = host
self.port = port
self.server = None
self.connections = {}
async def start(self):
self.server = server = await asyncio.start_server(self.handle_client, self.host, self.port)
addr = server.sockets[0].getsockname()
print(f'Serving on {addr}')
async with server:
await server.serve_forever()
async def handle_client(self, reader, writer):
addr = writer.get_extra_info('peername')
print(f'New connection from {addr}')
self.connections[addr] = writer
try:
while not reader.at_eof():
data = await reader.read(100)
message = data.decode()
if message:
print(f'Received {message!r} from {addr}')
await self.send(message, addr)
finally:
del self.connections[addr]
writer.close()
async def send(self, message, addr):
writer = self.connections.get(addr)
if not writer:
return
writer.write(message.encode())
await writer.drain()
async def recv(self, addr):
reader, _ = await asyncio.open_connection(addr[0], addr[1])
data = await reader.read(100)
return data.decode()
async def stop(self):
await self.server.close()
class Client:
def __init__(self, host, port):
self.host = host
self.port = port
self.reader = None
self.writer = None
async def connect(self):
self.reader, self.writer = await asyncio.open_connection(self.host, self.port)
async def send(self, message):
self.writer.write(message.encode())
async def recv(self):
data = await self.reader.read(100)
return data.decode()
async def close(self):
self.writer.close()
Test
import socket
import asyncio
from WebSockets.websocket import Server, Client
import pytest
def check_port(address, port):
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((address, port))
s.close()
return "server"
except OSError:
return "client"
async def serve():
server = Server('localhost', 8910)
await server.start()
async def client_connect():
client = Client('localhost', 8910)
await client.connect()
await client.send("Hello world!")
message = await client.recv()
if message:
print(f"Received {message!r} from server")
assert message == "Hello world!"
await client.close()
#pytest.mark.asyncio
async def test_sockets():
if check_port("localhost", 8910) == "server":
await serve()
# for reference on how to use in non pytest module
# loop = asyncio.new_event_loop()
# loop.run_until_complete(serve())
# loop.close()
else:
await client_connect()
# for reference on how to use in non pytest module
# loop = asyncio.new_event_loop()
# loop.run_until_complete(client_connect())
# loop.close()

pytest-asyncio tests not running async

I'm trying to implement a simple async test suite. If my understanding is correct of async, the tests below should only take about 2 seconds to run. However, it's taking 6 seconds. What am I missing to make these test to run async ("at the same time")?
import logging
import pytest
import asyncio
MSG_FORMAT = "%(asctime)s.%(msecs)03d %(module)s->%(funcName)-15s |%(levelname)s| %(message)s"
MSG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
LOG_LEVEL = logging.INFO
# Create logger
logger = logging.getLogger(__name__)
logger.setLevel(LOG_LEVEL)
# Create Stream Handler
log_stream = logging.StreamHandler()
log_format = logging.Formatter(fmt=MSG_FORMAT, datefmt=MSG_DATE_FORMAT)
log_stream.setFormatter(log_format)
log_stream.setLevel(LOG_LEVEL)
logger.addHandler(log_stream)
class TestMyStuff:
#staticmethod
async def foo(seconds):
await asyncio.sleep(seconds)
return 1
#pytest.mark.asyncio
async def test_1(self, event_loop):
logger.info("start")
assert await event_loop.create_task(self.foo(2)) == 1
logger.info("end")
#pytest.mark.asyncio
async def test_2(self, event_loop):
logger.info("start")
assert await event_loop.create_task(self.foo(2)) == 1
logger.info("end")
#pytest.mark.asyncio
async def test_3(self, event_loop):
logger.info("start")
# assert await event_loop.run_in_executor(None, self.foo) == 1
assert await event_loop.create_task(self.foo(2)) == 1
logger.info("end")
pytest extras:
plugins: asyncio-0.18.3, aiohttp-1.0.4
pytest-asyncio runs asynchronous tests serially. That plugin's goal is to make testing asynchronous code more convenient.
pytest-asyncio-cooperative on the other hand has the goal of running asyncio tests concurrently via cooperative multitasking (ie. all async coroutines sharing the same event loop and yielding to each other).
To try out pytest-asyncio-cooperative do the following:
Install the plugin
pip install pytest-asyncio-cooperative
Replace the #pytest.mark.asyncio marks with #pytest.mark.asyncio_cooperative
Remove all references to event_loop. pytest-asyncio-cooperative uses a single implicit event loop for asyncio interactions.
Run pytest with pytest-asyncio disabled. It is not compatible with pytest-asyncio-cooperative
pytest -p no:asyncio test_mytestfile.py
Here is the original code snippet with these modifications:
import logging
import pytest
import asyncio
MSG_FORMAT = "%(asctime)s.%(msecs)03d %(module)s->%(funcName)-15s |%(levelname)s| %(message)s"
MSG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
LOG_LEVEL = logging.INFO
# Create logger
logger = logging.getLogger(__name__)
logger.setLevel(LOG_LEVEL)
# Create Stream Handler
log_stream = logging.StreamHandler()
log_format = logging.Formatter(fmt=MSG_FORMAT, datefmt=MSG_DATE_FORMAT)
log_stream.setFormatter(log_format)
log_stream.setLevel(LOG_LEVEL)
logger.addHandler(log_stream)
class TestMyStuff:
#staticmethod
async def foo(seconds):
await asyncio.sleep(seconds)
return 1
#pytest.mark.asyncio_cooperative
async def test_1(self):
logger.info("start")
assert await self.foo(2) == 1
logger.info("end")
#pytest.mark.asyncio_cooperative
async def test_2(self):
logger.info("start")
assert await self.foo(2) == 1
logger.info("end")
#pytest.mark.asyncio_cooperative
async def test_3(self):
logger.info("start")
# assert await event_loop.run_in_executor(None, self.foo) == 1
assert await self.foo(2) == 1
logger.info("end")
And here are the test results:
plugins: hypothesis-6.39.4, asyncio-cooperative-0.28.0, anyio-3.4.0, typeguard-2.12.1, Faker-8.1.0
collected 3 items
test_mytestfile.py ... [100%]
================================ 3 passed in 2.18s =================================
Please checkout the README of pytest-asyncio-cooperative. Now that tests are run in a concurrent way you need to be wary of shared resources (eg. mocking)

PostgresSql update db used with Ayncio and Semaphore: couse deadlock error when Semaphore > 1

What I need: downloading data from API in async way and save/update it to pg database
Problem: geting an error - deadlock detaction: when BoundedSemaphore arg in main() > 1
Here is my async main function:
async def main(self, parameters, class_sql_alchemy, class_pydantic, url):
async with httpx.AsyncClient() as client:
resp = await client.post(url, headers=headers, json=data,
params={'per_page':100})
total_pages = resp.json()["pagination"]["total_pages"]
# BoundedSemaphore(1) > 1 cause deadlock error
sem = asyncio.BoundedSemaphore(1)
tasks = [
asyncio.ensure_future(
self.safe_download(i, headers, data, class_sql_alchemy, class_pydantic, url, sem))
# creating task starts coroutine
for i
in range(total_pages)
]
await asyncio.gather(*tasks) # await moment all downloads done
Here is my async safe_download function:
async def safe_download(self, i, headers, data, class_sql_alchemy, class_pydantic, url, sem):
async with sem: # semaphore limits num of simultaneous downloads
await self.download(i, headers, data, class_sql_alchemy, class_pydantic, url)
Here is my async download function:
async def download(self, num_page, headers, data, class_sql_alchemy, class_pydantic, url):
async with httpx.AsyncClient() as client:
resp = await client.post(url, headers=headers, json=data, params={'page': num_page + 1,'per_page': 100})
items: List[class_sql_alchemy] = [class_pydantic(**item).dict() for item in
resp.json()['people']]
list_of_item = []
for item in items:
list_of_item.append(item)
async with settings.async_orm_session.begin() as session:
await upsert(session, People, list_of_item)
Here my upsert function:
async def upsert(session, model, row):
table = model.__table__
stmt = postgresql.insert(table)
primary_keys = [key.name for key in inspect(table).primary_key]
update_dict = {c.name: c for c in stmt.excluded if not c.primary_key}
if not update_dict:
raise ValueError("insert_or_update resulted in an empty update_dict")
stmt = stmt.on_conflict_do_update(index_elements=primary_keys,
set_=update_dict)
await session.execute(stmt, row)
On startup:
- db connection
engine = create_async_engine(db_async_url, future=True, echo=True)
async_orm_session = sessionmaker(
engine, expire_on_commit=False, class_= AsyncSession
)

asyncpg: How to construct SET strings with parameters

What is the correct way to pass in parameters in a SET query?
this will return asyncpg.exceptions.PostgresSyntaxError: syntax error at or near "$1"
import asyncio
import asyncpg
async def main():
conn = await asyncpg.connect(user="xx", password="yy", host="127.0.0.1", database="my_db")
# works
await conn.execute("select $1", "1")
identity = "arn:aws:sts::123456:assumed-role/thing_1"
# fails
await conn.execute("set session iam.identity = $1", identity)
asyncio.run(main())
asyncpg has a function for this:
await conn.execute("select set_config('iam.identity', $1, false)", identity)

FastAPI pytest with arguments

I try to test fastAPI get route with pytest and the problem is how i can pass params to client.get
main.py
#app.get('/purpose'):
async def read_purpose(name, date):
"""some logic"""
return {'ok':'ok'}
test.py
client = TestClient(app)
def test_purpose():
response = client.get("/purpose", json={"name":"test_name", "date":"01.01.2020"})
assert response.status_code = 200
My test is failed. it can not find name, and date arguments.
How i can pass this arguments to my test.
Thank you
I have same problem when writing pytest for my first FastAPI demo.
#router.post('/item', tags=['items'], response_model=ShowItem)
async def create_item(item: ItemCreate,
user_id: int,
db: Session = Depends(get_db)):
date_posted = datetime.now().date()
# owner_id = 1
item = Items(**item.dict(),
date_posted=date_posted,
owner_id=user_id)
db.add(item)
db.commit()
db.refresh(item)
return item
You can try "params" instead of "json", because you are passing isolated query parameters
def test_create_item():
# wrong
data = {'title': 'Hot Boat', 'description': 'This is a boat', 'user_id': 1}
resp = client.post('/item', json.dumps(data))
# correct
data = {'title': 'Hot Boat', 'description': 'This is a boat'}
resp = client.post('/item', json.dumps(data), params={"user_id": 2})
assert resp.status_code == 200
Then i can by pass above error.
Try this fix:
client = TestClient(app)
def test_purpose():
response = client.get("/purpose", params={"name":"test_name", "date":"01.01.2020"})
assert response.status_code = 200
More detail refer Question 61383179