Seems the default timeout is 60 second in FastHttpLocust and HttpLocust. Is there a way to change it? Here is my locust file:
from locust import TaskSet, task, between
from locust.contrib.fasthttp import FastHttpLocust
class HTTPGet(TaskSet):
#task(1)
def index(self):
self.client.get("/")
class WebsiteUser(FastHttpLocust):
task_set = HTTPGet
host="http://www.google.com"
wait_time = between(0.5, 0.5)
In Locust 1.0 and later, you can use network_timeout and connection_timeout on FastHttpLocust.
See https://docs.locust.io/en/stable/increase-performance.html.
class WebsiteUser(FastHttpLocust):
network_timeout = 5.0
connection_timeout = 5.0
task_set = HTTPGet
host = "http://www.google.com"
wait_time = between(0.5, 0.5)
Related
I'm working on a basic client-server desktop app project using kivy & sockets & threading.
The client & server work on it's own, however when I try to integrate it with kivy, python & kivy don't want to respond & yet no definitive error pops up.
Could i have some ideas as to how to fix this?
This is the code that freezes when i run it, if i take away the import server_sock it works as a general gui and doesnt freeze.
import kivy
from kivy.app import App
from kivy.uix.label import Label
from kivy.uix.gridlayout import GridLayout
from kivy.uix.textinput import TextInput
from kivy.uix.button import Button
from kivy.uix.screenmanager import ScreenManager, Screen
from kivy.clock import Clock
import server_sock
import sys
kivy.require("2.1.0") #latest version
class ConnectPage(GridLayout):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.cols = 2
self.add_widget(Label(text="IP:"))
self.ip = TextInput(multiline=False)
self.add_widget(self.ip)
self.add_widget(Label(text="PORT:"))
self.port = TextInput(multiline=False)
self.add_widget(self.port)
self.add_widget(Label(text="USERNAME:"))
self.user = TextInput(multiline=False)
self.add_widget(self.user)
self.join = Button(text="Join")
self.join.bind(on_press=self.join_button)
self.add_widget(Label())
self.add_widget(self.join)
def join_button(self, instance):
port = self.port.text
ip = self.ip.text
user = self.user.text
info = f"Attempting to join {ip}:{port} as {user}"
chat_app.info_page.update_info(info)
chat_app.screen_manager.current = "Info"
Clock.shedule_once(self.connect,1)
def connect(self, _):
port = int(self.port.text)
ip = self.ip.text
user = self.user.text
try:
server_sock.connect(ip, port)
chat_app.create_chat_page()
chat_app.screen_manager.current = "Chat"
except:
show_error(message="not gonna happen")
class InfoPage(GridLayout):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.cols = 1
self.message = Label(halign="center", valign="middle", font_size="30")
self.message.bind(width=self.update_text_width)
self.add_widget(self.message)
def update_info(self,message):
self.message.text = message
def update_text_width(self, *_):
self.message.text_size = (self.message.width*0.9, None)
class ChatPage(GridLayout):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.cols = 1
self.add_widget(Label(text="Hey at least it works till now"))
class ChatApp(App):
def build(self):
self.screen_manager = ScreenManager()
self.connect_page = ConnectPage()
screen = Screen(name="Connect")
screen.add_widget(self.connect_page)
self.screen_manager.add_widget(screen)
self.info_page = InfoPage()
screen= Screen(name="Info")
screen.add_widget(self.info_page)
self.screen_manager.add_widget(screen)
return self.screen_manager
def create_chat_page(self):
self.chat_page = ChatPage()
screen = Screen(name="Chat")
screen.add_widget(self.chat_page)
self.screen_manager.add_widget(screen)
def show_error(message):
chat_app.info_page.update_info(message)
chat_app.screen_manager.current = "Info"
Clock.shedule_once(sys.exit, 10)
if __name__ == "__main__":
chat_app =ChatApp()
chat_app.run()
This is the server_sock file
import socket
import threading
import socket
HOST = '127.0.0.1'
PORT = 55555
server = socket.socket(
socket.AF_INET,
socket.SOCK_STREAM
)
try:
server.bind((HOST, PORT))
except:
print(f'unable to bind to {HOST} and {PORT}')
server.listen()
print(f"Listening for connections on {HOST}: {PORT}")
clients = []
nicknames = []
def broadcast(message):
for client in clients:
client.send(message)
def message_recv(client):
while True:
try:
message = client.recv(2048)
broadcast(message)
except:
index = clients.index(client)
clients.remove(client)
nickname = nicknames[index]
broadcast(f'{nickname} left the chat'.encode('ascii'))
nicknames.remove(nickname)
break
def recieve():
while True:
client, address = server.accept()
print(f"Connected with {str(address)}")
client.send("SOMETHING".encode('ascii'))
nickname = client.recv(2048).decode('ascii')
nicknames.append(nickname)
clients.append(client)
print(f"Nickname of the client is {nickname}")
broadcast(f"{nickname} joined the chat".encode('ascii'))
client.send("Connected to the server".encode('ascii'))
thread = threading.Thread(target=message_recv, args=(client,))
thread.start()
print("Server is listening")
recieve()
I'm trying to make FastAPI server which streams MJPEG from Raspberry Pi via picamera2 library. It works, but when I reload browser on /mjpeg multiple times, it will stuck. But with this example it works perfectly - you can reload browsers as many times as you want. Can you spot the problem?
For server I'm using uvicorn with default settings.
Thanks!
import io
import os
from threading import Condition
from fastapi import FastAPI, Request
from fastapi.middleware.cors import CORSMiddleware
from picamera2 import Picamera2
from picamera2.encoders import JpegEncoder
from picamera2.outputs import FileOutput
from fastapi.responses import HTMLResponse, StreamingResponse
app = FastAPI()
app.add_middleware(
CORSMiddleware,
allow_origins="http://localhost:8000",
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
class StreamingOutput(io.BufferedIOBase):
def __init__(self):
self.frame = None
self.condition = Condition()
def write(self, buf):
with self.condition:
self.frame = buf
self.condition.notify_all()
picam2 = Picamera2()
picam2.configure(picam2.create_video_configuration(main={"size": (640, 480)}))
output = StreamingOutput()
picam2.start_recording(JpegEncoder(), FileOutput(output))
def get_frame():
try:
while True:
with output.condition:
output.condition.wait()
frame = output.frame
yield (
b"--frame\r\n" b"Content-Type: image/jpeg\r\n\r\n" + frame + b"\r\n"
)
except Exception as e:
print("Error! Frames")
#app.get("/mjpeg", response_class=StreamingResponse)
def mjpeg(request: Request):
try:
frames = get_frame()
response = StreamingResponse(
frames,
headers={
"Cache-Control": "no-cache, private",
"Pragma": "no-cache",
},
media_type="multipart/x-mixed-replace; boundary=frame",
)
return response
except Exception as e:
print("Error! Route")
In pytest-django there is a builtin fixture live_server though it seems like this server (that is actually based on LiveServerTestCase) can't handle web-sockets or at least won't interact with my asgi.py module.
How can one mimic that fixture in order to use ChannelsLiveServerTestCase instead? Or anything else that will run a test-database and will be able to serve an ASGI application?
My goal eventually is to have as close to production environment as possible, for testing and being able to test interaction between different Consumers.
P.S: I know I can run manage.py testserver <Fixture> on another thread / process by overriding django_db_setup though I seek for a better solution.
You can implement a channels_live_server fixture based on the implementations of:
live_server fixture, which instantiates
LiveServer helper, which starts LiveServerThread, and
ChannelsLiveServerTestCase, which starts DaphneProcess.
#medihack implemented it at https://github.com/pytest-dev/pytest-django/issues/1027:
from functools import partial
from channels.routing import get_default_application
from daphne.testing import DaphneProcess
from django.contrib.staticfiles.handlers import ASGIStaticFilesHandler
from django.core.exceptions import ImproperlyConfigured
from django.db import connections
from django.test.utils import modify_settings
def make_application(*, static_wrapper):
# Module-level function for pickle-ability
application = get_default_application()
if static_wrapper is not None:
application = static_wrapper(application)
return application
class ChannelsLiveServer:
host = "localhost"
ProtocolServerProcess = DaphneProcess
static_wrapper = ASGIStaticFilesHandler
serve_static = True
def __init__(self) -> None:
for connection in connections.all():
if connection.vendor == "sqlite" and connection.is_in_memory_db():
raise ImproperlyConfigured(
"ChannelsLiveServer can not be used with in memory databases"
)
self._live_server_modified_settings = modify_settings(ALLOWED_HOSTS={"append": self.host})
self._live_server_modified_settings.enable()
get_application = partial(
make_application,
static_wrapper=self.static_wrapper if self.serve_static else None,
)
self._server_process = self.ProtocolServerProcess(self.host, get_application)
self._server_process.start()
self._server_process.ready.wait()
self._port = self._server_process.port.value
def stop(self) -> None:
self._server_process.terminate()
self._server_process.join()
self._live_server_modified_settings.disable()
#property
def url(self) -> str:
return f"http://{self.host}:{self._port}"
#pytest.fixture
def channels_live_server(request):
server = ChannelsLiveServer()
request.addfinalizer(server.stop)
return server
#aaron's solution can't work, due to pytest-django conservative approach for database access.
another proccess wouldn't be aware that your test has database access permissions therefore you won't have database access. (here is a POC)
Using a scoped fixture of daphne Server suffice for now.
import threading
import time
from functools import partial
from django.contrib.staticfiles.handlers import ASGIStaticFilesHandler
from django.core.exceptions import ImproperlyConfigured
from django.db import connections
from django.test.utils import modify_settings
from daphne.server import Server as DaphneServer
from daphne.endpoints import build_endpoint_description_strings
def get_open_port() -> int:
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("", 0))
s.listen(1)
port = s.getsockname()[1]
s.close()
return port
def make_application(*, static_wrapper):
# Module-level function for pickle-ability
if static_wrapper is not None:
application = static_wrapper(your_asgi_app)
return application
class ChannelsLiveServer:
port = get_open_port()
host = "localhost"
static_wrapper = ASGIStaticFilesHandler
serve_static = True
def __init__(self) -> None:
for connection in connections.all():
if connection.vendor == "sqlite" and connection.is_in_memory_db():
raise ImproperlyConfigured(
"ChannelsLiveServer can not be used with in memory databases"
)
self._live_server_modified_settings = modify_settings(ALLOWED_HOSTS={"append": self.host})
self._live_server_modified_settings.enable()
get_application = partial(
make_application,
static_wrapper=self.static_wrapper if self.serve_static else None,
)
endpoints = build_endpoint_description_strings(
host=self.host, port=self.port
)
self._server = DaphneServer(
application=get_application(),
endpoints=endpoints
)
t = threading.Thread(target=self._server.run)
t.start()
for i in range(10):
time.sleep(0.10)
if self._server.listening_addresses:
break
assert self._server.listening_addresses[0]
def stop(self) -> None:
self._server.stop()
self._live_server_modified_settings.disable()
#property
def url(self) -> str:
return f"ws://{self.host}:{self.port}"
#property
def http_url(self):
return f"http://{self.host}:{self.port}"
#pytest.fixture(scope='session')
def channels_live_server(request, live_server):
server = ChannelsLiveServer()
request.addfinalizer(server.stop)
return server
import falcon
import json
from tasks import add
from waitress import serve
class tasksresource:
def on_get(self, req, resp):
"""Handles GET requests"""
self.result = add.delay(1, 2)
self.context = {'ID': self.result.id, 'final result': self.result.ready()}
resp.body = json.dumps(self.context)
api = falcon.API()
api.add_route('/result', tasksresource())
# api.add_route('/result/task', taskresult())
if __name__ == '__main__':
serve(api, host='127.1.0.1', port=5555)
how do i get the Get the task id from json payload ( post data)
and add a route to it
Here a small example. Structure of files:
/project
__init__.py
app.py # routes, falcon etc.
tasks.py # celery
example.py # script for demonstration how it works
app.py:
import json
import falcon
from tasks import add
from celery.result import AsyncResult
class StartTask(object):
def on_get(self, req, resp):
# start task
task = add.delay(4, 4)
resp.status = falcon.HTTP_200
# return task_id to client
result = {'task_id': task.id}
resp.body = json.dumps(result)
class TaskStatus(object):
def on_get(self, req, resp, task_id):
# get result of task by task_id and generate content to client
task_result = AsyncResult(task_id)
result = {'status': task_result.status, 'result': task_result.result}
resp.status = falcon.HTTP_200
resp.body = json.dumps(result)
app = falcon.API()
# registration of routes
app.add_route('/start_task', StartTask())
app.add_route('/task_status/{task_id}', TaskStatus())
tasks.py:
from time import sleep
import celery
app = celery.Celery('tasks', broker='redis://localhost:6379/0', backend='redis://localhost:6379/0')
#app.task
def add(x, y):
"""
:param int x:
:param int y:
:return: int
"""
# sleep just for demonstration
sleep(5)
return x + y
Now we need to start celery application. Go to project folder and run:
celery -A tasks worker --loglevel=info
After this we need to start Falcon application. Go to project folder and run:
gunicorn app:app
Ok. Everything is ready.
example.py is small client side which can help to understand:
from time import sleep
import requests
# start new task
task_info = requests.get('http://127.0.0.1:8000/start_task')
task_info = task_info.json()
while True:
# check status of task by task_id while task is working
result = requests.get('http://127.0.0.1:8000/task_status/' + task_info['task_id'])
task_status = result.json()
print task_status
if task_status['status'] == 'SUCCESS' and task_status['result']:
print 'Task with id = %s is finished' % task_info['task_id']
print 'Result: %s' % task_status['result']
break
# sleep and check status one more time
sleep(1)
Just call python ./example.py and you should see something like this:
{u'status': u'PENDING', u'result': None}
{u'status': u'PENDING', u'result': None}
{u'status': u'PENDING', u'result': None}
{u'status': u'PENDING', u'result': None}
{u'status': u'PENDING', u'result': None}
{u'status': u'SUCCESS', u'result': 8}
Task with id = 76542904-6c22-4536-99d9-87efd66d9fe7 is finished
Result: 8
Hope this helps you.
The above example by Danila Ganchar is great and very helpful. I'm using celery version 4.3.0 with Python 3, and one of the errors I received from using the example above is on this line:
task_result = AsyncResult(task_id)
The error I would receive is:
AttributeError: 'DisabledBackend' object has no attribute '_get_task_meta_for'
This may be a recent change, but result.AsyncResult (or just AsyncResult in this example because he imported it from celery.result) doesn't know the backend you are using. There are 2 solutions to solving this problem:
1) You can take the AsyncResult of the actual task itself add.AsyncResult(task_id) because the add task already has the backend defined through the #app.task decorator. The downside to this in this example is you want to be able to get the result for any task by just passing in the task_id via the Falcon endpoint, so this is limited
2) The preferred method is to just pass in the app parameter to the AsyncResult function:
task = result.AsyncResult(id, app=app)
Hope this helps!
I have an application with Blueprints and Celery
the code is here:
config.py
import os
from celery.schedules import crontab
basedir = os.path.abspath(os.path.dirname(__file__))
class Config:
SECRET_KEY = os.environ.get('SECRET_KEY') or ''
SQLALCHEMY_COMMIT_ON_TEARDOWN = True
RECORDS_PER_PAGE = 40
SQLALCHEMY_DATABASE_URI = ''
CELERY_BROKER_URL = ''
CELERY_RESULT_BACKEND = ''
CELERY_RESULT_DBURI = ''
CELERY_TIMEZONE = 'Europe/Kiev'
CELERY_ENABLE_UTC = False
CELERYBEAT_SCHEDULE = {}
#staticmethod
def init_app(app):
pass
class DevelopmentConfig(Config):
DEBUG = True
WTF_CSRF_ENABLED = True
APP_HOME = ''
SQLALCHEMY_DATABASE_URI = 'mysql+mysqldb://...'
CELERY_BROKER_URL = 'sqla+mysql://...'
CELERY_RESULT_BACKEND = "database"
CELERY_RESULT_DBURI = 'mysql://...'
CELERY_TIMEZONE = 'Europe/Kiev'
CELERY_ENABLE_UTC = False
CELERYBEAT_SCHEDULE = {
'send-email-every-morning': {
'task': 'app.workers.tasks.send_email_task',
'schedule': crontab(hour=6, minute=15),
},
}
class TestConfig(Config):
DEBUG = True
WTF_CSRF_ENABLED = False
TESTING = True
SQLALCHEMY_DATABASE_URI = 'mysql+mysqldb://...'
class ProdConfig(Config):
DEBUG = False
WTF_CSRF_ENABLED = True
SQLALCHEMY_DATABASE_URI = 'mysql+mysqldb://...'
CELERY_BROKER_URL = 'sqla+mysql://...celery'
CELERY_RESULT_BACKEND = "database"
CELERY_RESULT_DBURI = 'mysql://.../celery'
CELERY_TIMEZONE = 'Europe/Kiev'
CELERY_ENABLE_UTC = False
CELERYBEAT_SCHEDULE = {
'send-email-every-morning': {
'task': 'app.workers.tasks.send_email_task',
'schedule': crontab(hour=6, minute=15),
},
}
config = {
'development': DevelopmentConfig,
'default': ProdConfig,
'production': ProdConfig,
'testing': TestConfig,
}
class AppConf:
"""
Class to store current config even out of context
"""
def __init__(self):
self.app = None
self.config = {}
def init_app(self, app):
if hasattr(app, 'config'):
self.app = app
self.config = app.config.copy()
else:
raise TypeError
init.py:
import os
from flask import Flask
from celery import Celery
from config import config, AppConf
def create_app(config_name):
app = Flask(__name__)
app.config.from_object(config[config_name])
config[config_name].init_app(app)
app_conf.init_app(app)
# Connect to Staging view
from staging.views import staging as staging_blueprint
app.register_blueprint(staging_blueprint)
return app
def make_celery(app=None):
app = app or create_app(os.getenv('FLASK_CONFIG') or 'default')
celery = Celery(__name__, broker=app.config.CELERY_BROKER_URL)
celery.conf.update(app.conf)
TaskBase = celery.Task
class ContextTask(TaskBase):
abstract = True
def __call__(self, *args, **kwargs):
with app.app_context():
return TaskBase.__call__(self, *args, **kwargs)
celery.Task = ContextTask
return celery
tasks.py:
from app import make_celery, app_conf
cel = make_celery(app_conf.app)
#cel.task
def send_realm_to_fabricdb(realm, form):
some actions...
and here is the problem:
The Blueprint "staging" uses task send_realm_to_fabricdb, so it makes: from tasks import send_realm_to_fabricdb
than, when I just run application, everything goes ok
BUT, when I'm trying to run celery: celery -A app.tasks worker -l info --beat, it goes to cel = make_celery(app_conf.app) in tasks.py, got app=None and trying to create application again: registering a blueprint... so I've got cycle import here.
Could you tell me how to break this cycle?
Thanks in advance.
I don't have the code to try this out, but I think things would work better if you move the creation of the Celery instance out of tasks.py and into the create_app function, so that it happens at the same time the app instance is created.
The argument you give to the Celery worker in the -A option does not need to have the tasks, Celery just needs the celery object, so for example, you could create a separate starter script, say celery_worker.py that calls create_app to create app and cel and then give it to the worker as -A celery_worker.cel, without involving the blueprint at all.
Hope this helps.
What I do to solve this error is that I create two Flask instance which one is for Web app, and another is for initial Celery instance.
Like #Miguel said, I have
celery_app.py for celery instance
manager.py for Flask instance
And in these two files, each module has it's own Flask instance.
So I can use celery.task in Views. And I can start celery worker separately.
Thanks Bob Jordan, you can find the answer from https://stackoverflow.com/a/50665633/2794539,
Key points:
1. make_celery do two things at the same time: create celery app and run celery with flask content, so you can create two functions to do make_celery job
2. celery app must init before blueprint register
Having the same problem, I ended up solving it very easily using shared_task (docs), keeping a single app.py file and not having to instantiate the flask app multiple times.
The original situation that led to the circular import:
from src.app import celery # src.app is ALSO importing the blueprints which are importing this file which causes the circular import.
#celery.task(bind=True)
def celery_test(self):
sleep(5)
logger.info("Task processed by Celery.")
The current code that works fine and avoids the circular import:
# from src.app import celery <- not needed anymore!
#shared_task(bind=True)
def celery_test(self):
sleep(5)
logger.info("Task processed by Celery.")
Please mind that I'm pretty new to Celery so I might be overseeing important stuff, it would be great if someone more experienced can give their opinion.