django celery queue routing won't work - celery

I have two tasks. the task "heavy_task" need a concurrency of 1 and the "lite_task" need a concurrency of 4
#task
def lite_task():
tabla = Proc_Carga()
sp = tabla.carga()
return None
#task()
def heavy_task(idprov,pfecha):
conci = Buscar_Conci()
spconc = conci.buscarcon(idprov,pfecha)
return None
I define the routes in my settings.py file:
BROKER_URL = 'redis://localhost:6379/0'
CELERY_IMPORTS = ("pc.tasks", )
CELERY_ACCEPT_CONTENT = ['json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_RESULT_BACKEND='djcelery.backends.cache:CacheBackend'
CELERY_ROUTES = {"tasks.heavy_task": {"queue": "heavy"},"tasks.lite_task": {"queue": "lite"}}
I try to excute two workers specifying the concurrency in this way
celery multi start heavy lite -A provcon -c:heavy 1 -c:lite 3
When first call the tasks heavy_task work fine and the concurrency works,
but after call the task lite_task the concurency for the queue heavy change.
I try this:
celery -A provcon worker -Q heavy -c 1
And when I execute the task heavy_task, the routing won't work and the task is not executed.
but if use this:
celery -A provcon -c 1
everything works fine, but I can only execute one task at time, and I need to be able to execute the heavy_task with a concurrency of 1 and the lite_task with a concurrency of 3
Any advice

I try different settings to make the queues works and finally I did it.
In the tasks.py file I harcoded the the queue in the task decorator
#task(queue = 'heavy')
To run the workers I use this:
celery multi start lite_w heavy_w -A provcon -Q:heavy_w heavy -Q:carga_w lite -l info -c:heavy_w 1 -c:lite_w 3 -E
And i delete the routing settings from the settings.py file. My settings are this:
BROKER_URL = 'redis://localhost:6379/0'
CELERY_IMPORTS = ("pc.tasks", )
CELERY_ACCEPT_CONTENT = ['json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_RESULT_BACKEND='djcelery.backends.cache:CacheBackend'

Related

Celery chain - if any tasks fail, do x, else y

I'm just getting into Celery chains in my Django project. I have the following function:
def orchestrate_tasks_for_account(account_id):
# Get the account, set status to 'SYNC' until the chain is complete
account = Account.objects.get(id=account_id)
account.status = "SYNC"
account.save()
chain = task1.s(account_id) | task2.s() | task3.s()
chain()
# if any of the tasks in the chain failed, set account.status = 'ERROR'
# else set the account.status = 'OK'
The chain works as expected, but I'm not sure how to take feedback from the chain and update the account based on the results
In other words, I'd like to set the account status to 'ERROR' if any of the tasks in the chain fail, otherwise I'd like to set the account status to 'OK'
I'm confused by the Celery documentation on how to handle an error with an if/else like I've commented in the last two lines above.
Does anyone have experience with this?
Ok - here's what I came up with
I've leveraged the waiting library in this solution
from celery import chain
from waiting import wait
def orchestrate_tasks_for_account(account_id):
account = Account.objects.get(id=account_id)
account.status = "SYNC"
account.save()
job = chain(
task1.s(account_id),
task2.s(),
task3.s()
)
result = job.apply_async()
wait(
lambda: result.ready(), # when async job is completed...
timeout_seconds=1800, # wait 1800 seconds (30 minutes)
waiting_for="task orchestration to complete"
)
if result.successful():
account.status = 'OK'
else:
account.status = 'ERROR'
account.save()
I am open to suggestions to make this better!

Celery broadcast task not working

I've tried to make a broadcast task but only one of my workers recieve it per each call. Would you please help me? (I'm using rabbitmq and node-celery)
default_exchange = Exchange('celery', type='direct')
celery.conf.update(
CELERY_RESULT_BACKEND = "amqp",
CELERY_RESULT_SERIALIZER='json',
CELERY_QUEUES = (
Queue('celery', default_exchange, routing_key='celery'),
Broadcast('broadcast_tasks'),
),
CELERY_ROUTES = (
{'my_tasks.sample_broadcast_task': {
'queue': 'broadcast_tasks',
}},
{'my_tasks.sample_normal_task': {
'queue': 'celery',
'exchange': 'celery',
'exchange_type': 'direct',
'routing_key': 'celery',
}}
),
)
I've also test following configurtion but not working.
celery.conf.update(
CELERY_RESULT_BACKEND = "amqp",
CELERY_RESULT_SERIALIZER='json',
CELERY_QUEUES=(
Queue('celery', Exchange('celery'), routing_key='celery'),
Broadcast('broadcast'),
),
)
#celery.task(ignore_result=True, queue='broadcast',
options=dict(queue='broadcast'))
def sample_broadcast_task():
print "test"
EDIT
after changing how to run worker by adding -Q broadcast, now i face to this error:
PreconditionFailed: Exchange.declare: (406) PRECONDITION_FAILED - inequivalent arg 'type' for exchange 'broadcast' in vhost '/': received 'direct' but current is 'fanout'
After trying many many many things, i finally find a solution. This work for me.
( celery 3.1.24 (Cipater) and Python 2.7.12 )
WORKER - tasks.py :
from celery import Celery
import celery_config
from kombu.common import Broadcast, Queue, Exchange
app = Celery()
app.config_from_object(sysadmin_celery_config)
#app.task
def print_prout(x):
print x
return x
WORKER - celery_config.py :
# coding=utf-8
from kombu.common import Broadcast, Queue, Exchange
BROKER_URL = 'amqp://login:pass#172.17.0.1//'
CELERY_RESULT_BACKEND = 'redis://:login#172.17.0.1'
CELERY_TIMEZONE = 'Europe/Paris'
CELERY_ENABLE_UTC = True
CELERY_TASK_SERIALIZER = 'pickle'
CELERY_RESULT_SERIALIZER = 'pickle'
CELERY_ACCEPT_CONTENT = ['pickle', 'json', 'msgpack', 'yaml']
CELERY_DISABLE_RATE_LIMITS = True
CELERY_ALWAYS_EAGER = False
CELERY_QUEUES = (Broadcast('broadcast_tasks'), )
worker lauched with :
celery -A celery_worker.tasks worker --loglevel=info --concurrency=1 -n worker_name_1
On the client (another docker container for me).
from celery import Celery
from celery_worker import tasks
result = tasks.print_prout.apply_async(['prout'], queue='broadcast_tasks')
print result.get()
The next step for me is how to retrieve and display results returned by all the workers. The "print result.get()" seems to return only the result of the last worker.
It does not seem obvious ( Have Celery broadcast return results from all workers )
according to your description:
I've tried to make a broadcast task but only one of my workers recieve it per each call
you may be using direct type exchange.
Try this
from celery import Celery
from kombu.common import Broadcast
BROKER_URL = 'amqp://guest:guest#localhost:5672//'
class CeleryConf:
# List of modules to import when celery starts.
CELERY_ACCEPT_CONTENT = ['json']
CELERY_IMPORTS = ('main.tasks')
CELERY_QUEUES = (Broadcast('q1'),)
CELERY_ROUTES = {
'tasks.sampletask': {'queue': 'q1'}
}
celeryapp = Celery('celeryapp', broker=BROKER_URL)
celeryapp.config_from_object(CeleryConf())
#celeryapp.task
def sampletask(form):
print form
To send the message, do
d= sampletask.apply_async(['4c5b678350fc643'],serializer="json", queue='q1')

Flask, blueprints uses celery task and got cycle import

I have an application with Blueprints and Celery
the code is here:
config.py
import os
from celery.schedules import crontab
basedir = os.path.abspath(os.path.dirname(__file__))
class Config:
SECRET_KEY = os.environ.get('SECRET_KEY') or ''
SQLALCHEMY_COMMIT_ON_TEARDOWN = True
RECORDS_PER_PAGE = 40
SQLALCHEMY_DATABASE_URI = ''
CELERY_BROKER_URL = ''
CELERY_RESULT_BACKEND = ''
CELERY_RESULT_DBURI = ''
CELERY_TIMEZONE = 'Europe/Kiev'
CELERY_ENABLE_UTC = False
CELERYBEAT_SCHEDULE = {}
#staticmethod
def init_app(app):
pass
class DevelopmentConfig(Config):
DEBUG = True
WTF_CSRF_ENABLED = True
APP_HOME = ''
SQLALCHEMY_DATABASE_URI = 'mysql+mysqldb://...'
CELERY_BROKER_URL = 'sqla+mysql://...'
CELERY_RESULT_BACKEND = "database"
CELERY_RESULT_DBURI = 'mysql://...'
CELERY_TIMEZONE = 'Europe/Kiev'
CELERY_ENABLE_UTC = False
CELERYBEAT_SCHEDULE = {
'send-email-every-morning': {
'task': 'app.workers.tasks.send_email_task',
'schedule': crontab(hour=6, minute=15),
},
}
class TestConfig(Config):
DEBUG = True
WTF_CSRF_ENABLED = False
TESTING = True
SQLALCHEMY_DATABASE_URI = 'mysql+mysqldb://...'
class ProdConfig(Config):
DEBUG = False
WTF_CSRF_ENABLED = True
SQLALCHEMY_DATABASE_URI = 'mysql+mysqldb://...'
CELERY_BROKER_URL = 'sqla+mysql://...celery'
CELERY_RESULT_BACKEND = "database"
CELERY_RESULT_DBURI = 'mysql://.../celery'
CELERY_TIMEZONE = 'Europe/Kiev'
CELERY_ENABLE_UTC = False
CELERYBEAT_SCHEDULE = {
'send-email-every-morning': {
'task': 'app.workers.tasks.send_email_task',
'schedule': crontab(hour=6, minute=15),
},
}
config = {
'development': DevelopmentConfig,
'default': ProdConfig,
'production': ProdConfig,
'testing': TestConfig,
}
class AppConf:
"""
Class to store current config even out of context
"""
def __init__(self):
self.app = None
self.config = {}
def init_app(self, app):
if hasattr(app, 'config'):
self.app = app
self.config = app.config.copy()
else:
raise TypeError
init.py:
import os
from flask import Flask
from celery import Celery
from config import config, AppConf
def create_app(config_name):
app = Flask(__name__)
app.config.from_object(config[config_name])
config[config_name].init_app(app)
app_conf.init_app(app)
# Connect to Staging view
from staging.views import staging as staging_blueprint
app.register_blueprint(staging_blueprint)
return app
def make_celery(app=None):
app = app or create_app(os.getenv('FLASK_CONFIG') or 'default')
celery = Celery(__name__, broker=app.config.CELERY_BROKER_URL)
celery.conf.update(app.conf)
TaskBase = celery.Task
class ContextTask(TaskBase):
abstract = True
def __call__(self, *args, **kwargs):
with app.app_context():
return TaskBase.__call__(self, *args, **kwargs)
celery.Task = ContextTask
return celery
tasks.py:
from app import make_celery, app_conf
cel = make_celery(app_conf.app)
#cel.task
def send_realm_to_fabricdb(realm, form):
some actions...
and here is the problem:
The Blueprint "staging" uses task send_realm_to_fabricdb, so it makes: from tasks import send_realm_to_fabricdb
than, when I just run application, everything goes ok
BUT, when I'm trying to run celery: celery -A app.tasks worker -l info --beat, it goes to cel = make_celery(app_conf.app) in tasks.py, got app=None and trying to create application again: registering a blueprint... so I've got cycle import here.
Could you tell me how to break this cycle?
Thanks in advance.
I don't have the code to try this out, but I think things would work better if you move the creation of the Celery instance out of tasks.py and into the create_app function, so that it happens at the same time the app instance is created.
The argument you give to the Celery worker in the -A option does not need to have the tasks, Celery just needs the celery object, so for example, you could create a separate starter script, say celery_worker.py that calls create_app to create app and cel and then give it to the worker as -A celery_worker.cel, without involving the blueprint at all.
Hope this helps.
What I do to solve this error is that I create two Flask instance which one is for Web app, and another is for initial Celery instance.
Like #Miguel said, I have
celery_app.py for celery instance
manager.py for Flask instance
And in these two files, each module has it's own Flask instance.
So I can use celery.task in Views. And I can start celery worker separately.
Thanks Bob Jordan, you can find the answer from https://stackoverflow.com/a/50665633/2794539,
Key points:
1. make_celery do two things at the same time: create celery app and run celery with flask content, so you can create two functions to do make_celery job
2. celery app must init before blueprint register
Having the same problem, I ended up solving it very easily using shared_task (docs), keeping a single app.py file and not having to instantiate the flask app multiple times.
The original situation that led to the circular import:
from src.app import celery # src.app is ALSO importing the blueprints which are importing this file which causes the circular import.
#celery.task(bind=True)
def celery_test(self):
sleep(5)
logger.info("Task processed by Celery.")
The current code that works fine and avoids the circular import:
# from src.app import celery <- not needed anymore!
#shared_task(bind=True)
def celery_test(self):
sleep(5)
logger.info("Task processed by Celery.")
Please mind that I'm pretty new to Celery so I might be overseeing important stuff, it would be great if someone more experienced can give their opinion.

Celery task subprocess stdout to log

I have a celery task which calls other python script external to Django application with subprocess. This program have some print's in it, and I want to have these print's in my celery log file or in my database. When I set CELERY_ALWAYS_EAGER = True in Django settings.py file, everything works fine. If I don't set this setting, celery task log subprocess stdout only when it exit. It seems like p.stdout.readline() is blocking.
run-test.py is a long process, couple of minutes, but it print what it's doing. I want to capture this.
#shared_task
def run_tests(scenario_path, vu):
basedir = os.path.abspath(os.path.dirname(__file__))
config_path = '%s/../../scripts/config.ini' % basedir
cmd = ['python', '%s/../../scripts/aws/run-test.py' % basedir, '%s' % config_path, scenario_path, str(vu), str(2)]
p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
while True:
line = p.stdout.readline()
if line != '':
logger.info(line)
else:
return
I found this to be very useful, using select for polling instead of blocking on readline.
https://gist.github.com/bgreenlee/1402841
child = subprocess.Popen(popenargs, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, **kwargs)
log_level = {child.stdout: stdout_log_level,
child.stderr: stderr_log_level}
def check_io():
ready_to_read = select.select([child.stdout, child.stderr], [], [], 1000)[0]
for io in ready_to_read:
line = io.readline()
logger.log(log_level[io], line[:-1])
# keep checking stdout/stderr until the child exits
while child.poll() is None:
check_io()
check_io() # check again to catch anything after the process exits

Celeryd does not execute my workers

I'm trying to daemonize my tasks in celery, i've tested without daemonizing and it's working very well.
But i can't daemonize like the tutorial said (http://docs.celeryproject.org/en/latest/tutorials/daemonizing.html#daemonizing)
I have my files:
solr_desa.py
from celery import Celery
celery = Celery('solrdesa')
celery.config_from_object('celeryconfig')
#celery.task(name = "doSomething")
def doSomething():
return 6
celeryconfig.py
from celery.schedules import crontab
BROKER_URL = '127.0.0.1'
BROKER_PORT = 5673
CELERY_RESULT_BACKEND = 'redis://127.0.0.1:6969/0'
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_TIMEZONE = 'America/Santiago'
CELERY_IMPORTS = ('solr_desa', )
CELERYBEAT_SCHEDULE = {
'solr_schedule': {
'task': 'doSomething',
'schedule': crontab(minute=9, hour=12)
},
}
And also
/etc/default/celeryd
CELERY_NODES="w1"
CELERYD_CHDIR="/opt/latam/script/solr"
CELERYD_OPTS="--time-limit=300 --concurrency=8"
CELERY_CONFIG_MODULE="celeryconfig"
CELERYD_LOG_FILE="/var/log/celery/%n.log"
CELERYD_PID_FILE="/var/run/celery/%n.pid"
I execute with the default celeryd in https://github.com/celery/celery/blob/3.0/extra/generic-init.d/celeryd, but the tasks are just enqueued but it looks like there are not workers :(
Where is my mistake in my configuration? :(
your broker url is wrong it should be something like this
BROKER_URL : transport://userid:password#hostname:port/virtual_host
#example
BROKER_URL = "amqp://{username}:{password}#{host}:{port}//"
read here for more details