celery giving Rate limit attempt for unknown task task name - celery

basically i am running two workers on celery, different module and different queue but same rabbitmq
celery worker -l info -A module_name.main.tasks -Q queue_one
celery worker -l info -A module_name.sub.sub_task -Q queue_two
when i try to rate limit 1st task present on 1 module i get this error from 2nd worker running the the other module..
app.control.rate_limit('module_name.main.tasks.method', '30/m')
Rate limit attempt for unknown task
I would prefer if the rate limit call would go to the worker that is working on that module and not to the other workers which are not working on that module.
any idea how to resolve this ??
Update: adding code:
celery_worker_base.py:
from __future__ import absolute_import
from celery import Celery
app = Celery('poc',
backend='mongodb://user:pass#ip:27017/collection',
broker='amqp://user:pass#ip/vhost',
include=['poc.main.proj.tasks'])
# Optional configuration, see the application user guide.
app.conf.update(
CELERY_TASK_RESULT_EXPIRES=3600,
CELERY_ROUTES = {'poc.main.proj.tasks': {'queue': 'proj_tasks'}}
)
app.control.rate_limit('poc.main.proj.tasks.get', '30/m')
app.control.rate_limit('poc.main.proj.tasks.compute', '30/m')
if __name__ == '__main__':
app.start()
celery worker code: tasks.py
from __future__ import absolute_import
from poc.celery.celery_worker_base import app
#app.task
def get(url):
print "calling get"
#app.task
def compute(info):
print "calling compute"
Another module: celery_master.py
from __future__ import absolute_import
from celery import Celery
from datetime import timedelta
from poc.config.config import *
from boto import ec2
master_app = Celery('poc',
backend='mongodb://user:pass#ip:27017/collection',
broker='amqp://user:pass#ip/vhost',
include=['poc.main.proj.tasks'])
# Optional configuration, see the application user guide.
master_app.conf.update(
CELERY_TASK_RESULT_EXPIRES=3600,
CELERYBEAT_SCHEDULE = {
'instance-check-every-fifteen-minute': {
'task': 'poc.main.instance.check.check_count',
'schedule': timedelta(seconds=900),
'options': {'queue' : 'instance_check'}
}
},
CELERY_ROUTES = {'poc.main.instance.check': {'queue': 'instance_check'}},
CELERY_TIMEZONE = 'UTC'
)
region = ec2.connect_to_region(
REGION,
aws_access_key_id=AWS_ACCESS_KEY,
aws_secret_access_key=AWS_SECRET_KEY
)
if __name__ == '__main__':
master_app.start()
master worker: check.py
from __future__ import absolute_import
from celery import Celery
from poc.config.config import *
from poc.celery.celery_master import master_app, region
#master_app.task
def check_count():
print "calling check"
PS: thanks for not down-voting the question.

Regarding celery not being able to find the task, I would ensure you are passing them according to how they are listed by app.control.app.tasks.
This provides a dict of known tasks, where the keys are what are eligible for passing to control.rate_limit().

Related

Flask Rest Api SQL Alchemy connection Cloud Sql Postgresq

I have a connection problem with Cloud Sql Postgres from my Flask Rest API app.
I have a db.py file:
import os
from flask_sqlalchemy import SQLAlchemy
import sqlalchemy
db = SQLAlchemy()
def connect_unix_socket() -> sqlalchemy.engine.base.Engine:
""" Initializes a Unix socket connection pool for a Cloud SQL instance of Postgres. """
# Note: Saving credentials in environment variables is convenient, but not
# secure - consider a more secure solution such as
# Cloud Secret Manager (https://cloud.google.com/secret-manager) to help
# keep secrets safe.
db_user = os.environ["DB_USER"] # e.g. 'my-database-user'
db_pass = os.environ["DB_PASS"] # e.g. 'my-database-password'
db_name = os.environ["DB_NAME"] # e.g. 'my-database'
unix_socket_path = os.environ["INSTANCE_UNIX_SOCKET"] # e.g. '/cloudsql/project:region:instance'
pool = sqlalchemy.create_engine(
# Equivalent URL:
# postgresql+pg8000://<db_user>:<db_pass>#/<db_name>
# ?unix_sock=<INSTANCE_UNIX_SOCKET>/.s.PGSQL.5432
# Note: Some drivers require the `unix_sock` query parameter to use a different key.
# For example, 'psycopg2' uses the path set to `host` in order to connect successfully.
sqlalchemy.engine.url.URL.create(
drivername="postgresql+pg8000",
username=db_user,
password=db_pass,
database=db_name,
query={"unix_sock": "{}/.s.PGSQL.5432".format(unix_socket_path)},
),
# [START_EXCLUDE]
# Pool size is the maximum number of permanent connections to keep.
pool_size=5,
# Temporarily exceeds the set pool_size if no connections are available.
max_overflow=2,
# The total number of concurrent connections for your application will be
# a total of pool_size and max_overflow.
# 'pool_timeout' is the maximum number of seconds to wait when retrieving a
# new connection from the pool. After the specified amount of time, an
# exception will be thrown.
pool_timeout=30, # 30 seconds
# 'pool_recycle' is the maximum number of seconds a connection can persist.
# Connections that live longer than the specified amount of time will be
# re-established
pool_recycle=1800, # 30 minutes
# [END_EXCLUDE]
)
return pool
I import the db.py file in my app.py file:
import os
import sqlalchemy
from flask import Flask
from flask_smorest import Api
from flask_sqlalchemy import SQLAlchemy
from db import db, connect_unix_socket
import models
from resources.user import blp as UserBlueprint
# pylint: disable=C0103
app = Flask(__name__)
def init_connection_pool() -> sqlalchemy.engine.base.Engine:
# use a Unix socket when INSTANCE_UNIX_SOCKET (e.g. /cloudsql/project:region:instance) is defined
if unix_socket_path:
return connect_unix_socket()
raise ValueError(
"Missing database connection type. Please define one of INSTANCE_HOST, INSTANCE_UNIX_SOCKET, or INSTANCE_CONNECTION_NAME"
)
db = None
#app.before_first_request
def init_db() -> sqlalchemy.engine.base.Engine:
global db
db = init_connection_pool()
api = Api(app)
#app.route("/api")
def user_route():
return "Welcome user API!"
api.register_blueprint(UserBlueprint)
if __name__ == '__main__':
server_port = os.environ.get('PORT', '8080')
app.run(debug=True, port=server_port, host='0.0.0.0')
The app run correctly, when i call the end point to Get or Post users, the app crash and give me this error:
"The current Flask app is not registered with this 'SQLAlchemy'"
RuntimeError: The current Flask app is not registered with this 'SQLAlchemy' instance. Did you forget to call 'init_app', or did you create multiple 'SQLAlchemy' instances?
This is my User.py class:
from sqlalchemy.exc import SQLAlchemyError, IntegrityError
from db import db
from models import UserModel
from schemas import UserSchema
blp = Blueprint("Users", "users", description="Operations on users")
#blp.route("/user/<string:user_id>")
class User(MethodView):
#blp.response(200, UserSchema)
def get(self, user_id):
user = UserModel.query.get_or_404(user_id)
return user
def delete(self, user_id):
user = UserModel.query.get_or_404(user_id)
db.session.delete(user)
db.session.commit()
return {"message": "User deleted"}, 200
#blp.route("/user")
class UserList(MethodView):
#blp.response(200, UserSchema(many=True))
def get(self):
return UserModel.query.all()
How i can fix this issue?
#dev_ Your issue is that your are trying to intermingle the use of SQLAlchemy Core with SQLAlchemy ORM as if they are the same thing, leading to your issues. SQLAlchemy connection pools created using sqlalchemy.create_engine use the CORE API while Flask-SQLAlchemy uses the SQLAlchemy ORM model. This is the core reason for you issue. It is easier to use one or the other.
I would recommend using purely Flask-SQLALchemy with the use of the cloud-sql-python-connector library for your use-case. It will make your life much easier.
For simplicity, I am getting rid of your db.py leading to your app.py file being as follows:
from flask import Flask
from flask_smorest import Api
from flask_sqlalchemy import SQLAlchemy
from google.cloud.sql.connector import Connector, IPTypes
from resources.user import blp as UserBlueprint
# load env vars
db_user = os.environ["DB_USER"] # e.g. 'my-database-user'
db_pass = os.environ["DB_PASS"] # e.g. 'my-database-password'
db_name = os.environ["DB_NAME"] # e.g. 'my-database'
instance_connection_name = os.environ["INSTANCE_CONNECTION_NAME"] # e.g. 'project:region:instance'
# Python Connector database connection function
def getconn():
with Connector() as connector:
conn = connector.connect(
instance_connection_name, # Cloud SQL Instance Connection Name
"pg8000",
user=db_user,
password=db_pass,
db=db_name,
ip_type= IPTypes.PUBLIC # IPTypes.PRIVATE for private IP
)
return conn
app = Flask(__name__)
# configure Flask-SQLAlchemy to use Python Connector
app.config['SQLALCHEMY_DATABASE_URI'] = "postgresql+pg8000://"
app.config['SQLALCHEMY_ENGINE_OPTIONS'] = {
"creator": getconn
}
# initialize db (using app!)
db = SQLAlchemy(app)
# rest of your code
api = Api(app)
# ...
Hope this helps resolve your issue!

Celery beat stuck on start

After celery launch, I have the following output:
[2022-12-24 17:42:25,851: INFO/MainProcess] Connected to redis://localhost:6379//
[2022-12-24 17:42:25,854: INFO/MainProcess] mingle: searching for neighbors
[2022-12-24 17:42:26,506: INFO/Beat] beat: Starting...
[2022-12-24 17:42:26,862: INFO/MainProcess] mingle: all alone
[2022-12-24 17:42:26,881: INFO/MainProcess] celery#Bulrathi-Mac-mini.local ready.
implying beat is stuck on start (indeed, periodic tasks are not executed).
I'm starting the celery like this:
celery -A app.celery worker -B -l info
My code is
from datetime import timedelta
from celery import Celery
from celery.utils.log import get_task_logger
celery = Celery(
__name__,
broker='redis://localhost:6379',
include=['tasks']
)
celery.conf.timezone = 'UTC'
logger = get_task_logger(__name__)
#celery.task
def periodic_task():
logger.debug('yay')
CELERYBEAT_SCHEDULE = {
'every-second': {
'task': 'periodic_task',
'schedule': timedelta(seconds=1),
}
}
I googled for the similar issues, but didn't find any suitable solution. Your help is highly appreciated, thank you in advance.

Dynamically change celery beat schedule params

I get schedule values from .env file. And sometimes parameters in .env file change.
Is it possible to change schedule values of already running celery beat tasks?
My celery.py:
import os
from celery import Celery
from celery.schedules import crontab
from dotenv import load_dotenv
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'myproj.settings')
app = Celery('myproj')
app.config_from_object('django.conf:settings', namespace='CELERY')
app.autodiscover_tasks()
load_dotenv()
orders_update_time = float(os.getenv("ORDERS_UPDATE_TIME"))
if not orders_update_time:
orders_update_time = 60.0
orders_update_time = float(os.getenv("REMAINS_SEND_TIME"))
if not remains_send_time:
remains_send_time = 60.0
app.conf.beat_schedule = {
'wb_orders_autosaver': {
'task': 'myapp.tasks.orders_autosave',
'schedule': orders_update_time,
},
'wb_remains_autosender': {
'task': 'myapp.tasks.remains_autosend',
'schedule': remains_send_time,
},
}
Yes, use django-celery-beat. That will allow you to save your schedule to the database and you can use the django admin ui to modify the schedule.
From django shell_plus, you can run the following commands to create your schedule:
schedule = CrontabSchedule(minute='0', hour='10')
schedule.save()
PeriodicTask.objects.create(
crontab=schedule,
task='myapp.tasks.orders_autosave',
name='autosave orders',
)
schedule = CrontabSchedule(minute='15', hour='10')
schedule.save()
PeriodicTask.objects.create(
crontab=schedule,
task='myapp.tasks.remains_autosend',
name='autosend remains',
)
PeriodicTasks.changed()
Or you can use the UI in the django admin panel:
Select add periodic task
Enter in the information about your task and select save

Airflow task not running on schedule with PrestoDB Query

I have defined a Airflow sample task where I wanted to run a PrestoDB Query followed by a Spark job to perform a simple word count example. Here is the DAG I defined:
from pandas import DataFrame
import logging
from datetime import timedelta
from operator import add
import airflow
from airflow import DAG
from airflow.operators.python_operator import PythonOperator
from airflow.hooks.presto_hook import PrestoHook
default_args = {
'owner': 'airflow',
'start_date': airflow.utils.dates.days_ago(1),
'depends_on_past': False,
'email': ['airflow#example.com'],
'email_on_failure': False,
'email_on_retry': False,
'retries': 1,
'retry_delay': timedelta(minutes=5),
}
dag = DAG(
'presto_dag',
default_args=default_args,
description='A simple tutorial DAG with PrestoDB and Spark',
# Continue to run DAG once per hour
schedule_interval='#daily',
)
def talk_to_presto():
ph = PrestoHook(host='presto.myhost.com', port=9988)
# Query PrestoDB
query = "show catalogs"
# Fetch Data
data = ph.get_records(query)
logging.info(data)
return data
def submit_to_spark():
# conf = SparkConf().setAppName("PySpark App").setMaster("http://sparkhost.com:18080/")
# sc = SparkContext(conf)
# data = sc.parallelize(list("Hello World"))
# counts = data.map(lambda x: (x, 1)).reduceByKey(add).sortBy(lambda x: x[1], ascending=False).collect()
# for (word, count) in counts:
# print("{}: {}".format(word, count))
# sc.stop()
return "Hello"
presto_task = PythonOperator(
task_id='talk_to_presto',
provide_context=True,
python_callable=talk_to_presto,
dag=dag,
)
spark_task = PythonOperator(
task_id='submit_to_spark',
provide_context=True,
python_callable=submit_to_spark,
dag=dag,
)
presto_task >> spark_task
When I submit the task, about 20 DAG instances stay in the running state:
But it never completes and no logs are generated, at least for the PrestoDB Query. I am able to run the same PrestoDB Query from the Airflow's Data Profiling > Ad-Hoc Query section correctly.
I have intentionally commented out the PySpark code as it wasn't running and not of focus in the question.
I have two questions:
Why aren't the tasks completed and stays in the running state?
What am I doing wrong with the PrestoHook as the query isn't running?

How to restart a dag when it fails on airflow 1.8?

With:
default_args = {
...
    'retries': 1,
'retry_delay': timedelta (seconds = 1),
...
}
I can get the task that fails to retry several times, but how can I get it when a task fails, the DAG starts again?
Of course, automatically...
You can run a second "Fail Check" DAG that queries for any task instances where the task_id matches what you want and the state is failed using the provide_session util. Then, you'll want to optionally clear downstream tasks as well and set the state of the relevant DagRun to running.
from datetime import datetime, timedelta
from sqlalchemy import and_
import json
from airflow import DAG
from airflow.models import TaskInstance, DagRun
from airflow.utils.db import provide_session
from airflow.operators.python_operator import PythonOperator
default_args = {'start_date': datetime(2018, 6, 11),
'retries': 2,
'retry_delay': timedelta(minutes=2),
'email': [],
'email_on_failure': True}
dag = DAG('__RESET__FAILED_TASKS',
default_args=default_args,
schedule_interval='#daily',
catchup=False
)
#provide_session
def check_py(session=None, **kwargs):
relevant_task_id = 'relevant_task_id'
obj = (session
.query(TaskInstance)
.filter(and_(TaskInstance.task_id == relevant_task_id,
TaskInstance.state == 'failed'))
.all())
if obj is None:
raise KeyError('No failed Task Instances of {} exist.'.format(relevant_task_id))
else:
# Clear the relevant tasks.
(session
.query(TaskInstance)
.filter(and_(TaskInstance.task_id == relevant_task_id,
TaskInstance.state == 'failed'))
.delete())
# Clear downstream tasks and set relevant DAG state to RUNNING
for _ in obj:
_ = json.loads(_.val)
# OPTIONAL: Clear downstream tasks in the specified Dag Run.
for task in _['downstream_tasks']:
(session
.query(TaskInstance)
.filter(and_(TaskInstance.task_id == task,
TaskInstance.dag_id == _['dag_id'],
TaskInstance.execution_date == datetime.strptime(_['ts'],
"%Y-%m-%dT%H:%M:%S")))
.delete())
# Set the Dag Run state to "running"
dag_run = (session
.query(DagRun)
.filter(and_(DagRun.dag_id == _['dag_id'],
DagRun.execution_date == datetime.strptime(_['ts'],
"%Y-%m-%dT%H:%M:%S")))
.first())
dag_run.set_state('running')
with dag:
run_check = PythonOperator(task_id='run_check',
python_callable=check_py,
provide_context=True)
run_check
The canonical solution to this in Airflow is to create a subdagoperator that wraps all the other tasks in the dag, and apply the retry to that.
You could potentially use the on_failure_callback feature to call a python / bash script that would restart the DAG. There is not currently a feature provided by Airflow to automatically restart the DAG upon task failure.