how to init arguments in User class - locust

I create a User class with some init arguments, but got TypeError as below when executed :"locust -f stress.py --headless -u 100 -r 100 -t 1m"
Traceback (most recent call last):
File "src/gevent/greenlet.py", line 906, in gevent._gevent_cgreenlet.Greenlet.run
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/locust/runners.py", line 406, in
lambda: super(LocalRunner, self).start(user_count, spawn_rate, wait=wait)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/locust/runners.py", line 314, in start
self.spawn_users(user_count, spawn_rate=spawn_rate, wait=wait)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/locust/runners.py", line 206, in spawn_users
spawn()
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/locust/runners.py", line 199, in spawn
new_user = user_class(self.environment)
TypeError: init() missing 4 required positional arguments: 'uid', 'room_id', 'action_uid', and 'appkey'
class JoinRoomUser(HttpUser):
wait_time = between(1, 2.5)
host = ''
def __init__(self, env, uid, room_id, action_uid, appkey):
self.env = env
self.uid = uid
self.room_id =room_id
self.action_id = action_uid
self.appkey = appkey
how to init these arguments?
the task in locust script as below.
#task
def join_room(self):
host = 'xxxxxxx'
body = {
"room_id": self.room_id,
"userid": self.action_uid,
"uid": self.action_uid
}
rsp = self.client.post(url, json=body, headers=get_header(self.appkey))
if I exectue "locust -f StressTest/stress.py 'Qa' 123456 '101013' 686868 'eos' --headless -u 100 -r 100 --run-time 1m" , then throw " bogon/ERROR/locust.main: Unknown User(s): Qa, 686868, 123456, 101013, eos" error.

Try use *args, **kwargs and super() when inherit from Parent class. For that particular case, you don't provide uid, room_id, action_uid, appkey variables on the step of initialization, because HttpUser inherited from User, which has only one 1 argument - environment
class SimpleSendRequest(User):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
for User class is more suggested to inherit from User class doc
if you really required to set variables to daughter object, you can set it on the step of initialization, just like that:
def __init__(self, env, *args, **kwargs):
super().__init__(env, *args, **kwargs)
self.uid = your_map['uid']
self.room_id = your_map['room_id']
self.action_id = your_map['action_uid']
self.appkey = your_map['appkey']
the only question, how are you going to use them
to add custom arguments, via CLI check github:
#events.init_command_line_parser.add_listener
def init_parser(parser):
parser.add_argument("--customarg", type=str, env_var="LOCUST_MY_ARGUMENT", default="1234", help="It's working")
#events.init.add_listener
def _(environment, **kw):
print("Custom argument supplied: %s" % environment.parsed_options.customarg)
to get it inside your class JoinRoomUser init, use:
print(self.environment.parsed_options.customarg)
and run like:
locust -f locustfile.py --customarg CUSTOM_VALUE --headless -u 10 -r 10 --run-time 5s

Related

Adjust wait_time in locust using custom arguments via CLI

I want to adjust wait_time parameter by passing it via CLI.
I have tried the following way:
custom_wait_time = None
# Add custom argument to locust
#events.init_command_line_parser.add_listener
def init_parser(parser):
parser.add_argument("--locust-wait-time",
type=int,
include_in_web_ui=True,
default=None,
help="Wait time per each request of a user.")
#events.init.add_listener
def _(environment, **kwargs):
global custom_wait_time
custom_wait_time = int(environment.parsed_options.locust_wait_time)
print(custom_wait_time) # First print
class MyUser(HttpUser):
global custom_wait_time
print(custom_wait_time) # Second print
wait_time = constant(custom_wait_time)
Assume that custom_wait_time=10 when I pass it via CLI, the First print gives me custom_wait_time=10 while the Second print gives me custom_wait_time=None instead of 10, so the wait_time = constant(custom_wait_time) will break and give me the error below:
Traceback (most recent call last):
File "src/gevent/greenlet.py", line 908, in gevent._gevent_cgreenlet.Greenlet.run
File "/Users/opt/anaconda3/envs/ai/lib/python3.7/site-packages/locust/user/users.py", line 176, in run_user
user.run()
File "/Users/opt/anaconda3/envs/ai/lib/python3.7/site-packages/locust/user/users.py", line 144, in run
self._taskset_instance.run()
File "/Users/opt/anaconda3/envs/ai/lib/python3.7/site-packages/locust/user/task.py", line 367, in run
self.wait()
File "/Users/opt/anaconda3/envs/ai/lib/python3.7/site-packages/locust/user/task.py", line 445, in wait
self._sleep(self.wait_time())
File "/Users/opt/anaconda3/envs/ai/lib/python3.7/site-packages/locust/user/task.py", line 451, in _sleep
gevent.sleep(seconds)
File "/Users/opt/anaconda3/envs/ai/lib/python3.7/site-packages/gevent/hub.py", line 157, in sleep
if seconds <= 0:
TypeError: '<=' not supported between instances of 'NoneType' and 'int'
Any help would be appreciated.
The problem is that the code will run in the wrong order - MyUser is defined before any of the init-methods are called.
If you instead do MyUser.wait_time = constant(custom_wait_time) inside your init handler (and dont set it at all in the class) it should work.
That way you dont need any globals either :)
I've just do the same work.
#events.init_command_line_parser.add_listener
def _(parser):
parser.add_argument("--waitTime", type=float, env_var="LOCUST_WAIT_TIME", default=1.0, help="wait time between each task of an user")
# Set `include_in_web_ui` to False if you want to hide from the web UI
#parser.add_argument("--my-ui-invisible-argument", include_in_web_ui=False, default="I am invisible")
and in the test class, just use it value like this
class GetInfoUser(HttpUser):
def wait_time(self):
return self.environment.parsed_options.waitTime

Chaining an iterable task result to a group, then chaining the results of this group to a chord

Related: How to chain a Celery task that returns a list into a group? and Chain a celery task's results into a distributed group, but they lack a MWE, to my understanding do not fully cover my needs, and are maybe outdated.
I need to dynamically pass the output of a task to a parallel group of tasks, then pass the results of this group to a final task. I would like to make this a single "meta task".
Here's my attempt so far:
# chaintasks.py
import random
from typing import List, Iterable, Callable
from celery import Celery, signature, group, chord
app = Celery(
"chaintasks",
backend="redis://localhost:6379",
broker="pyamqp://guest#localhost//",
)
app.conf.update(task_track_started=True, result_persistent=True)
#app.task
def select_items(items: List[str]):
print("In select_items, received", items)
selection = tuple(set(random.choices(items, k=random.randint(1, len(items)))))
print("In select_items, sending", selection)
return selection
#app.task
def process_item(item: str):
print("In process_item, received", item)
return item + "_processed"
#app.task
def group_items(items: List[str]):
print("In group_items, received", items)
return ":".join(items)
#app.task
def dynamic_map_group(iterable: Iterable, callback: Callable):
# Map a callback over an iterator and return as a group
# Credit to https://stackoverflow.com/a/13569873/5902284
callback = signature(callback)
return group(callback.clone((arg,)) for arg in iterable).delay()
#app.task
def dynamic_map_chord(iterable: Iterable, callback: Callable):
callback = signature(callback)
return chord(callback.clone((arg,)) for arg in iterable).delay()
Now, to try this out, let's spin up a celery worker and use docker to for rabbitmq and redis
$ docker run -p 5672:5672 rabbitmq
$ docker run -p 6379:6379 redis
$ celery -A chaintasks:app worker -E
Now, let's try to illustrate what I need:
items = ["item1", "item2", "item3", "item4"]
print(select_items.s(items).apply_async().get())
This outputs: ['item3'], great.
meta_task1 = select_items.s(items) | dynamic_map_group.s(process_item.s())
meta_task1_result = meta_task1.apply_async().get()
print(meta_task1_result)
It gets trickier here: the output is [['b916f5d3-4367-46c5-896f-f2d2e2e59a00', None], [[['3f78d31b-e374-484e-b05b-40c7a2038081', None], None], [['bce50d49-466a-43e9-b6ad-1b78b973b50f', None], None]]].
I am not sure what the Nones, mean, but I guess the UUIDs represents the subtasks of my meta task. But how do I get their results from here?
I tried:
subtasks_ids = [i[0][0] for i in meta_task1_result[1]]
processed_items = [app.AsyncResult(i).get() for i in subtasks_ids]
but this just hangs. What am I doing wrong here? I'm expecting an output looking like ['processed_item1', 'processed_item3']
What about trying to chord the output of select_items to group_items?
meta_task2 = select_items.s(items) | dynamic_map_chord.s(group_items.s())
print(meta_task2.apply_async().get())
Ouch, this raises an AttributeError that I don't really understand.
Traceback (most recent call last):
File "chaintasks.py", line 70, in <module>
print(meta_task2.apply_async().get())
File "/site-packages/celery/result.py", line 223, in get
return self.backend.wait_for_pending(
File "/site-packages/celery/backends/asynchronous.py", line 201, in wait_for_pending
return result.maybe_throw(callback=callback, propagate=propagate)
File "/site-packages/celery/result.py", line 335, in maybe_throw
self.throw(value, self._to_remote_traceback(tb))
File "/site-packages/celery/result.py", line 328, in throw
self.on_ready.throw(*args, **kwargs)
File "/site-packages/vine/promises.py", line 234, in throw
reraise(type(exc), exc, tb)
File "/site-packages/vine/utils.py", line 30, in reraise
raise value
AttributeError: 'NoneType' object has no attribute 'clone'
Process finished with exit code 1
And finally, what I really am trying to do is something like:
ultimate_meta_task = (
select_items.s(items)
| dynamic_map_group.s(process_item.s())
| dynamic_map_chord.s(group_items.s())
)
print(ultimate_meta_task.apply_async().get())
but this leads to:
Traceback (most recent call last):
File "chaintasks.py", line 77, in <module>
print(ultimate_meta_task.apply_async().get())
File "/site-packages/celery/result.py", line 223, in get
return self.backend.wait_for_pending(
File "/site-packages/celery/backends/asynchronous.py", line 199, in wait_for_pending
for _ in self._wait_for_pending(result, **kwargs):
File "/site-packages/celery/backends/asynchronous.py", line 265, in _wait_for_pending
for _ in self.drain_events_until(
File "/site-packages/celery/backends/asynchronous.py", line 58, in drain_events_until
on_interval()
File "/site-packages/vine/promises.py", line 160, in __call__
return self.throw()
File "/site-packages/vine/promises.py", line 157, in __call__
retval = fun(*final_args, **final_kwargs)
File "/site-packages/celery/result.py", line 236, in _maybe_reraise_parent_error
node.maybe_throw()
File "/site-packages/celery/result.py", line 335, in maybe_throw
self.throw(value, self._to_remote_traceback(tb))
File "/site-packages/celery/result.py", line 328, in throw
self.on_ready.throw(*args, **kwargs)
File "/site-packages/vine/promises.py", line 234, in throw
reraise(type(exc), exc, tb)
File "/site-packages/vine/utils.py", line 30, in reraise
raise value
kombu.exceptions.EncodeError: TypeError('Object of type GroupResult is not JSON serializable')
Process finished with exit code 1
What I try to achieve here, is to get an output looking like 'processed_item1:processed_item3'. Is this even doable using celery? Any help here is appreciated.

ERROR: Uncaught exception in luigi (TypeError: must be string or buffer, not None)

I am having trouble while calling /triggering Luigi Task from a python code.
Basically i need to trigger a luigi task just like we do on command line, but from a python code
I am using supbrocess.popen to call a luigi task using a shell
command
I have a test code named as test.py and have a test class in module
task_scheduler.py which contains my luigi task (both modules in same location/dir)
import luigi
class TestClass(luigi.Task):
# param = luigi.DictParameter(default=dict())
def requires(self):
print "I am TestClass req"
def run(self):
with open('myfile.txt', 'w') as f:
f.write("asasasas")
print "I am TestClass run"
import subprocess
p = subprocess.Popen("python -m luigi --module task_scheduler TestClass", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
print p.pid
(output, err) = p.communicate()
print "-------------O/P-------------"
print output
print "-------------error-------------"
print err
But I am getting the error as
52688
-------------O/P-------------
-------------error-------------
ERROR: Uncaught exception in luigi
Traceback (most recent call last):
File "/Library/Python/2.7/site-packages/luigi/retcodes.py", line 61, in run_with_retcodes
worker = luigi.interface._run(argv)['worker']
File "/Library/Python/2.7/site-packages/luigi/interface.py", line 238, in _run
return _schedule_and_run([cp.get_task_obj()], worker_scheduler_factory)
File "/Library/Python/2.7/site-packages/luigi/interface.py", line 172, in _schedule_and_run
not(lock.acquire_for(env_params.lock_pid_dir, env_params.lock_size, kill_signal))):
File "/Library/Python/2.7/site-packages/luigi/lock.py", line 82, in acquire_for
my_pid, my_cmd, pid_file = get_info(pid_dir)
File "/Library/Python/2.7/site-packages/luigi/lock.py", line 67, in get_info
pid_file = os.path.join(pid_dir, hashlib.md5(cmd_hash).hexdigest()) + '.pid'
TypeError: must be string or buffer, not None
Can anyone please suggest me what I am doing wrong here?
The command "python -m luigi --module task_scheduler TestClass" works perfectly if I use shell prompt
I just tried executing the test.py from command line.
It seems when I was using the IDE to run (PyCharm) then only it gives this issue
This is fixed in version 2.2.0. Check github issue #1654

Made Locust to login to a Web Application

I want locust to be able to login to my web application and start to click in the links inside the web application.
With this code I just get activity for the front page with the login and i don't get any notification from inside the application.
Code:
import random
from locust import HttpLocust, TaskSet, task
from pyquery import PyQuery
class WalkPages(TaskSet):
def on_start(self):
self.client.post("/", {
"UserName": "my#email.com",
"Password": "2Password!",
"submit": "Sign In"
})
self.index_page()
#task(10)
def index_page(self):
r = self.client.get("/Dashboard.mvc")
pq = PyQuery(r.content)
link_elements = pq("a")
self.urls_on_current_page = []
for l in link_elements:
if "href" in l.attrib:
self.urls_on_current_page.append(l.attrib["href"])
#task(30)
def load_page(self):
url = random.choice(self.urls_on_current_page)
r = self.client.get(url)
class AwesomeUser(HttpLocust):
task_set = WalkPages
host = "https://myenv.beta.webapp.com"
min_wait = 20 * 1000
max_wait = 60 * 1000
I get the follow msg in the terminal after the first round.
[2015-02-13 12:08:43,740] webapp-qa/ERROR/stderr: Traceback (most recent call last):
File "/usr/local/lib/python2.7/dist-packages/locust/core.py", line 267, in run
self.execute_next_task()
File "/usr/local/lib/python2.7/dist-packages/locust/core.py", line 293, in execute_next_task
self.execute_task(task["callable"], *task["args"], **task["kwargs"])
File "/usr/local/lib/python2.7/dist-packages/locust/core.py", line 305, in execute_task
task(self, *args, **kwargs)
File "/home/webapp/LoadTest/locustfile.py", line 31, in load_page
url = random.choice(self.urls_on_current_page)
File "/usr/lib/python2.7/random.py", line 273, in choice
return seq[int(self.random() * len(seq))] # raises IndexError if seq is empty
IndexError: list index out of range
[2015-02-13 12:08:43,752] webapp-qa/ERROR/stderr: Traceback (most recent call last):
File "/usr/local/lib/python2.7/dist-packages/locust/core.py", line 267, in run
self.execute_next_task()
File "/usr/local/lib/python2.7/dist-packages/locust/core.py", line 293, in execute_next_task
self.execute_task(task["callable"], *task["args"], **task["kwargs"])
File "/usr/local/lib/python2.7/dist-packages/locust/core.py", line 305, in execute_task
task(self, *args, **kwargs)
File "/home/webapp/LoadTest/locustfile.py", line 31, in load_page
url = random.choice(self.urls_on_current_page)
File "/usr/lib/python2.7/random.py", line 273, in choice
return seq[int(self.random() * len(seq))] # raises IndexError if seq is empty
IndexError: list index out of range
[2015-02-13 12:08:43,775] webapp-qa/ERROR/stderr: Traceback (most recent call last):
File "/usr/local/lib/python2.7/dist-packages/locust/core.py", line 267, in run
self.execute_next_task()
File "/usr/local/lib/python2.7/dist-packages/locust/core.py", line 293, in execute_next_task
self.execute_task(task["callable"], *task["args"], **task["kwargs"])
File "/usr/local/lib/python2.7/dist-packages/locust/core.py", line 305, in execute_task
task(self, *args, **kwargs)
File "/home/webapp/LoadTest/locustfile.py", line 31, in load_page
url = random.choice(self.urls_on_current_page)
File "/usr/lib/python2.7/random.py", line 273, in choice
return seq[int(self.random() * len(seq))] # raises IndexError if seq is empty
IndexError: list index out of range
Your list may be empty.
#task(30)
def load_page(self):
if self.urls_on_current_page:
url = random.choice(self.urls_on_current_page)
r = self.client.get(url)
I takes time but someone may need this. My findings in your code: login requests seems not correct (check mine if correct), you cannot reach a variable defined inside of a function from another function, giving task(10) is not suitable for data setter function. Set urls_on_current_page as a class variable to serve for other class members. See my code and comment:
import random
from locust import HttpLocust, TaskSet, task
from pyquery import PyQuery
class WalkPages(TaskSet):
# define variable here to access them from inside the functions
urls_on_current_page = []
def login(self):
self.client.post("/login", data = {"UserName": "mesutgunes#email.com", "Password": "password"})
def get_urls(self):
r = self.client.get("/Dashboard.mvc")
pq = PyQuery(r.content)
link_elements = pq("a")
for link in link_elements:
if key in link.attrib and "http" not in link.attrib[key]:
# there maybe external link on the page
self.urls_on_current_page.append(link.attrib[key])
def on_start(self):
self.login()
self.get_urls()
#task(30)
def load_page(self):
url = random.choice(self.urls_on_current_page)
r = self.client.get(url)
class AwesomeUser(HttpLocust):
task_set = WalkPages
host = "https://myenv.beta.webapp.com"
min_wait = 20 * 1000
max_wait = 60 * 1000

How to plug txyam on top of tornado IOLoop

I 've tried to use tornado.platform.twisted to integrate txyam memcached client, but when I try to check it for functioning, next error is thrown:
Traceback (most recent call last):
File "swcomet/tx_memcache_helper.py", line 32, in <module>
mem_helper = MemcacheHelper()
File "swcomet/tx_memcache_helper.py", line 19, in __init__
self.add(4)
File "/home/rustem/work/sw.services.swcomet.python/venv/local/lib/python2.7/site-packages/tornado/gen.py", line 117, in wrapper
gen = func(*args, **kwargs)
File "swcomet/tx_memcache_helper.py", line 25, in add
self.mem.getPickled(user_id, decompress=True)
File "/home/rustem/work/sw.services.swcomet.python/venv/lib/python2.7/site-packages/txyam/client.py", line 133, in getPickled
return self.get(key, **kwargs).addCallback(handleResult, uncompress)
File "/home/rustem/work/sw.services.swcomet.python/venv/lib/python2.7/site-packages/txyam/client.py", line 27, in wrapper
func = getattr(self.getClient(key), cmd)
File "/home/rustem/work/sw.services.swcomet.python/venv/lib/python2.7/site-packages/txyam/client.py", line 48, in getClient
raise NoServerError, "No connected servers remaining."
txyam.client.NoServerError: No connected servers remaining.
The source code which dumps that error:
import tornado.ioloop
import tornado.gen
from txyam.client import YamClient
from swtools.date import _ts
import tornado.platform.twisted
MEMHOSTS = ['127.0.0.1111']
USER_EXPIRATION_TIME = 61
class MemcacheHelper(object):
def __init__(self, *a, **kw):
try:
self.mem = YamClient(["127.0.0.1"])
except Exception, e:
print "ERror", e
self.clients = set()
self.add(4)
#tornado.gen.engine
def add(self, user_id, expire=None):
self.clients.add(user_id)
expire = expire or USER_EXPIRATION_TIME
self.mem.getPickled(user_id, decompress=True)
print "hmmm"
if __name__ == '__main__':
print "trying to start on top of IOLOOP"
ioloop = tornado.ioloop.IOLoop.instance()
#reactor = TornadoReactor(ioloop)
mem_helper = MemcacheHelper()
#mem_helper.add(4)
ioloop.start()
Please, help me to solve this problem!
txyam appears not to let you perform any memcache operations until after at least one connection has been established:
def getActiveConnections(self):
return [factory.client for factory in self.factories if not factory.client is None]
def getClient(self, key):
hosts = self.getActiveConnections()
log.msg("Using %i active hosts" % len(hosts))
if len(hosts) == 0:
raise NoServerError, "No connected servers remaining."
return hosts[ketama(key) % len(hosts)]
It attempts to set up these connections right away:
def __init__(self, hosts):
"""
#param hosts: A C{list} of C{tuple}s containing hosts and ports.
"""
self.connect(hosts)
But connection setup is asynchronous, and it doesn't expose an event to indicate when at least one connection has been established.
So your code fails because you call add right away, before any connections exist. A good long-term fix would be to file a bug report against txyam, because this isn't a very nice interface. YamClient could have a whenReady method that returns a Deferred that fires when you are actually allowed to use the YamClient instance. Or there could be an alternate constructor that returns a Deferred that fires with the YamClient instance, but only after it can be used.