I want to create Base class for all tasks in Celery my code is
tasks/all/test.py
from celery import app
from base.main import CeleryMain
#app.task(time_limit=10)
def test():
task = CeleryMain.delay()
return task
base/main.py
from celery import app
import requests
from celery import Celery, Task
class CeleryMain(app.Task):
abstract = True
def run(self, task):
data = task.apply_async()
s = data.get(timeout=10, interval=0.01)
return {'success': True, 'data': s}
task = CeleryMain()
app.register_task(task)
task.delay()
celery.py
imports = (
'tasks.all.test',
)
I have error:
celery.exceptions.NotRegistered: 'tasks.all.test'
Please help me on how to create main class for all tasks correctly without mistakes for each task Base class.
CeleryMain= app.register_task(CeleryMain())
task = CeleryMain.run(task)
print(task.id)
Related
I have the following code which run fine in locust 0.9. Now with 1.3 it throws the exception mentioned in the title. Can anyone see what's wrong?
import time
import random
import datetime
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
import logging
import json
import os
from random import randint, choice
from locust import HttpUser, TaskSet, task
from pyquery import PyQuery
requests.packages.urllib3.disable_warnings()
class FrontPage(TaskSet):
def on_start(self):
self.client.verify = False
#task(20)
def index(self):
self.client.get("/")
class DestinationPagesFixed(TaskSet):
de_paths = ["/belgien", "daenemark", "deutschland", "frankreich", "griechenland"
, "italien"
, "luxemburg"
]
def on_start(self):
self.client.verify = False
#task
def test_1(self):
paths = self.de_paths
path = choice(paths)
self.client.get(path, name="Static page")
class UserBehavior(TaskSet):
tasks = {FrontPage: 15, DestinationPagesFixed: 19}
class WebsiteUser(HttpUser):
task_set = UserBehavior
min_wait = 400
max_wait = 10000
Change
task_set = UserBehavior
to
tasks = [UserBehavior]
Or (skipping your UserBehaviour class entirely)
tasks = {FrontPage: 15, DestinationPagesFixed: 19}
I'm working on an application that has a couple of long-running streams going, where it subscribes to data about a certain entity and processes that data. These streams should be up 24/7, so we needed to handle failures (network issues etc).
For that purpose, we've wrapped our sources in RestartingSource.
I'm now trying to verify this behaviour, and while it looks like it functions, I'm struggling to create a test where I push in some data, verify that it processes correctly, then send an error, and verify that it reconnects after that and continues processing.
I've boiled that down to this minimal case:
import akka.actor.ActorSystem
import akka.stream.ActorMaterializer
import akka.stream.scaladsl.{RestartSource, Sink, Source}
import akka.stream.testkit.TestPublisher
import org.scalatest.concurrent.Eventually
import org.scalatest.{FlatSpec, Matchers}
import scala.concurrent.duration._
import scala.concurrent.ExecutionContext
class MinimalSpec extends FlatSpec with Matchers with Eventually {
"restarting a failed source" should "be testable" in {
implicit val sys: ActorSystem = ActorSystem("akka-grpc-measurements-for-test")
implicit val mat: ActorMaterializer = ActorMaterializer()
implicit val ec: ExecutionContext = sys.dispatcher
val probe = TestPublisher.probe[Int]()
val restartingSource = RestartSource
.onFailuresWithBackoff(1 second, 1 minute, 0d) { () => Source.fromPublisher(probe) }
var last: Int = 0
val sink = Sink.foreach { l: Int => last = l }
restartingSource.runWith(sink)
probe.sendNext(1)
eventually {
last shouldBe 1
}
probe.sendNext(2)
eventually {
last shouldBe 2
}
probe.sendError(new RuntimeException("boom"))
probe.expectSubscription()
probe.sendNext(3)
eventually {
last shouldBe 3
}
}
}
This test consistently fails on the last eventually block with Last failure message: 2 was not equal to 3. What am I missing here?
Edit: akka version is 2.5.31
I figured it out after having had a look at the TestPublisher code. Its subscription is a lazy val. So when RestartSource detects the error, and executes the factory method () => Source.fromPublisher(probe) again, it gets a new Source, but the subscription of the probe is still pointing to the old Source. Changing the code to initialize both a new Source and TestPublisher works.
I want to use Akka scheduler to do some cron jobs in my Play application.
Since Play 2.4 GlobalSettings is not recommended. Anyone has some sample code on how to do that?
Even I tried the simple code from Play docs and it still not working.
class CustomApplicationLoader extends GuiceApplicationLoader() {
val logger: Logger = Logger(this.getClass)
override def builder(context: ApplicationLoader.Context): GuiceApplicationBuilder = {
logger.info("start")
val extra = Configuration("a" -> 1)
initialBuilder
.in(context.environment)
.loadConfig(extra ++ context.initialConfiguration)
.overrides(overrides(context): _*)
}
}
play.application.loader = "com.xxx.CustomApplicationLoader"
Why I can't get the logging message to print?
How I can use Akka on Application start?
This is how do it in my app:
Start of by defining a trait Scheduled.
package scheduled
import akka.actor.Cancellable
trait Scheduled {
var cancellable: Option[Cancellable] = None
val defaultInterval = 60
val secondsToWait = {
import scala.concurrent.duration._
10 seconds
}
def start()
def stop() = {
cancellable.map(_.cancel())
}
}
Then implement Scheduled with Akka magic
package scheduled
import akka.actor.ActorSystem
import play.api.{Configuration, Logger}
import com.google.inject.{Singleton, Inject}
import play.api.libs.concurrent.Execution.Implicits._
trait ScheduledWorker extends Scheduled
#Singleton
class ScheduledWorkerImpl #Inject()(
actorSystem: ActorSystem,
configuration: Configuration
) extends ScheduledWorker {
start()
lazy val intervalKey = "worker.interval"
lazy val jobEnabled = "worker.enabled"
override def start(): Unit = {
import scala.concurrent.duration._
lazy val i = configuration.getInt(intervalKey).getOrElse(defaultInterval)
lazy val isEnabled = Option(System.getProperty(jobEnabled)).getOrElse(
configuration.getString(jobEnabled).getOrElse("false")
).equals("true")
cancellable = isEnabled match {
case true =>
Some(
actorSystem.scheduler.schedule(0 seconds, i minutes) {
.. MAJOR COOL CODE!!! ;))) ...
}
)
case _ => None
}
}
}
create a module to eagerly start the scheduled stuff
package modules
import play.api.{Configuration, Environment}
import play.api.inject.Module
import scheduled.{ScheduledWorker}
class ScheduledModule extends Module {
def bindings(environment: Environment,
configuration: Configuration) = Seq(
bind[ScheduledWorker].to[ScheduledWorkerImpl].eagerly()
)
}
make sure that your config specifies ScheduledModule.
play.modules.enabled += "modules.ScheduledModule"
And voila you have a working scheduled task when your play 2.4 app starts =)
I've recently started working in Scala and Play Framework and just upgraded a service I've been working on to Play 2.4.3. My end goal is to create a nightly process that starts up a service method in my play application for the purpose of scheduling events via a method call, which I'm currently calling with an Actor.
I had the basic idea of this working through a Global.scala file with an override onStart, but then I saw the play documentation about moving away from the use of GlobalSettings (https://www.playframework.com/documentation/2.4.x/GlobalSettings) and have been trying to move it to an injected dependency approach.
Here's what I've pieced together so far:
Module Code:
import javax.inject._
import com.myOrganization.myPackage.Actors.ScheduleActor
import play.api.libs.concurrent.AkkaGuiceSupport
import play.libs.Akka
import play.api.libs.concurrent.Execution.Implicits.defaultContext
import akka.actor.{ActorRef, ActorSystem}
import scala.concurrent.duration._
import play.Application
import com.google.inject.AbstractModule
#Singleton
class NightlyEvalSchedulerStartup #Inject()(system: ActorSystem, #Named("ScheduleActor") scheduleActor: ActorRef) {
Akka.system.scheduler.schedule(10.seconds, 20.seconds, scheduleActor, "ScheduleActor")
}
class ScheduleModule extends AbstractModule with AkkaGuiceSupport {
def configure() = {
bindActor[ScheduleActor]("ScheduleActor")
bind(classOf[NightlyEvalSchedulerStartup]).asEagerSingleton
}
}
Actor Class:
import akka.actor.{Actor, Props}
import com.myOrganization.myPackage.services.MySchedulingService
object ScheduleActor {
def props = Props[ScheduleActor]
class updateSchedules
}
class ScheduleActor extends Actor {
val MySchedulingService: MySchedulingService = new MySchedulingService
def receive = {
case "runScheduler" => MySchedulingService.nightlyScheduledUpdate()
}
}
Application.conf
play.modules.enabled += "com.myOrganization.myPackage.modules.ScheduleModule"
The service is calling down to a method that is primarily based on scala logic code and database interactions via Anorm.
Every time I try to start the service up with activator start (or run, once an Http request is received) I get the following error:
Oops, cannot start the server.
com.google.inject.CreationException: Unable to create injector, see the following errors:
1) Error injecting constructor, java.lang.RuntimeException: There is no started application
I've tried running the same code by replacing the Aka.system.scheduler... piece with a simple println() and everything seemed to work fine, meaning the service started up and I saw my message on the console. So I'm guessing there's some dependency that I'm missing for the Akka scheduler that is causing it to blowup. Any suggestions you can offer would be great, I've been banging my head against this all day.
EDIT (Solved Code Per Request):
Module Code, with some added code for getting a rough estimation of 3am the next night. This might change down the line, but it works for now:
package com.myOrganization.performanceManagement.modules
import com.myOrganization.performanceManagement.Actors.ScheduleActor
import com.myOrganization.performanceManagement.Actors.ScheduleActor.nightlySchedule
import org.joda.time.{Seconds, LocalDate, LocalTime, LocalDateTime}
import play.api.libs.concurrent.AkkaGuiceSupport
import play.api.libs.concurrent.Execution.Implicits.defaultContext
import akka.actor.{ActorRef, ActorSystem}
import scala.concurrent.duration.{FiniteDuration, SECONDS, HOURS }
import org.joda.time._
import com.google.inject.{Inject, Singleton, AbstractModule}
import com.google.inject.name.Named
class ScheduleModule extends AbstractModule with AkkaGuiceSupport {
override def configure() = {
bindActor[ScheduleActor]("ScheduleActor")
bind(classOf[NightlyEvalSchedulerStartup]).asEagerSingleton()
}
}
#Singleton
class NightlyEvalSchedulerStartup #Inject()(system: ActorSystem, #Named("ScheduleActor") scheduleActor: ActorRef) {
//Calculate initial delay to 3am the next day.
val currentTime: DateTime = DateTime.now
val targetDateTime = currentTime.plusDays(1).withTimeAtStartOfDay()
//Account for Daylight savings to an extent, not mandatory that it starts at 3am, just after midnight.
val initialDelaySeconds = targetDateTime.getHourOfDay match {
case 0 => new Duration(currentTime, targetDateTime.plusHours(3)).getStandardSeconds
case 1 => new Duration(currentTime, targetDateTime.plusHours(2)).getStandardSeconds
}
//Schedule first actor firing to occur at calculated delay and then every 24 hours.
system.scheduler.schedule(FiniteDuration(initialDelaySeconds, SECONDS), FiniteDuration(24, HOURS), scheduleActor, nightlySchedule)
}
Actor:
package com.myOrganization.performanceManagement.Actors
import akka.actor.{ActorSystem, Actor}
import com.google.inject.Inject
import com.myOrganization.performanceManagement.services.PMEvalSchedulingService
object ScheduleActor {
case object nightlySchedule
}
class ScheduleActor #Inject() (actorSystem: ActorSystem) extends Actor {
val pMEvalSchedulingService: PMEvalSchedulingService = new PMEvalSchedulingService
override def receive: Receive = {
case nightlySchedule =>
println("Called the scheduler")
pMEvalSchedulingService.nightlyScheduledEvaluationsUpdate()
}
}
Well in my case the biggest issue ended up being that when scheduling the actor call in NightlyEvalSchedulerStartup() I was calling Akka.system... which was causing the system to instantiate a new AkkaSystem before the app existed. By removing the akk system was came to represent the injected dependency which was ready to go. Hope this helps someone in the future!
I created the following Akka Actor code in Scala. Code works fine when a single workerActor is created. But code silently fails when I try to create a pool of worker actors using round robin logic. Any idea how to fix this? How do I get more debug info to be printed?
import scala.collection.immutable.Map
import scala.collection.mutable.ArrayBuffer
import akka.actor.actorRef2Scala
import akka.actor.ActorSystem
import akka.actor.Props
import scala.concurrent.Await
import scala.concurrent.duration._
import akka.pattern.ask
import akka.util.Timeout
import akka.actor._
import org.junit._
import org.junit.Assert._
import messaging.actors._
import akka.routing.RoundRobinRouter
import akka.routing._
class MainEngineActorTest {
#Test
def testMainActor () = {
val _system = ActorSystem("MainEngineActor")
val master = _system.actorOf(Props[MainEngineActor], name = "EngineActor")
println ("Created Main Engine Actor")
implicit val timeout = Timeout(5 seconds)
val userID = new UserID ("test1")
println ("Sending messages")
for (i <- ( 1 to 10)) {
master ! "Hello"
master ! "World"
}
}
}
class MainEngineActor extends Actor with ActorLogging{
// works if we create only a single workerActor
//val workerActors = context.actorOf(Props[WorkerActor], name = "WorkerActors")
// Doesn't work when we create a pool of worker actors - how do we fix this?
// why doesn't this work and why aren't any error messages printed?
val workerActors = context.actorOf(RoundRobinPool(5).props(Props[WorkerActor]), name = "WorkerActors")
def receive: Receive = {
case request => {
workerActors forward request
}
}
}
class WorkerActor extends Actor {
def receive: Receive = {
case request => {
println ("RequestReceived =" + request)
}
}
}
Try creating your pool like this instead:
val workerActors = context.actorOf(Props[WorkerActor].withRouter(RoundRobinPool(5)), name = "WorkerActors")
In addition, when running this as a Junit test, the program is terminating before the child actors have a chance to receive the message. I verified this by adding a Thread.sleep(5000) after the loop that is sending the Hello and World messages to master. I then tweaked your code a little bit to use Akka's TestActorRef from akka-testkit which will force everything to use the CallingThreadDispatcher to get synchronous execution throughout the test and everything works as expected. The two lines I changed are:
implicit val _system = ActorSystem("MainEngineActor")
val master = TestActorRef(new MainEngineActor())