I've created an akka-stream source by reading contents from a file.
implicit val system = ActorSystem("reactive-process")
implicit val materializer = ActorMaterializer()
implicit val ec: ExecutionContextExecutor = system.dispatcher
case class RequestCaseClass(data: String)
def httpPoolFlow(host: String, port: String) = Http().cachedHostConnectionPool[RequestCaseClass](host, port)(httpMat)
def makeFileSource(filePath: String) = FileIO.fromFile(new File(filePath))
.via(Framing.delimiter(ByteString(System.lineSeparator), 10000, allowTruncation = false))
.map(_.utf8String)
def requestFramer = Flow.fromFunction((dataRow: String) =>
(getCmsRequest(RequestCaseClass(dataRow)), RequestCaseClass(dataRow))
).filter(_._1.isSuccess).map(z => (z._1.get, z._2))
def responseHandler = Flow.fromFunction(
(in: (Try[HttpResponse], RequestCaseClass)) => {
in._1 match {
case Success(r) => r.status.intValue() match {
case 200 =>
r.entity.dataBytes.runWith(Sink.ignore)
logger.info(s"Success response for: ${in._2.id}")
true
case _ =>
val b = Await.result(r.entity.dataBytes.runFold(ByteString.empty)(_ ++ _)(materializer)
.map(bb => new String(bb.toArray))(materializer.executionContext), 60.seconds)
logger.warn(s"Non-200 response for: ${in._2.id} r: $b")
false
}
case Failure(t) =>
logger.error(s"Failed completing request for: ${in._2.id} e: ${t.getMessage}")
false
}
}
Now when using it in a runnable graph, I want the entire stream to stop on reaching End Of File being read using makeFileSource(fp).
makeFileSource(fp)
.via(requestFramer)
.via(httpPoolFlow)
.via(responseHandler)
.runWith(Sink.ignore)
The stream currently doesn't stop on reaching end of file.
Related
Suppose there is a stream of some files to be processed and only a specific file should be processed(consumed) when a condition is met.
i.e. Only if the stream contains a file named "aaa", process a file named "bbb"
SomeFile(name: String)
What would be the correct(recommended) way to do this?
Okay, here's an example. Be careful about building up too big a buffer here before the trigger hits
class FileFinder {
def matchFiles(triggerName: String,
matchName: String): Flow[SomeFile, SomeFile, NotUsed] =
Flow[SomeFile].statefulMapConcat(
statefulMatcher(matches(triggerName), matches(matchName)))
private def matches(matchName: String): SomeFile => Boolean = {
case SomeFile(name) if name == matchName => true
case _ => false
}
private def statefulMatcher(
triggerFilter: => SomeFile => Boolean,
sendFilter: SomeFile => Boolean): () => SomeFile => List[SomeFile] = {
var found = false
var sendFiles: List[SomeFile] = Nil
() => file: SomeFile =>
{
file match {
case f if triggerFilter(f) =>
found = true
val send = sendFiles
sendFiles = Nil
send
case f if sendFilter(f) =>
if (found)
List(f)
else {
sendFiles = f :: sendFiles
Nil
}
case _ => Nil
}
}
}
}
object FileFinder extends FileFinder {
def main(args: Array[String]): Unit = {
implicit val system: ActorSystem = ActorSystem("finder")
implicit val materializer: ActorMaterializer = ActorMaterializer()
implicit val executor: ExecutionContextExecutor =
materializer.executionContext
implicit val loggingAdapter: LoggingAdapter = system.log
val files = List(SomeFile("aaa"), SomeFile("bbb"), SomeFile("aaa"))
Source(files)
.via(matchFiles("bbb", "aaa"))
.runForeach(println(_))
.andThen({
case Success(_) =>
println("Success")
system.terminate()
case Failure(ex) =>
loggingAdapter.error("Shouldn't happen...", ex)
system.terminate()
})
}
}
case class SomeFile(name: String)
I am trying to send a request to remote actor using ask pattern. The local actor recieves some value and it performs some task on it and updates it.
Then when local actor try to send back the updated value to remote actor , error occurs while sending. How should i handle this error?
Error:
[INFO] [03/31/2017 17:28:18.383] [ClientSystem-akka.actor.default-dispatcher-3] [akka://ClientSystem/deadLetters]
Message [check.package$Rcvdcxt] from Actor[akka://ClientSystem/user/localA1#1050660737] to Actor[akka://ClientSystem/deadLetters] was not delivered.
[1] dead letters encountered.
Remote Actor:
class RemoteActor() extends Actor {
def receive = {
case TaskFromLocal() =>{
implicit val timeout: Timeout = 15000
val currentSender = sender
val f1 = currentSender ? RemoteActor.rtree.cxtA
f1.onComplete{
case Success(Rcvdcxt(cxtA))=>
println("Success"+cxtA)
case Success(s) =>
println("Success :"+s)
case Failure(ex) =>
println("failure:"+ex)
}
}
case _ => println("unknown msg")
}
}
object RemoteActor{
def createRndCxtC(count: Int):List[CxtC] = (for (i <- 1 to count) yield CxtC(Random.nextString(5), Random.nextInt())).toList
def createRndCxtB(count: Int): List[CxtB] = (for (i <- 1 to count) yield CxtB(createRndCxtC(count), Random.nextInt())).toList
def createRndCxtA(count: Int): List[CxtA] = (for (i <- 1 to count) yield CxtA(createRndCxtC(count), 5)).toList
var rtree = RCxt(createRndCxtA(1),createRndCxtB(2),1,"")
def main(args: Array[String]) {
val configFile = getClass.getClassLoader.getResource("remote_application.conf").getFile
val config = ConfigFactory.parseFile(new File(configFile))
val system = ActorSystem("RemoteSystem" , config)
val remoteActor = system.actorOf(Props[RemoteActor], name="remote")
println("remote is ready")
}
}
Local Actor :
class LocalActorA extends Actor{
#throws[Exception](classOf[Exception])
val remoteActor = context.actorSelection("akka.tcp://RemoteSystem#127.0.0.1:5150/user/remote")
def receive = {
case TaskLA1(taskA) => {
implicit val timeout: Timeout = 15000
val rCxt = remoteActor ? TaskFromLocal()
val currentSender = sender
rCxt.onComplete{
case Success(Rcvdcxt(cxtA))=>
println("Success"+cxtA)
println("Sender: "+ sender)
currentSender ! Rcvdcxt(cxtA)
case Success(s)=>
println("Got nothing from Remote"+s)
currentSender ! "Failuree"
case Failure(ex) =>
println("Failure in getting remote")
currentSender ! "Failure"
}
}
}
}
object LocalActorA {
def createRndCxtC(count: Int):List[CxtC] = (for (i <- 1 to count) yield CxtC(Random.nextString(5), Random.nextInt())).toList
def createRndCxtB(count: Int): List[CxtB] = (for (i <- 1 to count) yield CxtB(createRndCxtC(count), Random.nextInt())).toList
def createRndCxtA(count: Int): List[CxtA] = (for (i <- 1 to count) yield CxtA(createRndCxtC(count), 3)).toList
var tree = RCxt(createRndCxtA(2),createRndCxtB(2),1,"")
def main(args: Array[String]) {
val configFile = getClass.getClassLoader.getResource("local_application.conf").getFile
val config = ConfigFactory.parseFile(new File(configFile))
val system = ActorSystem("ClientSystem",config)
val localActorA1 = system.actorOf(Props[LocalActorA], name="localA1")
println("LocalActor A tree : "+tree)
localActorA1 ! TaskLA1(new DummySum())
}
}
Since you didn't post all the code I can't really tell exactly the error, but my best guess is related to the fact that you are calling sender in the onComplete in the LocalActor. This is unsafe and should be avoided at all costs. Instead, do something similar with the remote actor:
class LocalActor {
def receive = {
case TaskLA1(taskA) =>
val currentSender = sender
rCxt.onComplete {
case Success(Rcvdcxt(cxtA))=>
currentSender ! Rcvdcxt(cxtA)
...
}
}
}
How do you throttle Flow in the latest Akka (2.4.6) ? I'd like to throttle Http client flow to limit number of requests to 3 requests per second. I found following example online but it's for old Akka and akka-streams API changed so much that I can't figure out how to rewrite it.
def throttled[T](rate: FiniteDuration): Flow[T, T] = {
val tickSource: Source[Unit] = TickSource(rate, rate, () => ())
val zip = Zip[T, Unit]
val in = UndefinedSource[T]
val out = UndefinedSink[T]
PartialFlowGraph { implicit builder =>
import FlowGraphImplicits._
in ~> zip.left ~> Flow[(T, Unit)].map { case (t, _) => t } ~> out
tickSource ~> zip.right
}.toFlow(in, out)
}
Here is my best attempt so far
def throttleFlow[T](rate: FiniteDuration) = Flow.fromGraph(GraphDSL.create() { implicit builder =>
import GraphDSL.Implicits._
val ticker = Source.tick(rate, rate, Unit)
val zip = builder.add(Zip[T, Unit.type])
val map = Flow[(T, Unit.type)].map { case (value, _) => value }
val messageExtractor = builder.add(map)
val in = Inlet[T]("Req.in")
val out = Outlet[T]("Req.out")
out ~> zip.in0
ticker ~> zip.in1
zip.out ~> messageExtractor.in
FlowShape.of(in, messageExtractor.out)
})
it throws exception in my main flow though :)
private val queueHttp = Source.queue[(HttpRequest, (Any, Promise[(Try[HttpResponse], Any)]))](1000, OverflowStrategy.backpressure)
.via(throttleFlow(rate))
.via(poolClientFlow)
.mapAsync(4) {
case (util.Success(resp), any) =>
val strictFut = resp.entity.toStrict(5 seconds)
strictFut.map(ent => (util.Success(resp.copy(entity = ent)), any))
case other =>
Future.successful(other)
}
.toMat(Sink.foreach({
case (triedResp, (value: Any, p: Promise[(Try[HttpResponse], Any)])) =>
p.success(triedResp -> value)
case _ =>
throw new RuntimeException()
}))(Keep.left)
.run
where poolClientFlow is Http()(system).cachedHostConnectionPool[Any](baseDomain)
Exception is:
Caused by: java.lang.IllegalArgumentException: requirement failed: The output port [Req.out] is not part of the underlying graph.
at scala.Predef$.require(Predef.scala:219)
at akka.stream.impl.StreamLayout$Module$class.wire(StreamLayout.scala:204)
Here is an attempt that uses the throttle method as mentioned by #Qingwei. The key is to not use bindAndHandle(), but to use bind() and throttle the flow of incoming connections before handling them. The code is taken from the implementation of bindAndHandle(), but leaves out some error handling for simplicity. Please don't do that in production.
implicit val system = ActorSystem("test")
implicit val mat = ActorMaterializer()
import system.dispatcher
val maxConcurrentConnections = 4
val handler: Flow[HttpRequest, HttpResponse, NotUsed] = complete(LocalDateTime.now().toString)
def handleOneConnection(incomingConnection: IncomingConnection): Future[Done] =
incomingConnection.flow
.watchTermination()(Keep.right)
.joinMat(handler)(Keep.left)
.run()
Http().bind("127.0.0.1", 8080)
.throttle(3, 1.second, 1, ThrottleMode.Shaping)
.mapAsyncUnordered(maxConcurrentConnections)(handleOneConnection)
.to(Sink.ignore)
.run()
I am pretty new using PersistentActor ,
when I try to call updateState from a future onComplete, fails , nothing happanes , tried to debug it and I do get to the persist call but not into the updateState
trait Event
case class Cmd(data: String)
case class Evt(data: String) extends Event
class BarActor extends PersistentActor{
implicit val system = context.system
implicit val executionContext = system.dispatcher
def updateState(event: Evt): Unit ={
println("Updating state")
state = state.updated(event)
sender() ! state
}
def timeout(implicit ec: ExecutionContext) =
akka.pattern.after(duration = 2 seconds, using = system.scheduler)(Future.failed(new TimeoutException("Got timed out!")))
val receiveCommand: Receive = {
case Cmd(data) =>
def anotherFuture(i: Int)(implicit system: ActorSystem) = {
val realF = Future {
i % 2 match {
case 0 =>
Thread.sleep(100)
case _ =>
Thread.sleep(500)
}
i
}
Future.firstCompletedOf(Seq(realF, timeout))
.recover {
case _ => -1
}
}
val res = (1 to 10).map(anotherFuture(_))
val list = Future.sequence(res)
list.onComplete{
case _ =>
persist(Evt("testing"))(updateState)
}
}
}
You could try this:
list.onComplete {
case _ => self ! Evt("testing")
}
And add this to receiveCommand
case evt: Evt =>
persist(Evt("testing"))(updateStates)
I want to be able to make concurrent requests to multiple data repositories and consolidate the results. I am trying to understand if my approach is at all valid or if there is a better way to approach this problem. I am definitely new to Akka / Spray / Scala and really want to get a better understanding of how to properly build these components. Any suggestions / Tips would be greatly appreciated. Trying to wrap my head around the use of actors and futures for this type of implementation.
Spray Service:
trait DemoService extends HttpService with Actor with ActorLogging {
implicit val timeout = Timeout(5 seconds) // needed for `?` below
val mongoMasterActor = context.actorOf(Props[MongoMasterActor], "redisactor")
val dbMaster = context.actorOf(Props[DbMasterActor], "dbactor")
val messageApiRouting =
path("summary" / Segment / Segment) { (dataset, timeslice) =>
onComplete(getDbResponses(dbMaster, dataset, timeslice)) {
case Success(dbMessageResponse) => complete(s"The result was $dbMessageResponse")
case Failure(ex) => complete(s"An error occurred: ${ex.getMessage}")
}
}
/** Passes the desired actor reference for a specific dataset and timeslice for summary data retrieval
*
* #param mongoActor an actor reference to the RedisActor that will handle the appropriate request routing
* #param dataset The dataset for which the summary has been requested
* #param timeslice The timeslice (Month, Week, Day, etc.) for which the summary has been requested
*/
def getSummary(mongoActor: ActorRef, dataset: String, timeslice: String): Future[DbMessageResponse] = {
log.debug(s"dataset: $dataset timeslice: $timeslice")
val dbMessage = DbMessage("summary", dataset + timeslice)
(mongoActor ? dbMessage).mapTo[DbMessageResponse]
}
def getDbResponses(dbActor: ActorRef, dataset: String, timeslice: String): Future[SummaryResponse] = {
log.debug(s"dataset: $dataset timeslice: $timeslice")
val dbMessage = DbMessage("summary", dataset + timeslice)
(dbActor ? dbMessage).mapTo[SummaryResponse]
}
def getSummaryPayload(mongoSummary: DbMessageResponse, redisSummary: DbMessageResponse): String = {
mongoSummary.response + redisSummary.response
}
}
Akka Actor / Future mock db requests:
class DbMasterActor extends Actor with ActorLogging {
private var originalSender: ActorRef = _
//TODO: Need to add routing to the config to limit instances
val summaryActor = context.actorOf(Props(new SummaryActor), "summaryactor")
def receive = {
case msg: DbMessage => {
this.originalSender = sender
msg.query match {
case "summary" => {
getDbResults().onComplete{
case Success(result) => originalSender ! result
case Failure(ex) => log.error(ex.getMessage)
}
}
}
}
//If not match log an error
case _ => log.error("Received unknown message: {} ")
}
def getDbResults(): Future[SummaryResponse] = {
log.debug("hitting db results")
val mongoResult = Future{ Thread.sleep(500); "Mongo"}
val redisResult = Future{ Thread.sleep(800); "redis"}
for{
mResult <- mongoResult
rResult <- redisResult
} yield SummaryResponse(mResult, rResult)
}
}
Following the reading of Effective Akka by Jamie Allen, I am going to attempt to apply his "Cameo" pattern suggestion.
Slideshare:
http://www.slideshare.net/shinolajla/effective-akka-scalaio
Github:
https://github.com/jamie-allen/effective_akka
I think what I created will work, but doesn't sound like the best approach based on Jamie's comments in his talks. I will update / edit back to this post what I have implemented (or try to).
Summary Actor (Cameo Actor):
object SummaryResponseHandler {
case object DbRetrievalTimeout
def props(mongoDb: ActorRef, redisDb: ActorRef, originalSender: ActorRef): Props = {
Props(new SummaryResponseHandler(mongoDb, redisDb, originalSender))
}
}
class SummaryResponseHandler(mongoDb: ActorRef, redisDb: ActorRef,
originalSender: ActorRef) extends Actor with ActorLogging {
import SummaryResponseHandler._
var mongoSummary, redisSummary: Option[String] = None
def receive = LoggingReceive {
case MongoSummary(summary) =>
log.debug(s"Received mongo summary: $summary")
mongoSummary = summary
collectSummaries
case RedisSummary(summary) =>
log.debug(s"Received redis summary: $summary")
redisSummary = summary
collectSummaries
case DbRetrievalTimeout =>
log.debug("Timeout occurred")
sendResponseAndShutdown(DbRetrievalTimeout)
}
def collectSummaries = (mongoSummary, redisSummary) match {
case (Some(m), Some(r)) =>
log.debug(s"Values received for both databases")
timeoutMessager.cancel
sendResponseAndShutdown(DataSetSummary(mongoSummary, redisSummary))
case _ =>
}
def sendResponseAndShutdown(response: Any) = {
originalSender ! response
log.debug("Stopping context capturing actor")
context.stop(self)
}
import context.dispatcher
val timeoutMessager = context.system.scheduler.scheduleOnce(
250 milliseconds, self, DbRetrievalTimeout)
}
class SummaryRetriever(mongoDb: ActorRef, redisDb: ActorRef) extends Actor with ActorLogging {
def receive = {
case GetSummary(dataSet) =>
log.debug("received dataSet")
val originalSender = sender
val handler = context.actorOf(SummaryResponseHandler.props(mongoDb,redisDb, originalSender), "cameo-message-handler")
mongoDb.tell(GetSummary(dataSet), handler)
redisDb.tell(GetSummary(dataSet), handler)
case _ => log.debug(s"Unknown result $GetSummary(datset)")
}
}
Common:
case class GetSummary(dataSet: String)
case class DataSetSummary(
mongo: Option[String],
redis: Option[String]
)
case class MongoSummary(
summary: Option[String]
)
case class RedisSummary(
summary: Option[String]
)
trait MongoProxy extends Actor with ActorLogging
trait RedisProxy extends Actor with ActorLogging
Mock Stubs:
class MongoProxyStub extends RedisProxy {
val summaryData = Map[String, String](
"dataset1" -> "MongoData1",
"dataset2" -> "MongoData2")
def receive = LoggingReceive {
case GetSummary(dataSet: String) =>
log.debug(s"Received GetSummary for ID: $dataSet")
summaryData.get(dataSet) match {
case Some(data) => sender ! MongoSummary(Some(data))
case None => sender ! MongoSummary(Some(""))
}
}
}
class RedisProxyStub extends MongoProxy{
val summaryData = Map[String, String](
"dataset1" -> "RedisData1",
"dataset2" -> "RedisData2")
def receive = LoggingReceive {
case GetSummary(dataSet: String) =>
log.debug(s"Received GetSummary for ID: $dataSet")
summaryData.get(dataSet) match {
case Some(data) => sender ! RedisSummary(Some(data))
case None => sender ! RedisSummary(Some(""))
}
}
}
Boot (You should use test, but was just wanting to run from boot):
object Boot extends App{
val system = ActorSystem("DbSystem")
val redisProxy = system.actorOf(Props[RedisProxyStub], "cameo-success-mongo")
val mongoProxy = system.actorOf(Props[MongoProxyStub], "cameo-success-redis")
val summaryRetrieverActor = system.actorOf(Props(new SummaryRetriever(redisProxy, mongoProxy)), "cameo-retriever1")
implicit val timeout = Timeout(5 seconds)
val future = summaryRetrieverActor ? GetSummary("dataset1")
val result = Await.result(future, timeout.duration).asInstanceOf[DataSetSummary]
println(Some(result.mongo).x)
println(result.redis)
system.shutdown()
}
Application Config:
akka.loglevel = "DEBUG"
akka.event-handlers = ["akka.event.slf4j.Slf4jEventHandler"]
akka.actor.debug.autoreceive = on
akka.actor.debug.lifecycle = on
akka.actor.debug.receive = on
akka.actor.debug.event-stream = on