Given the following code:
class MigrationHandler #Inject()(database: Database, doUpdate: UpdateHandler)
(implicit #PropagateContext executor: ExecutionContext, actorSystem: ActorSystem)
extends Handler with Logging {
implicit val materializer: ActorMaterializer = ActorMaterializer()
val buttonTypeId = "someId1"
val promotionButtonTypeId = "someId2"
val typeId3 = "someId3"
val typeId4 = "someId4"
val typeIds = Seq[String](buttonTypeId, promotionButtonTypeId, typeId3, typeId4)
def apply(requestHeaders: Headers): Seq[Future[Done]] = {
for {
typeId <- typeIds
result = database.allByType(typeId)
.map(contents => contents.map(content => migrate(content, typeId))
.map(migratedContent => doUpdate(ContentId(migratedContent.id),
NewContent(migratedContent.raw, isDefaultContent = false), requestHeaders))
.runForeach(_ => ())).flatten
} yield result
}
def migrate(content: SomeContent, typeId: String): SomeContent = {
logger.info(s"$content with type $typeId")
content
}}
Future[Source[SomeContent, _]] is returned by the database.allByType(typeId)
In unit test where I am mocking database.allByType(typeId) to return Source.single(SomeContent()), I saw that I wasn't able reach the migrate function. Any idea what could be the problem here?
Related
I would like to have a Source.queue (or something analogous to it to push items on a materialized graph) that is instrumented to tell me the current level of saturation of the queue.
I'd like to do that without (re-)implementing the functionality provided by the QueueSource graph stage.
One possible solution I came up with is the following:
object InstrumentedSource {
final class InstrumentedSourceQueueWithComplete[T](
delegate: SourceQueueWithComplete[T],
bufferSize: Int,
)(implicit executionContext: ExecutionContext)
extends SourceQueueWithComplete[T] {
override def complete(): Unit = delegate.complete()
override def fail(ex: Throwable): Unit = delegate.fail(ex)
override def watchCompletion(): Future[Done] = delegate.watchCompletion()
private val buffered = new AtomicLong(0)
private[InstrumentedSource] def onDequeue(): Unit = {
val _ = buffered.decrementAndGet()
}
object BufferSaturationRatioGauge extends RatioGauge {
override def getRatio: RatioGauge.Ratio = RatioGauge.Ratio.of(buffered.get(), bufferSize)
}
lazy val bufferSaturationGauge: RatioGauge = BufferSaturationRatioGauge
override def offer(elem: T): Future[QueueOfferResult] = {
val result = delegate.offer(elem)
result.foreach {
case QueueOfferResult.Enqueued =>
val _ = buffered.incrementAndGet()
case _ => // do nothing
}
result
}
}
def queue[T](bufferSize: Int, overflowStrategy: OverflowStrategy)(
implicit executionContext: ExecutionContext,
materializer: Materializer,
): Source[T, InstrumentedSourceQueueWithComplete[T]] = {
val (queue, source) = Source.queue[T](bufferSize, overflowStrategy).preMaterialize()
val instrumentedQueue = new InstrumentedSourceQueueWithComplete[T](queue, bufferSize)
source.mapMaterializedValue(_ => instrumentedQueue).map { item =>
instrumentedQueue.onDequeue()
item
}
}
}
This mostly appears to work from some manual testing (apart from the fact that buffered is at most eventually consistent with the actual number of items in the queue, which should be fine in my case), but I was wondering if there is are solutions that perhaps make better use of built-in functionalities that I might have missed.
Below is simplified code snippet, where GraphStateLogic implementaion is passed to GraphStage as an constructor argument :-
package akka.shapes.examples.notworking
import akka.actor.ActorSystem
import akka.stream._
import akka.stream.scaladsl.{GraphDSL, RunnableGraph, Sink, Source}
import akka.stream.stage.{GraphStage, GraphStageLogic, InHandler}
//This is base graph stage, where GraphStageLogic and SinkShape are passed in constructor parameter
class BaseGraphStage[T](val shape: SinkShape[T], graphStageLogic: GraphStageLogic) extends GraphStage[ SinkShape[T] ] {
override def createLogic(inheritedAttributes: Attributes): GraphStageLogic = graphStageLogic
}
//this is a sample stateful extension of GraphStageLogic, that accepts first ten elements only
class CountLogic(sinkShape: SinkShape[Int], maxValue: Int) extends GraphStageLogic(sinkShape) {
var counter: Long = 0
override def preStart(): Unit = {
pull(sinkShape.in)
}
setHandler(sinkShape.in, new InHandler {
override def onPush(): Unit = {
val e = grab(sinkShape.in)
println("conditional sink : " + e)
counter = counter + 1
counter == maxValue match {
case true => completeStage()
case false => pull(sinkShape.in)
}
}
})
}
object SampleSinkNotWorking {
def main(args: Array[String]): Unit = {
implicit val actorSystem = ActorSystem("NotWroking")
implicit val actorMaterializer = ActorMaterializer()
val inlet = Inlet[Int](name = "sampleInlet")
val sinkShape = SinkShape( inlet )
val countGraphStateLogic = new CountLogic(sinkShape, 10)
val sinkGraphStage = new BaseGraphStage[Int](sinkShape, countGraphStateLogic)
val sink = Sink.fromGraph( sinkGraphStage )
val graph = GraphDSL.create() { implicit builder =>
import GraphDSL.Implicits._
Source(1 to 100) ~> sink
ClosedShape
}
val runnableGraph = RunnableGraph.fromGraph(graph)
runnableGraph.run()
}
}
Runnning above code is giving ArrayIndexOutOfBoundsException :-
Exception in thread "main" java.lang.ArrayIndexOutOfBoundsException:
-1 at akka.stream.stage.GraphStageLogic.setHandler(GraphStage.scala:439) at
akka.shapes.examples.notworking.CountLogic.(SampleSinkNotWorking.scala:24)
at
akka.shapes.examples.notworking.SampleSinkNotWorking$.main(SampleSinkNotWorking.scala:46)
at
akka.shapes.examples.notworking.SampleSinkNotWorking.main(SampleSinkNotWorking.scala)
I tried debugging, and it looks like, InLet id is -1, ant it's not getting reset.
But, why it's not getting reset, when GraphStateLogic is passed as an constructor argument to GraphState?
i am bit refactor your code and problem is gone, take a look :
class BaseGraphStage(maxValue: Int) extends GraphStage[SinkShape[Int]] {
val inlet = Inlet[Int](name = "sampleInlet")
override def createLogic(inheritedAttributes: Attributes): GraphStageLogic =
new GraphStageLogic(shape) with StageLogging {
var counter: Int = 0
setHandler(inlet, new InHandler {
override def onPush(): Unit = {
val e = grab(inlet)
log.info(s"$e is consumed")
counter += 1
if (counter == maxValue) {
completeStage()
} else {
pull(inlet)
}
}
})
override def preStart(): Unit =
pull(inlet)
override def postStop(): Unit =
counter = 0
}
override def shape: SinkShape[Int] = SinkShape(inlet)
}
object SampleSinkNotWorking {
def main(args: Array[String]): Unit = {
implicit val actorSystem = ActorSystem("NotWorking")
implicit val actorMaterializer = ActorMaterializer()
val sink = Sink.fromGraph(new BaseGraphStage(10))
Source(1 to 100).runWith(sink)
}
}
Can't answer fully on your last question, but i think all trick is about creating inlets in context of graph stage and not out of that, and use pre and post handlers. Hope that helps.
I am using reactivemongo-akka-stream and trying to transform the AkkaStreamCursor.documentSource. I am and am seeing two problems:
the documentation states that this operation returns a Source[T, NotUsed] but collection.find(query).cursor[BSONDocument].doccumentSource() returns a Source[BSONDocument, Future[State]]. Is there a way to avoid the State object?
Assuming I use Future[State] I am looking to get source of class as below
case class Inner(foo: String, baz: Int)
case class Outer(bar: Inner)
//
implicit object InnerReader extends BSONDocumentReader[Inner]//defined
val getCollection: Future[BSONCollection] = connection.database("db").map(_.collection("things")
def stream()(implicit m: Materializer): Source[Outer, Future[State]] = {
getCollection.map(_.find().cursor[Inner]().documentSource()).map(_.via(Flow[Inner].map(in => Outer(in))))
But instead of getting back a Future[Source[Outer, Future[State]] that i could deal with, this returns a Future[Source[Inner, Future[State]]#Repr[Outer]]
How can bson readers be used with this library?
As per the suggestion by cchantep, I needed to use fromFuture with flatMapConcat:
def stream()(implicit m: Materializer): Source[Outer, NotUsed] = {
val foo = getCollection.map(x => col2Source(x))
fix(foo).via(flowmap(map = Outer(_)))
}
def col2Source(col: BSONCollection): Source[Inner, Future[State]] = {
val cursor: AkkaStreamCursor[Inner] =
col.find(BSONDocument.empty).cursor[Inner]()
cursor.documentSource()
}
def flowmap[In, Out](
map: (In) => Out
): Flow[In, Out, NotUsed] = Flow[In].map(e => map(e))
def fix[Out, Mat](futureSource: Future[Source[Out, Mat]]): Source[Out, NotUsed] = {
Source.fromFuture(futureSource).flatMapConcat(identity)
}
I have an app that generate reports, with akka-http + akka-actors + akka-camel + akka-streams. When a post request arrives , the ActiveMqProducerActor enqueue the request into ActiveMq Broker. Then the ActiveMqConsumerActor consumes the message and start the task using akka-streams(in this actor i need the materializer) .
The main class create the ActorSystem and the ActorMaterializer, but i dont know how is the correct way to "inject" the materializer into the akka-actor
object ReportGeneratorApplication extends App {
implicit val system: ActorSystem = ActorSystem()
implicit val executor = system.dispatcher
implicit val materializer = ActorMaterializer()
val camelExtension: Camel = CamelExtension(system);
val amqc: ActiveMQComponent = ActiveMQComponent.activeMQComponent(env.getString("jms.url"))
amqc.setUsePooledConnection(true)
amqc.setAsyncConsumer(true)
amqc.setTrustAllPackages(true)
amqc.setConcurrentConsumers(1)
camelExtension.context.addComponent("jms", amqc);
val jmsProducer: ActorRef = system.actorOf(Props[ActiveMQProducerActor])
//Is this the correct way to pass the materializer?
val jmsConsumer: ActorRef = system.actorOf(Props(new ActiveMQConsumerActor()(materializer)), name = "jmsConsumer")
val endpoint: ReportEndpoint = new ReportEndpoint(jmsProducer);
Http().bindAndHandle(endpoint.routes, "localhost", 8881)
}
The ReportEndPoint class, that have the jmsProducerActor . Mongo is a trait with CRUD methods. JsonSupport(==SprayJsonSupport)
class ReportEndpoint(jmsProducer: ActorRef)
(implicit val system:ActorSystem,
implicit val executor: ExecutionContext,
implicit val materializer : ActorMaterializer)
extends JsonSupport with Mongo {
val routes =
pathPrefix("reports"){
post {
path("generate"){
entity(as[DataRequest]) { request =>
val id = java.util.UUID.randomUUID.toString
// **Enqueue the request into ActiveMq**
jmsProducer ! request
val future: Future[Seq[Completed]] = insertReport(request)
complete {
future.map[ToResponseMarshallable](r => r.head match {
case r : Completed => println(r); s"Reporte Generado con id $id"
case _ => HttpResponse(StatusCodes.InternalServerError, entity = "Error al generar reporte")
})
}
}
}
} ....
The idea of ActiveMqConsumerActor, is send the messages, with streams and backpressure, one by one,because ReportBuilderActor makes many mongo operations (and the datacenter it`s not very good).
//Is this the correct way to pass the materializer??
class ActiveMQConsumerActor (implicit materializer : ActorMaterializer) extends Consumer with Base {
override def endpointUri: String = env.getString("jms.queue")
val log = Logging(context.system, this)
val reportActor: ActorRef = context.actorOf(Props(new ReportBuilderActor()(materializer)), name = "reportActor")
override def receive: Receive = {
case msg: CamelMessage => msg.body match {
case data: DataRequest => {
//I need only one task running
Source.single(data).buffer(1, OverflowStrategy.backpressure).to(Sink.foreach(d => reportActor ! d)).run()
}
case _ => log.info("Invalid")
}
case _ => UnhandledMessage
}
}
Is a good idea have implicit values in companion objects?
Thanks!!
I'm creating an async library using Scala 2.10 futures. The constructor for the library takes a sequence of user-defined objects that implement a certain trait, and then a method on the library class sends some data one-by-one into the user-defined objects. I want the user to provide the ExecutionContext for the async operations when setting up the main instance, and then for that context to get passed into the user-defined objects as necessary. Simplified (pseudo?)code:
case class Response(thing: String)
class LibraryObject(stack: Seq[Processor])(implicit context: ExecutionContext) {
def entryPoint(data: String): Future[Response] = {
val response = Future(Response(""))
stack.foldLeft(response) { (resp, proc) => proc.process(data, resp) }
}
}
trait Processor {
def process(data: String, resp: Future[Response]): Future[Response]
}
It might be used something like this:
class ThingProcessor extends Processor {
override def process(data: String, response: Future[Response]) = {
response map { _.copy(thing = "THE THING") }
}
}
class PassThroughProcessor extends Processor {
override def process(request: Request, response: Future[Response]) = {
response
}
}
object TheApp extends App {
import ExecutionContext.Implicits.global
val stack = List(
new ThingProcessor,
new PassThroughProcessor
)
val libObj = new LibraryObject(stack)
val futureResponse = libObj.entryPoint("http://some/url")
// ...
}
I get a compile error for ThingProcessor:
Cannot find an implicit ExecutionContext, either require one yourself or import ExecutionContext.Implicits.global
My question is, how do I implicitly supply the ExecutionContext that LibraryObject has to the user-defined objects (ThingProcessor and PassThroughProcessor) or their methods without making the user (who will be writing the classes) worry about it--that is to say, I would prefer that the user did not have to type:
class MyFirstProcessor(implicit context: ExecutionContext)
or
override def process(...)(implicit context: ExecutionContext) = { ... }
The implicit scope includes companion objects and type parameters of base classes.
Or, library.submit(new library.Processor { def process() ... }).
This works, but wasn't my first thought, which was to be more clever:
import concurrent._
import concurrent.duration._
class Library(implicit xc: ExecutionContext = ExecutionContext.global) {
trait Processor {
implicit val myxc: ExecutionContext = xc
def process(i: Future[Int]): Future[Int]
}
def submit(p: Processor) = p process future(7)
}
object Test extends App {
val library = new Library
val p = new library.Processor {
def process(i: Future[Int]) = for (x <- i) yield 2 * x
}
val res = library submit p
val z = Await result (res, 10.seconds)
Console println z
}
Update:
import concurrent._
import concurrent.duration._
import java.util.concurrent.Executors
class Library()(implicit xc: ExecutionContext = ExecutionContext.global) {
trait Processor {
implicit val myxc: ExecutionContext = xc
def process(i: Future[Int]): Future[Int]
}
def submit(p: Processor) = p process future(7)
}
object ctx {
val xc = ExecutionContext fromExecutorService Executors.newSingleThreadExecutor
}
object library1 extends Library
object library2 extends Library()(ctx.xc)
object p extends library1.Processor {
def process(i: Future[Int]) = for (x <- i) yield 2 * x
}
object q extends library2.Processor {
def process(i: Future[Int]) = for (x <- i) yield 3 * x
}
object Test extends App {
val res = library1 submit p
//val oops = library2 submit p
//val oops = library1 submit q
val z = Await result (res, 10.seconds)
Console println z
Console println (Await result (library2 submit q, 10.seconds))
ctx.xc.shutdownNow()
}
It isn't much of a stretch to:
class Library(implicit xc: ExecutionContext = ExecutionContext.global) {
def submit(p: Processor): Future[Int] = p dueProcess future(7)
}
trait Processor {
implicit var myxc: ExecutionContext = _
def dueProcess(i: Future[Int])(implicit xc: ExecutionContext) = {
myxc = xc
process(i)
}
protected def process(i: Future[Int]): Future[Int]
}
object ctx {
val xc = ExecutionContext fromExecutorService Executors.newSingleThreadExecutor
}
object Test extends App {
def db() = Console println (new Throwable().getStackTrace mkString ("TRACE [\n ", "\n ", "\n]"))
val library = new Library()(ctx.xc)
val p = new Processor {
protected def process(i: Future[Int]) = for (x <- i) yield { db(); 2 * x }
}
val res = library submit p
val z = Await result (res, 10.seconds)
Console println z
ctx.xc.shutdownNow()
}