I am using Akka stream Source.queue as Source for websocket clients.
Reading from kafka topic with 10k records using kaka consumer API and offering it to Source.queue with buffer 100k.
I am using BroardcastHub for fan-out. The websocket client does not get any data but see records from kafka enqueued on offer result.
Appreciate any help.
def kafkaSourceQueue() = {
val sourceQueue = Source.queue[String](100000, OverflowStrategy.dropHead)
val (theQueue, queueSource)= sourceQueue.toMat(BroadcastHub.sink(bufferSize = 256))(Keep.both).run
val consumer = KafkaEventSource.initKafkaConsumer()
try {
while (true) {
val records = consumer.poll(polltimeout.toMillis)
for (record <- records.records(topic)) {
//println(record.value())
theQueue.offer(record.value()).onComplete{
case Success(QueueOfferResult.Enqueued) =>
println("enqueued")
case _ => println("Failed to enqueue")
}
}
}
}
finally {
if (consumer != null) {
println("consumer unsubscribed")
consumer.unsubscribe()
}
}
queueSource
}
private def logicStreamFlow: Flow[String, String, NotUsed] = {
Flow.fromSinkAndSourceCoupled(Sink.ignore, kafkaSourceQueue)
}
def websocketFlow: Flow[Message, Message, NotUsed] = {
Flow[Message]
.map{
case TextMessage.Strict(msg) => msg
case _ => throw new Exception("exception msg")
}
.via(logicStreamFlow)
.map { msg: String => TextMessage.Strict(msg) }
}
lazy private val streamRoute =
path("stream") {
handleWebSocketMessages {
websocketFlow
.watchTermination() { (_, done) =>
done.onComplete {
case Success(_) =>
log.info("Stream route completed successfully")
case Failure(ex) =>
log.error(s"Stream route completed with failure : $ex")
}
}
}
}
def startServer(): Unit = {
bindingFuture = Http().bindAndHandle(wsRoutes, HOST, PORT)
log.info(s"Server online at http://localhost:9000/")
}
def stopServer(): Unit = {
bindingFuture
.flatMap(_.unbind())
.onComplete{
_ => system.terminate()
log.info("terminated")
}
}
Related
I am beginner in akka and I have an problem statement to work with. I have an akka flow that's reading Kafka Events from some topic and doing some transformation before creating Commitable offset of the message.
I am not sure the best way to add a akka sink on top of this code to store the transformed events in some DB
def eventTransform : Flow[KafkaMessage,CommittableRecord[Either[Throwable,SomeEvent]],NotUsed]
def processEvents
: Flow[KafkaMessage, ConsumerMessage.CommittableOffset, NotUsed] =
Flow[KafkaMessage]
.via(eventTransform)
.filter({ x =>
x.value match {
case Right(event: SomeEvent) =>
event.status != "running"
case Left(_) => false
}
})
.map(_.message.committableOffset)
This is my akka source calling the akka flow
private val consumerSettings: ConsumerSettings[String, String] = ConsumerSettings(
system,
new StringDeserializer,
new StringDeserializer,
)
.withGroupId(groupId)
.withProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
private val committerSettings: CommitterSettings = CommitterSettings(system)
private val control = new AtomicReference[Consumer.Control](Consumer.NoopControl)
private val restartableSource = RestartSource
.withBackoff(restartSettings) { () =>
Consumer
.committableSource(consumerSettings, Subscriptions.topics(topicIn))
.mapMaterializedValue(control.set)
.via(processEvents) // calling the flow here
}
restartableSource
.toMat(Committer.sink(committerSettings))(Keep.both)
.run()
def api(): Behavior[Message] =
Behaviors.receive[Message] { (_, message) =>
message match {
case Stop =>
context.pipeToSelf(control.get().shutdown())(_ => Stopped)
Behaviors.same
case Stopped =>
Behaviors.stopped
}
}
.receiveSignal {
case (_, _ #(PostStop | PreRestart)) =>
control.get().shutdown()
Behaviors.same
}
}
I have the following test:
final class DetectorSpec extends BddSpec {
private val sap = Container.sap()
private val kafka = Container.kafka()
sap.start()
kafka.start()
override def afterAll(): Unit = {
sap.stop()
kafka.stop()
}
private def withKafkaAndSapOnline(testCode: TestProbe[ServerEvent] => Unit)
: Unit = {
val config = ConfigFactory.parseString(
s"""akka.actor.default-dispatcher = {
type = akka.testkit.CallingThreadDispatcherConfigurator
}
akka.actor.testkit.typed.single-expect-default = 0s
akka.loglevel = DEBUG
kafka {
servers = "${kafka.getBootstrapServers}"
zookeeper = "${kafka.getMappedPort(2181)}"
}
sap {
server = "ws://${sap.getContainerIpAddress}:${sap.getMappedPort(8080)}"
}""")
val testKit = ActorTestKit("DetectorSystem1", config)
testKit.spawn(DetectorSupervisor.create(), "DetectorSupervisor")
val inbox = testKit.createTestProbe[ServerEvent]("Receiver")
testKit.system.receptionist ! Receptionist.Register(ServerStateKey, inbox.ref)
testCode(inbox)
testKit.shutdownTestKit()
}
private def withKafkaAndSapOffline(testCode: (TestProbe[ServerEvent], TestProbe[ServerEvent]) => Unit)
: Unit = {
val config = ConfigFactory.parseString(
s"""akka.actor.default-dispatcher = {
type = akka.testkit.CallingThreadDispatcherConfigurator
}
akka.actor.testkit.typed.single-expect-default = 0s
akka.loglevel = DEBUG
kafka {
servers = "PLAINTEXT://localhost:9092"
zookeeper = "2181"
}
sap {
server = "ws://127.0.0.1"
}""")
val testKit = ActorTestKit("DetectorSystem2", config)
testKit.spawn(DetectorSupervisor.create(), "DetectorSupervisor")
val inbox1 = testKit.createTestProbe[ServerEvent]("Receiver1")
val inbox2 = testKit.createTestProbe[ServerEvent]("Receiver2")
testKit.system.receptionist ! Receptionist.Register(ServerStateKey, inbox1.ref)
testKit.system.receptionist ! Receptionist.Register(ServerStateKey, inbox2.ref)
testCode(inbox1, inbox2)
testKit.shutdownTestKit()
}
private def withKafkaOfflineSapOnline(testCode: TestProbe[ServerEvent] => Unit)
: Unit = {
val config = ConfigFactory.parseString(
s"""akka.actor.default-dispatcher = {
type = akka.testkit.CallingThreadDispatcherConfigurator
}
akka.actor.testkit.typed.single-expect-default = 0s
akka.loglevel = DEBUG
kafka {
servers = "PLAINTEXT://localhost:9092"
zookeeper = "2181"
}
sap {
server = "ws://${sap.getContainerIpAddress}:${sap.getMappedPort(8080)}"
}""")
val testKit = ActorTestKit("DetectorSystem3", config)
val inbox = testKit.createTestProbe[ServerEvent]("Receiver")
testKit.spawn(DetectorSupervisor.create(), "DetectorSupervisor")
testKit.system.receptionist ! Receptionist.Register(ServerStateKey, inbox.ref)
testCode(inbox)
testKit.shutdownTestKit()
}
private def withKafkaOnlineSapOffline(testCode: TestProbe[ServerEvent] => Unit)
: Unit = {
val config = ConfigFactory.parseString(
s"""akka.actor.default-dispatcher = {
type = akka.testkit.CallingThreadDispatcherConfigurator
}
akka.actor.testkit.typed.single-expect-default = 0s
akka.loglevel = DEBUG
kafka {
servers = "${kafka.getBootstrapServers}"
zookeeper = "${kafka.getMappedPort(2181)}"
}
sap {
server = "ws://127.0.0.1:8080"
}""")
val testKit = ActorTestKit("DetectorSystem4", config)
testKit.spawn(DetectorSupervisor.create(), "DetectorSupervisor")
val inbox = testKit.createTestProbe[ServerEvent]("Receiver")
testKit.system.receptionist ! Receptionist.Register(ServerStateKey, inbox.ref)
testCode(inbox)
testKit.shutdownTestKit()
}
feature("Detect Kafka and SAP availability") {
info("As a technical user, I want to be notified in real time, if Kafka and SAP is up and running or not.")
scenario("SAP and Kafka are available") {
withKafkaAndSapOnline { inbox =>
Given("I am waiting for the current state message")
When("I am receive the state message")
Then("it should contain `SAP and Kafka are online`")
inbox.fishForMessage(5.second){
case ServerOfflineApproved =>
FishingOutcomes.continue
case ServerOnlineApproved =>
FishingOutcomes.complete
case _ =>
FishingOutcomes.fail("Unexpected message")
}
}
}
scenario("SAP is online and Kafka is offline") {
withKafkaOfflineSapOnline { inbox =>
Given("I am waiting for the current state message")
When("I am receive the state message")
Then("it should contain `Kafka is offline`")
inbox.fishForMessage(5.second){
case ServerOfflineApproved =>
FishingOutcomes.complete
case _ =>
FishingOutcomes.fail("Unexpected message")
}
}
}
scenario("SAP is offline and Kafka is online") {
withKafkaOnlineSapOffline { inbox =>
Given("I am waiting for the current state message")
When("I am receive the state message")
Then("it should contain `SAP is offline`")
inbox.fishForMessage(5.second){
case ServerOfflineApproved =>
FishingOutcomes.complete
case _ =>
FishingOutcomes.fail("Unexpected message")
}
}
}
scenario("SAP and Kafka are offline") {
withKafkaAndSapOffline { (inbox1, inbox2) =>
Given("I am registering two listeners")
When("I am receive the state message")
Then("it should contain `Kafka and SAP are offline`")
inbox1.fishForMessage(5.second){
case ServerOfflineApproved =>
FishingOutcomes.complete
case _ =>
FishingOutcomes.fail("Unexpected message")
}
inbox2.fishForMessage(5.second){
case ServerOfflineApproved =>
FishingOutcomes.complete
case _ =>
FishingOutcomes.fail("Unexpected message")
}
}
}
}
}
As you can see after every test, it will shutdown the ActorTestKit. But sometimes it freezes, that means, I've got the following message:
[DEBUG] [07/19/2019 20:37:57.788] [DetectorSystem3-akka.actor.default-blocking-io-dispatcher-11] [akka://DetectorSystem3/system/IO-TCP/selectors/$a/0] Attempting connection to [localhost/127.0.0.1:32846]
[DEBUG] [07/19/2019 20:37:57.789] [DetectorSystem3-akka.io.pinned-dispatcher-2] [akka://DetectorSystem3/system/IO-TCP/selectors/$a/0] Connection established to [localhost:32846]
[INFO] [07/19/2019 20:37:58.807] [ScalaTest-run-running-DetectorSpec] [akka://DetectorSystem3/user/DetectorSupervisor/KafkaActor/KafkaStreamer] !!!!!!!!!!!!!!!!!!!!! Shutdown KafkaDetectorActor !!!!!!!!!!!!!!!!!!!!!
[INFO] [07/19/2019 20:37:58.810] [ScalaTest-run-running-DetectorSpec] [akka://DetectorSystem3/user/DetectorSupervisor/SapActor/SapStreamer] !!!!!!!!!!!!!!!!!!!!! Shutdown SapDetectorActor !!!!!!!!!!!!!!!!!!!!!
and the ActorTestKit never shuts down.
Here is the evidence:
It is just running and running...
How to stop it?
I am using akka-http websocket to push messages from a kafka topic to websocket clients.
For this purpose, i created a plain kafka consumer (using akka-streams-kafka connector) with offset set to "earliest" so that every new websocket client connecting gets all the data from the beginning.
The problem is that the first connected websocket client gets all the data and other ws clients (connecting after the first client has got all the data) do not get any. The kafka topic has 1million records.
I am using the BroadcastHub from Akka-streams.
Appreciate any suggestions.
lazy private val kafkaPlainSource: Source[String, NotUsed] = {
val consumerSettings = ConsumerSettings(system, new StringDeserializer, new StringDeserializer)
.withBootstrapServers(KAFKA_BROKERS)
.withGroupId(UUID.randomUUID().toString)
.withProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
val kafkaSource = Consumer.plainSource(consumerSettings, Subscriptions.topics(KAFKA_TOPIC))
.mapAsync(PARALLELISM) { cr =>
Future {
cr.value
}
}
kafkaSource.toMat(BroadcastHub.sink)(Keep.right).run
}
def logicFlow: Flow[String, String, NotUsed] =
Flow.fromSinkAndSourceCoupled(Sink.ignore, kafkaSource)
val websocketFlow: Flow[Message, Message, Any] = {
Flow[Message]
.map {
case TextMessage.Strict(msg) => msg
case _ => println("ignore streamed message")
}
.via(logicFlow)
.map { msg: String => TextMessage.Strict(msg) }
}
lazy private val streamRoute =
path("stream") {
handleWebSocketMessages {
websocketFlow
.watchTermination() { (_, done) =>
done.onComplete {
case Success(_) =>
log.info("Stream route completed successfully")
case Failure(ex) =>
log.error(s"Stream route completed with failure : $ex")
}
}
}
}
def startServer(): Unit = {
bindingFuture = Http().bindAndHandle(wsRoutes, HOST, PORT)
log.info(s"Server online at http://localhost:9000/")
}
def stopServer(): Unit = {
bindingFuture
.flatMap(_.unbind())
.onComplete{
_ => system.terminate()
log.info("terminated")
}
}
I am new to Akka and developed a sample Akka WebSocket server that streams a file's contents to clients using BroadcastHub (based on a sample from the Akka docs).
How can I measure the throughput (messages/second), assuming the clients are consuming as fast as the server?
// file source
val fileSource = FileIO.fromPath(Paths.get(path)
// Akka file source
val theFileSource = fileSource
.toMat(BroadcastHub.sink)(Keep.right)
.run
//Akka kafka file source
lazy val kafkaSourceActorStream = {
val (kafkaSourceActorRef, kafkaSource) = Source.actorRef[String](Int.MaxValue, OverflowStrategy.fail)
.toMat(BroadcastHub.sink)(Keep.both).run()
Consumer.plainSource(consumerSettings, Subscriptions.topics("perf-test-topic"))
.runForeach(record => kafkaSourceActorRef ! record.value().toString)
}
def logicFlow: Flow[String, String, NotUsed] = Flow.fromSinkAndSource(Sink.ignore, theFileSource)
val websocketFlow: Flow[Message, Message, Any] = {
Flow[Message]
.collect {
case TextMessage.Strict(msg) => Future.successful(msg)
case _ => println("ignore streamed message")
}
.mapAsync(parallelism = 2)(identity)
.via(logicFlow)
.map { msg: String => TextMessage.Strict(msg) }
}
val fileRoute =
path("file") {
handleWebSocketMessages(websocketFlow)
}
}
def startServer(): Unit = {
bindingFuture = Http().bindAndHandle(wsRoutes, HOST, PORT)
log.info(s"Server online at http://localhost:9000/")
}
def stopServer(): Unit = {
bindingFuture
.flatMap(_.unbind())
.onComplete{
_ => system.terminate()
log.info("terminated")
}
}
//ws client
def connectToWebSocket(url: String) = {
println("Connecting to websocket: " + url)
val (upgradeResponse, closed) = Http().singleWebSocketRequest(WebSocketRequest(url), websocketFlow)
val connected = upgradeResponse.flatMap{ upgrade =>
if(upgrade.response.status == StatusCodes.SwitchingProtocols )
{
println("Web socket connection success")
Future.successful(Done)
}else {
println("Web socket connection failed with error: {}", upgrade.response.status)
throw new RuntimeException(s"Web socket connection failed: ${upgrade.response.status}")
}
}
connected.onComplete { msg =>
println(msg)
}
}
def websocketFlow: Flow[Message, Message, _] = {
Flow.fromSinkAndSource(printFlowRate, Source.maybe)
}
lazy val printFlowRate =
Flow[Message]
.alsoTo(fileSink("output.txt"))
.via(flowRate(1.seconds))
.to(Sink.foreach(rate => println(s"$rate")))
def flowRate(sampleTime: FiniteDuration) =
Flow[Message]
.conflateWithSeed(_ ⇒ 1){ case (acc, _) ⇒ acc + 1 }
.zip(Source.tick(sampleTime, sampleTime, NotUsed))
.map(_._1.toDouble / sampleTime.toUnit(SECONDS))
def fileSink(file: String): Sink[Message, Future[IOResult]] = {
Flow[Message]
.map{
case TextMessage.Strict(msg) => msg
case TextMessage.Streamed(stream) => stream.runFold("")(_ + _).flatMap(msg => Future.successful(msg))
}
.map(s => ByteString(s + "\n"))
.toMat(FileIO.toFile(new File(file)))(Keep.right)
}
You could attach a throughput-measuring stream to your existing stream. Here is an example, inspired by this answer, that prints the number of integers that are emitted from the upstream source every second:
val rateSink = Flow[Int]
.conflateWithSeed(_ => 0){ case (acc, _) => acc + 1 }
.zip(Source.tick(1.second, 1.second, NotUsed))
.map(_._1)
.toMat(Sink.foreach(i => println(s"$i elements/second")))(Keep.right)
In the following example, we attach the above sink to a source that emits the integers 1 to 10 million. To prevent the rate-measuring stream from interfering with the main stream (which, in this case, simply converts every integer to a string and returns the last string processed as part of the materialized value), we use wireTapMat:
val (rateFut, mainFut) = Source(1 to 10000000)
.wireTapMat(rateSink)(Keep.right)
.map(_.toString)
.toMat(Sink.last[String])(Keep.both)
.run() // (Future[Done], Future[String])
rateFut onComplete {
case Success(x) => println(s"rateFut completed: $x")
case Failure(_) =>
}
mainFut onComplete {
case Success(s) => println(s"mainFut completed: $s")
case Failure(_) =>
}
Running the above sample prints something like the following:
0 elements/second
2597548 elements/second
3279052 elements/second
mainFut completed: 10000000
3516141 elements/second
607254 elements/second
rateFut completed: Done
If you don't need a reference to the materialized value of rateSink, use wireTap instead of wireTapMat. For example, attaching rateSink to your WebSocket flow could look like the following:
val websocketFlow: Flow[Message, Message, Any] = {
Flow[Message]
.wireTap(rateSink) // <---
.collect {
case TextMessage.Strict(msg) => Future.successful(msg)
case _ => println("ignore streamed message")
}
.mapAsync(parallelism = 2)(identity)
.via(logicFlow)
.map { msg: String => TextMessage.Strict(msg) }
}
wireTap is defined on both Source and Flow.
Where I last worked I implemented a performance benchmark of this nature.
Basically, it meant creating a simple client app that consumes messages from the websocket and outputs some metrics. The natural choice was to implement the client using akka-http client-side support for websockets. See:
https://doc.akka.io/docs/akka-http/current/client-side/websocket-support.html#singlewebsocketrequest
Then we used the micrometer library to expose metrics to Prometheus, which was our tool of choice for reporting and charting.
https://github.com/micrometer-metrics
https://micrometer.io/docs/concepts#_meters
I am trying to upload the file using akka-http, It works while I try to upload it on the same system or cluster but now how do I upload it to specific server ?
(path("/uploadFile") & post) {
extractRequestContext {
ctx => {
implicit val materializer = ctx.materializer
implicit val ec = ctx.executionContext
fileUpload("fileUpload") {
case (fileInfo, fileStream) =>
val sink = FileIO.toPath(Paths.get("/tmp/sample.jar") resolve fileInfo.fileName)
val writeResult = fileStream.runWith(sink)
onSuccess(writeResult) { result =>
result.status match {
case Success(_) => complete(s"Successfully written ${result.count} bytes")
case Failure(e) => throw e
}
}
}
}
}
}