Non-Terminating ZeroMQ Load Balancer - scala

Given the following slightly modified Load Balancer ZeroMQ code:
package net.broker
import org.zeromq.ZMQ
object LruQueue2 {
class ClientTask(name: String) extends Runnable {
override def run(): Unit = {
val context = ZMQ.context(1)
val client = context.socket(ZMQ.REQ)
client.setIdentity(name.getBytes)
client.connect("tcp://localhost:5555")
// send request, get reply
client.send("HELLO".getBytes, 0)
val reply = client.recv(0)
println(s"${new String(client.getIdentity)} received: ${new String(reply)}")
}
}
class WorkerTask(name: String) extends Runnable {
override def run(): Unit = {
val context = ZMQ.context(1)
val worker = context.socket(ZMQ.REQ)
worker.connect("tcp://localhost:5556")
worker.setIdentity(name.getBytes)
worker.send("READY".getBytes, 0)
while(true) {
val clientAddr = worker.recv(0)
val empty = worker.recv(0)
val clientMsg = worker.recv(0)
worker.send(clientAddr, ZMQ.SNDMORE)
worker.send("".getBytes, ZMQ.SNDMORE)
worker.send("WORLD".getBytes, 0)
println(s"${new String(worker.getIdentity)}: 3-frames to client: ${new String(clientAddr)}")
}
}
}
def main(args: Array[String]): Unit = {
val NOFLAGS = 0
// worker using REQ socket to do LRU routing
val NBR_CLIENTS = 1
val NBR_WORKERS = 1
val context = ZMQ.context(1)
val frontend = context.socket(ZMQ.ROUTER)
val backend = context.socket(ZMQ.ROUTER)
frontend.bind("tcp://*:5555")
backend.bind("tcp://*:5556")
val clients = (1 to NBR_CLIENTS).toList.map{ i => new Thread(new ClientTask(s"CLIENT$i"))}
val workers = (1 to NBR_CLIENTS).toList.map{ i => new Thread(new WorkerTask(s"WORKER$i"))}
clients.foreach(_.start)
workers.foreach(_.start)
val workerQueue = scala.collection.mutable.Queue[Array[Byte]]()
val poller = context.poller(2)
poller.register(backend, ZMQ.Poller.POLLIN)
poller.register(frontend, ZMQ.Poller.POLLIN)
var clientNbr = NBR_CLIENTS
while(true) {
println("begin to poll")
poller.poll()
println("done polling")
println("clientNbr:" + clientNbr)
println("workerQueue.length: " + workerQueue.length)
if(clientNbr == 0) {
sys.exit(0)
}
else if(poller.pollin(0) && clientNbr > 0) {
val workerAddr = backend.recv(NOFLAGS)
val empty = backend.recv(NOFLAGS)
val clientAddrOrReadyMsg = backend.recv(NOFLAGS)
workerQueue.enqueue(workerAddr)
if(new String(clientAddrOrReadyMsg) == "READY") {
// nothing to do - worker is letting us know that he's ready to work
}
else {
// retrieve remaining 2 frames of client message
// [Empty][Client Message of "HELLO"]
val empty = backend.recv(0)
val workerResponse = backend.recv(0)
frontend.send(clientAddrOrReadyMsg, ZMQ.SNDMORE)
frontend.send("".getBytes, ZMQ.SNDMORE)
frontend.send(workerResponse, NOFLAGS)
clientNbr -= 1
}
}
else if (poller.pollin(1) && workerQueue.nonEmpty) {
val clientAddr = frontend.recv(0)
val empty = frontend.recv(0)
val clientMsg = frontend.recv(0)
backend.send(workerQueue.dequeue(), ZMQ.SNDMORE)
backend.send("".getBytes, ZMQ.SNDMORE)
backend.send(clientAddr, ZMQ.SNDMORE)
backend.send("".getBytes, ZMQ.SNDMORE)
backend.send(clientMsg, NOFLAGS)
}
else {}
}
}
}
When I run the above code, I see the following output:
[info] Running net.broker.LruQueue2
[info] begin to poll
[info] done polling
[info] clientNbr:1
[info] workerQueue.length: 0
[info] begin to poll
[info] done polling
[info] clientNbr:1
[info] workerQueue.length: 1
[info] begin to poll
[info] WORKER1: 3-frames to client: CLIENT1
[info] done polling
[info] clientNbr:1
[info] workerQueue.length: 0
[info] begin to poll
[info] CLIENT1 received: WORLD
It appears that it's stuck on poller.poll(), per the output print statements.
How can I debug this behavior?

Related

Why am I getting this timeout during unit test of akka-stream?

I have an akka-gRPC service BiDirectional stream and I am testing it on a unit test. The service has uses akka-stream and I use the TestSink.probe to test the reply message. I am receiving back the messages from the service, but there is an error related to timeout that I cannot figure out what is the reason. This is the test:
object GreeterServiceConf {
// important to enable HTTP/2 in server ActorSystem's config
val configServer = ConfigFactory.parseString("akka.http.server.preview.enable-http2 = on")
.withFallback(ConfigFactory.defaultApplication())
val configString2 =
"""
|akka.grpc.client {
| "helloworld.GreeterService" {
| host = 127.0.0.1
| port = 8080
| }
|}
|""".stripMargin
val configClient = ConfigFactory.parseString(configString2)
}
class GreeterServiceImplSpec extends TestKit(ActorSystem("GreeterServiceImplSpec", ConfigFactory.load(GreeterServiceConf.configServer)))
with AnyWordSpecLike
with BeforeAndAfterAll
with Matchers
with ScalaFutures {
implicit val patience: PatienceConfig = PatienceConfig(scaled(5.seconds), scaled(100.millis))
// val testKit = ActorTestKit(conf)
val serverSystem: ActorSystem = system
val bound = new GreeterServer(serverSystem).run()
// make sure server is bound before using client
bound.futureValue
implicit val clientSystem: ActorSystem = ActorSystem("GreeterClient", ConfigFactory.load(GreeterServiceConf.configClient))
val client = GreeterServiceClient(
GrpcClientSettings
.fromConfig("helloworld.GreeterService")
.withTls(false)
)
override def afterAll: Unit = {
TestKit.shutdownActorSystem(system)
TestKit.shutdownActorSystem(clientSystem)
}
"GreeterService" should {
"reply to multiple requests" in {
import GreeterServiceData._
val names = List("John", "Michael", "Simone")
val expectedReply: immutable.Seq[HelloReply] = names.map { name =>
HelloReply(s"Hello, $name -> ${mapHelloReply.getOrElse(name, "this person does not exist =(")}")
}
val requestStream: Source[HelloRequest, NotUsed] = Source(names).map(name => HelloRequest(name))
val responseStream: Source[HelloReply, NotUsed] = client.sayHelloToAll(requestStream)
val sink = TestSink.probe[HelloReply]
val replyStream = responseStream.runWith(sink)
replyStream
.requestNext(HelloReply(s"Hello, John -> I killed Java"))
.requestNext(HelloReply(s"Hello, Michael -> We are the Jacksons 5"))
.requestNext(HelloReply(s"Hello, Simone -> I have found a job to work with Scala =)")) // THIS IS THE LINE 122 ON THE ERROR
// .request(3)
// .expectNextUnorderedN(expectedReply) // I also tested this but it did not work
.expectComplete()
}
}
}
The error is:
assertion failed: timeout (3 seconds) during expectMsg while waiting
for OnComplete java.lang.AssertionError: assertion failed: timeout (3
seconds) during expectMsg while waiting for OnComplete at
scala.Predef$.assert(Predef.scala:223) at
akka.testkit.TestKitBase.expectMsg_internal(TestKit.scala:459) at
akka.testkit.TestKitBase.expectMsg(TestKit.scala:436) at
akka.testkit.TestKitBase.expectMsg$(TestKit.scala:436) at
akka.testkit.TestKit.expectMsg(TestKit.scala:969) at
akka.stream.testkit.TestSubscriber$ManualProbe.expectComplete(StreamTestKit.scala:479)
at
com.example.helloworld.GreeterServiceImplSpec.$anonfun$new$5(GreeterServiceImplSpec.scala:121)
I got it to work based on the project akka-grpc-quickstart-scala.g8. I am executing runForeach to run the graph and have a materialized Sink on the response stream. Then, when the response is done I am doing an assert inside the Future[Done].
"reply to multiple requests" in {
import GreeterServiceData._
import system.dispatcher
val names = List("John", "Martin", "Michael", "UnknownPerson")
val expectedReplySeq: immutable.Seq[HelloReply] = names.map { name =>
HelloReply(s"Hello, $name -> ${mapHelloReply.getOrElse(name, "this person does not exist =(")}")
}
// println(s"expectedReplySeq: ${expectedReplySeq.foreach(println)}")
val requestStream: Source[HelloRequest, NotUsed] = Source(names).map(name => HelloRequest(name))
val responseStream: Source[HelloReply, NotUsed] = client.sayHelloToAll(requestStream)
val done: Future[Done] = responseStream.runForeach { reply: HelloReply =>
// println(s"got streaming reply: ${reply.message}")
assert(expectedReplySeq.contains(reply))
}
// OR USING Sink.foreach[HelloReply])(Keep.right)
val sinkHelloReply = Sink.foreach[HelloReply] { e =>
println(s"element: $e")
assert(expectedReplySeq.contains(e))
}
responseStream.toMat(sinkHelloReply)(Keep.right).run().onComplete {
case Success(value) => println(s"done")
case Failure(exception) => println(s"exception $exception")
}
}
Just to keep the reference of the whole code, the GreeterServiceImplSpec class is here.

Router Hanging in Dealer-Router Setup

Given the following attempt to connect 1 DEALER to 1 ROUTER:
package net.async
import org.zeromq.ZMQ
import org.zeromq.ZMQ.Socket
import scala.annotation.tailrec
object Client {
val Empty = "".getBytes
def message(x: Int) = s"HELLO_#$x".getBytes
val Count = 5
}
class Client(name: String) extends Runnable {
import Client._
import AsyncClientServer.Port
override def run(): Unit = {
val context = ZMQ.context(1)
val dealer = context.socket(ZMQ.DEALER)
dealer.setIdentity(name.getBytes)
dealer.connect(s"tcp://localhost:$Port")
runHelper(dealer, Count)
}
#tailrec
private def runHelper(dealer: Socket, count: Int): Unit = {
dealer.send(dealer.getIdentity, ZMQ.SNDMORE)
dealer.send(Empty, ZMQ.SNDMORE)
dealer.send(message(count), 0)
println(s"Dealer: ${dealer.getIdentity} received message: " + dealer.recv(0))
runHelper(dealer, count - 1)
}
}
object AsyncClientServer {
val Port = 5555
val context = ZMQ.context(1)
val router = context.socket(ZMQ.ROUTER)
def main(args: Array[String]): Unit = {
router.bind(s"tcp://*:$Port")
mainHelper()
new Thread(new Client("Joe")).start()
}
private def mainHelper(): Unit = {
println("Waiting to receive messages from Dealer.")
val identity = router.recv(0)
val empty = router.recv(0)
val message = router.recv(0)
println(s"Router received message, ${new String(message)} from sender: ${new String(identity)}.")
mainHelper()
}
}
I see the following output, hanging on the second message.
[info] Running net.async.AsyncClientServer
[info] Waiting to receive messages from Dealer.
Why is that?
Not sure if its the cause of your problem but you don't need to send the identity frame from your dealer, zeromq will do this for you. By adding it your actually sending a 4 part message.
IDENTITY
IDENTITY
EMPTY
CONTENT

How to mock TemporaryFile moveTo() method?

I'm trying to create a test case for the controller below but I'm encountering some issues that I believe to be related to mocking methods correctly.
Issue:
Every time my test runs through this line:
file.ref.moveTo(new File(s"${configuration.uploadFileLocation}/$filename").getCanonicalFile,true)
It throws the following error:
Error:
[info] - run test *** FAILED ***
[info] java.lang.NullPointerException:
[info] at java.nio.file.Files.provider(Files.java:97)
[info] at java.nio.file.Files.move(Files.java:1392)
[info] at play.api.libs.Files$TemporaryFile.moveTo(Files.scala:100)
Controller:
def run = Action.async(parse.multipartFormData) { implicit request =>
request.body.file("file").map { file =>
import java.io.File
val filename = randomFileName
val contentType = file.contentType
if (contentType == Some("text/csv")) {
file.ref.moveTo(new File(s"${configuration.uploadFileLocation}/$filename").getCanonicalFile, true)
val result = apiClient.match (filename)
Future.successful("ok")
} else {
Future.successful("ok")
}
}.getOrElse {
Future.successful("ok")
}
}
Test case:
test("run() test") {
val mockMessageApi = mock(classOf[MessagesApi])
val mockApiClient = mock(classOf[apiClient])
val mockConfiguration = mock(classOf[Configuration])
val mockFile = mock(classOf[File])
val buildFakeRequest = FakeRequest(POST, "", FakeHeaders(), {
val tempFile = new TemporaryFile(mockFile)
val filePart = FilePart("file", "test.file", Some("text/csv"), tempFile)
MultipartFormData(Map(), List(filePart), List())
}
)
val controller = new MultipleMatch(mockApiClient, mockMessageApi, mockConfiguration)
val response = controller.run()(buildFakeRequest)
}
Question:
How can I effectively mock the ref and the moveTo method? What is causing this to happen? How to avoid the same issue again in the future?

Understanding 5 REQs to Router

Looking at the Scala code for this 5 REQ <--> 1 ROUTER setup:
Worker
class WorkerTask extends Runnable {
override def run: Unit = {
val rand = new Random(System.currentTimeMillis())
val context = ZMQ.context(1)
val worker = context.socket(ZMQ.REQ)
worker.connect("tcp://localhost:5555")
var total = 0
var workload = ""
do {
worker.send("Ready".getBytes, 0)
workload = new String(worker.recv(0))
Thread.sleep (rand.nextInt(1) * 1000)
total += 1
} while (workload.equalsIgnoreCase("END") == false)
printf("Completed: %d tasks\n", total)
}
}
main (Router)
def main(args: Array[String]): Unit = {
val NBR_WORKERS = 5
val context = ZMQ.context(1)
val client = context.socket(ZMQ.ROUTER)
assert(client.getType > -1)
client.bind("tcp://*:5555")
val workers = List.fill(NBR_WORKERS)(new Thread(new WorkerTask))
workers.foreach (_.start)
for (i <- 1 to (NBR_WORKERS * 10)) {
// LRU worker is next waiting in queue
val address = client.recv(0)
val empty = client.recv(0)
val ready = client.recv(0)
client.send(address, ZMQ.SNDMORE)
client.send("".getBytes, ZMQ.SNDMORE)
client.send("This is the workload".getBytes, 0)
}
for (i <- 1 to NBR_WORKERS) {
val address = client.recv(0)
val empty = client.recv(0)
val ready = client.recv(0)
client.send(address, ZMQ.SNDMORE)
client.send("".getBytes, ZMQ.SNDMORE)
client.send("END".getBytes, 0)
}
}
Running on my machine:
[info] Running net.server.RouterToReq
Completed: 21 tasks
Completed: 1 tasks
Completed: 27 tasks
Completed: 5 tasks
Completed: 1 tasks
As I partially understand the above code, 5 REQ workers accept requests from the ROUTER on port 5555.
Lastly, in the following code:
for (i <- 1 to NBR_WORKERS) {
val address = client.recv(0)
val empty = client.recv(0)
val ready = client.recv(0)
What message is the client, i.e. ROUTER, receiving?
The worker sends the message "Ready" repeatedly as a single frame message.
The REQ socket will add a blank delimiter frame on the front.
The ROUTER socket will prepend an identifier frame to the front of any received message to identify where it came from.
Thus the single ready message becomes a 3 frame message when you receive it on the router, this is why 3 recv calls are required.
When you send on a router socket the first frame will be removed and used to identify which client to send the message to. The REQ socket will remove all frames until it finds an empty frame and thus you only need a single recv call on the worker side.

ThreadPoolExecution error with scala, slick, specs2

I am trying to write a db unit test for my module using slick and specs2 in scala language. I have written two tests at the moment but the second one fails with ThreadPoolExecution error.
I have tried many approaches but still can't manage to solve my problem.
Any help would be greatly appreciated.
Here is the test code:
class UserClientSpecs extends Specification{
"App should " should {
"add value to database" in new WithApplication {
println("before first test")
val recordEntry = new UserClient(None, "Lohs_atkal", 2)
val newRecord = UserService.addUser(recordEntry)
newRecord.onComplete {
case Success(value) => println(s"Got the callback, meaning = $value")
case Failure(e) => println(e.getMessage)
}
newRecord should not beNull
val count = UserService.listAllUsers.map {
v =>
println("the number of database entries are " + v.length)
}
}
"delete a record" in new WithApplication {
println("before second test")
val recordEntry = new UserClient(Some(0), "Lielaks Lohs", 5)
val newRecord = UserService.addUser(recordEntry)
newRecord.map {
v => println("newRecord value", v)
}
newRecord.onComplete {
case Success(value) => println(s"Got the callback, meaning = $value")
case Failure(e) => println(e.getMessage)
}
val recordEntry2 = new UserClient(Some(1), "Lielaks Lohs2", 50)
val newRecord2 = UserService.addUser(recordEntry2)
val countOne = UserService.listAllUsers.map {
res =>
println(res.length)
}
val deleteUser = UserService.deleteUser(1)
val countTwo = UserService.listAllUsers.map {
res =>
res should_==(1)
res should !==(2)
}
}
}
}
And the error I am getting when running my test through sbt -> testOnly:
[play-scala] $ testOnly models.UserClientSpecs
[info] UserClientSpecs
[info]
before first test
Got the callback, meaning = Some(UserClient(None,Lohs_atkal,2))
the number of database entries are 1
[info] App should should
[info] + add value to database
before second test
Task slick.backend.DatabaseComponent$DatabaseDef$$anon$2#6d01394 rejected from java.util.concurrent.ThreadPoolExecutor#7fa5be9b[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 2]
[info] + delete a record
[info]
[info] Total for specification UserClientSpecs
[info] Finished in 2 seconds, 739 ms
[info] 2 examples, 0 failure, 0 error
[info]
[info] Passed: Total 2, Failed 0, Errors 0, Passed 2
[success] Total time: 8 s, completed Mar 10, 2016 1:40:10 PM