Empty RDD while consuming messages from Rabbitmq in sparks scala - scala

Question

Below is the spark streaming code to consume messages from RabbitMQ.
import java.io.{ BufferedReader, InputStreamReader }
import java.net.Socket
import java.nio.charset.StandardCharsets
import org.apache.spark.SparkConf
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.{ Seconds, StreamingContext }
import org.apache.spark.streaming.receiver.Receiver
class CustomReceiver
extends Receiver[String](StorageLevel.MEMORY_AND_DISK_2) {
var factory: ConnectionFactory = _
var consumer: QueueingConsumer = _
var rabbitMQconnection: Connection = null;
var channel: Channel = null
var host = "****"
var port = "****"
var queueName = "test"
var virtualHost = "host"
var userName = "name"
var password = "****"
def onStart() {
// Start the thread that receives data over a connection
new Thread("Socket Receiver") {
override def run() { receive() }
}.start()
}
def onStop() {
// There is nothing much to do as the thread calling receive()
// is designed to stop by itself isStopped() returns false
}
/** Create a socket connection and receive data until receiver is stopped */
private def receive() {
var socket: Socket = null
var userInput: String = null
try {
val QUEUE_NAME = "test"
if (rabbitMQconnection == null) {
rabbitMQconnection = getRabbitMQConnection
channel = rabbitMQconnection.createChannel();
println("rabbitMQ Connection created");
if (rabbitMQconnection != null) {
println("rabbitMQ Connection created")
}
}
val delayArgs = Map[String, AnyRef]("x-message-ttl" -> Long.box(50000))
channel.queueDeclare("test",true,false,false,delayArgs)
consumer = new QueueingConsumer(channel);
channel.basicConsume("test",true,consumer)
val delivery = consumer.nextDelivery()
val ty = unzip(delivery.getBody())
println("at consumer : " +ty)
store(ty)
channel.basicAck(delivery.getEnvelope().getDeliveryTag(), false);
} catch {
case e: java.net.ConnectException =>
restart("Error connecting to ")
case t: Throwable =>
restart("Error receiving data", t)
}
}
def getRabbitMQConnection()={
val factory = new ConnectionFactory()
factory.setUsername("name")
factory.setPassword("password")
factory.setHost("****")
factory.setVirtualHost("*****")
factory.setAutomaticRecoveryEnabled(true)
factory.setTopologyRecoveryEnabled(true)
factory.newConnection()
}
def unzip(x: Array[Byte]) : String = {
val inputStream = new GZIPInputStream(new ByteArrayInputStream(x))
val output = scala.io.Source.fromInputStream(inputStream).mkString
return output
}
}
import org.apache.spark._
import org.apache.spark.storage._
import org.apache.spark.streaming._
import org.apache.spark.streaming.receiver._
import org.apache.spark.sql._
import org.apache.spark.sql.functions._
val batchInterval = Seconds(10)
val ssc = new StreamingContext(sc, batchInterval)
val stream = ssc.receiverStream(new CustomReceiver())
stream.foreachRDD(rdd =>{ print(rdd)
})
ssc.start()
ssc.awaitTermination()
This is giving me empty output as below.When I tried consuming messages using Python pika library(without using spark streaming context) I was able to consume the data and save the messages to storage.
Time: 1642473060000 ms
-------------------------------------------
-------------------------------------------
Time: 1642473070000 ms
-------------------------------------------
Could someone please let me know whats wrong in the above code.

Answer 1

Changing the x-message-ttl setting to the desired value helped me resolved the issue.x-message-ttl value should be same as the value which is declared in the rabbitmq queue.