I am trying to stream tweets with twitter4j3.0.3 with scala but it gives me these errors.
Here is my code:
import twitter4j._
import ch.qos.logback.core.status.StatusListener
import twitter4j.conf.ConfigurationBuilder
import ch.qos.logback.core.status
object stream {
def main(args: Array[String]) {
val cb: ConfigurationBuilder = new ConfigurationBuilder
cb.setDebugEnabled(true)
.setOAuthConsumerKey("1")
.setOAuthConsumerSecret("1")
.setOAuthAccessToken("1")
.setOAuthAccessTokenSecret("1")
def simpleStatusListener:StatusListener =new StatusListener() {
def addStatusEvent(status: Status) {println(x = status.getText)}
def onStatus(status: Status) { println(x = status.getText) }
def onDeletionNotice(statusDeletionNotice: StatusDeletionNotice) {}
def onTrackLimitationNotice(numberOfLimitedStatuses: Int) {}
def onException(ex: Exception) { ex.printStackTrace }
def onScrubGeo(arg0: Long, arg1: Long) {}
def onStallWarning(warning: StallWarning) {}
}
val twitterStream:TwitterStream= new TwitterStreamFactory(cb.build).getInstance()
twitterStream.addListener(simpleStatusListener)
twitterStream.sample()
}
}
and the error:
overloaded method value addListener with alternatives:
(twitter4j.RawStreamListener)Unit
(twitter4j.SiteStreamsListener)Unit
(twitter4j.StatusListener)Unit
(twitter4j.UserStreamListener)Unit
cannot be applied to (ch.qos.logback.core.status.StatusListener)
twitterStream.addListener(simpleStatusListener)
^
You're importing the wrong StatusListener interface.
Instead of
import ch.qos.logback.core.status.StatusListener
You need
import twitter4j.StatusListener
Related
Following the Akka tutorials https://doc.akka.io/docs/akka/current/typed/guide/tutorial_1.html I have modified sample code to send a message to an akka actor every 3 seconds:
scheduler.scheduleAtFixedRate(
initialDelay = Duration(0, TimeUnit.SECONDS),
interval = Duration(3, TimeUnit.SECONDS))(
runnable = task)
}
I'm unable to safely compiling changing the message return in Main from String to List[String]. So instead of firstRef ! "printit" , change to firstRef ! List("printit") but this causes compiler error:
To fix the compiler error I make changes to:
class Main(context: ActorContext[String]) extends AbstractBehavior[List[String]](context) {
override def onMessage(msg: String): Behavior[List[String]] =
def apply(): Behavior[List[String]] =
where previously this contained:
class Main(context: ActorContext[String]) extends AbstractBehavior[String](context) {
override def onMessage(msg: String): Behavior[String] =
def apply(): Behavior[String] =
What needs to change in order to return List[String] instead of String in the Main actor ?
complete code (without changes):
import java.util.concurrent.TimeUnit
import akka.actor.typed.ActorSystem
import akka.actor.typed.Behavior
import akka.actor.typed.scaladsl.AbstractBehavior
import akka.actor.typed.scaladsl.ActorContext
import akka.actor.typed.scaladsl.Behaviors
import map.QTableRow
import scala.concurrent.ExecutionContext
import scala.concurrent.duration._
import ExecutionContext.Implicits.global
import scala.collection.mutable.ListBuffer
object PrintMyActorRefActor {
def apply(): Behavior[String] =
Behaviors.setup(context => new PrintMyActorRefActor(context))
}
class PrintMyActorRefActor(context: ActorContext[String]) extends AbstractBehavior[String](context) {
override def onMessage(msg: String): Behavior[String] =
msg match {
case "printit" =>
val secondRef = context.spawn(Behaviors.empty[String], "second-actor")
println(s"Second: $secondRef")
this
}
}
object Main {
def apply(): Behavior[String] =
Behaviors.setup(context => new Main(context))
}
class Main(context: ActorContext[String]) extends AbstractBehavior[String](context) {
override def onMessage(msg: String): Behavior[String] =
msg match {
case "getdata" =>
val firstRef = context.spawn(PrintMyActorRefActor(), "first-actor"+String.valueOf(System.currentTimeMillis()))
println(s"First: $firstRef")
firstRef ! "printit"
this
}
}
object ActorHierarchyExperiments extends App {
val testSystem = ActorSystem(Main(), "testSystem")
val scheduler = testSystem.scheduler
val task = new Runnable { def run() {
testSystem ! "getdata" }
}
scheduler.scheduleAtFixedRate(
initialDelay = Duration(0, TimeUnit.SECONDS),
interval = Duration(3, TimeUnit.SECONDS))(
runnable = task)
}
The firstRef actor reference references an instance of PrintMyActorRefActor, which is a Behavior[ String ], so to send any List[ String ]to the ref, the PrintMyActorRefActor needs to be able to recieve it. Changing this, including the signature of its recieve message, should make the snippet compile. Now, of course, you'll need to change the behaviour of the PrintMyActorRefActor.onMessage( msg: List[ String ] ): Behavior[ List[ String ] ] to deal with the list instead of the single string...
Complete transformed code snippet:
import java.util.concurrent.TimeUnit
import akka.actor.typed.scaladsl.{AbstractBehavior, ActorContext, Behaviors}
import akka.actor.typed.{ActorSystem, Behavior}
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration._
object PrintMyActorRefActor {
def apply(): Behavior[List[String]] =
Behaviors.setup(context => new PrintMyActorRefActor(context))
}
class PrintMyActorRefActor(context: ActorContext[List[String]]) extends AbstractBehavior[List[String]](context) {
override def onMessage(msg: List[String]): Behavior[List[String]] =
msg match {
case "printit" :: xs => // ignores all but the head element
val secondRef = context.spawn(Behaviors.empty[String], "second-actor")
println(s"Second: $secondRef")
this
}
}
object Main {
def apply(): Behavior[String] =
Behaviors.setup(context => new Main(context))
}
class Main(context: ActorContext[String]) extends AbstractBehavior[String](context) {
override def onMessage(msg: String): Behavior[String] =
msg match {
case "getdata" =>
val firstRef = context.spawn(PrintMyActorRefActor(), "first-actor" + String.valueOf(System.currentTimeMillis()))
println(s"First: $firstRef")
firstRef ! List("printit")
this
}
}
object ActorHierarchyExperiments extends App {
val testSystem = ActorSystem(Main(), "testSystem")
val scheduler = testSystem.scheduler
val task = new Runnable {
def run() {
testSystem ! "getdata"
}
}
scheduler.scheduleAtFixedRate(
initialDelay = Duration(0, TimeUnit.SECONDS),
interval = Duration(3, TimeUnit.SECONDS))(
runnable = task)
}
I am using kafka_2.10-0.10.0.1 and scala_2.10.3.
I want to write custom Serializer and Deserializer using scala.
I tried with these Serializer (from CustomType) and Deserializer (obtain a CustomType):
class CustomTypeSerializer extends Serializer[CustomType] {
private val gson: Gson = new Gson()
override def configure(configs: util.Map[String, _], isKey: Boolean):
Unit = {
// nothing to do
}
override def serialize(topic: String, data: CustomType): Array[Byte] = {
if (data == null)
null
else
gson.toJson(data).getBytes
}
override def close(): Unit = {
//nothing to do
}
}
class CustomTypeDeserializer extends Deserializer[CustomType] {
private val gson: Gson = new Gson()
override def deserialize(topic: String, bytes: Array[Byte]): CustomType = {
val offerJson = gson.toJson(bytes.toString)
val psType: Type = new TypeToken[CustomType]() {}.getType()
val ps: CustomType = gson.fromJson(offerJson, psType)
ps
}
override def configure(configs: util.Map[String, _], isKey: Boolean):
Unit = {
// nothing to do
}
override def close(): Unit = {
//nothing to do
}
}
But, I got this error:
Exception in thread "main" org.apache.kafka.common.errors.SerializationException: Error deserializing key/value for partition topic_0_1-1 at offset 26
Caused by: com.google.gson.JsonSyntaxException: java.lang.IllegalStateException: Expected BEGIN_OBJECT but was BEGIN_ARRAY at line 1 column 2 path $
at com.google.gson.internal.bind.ReflectiveTypeAdapterFactory$Adapter.read(ReflectiveTypeAdapterFactory.java:224)
at com.google.gson.Gson.fromJson(Gson.java:887)
at com.google.gson.Gson.fromJson(Gson.java:852)
at com.google.gson.Gson.fromJson(Gson.java:801)
at kafka.PSDeserializer.deserialize(PSDeserializer.scala:24)
at kafka.PSDeserializer.deserialize(PSDeserializer.scala:18)
at org.apache.kafka.clients.consumer.internals.Fetcher.parseRecord(Fetcher.java:627)
at org.apache.kafka.clients.consumer.internals.Fetcher.parseFetchedData(Fetcher.java:548)
at org.apache.kafka.clients.consumer.internals.Fetcher.fetchedRecords(Fetcher.java:354)
at org.apache.kafka.clients.consumer.KafkaConsumer.pollOnce(KafkaConsumer.java:1000)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:938)
Can you help me please
Find below the custom serializer and deserializer for case class User, User(name:String,id:Int). Replace User in code with your case class. It will work.
import java.io.{ObjectInputStream, ByteArrayInputStream}
import java.util
import org.apache.kafka.common.serialization.{Deserializer, Serializer}
class CustomDeserializer extends Deserializer[User]{
override def configure(configs: util.Map[String,_],isKey: Boolean):Unit = {
}
override def deserialize(topic:String,bytes: Array[Byte]) = {
val byteIn = new ByteArrayInputStream(bytes)
val objIn = new ObjectInputStream(byteIn)
val obj = objIn.readObject().asInstanceOf[User]
byteIn.close()
objIn.close()
obj
}
override def close():Unit = {
}
}
import java.io.{ObjectOutputStream, ByteArrayOutputStream}
import java.util
import org.apache.kafka.common.serialization.Serializer
class CustomSerializer extends Serializer[User]{
override def configure(configs: util.Map[String,_],isKey: Boolean):Unit = {
}
override def serialize(topic:String, data:User):Array[Byte] = {
try {
val byteOut = new ByteArrayOutputStream()
val objOut = new ObjectOutputStream(byteOut)
objOut.writeObject(data)
objOut.close()
byteOut.close()
byteOut.toByteArray
}
catch {
case ex:Exception => throw new Exception(ex.getMessage)
}
}
override def close():Unit = {
}
}
When using Neo4j unmanaged extensions, one can stream results to the client while traversing the graph like this (in Scala):
import javax.ws.rs.core.{MediaType, Response, StreamingOutput}
val stream: StreamingOutput = ???
Response.ok().entity(stream).`type`(MediaType.APPLICATION_JSON).build()
I can't find a similar possibility when using Neo4j 3 used-defined stored procedures. They return Java 8 Streams but I can't see how I could add elements to such streams while they already being consumed, in parallel.
Is it possible?
I have an example of that in one of the APOC procedures.
https://github.com/neo4j-contrib/neo4j-apoc-procedures/blob/master/src/main/java/apoc/cypher/Cypher.java#L77
I want to add more / a more general example of that in the future.
Here is what I came up with based on Michael Hunger code (in Scala).
QueueBasedSpliterator:
import java.util.Spliterator
import java.util.concurrent.{BlockingQueue, TimeUnit}
import java.util.function.Consumer
import org.neo4j.kernel.api.KernelTransaction
private class QueueBasedSpliterator[T](queue: BlockingQueue[T],
tombstone: T,
tx: KernelTransaction) extends Spliterator[T] {
override def tryAdvance(action: Consumer[_ >: T]): Boolean =
try {
if (tx.shouldBeTerminated()) false
else {
val entry = queue.poll(100, TimeUnit.MILLISECONDS)
if (entry == null || entry == tombstone) false
else {
action.accept(entry)
true
}
}
} catch {
case e: InterruptedException => false
}
override def trySplit(): Spliterator[T] = null
override def estimateSize(): Long = Long.MaxValue
override def characteristics(): Int = Spliterator.ORDERED | Spliterator.NONNULL
}
Notice the 100 ms timeout value. Might require tuning.
ResultsStream (wrapper around blocking queue):
import java.util.concurrent.BlockingQueue
class ResultsStream[T](tombstone: T, queue: BlockingQueue[T]) extends AutoCloseable {
def put(t: T): Unit = {
queue.put(t)
}
override def close(): Unit = {
put(tombstone)
}
}
CommonUtil helper methods:
import java.util.concurrent.ArrayBlockingQueue
import java.util.stream.{Stream, StreamSupport}
import org.neo4j.kernel.api.KernelTransaction
import org.neo4j.kernel.internal.GraphDatabaseAPI
import scala.concurrent.{ExecutionContext, Future}
object CommonUtil {
def inTx(db: GraphDatabaseAPI)(f: => Unit): Unit =
Managed(db.beginTx()) { tx => f; tx.success() }
def inTxFuture(db: GraphDatabaseAPI)(f: => Unit)(implicit ec: ExecutionContext): Future[Unit] =
Future(inTx(db)(f))
def streamResults[T](tombstone: T, tx: KernelTransaction)
(f: ResultsStream[T] => Any): Stream[T] = {
val queue = new ArrayBlockingQueue[T](100)
f(new ResultsStream(tombstone, queue))
StreamSupport.stream(new QueueBasedSpliterator[T](queue, tombstone, tx), false)
}
}
Some more helpers:
object Managed {
type AutoCloseableView[T] = T => AutoCloseable
def apply[T : AutoCloseableView, V](resource: T)(op: T => V): V =
try {
op(resource)
} finally {
resource.close()
}
}
Pool:
import java.util.concurrent.{ArrayBlockingQueue, ThreadPoolExecutor, TimeUnit}
import scala.concurrent.{ExecutionContext, ExecutionContextExecutor}
object Pool {
lazy val DefaultExecutionContent: ExecutionContextExecutor =
ExecutionContext.fromExecutor(createDefaultExecutor())
// values might be tuned in production
def createDefaultExecutor(corePoolSize: Int = Runtime.getRuntime.availableProcessors() * 2,
keepAliveSeconds: Int = 30) = {
val queueSize = corePoolSize * 25
new ThreadPoolExecutor(
corePoolSize / 2,
corePoolSize,
keepAliveSeconds.toLong,
TimeUnit.SECONDS,
new ArrayBlockingQueue[Runnable](queueSize),
new ThreadPoolExecutor.CallerRunsPolicy()
)
}
}
Usage in a procedure:
#Procedure("example.readStream")
def readStream(#Name("nodeId") nodeId: NodeId): Stream[StreamingItem] =
CommonUtil.streamResults(StreamingItem.Tombstone, kernelTx) { results =>
CommonUtil.inTxFuture(db) { // uses Pool.DefaultExecutionContent
Managed(results) { _ =>
graphUtil.findTreeNode(nodeId).foreach { node =>
// add elements to the stream here
results.put(???)
}
}
}
}
StreamingItem.Tombstone is just a static StreamingItem instance with special meaning to close the stream. db and kernelTx are just context variable set by Neo4j:
#Context
public GraphDatabaseAPI db;
#Context
public KernelTransaction kernelTx;
I am learning twitter's RPC asynchronous system and it's important use filter and service deal with task logical.I have read the paper of https://twitter.github.io/finagle/guide/ServicesAndFilters.html and try write a simple code to make true I knows. But there some problem:
import com.twitter.util.Future
trait Service[Req, Rep] extends (Req => Future[Rep])
abstract class Filter[-ReqIn, +RepOut, +ReqOut, -RepIn]
extends ((ReqIn, Service[ReqOut, RepIn]) => Future[RepOut])
trait SimpleFilter[Req, Rep] extends Filter[Req, Rep, Req, Rep]
class DoubleFilter(times: Int) extends SimpleFilter[(Int,Int), Int] {
def apply(request: (Int,Int), service: Service[(Int,Int), Int]): Future[Int] = {
service((request._1 * times, request._2 * times))
}
}
class AddFilter(count: Int) extends SimpleFilter[(Int, Int), Int] {
def apply(request: (Int, Int), service: Service[(Int, Int),Int]): Future[Int] = {
service((request._1 + count, request._2 + count))
}
}
object Test extends App {
val doubleFilter = new DoubleFilter(2)
val addFilter = new AddFilter(10)
//req is tuple, rep is added count of int
val addELemsService = new Service[(Int,Int), Int] {
def apply(req: (Int,Int)) = {
Future {
req._1 + req._2
}
}
}
val serviceWithDoble = doubleFilter andThen addELemsService
val doubleWithAddFilter = doubleFilter andThen addFilter
}
Result with two compile error: value andThen is not a member of DoubleFilter!
What I miss understand? I'm straight think doc ignore something but I mislead.
I think You are importing wrong trait/class. I tried compiling following code with proper imports and it compiled without errors.
import com.twitter.finagle.{Service, Filter}
import com.twitter.util.Future
object Test extends App {
trait SimpleFilter[Req, Rep] extends Filter[Req, Rep, Req, Rep]
class DoubleFilter(times: Int) extends SimpleFilter[(Int, Int), Int] {
def apply(request: (Int, Int), service: Service[(Int, Int), Int]): Future[Int] = {
service((request._1 * times, request._2 * times))
}
}
class AddFilter(count: Int) extends SimpleFilter[(Int, Int), Int] {
def apply(request: (Int, Int), service: Service[(Int, Int), Int]): Future[Int] = {
service((request._1 + count, request._2 + count))
}
}
val doubleFilter = new DoubleFilter(2)
val addFilter = new AddFilter(10)
//req is tuple, rep is added count of int
val addELemsService = new Service[(Int, Int), Int] {
def apply(req: (Int, Int)) = {
Future {
req._1 + req._2
}
}
}
val serviceWithDoble = doubleFilter andThen addELemsService
val doubleWithAddFilter = doubleFilter andThen addFilter
}
Going from the following macro implementation, which is working as expected, i would like to remove the hard coded Account and replace it with a variable beeing passed as type parameter T:c.WeakTypeTag to the macro by the caller.
how should the macro be modified so that i can pass any type T?
`
import scala.language.experimental.macros
import scala.reflect.macros.whitebox.Context
import com.xy.iws.model.Model._
import com.xy.iws.repo.dao.DAO._
import com.xy.iws.service.run
import com.xy.iws.service.Api
import com.xy.iws.service.Request._
object makeService {
def make[T]:Api[Account] = macro makeImpl[T]
def makeImpl[T:c.WeakTypeTag](c: Context): c.Expr[Api[Account]] = c.universe.reify {
//val source = c.weakTypeOf[T]
import com.xy.iws.service.Request._
implicit val api = new Api[Account] {
def create():Int = {run.runG[Int](Create[Account])}
def all(): List[Account] = run.runG[List[Account]](new FindAll[Account])
def insert(model: List[Account]) = run.runG[Int](new Insert[Account](model))
def findSome(id: String): List[Account] = run.runG[List[Account]](new FindSome[Account](id))
def find(id: String): List[Account] = run.runG[List[Account]](new Find[Account](id))
def update(item: Account): List[Account] = {
val i = run.runG[List[Account]](new Find[Account](item.id))
if (i.size > 0) {
val updateAccount = run.runG[Int](new Update[Account](item.id, item.name))
}else {
insert(List(item))
}
run.runG[List[Account]](new FindAll[Account])
}
def delete(id: String): List[Account] = {
run.runG[Int](new Delete[Account](id))
run.runG[List[Account]](new FindAll[Account])
}
}
api
}
}
`
maybe you can use quasiquotes
abstract class Api[T] {
def a: Int
def b: Int
def t: List[T]
}
object Reify {
def apply[T](initValue: T): Api[T] = macro impl[T]
def impl[T: c.WeakTypeTag](c: Context)(initValue: c.Expr[T]): c.Tree = {
import c.universe._
val t = c.weakTypeOf[T].typeSymbol.name.toTypeName
q"""
val api = new Api[$t] {
def a = 1
def b = 2
override def t= List(${initValue})
}
api
"""
}
}
---use like this
object ReifyUsing extends App {
import macross.Reify
import macross.Api
println(Reify.apply[String]("String").t)
}
Thank you that was the hint i was looking for as answer to my question:
How to pass pass parameter in particular type parameter to the implementation of a macro?
Short answer: Use quasiquote
See the implementation below that did the job
`
import scala.language.experimental.macros
import scala.reflect.macros.whitebox.Context
import com.xy.iws.model.Model._
import com.xy.iws.repo.dao.DAO._
object makeService {
import com.xy.iws.service.Api
import com.xy.iws.service.Request
def make[T]:Api[T] = macro makeImpl[T]
def makeImpl[T:c.WeakTypeTag](c: Context): c.Tree = {
import c.universe._
val t = c.weakTypeOf[T].typeSymbol.name.toTypeName
q"""
implicit val api = new Api[$t] {
override def create():Int = {run.runG[Int](Create[$t])}
override def all(): List[$t] = run.runG[List[$t]](new FindAll[$t])
override def insert(model: List[$t]) = run.runG[Int](new Insert[$t](model))
override def findSome(id: String): List[$t] = run.runG[List[$t]](new FindSome[$t](id))
override def find(id: String): List[$t] = run.runG[List[$t]](new Find[$t](id))
override def update(item: $t): List[$t] = {
val i = run.runG[List[$t]](new Find[$t](item.id))
if (i.size > 0) {
run.runG[Int](new Update[$t](item.id, item.name))
}else {
insert(List(item))
}
run.runG[List[$t]](new FindAll[$t])
}
override def delete(id: String): List[$t] = {
run.runG[Int](new Delete[$t](id))
run.runG[List[$t]](new FindAll[$t])
}
}
api
"""
}
}
`
and i use the macro like this:
object Main {
def main(args: Array[String])
{
import com.xy.iws.service.Api
println(makeService.make[Account].all +" account objects")
println(makeService.make[Article].all +" article objects")
println(makeService.make[Supplier].all +" supplier objects")
}
}