Im trying to rewrite a httpclient through the java11 HttpClient in Scala
Here is my code:
import cats.effect._
import java.net.http._
import java.net.http.HttpResponse._
import java.net.http.HttpClient._
trait HttpClients[F[_]] {
def send(req: HttpRequest)(implicit F: Async[F]): F[HttpResponse[_]]
}
object HttpClients {
val client: HttpClient = HttpClient.newBuilder().followRedirects(Redirect.ALWAYS).build()
def newClient[F[_] : Async](): HttpClients[F] = new HttpClients[F] {
override def send(req: HttpRequest)(implicit F: Async[F]): F[HttpResponse[_]] = F.async { cb =>
val resp = client.sendAsync(req, BodyHandlers.ofString())
val s = resp.handle((res: HttpResponse[String], err: Throwable) => {
if (err == null)
cb(Right(res))
else
cb(Left(err))
})
s // TODO ?
// Type missmatch
// Required: F[Option[F[Unit]]]
// Found: Unit
}
}
}
the handle callback from this
I guess the error comes from here, but I don't know how to write next.
Then I make some change:
def newClient[F[_] : Async](): HttpClients[F] = new HttpClients[F] {
override def send(req: HttpRequest)(implicit F: Async[F]): F[HttpResponse[_]] = F.async[HttpResponse[_]] { cb =>
val s = Sync[F](F: Async[F]).delay {
val resp = client.sendAsync(req, BodyHandlers.ofString())
resp.handle((res: HttpResponse[String], err: Throwable) => {
if (err == null)
cb(Right(res))
else
cb(Left(err))
}).join()
}
F.delay(s.some)
}
}
This time, there is no error, but I don't know how to get the response's body
Thanks for your reply!
#OlegPyzhcov already provided insight in case you are using CE3, this answer is using CE2 in case that is what you wanted.
The first version of the code was correct, here is a full running example using Ammonite with some style improvements and ensuring a new client is created for each call and evaluation of newClient
// scala 2.13.5
import $ivy.`org.typelevel::cats-effect:2.5.0`
import cats.effect.{Async, IO}
import cats.syntax.all._
import java.net.URI
import java.net.http.{HttpClient, HttpRequest, HttpResponse}
trait HttpClients[F[_]] {
def send(req: HttpRequest): F[HttpResponse[String]]
}
object HttpClients {
def newClient[F[_]](implicit F: Async[F]): F[HttpClients[F]] =
F.delay {
HttpClient
.newBuilder
.followRedirects(HttpClient.Redirect.ALWAYS)
.build()
} map { client =>
new HttpClients[F] {
override def send(req: HttpRequest): F[HttpResponse[String]] =
F.async { cb =>
client.sendAsync(req, HttpResponse.BodyHandlers.ofString).handle {
(res: HttpResponse[String], err: Throwable) =>
if (err == null) cb(Right(res))
else cb(Left(err))
}
}
}
}
}
object Main {
private val request =
HttpRequest
.newBuilder
.GET
.uri(URI.create("https://stackoverflow.com/questions/tagged/scala?tab=Newest"))
.build()
private val program = for {
_ <- IO.delay(println("Hello, World!"))
client <- HttpClients.newClient[IO]
response <- client.send(request)
_ <- IO.delay(println(response))
_ <- IO.delay(println(response.body))
} yield ()
def run(): Unit = {
program.unsafeRunSync()
}
}
#main
def main(): Unit = {
Main.run()
}
Related
I have publisher with stream of messages. And i should send for consumers only messages which are based on consumer subsribers.
In my local is working for 2 tread without load, but on load testing is not working, many messages goes in not right way.
I have 2 versions of problem:
I new in akka stream and don't see in doc a little bit same case, joining 2 streams
I use not in right way scala tread safe options
What it can be?
import akka._
import akka.http.scaladsl.model.http2.PeerClosedStreamException
import akka.stream._
import akka.stream.scaladsl._
import scala.collection.mutable
import scala.concurrent.Future
import scala.util.matching.Regex
class RtkBrokerImpl(materializer: Materializer) extends MessageBroker {
private implicit val mat: Materializer = materializer
val mutableArray = mutable.Map[String, Source[ConsumeRequest, NotUsed]]()
val (inboundHub: Sink[ProduceRequest, NotUsed], outboundHub: Source[ConsumeResponse, NotUsed]) =
MergeHub.source[ProduceRequest]
.async
.mapAsyncUnordered(100)(x => Future.successful(ConsumeResponse(x.key, x.payload)))
.toMat(BroadcastHub.sink)(Keep.both)
.run()
val all: RunnableGraph[Source[ProduceRequest, NotUsed]] =
MergeHub.source[ProduceRequest]
.toMat(BroadcastHub.sink)(Keep.right)
val queue = Sink.queue[ProduceRequest](100)
override def produce(in: Source[ProduceRequest, NotUsed]): Future[ProduceResponse] = {
in.to(inboundHub).run()
Future.never
}
override def consume(in: Source[ConsumeRequest, NotUsed]): Source[ConsumeResponse, NotUsed] = {
val patternRepo = new PatternsRepository
in.runForeach { req =>
req.action match {
case ConsumeRequest.Action.SUBSCRIBE => patternRepo.putPatterns(req.keys)
case ConsumeRequest.Action.UNSUBSCRIBE => patternRepo.dropPatterns(req.keys)
}
}
outboundHub.async.filter(res => patternRepo.checkKey(res.key))
}
}
class PatternsRepository {
import RtkBrokerImpl._
import scala.collection.JavaConverters._
val concurrentSet: mutable.Set[String] = java.util.concurrent.ConcurrentHashMap.newKeySet[String]().asScala
def getPatterns(): mutable.Set[String] = {
concurrentSet
}
def checkKey(key: String): Boolean = {
concurrentSet.contains(key) ||
concurrentSet.exists(x => isInPattern(key, x))
}
def dropPatterns(string: Seq[String]) = {
string.foreach(concurrentSet.remove)
}
def putPatterns(string: Seq[String]) = {
concurrentSet.addAll(string)
}
}
object RtkBrokerImpl {
def isInPattern(key: String, pattern: String): Boolean = {
if (pattern.exists(x => x == '#' || x == '*')) {
val splittedPatten = pattern.split(".")
val splittedKey = key.split(".")
if (!splittedPatten.contains("#")) {
if (splittedKey.size != splittedKey.size)
return false
splittedPatten.zip(splittedKey)
.forall { case (key, pat) => if (pat == "*") true else key == pat }
} else {
val regExp = pattern.replaceAll(".", "\\.")
.replaceAll("\\*", "[A-Za-z0-9]+")
.replaceAll("#", "\\S+")
new Regex(regExp).matches(key)
}
} else {
key == pattern
}
}
}
The following Scala code uses cats EitherT to wrap results in a Future[Either[ServiceError, T]]:
package com.example
import com.example.AsyncResult.AsyncResult
import cats.implicits._
import scala.concurrent.ExecutionContext.Implicits.global
class ExternalService {
def doAction(): AsyncResult[Int] = {
AsyncResult.success(2)
}
def doException(): AsyncResult[Int] = {
println("do exception")
throw new NullPointerException("run time exception")
}
}
class ExceptionExample {
private val service = new ExternalService()
def callService(): AsyncResult[Int] = {
println("start callService")
val result = for {
num <- service.doException()
} yield num
result.recoverWith {
case ex: Throwable =>
println("recovered exception")
AsyncResult.success(99)
}
}
}
object ExceptionExample extends App {
private val me = new ExceptionExample()
private val result = me.callService()
result.value.map {
case Right(value) => println(value)
case Left(error) => println(error)
}
}
AsyncResult.scala contains:
package com.example
import cats.data.EitherT
import cats.implicits._
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.Future
object AsyncResult {
type AsyncResult[T] = EitherT[Future, ServiceError, T]
def apply[T](fe: => Future[Either[ServiceError, T]]): AsyncResult[T] = EitherT(fe)
def apply[T](either: Either[ServiceError, T]): AsyncResult[T] = EitherT.fromEither[Future](either)
def success[T](res: => T): AsyncResult[T] = EitherT.rightT[Future, ServiceError](res)
def error[T](error: ServiceError): AsyncResult[T] = EitherT.leftT[Future, T](error)
def futureSuccess[T](fres: => Future[T]): AsyncResult[T] = AsyncResult.apply(fres.map(res => Right(res)))
def expectTrue(cond: => Boolean, err: => ServiceError): AsyncResult[Boolean] = EitherT.cond[Future](cond, true, err)
def expectFalse(cond: => Boolean, err: => ServiceError): AsyncResult[Boolean] = EitherT.cond[Future](cond, false, err)
}
ServiceError.scala contains:
package com.example
sealed trait ServiceError {
val detail: String
}
In ExceptionExample, if it call service.doAction() it prints 2 as expected, but if it call service.doException() it throws an exception, but I expected it to print "recovered exception" and "99".
How do I recover from the exception correctly?
That is because doException is throwing exception inline. If you want to use Either, you have to return Future(Left(exception)) rather than throwing it.
I think, you are kinda overthinking this. It does not look like you need Either here ... or cats for that matter.
Why not do something simple, like this:
class ExternalService {
def doAction(): Future[Int] = Future.successful(2)
def doException(): AsyncResult[Int] = {
println("do exception")
Future.failed(NullPointerException("run time exception"))
// alternatively: Future { throw new NullPointerExceptioN() }
}
class ExceptionExample {
private val service = new ExternalService()
def callService(): AsyncResult[Int] = {
println("start callService")
val result = for {
num <- service.doException()
} yield num
// Note: the aboive is equivalent to just
// val result = service.doException
// You can write it as a chain without even needing a variable:
// service.doException.recover { ... }
result.recover { case ex: Throwable =>
println("recovered exception")
Future.successful(99)
}
}
I tend to agree that it seems a bit convoluted, but for the sake of the exercise, I believe there are a couple of things that don't quite click.
The first one is the fact that you are throwing the Exception instead of capturing it as part of the semantics of Future. ie. You should change your method doException from:
def doException(): AsyncResult[Int] = {
println("do exception")
throw new NullPointerException("run time exception")
}
To:
def doException(): AsyncResult[Int] = {
println("do exception")
AsyncResult(Future.failed(new NullPointerException("run time exception")))
}
The second bit that is not quite right, would be the recovery of the Exception. When you call recoverWith on an EitherT, you're defining a partial function from the Left of the EitherT to another EitherT. In your case, that'd be:
ServiceError => AsyncResult[Int]
If what you want is to recover the failed future, I think you'll need to explicitly recover on it. Something like:
AsyncResult {
result.value.recover {
case _: Throwable => {
println("recovered exception")
Right(99)
}
}
}
If you really want to use recoverWith, then you could write this instead:
AsyncResult {
result.value.recoverWith {
case _: Throwable =>
println("recovered exception")
Future.successful(Right(99))
}
}
I have a CSV file that I need to parse and do some action on every record. How do I use Free Monads with it? Currently, I'm loading the entire file into memory and would like to know if there is any better solution. Below is my program:
for {
reader <- F.getReader("my_file.csv")
csvRecords <- C.readCSV(reader)
_ <- I.processCSV(csvRecords)
_ <- F.close(reader)
} yield()
This code works for smaller files, but if I have very large files (over 1 GB), this wouldn't work very well. I'm using Commons CSV for reading the CSVRecords.
Looking into the code at your gist I think that the line with the comment is exactly the line you don't want at all:
object CSVIOInterpreter extends (CSVIO ~> Future) {
import scala.collection.JavaConverters._
override def apply[A](fa: CSVIO[A]): Future[A] = fa match {
case ReadCSV(reader) => Future.fromTry(Try {
CSVFormat.RFC4180
.withFirstRecordAsHeader()
.parse(reader)
.getRecords // Loads the complete file
.iterator().asScala.toStream
})
}
}
Just remove the whole getRecords line. CSVFormat.parse returns an instance of CSVParser which already implements Iterable<CSVRecord>. And the getRecords call is the only thing that force it to read the whole file.
Actually you can see CSVParser.getRecords implementation and it is
public List<CSVRecord> getRecords() throws IOException {
CSVRecord rec;
final List<CSVRecord> records = new ArrayList<>();
while ((rec = this.nextRecord()) != null) {
records.add(rec);
}
return records;
}
So it just materializes the whole file using this.nextRecord call which is obviously a more "core" part of the API.
So when I do a simplified version of your code without the getRecords call:
import cats._
import cats.free.Free
import java.io._
import org.apache.commons.csv._
import scala.collection.JavaConverters._
trait Action[A] {
def run(): A
}
object F {
import Free.liftF
case class GetReader(fileName: String) extends Action[Reader] {
override def run(): Reader = new FileReader(fileName)
}
case class CloseReader(reader: Reader) extends Action[Unit] {
override def run(): Unit = reader.close()
}
def getReader(fileName: String): Free[Action, Reader] = liftF(GetReader(fileName))
def close(reader: Reader): Free[Action, Unit] = liftF(CloseReader(reader))
}
object C {
import Free.liftF
case class ReadCSV(reader: Reader) extends Action[CSVParser] {
override def run(): CSVParser = CSVFormat.DEFAULT.parse(reader)
}
def readCSV(reader: Reader): Free[Action, CSVParser] = liftF(ReadCSV(reader))
}
object I {
import Free.liftF
case class ProcessCSV(parser: CSVParser) extends Action[Unit] {
override def run(): Unit = {
for (r <- parser.asScala)
println(r)
}
}
def processCSV(parser: CSVParser): Free[Action, Unit] = liftF(ProcessCSV(parser))
}
object Runner {
import cats.arrow.FunctionK
import cats.{Id, ~>}
val runner = new (Action ~> Id) {
def apply[A](fa: Action[A]): Id[A] = fa.run()
}
def run[A](free: Free[Action, A]): A = {
free.foldMap(runner)
}
}
def test() = {
val free = for {
// reader <- F.getReader("my_file.csv")
reader <- F.getReader("AssetsImportCompleteSample.csv")
csvRecords <- C.readCSV(reader)
_ <- I.processCSV(csvRecords)
_ <- F.close(reader)
} yield ()
Runner.run(free)
}
it seems to work OK in line-by-line mode.
Here how I use the CSV file to read and do some operation on that -
I use scala.io.Source.fromFile()
I create one case class of the type of header of CSV file to make the data more accessible and operational.
PS: I don't have knowledge of monads, as well as I am in beginner in Scala. I posted this as it may be helpful.
case class AirportData(id:Int, ident:String, name:String, typeAirport:String, latitude_deg:Double,
longitude_deg:Double, elevation_ft:Double, continent:String, iso_country:String, iso_region:String,
municipality:String)
object AirportData extends App {
def toDoubleOrNeg(s: String): Double = {
try {
s.toDouble
} catch {
case _: NumberFormatException => -1
}
}
val source = scala.io.Source.fromFile("resources/airportData/airports.csv")
val lines = source.getLines().drop(1)
val data = lines.flatMap { line =>
val p = line.split(",")
Seq(AirportData(p(0).toInt, p(1).toString, p(2).toString, p(3).toString, toDoubleOrNeg(p(4)), toDoubleOrNeg(p(5)),
toDoubleOrNeg(p(6)), p(7).toString, p(8).toString, p(9).toString, p(10).toString))
}.toArray
source.close()
println(data.length)
data.take(10) foreach println
}
I've got this Scala code:
Bot.dbUserCollection.find(/*filter*/).first().subscribe(new Observer[Document] {
override def onError(e: Throwable): Unit = Utils.handleError(msg, e)
override def onComplete(): Unit = {}
override def onNext(result: Document): Unit =
/*here im doing operations with the result, if found*/
})
What is the best way to execute some code only if no result was found? Can I somehow easily check if anything was found in onComplete() maybe? I know this has been asked and anwsered many times for other languages in many places all over the Internet but unfortunately I've found no Scala answer
Keeping in mind that onNext will be called when a new document is retrieved and onComplete will be called when the operation is complete regardless of whether or not the query returned any results, you can count the number of results in onNext and check that count in onComplete.
An MVCE follows:
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicLong
import org.mongodb.scala.bson.collection.immutable.Document
import org.mongodb.scala.{
MongoClient,
MongoCollection,
MongoDatabase,
Observable,
Observer
}
import scala.concurrent.Await
import scala.concurrent.duration.Duration
object SimpleInsertionAndRetrieval extends App {
import Helpers._
// To directly connect to the default server localhost on port 27017
val mongoClient: MongoClient = MongoClient()
val database: MongoDatabase = mongoClient.getDatabase("mydb")
val collection: MongoCollection[Document] = database.getCollection("test")
val doc: Document =
Document("_id" -> 0, "name" -> "MongoDB", "type" -> "database")
collection.insertOne(doc).results()
private def observer = new Observer[Document] {
private val counter = new AtomicLong(0)
override def onError(e: Throwable): Unit = {
println(s"Error: $e")
}
override def onComplete(): Unit = {
val numberOfElements = counter.get()
if (numberOfElements == 0) {
println("Nothing was found!")
} else {
println(s"There were $numberOfElements")
}
}
override def onNext(result: Document): Unit = {
counter.incrementAndGet()
// do other things
}
}
collection
.find()
.subscribe(observer)
collection
.find(Document("{ _id: 1 }")) // There is no document with _id = 0
.subscribe(observer)
Thread.sleep(5000)
/**
* From the documentation
*
* #see https://github.com/mongodb/mongo-scala-driver/blob/master/examples/src/test/scala/tour/Helpers.scala
*/
object Helpers {
implicit class DocumentObservable[C](val observable: Observable[Document])
extends ImplicitObservable[Document] {
override val converter: (Document) => String = (doc) => doc.toJson
}
implicit class GenericObservable[C](val observable: Observable[C])
extends ImplicitObservable[C] {
override val converter: (C) => String = (doc) => doc.toString
}
trait ImplicitObservable[C] {
val observable: Observable[C]
val converter: (C) => String
def results(): Seq[C] =
Await.result(observable.toFuture(), Duration(10, TimeUnit.SECONDS))
def headResult() =
Await.result(observable.head(), Duration(10, TimeUnit.SECONDS))
def printResults(initial: String = ""): Unit = {
if (initial.length > 0) print(initial)
results().foreach(res => println(converter(res)))
}
def printHeadResult(initial: String = ""): Unit =
println(s"${initial}${converter(headResult())}")
}
}
}
This prints
There were 1
Nothing was found!
When using Neo4j unmanaged extensions, one can stream results to the client while traversing the graph like this (in Scala):
import javax.ws.rs.core.{MediaType, Response, StreamingOutput}
val stream: StreamingOutput = ???
Response.ok().entity(stream).`type`(MediaType.APPLICATION_JSON).build()
I can't find a similar possibility when using Neo4j 3 used-defined stored procedures. They return Java 8 Streams but I can't see how I could add elements to such streams while they already being consumed, in parallel.
Is it possible?
I have an example of that in one of the APOC procedures.
https://github.com/neo4j-contrib/neo4j-apoc-procedures/blob/master/src/main/java/apoc/cypher/Cypher.java#L77
I want to add more / a more general example of that in the future.
Here is what I came up with based on Michael Hunger code (in Scala).
QueueBasedSpliterator:
import java.util.Spliterator
import java.util.concurrent.{BlockingQueue, TimeUnit}
import java.util.function.Consumer
import org.neo4j.kernel.api.KernelTransaction
private class QueueBasedSpliterator[T](queue: BlockingQueue[T],
tombstone: T,
tx: KernelTransaction) extends Spliterator[T] {
override def tryAdvance(action: Consumer[_ >: T]): Boolean =
try {
if (tx.shouldBeTerminated()) false
else {
val entry = queue.poll(100, TimeUnit.MILLISECONDS)
if (entry == null || entry == tombstone) false
else {
action.accept(entry)
true
}
}
} catch {
case e: InterruptedException => false
}
override def trySplit(): Spliterator[T] = null
override def estimateSize(): Long = Long.MaxValue
override def characteristics(): Int = Spliterator.ORDERED | Spliterator.NONNULL
}
Notice the 100 ms timeout value. Might require tuning.
ResultsStream (wrapper around blocking queue):
import java.util.concurrent.BlockingQueue
class ResultsStream[T](tombstone: T, queue: BlockingQueue[T]) extends AutoCloseable {
def put(t: T): Unit = {
queue.put(t)
}
override def close(): Unit = {
put(tombstone)
}
}
CommonUtil helper methods:
import java.util.concurrent.ArrayBlockingQueue
import java.util.stream.{Stream, StreamSupport}
import org.neo4j.kernel.api.KernelTransaction
import org.neo4j.kernel.internal.GraphDatabaseAPI
import scala.concurrent.{ExecutionContext, Future}
object CommonUtil {
def inTx(db: GraphDatabaseAPI)(f: => Unit): Unit =
Managed(db.beginTx()) { tx => f; tx.success() }
def inTxFuture(db: GraphDatabaseAPI)(f: => Unit)(implicit ec: ExecutionContext): Future[Unit] =
Future(inTx(db)(f))
def streamResults[T](tombstone: T, tx: KernelTransaction)
(f: ResultsStream[T] => Any): Stream[T] = {
val queue = new ArrayBlockingQueue[T](100)
f(new ResultsStream(tombstone, queue))
StreamSupport.stream(new QueueBasedSpliterator[T](queue, tombstone, tx), false)
}
}
Some more helpers:
object Managed {
type AutoCloseableView[T] = T => AutoCloseable
def apply[T : AutoCloseableView, V](resource: T)(op: T => V): V =
try {
op(resource)
} finally {
resource.close()
}
}
Pool:
import java.util.concurrent.{ArrayBlockingQueue, ThreadPoolExecutor, TimeUnit}
import scala.concurrent.{ExecutionContext, ExecutionContextExecutor}
object Pool {
lazy val DefaultExecutionContent: ExecutionContextExecutor =
ExecutionContext.fromExecutor(createDefaultExecutor())
// values might be tuned in production
def createDefaultExecutor(corePoolSize: Int = Runtime.getRuntime.availableProcessors() * 2,
keepAliveSeconds: Int = 30) = {
val queueSize = corePoolSize * 25
new ThreadPoolExecutor(
corePoolSize / 2,
corePoolSize,
keepAliveSeconds.toLong,
TimeUnit.SECONDS,
new ArrayBlockingQueue[Runnable](queueSize),
new ThreadPoolExecutor.CallerRunsPolicy()
)
}
}
Usage in a procedure:
#Procedure("example.readStream")
def readStream(#Name("nodeId") nodeId: NodeId): Stream[StreamingItem] =
CommonUtil.streamResults(StreamingItem.Tombstone, kernelTx) { results =>
CommonUtil.inTxFuture(db) { // uses Pool.DefaultExecutionContent
Managed(results) { _ =>
graphUtil.findTreeNode(nodeId).foreach { node =>
// add elements to the stream here
results.put(???)
}
}
}
}
StreamingItem.Tombstone is just a static StreamingItem instance with special meaning to close the stream. db and kernelTx are just context variable set by Neo4j:
#Context
public GraphDatabaseAPI db;
#Context
public KernelTransaction kernelTx;