how can I transform a result to Json in an reusable action?
Example:
object JsonAction {
def apply(block: Request[JsValue] => ???): Action[JsValue] = {
Action(BodyParsers.parse.json) { request =>
val result = block(request)
val finalResult = result.copy(body = Json.toJson(result.body))
finalResult
}
}
}
in my controller:
def index = JsonAction { req =>
Ok(new SomeModel(...))
}
The idea is to be able to seperate the result model and the representation as json (or xml for example).
I can not find a nice solution to this...
Something like this?
import play.api.libs.json._
import play.api.mvc._
object JsonAction extends Results {
def apply[A, B](block: A => B)(implicit reads: Reads[A], writes: Writes[B]): Action[JsValue] =
Action(BodyParsers.parse.json) { request =>
val result = for {
a <- Json.fromJson(request.body).asOpt
b = block(a)
} yield Ok(Json.toJson(b))
result getOrElse InternalServerError(???)
}
}
or you want to manually define resulting status in your block
object JsonAction extends Results {
def apply[A, B](block: A => (Option[(Status, B)]),
noneStatus: Result = BadRequest("could not parse json"))
(implicit reads: Reads[A], writes: Writes[B]): Action[JsValue] =
Action(BodyParsers.parse.json) { request =>
val result = for {
a <- Json.fromJson(request.body).asOpt
(status, b) <- block(a)
} yield status(Json.toJson(b))
result getOrElse noneStatus
}
}
object MathController {
import JsonAction._
def squareEquasion = JsonAction[Map[String, Double], Set[Double]] { map =>
for {a <- map get "a"
b <- map get "b"
c <- map get "c"
d = b * b - 4 * a * c} yield d match {
case d if d < 0 => (InternalServerError, Set.empty[Double])
case d if d == 0 => (Ok, Set(-b / 2 * a))
case d if d > 0 => (Ok, Set(1, -1) map (q => (-b + q * math.sqrt(d)) / 2 * a))
}
}
}
and final attempt - here we are providing instance of http.Writeable implicitly using an json.Writes instance and converting value to JSON inside this instance, so we could use Result builders ad-hock. This actually could cause some ambiguity if type have it's own Writeable instance (e.g. String):
import play.api.http.Writeable
import play.api.libs.json._
import play.api.mvc._
import scala.language.implicitConversions
object JsonAction {
private object Res extends Results
implicit def jsonWriteable[T](implicit writes: Writes[T]): Writeable[T] = {
val jsonWriteable = implicitly[Writeable[JsValue]]
def transform(obj: T) = jsonWriteable.transform(Json.toJson(obj))
new Writeable[T](transform, jsonWriteable.contentType)
}
def apply[A, B](block: A => Option[Result], noneStatus: Result = Res.BadRequest("could not parse json"))
(implicit reads: Reads[A], writes: Writes[B]): Action[JsValue] =
Action(BodyParsers.parse.json) { request =>
val result = for {
a <- Json.fromJson(request.body).asOpt
result <- block(a)
} yield result
result getOrElse noneStatus
}
}
object MathController extends Results{
import JsonAction._
def squareEquasion = JsonAction[Map[String, Double], Set[Double]] { map =>
for {a <- map get "a"
b <- map get "b"
c <- map get "c"
d = b * b - 4 * a * c} yield d match {
case d if d < 0 => InternalServerError("No answer")
case d if d == 0 => Ok(Set(-b / 2 * a))
case d if d > 0 => Ok(Set(1, -1) map (q => (-b + q * math.sqrt(d)) / 2 * a))
}
}
}
Related
I've written the following solution in scala for the three sum problem.
However the value in the map sums never gets updated.
Can someone explain why doesn't it get updated?
import collection.mutable.ListBuffer
object Solution {
def threeSum(nums: Array[Int]): List[List[Int]] = {
val sums = Map[Int, ListBuffer[(Int, Int)]]().withDefaultValue(ListBuffer())
for((a, aIndex) <- nums.view.zipWithIndex) {
for((b, bIndex) <- nums.view.zipWithIndex) {
if(aIndex != bIndex){
val sum = a + b
List(a, b).sorted match {
case List(a, b) => {
sums(sum) += ((a, b))
println(sums)
}
}
}
}
}
val result = ListBuffer[List[Int]]()
for(c <- nums){
val neg = c * -1
if(sums.contains(neg)){
val sumSet = sums(neg).toSet
for((a,b) <- sumSet) {
val triplet = List(a, b, c).sorted
result += triplet
}
}
}
result.toList
}
}
I want to group large Stream[F, A] into Stream[Stream[F, A]] with at most n element for inner stream.
This is what I did, basically pipe chunks into Queue[F, Queue[F, Chunk[A]], and then yields queue elements as result stream.
implicit class StreamSyntax[F[_], A](s: Stream[F, A])(
implicit F: Concurrent[F]) {
def groupedPipe(
lastQRef: Ref[F, Queue[F, Option[Chunk[A]]]],
n: Int): Pipe[F, A, Stream[F, A]] = { in =>
val initQs =
Queue.unbounded[F, Option[Queue[F, Option[Chunk[A]]]]].flatMap { qq =>
Queue.bounded[F, Option[Chunk[A]]](1).flatMap { q =>
lastQRef.set(q) *> qq.enqueue1(Some(q)).as(qq -> q)
}
}
Stream.eval(initQs).flatMap {
case (qq, initQ) =>
def newQueue = Queue.bounded[F, Option[Chunk[A]]](1).flatMap { q =>
qq.enqueue1(Some(q)) *> lastQRef.set(q).as(q)
}
val evalStream = {
in.chunks
.evalMapAccumulate((0, initQ)) {
case ((i, q), c) if i + c.size >= n =>
val (l, r) = c.splitAt(n - i)
q.enqueue1(Some(l)) >> q.enqueue1(None) >> q
.enqueue1(None) >> newQueue.flatMap { nq =>
nq.enqueue1(Some(r)).as(((r.size, nq), c))
}
case ((i, q), c) if (i + c.size) < n =>
q.enqueue1(Some(c)).as(((i + c.size, q), c))
}
.attempt ++ Stream.eval {
lastQRef.get.flatMap { last =>
last.enqueue1(None) *> last.enqueue1(None)
} *> qq.enqueue1(None)
}
}
qq.dequeue.unNoneTerminate
.map(
q =>
q.dequeue.unNoneTerminate
.flatMap(Stream.chunk)
.onFinalize(
q.dequeueChunk(Int.MaxValue).unNoneTerminate.compile.drain))
.concurrently(evalStream)
}
}
def grouped(n: Int) = {
Stream.eval {
Queue.unbounded[F, Option[Chunk[A]]].flatMap { empty =>
Ref.of[F, Queue[F, Option[Chunk[A]]]](empty)
}
}.flatMap { ref =>
val p = groupedPipe(ref, n)
s.through(p)
}
}
}
But it is very complicated, is there any simpler way ?
fs2 has chunkN chunkLimit methods that can help with grouping
stream.chunkN(n).map(Stream.chunk)
stream.chunkLimit(n).map(Stream.chunk)
chunkN produces chunks of size n until the end of a stream
chunkLimit splits existing chunks and can produce chunks with variable size.
scala> Stream(1,2,3).repeat.chunkN(2).take(5).toList
res0: List[Chunk[Int]] = List(Chunk(1, 2), Chunk(3, 1), Chunk(2, 3), Chunk(1, 2), Chunk(3, 1))
scala> (Stream(1) ++ Stream(2, 3) ++ Stream(4, 5, 6)).chunkLimit(2).toList
res0: List[Chunk[Int]] = List(Chunk(1), Chunk(2, 3), Chunk(4, 5), Chunk(6))
In addition to the already mentioned chunksN, also consider using groupWithin (fs2 1.0.1):
def groupWithin[F2[x] >: F[x]](n: Int, d: FiniteDuration)(implicit timer: Timer[F2], F: Concurrent[F2]): Stream[F2, Chunk[O]]
Divide this streams into groups of elements received within a time window, or limited by the number of the elements, whichever happens first. Empty groups, which can occur if no elements can be pulled from upstream in a given time window, will not be emitted.
Note: a time window starts each time downstream pulls.
I'm not sure why you'd want this to be nested streams, since the requirement is to have "at most n elements" in one batch - which implies that you're keeping track of a finite number of elements (which is exactly what a Chunk is for). Either way, a Chunk can always be represented as a Stream with Stream.chunk:
val chunks: Stream[F, Chunk[O]] = ???
val streamOfStreams: Stream[F, Stream[F, O]] = chunks.map(Stream.chunk)
Here's a complete example of how to use groupWithin:
import cats.implicits._
import cats.effect.{ExitCode, IO, IOApp}
import fs2._
import scala.concurrent.duration._
object GroupingDemo extends IOApp {
override def run(args: List[String]): IO[ExitCode] = {
Stream('a, 'b, 'c).covary[IO]
.groupWithin(2, 1.second)
.map(_.toList)
.showLinesStdOut
.compile.drain
.as(ExitCode.Success)
}
}
Outputs:
List('a, 'b)
List('c)
Finally I use a more reliable version (use Hotswap ensure queue termination) like this.
def grouped(
innerSize: Int
)(implicit F: Async[F]): Stream[F, Stream[F, A]] = {
type InnerQueue = Queue[F, Option[Chunk[A]]]
type OuterQueue = Queue[F, Option[InnerQueue]]
def swapperInner(swapper: Hotswap[F, InnerQueue], outer: OuterQueue) = {
val innerRes =
Resource.make(Queue.unbounded[F, Option[Chunk[A]]])(_.offer(None))
swapper.swap(innerRes).flatTap(q => outer.offer(q.some))
}
def loopChunk(
gathered: Int,
curr: Queue[F, Option[Chunk[A]]],
chunk: Chunk[A],
newInnerQueue: F[InnerQueue]
): F[(Int, Queue[F, Option[Chunk[A]]])] = {
if (gathered + chunk.size > innerSize) {
val (left, right) = chunk.splitAt(innerSize - gathered)
curr.offer(left.some) >> newInnerQueue.flatMap { nq =>
loopChunk(0, nq, right, newInnerQueue)
}
} else if (gathered + chunk.size == innerSize) {
curr.offer(chunk.some) >> newInnerQueue.tupleLeft(
0
)
} else {
curr.offer(chunk.some).as(gathered + chunk.size -> curr)
}
}
val prepare = for {
outer <- Resource.eval(Queue.unbounded[F, Option[InnerQueue]])
swapper <- Hotswap.create[F, InnerQueue]
} yield outer -> swapper
Stream.resource(prepare).flatMap {
case (outer, swapper) =>
val newInner = swapperInner(swapper, outer)
val background = Stream.eval(newInner).flatMap { initQueue =>
s.chunks
.filter(_.nonEmpty)
.evalMapAccumulate(0 -> initQueue) { (state, chunk) =>
val (gathered, curr) = state
loopChunk(gathered, curr, chunk, newInner).tupleRight({})
}
.onFinalize(swapper.clear *> outer.offer(None))
}
val foreground = Stream
.fromQueueNoneTerminated(outer)
.map(i => Stream.fromQueueNoneTerminatedChunk(i))
foreground.concurrently(background)
}
}
The code below contains various single-threaded implementations of reduceByKeyXXX methods and a few helper methods to create input sets and measure execution times. (Feel free to run the main-method)
The main purpose of reduceByKey (as in Spark) is to reduce key-value-pairs with the same key. Example:
scala> val xs = Seq( "a" -> 2, "b" -> 3, "a" -> 5)
xs: Seq[(String, Int)] = List((a,2), (b,3), (a,5))
scala> ReduceByKeyComparison.reduceByKey(xs, (x:Int, y:Int) ⇒ x+y )
res8: Seq[(String, Int)] = ArrayBuffer((b,3), (a,7))
Code
import java.util.HashMap
object Util {
def measure( body : => Unit ) : Long = {
val now = System.currentTimeMillis
body
val nowAfter = System.currentTimeMillis
nowAfter - now
}
def measureMultiple( body: => Unit, n: Int) : String = {
val executionTimes = (1 to n).toList.map( x => {
print(".")
measure(body)
} )
val avg = executionTimes.sum / executionTimes.size
executionTimes.mkString("", "ms, ", "ms") + s" Average: ${avg}ms."
}
}
object RandomUtil {
val AB = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
val r = new java.util.Random();
def randomString( len: Int ) : String = {
val sb = new StringBuilder( len );
for( i <- 0 to len-1 ) {
sb.append(AB.charAt(r.nextInt(AB.length())));
}
sb.toString();
}
def generateSeq(n: Int) : Seq[(String, Int)] = {
Seq.fill(n)( (randomString(2), r.nextInt(100)) )
}
}
object ReduceByKeyComparison {
def main(args: Array[String]) : Unit = {
implicit def iterableToPairedIterable[K, V](x: Iterable[(K, V)]) = { new PairedIterable(x) }
val runs = 10
val problemSize = 2000000
val ss = RandomUtil.generateSeq(problemSize)
println("ReduceByKey : " + Util.measureMultiple( reduceByKey(ss, (x:Int, y:Int) ⇒ x+y ), runs ))
println("ReduceByKey2: " + Util.measureMultiple( reduceByKey2(ss, (x:Int, y:Int) ⇒ x+y ), runs ))
println("ReduceByKey3: " + Util.measureMultiple( reduceByKey3(ss, (x:Int, y:Int) ⇒ x+y ), runs ))
println("ReduceByKeyPaired: " + Util.measureMultiple( ss.reduceByKey( (x:Int, y:Int) ⇒ x+y ), runs ))
println("ReduceByKeyA: " + Util.measureMultiple( reduceByKeyA( ss, (x:Int, y:Int) ⇒ x+y ), runs ))
}
// =============================================================================
// Different implementations
// =============================================================================
def reduceByKey[A,B]( s: Seq[(A,B)], fnc: (B, B) ⇒ B) : Seq[(A,B)] = {
val t = s.groupBy(x => x._1)
val u = t.map { case (k,v) => (k, v.map(_._2).reduce(fnc))}
u.toSeq
}
def reduceByKey2[A,B]( s: Seq[(A,B)], fnc: (B, B) ⇒ B) : Seq[(A,B)] = {
val r = s.foldLeft( Map[A,B]() ){ (m,a) ⇒
val k = a._1
val v = a._2
m.get(k) match {
case Some(pv) ⇒ m + ((k, fnc(pv, v)))
case None ⇒ m + ((k, v))
}
}
r.toSeq
}
def reduceByKey3[A,B]( s: Seq[(A,B)], fnc: (B, B) ⇒ B) : Seq[(A,B)] = {
var m = scala.collection.mutable.Map[A,B]()
s.foreach{ e ⇒
val k = e._1
val v = e._2
m.get(k) match {
case Some(pv) ⇒ m(k) = fnc(pv, v)
case None ⇒ m(k) = v
}
}
m.toSeq
}
/**
* Method code from [[http://ideone.com/dyrkYM]]
* All rights to Muhammad-Ali A'rabi according to [[https://issues.scala-lang.org/browse/SI-9064]]
*/
def reduceByKeyA[A,B]( s: Seq[(A,B)], fnc: (B, B) ⇒ B): Map[A, B] = {
s.groupBy(_._1).map(l => (l._1, l._2.map(_._2).reduce( fnc )))
}
/**
* Method code from [[http://ideone.com/dyrkYM]]
* All rights to Muhammad-Ali A'rabi according to [[https://issues.scala-lang.org/browse/SI-9064]]
*/
class PairedIterable[K, V](x: Iterable[(K, V)]) {
def reduceByKey(func: (V,V) => V) = {
val map = new HashMap[K, V]
x.foreach { pair =>
val old = map.get(pair._1)
map.put(pair._1, if (old == null) pair._2 else func(old, pair._2))
}
map
}
}
}
yielding the following results on my machine
..........ReduceByKey : 723ms, 782ms, 761ms, 617ms, 640ms, 707ms, 634ms, 611ms, 380ms, 458ms Average: 631ms.
..........ReduceByKey2: 580ms, 458ms, 452ms, 463ms, 462ms, 470ms, 463ms, 465ms, 458ms, 462ms Average: 473ms.
..........ReduceByKey3: 489ms, 466ms, 461ms, 468ms, 555ms, 474ms, 469ms, 457ms, 461ms, 468ms Average: 476ms.
..........ReduceByKeyPaired: 140ms, 124ms, 124ms, 120ms, 122ms, 124ms, 118ms, 126ms, 121ms, 119ms Average: 123ms.
..........ReduceByKeyA: 628ms, 694ms, 666ms, 656ms, 616ms, 660ms, 594ms, 659ms, 445ms, 399ms Average: 601ms.
and ReduceByKeyPaired currently being the fastest.
Question / Task
Is there a faster single-threaded (Scala) implementation?
Rewritting reduceByKey method of PairedIterable to recursion gives around 5-10% performance improvement.
That all i was able to get.
I've also tryed to increase initial capacity allocation for HashMap - but it does not show any significant changes.
class PairedIterable[K, V](x: Iterable[(K, V)]) {
def reduceByKey(func: (V,V) => V) = {
val map = new HashMap[K, V]()
#tailrec
def reduce(it: Iterable[(K, V)]): HashMap[K, V] = {
it match {
case Nil => map
case (k, v) :: tail =>
val old = map.get(k)
map.put(k, if (old == null) v else func(old, v))
reduce(tail)
}
}
val r = reduce(x)
r
}
}
In general, making some comparison analysis of provided methods - they can be splitted onto two categories.
First set of reduces are with sorting (grouping) - as we can see those methods add extra O(n*log[n]) complexity and are not effective for this scenario.
Seconds are with linear looping across all enries of Iterable. Those set of methods has extra get/put operations to temp map. But those gets/puts are not so time consuming - O(n)*O(c).
Moreover necessity to work with Options in scala collections makes it less effective.
I want to define a method, which will return a Future. And in this method, it will call another service which returns also a Future.
We have defined a BusinessResult to represent Success and Fail:
object validation {
trait BusinessResult[+V] {
def flatMap[T](f: V => BusinessResult[T]):BusinessResult[T]
def map[T](f: V => T): BusinessResult[T]
}
sealed case class Success[V](t:V) extends BusinessResult[V] {
def flatMap[T](f: V => BusinessResult[T]):BusinessResult[T] = {
f(t)
}
def map[T](f: V => T): BusinessResult[T] = {
Success(f(t))
}
}
sealed case class Fail(e:String) extends BusinessResult[Nothing] {
def flatMap[T](f: Nothing => BusinessResult[T]):BusinessResult[T] = this
def map[T](f: Nothing => T): BusinessResult[T] = this
}
}
And define the method:
import scala.concurrent._
import scala.concurrent.ExecutionContext.Implicits.global
import validation._
def name: BusinessResult[String] = Success("my name")
def externalService(name:String):Future[String] = future(name)
def myservice:Future[Int] = {
for {
n <- name
res <- externalService(n)
} yield res match {
case "ok" => 1
case _ => 0
}
}
But which is not compilable. The code in myservice can't return a Future[Int] type.
I also tried to wrap the name with Future:
def myservice:Future[Int] = {
for {
nn <- Future.successful(name)
n <- nn
res <- externalService(n)
} yield res match {
case "ok" => 1
case _ => 0
}
}
Which is also not compilable.
I know there must be a lot of issues in this code. How can I adjust them to make it compilable?
If you change the n with some hardcoded string it works, the problem is that in the for comprehension the variable n has type BusinessResult[String], as you probably already know for comprehension desugarize to map, flatMap and filter so the first part n <- name desugarize to a map on name:
val test: BusinessResult[String] = name.map(x => x)
Intellij thinks n is a String but the scala compiler disagree:
type mismatch;
[error] found : validation.BusinessResult[Nothing]
[error] required: scala.concurrent.Future[Int]
[error] n <- name
[error] ^
Easy solution could be to add a getter method to get back the string and do something like Option does:
object validation {
trait BusinessResult[+V] {
def flatMap[T](f: V => BusinessResult[T]):BusinessResult[T]
def map[T](f: V => T): BusinessResult[T]
def getVal: V
}
sealed case class Success[V](t:V) extends BusinessResult[V] {
def flatMap[T](f: V => BusinessResult[T]):BusinessResult[T] = {
f(t)
}
def map[T](f: V => T): BusinessResult[T] = {
Success(f(t))
}
def getVal: V = t
}
sealed case class Fail(e:String) extends BusinessResult[Nothing] {
def flatMap[T](f: Nothing => BusinessResult[T]):BusinessResult[T] = this
def map[T](f: Nothing => T): BusinessResult[T] = this
def getVal = throw new Exception("some message")
}
}
def myservice: Future[Int] = {
val value = name.getVal
for {
res <- externalService(value)
} yield res match {
case "ok" => 1
case _ => 0
}
}
Note that you can't extract the name in the for comprehension since map on String return Chars
Given this RichFile class and its companion object:
class RichFile(filePath: String) {
val file = new File(filePath)
}
object RichFile {
def apply(filePath: String) = new RichFile(filePath)
def unapply(rf: RichFile) = {
if (rf == null || rf.file.getAbsolutePath().length() == 0)
None
else {
val basename = rf.file.getName()
val dirname = rf.file.getParent()
val ei = basename.indexOf(".")
if (ei >= 0) {
Some((dirname, basename.substring(0, ei), basename.substring(ei + 1)))
} else {
Some((dirname, basename, ""))
}
}
}
def unapplySeq(rf: RichFile): Option[Seq[String]] = {
val filePath = rf.file.getAbsolutePath()
if (filePath.length() == 0)
None
else
Some(filePath.split("/"))
}
}
Basically I want to extract all the components of a file path as a sequence. Why does wild-card match doesn't work in the following code? Specifically the first case statement I am getting the error star patterns must correspond with varargs parameters.
val l = List(
RichFile("/abc/def/name.txt"),
RichFile("/home/cay/name.txt"),
RichFile("/a/b/c/d/e"))
l.foreach { f =>
f match {
case RichFile(_*) => println((x, y))
case RichFile(a, b, c) => println((a, b, c))
}
}
I also want to match them just like we match Lists in Scala, something like this:
l.foreach { f =>
f match {
case a::b::"def"::tail => println((a, tail))
case RichFile(_*) => println((x, y))
case RichFile(a, b, c) => println((a, b, c))
}
}
How can I do that using unapplySeq?
It looks like the issue is just that you have both unapply and unapplySeq, so when you have just one argument in the case RichFile, scala is confused about which kind of unapply you are trying to do. The way to resolve this would be to have two objects, one with unapply and one with unapplySeq, so that the usage is unambiguous.
class RichFile(filePath: String) {
val file = new File(filePath)
override def toString = f"RichFile($filePath)"
}
object RichFile {
def apply(filePath: String) = new RichFile(filePath)
def unapply(rf: RichFile) = {
if (rf == null || rf.file.getAbsolutePath.isEmpty) None
else {
val basename = rf.file.getName
val dirname = rf.file.getParent
val (name, ext) = basename.split("\\.", 2) match {
case Array(name, ext) => (name, ext)
case Array(name) => (name, "")
}
Some((dirname, name, ext))
}
}
}
object RichFilePath {
def unapplySeq(rf: RichFile): Option[Seq[String]] = {
val filePath = rf.file.getAbsolutePath()
if (filePath.isEmpty) None
else Some(filePath.split("/"))
}
}
val l = List(
RichFile("/abc/def/name.txt"),
RichFile("/home/cay/name.txt"),
RichFile("/a/b/c/d/e"),
RichFile("/y/z"))
l.foreach { f =>
f match {
case RichFilePath(a, b, c) => println("RichFilePath -> " + (a, b, c))
case RichFile(f) => println("RichFile -> " + f)
}
}
prints:
RichFile -> (/abc/def,name,txt)
RichFile -> (/home/cay,name,txt)
RichFile -> (/a/b/c/d,e,)
RichFilePath -> (,y,z)
Regarding the :: syntax, you can't use :: because it's already defined and only works on Lists. Further, you wouldn't want :: because operators ending in : are right-associative. This makes sense for List matching because the item is on the left and the rest is on the right. For your application, I assume that you want matching to read the same way as the directory structure: with the filename on the right and the "rest" on the left. Thus, you can def define your own operator for this:
object --> {
def unapply(rf: RichFile): Option[(RichFile, String)] = {
if (rf.file.getAbsolutePath.isEmpty)
None
else {
val f = rf.file.getAbsoluteFile
Some((RichFile(f.getParent), f.getName))
}
}
}
l.foreach { f =>
f match {
case a --> b --> c => println(f"$a\t$b\t$c")
}
}
prints
RichFile(/abc) def name.txt
RichFile(/home) cay name.txt
RichFile(/a/b/c) d e
RichFile(/) y z