how to delete a java file created from akka streams - scala

I am using akka streams and am following tutorial. The code is trying to delete the created file but it's not getting deleted. I was unable to know which resource is left opened. file.delete() is returning false. the file has the permission of read, write and execute
Here is the example code
final case class FileField(fieldName: String, fileNameF: FileInfo ⇒ File)
final case class PartsAndFiles(form: immutable.Map[String, List[String]], files: immutable.Seq[(FileInfo, File)]) {
final def addForm(fieldName: String, content: String): PartsAndFiles = this.copy(
form = {
val existingContent: List[String] = this.form.getOrElse(fieldName, List.empty)
val newContents: List[String] = content :: existingContent
this.form + (fieldName -> newContents)
}
)
final def addFile(info: FileInfo, file: File): PartsAndFiles = this.copy(
files = this.files :+ ((info, file))
)
}
object PartsAndFiles {
val Empty = PartsAndFiles(immutable.Map.empty, immutable.Seq.empty)
}
def formAndFiles(
fileFields: immutable.Seq[FileField]
): Directive1[PartsAndFiles] = entity(as[Multipart.FormData]).flatMap { formData ⇒
extractRequestContext.flatMap { ctx ⇒
implicit val mat = ctx.materializer
implicit val ec = ctx.executionContext
val uploadingSink =
Sink.foldAsync[PartsAndFiles, Multipart.FormData.BodyPart](PartsAndFiles.Empty) {
(acc, part) ⇒
def discard(p: Multipart.FormData.BodyPart): Future[PartsAndFiles] = {
p.entity.discardBytes()
Future.successful(acc)
}
part.filename.map { fileName ⇒
fileFields.find(_.fieldName == part.name)
.map {
case FileField(fieldName, destFn) ⇒
val fileInfo = FileInfo(part.name, fileName, part.entity.contentType)
val dest = destFn(fileInfo)
part.entity.dataBytes.runWith(FileIO.toPath(dest.toPath)).map { _ ⇒
acc.addFile(fileInfo, dest)
}
}.getOrElse(discard(part))
} getOrElse {
part.entity match {
case HttpEntity.Strict(ct, data) if ct.isInstanceOf[ContentType.NonBinary] ⇒
val charsetName = ct.asInstanceOf[ContentType.NonBinary].charset.nioCharset.name
val partContent = data.decodeString(charsetName)
Future.successful(acc.addForm(part.name, partContent))
case _ ⇒
discard(part)
}
}
}
val uploadedF = formData.parts.runWith(uploadingSink)
onSuccess(uploadedF)
}
}
val fileFields = scala.collection.immutable.Seq(FileField("picture", _ => new File("/tmp/uploaded")))
val routes: Route = formAndFiles(fileFields) {
case PartsAndFiles(fields, files) =>
files.foreach(_._2.delete())
val body = s"""
|File: ${files.head._2.getAbsolutePath}
|Form: ${fields}
""".stripMargin
complete(OK, body)
}
here
files.foreach(_._2.delete())
is returning false
files.foreach(_._2.isExists)
is returning true
i want to delete this file how can i do it ?

Related

How to process Akka stream based on some condition?

Suppose there is a stream of some files to be processed and only a specific file should be processed(consumed) when a condition is met.
i.e. Only if the stream contains a file named "aaa", process a file named "bbb"
SomeFile(name: String)
What would be the correct(recommended) way to do this?
Okay, here's an example. Be careful about building up too big a buffer here before the trigger hits
class FileFinder {
def matchFiles(triggerName: String,
matchName: String): Flow[SomeFile, SomeFile, NotUsed] =
Flow[SomeFile].statefulMapConcat(
statefulMatcher(matches(triggerName), matches(matchName)))
private def matches(matchName: String): SomeFile => Boolean = {
case SomeFile(name) if name == matchName => true
case _ => false
}
private def statefulMatcher(
triggerFilter: => SomeFile => Boolean,
sendFilter: SomeFile => Boolean): () => SomeFile => List[SomeFile] = {
var found = false
var sendFiles: List[SomeFile] = Nil
() => file: SomeFile =>
{
file match {
case f if triggerFilter(f) =>
found = true
val send = sendFiles
sendFiles = Nil
send
case f if sendFilter(f) =>
if (found)
List(f)
else {
sendFiles = f :: sendFiles
Nil
}
case _ => Nil
}
}
}
}
object FileFinder extends FileFinder {
def main(args: Array[String]): Unit = {
implicit val system: ActorSystem = ActorSystem("finder")
implicit val materializer: ActorMaterializer = ActorMaterializer()
implicit val executor: ExecutionContextExecutor =
materializer.executionContext
implicit val loggingAdapter: LoggingAdapter = system.log
val files = List(SomeFile("aaa"), SomeFile("bbb"), SomeFile("aaa"))
Source(files)
.via(matchFiles("bbb", "aaa"))
.runForeach(println(_))
.andThen({
case Success(_) =>
println("Success")
system.terminate()
case Failure(ex) =>
loggingAdapter.error("Shouldn't happen...", ex)
system.terminate()
})
}
}
case class SomeFile(name: String)

Akka Streams recursive flow call

I'm trying to do implement paging using Akka Streams. Currently I have
case class SomeObject(id:Long, next_page:Option[Map[String,String]])
def chainRequests(uri: Uri): Future[Option[(Uri, T)]] = {
if (uri.isEmpty) return Future.successful(None)
val response: Future[Response[T]] = sendWithRetry(prepareRequest(HttpMethods.GET, uri)).flatMap(unmarshal)
response.map { resp =>
resp.next_page match {
case Some(next_page) => Some(next_page("uri"), resp.data)
case _ => Some(Uri.Empty, resp.data)
}
}
}
Source.single(SomeObject).map(Uri(s"object/${_.id}")).map(uri => Source.unfoldAsync(url)(chainRequest)).map(...some processing goes here)
The problem is that if I do source.take(1000) and paging has a lot of elements(pages) than downstream does not gets new elements until Source.unfoldAsync finishes.
I was trying to use cycles in Flows like
val in = builder.add(Flow[Uri])
val out = builder.add[Flow[T]]
val partition = b.add(Partition[Response[T]](2,r => r.next_page match {case Some(_)=>1; case None => 0}))
val merge = b.add(Merge[Response[T]],2)
in ~> mergeUri ~> sendRequest ~> partition
mergeUri.preferred <~ extractNextUri <~ partition.out(1)
partition.out(0) ~> Flow[Response[T]].map(_.data) ~> out
FlowShape(in.in, out.out)
But above code does not work.
I'm stuck with creating my own GraphStage. UnfoldAsync takes first element, but with Flow solution I don't have "first" element. Any suggestions?
Thanks
Found the solution with writing my own GraphStage
final class PaginationGraphStage[S <: Uri, E](f: S => Future[Option[(S, E)]])(implicit ec: ExecutionContextExecutor)
extends GraphStage[FlowShape[S, E]]{
val in: Inlet[S] = Inlet[S]("PaginationGraphStage.in")
val out: Outlet[E] = Outlet[E]("PaginationGraphStage.out")
override val shape: FlowShape[S, E] = FlowShape.of(in, out)
override def createLogic(inheritedAttributes: Attributes): GraphStageLogic =
new GraphStageLogic(shape) with OutHandler with InHandler {
private[this] var state: S = _
private[this] var inFlight = 0
private[this] var asyncFinished = false
private[this] def todo: Int = inFlight
def futureCompleted(result: Try[Option[(Uri, E)]]): Unit = {
inFlight -= 1
result match {
case Failure(ex) => fail(out, ex)
case Success(None) =>
asyncFinished = true
complete(out)
case Success(Some((newS: S, elem: E))) if !newS.isEmpty =>
push(out, elem)
state = newS
case Success(Some((newS: Uri, elem: E))) =>
push(out, elem)
asyncFinished = true
if (isAvailable(in)) getHandler(in).onPush()
else completeStage()
}
}
private val futureCB = getAsyncCallback(futureCompleted)
private val invokeFutureCB: Try[Option[(S, E)]] => Unit = futureCB.invoke
private def pullIfNeeded(): Unit = {
if (!hasBeenPulled(in)) tryPull(in)
}
override def onUpstreamFinish(): Unit = {
if (todo == 0) completeStage()
}
def onPull(): Unit = {
if (state != null) {
asyncFinished = false
inFlight += 1
val future = f(state)
future.value match {
case None => future.onComplete(invokeFutureCB)
case Some(v) => futureCompleted(v)
}
} else {
pullIfNeeded()
}
}
override def onPush(): Unit = {
if (state == null) {
inFlight += 1
state = grab(in)
pullIfNeeded()
getHandler(out).onPull()
}
if (asyncFinished) {
inFlight += 1
state = grab(in)
pullIfNeeded()
}
}
setHandlers(in, out, this)
}
}

Akka Streams only processing one Collection

I have the following situation, I have bunch of directories which have bunch of files. I'm processing them using AKKA but for some reasons only the last sequence is processed, Here is the code of the method I have, let me know if you see something wrong
def read(): Unit = {
implicit val system = ActorSystem("LiveS3Parser")
implicit val materializer = ActorMaterializer()
val reader = new LiveSequenceFileReader(conf.getString("s3url"))
val dateList = generateDates(conf.getString("startDate"), conf.getString("endDate"))
reader.readAllFilesFromPath(conf.getString("s3url"))
val seqElements = generateURLS(dateList, conf).via(readDataFromS3(reader)).via(parseJsonSeq())
val sinkseq = Sink.fold(0)(persistDataSeq)
val dataCounter = seqElements.toMat(sinkseq)(Keep.right)
val sum: Future[Int] = dataCounter.run()
sum.andThen({
case _ =>
sum.foreach(c => println(s"Total records Loaded: $c"))
})
Await.result(sum,Duration.Inf)
}
def generateURLS(data: Seq[Long], conf: Config): Source[String, NotUsed] = {
val s3URL = conf.getString("s3url")
val dataWithURLs = data.map(x => s3URL.concat("dt=").concat(DateUtils.formatDate(new Date(x), "yyyy-MM-dd")))
Source(dataWithURLs.to[scala.collection.immutable.Seq])
}
def readDataFromS3(lv: LiveSequenceFileReader)(implicit ec: ExecutionContext): Flow[String, Seq[KeyValue], NotUsed] = {
Flow[String].mapAsyncUnordered(Runtime.getRuntime().availableProcessors())(url => Future(readFiles(url, lv)))
}
def parseJsonSeq()(implicit ec: ExecutionContext): Flow[Seq[KeyValue], Seq[Try[OptimizedSearchQueryEventMessage]], NotUsed] = {
Flow[Seq[KeyValue]].mapAsyncUnordered(Runtime.getRuntime().availableProcessors())(line => Future(parseAllItems(line)))
}
def readFiles(url: String, lv: LiveSequenceFileReader): Seq[KeyValue] = {
println("Reading Files from " + url)
val files = lv.readAllFilesFromPath(url)
println("Records to process" + files.size())
files
}
def parseAllItems(seq: Seq[KeyValue]) = {
seq.map(kv => parseItem(kv.getValue))
}
def parseItem(data: String): Try[OptimizedSearchQueryEventMessage] = {
val retVal = Try(mapper.readValue(data, classOf[OptimizedSearchQueryEventMessage]))
retVal
}
def generateDates(startingDate: String, endDate: String): Seq[Long] = {
val fmt = new SimpleDateFormat("yyyy-MM-dd")
val startDate = fmt.parse(startingDate).getTime
val endingDate = fmt.parse(endDate).getTime
val list = for (currentDate <- startDate to endingDate by TimeUnit.DAYS.toMillis(1)) yield currentDate
list
}

How to get the class of Option[_] on a None with Reflection

I have this setup where I want to populate an object fields from an XML NodeSeq. A cutdown version of my setup is shown. It works well, but when I put None in testObj, I get a java.util.NoSuchElementException: None.get.
The problem I believe is that I cannot find a way to get the class of the Option[_] when it is None. I've been stuck on this for ages and I cannot see a way out. I feel there must be a way to branch and populate the object field when the original value is None, but how?
import xml.NodeSeq
object test {
case class TestObject(name: Option[String] = None, value: Option[Double] = None)
def main(args: Array[String]): Unit = {
val testObj = TestObject(Some("xxx"), Some(12.0))
// val testObj = TestObject(None, Some(12.0))
val nodeSeq = <test> <name> yyy </name> <value> 34.0 </value> </test>
val result = getObject[TestObject](nodeSeq, Option(testObj))
println("result = " + result) // result = Some(TestObject(Some(yyy),Some(34.0)))
}
def getObject[A](nodeSeq: NodeSeq, obj: Option[A]): Option[A] = {
if ((nodeSeq.isEmpty) || (!obj.isDefined)) None else {
for (field <- obj.get.getClass.getDeclaredFields) {
field.setAccessible(true)
val fieldValue = field.get(obj.get)
if (fieldValue == null || !fieldValue.isInstanceOf[Option[_]]) None
var newValue: Option[_] = None
fieldValue.asInstanceOf[Option[_]].get match { // <---- None.get is the error here
case x: Double => newValue = Some((nodeSeq \ field.getName).text.trim.toDouble)
case x: String => newValue = Some((nodeSeq \ field.getName).text.trim)
}
val decField = obj.get.getClass.getDeclaredField(field.getName)
decField.setAccessible(true)
decField.set(obj.get, newValue)
}
obj
}
}
}
I'm ignoring some other parts of this code that I don't like, however, you can't call .get on a None. You can call map and it will return what I assume you want. Here's the modified code:
def getObject[A](nodeSeq: NodeSeq, obj: Option[A]): Option[A] = {
if (nodeSeq.isEmpty || obj.isEmpty) {
None
}
else {
for (field <- obj.get.getClass.getDeclaredFields) {
field.setAccessible(true)
val fieldValue = field.get(obj.get)
if (fieldValue == null || !fieldValue.isInstanceOf[Option[_]]) None
val newValue = fieldValue.asInstanceOf[Option[_]].map(a => {
a match {
case x: Double => Some((nodeSeq \ field.getName).text.trim.toDouble)
case x: String => Some((nodeSeq \ field.getName).text.trim)
}
})
val decField = obj.get.getClass.getDeclaredField(field.getName)
decField.setAccessible(true)
decField.set(obj.get, newValue)
}
obj
}
}
In practice, I usually avoid calling .get on options as it results in bugs quite often.
UPDATE:
This version is a little more functional, though it still alters the obj that you send in:
def getObject[A](nodeSeq: NodeSeq, obj: Option[A]): Option[A] = {
obj.map(o => {
for {
field <- o.getClass.getDeclaredFields
_ = field.setAccessible(true)
fieldValue <- Option(field.get(o))
} yield {
val newValue = fieldValue match {
case Some(a) => {
a match {
case x: Double => Some((nodeSeq \ field.getName).text.trim.toDouble)
case x: String => Some((nodeSeq \ field.getName).text.trim)
}
}
case _ => None
}
val decField = o.getClass.getDeclaredField(field.getName)
decField.setAccessible(true)
decField.set(o, newValue)
}
o
})
}
Actually, this might be closer to what you want: check the type of the field instead.
import xml.NodeSeq
object test {
case class TestObject(name: Option[String] = None, value: Option[Double] = None)
def main(args: Array[String]): Unit = {
val testObj = TestObject(Some("xxx"), Some(12.0))
// val testObj = TestObject(None, Some(12.0))
val nodeSeq = <test> <name> yyy </name> <value> 34.0 </value> </test>
val result = getObject[TestObject](nodeSeq, Option(testObj))
println("result = " + result) // result = Some(TestObject(Some(yyy),Some(34.0)))
}
def getObject[A](nodeSeq: NodeSeq, obj: Option[A]): Option[A] = {
if ((nodeSeq.isEmpty) || (!obj.isDefined)) None else {
for (field <- obj.get.getClass.getDeclaredFields) {
field.setAccessible(true)
val newValue = field.getGenericType match {
case t: ParametrizedType if (t.getActualType == classOf[Option[_]]) =>
val paramType = t.getActualTypeArguments()(0)
if (paramType == classOf[java.lang.Double]) // since Double can't be a generic type parameter on JVM
// from your original code, but it probably needs to be modified
// to handle cases where field isn't in nodeSeq or not a number
Some((nodeSeq \ field.getName).text.trim.toDouble)
else if (paramType == classOf[String])
Some((nodeSeq \ field.getName).text.trim)
else None
case _ => None
}
if (newValue.isDefined)
field.set(obj.get, newValue)
}
obj
}
}
}
Version creating a new object instead (might need modification for your requirements):
def getObject[A](nodeSeq: NodeSeq, objClass: Class[A]): Option[A] = {
if (nodeSeq.isEmpty) None else {
try {
val fieldValues = objClass.getDeclaredFields.flatMap { field =>
field.setAccessible(true)
field.getGenericType match {
case t: ParametrizedType if (t.getActualType == classOf[Option[_]]) =>
val paramType = t.getActualTypeArguments()(0)
if (paramType == classOf[java.lang.Double])
Some(Some((nodeSeq \ field.getName).text.trim.toDouble))
else if (paramType == classOf[String])
Some(Some((nodeSeq \ field.getName).text.trim))
else None
case _ => None
}
}
// assumes only one constructor, otherwise get the desired one
val ctor = objClass.getConstructors()(0)
Some(ctor.newInstance(fieldValues))
} catch {
case _ => None
}
}
}

How to get a name of a class member?

I want to be able to do something like this:
prepare form:
val formDescription = formBuilder(_.textField[User](_.firstName)
.textField[User](_.lastName)
).build
showForm(formDescription)
extract data from user filled form, using User:
//contains data of a form submitted by a user:
val formData: Map[String, String] = getFormData
val newUser = User(id = randomUuid, firstName = formData.extract[User](_.firstName))
One solution I see is to use a dynamic proxy that extends provided class and remembers what was invoked on him:
def getFieldName[T:Manifest](foo: T => Any) = {
val clazz = implicitly[Manifest[T]].erasure
val proxy = createDynamicProxy(clazz)
foo(proxy)
proxy.lastInvokedMethodName
}
Is there a better way to do it? Is there any lib that implements it already?
This reflective approach takes a case class and invokes its companion apply, calling getField and fetching default args if the field is not in the data.
import scala.reflect.runtime.{currentMirror => cm, universe => uni}
import uni._
def fromXML(xml: Node): Option[PluginDescription] = {
def extract[A]()(implicit tt: TypeTag[A]): Option[A] = {
// extract one field
def getField(field: String): Option[String] = {
val text = (xml \\ field).text.trim
if (text == "") None else Some(text)
}
val apply = uni.newTermName("apply")
val module = uni.typeOf[A].typeSymbol.companionSymbol.asModule
val ts = module.moduleClass.typeSignature
val m = (ts member apply).asMethod
val im = cm reflect (cm reflectModule module).instance
val mm = im reflectMethod m
def getDefault(i: Int): Option[Any] = {
val n = uni.newTermName("apply$default$" + (i+1))
val m = ts member n
if (m == NoSymbol) None
else Some((im reflectMethod m.asMethod)())
}
def extractArgs(pss: List[List[Symbol]]): List[Option[Any]] =
pss.flatten.zipWithIndex map (p => getField(p._1.name.encoded) orElse getDefault(p._2))
val args = extractArgs(m.paramss)
if (args exists (!_.isDefined)) None
else Some(mm(args.flatten: _*).asInstanceOf[A])
}
// check the top-level tag
xml match {
case <plugin>{_*}</plugin> => extract[PluginDescription]()
case _ => None
}
}
The idea was to do something like:
case class User(id: Int = randomUuid, firstName: String, lastName: String)
val user = extract[User]()
That's my own solution:
package utils
import javassist.util.proxy.{MethodHandler, MethodFilter, ProxyFactory}
import org.specs2.mutable._
import javassist.util.proxy.Proxy
import java.lang.reflect.{Constructor, Method}
class DynamicProxyTest extends Specification with MemberNameGetter {
"Dynamic proxy" should {
"extract field name" in {
memberName[TestClass](_.a) must ===("a")
memberName[TestClass](_.i) must ===("i")
memberName[TestClass](_.b) must ===("b")
memberName[TestClass](_.variable) must ===("variable")
memberName[TestClass](_.value) must ===("value")
memberName[TestClass](_.method) must ===("method")
}
}
}
trait MemberNameGetter {
def memberName[T: Manifest](foo: T => Any) = {
val mf = manifest[T]
val clazz = mf.erasure
val proxyFactory = new ProxyFactory
proxyFactory.setSuperclass(clazz)
proxyFactory.setFilter(new MethodFilter {
def isHandled(p1: Method) = true
})
val newClass = proxyFactory.createClass()
var lastInvokedMethod: String = null
val mh = new MethodHandler {
def invoke(p1: Any, p2: Method, p3: Method, p4: Array[AnyRef]) = {
lastInvokedMethod = p2.getName
p3.invoke(p1, p4: _*)
}
}
val constructor = defaultConstructor(newClass)
val parameters = defaultConstructorParameters(constructor)
// val proxy = constructor.newInstance("dsf", new Integer(0))
val proxy2 = constructor.newInstance(parameters: _*)
proxy2.asInstanceOf[Proxy].setHandler(mh)
foo(proxy2.asInstanceOf[T])
lastInvokedMethod
}
private def defaultConstructor(c: Class[_]) = c.getConstructors.head
private def defaultConstructorParameters(constructor: Constructor[_]) = {
val parameterTypes = constructor.getParameterTypes
parameterTypes.map{
case Integer.TYPE => Integer.valueOf(0)
case _ => null
}
}
}
case class TestClass(a: String, i: Int, b: Boolean) {
var variable = "asdf"
val value = "asdfasdfasd"
def method = "method"
}
val mh = new MethodHandler {
def invoke(p1: Any, p2: Method, p3: Method, p4: Array[AnyRef]) = {
lastInvokedMethod = p2.getName
p3.invoke(p1, p4: _*)
}
}
val constructor = defaultConstructor(newClass)
val parameters = defaultConstructorParameters(constructor)
// val proxy = constructor.newInstance("dsf", new Integer(0))
val proxy2 = constructor.newInstance(parameters: _*)
proxy2.asInstanceOf[Proxy].setHandler(mh)
foo(proxy2.asInstanceOf[T])
lastInvokedMethod
}
private def defaultConstructor(c: Class[_]) = c.getConstructors.head
private def defaultConstructorParameters(constructor: Constructor[_]) = {
val parameterTypes = constructor.getParameterTypes
parameterTypes.map{
case Integer.TYPE => Integer.valueOf(0)
case java.lang.Double.TYPE => java.lang.Double.valueOf(0)
case java.lang.Long.TYPE => java.lang.Long.valueOf(0)
case java.lang.Boolean.TYPE => java.lang.Boolean.FALSE
case _ => null
}
}
}
case class TestClass(a: String, i: Int, b: Boolean) {
var variable = "asdf"
val value = "asdfasdfasd"
def method = "method"
}