I have a datastrean in flink and I generate my owns metrics using gauge in a ProcessFunction.
As these metrics are important for my activity, i would like to unit test them once the flow is executed.
Unfortunately, I didn't find a way to implement a proper test reporter.
Here is a simple code explaining my issue.
Two concerns with this code:
how do i trigger the gauge
how do I get the reporter instiantiated by env.execute
Here is the sample
import java.util.concurrent.atomic.AtomicInteger
import org.apache.flink.api.scala.metrics.ScalaGauge
import org.apache.flink.configuration.{ConfigConstants, Configuration}
import org.apache.flink.metrics.reporter.AbstractReporter
import org.apache.flink.metrics.{Gauge, Metric, MetricConfig}
import org.apache.flink.streaming.api.TimeCharacteristic
import org.apache.flink.streaming.api.functions.ProcessFunction
import org.apache.flink.streaming.api.functions.sink.SinkFunction
import org.apache.flink.streaming.api.scala.{StreamExecutionEnvironment, _}
import org.apache.flink.util.Collector
import org.scalatest.FunSuite
import org.scalatest.Matchers._
import org.scalatest.PartialFunctionValues._
import scala.collection.JavaConverters._
import scala.collection.mutable
/* Test based on Flink test example https://ci.apache.org/projects/flink/flink-docs-master/dev/stream/testing.html */
class MultiplyByTwo extends ProcessFunction[Long, Long] {
override def processElement(data: Long, context: ProcessFunction[Long, Long]#Context, collector: Collector[Long]): Unit = {
collector.collect(data * 2L)
}
val nbrCalls = new AtomicInteger(0)
override def open(parameters: Configuration): Unit = {
getRuntimeContext.getMetricGroup
.addGroup("counter")
.gauge[Int, ScalaGauge[Int]]("call" , ScalaGauge[Int]( () => nbrCalls.get()))
}
}
// create a testing sink
class CollectSink extends SinkFunction[Long] {
override def invoke(value: Long): Unit = {
synchronized {
CollectSink.values.add(value)
}
}
}
object CollectSink {
val values: java.util.ArrayList[Long] = new java.util.ArrayList[Long]()
}
class StackOverflowTestReporter extends AbstractReporter {
var gaugesMetrics : mutable.Map[String, String] = mutable.Map[String, String]()
override def open(metricConfig: MetricConfig): Unit = {}
override def close(): Unit = {}
override def filterCharacters(s: String): String = s
def report(): Unit = {
gaugesMetrics = this.gauges.asScala.map(t => (metricValue(t._1), t._2))
}
private def metricValue(m: Metric): String = {
m match {
case g: Gauge[_] => g.getValue.toString
case _ => ""
}
}
}
class StackOverflowTest extends FunSuite with StreamingMultipleProgramsTestBase{
def createConfigForReporter(reporterName : String) : Configuration = {
val cfg : Configuration = new Configuration()
cfg.setString(ConfigConstants.METRICS_REPORTER_PREFIX + reporterName + "." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, classOf[StackOverflowTestReporter].getName)
cfg
}
test("test_metrics") {
val env = StreamExecutionEnvironment.createLocalEnvironment(
StreamExecutionEnvironment.getDefaultLocalParallelism,
createConfigForReporter("reporter"))
// configure your test environment
env.setParallelism(1)
env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime)
// values are collected in a static variable
CollectSink.values.clear()
// create a stream of custom elements and apply transformations
env.fromElements[Long](1L, 21L, 22L)
.process(new MultiplyByTwo())
.addSink(new CollectSink())
// execute
env.execute()
// verify your results
CollectSink.values should have length 3
CollectSink.values should contain (2L)
CollectSink.values should contain (42L)
CollectSink.values should contain (44L)
//verify gauge counter
//pseudo code ...
val testReporter : StackOverflowTestReporter = _ // how to get testReporter instantiate in env
testReporter.gaugesMetrics should have size 1
testReporter.gaugesMetrics should contain key "count.call"
testReporter.gaugesMetrics.valueAt("count.call") should be equals("3")
}
}
Solution thanks to Chesnay Schepler
import java.util.concurrent.atomic.AtomicInteger
import org.apache.flink.api.common.time.Time
import org.apache.flink.api.scala.metrics.ScalaGauge
import org.apache.flink.configuration.{ConfigConstants, Configuration}
import org.apache.flink.metrics.reporter.MetricReporter
import org.apache.flink.metrics.{Metric, MetricConfig, MetricGroup}
import org.apache.flink.streaming.api.functions.ProcessFunction
import org.apache.flink.streaming.api.functions.sink.SinkFunction
import org.apache.flink.streaming.api.scala.{StreamExecutionEnvironment, _}
import org.apache.flink.test.util.MiniClusterResource
import org.apache.flink.util.Collector
import org.scalatest.Matchers._
import org.scalatest.PartialFunctionValues._
import org.scalatest.{BeforeAndAfterAll, FunSuite}
import scala.collection.mutable
/* Test based on Flink test example https://ci.apache.org/projects/flink/flink-docs-master/dev/stream/testing.html */
class MultiplyByTwo extends ProcessFunction[Long, Long] {
override def processElement(data: Long, context: ProcessFunction[Long, Long]#Context, collector: Collector[Long]): Unit = {
nbrCalls.incrementAndGet()
collector.collect(data * 2L)
}
val nbrCalls = new AtomicInteger(0)
override def open(parameters: Configuration): Unit = {
getRuntimeContext.getMetricGroup
.addGroup("counter")
.gauge[Int, ScalaGauge[Int]]("call" , ScalaGauge[Int]( () => nbrCalls.get()))
}
}
// create a testing sink
class CollectSink extends SinkFunction[Long] {
import CollectSink._
override def invoke(value: Long): Unit = {
synchronized {
values.add(value)
}
}
}
object CollectSink {
val values: java.util.ArrayList[Long] = new java.util.ArrayList[Long]()
}
class StackOverflowTestReporter extends MetricReporter {
import StackOverflowTestReporter._
override def open(metricConfig: MetricConfig): Unit = {}
override def close(): Unit = {}
override def notifyOfAddedMetric(metric: Metric, metricName: String, group: MetricGroup) : Unit = {
metric match {
case gauge: ScalaGauge[_] => {
//drop group metrics meaningless for the test, seem's to be the first 6 items
val gaugeKey = group.getScopeComponents.toSeq.drop(6).mkString(".") + "." + metricName
gaugesMetrics(gaugeKey) = gauge.asInstanceOf[ScalaGauge[Int]]
}
case _ =>
}
}
override def notifyOfRemovedMetric(metric: Metric, metricName: String, group: MetricGroup): Unit = {}
}
object StackOverflowTestReporter {
var gaugesMetrics : mutable.Map[String, ScalaGauge[Int]] = mutable.Map[String, ScalaGauge[Int]]()
}
class StackOverflowTest extends FunSuite with BeforeAndAfterAll{
val miniClusterResource : MiniClusterResource = buildMiniClusterResource()
override def beforeAll(): Unit = {
CollectSink.values.clear()
StackOverflowTestReporter.gaugesMetrics.clear()
miniClusterResource.before()
}
override def afterAll(): Unit = {
miniClusterResource.after()
}
def createConfigForReporter() : Configuration = {
val cfg : Configuration = new Configuration()
cfg.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "reporter" + "." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, classOf[StackOverflowTestReporter].getName)
cfg
}
def buildMiniClusterResource() : MiniClusterResource = new MiniClusterResource(
new MiniClusterResource.MiniClusterResourceConfiguration(
createConfigForReporter(),1,1, Time.milliseconds(50L)))
test("test_metrics") {
val env = StreamExecutionEnvironment.getExecutionEnvironment
env.fromElements[Long](1L, 21L, 22L)
.process(new MultiplyByTwo())
.addSink(new CollectSink())
env.execute()
CollectSink.values should have length 3
CollectSink.values should contain (2L)
CollectSink.values should contain (42L)
CollectSink.values should contain (44L)
//verify gauge counter
val gaugeValues = StackOverflowTestReporter.gaugesMetrics.map(t => (t._1, t._2.getValue()))
gaugeValues should have size 1
gaugeValues should contain ("counter.call" -> 3)
}
}
your best bet is to use a MiniClusterResource to explicitly start a cluster before the job and configure a reporter that checks for specific metrics and exposes them through static fields.
#Rule
public final MiniClusterResource clusterResource = new MiniClusterResource(
new MiniClusterResourceConfiguration.Builder()
.setConfiguration(getConfig()));
private static Configuration getConfig() {
Configuration config = new Configuration();
config.setString(
ConfigConstants.METRICS_REPORTER_PREFIX +
"myTestReporter." +
ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX,
MyTestReporter.class.getName());
return config;
}
public static class MyTestReporter implements MetricReporter {
static volatile Gauge<?> myGauge = null;
#Override
public void open(MetricConfig metricConfig) {
}
#Override
public void close() {
}
#Override
public void notifyOfAddedMetric(Metric metric, String name, MetricGroup metricGroup) {
if ("myMetric".equals(name)) {
myGauge = (Gauge<?>) metric;
}
}
#Override
public void notifyOfRemovedMetric(Metric metric, String s, MetricGroup metricGroup) {
}
}
I am a Scala newbie, and I am moving my first steps with Spray.io. I have defined a bunch of routes that seem to work just fine when I manually run them through Curl. However, I do get a nasty NullPointerException when I try to unit test the 'GET /documents?q=' route using spray.testkit.Specs2RouteTest.
Here is the route definition:
trait MyService extends HttpService {
import DocumentJsonProtocol._
val searchService: SearchService
val myRoute =
path("documents") {
post {
entity(as[Document]) { doc =>
detach() {
val docId = searchService.indexDocument(doc)
complete(201, s"document created: $docId")
}
}
} ~
get {
parameter('q) { q =>
detach() {
val docs = searchService.matchingQuery(q)
complete(docs)
}
}
}
} ~
path("documents" / Segment ) { docId =>
get {
detach() {
searchService.getDocument(docId) match {
case Some(doc) => {
complete(doc)
}
case None => {
complete(404, s"Cannot find a document with id: $docId")
}
}
}
}
}
}
And here is the failing test:
package example.com
import org.specs2.mutable.Specification
import org.specs2.specification.Scope
import org.specs2.mock._
import spray.testkit.Specs2RouteTest
import spray.http._
import spray.httpx.marshalling._
import StatusCodes._
import com.example.services.SearchService
import com.example.data._
import DocumentJsonProtocol._
class MyServiceSpec extends Specification
with Specs2RouteTest
with Mockito
with MyService {
val searchService = mock[SearchService]
def actorRefFactory = system
trait testDoc extends Scope {
val docId = "test-id"
val testDoc =
Document(Some(docId), "test-title", "test-author", "body", None)
searchService.indexDocument(testDoc).returns(docId)
searchService.matchingQuery("test-title").returns(List(testDoc))
}
"MyService" should {
"allows to search documents by keyword" in new testDoc {
Get("/documents?q=hello-test") ~> myRoute ~> check {
Right(entity) === marshal(List(testDoc))
}
}
This fails with the following error:
[ERROR] [04/07/2014 23:56:50.689] [io-composable-MyServiceSpec-akka.actor.default-dispatcher-3] [ActorSystem(io-composable-MyServiceSpec)] Error during processing of request HttpRequest(GET,http://ex
ample.com/documents?q=hello-test,List(),Empty,HTTP/1.1)
java.lang.NullPointerException
at spray.json.CollectionFormats$$anon$1.write(CollectionFormats.scala:26)
at spray.json.CollectionFormats$$anon$1.write(CollectionFormats.scala:25)
at spray.httpx.SprayJsonSupport$$anonfun$sprayJsonMarshaller$1.apply(SprayJsonSupport.scala:43)
at spray.httpx.SprayJsonSupport$$anonfun$sprayJsonMarshaller$1.apply(SprayJsonSupport.scala:42)
at spray.httpx.marshalling.Marshaller$MarshallerDelegation$$anonfun$apply$1.apply(Marshaller.scala:58)
at spray.httpx.marshalling.Marshaller$MarshallerDelegation$$anonfun$apply$1.apply(Marshaller.scala:58)
at spray.httpx.marshalling.Marshaller$MarshallerDelegation$$anonfun$apply$2.apply(Marshaller.scala:61)
at spray.httpx.marshalling.Marshaller$MarshallerDelegation$$anonfun$apply$2.apply(Marshaller.scala:60)
at spray.httpx.marshalling.Marshaller$$anon$2.apply(Marshaller.scala:47)
at spray.httpx.marshalling.BasicToResponseMarshallers$$anon$1.apply(BasicToResponseMarshallers.scala:35)
at spray.httpx.marshalling.BasicToResponseMarshallers$$anon$1.apply(BasicToResponseMarshallers.scala:22)
at spray.httpx.marshalling.ToResponseMarshaller$$anonfun$compose$1.apply(Marshaller.scala:69)
at spray.httpx.marshalling.ToResponseMarshaller$$anonfun$compose$1.apply(Marshaller.scala:69)
at spray.httpx.marshalling.ToResponseMarshaller$$anon$3.apply(Marshaller.scala:81)
at spray.httpx.marshalling.ToResponseMarshallable$$anon$6.marshal(Marshaller.scala:141)
at spray.httpx.marshalling.ToResponseMarshallable$$anon$7.apply(Marshaller.scala:145)
at spray.httpx.marshalling.ToResponseMarshallable$$anon$7.apply(Marshaller.scala:144)
at spray.routing.RequestContext.complete(RequestContext.scala:235)
The problem I could see after reading your code is that you mock matchingQuery function with "test-title" argument and make request with "hello-test" query parameter which leads mock to return null.
How to add a extra field in todo application created using Play framework using Scala? I am using anorm DB... I am getting an error named "not found: value Task" in Application.scala at line 24. I have tried it out, please point out my mistake. Thanks in advance!
task.scala:
package models
import anorm._
import anorm.SqlParser._
import play.api.db._
import play.api.Play.current
case class Task(id: Long, label: String, name: String)
object Task {
val task = {
get[Long]("id") ~
get[String]("label") ~
get[String]("name") map {
case label~name => Task(id, name)
case id~label => Task(id, label)
}
}
def all(): List[Task] = DB.withConnection { implicit c =>
SQL("select * from task").as(task *)
}
def create(task: Task): Unit= {
DB.withConnection { implicit c =>
SQL("insert into task (label,name) values ({label},{name})").on(
'label -> label,
'name -> name
).executeUpdate()
}
}
def delete(id: Long) {
DB.withConnection { implicit c =>
SQL("delete from task where id = {id}").on(
'id -> id
).executeUpdate()
}
}
}
application.scala (controller class):
package controllers
import play.api._
import play.api.mvc._
import play.api.data._
import play.api.data.Forms._
import play.api.data.Form
import play.api.data.Forms.{tuple,nonEmptyText}
import play.api.mvc.{Action, Controller}
import anorm.NotAssigned
import models.Task
object Application extends Controller {
def index = Action {
Redirect(routes.Application.tasks)
}
val taskForm = Form(
tuple(
"label" -> nonEmptyText,
"name" -> nonEmptyText
)
)
def tasks = Action {
Ok(views.html.index(Task.all(), taskForm))
}
def showTask= Action {
Ok(views.html.test(Task.all(), taskForm))
}
def newTask = Action { implicit request =>
taskForm.bindFromRequest.fold(
errors => BadRequest(views.html.index(Task.all(), errors)),
{
case (label, name) => {
Task.create(Task(NotAssigned, label, name))
Redirect(routes.Application.showTask)
}
}
)
}
def deleteTask(id: Long) = Action {
Task.delete(id)
Redirect(routes.Application.showTask)
}
}
Index (view file):
#(tasks: List[Task], taskForm: Form[(String, String)])
#import helper._
<h2>Add a new task</h2>
#form(routes.Application.newTask) {
#inputText(taskForm("label"))
#inputText(taskForm("name"))
<input type="submit" value="Create">
}
test.html (view file 2):
#(tasks: List[Task], taskForm: Form[(String,String)])
#import helper._
#main("Todo list") {
<h1>#tasks.size task(s)</h1>
<ul>
#tasks.map { task =>
<li>
<b>#task.label</b>
<b>#task.name</b>
#form(routes.Application.deleteTask(task.id)) {
<input type="submit" value="Delete">
}
</li>
}
</ul>
}
Try to use :
(apply) and (unapply)
methods properly for form element.
(Task.apply)(Task.unapply)
import models.Task._ imports all the methods on the companion object models.Task into the current scope, not the Task class and object themselves. So the current code would allow you to just call all and it would refer to Task.all
Change the import to import models.Task to instead get Task into the scope in your Application object and you will be able to use the task methods like you are trying to.
I am using Play framework 2.1.1 with scala.I query a database table return to controller as list and then convert list to string and return to ajax call from javascript code.
How to return query result as json and return to ajax call throught controller?
Application.scala
import play.api._
import play.api.mvc._
import play.api.data._
import views.html._
import models._
object Application extends Controller {
def index = Action {
Ok(views.html.index())
}
def getSummaryTable = Action{
var sum="Summary Table";
Ok(ajax_result.render((Timesheet.getAll).mkString("\n")))
}
def javascriptRoutes = Action { implicit request =>
import routes.javascript._
Ok(
Routes.javascriptRouter("jsRoutes")(
// Routes
controllers.routes.javascript.Application.getSummaryTable
)
).as("text/javascript")
}
}
TimeSheet.scala
// Use PostgresDriver to connect to a Postgres database
import scala.slick.driver.PostgresDriver.simple._
import scala.slick.lifted.{MappedTypeMapper,BaseTypeMapper,TypeMapperDelegate}
import scala.slick.driver.BasicProfile
import scala.slick.session.{PositionedParameters,PositionedResult}
// Use the implicit threadLocalSession
import Database.threadLocalSession
import java.sql.Date
import java.sql.Time
case class Timesheet(ID: Int, dateVal: String, entryTime: Time, exitTime: Time, someVal: String)
object Timesheet {
//Definition of Timesheet table
// object TS extends Table[(Int,String,Time,Time,String)]("timesheet"){
val TSTable = new Table[Timesheet]("timesheet"){
def ID = column[Int]("id")
def dateVal = column[String]("date")
def entryTime = column[Time]("entry_time")
def exitTime = column[Time]("exit_time")
def someVal = column[String]("someval")
def * = ID ~ dateVal ~ entryTime ~ exitTime ~ someVal <> (Timesheet.apply _, Timesheet.unapply _)
}
def getAll: Seq[Timesheet] = {
Database.forURL("jdbc:postgresql://localhost:5432/my_db", "postgres", "password",null, driver="org.postgresql.Driver") withSession{
val q = Query(TSTable)
val qFiltered = q.filter(_.ID === 41 )
val qDateFilter = qFiltered.filter(_.dateVal === "01/03/2013")
val qSorted = qDateFilter.sortBy(_.entryTime)
qSorted.list
}
}
}
Also, don't forget to provide an implicit (or not) Json deserializer for your model, otherwise, Scala compiler will yell at you :-). You can do something like :
def allTimesheet = Action {
val timesheetWrites = Json.writes[Timesheet] // here it's the deserializer
val listofTimeSheet = Timesheet.getAll
Ok( Json.toJson( listofTimeSheet )( timesheetWrites ) )
}
or you can use implicits like :
def allTimesheet = Action {
implicit val timesheetWrites = Json.writes[Timesheet] // here it's the deserializer
val listofTimeSheet = Timesheet.getAll
Ok( Json.toJson( listofTimeSheet ) )
}
and even declare your deserializer in your model companion object like :
companion object
object Timesheet {
implicit val timesheetWrites = Json.writes[Timesheet] // here it's the deserializer
....
}
and in the controller
import models.Timesheet.timesheetWrites
def allTimesheet = Action {
val listofTimeSheet = Timesheet.getAll
Ok( Json.toJson( listofTimeSheet ) )
}
I recommend you use play.api.libs.Json.toJson.
Here's an example:
object Products extends Controller {
def list = Action {
val productCodes = Product.findAll.map(_.ean)
Ok(Json.toJson(productCodes))
}
Json.toJson returns a JsValue for which Play will automatically add a application/json header.
See Play For Scala chapter 8.
I found the following question/answer:
Test MultipartFormData in Play 2.0 FakeRequest
But it seems things have changed in Play 2.1. I've tried adapting the example like so:
"Application" should {
"Upload Photo" in {
running(FakeApplication()) {
val data = new MultipartFormData(Map(), List(
FilePart("qqfile", "message", Some("Content-Type: multipart/form-data"),
TemporaryFile(getClass().getResource("/test/photos/DSC03024.JPG").getFile()))
), List())
val Some(result) = routeAndCall(FakeRequest(POST, "/admin/photo/upload", FakeHeaders(), data))
status(result) must equalTo(CREATED)
headers(result) must contain(LOCATION)
contentType(result) must beSome("application/json")
However whenever I attempt to run the request, I get a null-pointer exception:
[error] ! Upload Photo
[error] NullPointerException: null (PhotoManagementSpec.scala:25)
[error] test.PhotoManagementSpec$$anonfun$1$$anonfun$apply$3$$anonfun$apply$4.apply(PhotoManagementSpec.scala:28)
[error] test.PhotoManagementSpec$$anonfun$1$$anonfun$apply$3$$anonfun$apply$4.apply(PhotoManagementSpec.scala:25)
[error] play.api.test.Helpers$.running(Helpers.scala:40)
[error] test.PhotoManagementSpec$$anonfun$1$$anonfun$apply$3.apply(PhotoManagementSpec.scala:25)
[error] test.PhotoManagementSpec$$anonfun$1$$anonfun$apply$3.apply(PhotoManagementSpec.scala:25)
If I try to replace the deprecated routeAndCall with just route (and remove the Option around result), I get a compile error stating that it can't write an instance of MultipartFormData[TemporaryFile] to the HTTP response.
What's the right way to design this test in Play 2.1 with Scala?
Edit: Tried to modify the code to test just the controller:
"Application" should {
"Upload Photo" in {
val data = new MultipartFormData(Map(), List(
FilePart("qqfile", "message", Some("Content-Type: multipart/form-data"),
TemporaryFile(getClass().getResource("/test/photos/DSC03024.JPG").getFile()))
), List())
val result = controllers.Photo.upload()(FakeRequest(POST, "/admin/photo/upload",FakeHeaders(),data))
status(result) must equalTo(OK)
contentType(result) must beSome("text/html")
charset(result) must beSome("utf-8")
contentAsString(result) must contain("Hello Bob")
}
But I now get a type error on all the test conditions around the results like so:
[error] found : play.api.libs.iteratee.Iteratee[Array[Byte],play.api.mvc.Result]
[error] required: play.api.mvc.Result
I don't understand why I'm getting an Interator for byte arrays mapped to Results. Could this have something to do with how I'm using a custom body parser? My controller's definition looks like this:
def upload = Action(CustomParsers.multipartFormDataAsBytes) { request =>
request.body.file("qqfile").map { upload =>
Using the form parser from this post: Pulling files from MultipartFormData in memory in Play2 / Scala
Play 2.3 includes a newer version of httpmime.jar, requiring some minor corrections. Building on Marcus's solution using Play's Writeable mechanism, while retaining some of the syntactic sugar from my Play 2.1 solution, this is what I've come up with:
import scala.language.implicitConversions
import java.io.{ByteArrayOutputStream, File}
import org.apache.http.entity.ContentType
import org.apache.http.entity.mime.MultipartEntityBuilder
import org.apache.http.entity.mime.content._
import org.specs2.mutable.Specification
import play.api.http._
import play.api.libs.Files.TemporaryFile
import play.api.mvc.MultipartFormData.FilePart
import play.api.mvc.{Codec, MultipartFormData}
import play.api.test.Helpers._
import play.api.test.{FakeApplication, FakeRequest}
trait FakeMultipartUpload {
implicit def writeableOf_multiPartFormData(implicit codec: Codec): Writeable[MultipartFormData[TemporaryFile]] = {
val builder = MultipartEntityBuilder.create().setBoundary("12345678")
def transform(multipart: MultipartFormData[TemporaryFile]): Array[Byte] = {
multipart.dataParts.foreach { part =>
part._2.foreach { p2 =>
builder.addPart(part._1, new StringBody(p2, ContentType.create("text/plain", "UTF-8")))
}
}
multipart.files.foreach { file =>
val part = new FileBody(file.ref.file, ContentType.create(file.contentType.getOrElse("application/octet-stream")), file.filename)
builder.addPart(file.key, part)
}
val outputStream = new ByteArrayOutputStream
builder.build.writeTo(outputStream)
outputStream.toByteArray
}
new Writeable[MultipartFormData[TemporaryFile]](transform, Some(builder.build.getContentType.getValue))
}
/** shortcut for generating a MultipartFormData with one file part which more fields can be added to */
def fileUpload(key: String, file: File, contentType: String): MultipartFormData[TemporaryFile] = {
MultipartFormData(
dataParts = Map(),
files = Seq(FilePart[TemporaryFile](key, file.getName, Some(contentType), TemporaryFile(file))),
badParts = Seq(),
missingFileParts = Seq())
}
/** shortcut for a request body containing a single file attachment */
case class WrappedFakeRequest[A](fr: FakeRequest[A]) {
def withFileUpload(key: String, file: File, contentType: String) = {
fr.withBody(fileUpload(key, file, contentType))
}
}
implicit def toWrappedFakeRequest[A](fr: FakeRequest[A]) = WrappedFakeRequest(fr)
}
class MyTest extends Specification with FakeMultipartUpload {
"uploading" should {
"be easier than this" in {
running(FakeApplication()) {
val uploadFile = new File("/tmp/file.txt")
val req = FakeRequest(POST, "/upload/path").
withFileUpload("image", uploadFile, "image/gif")
val response = route(req).get
status(response) must equalTo(OK)
}
}
}
}
I managed to get this working with Play 2.1 based on various mailing list suggestions. Here's how I do it:
import scala.language.implicitConversions
import java.io.{ ByteArrayOutputStream, File }
import org.apache.http.entity.mime.MultipartEntity
import org.apache.http.entity.mime.content.{ ContentBody, FileBody }
import org.specs2.mutable.Specification
import play.api.http.Writeable
import play.api.test.{ FakeApplication, FakeRequest }
import play.api.test.Helpers._
trait FakeMultipartUpload {
case class WrappedFakeRequest[A](fr: FakeRequest[A]) {
def withMultipart(parts: (String, ContentBody)*) = {
// create a multipart form
val entity = new MultipartEntity()
parts.foreach { part =>
entity.addPart(part._1, part._2)
}
// serialize the form
val outputStream = new ByteArrayOutputStream
entity.writeTo(outputStream)
val bytes = outputStream.toByteArray
// inject the form into our request
val headerContentType = entity.getContentType.getValue
fr.withBody(bytes).withHeaders(CONTENT_TYPE -> headerContentType)
}
def withFileUpload(fileParam: String, file: File, contentType: String) = {
withMultipart(fileParam -> new FileBody(file, contentType))
}
}
implicit def toWrappedFakeRequest[A](fr: FakeRequest[A]) = WrappedFakeRequest(fr)
// override Play's equivalent Writeable so that the content-type header from the FakeRequest is used instead of application/octet-stream
implicit val wBytes: Writeable[Array[Byte]] = Writeable(identity, None)
}
class MyTest extends Specification with FakeMultipartUpload {
"uploading" should {
"be easier than this" in {
running(FakeApplication()) {
val uploadFile = new File("/tmp/file.txt")
val req = FakeRequest(POST, "/upload/path").
withFileUpload("image", uploadFile, "image/gif")
val response = route(req).get
status(response) must equalTo(OK)
}
}
}
}
I've modified Alex's code to act as a Writable which better integrates into Play 2.2.2
package test
import play.api.http._
import play.api.mvc.MultipartFormData.FilePart
import play.api.libs.iteratee._
import play.api.libs.Files.TemporaryFile
import play.api.mvc.{Codec, MultipartFormData }
import java.io.{FileInputStream, ByteArrayOutputStream}
import org.apache.commons.io.IOUtils
import org.apache.http.entity.mime.MultipartEntity
import org.apache.http.entity.mime.content._
object MultipartWriteable {
/**
* `Writeable` for multipart/form-data.
*
*/
implicit def writeableOf_multiPartFormData(implicit codec: Codec): Writeable[MultipartFormData[TemporaryFile]] = {
val entity = new MultipartEntity()
def transform(multipart: MultipartFormData[TemporaryFile]):Array[Byte] = {
multipart.dataParts.foreach { part =>
part._2.foreach { p2 =>
entity.addPart(part._1, new StringBody(p2))
}
}
multipart.files.foreach { file =>
val part = new FileBody(file.ref.file, file.filename, file.contentType.getOrElse("application/octet-stream"), null)
entity.addPart(file.key, part)
}
val outputStream = new ByteArrayOutputStream
entity.writeTo(outputStream)
val bytes = outputStream.toByteArray
outputStream.close
bytes
}
new Writeable[MultipartFormData[TemporaryFile]](transform, Some(entity.getContentType.getValue))
}
}
This way it is possible to write something like this:
val filePart:MultipartFormData.FilePart[TemporaryFile] = MultipartFormData.FilePart(...)
val fileParts:Seq[MultipartFormData.FilePart[TemporaryFile]] = Seq(filePart)
val dataParts:Map[String, Seq[String]] = ...
val multipart = new MultipartFormData[TemporaryFile](dataParts, fileParts, List(), List())
val request = FakeRequest(POST, "/url", FakeHeaders(), multipart)
var result = route(request).get
Following EEColor's suggestion, I got the following to work:
"Upload Photo" in {
val file = scala.io.Source.fromFile(getClass().getResource("/photos/DSC03024.JPG").getFile())(scala.io.Codec.ISO8859).map(_.toByte).toArray
val data = new MultipartFormData(Map(), List(
FilePart("qqfile", "DSC03024.JPG", Some("image/jpeg"),
file)
), List())
val result = controllers.Photo.upload()(FakeRequest(POST, "/admin/photos/upload",FakeHeaders(),data))
status(result) must equalTo(CREATED)
headers(result) must haveKeys(LOCATION)
contentType(result) must beSome("application/json")
}
Here's my version of Writeable[AnyContentAsMultipartFormData]:
import java.io.File
import play.api.http.{HeaderNames, Writeable}
import play.api.libs.Files.TemporaryFile
import play.api.mvc.MultipartFormData.FilePart
import play.api.mvc.{AnyContentAsMultipartFormData, Codec, MultipartFormData}
object MultipartFormDataWritable {
val boundary = "--------ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890"
def formatDataParts(data: Map[String, Seq[String]]) = {
val dataParts = data.flatMap { case (key, values) =>
values.map { value =>
val name = s""""$key""""
s"--$boundary\r\n${HeaderNames.CONTENT_DISPOSITION}: form-data; name=$name\r\n\r\n$value\r\n"
}
}.mkString("")
Codec.utf_8.encode(dataParts)
}
def filePartHeader(file: FilePart[TemporaryFile]) = {
val name = s""""${file.key}""""
val filename = s""""${file.filename}""""
val contentType = file.contentType.map { ct =>
s"${HeaderNames.CONTENT_TYPE}: $ct\r\n"
}.getOrElse("")
Codec.utf_8.encode(s"--$boundary\r\n${HeaderNames.CONTENT_DISPOSITION}: form-data; name=$name; filename=$filename\r\n$contentType\r\n")
}
val singleton = Writeable[MultipartFormData[TemporaryFile]](
transform = { form: MultipartFormData[TemporaryFile] =>
formatDataParts(form.dataParts) ++
form.files.flatMap { file =>
val fileBytes = Files.readAllBytes(Paths.get(file.ref.file.getAbsolutePath))
filePartHeader(file) ++ fileBytes ++ Codec.utf_8.encode("\r\n")
} ++
Codec.utf_8.encode(s"--$boundary--")
},
contentType = Some(s"multipart/form-data; boundary=$boundary")
)
}
implicit val anyContentAsMultipartFormWritable: Writeable[AnyContentAsMultipartFormData] = {
MultipartFormDataWritable.singleton.map(_.mdf)
}
It's adapted from (and some bugs fixed): https://github.com/jroper/playframework/blob/multpart-form-data-writeable/framework/src/play/src/main/scala/play/api/http/Writeable.scala#L108
See the whole post here, if you are interested: http://tech.fongmun.com/post/125479939452/test-multipartformdata-in-play
For me, the best solution for this problem is the Alex Varju one
Here is a version updated for Play 2.5:
object FakeMultipartUpload {
implicit def writeableOf_multiPartFormData(implicit codec: Codec): Writeable[AnyContentAsMultipartFormData] = {
val builder = MultipartEntityBuilder.create().setBoundary("12345678")
def transform(multipart: AnyContentAsMultipartFormData): ByteString = {
multipart.mdf.dataParts.foreach { part =>
part._2.foreach { p2 =>
builder.addPart(part._1, new StringBody(p2, ContentType.create("text/plain", "UTF-8")))
}
}
multipart.mdf.files.foreach { file =>
val part = new FileBody(file.ref.file, ContentType.create(file.contentType.getOrElse("application/octet-stream")), file.filename)
builder.addPart(file.key, part)
}
val outputStream = new ByteArrayOutputStream
builder.build.writeTo(outputStream)
ByteString(outputStream.toByteArray)
}
new Writeable(transform, Some(builder.build.getContentType.getValue))
}
}
In Play 2.6.x you can write test cases in the following way to test file upload API:
class HDFSControllerTest extends Specification {
"HDFSController" should {
"return 200 Status for file Upload" in new WithApplication {
val tempFile = SingletonTemporaryFileCreator.create("txt","csv")
tempFile.deleteOnExit()
val data = new MultipartFormData[TemporaryFile](Map(),
List(FilePart("metadata", "text1.csv", Some("text/plain"), tempFile)), List())
val res: Option[Future[Result]] = route(app, FakeRequest(POST, "/api/hdfs").withMultipartFormDataBody(data))
print(contentAsString(res.get))
res must beSome.which(status(_) == OK)
}
}
}
Made Alex's version compatible with Play 2.8
import akka.util.ByteString
import java.io.ByteArrayOutputStream
import org.apache.http.entity.mime.content.StringBody
import org.apache.http.entity.ContentType
import org.apache.http.entity.mime.content.FileBody
import org.apache.http.entity.mime.MultipartEntityBuilder
import play.api.http.Writeable
import play.api.libs.Files.TemporaryFile
import play.api.mvc.Codec
import play.api.mvc.MultipartFormData
import play.api.mvc.MultipartFormData.FilePart
import play.api.test.FakeRequest
trait FakeMultipartUpload {
implicit def writeableOf_multiPartFormData(
implicit codec: Codec
): Writeable[MultipartFormData[TemporaryFile]] = {
val builder = MultipartEntityBuilder.create().setBoundary("12345678")
def transform(multipart: MultipartFormData[TemporaryFile]): ByteString = {
multipart.dataParts.foreach { part =>
part._2.foreach { p2 =>
builder.addPart(part._1, new StringBody(p2, ContentType.create("text/plain", "UTF-8")))
}
}
multipart.files.foreach { file =>
val part = new FileBody(
file.ref.file,
ContentType.create(file.contentType.getOrElse("application/octet-stream")),
file.filename
)
builder.addPart(file.key, part)
}
val outputStream = new ByteArrayOutputStream
builder.build.writeTo(outputStream)
ByteString(outputStream.toByteArray)
}
new Writeable(transform, Some(builder.build.getContentType.getValue))
}
/** shortcut for generating a MultipartFormData with one file part which more fields can be added to */
def fileUpload(
key: String,
file: TemporaryFile,
contentType: String
): MultipartFormData[TemporaryFile] = {
MultipartFormData(
dataParts = Map(),
files = Seq(FilePart[TemporaryFile](key, file.file.getName, Some(contentType), file)),
badParts = Seq()
)
}
/** shortcut for a request body containing a single file attachment */
case class WrappedFakeRequest[A](fr: FakeRequest[A]) {
def withFileUpload(key: String, file: TemporaryFile, contentType: String) = {
fr.withBody(fileUpload(key, file, contentType))
}
}
implicit def toWrappedFakeRequest[A](fr: FakeRequest[A]) = WrappedFakeRequest(fr)
}