I'm trying to get started with avro & avro4s and I'm running into trouble getting a silly example working.
I can't even get my own version of a test found in the source repo to compile - https://github.com/sksamuel/avro4s/blob/master/avro4s-core/src/test/scala/com/sksamuel/avro4s/AvroJsonInputTest.scala#L14-L20
I'm getting the error mentioned in the title could not find implicit value for parameter fromRecord: com.sksamuel.avro4s.FromRecord[Foo].
package test
import java.io.ByteArrayInputStream
import com.sksamuel.avro4s._
import org.specs2.mutable.Specification
class AvroSpec extends Specification {
"My classes" should {
"be deserialized by avro" in {
case class Foo(num: Int, name: String)
val json = """{ "num": 17, "name": "jibba-jabbba" }"""
val inst = Foo(num = 17, name = "jibba-jabba")
val in = new ByteArrayInputStream(json.getBytes("UTF-8"), 0, json.length)
val foo = new AvroJsonInputStream[Foo](in).iterator.toSet
in.close()
foo mustEqual Set(inst)
}
}
}
I'm using Scala 2.11.8 and avro 1.7.0. What am I doing wrong?
Move the case class declaration outside the method definition:
package test
import java.io.ByteArrayInputStream
import com.sksamuel.avro4s._
import org.specs2.mutable.Specification
case class Foo(num: Int, name: String)
class AvroSpec extends Specification {
"My classes" should {
"be deserialized by avro" in {
val json = """{ "num": 17, "name": "jibba-jabbba" }"""
val inst = Foo(num = 17, name = "jibba-jabba")
val in = new ByteArrayInputStream(json.getBytes("UTF-8"), 0, json.length)
val foo = new AvroJsonInputStream[Foo](in).iterator.toSet
in.close()
foo mustEqual Set(inst)
}
}
}
Related
I want to use JCommander to parse args.
I wrote some code:
import com.beust.jcommander.{JCommander, Parameter}
import scala.collection.mutable.ArrayBuffer
object Config {
#Parameter(names = Array("--categories"), required = true)
var categories = new ArrayBuffer[String]
}
object Main {
def main(args: Array[String]): Unit = {
val cfg = Config
JCommander
.newBuilder()
.addObject(cfg)
.build()
.parse(args.toArray: _*)
println(cfg.categories)
}
}
Howewer it fails with
com.beust.jcommander.ParameterException: Could not invoke null
Reason: Can not set static scala.collection.mutable.ArrayBuffer field InterestRulesConfig$.categories to java.lang.String
What am i doing wrong?
JCommander uses knowledge about types in Java to map values to parameters. But Java doesn't have a type scala.collection.mutable.ArrayBuffer. It has a type java.util.List. If you want to use JCommander you have to stick to Java's build-in types.
If you want to use Scala's types use one of Scala's libraries that handle in in more idiomatic manner: scopt or decline.
Working example
import java.util
import com.beust.jcommander.{JCommander, Parameter}
import scala.jdk.CollectionConverters._
object Config {
#Parameter(names = Array("--categories"), required = true)
var categories: java.util.List[Integer] = new util.ArrayList[Integer]()
}
object Hello {
def main(args: Array[String]): Unit = {
val cfg = Config
JCommander
.newBuilder()
.addObject(cfg)
.build()
.parse(args.toArray: _*)
println(cfg.categories)
println(cfg.categories.getClass())
val a = cfg.categories.asScala
for (x <- a) {
println(x.toInt)
println(x.toInt.getClass())
}
}
}
I am trying to run the following program
import akka.actor.ActorSystem
import akka.http.scaladsl.Http
import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport
import akka.http.scaladsl.marshalling.ToResponseMarshallable
import akka.http.scaladsl.model.{ContentTypes, HttpEntity}
import akka.http.scaladsl.server.Directives._
import akka.http.scaladsl.server.directives.BasicDirectives
import akka.io.IO
import akka.stream.ActorMaterializer
import com.typesafe.config.ConfigFactory
import spray.json.DefaultJsonProtocol
import scala.concurrent.duration._
import scala.concurrent.Await
trait JsonSupport extends SprayJsonSupport with DefaultJsonProtocol {
implicit val itemFormat = jsonFormat2(Item)
}
final case class Item(name: String, id: Long)
object Main extends App with RestInterface {
val config = ConfigFactory.load()
val host = config.getString("http.host")
val port = config.getInt("http.port")
implicit val system = ActorSystem("My-ActorSystem")
implicit val executionContext = system.dispatcher
implicit val materializer = ActorMaterializer()
//implicit val timeout = Timeout(10 seconds)
val api = routes
Http().bindAndHandle(api, host, port) map {
binding => println(s"The binding local address is ${binding.localAddress}")
}
// recover
// { case ex => println(s"some shit occurred and it is"+ex.getMessage) }
// sys.addShutdownHook {
// system.log.info("Shutting down Spray HTTP in 10 seconds...")
// // NOTE(soakley): We are waiting to unbind to allow the VIP time to query healthcheck status.
// Thread.sleep(10000)
// system.log.info("Shutting down Spray HTTP and allowing 10 seconds for connections to drain...")
// val unbindFuture = Http.unbind(10.seconds)
//
// Await.ready(unbindFuture, 10.seconds)
// }
}
case class Stringer(str1 : String, str2 : String, str3 : String)
trait RestInterface extends Resource {
val routes = questionroutes ~ mydirectiveroute01 ~ mydirectiveroute02
}
trait Resource extends QuestionResource with mydirective01 with mydirective02
trait QuestionResource {
val questionroutes = {
path("hi") {
get {
val stringer_instance = Stringer("questionairre created","whatever","whatever-whatever")
complete(ToResponseMarshallable(stringer_instance.toString()))
}
}
}
}
trait mydirective01 extends BasicDirectives {
val getandputdirective = get | put
val mydirectiveroute01 = {
path("customhi" / IntNumber) {
passednumber1 => {
path(IntNumber) {
passednumber2 => {
getandputdirective {
ctx =>
ctx.complete("Testing get and put directive" + s"The passed string is " + passednumber2 + passednumber1)
}
}
}
}
}
}
}
trait mydirective02 extends BasicDirectives with JsonSupport {
val mydirectiveroute02 =
path("hello") {
get {
complete(HttpEntity(
ContentTypes.`text/html(UTF-8)`,
"<h1>Say hello to akka-http</h1>"))
}
} ~
path("randomitem") {
get {
// will marshal Item to JSON
complete(Item("thing", 42))
}
} ~
path("saveitem") {
post {
// will unmarshal JSON to Item
entity(as[Item]) { item =>
println(s"Server saw Item : $item")
complete(item)
}
}
}
}
But I get the following error
Exception in thread "main" java.lang.NoSuchMethodError: akka.util.Helpers$.toRootLowerCase(Ljava/lang/String;)Ljava/lang/String;
The error seems to be because of this line
implicit val materializer = ActorMaterializer()
May I know where I am going wrong. I have identical pom.xml file and the program in another module and it seems to run fine. I dont understand where I am going wrong. Thanks
I get errors like that when something that should have been re-compiled has not been re-compiled. clean" followed by "compile" in sbt resolves it.
The issue is due to the conflict in dependencies make sure you are using same version for all the related packages you are importing.
example:
<dependency>
<groupId>com.typesafe.akka</groupId>
<artifactId>akka-http-core_2.11</artifactId>
<version>10.0.0</version>
</dependency>
2.11 make sure you have other dependencies with same version
then make a clean build.
Your project dependencies have conflicting versions. The version that has that function gets discarded, and the other version included in the jar (or classpath - depending on how you execute).
Please provide your dependencies file (i.e. sbt.build or gradle.build etc.)
I'd like to build a generic method for transforming Scala Case Classes to Mongo Documents.
A promising Document constructor is
fromSeq(ts: Seq[(String, BsonValue)]): Document
I can turn a case class into a Map[String -> Any], but then I've lost the type information I need to use the implicit conversions to BsonValues. Maybe TypeTags can help with this?
Here's what I've tried:
import org.mongodb.scala.bson.BsonTransformer
import org.mongodb.scala.bson.collection.immutable.Document
import org.mongodb.scala.bson.BsonValue
case class Person(age: Int, name: String)
//transform scala values into BsonValues
def transform[T](v: T)(implicit transformer: BsonTransformer[T]): BsonValue = transformer(v)
// turn any case class into a Map[String, Any]
def caseClassToMap(cc: Product) = {
val values = cc.productIterator
cc.getClass.getDeclaredFields.map( _.getName -> values.next).toMap
}
// transform a Person into a Document
def personToDocument(person: Person): Document = {
val map = caseClassToMap(person)
val bsonValues = map.toSeq.map { case (key, value) =>
(key, transform(value))
}
Document.fromSeq(bsonValues)
}
<console>:24: error: No bson implicit transformer found for type Any. Implement or import an implicit BsonTransformer for this type.
(key, transform(value))
def personToDocument(person: Person): Document = {
Document("age" -> person.age, "name" -> person.name)
}
Below code works without manual conversion of an object.
import reactivemongo.api.bson.{BSON, BSONDocument, Macros}
case class Person(name:String = "SomeName", age:Int = 20)
implicit val personHandler = Macros.handler[Person]
val bsonPerson = BSON.writeDocument[Person](Person())
println(s"${BSONDocument.pretty(bsonPerson.getOrElse(BSONDocument.empty))}")
You can use Salat https://github.com/salat/salat. A nice example can be found here - https://gist.github.com/bhameyie/8276017. This is the piece of code that will help you -
import salat._
val dBObject = grater[Artist].asDBObject(artist)
artistsCollection.save(dBObject, WriteConcern.Safe)
I was able to serialize a case class to a BsonDocument using the org.bson.BsonDocumentWriter. The below code runs using scala 2.12 and mongo-scala-driver_2.12 version 2.6.0
My quest for this solution was aided by this answer (where they are trying to serialize in the opposite direction): Serialize to object using scala mongo driver?
import org.mongodb.scala.bson.codecs.Macros
import org.mongodb.scala.bson.codecs.DEFAULT_CODEC_REGISTRY
import org.bson.codecs.configuration.CodecRegistries.{fromRegistries, fromProviders}
import org.bson.codecs.EncoderContext
import org.bson.BsonDocumentWriter
import org.mongodb.scala.bson.BsonDocument
import org.bson.codecs.configuration.CodecRegistry
import org.bson.codecs.Codec
case class Animal(name : String, species: String, genus: String, weight: Int)
object TempApp {
def main(args: Array[String]) {
val jaguar = Animal("Jenny", "Jaguar", "Panthera", 190)
val codecProvider = Macros.createCodecProvider[Animal]()
val codecRegistry: CodecRegistry = fromRegistries(fromProviders(codecProvider), DEFAULT_CODEC_REGISTRY)
val codec = Macros.createCodec[Animal](codecRegistry)
val encoderContext = EncoderContext.builder.isEncodingCollectibleDocument(true).build()
var doc = BsonDocument()
val writr = new BsonDocumentWriter(doc) // need to call new since Java lib w/o companion object
codec.encode(writr, jaguar, encoderContext)
print(doc)
}
};
I'm trying to prevent one of the properties of a Scala case class being serialised. I've tried annotating the property in question with the usual #JsonIgnore and I've also tried attaching the #JsonIgnoreProperties(Array("property_name")) to the case class. Neither of which seem to achieve what I want.
Here's a small example:
import org.json4s._
import org.json4s.jackson._
import org.json4s.jackson.Serialization
import org.json4s.jackson.Serialization.{read, write}
import com.fasterxml.jackson.annotation._
object Example extends App {
#JsonIgnoreProperties(Array("b"))
case class Message(a: String, #JsonIgnore b: String)
implicit val formats = Serialization.formats(NoTypeHints)
val jsonInput = """{ "a": "Hello", "b":"World!" }"""
val message = read[Message](jsonInput)
println("Read " + message) // "Read Message(Hello,World!)
val output = write(message)
println("Wrote " + output) // "Wrote {"a":"Hello","b":"World!"}"
}
Change your #JsonIgnore to #JsonProperty("b"). You have correctly stated to Ignore the property 'b but 'b has not yet been annotated as a property.
#JsonIgnoreProperties(Array("b"))
case class Message(a: String, #JsonProperty("b") b: String)
With jackson-databind 2.8.6 and jackson-module-scala 2.8.4
"com.fasterxml.jackson.core" % "jackson-databind" % "2.8.6",
"com.fasterxml.jackson.module" % "jackson-module-scala_2.11" % "2.8.4"
Only #JsonIgnoreProperties works fine,
Example case class as below where I'm ignoring "eventOffset" and "hashValue",
import java.util.Date
import com.fasterxml.jackson.annotation.{JsonIgnore, JsonIgnoreProperties}
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper
#JsonIgnoreProperties(Array("eventOffset", "hashValue"))
case class TestHappenedEvent(eventOffset: Long, hashValue: Long, eventType: String,
createdDate: Date, testField: String) {
def this() {
this(0, 0, "", new Date(), "")
}
def toJSON(): String = {
val objectMapper = new ObjectMapper() with ScalaObjectMapper
objectMapper.registerModule(DefaultScalaModule)
val data = this.copy()
val stream = new ByteArrayOutputStream()
objectMapper.writeValue(stream, data)
stream.toString
}
}
test
import org.scalatest.FunSuite
import spray.json._
class BaseEventSpecs extends FunSuite {
val abstractEvent = TestHappenedEvent(0, 1, "TestHappenedEvent", new Date(2017, 10, 28), "item is sold")
test("converts itself to JSON") {
assert(abstractEvent.toJSON().parseJson ==
"""
{
"eventType":"TestHappenedEvent",
"createdDate":61470000000000,
"testField":"item is sold"
}
""".stripMargin.parseJson)
}
}
Looking for a good example of polymorphic serialization deserialization using jackson with scala
got an exception :
Exception in thread "main"
Blockquote
org.codehaus.jackson.map.exc.UnrecognizedPropertyException:
Unrecognized field "animals" (Class Zoo), not marked as ignorable
after trying the following code :
import org.codehaus.jackson.annotate.{ JsonTypeInfo, JsonSubTypes }
import org.codehaus.jackson.annotate.JsonSubTypes.Type
#JsonTypeInfo(
use = JsonTypeInfo.Id.NAME,
include= JsonTypeInfo.As.PROPERTY,
property = "type"
)
#JsonSubTypes(Array(
new Type(value= classOf[Cat] , name = "cat"),
new Type(value= classOf[Dog] , name = "dog")
)
)
abstract class Animal {
val name:String = "NoName"
}
class Cat extends Animal{
val favoriteToy = "edi"
}
class Dog extends Animal{
val breed = "German Shepherd"
val color = "brown"
}
class Zoo {
val animals = new scala.collection.mutable.ListBuffer[Animal]
}
import org.codehaus.jackson.map.ObjectMapper
object Foo {
def main (args:Array[String]) {
val mapper = new ObjectMapper()
mapper.setPropertyNamingStrategy(CamelCaseNamingStrategy )
val source = scala.io.Source.fromFile("input.json" )
val input = source.mkString
source.close
val zoo = mapper.readValue(input,classOf[Zoo])
println(mapper.writeValueAsString(zoo))
}
import org.codehaus.jackson.map.introspect.{AnnotatedField, AnnotatedMethod}
import org.codehaus.jackson.map.{MapperConfig, PropertyNamingStrategy}
object CamelCaseNamingStrategy extends PropertyNamingStrategy{
override def nameForGetterMethod (config: MapperConfig[_], method: AnnotatedMethod, defaultName: String) =
{
translate(defaultName)
}
override def nameForSetterMethod (config: MapperConfig[_], method: AnnotatedMethod, defaultName: String) = {
translate(defaultName)
}
override def nameForField (config: MapperConfig[_], field: AnnotatedField, defaultName: String) = {
translate(defaultName)
}
def translate(defaultName:String) = {
val nameChars = defaultName.toCharArray
val nameTranslated = new StringBuilder(nameChars.length*2)
for ( c <- nameChars){
if (Character.isUpperCase(c)){
nameTranslated.append("_")
}
nameTranslated.append( Character.toLowerCase(c))
}
nameTranslated.toString
}
}
file input.json
{
"animals":
[
{"type":"dog","name":"Spike","breed":"mutt","color":"red"},
{"type":"cat","name":"Fluffy","favoriteToy":"spider ring"}
]
}
If you're doing polymorphic deserialization in Scala I'd strongly recommend using case classes and Jackson's scala module.
object Test {
import com.fasterxml.jackson.annotation.JsonSubTypes.Type
import com.fasterxml.jackson.annotation.{JsonSubTypes, JsonTypeInfo}
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper
#JsonTypeInfo(
use = JsonTypeInfo.Id.NAME,
include = JsonTypeInfo.As.PROPERTY,
property = "type"
)
#JsonSubTypes(Array(
new Type(value = classOf[Cat], name = "cat"),
new Type(value = classOf[Dog], name = "dog")
))
trait Animal
case class Dog(name: String, breed: String, leash_color: String) extends Animal
case class Cat(name: String, favorite_toy: String) extends Animal
case class Zoo(animals: Iterable[Animal])
def main(args: Array[String]): Unit = {
val objectMapper = new ObjectMapper with ScalaObjectMapper
objectMapper.registerModule(DefaultScalaModule)
val dogStr = """{"type": "dog", "name": "Spike", "breed": "mutt", "leash_color": "red"}"""
val catStr = """{"type": "cat", "name": "Fluffy", "favorite_toy": "spider ring"}"""
val zooStr = s"""{"animals":[$dogStr, $catStr]}"""
val zoo = objectMapper.readValue[Zoo](zooStr)
println(zoo)
// Prints: Zoo(List(Dog(Spike,mutt,red), Cat(Fluffy,spider ring)))
}
}
Ok, Got it here is a working example with scala based on Deserialize JSON with Jackson into Polymorphic by Programmer Bruce:
import org.codehaus.jackson.annotate.JsonSubTypes.Type
import org.codehaus.jackson.annotate.{JsonSubTypes, JsonTypeInfo}
#JsonTypeInfo(
use = JsonTypeInfo.Id.NAME,
include= JsonTypeInfo.As.PROPERTY,
property = "type"
)
#JsonSubTypes(Array(
new Type(value= classOf[Cat] , name = "cat"),
new Type(value= classOf[Dog] , name = "dog")
)
)
abstract class Animal {
var name:String =""
}
class Dog extends Animal{
var breed= "German Shepherd"
var color = "brown"
}
class Cat extends Animal{
var favoriteToy:String = "nothing"
}
class Zoo {
var animals = new Array[Animal](5)
}
import org.codehaus.jackson.annotate.JsonAutoDetect.Visibility
import org.codehaus.jackson.annotate.JsonMethod
import org.codehaus.jackson.map.{DeserializationConfig, ObjectMapper}
object Foo {
def main (args:Array[String]) {
val mapper = new ObjectMapper().setVisibility(JsonMethod.FIELD,Visibility.ANY)
mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES,false)
val source = scala.io.Source.fromFile("/input.json" )
val input = source.mkString
println("input " + input)
source.close
val zoo = mapper.readValue(input,classOf[Zoo])
println(mapper.writeValueAsString(zoo))
}
}
file:input.json { "animals": [
{"type":"dog","name":"Spike","breed":"mutt","color":"red"},
{"type":"cat","name":"Fluffy","favoriteToy":"spider ring"}
] }