case class ACase( name: String)
case class BCase( name: String, num: Int)
I have ACase's dataset and would like to change it as BCase type with adding column. Here is my try.
def makeBCase( aCase: ACase, num: Int) : BCase = {
BCase( aCase.name, num)
}
def change( aDS: Dataset[ACase]): Dataset[BCase] = {
val bDS = aDS.map { a =>
makeBCase( a, 123)
}
....
bDS // error - type mismatch; required : org.apache.spark.sql.Dataset[ACase]
}
How can I change Its type?
Related
Using this class I'm attempting to modify the String parameter of the constructor and convert it to date :
class OrderInstance(var size: Double,
var side: String,
var trade_id: Int,
var price: Double,
var time: java.util.Date,
var code: String) {
def getTime(time: String): java.util.Date = new java.util.Date()
def this(size: String,
side: String,
trade_id: Int,
price: String,
time: String, code: String) = this(size.toDouble: Double,
side: String,
trade_id: Int,
price.toDouble: Double,
getTime(time: String) : java.util.Date, code: String)
// Step 3 - proper signature for `equals`
// Steps 4 thru 7 - implement a `match` expression
override def equals(that: Any): Boolean =
that match {
case that: OrderInstance => {
that.canEqual(this) &&
this.trade_id == that.trade_id
}
case _ => false
}
// Step 1 - proper signature for `canEqual`
// Step 2 - compare `a` to the current class
def canEqual(a: Any) = a.isInstanceOf[OrderInstance]
// Step 8 - implement a corresponding hashCode c=method
override def hashCode: Int = {
val prime = 31
var result = 1
result = prime * result + trade_id;
result = prime * result + (if (code == null) 0 else code.hashCode)
result
}
override def toString = {
String.valueOf(trade_id)
}
}
This line :
def getTime(time: String): java.util.Date = new java.util.Date()
should be called in order to return a date instead of String. But I receive error :
Error:(54, 7) not found: value getTime
getTime(time: String) : java.util.Date, code: String)
This error is reported at run time, not compile time. Have I not defined the function getTime correctly ?
Update :
I've attempted to make more clear what I'm trying to achieve :
class OrderInstance(var size: Double,
var side: String,
var trade_id: Int,
var price: Double,
var time: String,
var code: String){
this(time) = this(OrderInstance.getTime(time))
def this(size: String,
side: String,
trade_id: Int,
price: String,
time: String, code: String) = this(size.toDouble: Double,
side: String,
trade_id: Int,
price.toDouble: Double,
time : String, code: String)
// Step 3 - proper signature for `equals`
// Steps 4 thru 7 - implement a `match` expression
override def equals(that: Any): Boolean =
that match {
case that: OrderInstance => {
that.canEqual(this) &&
this.trade_id == that.trade_id
}
case _ => false
}
// Step 1 - proper signature for `canEqual`
// Step 2 - compare `a` to the current class
def canEqual(a: Any) = a.isInstanceOf[OrderInstance]
// Step 8 - implement a corresponding hashCode c=method
override def hashCode: Int = {
val prime = 31
var result = 1
result = prime * result + trade_id;
result = prime * result + (if (code == null) 0 else code.hashCode)
result
}
override def toString = {
String.valueOf(trade_id)
}
}
object OrderInstance {
def getTime(time: String): java.util.Date = new java.util.Date()
}
new OrderInstance(1.0 , "" , 1 , 1.0 , "2019-09-13T16:27:19.881Z" , "")
This returns error :
Error:(60, 26) OrderInstance does not take parameters
this(time) = this(OrderInstance.getTime(time))
for line
this(time) = this(OrderInstance.getTime(time))
How to update time to be converted to Date instead of String ?
getTime is an instance method, which you're trying to invoke from a constructor, i.e. before the instance has been created. Since getTime doesn't actually use any instance variables, you'll want to place it in a companion object (the scala equivalent of static)
class OrderInstance(
//...
) {
this(string) = this(OrderInstance.getTime(string))
}
object OrderInstance {
def getTime(time: String): java.util.Date = new java.util.Date()
}
Thanks to answer from user user1186491 this works as expected :
class OrderInstance(var time: java.util.Date){
def this(time: String) = this(
OrderInstance.getTime(time))
}
object OrderInstance {
def getTime(time: String): java.util.Date = new java.util.Date()
}
val o = new OrderInstance("2019-09-13T16:27:19.881Z" )
println(o.time)
I have the following enum.
object Smth extends Enumeration {
class Value(name: String, func: (String, Int) => Boolean) extends super.Val(name)
private def opA(index: Int) : Boolean ={
index > 0
}
private def opB(x: String, index: Int) : Boolean ={
index > 0 && x.length > 0
}
val x = new Value("X", opA) // type mismatch error for opA
val y = new Value("Y", opB)
}
Enum constructor takes as an argument a function, of type (String, Int) => Boolean. Is it possible to create enum constructor in such a way that it will accept functions with 2 distinct contracts, for example:
(String, Int) => Boolean
(Int) => Boolean
I am trying to avoid using default value for argument, in function definition.
This is how I would like to use it.
if(Smth.x.func(0)) { do smth }
else if(Smth.y.func("str", 0)) { do smthElse }
If you just want default value you can do this:
class Value(val name: String, val func: (String, Int) => Boolean) extends super.Val(name) {
def apply(name: String, func: (Int) => Boolean): Value =
new Value(name, (x -> func("default value", x)))
}
Let's consider a classification problem :
object Classify extends App {
type Tag = String
type Classifier[A] = A => Set[Tag]
case class Model(a: Int, b: String, c: String, d: String)
def aClassifier : Classifier[Int] = _ => Set("A", "a")
def bClassifier : Classifier[String] = _ => Set("B")
def cClassifier : Classifier[String] = _ => Set("C")
def modelClassifier : Classifier[Model] = {
m => aClassifier(m.a) ++ bClassifier(m.b) ++ cClassifier(m.c)
}
println(modelClassifier(Model(1,"b", "c", "d")))
}
Is there a smarter way to implement modelClassifier using scalaz ?
As an idea, consider this code:
for (i <- 0 until model.productArity) yield {
val fieldValue = model.productElement(i)
fieldValue match {
case x: Int => //use integer classifier
case s: String => //use string classifier
case _ =>
}
}
scalaz library hasn't any macro case class introspection by design, but shapeless has
Consider such definitions:
import shapeless._
import shapeless.tag._
import shapeless.labelled._
trait Omit
val omit = tag[Omit]
case class Model(a: Int, b: String, c: String, d: String ## Omit)
Let define following polymorphic function
object classifiers extends Poly1 {
implicit def stringClassifier[K <: Symbol](implicit witness: Witness.Aux[K]) =
at[FieldType[K, String]](value => Set(witness.value.name.toUpperCase))
implicit def intClassifier[K <: Symbol](implicit witness: Witness.Aux[K]) =
at[FieldType[K, Int]](value => {
val name = witness.value.name
Set(name.toUpperCase, name.toLowerCase)
})
implicit def omitClassifier[K, T] =
at[FieldType[K, T ## Omit]](_ => Set.empty[String])
}
Now your modelClassifier could be done as:
def modelClassifier: Classifier[Model] =
m => LabelledGeneric[Model].to(m).map(classifiers).toList.reduce(_ union _)
you can check it via
println(modelClassifier(Model(1, "b", "c", omit("d"))))
Note that Type ## Tag is subtype of Type so model.d still could be used as String everywhere
How do you intend to distinguish between bClassifier and cClassifier? By name? By order of declaration? That does not sound very "smart" or reliable. Consider encoding your intent explicitly instead. Something like this, perhaps:
case class Classifiable[T](data: T, classifier: Classifier[T])
object Classifiable {
def Null[T](data: T) = Classifiable(data, _ => Nil)
}
case class Model(a: Classifiable[Int], b: Classifiable[String], c: Classifiable[String], d: Classifiable[String])
object Model {
def apply(a: Int, b: String, c: String, d: String) =
Model(
Classifiable(a, aClassifier),
Classifiable(b, bClassifier),
Classifiable(c, cClassifier),
Classifiable.Null(d)
)
}
def modelClassifier(m: Model) = m
.productIterator
.collect { case x: Classifiable[_] =>
x.classifier()(x)
}
.reduce(_ ++ _)
println(modelClassifier(Model(1,"b", "c", "d")))
I'd like to put some data into a HashMap and retrieve these as typed values using a function. The function takes the expected type and also a default value in case the value is not stored in the HashMap. Type erasure of the JVM makes this a tricky thing.
Q: How can I retrieve a typed value?
Code and results below.
abstract class Parameters(val name: String) {
val parameters = new HashMap[String, Any]()
def put(key: String, value: Any) = parameters(key) = value
def get(key: String) = parameters.getOrElse(key, None)
def remove(key: String) = parameters.remove(key)
def g0[T: TypeTag](key: String, defaultValue: T) = {
get(key) match {
case x: T => x
case None => defaultValue
case _ => defaultValue
}
}
def g1[T: ClassTag](key: String, defaultValue: T) = {
val compareClass = implicitly[ClassTag[T]].runtimeClass
get(key) match {
case None => defaultValue
case x if compareClass.isInstance(x) => x.asInstanceOf[T]
}
}
}
class P extends Parameters("AParmList") {
put("1", 1)
put("3", "three")
put("4", 4.0)
put("width", 600)
println(g0[Int]("width", -1))
println(g0[Int]("fail", -2))
println(g1[Int]("width", -3))
println(g1[Int]("fail", -4))
}
object TypeMatching {
def main(args: Array[String]) {
new P
}
}
The output is (comments in parenthesis):
600 (as expected)
None (expected -2)
and a match error (java.lang.Integer stored, Int required)
I'm not sure if i got the topic right. I'll try to describe the problem.
I have one common field trait. StringField and IntField extend this class:
trait BaseField[T] {
def name = "field"
var owner : FieldContainer
var value : T
def set(value : T) {
this.value = value
this.owner.fields.put(name, this)
}
}
class StringField extends BaseField[String]
class IntField extends BaseField[Int]
How do i implement the FieldContainer class? What i want is to match the FieldTypes later on:
val fieldContainer = {...init code here...}
fieldContainer.fields foreach {
field -> {
field match {
case f: StringField => println("String")
case f: IntField => println("Int")
case _ => println("Unknown")
}
}
}
This is my FieldContainer (so far)
trait FieldContainer {
private metaFields : HashMap[String, Any] = new HashMap[String, Any]
def fields : HashMap[String, Any] = this.metaFields
}
And i use it in that way:
class Pizza extends FieldContainer {
object name extends StringField(this) {
override def name = "pizza_name"
}
object pieces extends IntField(this) {
override def name = "pieces_count"
}
}
Fields don't need to know their owners.
class BaseField[T](initValue: T, val name: String = "field") {
private[this] var _value: T = initValue
def apply() = _value
def update(v: T) { _value = v }
override def toString(): String = name + "(" + apply() + ")"
}
class StringField(initValue: String, name: String = "field") extends BaseField[String](initValue, name)
class IntField(initValue: Int, name: String = "field") extends BaseField[Int](initValue, name)
trait FieldContainer {
protected def addField[C <: BaseField[_]](field: C): C = {
_fields += (field.name -> field)
field
}
protected def stringField(initValue: String, name: String): StringField =
addField(new StringField(initValue, name))
protected def intField(initValue: Int, name: String): IntField =
addField(new IntField(initValue, name))
private var _fields : Map[String, Any] = Map[String, Any]()
def fields : Map[String, Any] = _fields
}
Objects (singletons) initialized when first accessed, so you should use val instead of object for fields:
class Pizza extends FieldContainer {
val name = stringField("", "pizza_name")
val pieces = intField(0, "pieces_count")
val mass: BaseField[Double] = addField(new BaseField[Double](0, "mass"))
}
Usage:
scala> val p = new Pizza()
p: Pizza = Pizza#8c61644
scala> p.fields
res0: Map[String,Any] = Map(pizza_name -> pizza_name(), pieces_count -> pieces_count(0), mass -> mass(0.0))
scala> p.name() = "new name"
scala> p.pieces() = 10
scala> p.mass() = 0.5
scala> p.fields
res4: Map[String,Any] = Map(pizza_name -> pizza_name(new name), pieces_count -> pieces_count(10), mass -> mass(0.5))
scala> p.name()
res5: String = new name
scala> p.pieces()
res6: Int = 10
scala> p.mass
res7: BaseField[Double] = mass(0.5)