I have a protobuf file ...
which I transform to a Scala file using ScalaPB. This way I can then ...
use it inside my Juypter notebook* for transformation. Sadly, when I ...
run the specific cell I get an type
mismatch error and I don't know why?
As the protobuf file is working with Python and the Scala code is generated, what is not right here? Could this be a bug?
*The notebook uses com.google.protobuf:protobuf-java:3.5.0,com.thesamet.scalapb:sparksql-scalapb_2.11:0.7.0 as imports
Sources & Error
protobuf file:
syntax = "proto2";
import "scalapb/scalapb.proto";
option (scalapb.options) = {
flat_package: true
single_file: true
};
message JSONEntry {
required uint64 ts = 1;
required string data = 2;
}
message JSONOutput {
optional string metadata = 1;
repeated JSONEntry entry = 2;
}
Scala (generated) code
// Generated by the Scala Plugin for the Protocol Buffer Compiler.
// Do not edit!
//
// Protofile syntax: PROTO2
#SerialVersionUID(0L)
final case class JSONEntry(
ts: _root_.scala.Long,
data: _root_.scala.Predef.String
) extends scalapb.GeneratedMessage with scalapb.Message[JSONEntry] with scalapb.lenses.Updatable[JSONEntry] {
#transient
private[this] var __serializedSizeCachedValue: _root_.scala.Int = 0
private[this] def __computeSerializedValue(): _root_.scala.Int = {
var __size = 0
__size += _root_.com.google.protobuf.CodedOutputStream.computeUInt64Size(1, ts)
__size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(2, data)
__size
}
final override def serializedSize: _root_.scala.Int = {
var read = __serializedSizeCachedValue
if (read == 0) {
read = __computeSerializedValue()
__serializedSizeCachedValue = read
}
read
}
def writeTo(`_output__`: _root_.com.google.protobuf.CodedOutputStream): Unit = {
_output__.writeUInt64(1, ts)
_output__.writeString(2, data)
}
def mergeFrom(`_input__`: _root_.com.google.protobuf.CodedInputStream): JSONEntry = {
var __ts = this.ts
var __data = this.data
var __requiredFields0: _root_.scala.Long = 0x3L
var _done__ = false
while (!_done__) {
val _tag__ = _input__.readTag()
_tag__ match {
case 0 => _done__ = true
case 8 =>
__ts = _input__.readUInt64()
__requiredFields0 &= 0xfffffffffffffffeL
case 18 =>
__data = _input__.readString()
__requiredFields0 &= 0xfffffffffffffffdL
case tag => _input__.skipField(tag)
}
}
if (__requiredFields0 != 0L) { throw new _root_.com.google.protobuf.InvalidProtocolBufferException("Message missing required fields.") }
JSONEntry(
ts = __ts,
data = __data
)
}
def withTs(__v: _root_.scala.Long): JSONEntry = copy(ts = __v)
def withData(__v: _root_.scala.Predef.String): JSONEntry = copy(data = __v)
def getFieldByNumber(__fieldNumber: _root_.scala.Int): scala.Any = {
(__fieldNumber: #_root_.scala.unchecked) match {
case 1 => ts
case 2 => data
}
}
def getField(__field: _root_.scalapb.descriptors.FieldDescriptor): _root_.scalapb.descriptors.PValue = {
require(__field.containingMessage eq companion.scalaDescriptor)
(__field.number: #_root_.scala.unchecked) match {
case 1 => _root_.scalapb.descriptors.PLong(ts)
case 2 => _root_.scalapb.descriptors.PString(data)
}
}
def toProtoString: _root_.scala.Predef.String = _root_.scalapb.TextFormat.printToUnicodeString(this)
def companion = JSONEntry
}
object JSONEntry extends scalapb.GeneratedMessageCompanion[JSONEntry] {
implicit def messageCompanion: scalapb.GeneratedMessageCompanion[JSONEntry] = this
def fromFieldsMap(__fieldsMap: scala.collection.immutable.Map[_root_.com.google.protobuf.Descriptors.FieldDescriptor, scala.Any]): JSONEntry = {
require(__fieldsMap.keys.forall(_.getContainingType() == javaDescriptor), "FieldDescriptor does not match message type.")
val __fields = javaDescriptor.getFields
JSONEntry(
__fieldsMap(__fields.get(0)).asInstanceOf[_root_.scala.Long],
__fieldsMap(__fields.get(1)).asInstanceOf[_root_.scala.Predef.String]
)
}
implicit def messageReads: _root_.scalapb.descriptors.Reads[JSONEntry] = _root_.scalapb.descriptors.Reads{
case _root_.scalapb.descriptors.PMessage(__fieldsMap) =>
require(__fieldsMap.keys.forall(_.containingMessage == scalaDescriptor), "FieldDescriptor does not match message type.")
JSONEntry(
__fieldsMap.get(scalaDescriptor.findFieldByNumber(1).get).get.as[_root_.scala.Long],
__fieldsMap.get(scalaDescriptor.findFieldByNumber(2).get).get.as[_root_.scala.Predef.String]
)
case _ => throw new RuntimeException("Expected PMessage")
}
def javaDescriptor: _root_.com.google.protobuf.Descriptors.Descriptor = DataProto.javaDescriptor.getMessageTypes.get(0)
def scalaDescriptor: _root_.scalapb.descriptors.Descriptor = DataProto.scalaDescriptor.messages(0)
def messageCompanionForFieldNumber(__number: _root_.scala.Int): _root_.scalapb.GeneratedMessageCompanion[_] = throw new MatchError(__number)
lazy val nestedMessagesCompanions: Seq[_root_.scalapb.GeneratedMessageCompanion[_]] = Seq.empty
def enumCompanionForFieldNumber(__fieldNumber: _root_.scala.Int): _root_.scalapb.GeneratedEnumCompanion[_] = throw new MatchError(__fieldNumber)
lazy val defaultInstance = JSONEntry(
ts = 0L,
data = ""
)
implicit class JSONEntryLens[UpperPB](_l: _root_.scalapb.lenses.Lens[UpperPB, JSONEntry]) extends _root_.scalapb.lenses.ObjectLens[UpperPB, JSONEntry](_l) {
def ts: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Long] = field(_.ts)((c_, f_) => c_.copy(ts = f_))
def data: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.data)((c_, f_) => c_.copy(data = f_))
}
final val TS_FIELD_NUMBER = 1
final val DATA_FIELD_NUMBER = 2
}
#SerialVersionUID(0L)
final case class JSONOutput(
metadata: scala.Option[_root_.scala.Predef.String] = None,
entry: _root_.scala.collection.Seq[JSONEntry] = _root_.scala.collection.Seq.empty
) extends scalapb.GeneratedMessage with scalapb.Message[JSONOutput] with scalapb.lenses.Updatable[JSONOutput] {
#transient
private[this] var __serializedSizeCachedValue: _root_.scala.Int = 0
private[this] def __computeSerializedValue(): _root_.scala.Int = {
var __size = 0
if (metadata.isDefined) { __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(1, metadata.get) }
entry.foreach(entry => __size += 1 + _root_.com.google.protobuf.CodedOutputStream.computeUInt32SizeNoTag(entry.serializedSize) + entry.serializedSize)
__size
}
final override def serializedSize: _root_.scala.Int = {
var read = __serializedSizeCachedValue
if (read == 0) {
read = __computeSerializedValue()
__serializedSizeCachedValue = read
}
read
}
def writeTo(`_output__`: _root_.com.google.protobuf.CodedOutputStream): Unit = {
metadata.foreach { __v =>
_output__.writeString(1, __v)
};
entry.foreach { __v =>
_output__.writeTag(2, 2)
_output__.writeUInt32NoTag(__v.serializedSize)
__v.writeTo(_output__)
};
}
def mergeFrom(`_input__`: _root_.com.google.protobuf.CodedInputStream): JSONOutput = {
var __metadata = this.metadata
val __entry = (_root_.scala.collection.immutable.Vector.newBuilder[JSONEntry] ++= this.entry)
var _done__ = false
while (!_done__) {
val _tag__ = _input__.readTag()
_tag__ match {
case 0 => _done__ = true
case 10 =>
__metadata = Option(_input__.readString())
case 18 =>
__entry += _root_.scalapb.LiteParser.readMessage(_input__, JSONEntry.defaultInstance)
case tag => _input__.skipField(tag)
}
}
JSONOutput(
metadata = __metadata,
entry = __entry.result()
)
}
def getMetadata: _root_.scala.Predef.String = metadata.getOrElse("")
def clearMetadata: JSONOutput = copy(metadata = None)
def withMetadata(__v: _root_.scala.Predef.String): JSONOutput = copy(metadata = Option(__v))
def clearEntry = copy(entry = _root_.scala.collection.Seq.empty)
def addEntry(__vs: JSONEntry*): JSONOutput = addAllEntry(__vs)
def addAllEntry(__vs: TraversableOnce[JSONEntry]): JSONOutput = copy(entry = entry ++ __vs)
def withEntry(__v: _root_.scala.collection.Seq[JSONEntry]): JSONOutput = copy(entry = __v)
def getFieldByNumber(__fieldNumber: _root_.scala.Int): scala.Any = {
(__fieldNumber: #_root_.scala.unchecked) match {
case 1 => metadata.orNull
case 2 => entry
}
}
def getField(__field: _root_.scalapb.descriptors.FieldDescriptor): _root_.scalapb.descriptors.PValue = {
require(__field.containingMessage eq companion.scalaDescriptor)
(__field.number: #_root_.scala.unchecked) match {
case 1 => metadata.map(_root_.scalapb.descriptors.PString).getOrElse(_root_.scalapb.descriptors.PEmpty)
case 2 => _root_.scalapb.descriptors.PRepeated(entry.map(_.toPMessage)(_root_.scala.collection.breakOut))
}
}
def toProtoString: _root_.scala.Predef.String = _root_.scalapb.TextFormat.printToUnicodeString(this)
def companion = JSONOutput
}
object JSONOutput extends scalapb.GeneratedMessageCompanion[JSONOutput] {
implicit def messageCompanion: scalapb.GeneratedMessageCompanion[JSONOutput] = this
def fromFieldsMap(__fieldsMap: scala.collection.immutable.Map[_root_.com.google.protobuf.Descriptors.FieldDescriptor, scala.Any]): JSONOutput = {
require(__fieldsMap.keys.forall(_.getContainingType() == javaDescriptor), "FieldDescriptor does not match message type.")
val __fields = javaDescriptor.getFields
JSONOutput(
__fieldsMap.get(__fields.get(0)).asInstanceOf[scala.Option[_root_.scala.Predef.String]],
__fieldsMap.getOrElse(__fields.get(1), Nil).asInstanceOf[_root_.scala.collection.Seq[JSONEntry]]
)
}
implicit def messageReads: _root_.scalapb.descriptors.Reads[JSONOutput] = _root_.scalapb.descriptors.Reads{
case _root_.scalapb.descriptors.PMessage(__fieldsMap) =>
require(__fieldsMap.keys.forall(_.containingMessage == scalaDescriptor), "FieldDescriptor does not match message type.")
JSONOutput(
__fieldsMap.get(scalaDescriptor.findFieldByNumber(1).get).flatMap(_.as[scala.Option[_root_.scala.Predef.String]]),
__fieldsMap.get(scalaDescriptor.findFieldByNumber(2).get).map(_.as[_root_.scala.collection.Seq[JSONEntry]]).getOrElse(_root_.scala.collection.Seq.empty)
)
case _ => throw new RuntimeException("Expected PMessage")
}
def javaDescriptor: _root_.com.google.protobuf.Descriptors.Descriptor = DataProto.javaDescriptor.getMessageTypes.get(1)
def scalaDescriptor: _root_.scalapb.descriptors.Descriptor = DataProto.scalaDescriptor.messages(1)
def messageCompanionForFieldNumber(__number: _root_.scala.Int): _root_.scalapb.GeneratedMessageCompanion[_] = {
var __out: _root_.scalapb.GeneratedMessageCompanion[_] = null
(__number: #_root_.scala.unchecked) match {
case 2 => __out = JSONEntry
}
__out
}
lazy val nestedMessagesCompanions: Seq[_root_.scalapb.GeneratedMessageCompanion[_]] = Seq.empty
def enumCompanionForFieldNumber(__fieldNumber: _root_.scala.Int): _root_.scalapb.GeneratedEnumCompanion[_] = throw new MatchError(__fieldNumber)
lazy val defaultInstance = JSONOutput(
)
implicit class JSONOutputLens[UpperPB](_l: _root_.scalapb.lenses.Lens[UpperPB, JSONOutput]) extends _root_.scalapb.lenses.ObjectLens[UpperPB, JSONOutput](_l) {
def metadata: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.getMetadata)((c_, f_) => c_.copy(metadata = Option(f_)))
def optionalMetadata: _root_.scalapb.lenses.Lens[UpperPB, scala.Option[_root_.scala.Predef.String]] = field(_.metadata)((c_, f_) => c_.copy(metadata = f_))
def entry: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.collection.Seq[JSONEntry]] = field(_.entry)((c_, f_) => c_.copy(entry = f_))
}
final val METADATA_FIELD_NUMBER = 1
final val ENTRY_FIELD_NUMBER = 2
}
object DataProto extends _root_.scalapb.GeneratedFileObject {
lazy val dependencies: Seq[_root_.scalapb.GeneratedFileObject] = Seq(
scalapb.options.ScalapbProto
)
lazy val messagesCompanions: Seq[_root_.scalapb.GeneratedMessageCompanion[_]] = Seq(
JSONEntry,
JSONOutput
)
private lazy val ProtoBytes: Array[Byte] =
scalapb.Encoding.fromBase64(scala.collection.Seq(
"""CgpkYXRhLnByb3RvGhVzY2FsYXBiL3NjYWxhcGIucHJvdG8iLwoJSlNPTkVudHJ5Eg4KAnRzGAEgAigEUgJ0cxISCgRkYXRhG
AIgAigJUgRkYXRhIkoKCkpTT05PdXRwdXQSGgoIbWV0YWRhdGEYASABKAlSCG1ldGFkYXRhEiAKBWVudHJ5GAIgAygLMgouSlNPT
kVudHJ5UgVlbnRyeUIH4j8EEAEoAQ=="""
).mkString)
lazy val scalaDescriptor: _root_.scalapb.descriptors.FileDescriptor = {
val scalaProto = com.google.protobuf.descriptor.FileDescriptorProto.parseFrom(ProtoBytes)
_root_.scalapb.descriptors.FileDescriptor.buildFrom(scalaProto, dependencies.map(_.scalaDescriptor))
}
lazy val javaDescriptor: com.google.protobuf.Descriptors.FileDescriptor = {
val javaProto = com.google.protobuf.DescriptorProtos.FileDescriptorProto.parseFrom(ProtoBytes)
com.google.protobuf.Descriptors.FileDescriptor.buildFrom(javaProto, Array(
scalapb.options.ScalapbProto.javaDescriptor
))
}
#deprecated("Use javaDescriptor instead. In a future version this will refer to scalaDescriptor.", "ScalaPB 0.5.47")
def descriptor: com.google.protobuf.Descriptors.FileDescriptor = javaDescriptor
}
Error
<console>:82: error: type mismatch;
found : JSONEntry.type
required: scalapb.GeneratedMessageCompanion[_]
def companion = JSONEntry
^
I was able to successfully compile your proto file with the following code
project/scalapb.sbt
addSbtPlugin("com.thesamet" % "sbt-protoc" % "0.99.16")
libraryDependencies += "com.thesamet.scalapb" %% "compilerplugin" % "0.7.0"
build.sbt
lazy val root = (project in file(".")).
settings(
inThisBuild(List(
organization := "com.example",
scalaVersion := "2.12.4",
version := "0.1.0-SNAPSHOT"
)),
name := "protobuf",
libraryDependencies ++= Seq(
"com.thesamet.scalapb" %% "scalapb-runtime" % scalapb.compiler.Version.scalapbVersion % "protobuf"
),
PB.targets in (Compile) := Seq(
scalapb.gen() -> (sourceManaged in Compile).value
)
)
Now if you copy and paste your photo file into src/main/protobuf as hello.proto and do a sbt clean compile
The only thing I did differently is that I added a package to the photo file
syntax = "proto2";
import "scalapb/scalapb.proto";
option (scalapb.options) = {
package_name: "com.abhi"
flat_package: true
single_file: true
};
message JSONEntry {
required uint64 ts = 1;
required string data = 2;
}
message JSONOutput {
optional string metadata = 1;
repeated JSONEntry entry = 2;
}
Now finally use the generated code in your app
package example
import com.abhi.JSONEntry
import java.io._
object Hello extends App {
val jsonEntry = JSONEntry(10L, "foo")
val target = new FileOutputStream(new File("foo.bin"))
jsonEntry.writeTo(target)
target.close()
}
The code compiles correctly and there is no compilation error
Related
I define val like this :
val config = Config(args)
val product_type = config.product_type
then I send product_type as "AA"
and my code is this :
val scores = df.mapPartitions(iterator => {
val inputStream =
if(product_type == "AA" ) {
getClass().getClassLoader().getResourceAsStream("my_aa.hdf5")
}
else {
getClass().getClassLoader().getResourceAsStream("my_bb.hdf5")
}
val multiLayerNetwork: MultiLayerNetwork = KerasModelImport.importKerasSequentialModelAndWeights(inputStream, false)
val wrapped: ParallelInference = new ParallelInference.Builder(multiLayerNetwork).build()
val res = iterator.map(row => {
wrapped.output(row).toDoubleVector
})
res
})
But my inputStream equals "my_bb.hdf5" which is not correct. This value comes from else statement. So why my product_type variable cant read in mappartition?
I print my product_type value before code and I checked it , it is : "AA"
it occurs because of i get this variable from argument in spark submit.sh
and it can not read from mappartition.
It works like this:
val scores =
if (product_type == "AA") {
df.mapPartitions(iterator => {
val inputStream = getClass().getClassLoader().getResourceAsStream("AA.hdf5")
val multiLayerNetwork: MultiLayerNetwork = KerasModelImport.importKerasSequentialModelAndWeights(inputStream, false)
val wrapped: ParallelInference = new ParallelInference.Builder(multiLayerNetwork).build()
val res = iterator.map(row => {
wrapped.output(row).toDoubleVector
})
res
})
} else {
df.mapPartitions(iterator => {
val inputStream = getClass().getClassLoader().getResourceAsStream("BB.hdf5")
val multiLayerNetwork: MultiLayerNetwork = KerasModelImport.importKerasSequentialModelAndWeights(inputStream, false)
val wrapped: ParallelInference = new ParallelInference.Builder(multiLayerNetwork).build()
val res = iterator.map(row => {
wrapped.output(row).toDoubleVector
})
res
})
}
val patterns = ctx.getBroadcastState(patternStateDescriptor)
The imports I made
import org.apache.flink.api.common.serialization.SimpleStringSchema
import org.apache.flink.api.common.state.{MapStateDescriptor, ValueState, ValueStateDescriptor}
import org.apache.flink.api.scala.typeutils.Types
import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.datastream.BroadcastStream
import org.apache.flink.streaming.api.functions.co.KeyedBroadcastProcessFunction
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer010
import org.apache.flink.streaming.api.scala._
import org.apache.flink.util.Collector
Here's the code
val env = StreamExecutionEnvironment.getExecutionEnvironment
val properties = new Properties()
properties.setProperty("bootstrap.servers","localhost:9092")
val patternStream = new FlinkKafkaConsumer010("patterns", new SimpleStringSchema, properties)
val patterns = env.addSource(patternStream)
var patternData = patterns.map {
str =>
val splitted_str = str.split(",")
PatternStream(splitted_str(0).trim, splitted_str(1).trim, splitted_str(2).trim)
}
val logsStream = new FlinkKafkaConsumer010("logs", new SimpleStringSchema, properties)
// logsStream.setStartFromEarliest()
val logs = env.addSource(logsStream)
var data = logs.map {
str =>
val splitted_str = str.split(",")
LogsTest(splitted_str.head.trim, splitted_str(1).trim, splitted_str(2).trim)
}
val keyedData: KeyedStream[LogsTest, String] = data.keyBy(_.metric)
val bcStateDescriptor = new MapStateDescriptor[Unit, PatternStream]("patterns", Types.UNIT, Types.of[PatternStream]) // first type defined is for the key and second data type defined is for the value
val broadcastPatterns: BroadcastStream[PatternStream] = patternData.broadcast(bcStateDescriptor)
val alerts = keyedData
.connect(broadcastPatterns)
.process(new PatternEvaluator())
alerts.print()
// println(alerts.getClass)
// val sinkProducer = new FlinkKafkaProducer010("output", new SimpleStringSchema(), properties)
env.execute("Flink Broadcast State Job")
}
class PatternEvaluator()
extends KeyedBroadcastProcessFunction[String, LogsTest, PatternStream, (String, String, String)] {
private lazy val patternStateDescriptor = new MapStateDescriptor("patterns", classOf[String], classOf[String])
private var lastMetricState: ValueState[String] = _
override def open(parameters: Configuration): Unit = {
val lastMetricDescriptor = new ValueStateDescriptor("last-metric", classOf[String])
lastMetricState = getRuntimeContext.getState(lastMetricDescriptor)
}
override def processElement(reading: LogsTest,
readOnlyCtx: KeyedBroadcastProcessFunction[String, LogsTest, PatternStream, (String, String, String)]#ReadOnlyContext,
out: Collector[(String, String, String)]): Unit = {
val metrics = readOnlyCtx.getBroadcastState(patternStateDescriptor)
if (metrics.contains(reading.metric)) {
val metricPattern: String = metrics.get(reading.metric)
val metricPatternValue: String = metrics.get(reading.value)
val lastMetric = lastMetricState.value()
val logsMetric = (reading.metric)
val logsValue = (reading.value)
if (logsMetric == metricPattern) {
if (metricPatternValue == logsValue) {
out.collect((reading.timestamp, reading.value, reading.metric))
}
}
}
}
override def processBroadcastElement(
update: PatternStream,
ctx: KeyedBroadcastProcessFunction[String, LogsTest, PatternStream, (String, String, String)]#Context,
out: Collector[(String, String, String)]
): Unit = {
val patterns = ctx.getBroadcastState(patternStateDescriptor)
if (update.metric == "IP") {
patterns.put(update.metric /*,update.operator*/ , update.value)
}
// else if (update.metric == "username"){
// patterns.put(update.metric, update.value)
// }
// else {
// println("No required data found")
// }
// }
}
}
Sample Data :- Logs Stream
"21/09/98","IP", "5.5.5.5"
Pattern Stream
"IP","==","5.5.5.5"
I'm unable to analyse data by getting desired result, i.e = 21/09/98,IP,5.5.5.5
There's no error as of now, it's just not analysing the data
The code is reading streams (Checked)
One common source of trouble in cases like this is that the API offers no control over the order in which the patterns and the data are ingested. It could be that processElement is being called before processBroadcastElement.
I have the following list,
10,44,22
10,47,12
15,38,3
15,41,30
16,44,15
16,47,18
22,38,21
22,41,42
34,44,40
34,47,36
40,38,39
40,41,42
45,38,27
45,41,30
46,44,45
46,47,48
Then I am creating one file with it is content with the following code:
val fstream:FileWriter = new FileWriter("patSPO.csv")
var out:BufferedWriter = new BufferedWriter(fstream)
val sl = listSPO.sortBy(l => (l.sub, l.pre))
for ( a <- 0 to listSPO.size-1){
out.write(sl(a).sub.toString+","+sl(a).pre.toString+","+sl(a).obj.toString+"\n")
}
out.close()
However I want to divide the content in a n files, then I try the following for 4 files:
val fstream:FileWriter = new FileWriter("patSPO.csv")
val fstream1:FileWriter = new FileWriter("patSPO1.csv")
val fstream2:FileWriter = new FileWriter("patSPO2.csv")
val fstream3:FileWriter = new FileWriter("patSPO3.csv")
val fstream4:FileWriter = new FileWriter("patSPO4.csv")
var out:BufferedWriter = new BufferedWriter(fstream)
var out1:BufferedWriter = new BufferedWriter(fstream1)
var out2:BufferedWriter = new BufferedWriter(fstream2)
var out3:BufferedWriter = new BufferedWriter(fstream3)
var out4:BufferedWriter = new BufferedWriter(fstream4)
val b :Int = listSPO.size/4
val sl = listSPO.sortBy(l => (l.sub, l.pre))
for ( a <- 0 to listSPO.size-1){
out.write(sl(a).sub.toString+","+sl(a).pre.toString+","+sl(a).obj.toString+"\n")
}
for ( a <- 0 to b-1){
out1.write(sl(a).sub.toString+","+sl(a).pre.toString+","+sl(a).obj.toString+"\n")
}
for ( a <- b to (b*2)-1){
out2.write(sl(a).sub.toString+","+sl(a).pre.toString+","+sl(a).obj.toString+"\n")
}
for ( a <- b*2 to (b*3)-1){
out3.write(sl(a).sub.toString+","+sl(a).pre.toString+","+sl(a).obj.toString+"\n")
}
for ( a <- b*3 to (b*4)-1){
out4.write(sl(a).sub.toString+","+sl(a).pre.toString+","+sl(a).obj.toString+"\n")
}
out.close()
out1.close()
out2.close()
out3.close()
out4.close()
Then my question is if exist a general code where I put the number of files to generate, for example 32, and not to write 32 times the out, the for and the fstream?
First, let's make some utilities to eliminate ugly boilerplate:
def open(
number: Int = 1,
prefix: String,
ext: String
) = (0 to n-1)
.iterator
.map { i =>
"%s%s%s".format(prefix, if(i == 0) "" else i.toString, postfix)
}.map(new FileWriter(_))
.map(new BufferedWriter(_))
def toString(l: WhateverTheType) = Seq(
l.sub,
l.pre,
l.obj
).mkString(",") + "\n"
And now to the implementation:
writeToFiles(
listSPO: List[WhateverTheType],
numFiles: Int = 1,
prefix: String = "pasSPO",
ext: String = ".csv"
) = listSPO
.grouped((listSPO.size+1)/numFiles)
.zip(open(numFiles, prefix, ext))
.foreach { case (input, file) =>
try {
input.foreach(file.write(toString(input)))
} finally {
file.close
}
}
Assuming the objects in the list have this type (otherwise, relpace SPO in the below code with your type):
case class SPO(sub: Int, pre: Int, obj: Int)
This should do it:
val sl = listSPO.sortBy(l => (l.sub, l.pre))
val files = 5 // whatever number you want
// helper function to write a single list - similar to your implementation but shorter
def writeListToFile(name: String, list: List[SPO]): Unit = {
val writer = new BufferedWriter(new FileWriter(name))
list.foreach(spo => writer.write(s"${spo.sub},${spo.pre},${spo.obj}\n"))
writer.close()
}
sl.grouped(sl.size / files) // split into sublists
.zipWithIndex // add index of sublists for file name
.foreach {
case (sublist, 0) => writeListToFile(s"pasSPO.csv", sublist) // if indeed you want the first file name NOT to include the index
case (sublist, index) => writeListToFile(s"pasSPO$index.csv", sublist)
}
What I did works fine (at least it looks look it does), but I am not convinced this is the best way to do it...
Basically, I wanted to have my i18n translations in a database instead of property files so users can easily edit those translations and a cache can serve them to the other users within a short period of time - I use an Akka Actor to read from the database and create a cache used by the messageApi (before I always needed to redeploy with the changes in the property files).
Basically, is what I did totally the wrong way to do it?
TranslationActor.scala :
class TranslationActor extends Actor {
def receive = {
case _ => {
Logger.info("Starting to cache the translations")
TranslationActor.tempCache = ListMap.empty
var translations: ListMap[String, String] = ListMap.empty
for (acceptedLanguage <- TranslationActor.acceptedLanguages) {
val translationLanguageId: Long = TranslationLanguage.findByCode(acceptedLanguage).get.id
val languageTranslations: Seq[Translation] = Translation.findAllByLanguageId(translationLanguageId)
translations = new ListMap[String, String]
for (languageTranslation <- languageTranslations) {
val tag = EnglishTranslation.findById(languageTranslation.englishTranslationId).get.tag
var returnedTranslation: String = languageTranslation.translation
if (returnedTranslation.isEmpty) {
returnedTranslation = tag
}
translations += tag -> new CacheValue(new Locale(acceptedLanguage), returnedTranslation).stringVar
}
TranslationActor.tempCache += acceptedLanguage -> translations
}
TranslationActor.cache = TranslationActor.tempCache
Logger.info("Finished to cache the translations")
}
}
}
object TranslationActor {
var acceptedLanguages: Seq[String] = Seq("fr", "en")
var cache: ListMap[String, ListMap[String, String]] = ListMap.empty
var tempCache: ListMap[String, ListMap[String, String]] = ListMap.empty
}
class CacheValue(locale: Locale, string: String) {
val created: Long = System.currentTimeMillis
var messageFormat: MessageFormat = null
var localeVar: Locale = locale
var stringVar: String = string
def isOlderThan(period: Long): Boolean = {
(System.currentTimeMillis - created) > (period * 1000)
}
def getMessageFormat: MessageFormat = {
if (messageFormat == null) {
if (stringVar != null) {
messageFormat = new MessageFormat(stringVar, localeVar)
} else {
messageFormat = new MessageFormat("", localeVar)
}
}
messageFormat
}
}
ManageTranslationDaemon.scala :
#Singleton
class ManageTranslationDaemon #Inject() (actorSystem: ActorSystem, applicationLifecycle: ApplicationLifecycle) {
Logger.info("Scheduling the translation daemon")
val translationActor = actorSystem.actorOf(Props(new TranslationActor()))
actorSystem.scheduler.schedule(1 seconds, 30 minutes, translationActor, "translationDaemon")
applicationLifecycle.addStopHook { () =>
Logger.info("Shutting down translation daemon")
Future.successful(actorSystem.shutdown())
}
}
TranslationGuiceConfiguration.scala : (from an com.google.inject.AbstractModule)
class TranslationGuiceConfiguration extends AbstractModule {
def configure() : Unit = {
bind(classOf[ManageTranslationDaemon]).asEagerSingleton()
}
}
Then I extended parts of the DefaultMessagesApi (by looking at the code of MessagesApi) in
MessagesPersoApi.scala :
class MessagesPersoApi #Inject() (environment: Environment, configuration: Configuration, langs: Langs) extends DefaultMessagesApi(environment: Environment, configuration: Configuration, langs: Langs) {
private def joinPaths(first: Option[String], second: String) = first match {
case Some(parent) => new java.io.File(parent, second).getPath
case None => second
}
override protected def loadMessages(langCode: String): Map[String, String] = {
TranslationActor.cache.getOrElse(langCode, loadMessagesFromFile("messages." + langCode))
}
protected def loadMessagesFromFile(langCode: String): Map[String, String] = {
import scala.collection.JavaConverters._
environment.classLoader.getResources(joinPaths(messagesPrefix, langCode)).asScala.toList
.filterNot(url => Resources.isDirectory(environment.classLoader, url)).reverse
.map { messageFile =>
Messages.parse(Messages.UrlMessageSource(messageFile), messageFile.toString).fold(e => throw e, identity)
}.foldLeft(Map.empty[String, String]) {_ ++ _}
}
override protected def loadAllMessages: Map[String, Map[String, String]] = {
langs.availables.map(_.code).map { lang =>
(lang, loadMessages(lang))
}.toMap
.+("default" -> loadMessagesFromFile("messages"))
.+("default.play" -> loadMessagesFromFile("messages.default"))
}
}
And finally created a module (play.api.inject.Module)
MessagesPersoModule.scala :
class MessagesPersoModule extends Module {
def bindings(environment: Environment, configuration: Configuration) = {
Seq(
bind[Langs].to[DefaultLangs],
bind[MessagesApi].to[MessagesPersoApi]
)
}
}
And at the end, used it in my application.conf :
play.modules.disabled += "play.api.i18n.I18nModule"
play.modules.enabled += "modules.MessagesPersoModule"
play.modules.enabled += "modules.TranslationGuiceConfiguration"
Does that actually make sense? It seemed to me that it was a bit "complicated" to write. Is there an easier way to do that same logic with less code/classes ?
Thanks,
Yoann
How does one access the parameters used to construct a Module from inside the Tester that is testing it?
In the test below I am passing the parameters explicitly both to the Module and to the Tester. I would prefer not to have to pass them to the Tester but instead extract them from the module that was also passed in.
Also I am new to scala/chisel so any tips on bad techniques I'm using would be appreciated :).
import Chisel._
import math.pow
class TestA(dataWidth: Int, arrayLength: Int) extends Module {
val dataType = Bits(INPUT, width = dataWidth)
val arrayType = Vec(gen = dataType, n = arrayLength)
val io = new Bundle {
val i_valid = Bool(INPUT)
val i_data = dataType
val i_array = arrayType
val o_valid = Bool(OUTPUT)
val o_data = dataType.flip
val o_array = arrayType.flip
}
io.o_valid := io.i_valid
io.o_data := io.i_data
io.o_array := io.i_array
}
class TestATests(c: TestA, dataWidth: Int, arrayLength: Int) extends Tester(c) {
val maxData = pow(2, dataWidth).toInt
for (t <- 0 until 16) {
val i_valid = rnd.nextInt(2)
val i_data = rnd.nextInt(maxData)
val i_array = List.fill(arrayLength)(rnd.nextInt(maxData))
poke(c.io.i_valid, i_valid)
poke(c.io.i_data, i_data)
(c.io.i_array, i_array).zipped foreach {
(element,value) => poke(element, value)
}
expect(c.io.o_valid, i_valid)
expect(c.io.o_data, i_data)
(c.io.o_array, i_array).zipped foreach {
(element,value) => poke(element, value)
}
step(1)
}
}
object TestAObject {
def main(args: Array[String]): Unit = {
val tutArgs = args.slice(0, args.length)
val dataWidth = 5
val arrayLength = 6
chiselMainTest(tutArgs, () => Module(
new TestA(dataWidth=dataWidth, arrayLength=arrayLength))){
c => new TestATests(c, dataWidth=dataWidth, arrayLength=arrayLength)
}
}
}
If you make the arguments dataWidth and arrayLength members of TestA you can just reference them. In Scala this can be accomplished by inserting val into the argument list:
class TestA(val dataWidth: Int, val arrayLength: Int) extends Module ...
Then you can reference them from the test as members with c.dataWidth or c.arrayLength