parsing issue in apache beam (beamSql) - apache-beam

Question

I have a below code in which I'm reading a file in string format, then converting it into class format then converting it to BeamRecord and at the end converting back it to string format and writing the output in google storage.
DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
options.setProject("beta-194409");
options.setStagingLocation("gs://clrtegbucket/staging");
options.setRunner(DataflowRunner.class);
DataflowRunner.fromOptions(options);
Pipeline p = Pipeline.create(options);
PCollection<String> weekly = p.apply(TextIO.read().from("gs://gcp/input/WeeklyDueto.csv"));
PCollection<ClassWeeklyDueto> pojos = weekly.apply(ParDo.of(new DoFn<String, ClassWeeklyDueto>() { // converting String into class
// typ
private static final long serialVersionUID = 1L;
#ProcessElement
public void processElement(ProcessContext c) {
String[] strArr = c.element().split(",");
ClassWeeklyDueto clr = new ClassWeeklyDueto();
clr.setCatLib(strArr[1]);
clr.setCausalValue(strArr[7]);
clr.setDuetoValue(strArr[5]);
clr.setModelIteration(strArr[8]);
clr.setOutlet(strArr[0]);
clr.setPrimaryCausalKey(strArr[6]);
clr.setProdKey(strArr[2]);
clr.setPublished(strArr[9]);
clr.setSalesComponent(strArr[4]);
clr.setWeek(strArr[3]);
global_Weekly.add(clr);
c.output(clr);
}
}));
BeamRecordSqlType appType = BeamRecordSqlType.create(
Arrays.asList("Outlet", "CatLib", "ProdKey", "Week", "SalesComponent", "DuetoValue","PrimaryCausalKey", "CausalValue", "ModelIteration", "Published"),
Arrays.asList(Types.VARCHAR, Types.VARCHAR, Types.VARCHAR, Types.VARCHAR, Types.VARCHAR, Types.FLOAT, Types.VARCHAR, Types.VARCHAR, Types.VARCHAR, Types.VARCHAR));
PCollection<BeamRecord> apps = pojos.apply(ParDo.of(new DoFn<ClassWeeklyDueto, BeamRecord>() {
private static final long serialVersionUID = 1L;
#ProcessElement
public void processElement(ProcessContext c) {
BeamRecord br = new BeamRecord(appType, {
BeamRecord br = new BeamRecord(appType, c.element().Outlet, c.element().CatLib, c.element().ProdKey,
c.element().Week, c.element().SalesComponent, c.element().DuetoValue,
c.element().PrimaryCausalKey, c.element().CausalValue, c.element().ModelIteration,
c.element().Published);
c.output(br); }
})).setCoder(appType.getRecordCoder());
PCollection<String> gs_output_final = apps.apply(ParDo.of(new DoFn<BeamRecord, String>() {
private static final long serialVersionUID = 1L;
#ProcessElement
public void processElement(ProcessContext c) {
c.output(c.element().toString());
System.out.println(c.element().toString());
}
}));
gs_output_final.apply(TextIO.write().to("gs://gcp/output/Q"));
I have created class ClassWeeklyDueto below :
package com.pojo;
import java.io.Serializable;
public class ClassWeeklyDueto implements Serializable {
private static final long serialVersionUID = 1L;
public String Outlet;
public String CatLib;
public String ProdKey;
public String Week;
public String SalesComponent;
public float DuetoValue;
public String PrimaryCausalKey;
public String CausalValue;
public String ModelIteration;
public String Published;
public String getOutlet() {
return Outlet;
}
public void setOutlet(String outlet) {
Outlet = outlet;
}
public String getCatLib() {
return CatLib;
}
public void setCatLib(String catLib) {
CatLib = catLib;
}
public String getProdKey() {
return ProdKey;
}
public void setProdKey(String prodKey) {
ProdKey = prodKey;
}
public String getWeek() {
return Week;
}
public void setWeek(String week) {
Week = week;
}
public String getSalesComponent() {
return SalesComponent;
}
public void setSalesComponent(String salesComponent) {
SalesComponent = salesComponent;
}
public float getDuetoValue() {
return DuetoValue;
}
public void setDuetoValue(float duetoValue) {
DuetoValue = duetoValue;
}
public String getPrimaryCausalKey() {
return PrimaryCausalKey;
}
public void setPrimaryCausalKey(String primaryCausalKey) {
PrimaryCausalKey = primaryCausalKey;
}
public String getCausalValue() {
return CausalValue;
}
public void setCausalValue(String causalValue) {
CausalValue = causalValue;
}
public String getModelIteration() {
return ModelIteration;
}
public void setModelIteration(String modelIteration) {
ModelIteration = modelIteration;
}
public String getPublished() {
return Published;
}
public void setPublished(String published) {
Published = published;
}
public float setDuetoValue(String string) {
// TODO Auto-generated method stub
float f = Float.valueOf(string.trim()).floatValue();
return f;
}
}
The DueToValue field is declared float type, Only the field declared as varchar is getting parsed rest none of the datatypes are getting parsed.
So how shall I parse field declared as Int or float or even Date ?

Answer 1

When you manually split a string line from CSV, you get an array of strings. Then you have to manually parse the values from strings. Java doesn't handle it automatically.
In your case to handle floats you need to change clr.setDueToValue(strArr[5]) to clr.setDueToValue(Float.parseFloat(strArr[5])), see the doc.
Similarly you can use Integer.parseInt() to parse integers.
For parsing dates you will likely need to use a SimpleDateFormat.