Unable to load the data to elasticsearch using kafka-structured streaming - scala

Question

I am able to consume the messages from kafka using structured streaming and able to print on the console mode but unable to write it to elastic search. Tried all the available options nothing is working. Unfortunately, I am not getting any error also. Any suggestions ? Am I missing anything?
val kafkaDF = spark
.readStream
.format("kafka")
.option("kafka.bootstrap.servers", config.KafkaServer) //"x-sin-edp-d-kfk-8-s01.dfs:6667,x-sin-edp-d-kfk-8-s02.dfs:6667"
.option("subscribe", config.topic) //"uat.push.mulesoft")
.option("startingOffsets", "earliest")
.option("failOnDataLoss", "false")
//.option("auto.offset.reset", "earliest")
.load()
val invoiceWriterQuery = outputDF.writeStream
//.queryName("Flattened Invoice Writer")
.outputMode("append")
// .format("console")
.format("org.elasticsearch.spark.sql")
//.format("es")
.option("es.nodes.wan.only","true")
.option("es.nodes.discovery", "false")
.option("es.nodes.client.only", "false")
.option("es.net.ssl","true")
.option("es.mapping.id", outputDF.col("key").toString())
.option("es.write.operation", "upsert")
.option("es.nodes", config.esnodes )//"https://vpc-dfs-datahub-elastic-dev-j4qxkivimyqieuuzed2ntomviu.ap-southeast-1.es.amazonaws.com/"
.option("es.port", config.esport)
.option("es.resource", "loyalty_vw_eod_sls_dtl_test1/doc")
// .option("path", "output")
.option("checkpointLocation", "chk-point-dir")
.trigger(Trigger.ProcessingTime("1 minute"))
.start()
//sf_df.unpersist()
invoiceWriterQuery.awaitTermination();
--------------------------------------------------------------------------------------------------
4.0.0
<groupId>org.example</groupId>
<artifactId>LoyaltyESWrite</artifactId>
<version>0.0.1-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Loyalty_Write_To_ElasticSearch</name>
<url>http://maven.apache.org</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<encoding>UTF-8</encoding>
<scala.version>2.11.8</scala.version>
<scala.compat.version>2.11</scala.compat.version>
<java.version>1.8</java.version>
<spark.core.version>2.3.1</spark.core.version>
<hadoop.version>2.7.3</hadoop.version>
</properties>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.compat.version}</artifactId>
<version>${spark.core.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.compat.version}</artifactId>
<version>${spark.core.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_${scala.compat.version}</artifactId>
<version>${spark.core.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming-kafka-0-10_2.11</artifactId>
<!--<version>${spark.core.version}</version> -->
<version>2.3.1</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql-kafka-0-10_2.11</artifactId>
<version>2.3.1</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-compiler</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>net.ceedubs</groupId>
<artifactId>ficus_2.11</artifactId>
<version>1.1.1</version>
</dependency>
<dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>net.liftweb</groupId>
<artifactId>lift-json_2.11</artifactId>
<version>3.2.0-M2</version>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${scala.compat.version}</artifactId>
<version>2.2.6</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>jdk.tools</groupId>
<artifactId>jdk.tools</artifactId>
<version>1.8</version>
<scope>system</scope>
<systemPath>${JAVA_HOME}/lib/tools.jar</systemPath>
</dependency>
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch-spark-20_2.11</artifactId>
<version>7.1.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/net.snowflake/spark-snowflake -->
<dependency>
<groupId>net.snowflake</groupId>
<artifactId>spark-snowflake_2.11</artifactId>
<version>2.8.5-spark_2.3</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>3.2.2</version>
<executions>
<execution>
<id>scala-compile-first</id>
<phase>process-resources</phase>
<goals>
<goal>add-source</goal>
<goal>compile</goal>
</goals>
<configuration>
<sources>
<source>src/main/scala</source>
</sources>
</configuration>
</execution>
<execution>
<id>scala-test-compile</id>
<phase>process-test-resources</phase>
<goals>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
<configuration>
<recompileMode>incremental</recompileMode>
<args>
<arg>-deprecation</arg>
<arg>-explaintypes</arg>
<arg>-target:jvm-1.8</arg>
<arg>${project.build.directory}/.scala_dependencies</arg>
</args>
</configuration>
</plugin>
<plugin>
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
<version>1.0</version>
<configuration>
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
<stdout>W</stdout>
</configuration>
<executions>
<execution>
<id>scala-test</id>
<goals>
<goal>test</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.5.1</version>
<configuration>
<source>${java.version}</source>
<target>${java.version}</target>
<encoding>UTF-8</encoding>
<maxmem>1024m</maxmem>
<fork>true</fork>
<compilerArgs>
<arg>-Xlint:all,-serial,-path</arg>
</compilerArgs>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.12.4</version>
<configuration>
<argLine>-Xmx512m</argLine>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.6</version>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<archive>
<manifest>
<mainClass>org.example.WriteToES</mainClass>
</manifest>
</archive>
</configuration>
<executions>
<execution>
<id>create-my-bundle</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>