Not able run code on spark-submit - scala

Question

I have this code of scala which I want to run on terminal using spark-submit command. There seems to no problem while running it in intellij IDE.
Code
package com.scryAnalytics.NLPAnnotationController.Work
import java.net.MalformedURLException
import java.util.{ArrayList, Arrays}
import com.scryAnalytics.NLPAnnotationController.Configuration.{VOCPConstants, VocpConfiguration}
import com.scryAnalytics.NLPAnnotationController.DAO.NLPEntitiesDAO
import com.scryAnalytics.NLPGeneric.{NLPEntities, _}
import com.vocp.ner.main.GateNERImpl
import gate.util.GateException
import org.apache.hadoop.hbase.client.{HBaseAdmin, Put}
import org.apache.hadoop.hbase.io.ImmutableBytesWritable
import org.apache.hadoop.hbase.mapreduce.{MultiTableOutputFormat, TableInputFormat, TableOutputFormat}
import org.apache.hadoop.hbase.util.Bytes
import org.apache.hadoop.hbase.{HTableDescriptor, TableName}
import org.apache.hadoop.mapreduce.Job
import org.apache.log4j.Logger
import org.apache.spark.{SparkConf, SparkContext}
class NLPProcessingLog {
var log: Logger = Logger.getLogger(classOf[NLPProcessingLog])
log.info("Logger Initialized .....")
}
object NlpProcessing {
val logger = new NLPProcessingLog
#throws(classOf[Exception])
def nlpAnnotationExtraction(conf: org.apache.hadoop.conf.Configuration, batchString: String): Int = {
logger.log.info("In Main Object..")
//Initializing Spark Context
val sc = new SparkContext(new SparkConf().setAppName("NLPAnnotationController").setMaster("local"))
val batchId =
if (batchString == "newbatch")
java.lang.Long.toString(System.currentTimeMillis())
else batchString
conf.set("batchId", batchId)
val inputCfs = Arrays.asList(conf.get(VOCPConstants.INPUTCOLUMNFAMILIES).split(","): _*)
try {
conf.set(TableInputFormat.INPUT_TABLE, conf.get(VOCPConstants.INPUTTABLE))
conf.set(TableOutputFormat.OUTPUT_TABLE, conf.get(VOCPConstants.OUTPUTTABLE))
val job: Job = Job.getInstance(conf, "NLPAnnotationJob")
job.getConfiguration().set(TableOutputFormat.OUTPUT_TABLE, conf.get(VOCPConstants.OUTPUTTABLE))
job.setOutputFormatClass(classOf[MultiTableOutputFormat])
val admin = new HBaseAdmin(conf)
if (!admin.isTableAvailable(conf.get(VOCPConstants.OUTPUTTABLE))) {
val tableDesc = new HTableDescriptor(TableName.valueOf(conf.get(VOCPConstants.OUTPUTTABLE)))
admin.createTable(tableDesc)
}
val hBaseRDD = sc.newAPIHadoopRDD(conf, classOf[TableInputFormat],
classOf[org.apache.hadoop.hbase.io.ImmutableBytesWritable],
classOf[org.apache.hadoop.hbase.client.Result])
val processedFilteredRDD = hBaseRDD.map(x => x._2).filter { result =>
val flag = Bytes.toString(result.getValue(Bytes.toBytes("f"),
Bytes.toBytes("is_processed")))
(flag == null) || (flag == 0)
}
println(processedFilteredRDD.count())
val messageRDD = processedFilteredRDD.filter { x => x != null }.map { result =>
val message = Bytes.toString(result.getValue(Bytes.toBytes("p"),
Bytes.toBytes("message")))
(Bytes.toString(result.getRow()), message)
}
println("Number of partitions " + messageRDD.getNumPartitions)
val pluginHome = conf.get(VOCPConstants.GATE_PLUGIN_ARCHIVE)
val requiredNLPEntities = new ArrayList[NLPEntities]()
requiredNLPEntities.add(NLPEntities.POS_TAGGER)
requiredNLPEntities.add(NLPEntities.VP_CHUNKER)
requiredNLPEntities.add(NLPEntities.NP_CHUNKER)
val nlpGenericRDD = messageRDD.mapPartitions { iter =>
val nlpModule = new GateGenericNLP(pluginHome, requiredNLPEntities)
iter.map { x =>
val nlpGenericJson = nlpModule.generateNLPEntities(x._2)
val genericNLPObject = Utility.jsonToGenericNLP(nlpGenericJson)
(x._1, x._2, genericNLPObject)
}
}
val requiredNEREntities = new ArrayList[String]()
requiredNEREntities.add("DRUG")
requiredNEREntities.add("SE")
requiredNEREntities.add("REG")
requiredNEREntities.add("ALT_THERAPY")
requiredNEREntities.add("ALT_DRUG")
val nlpRDD = nlpGenericRDD.mapPartitions { iter =>
val nerModule = new GateNERImpl(pluginHome, requiredNEREntities)
iter.map { x =>
val nerJson = nerModule.generateNER(x._2, Utility.objectToJson(x._3))
val nerJsonObject = Utility.jsonToGateNer(nerJson)
val nlpEntities: NLPEntitiesDAO = new NLPEntitiesDAO
nlpEntities.setToken(x._3.getToken())
nlpEntities.setSpaceToken(x._3.getSpaceToken())
nlpEntities.setSentence(x._3.getSentence())
nlpEntities.setSplit(x._3.getSplit())
nlpEntities.setVG(x._3.getVG)
nlpEntities.setNounChunk(x._3.getNounChunk)
nlpEntities.setDRUG(nerJsonObject.getDRUG())
nlpEntities.setREG(nerJsonObject.getREG())
nlpEntities.setSE(nerJsonObject.getSE())
nlpEntities.setALT_DRUG(nerJsonObject.getALT_DRUG())
nlpEntities.setALT_THERAPY(nerJsonObject.getALT_THERAPY())
(x._1, nlpEntities)
}
}
//outputRDD.foreach(println)
val newRDD = nlpRDD.map { k => convertToPut(k) }
newRDD.saveAsNewAPIHadoopDataset(job.getConfiguration())
return 0
} catch {
case e: MalformedURLException => {
e.printStackTrace()
return 1
}
case e: GateException =>
{
e.printStackTrace()
return 1
}
}
}
def convertToPut(genericNlpWithRowKey: (String, NLPEntitiesDAO)): (ImmutableBytesWritable, Put) = {
val rowkey = genericNlpWithRowKey._1
val genericNLP = genericNlpWithRowKey._2
val put = new Put(Bytes.toBytes(rowkey))
val genCFDataBytes = Bytes.toBytes("gen")
val nerCFDataBytes = Bytes.toBytes("ner")
val flagCFDataBytes = Bytes.toBytes("f")
put.add(genCFDataBytes, Bytes.toBytes("token"),
Bytes.toBytes(Utility.objectToJson((genericNLP.getToken()))));
put.add(genCFDataBytes, Bytes.toBytes("spaceToken"),
Bytes.toBytes(Utility.objectToJson((genericNLP.getSpaceToken()))));
put.add(genCFDataBytes, Bytes.toBytes("sentence"),
Bytes.toBytes(Utility.objectToJson((genericNLP.getSentence()))));
put.add(genCFDataBytes, Bytes.toBytes("verbGroup"),
Bytes.toBytes(Utility.objectToJson((genericNLP.getVG()))));
put.add(genCFDataBytes, Bytes.toBytes("split"),
Bytes.toBytes(Utility.objectToJson((genericNLP.getSplit()))));
put.add(genCFDataBytes, Bytes.toBytes("nounChunk"),
Bytes.toBytes(Utility.objectToJson((genericNLP.getNounChunk()))));
put.add(nerCFDataBytes, Bytes.toBytes("drug"),
Bytes.toBytes(Utility.objectToJson((genericNLP.getDRUG()))))
put.add(nerCFDataBytes, Bytes.toBytes("sideEffect"),
Bytes.toBytes(Utility.objectToJson((genericNLP.getSE()))))
put.add(nerCFDataBytes, Bytes.toBytes("regimen"),
Bytes.toBytes(Utility.objectToJson((genericNLP.getREG()))))
put.add(nerCFDataBytes, Bytes.toBytes("altTherapy"),
Bytes.toBytes(Utility.objectToJson((genericNLP.getALT_THERAPY()))))
put.add(nerCFDataBytes, Bytes.toBytes("altDrug"),
Bytes.toBytes(Utility.objectToJson((genericNLP.getALT_DRUG()))))
put.add(flagCFDataBytes, Bytes.toBytes("is_processed"),
Bytes.toBytes("1"))
put.add(flagCFDataBytes, Bytes.toBytes("dStatus"),
Bytes.toBytes("0"))
put.add(flagCFDataBytes, Bytes.toBytes("rStatus"),
Bytes.toBytes("0"))
put.add(flagCFDataBytes, Bytes.toBytes("adStatus"),
Bytes.toBytes("0"))
put.add(flagCFDataBytes, Bytes.toBytes("atStatus"),
Bytes.toBytes("0"))
(new ImmutableBytesWritable(Bytes.toBytes(rowkey)), put)
}
def pipeLineExecute(args: Array[String]): Int = {
var batchString = ""
val usage = "Usage: NLPAnnotationController" + " -inputTable tableName -outputTable tableName" +
" -batchId batchId / -newbatch \n"
if (args.length == 0) {
System.err.println(usage)
return -1
}
val conf = VocpConfiguration.create
for (i <- 0 until args.length by 2) {
if ("-inputTable" == args(i)) {
conf.set(VOCPConstants.INPUTTABLE, args(i + 1))
} else if ("-outputTable" == args(i)) {
conf.set(VOCPConstants.OUTPUTTABLE, args(i + 1))
} else if ("-batchId" == args(i)) {
batchString = args(i)
} else if ("-newbatch" == args(i)) {
batchString = "newbatch"
} else {
throw new IllegalArgumentException("arg " + args(i) + " not recognized")
}
}
val result = nlpAnnotationExtraction(conf, batchString)
result
}
def main(args: Array[String]) {
val res = pipeLineExecute(args)
System.exit(res)
}
}
I am trying to make a fat jar file to be executed using spark-submit.
pom.xml
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.scryAnalytics</groupId>
<artifactId>NLPAnnotationController</artifactId>
<version>1.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>NLPAnnotationController2</name>
<url>http://maven.apache.org</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<hadoop.version>2.6.0-cdh5.7.2</hadoop.version>
<jdk.version>1.7</jdk.version>
<sdk.version>2.10.5</sdk.version>
<hbase.version>0.98.16-hadoop2</hbase.version>
</properties>
<repositories>
<repository>
<id>cloudera</id>
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
</repository>
</repositories>
<pluginRepositories>
<pluginRepository>
<id>scala-tools.org</id>
<name>Scala-tools Maven2 Repository</name>
<url>http://scala-tools.org/repo-releases</url>
</pluginRepository>
</pluginRepositories>
<dependencies>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>2.10.5</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.10</artifactId>
<version>1.6.1</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-mllib_2.10</artifactId>
<version>1.6.1</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>${hbase.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-common</artifactId>
<version>${hbase.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-spark</artifactId>
<version>1.2.0-cdh5.7.2</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
<version>${hbase.version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit-dep</artifactId>
<version>4.8.2</version>
</dependency>
<dependency>
<groupId>uk.ac.gate</groupId>
<artifactId>gate-core</artifactId>
<version>8.1</version>
</dependency>
<dependency>
<groupId>uk.ac.gate</groupId>
<artifactId>gate-compiler-jdt</artifactId>
<version>4.3.2-P20140317-1600</version>
</dependency>
<dependency>
<groupId>com.thoughtworks.xstream</groupId>
<artifactId>xstream</artifactId>
<version>1.4.8</version>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
<version>1.9.13</version>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
<version>1.9.13</version>
</dependency>
<dependency>
<groupId>com.scryAnalytics</groupId>
<artifactId>NLPGeneric</artifactId>
<version>1.1</version>
</dependency>
<dependency>
<groupId>NER</groupId>
<artifactId>NER</artifactId>
<version>1.2</version>
</dependency>
</dependencies>
<build>
<finalName>NLPAnnotationController</finalName>
<plugins>
<!-- download source code in Eclipse, best practice -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-eclipse-plugin</artifactId>
<version>2.9</version>
<configuration>
<downloadSources>true</downloadSources>
<downloadJavadocs>false</downloadJavadocs>
</configuration>
</plugin>
<!-- Set a compiler level -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.3</version>
<configuration>
<source>${jdk.version}</source>
<target>${jdk.version}</target>
</configuration>
</plugin>
<!-- Maven Assembly Plugin -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4.1</version>
<configuration>
<!-- get all project dependencies -->
<descriptors>
<descriptor>src/main/assembly/hadoop-job.xml</descriptor>
</descriptors>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<!-- MainClass in mainfest make a executable jar -->
<archive>
<manifest>
<mainClass>com.scryAnalytics.NLPAnnotationController.Work.NlpProcessing</mainClass>
</manifest>
</archive>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<!-- bind to the packaging phase -->
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
<resources>
<resource>
<directory>conf</directory>
</resource>
</resources>
</build>
Error
spark-submit target/NLPAnnotationController-job.jar -inputTable posts -outputTable posts -batchId 1
java.lang.ClassNotFoundException: com.scryAnalytics.NLPAnnotationController.Work.NlpProcessing
at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:278)
at org.apache.spark.util.Utils$.classForName(Utils.scala:174)
at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:689)
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181)
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:121)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
As I have said, the works perfectly fine on intellij.
Any help would be appreciated.