Can't create stream from Kafka topic using Siddhi. Even if I create string with Design View.
I copied all required jars to lib and bundle folders. Even started Kafka with Zookeeper locally (dunno why I need it locally but nwm).
On tooling.sh start I have following error:
[2020-02-26 22:15:43,041] WARNING {org.wso2.carbon.launcher.extensions.OSGiLibBundleDeployerUtils lambda$getBundlesInfo$1} - Error when loading the OSGi bundle information from /home/Hed/StreamProcessor/siddhi-tooling-5.1.2/lib/kafka-clients-2.3.0.jar
java.io.IOException: Required bundle manifest headers do not exist
at org.wso2.carbon.launcher.extensions.OSGiLibBundleDeployerUtils.getBundleInfo(OSGiLibBundleDeployerUtils.java:183)
at org.wso2.carbon.launcher.extensions.OSGiLibBundleDeployerUtils.lambda$getBundlesInfo$1(OSGiLibBundleDeployerUtils.java:135)
at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
at java.util.Spliterators$ArraySpliterator.forEachRemaining(Spliterators.java:948)
at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482)
at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
at java.util.stream.StreamSpliterators$WrappingSpliterator.forEachRemaining(StreamSpliterators.java:313)
at java.util.stream.StreamSpliterators$DistinctSpliterator.forEachRemaining(StreamSpliterators.java:1291)
at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482)
at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
at java.util.stream.ReduceOps$ReduceTask.doLeaf(ReduceOps.java:747)
at java.util.stream.ReduceOps$ReduceTask.doLeaf(ReduceOps.java:721)
at java.util.stream.AbstractTask.compute(AbstractTask.java:327)
at java.util.concurrent.CountedCompleter.exec(CountedCompleter.java:731)
at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289)
at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056)
at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692)
at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:157)
For this script:
#App:name("HelloKafka")
#App:description('Consume events from a Kafka Topic and publish to a different Kafka Topic')
#source(type='kafka',
topic.list='kafka_topic',
partition.no.list='0',
threading.option='single.thread',
group.id="group",
bootstrap.servers='localhost:9092',
#map(type='json'))
define stream SweetProductionStream (name string, amount double);
I have see error on Run command:
io.siddhi.core.exception.SiddhiAppCreationException: Error on 'HelloKafka' # Line: 10. Position: 26, near '#source(type='kafka',
topic.list='kafka_topic',
partition.no.list='0',
threading.option='single.thread',
group.id="group",
bootstrap.servers='localhost:9092',
#map(type='json'))'. org/apache/kafka/clients/producer/Producer
at io.siddhi.core.util.ExceptionUtil.populateQueryContext(ExceptionUtil.java:43)
at io.siddhi.core.util.parser.helper.DefinitionParserHelper.addEventSource(DefinitionParserHelper.java:388)
at io.siddhi.core.util.SiddhiAppRuntimeBuilder.defineStream(SiddhiAppRuntimeBuilder.java:117)
at io.siddhi.core.util.parser.SiddhiAppParser.defineStreamDefinitions(SiddhiAppParser.java:374)
at io.siddhi.core.util.parser.SiddhiAppParser.parse(SiddhiAppParser.java:230)
at io.siddhi.core.SiddhiManager.createSiddhiAppRuntime(SiddhiManager.java:85)
at io.siddhi.core.SiddhiManager.createSiddhiAppRuntime(SiddhiManager.java:95)
at io.siddhi.distribution.editor.core.internal.DebugRuntime.createRuntime(DebugRuntime.java:201)
at io.siddhi.distribution.editor.core.internal.DebugRuntime.(DebugRuntime.java:56)
at io.siddhi.distribution.editor.core.internal.DebugProcessorService.start(DebugProcessorService.java:38)
at io.siddhi.distribution.editor.core.internal.EditorMicroservice.start(EditorMicroservice.java:761)
at io.siddhi.distribution.editor.core.internal.EditorMicroservice.startWithVariables(EditorMicroservice.java:781)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.wso2.msf4j.internal.router.HttpMethodInfo.invokeResource(HttpMethodInfo.java:187)
at org.wso2.msf4j.internal.router.HttpMethodInfo.invoke(HttpMethodInfo.java:143)
at org.wso2.msf4j.internal.MSF4JHttpConnectorListener.dispatchMethod(MSF4JHttpConnectorListener.java:218)
at org.wso2.msf4j.internal.MSF4JHttpConnectorListener.lambda$onMessage$58(MSF4JHttpConnectorListener.java:129)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.NoClassDefFoundError: org/apache/kafka/clients/producer/Producer
at java.lang.Class.getDeclaredConstructors0(Native Method)
at java.lang.Class.privateGetDeclaredConstructors(Class.java:2671)
at java.lang.Class.getConstructor0(Class.java:3075)
at java.lang.Class.newInstance(Class.java:412)
at io.siddhi.core.util.SiddhiClassLoader.loadClass(SiddhiClassLoader.java:32)
at io.siddhi.core.util.SiddhiClassLoader.loadExtensionImplementation(SiddhiClassLoader.java:48)
at io.siddhi.core.util.parser.helper.DefinitionParserHelper.addEventSource(DefinitionParserHelper.java:346)
... 21 more
Caused by: java.lang.ClassNotFoundException: org.apache.kafka.clients.producer.Producer cannot be found by siddhi-io-kafka_5.0.7
at org.eclipse.osgi.internal.loader.BundleLoader.findClassInternal(BundleLoader.java:448)
at org.eclipse.osgi.internal.loader.BundleLoader.findClass(BundleLoader.java:361)
at org.eclipse.osgi.internal.loader.BundleLoader.findClass(BundleLoader.java:353)
at org.eclipse.osgi.internal.loader.ModuleClassLoader.loadClass(ModuleClassLoader.java:161)
at java.lang.ClassLoader.loadClass(ClassLoader.java:352)
... 28 more
Can somebody tell me what am I doing wrong? :(
Please make sure you had the OSGi-converted jars to the "C:\Program Files\WSO2\Enterprise Integrator\7.0.2\streaming-integrator\lib".
The OSGi-converted jar list:
kafka_2.12_2.3.0_1.0.0
kafka_clients_2.3.0_1.0.0
metrics_core_2.2.0_1.0.0
scala_library_2.12.8_1.0.0
zkclient_0.11_1.0.0
zookeeper_3.4.14_1.0.0
The, copy the original jars to to the "C:\Program Files\WSO2\Enterprise Integrator\7.0.2\streaming-integrator\samples\sample-clients\lib"
The list of original jars:
kafka_2.12-2.3.0
kafka-clients-2.3.0
metrics-core-2.2.0
scala-library-2.12.8
zkclient-0.11
zookeeper-3.4.14
In order to generate the OSGi-converted jars, copy all original jars to a folder called "source" and create an empty folder called "destination". Then run the following command in the terminal:
MINGW32 /c/Program Files/WSO2/Enterprise Integrator/7.0.2/streaming-integrator/bin
$ ./jartobundle.sh C:/DevTools/source C:/DevTools/destination
Finally, distribute the OSGis and original in accordance with the directories above.
PS1: in my case i am using kafka_2.12-2.4.1, but the basename of the jars does not change.
PS2: adapt the directories to your installation path
For more details check WSO2 documentation: Kafka transport
I tried to analyze a DB2 database with SchemaSpy, but got a warning 'Connection Failure'. I tried this way:
java -jar schemaspy-6.0.0.jar -configFile schemaspy.properties --logging.pattern.console="%d{HH:mm:ss.SSS} %clr(%-5level) - %msg%n" --logging.level.org.schemaspy=TRACE
(found the logging part on https://github.com/schemaspy/schemaspy/issues/250)
The .properties file looks like this:
schemaspy.t=db2
schemaspy.dp=C:\tmp\db2jcc.jar
schemaspy.host=**host**
schemaspy.port=50000
schemaspy.db=**db**
schemaspy.u=**user**
schemaspy.p=**password**
schemaspy.o=D:\**\schemaspy-output\
schemaspy.s=**schema**
The error I got was:
14:24:20.297 DEBUG - Unable to find driverClass COM.ibm.db2.jdbc.app.DB2Driver'
14:24:20.308 WARN - Connection Failure
org.schemaspy.model.ConnectionFailure: Failed to connect to database URL [jdbc:db2:zumtest] Failed to create any of 'COM.ibm.db2.jdbc.app.DB2Driver' driver from driverPath 'C:\tmp\db2jcc.jar' with sibling jars no.
Resulting in classpath:
file:/C:/tmp/db2jcc.jar
at org.schemaspy.DbDriverLoader.getConnection(DbDriverLoader.java:101)
at org.schemaspy.DbDriverLoader.getConnection(DbDriverLoader.java:75)
at org.schemaspy.service.SqlService.connect(SqlService.java:68)
at org.schemaspy.SchemaAnalyzer.analyze(SchemaAnalyzer.java:186)
at org.schemaspy.SchemaAnalyzer.analyze(SchemaAnalyzer.java:107)
at org.schemaspy.cli.SchemaSpyRunner.runAnalyzer(SchemaSpyRunner.java:97)
at org.schemaspy.cli.SchemaSpyRunner.run(SchemaSpyRunner.java:86)
at org.schemaspy.Main.main(Main.java:48)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
at java.lang.reflect.Method.invoke(Unknown Source)
at org.springframework.boot.loader.MainMethodRunner.run(MainMethodRunner.java:48)
at org.springframework.boot.loader.Launcher.launch(Launcher.java:87)
at org.springframework.boot.loader.Launcher.launch(Launcher.java:50)
at org.springframework.boot.loader.JarLauncher.main(JarLauncher.java:51)
Caused by: org.schemaspy.model.ConnectionFailure: Failed to create any of 'COM.ibm.db2.jdbc.app.DB2Driver' driver from driverPath 'C:\tmp\db2jcc.jar' with sibling jars no.
Resulting in classpath:
file:/C:/tmp/db2jcc.jar
at org.schemaspy.DbDriverLoader.getDriver(DbDriverLoader.java:147)
at org.schemaspy.DbDriverLoader.getConnection(DbDriverLoader.java:93)
... 15 common frames omitted
I guess, the error comes from the wrong class path? But how can I fix this? I tried to change the line in db2.properties
driver=COM.ibm.db2.jdbc.app.DB2Driver
to
driver=COM.ibm.db2.jcc.DB2Driver
, because I extracted this class path from the driver's .jar flie, but it did not help.
this worked for me using db2jcc.jar
# type of database. Run with -dbhelp for details
schemaspy.t=db2
# optional path to alternative jdbc drivers.
schemaspy.dp=/lib/db2jcc.jar
# database properties: host, port number, name user, password
schemaspy.host=xxxxxx
schemaspy.port=xxxxx
schemaspy.db=xxx
schemaspy.u=xx
schemaspy.p=xx
# output dir to save generated files
schemaspy.o=/output
# db scheme for which generate diagrams
schemaspy.s=xxxx
driver=com.ibm.db2.jcc.DB2Driver
connectionSpec=jdbc:db2://xxx.xx.x.xxx:[PORT]/[DBNAME]
schemaspy.cat=%
The command used:
java -jar schemaspy-6.1.0.jar -configFile db2.properties --logging.pattern.console="%d{HH:mm:ss.SSS} %clr(%-5level) - %msg%n" --logging.level.org.schemaspy=TRACE
I made an installer with latest Install4J and want to install the package on a raspberry pi for testing.
It's running latest Raspbian Image with latest updates and openJDK v.1.8.0.40
If I try to start the installer in unattended mode (-q) I get a "Starting Installer..." message and then I get a NullPointerException.
This is the console output:
pi#raspberrypi:~ $ ./plentyBase.sh -q
Starting Installer ...
java.lang.NullPointerException
at javax.swing.RepaintManager.currentManager(RepaintManager.java:262)
at javax.swing.JComponent.repaint(JComponent.java:4799)
at java.awt.Component.repaint(Component.java:3303)
at javax.swing.JComponent.setFont(JComponent.java:2756)
at javax.swing.LookAndFeel.installColorsAndFont(LookAndFeel.java:208)
at javax.swing.plaf.basic.BasicPanelUI.installDefaults(BasicPanelUI.java:66)
at javax.swing.plaf.basic.BasicPanelUI.installUI(BasicPanelUI.java:56)
at javax.swing.JComponent.setUI(JComponent.java:664)
at javax.swing.JPanel.setUI(JPanel.java:153)
at javax.swing.JPanel.updateUI(JPanel.java:126)
at javax.swing.JPanel.<init>(JPanel.java:86)
at javax.swing.JPanel.<init>(JPanel.java:109)
at javax.swing.JPanel.<init>(JPanel.java:117)
at com.install4j.runtime.installer.frontend.FormPanel.<init>(FormPanel.java:50)
at com.install4j.runtime.installer.frontend.headless.AbstractHeadlessScreenExecutor.init(AbstractHeadlessScreenExecutor.java:67)
at com.install4j.runtime.installer.frontend.headless.UnattendedScreenExecutor.<init>(UnattendedScreenExecutor.java:48)
at com.install4j.runtime.installer.frontend.headless.InstallerUnattendedScreenExecutor.<init>(InstallerUnattendedScreenExecutor.java:10)
at com.install4j.runtime.installer.Installer.getScreenExecutor(Installer.java:88)
at com.install4j.runtime.installer.Installer.runInProcess(Installer.java:58)
at com.install4j.runtime.installer.Installer.main(Installer.java:46)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:483)
at com.exe4j.runtime.LauncherEngine.launch(LauncherEngine.java:65)
at com.install4j.runtime.launcher.UnixLauncher.main(UnixLauncher.java:57)
An error occurred:
java.lang.NullPointerException
Error log: /tmp/install4jError7183942791429390239.log
java.lang.NullPointerException
at javax.swing.RepaintManager.currentManager(RepaintManager.java:262)
at javax.swing.JComponent.repaint(JComponent.java:4799)
at java.awt.Component.repaint(Component.java:3303)
at javax.swing.JComponent.setFont(JComponent.java:2756)
at javax.swing.LookAndFeel.installColorsAndFont(LookAndFeel.java:208)
at javax.swing.plaf.basic.BasicPanelUI.installDefaults(BasicPanelUI.java:66)
at javax.swing.plaf.basic.BasicPanelUI.installUI(BasicPanelUI.java:56)
at javax.swing.JComponent.setUI(JComponent.java:664)
at javax.swing.JPanel.setUI(JPanel.java:153)
at javax.swing.JPanel.updateUI(JPanel.java:126)
at javax.swing.JPanel.<init>(JPanel.java:86)
at javax.swing.JPanel.<init>(JPanel.java:109)
at javax.swing.JPanel.<init>(JPanel.java:117)
at com.install4j.runtime.installer.frontend.FormPanel.<init>(FormPanel.java:50)
at com.install4j.runtime.installer.frontend.headless.AbstractHeadlessScreenExecutor.init(AbstractHeadlessScreenExecutor.java:67)
at com.install4j.runtime.installer.frontend.headless.UnattendedScreenExecutor.<init>(UnattendedScreenExecutor.java:48)
at com.install4j.runtime.installer.frontend.headless.InstallerUnattendedScreenExecutor.<init>(InstallerUnattendedScreenExecutor.java:10)
at com.install4j.runtime.installer.Installer.getScreenExecutor(Installer.java:88)
at com.install4j.runtime.installer.Installer.runInProcess(Installer.java:58)
at com.install4j.runtime.installer.Installer.main(Installer.java:46)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:483)
at com.exe4j.runtime.LauncherEngine.launch(LauncherEngine.java:65)
at com.install4j.runtime.launcher.UnixLauncher.main(UnixLauncher.java:57)
This is the content of the error log:
pi#raspberrypi:~ $ cat /tmp/install4jError7183942791429390239.log
Exception:
java.lang.NullPointerException
at javax.swing.RepaintManager.currentManager(RepaintManager.java:262)
at javax.swing.JComponent.repaint(JComponent.java:4799)
at java.awt.Component.repaint(Component.java:3303)
at javax.swing.JComponent.setFont(JComponent.java:2756)
at javax.swing.LookAndFeel.installColorsAndFont(LookAndFeel.java:208)
at javax.swing.plaf.basic.BasicPanelUI.installDefaults(BasicPanelUI.java:66)
at javax.swing.plaf.basic.BasicPanelUI.installUI(BasicPanelUI.java:56)
at javax.swing.JComponent.setUI(JComponent.java:664)
at javax.swing.JPanel.setUI(JPanel.java:153)
at javax.swing.JPanel.updateUI(JPanel.java:126)
at javax.swing.JPanel.<init>(JPanel.java:86)
at javax.swing.JPanel.<init>(JPanel.java:109)
at javax.swing.JPanel.<init>(JPanel.java:117)
at com.install4j.runtime.installer.frontend.FormPanel.<init>(FormPanel.java:50)
at com.install4j.runtime.installer.frontend.headless.AbstractHeadlessScreenExecutor.init(AbstractHeadlessScreenExecutor.java:67)
at com.install4j.runtime.installer.frontend.headless.UnattendedScreenExecutor.<init>(UnattendedScreenExecutor.java:48)
at com.install4j.runtime.installer.frontend.headless.InstallerUnattendedScreenExecutor.<init>(InstallerUnattendedScreenExecutor.java:10)
at com.install4j.runtime.installer.Installer.getScreenExecutor(Installer.java:88)
at com.install4j.runtime.installer.Installer.runInProcess(Installer.java:58)
at com.install4j.runtime.installer.Installer.main(Installer.java:46)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:483)
at com.exe4j.runtime.LauncherEngine.launch(LauncherEngine.java:65)
at com.install4j.runtime.launcher.UnixLauncher.main(UnixLauncher.java:57)
System properties:
java.runtime.name=OpenJDK Runtime Environment
exe4j.moduleName=/home/pi/plentyBase.sh
sun.boot.library.path=/usr/lib/jvm/java-8-openjdk-armhf/jre/lib/arm
java.vm.version=25.40-b08
java.vm.vendor=Oracle Corporation
java.vendor.url=http://java.oracle.com/
path.separator=:
java.vm.name=OpenJDK Zero VM
file.encoding.pkg=sun.io
sun.java.launcher=SUN_STANDARD
sun.os.patch.level=unknown
install4j.exeDir=/home/pi/
java.vm.specification.name=Java Virtual Machine Specification
user.dir=/home/pi/plentyBase.sh.939.dir
java.runtime.version=1.8.0_40-internal-b04
java.awt.graphicsenv=sun.awt.X11GraphicsEnvironment
i4jv=0
java.endorsed.dirs=/usr/lib/jvm/java-8-openjdk-armhf/jre/lib/endorsed
os.arch=arm
java.io.tmpdir=/tmp
line.separator=
java.vm.specification.vendor=Oracle Corporation
os.name=Linux
exe4j.totalDataLength=36451286
sun.java2d.noddraw=true
sun.jnu.encoding=UTF-8
java.library.path=/home/pi/plentyBase.sh.939.dir/user::/usr/java/packages/lib/arm:/usr/lib/arm-linux-gnueabihf/jni:/lib/arm-linux-gnueabihf:/usr/lib/arm-linux-gnueabihf:/usr/lib/jni:/lib:/usr/lib
java.specification.name=Java Platform API Specification
java.class.version=52.0
os.version=4.4.50+
i4j.vpt=true
install4j.cwd=/home/pi
user.home=/home/pi
user.timezone=Etc/UTC
java.awt.printerjob=sun.print.PSPrinterJob
file.encoding=UTF-8
install4j.jvmDir=/usr/lib/jvm/java-8-openjdk-armhf/jre
java.specification.version=1.8
java.class.path=i4jruntime.jar
user.name=pi
java.vm.specification.version=1.8
sun.java.command=com.install4j.runtime.launcher.UnixLauncher launch 0 com.install4j.runtime.installer.Installer -q
java.home=/usr/lib/jvm/java-8-openjdk-armhf/jre
sun.arch.data.model=32
user.language=en
java.specification.vendor=Oracle Corporation
awt.toolkit=sun.awt.X11.XToolkit
java.vm.info=interpreted mode
java.version=1.8.0_40-internal
java.ext.dirs=/usr/lib/jvm/java-8-openjdk-armhf/jre/lib/ext:/usr/java/packages/lib/ext
sun.boot.class.path=/usr/lib/jvm/java-8-openjdk-armhf/jre/lib/resources.jar:/usr/lib/jvm/java-8-openjdk-armhf/jre/lib/rt.jar:/usr/lib/jvm/java-8-openjdk-armhf/jre/lib/sunrsasign.jar:/usr/lib/jvm/java-8-openjdk-armhf/jre/lib/jsse.jar:/usr/lib/jvm/java-8-openjdk-armhf/jre/lib/jce.jar:/usr/lib/jvm/java-8-openjdk-armhf/jre/lib/charsets.jar:/usr/lib/jvm/java-8-openjdk-armhf/jre/lib/jfr.jar:/usr/lib/jvm/java-8-openjdk-armhf/jre/classes
java.awt.headless=true
install4j.appDir=/home/pi/
java.vendor=Oracle Corporation
file.separator=/
java.vendor.url.bug=http://bugreport.sun.com/bugreport/
sun.font.fontmanager=sun.awt.X11FontManager
sun.io.unicode.encoding=UnicodeLittle
sun.cpu.endian=little
sun.cpu.isalist=
The same installer is working fine on Ubuntu 16.04 LTS in a virtual machine.
Edit:
I think it comes from JPanel init to get FontMetrics?
Found it here: NPE when calling new JPanel() in headless (RepaintManager.currentManager)
So it looks like a bug in install4J.
If I open the installer on desktop/gui mode on raspberry pi the installer is working fine.
Any ideas what's wrong?
I try to execute spark-shell on Windows 10, but I keep getting this error every time I run it.
I used both latest and spark-1.5.0-bin-hadoop2.4 versions.
15/09/22 18:46:24 WARN Connection: BoneCP specified but not present in
CLASSPATH (or one of dependencies)
15/09/22 18:46:24 WARN Connection: BoneCP specified but not present in CLASSPATH (or one of dependencies)
15/09/22 18:46:27 WARN ObjectStore: Version information not found in
metastore. hive.metastore.schema.verification is not enabled so recording the schema version 1.2.0
15/09/22 18:46:27 WARN ObjectStore: Failed to get database default, returning NoSuchObjectException
15/09/22 18:46:27 WARN : Your hostname, DESKTOP-8JS2RD5 resolves to a loopback/non-reachable address: fe80:0:0:0:0:5efe:c0a8:103%net1, but we couldn't find any external IP address!
java.lang.RuntimeException: java.lang.NullPointerException
at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:522)
at org.apache.spark.sql.hive.client.ClientWrapper.<init> (ClientWrapper.scala:171)
at org.apache.spark.sql.hive.HiveContext.executionHive$lzycompute(HiveContext.scala :163)
at org.apache.spark.sql.hive.HiveContext.executionHive(HiveContext.scala:161)
at org.apache.spark.sql.hive.HiveContext.<init>(HiveContext.scala:168)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source)
at java.lang.reflect.Constructor.newInstance(Unknown Source)
at org.apache.spark.repl.SparkILoop.createSQLContext(SparkILoop.scala:1028)
at $iwC$$iwC.<init>(<console>:9)
at $iwC.<init>(<console>:18)
at <init>(<console>:20)
at .<init>(<console>:24)
at .<clinit>(<console>)
at .<init>(<console>:7)
at .<clinit>(<console>)
at $print(<console>)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
at java.lang.reflect.Method.invoke(Unknown Source)
at org.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:1065)
at org.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1340)
at org.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:840)
at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:871)
at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:819)
at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:857)
at org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:902)
at org.apache.spark.repl.SparkILoop.command(SparkILoop.scala:814)
at org.apache.spark.repl.SparkILoopInit$$anonfun$initializeSpark$1.apply(SparkILoopInit.scala:132)
at org.apache.spark.repl.SparkILoopInit$$anonfun$initializeSpark$1.apply(SparkILoopInit.scala:124)
at org.apache.spark.repl.SparkIMain.beQuietDuring(SparkIMain.scala:324)
at org.apache.spark.repl.SparkILoopInit$class.initializeSpark(SparkILoopInit.scala:124)
at org.apache.spark.repl.SparkILoop.initializeSpark(SparkILoop.scala:64)
at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1$$anonfun$apply$mcZ$sp$5.apply$mcV$sp(SparkILoop.scala:974)
at org.apache.spark.repl.SparkILoopInit$class.runThunks(SparkILoopInit.scala:159)
at org.apache.spark.repl.SparkILoop.runThunks(SparkILoop.scala:64)
at org.apache.spark.repl.SparkILoopInit$class.postInitialization(SparkILoopInit.sca la:108)
at org.apache.spark.repl.SparkILoop.postInitialization(SparkILoop.scala:64)
at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$proc ess$1.apply$mcZ$sp(SparkILoop.scala:991)
at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$proc ess$1.apply(SparkILoop.scala:945)
at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$proc ess$1.apply(SparkILoop.scala:945)
at scala.tools.nsc.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scal a:135)
at org.apache.spark.repl.SparkILoop.org$apache$spark$repl$SparkILoop$$process(SparkILoop.scala:945)
at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:1059)
at org.apache.spark.repl.Main$.main(Main.scala:31)
at org.apache.spark.repl.Main.main(Main.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
at java.lang.reflect.Method.invoke(Unknown Source)
at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:672)
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180)
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:120)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
Caused by: java.lang.NullPointerException
at java.lang.ProcessBuilder.start(Unknown Source)
at org.apache.hadoop.util.Shell.runCommand(Shell.java:445)
at org.apache.hadoop.util.Shell.run(Shell.java:418)
at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:650)
at org.apache.hadoop.util.Shell.execCommand(Shell.java:739)
at org.apache.hadoop.util.Shell.execCommand(Shell.java:722)
at org.apache.hadoop.fs.FileUtil.execCommand(FileUtil.java:1097)
at org.apache.hadoop.fs.RawLocalFileSystem$DeprecatedRawLocalFileStatus.loadPermissionInfo(RawLocalFileSystem.java:559)
at org.apache.hadoop.fs.RawLocalFileSystem$DeprecatedRawLocalFileStatus.getPermission(RawLocalFileSystem.java:534)
org.apache.hadoop.hive.ql.session.SessionState.createRootHDFSDir(SessionState.java:599)
at org.apache.hadoop.hive.ql.session.SessionState.createSessionDirs(SessionState.java:554)
org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:508)
... 56 more
<console>:10: error: not found: value sqlContext
import sqlContext.implicits._
^
<console>:10: error: not found: value sqlContext
import sqlContext.sql
^
I used Spark 1.5.2 with Hadoop 2.6 and had similar problems. Solved by doing the following steps:
Download winutils.exe from the repository to some local folder, e.g. C:\hadoop\bin.
Set HADOOP_HOME to C:\hadoop.
Create c:\tmp\hive directory (using Windows Explorer or any other tool).
Open command prompt with admin rights.
Run C:\hadoop\bin\winutils.exe chmod 777 /tmp/hive
With that, I am still getting some warnings, but no ERRORs and can run Spark applications just fine.
I was facing a similar issue, got it resolved by putting the winutil inside bin folder. The Hadoop_home should be set as C:\Winutils and winutil to be placed in C:\Winutils\bin.
Windows 10 64 bit Winutils are available in https://github.com/steveloughran/winutils/tree/master/hadoop-2.6.0/bin
Also ensure that command line has administrative access.
Refer https://wiki.apache.org/hadoop/WindowsProblems
My guess is that you're running into https://issues.apache.org/jira/browse/SPARK-10528. I was seeing the same issue running on Windows 7. Initially I was getting the NullPointerException as you did. When I put winutils into the bin directory and set HADOOP_HOME to point to the Spark directory, I got the error described in the JIRA issue.
Or perhaps this link here below be easier to follow,
https://wiki.apache.org/hadoop/WindowsProblems
Basically download and copy winutils.exe to your spark\bin folder. Re-run spark-shell
If you have not set your /tmp/hive to a writable state, please do so.
You need to give permission to /tmp/hive directory to resolve this exception.
Hope you already have winutils.exe and set HADOOP_HOME environment variable. Then open the command prompt and run following command as administrator:
If winutils.exe is present in D:\winutils\bin location and \tmp\hive is also in D drive:
D:\winutils\bin\winutils.exe chmod 777 D:\tmp\hive
For more details,you can refer the following links :
Frequent Issues occurred during Spark Development
How to run Apache Spark on Windows7 in standalone mode
You can resolve this issue by placing mysqlconnector jar in spark-1.6.0/libs folder and restart it again.It works.
The important thing is here instead of running spark-shell you should do
spark-shell --driver-class-path /home/username/spark-1.6.0-libs-mysqlconnector.jar
Hope it should work.
For Python - Create a SparkSession in your python (This config section is only for Windows)
spark = SparkSession.builder.config("spark.sql.warehouse.dir", "C:/temp").appName("SparkSQL").getOrCreate()
Copy winutils.exe and keep in C:\winutils\bin and execute the bellow commands
C:\Windows\system32>C:\winutils\bin\winutils.exe chmod 777 C:/temp
Run command prompt in ADMIN mode ( Run as Administrator)
My issue was having other .exe's/Jars inside the winutils/bin folder. So I cleared all the others and was left with winutils.exe alone. Was using spark 2.1.1
Issue was resolved after installing correct Java version in my case its java 8 and setting the environmental variables. Make sure you run the winutils.exe to create a temporary directory as below.
c:\winutils\bin\winutils.exe chmod 777 \tmp\hive
Above should not return any error. Use java -version to verify the version of java you are using before invoking spark-shell.
In Windows, you need to clone "winutils"
git clone https://github.com/steveloughran/winutils.git
And
set var HADOOP_HOME to DIR_CLONED\hadoop-{version}
Remember to choose the version of your hadoop.
Setting SPARK_LOCAL_HOSTNAME as localhost (on Windows 10) resolved the problem for me
I wrote a JavaFx app (JavaFx windows based form) on my desktop running
Win 7 32 Bit
Netbeans 7.3 Beta
jdk-7u9-windows-i586
and it runs successfully.
I recently got a laptop running
Win 7 64 Bit
Netbeans 7.3 Beta
jdk-7u9-windows-x64
I just copied the code over and switch the Java Platform.... currently its set to the default "Default JavaFx Platform".
If i run the app I get the following error dialog
JavaFx launcher error - Exception while running Application
Anyone know what I need to change here...
Full stack trace when running application
ant -f "C:\\DEV\\Projects\\Java Apps\\BaseAppPlatform" jfxsa-run
init:
Deleting: C:\DEV\Projects\Java Apps\BaseAppPlatform\build\built-jar.properties
deps-jar:
Updating property file: C:\DEV\Projects\Java Apps\BaseAppPlatform\build\built-jar.properties
compile:
Detected JavaFX Ant API version 1.2
Launching <fx:jar> task from C:\Program Files\Java\jdk1.7.0_09\lib\ant-javafx.jar
Signing JAR: C:\DEV\Projects\Java Apps\BaseAppPlatform\dist\BaseAppPlatform.jar to C:\DEV\Projects\Java Apps\BaseAppPlatform\dist\BaseAppPlatform.jar as nb-jfx
Warning:
The signer certificate will expire within six months.
Enter Passphrase for keystore: Enter key password for nb-jfx:
Launching <fx:deploy> task from C:\Program Files\Java\jdk1.7.0_09\lib\ant-javafx.jar
jfx-deployment-script:
jfx-deployment:
jar:
Copying 12 files to C:\DEV\Projects\Java Apps\BaseAppPlatform\dist\run858846669
jfx-project-run:
Executing com.javafx.main.Main from C:\DEV\Projects\Java Apps\BaseAppPlatform\dist\run858846669\BaseAppPlatform.jar using platform C:\Program Files\Java\jdk1.7.0_09/bin/java
java.lang.ClassNotFoundException: co.za.chrispie.LoginController
file:/C:/DEV/Projects/Java%20Apps/BaseAppPlatform/dist/run858846669/BaseAppPlatform.jar!/BaseAppPlatform/Login.fxml:9
at javafx.fxml.FXMLLoader$ValueElement.processAttribute(FXMLLoader.java:728)
at javafx.fxml.FXMLLoader$InstanceDeclarationElement.processAttribute(FXMLLoader.java:777)
at javafx.fxml.FXMLLoader$Element.processStartElement(FXMLLoader.java:182)
at javafx.fxml.FXMLLoader$ValueElement.processStartElement(FXMLLoader.java:565)
at javafx.fxml.FXMLLoader.processStartElement(FXMLLoader.java:2314)
at javafx.fxml.FXMLLoader.load(FXMLLoader.java:2131)
at javafx.fxml.FXMLLoader.load(FXMLLoader.java:2028)
at javafx.fxml.FXMLLoader.load(FXMLLoader.java:2742)
at javafx.fxml.FXMLLoader.load(FXMLLoader.java:2721)
at javafx.fxml.FXMLLoader.load(FXMLLoader.java:2707)
at javafx.fxml.FXMLLoader.load(FXMLLoader.java:2694)
at javafx.fxml.FXMLLoader.load(FXMLLoader.java:2683)
at BaseAppPlatform.BaseAppPlatform.start(BaseAppPlatform.java:21)
at com.sun.javafx.application.LauncherImpl$5.run(LauncherImpl.java:319)
at com.sun.javafx.application.PlatformImpl$5.run(PlatformImpl.java:206)
at com.sun.javafx.application.PlatformImpl$4.run(PlatformImpl.java:173)
at com.sun.glass.ui.win.WinApplication._runLoop(Native Method)
at com.sun.glass.ui.win.WinApplication.access$100(WinApplication.java:29)
at com.sun.glass.ui.win.WinApplication$3$1.run(WinApplication.java:73)
at java.lang.Thread.run(Thread.java:722)
Exception in Application start method
java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:601)
at com.javafx.main.Main.launchApp(Main.java:642)
at com.javafx.main.Main.main(Main.java:805)
Caused by: java.lang.RuntimeException: Exception in Application start method
at com.sun.javafx.application.LauncherImpl.launchApplication1(LauncherImpl.java:403)
at com.sun.javafx.application.LauncherImpl.access$000(LauncherImpl.java:47)
at com.sun.javafx.application.LauncherImpl$1.run(LauncherImpl.java:115)
at java.lang.Thread.run(Thread.java:722)
Caused by: javafx.fxml.LoadException: java.lang.ClassNotFoundException: co.za.chrispie.LoginController
at javafx.fxml.FXMLLoader$ValueElement.processAttribute(FXMLLoader.java:728)
at javafx.fxml.FXMLLoader$InstanceDeclarationElement.processAttribute(FXMLLoader.java:777)
at javafx.fxml.FXMLLoader$Element.processStartElement(FXMLLoader.java:182)
at javafx.fxml.FXMLLoader$ValueElement.processStartElement(FXMLLoader.java:565)
at javafx.fxml.FXMLLoader.processStartElement(FXMLLoader.java:2314)
at javafx.fxml.FXMLLoader.load(FXMLLoader.java:2131)
at javafx.fxml.FXMLLoader.load(FXMLLoader.java:2028)
at javafx.fxml.FXMLLoader.load(FXMLLoader.java:2742)
at javafx.fxml.FXMLLoader.load(FXMLLoader.java:2721)
at javafx.fxml.FXMLLoader.load(FXMLLoader.java:2707)
at javafx.fxml.FXMLLoader.load(FXMLLoader.java:2694)
at javafx.fxml.FXMLLoader.load(FXMLLoader.java:2683)
at BaseAppPlatform.BaseAppPlatform.start(BaseAppPlatform.java:21)
at com.sun.javafx.application.LauncherImpl$5.run(LauncherImpl.java:319)
at com.sun.javafx.application.PlatformImpl$5.run(PlatformImpl.java:206)
at com.sun.javafx.application.PlatformImpl$4.run(PlatformImpl.java:173)
at com.sun.glass.ui.win.WinApplication._runLoop(Native Method)
at com.sun.glass.ui.win.WinApplication.access$100(WinApplication.java:29)
at com.sun.glass.ui.win.WinApplication$3$1.run(WinApplication.java:73)
... 1 more
Caused by: java.lang.ClassNotFoundException: co.za.chrispie.LoginController
at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:423)
at java.lang.ClassLoader.loadClass(ClassLoader.java:356)
at javafx.fxml.FXMLLoader$ValueElement.processAttribute(FXMLLoader.java:726)
... 19 more
Judging by java.lang.ClassNotFoundException: co.za.chrispie.LoginController you missed few code or dependent library while copying.
Find java or jar file responsible for class co.za.chrispie.LoginController and make sure:
it exists on your laptop
it's in the correct location package-wise
it's included into your new project (sources folder)