Kafka Connect Debezium error while filtering - apache-kafka
For config:
curl -X POST "${KAFKA_CONNECT_HOST}/connectors" -H "Content-Type: application/json" -d '{
"name": "DebeziumSMS",
"config": {
"connector.class": "io.debezium.connector.sqlserver.SqlServerConnector",
"tasks.max": 1,
"database.hostname": "bill-srv02.corp.oblakowifi.ru",
"database.port": 1433,
"database.user": "sa",
"database.password": "******",
"database.dbname": "sms",
"database.server.name": "server-test",
"database.history.kafka.bootstrap.servers": "192.168.26.142:9092",
"database.history.kafka.topic": "schema-changes.sms",
"errors.log.enable": "true",
"database.history.skip.unparseable.ddl": "true",
"time.precision.mode": "connect",
"transforms": "filter",
"transforms.filter.type": "io.debezium.transforms.Filter",
"transforms.filter.language": "jsr223.groovy",
"transforms.filter.condition": "value.op == ''c''"
}
}'
this error:
[2021-10-19 12:50:46,234] ERROR Error encountered in task DebeziumSMS-0. Executing stage 'TRANSFORMATION' with class 'io.debezium.transforms.Filter'. (org.apache.kafka.connect.runtime.errors.LogReporter:66)
io.debezium.DebeziumException: Error while evaluating expression 'value.op == 'c'' for record 'SourceRecord{sourcePartition={server=server-test}, sourceOffset={transaction_id=null, event_serial_no=1, commit_lsn=00000064:0000420c:0003, change_lsn=00000064:0000420c:0002}} ConnectRecord{topic='server-test', kafkaPartition=0, key=Struct{databaseName=sms}, keySchema=Schema{io.debezium.connector.sqlserver.SchemaChangeKey:STRUCT}, value=Struct{source=Struct{version=1.5.0.Final,connector=sqlserver,name=server-test,ts_ms=1634637045699,db=,schema=,table=,change_lsn=00000064:0000420c:0002,commit_lsn=00000064:0000420c:0003},databaseName=sms,schemaName=dbo,ddl=N/A,tableChanges=[Struct{type=ALTER,id="sms"."dbo"."sms",table=Struct{primaryKeyColumnNames=[id],columns=[Struct{name=id,jdbcType=1,typeName=uniqueidentifier,typeExpression=uniqueidentifier,length=36,position=1,optional=false,autoIncremented=false,generated=false}, Struct{name=creator_login,jdbcType=12,typeName=varchar,typeExpression=varchar,length=64,position=2,optional=false,autoIncremented=false,generated=false}, Struct{name=system_ip_address,jdbcType=12,typeName=varchar,typeExpression=varchar,length=128,position=3,optional=false,autoIncremented=false,generated=false}, Struct{name=system_id,jdbcType=4,typeName=int,typeExpression=int,length=10,scale=0,position=4,optional=false,autoIncremented=false,generated=false}, Struct{name=system_sub_id,jdbcType=12,typeName=varchar,typeExpression=varchar,length=128,position=5,optional=false,autoIncremented=false,generated=false}, Struct{name=type_id,jdbcType=12,typeName=varchar,typeExpression=varchar,length=128,position=6,optional=true,autoIncremented=false,generated=false}, Struct{name=date_create,jdbcType=93,typeName=datetime,typeExpression=datetime,length=23,scale=3,position=7,optional=false,autoIncremented=false,generated=false}, Struct{name=date_send,jdbcType=93,typeName=datetime,typeExpression=datetime,length=23,scale=3,position=8,optional=false,autoIncremented=false,generated=false}, Struct{name=date_state,jdbcType=93,typeName=datetime,typeExpression=datetime,length=23,scale=3,position=9,optional=true,autoIncremented=false,generated=false}, Struct{name=time_to_live,jdbcType=4,typeName=int,typeExpression=int,length=10,scale=0,position=10,optional=false,autoIncremented=false,generated=false}, Struct{name=state_id,jdbcType=4,typeName=int,typeExpression=int,length=10,scale=0,position=11,optional=false,autoIncremented=false,generated=false}, Struct{name=phone,jdbcType=12,typeName=varchar,typeExpression=varchar,length=32,position=12,optional=false,autoIncremented=false,generated=false}, Struct{name=target_id,jdbcType=12,typeName=varchar,typeExpression=varchar,length=128,position=13,optional=true,autoIncremented=false,generated=false}, Struct{name=provider_id,jdbcType=4,typeName=int,typeExpression=int,length=10,scale=0,position=14,optional=false,autoIncremented=false,generated=false}, Struct{name=message,jdbcType=12,typeName=varchar,typeExpression=varchar,length=2147483647,position=15,optional=false,autoIncremented=false,generated=false}, Struct{name=linked_file,jdbcType=12,typeName=varchar,typeExpression=varchar,length=4000,position=16,optional=true,autoIncremented=false,generated=false}, Struct{name=error_id,jdbcType=4,typeName=int,typeExpression=int,length=10,scale=0,position=17,optional=true,autoIncremented=false,generated=false}, Struct{name=error_msg,jdbcType=12,typeName=varchar,typeExpression=varchar,length=4000,position=18,optional=true,autoIncremented=false,generated=false}]}}]}, valueSchema=Schema{io.debezium.connector.sqlserver.SchemaChangeValue:STRUCT}, timestamp=null, headers=ConnectHeaders(headers=)}'
at io.debezium.transforms.scripting.Jsr223Engine.eval(Jsr223Engine.java:116)
at io.debezium.transforms.Filter.doApply(Filter.java:33)
at io.debezium.transforms.ScriptingTransformation.apply(ScriptingTransformation.java:189)
at org.apache.kafka.connect.runtime.TransformationChain.lambda$apply$0(TransformationChain.java:50)
at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndRetry(RetryWithToleranceOperator.java:156)
at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndHandleError(RetryWithToleranceOperator.java:190)
at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execute(RetryWithToleranceOperator.java:132)
at org.apache.kafka.connect.runtime.TransformationChain.apply(TransformationChain.java:50)
at org.apache.kafka.connect.runtime.WorkerSourceTask.sendRecords(WorkerSourceTask.java:323)
at org.apache.kafka.connect.runtime.WorkerSourceTask.execute(WorkerSourceTask.java:248)
at org.apache.kafka.connect.runtime.WorkerTask.doRun(WorkerTask.java:182)
at org.apache.kafka.connect.runtime.WorkerTask.run(WorkerTask.java:231)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: javax.script.ScriptException: org.apache.kafka.connect.errors.DataException: op is not a valid field name
at org.codehaus.groovy.jsr223.GroovyScriptEngineImpl.eval(GroovyScriptEngineImpl.java:320)
at org.codehaus.groovy.jsr223.GroovyCompiledScript.eval(GroovyCompiledScript.java:71)
at javax.script.CompiledScript.eval(CompiledScript.java:92)
at io.debezium.transforms.scripting.Jsr223Engine.eval(Jsr223Engine.java:107)
... 16 more
Caused by: org.apache.kafka.connect.errors.DataException: op is not a valid field name
at org.apache.kafka.connect.data.Struct.lookupField(Struct.java:254)
at org.apache.kafka.connect.data.Struct.get(Struct.java:74)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:107)
at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:323)
at org.codehaus.groovy.runtime.metaclass.MethodMetaProperty$GetMethodMetaProperty.getProperty(MethodMetaProperty.java:62)
at org.codehaus.groovy.runtime.callsite.GetEffectivePojoPropertySite.getProperty(GetEffectivePojoPropertySite.java:63)
at org.codehaus.groovy.runtime.callsite.AbstractCallSite.callGetProperty(AbstractCallSite.java:329)
at Script1.run(Script1.groovy:1)
at org.codehaus.groovy.jsr223.GroovyScriptEngineImpl.eval(GroovyScriptEngineImpl.java:317)
... 19 more
[2021-10-19 12:50:46,237] INFO WorkerSourceTask{id=DebeziumSMS-0} flushing 0 outstanding messages for offset commit (org.apache.kafka.connect.runtime.WorkerSourceTask:487)
[2021-10-19 12:50:46,237] ERROR WorkerSourceTask{id=DebeziumSMS-0} Task threw an uncaught and unrecoverable exception. Task is being killed and will not recover until manually restarted (org.apache.kafka.connect.runtime.WorkerTask:184)
org.apache.kafka.connect.errors.ConnectException: Tolerance exceeded in error handler
at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndHandleError(RetryWithToleranceOperator.java:206)
at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execute(RetryWithToleranceOperator.java:132)
at org.apache.kafka.connect.runtime.TransformationChain.apply(TransformationChain.java:50)
at org.apache.kafka.connect.runtime.WorkerSourceTask.sendRecords(WorkerSourceTask.java:323)
at org.apache.kafka.connect.runtime.WorkerSourceTask.execute(WorkerSourceTask.java:248)
at org.apache.kafka.connect.runtime.WorkerTask.doRun(WorkerTask.java:182)
at org.apache.kafka.connect.runtime.WorkerTask.run(WorkerTask.java:231)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: io.debezium.DebeziumException: Error while evaluating expression 'value.op == 'c'' for record 'SourceRecord{sourcePartition={server=server-test}, sourceOffset={transaction_id=null, event_serial_no=1, commit_lsn=00000064:0000420c:0003, change_lsn=00000064:0000420c:0002}} ConnectRecord{topic='server-test', kafkaPartition=0, key=Struct{databaseName=sms}, keySchema=Schema{io.debezium.connector.sqlserver.SchemaChangeKey:STRUCT}, value=Struct{source=Struct{version=1.5.0.Final,connector=sqlserver,name=server-test,ts_ms=1634637045699,db=,schema=,table=,change_lsn=00000064:0000420c:0002,commit_lsn=00000064:0000420c:0003},databaseName=sms,schemaName=dbo,ddl=N/A,tableChanges=[Struct{type=ALTER,id="sms"."dbo"."sms",table=Struct{primaryKeyColumnNames=[id],columns=[Struct{name=id,jdbcType=1,typeName=uniqueidentifier,typeExpression=uniqueidentifier,length=36,position=1,optional=false,autoIncremented=false,generated=false}, Struct{name=creator_login,jdbcType=12,typeName=varchar,typeExpression=varchar,length=64,position=2,optional=false,autoIncremented=false,generated=false}, Struct{name=system_ip_address,jdbcType=12,typeName=varchar,typeExpression=varchar,length=128,position=3,optional=false,autoIncremented=false,generated=false}, Struct{name=system_id,jdbcType=4,typeName=int,typeExpression=int,length=10,scale=0,position=4,optional=false,autoIncremented=false,generated=false}, Struct{name=system_sub_id,jdbcType=12,typeName=varchar,typeExpression=varchar,length=128,position=5,optional=false,autoIncremented=false,generated=false}, Struct{name=type_id,jdbcType=12,typeName=varchar,typeExpression=varchar,length=128,position=6,optional=true,autoIncremented=false,generated=false}, Struct{name=date_create,jdbcType=93,typeName=datetime,typeExpression=datetime,length=23,scale=3,position=7,optional=false,autoIncremented=false,generated=false}, Struct{name=date_send,jdbcType=93,typeName=datetime,typeExpression=datetime,length=23,scale=3,position=8,optional=false,autoIncremented=false,generated=false}, Struct{name=date_state,jdbcType=93,typeName=datetime,typeExpression=datetime,length=23,scale=3,position=9,optional=true,autoIncremented=false,generated=false}, Struct{name=time_to_live,jdbcType=4,typeName=int,typeExpression=int,length=10,scale=0,position=10,optional=false,autoIncremented=false,generated=false}, Struct{name=state_id,jdbcType=4,typeName=int,typeExpression=int,length=10,scale=0,position=11,optional=false,autoIncremented=false,generated=false}, Struct{name=phone,jdbcType=12,typeName=varchar,typeExpression=varchar,length=32,position=12,optional=false,autoIncremented=false,generated=false}, Struct{name=target_id,jdbcType=12,typeName=varchar,typeExpression=varchar,length=128,position=13,optional=true,autoIncremented=false,generated=false}, Struct{name=provider_id,jdbcType=4,typeName=int,typeExpression=int,length=10,scale=0,position=14,optional=false,autoIncremented=false,generated=false}, Struct{name=message,jdbcType=12,typeName=varchar,typeExpression=varchar,length=2147483647,position=15,optional=false,autoIncremented=false,generated=false}, Struct{name=linked_file,jdbcType=12,typeName=varchar,typeExpression=varchar,length=4000,position=16,optional=true,autoIncremented=false,generated=false}, Struct{name=error_id,jdbcType=4,typeName=int,typeExpression=int,length=10,scale=0,position=17,optional=true,autoIncremented=false,generated=false}, Struct{name=error_msg,jdbcType=12,typeName=varchar,typeExpression=varchar,length=4000,position=18,optional=true,autoIncremented=false,generated=false}]}}]}, valueSchema=Schema{io.debezium.connector.sqlserver.SchemaChangeValue:STRUCT}, timestamp=null, headers=ConnectHeaders(headers=)}'
at io.debezium.transforms.scripting.Jsr223Engine.eval(Jsr223Engine.java:116)
at io.debezium.transforms.Filter.doApply(Filter.java:33)
at io.debezium.transforms.ScriptingTransformation.apply(ScriptingTransformation.java:189)
at org.apache.kafka.connect.runtime.TransformationChain.lambda$apply$0(TransformationChain.java:50)
at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndRetry(RetryWithToleranceOperator.java:156)
at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndHandleError(RetryWithToleranceOperator.java:190)
... 11 more
Caused by: javax.script.ScriptException: org.apache.kafka.connect.errors.DataException: op is not a valid field name
at org.codehaus.groovy.jsr223.GroovyScriptEngineImpl.eval(GroovyScriptEngineImpl.java:320)
at org.codehaus.groovy.jsr223.GroovyCompiledScript.eval(GroovyCompiledScript.java:71)
at javax.script.CompiledScript.eval(CompiledScript.java:92)
at io.debezium.transforms.scripting.Jsr223Engine.eval(Jsr223Engine.java:107)
... 16 more
Caused by: org.apache.kafka.connect.errors.DataException: op is not a valid field name
at org.apache.kafka.connect.data.Struct.lookupField(Struct.java:254)
at org.apache.kafka.connect.data.Struct.get(Struct.java:74)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:107)
at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:323)
at org.codehaus.groovy.runtime.metaclass.MethodMetaProperty$GetMethodMetaProperty.getProperty(MethodMetaProperty.java:62)
at org.codehaus.groovy.runtime.callsite.GetEffectivePojoPropertySite.getProperty(GetEffectivePojoPropertySite.java:63)
at org.codehaus.groovy.runtime.callsite.AbstractCallSite.callGetProperty(AbstractCallSite.java:329)
at Script1.run(Script1.groovy:1)
at org.codehaus.groovy.jsr223.GroovyScriptEngineImpl.eval(GroovyScriptEngineImpl.java:317)
... 19 more
[2021-10-19 12:50:46,237] INFO Stopping down connector (io.debezium.connector.common.BaseSourceTask:238)
After recreating Debezium Connector by DELETE and next POST, Debezium works fine a few days. But then error repeats, and connector stopps with this error.
And after reboot Kafka Connect (distributed) service this error again.
Trace for connector error by REST :
"trace":"org.apache.kafka.connect.errors.ConnectException: Tolerance exceeded in error handler\n\tat org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndHandleError(RetryWithToleranceOperator.java:206)\n\tat org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execute(RetryWithToleranceOperator.java:132)\n\tat org.apache.kafka.connect.runtime.TransformationChain.apply(TransformationChain.java:50)\n\tat org.apache.kafka.connect.runtime.WorkerSourceTask.sendRecords(WorkerSourceTask.java:323)\n\tat org.apache.kafka.connect.runtime.WorkerSourceTask.execute(WorkerSourceTask.java:248)\n\tat org.apache.kafka.connect.runtime.WorkerTask.doRun(WorkerTask.java:182)\n\tat org.apache.kafka.connect.runtime.WorkerTask.run(WorkerTask.java:231)\n\tat java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:266)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:748)\nCaused by: io.debezium.DebeziumException: Error while evaluating expression 'value && value.op == 'c'' for record 'SourceRecord{sourcePartition={server=server-test}, sourceOffset={transaction_id=null, event_serial_no=1, commit_lsn=00000064:0000420c:0003, change_lsn=00000064:0000420c:0002}} ConnectRecord{topic='server-test', kafkaPartition=0, key=Struct{databaseName=sms}, keySchema=Schema{io.debezium.connector.sqlserver.SchemaChangeKey:STRUCT}, value=Struct{source=Struct{version=1.5.0.Final,connector=sqlserver,name=server-test,ts_ms=1634805618423,db=,schema=,table=,change_lsn=00000064:0000420c:0002,commit_lsn=00000064:0000420c:0003},databaseName=sms,schemaName=dbo,ddl=N/A,tableChanges=[Struct{type=ALTER,id=\"sms\".\"dbo\".\"sms\",table=Struct{primaryKeyColumnNames=[id],columns=[Struct{name=id,jdbcType=1,typeName=uniqueidentifier,typeExpression=uniqueidentifier,length=36,position=1,optional=false,autoIncremented=false,generated=false}, Struct{name=creator_login,jdbcType=12,typeName=varchar,typeExpression=varchar,length=64,position=2,optional=false,autoIncremented=false,generated=false}, Struct{name=system_ip_address,jdbcType=12,typeName=varchar,typeExpression=varchar,length=128,position=3,optional=false,autoIncremented=false,generated=false}, Struct{name=system_id,jdbcType=4,typeName=int,typeExpression=int,length=10,scale=0,position=4,optional=false,autoIncremented=false,generated=false}, Struct{name=system_sub_id,jdbcType=12,typeName=varchar,typeExpression=varchar,length=128,position=5,optional=false,autoIncremented=false,generated=false}, Struct{name=type_id,jdbcType=12,typeName=varchar,typeExpression=varchar,length=128,position=6,optional=true,autoIncremented=false,generated=false}, Struct{name=date_create,jdbcType=93,typeName=datetime,typeExpression=datetime,length=23,scale=3,position=7,optional=false,autoIncremented=false,generated=false}, Struct{name=date_send,jdbcType=93,typeName=datetime,typeExpression=datetime,length=23,scale=3,position=8,optional=false,autoIncremented=false,generated=false}, Struct{name=date_state,jdbcType=93,typeName=datetime,typeExpression=datetime,length=23,scale=3,position=9,optional=true,autoIncremented=false,generated=false}, Struct{name=time_to_live,jdbcType=4,typeName=int,typeExpression=int,length=10,scale=0,position=10,optional=false,autoIncremented=false,generated=false}, Struct{name=state_id,jdbcType=4,typeName=int,typeExpression=int,length=10,scale=0,position=11,optional=false,autoIncremented=false,generated=false}, Struct{name=phone,jdbcType=12,typeName=varchar,typeExpression=varchar,length=32,position=12,optional=false,autoIncremented=false,generated=false}, Struct{name=target_id,jdbcType=12,typeName=varchar,typeExpression=varchar,length=128,position=13,optional=true,autoIncremented=false,generated=false}, Struct{name=provider_id,jdbcType=4,typeName=int,typeExpression=int,length=10,scale=0,position=14,optional=false,autoIncremented=false,generated=false}, Struct{name=message,jdbcType=12,typeName=varchar,typeExpression=varchar,length=2147483647,position=15,optional=false,autoIncremented=false,generated=false}, Struct{name=linked_file,jdbcType=12,typeName=varchar,typeExpression=varchar,length=4000,position=16,optional=true,autoIncremented=false,generated=false}, Struct{name=error_id,jdbcType=4,typeName=int,typeExpression=int,length=10,scale=0,position=17,optional=true,autoIncremented=false,generated=false}, Struct{name=error_msg,jdbcType=12,typeName=varchar,typeExpression=varchar,length=4000,position=18,optional=true,autoIncremented=false,generated=false}]}}]}, valueSchema=Schema{io.debezium.connector.sqlserver.SchemaChangeValue:STRUCT}, timestamp=null, headers=ConnectHeaders(headers=)}'\n\tat io.debezium.transforms.scripting.Jsr223Engine.eval(Jsr223Engine.java:116)\n\tat io.debezium.transforms.Filter.doApply(Filter.java:33)\n\tat io.debezium.transforms.ScriptingTransformation.apply(ScriptingTransformation.java:189)\n\tat org.apache.kafka.connect.runtime.TransformationChain.lambda$apply$0(TransformationChain.java:50)\n\tat org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndRetry(RetryWithToleranceOperator.java:156)\n\tat org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndHandleError(RetryWithToleranceOperator.java:190)\n\t... 11 more\nCaused by: javax.script.ScriptException: org.apache.kafka.connect.errors.DataException: op is not a valid field name\n\tat org.codehaus.groovy.jsr223.GroovyScriptEngineImpl.eval(GroovyScriptEngineImpl.java:320)\n\tat org.codehaus.groovy.jsr223.GroovyCompiledScript.eval(GroovyCompiledScript.java:71)\n\tat javax.script.CompiledScript.eval(CompiledScript.java:92)\n\tat io.debezium.transforms.scripting.Jsr223Engine.eval(Jsr223Engine.java:107)\n\t... 16 more\nCaused by: org.apache.kafka.connect.errors.DataException: op is not a valid field name\n\tat org.apache.kafka.connect.data.Struct.lookupField(Struct.java:254)\n\tat org.apache.kafka.connect.data.Struct.get(Struct.java:74)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:498)\n\tat org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:107)\n\tat groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:323)\n\tat org.codehaus.groovy.runtime.metaclass.MethodMetaProperty$GetMethodMetaProperty.getProperty(MethodMetaProperty.java:62)\n\tat org.codehaus.groovy.runtime.callsite.GetEffectivePojoPropertySite.getProperty(GetEffectivePojoPropertySite.java:63)\n\tat org.codehaus.groovy.runtime.callsite.AbstractCallSite.callGetProperty(AbstractCallSite.java:329)\n\tat Script3.run(Script3.groovy:1)\n\tat org.codehaus.groovy.jsr223.GroovyScriptEngineImpl.eval(GroovyScriptEngineImpl.java:317)\n\t... 19 more\n"}
Caused by: org.apache.kafka.connect.errors.DataException: op is not a valid field name
Why?
What can be wrong ? Thank you.
Related
Debezium: RestClientException: Leader not known; error code: 50004
I am getting exceptions when i try to deploy debezium connector to kafka-connect. As a result, snapshots are not created and also cdc streaming is blocked. The problem is I am not able to find where the issue is i.e. is it in kafka-connect, schema-registry or debezium connector { "name": "my-connector", "config": { "connector.class": "io.debezium.connector.mysql.MySqlConnector", "database.hostname": "root", "database.port": "3306", "database.user": "USER", "database.password": "PASSWORD", "database.server.id": "184056", "database.server.name": "dbz", "table.include.list": "T1,T2", "snapshot.mode": "initial", "database.history.kafka.bootstrap.servers": "xxx:9092", "database.history.kafka.topic": "dbz.myhistory", "plugin.path": "/usr/share/java/debezium-mysql-connect/", "database.serverTimezone": "America/Los_Angeles", "snapshot.locking.mode": "none", "include.query": "true", "key.converter": "io.confluent.connect.avro.AvroConverter", "key.converter.schema.registry.url": "XXX", "value.converter": "io.confluent.connect.avro.AvroConverter", "value.converter.schema.registry.url": "XXX" } } kafka-connect exception logs: org.apache.kafka.connect.errors.ConnectException: Tolerance exceeded in error handler at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndHandleError(RetryWithToleranceOperator.java:178) at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execute(RetryWithToleranceOperator.java:104) at org.apache.kafka.connect.runtime.WorkerSourceTask.convertTransformedRecord(WorkerSourceTask.java:294) at org.apache.kafka.connect.runtime.WorkerSourceTask.sendRecords(WorkerSourceTask.java:323) at org.apache.kafka.connect.runtime.WorkerSourceTask.execute(WorkerSourceTask.java:247) at org.apache.kafka.connect.runtime.WorkerTask.doRun(WorkerTask.java:184) at org.apache.kafka.connect.runtime.WorkerTask.run(WorkerTask.java:234) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.kafka.connect.errors.DataException: Failed to serialize Avro data from topic dbz : at io.confluent.connect.avro.AvroConverter.fromConnectData(AvroConverter.java:91) at org.apache.kafka.connect.storage.Converter.fromConnectData(Converter.java:63) at org.apache.kafka.connect.runtime.WorkerSourceTask.lambda$convertTransformedRecord$1(WorkerSourceTask.java:294) at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndRetry(RetryWithToleranceOperator.java:128) at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndHandleError(RetryWithToleranceOperator.java:162) ... 11 more Caused by: org.apache.kafka.common.errors.SerializationException: Error registering Avro schema: {"type":"record","name":"SchemaChangeKey","namespace":"io.debezium.connector.mysql","fields":[{"name":"databaseName","type":"string"}],"connect.name":"io.debezium.connector.mysql.SchemaChangeKey"} Caused by: io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException: Leader not known.; error code: 50004 at io.confluent.kafka.schemaregistry.client.rest.RestService.sendHttpRequest(RestService.java:292) at io.confluent.kafka.schemaregistry.client.rest.RestService.httpRequest(RestService.java:351) at io.confluent.kafka.schemaregistry.client.rest.RestService.registerSchema(RestService.java:494) at io.confluent.kafka.schemaregistry.client.rest.RestService.registerSchema(RestService.java:485) at io.confluent.kafka.schemaregistry.client.rest.RestService.registerSchema(RestService.java:458) at io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient.registerAndGetId(CachedSchemaRegistryClient.java:206) at io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient.register(CachedSchemaRegistryClient.java:268) at io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient.register(CachedSchemaRegistryClient.java:244) at io.confluent.kafka.serializers.AbstractKafkaAvroSerializer.serializeImpl(AbstractKafkaAvroSerializer.java:74) at io.confluent.connect.avro.AvroConverter$Serializer.serialize(AvroConverter.java:143) at io.confluent.connect.avro.AvroConverter.fromConnectData(AvroConverter.java:84) at org.apache.kafka.connect.storage.Converter.fromConnectData(Converter.java:63) at org.apache.kafka.connect.runtime.WorkerSourceTask.lambda$convertTransformedRecord$1(WorkerSourceTask.java:294) at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndRetry(RetryWithToleranceOperator.java:128) at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndHandleError(RetryWithToleranceOperator.java:162) at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execute(RetryWithToleranceOperator.java:104) at org.apache.kafka.connect.runtime.WorkerSourceTask.convertTransformedRecord(WorkerSourceTask.java:294) at org.apache.kafka.connect.runtime.WorkerSourceTask.sendRecords(WorkerSourceTask.java:323) at org.apache.kafka.connect.runtime.WorkerSourceTask.execute(WorkerSourceTask.java:247) at org.apache.kafka.connect.runtime.WorkerTask.doRun(WorkerTask.java:184) at org.apache.kafka.connect.runtime.WorkerTask.run(WorkerTask.java:234) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) [2021-09-20 05:29:23,849] ERROR WorkerSourceTask{id=my-connector-0} Task is being killed and will not recover until manually restarted (org.apache.kafka.connect.runtime.WorkerTask:187) [2021-09-20 05:29:23,849] INFO Stopping down connector (io.debezium.connector.common.BaseSourceTask:241) [2021-09-20 05:29:23,853] WARN Snapshot was interrupted before completion (io.debezium.pipeline.source.AbstractSnapshotChangeEventSource:74) [2021-09-20 05:29:23,853] INFO Snapshot - Final stage (io.debezium.pipeline.source.AbstractSnapshotChangeEventSource:82) [2021-09-20 05:29:23,854] WARN Change event source executor was interrupted (io.debezium.pipeline.ChangeEventSourceCoordinator:132) java.lang.InterruptedException: Interrupted while processing event SchemaChangeEvent [database=mydb, schema=null, ddl=DROP TABLE IF EXISTS `db`.`t3`, tables=[], type=DROP] at io.debezium.connector.mysql.MySqlSnapshotChangeEventSource.createSchemaChangeEventsForTables(MySqlSnapshotChangeEventSource.java:539) at io.debezium.relational.RelationalSnapshotChangeEventSource.doExecute(RelationalSnapshotChangeEventSource.java:127) at io.debezium.pipeline.source.AbstractSnapshotChangeEventSource.execute(AbstractSnapshotChangeEventSource.java:70) at io.debezium.pipeline.ChangeEventSourceCoordinator.lambda$start$0(ChangeEventSourceCoordinator.java:118) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Schema-Registry Exception error logs: io.confluent.kafka.schemaregistry.rest.exceptions.RestUnknownLeaderException: Leader not known. at io.confluent.kafka.schemaregistry.rest.exceptions.Errors.unknownLeaderException(Errors.java:153) at io.confluent.kafka.schemaregistry.rest.resources.SubjectVersionsResource.register(SubjectVersionsResource.java:286) at jdk.internal.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:566) at org.glassfish.jersey.server.model.internal.ResourceMethodInvocationHandlerFactory.lambda$static$0(ResourceMethodInvocationHandlerFactory.java:52) at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher$1.run(AbstractJavaResourceMethodDispatcher.java:124) at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.invoke(AbstractJavaResourceMethodDispatcher.java:167) at org.glassfish.jersey.server.model.internal.JavaResourceMethodDispatcherProvider$VoidOutInvoker.doDispatch(JavaResourceMethodDispatcherProvider.java:159) at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.dispatch(AbstractJavaResourceMethodDispatcher.java:79) at org.glassfish.jersey.server.model.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:475) at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:397) at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:81) at org.glassfish.jersey.server.ServerRuntime$1.run(ServerRuntime.java:255) at org.glassfish.jersey.internal.Errors$1.call(Errors.java:248) at org.glassfish.jersey.internal.Errors$1.call(Errors.java:244) at org.glassfish.jersey.internal.Errors.process(Errors.java:292) at org.glassfish.jersey.internal.Errors.process(Errors.java:274) at org.glassfish.jersey.internal.Errors.process(Errors.java:244) at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:265) at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:234) at org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:680) at org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.java:394) at org.glassfish.jersey.servlet.ServletContainer.serviceImpl(ServletContainer.java:386) at org.glassfish.jersey.servlet.ServletContainer.doFilter(ServletContainer.java:561) at org.glassfish.jersey.servlet.ServletContainer.doFilter(ServletContainer.java:502) at org.glassfish.jersey.servlet.ServletContainer.doFilter(ServletContainer.java:439) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1435) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1350) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:146) at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:146) at org.eclipse.jetty.server.handler.StatisticsHandler.handle(StatisticsHandler.java:179) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:234) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:763) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) at org.eclipse.jetty.server.Server.handle(Server.java:516) at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:388) at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:633) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:380) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105) at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:336) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:313) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:129) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:383) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base/java.lang.Thread.run(Thread.java:829) Caused by: io.confluent.kafka.schemaregistry.exceptions.UnknownLeaderException: Register schema request failed since leader is unknown at io.confluent.kafka.schemaregistry.storage.KafkaSchemaRegistry.registerOrForward(KafkaSchemaRegistry.java:594) at io.confluent.kafka.schemaregistry.rest.resources.SubjectVersionsResource.register(SubjectVersionsResource.java:266) ... 60 more Can someone help me what this exception means and how to resolve it?
How to resolve JsonParseException for InfluxDB sink connector
Here is json data in test topic [kafka] This is a connector property curl -i -X PUT -H "Content-Type:application/json" \ http://localhost:8089/connectors/influxdb-sink-test-20210615/config \ -d '{ "connector.class" : "io.confluent.influxdb.InfluxDBSinkConnector", "value.converter" : "org.apache.kafka.connect.json.JsonConverter", "value.converter.schemas.enable": "false", "key.converter" : "org.apache.kafka.connect.storage.StringConverter", "topics" : "test3", "influxdb.url" : "http://localhost:8086", "influxdb.db" : "exemDB", "measurement.name.format" : "${topic}" }' An exception is raised in the Kafka connector because the tags field is not converted properly. How can I properly convert the tags field? Removing the tags field and producing data on the topic normally inserts data into the influxDB. This is Exception.. If i remove the field tag and produce it, it works well on influxdb. How can I fill in the tags field? [2021-06-17 09:24:59,521] ERROR WorkerSinkTask{id=influxdb-sink-tcore-test-20210615-0} Task threw an uncaught and unrecoverable exception (org.apache.kafka.connect.runtime.WorkerTask:177) org.apache.kafka.connect.errors.ConnectException: Tolerance exceeded in error handler at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndHandleError(RetryWithToleranceOperator.java:178) at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execute(RetryWithToleranceOperator.java:104) at org.apache.kafka.connect.runtime.WorkerSinkTask.convertAndTransformRecord(WorkerSinkTask.java:513) at org.apache.kafka.connect.runtime.WorkerSinkTask.convertMessages(WorkerSinkTask.java:490) at org.apache.kafka.connect.runtime.WorkerSinkTask.poll(WorkerSinkTask.java:321) at org.apache.kafka.connect.runtime.WorkerSinkTask.iteration(WorkerSinkTask.java:225) at org.apache.kafka.connect.runtime.WorkerSinkTask.execute(WorkerSinkTask.java:193) at org.apache.kafka.connect.runtime.WorkerTask.doRun(WorkerTask.java:175) at org.apache.kafka.connect.runtime.WorkerTask.run(WorkerTask.java:219) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.kafka.connect.errors.DataException: Converting byte[] to Kafka Connect data failed due to serialization error: at org.apache.kafka.connect.json.JsonConverter.toConnectData(JsonConverter.java:334) at org.apache.kafka.connect.runtime.WorkerSinkTask.lambda$convertAndTransformRecord$1(WorkerSinkTask.java:513) at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndRetry(RetryWithToleranceOperator.java:128) at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndHandleError(RetryWithToleranceOperator.java:162) ... 13 more Caused by: org.apache.kafka.common.errors.SerializationException: com.fasterxml.jackson.core.JsonParseException: Unexpected character ('t' (code 116)): was expecting comma to separate Object entries at [Source: (byte[])"{"dataType":"numeric","parentResourceId":0,"resourceId":1037092,"resourceUUID":"1037092","resourceName":"tcore-master-01","resourceTypeId":5,"resourceTypeName":"물리서버","resourceLocation":"미지정","ip":"localhost","tags":"[{"tagId":0,"tagValueId":0,"tag":"미지정","value":"미지정"},{"tagId":7,"tagValueId":2103044,"tag":"물리서버","value":"tcore-master-01"},{"tagId":52,"tagValueId":2103045,"tag":"미지정","value":"tcore-master01"}]","metricSeq":930,"metricName":"command_"[truncated 279 bytes]; line: 1, column: 242] Caused by: com.fasterxml.jackson.core.JsonParseException: Unexpected character ('t' (code 116)): was expecting comma to separate Object entries at [Source: (byte[])"{"dataType":"numeric","parentResourceId":0,"resourceId":1037092,"resourceUUID":"1037092","resourceName":"tcore-master-01","resourceTypeId":5,"resourceTypeName":"물리서버","resourceLocation":"미지정","ip":"localhost","tags":"[{"tagId":0,"tagValueId":0,"tag":"미지정","value":"미지정"},{"tagId":7,"tagValueId":2103044,"tag":"물리서버","value":"tcore-master-01"},{"tagId":52,"tagValueId":2103045,"tag":"미지정","value":"tcore-master01"}]","metricSeq":930,"metricName":"command_"[truncated 279 bytes]; line: 1, column: 242] at com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:1804) at com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:669) at com.fasterxml.jackson.core.base.ParserMinimalBase._reportUnexpectedChar(ParserMinimalBase.java:567) at com.fasterxml.jackson.core.json.UTF8StreamJsonParser.nextFieldName(UTF8StreamJsonParser.java:980) at com.fasterxml.jackson.databind.deser.std.BaseNodeDeserializer.deserializeObject(JsonNodeDeserializer.java:247) at com.fasterxml.jackson.databind.deser.std.JsonNodeDeserializer.deserialize(JsonNodeDeserializer.java:68) at com.fasterxml.jackson.databind.deser.std.JsonNodeDeserializer.deserialize(JsonNodeDeserializer.java:15) at com.fasterxml.jackson.databind.ObjectMapper._readTreeAndClose(ObjectMapper.java:4056) at com.fasterxml.jackson.databind.ObjectMapper.readTree(ObjectMapper.java:2571) at org.apache.kafka.connect.json.JsonDeserializer.deserialize(JsonDeserializer.java:50) at org.apache.kafka.connect.json.JsonConverter.toConnectData(JsonConverter.java:332) at org.apache.kafka.connect.runtime.WorkerSinkTask.lambda$convertAndTransformRecord$1(WorkerSinkTask.java:513) at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndRetry(RetryWithToleranceOperator.java:128) at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndHandleError(RetryWithToleranceOperator.java:162) at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execute(RetryWithToleranceOperator.java:104) at org.apache.kafka.connect.runtime.WorkerSinkTask.convertAndTransformRecord(WorkerSinkTask.java:513) at org.apache.kafka.connect.runtime.WorkerSinkTask.convertMessages(WorkerSinkTask.java:490) at org.apache.kafka.connect.runtime.WorkerSinkTask.poll(WorkerSinkTask.java:321) at org.apache.kafka.connect.runtime.WorkerSinkTask.iteration(WorkerSinkTask.java:225) at org.apache.kafka.connect.runtime.WorkerSinkTask.execute(WorkerSinkTask.java:193) at org.apache.kafka.connect.runtime.WorkerTask.doRun(WorkerTask.java:175) at org.apache.kafka.connect.runtime.WorkerTask.run(WorkerTask.java:219) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) [2021-06-17 09:24:59,522] ERROR WorkerSinkTask{id=influxdb-sink-tcore-test-20210615-0} Task is being killed and will not recover until manually restarted (org.apache.kafka.connect.runtime.WorkerTask:178) [2021-06-17 09:24:59,522] INFO Closing InfluxDB client. (io.confluent.influxdb.sink.writer.InfluxDBWriter:406)
Facing issues while using camel-sftp-kafka-connector in Apache kafka?
i am trying to create a sftp connector in Apache kafka using camel-sftp-kafka-connector. i just downloaded the sftp-connector jar and pasted into the kafka's lib folder. while hitting /connector-plugins endpoint i can able to see the sftp-connector is available so below is my sample connector configuration { "name": "camel-sftp-connector-1", "config": { "connector.class": "org.apache.camel.kafkaconnector.sftp.CamelSftpSourceConnector", "key.converter": "org.apache.kafka.connect.storage.StringConverter", "value.converter": "org.apache.kafka.connect.converters.ByteArrayConverter", "transforms": "RemoteTransformer", "transforms.RemoteTransformer.type": "org.apache.camel.kafkaconnector.RemoteFileTransforms", "topics": "came-sftp-test-1", "camel.source.path.host": "192.168.30.167", "camel.source.path.port": "66", "camel.source.path.directoryName": "/csv/sample", "camel.source.endpoint.recursive": "true", "camel.source.endpoint.username": "sftpuser", "camel.source.endpoint.password": "sftppass", "camel.source.endpoint.noop": "/false", "camel.source.endpoint.move": "/done" } } I got the below exception and dont know how to fix it. [2021-05-05 19:20:58,560] WARN /connectors (org.eclipse.jetty.server.HttpChannel) javax.servlet.ServletException: org.glassfish.jersey.server.ContainerException: java.lang.NoClassDefFoundError: org/jctools/queues/MessagePassingQueue$Supplier at org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.java:410) at org.glassfish.jersey.servlet.WebComponent.service(WebComponent.java:346) at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:366) at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:319) at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:205) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:763) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:563) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1612) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1582) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:234) at org.eclipse.jetty.server.handler.StatisticsHandler.handle(StatisticsHandler.java:179) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) at org.eclipse.jetty.server.Server.handle(Server.java:516) at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383) at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:556) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:273) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105) at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905) at java.base/java.lang.Thread.run(Thread.java:834) Caused by: org.glassfish.jersey.server.ContainerException: java.lang.NoClassDefFoundError: org/jctools/queues/MessagePassingQueue$Supplier at org.glassfish.jersey.servlet.internal.ResponseWriter.rethrow(ResponseWriter.java:254) at org.glassfish.jersey.servlet.internal.ResponseWriter.failure(ResponseWriter.java:236) at org.glassfish.jersey.server.ServerRuntime$Responder.process(ServerRuntime.java:436) at org.glassfish.jersey.server.ServerRuntime$1.run(ServerRuntime.java:261) at org.glassfish.jersey.internal.Errors$1.call(Errors.java:248) at org.glassfish.jersey.internal.Errors$1.call(Errors.java:244) at org.glassfish.jersey.internal.Errors.process(Errors.java:292) at org.glassfish.jersey.internal.Errors.process(Errors.java:274) at org.glassfish.jersey.internal.Errors.process(Errors.java:244) at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:265) at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:232) at org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:680) at org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.java:394) ... 30 more Caused by: java.lang.NoClassDefFoundError: org/jctools/queues/MessagePassingQueue$Supplier at org.apache.camel.kafkaconnector.CamelSourceConnectorConfig.<clinit>(CamelSourceConnectorConfig.java:43) at org.apache.camel.kafkaconnector.sftp.CamelSftpSourceConnector.config(CamelSftpSourceConnector.java:29) at org.apache.kafka.connect.runtime.AbstractHerder.validateConnectorConfig(AbstractHerder.java:366) at org.apache.kafka.connect.runtime.AbstractHerder.lambda$validateConnectorConfig$1(AbstractHerder.java:326) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ... 1 more Caused by: java.lang.ClassNotFoundException: org.jctools.queues.MessagePassingQueue$Supplier at java.base/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:581) at java.base/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:178) at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:521) ... 9 more [2021-05-05 19:20:58,562] WARN unhandled due to prior sendError (org.eclipse.jetty.server.HttpChannelState) javax.servlet.ServletException: org.glassfish.jersey.server.ContainerException: java.lang.NoClassDefFoundError: org/jctools/queues/MessagePassingQueue$Supplier at org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.java:410) at org.glassfish.jersey.servlet.WebComponent.service(WebComponent.java:346) at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:366) at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:319) at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:205) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:763) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:563) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1612) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1582) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:234) at org.eclipse.jetty.server.handler.StatisticsHandler.handle(StatisticsHandler.java:179) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) at org.eclipse.jetty.server.Server.handle(Server.java:516) at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383) at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:556) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:273) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105) at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905) at java.base/java.lang.Thread.run(Thread.java:834) Caused by: org.glassfish.jersey.server.ContainerException: java.lang.NoClassDefFoundError: org/jctools/queues/MessagePassingQueue$Supplier at org.glassfish.jersey.servlet.internal.ResponseWriter.rethrow(ResponseWriter.java:254) at org.glassfish.jersey.servlet.internal.ResponseWriter.failure(ResponseWriter.java:236) at org.glassfish.jersey.server.ServerRuntime$Responder.process(ServerRuntime.java:436) at org.glassfish.jersey.server.ServerRuntime$1.run(ServerRuntime.java:261) at org.glassfish.jersey.internal.Errors$1.call(Errors.java:248) at org.glassfish.jersey.internal.Errors$1.call(Errors.java:244) at org.glassfish.jersey.internal.Errors.process(Errors.java:292) at org.glassfish.jersey.internal.Errors.process(Errors.java:274) at org.glassfish.jersey.internal.Errors.process(Errors.java:244) at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:265) at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:232) at org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:680) at org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.java:394) ... 30 more Caused by: java.lang.NoClassDefFoundError: org/jctools/queues/MessagePassingQueue$Supplier at org.apache.camel.kafkaconnector.CamelSourceConnectorConfig.<clinit>(CamelSourceConnectorConfig.java:43) at org.apache.camel.kafkaconnector.sftp.CamelSftpSourceConnector.config(CamelSftpSourceConnector.java:29) at org.apache.kafka.connect.runtime.AbstractHerder.validateConnectorConfig(AbstractHerder.java:366) at org.apache.kafka.connect.runtime.AbstractHerder.lambda$validateConnectorConfig$1(AbstractHerder.java:326) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ... 1 more Caused by: java.lang.ClassNotFoundException: org.jctools.queues.MessagePassingQueue$Supplier at java.base/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:581) at java.base/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:178) at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:521) ... 9 more could anyone tell me what is the issue ? whether i am configuring in wrong way or else any extra jars need to be added?
I've not used Camel connectors, but based on the error, you'll need these jars for that specific package. There's no guarantee you'll be missing other classes needed for the connector https://github.com/JCTools/JCTools#get-it-now JARs pasted into the kafka's lib folder. Modifying the broker classpath is not the appropriate way to configure Connect's plugin.path setting. Connector plugin JARs should be in isolated directories per connector
Could not initialize class io.debezium.connector.oracle.OracleConnectorConfig
Set-up/Configuration In case, it's needed, it's at this SO question. Issue localhost:8083 is working fine as i am getting: {"version":"2.6.0","commit":"62abe01bee039651","kafka_cluster_id":"k6c8D2yvR5OcVFMVZayP9A"} But when i post a connector configuration to localhost:8083/connectors, i am getting a 500 Server Error. I am not posting the json body as it's not relevant. Error WARN /connectors (org.eclipse.jetty.server.HttpChannel) javax.servlet.ServletException: org.glassfish.jersey.server.ContainerException: java.lang.NoClassDefFoundError: io/debezium/DebeziumException at org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.java:408) at org.glassfish.jersey.servlet.WebComponent.service(WebComponent.java:346) at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:365) at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:318) at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:205) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:760) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:547) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1607) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1297) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1577) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1212) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:221) at org.eclipse.jetty.server.handler.StatisticsHandler.handle(StatisticsHandler.java:173) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) at org.eclipse.jetty.server.Server.handle(Server.java:500) at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383) at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:270) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at java.lang.Thread.run(Thread.java:748) Caused by: org.glassfish.jersey.server.ContainerException: java.lang.NoClassDefFoundError: io/debezium/DebeziumException at org.glassfish.jersey.servlet.internal.ResponseWriter.rethrow(ResponseWriter.java:254) at org.glassfish.jersey.servlet.internal.ResponseWriter.failure(ResponseWriter.java:236) at org.glassfish.jersey.server.ServerRuntime$Responder.process(ServerRuntime.java:436) at org.glassfish.jersey.server.ServerRuntime$1.run(ServerRuntime.java:261) at org.glassfish.jersey.internal.Errors$1.call(Errors.java:248) at org.glassfish.jersey.internal.Errors$1.call(Errors.java:244) at org.glassfish.jersey.internal.Errors.process(Errors.java:292) at org.glassfish.jersey.internal.Errors.process(Errors.java:274) at org.glassfish.jersey.internal.Errors.process(Errors.java:244) at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:265) at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:232) at org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:679) at org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.java:392) ... 30 more Caused by: java.lang.NoClassDefFoundError: io/debezium/DebeziumException at io.debezium.relational.HistorizedRelationalDatabaseConnectorConfig.(HistorizedRelationalDatabaseConnectorConfig.java:52) at io.debezium.connector.oracle.OracleConnector.config(OracleConnector.java:51) at org.apache.kafka.connect.runtime.AbstractHerder.validateConnectorConfig(AbstractHerder.java:366) at org.apache.kafka.connect.runtime.AbstractHerder.lambda$validateConnectorConfig$1(AbstractHerder.java:326) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ... 1 more Caused by: java.lang.ClassNotFoundException: io.debezium.DebeziumException at java.net.URLClassLoader.findClass(URLClassLoader.java:382) at java.lang.ClassLoader.loadClass(ClassLoader.java:418) at org.apache.kafka.connect.runtime.isolation.PluginClassLoader.loadClass(PluginClassLoader.java:104) at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ... 9 more [2020-09-23 17:51:57,186] WARN unhandled due to prior sendError (org.eclipse.jetty.server.HttpChannelState) javax.servlet.ServletException: org.glassfish.jersey.server.ContainerException: java.lang.NoClassDefFoundError: io/debezium/DebeziumException at org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.java:408) at org.glassfish.jersey.servlet.WebComponent.service(WebComponent.java:346) at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:365) at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:318) at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:205) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:760) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:547) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1607) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1297) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1577) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1212) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:221) at org.eclipse.jetty.server.handler.StatisticsHandler.handle(StatisticsHandler.java:173) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) at org.eclipse.jetty.server.Server.handle(Server.java:500) at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383) at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:270) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at java.lang.Thread.run(Thread.java:748) Caused by: org.glassfish.jersey.server.ContainerException: java.lang.NoClassDefFoundError: io/debezium/DebeziumException at org.glassfish.jersey.servlet.internal.ResponseWriter.rethrow(ResponseWriter.java:254) at org.glassfish.jersey.servlet.internal.ResponseWriter.failure(ResponseWriter.java:236) at org.glassfish.jersey.server.ServerRuntime$Responder.process(ServerRuntime.java:436) at org.glassfish.jersey.server.ServerRuntime$1.run(ServerRuntime.java:261) at org.glassfish.jersey.internal.Errors$1.call(Errors.java:248) at org.glassfish.jersey.internal.Errors$1.call(Errors.java:244) at org.glassfish.jersey.internal.Errors.process(Errors.java:292) at org.glassfish.jersey.internal.Errors.process(Errors.java:274) at org.glassfish.jersey.internal.Errors.process(Errors.java:244) at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:265) at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:232) at org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:679) at org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.java:392) ... 30 more Caused by: java.lang.NoClassDefFoundError: io/debezium/DebeziumException at io.debezium.relational.HistorizedRelationalDatabaseConnectorConfig.(HistorizedRelationalDatabaseConnectorConfig.java:52) at io.debezium.connector.oracle.OracleConnector.config(OracleConnector.java:51) at org.apache.kafka.connect.runtime.AbstractHerder.validateConnectorConfig(AbstractHerder.java:366) at org.apache.kafka.connect.runtime.AbstractHerder.lambda$validateConnectorConfig$1(AbstractHerder.java:326) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ... 1 more Caused by: java.lang.ClassNotFoundException: io.debezium.DebeziumException at java.net.URLClassLoader.findClass(URLClassLoader.java:382) at java.lang.ClassLoader.loadClass(ClassLoader.java:418) at org.apache.kafka.connect.runtime.isolation.PluginClassLoader.loadClass(PluginClassLoader.java:104) at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ... 9 more instantclient echo %PATH% | findstr instantclient XXX;C:\Users\username\Downloads\instantclient_19_8;XXX Any pointers would be much appreciated. Thanks!
Issue solved, i had to copy all the jars found in the plug-in archive here.
Caused by: java.lang.ClassNotFoundException: io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor
Trying to publish message to kafka topic using rest proxy by Confluent platform using this command and responds with an error as mentioned below Request: $ curl -X POST -H "Content-Type: application/vnd.kafka.avro.v2+json" \ -H "Accept: application/vnd.kafka.v2+json" \ --data '{"value_schema": "{\"type\": \"record\", \"name\": \"User\", \"fields\": [{\"name\": \"name\", \"type\": \"string\"}]}", "records": [{"value": {"name": "test name"}}]}' \ "http://${RESTPROXY_HOST}:8082/topics/${TOPIC}" Response: {"error_code":500,"message":"Internal Server Error"} Error log from ./kafka-rest/logs/kafka-rest.log [2018-08-06 14:29:00,734] INFO [Producer clientId=producer-4] Closing the Kafka producer with timeoutMillis = 0 ms. (org.apache.kafka.clients.producer.KafkaProducer) [2018-08-06 14:29:00,734] ERROR Unhandled exception resulting in internal server error response (io.confluent.rest.exceptions.GenericExceptionMapper) java.lang.reflect.UndeclaredThrowableException at com.sun.proxy.$Proxy12.getProducerPool(Unknown Source) at io.confluent.kafkarest.resources.TopicsResource.produce(TopicsResource.java:147) at io.confluent.kafkarest.resources.TopicsResource.produceAvro(TopicsResource.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.glassfish.jersey.server.model.internal.ResourceMethodInvocationHandlerFactory$1.invoke(ResourceMethodInvocationHandlerFactory.java:81) at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher$1.run(AbstractJavaResourceMethodDispatcher.java:144) at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.invoke(AbstractJavaResourceMethodDispatcher.java:161) at org.glassfish.jersey.server.model.internal.JavaResourceMethodDispatcherProvider$VoidOutInvoker.doDispatch(JavaResourceMethodDispatcherProvider.java:143) at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.dispatch(AbstractJavaResourceMethodDispatcher.java:99) at org.glassfish.jersey.server.model.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:389) at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:347) at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:102) at org.glassfish.jersey.server.ServerRuntime$2.run(ServerRuntime.java:326) at org.glassfish.jersey.internal.Errors$1.call(Errors.java:271) at org.glassfish.jersey.internal.Errors$1.call(Errors.java:267) at org.glassfish.jersey.internal.Errors.process(Errors.java:315) at org.glassfish.jersey.internal.Errors.process(Errors.java:297) at org.glassfish.jersey.internal.Errors.process(Errors.java:267) at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:317) at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:305) at org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:1154) at org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.java:473) at org.glassfish.jersey.servlet.ServletContainer.serviceImpl(ServletContainer.java:408) at org.glassfish.jersey.servlet.ServletContainer.doFilter(ServletContainer.java:583) at org.glassfish.jersey.servlet.ServletContainer.doFilter(ServletContainer.java:524) at org.glassfish.jersey.servlet.ServletContainer.doFilter(ServletContainer.java:461) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:110) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97) at org.eclipse.jetty.server.handler.StatisticsHandler.handle(StatisticsHandler.java:159) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97) at org.eclipse.jetty.server.Server.handle(Server.java:499) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:311) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:258) at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:544) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Thread.java:748) Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at io.confluent.kafkarest.extension.ContextInvocationHandler.invoke(ContextInvocationHandler.java:33) ... 48 more Caused by: org.apache.kafka.common.KafkaException: Failed to construct kafka producer at org.apache.kafka.clients.producer.KafkaProducer.<init>(KafkaProducer.java:459) at org.apache.kafka.clients.producer.KafkaProducer.<init>(KafkaProducer.java:289) at io.confluent.kafkarest.ProducerPool.buildNoSchemaProducer(ProducerPool.java:109) at io.confluent.kafkarest.ProducerPool.buildBinaryProducer(ProducerPool.java:95) at io.confluent.kafkarest.ProducerPool.<init>(ProducerPool.java:68) at io.confluent.kafkarest.ProducerPool.<init>(ProducerPool.java:57) at io.confluent.kafkarest.ProducerPool.<init>(ProducerPool.java:50) at io.confluent.kafkarest.DefaultKafkaRestContext.getProducerPool(DefaultKafkaRestContext.java:70) ... 53 more Caused by: org.apache.kafka.common.KafkaException: io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor ClassNotFoundException exception occurred at org.apache.kafka.common.config.AbstractConfig.getConfiguredInstances(AbstractConfig.java:357) at org.apache.kafka.common.config.AbstractConfig.getConfiguredInstances(AbstractConfig.java:332) at org.apache.kafka.common.config.AbstractConfig.getConfiguredInstances(AbstractConfig.java:319) at org.apache.kafka.clients.producer.KafkaProducer.<init>(KafkaProducer.java:380) ... 60 more Caused by: java.lang.ClassNotFoundException: io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor at java.net.URLClassLoader.findClass(URLClassLoader.java:381) at java.lang.ClassLoader.loadClass(ClassLoader.java:424) at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:338) at java.lang.ClassLoader.loadClass(ClassLoader.java:357) at java.lang.Class.forName0(Native Method) at java.lang.Class.forName(Class.java:348) at org.apache.kafka.common.utils.Utils.loadClass(Utils.java:322) at org.apache.kafka.common.utils.Utils.newInstance(Utils.java:311) at org.apache.kafka.common.config.AbstractConfig.getConfiguredInstances(AbstractConfig.java:355) ... 63 more
Short io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor class does not exists in Kafka rest proxy class path. placing monitoring-interceptors.jar in Kafka rest proxy(or which ever confluent service failing due to this error) classpath should solve the issue Detail This jar can found at local $CONFLUENT_HOME/share/java/monitoring-interceptors/monitoring-interceptors-*.jar or confluent maven repo with group: "io.confluent", artifact: "monitoring-interceptors", version: "4.1.0" $ cd $CONFLUENT_HOME $ cp ./share/java/monitoring-interceptors/monitoring-interceptors-4.1.0.jar \ ./share/java/kafka-rest/ $ ./bin/confluent stop kafka-rest $ ./bin/confluent start kafka-rest