How to resolve JsonParseException for InfluxDB sink connector - apache-kafka
Here is json data in test topic [kafka]
This is a connector property
curl -i -X PUT -H "Content-Type:application/json" \
http://localhost:8089/connectors/influxdb-sink-test-20210615/config \
-d '{
"connector.class" : "io.confluent.influxdb.InfluxDBSinkConnector",
"value.converter" : "org.apache.kafka.connect.json.JsonConverter",
"value.converter.schemas.enable": "false",
"key.converter" : "org.apache.kafka.connect.storage.StringConverter",
"topics" : "test3",
"influxdb.url" : "http://localhost:8086",
"influxdb.db" : "exemDB",
"measurement.name.format" : "${topic}"
}'
An exception is raised in the Kafka connector because the tags field is not converted properly. How can I properly convert the tags field?
Removing the tags field and producing data on the topic normally inserts data into the influxDB.
This is Exception..
If i remove the field tag and produce it, it works well on influxdb. How can I fill in the tags field?
[2021-06-17 09:24:59,521] ERROR WorkerSinkTask{id=influxdb-sink-tcore-test-20210615-0} Task threw an uncaught and unrecoverable exception (org.apache.kafka.connect.runtime.WorkerTask:177)
org.apache.kafka.connect.errors.ConnectException: Tolerance exceeded in error handler
at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndHandleError(RetryWithToleranceOperator.java:178)
at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execute(RetryWithToleranceOperator.java:104)
at org.apache.kafka.connect.runtime.WorkerSinkTask.convertAndTransformRecord(WorkerSinkTask.java:513)
at org.apache.kafka.connect.runtime.WorkerSinkTask.convertMessages(WorkerSinkTask.java:490)
at org.apache.kafka.connect.runtime.WorkerSinkTask.poll(WorkerSinkTask.java:321)
at org.apache.kafka.connect.runtime.WorkerSinkTask.iteration(WorkerSinkTask.java:225)
at org.apache.kafka.connect.runtime.WorkerSinkTask.execute(WorkerSinkTask.java:193)
at org.apache.kafka.connect.runtime.WorkerTask.doRun(WorkerTask.java:175)
at org.apache.kafka.connect.runtime.WorkerTask.run(WorkerTask.java:219)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.kafka.connect.errors.DataException: Converting byte[] to Kafka Connect data failed due to serialization error:
at org.apache.kafka.connect.json.JsonConverter.toConnectData(JsonConverter.java:334)
at org.apache.kafka.connect.runtime.WorkerSinkTask.lambda$convertAndTransformRecord$1(WorkerSinkTask.java:513)
at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndRetry(RetryWithToleranceOperator.java:128)
at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndHandleError(RetryWithToleranceOperator.java:162)
... 13 more
Caused by: org.apache.kafka.common.errors.SerializationException: com.fasterxml.jackson.core.JsonParseException: Unexpected character ('t' (code 116)): was expecting comma to separate Object entries
at [Source: (byte[])"{"dataType":"numeric","parentResourceId":0,"resourceId":1037092,"resourceUUID":"1037092","resourceName":"tcore-master-01","resourceTypeId":5,"resourceTypeName":"물리서버","resourceLocation":"미지정","ip":"localhost","tags":"[{"tagId":0,"tagValueId":0,"tag":"미지정","value":"미지정"},{"tagId":7,"tagValueId":2103044,"tag":"물리서버","value":"tcore-master-01"},{"tagId":52,"tagValueId":2103045,"tag":"미지정","value":"tcore-master01"}]","metricSeq":930,"metricName":"command_"[truncated 279 bytes]; line: 1, column: 242]
Caused by: com.fasterxml.jackson.core.JsonParseException: Unexpected character ('t' (code 116)): was expecting comma to separate Object entries
at [Source: (byte[])"{"dataType":"numeric","parentResourceId":0,"resourceId":1037092,"resourceUUID":"1037092","resourceName":"tcore-master-01","resourceTypeId":5,"resourceTypeName":"물리서버","resourceLocation":"미지정","ip":"localhost","tags":"[{"tagId":0,"tagValueId":0,"tag":"미지정","value":"미지정"},{"tagId":7,"tagValueId":2103044,"tag":"물리서버","value":"tcore-master-01"},{"tagId":52,"tagValueId":2103045,"tag":"미지정","value":"tcore-master01"}]","metricSeq":930,"metricName":"command_"[truncated 279 bytes]; line: 1, column: 242]
at com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:1804)
at com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:669)
at com.fasterxml.jackson.core.base.ParserMinimalBase._reportUnexpectedChar(ParserMinimalBase.java:567)
at com.fasterxml.jackson.core.json.UTF8StreamJsonParser.nextFieldName(UTF8StreamJsonParser.java:980)
at com.fasterxml.jackson.databind.deser.std.BaseNodeDeserializer.deserializeObject(JsonNodeDeserializer.java:247)
at com.fasterxml.jackson.databind.deser.std.JsonNodeDeserializer.deserialize(JsonNodeDeserializer.java:68)
at com.fasterxml.jackson.databind.deser.std.JsonNodeDeserializer.deserialize(JsonNodeDeserializer.java:15)
at com.fasterxml.jackson.databind.ObjectMapper._readTreeAndClose(ObjectMapper.java:4056)
at com.fasterxml.jackson.databind.ObjectMapper.readTree(ObjectMapper.java:2571)
at org.apache.kafka.connect.json.JsonDeserializer.deserialize(JsonDeserializer.java:50)
at org.apache.kafka.connect.json.JsonConverter.toConnectData(JsonConverter.java:332)
at org.apache.kafka.connect.runtime.WorkerSinkTask.lambda$convertAndTransformRecord$1(WorkerSinkTask.java:513)
at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndRetry(RetryWithToleranceOperator.java:128)
at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndHandleError(RetryWithToleranceOperator.java:162)
at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execute(RetryWithToleranceOperator.java:104)
at org.apache.kafka.connect.runtime.WorkerSinkTask.convertAndTransformRecord(WorkerSinkTask.java:513)
at org.apache.kafka.connect.runtime.WorkerSinkTask.convertMessages(WorkerSinkTask.java:490)
at org.apache.kafka.connect.runtime.WorkerSinkTask.poll(WorkerSinkTask.java:321)
at org.apache.kafka.connect.runtime.WorkerSinkTask.iteration(WorkerSinkTask.java:225)
at org.apache.kafka.connect.runtime.WorkerSinkTask.execute(WorkerSinkTask.java:193)
at org.apache.kafka.connect.runtime.WorkerTask.doRun(WorkerTask.java:175)
at org.apache.kafka.connect.runtime.WorkerTask.run(WorkerTask.java:219)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
[2021-06-17 09:24:59,522] ERROR WorkerSinkTask{id=influxdb-sink-tcore-test-20210615-0} Task is being killed and will not recover until manually restarted (org.apache.kafka.connect.runtime.WorkerTask:178)
[2021-06-17 09:24:59,522] INFO Closing InfluxDB client. (io.confluent.influxdb.sink.writer.InfluxDBWriter:406)
Related
Kafka Connect Debezium error while filtering
For config: curl -X POST "${KAFKA_CONNECT_HOST}/connectors" -H "Content-Type: application/json" -d '{ "name": "DebeziumSMS", "config": { "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", "tasks.max": 1, "database.hostname": "bill-srv02.corp.oblakowifi.ru", "database.port": 1433, "database.user": "sa", "database.password": "******", "database.dbname": "sms", "database.server.name": "server-test", "database.history.kafka.bootstrap.servers": "192.168.26.142:9092", "database.history.kafka.topic": "schema-changes.sms", "errors.log.enable": "true", "database.history.skip.unparseable.ddl": "true", "time.precision.mode": "connect", "transforms": "filter", "transforms.filter.type": "io.debezium.transforms.Filter", "transforms.filter.language": "jsr223.groovy", "transforms.filter.condition": "value.op == ''c''" } }' this error: [2021-10-19 12:50:46,234] ERROR Error encountered in task DebeziumSMS-0. Executing stage 'TRANSFORMATION' with class 'io.debezium.transforms.Filter'. (org.apache.kafka.connect.runtime.errors.LogReporter:66) io.debezium.DebeziumException: Error while evaluating expression 'value.op == 'c'' for record 'SourceRecord{sourcePartition={server=server-test}, sourceOffset={transaction_id=null, event_serial_no=1, commit_lsn=00000064:0000420c:0003, change_lsn=00000064:0000420c:0002}} ConnectRecord{topic='server-test', kafkaPartition=0, key=Struct{databaseName=sms}, keySchema=Schema{io.debezium.connector.sqlserver.SchemaChangeKey:STRUCT}, value=Struct{source=Struct{version=1.5.0.Final,connector=sqlserver,name=server-test,ts_ms=1634637045699,db=,schema=,table=,change_lsn=00000064:0000420c:0002,commit_lsn=00000064:0000420c:0003},databaseName=sms,schemaName=dbo,ddl=N/A,tableChanges=[Struct{type=ALTER,id="sms"."dbo"."sms",table=Struct{primaryKeyColumnNames=[id],columns=[Struct{name=id,jdbcType=1,typeName=uniqueidentifier,typeExpression=uniqueidentifier,length=36,position=1,optional=false,autoIncremented=false,generated=false}, Struct{name=creator_login,jdbcType=12,typeName=varchar,typeExpression=varchar,length=64,position=2,optional=false,autoIncremented=false,generated=false}, Struct{name=system_ip_address,jdbcType=12,typeName=varchar,typeExpression=varchar,length=128,position=3,optional=false,autoIncremented=false,generated=false}, Struct{name=system_id,jdbcType=4,typeName=int,typeExpression=int,length=10,scale=0,position=4,optional=false,autoIncremented=false,generated=false}, Struct{name=system_sub_id,jdbcType=12,typeName=varchar,typeExpression=varchar,length=128,position=5,optional=false,autoIncremented=false,generated=false}, Struct{name=type_id,jdbcType=12,typeName=varchar,typeExpression=varchar,length=128,position=6,optional=true,autoIncremented=false,generated=false}, Struct{name=date_create,jdbcType=93,typeName=datetime,typeExpression=datetime,length=23,scale=3,position=7,optional=false,autoIncremented=false,generated=false}, Struct{name=date_send,jdbcType=93,typeName=datetime,typeExpression=datetime,length=23,scale=3,position=8,optional=false,autoIncremented=false,generated=false}, Struct{name=date_state,jdbcType=93,typeName=datetime,typeExpression=datetime,length=23,scale=3,position=9,optional=true,autoIncremented=false,generated=false}, Struct{name=time_to_live,jdbcType=4,typeName=int,typeExpression=int,length=10,scale=0,position=10,optional=false,autoIncremented=false,generated=false}, Struct{name=state_id,jdbcType=4,typeName=int,typeExpression=int,length=10,scale=0,position=11,optional=false,autoIncremented=false,generated=false}, Struct{name=phone,jdbcType=12,typeName=varchar,typeExpression=varchar,length=32,position=12,optional=false,autoIncremented=false,generated=false}, Struct{name=target_id,jdbcType=12,typeName=varchar,typeExpression=varchar,length=128,position=13,optional=true,autoIncremented=false,generated=false}, Struct{name=provider_id,jdbcType=4,typeName=int,typeExpression=int,length=10,scale=0,position=14,optional=false,autoIncremented=false,generated=false}, Struct{name=message,jdbcType=12,typeName=varchar,typeExpression=varchar,length=2147483647,position=15,optional=false,autoIncremented=false,generated=false}, Struct{name=linked_file,jdbcType=12,typeName=varchar,typeExpression=varchar,length=4000,position=16,optional=true,autoIncremented=false,generated=false}, Struct{name=error_id,jdbcType=4,typeName=int,typeExpression=int,length=10,scale=0,position=17,optional=true,autoIncremented=false,generated=false}, Struct{name=error_msg,jdbcType=12,typeName=varchar,typeExpression=varchar,length=4000,position=18,optional=true,autoIncremented=false,generated=false}]}}]}, valueSchema=Schema{io.debezium.connector.sqlserver.SchemaChangeValue:STRUCT}, timestamp=null, headers=ConnectHeaders(headers=)}' at io.debezium.transforms.scripting.Jsr223Engine.eval(Jsr223Engine.java:116) at io.debezium.transforms.Filter.doApply(Filter.java:33) at io.debezium.transforms.ScriptingTransformation.apply(ScriptingTransformation.java:189) at org.apache.kafka.connect.runtime.TransformationChain.lambda$apply$0(TransformationChain.java:50) at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndRetry(RetryWithToleranceOperator.java:156) at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndHandleError(RetryWithToleranceOperator.java:190) at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execute(RetryWithToleranceOperator.java:132) at org.apache.kafka.connect.runtime.TransformationChain.apply(TransformationChain.java:50) at org.apache.kafka.connect.runtime.WorkerSourceTask.sendRecords(WorkerSourceTask.java:323) at org.apache.kafka.connect.runtime.WorkerSourceTask.execute(WorkerSourceTask.java:248) at org.apache.kafka.connect.runtime.WorkerTask.doRun(WorkerTask.java:182) at org.apache.kafka.connect.runtime.WorkerTask.run(WorkerTask.java:231) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: javax.script.ScriptException: org.apache.kafka.connect.errors.DataException: op is not a valid field name at org.codehaus.groovy.jsr223.GroovyScriptEngineImpl.eval(GroovyScriptEngineImpl.java:320) at org.codehaus.groovy.jsr223.GroovyCompiledScript.eval(GroovyCompiledScript.java:71) at javax.script.CompiledScript.eval(CompiledScript.java:92) at io.debezium.transforms.scripting.Jsr223Engine.eval(Jsr223Engine.java:107) ... 16 more Caused by: org.apache.kafka.connect.errors.DataException: op is not a valid field name at org.apache.kafka.connect.data.Struct.lookupField(Struct.java:254) at org.apache.kafka.connect.data.Struct.get(Struct.java:74) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:107) at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:323) at org.codehaus.groovy.runtime.metaclass.MethodMetaProperty$GetMethodMetaProperty.getProperty(MethodMetaProperty.java:62) at org.codehaus.groovy.runtime.callsite.GetEffectivePojoPropertySite.getProperty(GetEffectivePojoPropertySite.java:63) at org.codehaus.groovy.runtime.callsite.AbstractCallSite.callGetProperty(AbstractCallSite.java:329) at Script1.run(Script1.groovy:1) at org.codehaus.groovy.jsr223.GroovyScriptEngineImpl.eval(GroovyScriptEngineImpl.java:317) ... 19 more [2021-10-19 12:50:46,237] INFO WorkerSourceTask{id=DebeziumSMS-0} flushing 0 outstanding messages for offset commit (org.apache.kafka.connect.runtime.WorkerSourceTask:487) [2021-10-19 12:50:46,237] ERROR WorkerSourceTask{id=DebeziumSMS-0} Task threw an uncaught and unrecoverable exception. Task is being killed and will not recover until manually restarted (org.apache.kafka.connect.runtime.WorkerTask:184) org.apache.kafka.connect.errors.ConnectException: Tolerance exceeded in error handler at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndHandleError(RetryWithToleranceOperator.java:206) at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execute(RetryWithToleranceOperator.java:132) at org.apache.kafka.connect.runtime.TransformationChain.apply(TransformationChain.java:50) at org.apache.kafka.connect.runtime.WorkerSourceTask.sendRecords(WorkerSourceTask.java:323) at org.apache.kafka.connect.runtime.WorkerSourceTask.execute(WorkerSourceTask.java:248) at org.apache.kafka.connect.runtime.WorkerTask.doRun(WorkerTask.java:182) at org.apache.kafka.connect.runtime.WorkerTask.run(WorkerTask.java:231) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: io.debezium.DebeziumException: Error while evaluating expression 'value.op == 'c'' for record 'SourceRecord{sourcePartition={server=server-test}, sourceOffset={transaction_id=null, event_serial_no=1, commit_lsn=00000064:0000420c:0003, change_lsn=00000064:0000420c:0002}} ConnectRecord{topic='server-test', kafkaPartition=0, key=Struct{databaseName=sms}, keySchema=Schema{io.debezium.connector.sqlserver.SchemaChangeKey:STRUCT}, value=Struct{source=Struct{version=1.5.0.Final,connector=sqlserver,name=server-test,ts_ms=1634637045699,db=,schema=,table=,change_lsn=00000064:0000420c:0002,commit_lsn=00000064:0000420c:0003},databaseName=sms,schemaName=dbo,ddl=N/A,tableChanges=[Struct{type=ALTER,id="sms"."dbo"."sms",table=Struct{primaryKeyColumnNames=[id],columns=[Struct{name=id,jdbcType=1,typeName=uniqueidentifier,typeExpression=uniqueidentifier,length=36,position=1,optional=false,autoIncremented=false,generated=false}, Struct{name=creator_login,jdbcType=12,typeName=varchar,typeExpression=varchar,length=64,position=2,optional=false,autoIncremented=false,generated=false}, Struct{name=system_ip_address,jdbcType=12,typeName=varchar,typeExpression=varchar,length=128,position=3,optional=false,autoIncremented=false,generated=false}, Struct{name=system_id,jdbcType=4,typeName=int,typeExpression=int,length=10,scale=0,position=4,optional=false,autoIncremented=false,generated=false}, Struct{name=system_sub_id,jdbcType=12,typeName=varchar,typeExpression=varchar,length=128,position=5,optional=false,autoIncremented=false,generated=false}, Struct{name=type_id,jdbcType=12,typeName=varchar,typeExpression=varchar,length=128,position=6,optional=true,autoIncremented=false,generated=false}, Struct{name=date_create,jdbcType=93,typeName=datetime,typeExpression=datetime,length=23,scale=3,position=7,optional=false,autoIncremented=false,generated=false}, Struct{name=date_send,jdbcType=93,typeName=datetime,typeExpression=datetime,length=23,scale=3,position=8,optional=false,autoIncremented=false,generated=false}, Struct{name=date_state,jdbcType=93,typeName=datetime,typeExpression=datetime,length=23,scale=3,position=9,optional=true,autoIncremented=false,generated=false}, Struct{name=time_to_live,jdbcType=4,typeName=int,typeExpression=int,length=10,scale=0,position=10,optional=false,autoIncremented=false,generated=false}, Struct{name=state_id,jdbcType=4,typeName=int,typeExpression=int,length=10,scale=0,position=11,optional=false,autoIncremented=false,generated=false}, Struct{name=phone,jdbcType=12,typeName=varchar,typeExpression=varchar,length=32,position=12,optional=false,autoIncremented=false,generated=false}, Struct{name=target_id,jdbcType=12,typeName=varchar,typeExpression=varchar,length=128,position=13,optional=true,autoIncremented=false,generated=false}, Struct{name=provider_id,jdbcType=4,typeName=int,typeExpression=int,length=10,scale=0,position=14,optional=false,autoIncremented=false,generated=false}, Struct{name=message,jdbcType=12,typeName=varchar,typeExpression=varchar,length=2147483647,position=15,optional=false,autoIncremented=false,generated=false}, Struct{name=linked_file,jdbcType=12,typeName=varchar,typeExpression=varchar,length=4000,position=16,optional=true,autoIncremented=false,generated=false}, Struct{name=error_id,jdbcType=4,typeName=int,typeExpression=int,length=10,scale=0,position=17,optional=true,autoIncremented=false,generated=false}, Struct{name=error_msg,jdbcType=12,typeName=varchar,typeExpression=varchar,length=4000,position=18,optional=true,autoIncremented=false,generated=false}]}}]}, valueSchema=Schema{io.debezium.connector.sqlserver.SchemaChangeValue:STRUCT}, timestamp=null, headers=ConnectHeaders(headers=)}' at io.debezium.transforms.scripting.Jsr223Engine.eval(Jsr223Engine.java:116) at io.debezium.transforms.Filter.doApply(Filter.java:33) at io.debezium.transforms.ScriptingTransformation.apply(ScriptingTransformation.java:189) at org.apache.kafka.connect.runtime.TransformationChain.lambda$apply$0(TransformationChain.java:50) at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndRetry(RetryWithToleranceOperator.java:156) at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndHandleError(RetryWithToleranceOperator.java:190) ... 11 more Caused by: javax.script.ScriptException: org.apache.kafka.connect.errors.DataException: op is not a valid field name at org.codehaus.groovy.jsr223.GroovyScriptEngineImpl.eval(GroovyScriptEngineImpl.java:320) at org.codehaus.groovy.jsr223.GroovyCompiledScript.eval(GroovyCompiledScript.java:71) at javax.script.CompiledScript.eval(CompiledScript.java:92) at io.debezium.transforms.scripting.Jsr223Engine.eval(Jsr223Engine.java:107) ... 16 more Caused by: org.apache.kafka.connect.errors.DataException: op is not a valid field name at org.apache.kafka.connect.data.Struct.lookupField(Struct.java:254) at org.apache.kafka.connect.data.Struct.get(Struct.java:74) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:107) at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:323) at org.codehaus.groovy.runtime.metaclass.MethodMetaProperty$GetMethodMetaProperty.getProperty(MethodMetaProperty.java:62) at org.codehaus.groovy.runtime.callsite.GetEffectivePojoPropertySite.getProperty(GetEffectivePojoPropertySite.java:63) at org.codehaus.groovy.runtime.callsite.AbstractCallSite.callGetProperty(AbstractCallSite.java:329) at Script1.run(Script1.groovy:1) at org.codehaus.groovy.jsr223.GroovyScriptEngineImpl.eval(GroovyScriptEngineImpl.java:317) ... 19 more [2021-10-19 12:50:46,237] INFO Stopping down connector (io.debezium.connector.common.BaseSourceTask:238) After recreating Debezium Connector by DELETE and next POST, Debezium works fine a few days. But then error repeats, and connector stopps with this error. And after reboot Kafka Connect (distributed) service this error again. Trace for connector error by REST : "trace":"org.apache.kafka.connect.errors.ConnectException: Tolerance exceeded in error handler\n\tat org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndHandleError(RetryWithToleranceOperator.java:206)\n\tat org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execute(RetryWithToleranceOperator.java:132)\n\tat org.apache.kafka.connect.runtime.TransformationChain.apply(TransformationChain.java:50)\n\tat org.apache.kafka.connect.runtime.WorkerSourceTask.sendRecords(WorkerSourceTask.java:323)\n\tat org.apache.kafka.connect.runtime.WorkerSourceTask.execute(WorkerSourceTask.java:248)\n\tat org.apache.kafka.connect.runtime.WorkerTask.doRun(WorkerTask.java:182)\n\tat org.apache.kafka.connect.runtime.WorkerTask.run(WorkerTask.java:231)\n\tat java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:266)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:748)\nCaused by: io.debezium.DebeziumException: Error while evaluating expression 'value && value.op == 'c'' for record 'SourceRecord{sourcePartition={server=server-test}, sourceOffset={transaction_id=null, event_serial_no=1, commit_lsn=00000064:0000420c:0003, change_lsn=00000064:0000420c:0002}} ConnectRecord{topic='server-test', kafkaPartition=0, key=Struct{databaseName=sms}, keySchema=Schema{io.debezium.connector.sqlserver.SchemaChangeKey:STRUCT}, value=Struct{source=Struct{version=1.5.0.Final,connector=sqlserver,name=server-test,ts_ms=1634805618423,db=,schema=,table=,change_lsn=00000064:0000420c:0002,commit_lsn=00000064:0000420c:0003},databaseName=sms,schemaName=dbo,ddl=N/A,tableChanges=[Struct{type=ALTER,id=\"sms\".\"dbo\".\"sms\",table=Struct{primaryKeyColumnNames=[id],columns=[Struct{name=id,jdbcType=1,typeName=uniqueidentifier,typeExpression=uniqueidentifier,length=36,position=1,optional=false,autoIncremented=false,generated=false}, Struct{name=creator_login,jdbcType=12,typeName=varchar,typeExpression=varchar,length=64,position=2,optional=false,autoIncremented=false,generated=false}, Struct{name=system_ip_address,jdbcType=12,typeName=varchar,typeExpression=varchar,length=128,position=3,optional=false,autoIncremented=false,generated=false}, Struct{name=system_id,jdbcType=4,typeName=int,typeExpression=int,length=10,scale=0,position=4,optional=false,autoIncremented=false,generated=false}, Struct{name=system_sub_id,jdbcType=12,typeName=varchar,typeExpression=varchar,length=128,position=5,optional=false,autoIncremented=false,generated=false}, Struct{name=type_id,jdbcType=12,typeName=varchar,typeExpression=varchar,length=128,position=6,optional=true,autoIncremented=false,generated=false}, Struct{name=date_create,jdbcType=93,typeName=datetime,typeExpression=datetime,length=23,scale=3,position=7,optional=false,autoIncremented=false,generated=false}, Struct{name=date_send,jdbcType=93,typeName=datetime,typeExpression=datetime,length=23,scale=3,position=8,optional=false,autoIncremented=false,generated=false}, Struct{name=date_state,jdbcType=93,typeName=datetime,typeExpression=datetime,length=23,scale=3,position=9,optional=true,autoIncremented=false,generated=false}, Struct{name=time_to_live,jdbcType=4,typeName=int,typeExpression=int,length=10,scale=0,position=10,optional=false,autoIncremented=false,generated=false}, Struct{name=state_id,jdbcType=4,typeName=int,typeExpression=int,length=10,scale=0,position=11,optional=false,autoIncremented=false,generated=false}, Struct{name=phone,jdbcType=12,typeName=varchar,typeExpression=varchar,length=32,position=12,optional=false,autoIncremented=false,generated=false}, Struct{name=target_id,jdbcType=12,typeName=varchar,typeExpression=varchar,length=128,position=13,optional=true,autoIncremented=false,generated=false}, Struct{name=provider_id,jdbcType=4,typeName=int,typeExpression=int,length=10,scale=0,position=14,optional=false,autoIncremented=false,generated=false}, Struct{name=message,jdbcType=12,typeName=varchar,typeExpression=varchar,length=2147483647,position=15,optional=false,autoIncremented=false,generated=false}, Struct{name=linked_file,jdbcType=12,typeName=varchar,typeExpression=varchar,length=4000,position=16,optional=true,autoIncremented=false,generated=false}, Struct{name=error_id,jdbcType=4,typeName=int,typeExpression=int,length=10,scale=0,position=17,optional=true,autoIncremented=false,generated=false}, Struct{name=error_msg,jdbcType=12,typeName=varchar,typeExpression=varchar,length=4000,position=18,optional=true,autoIncremented=false,generated=false}]}}]}, valueSchema=Schema{io.debezium.connector.sqlserver.SchemaChangeValue:STRUCT}, timestamp=null, headers=ConnectHeaders(headers=)}'\n\tat io.debezium.transforms.scripting.Jsr223Engine.eval(Jsr223Engine.java:116)\n\tat io.debezium.transforms.Filter.doApply(Filter.java:33)\n\tat io.debezium.transforms.ScriptingTransformation.apply(ScriptingTransformation.java:189)\n\tat org.apache.kafka.connect.runtime.TransformationChain.lambda$apply$0(TransformationChain.java:50)\n\tat org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndRetry(RetryWithToleranceOperator.java:156)\n\tat org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndHandleError(RetryWithToleranceOperator.java:190)\n\t... 11 more\nCaused by: javax.script.ScriptException: org.apache.kafka.connect.errors.DataException: op is not a valid field name\n\tat org.codehaus.groovy.jsr223.GroovyScriptEngineImpl.eval(GroovyScriptEngineImpl.java:320)\n\tat org.codehaus.groovy.jsr223.GroovyCompiledScript.eval(GroovyCompiledScript.java:71)\n\tat javax.script.CompiledScript.eval(CompiledScript.java:92)\n\tat io.debezium.transforms.scripting.Jsr223Engine.eval(Jsr223Engine.java:107)\n\t... 16 more\nCaused by: org.apache.kafka.connect.errors.DataException: op is not a valid field name\n\tat org.apache.kafka.connect.data.Struct.lookupField(Struct.java:254)\n\tat org.apache.kafka.connect.data.Struct.get(Struct.java:74)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:498)\n\tat org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:107)\n\tat groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:323)\n\tat org.codehaus.groovy.runtime.metaclass.MethodMetaProperty$GetMethodMetaProperty.getProperty(MethodMetaProperty.java:62)\n\tat org.codehaus.groovy.runtime.callsite.GetEffectivePojoPropertySite.getProperty(GetEffectivePojoPropertySite.java:63)\n\tat org.codehaus.groovy.runtime.callsite.AbstractCallSite.callGetProperty(AbstractCallSite.java:329)\n\tat Script3.run(Script3.groovy:1)\n\tat org.codehaus.groovy.jsr223.GroovyScriptEngineImpl.eval(GroovyScriptEngineImpl.java:317)\n\t... 19 more\n"} Caused by: org.apache.kafka.connect.errors.DataException: op is not a valid field name Why? What can be wrong ? Thank you.
Debezium: RestClientException: Leader not known; error code: 50004
I am getting exceptions when i try to deploy debezium connector to kafka-connect. As a result, snapshots are not created and also cdc streaming is blocked. The problem is I am not able to find where the issue is i.e. is it in kafka-connect, schema-registry or debezium connector { "name": "my-connector", "config": { "connector.class": "io.debezium.connector.mysql.MySqlConnector", "database.hostname": "root", "database.port": "3306", "database.user": "USER", "database.password": "PASSWORD", "database.server.id": "184056", "database.server.name": "dbz", "table.include.list": "T1,T2", "snapshot.mode": "initial", "database.history.kafka.bootstrap.servers": "xxx:9092", "database.history.kafka.topic": "dbz.myhistory", "plugin.path": "/usr/share/java/debezium-mysql-connect/", "database.serverTimezone": "America/Los_Angeles", "snapshot.locking.mode": "none", "include.query": "true", "key.converter": "io.confluent.connect.avro.AvroConverter", "key.converter.schema.registry.url": "XXX", "value.converter": "io.confluent.connect.avro.AvroConverter", "value.converter.schema.registry.url": "XXX" } } kafka-connect exception logs: org.apache.kafka.connect.errors.ConnectException: Tolerance exceeded in error handler at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndHandleError(RetryWithToleranceOperator.java:178) at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execute(RetryWithToleranceOperator.java:104) at org.apache.kafka.connect.runtime.WorkerSourceTask.convertTransformedRecord(WorkerSourceTask.java:294) at org.apache.kafka.connect.runtime.WorkerSourceTask.sendRecords(WorkerSourceTask.java:323) at org.apache.kafka.connect.runtime.WorkerSourceTask.execute(WorkerSourceTask.java:247) at org.apache.kafka.connect.runtime.WorkerTask.doRun(WorkerTask.java:184) at org.apache.kafka.connect.runtime.WorkerTask.run(WorkerTask.java:234) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.kafka.connect.errors.DataException: Failed to serialize Avro data from topic dbz : at io.confluent.connect.avro.AvroConverter.fromConnectData(AvroConverter.java:91) at org.apache.kafka.connect.storage.Converter.fromConnectData(Converter.java:63) at org.apache.kafka.connect.runtime.WorkerSourceTask.lambda$convertTransformedRecord$1(WorkerSourceTask.java:294) at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndRetry(RetryWithToleranceOperator.java:128) at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndHandleError(RetryWithToleranceOperator.java:162) ... 11 more Caused by: org.apache.kafka.common.errors.SerializationException: Error registering Avro schema: {"type":"record","name":"SchemaChangeKey","namespace":"io.debezium.connector.mysql","fields":[{"name":"databaseName","type":"string"}],"connect.name":"io.debezium.connector.mysql.SchemaChangeKey"} Caused by: io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException: Leader not known.; error code: 50004 at io.confluent.kafka.schemaregistry.client.rest.RestService.sendHttpRequest(RestService.java:292) at io.confluent.kafka.schemaregistry.client.rest.RestService.httpRequest(RestService.java:351) at io.confluent.kafka.schemaregistry.client.rest.RestService.registerSchema(RestService.java:494) at io.confluent.kafka.schemaregistry.client.rest.RestService.registerSchema(RestService.java:485) at io.confluent.kafka.schemaregistry.client.rest.RestService.registerSchema(RestService.java:458) at io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient.registerAndGetId(CachedSchemaRegistryClient.java:206) at io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient.register(CachedSchemaRegistryClient.java:268) at io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient.register(CachedSchemaRegistryClient.java:244) at io.confluent.kafka.serializers.AbstractKafkaAvroSerializer.serializeImpl(AbstractKafkaAvroSerializer.java:74) at io.confluent.connect.avro.AvroConverter$Serializer.serialize(AvroConverter.java:143) at io.confluent.connect.avro.AvroConverter.fromConnectData(AvroConverter.java:84) at org.apache.kafka.connect.storage.Converter.fromConnectData(Converter.java:63) at org.apache.kafka.connect.runtime.WorkerSourceTask.lambda$convertTransformedRecord$1(WorkerSourceTask.java:294) at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndRetry(RetryWithToleranceOperator.java:128) at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execAndHandleError(RetryWithToleranceOperator.java:162) at org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator.execute(RetryWithToleranceOperator.java:104) at org.apache.kafka.connect.runtime.WorkerSourceTask.convertTransformedRecord(WorkerSourceTask.java:294) at org.apache.kafka.connect.runtime.WorkerSourceTask.sendRecords(WorkerSourceTask.java:323) at org.apache.kafka.connect.runtime.WorkerSourceTask.execute(WorkerSourceTask.java:247) at org.apache.kafka.connect.runtime.WorkerTask.doRun(WorkerTask.java:184) at org.apache.kafka.connect.runtime.WorkerTask.run(WorkerTask.java:234) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) [2021-09-20 05:29:23,849] ERROR WorkerSourceTask{id=my-connector-0} Task is being killed and will not recover until manually restarted (org.apache.kafka.connect.runtime.WorkerTask:187) [2021-09-20 05:29:23,849] INFO Stopping down connector (io.debezium.connector.common.BaseSourceTask:241) [2021-09-20 05:29:23,853] WARN Snapshot was interrupted before completion (io.debezium.pipeline.source.AbstractSnapshotChangeEventSource:74) [2021-09-20 05:29:23,853] INFO Snapshot - Final stage (io.debezium.pipeline.source.AbstractSnapshotChangeEventSource:82) [2021-09-20 05:29:23,854] WARN Change event source executor was interrupted (io.debezium.pipeline.ChangeEventSourceCoordinator:132) java.lang.InterruptedException: Interrupted while processing event SchemaChangeEvent [database=mydb, schema=null, ddl=DROP TABLE IF EXISTS `db`.`t3`, tables=[], type=DROP] at io.debezium.connector.mysql.MySqlSnapshotChangeEventSource.createSchemaChangeEventsForTables(MySqlSnapshotChangeEventSource.java:539) at io.debezium.relational.RelationalSnapshotChangeEventSource.doExecute(RelationalSnapshotChangeEventSource.java:127) at io.debezium.pipeline.source.AbstractSnapshotChangeEventSource.execute(AbstractSnapshotChangeEventSource.java:70) at io.debezium.pipeline.ChangeEventSourceCoordinator.lambda$start$0(ChangeEventSourceCoordinator.java:118) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Schema-Registry Exception error logs: io.confluent.kafka.schemaregistry.rest.exceptions.RestUnknownLeaderException: Leader not known. at io.confluent.kafka.schemaregistry.rest.exceptions.Errors.unknownLeaderException(Errors.java:153) at io.confluent.kafka.schemaregistry.rest.resources.SubjectVersionsResource.register(SubjectVersionsResource.java:286) at jdk.internal.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:566) at org.glassfish.jersey.server.model.internal.ResourceMethodInvocationHandlerFactory.lambda$static$0(ResourceMethodInvocationHandlerFactory.java:52) at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher$1.run(AbstractJavaResourceMethodDispatcher.java:124) at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.invoke(AbstractJavaResourceMethodDispatcher.java:167) at org.glassfish.jersey.server.model.internal.JavaResourceMethodDispatcherProvider$VoidOutInvoker.doDispatch(JavaResourceMethodDispatcherProvider.java:159) at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.dispatch(AbstractJavaResourceMethodDispatcher.java:79) at org.glassfish.jersey.server.model.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:475) at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:397) at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:81) at org.glassfish.jersey.server.ServerRuntime$1.run(ServerRuntime.java:255) at org.glassfish.jersey.internal.Errors$1.call(Errors.java:248) at org.glassfish.jersey.internal.Errors$1.call(Errors.java:244) at org.glassfish.jersey.internal.Errors.process(Errors.java:292) at org.glassfish.jersey.internal.Errors.process(Errors.java:274) at org.glassfish.jersey.internal.Errors.process(Errors.java:244) at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:265) at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:234) at org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:680) at org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.java:394) at org.glassfish.jersey.servlet.ServletContainer.serviceImpl(ServletContainer.java:386) at org.glassfish.jersey.servlet.ServletContainer.doFilter(ServletContainer.java:561) at org.glassfish.jersey.servlet.ServletContainer.doFilter(ServletContainer.java:502) at org.glassfish.jersey.servlet.ServletContainer.doFilter(ServletContainer.java:439) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1435) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1350) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:146) at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:146) at org.eclipse.jetty.server.handler.StatisticsHandler.handle(StatisticsHandler.java:179) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:234) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:763) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) at org.eclipse.jetty.server.Server.handle(Server.java:516) at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:388) at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:633) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:380) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105) at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:336) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:313) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:129) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:383) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base/java.lang.Thread.run(Thread.java:829) Caused by: io.confluent.kafka.schemaregistry.exceptions.UnknownLeaderException: Register schema request failed since leader is unknown at io.confluent.kafka.schemaregistry.storage.KafkaSchemaRegistry.registerOrForward(KafkaSchemaRegistry.java:594) at io.confluent.kafka.schemaregistry.rest.resources.SubjectVersionsResource.register(SubjectVersionsResource.java:266) ... 60 more Can someone help me what this exception means and how to resolve it?
Facing issues while using camel-sftp-kafka-connector in Apache kafka?
i am trying to create a sftp connector in Apache kafka using camel-sftp-kafka-connector. i just downloaded the sftp-connector jar and pasted into the kafka's lib folder. while hitting /connector-plugins endpoint i can able to see the sftp-connector is available so below is my sample connector configuration { "name": "camel-sftp-connector-1", "config": { "connector.class": "org.apache.camel.kafkaconnector.sftp.CamelSftpSourceConnector", "key.converter": "org.apache.kafka.connect.storage.StringConverter", "value.converter": "org.apache.kafka.connect.converters.ByteArrayConverter", "transforms": "RemoteTransformer", "transforms.RemoteTransformer.type": "org.apache.camel.kafkaconnector.RemoteFileTransforms", "topics": "came-sftp-test-1", "camel.source.path.host": "192.168.30.167", "camel.source.path.port": "66", "camel.source.path.directoryName": "/csv/sample", "camel.source.endpoint.recursive": "true", "camel.source.endpoint.username": "sftpuser", "camel.source.endpoint.password": "sftppass", "camel.source.endpoint.noop": "/false", "camel.source.endpoint.move": "/done" } } I got the below exception and dont know how to fix it. [2021-05-05 19:20:58,560] WARN /connectors (org.eclipse.jetty.server.HttpChannel) javax.servlet.ServletException: org.glassfish.jersey.server.ContainerException: java.lang.NoClassDefFoundError: org/jctools/queues/MessagePassingQueue$Supplier at org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.java:410) at org.glassfish.jersey.servlet.WebComponent.service(WebComponent.java:346) at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:366) at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:319) at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:205) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:763) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:563) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1612) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1582) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:234) at org.eclipse.jetty.server.handler.StatisticsHandler.handle(StatisticsHandler.java:179) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) at org.eclipse.jetty.server.Server.handle(Server.java:516) at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383) at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:556) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:273) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105) at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905) at java.base/java.lang.Thread.run(Thread.java:834) Caused by: org.glassfish.jersey.server.ContainerException: java.lang.NoClassDefFoundError: org/jctools/queues/MessagePassingQueue$Supplier at org.glassfish.jersey.servlet.internal.ResponseWriter.rethrow(ResponseWriter.java:254) at org.glassfish.jersey.servlet.internal.ResponseWriter.failure(ResponseWriter.java:236) at org.glassfish.jersey.server.ServerRuntime$Responder.process(ServerRuntime.java:436) at org.glassfish.jersey.server.ServerRuntime$1.run(ServerRuntime.java:261) at org.glassfish.jersey.internal.Errors$1.call(Errors.java:248) at org.glassfish.jersey.internal.Errors$1.call(Errors.java:244) at org.glassfish.jersey.internal.Errors.process(Errors.java:292) at org.glassfish.jersey.internal.Errors.process(Errors.java:274) at org.glassfish.jersey.internal.Errors.process(Errors.java:244) at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:265) at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:232) at org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:680) at org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.java:394) ... 30 more Caused by: java.lang.NoClassDefFoundError: org/jctools/queues/MessagePassingQueue$Supplier at org.apache.camel.kafkaconnector.CamelSourceConnectorConfig.<clinit>(CamelSourceConnectorConfig.java:43) at org.apache.camel.kafkaconnector.sftp.CamelSftpSourceConnector.config(CamelSftpSourceConnector.java:29) at org.apache.kafka.connect.runtime.AbstractHerder.validateConnectorConfig(AbstractHerder.java:366) at org.apache.kafka.connect.runtime.AbstractHerder.lambda$validateConnectorConfig$1(AbstractHerder.java:326) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ... 1 more Caused by: java.lang.ClassNotFoundException: org.jctools.queues.MessagePassingQueue$Supplier at java.base/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:581) at java.base/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:178) at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:521) ... 9 more [2021-05-05 19:20:58,562] WARN unhandled due to prior sendError (org.eclipse.jetty.server.HttpChannelState) javax.servlet.ServletException: org.glassfish.jersey.server.ContainerException: java.lang.NoClassDefFoundError: org/jctools/queues/MessagePassingQueue$Supplier at org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.java:410) at org.glassfish.jersey.servlet.WebComponent.service(WebComponent.java:346) at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:366) at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:319) at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:205) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:763) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:563) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1612) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1582) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:234) at org.eclipse.jetty.server.handler.StatisticsHandler.handle(StatisticsHandler.java:179) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) at org.eclipse.jetty.server.Server.handle(Server.java:516) at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383) at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:556) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:273) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105) at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905) at java.base/java.lang.Thread.run(Thread.java:834) Caused by: org.glassfish.jersey.server.ContainerException: java.lang.NoClassDefFoundError: org/jctools/queues/MessagePassingQueue$Supplier at org.glassfish.jersey.servlet.internal.ResponseWriter.rethrow(ResponseWriter.java:254) at org.glassfish.jersey.servlet.internal.ResponseWriter.failure(ResponseWriter.java:236) at org.glassfish.jersey.server.ServerRuntime$Responder.process(ServerRuntime.java:436) at org.glassfish.jersey.server.ServerRuntime$1.run(ServerRuntime.java:261) at org.glassfish.jersey.internal.Errors$1.call(Errors.java:248) at org.glassfish.jersey.internal.Errors$1.call(Errors.java:244) at org.glassfish.jersey.internal.Errors.process(Errors.java:292) at org.glassfish.jersey.internal.Errors.process(Errors.java:274) at org.glassfish.jersey.internal.Errors.process(Errors.java:244) at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:265) at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:232) at org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:680) at org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.java:394) ... 30 more Caused by: java.lang.NoClassDefFoundError: org/jctools/queues/MessagePassingQueue$Supplier at org.apache.camel.kafkaconnector.CamelSourceConnectorConfig.<clinit>(CamelSourceConnectorConfig.java:43) at org.apache.camel.kafkaconnector.sftp.CamelSftpSourceConnector.config(CamelSftpSourceConnector.java:29) at org.apache.kafka.connect.runtime.AbstractHerder.validateConnectorConfig(AbstractHerder.java:366) at org.apache.kafka.connect.runtime.AbstractHerder.lambda$validateConnectorConfig$1(AbstractHerder.java:326) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ... 1 more Caused by: java.lang.ClassNotFoundException: org.jctools.queues.MessagePassingQueue$Supplier at java.base/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:581) at java.base/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:178) at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:521) ... 9 more could anyone tell me what is the issue ? whether i am configuring in wrong way or else any extra jars need to be added?
I've not used Camel connectors, but based on the error, you'll need these jars for that specific package. There's no guarantee you'll be missing other classes needed for the connector https://github.com/JCTools/JCTools#get-it-now JARs pasted into the kafka's lib folder. Modifying the broker classpath is not the appropriate way to configure Connect's plugin.path setting. Connector plugin JARs should be in isolated directories per connector
Spring Batch:Non-skippable exception in recoverer while processing;java.lang.NullPointerException at FaultTolerantChunkProcessor.java:289
I have one step configurations as below stepBuilderFactory.get("step2") .<EventLogWrapper, BaseCscEvent>chunk(100) .reader(eventLogReader) .processor(eventLogItemProcessor) .writer(compositeItemWriter) .faultTolerant().retryLimit(10).retry(Exception.class) .stream(dataBaseItemWriter) .stream(rabbitAmqpItemWriter) .taskExecutor(taskExecutor()) .build(); My Reader configuration is as below AmqpItemReader<EventLog> eventLogReader(AmqpTemplate eventLogReaderRabbitTemplate){ AmqpItemReader amqpItemReader = new AmqpItemReader<EventLog>(eventLogReaderRabbitTemplate); amqpItemReader.setItemType(EventLogWrapper.class); return amqpItemReader; } I am getting one intermittent error as below ExitMessage org.springframework.retry.RetryException: Non-skippable exception in recoverer while processing; nested exception is java.lang.NullPointerException at org.springframework.batch.core.step.item.FaultTolerantChunkProcessor$2.recover(FaultTolerantChunkProcessor.java:289) at org.springframework.retry.support.RetryTemplate.handleRetryExhausted(RetryTemplate.java:512) at org.springframework.retry.support.RetryTemplate.doExecute(RetryTemplate.java:351) at org.springframework.retry.support.RetryTemplate.execute(RetryTemplate.java:211) at org.springframework.batch.core.step.item.BatchRetryTemplate.execute(BatchRetryTemplate.java:217) at org.springframework.batch.core.step.item.FaultTolerantChunkProcessor.transform(FaultTolerantChunkProcessor.java:298) at org.springframework.batch.core.step.item.SimpleChunkProcessor.process(SimpleChunkProcessor.java:210) at org.springframework.batch.core.step.item.ChunkOrientedTasklet.execute(ChunkOrientedTasklet.java:77) at org.springframework.batch.core.step.tasklet.TaskletStep$ChunkTransactionCallback.doInTransaction(TaskletStep.java:407) at org.springframework.batch.core.step.tasklet.TaskletStep$ChunkTransactionCallback.doInTransaction(TaskletStep.java:331) at org.springframework.transaction.support.TransactionTemplate.execute(TransactionTemplate.java:140) at org.springframework.batch.core.step.tasklet.TaskletStep$2.doInChunkContext(TaskletStep.java:273) at org.springframework.batch.core.scope.context.StepContextRepeatCallback.doInIteration(StepContextRepeatCallback.java:82) at org.springframework.batch.repeat.support.TaskExecutorRepeatTemplate$ExecutingRunnable.run(TaskExecutorRepeatTemplate.java:262) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: java.lang.NullPointerException What is the reason behind this behavior ? What should I do to resolve this ? Thanks for reading!
Internal Error: null while running Scala Worksheet in Intellij
I am getting exception while running Scala Worksheet in intellij Internal error: null org.jetbrains.jps.incremental.scala.remote.ClientEventProcessor.process(ClientEventProcessor.scala:22) org.jetbrains.jps.incremental.scala.remote.RemoteResourceOwner$class.handle(RemoteResourceOwner.scala:47) org.jetbrains.plugins.scala.compiler.RemoteServerRunner.handle(RemoteServerRunner.scala:14) org.jetbrains.jps.incremental.scala.remote.RemoteResourceOwner$$anonfun$send$1$$anonfun$apply$1$$anonfun$apply$3.apply(RemoteResourceOwner.scala:30) org.jetbrains.jps.incremental.scala.remote.RemoteResourceOwner$$anonfun$send$1$$anonfun$apply$1$$anonfun$apply$3.apply(RemoteResourceOwner.scala:29) org.jetbrains.jps.incremental.scala.package$.using(package.scala:21) org.jetbrains.jps.incremental.scala.remote.RemoteResourceOwner$$anonfun$send$1$$anonfun$apply$1.apply(RemoteResourceOwner.scala:29) org.jetbrains.jps.incremental.scala.remote.RemoteResourceOwner$$anonfun$send$1$$anonfun$apply$1.apply(RemoteResourceOwner.scala:25) org.jetbrains.jps.incremental.scala.package$.using(package.scala:21) org.jetbrains.jps.incremental.scala.remote.RemoteResourceOwner$$anonfun$send$1.apply(RemoteResourceOwner.scala:25) org.jetbrains.jps.incremental.scala.remote.RemoteResourceOwner$$anonfun$send$1.apply(RemoteResourceOwner.scala:24) org.jetbrains.jps.incremental.scala.package$.using(package.scala:21) org.jetbrains.jps.incremental.scala.remote.RemoteResourceOwner$class.send(RemoteResourceOwner.scala:24) org.jetbrains.plugins.scala.compiler.RemoteServerRunner.send(RemoteServerRunner.scala:14) org.jetbrains.plugins.scala.compiler.RemoteServerRunner$$anon$1$$anonfun$run$1.apply$mcVI$sp(RemoteServerRunner.scala:33) scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:166) org.jetbrains.plugins.scala.compiler.RemoteServerRunner$$anon$1.run(RemoteServerRunner.scala:30) org.jetbrains.plugins.scala.worksheet.server.RemoteServerConnector.compileAndRun(RemoteServerConnector.scala:85) org.jetbrains.plugins.scala.worksheet.processor.WorksheetCompiler$$anon$1.run(WorksheetCompiler.scala:78) com.intellij.compiler.progress.CompilerTask.run(CompilerTask.java:196) com.intellij.openapi.progress.impl.CoreProgressManager$TaskRunnable.run(CoreProgressManager.java:635) com.intellij.openapi.progress.impl.CoreProgressManager$3.run(CoreProgressManager.java:170) com.intellij.openapi.progress.impl.CoreProgressManager.registerIndicatorAndRun(CoreProgressManager.java:494) com.intellij.openapi.progress.impl.CoreProgressManager.executeProcessUnderProgress(CoreProgressManager.java:443) com.intellij.openapi.progress.impl.ProgressManagerImpl.executeProcessUnderProgress(ProgressManagerImpl.java:54) com.intellij.openapi.progress.impl.CoreProgressManager.runProcess(CoreProgressManager.java:155) com.intellij.openapi.progress.impl.ProgressManagerImpl$1.run(ProgressManagerImpl.java:128) com.intellij.openapi.application.impl.ApplicationImpl$2.run(ApplicationImpl.java:307) java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) java.util.concurrent.FutureTask.run(FutureTask.java:266) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) java.lang.Thread.run(Thread.java:745) Please help to resolve it.