$HADOOP_HOME/bin/hdfs --config $HADOOP_CONF_DIR namenode -bootstrapStandby
getting below error in Hadoop HA 3.3.3 on standbyNode for the above command.
{"name":"org.apache.hadoop.hdfs.server.namenode.NameNode","time":1659853529569,"date":"2022-08-07 06:25:29,569","level":"INFO","thread":"main","message":"registered UNIX signal handlers for [TERM, HUP, INT]"}
{"name":"org.apache.hadoop.hdfs.server.namenode.NameNode","time":1659853529682,"date":"2022-08-07 06:25:29,682","level":"INFO","thread":"main","message":"createNameNode [-bootstrapStandby]"}
{"name":"org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby","time":1659853530068,"date":"2022-08-07 06:25:30,068","level":"INFO","thread":"main","message":"Found nn: apache-hadoop-namenode-0.apache-hadoop-namenode.backend.svc.cluster.local, ipc: hdfs:8020"}
{"name":"org.apache.hadoop.hdfs.server.namenode.NameNode","time":1659853530069,"date":"2022-08-07 06:25:30,069","level":"ERROR","thread":"main","message":"Failed to start namenode.","exceptionclass":"java.io.IOException","stack":["java.io.IOException: java.lang.NullPointerException","\tat org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby.run(BootstrapStandby.java:549)","\tat org.apache.hadoop.hdfs.server.namenode.NameNode.createNameNode(NameNode.java:1741)","\tat org.apache.hadoop.hdfs.server.namenode.NameNode.main(NameNode.java:1834)","Caused by: java.lang.NullPointerException","\tat org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby.parseConfAndFindOtherNN(BootstrapStandby.java:435)","\tat org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby.run(BootstrapStandby.java:114)","\tat org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:81)","\tat org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:95)","\tat org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby.run(BootstrapStandby.java:544)","\t... 2 more"]}{"name":"org.apache.hadoop.util.ExitUtil","time":1659853530074,"date":"2022-08-07 06:25:30,074","level":"INFO","thread":"main","message":"Exiting with status 1: java.io.IOException: java.lang.NullPointerException"}
{"name":"org.apache.hadoop.hdfs.server.namenode.NameNode","time":1659853530083,"date":"2022-08-07 06:25:30,083","level":"INFO","thread":"shutdown-hook-0","message":"SHUTDOWN_MSG: \n\nSHUTDOWN_MSG: Shutting down NameNode at apache-hadoop-namenode-1.apache-hadoop-namenode.backend.svc.cluster.local"}
We are trying to upgrade from Wildfly 18 to 22 and are experiencing some trouble with threads left dangling in the jgroups thread group. When the number of threads reaches the default max number (200) everything stops up (naturally). This takes two to three days in our test environment.
In the thread dump we are left with 200 dangling threads that look like this:
at jdk.internal.misc.Unsafe.park(java.base#11.0.8/Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(java.base#11.0.8/LockSupport.java:234)
at java.util.concurrent.CompletableFuture$Signaller.block(java.base#11.0.8/CompletableFuture.java:1798)
at java.util.concurrent.ForkJoinPool.managedBlock(java.base#11.0.8/ForkJoinPool.java:3128)
at java.util.concurrent.CompletableFuture.timedGet(java.base#11.0.8/CompletableFuture.java:1868)
at java.util.concurrent.CompletableFuture.get(java.base#11.0.8/CompletableFuture.java:2021)
at org.infinispan.util.concurrent.CompletableFutures.await(CompletableFutures.java:125)
at org.infinispan.util.concurrent.CompletionStages.join(CompletionStages.java:80)
at org.infinispan.interceptors.distribution.L1NonTxInterceptor.lambda$handleDataWriteCommand$10(L1NonTxInterceptor.java:399)
at org.infinispan.interceptors.distribution.L1NonTxInterceptor$$Lambda$1806/0x0000000841ace040.apply(Unknown Source)
at org.infinispan.interceptors.impl.QueueAsyncInvocationStage.invokeQueuedHandlers(QueueAsyncInvocationStage.java:125)
at org.infinispan.interceptors.impl.QueueAsyncInvocationStage.accept(QueueAsyncInvocationStage.java:88)
at org.infinispan.interceptors.impl.QueueAsyncInvocationStage.accept(QueueAsyncInvocationStage.java:33)
at java.util.concurrent.CompletableFuture.uniWhenComplete(java.base#11.0.8/CompletableFuture.java:859)
at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(java.base#11.0.8/CompletableFuture.java:837)
at java.util.concurrent.CompletableFuture.postComplete(java.base#11.0.8/CompletableFuture.java:506)
at java.util.concurrent.CompletableFuture.complete(java.base#11.0.8/CompletableFuture.java:2073)
at org.infinispan.interceptors.distribution.PrimaryOwnerOnlyCollector.primaryResult(PrimaryOwnerOnlyCollector.java:30)
at org.infinispan.interceptors.distribution.TriangleDistributionInterceptor.lambda$forwardToPrimary$6(TriangleDistributionInterceptor.java:526)
at org.infinispan.interceptors.distribution.TriangleDistributionInterceptor$$Lambda$1953/0x0000000841d51040.apply(Unknown Source)
at java.util.concurrent.CompletableFuture.uniHandle(java.base#11.0.8/CompletableFuture.java:930)
at java.util.concurrent.CompletableFuture$UniHandle.tryFire(java.base#11.0.8/CompletableFuture.java:907)
at java.util.concurrent.CompletableFuture.postComplete(java.base#11.0.8/CompletableFuture.java:506)
at java.util.concurrent.CompletableFuture.complete(java.base#11.0.8/CompletableFuture.java:2073)
at org.infinispan.remoting.transport.AbstractRequest.complete(AbstractRequest.java:67)
at org.infinispan.remoting.transport.impl.SingleTargetRequest.onResponse(SingleTargetRequest.java:45)
at org.infinispan.remoting.transport.impl.RequestRepository.addResponse(RequestRepository.java:52)
at org.infinispan.remoting.transport.jgroups.JGroupsTransport.processResponse(JGroupsTransport.java:1402)
at org.infinispan.remoting.transport.jgroups.JGroupsTransport.processMessage(JGroupsTransport.java:1305)
at org.infinispan.remoting.transport.jgroups.JGroupsTransport.access$300(JGroupsTransport.java:131)
at org.infinispan.remoting.transport.jgroups.JGroupsTransport$ChannelCallbacks.up(JGroupsTransport.java:1445)
at org.jgroups.JChannel.up(JChannel.java:784)
at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:913)
at org.jgroups.protocols.FRAG3.up(FRAG3.java:165)
at org.jgroups.protocols.FlowControl.up(FlowControl.java:343)
at org.jgroups.protocols.FlowControl.up(FlowControl.java:343)
at org.jgroups.protocols.pbcast.GMS.up(GMS.java:876)
at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:243)
at org.jgroups.protocols.UNICAST3.deliverMessage(UNICAST3.java:1049)
at org.jgroups.protocols.UNICAST3.addMessage(UNICAST3.java:772)
at org.jgroups.protocols.UNICAST3.handleDataReceived(UNICAST3.java:753)
at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:405)
at org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:592)
at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:132)
at org.jgroups.protocols.FailureDetection.up(FailureDetection.java:186)
at org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:254)
at org.jgroups.protocols.MERGE3.up(MERGE3.java:281)
at org.jgroups.protocols.Discovery.up(Discovery.java:300)
at org.jgroups.protocols.TP.passMessageUp(TP.java:1396)
at org.jgroups.util.SubmitToThreadPool$SingleMessageHandler.run(SubmitToThreadPool.java:87)
at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base#11.0.8/ThreadPoolExecutor.java:1128)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base#11.0.8/ThreadPoolExecutor.java:628)
at java.lang.Thread.run(java.base#11.0.8/Thread.java:834)
These threads are hanging on a CompletableFuture.get(...) with a infinispan-hardcoded timeout of 24 hours. This is probably not something that should happen, which is confirmed by the exception we get when the timeout is finally reached:
2021-03-03 09:00:46,638 ERROR [org.infinispan.interceptors.impl.InvocationContextInterceptor] (jgroups-263,xxxxxxxxxxxx) ISPN000136: Error executing command ReadWriteKeyCommand on Cache 'sessionmanager.session', writing keys [xxxx#xxxxxx]: java.lang.IllegalStateException: This should never happen!
at org.infinispan#11.0.8.Final//org.infinispan.util.concurrent.CompletableFutures.await(CompletableFutures.java:127)
at org.infinispan#11.0.8.Final//org.infinispan.util.concurrent.CompletionStages.join(CompletionStages.java:80)
at org.infinispan#11.0.8.Final//org.infinispan.interceptors.distribution.L1NonTxInterceptor.lambda$handleDataWriteCommand$10(L1NonTxInterceptor.java:399)
at org.infinispan#11.0.8.Final//org.infinispan.interceptors.impl.QueueAsyncInvocationStage.invokeQueuedHandlers(QueueAsyncInvocationStage.java:125)
at org.infinispan#11.0.8.Final//org.infinispan.interceptors.impl.QueueAsyncInvocationStage.accept(QueueAsyncInvocationStage.java:88)
at org.infinispan#11.0.8.Final//org.infinispan.interceptors.impl.QueueAsyncInvocationStage.accept(QueueAsyncInvocationStage.java:33)
at java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859)
at java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837)
at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506)
at java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2073)
at org.infinispan#11.0.8.Final//org.infinispan.interceptors.distribution.PrimaryOwnerOnlyCollector.primaryResult(PrimaryOwnerOnlyCollector.java:30)
at org.infinispan#11.0.8.Final//org.infinispan.interceptors.distribution.TriangleDistributionInterceptor.lambda$forwardToPrimary$6(TriangleDistributionInterceptor.java:526)
at java.base/java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:930)
at java.base/java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:907)
at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506)
at java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2073)
at org.infinispan#11.0.8.Final//org.infinispan.remoting.transport.AbstractRequest.complete(AbstractRequest.java:67)
at org.infinispan#11.0.8.Final//org.infinispan.remoting.transport.impl.SingleTargetRequest.onResponse(SingleTargetRequest.java:45)
at org.infinispan#11.0.8.Final//org.infinispan.remoting.transport.impl.RequestRepository.addResponse(RequestRepository.java:52)
at org.infinispan#11.0.8.Final//org.infinispan.remoting.transport.jgroups.JGroupsTransport.processResponse(JGroupsTransport.java:1402)
at org.infinispan#11.0.8.Final//org.infinispan.remoting.transport.jgroups.JGroupsTransport.processMessage(JGroupsTransport.java:1305)
at org.infinispan#11.0.8.Final//org.infinispan.remoting.transport.jgroups.JGroupsTransport.access$300(JGroupsTransport.java:131)
at org.infinispan#11.0.8.Final//org.infinispan.remoting.transport.jgroups.JGroupsTransport$ChannelCallbacks.up(JGroupsTransport.java:1445)
at org.jgroups#4.2.10.Final//org.jgroups.JChannel.up(JChannel.java:784)
at org.jgroups#4.2.10.Final//org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:913)
at org.jgroups#4.2.10.Final//org.jgroups.protocols.FRAG3.up(FRAG3.java:165)
at org.jgroups#4.2.10.Final//org.jgroups.protocols.FlowControl.up(FlowControl.java:343)
at org.jgroups#4.2.10.Final//org.jgroups.protocols.FlowControl.up(FlowControl.java:343)
at org.jgroups#4.2.10.Final//org.jgroups.protocols.pbcast.GMS.up(GMS.java:876)
at org.jgroups#4.2.10.Final//org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:243)
at org.jgroups#4.2.10.Final//org.jgroups.protocols.UNICAST3.deliverMessage(UNICAST3.java:1049)
at org.jgroups#4.2.10.Final//org.jgroups.protocols.UNICAST3.addMessage(UNICAST3.java:772)
at org.jgroups#4.2.10.Final//org.jgroups.protocols.UNICAST3.handleDataReceived(UNICAST3.java:753)
at org.jgroups#4.2.10.Final//org.jgroups.protocols.UNICAST3.up(UNICAST3.java:405)
at org.jgroups#4.2.10.Final//org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:592)
at org.jgroups#4.2.10.Final//org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:132)
at org.jgroups#4.2.10.Final//org.jgroups.protocols.FailureDetection.up(FailureDetection.java:186)
at org.jgroups#4.2.10.Final//org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:254)
at org.jgroups#4.2.10.Final//org.jgroups.protocols.MERGE3.up(MERGE3.java:281)
at org.jgroups#4.2.10.Final//org.jgroups.protocols.Discovery.up(Discovery.java:300)
at org.jgroups#4.2.10.Final//org.jgroups.protocols.TP.passMessageUp(TP.java:1396)
at org.jgroups#4.2.10.Final//org.jgroups.util.SubmitToThreadPool$SingleMessageHandler.run(SubmitToThreadPool.java:87)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:834)
Caused by: java.util.concurrent.TimeoutException
at java.base/java.util.concurrent.CompletableFuture.timedGet(CompletableFuture.java:1886)
at java.base/java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2021)
at org.infinispan#11.0.8.Final//org.infinispan.util.concurrent.CompletableFutures.await(CompletableFutures.java:125)
... 44 more
Have anyone encountered a similar problem? What can possibly lead to this situation?
I'm making a reactive/flux/mono rest service. Backend is Oracle, and using rxjava2-jdbc.
How to get past this blocking error?
I'm learning rx by example, so would be great to know the conceptual details that prevents list manipulation which feels routine-use.
Repository returns a Flux from rx/database:
Handler tries to add that list/flux into another Protobuf object SearchResponse, but fails.
Short stack trance:
Full stack trace:
2020-02-23T10:43:37,967 INFO [main] o.s.d.r.c.RepositoryConfigurationDelegate: Bootstrapping Spring Data JDBC repositories in DEFAULT mode.
2020-02-23T10:43:38,046 INFO [main] o.s.d.r.c.RepositoryConfigurationDelegate: Finished Spring Data repository scanning in 69ms. Found 0 JDBC repository interfaces.
2020-02-23T10:43:38,988 INFO [main] c.z.h.HikariDataSource: HikariPool-1 - Starting...
2020-02-23T10:43:39,334 INFO [main] c.z.h.HikariDataSource: HikariPool-1 - Start completed.
2020-02-23T10:43:40,196 INFO [main] o.s.b.w.e.n.NettyWebServer: Netty started on port(s): 8080
2020-02-23T10:43:40,199 INFO [main] o.s.b.StartupInfoLogger: Started App in 4.298 seconds (JVM running for 5.988)
2020-02-23T10:44:01,307 ERROR [reactor-http-nio-2] o.s.c.l.CompositeLog: [d806b05e] 500 Server Error for HTTP GET "/webflux/customers"
java.lang.IllegalStateException: Iterating over a toIterable() / toStream() is blocking, which is not supported in thread reactor-http-nio-2
at reactor.core.publisher.BlockingIterable$SubscriberIterator.hasNext(BlockingIterable.java:160)
Suppressed: reactor.core.publisher.FluxOnAssembly$OnAssemblyException:
Error has been observed at the following site(s):
|_ checkpoint ⇢ HTTP GET "/webflux/customers" [ExceptionHandlingWebHandler]
Stack trace:
at reactor.core.publisher.BlockingIterable$SubscriberIterator.hasNext(BlockingIterable.java:160)
at com.google.protobuf.AbstractMessageLite$Builder.addAllCheckingNulls(AbstractMessageLite.java:372)
at com.google.protobuf.AbstractMessageLite$Builder.addAll(AbstractMessageLite.java:434)
at pn.api.protobuf.Proto$SearchResponse$Builder.addAllCustomers(Proto.java:3758)
at pn.api.controller.AppHandler.getAllCustomers(AppHandler.java:24)
at org.springframework.web.reactive.function.server.support.HandlerFunctionAdapter.handle(HandlerFunctionAdapter.java:61)
at org.springframework.web.reactive.DispatcherHandler.invokeHandler(DispatcherHandler.java:161)
at org.springframework.web.reactive.DispatcherHandler.lambda$handle$1(DispatcherHandler.java:146)
at reactor.core.publisher.MonoFlatMap$FlatMapMain.onNext(MonoFlatMap.java:118)
at reactor.core.publisher.FluxSwitchIfEmpty$SwitchIfEmptySubscriber.onNext(FluxSwitchIfEmpty.java:67)
at reactor.core.publisher.MonoNext$NextSubscriber.onNext(MonoNext.java:76)
at reactor.core.publisher.FluxConcatMap$ConcatMapImmediate.innerNext(FluxConcatMap.java:274)
at reactor.core.publisher.FluxConcatMap$ConcatMapInner.onNext(FluxConcatMap.java:851)
at reactor.core.publisher.FluxMapFuseable$MapFuseableSubscriber.onNext(FluxMapFuseable.java:121)
at reactor.core.publisher.FluxPeekFuseable$PeekFuseableSubscriber.onNext(FluxPeekFuseable.java:203)
at reactor.core.publisher.Operators$ScalarSubscription.request(Operators.java:2199)
at reactor.core.publisher.FluxPeekFuseable$PeekFuseableSubscriber.request(FluxPeekFuseable.java:137)
at reactor.core.publisher.FluxMapFuseable$MapFuseableSubscriber.request(FluxMapFuseable.java:162)
at reactor.core.publisher.Operators$MultiSubscriptionSubscriber.set(Operators.java:2007)
at reactor.core.publisher.Operators$MultiSubscriptionSubscriber.onSubscribe(Operators.java:1881)
at reactor.core.publisher.FluxMapFuseable$MapFuseableSubscriber.onSubscribe(FluxMapFuseable.java:90)
at reactor.core.publisher.FluxPeekFuseable$PeekFuseableSubscriber.onSubscribe(FluxPeekFuseable.java:171)
at reactor.core.publisher.MonoJust.subscribe(MonoJust.java:54)
at reactor.core.publisher.Mono.subscribe(Mono.java:4105)
at reactor.core.publisher.FluxConcatMap$ConcatMapImmediate.drain(FluxConcatMap.java:441)
at reactor.core.publisher.FluxConcatMap$ConcatMapImmediate.onSubscribe(FluxConcatMap.java:211)
at reactor.core.publisher.FluxIterable.subscribe(FluxIterable.java:139)
at reactor.core.publisher.FluxIterable.subscribe(FluxIterable.java:63)
at reactor.core.publisher.InternalMonoOperator.subscribe(InternalMonoOperator.java:55)
at reactor.core.publisher.MonoDefer.subscribe(MonoDefer.java:52)
at reactor.core.publisher.Mono.subscribe(Mono.java:4105)
at reactor.core.publisher.MonoIgnoreThen$ThenIgnoreMain.drain(MonoIgnoreThen.java:172)
at reactor.core.publisher.MonoIgnoreThen.subscribe(MonoIgnoreThen.java:56)
at reactor.core.publisher.InternalMonoOperator.subscribe(InternalMonoOperator.java:55)
at reactor.netty.http.server.HttpServerHandle.onStateChange(HttpServerHandle.java:64)
at reactor.netty.tcp.TcpServerBind$ChildObserver.onStateChange(TcpServerBind.java:228)
at reactor.netty.http.server.HttpServerOperations.onInboundNext(HttpServerOperations.java:465)
at reactor.netty.channel.ChannelOperationsHandler.channelRead(ChannelOperationsHandler.java:90)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:355)
at reactor.netty.http.server.HttpTrafficHandler.channelRead(HttpTrafficHandler.java:167)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:355)
at io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:436)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:321)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:295)
at io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:251)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:355)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:714)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.base/java.lang.Thread.run(Thread.java:830)
Repository.java
public Flux<Proto.Customer> allCustomers() {//rxjava2 returns Flowable<> ... Flux<>
Flowable<Proto.Customer> customerFlowable =
db.select(queryAllCustomers).get(new CustomerResultSetMapper());
return Flux.from(customerFlowable);
}
AppHandler.java
public Mono<ServerResponse> getAllCustomers(ServerRequest request) {
Flux<Proto.Customer> customers = repository.allCustomers();
Proto.SearchResponse out = Proto.SearchResponse.newBuilder()
.addAllCustomers(customers.toIterable()).build();
return ServerResponse.ok()
.contentType(MediaType.APPLICATION_JSON)
.body(out, Proto.SearchResponse.class);
}
Description:
I am unable to pass MongoDB authorization verification in springboot,but I can pass in the mongo shell environment.
I confirm that the password is correct in SpringBoot
It works well in Robo 3T client
Help:
Did I ignore any other operations?
Env: mongo shell
[dev#local ~]$ mongo 127.0.0.1:27017/wm-mongo -u wm-mongo -p SOh3TbYhx8ypJPxmt1oOfL
MongoDB shell version v3.4.20
connecting to: mongodb://127.0.0.1:27017/wm-mongo
MongoDB server version: 3.4.20
> show collections
ShuJuMoHeMessage
ShuJuMoHeReport
system.users
> db.ShuJuMoHeReport.find()
> db.ShuJuMoHeReport.count()
0
>
Env: Spring Boot
spring.data.mongodb.uri=mongodb://wm-mongo:SOh3TbYhx8ypJPxmt1oOfL#127.0.0.1:27017/wm-mongo
Caused by: com.mongodb.MongoSecurityException: Exception authenticating MongoCredential{mechanism=SCRAM-SHA-1, userName='wm-mongo', source='wm-mongo', password=<hidden>, mechanismProperties={}}
at com.mongodb.internal.connection.SaslAuthenticator.wrapException(SaslAuthenticator.java:173)
at com.mongodb.internal.connection.SaslAuthenticator.access$300(SaslAuthenticator.java:40)
at com.mongodb.internal.connection.SaslAuthenticator$1.run(SaslAuthenticator.java:70)
at com.mongodb.internal.connection.SaslAuthenticator$1.run(SaslAuthenticator.java:47)
at com.mongodb.internal.connection.SaslAuthenticator.doAsSubject(SaslAuthenticator.java:179)
at com.mongodb.internal.connection.SaslAuthenticator.authenticate(SaslAuthenticator.java:47)
at com.mongodb.internal.connection.InternalStreamConnectionInitializer.authenticateAll(InternalStreamConnectionInitializer.java:151)
at com.mongodb.internal.connection.InternalStreamConnectionInitializer.initialize(InternalStreamConnectionInitializer.java:64)
at com.mongodb.internal.connection.InternalStreamConnection.open(InternalStreamConnection.java:127)
at com.mongodb.internal.connection.UsageTrackingInternalConnection.open(UsageTrackingInternalConnection.java:50)
at com.mongodb.internal.connection.DefaultConnectionPool$PooledConnection.open(DefaultConnectionPool.java:390)
at com.mongodb.internal.connection.DefaultConnectionPool.get(DefaultConnectionPool.java:106)
at com.mongodb.internal.connection.DefaultConnectionPool.get(DefaultConnectionPool.java:92)
at com.mongodb.internal.connection.DefaultServer.getConnection(DefaultServer.java:85)
at com.mongodb.binding.ClusterBinding$ClusterBindingConnectionSource.getConnection(ClusterBinding.java:114)
at com.mongodb.operation.OperationHelper.withConnectionSource(OperationHelper.java:451)
at com.mongodb.operation.OperationHelper.withConnection(OperationHelper.java:415)
at com.mongodb.operation.CreateIndexesOperation.execute(CreateIndexesOperation.java:169)
at com.mongodb.operation.CreateIndexesOperation.execute(CreateIndexesOperation.java:70)
at com.mongodb.client.internal.MongoClientDelegate$DelegateOperationExecutor.execute(MongoClientDelegate.java:193)
at com.mongodb.client.internal.MongoCollectionImpl.executeCreateIndexes(MongoCollectionImpl.java:805)
at com.mongodb.client.internal.MongoCollectionImpl.createIndexes(MongoCollectionImpl.java:788)
at com.mongodb.client.internal.MongoCollectionImpl.createIndexes(MongoCollectionImpl.java:783)
at com.mongodb.client.internal.MongoCollectionImpl.createIndex(MongoCollectionImpl.java:768)
at org.springframework.data.mongodb.core.DefaultIndexOperations.lambda$ensureIndex$0(DefaultIndexOperations.java:135)
at org.springframework.data.mongodb.core.MongoTemplate.execute(MongoTemplate.java:535)
... 83 common frames omitted
Caused by: com.mongodb.MongoCommandException: Command failed with error 18 (AuthenticationFailed): 'Authentication failed.' on server 127.0.0.1:27017. The full response is { "ok" : 0.0, "errmsg" : "Authentication failed.", "code" : 18, "codeName" : "AuthenticationFailed" }
at com.mongodb.internal.connection.ProtocolHelper.getCommandFailureException(ProtocolHelper.java:179)
at com.mongodb.internal.connection.InternalStreamConnection.receiveCommandMessageResponse(InternalStreamConnection.java:293)
at com.mongodb.internal.connection.InternalStreamConnection.sendAndReceive(InternalStreamConnection.java:255)
at com.mongodb.internal.connection.CommandHelper.sendAndReceive(CommandHelper.java:83)
at com.mongodb.internal.connection.CommandHelper.executeCommand(CommandHelper.java:33)
at com.mongodb.internal.connection.SaslAuthenticator.sendSaslContinue(SaslAuthenticator.java:134)
at com.mongodb.internal.connection.SaslAuthenticator.access$200(SaslAuthenticator.java:40)
at com.mongodb.internal.connection.SaslAuthenticator$1.run(SaslAuthenticator.java:67)
... 106 common frames omitted
When i run the JDeveloper( version 12c) and use ADF framework to create a web application, i create a business components and use the data control to drag and drop table from database, but when i run the application the web-logic give me this error (Error 500--Internal Server Error), Below is the full Error Log. Please help me to solve the problem
Error 500--Internal Server Error
oracle.jbo.JboException: JBO-29114 ADFContext is not setup to process messages for this exception. Use the exception stack trace and error code to investigate the root cause of this exception. Root cause error code is JBO-29000. Error message parameters are {0=oracle.adf.model.adapter.AdapterException, 1=DCA-40002: The WSDL document is invalid due to the following reason : null.}
at oracle.adf.model.adapter.DataControlFactoryImpl.createSession(DataControlFactoryImpl.java:228)
at oracle.adf.model.binding.DCDataControlReference.getDataControl(DCDataControlReference.java:274)
at oracle.adf.model.BindingContext.instantiateDataControl(BindingContext.java:1416)
at oracle.adf.model.dcframe.DataControlFrameImpl.doFindDataControl(DataControlFrameImpl.java:1906)
at oracle.adf.model.dcframe.DataControlFrameImpl.internalFindDataControl(DataControlFrameImpl.java:1768)
at oracle.adf.model.dcframe.DataControlFrameImpl.findDataControl(DataControlFrameImpl.java:1728)
at oracle.adf.model.BindingContext.internalFindDataControl(BindingContext.java:1549)
at oracle.adf.model.BindingContext.get(BindingContext.java:1499)
at oracle.adf.model.binding.DCUtil.findSpelObject(DCUtil.java:340)
at oracle.adf.model.binding.DCBindingContainer.evaluateParameterWithElCheck(DCBindingContainer.java:1521)
at oracle.adf.model.binding.DCBindingContainer.findDataControl(DCBindingContainer.java:1627)
at oracle.jbo.uicli.binding.JUCtrlActionBinding.internalGetDataControl(JUCtrlActionBinding.java:588)
at oracle.jbo.uicli.binding.JUCtrlActionBinding.getDataControl(JUCtrlActionBinding.java:648)
at oracle.jbo.uicli.binding.JUCtrlActionBinding.isOperationEnabled(JUCtrlActionBinding.java:336)
at oracle.jbo.uicli.binding.JUCtrlActionBinding.isActionEnabled(JUCtrlActionBinding.java:300)
at oracle.jbo.uicli.binding.JUCtrlActionBinding.getEnabled(JUCtrlActionBinding.java:1922)
at oracle.jbo.uicli.binding.JUCtrlActionBinding.internalGet(JUCtrlActionBinding.java:2016)
at oracle.adfinternal.view.faces.model.binding.FacesCtrlActionBinding.internalGet(FacesCtrlActionBinding.java:368)
at oracle.adf.model.binding.DCControlBinding.get(DCControlBinding.java:763)
at javax.el.MapELResolver.getValue(MapELResolver.java:199)
at com.sun.faces.el.DemuxCompositeELResolver._getValue(DemuxCompositeELResolver.java:176)
at com.sun.faces.el.DemuxCompositeELResolver.getValue(DemuxCompositeELResolver.java:203)
at com.sun.el.parser.AstValue.getValue(AstValue.java:139)
at com.sun.el.parser.AstValue.getValue(AstValue.java:203)
at com.sun.el.parser.AstNot.getValue(AstNot.java:63)
at com.sun.el.ValueExpressionImpl.getValue(ValueExpressionImpl.java:226)
at com.sun.faces.facelets.el.TagValueExpression.getValue(TagValueExpression.java:109)
at org.apache.myfaces.trinidad.bean.FacesBeanImpl.getProperty(FacesBeanImpl.java:73)
at oracle.adfinternal.view.faces.renderkit.rich.ButtonRenderer.getDisabled(ButtonRenderer.java:449)
at oracle.adfinternal.view.faces.renderkit.rich.ButtonRenderer.encodeAll(ButtonRenderer.java:284)
at oracle.adf.view.rich.render.RichRenderer.encodeAll(RichRenderer.java:1650)
at org.apache.myfaces.trinidad.render.CoreRenderer.encodeEnd(CoreRenderer.java:538)
at org.apache.myfaces.trinidad.component.UIXComponentBase.encodeEnd(UIXComponentBase.java:1230)
at javax.faces.component.UIComponent.encodeAll(UIComponent.java:1863)
at org.apache.myfaces.trinidad.render.CoreRenderer.encodeChild(CoreRenderer.java:660)
at org.apache.myfaces.trinidad.render.CoreRenderer.encodeAllChildren(CoreRenderer.java:677)
at oracle.adf.view.rich.render.RichRenderer.encodeAllChildrenInContext(RichRenderer.java:3284)
at oracle.adfinternal.view.faces.renderkit.rich.FormRenderer.encodeAll(FormRenderer.java:275)
at oracle.adf.view.rich.render.RichRenderer.encodeAll(RichRenderer.java:1650)
at org.apache.myfaces.trinidad.render.CoreRenderer.encodeEnd(CoreRenderer.java:538)
at org.apache.myfaces.trinidad.component.UIXComponentBase.encodeEnd(UIXComponentBase.java:1230)
at javax.faces.component.UIComponent.encodeAll(UIComponent.java:1863)
at org.apache.myfaces.trinidad.render.CoreRenderer.encodeChild(CoreRenderer.java:660)
at org.apache.myfaces.trinidad.render.CoreRenderer.encodeAllChildren(CoreRenderer.java:677)
at oracle.adf.view.rich.render.RichRenderer.encodeAllChildrenInContext(RichRenderer.java:3284)
at oracle.adfinternal.view.faces.renderkit.rich.DocumentRenderer.encodeAll(DocumentRenderer.java:1428)
at oracle.adf.view.rich.render.RichRenderer.encodeAll(RichRenderer.java:1650)
at org.apache.myfaces.trinidad.render.CoreRenderer.encodeEnd(CoreRenderer.java:538)
at org.apache.myfaces.trinidad.component.UIXComponentBase.encodeEnd(UIXComponentBase.java:1230)
at javax.faces.component.UIComponent.encodeAll(UIComponent.java:1863)
at javax.faces.component.UIComponent.encodeAll(UIComponent.java:1859)
at oracle.adfinternal.view.faces.component.AdfViewRoot.encodeAll(AdfViewRoot.java:102)
at com.sun.faces.application.view.FaceletViewHandlingStrategy.renderView(FaceletViewHandlingStrategy.java:458)
at org.apache.myfaces.trinidad.view.ViewDeclarationLanguageWrapper.renderView(ViewDeclarationLanguageWrapper.java:101)
at org.apache.myfaces.trinidad.view.ViewDeclarationLanguageWrapper.renderView(ViewDeclarationLanguageWrapper.java:101)
at org.apache.myfaces.trinidadinternal.application.ViewDeclarationLanguageFactoryImpl$ChangeApplyingVDLWrapper.renderView(ViewDeclarationLanguageFactoryImpl.java:338)
at com.sun.faces.application.view.MultiViewHandler.renderView(MultiViewHandler.java:134)
at javax.faces.application.ViewHandlerWrapper.renderView(ViewHandlerWrapper.java:337)
at org.apache.myfaces.trinidadinternal.application.ViewHandlerImpl.renderView(ViewHandlerImpl.java:170)
at oracle.adfinternal.view.faces.lifecycle.ResponseRenderManager.runRenderView(ResponseRenderManager.java:52)
at oracle.adfinternal.view.faces.lifecycle.LifecycleImpl._renderResponse(LifecycleImpl.java:1228)
at oracle.adfinternal.view.faces.lifecycle.LifecycleImpl._executeRenderResponse(LifecycleImpl.java:1040)
at oracle.adfinternal.view.faces.lifecycle.LifecycleImpl._executePhase(LifecycleImpl.java:332)
at oracle.adfinternal.view.faces.lifecycle.LifecycleImpl.render(LifecycleImpl.java:254)
at javax.faces.webapp.FacesServlet.service(FacesServlet.java:651)
at weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:286)
at weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:260)
at weblogic.servlet.internal.StubSecurityHelper.invokeServlet(StubSecurityHelper.java:137)
at weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:350)
at weblogic.servlet.internal.TailFilter.doFilter(TailFilter.java:25)
at weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:78)
at oracle.adf.model.servlet.ADFBindingFilter.doFilter(ADFBindingFilter.java:194)
at weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:78)
at oracle.adfinternal.view.faces.webapp.rich.RegistrationFilter.doFilter(RegistrationFilter.java:105)
at org.apache.myfaces.trinidadinternal.webapp.TrinidadFilterImpl$FilterListChain.doFilter(TrinidadFilterImpl.java:529)
at oracle.adfinternal.view.faces.activedata.AdsFilter.doFilter(AdsFilter.java:60)
at org.apache.myfaces.trinidadinternal.webapp.TrinidadFilterImpl$FilterListChain.doFilter(TrinidadFilterImpl.java:529)
at org.apache.myfaces.trinidadinternal.webapp.TrinidadFilterImpl._doFilterImpl(TrinidadFilterImpl.java:354)
at org.apache.myfaces.trinidadinternal.webapp.TrinidadFilterImpl.doFilter(TrinidadFilterImpl.java:232)
at org.apache.myfaces.trinidad.webapp.TrinidadFilter.doFilter(TrinidadFilter.java:92)
at weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:78)
at oracle.security.jps.ee.http.JpsAbsFilter$1.run(JpsAbsFilter.java:141)
at java.security.AccessController.doPrivileged(Native Method)
at oracle.security.jps.util.JpsSubject.doAsPrivileged(JpsSubject.java:315)
at oracle.security.jps.ee.util.JpsPlatformUtil.runJaasMode(JpsPlatformUtil.java:649)
at oracle.security.jps.ee.http.JpsAbsFilter.runJaasMode(JpsAbsFilter.java:124)
at oracle.security.jps.ee.http.JpsAbsFilter.doFilter(JpsAbsFilter.java:232)
at oracle.security.jps.ee.http.JpsFilter.doFilter(JpsFilter.java:94)
at weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:78)
at oracle.dms.servlet.DMSServletFilter.doFilter(DMSServletFilter.java:224)
at weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:78)
at weblogic.servlet.internal.RequestEventsFilter.doFilter(RequestEventsFilter.java:32)
at weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:78)
at weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.wrapRun(WebAppServletContext.java:3654)
at weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.run(WebAppServletContext.java:3620)
at weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:326)
at weblogic.security.service.SecurityManager.runAsForUserCode(SecurityManager.java:196)
at weblogic.servlet.provider.WlsSecurityProvider.runAsForUserCode(WlsSecurityProvider.java:203)
at weblogic.servlet.provider.WlsSubjectHandle.run(WlsSubjectHandle.java:71)
at weblogic.servlet.internal.WebAppServletContext.doSecuredExecute(WebAppServletContext.java:2423)
at weblogic.servlet.internal.WebAppServletContext.securedExecute(WebAppServletContext.java:2280)
at weblogic.servlet.internal.WebAppServletContext.execute(WebAppServletContext.java:2258)
at weblogic.servlet.internal.ServletRequestImpl.runInternal(ServletRequestImpl.java:1626)
at weblogic.servlet.internal.ServletRequestImpl.run(ServletRequestImpl.java:1586)
at weblogic.servlet.provider.ContainerSupportProviderImpl$WlsRequestExecutor.run(ContainerSupportProviderImpl.java:270)
at weblogic.invocation.ComponentInvocationContextManager._runAs(ComponentInvocationContextManager.java:348)
at weblogic.invocation.ComponentInvocationContextManager.runAs(ComponentInvocationContextManager.java:333)
at weblogic.work.LivePartitionUtility.doRunWorkUnderContext(LivePartitionUtility.java:54)
at weblogic.work.PartitionUtility.runWorkUnderContext(PartitionUtility.java:41)
at weblogic.work.SelfTuningWorkManagerImpl.runWorkUnderContext(SelfTuningWorkManagerImpl.java:617)
at weblogic.work.ExecuteThread.execute(ExecuteThread.java:397)
at weblogic.work.ExecuteThread.run(ExecuteThread.java:346)
Caused by: oracle.adf.model.adapter.AdapterException: JBO-29114 ADFContext is not setup to process messages for this exception. Use the exception stack trace and error code to investigate the root cause of this exception. Root cause error code is DCA-40002. Error message parameters are {0=null}
at oracle.adfinternal.model.adapter.webservice.WSModel.createModelFromWSDL(WSModel.java:725)
at oracle.adfinternal.model.adapter.webservice.WSModel.(WSModel.java:215)
at oracle.adfinternal.model.adapter.webservice.WSDefinition.createWSModel(WSDefinition.java:1922)
at oracle.adfinternal.model.adapter.webservice.WSDefinition.loadDefaultModel(WSDefinition.java:1917)
at oracle.adfinternal.model.adapter.webservice.WSDefinition.loadDCOperations(WSDefinition.java:1905)
at oracle.adfinternal.model.adapter.webservice.WSDefinition.createDataControl(WSDefinition.java:800)
at oracle.adf.model.adapter.DataControlFactoryImpl.createSession(DataControlFactoryImpl.java:202)
... 111 more
Try using https://www.wsdl-analyzer.com/ to analyze whether WSDL is correct / service is up etc .