ActiveMQ Artemis + JGroups + Getting OutOfMemory Error - activemq-artemis

Question

I am using ActiveMQ Artemis 2.11.0 (having JGroups version 3.6.13) and run Artemis in a master/slave configuration. There are two masters and each master has a slave. Please find attached screenshot of OutOfMemory error.
I have tried to reproduce this issue but not able to reproduce it.
Please find below jgroups configuration:
<config xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="urn:org:jgroups"
xsi:schemaLocation="urn:org:jgroups http://www.jgroups.org/schema/jgroups.xsd">
<TCP loopback="true" recv_buf_size="${tcp.recv_buf_size:5M}"
send_buf_size="${tcp.send_buf_size:5M}" max_bundle_size="64K"
max_bundle_timeout="30" use_send_queues="true" sock_conn_timeout="300"
timer_type="new3" timer.min_threads="4" timer.max_threads="10"
timer.keep_alive_time="3000" timer.queue_max_size="500"
thread_pool.enabled="true" thread_pool.min_threads="2"
thread_pool.max_threads="8" thread_pool.keep_alive_time="5000"
thread_pool.queue_enabled="true" thread_pool.queue_max_size="10000"
thread_pool.rejection_policy="discard" oob_thread_pool.enabled="true"
oob_thread_pool.min_threads="1" oob_thread_pool.max_threads="8"
oob_thread_pool.keep_alive_time="5000"
oob_thread_pool.queue_enabled="false"
oob_thread_pool.queue_max_size="100"
oob_thread_pool.rejection_policy="discard" />
<!-- a location that can be found by both server's running -->
<FILE_PING location="/opt/shared/fileping/artemis"
remove_all_files_on_view_change="true" />
<MERGE3 min_interval="60000" max_interval="90000" />
<FD_SOCK />
<FD_ALL timeout="30000" interval="15000"
timeout_check_interval="30000" />
<VERIFY_SUSPECT timeout="15000" />
<RSVP />
<pbcast.NAKACK2 use_mcast_xmit="false"
log_discard_msgs="false" max_rebroadcast_timeout="10000"
log_not_found_msgs="false" />
<UNICAST2 log_not_found_msgs="false" />
<pbcast.STABLE stability_delay="10000"
desired_avg_gossip="50000" max_bytes="4M" />
<pbcast.GMS print_local_addr="true" join_timeout="10000"
leave_timeout="10000" view_ack_collection_timeout="10000"
view_bundling="true" />
<UFC max_credits="2M" min_threshold="0.4" />
<FRAG2 frag_size="60K" />
<pbcast.STATE_TRANSFER />
master-broker.xml
<configuration xmlns="urn:activemq"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:xi="http://www.w3.org/2001/XInclude"
xsi:schemaLocation="urn:activemq /schema/artemis-configuration.xsd">
<core xmlns="urn:activemq:core"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="urn:activemq:core ">
<persistence-enabled>true</persistence-enabled>
<thread-pool-max-size>200</thread-pool-max-size>
<address-queue-scan-period>-1</address-queue-scan-period>
<journal-type>NIO</journal-type>
<paging-directory>${data.dir}/paging</paging-directory>
<bindings-directory>${data.dir}/bindings</bindings-directory>
<journal-directory>${data.dir}/journal</journal-directory>
<node-manager-lock-directory>${lock.dir}/artemis1-lock</node-manager-lock-directory>
<large-messages-directory>${data.dir}/large-messages</large-messages-directory>
<journal-datasync>true</journal-datasync>
<journal-min-files>50</journal-min-files>
<journal-pool-files>100</journal-pool-files>
<journal-file-size>10M</journal-file-size>
<page-max-concurrent-io>10</page-max-concurrent-io>
<journal-compact-min-files>150</journal-compact-min-files>
<journal-buffer-timeout>23480000</journal-buffer-timeout>
<journal-max-io>1</journal-max-io>
<disk-scan-period>5000</disk-scan-period>
<max-disk-usage>90</max-disk-usage>
<critical-analyzer>true</critical-analyzer>
<critical-analyzer-timeout>120000</critical-analyzer-timeout>
<critical-analyzer-check-period>60000</critical-analyzer-check-period>
<critical-analyzer-policy>LOG</critical-analyzer-policy>
<transaction-timeout>1800000</transaction-timeout>
<connectors>
<connector name="netty-connector">tcp://localhost:61616</connector>
</connectors>
<!-- Acceptors -->
<acceptors>
<acceptor name="netty-acceptor">tcp://localhost:61616?anycastPrefix=jms.queue.;multicastPrefix=jms.topic.
</acceptor>
</acceptors>
<ha-policy>
<shared-store>
<master>
<failover-on-shutdown>true</failover-on-shutdown>
</master>
</shared-store>
</ha-policy>
<broadcast-groups>
<broadcast-group name="my-broadcast-group">
<broadcast-period>5000</broadcast-period>
<jgroups-file>jgroups.xml</jgroups-file>
<jgroups-channel>persistence-fs</jgroups-channel>
<connector-ref>netty-connector</connector-ref>
</broadcast-group>
</broadcast-groups>
<discovery-groups>
<discovery-group name="my-discovery-group">
<jgroups-file>idsk-jgroups.xml</jgroups-file>
<jgroups-channel>persistence-fs</jgroups-channel>
<refresh-timeout>10000</refresh-timeout>
</discovery-group>
</discovery-groups>
<cluster-connections>
<cluster-connection name="my-cluster">
<connector-ref>netty-connector</connector-ref>
<retry-interval>500</retry-interval>
<use-duplicate-detection>true</use-duplicate-detection>
<message-load-balancing>ON_DEMAND</message-load-balancing>
<max-hops>1</max-hops>
<discovery-group-ref discovery-group-name="my-discovery-group" />
</cluster-connection>
</cluster-connections>
<cluster-user>admin</cluster-user>
<cluster-password>admin</cluster-password>
<security-settings>
<security-setting match="#">
<permission type="createNonDurableQueue" roles="amq" />
<permission type="deleteNonDurableQueue" roles="amq" />
<permission type="createDurableQueue" roles="amq" />
<permission type="deleteDurableQueue" roles="amq" />
<permission type="createAddress" roles="amq" />
<permission type="deleteAddress" roles="amq" />
<permission type="consume" roles="amq" />
<permission type="browse" roles="amq" />
<permission type="send" roles="amq" />
<permission type="manage" roles="amq" />
</security-setting>
</security-settings>
<address-settings>
<address-setting match="activemq.management#">
<dead-letter-address>DLQ</dead-letter-address>
<expiry-address>ExpiryQueue</expiry-address>
<redelivery-delay>60000</redelivery-delay>
<max-delivery-attempts>5</max-delivery-attempts>
<max-size-bytes>50485760</max-size-bytes>
<page-size-bytes>10485760</page-size-bytes>
<message-counter-history-day-limit>10
</message-counter-history-day-limit>
<address-full-policy>PAGE</address-full-policy>
<auto-create-queues>true</auto-create-queues>
<auto-create-addresses>false</auto-create-addresses>
</address-setting>
<address-setting match="#">
<dead-letter-address>DLQ</dead-letter-address>
<expiry-address>ExpiryQueue</expiry-address>
<redelivery-delay>60000</redelivery-delay>
<max-delivery-attempts>5</max-delivery-attempts>
<redistribution-delay>10000</redistribution-delay>
<max-size-bytes>50485760</max-size-bytes>
<page-size-bytes>10485760</page-size-bytes>
<message-counter-history-day-limit>10
</message-counter-history-day-limit>
<address-full-policy>PAGE</address-full-policy>
<auto-create-queues>true</auto-create-queues>
<auto-create-addresses>false</auto-create-addresses>
</address-setting>
</address-settings>
<addresses>
<address name="DLQ">
<anycast>
<queue name="DLQ" />
</anycast>
</address>
<address name="ExpiryQueue">
<anycast>
<queue name="ExpiryQueue" />
</anycast>
</address>
</addresses>
</core>
slave-broker.xml
<configuration xmlns="urn:activemq"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:xi="http://www.w3.org/2001/XInclude"
xsi:schemaLocation="urn:activemq /schema/artemis-configuration.xsd">
<core xmlns="urn:activemq:core"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="urn:activemq:core ">
<persistence-enabled>true</persistence-enabled>
<thread-pool-max-size>200</thread-pool-max-size>
<address-queue-scan-period>-1</address-queue-scan-period>
<journal-type>NIO</journal-type>
<paging-directory>${data.dir}/paging</paging-directory>
<bindings-directory>${data.dir}/bindings</bindings-directory>
<journal-directory>${data.dir}/journal</journal-directory>
<node-manager-lock-directory>${lock.dir}/artemis1-lock</node-manager-lock-directory>
<large-messages-directory>${data.dir}/large-messages</large-messages-directory>
<journal-datasync>true</journal-datasync>
<journal-min-files>50</journal-min-files>
<journal-pool-files>100</journal-pool-files>
<journal-file-size>10M</journal-file-size>
<page-max-concurrent-io>10</page-max-concurrent-io>
<journal-compact-min-files>150</journal-compact-min-files>
<journal-buffer-timeout>23480000</journal-buffer-timeout>
<journal-max-io>1</journal-max-io>
<server-dump-interval>60000</server-dump-interval>
<disk-scan-period>5000</disk-scan-period>
<max-disk-usage>90</max-disk-usage>
<critical-analyzer>true</critical-analyzer>
<critical-analyzer-timeout>120000</critical-analyzer-timeout>
<critical-analyzer-check-period>60000</critical-analyzer-check-period>
<critical-analyzer-policy>LOG</critical-analyzer-policy>
<transaction-timeout>1800000</transaction-timeout>
<connectors>
<connector name="netty-connector">tcp://localhost:61618</connector>
</connectors>
<!-- Acceptors -->
<acceptors>
<acceptor name="netty-acceptor">tcp://localhost:61618?anycastPrefix=jms.queue.;multicastPrefix=jms.topic.
</acceptor>
</acceptors>
<ha-policy>
<shared-store>
<slave>
<failover-on-shutdown>true</failover-on-shutdown>
<restart-backup>true</restart-backup>
<allow-failback>true</allow-failback>
</slave>
</shared-store>
</ha-policy>
<broadcast-groups>
<broadcast-group name="my-broadcast-group">
<broadcast-period>5000</broadcast-period>
<jgroups-file>idsk-jgroups.xml</jgroups-file>
<jgroups-channel>persistence-fs</jgroups-channel>
<connector-ref>netty-connector</connector-ref>
</broadcast-group>
</broadcast-groups>
<discovery-groups>
<discovery-group name="my-discovery-group">
<jgroups-file>jgroups.xml</jgroups-file>
<jgroups-channel>persistence-fs</jgroups-channel>
<refresh-timeout>10000</refresh-timeout>
</discovery-group>
</discovery-groups>
<cluster-connections>
<cluster-connection name="my-cluster">
<connector-ref>netty-connector</connector-ref>
<retry-interval>500</retry-interval>
<use-duplicate-detection>true</use-duplicate-detection>
<message-load-balancing>ON_DEMAND</message-load-balancing>
<max-hops>1</max-hops>
<discovery-group-ref discovery-group-name="my-discovery-group" />
</cluster-connection>
</cluster-connections>
<cluster-user>admin</cluster-user>
<cluster-password>admin</cluster-password>
<security-settings>
<security-setting match="#">
<permission type="createNonDurableQueue" roles="amq" />
<permission type="deleteNonDurableQueue" roles="amq" />
<permission type="createDurableQueue" roles="amq" />
<permission type="deleteDurableQueue" roles="amq" />
<permission type="createAddress" roles="amq" />
<permission type="deleteAddress" roles="amq" />
<permission type="consume" roles="amq" />
<permission type="browse" roles="amq" />
<permission type="send" roles="amq" />
<permission type="manage" roles="amq" />
</security-setting>
</security-settings>
<address-settings>
<address-setting match="activemq.management#">
<dead-letter-address>DLQ</dead-letter-address>
<expiry-address>ExpiryQueue</expiry-address>
<redelivery-delay>60000</redelivery-delay>
<max-delivery-attempts>5</max-delivery-attempts>
<!-- with -1 only the global-max-size is in use for limiting -->
<max-size-bytes>50485760</max-size-bytes>
<page-size-bytes>10485760</page-size-bytes>
<message-counter-history-day-limit>10
</message-counter-history-day-limit>
<address-full-policy>PAGE</address-full-policy>
<auto-create-queues>true</auto-create-queues>
<auto-create-addresses>false</auto-create-addresses>
</address-setting>
<!--default for catch all -->
<address-setting match="#">
<dead-letter-address>DLQ</dead-letter-address>
<expiry-address>ExpiryQueue</expiry-address>
<redelivery-delay>60000</redelivery-delay>
<max-delivery-attempts>5</max-delivery-attempts>
<redistribution-delay>10000</redistribution-delay>
<!-- with -1 only the global-max-size is in use for limiting -->
<max-size-bytes>50485760</max-size-bytes>
<page-size-bytes>10485760</page-size-bytes>
<message-counter-history-day-limit>10</message-counter-history-day-limit>
<address-full-policy>PAGE</address-full-policy>
<auto-create-queues>true</auto-create-queues>
<auto-create-addresses>false</auto-create-addresses>
</address-setting>
</address-settings>
</core>

Answer 1

Check if the cluster have started first, it can be an issue of UDP multicast, my friend one question the only shared directory you use is ther artemis1-lock??