How to setup Spring Batch with DbUnit? - spring-batch

I'm trying to test spring batch using dbunit but when batch runs HibernateCursorItemReader it doesn't see temporary created objects in the db. But if I create entities using HibernateDaoSupport everything is fine and I can work with test data. Does anybody know how to force batch see my test entities?
My test classes:
#RunWith(SpringEasyMockRunner.class)
#TestExecutionListeners({DependencyInjectionTestExecutionListener.class, TransactionalTestExecutionListener.class, DbUnitTestExecutionListener.class})
#Transactional
#ContextConfiguration({"classpath:/integration-test-context.xml"})
#DbUnitConfiguration(
databaseConnection = {"dbUnitDatabaseConnection"}
)
public abstract class BaseIT implements ApplicationContextAware {
...
}
#RunWith(SpringEasyMockRunner.class)
#ContextConfiguration("classpath:/sopl-test-context.xml")
#DatabaseSetup(value = "classpath:data/SoplTestDataIT.xml", type = DatabaseOperation.CLEAN_INSERT)
#Slf4j
public class SoplOrderTestIT extends BaseIT {
...
}
And configs integration-test-context.xml:
<bean id="dataSource" class="com.mchange.v2.c3p0.ComboPooledDataSource" lazy-init="true" destroy-method="close">
<property name="driverClass" value="${hibernate.driver}"/>
<property name="jdbcUrl" value="${hibernate.connection.url}"/>
<property name="user" value="${hibernate.connection.username}"/>
<property name="password" value="${hibernate.connection.password}"/>
<property name="initialPoolSize" value="${c3p0.initialPoolSize:1}"/>
<property name="minPoolSize" value="${c3p0.minPoolSize:0}"/>
<property name="maxPoolSize" value="${c3p0.maxPoolSize:200}"/>
<property name="maxIdleTime" value="${c3p0.maxIdleTime:120}"/>
<property name="maxStatementsPerConnection" value="${c3p0.maxStatementsPerConnection:100}"/>
<property name="checkoutTimeout" value="${c3p0.checkoutTimeout:30000}"/>
<property name="idleConnectionTestPeriod" value="${c3p0.idleConnectionTestPeriod:3600}"/>
<property name="unreturnedConnectionTimeout" value="${c3p0.unreturnedConnectionTimeout:7200}"/>
<property name="debugUnreturnedConnectionStackTraces"
value="${c3p0.debugUnreturnedConnectionStackTraces:true}"/>
</bean>
<bean name="sessionFactory" class="org.springframework.orm.hibernate4.LocalSessionFactoryBean">
<property name="dataSource" ref="dataSource"/>
<property name="namingStrategy">
<bean class="org.hibernate.cfg.DefaultComponentSafeNamingStrategy"/>
</property>
<property name="configLocations" value="classpath:/hibernate/*-hibernate.cfg.xml"/>
<property name="hibernateProperties">
<props merge="true">
<prop key="hibernate.dialect">org.hibernate.dialect.MySQLInnoDBDialect</prop>
<prop key="hibernate.cache.use_second_level_cache">false</prop>
<prop key="hibernate.show_sql">${hibernate.show_sql:false}</prop>
<prop key="hibernate.format_sql">${hibernate.format_sql:false}</prop>
<prop key="hibernate.use_sql_comments">${hibernate.use_sql_comments:false}</prop>
<prop key="hibernate.generate_statistics">${hibernate.generate_statistics:false}</prop>
<prop key="hibernate.hbm2ddl.auto"/>
<prop key="hibernate.search.autoregister_listeners">false</prop>
</props>
</property>
</bean>
<!-- transaction manager for hibernate -->
<bean id="txManager" class="org.springframework.orm.hibernate4.HibernateTransactionManager" primary="true">
<property name="sessionFactory" ref="sessionFactory"/>
</bean>
<!-- spring-test-dbunit configuration: -->
<bean id="dbUnitDatabaseConfig" class="com.github.springtestdbunit.bean.DatabaseConfigBean">
<property name="qualifiedTableNames" value="true"/>
<property name="metadataHandler">
<bean class="org.dbunit.ext.mysql.MySqlMetadataHandler"/>
</property>
<property name="datatypeFactory">
<bean class="org.dbunit.ext.mysql.MySqlDataTypeFactory"/>
</property>
</bean>
<bean id="dbUnitDatabaseConnection"
class="com.github.springtestdbunit.bean.DatabaseDataSourceConnectionFactoryBean">
<property name="databaseConfig" ref="dbUnitDatabaseConfig"/>
<property name="dataSource" ref="dataSource"/>
</bean>
...

Related

Transaction in jboss eap JmsComponent camel context

I'm having troubles setting up transactions using JtaTransactionManager. I've tried everything I could find in internet.
I want to make rollback if there is any Exception in the route or in the routes coming from this route.
Here is my camel context beans:
<bean id="jmsConnectionFactory" class="org.springframework.jndi.JndiObjectFactoryBean">
<property name="jndiName" value="java:/JmsMQXA" />
<property name="lookupOnStartup" value="false" />
<property name="cache" value="true" />
<property name="proxyInterface" value="javax.jms.ConnectionFactory" />
</bean>
<bean id="jmsTransactionManager" class="org.springframework.transaction.jta.JtaTransactionManager">
<property name="transactionManagerName" value="java:/TransactionManager" />
</bean>
<bean id="mq" class="org.apache.camel.component.jms.JmsComponent">
<property name="connectionFactory" ref="jmsConnectionFactory" />
<property name="transacted" value="true"/>
<property name="cacheLevelName" value="CACHE_NONE" />
<property name="transactionManager" ref="jmsTransactionManager" />
</bean>
<bean id="PROPAGATION_REQUIRED" class="org.apache.camel.spring.spi.SpringTransactionPolicy">
<property name="transactionManager" ref="jmsTransactionManager" />
<property name="propagationBehaviorName" value="PROPAGATION_REQUIRES_NEW" />
</bean>
My route:
<route>
<from uri="mq:TEST.QUEUE" />
<transacted ref="PROPAGATION_REQUIRED"/>
MY standalone
<subsystem xmlns="urn:jboss:domain:resource-adapters:5.0">
<resource-adapters>
<resource-adapter id="wmq.jmsra.rar">
<archive>
wmq.jmsra.rar
</archive>
<transaction-support>XATransaction</transaction-support>...
And the problems I get:
WARN [org.apache.camel.spring.spi.TransactionErrorHandler] (Camel (camel-1) thread #1 - JmsConsumer[TEST.QUEUE]) Transaction rollback (0x68e69794) redelivered(true) for (MessageId: ID:414d51205141424749434c4441202020ab7e795dfb630223 on ExchangeId: ID-M0AB1I4O-1572259802058-0-7) caught
INFO [org.jboss.as.connector.deployers.RaXmlDeployer] (Camel (camel-1) thread #1 - JmsConsumer[TEST.QUEUE]) wmq.jmsra.rar: MQJCA4026:Transaction backed out with reason: 'The method 'xa_end' has failed with errorCode '100'.'.
WARN [com.arjuna.ats.jta] (Camel (camel-1) thread #1 - JmsConsumer[TEST.QUEUE]) ARJUNA016045: attempted rollback of < formatId=131077, gtrid_length=29, bqual_length=36, tx_uid=0:ffff0ae317d6:-21705625:5db6b586:1f48, node_name=1, branch_uid=0:ffff0ae317d6:-21705625:5db6b586:1f4c, subordinatenodename=null, eis_name=java:/JmsMQXA > (XAResourceWrapperImpl#258df727[xaResource=com.ibm.mq.connector.xa.XARWrapper#4e359608 pad=false overrideRmValue=null productName=WebSphere MQ productVersion=%I% %E% %U% jndiName=java:/JmsMQXA]) failed with exception code XAException.XAER_NOTA: javax.transaction.xa.XAException: The method 'xa_rollback' has failed with errorCode '-4'.
at com.ibm.mq.jmqi.JmqiXAResource.rollback(JmqiXAResource.java:874)
at com.ibm.mq.connector.xa.XARWrapper.rollback(XARWrapper.java:598)
at org.jboss.jca.core.tx.jbossts.XAResourceWrapperImpl.rollback(XAResourceWrapperImpl.java:196)
at com.arjuna.ats.internal.jta.resources.arjunacore.XAResourceRecord.topLevelAbort(XAResourceRecord.java:362)
at com.arjuna.ats.arjuna.coordinator.BasicAction.doAbort(BasicAction.java:3023)
at com.arjuna.ats.arjuna.coordinator.BasicAction.doAbort(BasicAction.java:3002)
at com.arjuna.ats.arjuna.coordinator.BasicAction.Abort(BasicAction.java:1674)
at com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.cancel(TwoPhaseCoordinator.java:124)
at com.arjuna.ats.arjuna.AtomicAction.abort(AtomicAction.java:186)
at com.arjuna.ats.internal.jta.transaction.arjunacore.TransactionImple.rollbackAndDisassociate(TransactionImple.java:1371)
at com.arjuna.ats.internal.jta.transaction.arjunacore.BaseTransaction.rollback(BaseTransaction.java:143)
at com.arjuna.ats.jbossatx.BaseTransactionManagerDelegate.rollback(BaseTransactionManagerDelegate.java:134)
at org.wildfly.transaction.client.LocalTransaction.rollbackAndDissociate(LocalTransaction.java:104)
at org.wildfly.transaction.client.ContextTransactionManager.rollback(ContextTransactionManager.java:83)
at org.wildfly.transaction.client.LocalUserTransaction.rollback(LocalUserTransaction.java:58)
at org.springframework.transaction.jta.JtaTransactionManager.doRollback(JtaTransactionManager.java:1048)
at org.springframework.transaction.support.AbstractPlatformTransactionManager.processRollback(AbstractPlatformTransactionManager.java:857)
at org.springframework.transaction.support.AbstractPlatformTransactionManager.commit(AbstractPlatformTransactionManager.java:717)
at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.receiveAndExecute(AbstractPollingMessageListenerContainer.java:249)
at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:1168)
at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.executeOngoingLoop(DefaultMessageListenerContainer.java:1160)
at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:1057)
at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.lang.Thread.run(Unknown Source)
There was an older issue with WebSphere MQ, I can now only find quickly this reference https://developer.jboss.org/thread/250976
Maybe it gives you a hint (and in any case, trace logs on the MQ server could show more info)
There are a couple of problems with your code snippet. The JmsTransactionManager should not be assigned from the NDI transaction manager - which is a JTA transaction manager. Note that if you set you component as 'transacted' and provide a connection factory, then Spring JMS will instantiate a JmsTransactionManager for you and wire it up to the connection factory. So, just:
<bean id="mq" class="org.apache.camel.component.jms.JmsComponent">
<property name="connectionFactory" ref="jmsConnectionFactory" />
<property name="transacted" value="true"/>
<property name="cacheLevelName" value="CACHE_NONE" />
<property name="receiveTimeout" value="100000"/>
And then drop:
<bean id="jmsTransactionManager" class="org.springframework.transaction.jta.JtaTransactionManager">
<property name="transactionManagerName" value="java:/TransactionManager" />
and:
<bean id="PROPAGATION_REQUIRED"
class="org.apache.camel.spring.spi.SpringTransactionPolicy">
<property name="transactionManager" ref="jmsTransactionManager" />
<property name="propagationBehaviorName" value="PROPAGATION_REQUIRES_NEW" />
Maybe try something like:
<bean class="com.ibm.mq.jms.MQConnectionFactory" id="source.mqConnectionFactory">
<property name="connectionNameList" value="10.0.0.207(1414)"/>
<property name="queueManager" value="MY.QUEUE.MANAGER"/>
<property name="channel" value="SYSTEM.DEF.SVRCONN"/>
<property name="transportType" value="1"/>
<property name="clientReconnectOptions" value="67108864"/>
</bean>
<bean
class="org.springframework.jms.connection.UserCredentialsConnectionFactoryAdapter" id="source.mqUserCred">
<property name="targetConnectionFactory" ref="source.mqConnectionFactory"/>
<property name="username" value="my-password"/>
</bean>
<bean class="org.apache.activemq.jms.pool.PooledConnectionFactory"
id="source.pooledConnectionFactory" primary="true">
<property name="maxConnections" value="1"/>
<property name="idleTimeout" value="0"/>
<property name="connectionFactory" ref="source.mqUserCred"/>
</bean>
<bean class="org.apache.camel.component.jms.JmsComponent" id="source">
<property name="configuration">
<bean class="org.apache.camel.component.jms.JmsConfiguration">
<property name="connectionFactory" ref="source.pooledConnectionFactory"/>
<property name="transacted" value="true"/>
<property name="receiveTimeout" value="100000"/>
<property name="maxConcurrentConsumers" value="5"/>
</bean>
</property>
</bean>

Spring batch: how to configure xa transactions

This is my situation:
a Postgres database where I have only Spring Batch tables
an Oracle database where I have business data to read, write and update
I have tried to configure an XA enviroment with Atomikos and it seems to work but honestly I haven't understand what is exactly happening.
Can you check my configuration, please? I'm totally noob in these things...
If I set "hibernate.transaction.jta.platform" to "com.atomikos.icatch.jta.hibernate4.AtomikosPlatform" instead of my class "SpringJtaPlatformAdapter" it seems that the batch doesn't commit on my Postgres DB. Why?
For testing I was using Spring "JpaPagingItemReader". With this reader I get always the exception "java.lang.IllegalStateException: A JTA EntityManager cannot use getTransaction()" so I copied this solution: solution
Why I need to do this? Is there another JpaReader for XA?
Thanks a lot for your help.
Here my configuration:
spring-batch-core 3.0.7.RELEASE
spring-jdbc and spring-orm 4.0.5.RELEASE
hibernate-entitymanager 5.0.7.Final
Atomikos transactions-jta, transactions-jdbc, transactions-hibernate4 4.0.6
database.xml
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:p="http://www.springframework.org/schema/p" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:jdbc="http://www.springframework.org/schema/jdbc" xmlns:jpa="http://www.springframework.org/schema/data/jpa"
xmlns:util="http://www.springframework.org/schema/util"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-4.0.xsd
http://www.springframework.org/schema/jdbc http://www.springframework.org/schema/jdbc/spring-jdbc.xsd
http://www.springframework.org/schema/data/jpa http://www.springframework.org/schema/data/jpa/spring-jpa.xsd
http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
<bean id="oracleDataSource" class="com.atomikos.jdbc.AtomikosDataSourceBean"
init-method="init" destroy-method="close">
<property name="xaDataSourceClassName" value="oracle.jdbc.xa.client.OracleXADataSource" />
<property name="uniqueResourceName" value="oracleDS" />
<property name="minPoolSize" value="1" />
<property name="maxPoolSize" value="3"/>
<property name="testQuery" value="select * from dual" />
<property name="xaProperties">
<props>
<prop key="URL">${database.oracle.url}</prop>
<prop key="user">${database.oracle.username}</prop>
<prop key="password">${database.oracle.password}</prop>
</props>
</property>
</bean>
<bean id="postgresDataSource" class="com.atomikos.jdbc.AtomikosDataSourceBean"
init-method="init" destroy-method="close">
<property name="xaDataSourceClassName" value="org.postgresql.xa.PGXADataSource" />
<property name="uniqueResourceName" value="postgresDS" />
<property name="minPoolSize" value="1" />
<property name="maxPoolSize" value="3"/>
<property name="testQuery" value="select * from batch_job_execution" />
<property name="xaProperties">
<props>
<prop key="serverName">localhost</prop>
<prop key="databaseName">postgres</prop>
<prop key="user">${database.postgresql.username}</prop>
<prop key="password">${database.postgresql.password}</prop>
</props>
</property>
</bean>
<bean id="atomikosTransactionService" class="com.atomikos.icatch.config.UserTransactionServiceImp"
init-method="init" destroy-method="shutdownForce">
<constructor-arg>
<props>
<prop key="com.atomikos.icatch.service">com.atomikos.icatch.standalone.UserTransactionServiceFactory</prop>
<prop key="com.atomikos.icatch.tm_unique_name">coriTransactionManager</prop>
</props>
</constructor-arg>
</bean>
<bean id="atomikosUserTransaction" class="com.atomikos.icatch.jta.UserTransactionImp" depends-on="atomikosTransactionService">
<property name="transactionTimeout" value="300" />
</bean>
<bean id="atomikosTransactionManager" class="com.atomikos.icatch.jta.UserTransactionManager"
init-method="init" depends-on="atomikosTransactionService" destroy-method="close">
<property name="forceShutdown" value="true" />
<property name="startupTransactionService" value="false" />
</bean>
<bean id="mainTransactionManager" class="org.springframework.transaction.jta.JtaTransactionManager">
<property name="transactionManager" ref="atomikosTransactionManager" />
<property name="userTransaction" ref="atomikosUserTransaction" />
</bean>
<alias name="mainTransactionManager" alias="transactionManager" />
<!-- inject the Atomikos transaction manager into a Spring Hibernate adapter for JTA Platform -->
<bean id="springJtaPlatformAdapter" class="com.mydomain.jta.SpringJtaPlatformAdapter">
<property name="jtaTransactionManager" ref="mainTransactionManager" />
</bean>
<bean id="entityManagerFactoryOracle" class="org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean"
depends-on="mainTransactionManager,springJtaPlatformAdapter">
<property name="persistenceXmlLocation" value="classpath:persistence.xml" />
<property name="persistenceUnitName" value="oraclePersistenceUnit" />
<property name="jpaVendorAdapter" ref="jpaVendorAdapterOracle"/>
<property name="dataSource" ref="oracleDataSource" />
<property name="jpaPropertyMap" ref="jpaPropertyMapOracle"></property>
</bean>
<bean id="jpaVendorAdapterOracle" class="org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter">
<property name="generateDdl" value="false"/>
<property name="showSql" value="true" />
<property name="databasePlatform" value="org.hibernate.dialect.Oracle10gDialect" />
</bean>
<util:map id="jpaPropertyMapOracle">
<entry key="hibernate.transaction.jta.platform" value="com.mydomain.jta.SpringJtaPlatformAdapter" />
<!-- <entry key="hibernate.transaction.jta.platform" value="com.atomikos.icatch.jta.hibernate4.AtomikosPlatform"/> -->
</util:map>
</beans>
context.xml
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:jdbc="http://www.springframework.org/schema/jdbc"
xsi:schemaLocation="
http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans-4.0.xsd
http://www.springframework.org/schema/jdbc
http://www.springframework.org/schema/jdbc/spring-jdbc.xsd">
<bean id="jobRepository" class="org.springframework.batch.core.repository.support.JobRepositoryFactoryBean">
<property name="dataSource" ref="postgresDataSource" />
<property name="transactionManager" ref="transactionManager" />
<property name="databaseType" value="POSTGRES" />
<property name="isolationLevelForCreate" value="ISOLATION_DEFAULT"/>
</bean>
<bean id="jobLauncher" class="org.springframework.batch.core.launch.support.SimpleJobLauncher">
<property name="jobRepository" ref="jobRepository" />
</bean>
</beans>
SpringJtaPlatformAdapter
public class SpringJtaPlatformAdapter extends AbstractJtaPlatform {
private static final long serialVersionUID = 1L;
private static TransactionManager sTransactionManager;
private static UserTransaction sUserTransaction;
#Override
protected TransactionManager locateTransactionManager() {
return sTransactionManager;
}
#Override
protected UserTransaction locateUserTransaction() {
return sUserTransaction;
}
public void setJtaTransactionManager(JtaTransactionManager jtaTransactionManager) {
sTransactionManager = jtaTransactionManager.getTransactionManager();
sUserTransaction = jtaTransactionManager.getUserTransaction();
}
}

maxItemCount property not working for JdbcPagingItemReader

I am setting the maxItemCount property of JdbcPagingItemReader.
I am setting it to 200 but I am getting read/processed/written of 203, 205.
Most of the time I am getting 200 but I get around 200+ commonly.
Why is this happening??
I've checked and there are no same timestamp value for the the sortkey in the 203-205 processed and the max.item.count field is not present in the batch_execution_context entry in the database table.
There is a JdbcPagingItemReader.read.count.max field but it is set to 200.
I am using oracle.
<bean id="batchReader" class="org.springframework.batch.item.database.JdbcPagingItemReader" scope="step">
<property name="dataSource" ref="myDataSource"/>
<property name="queryProvider">
<bean class="org.springframework.batch.item.database.support.SqlPagingQueryProviderFactoryBean">
<property name="dataSource" ref="myDataSource"/>
<property name="selectClause" value="select *" />
<property name="fromClause" value="from TRANSACTION" />
<property name="whereClause" value="where STATUS = 'OK' and TYPE = 200 " />
<property name="sortKey" value="TRANSACTION_TIMESTAMP"/>
</bean>
</property>
<!-- Inject via the ExecutionContext in rangePartitioner -->
<property name="parameterValues">
<map>
</map>
</property>
<property name="maxItemCount" value="200"/>
<property name="pageSize" value="50"/>
<property name="rowMapper">
<bean class="com.mappers.TransactionMapper" scope="step"/>
</property>
</bean>

Processing a large file using spring batch

I have a large file which may contain 100K to 500K records. I am planning to use chunk oriented processing and my thought is
1) Split the large file into smaller based on the count let say 10K in each file.
2) If there are 100K records then I will get 10 files each containing 10K reocrds
3) I would like to partition these 10 files and would like to process using 5 threads. I am thinking to use custom MultiResourcePartioner
4) The 5 threads should process all the 10 files created in split process.
5) I don't want to create same number of threads equal to file count as in that case I may face memory issues. What I am looking is whatever the number of files I would like to process them using only 5 threads (I can increase based on my requirements).
Expert could you let me know this can be achieved using spring batch? If yes could you please share pointers or reference implementations
Thanks in advance
The working job-config xml
<description>Spring Batch File Chunk Processing</description>
<import resource="../config/batch-context.xml" />
<batch:job id="file-partition-batch" job-repository="jobRepository" restartable="false">
<batch:step id="master">
<batch:partition partitioner="partitioner" handler="partitionHandler" />
</batch:step>
</batch:job>
<batch:step id="slave">
<batch:tasklet>
<batch:chunk reader="reader" processor="compositeProcessor"
writer="compositeWriter" commit-interval="5">
</batch:chunk>
</batch:tasklet>
</batch:step>
<bean id="partitionHandler" class="org.springframework.batch.core.partition.support.TaskExecutorPartitionHandler">
<property name="taskExecutor" ref="taskExecutor"/>
<property name="step" ref="slave" />
<property name="gridSize" value="5" />
</bean>
<bean id="partitioner" class="com.poc.partitioner.FileMultiResourcePartitioner">
<property name="resources" value="file:/Users/anupghosh/Documents/Spring_Batch/FilePartitionBatch/*.txt" />
<property name="threadName" value="feed-processor" />
</bean>
<bean id="taskExecutor" class="org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor">
<property name="corePoolSize" value="5" />
<property name="maxPoolSize" value="5" />
</bean>
<bean id="reader" class="org.springframework.batch.item.file.FlatFileItemReader" scope="step">
<property name="resource" value="#{stepExecutionContext['fileName']}" />
<property name="lineMapper">
<bean class="org.springframework.batch.item.file.mapping.DefaultLineMapper">
<property name="lineTokenizer">
<bean class="org.springframework.batch.item.file.transform.DelimitedLineTokenizer">
<property name="delimiter" value="|"/>
<property name="names" value="key,docName,docTypCD,itemType,itemNum,launchDate,status" />
</bean>
</property>
<property name="fieldSetMapper">
<bean class="com.poc.mapper.FileRowMapper" />
</property>
</bean>
</property>
</bean>
<bean id="validatingProcessor" class="org.springframework.batch.item.validator.ValidatingItemProcessor">
<constructor-arg ref="feedRowValidator" />
</bean>
<bean id="feedProcesor" class="com.poc.processor.FeedProcessor" />
<bean id="compositeProcessor" class="org.springframework.batch.item.support.CompositeItemProcessor" scope="step">
<property name="delegates">
<list>
<ref bean="validatingProcessor" />
<ref bean="feedProcesor" />
</list>
</property>
</bean>
<bean id="recordDecWriter" class="com.poc.writer.RecordDecWriter" />
<bean id="reconFlatFileCustomWriter" class="com.poc.writer.ReconFileWriter">
<property name="reconFlatFileWriter" ref="reconFlatFileWriter" />
</bean>
<bean id="reconFlatFileWriter" class="org.springframework.batch.item.file.FlatFileItemWriter" scope="step">
<property name="resource" value="file:/Users/anupghosh/Documents/Spring_Batch/recon-#{stepExecutionContext[threadName]}.txt" />
<property name="shouldDeleteIfExists" value="true" />
<property name="lineAggregator">
<bean class="org.springframework.batch.item.file.transform.DelimitedLineAggregator">
<property name="delimiter" value="|" />
<property name="fieldExtractor">
<bean class="org.springframework.batch.item.file.transform.BeanWrapperFieldExtractor">
<property name="names" value="validationError" />
</bean>
</property>
</bean>
</property>
</bean>
<bean id="compositeWriter" class="org.springframework.batch.item.support.CompositeItemWriter">
<property name="delegates">
<list>
<ref bean="recordDecWriter" />
<ref bean="reconFlatFileCustomWriter" />
</list>
</property>
</bean>
<bean id="feedRowValidator" class="org.springframework.batch.item.validator.SpringValidator">
<property name="validator">
<bean class="com.poc.validator.FeedRowValidator"/>
</property>
</bean>
was able to solve this using MultiResourcePartitioner. below are java config
#Bean
public Partitioner partitioner() {
MultiResourcePartitioner partitioner = new MultiResourcePartitioner();
ClassLoader cl = this.getClass().getClassLoader();
ResourcePatternResolver resolver = new PathMatchingResourcePatternResolver(cl);
Resource[] resources = resolver.getResources("file:" + filePath + "/"+"*.csv");
partitioner.setResources(resources);
partitioner.partition(10);
return partitioner;
}
#Bean
public TaskExecutor taskExecutor() {
ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
taskExecutor.setMaxPoolSize(4);
taskExecutor.afterPropertiesSet();
return taskExecutor;
}
#Bean
#Qualifier("masterStep")
public Step masterStep() {
return stepBuilderFactory.get("masterStep")
.partitioner(ProcessDataStep())
.partitioner("ProcessDataStep",partitioner())
.taskExecutor(taskExecutor())
.listener(pcStressStepListener)
.build();
}
#Bean
#Qualifier("processData")
public Step processData() {
return stepBuilderFactory.get("processData")
.<pojo, pojo> chunk(5000)
.reader(reader)
.processor(processor())
.writer(writer)
.build();
}
#Bean(name="reader")
#StepScope
public FlatFileItemReader<pojo> reader(#Value("#{stepExecutionContext['fileName']}") String filename) {
FlatFileItemReader<pojo> reader = new FlatFileItemReader<>();
reader.setResource(new UrlResource(filename));
reader.setLineMapper(new DefaultLineMapper<pojo>() {
{
setLineTokenizer(new DelimitedLineTokenizer() {
{
setNames(FILE HEADER);
}
});
setFieldSetMapper(new BeanWrapperFieldSetMapper<pojo>() {
{
setTargetType(pojo.class);
}
});
}
});
return reader;
}

Spring Data JPA with a custom RepositoryFactory and Multiple EntityManagers

I'm currently trying to get Spring Data JPA to work with two DataSources and thus two EntityManagers. My Repositories all inherit from a custom base class "BaseRepositoryImpl".
Problem:
I always get the following exception:
org.springframework.beans.factory.NoSuchBeanDefinitionException:
No unique bean of type [javax.persistence.EntityManagerFactory] is defined:
expected single bean but found 2: entityManagerFactory1,entityManagerFactory2
Question:
How can I tell my custom RepositoryFactory which EntityManagerFactory to use?
Here's my Spring configuration:
<jpa:repositories base-package="package1" transaction-manager-ref="..."
factory-class="MyFactoryBean"
entity-manager-factory-ref="entityManagerFactory1"/>
<bean id="dataSource1" class="org.springframework.jndi.JndiObjectFactoryBean">
<property name="jndiName" value="..." />
</bean>
<bean id="entityManagerFactory1" class="org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean">
<property name="dataSource" ref="dataSource1" />
<property name="persistenceUnitName" value="unit1" />
...
</bean>
<jpa:repositories base-package="package2" transaction-manager-ref="..."
factory-class="MyFactoryBean"
entity-manager-factory-ref="entityManagerFactory2"/>
<bean id="dataSource2" class="org.springframework.jndi.JndiObjectFactoryBean">
<property name="jndiName" value="..." />
</bean>
<bean id="entityManagerFactory2" class="org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean">
<property name="dataSource" ref="dataSource2" />
<property name="persistenceUnitName" value="unit2" />
...
</bean>
The class "MyFactoryBean" is implemented exactly as described at http://docs.spring.io/spring-data/jpa/docs/1.4.x/reference/htmlsingle/#repositories.custom-behaviour-for-all-repositories.
The configuration "entity-manager-factory-ref" seems to be ignored by Spring, at least when using a custom RepositoryFactory.
Any hints?
Entity manager factory is used to create entity managers for different persistence units. You declare your entity manager bean and then give entity manager factory bean as it's class.
This is what worked for me:
<tx:annotation-driven transaction-manager="transactionManager" />
<bean class="org.springframework.orm.jpa.support.PersistenceAnnotationBeanPostProcessor"/>
<bean id="transactionManager" class="org.springframework.orm.jpa.JpaTransactionManager">
<property name="entityManagerFactory" ref="defaultEntityManager"/>
</bean>
<bean id="defaultDataSource" class="org.apache.commons.dbcp.BasicDataSource" destroy-method="close">
<property name="driverClassName" value="${hibernate.connection.driver_class}"/>
<property name="url" value="${db.default.url}"/>
<property name="username" value="${db.default.username}"/>
<property name="password" value="${db.default.password}"/>
</bean>
<bean id="defaultEntityManager" class="org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean">
<property name="dataSource" ref="defaultDataSource"/>
<property name="persistenceUnitName" value="defaultPersistenceUnit"/>
</bean>
<bean id="otherDataSource" class="org.apache.commons.dbcp.BasicDataSource" destroy-method="close">
<property name="driverClassName" value="${hibernate.connection.driver_class}"/>
<property name="url" value="${db.other.url}"/>
<property name="username" value="${db.other.username}"/>
<property name="password" value="${db.other.password}"/>
</bean>
<bean id="otherEntityManager" class="org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean">
<property name="dataSource" ref="otherDataSource"/>
<property name="persistenceUnitName" value="otherPersistenceUnit"></property>
</bean>
Then when you are implementing your data access classes just declare:
#Repository
#Transactional
public class MyDAOImpl implements MyDAO
{
#PersistenceContext(unitName = "defaultPersistenceUnit")
private EntityManager entityManager;
}
and for DAO that uses other persistence unit:
#Repository
#Transactional
public class MyOtherDAOImpl implements MyOtherDAO
{
#PersistenceContext(unitName = "otherPersistenceUnit")
private EntityManager entityManager;
}