I have this scheduler :
<bean id="odilQuartzScheduler"
class="org.springframework.scheduling.quartz.SchedulerFactoryBean"
lazy-init="false">
<property name="jobFactory">
<bean class="org.springframework.scheduling.quartz.SpringBeanJobFactory"/>
</property>
<property name="schedulerName" value="OdilScheduler"/>
<property name="applicationContextSchedulerContextKey" value="applicationContext"/>
<property name="autoStartup" value="${popo.scheduler}"/>
<property name="startupDelay" value="60"/>
<property name="overwriteExistingJobs" value="true"/>
<property name="configLocation" value="classpath:quartz-odil.properties"/>
<property name="transactionManager" ref="applicationTransactionManager"/>
<property name="schedulerContextAsMap">
<map>
<entry key="globalConfiguration" value-ref="globalConfiguration"/>
<entry key="odilFileImporter" value-ref="odilFileImporter"/>
<entry key="odilRemoteDispatcher" value-ref="odilRemoteDispatcher"/>
<entry key="odilService" value-ref="odilService"/>
<entry key="centreReferenceService" value-ref="centreReferenceService"/>
<entry key="envoiParserFactory" value-ref="envoiParserFactory"/>
<entry key="versionReferenceService" value-ref="versionReferenceService"/>
<entry key="delPublisherService" value-ref="delPublisherService"/>
<entry key="capteurPublisherService" value-ref="capteurPublisherService"/>
</map>
</property>
<property name="triggers">
<list>
<ref bean="piloteOdilTrigger-1"/>
<ref bean="piloteOdilTrigger-2"/>
<ref bean="piloteOdilTrigger-3"/>
<ref bean="piloteOdilTrigger-4"/>
<ref bean="piloteOdilTrigger-5"/>
<ref bean="piloteOdilTrigger-6"/>
</list>
</property>
</bean>
<bean id="piloteOdilTrigger-1" class="org.springframework.scheduling.quartz.SimpleTriggerBean">
<property name="name" value="Trigger 1"/>
<property name="jobDetail" ref="piloteOdilJobBean1"/>
<property name="repeatInterval" value="60000"/>
<property name="group" value="POPO"/>
</bean>
<bean id="piloteOdilTrigger-2" class="org.springframework.scheduling.quartz.CronTriggerBean">
<property name="name" value="Trigger 2"/>
<property name="jobDetail" ref="piloteOdilJobBean2"/>
<property name="cronExpression" value="${popo.odil.rushhour.cronexpression}" />
<property name="group" value="POPO"/>
</bean>
<bean id="piloteOdilTrigger-3" class="org.springframework.scheduling.quartz.CronTriggerBean">
<property name="name" value="Trigger 3"/>
<property name="jobDetail" ref="piloteOdilJobBean3"/>
<property name="cronExpression" value="${popo.odil.rushhour.cronexpression}" />
<property name="group" value="POPO"/>
</bean>
<bean id="piloteOdilTrigger-4" class="org.springframework.scheduling.quartz.CronTriggerBean">
<property name="name" value="Trigger 4"/>
<property name="jobDetail" ref="piloteOdilJobBean4"/>
<property name="cronExpression" value="${popo.odil.rushhour.cronexpression}" />
<property name="group" value="POPO"/>
</bean>
<bean id="piloteOdilTrigger-5" class="org.springframework.scheduling.quartz.CronTriggerBean">
<property name="name" value="Trigger 5"/>
<property name="jobDetail" ref="piloteOdilJobBean5"/>
<property name="cronExpression" value="${popo.odil.rushhour.cronexpression}" />
<property name="group" value="POPO"/>
</bean>
<bean id="piloteOdilTrigger-6" class="org.springframework.scheduling.quartz.CronTriggerBean">
<property name="name" value="Trigger 6"/>
<property name="jobDetail" ref="piloteOdilJobBean6"/>
<property name="cronExpression" value="${popo.odil.rushhour.cronexpression}" />
<property name="group" value="POPO"/>
</bean>
<bean id="piloteOdilJobBean-template" abstract="true"
class="org.springframework.scheduling.quartz.JobDetailBean">
<property name="name" value="Pilote ODIL"/>
<property name="jobClass" value="fr.xxxx.popo.batch.PiloteOdilJob"/> <!-- implements StatefulJob -->
<property name="requestsRecovery" value="true"/>
<property name="group" value="POPO"/>
</bean>
<bean id="piloteOdilJobBean1" parent="piloteOdilJobBean-template" />
<bean id="piloteOdilJobBean2" parent="piloteOdilJobBean-template" />
<bean id="piloteOdilJobBean3" parent="piloteOdilJobBean-template" />
<bean id="piloteOdilJobBean4" parent="piloteOdilJobBean-template" />
<bean id="piloteOdilJobBean5" parent="piloteOdilJobBean-template" />
<bean id="piloteOdilJobBean6" parent="piloteOdilJobBean-template" />
this is it's configuration file
#============================================================================
# Main Scheduler Properties
#============================================================================
org.quartz.scheduler.instanceName = OdilScheduler
org.quartz.scheduler.instanceId = AUTO
org.quartz.scheduler.rmi.export = false
org.quartz.scheduler.rmi.proxy = false
#============================================================================
# ThreadPool
#============================================================================
org.quartz.threadPool.class = org.quartz.simpl.SimpleThreadPool
org.quartz.threadPool.threadCount = 6
org.quartz.threadPool.threadPriority = 5
#============================================================================
# JobStore
#============================================================================
org.quartz.jobStore.class = org.quartz.simpl.RAMJobStore
org.quartz.jobStore.misfireThreshold = 3
#============================================================================
# Plugins
#============================================================================
org.quartz.plugin.shutdownhook.class = org.quartz.plugins.management.ShutdownHookPlugin
org.quartz.plugin.shutdownhook.cleanShutdown = true
From what I understood using a ramJobStore make it non clusterable and thus load balancing should not work.
But in reallity when I have 2 servers running, each one has it's own OdilScheduler, with 6 triggers each.
However a total of only 6 trigger will work at the same time.
3 on instance 1 and 3 on instance 2 for example, I want all the 12 trigger to work simultaniously.
What am I missing?
Thanks
ps : quartz version : 1.8.5
I was mislead, in fact I only tried with 3 trigger on each instance.
1 normal and 2 cron trigger.
The cron trigger were using the RAMJobStore, but because of the template, each trigger had the same name so each instance could only use 1 cron trigger (they are identified by their name).
Which is why it looked like load balancing was happening, in fact what was happening is that since they had the same name only one trigger was triggered per instance.
If I had set 5 cron trigger for each instance I would have had only 1 trigger per instance.
Related
I am trying get job-parameteres-in-to-item-processor-using-spring-batch-annotation.
I implemented it by referring below link; but for me the variable (batchRunName) is coming as null when I try to access it in my Processor class.
Can any one please look at it. I am sure; I am missing some small thing.
How to get Job parameteres in to item processor using spring Batch annotation
public static void main(String[] args) {
contextObj = new ClassPathXmlApplicationContext(springConfig);
jobObj = (Job) contextObj.getBean("XYZ-1001-DD-01");
JobParametersBuilder jobBuilder = new JobParametersBuilder();
System.out.println("args[0] is " + args[0] );
jobBuilder.addString("batchRunName", args[0]);
public class TimeProcessor implements ItemProcessor<Time, TimeMetric> {
private DataSource dataSource;
#Value("#{jobParameters['batchRunNumber']}")
private String batchRunNumber;
public void setBatchRunNumber(String batchRunNumber) {
this.batchRunNumber = batchRunNumber;
}
<bean id="timeProcessor"
class="com.xyz.processor.TimeProcessor" scope="step">
<property name="dataSource" ref="oracledataSource" />
</bean>
=================FULL XML CONFIGURATION========================
<import resource="classpath:/batch/utility/skip/batch_skip.xml" />
<import resource="classpath:/batch/config/context-postgres.xml" />
<import resource="classpath:/batch/config/oracle-database.xml" />
<context:property-placeholder
location="classpath:/batch/jobs/TPF-1001-DD-01/TPF-1001-DD-01.properties" />
<bean id="gridSizePartitioner"
class="com.tpf.partitioner.GridSizePartitioner" />
<task:executor id="taskExecutor" pool-size="${pool.size}" />
<batch:job id="XYZJob" job-repository="jobRepository"
restartable="true">
<batch:step id="XYZSTEP">
<batch:description>Convert TIF files to PDF</batch:description>
<batch:partition partitioner="gridSizePartitioner">
<batch:handler task-executor="taskExecutor"
grid-size="${pool.size}" />
<batch:step>
<batch:tasklet allow-start-if-complete="true">
<batch:chunk commit-interval="${commit.interval}"
skip-limit="${job.skip.limit}">
<batch:reader>
<bean id="timeReader"
class="org.springframework.batch.item.database.JdbcCursorItemReader"
scope="step">
<property name="dataSource" ref="oracledataSource" />
<property name="sql">
<value>
select TIME_ID as timesheetId,count(*),max(CREATION_DATETIME) as creationDateTime , ILN_NUMBER as ilnNumber
from TS_FAKE_NAME
where creation_datetime >= '#{jobParameters['creation_start_date1']} 12.00.00.000000000 AM'
and creation_datetime < '#{jobParameters['creation_start_date2']} 11.59.59.999999999 PM'
and mod(time_id,${pool.size})=#{stepExecutionContext['partition.id']}
group by time_id ,ILN_NUMBER
</value>
</property>
<property name="rowMapper">
<bean
class="org.springframework.jdbc.core.BeanPropertyRowMapper">
<property name="mappedClass"
value="com.tpf.model.Time" />
</bean>
</property>
</bean>
</batch:reader>
<batch:processor>
<bean id="compositeItemProcessor"
class="org.springframework.batch.item.support.CompositeItemProcessor">
<property name="delegates">
<list>
<ref bean="timeProcessor" />
</list>
</property>
</bean>
</batch:processor>
<batch:writer>
<bean id="compositeItemWriter"
class="org.springframework.batch.item.support.CompositeItemWriter">
<property name="delegates">
<list>
<ref bean="timeWriter" />
</list>
</property>
</bean>
</batch:writer>
<batch:skippable-exception-classes>
<batch:include
class="com.utility.skip.BatchSkipException" />
</batch:skippable-exception-classes>
<batch:listeners>
<batch:listener ref="batchSkipListener" />
</batch:listeners>
</batch:chunk>
</batch:tasklet>
</batch:step>
</batch:partition>
</batch:step>
<batch:validator>
<bean
class="org.springframework.batch.core.job.DefaultJobParametersValidator">
<property name="requiredKeys">
<list>
<value>batchRunNumber</value>
<value>creation_start_date1</value>
<value>creation_start_date2</value>
</list>
</property>
</bean>
</batch:validator>
</batch:job>
<bean id="timesheetWriter" class="com.tpf.writer.TimeWriter"
scope="step">
<property name="dataSource" ref="dataSource" />
</bean>
<bean id="timeProcessor"
class="com.tpf.processor.TimeProcessor" scope="step">
<property name="dataSource" ref="oracledataSource" />
</bean>
I think you are facing the issue reported in BATCH-2351.
You can try to provide the job parameter via XML instead of annotation (Since the majority of your config is XML based):
<bean id="timeProcessor" class="com.xyz.processor.TimeProcessor" scope="step">
<property name="dataSource" ref="oracledataSource" />
<property name="batchRunNumber" value="#{jobParameters['batchRunNumber']}" />
</bean>
Hope this helps.
This is my situation:
a Postgres database where I have only Spring Batch tables
an Oracle database where I have business data to read, write and update
I have tried to configure an XA enviroment with Atomikos and it seems to work but honestly I haven't understand what is exactly happening.
Can you check my configuration, please? I'm totally noob in these things...
If I set "hibernate.transaction.jta.platform" to "com.atomikos.icatch.jta.hibernate4.AtomikosPlatform" instead of my class "SpringJtaPlatformAdapter" it seems that the batch doesn't commit on my Postgres DB. Why?
For testing I was using Spring "JpaPagingItemReader". With this reader I get always the exception "java.lang.IllegalStateException: A JTA EntityManager cannot use getTransaction()" so I copied this solution: solution
Why I need to do this? Is there another JpaReader for XA?
Thanks a lot for your help.
Here my configuration:
spring-batch-core 3.0.7.RELEASE
spring-jdbc and spring-orm 4.0.5.RELEASE
hibernate-entitymanager 5.0.7.Final
Atomikos transactions-jta, transactions-jdbc, transactions-hibernate4 4.0.6
database.xml
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:p="http://www.springframework.org/schema/p" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:jdbc="http://www.springframework.org/schema/jdbc" xmlns:jpa="http://www.springframework.org/schema/data/jpa"
xmlns:util="http://www.springframework.org/schema/util"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-4.0.xsd
http://www.springframework.org/schema/jdbc http://www.springframework.org/schema/jdbc/spring-jdbc.xsd
http://www.springframework.org/schema/data/jpa http://www.springframework.org/schema/data/jpa/spring-jpa.xsd
http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
<bean id="oracleDataSource" class="com.atomikos.jdbc.AtomikosDataSourceBean"
init-method="init" destroy-method="close">
<property name="xaDataSourceClassName" value="oracle.jdbc.xa.client.OracleXADataSource" />
<property name="uniqueResourceName" value="oracleDS" />
<property name="minPoolSize" value="1" />
<property name="maxPoolSize" value="3"/>
<property name="testQuery" value="select * from dual" />
<property name="xaProperties">
<props>
<prop key="URL">${database.oracle.url}</prop>
<prop key="user">${database.oracle.username}</prop>
<prop key="password">${database.oracle.password}</prop>
</props>
</property>
</bean>
<bean id="postgresDataSource" class="com.atomikos.jdbc.AtomikosDataSourceBean"
init-method="init" destroy-method="close">
<property name="xaDataSourceClassName" value="org.postgresql.xa.PGXADataSource" />
<property name="uniqueResourceName" value="postgresDS" />
<property name="minPoolSize" value="1" />
<property name="maxPoolSize" value="3"/>
<property name="testQuery" value="select * from batch_job_execution" />
<property name="xaProperties">
<props>
<prop key="serverName">localhost</prop>
<prop key="databaseName">postgres</prop>
<prop key="user">${database.postgresql.username}</prop>
<prop key="password">${database.postgresql.password}</prop>
</props>
</property>
</bean>
<bean id="atomikosTransactionService" class="com.atomikos.icatch.config.UserTransactionServiceImp"
init-method="init" destroy-method="shutdownForce">
<constructor-arg>
<props>
<prop key="com.atomikos.icatch.service">com.atomikos.icatch.standalone.UserTransactionServiceFactory</prop>
<prop key="com.atomikos.icatch.tm_unique_name">coriTransactionManager</prop>
</props>
</constructor-arg>
</bean>
<bean id="atomikosUserTransaction" class="com.atomikos.icatch.jta.UserTransactionImp" depends-on="atomikosTransactionService">
<property name="transactionTimeout" value="300" />
</bean>
<bean id="atomikosTransactionManager" class="com.atomikos.icatch.jta.UserTransactionManager"
init-method="init" depends-on="atomikosTransactionService" destroy-method="close">
<property name="forceShutdown" value="true" />
<property name="startupTransactionService" value="false" />
</bean>
<bean id="mainTransactionManager" class="org.springframework.transaction.jta.JtaTransactionManager">
<property name="transactionManager" ref="atomikosTransactionManager" />
<property name="userTransaction" ref="atomikosUserTransaction" />
</bean>
<alias name="mainTransactionManager" alias="transactionManager" />
<!-- inject the Atomikos transaction manager into a Spring Hibernate adapter for JTA Platform -->
<bean id="springJtaPlatformAdapter" class="com.mydomain.jta.SpringJtaPlatformAdapter">
<property name="jtaTransactionManager" ref="mainTransactionManager" />
</bean>
<bean id="entityManagerFactoryOracle" class="org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean"
depends-on="mainTransactionManager,springJtaPlatformAdapter">
<property name="persistenceXmlLocation" value="classpath:persistence.xml" />
<property name="persistenceUnitName" value="oraclePersistenceUnit" />
<property name="jpaVendorAdapter" ref="jpaVendorAdapterOracle"/>
<property name="dataSource" ref="oracleDataSource" />
<property name="jpaPropertyMap" ref="jpaPropertyMapOracle"></property>
</bean>
<bean id="jpaVendorAdapterOracle" class="org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter">
<property name="generateDdl" value="false"/>
<property name="showSql" value="true" />
<property name="databasePlatform" value="org.hibernate.dialect.Oracle10gDialect" />
</bean>
<util:map id="jpaPropertyMapOracle">
<entry key="hibernate.transaction.jta.platform" value="com.mydomain.jta.SpringJtaPlatformAdapter" />
<!-- <entry key="hibernate.transaction.jta.platform" value="com.atomikos.icatch.jta.hibernate4.AtomikosPlatform"/> -->
</util:map>
</beans>
context.xml
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:jdbc="http://www.springframework.org/schema/jdbc"
xsi:schemaLocation="
http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans-4.0.xsd
http://www.springframework.org/schema/jdbc
http://www.springframework.org/schema/jdbc/spring-jdbc.xsd">
<bean id="jobRepository" class="org.springframework.batch.core.repository.support.JobRepositoryFactoryBean">
<property name="dataSource" ref="postgresDataSource" />
<property name="transactionManager" ref="transactionManager" />
<property name="databaseType" value="POSTGRES" />
<property name="isolationLevelForCreate" value="ISOLATION_DEFAULT"/>
</bean>
<bean id="jobLauncher" class="org.springframework.batch.core.launch.support.SimpleJobLauncher">
<property name="jobRepository" ref="jobRepository" />
</bean>
</beans>
SpringJtaPlatformAdapter
public class SpringJtaPlatformAdapter extends AbstractJtaPlatform {
private static final long serialVersionUID = 1L;
private static TransactionManager sTransactionManager;
private static UserTransaction sUserTransaction;
#Override
protected TransactionManager locateTransactionManager() {
return sTransactionManager;
}
#Override
protected UserTransaction locateUserTransaction() {
return sUserTransaction;
}
public void setJtaTransactionManager(JtaTransactionManager jtaTransactionManager) {
sTransactionManager = jtaTransactionManager.getTransactionManager();
sUserTransaction = jtaTransactionManager.getUserTransaction();
}
}
I am setting the maxItemCount property of JdbcPagingItemReader.
I am setting it to 200 but I am getting read/processed/written of 203, 205.
Most of the time I am getting 200 but I get around 200+ commonly.
Why is this happening??
I've checked and there are no same timestamp value for the the sortkey in the 203-205 processed and the max.item.count field is not present in the batch_execution_context entry in the database table.
There is a JdbcPagingItemReader.read.count.max field but it is set to 200.
I am using oracle.
<bean id="batchReader" class="org.springframework.batch.item.database.JdbcPagingItemReader" scope="step">
<property name="dataSource" ref="myDataSource"/>
<property name="queryProvider">
<bean class="org.springframework.batch.item.database.support.SqlPagingQueryProviderFactoryBean">
<property name="dataSource" ref="myDataSource"/>
<property name="selectClause" value="select *" />
<property name="fromClause" value="from TRANSACTION" />
<property name="whereClause" value="where STATUS = 'OK' and TYPE = 200 " />
<property name="sortKey" value="TRANSACTION_TIMESTAMP"/>
</bean>
</property>
<!-- Inject via the ExecutionContext in rangePartitioner -->
<property name="parameterValues">
<map>
</map>
</property>
<property name="maxItemCount" value="200"/>
<property name="pageSize" value="50"/>
<property name="rowMapper">
<bean class="com.mappers.TransactionMapper" scope="step"/>
</property>
</bean>
I have a large file which may contain 100K to 500K records. I am planning to use chunk oriented processing and my thought is
1) Split the large file into smaller based on the count let say 10K in each file.
2) If there are 100K records then I will get 10 files each containing 10K reocrds
3) I would like to partition these 10 files and would like to process using 5 threads. I am thinking to use custom MultiResourcePartioner
4) The 5 threads should process all the 10 files created in split process.
5) I don't want to create same number of threads equal to file count as in that case I may face memory issues. What I am looking is whatever the number of files I would like to process them using only 5 threads (I can increase based on my requirements).
Expert could you let me know this can be achieved using spring batch? If yes could you please share pointers or reference implementations
Thanks in advance
The working job-config xml
<description>Spring Batch File Chunk Processing</description>
<import resource="../config/batch-context.xml" />
<batch:job id="file-partition-batch" job-repository="jobRepository" restartable="false">
<batch:step id="master">
<batch:partition partitioner="partitioner" handler="partitionHandler" />
</batch:step>
</batch:job>
<batch:step id="slave">
<batch:tasklet>
<batch:chunk reader="reader" processor="compositeProcessor"
writer="compositeWriter" commit-interval="5">
</batch:chunk>
</batch:tasklet>
</batch:step>
<bean id="partitionHandler" class="org.springframework.batch.core.partition.support.TaskExecutorPartitionHandler">
<property name="taskExecutor" ref="taskExecutor"/>
<property name="step" ref="slave" />
<property name="gridSize" value="5" />
</bean>
<bean id="partitioner" class="com.poc.partitioner.FileMultiResourcePartitioner">
<property name="resources" value="file:/Users/anupghosh/Documents/Spring_Batch/FilePartitionBatch/*.txt" />
<property name="threadName" value="feed-processor" />
</bean>
<bean id="taskExecutor" class="org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor">
<property name="corePoolSize" value="5" />
<property name="maxPoolSize" value="5" />
</bean>
<bean id="reader" class="org.springframework.batch.item.file.FlatFileItemReader" scope="step">
<property name="resource" value="#{stepExecutionContext['fileName']}" />
<property name="lineMapper">
<bean class="org.springframework.batch.item.file.mapping.DefaultLineMapper">
<property name="lineTokenizer">
<bean class="org.springframework.batch.item.file.transform.DelimitedLineTokenizer">
<property name="delimiter" value="|"/>
<property name="names" value="key,docName,docTypCD,itemType,itemNum,launchDate,status" />
</bean>
</property>
<property name="fieldSetMapper">
<bean class="com.poc.mapper.FileRowMapper" />
</property>
</bean>
</property>
</bean>
<bean id="validatingProcessor" class="org.springframework.batch.item.validator.ValidatingItemProcessor">
<constructor-arg ref="feedRowValidator" />
</bean>
<bean id="feedProcesor" class="com.poc.processor.FeedProcessor" />
<bean id="compositeProcessor" class="org.springframework.batch.item.support.CompositeItemProcessor" scope="step">
<property name="delegates">
<list>
<ref bean="validatingProcessor" />
<ref bean="feedProcesor" />
</list>
</property>
</bean>
<bean id="recordDecWriter" class="com.poc.writer.RecordDecWriter" />
<bean id="reconFlatFileCustomWriter" class="com.poc.writer.ReconFileWriter">
<property name="reconFlatFileWriter" ref="reconFlatFileWriter" />
</bean>
<bean id="reconFlatFileWriter" class="org.springframework.batch.item.file.FlatFileItemWriter" scope="step">
<property name="resource" value="file:/Users/anupghosh/Documents/Spring_Batch/recon-#{stepExecutionContext[threadName]}.txt" />
<property name="shouldDeleteIfExists" value="true" />
<property name="lineAggregator">
<bean class="org.springframework.batch.item.file.transform.DelimitedLineAggregator">
<property name="delimiter" value="|" />
<property name="fieldExtractor">
<bean class="org.springframework.batch.item.file.transform.BeanWrapperFieldExtractor">
<property name="names" value="validationError" />
</bean>
</property>
</bean>
</property>
</bean>
<bean id="compositeWriter" class="org.springframework.batch.item.support.CompositeItemWriter">
<property name="delegates">
<list>
<ref bean="recordDecWriter" />
<ref bean="reconFlatFileCustomWriter" />
</list>
</property>
</bean>
<bean id="feedRowValidator" class="org.springframework.batch.item.validator.SpringValidator">
<property name="validator">
<bean class="com.poc.validator.FeedRowValidator"/>
</property>
</bean>
was able to solve this using MultiResourcePartitioner. below are java config
#Bean
public Partitioner partitioner() {
MultiResourcePartitioner partitioner = new MultiResourcePartitioner();
ClassLoader cl = this.getClass().getClassLoader();
ResourcePatternResolver resolver = new PathMatchingResourcePatternResolver(cl);
Resource[] resources = resolver.getResources("file:" + filePath + "/"+"*.csv");
partitioner.setResources(resources);
partitioner.partition(10);
return partitioner;
}
#Bean
public TaskExecutor taskExecutor() {
ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
taskExecutor.setMaxPoolSize(4);
taskExecutor.afterPropertiesSet();
return taskExecutor;
}
#Bean
#Qualifier("masterStep")
public Step masterStep() {
return stepBuilderFactory.get("masterStep")
.partitioner(ProcessDataStep())
.partitioner("ProcessDataStep",partitioner())
.taskExecutor(taskExecutor())
.listener(pcStressStepListener)
.build();
}
#Bean
#Qualifier("processData")
public Step processData() {
return stepBuilderFactory.get("processData")
.<pojo, pojo> chunk(5000)
.reader(reader)
.processor(processor())
.writer(writer)
.build();
}
#Bean(name="reader")
#StepScope
public FlatFileItemReader<pojo> reader(#Value("#{stepExecutionContext['fileName']}") String filename) {
FlatFileItemReader<pojo> reader = new FlatFileItemReader<>();
reader.setResource(new UrlResource(filename));
reader.setLineMapper(new DefaultLineMapper<pojo>() {
{
setLineTokenizer(new DelimitedLineTokenizer() {
{
setNames(FILE HEADER);
}
});
setFieldSetMapper(new BeanWrapperFieldSetMapper<pojo>() {
{
setTargetType(pojo.class);
}
});
}
});
return reader;
}
I have one basis job with one basic step. This jobs is executing every x second (I am using quartz for this). Then in my config class I also have variable "runStep". Where should I add this attribute and run my step only if runStep is true ?
<batch:job id="export1" parent="baseJob">
<batch:step id="registruj" parent="baseStep">
<tasklet>
<chunk reader="registrujReader" processor="registrujProcessor" writer="registrujWriter"
commit-interval="1" />
</tasklet>
</batch:step>
</batch:job>
<bean id="baseJob" class="org.springframework.batch.core.job.SimpleJob" abstract="true">
<property name="jobRepository" ref="jobRepository" />
</bean>
<bean id="baseStep" class="org.springframework.batch.core.step.factory.SimpleStepFactoryBean" abstract="true">
<property name="transactionManager" ref="transactionManager" />
<property name="jobRepository" ref="jobRepository" />
<property name="startLimit" value="100" />
<property name="commitInterval" value="1" />
</bean>
<bean id="jobRepository" class="org.springframework.batch.core.repository.support.JobRepositoryFactoryBean"
p:dataSource-ref="expDataSource" p:transactionManager-ref="transactionManager" />
<bean id="jobLauncher" class="org.springframework.batch.core.launch.support.SimpleJobLauncher">
<property name="jobRepository" ref="jobRepository" />
</bean>
<bean class="org.springframework.batch.core.configuration.support.JobRegistryBeanPostProcessor">
<property name="jobRegistry" ref="jobRegistry" />
</bean>
<bean id="jobRegistry" class="org.springframework.batch.core.configuration.support.MapJobRegistry" />
<bean id="registrujWriter" class="cz.isvs.reg.rob.util.export.batch.RegistrujItemWriter" scope="step" />
<bean id="registrujReader" class="cz.isvs.reg.rob.util.export.batch.RegistrujItemReader" scope="step" />
<bean id="registrujProcessor" class="cz.isvs.reg.rob.util.export.batch.RegistrujItemProcessor" scope="step" />
<!-- run every 10 seconds -->
<bean class="org.springframework.scheduling.quartz.SchedulerFactoryBean">
<property name="triggers">
<bean id="cronTrigger" class="org.springframework.scheduling.quartz.CronTriggerBean">
<property name="jobDetail" ref="jobDetail" />
<property name="cronExpression" value="*/10 * * * * ?" />
</bean>
</property>
</bean>
<bean id="jobDetail" class="org.springframework.scheduling.quartz.JobDetailBean">
<property name="jobClass" value="cz.isvs.reg.rob.util.export.batch.JobLauncherDetails" />
<property name="group" value="quartz-batch" />
<property name="jobDataAsMap">
<map>
<entry key="jobName" value="export1" />
<entry key="jobLocator" value-ref="jobRegistry" />
<entry key="jobLauncher" value-ref="jobLauncher" />
</map>
</property>
</bean>
Use a JobExecutionDecider
public class RunStepDecider implements JobExecutionDecider {
public FlowExecutionStatus decide(JobExecution jobExecution, StepExecution stepExecution) {
final String runStep = jobExecution.getJobParameters().getString("runStep");
//Depending on the above condition you can return Completed or Failed.
return new FlowExecutionStatus.COMPLETED;
}
}
<batch:job id="export1" parent="baseJob">
<decision id="decision" decider="decider">
<next on="COMPLETED" to="registruj" />
</decision>
<batch:step id="registruj" parent="baseStep">
<tasklet>
<chunk reader="registrujReader" processor="registrujProcessor" writer="registrujWriter" commit-interval="1" />
</tasklet>
</batch:step>
</batch:job>
<bean id="decider" class="RunStepDecider" />
and pass runStep as JobParameter.
Hope can help to solve your problem.