Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 5730f38

Browse files
fmbenhassinemminella
fmbenhassine
authored and
mminella
committed
Add remote partitioning samples
This commit also fixes the code examples in the remote partitioning section of the documentation Resolves BATCH-2730
1 parent 1046862 commit 5730f38

File tree

13 files changed

+913
-34
lines changed

13 files changed

+913
-34
lines changed

‎spring-batch-docs/asciidoc/spring-batch-integration.adoc‎

Lines changed: 44 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1128,6 +1128,10 @@ configuration:
11281128
.Java Configuration
11291129
[source, java, role="javaContent"]
11301130
----
1131+
1132+
/*
1133+
* Configuration of the master side
1134+
*/
11311135
@Bean
11321136
public PartitionHandler partitionHandler() {
11331137
MessageChannelPartitionHandler partitionHandler = new MessageChannelPartitionHandler();
@@ -1141,6 +1145,11 @@ public PartitionHandler partitionHandler() {
11411145
return partitionHandler;
11421146
}
11431147
1148+
@Bean
1149+
public QueueChannel outboundReplies() {
1150+
return new QueueChannel();
1151+
}
1152+
11441153
@Bean
11451154
public DirectChannel outboundRequests() {
11461155
return new DirectChannel();
@@ -1155,19 +1164,33 @@ public IntegrationFlow outboundJmsRequests() {
11551164
}
11561165
11571166
@Bean
1158-
public DirectChannel inboundRequests() {
1167+
@ServiceActivator(inputChannel = "inboundStaging")
1168+
public AggregatorFactoryBean partitioningMessageHandler() throws Exception {
1169+
AggregatorFactoryBean aggregatorFactoryBean = new AggregatorFactoryBean();
1170+
aggregatorFactoryBean.setProcessorBean(partitionHandler());
1171+
aggregatorFactoryBean.setOutputChannel(outboundReplies());
1172+
// configure other propeties of the aggregatorFactoryBean
1173+
return aggregatorFactoryBean;
1174+
}
1175+
1176+
@Bean
1177+
public DirectChannel inboundStaging() {
11591178
return new DirectChannel();
11601179
}
11611180
1162-
public IntegrationFlow inboundJmsRequests() {
1181+
@Bean
1182+
public IntegrationFlow inboundJmsStaging() {
11631183
return IntegrationFlows
11641184
.from(Jms.messageDrivenChannelAdapter(connectionFactory())
11651185
.configureListenerContainer(c -> c.subscriptionDurable(false))
1166-
.destination("requestsQueue"))
1167-
.channel(inboundRequests())
1186+
.destination("stagingQueue"))
1187+
.channel(inboundStaging())
11681188
.get();
1169-
}
1189+
}
11701190
1191+
/*
1192+
* Configuration of the worker side
1193+
*/
11711194
@Bean
11721195
public StepExecutionRequestHandler stepExecutionRequestHandler() {
11731196
StepExecutionRequestHandler stepExecutionRequestHandler = new StepExecutionRequestHandler();
@@ -1183,46 +1206,30 @@ public StepExecutionRequestHandler serviceActivator() throws Exception {
11831206
}
11841207
11851208
@Bean
1186-
public DirectChannel outboundStaging() {
1187-
return new DirectChannel();
1188-
}
1189-
1190-
@Bean
1191-
public IntegrationFlow outboundJmsStaging() {
1192-
return IntegrationFlows.from("outboundStaging")
1193-
.handle(Jms.outboundGateway(connectionFactory())
1194-
.requestDestination("stagingQueue"))
1195-
.get();
1196-
}
1197-
1198-
@Bean
1199-
public DirectChannel inboundStaging() {
1209+
public DirectChannel inboundRequests() {
12001210
return new DirectChannel();
12011211
}
12021212
1203-
@Bean
1204-
public IntegrationFlow inboundJmsStaging() {
1213+
public IntegrationFlow inboundJmsRequests() {
12051214
return IntegrationFlows
12061215
.from(Jms.messageDrivenChannelAdapter(connectionFactory())
12071216
.configureListenerContainer(c -> c.subscriptionDurable(false))
1208-
.destination("stagingQueue"))
1209-
.channel(inboundStaging())
1217+
.destination("requestsQueue"))
1218+
.channel(inboundRequests())
12101219
.get();
12111220
}
12121221
12131222
@Bean
1214-
@ServiceActivator(inputChannel = "inboundStaging")
1215-
public AggregatorFactoryBean partitioningMessageHandler() throws Exception {
1216-
AggregatorFactoryBean aggregatorFactoryBean = new AggregatorFactoryBean();
1217-
aggregatorFactoryBean.setProcessorBean(partitionHandler());
1218-
aggregatorFactoryBean.setOutputChannel(outboundReplies());
1219-
...
1220-
return aggregatorFactoryBean;
1223+
public DirectChannel outboundStaging() {
1224+
return new DirectChannel();
12211225
}
12221226
12231227
@Bean
1224-
public QueueChannel outboundReplies() {
1225-
return new QueueChannel();
1228+
public IntegrationFlow outboundJmsStaging() {
1229+
return IntegrationFlows.from("outboundStaging")
1230+
.handle(Jms.outboundGateway(connectionFactory())
1231+
.requestDestination("stagingQueue"))
1232+
.get();
12261233
}
12271234
----
12281235

@@ -1245,8 +1252,12 @@ You must also ensure that the partition `handler` attribute maps to the `partiti
12451252
public Job personJob() {
12461253
return jobBuilderFactory.get("personJob")
12471254
.start(stepBuilderFactory.get("step1.master")
1248-
.partitioner(partitionHandler())
1255+
.partitioner("step1.worker", partitioner())
1256+
.partitionHandler(partitionHandler())
12491257
.build())
12501258
.build();
12511259
}
12521260
----
1261+
1262+
You can find a complete example of a remote partitioning job
1263+
link:$$https://github.com/spring-projects/spring-batch/tree/master/spring-batch-samples#remote-partitioning-sample$$[here].

‎spring-batch-samples/README.md‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -638,13 +638,27 @@ happens inside the main step transaction).
638638
The purpose of this sample is to show multi-threaded step execution
639639
using the `PartitionHandler` SPI. The example uses a
640640
`TaskExecutorPartitionHandler` to spread the work of reading
641-
some files acrosss multiple threads, with one `Step` execution
641+
some files across multiple threads, with one `Step` execution
642642
per thread. The key components are the `PartitionStep` and the
643643
`MultiResourcePartitioner` which is responsible for dividing up
644644
the work. Notice that the readers and writers in the `Step`
645645
that is being partitioned are step-scoped, so that their state does
646646
not get shared across threads of execution.
647647

648+
### [Remote Partitioning Sample](id:remotePartitioning)
649+
650+
This sample shows how to configure a remote partitioning job. The master step
651+
uses a `MessageChannelPartitionHandler` to send partitions to and receive
652+
replies from workers. Two examples are shown:
653+
654+
* A master step that polls the job repository to see if all workers have finished
655+
their work
656+
* A master step that aggregates replies from workers to notify work completion
657+
658+
The sample uses an embedded JMS broker and an embedded database for simplicity
659+
but any option supported via Spring Integration for communication is technically
660+
acceptable.
661+
648662
### [Remote Chunking Sample](id:remoteChunking)
649663

650664
This sample shows how to configure a remote chunking job. The master step will

‎spring-batch-samples/src/main/java/org/springframework/batch/sample/remotepartitioning/BasicPartitioner.java‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
/*
2+
* Copyright 2018 the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.springframework.batch.sample.remotepartitioning;
18+
19+
import java.util.Map;
20+
21+
import org.springframework.batch.core.partition.support.SimplePartitioner;
22+
import org.springframework.batch.item.ExecutionContext;
23+
24+
/**
25+
* Simple partitioner for demonstration purpose.
26+
*
27+
* @author Mahmoud Ben Hassine
28+
*/
29+
public class BasicPartitioner extends SimplePartitioner {
30+
31+
private static final String PARTITION_KEY = "partition";
32+
33+
@Override
34+
public Map<String, ExecutionContext> partition(int gridSize) {
35+
Map<String, ExecutionContext> partitions = super.partition(gridSize);
36+
int i = 0;
37+
for (ExecutionContext context : partitions.values()) {
38+
context.put(PARTITION_KEY, PARTITION_KEY + (i++));
39+
}
40+
return partitions;
41+
}
42+
43+
}

‎spring-batch-samples/src/main/java/org/springframework/batch/sample/remotepartitioning/BrokerConfiguration.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/*
2+
* Copyright 2018 the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.springframework.batch.sample.remotepartitioning;
18+
19+
import org.apache.activemq.ActiveMQConnectionFactory;
20+
21+
import org.springframework.beans.factory.annotation.Value;
22+
import org.springframework.context.annotation.Bean;
23+
import org.springframework.context.annotation.Configuration;
24+
import org.springframework.context.annotation.PropertySource;
25+
26+
/**
27+
* @author Mahmoud Ben Hassine
28+
*/
29+
@Configuration
30+
@PropertySource("classpath:remote-partitioning.properties")
31+
public class BrokerConfiguration {
32+
33+
@Value("${broker.url}")
34+
private String brokerUrl;
35+
36+
@Bean
37+
public ActiveMQConnectionFactory connectionFactory() {
38+
ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory();
39+
connectionFactory.setBrokerURL(this.brokerUrl);
40+
connectionFactory.setTrustAllPackages(true);
41+
return connectionFactory;
42+
}
43+
44+
}

‎spring-batch-samples/src/main/java/org/springframework/batch/sample/remotepartitioning/DataSourceConfiguration.java‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/*
2+
* Copyright 2018 the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.springframework.batch.sample.remotepartitioning;
18+
19+
import javax.sql.DataSource;
20+
21+
import org.apache.commons.dbcp2.BasicDataSource;
22+
23+
import org.springframework.beans.factory.annotation.Value;
24+
import org.springframework.context.annotation.Bean;
25+
import org.springframework.context.annotation.Configuration;
26+
import org.springframework.context.annotation.PropertySource;
27+
28+
/**
29+
* @author Mahmoud Ben Hassine
30+
*/
31+
@Configuration
32+
@PropertySource("classpath:remote-partitioning.properties")
33+
public class DataSourceConfiguration {
34+
35+
@Value("${datasource.url}")
36+
private String url;
37+
38+
@Value("${datasource.username}")
39+
private String username;
40+
41+
@Value("${datasource.password}")
42+
private String password;
43+
44+
@Bean
45+
public DataSource dataSource() {
46+
BasicDataSource dataSource = new BasicDataSource();
47+
dataSource.setUrl(url);
48+
dataSource.setUsername(username);
49+
dataSource.setPassword(password);
50+
return dataSource;
51+
}
52+
53+
}

0 commit comments

Comments
 (0)