JHipster with kafka real app usage example - apache-kafka

Question

In JHipster version 6.6.0 the Kafka usage model has been changed from the standard Producer/Consumer classes to the WebResource level. There are no real examples what is an advantage of this change and how this change can be used in real apps.
Let`s say we have Service A and Service B. The communication between these two services has to be accomplished via Kafka events.
The question is - what I have to do so Service B starts to listen to events from the Service A topic. In the current configuration, it looks like I have to manually trigger /consumes endpoint, but it makes no sense because I'm expecting that the service will start to listen for the specified list of topics after the app is up and running.
I would appreciate any comment on this topic to help me understand this.
Example:
jhipster 7.1.0 generates this resources:
Service A - gateway
package com.stukans.refirmware.gateway.web.rest;
import com.stukans.refirmware.gateway.config.KafkaProperties;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.web.bind.annotation.*;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.kafka.receiver.KafkaReceiver;
import reactor.kafka.receiver.ReceiverOptions;
import reactor.kafka.sender.KafkaSender;
import reactor.kafka.sender.SenderOptions;
import reactor.kafka.sender.SenderRecord;
import reactor.kafka.sender.SenderResult;
#RestController
#RequestMapping("/api/gateway-kafka")
public class GatewayKafkaResource {
private final Logger log = LoggerFactory.getLogger(GatewayKafkaResource.class);
private final KafkaProperties kafkaProperties;
private KafkaSender<String, String> sender;
public GatewayKafkaResource(KafkaProperties kafkaProperties) {
this.kafkaProperties = kafkaProperties;
this.sender = KafkaSender.create(SenderOptions.create(kafkaProperties.getProducerProps()));
}
#PostMapping("/publish/{topic}")
public Mono<PublishResult> publish(
#PathVariable String topic,
#RequestParam String message,
#RequestParam(required = false) String key
) {
log.debug("REST request to send to Kafka topic {} with key {} the message : {}", topic, key, message);
return Mono
.just(SenderRecord.create(topic, null, null, key, message, null))
.as(sender::send)
.next()
.map(SenderResult::recordMetadata)
.map(
metadata ->
new PublishResult(metadata.topic(), metadata.partition(), metadata.offset(), Instant.ofEpochMilli(metadata.timestamp()))
);
}
#GetMapping("/consume")
public Flux<String> consume(#RequestParam("topic") List<String> topics, #RequestParam Map<String, String> consumerParams) {
log.debug("REST request to consume records from Kafka topics {}", topics);
Map<String, Object> consumerProps = kafkaProperties.getConsumerProps();
consumerProps.putAll(consumerParams);
consumerProps.remove("topic");
ReceiverOptions<String, String> receiverOptions = ReceiverOptions.<String, String>create(consumerProps).subscription(topics);
return KafkaReceiver.create(receiverOptions).receive().map(ConsumerRecord::value);
}
private static class PublishResult {
public final String topic;
public final int partition;
public final long offset;
public final Instant timestamp;
private PublishResult(String topic, int partition, long offset, Instant timestamp) {
this.topic = topic;
this.partition = partition;
this.offset = offset;
this.timestamp = timestamp;
}
}
}
Service B - agent
package com.stukans.refirmware.agent.web.rest;
import com.stukans.refirmware.agent.config.KafkaProperties;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.web.bind.annotation.*;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.kafka.receiver.KafkaReceiver;
import reactor.kafka.receiver.ReceiverOptions;
import reactor.kafka.sender.KafkaSender;
import reactor.kafka.sender.SenderOptions;
import reactor.kafka.sender.SenderRecord;
import reactor.kafka.sender.SenderResult;
#RestController
#RequestMapping("/api/agent-kafka")
public class AgentKafkaResource {
private final Logger log = LoggerFactory.getLogger(AgentKafkaResource.class);
private final KafkaProperties kafkaProperties;
private KafkaSender<String, String> sender;
public AgentKafkaResource(KafkaProperties kafkaProperties) {
this.kafkaProperties = kafkaProperties;
this.sender = KafkaSender.create(SenderOptions.create(kafkaProperties.getProducerProps()));
}
#PostMapping("/publish/{topic}")
public Mono<PublishResult> publish(
#PathVariable String topic,
#RequestParam String message,
#RequestParam(required = false) String key
) {
log.debug("REST request to send to Kafka topic {} with key {} the message : {}", topic, key, message);
return Mono
.just(SenderRecord.create(topic, null, null, key, message, null))
.as(sender::send)
.next()
.map(SenderResult::recordMetadata)
.map(
metadata ->
new PublishResult(metadata.topic(), metadata.partition(), metadata.offset(), Instant.ofEpochMilli(metadata.timestamp()))
);
}
#GetMapping("/consume")
public Flux<String> consume(#RequestParam("topic") List<String> topics, #RequestParam Map<String, String> consumerParams) {
log.debug("REST request to consume records from Kafka topics {}", topics);
Map<String, Object> consumerProps = kafkaProperties.getConsumerProps();
consumerProps.putAll(consumerParams);
consumerProps.remove("topic");
ReceiverOptions<String, String> receiverOptions = ReceiverOptions.<String, String>create(consumerProps).subscription(topics);
return KafkaReceiver.create(receiverOptions).receive().map(ConsumerRecord::value);
}
private static class PublishResult {
public final String topic;
public final int partition;
public final long offset;
public final Instant timestamp;
private PublishResult(String topic, int partition, long offset, Instant timestamp) {
this.topic = topic;
this.partition = partition;
this.offset = offset;
this.timestamp = timestamp;
}
}
}
That`s the only code related to Kafka that is available.
Prior to version 6.6.0, JHipster generated standard Producer/Consumer classes which I could use to define which topics to listen to. Now it is not clear how to use generated code to emit/listen to the events.

Answer 1

First, since we cannot see your code, we don't know how it does (or should) work...
In any case, you should not be "triggering" consumers via REST; they should be automatically started when the service starts. If there are no messages, then they are polling in the background and idling until the "producer service" pushes a message to the topic.

Answer 2

I make a service like this auto triggering consumers:
#Service
public class KafkaConsumerService {
private final Logger log = LoggerFactory.getLogger(KafkaConsumerService.class);
private static final String GROUP_USER_CREATE_TOPIC = "GROUP_STORE.GROUP_USER.SAVE"; //<application name>.<dataset name>.<event>
private final KafkaProperties kafkaProperties;
private KafkaReceiver<String, String> kafkaReceiver;
private final ObjectMapper objectMapper = new ObjectMapper();
private final GroupMemberService groupMemberService;
public KafkaConsumerService(KafkaProperties kafkaProperties, GroupMemberService groupMemberService) {
this.kafkaProperties = kafkaProperties;
this.groupMemberService = groupMemberService;
}
#PostConstruct
public void start() {
log.info("Kafka consumer starting...");
Map<String, Object> consumerProps = kafkaProperties.getConsumerProps();
ReceiverOptions<String, String> receiverOptions = ReceiverOptions.<String, String>create(consumerProps)
.subscription(Collections.singletonList(GROUP_USER_CREATE_TOPIC));
this.kafkaReceiver = KafkaReceiver.create(receiverOptions);
consumeGroupMember().subscribe();
}
public Flux<GroupMemberDTO> consumeGroupMember() {
log.debug("consumer group member....");
return this.kafkaReceiver
.receive()
.map(ConsumerRecord::value)
.flatMap(
record -> {
try {
GroupMemberDTO groupMemberDTO = objectMapper.readValue(record, GroupMemberDTO.class);
log.debug("Complete convert object: {}", groupMemberDTO);
return groupMemberService.insert(groupMemberDTO);
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
}
);
}
public void shutdown() {
log.info("Shutdown kafka consumer");
}
}