I am trying the following Kafka stream sample code.
http://kafka.apache.org/0110/documentation/streams/tutorial
import java.io.*;
import java.net.URL;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.CountDownLatch;
import javax.servlet.*;
import javax.servlet.annotation.WebServlet;
import javax.servlet.http.*;
import org.json.JSONObject;
import org.apache.kafka.common.serialization.Serdes;
import org.apache.kafka.streams.KafkaStreams;
import org.apache.kafka.streams.StreamsConfig;
import org.apache.kafka.streams.kstream.KStreamBuilder;
#WebServlet("/KafkaStreamsTest")
public class KafkaStreamsTest extends HttpServlet {
/**
*
*/
private final String topic = "testtopic";
private final String streamouttopic = "streamtesttopic";
public static final String CLIENT_ID = "SampleStreamProducer";
JSONObject jsonObject;
#Override
public void doGet(HttpServletRequest request, HttpServletResponse response)
throws IOException, ServletException {
}
// Do the same thing for GET and POST requests
#Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws IOException, ServletException {
System.out.println("Server called");
Properties props = new Properties();
props.put("bootstrap.servers", "localhost");
props.put("group.id", CLIENT_ID);
props.put(StreamsConfig.APPLICATION_ID_CONFIG, CLIENT_ID);
props.put(StreamsConfig.KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
props.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
KStreamBuilder builder = new KStreamBuilder();
builder.stream(topic).to(streamouttopic);
final KafkaStreams streams = new KafkaStreams(builder, props);
final CountDownLatch latch = new CountDownLatch(1);
// attach shutdown handler to catch control-c
Runtime.getRuntime().addShutdownHook(new Thread("streams-shutdown-hook") {
#Override
public void run() {
streams.close();
latch.countDown();
}
});
try {
streams.start();
latch.await();
} catch (Throwable e) {
System.exit(1);
}
System.exit(0);
}
}
When I run this, I am getting error as
Exception in thread "StreamThread-1" java.lang.NoClassDefFoundError: org/apache/kafka/common/record/ByteBufferInputStream
Could someone guide me what could the reason? I have kafka-clients-0.11.0.0-cp1.jar, kafka-streams-0.10.0.0.jar and other required JAR files included in this Eclipse project.
Related
I am using kafka I have a Notification class that i am serializing using spring-kafka.
package com.code2hack.notification;
public class Notification {
private Object message;
private NotificationType type;
public static Notification create(NotificationType type, Object message){
return new Notification(message,type);
}
public Notification(){
}
public Notification(Object message, NotificationType type){
this.message = message;
this.type = type;
}
#Override
public String toString() {
return "Notification{" +
"message=" + message +
", type=" + type +
'}';
}
public <T> T getMessage(Class<T> type){
return (T)this.message;
}
public NotificationType getType(){
return this.type;
}
public void setType(NotificationType type){
this.type = type;
}
public void setMessage(Object message){
this.message = message;
}
}
here is my configuration
spring:
kafka:
producer:
bootstrap-servers: localhost:9092
key-serializer: org.apache.kafka.common.serialization.StringSerializer
value-serializer: org.springframework.kafka.support.serializer.JsonSerializer
When i try to consume the notification from consumer my message part is missing in Notification I am able to to receive type.
I even tried kafka console-consumer there also it prints only type field from my notification message is missing here also.
I don't know what i am missing.
My Consumer configuration is
package com.code2hack.booking;
import com.code2hack.notification.Notification;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.annotation.EnableKafka;
import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
import org.springframework.kafka.core.ConsumerFactory;
import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
import org.springframework.kafka.support.serializer.JsonDeserializer;
import java.util.HashMap;
import java.util.Map;
#Configuration
#EnableKafka
public class KafkaConfiguration {
#Value("${spring.kafka.consumer.bootstrap-servers}")
private String address;
#Bean
public ConsumerFactory<String, Notification> consumerFactory() {
Map<String, Object> props = new HashMap<>();
props.put(
ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,
address);
props.put(
ConsumerConfig.GROUP_ID_CONFIG,
"booking");
JsonDeserializer<Notification> ds = new JsonDeserializer<>();
ds.addTrustedPackages("*");
return new DefaultKafkaConsumerFactory<>(props,
new StringDeserializer(),
ds);
}
#Bean
public ConcurrentKafkaListenerContainerFactory<String, Notification>
kafkaListenerContainerFactory() {
ConcurrentKafkaListenerContainerFactory<String, Notification> factory =
new ConcurrentKafkaListenerContainerFactory<>();
factory.setConsumerFactory(consumerFactory());
return factory;
}
}
Here is my consumer
#KafkaListener(topics = "sql-insert",groupId = "booking")
public void onNotification(#Payload Notification notification){
handleNotification(notification);
}
Please help me.
Note: Actually problem is with JsonSerializer in Kafka. i tried below code and it is not serializing the object properly.
public static void main(String[] args) {
//SpringApplication.run(BookingServiceApplication.class, args);
Notification notification = Notification.create(NotificationType.NEW_SCHEDULED,"Hellow how are you");
byte[] serialize = new JsonSerializer<Notification>().serialize("sql-insert", notification);
System.out.println(new String(serialize));
}
It is giving me the output.
{"type":"NEW_SCHEDULED"}
Is there any way to fix it.
Unless you create a custom serializer, Jackson only works with JavaBean semantics; there is no getter for message; you need to add a simple getter for the message property.
I'm new to javaFX 8 and working with a simple project, but im having some issues with scene.
I get this error:
Error:(35, 25) java: method getScene in class javafx.stage.Window cannot be applied to given types;
required: no arguments
found: javafx.scene.Scene
reason: actual and formal argument lists differ in length
And this is the code:
package ch.makery.address;
import java.io.IOException;
import javafx.application.Application;
import javafx.fxml.FXMLLoader;
import javafx.scene.Scene;
import javafx.scene.layout.AnchorPane;
import javafx.scene.layout.BorderPane;
import javafx.stage.Stage;
public class MainApp extends Application{
private Stage primaryStage;
private BorderPane rootLayout;
#Override
public void start(Stage primaryStage) {
this.primaryStage = primaryStage;
this.primaryStage.setTitle("AddressApp");
initRootLayout();
showPersonOverview();
}
public void initRootLayout() {
try {
FXMLLoader loader = new FXMLLoader();
loader.setLocation(MainApp.class.getResource("view/RootLayout.fxml"));
rootLayout = (BorderPane) loader.load();
Scene scene = new Scene(rootLayout);
primaryStage.getScene(scene);
primaryStage.show();
}catch (IOException e) {
e.printStackTrace();
}
}
public void showPersonOverview() {
try {
FXMLLoader loader = new FXMLLoader();
loader.setLocation(MainApp.class.getResource("view.Personview.fxml"));
AnchorPane personOverview = (AnchorPane) loader.load();
rootLayout.setCenter(personOverview);
}catch (IOException e) {
e.printStackTrace();
}
}
public Stage getPrimaryStage() {
return primaryStage;
}
public static void main(String[] args) {
launch(args);
}
}
Thanks in advance.
I'm using multi-threaded step while reading a file from the resources. Let's say I have several files to be processed & multiple-threads are processing the same file so, I'm not sure at which point in time my whole files get processed.
Once my file successfully processed, I need to archive/delete the file. Can someone guide me what should I use?
Here is my sample code.
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
import java.util.stream.Stream;
import javax.sql.DataSource;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobExecutionListener;
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.JobParametersBuilder;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.StepExecution;
import org.springframework.batch.core.annotation.AfterStep;
import org.springframework.batch.core.annotation.BeforeStep;
import org.springframework.batch.core.configuration.annotation.BatchConfigurer;
import org.springframework.batch.core.configuration.annotation.DefaultBatchConfigurer;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.launch.JobLauncher;
import org.springframework.batch.core.launch.support.RunIdIncrementer;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.NonTransientResourceException;
import org.springframework.batch.item.ParseException;
import org.springframework.batch.item.UnexpectedInputException;
import org.springframework.batch.support.transaction.ResourcelessTransactionManager;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.Resource;
import org.springframework.core.task.SimpleAsyncTaskExecutor;
import org.springframework.core.task.TaskExecutor;
import com.iana.spring.batch.dao.GenericDAO;
import com.iana.spring.batch.listener.BatchJobCompletionListener;
#Configuration
public class BatchConfig {
#Autowired
private JobBuilderFactory jobBuilderFactory;
#Autowired
private StepBuilderFactory stepBuilderFactory;
#Autowired
private JobLauncher jobLauncher;
#Autowired
private Job processJob;
#Value("classpath*:/final/HYUMER_SI_*.txt")
private Resource[] inputFiles;
#Autowired
#Qualifier("test2DataSource")
private DataSource test2DataSource;
public void saveFileLog(String fileLog) throws Exception{
String query = "INSERT INTO FILE_LOG(LOG_INFO) VALUES (?)";
new GenericDAO().saveOrUpdate(test2DataSource, query, false, fileLog);
}
// This job runs in every 5 seconds
//#Scheduled(fixedRate = 150000000)
public void fixedRatedCallingMethod() {
try {
JobParameters jobParameters = new JobParametersBuilder()
.addLong("time", System.currentTimeMillis())
.toJobParameters();
jobLauncher.run(processJob, jobParameters);
System.out.println("I have been scheduled with Spring scheduler");
} catch (Exception e) {
e.printStackTrace();
}
}
/* In case of multiple DataSources configuration- we need to add following code.
* - It is a good practice to provide Spring Batch database as #Primary to get the benefits of all default functionalities
* implemented by Spring Batch Statistics.
* - All insert and update batch job running statistics will be maintained by Spring Batch Itself.
* - No need to write any extra line of codes.
* Error: To use the default BatchConfigurer the context must contain no more than one DataSource, found 2
*/
#Bean
BatchConfigurer configurer(#Qualifier("testDataSource") DataSource dataSource){
return new DefaultBatchConfigurer(dataSource);
}
#Bean
public Job processJob() throws Exception{
return jobBuilderFactory.get("processJob")
.incrementer(new RunIdIncrementer())
.listener(listener())
.flow(orderStep1())
.end()
.build();
}
#Bean
public TaskExecutor taskExecutor(){
SimpleAsyncTaskExecutor asyncTaskExecutor=new SimpleAsyncTaskExecutor("spring_batch");
asyncTaskExecutor.setConcurrencyLimit(20);
return asyncTaskExecutor;
}
#Bean
public ItemReader<String> batchItemReader() {
Queue<String> dataList = new LinkedList<String>();
return new ItemReader<String>() {
#BeforeStep
public void beforeStep(StepExecution stepExecution) {
System.err.println("in before step...");
try {
if(inputFiles != null) {
for (int i = 0; i < inputFiles.length; i++) {
String fileName = inputFiles[i].getFile().getAbsolutePath();
try (Stream<String> stream = Files.lines(Paths.get(fileName))) {
stream.forEach( s -> dataList.add(s));
} catch (IOException e) {
e.printStackTrace();
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
System.out.println("fileList Size::"+dataList.size());
}
#Override
public synchronized String read()throws Exception, UnexpectedInputException, ParseException, NonTransientResourceException {
System.out.println("--> in item reader.........");
String fileName = null;
if (dataList.size() > 0) {
fileName = dataList.remove();
file_reading_cnt++;
}
return fileName;
}
#AfterStep
public void afterStep(StepExecution stepExecution) {
System.err.println("in after step..."+file_reading_cnt);
}
};
}
volatile int file_reading_cnt = 0;
#Bean
public ItemWriter<String> batchItemWriter(){
return new ItemWriter<String>() {
#Override
public void write(List<? extends String> fileList) throws Exception {
System.out.println("----- in item writer.........");
fileList.forEach(data -> {
try {
saveFileLog(data);
} catch (Exception e) {
e.printStackTrace();
}
});
}
};
}
/**
* To create a step, reader, processor and writer has been passed serially
*
* #return
*/
#Bean
public Step orderStep1() throws Exception{
return stepBuilderFactory.get("orderStep1").<String, String>chunk(20)
.reader(batchItemReader())
.writer(batchItemWriter())
.taskExecutor(taskExecutor())
.throttleLimit(20)
.build();
}
#Bean
public JobExecutionListener listener() {
return new BatchJobCompletionListener();
}
#Bean
public ResourcelessTransactionManager transactionManager() {
return new ResourcelessTransactionManager();
}
}
In my project, I used #KakfaListener configure kafka containerFactory and topic.
Topic Name:
public static final String CONNECT_DEVICE_MESSAGE_TOPIC = "connectDeviceMessageTopic";
Topic Listen:
#KafkaListener(containerFactory = "receiveKafkaListenerContainerFactory", topics = KafkaQueueName.CONNECT_DEVICE_MESSAGE_TOPIC)
public void onMessageListener(MessageTemplate message){
}
Kafka Config:
package me.hekr.bot.parse.core.kafka;
import lombok.extern.slf4j.Slf4j;
import me.hekr.bot.utils.IpUtil;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.annotation.EnableKafka;
import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
import org.springframework.kafka.config.KafkaListenerContainerFactory;
import org.springframework.kafka.core.*;
import org.springframework.kafka.listener.ConcurrentMessageListenerContainer;
import org.springframework.kafka.support.converter.StringJsonMessageConverter;
import java.util.HashMap;
import java.util.Map;
/**
* Created by Neon Wang on 2016/10/20.
*/
#EnableKafka
#Configuration
#Slf4j
public class KafkaConfig {
#Value("${bot.kafka.servers}")
private String servers;
/*********************** Producer Config ***************************/
private ProducerFactory<String, String> producerFactory() {
return new DefaultKafkaProducerFactory<>(producerConfigs());
}
private Map<String, Object> producerConfigs() {
return new CustomHashMap().put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, servers)
.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer")
.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer")
.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 1000 * 2);
}
#Bean
public KafkaTemplate<String, String> kafkaTemplate() {
return new KafkaTemplate<>(producerFactory());
}
/*********************** Consumer Config ***************************/
private Map<String, Object> consumerProps() {
return new CustomHashMap()
.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, servers)
.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true)
.put(ConsumerConfig.GROUP_ID_CONFIG, "parseReceiveMessageFormConnection")
.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "100")
.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "15000")
.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class)
.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
}
#Bean
KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>>
receiveKafkaListenerContainerFactory() {
ConcurrentKafkaListenerContainerFactory<String, String> factory =
new ConcurrentKafkaListenerContainerFactory<>();
factory.setConsumerFactory(consumerFactory());
factory.setConcurrency(3);
factory.setMessageConverter(new StringJsonMessageConverter());
factory.getContainerProperties().setPollTimeout(3000L);
return factory;
}
private ConsumerFactory<String, String> consumerFactory() {
return new DefaultKafkaConsumerFactory<>(consumerProps());
}
class CustomHashMap extends HashMap<String, Object> {
CustomHashMap(){
super();
}
#Override
public CustomHashMap put(String key, Object value) {
super.put(key, value);
return this;
}
}
}
Started project, kafka configuration information was successed, but I found every topic information had logged three times, is it normal？
And when the first message reviced, turn out print
2017-08-16T11:12:43.633+0800 INFO [org.springframework.kafka.KafkaListenerEndpointContainer#0-2-kafka-consumer-1] o.a.k.c.c.i.AbstractCoordinator.handle:623 - Attempt to heart beat failed since the group is rebalancing, try to re-join group.
Then the first message had received again and again, all result turn out was same！
After three times, hadn't received any message, I had no idea, who can help me?
The group id had changed twice， in last version had used localhost ip.
IpUtil.getLocalhostAddress().replace(".", "")
I want to print the Shipping rate those come on the checkout process, Shipping is calculated dynamically, on the bases of address or may be product in the Cart. I want to print the Shipping rate those come on the 2nd checkout page after calculation.
import com.thoughtworks.selenium.*;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.*;
import java.util.regex.Pattern;
public class testing {
private Selenium selenium;
#Before
public void setUp() throws Exception {
selenium = new DefaultSelenium("localhost", 4444, "*chrome", "http://testing-671.myshopify.com/");
selenium.start();
}
#Test
public void testtesting() throws Exception {
selenium.open("/");
selenium.click("link=Testing");
selenium.waitForPageToLoad("30000");
selenium.click("//a[contains(text(),'Catalog')]");
selenium.waitForPageToLoad("30000");
selenium.click("css=img[alt=\"testing\"]");
selenium.waitForPageToLoad("30000");
selenium.click("id=add-to-cart");
selenium.waitForPageToLoad("30000");
selenium.click("id=checkout");
selenium.waitForPageToLoad("30000");
selenium.type("id=checkout_email", "tester#gmail.com");
selenium.type("id=checkout_shipping_address_first_name", "test");
selenium.type("id=checkout_shipping_address_last_name", "test");
selenium.type("id=checkout_shipping_address_company", "test");
selenium.type("id=checkout_shipping_address_address1", "test");
selenium.type("id=checkout_shipping_address_address2", "test");
selenium.type("id=checkout_shipping_address_city", "test");
selenium.select("id=checkout_shipping_address_country", "label=Albania");
selenium.click("css=option[value=\"Albania\"]");
selenium.select("id=checkout_shipping_address_country", "label=India");
selenium.select("id=checkout_shipping_address_province", "label=Chandigarh");
selenium.type("id=checkout_shipping_address_zip", "160062");
selenium.click("name=commit");
selenium.waitForPageToLoad("30000");
}
#After
public void tearDown() throws Exception {
selenium.stop();
}
}
This is the updated code that I've retrieved by running in Selenium IDE and exporting the testcase as program for Selenium RC and junit:
import com.thoughtworks.selenium.*;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.*;
import java.util.regex.Pattern;
public class Testcases {
private Selenium selenium;
#Before
public void setUp() throws Exception {
selenium = new DefaultSelenium("localhost", 4444, "*chrome", "http://testing-671.myshopify.com");
selenium.start();
}
#Test
public void testCases() throws Exception {
selenium.open("/");
selenium.click("link=Testing");
selenium.waitForPageToLoad("");
selenium.click("//a[contains(text(),'Catalog')]");
selenium.waitForPageToLoad("30000");
selenium.click("css=img[alt=\"testing\"]");
selenium.waitForPageToLoad("30000");
selenium.click("id=add-to-cart");
selenium.waitForPageToLoad("30000");
selenium.click("id=checkout");
selenium.waitForPageToLoad("30000");
selenium.type("id=checkout_email", "tester#gmail.com");
selenium.type("id=checkout_shipping_address_first_name", "test");
selenium.type("id=checkout_shipping_address_last_name", "test");
selenium.type("id=checkout_shipping_address_company", "test");
selenium.type("id=checkout_shipping_address_address1", "test");
selenium.type("id=checkout_shipping_address_address2", "test");
selenium.type("id=checkout_shipping_address_city", "test");
selenium.select("id=checkout_shipping_address_country", "label=Albania");
selenium.select("id=checkout_shipping_address_province", "label=Chandigarh");
selenium.type("id=checkout_shipping_address_zip", "160062");
selenium.click("name=commit");
selenium.waitForPageToLoad("30000");
for (int second = 0;; second++) {
if (second >= 60) fail("timeout");
try { if (selenium.isElementPresent("//span[#class='total-line__name' and contains(text(),'Shipping')]/following-sibling::strong[contains(text(),'Rs.')]")) break; } catch (Exception e) {}
Thread.sleep(1000);
}
String Price = selenium.getText("//span[#class='total-line__name' and contains(text(),'Shipping')]/following-sibling::strong");
System.out.println(Price);
}
#After
public void tearDown() throws Exception {
selenium.stop();
}
}