Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit cb3838e

Browse files
author
Weichao Mao
authored
Merge pull request #1 from lavline/master
PhSHI
2 parents 8b80546 + 3029f2b commit cb3838e

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+5584
-0
lines changed

‎PhSHI-Kafka/README.txt‎

Whitespace-only changes.

‎PhSHI-Kafka/lib/pure_lib.jar‎

17.6 MB
Binary file not shown.

‎PhSHI-Kafka/resources/log4j.properties‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
log4j.rootLogger=INFO, stderr
2+
3+
log4j.appender.stderr=org.apache.log4j.ConsoleAppender
4+
log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
5+
log4j.appender.stderr.layout.ConversionPattern=[%d] %p %m (%c)%n
6+
log4j.appender.stderr.Target=System.err

‎PhSHI-Kafka/src/Client/SubConsumer.java‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
package Client;
2+
3+
import MySerdes.ValueSerde;
4+
import Structure.EventVal;
5+
import org.apache.kafka.clients.consumer.ConsumerRecord;
6+
import org.apache.kafka.clients.consumer.ConsumerRecords;
7+
import org.apache.kafka.clients.consumer.KafkaConsumer;
8+
import org.apache.kafka.common.errors.WakeupException;
9+
10+
import java.io.BufferedWriter;
11+
import java.io.File;
12+
import java.io.FileWriter;
13+
import java.util.Arrays;
14+
import java.util.Properties;
15+
16+
public class SubConsumer {
17+
18+
19+
public static void main(String[] args) {
20+
Properties props = new Properties();
21+
props.put("bootstrap.servers", "192.168.101.15:9092,192.168.101.12:9092,192.168.101.28:9092");
22+
props.put("group.id", "Consumer");
23+
props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
24+
props.put("value.deserializer", ValueSerde.EventValDeserde.class.getName());
25+
26+
KafkaConsumer<String, EventVal> consumer = new KafkaConsumer<>(props);
27+
28+
/*
29+
Scanner input = new Scanner(System.in);
30+
String name;
31+
name = input.next();
32+
input.close();
33+
*/
34+
consumer.subscribe(Arrays.asList("TestClient"));
35+
36+
//TopicPartition topicPartition = new TopicPartition("", 0);
37+
38+
// attach shutdown handler to catch control-c
39+
Runtime.getRuntime().addShutdownHook(new Thread("consumer-shutdown-hook") {
40+
@Override
41+
public void run() {
42+
consumer.wakeup();
43+
//latch.countDown();
44+
}
45+
});
46+
BufferedWriter bw = null;
47+
//BufferedWriter bw1 = null;
48+
try {
49+
//read Record
50+
51+
//File file1 = new File("filepath");
52+
File file = new File("/home/ubuntu/Stream-data/rcv-time-1.txt");
53+
54+
FileWriter fw = new FileWriter(file, true);
55+
//FileWriter fw1 = new FileWriter(file1, true);
56+
57+
bw = new BufferedWriter(fw);
58+
//bw1 = new BufferedWriter(fw1);
59+
long tmptime = 0;
60+
double aver_rcv_time = 0;
61+
int x = 0;
62+
while(true) {
63+
ConsumerRecords<String, EventVal> records = consumer.poll(70);
64+
// move offset to begin
65+
//consumer.seekToBeginning(Collections.singleton(topicPartition));
66+
//consumer.seek(topicPartition, 4);
67+
tmptime = System.currentTimeMillis();
68+
for(ConsumerRecord<String, EventVal> record : records) {
69+
70+
EventVal eVal = record.value();
71+
eVal.EventGetTime = tmptime - eVal.EventStartSendTime;
72+
long tmp = tmptime - eVal.EventProduceTime;
73+
74+
//save rcv_time
75+
String s = String.valueOf(eVal.StockId) + " "
76+
+ String.valueOf(eVal.EventArriveTime) + " "
77+
+ String.valueOf(eVal.EventMatchTime / 1000000.0) + " "
78+
+ String.valueOf(eVal.EventGetTime) + " "
79+
+ String.valueOf(tmp);
80+
81+
bw.write(s + "\n");
82+
83+
/*save event
84+
int n = eVal.AttributeNum;
85+
s = "";
86+
for(int i = 0; i < n; i++) {
87+
s = s + String.valueOf(eVal.eventVals[i].attributeId) + " " + eVal.eventVals[i].val + " ";
88+
//System.out.println("attribute id: " + eVal.eventVals[i].attributeId + " value: " + eVal.eventVals[i].val);
89+
}
90+
bw1.write(s + "\n");
91+
*/
92+
93+
aver_rcv_time += tmp;
94+
x++;
95+
if(x % 1000 == 0){
96+
System.out.println("1000rcv aver_ time: " + (aver_rcv_time / 1000.0));
97+
aver_rcv_time = 0;
98+
x = 0;
99+
}
100+
}
101+
}
102+
} catch (WakeupException e) {
103+
104+
} catch (Throwable e) {
105+
System.exit(1);
106+
} finally {
107+
try {
108+
bw.close();
109+
//bw1.close();
110+
}catch (Exception e){
111+
e.printStackTrace();
112+
}
113+
consumer.close();
114+
}
115+
System.exit(0);
116+
}
117+
}

‎PhSHI-Kafka/src/Client/SubProducer.java‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
package Client;
2+
3+
import MySerdes.ValueSerde;
4+
import Structure.SubscribeVal;
5+
import org.apache.kafka.clients.producer.KafkaProducer;
6+
import org.apache.kafka.clients.producer.ProducerRecord;
7+
8+
import java.io.File;
9+
import java.util.Properties;
10+
import java.util.Scanner;
11+
12+
public class SubProducer {
13+
//read file
14+
public static void main(String[] args) {
15+
Scanner input = new Scanner(System.in);
16+
Properties Props = new Properties();
17+
Props.put("bootstrap.servers", "192.168.101.15:9092,192.168.101.12:9092,192.168.101.28:9092");
18+
Props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
19+
Props.put("value.serializer", ValueSerde.SubValSerde.class.getName());
20+
21+
KafkaProducer<String, SubscribeVal> producer = new KafkaProducer<>(Props);
22+
Scanner s = null;
23+
try{
24+
File file = new File("resources/");
25+
s = new Scanner(file);
26+
s.nextInt();
27+
s.nextInt();
28+
s.nextInt();
29+
}catch (Throwable e){
30+
System.exit(1);
31+
}
32+
System.out.println("input sub num: ");
33+
int num = input.nextInt();
34+
for(int i = 0; i < num; i++) {
35+
String SubId = s.next();
36+
int StockId = s.nextInt();
37+
int AttributeNum = s.nextInt();
38+
SubscribeVal sVal = new SubscribeVal(AttributeNum);
39+
sVal.SubId = SubId;
40+
sVal.StockId = StockId;
41+
for(int j = 0; j < sVal.AttributeNum; j++){
42+
sVal.subVals.get(j).attributeId = s.nextInt();
43+
sVal.subVals.get(j).min_val = s.nextDouble();
44+
sVal.subVals.get(j).max_val = s.nextDouble();
45+
}
46+
//Record
47+
ProducerRecord<String, SubscribeVal> record = new ProducerRecord<>("NewSub", sVal);
48+
//send
49+
try {
50+
producer.send(record).get();
51+
System.err.println("Producer Send " + i + " Success!");
52+
Thread.sleep(10);
53+
} catch (Exception e) {
54+
e.printStackTrace();
55+
}
56+
}
57+
producer.close();
58+
s.close();
59+
}
60+
}

‎PhSHI-Kafka/src/Client/creatEvent.java‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
package Client;
2+
3+
import Structure.EventVal;
4+
5+
import java.io.BufferedWriter;
6+
import java.io.File;
7+
import java.io.FileWriter;
8+
import java.util.Random;
9+
import java.util.Scanner;
10+
11+
public class creatEvent {
12+
public static void main(String[] args) {
13+
Scanner input = new Scanner(System.in);
14+
BufferedWriter bw = null;
15+
try{
16+
File file = new File("resources/eventData.txt");
17+
FileWriter fw = new FileWriter(file, true);
18+
bw = new BufferedWriter(fw);
19+
}catch (Throwable e){
20+
System.exit(1);
21+
}
22+
System.out.println("input event num: ");
23+
int num = input.nextInt();
24+
input.nextLine();
25+
System.out.println("input event attribute num: ");
26+
int attribute_num = input.nextInt();
27+
int stock_id = 1;
28+
Random r = new Random();
29+
String s = num + " " + stock_id + " " + attribute_num;
30+
try {
31+
bw.write(s + "\n");
32+
} catch (Exception e) {
33+
e.printStackTrace();
34+
}
35+
for (int i = 0; i < num; i++) {
36+
EventVal eVal = new EventVal(attribute_num, stock_id);
37+
eVal.eventVals[0].attributeId = 0;
38+
eVal.eventVals[0].val = r.nextDouble() * 997 + 2;
39+
s = eVal.eventVals[0].attributeId + " " + eVal.eventVals[0].val;
40+
for(int j = 1; j < attribute_num; j++) {
41+
eVal.eventVals[j].attributeId = j;
42+
eVal.eventVals[j].val = r.nextDouble() * 997 + 2;
43+
s = s + " " + eVal.eventVals[j].attributeId + " " + eVal.eventVals[j].val;
44+
}
45+
try {
46+
bw.write(s + "\n");
47+
} catch (Exception e) {
48+
e.printStackTrace();
49+
}
50+
}
51+
input.close();
52+
try {
53+
bw.flush();
54+
bw.close();
55+
}catch (Exception e){
56+
System.exit(1);
57+
}
58+
}
59+
}

0 commit comments

Comments
 (0)