This commit is contained in:
959814898@qq.com
2023-07-22 22:35:18 +08:00
parent f978245495
commit ec6035d80f
35 changed files with 1076 additions and 202 deletions
+14 -2
View File
@@ -11,13 +11,25 @@
<artifactId>kafka</artifactId>
<properties>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>3.22.3</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java-util -->
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java-util</artifactId>
<version>3.22.3</version>
</dependency>
</dependencies>
</project>
@@ -14,15 +14,17 @@ import java.util.stream.Stream;
/**
* kafka发送消息测试
* @ClassName: KafkaProducerSend
* @Date: 2022/4/12 15:18
*
* @author wangyl
* @version V1.0
* @ClassName: KafkaProducerSend
* @Date: 2022/4/12 15:18
*/
@Slf4j
public class KafkaConsumerPoll {
/**
* 消费者消费消息
*
* @param kafkaConsumerNormal
* @param topic
* @return void
@@ -32,18 +34,21 @@ public class KafkaConsumerPoll {
*/
public static void PollMessage(Consumer<String, String> kafkaConsumerNormal, String topic) {
//消费者消费消息
kafkaConsumerNormal.subscribe(Arrays.asList(topic));
// kafkaConsumerNormal.subscribe(Arrays.asList(topic));
TopicPartition topicPartition = new TopicPartition(topic, 1);
kafkaConsumerNormal.assign(Arrays.asList(topicPartition));
while (true) {
//消费消息
kafkaConsumerNormal.poll(Duration.ofMillis(100))
.forEach(record -> {
log.info("消费者消费消息:{},分区:{},topic{},offset:{}", record.value(), record.partition(), record.topic(), record.offset());
});
.forEach(record -> {
log.info("消费者消费消息:{},分区:{},topic{},offset:{}", record.value(), record.partition(), record.topic(), record.offset());
});
}
}
/**
* 同步手动提交
*
* @param kafkaConsumerNormal
* @param topic
* @return void
@@ -57,9 +62,9 @@ public class KafkaConsumerPoll {
while (true) {
//消费消息
kafkaConsumerNormal.poll(Duration.ofMillis(100))
.forEach(record -> {
log.info("消费者消费消息:{},分区:{},topic{},offset:{}", record.value(), record.partition(), record.topic(), record.offset());
});
.forEach(record -> {
log.info("消费者消费消息:{},分区:{},topic{},offset:{}", record.value(), record.partition(), record.topic(), record.offset());
});
kafkaConsumerNormal.commitSync();
}
@@ -67,6 +72,7 @@ public class KafkaConsumerPoll {
/**
* 同步手动提交offset
*
* @param kafkaConsumerNormal
* @param topic
* @return void
@@ -81,10 +87,10 @@ public class KafkaConsumerPoll {
while (true) {
//消费消息
kafkaConsumerNormal.poll(Duration.ofMillis(100))
.forEach(record -> {
log.info("消费者消费消息:{},分区:{},topic{},offset:{}", record.value(), record.partition(), record.topic(), record.offset());
offsets.put(new TopicPartition(record.topic(), record.partition()), new OffsetAndMetadata(record.offset() + 1));
});
.forEach(record -> {
log.info("消费者消费消息:{},分区:{},topic{},offset:{}", record.value(), record.partition(), record.topic(), record.offset());
offsets.put(new TopicPartition(record.topic(), record.partition()), new OffsetAndMetadata(record.offset() + 1));
});
kafkaConsumerNormal.commitSync(offsets);
offsets.clear();
}
@@ -92,6 +98,7 @@ public class KafkaConsumerPoll {
/**
* 异步提交,有回调
*
* @param kafkaConsumerNormal
* @param topic
* @return void
@@ -106,10 +113,10 @@ public class KafkaConsumerPoll {
while (true) {
//消费消息
kafkaConsumerNormal.poll(Duration.ofMillis(100))
.forEach(record -> {
log.info("消费者消费消息:{},分区:{},topic{},offset:{}", record.value(), record.partition(), record.topic(), record.offset());
offsets.put(new TopicPartition(record.topic(), record.partition()), new OffsetAndMetadata(record.offset() + 1));
});
.forEach(record -> {
log.info("消费者消费消息:{},分区:{},topic{},offset:{}", record.value(), record.partition(), record.topic(), record.offset());
offsets.put(new TopicPartition(record.topic(), record.partition()), new OffsetAndMetadata(record.offset() + 1));
});
kafkaConsumerNormal.commitAsync(offsets, (offsetsInfo, exception) -> {
if (exception != null) {
log.error("提交消息异常:{}", exception.getMessage());
@@ -122,6 +129,7 @@ public class KafkaConsumerPoll {
/**
* 异步回调offsets
*
* @param kafkaConsumerNormal
* @param topic
* @return void
@@ -135,15 +143,15 @@ public class KafkaConsumerPoll {
while (true) {
//消费消息
kafkaConsumerNormal.poll(Duration.ofMillis(100))
.forEach(record -> {
log.info("消费者消费消息:{},分区:{},topic{},offset:{}", record.value(), record.partition(), record.topic(), record.offset());
});
.forEach(record -> {
log.info("消费者消费消息:{},分区:{},topic{},offset:{}", record.value(), record.partition(), record.topic(), record.offset());
});
kafkaConsumerNormal.commitAsync();
}
}
public static void PollMessageSeek(Consumer<String, String> kafkaConsumerNormal, String topic,int partition,int offset) {
public static void PollMessageSeek(Consumer<String, String> kafkaConsumerNormal, String topic, int partition, int offset) {
//消费者消费消息
kafkaConsumerNormal.subscribe(Arrays.asList(topic), new ConsumerRebalanceListener() {
@Override
@@ -160,14 +168,14 @@ public class KafkaConsumerPoll {
while (true) {
//消费消息
kafkaConsumerNormal.poll(Duration.ofMillis(100))
.forEach(record -> {
log.info("消费者消费消息:{},分区:{},topic{},offset:{}", record.value(), record.partition(), record.topic(), record.offset());
});
.forEach(record -> {
log.info("消费者消费消息:{},分区:{},topic{},offset:{}", record.value(), record.partition(), record.topic(), record.offset());
});
kafkaConsumerNormal.commitAsync();
}
}
public static void PollMessageSeekBegin(Consumer<String, String> kafkaConsumerNormal, String topic,int partition) {
public static void PollMessageSeekBegin(Consumer<String, String> kafkaConsumerNormal, String topic, int partition) {
final TopicPartition topicPartition = new TopicPartition(topic, partition);
final List<TopicPartition> topicPartitions = Arrays.asList(topicPartition);
kafkaConsumerNormal.subscribe(Arrays.asList(topic), new ConsumerRebalanceListener() {
@@ -185,14 +193,14 @@ public class KafkaConsumerPoll {
while (true) {
//消费消息
kafkaConsumerNormal.poll(Duration.ofMillis(100))
.forEach(record -> {
log.info("消费者消费消息:{},分区:{},topic{},offset:{}", record.value(), record.partition(), record.topic(), record.offset());
});
.forEach(record -> {
log.info("消费者消费消息:{},分区:{},topic{},offset:{}", record.value(), record.partition(), record.topic(), record.offset());
});
kafkaConsumerNormal.commitAsync();
}
}
public static void PollMessageSeekEnd(Consumer<String, String> kafkaConsumerNormal, String topic,int partition) {
public static void PollMessageSeekEnd(Consumer<String, String> kafkaConsumerNormal, String topic, int partition) {
//消费者消费消息
final TopicPartition topicPartition = new TopicPartition(topic, partition);
final List<TopicPartition> topicPartitions = Arrays.asList(topicPartition);
@@ -211,9 +219,9 @@ public class KafkaConsumerPoll {
while (true) {
//消费消息
kafkaConsumerNormal.poll(Duration.ofMillis(100))
.forEach(record -> {
log.info("消费者消费消息:{},分区:{},topic{},offset:{}", record.value(), record.partition(), record.topic(), record.offset());
});
.forEach(record -> {
log.info("消费者消费消息:{},分区:{},topic{},offset:{}", record.value(), record.partition(), record.topic(), record.offset());
});
kafkaConsumerNormal.commitAsync();
}
}
@@ -8,10 +8,11 @@ import java.util.Properties;
/**
* kafak生产者示例
* @ClassName: KafkaProducer
* @Date: 2022/4/12 14:36
*
* @author wangyl
* @version V1.0
* @ClassName: KafkaProducer
* @Date: 2022/4/12 14:36
*/
public class KafkaProducerExample {
+107
View File
@@ -0,0 +1,107 @@
syntax = "proto3";
package ivf_tl_Entity.DTO;
//生成的java文件名
option java_package = "com.aivfo.data.transmission.entity.dto";
option java_outer_classname = "ImageDataDTO";
message ImageDTO {
/**
* tl设备编号
*/
string tlSn = 1;
/**
* 仓室编号:1-10
*/
int32 houseSn = 2;
/**
* well编号 1-16
*/
int32 wellSn = 3;
/**
* ccd编号
*/
string ccdSn = 4;
/**
* 原图名字
*/
string sourceImageName = 5;
/**
* 原图的本地路径(不是上传后的保存路径)
*/
string sourceImagePath = 6;
/**
* 原图宽
*/
int32 sourceImageWidth = 7;
/**
* 原图高
*/
int32 sourceImageHeight = 8;
/**
* 设备拍摄时间 格式2020-10-10 00:00:00
*/
string imageTime = 9;
/**
* 受精时间 格式2020-10-10 00:00:00
*/
string fertilizationTime = 10;
/**
* 对焦还是ccd拍照 1 自动对焦 0 CCD拍照
*/
int32 photographType = 11;
/**
* 拍照总层数
*/
int32 totalLayer = 12;
/**
* 图片层
*/
int32 pictureLayer = 13;
/**
* 垂直电机位置
*/
int32 shootingPosition = 14;
/**
* 是否是最清晰的(CCD拍照才有)0不是 1最清晰
*/
int32 clearest = 15;
/**
* 拍照结束标记 1 结束 0 未结束 (结束后开始更新对焦起点,ccd拍照结束开始合成视频)
*/
int32 end = 16;
/**
* 培养记录id
*/
uint64 embryoCultureRecordId = 17;
/**
* 胚胎id
*/
uint64 embryoId = 18;
/**
* 图片数据
*/
bytes imageData = 19;
/**
* 水平电机位置
*/
int32 horizontalPosition = 20;
}
@@ -12,9 +12,11 @@ import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.TopicPartition;
import org.junit.jupiter.api.Test;
import java.io.*;
import java.time.Duration;
import java.util.Arrays;
import java.util.Properties;
@@ -70,7 +72,7 @@ public class ConsumerTest {
}
/**
* 移动到指定问位置消费消息
* 移动到指定问位置消费消息
*/
@Test
public void pollMessageSeekTest() {
@@ -79,7 +81,7 @@ public class ConsumerTest {
}
/**
* 移动到开始位置消费消息
* 移动到开始位置消费消息
*/
@Test
public void pollMessageSeekBeginTest() {
@@ -88,7 +90,7 @@ public class ConsumerTest {
}
/**
* 移动到结束位置消费消息
* 移动到结束位置消费消息
*/
@Test
public void pollMessageCommitSeekEndTest() {
@@ -102,27 +104,21 @@ public class ConsumerTest {
// 创建 Kafka 消费者配置
Properties consumerProps = new Properties();
consumerProps.put("bootstrap.servers", "localhost:9092");
consumerProps.put("bootstrap.servers", "192.168.31.89:9092");
consumerProps.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
consumerProps.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
consumerProps.put("group.id","test");
consumerProps.put("group.id", "test");
consumerProps.put("max.request.size", "2097152");
// 创建 Kafka 生产者和消费者
KafkaConsumer<String, String> consumer = new KafkaConsumer<>(consumerProps);
TopicPartition topicPartition = new TopicPartition("wyl_topic", 1);
consumer.assign(Arrays.asList(topicPartition));
// 接收图片消息从 Kafka
consumer.subscribe(Arrays.asList("wyl-events"));
while (true) {
ConsumerRecords<String, String> records = consumer.poll(100);
ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(100));
for (ConsumerRecord<String, String> record : records) {
byte[] imageBytes = record.value().getBytes(ISO_8859_1);
// 将图片字节数组保存到文件中
// 这里假设图片文件名为 received-image.jpg
File receivedFile = new File("/Users/wyl/Desktop/wyl01.png");
FileOutputStream fileOutputStream = new FileOutputStream(receivedFile);
fileOutputStream.write(imageBytes);
fileOutputStream.close();
System.out.println(11);
String value = record.value();
System.out.println(value);
}
}
}
@@ -1,18 +1,22 @@
package com.wyl.kafka.test;
import com.google.protobuf.ByteString;
import com.wyl.kafka.producers.KafkaProducerExample;
import com.wyl.kafka.producers.KafkaProducerSend;
import jdk.jfr.internal.tool.Main;
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.junit.jupiter.api.Test;
import java.io.*;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.Properties;
@@ -31,73 +35,4 @@ public class ProducerTest {
final Producer<String, String> stringStringProducer = KafkaProducerExample.KafkaProducerTransactional();
KafkaProducerSend.sendMessageTransactional(stringStringProducer, TOPIC_NAME, "test123");
}
@Test
public void imageSend() throws IOException, InterruptedException {
// 创建 Kafka 生产者配置
Properties producerProps = new Properties();
producerProps.put("bootstrap.servers", "localhost:9092");
producerProps.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
producerProps.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
producerProps.put("max.request.size", "2097152");
// 创建 Kafka 生产者和消费者
KafkaProducer<String, String> producer = new KafkaProducer<>(producerProps);
// 发送图片消息到 Kafka
File file = new File("/Users/wyl/Desktop/wyl.png");
InputStream inputStream = new FileInputStream(file);
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
byte[] buffer = new byte[1024];
int bytesRead;
while ((bytesRead = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, bytesRead);
}
byte[] imageData = outputStream.toByteArray();
String s = new String(imageData,StandardCharsets.ISO_8859_1);
ProducerRecord<String, String> record = new ProducerRecord<>("wyl-events", s);
producer.send(record,(m,e)->{
if(Objects.nonNull(e))
{
e.printStackTrace();
}
System.out.println(m);
});
Thread.sleep(100000);
}
public static void main(String[] args) throws IOException {
byte[] byteArray = { -119, 66, 67 };
String str = new String(byteArray,StandardCharsets.ISO_8859_1);
System.out.println(str);
byte[] bytes = str.getBytes(StandardCharsets.ISO_8859_1);
System.out.println(bytes);
}
public static void main1(String[] args) throws IOException {
File file = new File("/Users/wyl/Desktop/wyl.png");
InputStream inputStream = new FileInputStream(file);
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
byte[] buffer = new byte[1024];
int bytesRead;
while ((bytesRead = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, bytesRead);
}
byte[] imageData = outputStream.toByteArray();
String s = new String(imageData);
}
public static void main2(String[] args) {
byte[] byteArray = { -119, 66, 67 };
Charset[] charsets = { StandardCharsets.UTF_8, Charset.forName("ISO-8859-1"), Charset.forName("GBK") };
for (Charset charset : charsets) {
String str = new String(byteArray, charset);
System.out.println(charset.name() + ": " + str);
byte[] newByteArray = str.getBytes(charset);
System.out.println(charset.name() + ": " + Arrays.toString(newByteArray));
}
}
}
@@ -0,0 +1,216 @@
package com.wyl.kafka.test;
import com.google.protobuf.ByteString;
import lombok.Data;
import lombok.SneakyThrows;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.junit.jupiter.api.Test;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.concurrent.atomic.AtomicReference;
public class TlPictureProducerTest {
@SneakyThrows
@Test
public void autofocus() {
/**
* 构建kafka 生产者
*/
KafkaProducer<String, byte[]> kafkaProducer = buildProducer();
String topic = "CCD-PICTURE-NEO-1-wyltest";
AtomicReference<Integer> partition = new AtomicReference<>(0);
LocalDateTime now = LocalDateTime.now();
String format = now.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"));
List<Embryo> embryoList = bulidEmbtyo();
Integer shootingPosition = 10000;
embryoList.stream().forEach(embryo -> {
embryo.embryoIds.entrySet().stream().forEach(id -> {
for (int i = 1; i < 81; i++) {
Integer end = 0;
if (i == 80) {
end = 1;
}
ImageDataDTO.ImageDTO image = makeSendData(null, embryo.getHouseSn(), id.getKey(), 1, format, embryo.getFertilizationTime(), i, shootingPosition * (125 + i), end, id.getValue(), embryo.getId());
partition.set(image.getHouseSn() % 3);
ProducerRecord<String, byte[]> record = new ProducerRecord<>(topic, partition.get(), null, image.toByteArray());
/**
* 发送消息
*/
kafkaProducer.send(record, (m, e) -> {
if (!Objects.isNull(e)) {
e.printStackTrace();
} else {
System.out.println(m.timestamp());
}
});
}
});
});
Thread.sleep(100000);
}
@SneakyThrows
@Test
public void ccd() {
/**
* 构建kafka 生产者
*/
KafkaProducer<String, byte[]> kafkaProducer = buildProducer();
String topic = "CCD-PICTURE-NEO-1-wyltest";
AtomicReference<Integer> partition = new AtomicReference<>(0);
LocalDateTime now = LocalDateTime.now();
String format = now.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"));
List<Embryo> embryoList = bulidEmbtyo();
Integer shootingPosition = 40000;
embryoList.stream().forEach(embryo -> {
embryo.embryoIds.entrySet().stream().forEach(id -> {
for (int i = 1; i < 22; i++) {
Integer end = 0;
if (i == 21) {
end = 1;
}
int a = (i + 1) / 2;
int c = 0;
if (a == 11) {
c = 1;
}
ImageDataDTO.ImageDTO image = makeSendData(c, embryo.getHouseSn(), id.getKey(), 0, format,
embryo.getFertilizationTime(), i, shootingPosition * (125 + i),
end, id.getValue(), embryo.getId());
partition.set(image.getHouseSn() % 3);
ProducerRecord<String, byte[]> record = new ProducerRecord<>(topic, partition.get(), null, image.toByteArray());
/**
* 发送消息
*/
kafkaProducer.send(record, (m, e) -> {
if (!Objects.isNull(e)) {
e.printStackTrace();
} else {
System.out.println(m.timestamp());
}
});
}
});
});
Thread.sleep(100000);
}
/**
* 构建kafka生产者
*
* @param
* @return org.apache.kafka.clients.producer.KafkaProducer<java.lang.String, byte [ ]>
* @Date 2023/7/18
* @Author wangyl
*/
private static KafkaProducer<String, byte[]> buildProducer() {
Properties producerProps = new Properties();
producerProps.put("bootstrap.servers", "192.168.31.89:9092");
producerProps.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
producerProps.put("value.serializer", "org.apache.kafka.common.serialization.ByteArraySerializer");
producerProps.put("max.request.size", "2097152");
KafkaProducer<String, byte[]> producer = new KafkaProducer<>(producerProps);
return producer;
}
/**
* 获取图片数据
*
* @param
* @return byte[]
* @Date 2023/7/18
* @Author wangyl
*/
public static byte[] getImageData() throws IOException {
File file = new File("C:\\Users\\95981\\Desktop\\wangyongliang.jpg");
InputStream inputStream = new FileInputStream(file);
byte[] bytes = inputStream.readAllBytes();
return bytes;
}
/**
* 构建图片传输数据
*
* @param imageTime
* @param pictureLayer
* @param shootingPosition
* @param end
* @return com.wyl.kafka.test.ImageDataDTO.ImageDTO
* @Date 2023/7/18
* @Author wangyl
*/
@SneakyThrows
public static ImageDataDTO.ImageDTO makeSendData(Integer c, Integer houseSn, Integer wellSn, Integer photographType, String imageTime, String fertilizationTime, Integer pictureLayer, Integer shootingPosition, Integer end, long embryoCultureRecordId, long embryoId) {
ImageDataDTO.ImageDTO.Builder builder = ImageDataDTO.ImageDTO.newBuilder()
.setTlSn("NEO-1-wyltest")
.setHouseSn(houseSn)
.setWellSn(wellSn)
.setCcdSn("dfbb63a9-20ee-4b9b-9f0d-ae8205f4fa1f")
.setSourceImageName(UUID.randomUUID().toString())
.setSourceImagePath("TLData/Embryos/1_558/13_6578/1/")
.setSourceImageWidth(753)
.setSourceImageHeight(895)
.setImageTime(imageTime)
.setFertilizationTime(fertilizationTime)
.setPhotographType(photographType)
.setTotalLayer(40)
.setPictureLayer(pictureLayer)
.setShootingPosition(shootingPosition)
.setEnd(end)
.setEmbryoCultureRecordId(embryoCultureRecordId)
.setEmbryoId(embryoId)
.setImageData(ByteString.copyFrom(getImageData()))
.setHorizontalPosition(0);
if (Objects.nonNull(c)) {
builder.setClearest(c);
}
return builder.build();
}
public static List<Embryo> bulidEmbtyo() {
List<Embryo> embryoList = new ArrayList<>();
Embryo embryo = new Embryo();
embryo.setId(10L);
embryo.setHouseSn(1);
embryo.setFertilizationTime("2023-07-13 11:06:23");
Map<Integer, Long> embryoIds = new HashMap<>();
embryoIds.put(1, 33L);
embryoIds.put(2, 34L);
embryoIds.put(3, 35L);
embryo.setEmbryoIds(embryoIds);
embryoList.add(embryo);
Embryo embryo2 = new Embryo();
embryo2.setId(11L);
embryo2.setHouseSn(2);
embryo2.setFertilizationTime("2023-07-13 11:06:23");
Map<Integer, Long> embryoIds2 = new HashMap<>();
embryoIds2.put(2, 36L);
embryoIds2.put(3, 37L);
embryoIds2.put(4, 38L);
embryo2.setEmbryoIds(embryoIds2);
embryoList.add(embryo2);
return embryoList;
}
@Data
public static class Embryo {
private Long id;
private Integer houseSn;
private Map<Integer, Long> embryoIds;
private String fertilizationTime;
}
}