wyl/canal
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

修改 kafka 异步发送 flat message

Browse Source
This commit is contained in:
mcy
2018-12-27 17:52:33 +08:00
parent d9e259dde4
commit 150a13824b
1 changed files with 27 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java
+27 -38
View File
@@ -124,6 +124,7 @@ public class CanalKafkaProducer implements CanalMQProducer {
}
if (record != null) {
// 同步发送原生message
producer.send(record).get();
if (logger.isDebugEnabled()) {
@@ -141,13 +142,17 @@ public class CanalKafkaProducer implements CanalMQProducer {
List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(message);
if (flatMessages != null) {
for (FlatMessage flatMessage : flatMessages) {
if (canalDestination.getPartition() != null) {
if (StringUtils.isEmpty(canalDestination.getPartitionHash())) {
try {
Integer partition = canalDestination.getPartition();
if (partition == null) {
partition = 0;
}
ProducerRecord<String, String> record = new ProducerRecord<String, String>(topicName,
canalDestination.getPartition(),
partition,
null,
JSON.toJSONString(flatMessage, SerializerFeature.WriteMapNullValue));
producer2.send(record).get();
producer2.send(record);
} catch (Exception e) {
logger.error(e.getMessage(), e);
// producer.abortTransaction();
@@ -155,43 +160,27 @@ public class CanalKafkaProducer implements CanalMQProducer {
return;
}
} else {
if (canalDestination.getPartitionHash() != null
&& !canalDestination.getPartitionHash().isEmpty()) {
FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage,
canalDestination.getPartitionsNum(),
canalDestination.getPartitionHash());
int length = partitionFlatMessage.length;
for (int i = 0; i < length; i++) {
FlatMessage flatMessagePart = partitionFlatMessage[i];
if (flatMessagePart != null) {
try {
ProducerRecord<String, String> record = new ProducerRecord<String, String>(
topicName,
i,
null,
JSON.toJSONString(flatMessagePart, SerializerFeature.WriteMapNullValue));
producer2.send(record).get();
} catch (Exception e) {
logger.error(e.getMessage(), e);
// producer.abortTransaction();
callback.rollback();
return;
}
FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage,
canalDestination.getPartitionsNum(),
canalDestination.getPartitionHash());
int length = partitionFlatMessage.length;
for (int i = 0; i < length; i++) {
FlatMessage flatMessagePart = partitionFlatMessage[i];
if (flatMessagePart != null) {
try {
ProducerRecord<String, String> record = new ProducerRecord<String, String>(
topicName,
i,
null,
JSON.toJSONString(flatMessagePart, SerializerFeature.WriteMapNullValue));
producer2.send(record);
} catch (Exception e) {
logger.error(e.getMessage(), e);
// producer.abortTransaction();
callback.rollback();
return;
}
}
} else {
try {
ProducerRecord<String, String> record = new ProducerRecord<String, String>(topicName,
0,
null,
JSON.toJSONString(flatMessage, SerializerFeature.WriteMapNullValue));
producer2.send(record).get();
} catch (Exception e) {
logger.error(e.getMessage(), e);
// producer.abortTransaction();
callback.rollback();
return;
}
}
}
if (logger.isDebugEnabled()) {