wyl/canal
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

模块化mq connector (#2562)

Browse Source 
* add connector module, add kafka producer connector module for server

* add rocketMQ producer connector module for server

* add rabbitMQ producer connector module for server

* optimize code, add annotation

* add canal message consumer connector for canal-adapter module
This commit is contained in:
rewerma
2020-02-28 11:00:08 +08:00
committed by GitHub
parent 4fc7e5f7d9
commit 265a8e8594
82 changed files with 4932 additions and 2412 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client-adapter/common/pom.xml
+8 -2
View File
@@ -1,5 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>canal.client-adapter</artifactId>
<groupId>com.alibaba.otter</groupId>
@@ -17,6 +18,11 @@
<artifactId>canal.protocol</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-core</artifactId>
@@ -49,7 +55,7 @@
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>druid</artifactId>
</dependency>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java
+12 -1
View File
@@ -3,6 +3,7 @@ package com.alibaba.otter.canal.client.adapter.support;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
/**
* 配置信息类
@@ -27,7 +28,7 @@ public class CanalClientConfig {
// 重试次数
private Integer retries;
// 消费超时时间
private Long timeout;
private Long timeout = 500L;
// 模式 tcp kafka rocketMQ
private String mode = "tcp";
// aliyun ak/sk
@@ -39,6 +40,8 @@ public class CanalClientConfig {
// rabbitmq vhost
private String vhost = "/";
private Properties consumerProperties;
private Long resourceOwnerId;
// 是否启用消息轨迹
private boolean enableMessageTrace;
@@ -163,6 +166,14 @@ public class CanalClientConfig {
this.vhost = vhost;
}
public Properties getConsumerProperties() {
return consumerProperties;
}
public void setConsumerProperties(Properties consumerProperties) {
this.consumerProperties = consumerProperties;
}
public Long getResourceOwnerId() {
return resourceOwnerId;
}
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java
+22 -18
View File
@@ -2,6 +2,7 @@ package com.alibaba.otter.canal.client.adapter.support;
import java.util.*;
import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message;
@@ -131,10 +132,13 @@ public class MessageUtil {
return dmls;
}
public static List<Dml> flatMessage2Dml(String destination, String groupId, List<FlatMessage> flatMessages) {
List<Dml> dmls = new ArrayList<Dml>(flatMessages.size());
for (FlatMessage flatMessage : flatMessages) {
Dml dml = flatMessage2Dml(destination, groupId, flatMessage);
public static List<Dml> flatMessage2Dml(String destination, String groupId, List<CommonMessage> commonMessages) {
if (commonMessages == null) {
return new ArrayList<>();
}
List<Dml> dmls = new ArrayList<Dml>(commonMessages.size());
for (CommonMessage commonMessage : commonMessages) {
Dml dml = flatMessage2Dml(destination, groupId, commonMessage);
if (dml != null) {
dmls.add(dml);
}
@@ -143,31 +147,31 @@ public class MessageUtil {
return dmls;
}
public static Dml flatMessage2Dml(String destination, String groupId, FlatMessage flatMessage) {
if (flatMessage == null) {
public static Dml flatMessage2Dml(String destination, String groupId, CommonMessage commonMessage) {
if (commonMessage == null) {
return null;
}
Dml dml = new Dml();
dml.setDestination(destination);
dml.setGroupId(groupId);
dml.setDatabase(flatMessage.getDatabase());
dml.setTable(flatMessage.getTable());
dml.setPkNames(flatMessage.getPkNames());
dml.setIsDdl(flatMessage.getIsDdl());
dml.setType(flatMessage.getType());
dml.setTs(flatMessage.getTs());
dml.setEs(flatMessage.getEs());
dml.setSql(flatMessage.getSql());
dml.setDatabase(commonMessage.getDatabase());
dml.setTable(commonMessage.getTable());
dml.setPkNames(commonMessage.getPkNames());
dml.setIsDdl(commonMessage.getIsDdl());
dml.setType(commonMessage.getType());
dml.setTs(commonMessage.getTs());
dml.setEs(commonMessage.getEs());
dml.setSql(commonMessage.getSql());
// if (flatMessage.getSqlType() == null || flatMessage.getMysqlType() == null) {
// throw new RuntimeException("SqlType or mysqlType is null");
// }
List<Map<String, String>> data = flatMessage.getData();
List<Map<String, Object>> data = commonMessage.getData();
if (data != null) {
dml.setData(changeRows(dml.getTable(), data, flatMessage.getSqlType(), flatMessage.getMysqlType()));
dml.setData(data);
}
List<Map<String, String>> old = flatMessage.getOld();
List<Map<String, Object>> old = commonMessage.getOld();
if (old != null) {
dml.setOld(changeRows(dml.getTable(), old, flatMessage.getSqlType(), flatMessage.getMysqlType()));
dml.setOld(old);
}
return dml;
}
client-adapter/launcher/pom.xml
+53 -24
View File
@@ -30,11 +30,6 @@
<artifactId>client-adapter.common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.client</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
@@ -56,25 +51,6 @@
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
</dependency>
<!-- 单独引入rocketmq依赖 -->
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-client</artifactId>
</dependency>
<!-- 单独引入kafka依赖 -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
</dependency>
<dependency>
<groupId>com.rabbitmq</groupId>
<artifactId>amqp-client</artifactId>
</dependency>
<dependency>
<groupId>com.alibaba.mq-amqp</groupId>
<artifactId>mq-amqp-client</artifactId>
</dependency>
<!-- jdbc -->
<dependency>
<groupId>mysql</groupId>
@@ -159,6 +135,59 @@
<classifier>jar-with-dependencies</classifier>
<scope>provided</scope>
</dependency>
<!-- connector plugin -->
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.tcp</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
<artifactId>*</artifactId>
<groupId>*</groupId>
</exclusion>
</exclusions>
<classifier>jar-with-dependencies</classifier>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.kafka</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
<artifactId>*</artifactId>
<groupId>*</groupId>
</exclusion>
</exclusions>
<classifier>jar-with-dependencies</classifier>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.rocketmq</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
<artifactId>*</artifactId>
<groupId>*</groupId>
</exclusion>
</exclusions>
<classifier>jar-with-dependencies</classifier>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.rabbitmq</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
<artifactId>*</artifactId>
<groupId>*</groupId>
</exclusion>
</exclusions>
<classifier>jar-with-dependencies</classifier>
<scope>provided</scope>
</dependency>
</dependencies>
<build>
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java
-263
View File
@@ -1,263 +0,0 @@
package com.alibaba.otter.canal.adapter.launcher.loader;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch;
import com.alibaba.otter.canal.adapter.launcher.config.SpringContext;
import com.alibaba.otter.canal.client.CanalMQConnector;
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
import com.alibaba.otter.canal.client.adapter.support.Dml;
import com.alibaba.otter.canal.client.adapter.support.MessageUtil;
import com.alibaba.otter.canal.client.adapter.support.Util;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message;
/**
* 适配器工作线程抽象类
*
* @author rewerma 2018-8-19 下午11:30:49
* @version 1.0.0
*/
public abstract class AbstractCanalAdapterWorker {
protected final Logger logger = LoggerFactory.getLogger(this.getClass());
protected String canalDestination; // canal实例
protected String groupId = null; // groupId
protected List<List<OuterAdapter>> canalOuterAdapters; // 外部适配器
protected CanalClientConfig canalClientConfig; // 配置
protected ExecutorService groupInnerExecutorService; // 组内工作线程池
protected volatile boolean running = false; // 是否运行中
protected Thread thread = null;
protected Thread.UncaughtExceptionHandler handler = (t, e) -> logger.error("parse events has an error", e);
protected SyncSwitch syncSwitch;
public AbstractCanalAdapterWorker(List<List<OuterAdapter>> canalOuterAdapters){
this.canalOuterAdapters = canalOuterAdapters;
this.groupInnerExecutorService = Util.newFixedThreadPool(canalOuterAdapters.size(), 5000L);
syncSwitch = (SyncSwitch) SpringContext.getBean(SyncSwitch.class);
}
protected void writeOut(final Message message) {
List<Future<Boolean>> futures = new ArrayList<>();
// 组间适配器并行运行
canalOuterAdapters.forEach(outerAdapters -> {
final List<OuterAdapter> adapters = outerAdapters;
futures.add(groupInnerExecutorService.submit(() -> {
try {
// 组内适配器穿行运行，尽量不要配置组内适配器
adapters.forEach(adapter -> {
long begin = System.currentTimeMillis();
List<Dml> dmls = MessageUtil.parse4Dml(canalDestination, groupId, message);
if (dmls != null) {
batchSync(dmls, adapter);
if (logger.isDebugEnabled()) {
logger.debug("{} elapsed time: {}",
adapter.getClass().getName(),
(System.currentTimeMillis() - begin));
}
}
});
return true;
} catch (Exception e) {
logger.error(e.getMessage(), e);
return false;
}
}));
// 等待所有适配器写入完成
// 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成
RuntimeException exception = null;
for (Future<Boolean> future : futures) {
try {
if (!future.get()) {
exception = new RuntimeException("Outer adapter sync failed! ");
}
} catch (Exception e) {
exception = new RuntimeException(e);
}
}
if (exception != null) {
throw exception;
}
});
}
protected void writeOut(final List<FlatMessage> flatMessages) {
List<Future<Boolean>> futures = new ArrayList<>();
// 组间适配器并行运行
canalOuterAdapters.forEach(outerAdapters -> {
futures.add(groupInnerExecutorService.submit(() -> {
try {
// 组内适配器穿行运行，尽量不要配置组内适配器
outerAdapters.forEach(adapter -> {
long begin = System.currentTimeMillis();
List<Dml> dmls = MessageUtil.flatMessage2Dml(canalDestination, groupId, flatMessages);
batchSync(dmls, adapter);
if (logger.isDebugEnabled()) {
logger.debug("{} elapsed time: {}",
adapter.getClass().getName(),
(System.currentTimeMillis() - begin));
}
});
return true;
} catch (Exception e) {
logger.error(e.getMessage(), e);
return false;
}
}));
// 等待所有适配器写入完成
// 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成
RuntimeException exception = null;
for (Future<Boolean> future : futures) {
try {
if (!future.get()) {
exception = new RuntimeException("Outer adapter sync failed! ");
}
} catch (Exception e) {
exception = new RuntimeException(e);
}
}
if (exception != null) {
throw exception;
}
});
}
@SuppressWarnings("unchecked")
protected boolean mqWriteOutData(int retry, long timeout, int i, final boolean flatMessage,
CanalMQConnector connector) {
ExecutorService workerExecutor = Util.newSingleDaemonThreadExecutor(5000);
try {
List<?> messages;
if (!flatMessage) {
messages = connector.getListWithoutAck(100L, TimeUnit.MILLISECONDS);
} else {
messages = connector.getFlatListWithoutAck(100L, TimeUnit.MILLISECONDS);
}
if (messages != null && !messages.isEmpty()) {
Future<Boolean> future = workerExecutor.submit(() -> {
if (flatMessage) {
// batch write
writeOut((List<FlatMessage>) messages);
// FIXME xxx
// messages.forEach((message ->
// System.out.println(JSON.toJSONString(message))));
} else {
for (final Object message : messages) {
writeOut((Message) message);
}
}
return true;
});
try {
future.get(timeout, TimeUnit.MILLISECONDS);
} catch (Exception e) {
future.cancel(true);
throw e;
}
connector.ack();
}
return true;
} catch (Throwable e) {
if (i == retry - 1) {
connector.ack();
logger.error(e.getMessage() + " Error sync but ACK!");
return true;
} else {
connector.rollback();
logger.error(e.getMessage() + " Error sync and rollback, execute times: " + (i + 1));
}
try {
Thread.sleep(500);
} catch (InterruptedException e1) {
// ignore
}
} finally {
workerExecutor.shutdown();
}
return false;
}
/**
* 分批同步
*
* @param dmls
* @param adapter
*/
private void batchSync(List<Dml> dmls, OuterAdapter adapter) {
// 分批同步
if (dmls.size() <= canalClientConfig.getSyncBatchSize()) {
adapter.sync(dmls);
} else {
int len = 0;
List<Dml> dmlsBatch = new ArrayList<>();
for (Dml dml : dmls) {
dmlsBatch.add(dml);
if (dml.getData() == null || dml.getData().isEmpty()) {
len += 1;
} else {
len += dml.getData().size();
}
if (len >= canalClientConfig.getSyncBatchSize()) {
adapter.sync(dmlsBatch);
dmlsBatch.clear();
len = 0;
}
}
if (!dmlsBatch.isEmpty()) {
adapter.sync(dmlsBatch);
}
}
}
public void start() {
if (!running) {
thread = new Thread(this::process);
thread.setUncaughtExceptionHandler(handler);
thread.start();
running = true;
}
}
protected abstract void process();
public void stop() {
try {
if (!running) {
return;
}
running = false;
syncSwitch.release(canalDestination);
logger.info("destination {} is waiting for adapters' worker thread die!", canalDestination);
if (thread != null) {
try {
thread.join();
} catch (InterruptedException e) {
// ignore
}
}
groupInnerExecutorService.shutdown();
logger.info("destination {} adapters worker thread dead!", canalDestination);
canalOuterAdapters.forEach(outerAdapters -> outerAdapters.forEach(OuterAdapter::destroy));
logger.info("destination {} all adapters destroyed!", canalDestination);
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
}
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AdapterProcessor.java
+268
View File
@@ -0,0 +1,268 @@
package com.alibaba.otter.canal.adapter.launcher.loader;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import com.alibaba.otter.canal.adapter.launcher.config.SpringContext;
import com.alibaba.otter.canal.client.adapter.support.Util;
import com.alibaba.otter.canal.connector.core.config.CanalConstants;
import com.alibaba.otter.canal.connector.core.spi.ExtensionLoader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch;
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
import com.alibaba.otter.canal.client.adapter.support.Dml;
import com.alibaba.otter.canal.client.adapter.support.MessageUtil;
import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
/**
* 适配处理器
*
* @author rewerma 2020-02-01
* @version 1.0.0
*/
public class AdapterProcessor {
private static final Logger logger = LoggerFactory.getLogger(AdapterProcessor.class);
private static final String CONNECTOR_SPI_DIR = "/plugin";
private static final String CONNECTOR_STANDBY_SPI_DIR = "/canal-adapter/plugin";
private CanalMsgConsumer canalMsgConsumer;
private String canalDestination; // canal实例
private String groupId = null; // groupId
private List<List<OuterAdapter>> canalOuterAdapters; // 外部适配器
private CanalClientConfig canalClientConfig; // 配置
private ExecutorService groupInnerExecutorService; // 组内工作线程池
private volatile boolean running = false; // 是否运行中
private Thread thread = null;
private Thread.UncaughtExceptionHandler handler = (t, e) -> logger
.error("parse events has an error", e);
private SyncSwitch syncSwitch;
public AdapterProcessor(CanalClientConfig canalClientConfig, String destination, String groupId,
List<List<OuterAdapter>> canalOuterAdapters){
this.canalClientConfig = canalClientConfig;
this.canalDestination = destination;
this.groupId = groupId;
this.canalOuterAdapters = canalOuterAdapters;
this.groupInnerExecutorService = Util.newFixedThreadPool(canalOuterAdapters.size(), 5000L);
syncSwitch = (SyncSwitch) SpringContext.getBean(SyncSwitch.class);
// load connector consumer
ExtensionLoader<CanalMsgConsumer> loader = new ExtensionLoader<>(CanalMsgConsumer.class);
canalMsgConsumer = loader
.getExtension(canalClientConfig.getMode().toLowerCase(), CONNECTOR_SPI_DIR, CONNECTOR_STANDBY_SPI_DIR);
Properties properties = canalClientConfig.getConsumerProperties();
properties.put(CanalConstants.CANAL_MQ_FLAT_MESSAGE, canalClientConfig.getFlatMessage());
properties.put(CanalConstants.CANAL_ALIYUN_ACCESS_KEY, canalClientConfig.getAccessKey());
properties.put(CanalConstants.CANAL_ALIYUN_SECRET_KEY, canalClientConfig.getSecretKey());
ClassLoader cl = Thread.currentThread().getContextClassLoader();
Thread.currentThread().setContextClassLoader(canalMsgConsumer.getClass().getClassLoader());
canalMsgConsumer.init(properties, canalDestination, groupId);
Thread.currentThread().setContextClassLoader(cl);
}
public void start() {
if (!running) {
thread = new Thread(this::process);
thread.setUncaughtExceptionHandler(handler);
thread.start();
running = true;
}
}
public void writeOut(final List<CommonMessage> commonMessages) {
List<Future<Boolean>> futures = new ArrayList<>();
// 组间适配器并行运行
canalOuterAdapters.forEach(outerAdapters -> {
futures.add(groupInnerExecutorService.submit(() -> {
try {
// 组内适配器穿行运行，尽量不要配置组内适配器
outerAdapters.forEach(adapter -> {
long begin = System.currentTimeMillis();
List<Dml> dmls = MessageUtil.flatMessage2Dml(canalDestination, groupId, commonMessages);
batchSync(dmls, adapter);
if (logger.isDebugEnabled()) {
logger.debug("{} elapsed time: {}",
adapter.getClass().getName(),
(System.currentTimeMillis() - begin));
}
});
return true;
} catch (Exception e) {
logger.error(e.getMessage(), e);
return false;
}
}));
// 等待所有适配器写入完成
// 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成
RuntimeException exception = null;
for (Future<Boolean> future : futures) {
try {
if (!future.get()) {
exception = new RuntimeException("Outer adapter sync failed! ");
}
} catch (Exception e) {
exception = new RuntimeException(e);
}
}
if (exception != null) {
throw exception;
}
});
}
/**
* 分批同步
*
* @param dmls
* @param adapter
*/
private void batchSync(List<Dml> dmls, OuterAdapter adapter) {
// 分批同步
if (dmls.size() <= canalClientConfig.getSyncBatchSize()) {
adapter.sync(dmls);
} else {
int len = 0;
List<Dml> dmlsBatch = new ArrayList<>();
for (Dml dml : dmls) {
dmlsBatch.add(dml);
if (dml.getData() == null || dml.getData().isEmpty()) {
len += 1;
} else {
len += dml.getData().size();
}
if (len >= canalClientConfig.getSyncBatchSize()) {
adapter.sync(dmlsBatch);
dmlsBatch.clear();
len = 0;
}
}
if (!dmlsBatch.isEmpty()) {
adapter.sync(dmlsBatch);
}
}
}
private void process() {
while (!running) { // waiting until running == true
while (!running) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
}
}
int retry = canalClientConfig.getRetries() == null
|| canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
if (retry == -1) {
// 重试次数-1代表异常时一直阻塞重试
retry = Integer.MAX_VALUE;
}
while (running) {
try {
syncSwitch.get(canalDestination);
logger.info("=============> Start to connect destination: {} <=============", this.canalDestination);
canalMsgConsumer.connect();
logger.info("=============> Subscribe destination: {} succeed <=============", this.canalDestination);
while (running) {
try {
syncSwitch.get(canalDestination, 1L, TimeUnit.MINUTES);
} catch (TimeoutException e) {
break;
}
if (!running) {
break;
}
for (int i = 0; i < retry; i++) {
if (!running) {
break;
}
try {
if (logger.isDebugEnabled()) {
logger.debug("destination: {} ", canalDestination);
}
long begin = System.currentTimeMillis();
List<CommonMessage> commonMessages = canalMsgConsumer
.getMessage(this.canalClientConfig.getTimeout(), TimeUnit.MILLISECONDS);
writeOut(commonMessages);
canalMsgConsumer.ack();
if (logger.isDebugEnabled()) {
logger.debug("destination: {} elapsed time: {} ms",
canalDestination,
System.currentTimeMillis() - begin);
}
} catch (Exception e) {
if (i != retry - 1) {
canalMsgConsumer.rollback(); // 处理失败, 回滚数据
logger.error(e.getMessage() + " Error sync and rollback, execute times: " + (i + 1));
} else {
canalMsgConsumer.ack();
logger.error(e.getMessage() + " Error sync but ACK!");
}
Thread.sleep(500);
}
}
}
canalMsgConsumer.disconnect();
} catch (Throwable e) {
logger.error("process error!", e);
}
if (running) { // is reconnect
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
// ignore
}
}
}
}
public void stop() {
try {
if (!running) {
return;
}
running = false;
syncSwitch.release(canalDestination);
logger.info("destination {} is waiting for adapters' worker thread die!", canalDestination);
if (thread != null) {
try {
thread.join();
} catch (InterruptedException e) {
// ignore
}
}
groupInnerExecutorService.shutdown();
logger.info("destination {} adapters worker thread dead!", canalDestination);
canalOuterAdapters.forEach(outerAdapters -> outerAdapters.forEach(OuterAdapter::destroy));
logger.info("destination {} all adapters destroyed!", canalDestination);
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
}
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java
-96
View File
@@ -1,96 +0,0 @@
package com.alibaba.otter.canal.adapter.launcher.loader;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import org.apache.kafka.common.errors.WakeupException;
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
import com.alibaba.otter.canal.client.adapter.support.Util;
import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector;
/**
* kafka对应的client适配器工作线程
*
* @author rewerma 2018-8-19 下午11:30:49
* @version 1.0.0
*/
public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
private KafkaCanalConnector connector;
private String topic;
private boolean flatMessage;
public CanalAdapterKafkaWorker(CanalClientConfig canalClientConfig, String bootstrapServers, String topic,
String groupId, List<List<OuterAdapter>> canalOuterAdapters, boolean flatMessage){
super(canalOuterAdapters);
this.canalClientConfig = canalClientConfig;
this.topic = topic;
super.canalDestination = topic;
super.groupId = groupId;
this.flatMessage = flatMessage;
this.connector = new KafkaCanalConnector(bootstrapServers,
topic,
null,
groupId,
canalClientConfig.getBatchSize(),
flatMessage);
connector.setSessionTimeout(30L, TimeUnit.SECONDS);
}
@Override
protected void process() {
while (!running) {
try {
Thread.sleep(500);
} catch (InterruptedException e) {
// ignore
}
}
int retry = canalClientConfig.getRetries() == null
|| canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒
while (running) {
try {
syncSwitch.get(canalDestination);
logger.info("=============> Start to connect topic: {} <=============", this.topic);
connector.connect();
logger.info("=============> Start to subscribe topic: {} <=============", this.topic);
connector.subscribe();
logger.info("=============> Subscribe topic: {} succeed <=============", this.topic);
while (running) {
boolean status = syncSwitch.status(canalDestination);
if (!status) {
connector.disconnect();
break;
}
if (retry == -1) {
retry = Integer.MAX_VALUE;
}
for (int i = 0; i < retry; i++) {
if (!running) {
break;
}
if (mqWriteOutData(retry, timeout, i, flatMessage, connector)) {
break;
}
}
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
try {
connector.unsubscribe();
} catch (WakeupException e) {
// No-op. Continue process
}
connector.disconnect();
logger.info("=============> Disconnect topic: {} <=============", this.topic);
}
}
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java
+125 -130
View File
@@ -37,9 +37,7 @@ public class CanalAdapterLoader {
private CanalClientConfig canalClientConfig;
private Map<String, CanalAdapterWorker> canalWorkers = new HashMap<>();
private Map<String, AbstractCanalAdapterWorker> canalMQWorker = new HashMap<>();
private Map<String, AdapterProcessor> canalAdapterProcessors = new HashMap<>();
private ExtensionLoader<OuterAdapter> loader;
@@ -53,120 +51,132 @@ public class CanalAdapterLoader {
public void init() {
loader = ExtensionLoader.getExtensionLoader(OuterAdapter.class);
String canalServerHost = this.canalClientConfig.getCanalServerHost();
SocketAddress sa = null;
if (canalServerHost != null) {
String[] ipPort = canalServerHost.split(":");
sa = new InetSocketAddress(ipPort[0], Integer.parseInt(ipPort[1]));
}
String zkHosts = this.canalClientConfig.getZookeeperHosts();
if ("tcp".equalsIgnoreCase(canalClientConfig.getMode())) {
// 初始化canal-client的适配器
for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
for (CanalClientConfig.Group connectorGroup : canalAdapter.getGroups()) {
List<OuterAdapter> canalOutConnectors = new CopyOnWriteArrayList<>();
for (OuterAdapterConfig c : connectorGroup.getOuterAdapters()) {
loadAdapter(c, canalOutConnectors);
}
canalOuterAdapterGroups.add(canalOutConnectors);
List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
for (OuterAdapterConfig config : group.getOuterAdapters()) {
loadAdapter(config, canalOuterAdapters);
}
CanalAdapterWorker worker;
if (sa != null) {
worker = new CanalAdapterWorker(canalClientConfig,
canalAdapter.getInstance(),
sa,
canalOuterAdapterGroups);
} else if (zkHosts != null) {
worker = new CanalAdapterWorker(canalClientConfig,
canalAdapter.getInstance(),
zkHosts,
canalOuterAdapterGroups);
} else {
throw new RuntimeException("No canal server connector found");
}
canalWorkers.put(canalAdapter.getInstance(), worker);
worker.start();
logger.info("Start adapter for canal instance: {} succeed", canalAdapter.getInstance());
}
} else if ("kafka".equalsIgnoreCase(canalClientConfig.getMode())) {
// 初始化canal-client-kafka的适配器
for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
for (OuterAdapterConfig config : group.getOuterAdapters()) {
loadAdapter(config, canalOuterAdapters);
}
canalOuterAdapterGroups.add(canalOuterAdapters);
canalOuterAdapterGroups.add(canalOuterAdapters);
CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(canalClientConfig,
canalClientConfig.getMqServers(),
canalAdapter.getInstance(),
group.getGroupId(),
canalOuterAdapterGroups,
canalClientConfig.getFlatMessage());
canalMQWorker.put(canalAdapter.getInstance() + "-kafka-" + group.getGroupId(), canalKafkaWorker);
canalKafkaWorker.start();
logger.info("Start adapter for canal-client mq topic: {} succeed",
AdapterProcessor adapterProcessor = canalAdapterProcessors.computeIfAbsent(canalAdapter.getInstance()+"|"+StringUtils.trimToEmpty(group.getGroupId()),
f-> new AdapterProcessor(canalClientConfig,canalAdapter.getInstance(),
group.getGroupId(), canalOuterAdapterGroups));
adapterProcessor.start();
logger.info("Start adapter for canal-client mq topic: {} succeed",
canalAdapter.getInstance() + "-" + group.getGroupId());
}
}
} else if ("rocketMQ".equalsIgnoreCase(canalClientConfig.getMode())) {
// 初始化canal-client-rocketMQ的适配器
for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
for (OuterAdapterConfig config : group.getOuterAdapters()) {
loadAdapter(config, canalOuterAdapters);
}
canalOuterAdapterGroups.add(canalOuterAdapters);
CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig,
canalClientConfig.getMqServers(),
canalAdapter.getInstance(),
group.getGroupId(),
canalOuterAdapterGroups,
canalClientConfig.getAccessKey(),
canalClientConfig.getSecretKey(),
canalClientConfig.getFlatMessage(),
canalClientConfig.isEnableMessageTrace(),
canalClientConfig.getCustomizedTraceTopic(),
canalClientConfig.getAccessChannel(),
canalClientConfig.getNamespace());
canalMQWorker.put(canalAdapter.getInstance() + "-rocketmq-" + group.getGroupId(), rocketMQWorker);
rocketMQWorker.start();
logger.info("Start adapter for canal-client mq topic: {} succeed",
canalAdapter.getInstance() + "-" + group.getGroupId());
}
}
} else if ("rabbitMQ".equalsIgnoreCase(canalClientConfig.getMode())) {
// 初始化canal-client-rabbitMQ的适配器
for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
for (OuterAdapterConfig config : group.getOuterAdapters()) {
loadAdapter(config, canalOuterAdapters);
}
canalOuterAdapterGroups.add(canalOuterAdapters);
CanalAdapterRabbitMQWorker rabbitMQWork = new CanalAdapterRabbitMQWorker(canalClientConfig,
canalOuterAdapterGroups,
canalAdapter.getInstance(),
group.getGroupId(),
canalClientConfig.getFlatMessage());
canalMQWorker.put(canalAdapter.getInstance() + "-rabbitmq-" + group.getGroupId(), rabbitMQWork);
rabbitMQWork.start();
logger.info("Start adapter for canal-client mq topic: {} succeed",
canalAdapter.getInstance() + "-" + group.getGroupId());
}
}
// CanalAdapterRabbitMQWork
}
// if ("tcp".equalsIgnoreCase(canalClientConfig.getMode())) {
// // 初始化canal-client的适配器
// for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
// List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
//
// for (CanalClientConfig.Group connectorGroup : canalAdapter.getGroups()) {
// List<OuterAdapter> canalOutConnectors = new CopyOnWriteArrayList<>();
// for (OuterAdapterConfig c : connectorGroup.getOuterAdapters()) {
// loadAdapter(c, canalOutConnectors);
// }
// canalOuterAdapterGroups.add(canalOutConnectors);
// }
// CanalAdapterWorker worker;
// if (StringUtils.isNotEmpty(canalServerHost)) {
// worker = new CanalAdapterWorker(canalClientConfig,
// canalAdapter.getInstance(),
// canalServerHost,
// zkHosts,
// canalOuterAdapterGroups);
// } else if (zkHosts != null) {
// worker = new CanalAdapterWorker(canalClientConfig,
// canalAdapter.getInstance(),
// zkHosts,
// canalOuterAdapterGroups);
// } else {
// throw new RuntimeException("No canal server connector found");
// }
// canalWorkers.put(canalAdapter.getInstance(), worker);
// worker.start();
// logger.info("Start adapter for canal instance: {} succeed", canalAdapter.getInstance());
// }
// } else if ("kafka".equalsIgnoreCase(canalClientConfig.getMode())) {
// // 初始化canal-client-kafka的适配器
// for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
// for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
// List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
// List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
// for (OuterAdapterConfig config : group.getOuterAdapters()) {
// loadAdapter(config, canalOuterAdapters);
// }
// canalOuterAdapterGroups.add(canalOuterAdapters);
//
// CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(canalClientConfig,
// canalClientConfig.getMqServers(),
// canalAdapter.getInstance(),
// group.getGroupId(),
// canalOuterAdapterGroups,
// canalClientConfig.getFlatMessage());
// canalMQWorker.put(canalAdapter.getInstance() + "-kafka-" + group.getGroupId(), canalKafkaWorker);
// canalKafkaWorker.start();
// logger.info("Start adapter for canal-client mq topic: {} succeed",
// canalAdapter.getInstance() + "-" + group.getGroupId());
// }
// }
// } else if ("rocketMQ".equalsIgnoreCase(canalClientConfig.getMode())) {
// // 初始化canal-client-rocketMQ的适配器
// for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
// for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
// List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
// List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
// for (OuterAdapterConfig config : group.getOuterAdapters()) {
// loadAdapter(config, canalOuterAdapters);
// }
// canalOuterAdapterGroups.add(canalOuterAdapters);
// CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig,
// canalClientConfig.getMqServers(),
// canalAdapter.getInstance(),
// group.getGroupId(),
// canalOuterAdapterGroups,
// canalClientConfig.getAccessKey(),
// canalClientConfig.getSecretKey(),
// canalClientConfig.getFlatMessage(),
// canalClientConfig.isEnableMessageTrace(),
// canalClientConfig.getCustomizedTraceTopic(),
// canalClientConfig.getAccessChannel(),
// canalClientConfig.getNamespace());
// canalMQWorker.put(canalAdapter.getInstance() + "-rocketmq-" + group.getGroupId(), rocketMQWorker);
// rocketMQWorker.start();
//
// logger.info("Start adapter for canal-client mq topic: {} succeed",
// canalAdapter.getInstance() + "-" + group.getGroupId());
// }
// }
// } else if ("rabbitMQ".equalsIgnoreCase(canalClientConfig.getMode())) {
// // 初始化canal-client-rabbitMQ的适配器
// for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
// for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
// List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
// List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
// for (OuterAdapterConfig config : group.getOuterAdapters()) {
// loadAdapter(config, canalOuterAdapters);
// }
// canalOuterAdapterGroups.add(canalOuterAdapters);
// CanalAdapterRabbitMQWorker rabbitMQWork = new CanalAdapterRabbitMQWorker(canalClientConfig,
// canalOuterAdapterGroups,
// canalAdapter.getInstance(),
// group.getGroupId(),
// canalClientConfig.getFlatMessage());
// canalMQWorker.put(canalAdapter.getInstance() + "-rabbitmq-" + group.getGroupId(), rabbitMQWork);
// rabbitMQWork.start();
//
// logger.info("Start adapter for canal-client mq topic: {} succeed",
// canalAdapter.getInstance() + "-" + group.getGroupId());
// }
// }
// // CanalAdapterRabbitMQWork
// }
}
private void loadAdapter(OuterAdapterConfig config, List<OuterAdapter> canalOutConnectors) {
@@ -206,10 +216,10 @@ public class CanalAdapterLoader {
* 销毁所有适配器 为防止canal实例太多造成销毁阻塞, 并行销毁
*/
public void destroy() {
if (!canalWorkers.isEmpty()) {
ExecutorService stopExecutorService = Executors.newFixedThreadPool(canalWorkers.size());
for (CanalAdapterWorker canalAdapterWorker : canalWorkers.values()) {
stopExecutorService.execute(canalAdapterWorker::stop);
if (!canalAdapterProcessors.isEmpty()) {
ExecutorService stopExecutorService = Executors.newFixedThreadPool(canalAdapterProcessors.size());
for (AdapterProcessor adapterProcessor : canalAdapterProcessors.values()) {
stopExecutorService.execute(adapterProcessor::stop);
}
stopExecutorService.shutdown();
try {
@@ -220,21 +230,6 @@ public class CanalAdapterLoader {
// ignore
}
}
if (!canalMQWorker.isEmpty()) {
ExecutorService stopMQWorkerService = Executors.newFixedThreadPool(canalMQWorker.size());
for (AbstractCanalAdapterWorker canalAdapterMQWorker : canalMQWorker.values()) {
stopMQWorkerService.execute(canalAdapterMQWorker::stop);
}
stopMQWorkerService.shutdown();
try {
while (!stopMQWorkerService.awaitTermination(1, TimeUnit.SECONDS)) {
// ignore
}
} catch (InterruptedException e) {
// ignore
}
}
logger.info("All canal adapters destroyed");
}
}
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRabbitMQWorker.java
-90
View File
@@ -1,90 +0,0 @@
package com.alibaba.otter.canal.adapter.launcher.loader;
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
import com.alibaba.otter.canal.client.adapter.support.Util;
import com.alibaba.otter.canal.client.rabbitmq.RabbitMQCanalConnector;
import org.apache.kafka.common.errors.WakeupException;
import java.util.List;
import java.util.concurrent.ExecutorService;
public class CanalAdapterRabbitMQWorker extends AbstractCanalAdapterWorker {
private RabbitMQCanalConnector connector;
private String topic;
private boolean flatMessage;
public CanalAdapterRabbitMQWorker(CanalClientConfig canalClientConfig, List<List<OuterAdapter>> canalOuterAdapters,
String topic, String groupId, boolean flatMessage){
super(canalOuterAdapters);
this.canalClientConfig = canalClientConfig;
this.topic = topic;
this.flatMessage = flatMessage;
this.canalDestination = topic;
this.groupId = groupId;
this.connector = new RabbitMQCanalConnector(canalClientConfig.getMqServers(),
canalClientConfig.getVhost(),
topic,
canalClientConfig.getAccessKey(),
canalClientConfig.getSecretKey(),
canalClientConfig.getUsername(),
canalClientConfig.getPassword(),
canalClientConfig.getResourceOwnerId(),
flatMessage);
}
@Override
protected void process() {
while (!running) {
try {
Thread.sleep(1000);
} catch (InterruptedException ignored) {
}
}
int retry = canalClientConfig.getRetries() == null
|| canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒
while (running) {
try {
syncSwitch.get(canalDestination);
logger.info("=============> Start to connect topic: {} <=============", this.topic);
connector.connect();
logger.info("=============> Start to subscribe topic: {}<=============", this.topic);
connector.subscribe();
logger.info("=============> Subscribe topic: {} succeed<=============", this.topic);
while (running) {
boolean status = syncSwitch.status(canalDestination);
if (!status) {
connector.disconnect();
break;
}
if (retry == -1) {
retry = Integer.MAX_VALUE;
}
for (int i = 0; i < retry; i++) {
if (!running) {
break;
}
if (mqWriteOutData(retry, timeout, i, flatMessage, connector)) {
break;
}
}
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
try {
connector.unsubscribe();
} catch (WakeupException e) {
// No-op. Continue process
}
connector.disconnect();
logger.info("=============> Disconnect topic: {} <=============", this.topic);
}
}
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java
-119
View File
@@ -1,119 +0,0 @@
package com.alibaba.otter.canal.adapter.launcher.loader;
import java.util.List;
import java.util.concurrent.ExecutorService;
import org.apache.kafka.common.errors.WakeupException;
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
import com.alibaba.otter.canal.client.adapter.support.Util;
import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnector;
/**
* rocketmq对应的client适配器工作线程
*
* @version 1.0.0
*/
public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker {
private RocketMQCanalConnector connector;
private String topic;
private boolean flatMessage;
public CanalAdapterRocketMQWorker(CanalClientConfig canalClientConfig, String nameServers, String topic,
String groupId, List<List<OuterAdapter>> canalOuterAdapters, String accessKey,
String secretKey, boolean flatMessage){
super(canalOuterAdapters);
this.canalClientConfig = canalClientConfig;
this.topic = topic;
this.flatMessage = flatMessage;
super.canalDestination = topic;
super.groupId = groupId;
this.connector = new RocketMQCanalConnector(nameServers,
topic,
groupId,
accessKey,
secretKey,
canalClientConfig.getBatchSize(),
flatMessage);
logger.info("RocketMQ consumer config topic:{}, nameServer:{}, groupId:{}", topic, nameServers, groupId);
}
public CanalAdapterRocketMQWorker(CanalClientConfig canalClientConfig, String nameServers, String topic,
String groupId, List<List<OuterAdapter>> canalOuterAdapters, String accessKey,
String secretKey, boolean flatMessage, boolean enableMessageTrace,
String customizedTraceTopic, String accessChannel, String namespace) {
super(canalOuterAdapters);
this.canalClientConfig = canalClientConfig;
this.topic = topic;
this.flatMessage = flatMessage;
super.canalDestination = topic;
super.groupId = groupId;
this.connector = new RocketMQCanalConnector(nameServers,
topic,
groupId,
accessKey,
secretKey,
canalClientConfig.getBatchSize(),
flatMessage,
enableMessageTrace,
customizedTraceTopic,
accessChannel,
namespace);
logger.info("RocketMQ consumer config topic:{}, nameServer:{}, groupId:{}", topic, nameServers, groupId);
}
@Override
protected void process() {
while (!running) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
}
int retry = canalClientConfig.getRetries() == null
|| canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒
while (running) {
try {
syncSwitch.get(canalDestination);
logger.info("=============> Start to connect topic: {} <=============", this.topic);
connector.connect();
logger.info("=============> Start to subscribe topic: {}<=============", this.topic);
connector.subscribe();
logger.info("=============> Subscribe topic: {} succeed<=============", this.topic);
while (running) {
boolean status = syncSwitch.status(canalDestination);
if (!status) {
connector.disconnect();
break;
}
if (retry == -1) {
retry = Integer.MAX_VALUE;
}
for (int i = 0; i < retry; i++) {
if (!running) {
break;
}
if (mqWriteOutData(retry, timeout, i, flatMessage, connector)) {
break;
}
}
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
try {
connector.unsubscribe();
} catch (WakeupException e) {
// No-op. Continue process
}
connector.disconnect();
logger.info("=============> Disconnect topic: {} <=============", this.topic);
}
}
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java
-193
View File
@@ -1,193 +0,0 @@
package com.alibaba.otter.canal.adapter.launcher.loader;
import java.net.SocketAddress;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import com.alibaba.otter.canal.client.CanalConnector;
import com.alibaba.otter.canal.client.CanalConnectors;
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
import com.alibaba.otter.canal.client.impl.ClusterCanalConnector;
import com.alibaba.otter.canal.client.impl.SimpleCanalConnector;
import com.alibaba.otter.canal.protocol.Message;
/**
* 原生canal-server对应的client适配器工作线程
*
* @author rewrema 2018-8-19 下午11:30:49
* @version 1.0.0
*/
public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
private static final int BATCH_SIZE = 50;
private static final int SO_TIMEOUT = 0;
private CanalConnector connector;
/**
* 单台client适配器worker的构造方法
*
* @param canalDestination canal实例名
* @param address canal-server地址
* @param canalOuterAdapters 外部适配器组
*/
public CanalAdapterWorker(CanalClientConfig canalClientConfig, String canalDestination, SocketAddress address,
List<List<OuterAdapter>> canalOuterAdapters){
super(canalOuterAdapters);
this.canalClientConfig = canalClientConfig;
this.canalDestination = canalDestination;
connector = CanalConnectors.newSingleConnector(address, canalDestination, "", "");
}
/**
* HA模式下client适配器worker的构造方法
*
* @param canalDestination canal实例名
* @param zookeeperHosts zookeeper地址
* @param canalOuterAdapters 外部适配器组
*/
public CanalAdapterWorker(CanalClientConfig canalClientConfig, String canalDestination, String zookeeperHosts,
List<List<OuterAdapter>> canalOuterAdapters){
super(canalOuterAdapters);
this.canalDestination = canalDestination;
this.canalClientConfig = canalClientConfig;
connector = CanalConnectors.newClusterConnector(zookeeperHosts, canalDestination, "", "");
((ClusterCanalConnector) connector).setSoTimeout(SO_TIMEOUT);
}
@Override
protected void process() {
while (!running) { // waiting until running == true
while (!running) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
}
}
int retry = canalClientConfig.getRetries() == null || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
if (retry == -1) {
// 重试次数-1代表异常时一直阻塞重试
retry = Integer.MAX_VALUE;
}
// long timeout = canalClientConfig.getTimeout() == null ? 300000 :
// canalClientConfig.getTimeout(); // 默认超时5分钟
Integer batchSize = canalClientConfig.getBatchSize();
if (batchSize == null) {
batchSize = BATCH_SIZE;
}
while (running) {
try {
syncSwitch.get(canalDestination);
logger.info("=============> Start to connect destination: {} <=============", this.canalDestination);
connector.connect();
logger.info("=============> Start to subscribe destination: {} <=============", this.canalDestination);
connector.subscribe();
logger.info("=============> Subscribe destination: {} succeed <=============", this.canalDestination);
while (running) {
try {
syncSwitch.get(canalDestination, 1L, TimeUnit.MINUTES);
} catch (TimeoutException e) {
break;
}
if (!running) {
break;
}
for (int i = 0; i < retry; i++) {
if (!running) {
break;
}
Message message = connector.getWithoutAck(batchSize); // 获取指定数量的数据
long batchId = message.getId();
try {
int size = message.getEntries().size();
if (batchId == -1 || size == 0) {
Thread.sleep(500);
} else {
if (logger.isDebugEnabled()) {
logger.debug("destination: {} batchId: {} batchSize: {} ",
canalDestination,
batchId,
size);
}
long begin = System.currentTimeMillis();
writeOut(message);
if (logger.isDebugEnabled()) {
logger.debug("destination: {} batchId: {} elapsed time: {} ms",
canalDestination,
batchId,
System.currentTimeMillis() - begin);
}
}
connector.ack(batchId); // 提交确认
break;
} catch (Exception e) {
if (i != retry - 1) {
connector.rollback(batchId); // 处理失败, 回滚数据
logger.error(e.getMessage() + " Error sync and rollback, execute times: " + (i + 1));
} else {
connector.ack(batchId);
logger.error(e.getMessage() + " Error sync but ACK!");
}
Thread.sleep(500);
}
}
}
} catch (Throwable e) {
logger.error("process error!", e);
} finally {
connector.disconnect();
logger.info("=============> Disconnect destination: {} <=============", this.canalDestination);
}
if (running) { // is reconnect
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
// ignore
}
}
}
}
@Override
public void stop() {
try {
if (!running) {
return;
}
if (connector instanceof ClusterCanalConnector) {
((ClusterCanalConnector) connector).stopRunning();
} else if (connector instanceof SimpleCanalConnector) {
((SimpleCanalConnector) connector).stopRunning();
}
running = false;
syncSwitch.release(canalDestination);
logger.info("destination {} is waiting for adapters' worker thread die!", canalDestination);
if (thread != null) {
try {
thread.join();
} catch (InterruptedException e) {
// ignore
}
}
groupInnerExecutorService.shutdown();
logger.info("destination {} adapters worker thread dead!", canalDestination);
canalOuterAdapters.forEach(outerAdapters -> outerAdapters.forEach(OuterAdapter::destroy));
logger.info("destination {} all adapters destroyed!", canalDestination);
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
}
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/DbRemoteConfigLoader.java
+2 -2
View File
@@ -15,14 +15,14 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import com.alibaba.otter.canal.common.utils.CommonUtils;
import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.otter.canal.adapter.launcher.config.AdapterConfigHolder;
import com.alibaba.otter.canal.common.utils.CommonUtils;
import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
import com.google.common.base.Joiner;
/**
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/RemoteAdapterMonitorImpl.java
+1 -2
View File
@@ -5,11 +5,10 @@ import java.io.FileOutputStream;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
import com.alibaba.otter.canal.common.utils.CommonUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.common.utils.CommonUtils;
/**
* 远程配置监听器实现
*
client-adapter/launcher/src/main/resources/application.yml
+34 -9
View File
@@ -7,20 +7,45 @@ spring:
default-property-inclusion: non_null
canal.conf:
mode: tcp # kafka rocketMQ
canalServerHost: 127.0.0.1:11111
# zookeeperHosts: slave1:2181
# mqServers: 127.0.0.1:9092 #or rocketmq
# flatMessage: true
batchSize: 500
mode: tcp #tcp kafka rocketMQ rabbitMQ
flatMessage: true
zookeeperHosts:
syncBatchSize: 1000
retries: 0
timeout:
accessKey:
secretKey:
username:
password:
vhost:
consumerProperties:
# canal tcp consumer
canal.tcp.server.host: 127.0.0.1:11111
canal.tcp.zookeeper.hosts:
canal.tcp.batch.size: 500
canal.tcp.username:
canal.tcp.password:
# kafka consumer
kafka.bootstrap.servers: 127.0.0.1:9092
kafka.enable.auto.commit: false
kafka.auto.commit.interval.ms: 1000
kafka.auto.offset.reset: latest
kafka.request.timeout.ms: 40000
kafka.session.timeout.ms: 30000
kafka.isolation.level: read_committed
kafka.max.poll.records: 1000
# rocketMQ consumer
rocketmq.namespace:
rocketmq.namesrv.addr: 127.0.0.1:9876
rocketmq.batch.size: 1000
rocketmq.enable.message.trace: false
rocketmq.customized.trace.topic:
rocketmq.access.channel:
rocketmq.subscribe.filter:
# rabbitMQ consumer
rabbitmq.host:
rabbitmq.virtual.host:
rabbitmq.username:
rabbitmq.password:
rabbitmq.resource.ownerId:
# srcDataSources:
# defaultDS:
# url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true
client/pom.xml
+10 -2
View File
@@ -101,23 +101,31 @@
<version>${spring_version}</version>
<scope>test</scope>
</dependency>
<!-- 客户端要使用请单独引入mq-clients依赖 -->
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-client</artifactId>
<version>4.5.2</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-acl</artifactId>
<version>4.5.2</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.rabbitmq</groupId>
<artifactId>amqp-client</artifactId>
<version>5.5.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.alibaba.mq-amqp</groupId>
<artifactId>mq-amqp-client</artifactId>
<version>1.0.3</version>
<scope>provided</scope>
</dependency>
<!-- 客户端要使用请单独引入kafka-clients依赖 -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
@@ -143,7 +151,7 @@
<createDependencyReducedPom>false</createDependencyReducedPom>
<artifactSet>
<includes>
<include>com.alibaba.otter:*</include>
<include>com.alibaba.otter:canal.client</include>
<include>com.google.guava:*</include>
<include>org.joda:*</include>
<include>com.google.common:*</include>
connector/core/pom.xml
+40
View File
@@ -0,0 +1,40 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>canal.connector</artifactId>
<groupId>com.alibaba.otter</groupId>
<version>1.1.5-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.core</artifactId>
<packaging>jar</packaging>
<name>canal connector core module for otter ${project.version}</name>
<dependencies>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.protocol</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.googlecode.aviator</groupId>
<artifactId>aviator</artifactId>
<version>2.2.1</version>
</dependency>
<dependency>
<groupId>oro</groupId>
<artifactId>oro</artifactId>
<version>2.0.8</version>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.9.4</version>
</dependency>
</dependencies>
</project>
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/config/CanalConstants.java
+26
View File
@@ -0,0 +1,26 @@
package com.alibaba.otter.canal.connector.core.config;
/**
* MQ配置常量
*
* @author rewerma 2020-01-27
* @version 1.0.0
*/
public class CanalConstants {
public static final String ROOT = "canal";
public static final String CANAL_FILTER_TRANSACTION_ENTRY = ROOT + "." + "instance.filter.transaction.entry";
public static final String CANAL_MQ_FLAT_MESSAGE = ROOT + "." + "mq.flat.message";
public static final String CANAL_MQ_DATABASE_HASH = ROOT + "." + "mq.database.hash";
public static final String CANAL_MQ_PARALLEL_THREAD_SIZE = ROOT + "." + "mq.parallel.thread.size";
public static final String CANAL_MQ_CANAL_BATCH_SIZE = ROOT + "." + "mq.canal.batch.size";
public static final String CANAL_MQ_CANAL_FETCH_TIMEOUT = ROOT + "." + "mq.canal.fetch.timeout";
public static final String CANAL_MQ_ACCESS_CHANNEL = ROOT + "." + "mq.access.channel";
public static final String CANAL_ALIYUN_ACCESS_KEY = ROOT + "." + "aliyun.accessKey";
public static final String CANAL_ALIYUN_SECRET_KEY = ROOT + "." + "aliyun.secretKey";
public static final String CANAL_ALIYUN_UID = ROOT + "." + "aliyun.uid";
}
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/config/MQProperties.java
+102
View File
@@ -0,0 +1,102 @@
package com.alibaba.otter.canal.connector.core.config;
/**
* MQ配置类
*
* @author rewerma 2020-01-27
* @version 1.0.0
*/
public class MQProperties {
private boolean flatMessage = true;
private boolean databaseHash = true;
private boolean filterTransactionEntry = true;
private Integer parallelThreadSize = 8;
private Integer fetchTimeout = 100;
private Integer batchSize = 50;
private String accessChannel = "local";
private String aliyunAccessKey = "";
private String aliyunSecretKey = "";
private int aliyunUid = 0;
public boolean isFlatMessage() {
return flatMessage;
}
public void setFlatMessage(boolean flatMessage) {
this.flatMessage = flatMessage;
}
public boolean isDatabaseHash() {
return databaseHash;
}
public void setDatabaseHash(boolean databaseHash) {
this.databaseHash = databaseHash;
}
public boolean isFilterTransactionEntry() {
return filterTransactionEntry;
}
public void setFilterTransactionEntry(boolean filterTransactionEntry) {
this.filterTransactionEntry = filterTransactionEntry;
}
public Integer getParallelThreadSize() {
return parallelThreadSize;
}
public void setParallelThreadSize(Integer parallelThreadSize) {
this.parallelThreadSize = parallelThreadSize;
}
public Integer getFetchTimeout() {
return fetchTimeout;
}
public void setFetchTimeout(Integer fetchTimeout) {
this.fetchTimeout = fetchTimeout;
}
public Integer getBatchSize() {
return batchSize;
}
public void setBatchSize(Integer batchSize) {
this.batchSize = batchSize;
}
public String getAccessChannel() {
return accessChannel;
}
public void setAccessChannel(String accessChannel) {
this.accessChannel = accessChannel;
}
public String getAliyunAccessKey() {
return aliyunAccessKey;
}
public void setAliyunAccessKey(String aliyunAccessKey) {
this.aliyunAccessKey = aliyunAccessKey;
}
public String getAliyunSecretKey() {
return aliyunSecretKey;
}
public void setAliyunSecretKey(String aliyunSecretKey) {
this.aliyunSecretKey = aliyunSecretKey;
}
public int getAliyunUid() {
return aliyunUid;
}
public void setAliyunUid(int aliyunUid) {
this.aliyunUid = aliyunUid;
}
}
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/consumer/CommonMessage.java
+131
View File
@@ -0,0 +1,131 @@
package com.alibaba.otter.canal.connector.core.consumer;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
public class CommonMessage implements Serializable {
private static final long serialVersionUID = 2611556444074013268L;
private String database; // 数据库或schema
private String table; // 表名
private List<String> pkNames;
private Boolean isDdl;
private String type; // 类型: INSERT UPDATE DELETE
// binlog executeTime
private Long es; // 执行耗时
// dml build timeStamp
private Long ts; // 同步时间
private String sql; // 执行的sql, dml sql为空
private List<Map<String, Object>> data; // 数据列表
private List<Map<String, Object>> old; // 旧数据列表, 用于update, size和data的size一一对应
public String getDatabase() {
return database;
}
public void setDatabase(String database) {
this.database = database;
}
public String getTable() {
return table;
}
public void setTable(String table) {
this.table = table;
}
public List<String> getPkNames() {
return pkNames;
}
public void setPkNames(List<String> pkNames) {
this.pkNames = pkNames;
}
public Boolean getIsDdl() {
return isDdl;
}
public void setIsDdl(Boolean isDdl) {
this.isDdl = isDdl;
}
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public Long getTs() {
return ts;
}
public void setTs(Long ts) {
this.ts = ts;
}
public String getSql() {
return sql;
}
public void setSql(String sql) {
this.sql = sql;
}
public List<Map<String, Object>> getData() {
return data;
}
public void setData(List<Map<String, Object>> data) {
this.data = data;
}
public List<Map<String, Object>> getOld() {
return old;
}
public void setOld(List<Map<String, Object>> old) {
this.old = old;
}
public Long getEs() {
return es;
}
public void setEs(Long es) {
this.es = es;
}
public void clear() {
database = null;
table = null;
type = null;
ts = null;
es = null;
data = null;
old = null;
sql = null;
}
@Override
public String toString() {
return "CommonMessage{" +
"database='" + database + '\'' +
", table='" + table + '\'' +
", pkNames=" + pkNames +
", isDdl=" + isDdl +
", type='" + type + '\'' +
", es=" + es +
", ts=" + ts +
", sql='" + sql + '\'' +
", data=" + data +
", old=" + old +
'}';
}
}
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/AviaterRegexFilter.java
+125
View File
@@ -0,0 +1,125 @@
package com.alibaba.otter.canal.connector.core.filter;
import java.util.*;
import org.apache.commons.lang.StringUtils;
import com.googlecode.aviator.AviatorEvaluator;
import com.googlecode.aviator.Expression;
/**
* 基于aviater进行tableName正则匹配的过滤算法
*
* @author jianghang 2012-7-20 下午06:01:34
*/
public class AviaterRegexFilter {
private static final String SPLIT = ",";
private static final String PATTERN_SPLIT = "|";
private static final String FILTER_EXPRESSION = "regex(pattern,target)";
private static final RegexFunction regexFunction = new RegexFunction();
private final Expression exp = AviatorEvaluator.compile(FILTER_EXPRESSION, true);
static {
AviatorEvaluator.addFunction(regexFunction);
}
private static final Comparator<String> COMPARATOR = new StringComparator();
final private String pattern;
final private boolean defaultEmptyValue;
public AviaterRegexFilter(String pattern){
this(pattern, true);
}
public AviaterRegexFilter(String pattern, boolean defaultEmptyValue){
this.defaultEmptyValue = defaultEmptyValue;
List<String> list = null;
if (StringUtils.isEmpty(pattern)) {
list = new ArrayList<String>();
} else {
String[] ss = StringUtils.split(pattern, SPLIT);
list = Arrays.asList(ss);
}
// 对pattern按照从长到短的排序
// 因为 foo|foot 匹配 foot 会出错，原因是 foot 匹配了 foo 之后，会返回 foo，但是 foo 的长度和 foot
// 的长度不一样
Collections.sort(list, COMPARATOR);
// 对pattern进行头尾完全匹配
list = completionPattern(list);
this.pattern = StringUtils.join(list, PATTERN_SPLIT);
}
public boolean filter(String filtered) {
if (StringUtils.isEmpty(pattern)) {
return defaultEmptyValue;
}
if (StringUtils.isEmpty(filtered)) {
return defaultEmptyValue;
}
Map<String, Object> env = new HashMap<String, Object>();
env.put("pattern", pattern);
env.put("target", filtered.toLowerCase());
return (Boolean) exp.execute(env);
}
/**
* 修复正则表达式匹配的问题，因为使用了 oro 的 matches，会出现：
*
* <pre>
* foo|foot 匹配 foot 出错，原因是 foot 匹配了 foo 之后，会返回 foo，但是 foo 的长度和 foot 的长度不一样
* </pre>
*
* 因此此类对正则表达式进行了从长到短的排序
*
* @author zebin.xuzb 2012-10-22 下午2:02:26
* @version 1.0.0
*/
private static class StringComparator implements Comparator<String> {
@Override
public int compare(String str1, String str2) {
if (str1.length() > str2.length()) {
return -1;
} else if (str1.length() < str2.length()) {
return 1;
} else {
return 0;
}
}
}
/**
* 修复正则表达式匹配的问题，即使按照长度递减排序，还是会出现以下问题：
*
* <pre>
* foooo|f.*t 匹配 fooooot 出错，原因是 fooooot 匹配了 foooo 之后，会将 fooo 和数据进行匹配，但是 foooo 的长度和 fooooot 的长度不一样
* </pre>
*
* 因此此类对正则表达式进行头尾完全匹配
*
* @author simon
* @version 1.0.0
*/
private List<String> completionPattern(List<String> patterns) {
List<String> result = new ArrayList<String>();
for (String pattern : patterns) {
StringBuffer stringBuffer = new StringBuffer();
stringBuffer.append("^");
stringBuffer.append(pattern);
stringBuffer.append("$");
result.add(stringBuffer.toString());
}
return result;
}
@Override
public String toString() {
return pattern;
}
}
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/PatternUtils.java
+33
View File
@@ -0,0 +1,33 @@
package com.alibaba.otter.canal.connector.core.filter;
import java.util.Map;
import org.apache.oro.text.regex.MalformedPatternException;
import org.apache.oro.text.regex.Pattern;
import org.apache.oro.text.regex.PatternCompiler;
import org.apache.oro.text.regex.Perl5Compiler;
import com.google.common.collect.MapMaker;
import com.google.common.collect.MigrateMap;
public class PatternUtils {
@SuppressWarnings("deprecation")
private static Map<String, Pattern> patterns = MigrateMap.makeComputingMap(new MapMaker().softValues(), pattern -> {
try {
PatternCompiler pc = new Perl5Compiler();
return pc.compile(pattern,
Perl5Compiler.CASE_INSENSITIVE_MASK | Perl5Compiler.READ_ONLY_MASK | Perl5Compiler.SINGLELINE_MASK);
} catch (MalformedPatternException e) {
throw new RuntimeException(e);
}
});
public static Pattern getPattern(String pattern) {
return patterns.get(pattern);
}
public static void clear() {
patterns.clear();
}
}
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/RegexFunction.java
+31
View File
@@ -0,0 +1,31 @@
package com.alibaba.otter.canal.connector.core.filter;
import java.util.Map;
import org.apache.oro.text.regex.Perl5Matcher;
import com.googlecode.aviator.runtime.function.AbstractFunction;
import com.googlecode.aviator.runtime.function.FunctionUtils;
import com.googlecode.aviator.runtime.type.AviatorBoolean;
import com.googlecode.aviator.runtime.type.AviatorObject;
/**
* 提供aviator regex的代码扩展
*
* @author jianghang 2012-7-23 上午10:29:23
*/
public class RegexFunction extends AbstractFunction {
public AviatorObject call(Map<String, Object> env, AviatorObject arg1, AviatorObject arg2) {
String pattern = FunctionUtils.getStringValue(arg1, env);
String text = FunctionUtils.getStringValue(arg2, env);
Perl5Matcher matcher = new Perl5Matcher();
boolean isMatch = matcher.matches(text, PatternUtils.getPattern(pattern));
return AviatorBoolean.valueOf(isMatch);
}
public String getName() {
return "regex";
}
}
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/AbstractMQProducer.java
+110
View File
@@ -0,0 +1,110 @@
package com.alibaba.otter.canal.connector.core.producer;
import java.util.Properties;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import com.alibaba.otter.canal.connector.core.config.CanalConstants;
import org.apache.commons.lang.StringUtils;
import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
import com.alibaba.otter.canal.connector.core.config.MQProperties;
import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
/**
* MQ producer 抽象类
*
* @author rewerma 2020-01-27
* @version 1.0.0
*/
public abstract class AbstractMQProducer implements CanalMQProducer {
protected MQProperties mqProperties;
protected ThreadPoolExecutor executor;
@Override
public void init(Properties properties) {
// parse canal mq properties
loadCanalMqProperties(properties);
int parallelThreadSize = mqProperties.getParallelThreadSize();
executor = new ThreadPoolExecutor(parallelThreadSize,
parallelThreadSize,
0,
TimeUnit.SECONDS,
new ArrayBlockingQueue<Runnable>(parallelThreadSize * 2),
new NamedThreadFactory("MQParallel"),
new ThreadPoolExecutor.CallerRunsPolicy());
}
@Override
public MQProperties getMqProperties() {
return this.mqProperties;
}
@Override
public void stop() {
if (executor != null) {
executor.shutdownNow();
}
}
/**
* 初始化配置
* <p>
* canal.mq.flat.message = true <br/>
* canal.mq.database.hash = true <br/>
* canal.mq.filter.transaction.entry = true <br/>
* canal.mq.parallel.thread.size = 8 <br/>
* canal.mq.batch.size = 50 <br/>
* canal.mq.timeout = 100 <br/>
* canal.mq.access.channel = local <br/>
* </p>
*
* @param properties 总配置对象
*/
private void loadCanalMqProperties(Properties properties) {
String flatMessage = properties.getProperty(CanalConstants.CANAL_MQ_FLAT_MESSAGE);
if (!StringUtils.isEmpty(flatMessage)) {
mqProperties.setFlatMessage(Boolean.parseBoolean(flatMessage));
}
String databaseHash = properties.getProperty(CanalConstants.CANAL_MQ_DATABASE_HASH);
if (!StringUtils.isEmpty(databaseHash)) {
mqProperties.setDatabaseHash(Boolean.parseBoolean(databaseHash));
}
String filterTranEntry = properties.getProperty(CanalConstants.CANAL_FILTER_TRANSACTION_ENTRY);
if (!StringUtils.isEmpty(filterTranEntry)) {
mqProperties.setFilterTransactionEntry(Boolean.parseBoolean(filterTranEntry));
}
String parallelThreadSize = properties.getProperty(CanalConstants.CANAL_MQ_PARALLEL_THREAD_SIZE);
if (!StringUtils.isEmpty(parallelThreadSize)) {
mqProperties.setParallelThreadSize(Integer.parseInt(parallelThreadSize));
}
String batchSize = properties.getProperty(CanalConstants.CANAL_MQ_CANAL_BATCH_SIZE);
if (!StringUtils.isEmpty(batchSize)) {
mqProperties.setBatchSize(Integer.parseInt(batchSize));
}
String timeOut = properties.getProperty(CanalConstants.CANAL_MQ_CANAL_FETCH_TIMEOUT);
if (!StringUtils.isEmpty(timeOut)) {
mqProperties.setFetchTimeout(Integer.parseInt(timeOut));
}
String accessChannel = properties.getProperty(CanalConstants.CANAL_MQ_ACCESS_CHANNEL);
if (!StringUtils.isEmpty(accessChannel)) {
mqProperties.setAccessChannel(accessChannel);
}
String aliyunAccessKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_ACCESS_KEY);
if (!StringUtils.isEmpty(aliyunAccessKey)) {
mqProperties.setAliyunAccessKey(aliyunAccessKey);
}
String aliyunSecretKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_SECRET_KEY);
if (!StringUtils.isEmpty(aliyunSecretKey)) {
mqProperties.setAliyunAccessKey(aliyunSecretKey);
}
String aliyunUid = properties.getProperty(CanalConstants.CANAL_ALIYUN_UID);
if (!StringUtils.isEmpty(aliyunUid)) {
mqProperties.setAliyunUid(Integer.parseInt(aliyunUid));
}
}
}
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/MQDestination.java
+64
View File
@@ -0,0 +1,64 @@
package com.alibaba.otter.canal.connector.core.producer;
/**
* MQ producer destination
*
* @author rewerma 2020-01-27
* @version 1.0.0
*/
public class MQDestination {
private String canalDestination;
private String topic;
private Integer partition;
private Integer partitionsNum;
private String partitionHash;
private String dynamicTopic;
public String getCanalDestination() {
return canalDestination;
}
public void setCanalDestination(String canalDestination) {
this.canalDestination = canalDestination;
}
public String getTopic() {
return topic;
}
public void setTopic(String topic) {
this.topic = topic;
}
public Integer getPartition() {
return partition;
}
public void setPartition(Integer partition) {
this.partition = partition;
}
public Integer getPartitionsNum() {
return partitionsNum;
}
public void setPartitionsNum(Integer partitionsNum) {
this.partitionsNum = partitionsNum;
}
public String getPartitionHash() {
return partitionHash;
}
public void setPartitionHash(String partitionHash) {
this.partitionHash = partitionHash;
}
public String getDynamicTopic() {
return dynamicTopic;
}
public void setDynamicTopic(String dynamicTopic) {
this.dynamicTopic = dynamicTopic;
}
}
server/src/main/java/com/alibaba/otter/canal/common/MQMessageUtils.java → connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/MQMessageUtils.java
+70 -66
View File
@@ -1,25 +1,18 @@
package com.alibaba.otter.canal.common;
package com.alibaba.otter.canal.connector.core.producer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.*;
import java.util.concurrent.ThreadPoolExecutor;
import com.google.common.base.Function;
import org.apache.commons.lang.StringUtils;
import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
import com.alibaba.otter.canal.connector.core.filter.AviaterRegexFilter;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
import com.alibaba.otter.canal.protocol.CanalEntry.RowChange;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message;
import com.google.common.base.Function;
import com.google.common.collect.Lists;
import com.google.common.collect.MapMaker;
import com.google.common.collect.MigrateMap;
@@ -29,77 +22,88 @@ import com.google.protobuf.InvalidProtocolBufferException;
/**
* process MQ Message utils
*
* @author agapple 2018年12月11日 下午1:28:32
*/
/**
* @author agapple 2019年9月29日 下午12:36:26
* @since 5.0.0
*/
public class MQMessageUtils {
@SuppressWarnings("deprecation")
private static Map<String, List<PartitionData>> partitionDatas = MigrateMap.makeComputingMap(new MapMaker().softValues(),
new Function<String, List<PartitionData>>() {
private static Map<String, List<PartitionData>> partitionDatas = MigrateMap
.makeComputingMap(new MapMaker().softValues(), pkHashConfigs -> {
List<PartitionData> datas = Lists
.newArrayList();
public List<PartitionData> apply(String pkHashConfigs) {
List<PartitionData> datas = Lists.newArrayList();
String[] pkHashConfigArray = StringUtils
.split(StringUtils.replace(
pkHashConfigs,
",",
";"), ";");
// schema.table:id^name
for (String pkHashConfig : pkHashConfigArray) {
PartitionData data = new PartitionData();
int i = pkHashConfig
.lastIndexOf(":");
if (i > 0) {
String pkStr = pkHashConfig
.substring(i + 1);
if (pkStr.equalsIgnoreCase(
"$pk$")) {
data.hashMode.autoPkHash = true;
} else {
data.hashMode.pkNames = Lists
.newArrayList(
StringUtils
.split(
pkStr,
'^'));
}
String[] pkHashConfigArray = StringUtils.split(StringUtils.replace(pkHashConfigs,
",",
";"),
";");
// schema.table:id^name
for (String pkHashConfig : pkHashConfigArray) {
PartitionData data = new PartitionData();
int i = pkHashConfig.lastIndexOf(":");
if (i > 0) {
String pkStr = pkHashConfig.substring(i + 1);
if (pkStr.equalsIgnoreCase("$pk$")) {
data.hashMode.autoPkHash = true;
} else {
data.hashMode.pkNames = Lists.newArrayList(StringUtils.split(pkStr,
'^'));
}
pkHashConfig = pkHashConfig
.substring(0, i);
} else {
data.hashMode.tableHash = true;
}
pkHashConfig = pkHashConfig.substring(0,
i);
} else {
data.hashMode.tableHash = true;
}
if (!isWildCard(
pkHashConfig)) {
data.simpleName = pkHashConfig;
} else {
data.regexFilter = new AviaterRegexFilter(
pkHashConfig);
}
datas.add(data);
}
if (!isWildCard(pkHashConfig)) {
data.simpleName = pkHashConfig;
} else {
data.regexFilter = new AviaterRegexFilter(pkHashConfig);
}
datas.add(data);
}
return datas;
}
});
return datas;
});
@SuppressWarnings("deprecation")
private static Map<String, List<DynamicTopicData>> dynamicTopicDatas = MigrateMap.makeComputingMap(new MapMaker().softValues(),
new Function<String, List<DynamicTopicData>>() {
private static Map<String, List<DynamicTopicData>> dynamicTopicDatas = MigrateMap
.makeComputingMap(new MapMaker().softValues(), new Function<String, List<DynamicTopicData>>() {
public List<DynamicTopicData> apply(String pkHashConfigs) {
List<DynamicTopicData> datas = Lists.newArrayList();
String[] dynamicTopicArray = StringUtils.split(StringUtils.replace(pkHashConfigs,
",",
";"),
";");
List<DynamicTopicData> datas = Lists
.newArrayList();
String[] dynamicTopicArray = StringUtils
.split(StringUtils.replace(
pkHashConfigs,
",",
";"), ";");
// schema.table
for (String dynamicTopic : dynamicTopicArray) {
DynamicTopicData data = new DynamicTopicData();
if (!isWildCard(dynamicTopic)) {
if (!isWildCard(
dynamicTopic)) {
data.simpleName = dynamicTopic;
} else {
if (dynamicTopic.contains("\\.")) {
data.tableRegexFilter = new AviaterRegexFilter(dynamicTopic);
if (dynamicTopic
.contains("\\.")) {
data.tableRegexFilter = new AviaterRegexFilter(
dynamicTopic);
} else {
data.schemaRegexFilter = new AviaterRegexFilter(dynamicTopic);
data.schemaRegexFilter = new AviaterRegexFilter(
dynamicTopic);
}
}
datas.add(data);
@@ -238,8 +242,8 @@ public class MQMessageUtils {
* @return 分区message数组
*/
@SuppressWarnings("unchecked")
public static Message[] messagePartition(EntryRowData[] datas, long id, Integer partitionsNum,
String pkHashConfigs, boolean databaseHash) {
public static Message[] messagePartition(EntryRowData[] datas, long id, Integer partitionsNum, String pkHashConfigs,
boolean databaseHash) {
if (partitionsNum == null) {
partitionsNum = 1;
}
@@ -338,8 +342,8 @@ public class MQMessageUtils {
/**
* 将Message转换为FlatMessage
*
* @param message 原生message
* @return FlatMessage列表
* @author agapple 2018年12月11日 下午1:28:32
*/
public static List<FlatMessage> messageConverter(EntryRowData[] datas, long id) {
List<FlatMessage> flatMessages = new ArrayList<>();
@@ -627,8 +631,8 @@ public class MQMessageUtils {
private static boolean isWildCard(String value) {
// not contaiins '.' ?
return StringUtils.containsAny(value, new char[] { '*', '?', '+', '|', '(', ')', '{', '}', '[', ']', '\\', '$',
'^' });
return StringUtils.containsAny(value,
new char[] { '*', '?', '+', '|', '(', ')', '{', '}', '[', ']', '\\', '$', '^' });
}
private static void put2MapMessage(Map<String, Message> messageMap, Long messageId, String topicName,
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/CanalMQProducer.java
+43
View File
@@ -0,0 +1,43 @@
package com.alibaba.otter.canal.connector.core.spi;
import java.util.Properties;
import com.alibaba.otter.canal.connector.core.config.MQProperties;
import com.alibaba.otter.canal.connector.core.util.Callback;
import com.alibaba.otter.canal.connector.core.producer.MQDestination;
import com.alibaba.otter.canal.protocol.Message;
/**
* MQ producer SPI 接口
*
* @author rewerma 2020-01-27
* @version 1.0.0
*/
@SPI("kafka")
public interface CanalMQProducer {
/**
* Init producer.
*/
void init(Properties properties);
/**
* Get base mq properties
*
* @return MQProperties
*/
MQProperties getMqProperties();
/**
* Send canal message to related topic
*
* @param canalDestination canal mq destination
* @param message canal message
*/
void send(MQDestination canalDestination, Message message, Callback callback);
/**
* Stop MQ producer service
*/
void stop();
}
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/CanalMsgConsumer.java
+55
View File
@@ -0,0 +1,55 @@
package com.alibaba.otter.canal.connector.core.spi;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
/**
* Canal/MQ consumer SPI 接口
*
* @author rewerma @ 2020-02-01
* @version 1.0.0
*/
@SPI("kafka")
public interface CanalMsgConsumer {
/**
* 初始化
*
* @param properties consumer properties
* @param topic topic/destination
* @param groupId mq group id
*/
void init(Properties properties, String topic, String groupId);
/**
* 连接Canal/MQ
*/
void connect();
/**
* 批量拉取数据
*
* @param timeout 超时时间
* @param unit 时间单位
* @return Message列表
*/
List<CommonMessage> getMessage(Long timeout, TimeUnit unit);
/**
* 提交
*/
void ack();
/**
* 回滚
*/
void rollback();
/**
* 断开连接
*/
void disconnect();
}
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/ExtensionLoader.java
+435
View File
@@ -0,0 +1,435 @@
package com.alibaba.otter.canal.connector.core.spi;
import java.io.BufferedReader;
import java.io.File;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* SPI 类加载器
*
* @author rewerma 2018-8-19 下午11:30:49
* @version 1.0.0
*/
public class ExtensionLoader<T> {
private static final Logger logger = LoggerFactory
.getLogger(ExtensionLoader.class);
private static final String SERVICES_DIRECTORY = "META-INF/services/";
private static final String CANAL_DIRECTORY = "META-INF/canal/";
private static final String DEFAULT_CLASSLOADER_POLICY = "internal";
private static final Pattern NAME_SEPARATOR = Pattern
.compile("\\s*[,]+\\s*");
private static final ConcurrentMap<Class<?>, ExtensionLoader<?>> EXTENSION_LOADERS = new ConcurrentHashMap<>();
private static final ConcurrentMap<Class<?>, Object> EXTENSION_INSTANCES = new ConcurrentHashMap<>();
private static final ConcurrentMap<String, Object> EXTENSION_KEY_INSTANCE = new ConcurrentHashMap<>();
private final Class<?> type;
private final String classLoaderPolicy;
private final ConcurrentMap<Class<?>, String> cachedNames = new ConcurrentHashMap<>();
private final Holder<Map<String, Class<?>>> cachedClasses = new Holder<>();
private final ConcurrentMap<String, Holder<Object>> cachedInstances = new ConcurrentHashMap<>();
private String cachedDefaultName;
private ConcurrentHashMap<String, IllegalStateException> exceptions = new ConcurrentHashMap<>();
private static <T> boolean withExtensionAnnotation(Class<T> type) {
return type.isAnnotationPresent(SPI.class);
}
public static <T> ExtensionLoader<T> getExtensionLoader(Class<T> type) {
return getExtensionLoader(type, DEFAULT_CLASSLOADER_POLICY);
}
@SuppressWarnings("unchecked")
public static <T> ExtensionLoader<T> getExtensionLoader(Class<T> type, String classLoaderPolicy) {
if (type == null) throw new IllegalArgumentException("Extension type == null");
if (!type.isInterface()) {
throw new IllegalArgumentException("Extension type(" + type + ") is not interface!");
}
if (!withExtensionAnnotation(type)) {
throw new IllegalArgumentException("Extension type(" + type + ") is not extension, because WITHOUT @"
+ SPI.class.getSimpleName() + " Annotation!");
}
ExtensionLoader<T> loader = (ExtensionLoader<T>) EXTENSION_LOADERS.get(type);
if (loader == null) {
EXTENSION_LOADERS.putIfAbsent(type, new ExtensionLoader<T>(type, classLoaderPolicy));
loader = (ExtensionLoader<T>) EXTENSION_LOADERS.get(type);
}
return loader;
}
public ExtensionLoader(Class<?> type){
this.type = type;
this.classLoaderPolicy = DEFAULT_CLASSLOADER_POLICY;
}
public ExtensionLoader(Class<?> type, String classLoaderPolicy){
this.type = type;
this.classLoaderPolicy = classLoaderPolicy;
}
/**
* 返回指定名字的扩展
*
* @param name
* @return
*/
@SuppressWarnings("unchecked")
public T getExtension(String name, String spiDir, String standbyDir) {
if (name == null || name.length() == 0) throw new IllegalArgumentException("Extension name == null");
if ("true".equals(name)) {
return getDefaultExtension(spiDir, standbyDir);
}
Holder<Object> holder = cachedInstances.get(name);
if (holder == null) {
cachedInstances.putIfAbsent(name, new Holder<>());
holder = cachedInstances.get(name);
}
Object instance = holder.get();
if (instance == null) {
synchronized (holder) {
instance = holder.get();
if (instance == null) {
instance = createExtension(name, spiDir, standbyDir);
holder.set(instance);
}
}
}
return (T) instance;
}
@SuppressWarnings("unchecked")
public T getExtension(String name, String key, String spiDir, String standbyDir) {
if (name == null || name.length() == 0) throw new IllegalArgumentException("Extension name == null");
if ("true".equals(name)) {
return getDefaultExtension(spiDir, standbyDir);
}
String extKey = name + "-" + StringUtils.trimToEmpty(key);
Holder<Object> holder = cachedInstances.get(extKey);
if (holder == null) {
cachedInstances.putIfAbsent(extKey, new Holder<>());
holder = cachedInstances.get(extKey);
}
Object instance = holder.get();
if (instance == null) {
synchronized (holder) {
instance = holder.get();
if (instance == null) {
instance = createExtension(name, key, spiDir, standbyDir);
holder.set(instance);
}
}
}
return (T) instance;
}
/**
* 返回缺省的扩展，如果没有设置则返回<code>null</code>
*/
public T getDefaultExtension(String spiDir, String standbyDir) {
getExtensionClasses(spiDir, standbyDir);
if (null == cachedDefaultName || cachedDefaultName.length() == 0 || "true".equals(cachedDefaultName)) {
return null;
}
return getExtension(cachedDefaultName, spiDir, standbyDir);
}
@SuppressWarnings("unchecked")
public T createExtension(String name, String spiDir, String standbyDir) {
Class<?> clazz = getExtensionClasses(spiDir, standbyDir).get(name);
if (clazz == null) {
throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type
+ ") could not be instantiated: class could not be found");
}
try {
T instance = (T) EXTENSION_INSTANCES.get(clazz);
if (instance == null) {
EXTENSION_INSTANCES.putIfAbsent(clazz, (T) clazz.newInstance());
instance = (T) EXTENSION_INSTANCES.get(clazz);
}
return instance;
} catch (Throwable t) {
throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type
+ ") could not be instantiated: " + t.getMessage(),
t);
}
}
@SuppressWarnings("unchecked")
private T createExtension(String name, String key, String spiDir, String standbyDir) {
Class<?> clazz = getExtensionClasses(spiDir, standbyDir).get(name);
if (clazz == null) {
throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type
+ ") could not be instantiated: class could not be found");
}
try {
T instance = (T) EXTENSION_KEY_INSTANCE.get(name + "-" + key);
if (instance == null) {
EXTENSION_KEY_INSTANCE.putIfAbsent(name + "-" + key, clazz.newInstance());
instance = (T) EXTENSION_KEY_INSTANCE.get(name + "-" + key);
}
return instance;
} catch (Throwable t) {
throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type
+ ") could not be instantiated: " + t.getMessage(),
t);
}
}
private Map<String, Class<?>> getExtensionClasses(String spiDir, String standbyDir) {
Map<String, Class<?>> classes = cachedClasses.get();
if (classes == null) {
synchronized (cachedClasses) {
classes = cachedClasses.get();
if (classes == null) {
classes = loadExtensionClasses(spiDir, standbyDir);
cachedClasses.set(classes);
}
}
}
return classes;
}
private String getJarDirectoryPath() {
URL url = Thread.currentThread().getContextClassLoader().getResource("");
String dirtyPath;
if (url != null) {
dirtyPath = url.toString();
} else {
File file = new File("");
dirtyPath = file.getAbsolutePath();
}
String jarPath = dirtyPath.replaceAll("^.*file:/", ""); // removes
// file:/ and
// everything
// before it
jarPath = jarPath.replaceAll("jar!.*", "jar"); // removes everything
// after .jar, if .jar
// exists in dirtyPath
jarPath = jarPath.replaceAll("%20", " "); // necessary if path has
// spaces within
if (!jarPath.endsWith(".jar")) { // this is needed if you plan to run
// the app using Spring Tools Suit play
// button.
jarPath = jarPath.replaceAll("/classes/.*", "/classes/");
}
Path path = Paths.get(jarPath).getParent(); // Paths - from java 8
if (path != null) {
return path.toString();
}
return null;
}
private Map<String, Class<?>> loadExtensionClasses(String spiDir, String standbyDir) {
final SPI defaultAnnotation = type.getAnnotation(SPI.class);
if (defaultAnnotation != null) {
String value = defaultAnnotation.value();
if ((value = value.trim()).length() > 0) {
String[] names = NAME_SEPARATOR.split(value);
if (names.length > 1) {
throw new IllegalStateException("more than 1 default extension name on extension " + type.getName()
+ ": " + Arrays.toString(names));
}
if (names.length == 1) cachedDefaultName = names[0];
}
}
Map<String, Class<?>> extensionClasses = new HashMap<>();
if (spiDir != null && standbyDir != null) {
// 1. plugin folder，customized extension classLoader （jar_dir/plugin）
String dir = File.separator + this.getJarDirectoryPath() + spiDir; // + "plugin";
File externalLibDir = new File(dir);
if (!externalLibDir.exists()) {
externalLibDir = new File(File.separator + this.getJarDirectoryPath() + standbyDir);
}
logger.info("extension classpath dir: " + externalLibDir.getAbsolutePath());
if (externalLibDir.exists()) {
File[] files = externalLibDir.listFiles((dir1, name) -> name.endsWith(".jar"));
if (files != null) {
for (File f : files) {
URL url;
try {
url = f.toURI().toURL();
} catch (MalformedURLException e) {
throw new RuntimeException("load extension jar failed!", e);
}
ClassLoader parent = Thread.currentThread().getContextClassLoader();
URLClassLoader localClassLoader;
if (classLoaderPolicy == null || "".equals(classLoaderPolicy)
|| DEFAULT_CLASSLOADER_POLICY.equalsIgnoreCase(classLoaderPolicy)) {
localClassLoader = new URLClassExtensionLoader(new URL[] { url });
} else {
localClassLoader = new URLClassLoader(new URL[] { url }, parent);
}
loadFile(extensionClasses, CANAL_DIRECTORY, localClassLoader);
loadFile(extensionClasses, SERVICES_DIRECTORY, localClassLoader);
}
}
}
}
// 2. load inner extension class with default classLoader
ClassLoader classLoader = findClassLoader();
loadFile(extensionClasses, CANAL_DIRECTORY, classLoader);
loadFile(extensionClasses, SERVICES_DIRECTORY, classLoader);
return extensionClasses;
}
private void loadFile(Map<String, Class<?>> extensionClasses, String dir, ClassLoader classLoader) {
String fileName = dir + type.getName();
try {
Enumeration<URL> urls;
if (classLoader != null) {
urls = classLoader.getResources(fileName);
} else {
urls = ClassLoader.getSystemResources(fileName);
}
if (urls != null) {
while (urls.hasMoreElements()) {
URL url = urls.nextElement();
try {
BufferedReader reader = null;
try {
reader = new BufferedReader(
new InputStreamReader(url.openStream(), StandardCharsets.UTF_8));
String line = null;
while ((line = reader.readLine()) != null) {
final int ci = line.indexOf('#');
if (ci >= 0) line = line.substring(0, ci);
line = line.trim();
if (line.length() > 0) {
try {
String name = null;
int i = line.indexOf('=');
if (i > 0) {
name = line.substring(0, i).trim();
line = line.substring(i + 1).trim();
}
if (line.length() > 0) {
Class<?> clazz = classLoader.loadClass(line);
// Class<?> clazz =
// Class.forName(line, true,
// classLoader);
if (!type.isAssignableFrom(clazz)) {
throw new IllegalStateException(
"Error when load extension class(interface: " + type
+ ", class line: " + clazz.getName()
+ "), class " + clazz.getName()
+ "is not subtype of interface.");
} else {
try {
clazz.getConstructor(type);
} catch (NoSuchMethodException e) {
clazz.getConstructor();
String[] names = NAME_SEPARATOR.split(name);
if (names != null && names.length > 0) {
for (String n : names) {
if (!cachedNames.containsKey(clazz)) {
cachedNames.put(clazz, n);
}
Class<?> c = extensionClasses.get(n);
if (c == null) {
extensionClasses.put(n, clazz);
} else if (c != clazz) {
cachedNames.remove(clazz);
throw new IllegalStateException(
"Duplicate extension " + type.getName() + " name "
+ n + " on "
+ c.getName() + " and "
+ clazz.getName());
}
}
}
}
}
}
} catch (Throwable t) {
IllegalStateException e = new IllegalStateException(
"Failed to load extension class(interface: " + type + ", class line: "
+ line + ") in " + url
+ ", cause: "
+ t.getMessage(),
t);
exceptions.put(line, e);
}
}
} // end of while read lines
} finally {
if (reader != null) {
reader.close();
}
}
} catch (Throwable t) {
logger.error("Exception when load extension class(interface: " + type + ", class file: " + url
+ ") in " + url,
t);
}
} // end of while urls
}
} catch (Throwable t) {
logger.error(
"Exception when load extension class(interface: " + type + ", description file: " + fileName + ").",
t);
}
}
@SuppressWarnings("unused")
private static ClassLoader findClassLoader() {
return ExtensionLoader.class.getClassLoader();
}
@Override
public String toString() {
return this.getClass().getName() + "[" + type.getName() + "]";
}
private static class Holder<T> {
private volatile T value;
private void set(T value) {
this.value = value;
}
private T get() {
return value;
}
}
}
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/SPI.java
+18
View File
@@ -0,0 +1,18 @@
package com.alibaba.otter.canal.connector.core.spi;
import java.lang.annotation.*;
/**
* SPI装载器注解
*
* @author rewerma @ 2018-10-20
* @version 1.0.0
*/
@Documented
@Retention(RetentionPolicy.RUNTIME)
@Target({ ElementType.TYPE })
public @interface SPI {
// Default SPI name
String value() default "";
}
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/URLClassExtensionLoader.java
+88
View File
@@ -0,0 +1,88 @@
package com.alibaba.otter.canal.connector.core.spi;
import java.io.IOException;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.Enumeration;
import java.util.NoSuchElementException;
public class URLClassExtensionLoader extends URLClassLoader {
public URLClassExtensionLoader(URL[] urls) {
super(urls);
}
@Override
public Class<?> loadClass(String name) throws ClassNotFoundException {
Class<?> c = findLoadedClass(name);
if (c != null) {
return c;
}
if (name.startsWith("java.") || name.startsWith("org.slf4j.")
|| name.startsWith("org.apache.logging")
|| name.startsWith("org.apache.commons.logging.")) {
// || name.startsWith("org.apache.hadoop."))
// {
c = super.loadClass(name);
}
if (c != null) return c;
try {
// 先加载jar内的class，可避免jar冲突
c = findClass(name);
} catch (ClassNotFoundException e) {
c = null;
}
if (c != null) {
return c;
}
return super.loadClass(name);
}
@Override
public Enumeration<URL> getResources(String name) throws IOException {
@SuppressWarnings("unchecked")
Enumeration<URL>[] tmp = (Enumeration<URL>[]) new Enumeration<?>[2];
tmp[0] = findResources(name); // local class
// path first
// tmp[1] = super.getResources(name);
return new CompoundEnumeration<>(tmp);
}
private static class CompoundEnumeration<E> implements Enumeration<E> {
private Enumeration<E>[] enums;
private int index = 0;
public CompoundEnumeration(Enumeration<E>[] enums){
this.enums = enums;
}
private boolean next() {
while (this.index < this.enums.length) {
if (this.enums[this.index] != null && this.enums[this.index].hasMoreElements()) {
return true;
}
++this.index;
}
return false;
}
public boolean hasMoreElements() {
return this.next();
}
public E nextElement() {
if (!this.next()) {
throw new NoSuchElementException();
} else {
return this.enums[this.index].nextElement();
}
}
}
}
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/Callback.java
+13
View File
@@ -0,0 +1,13 @@
package com.alibaba.otter.canal.connector.core.util;
/**
* MQ 回调类
*
* @author rewerma 2020-01-27
* @version 1.0.0
*/
public interface Callback {
void commit();
void rollback();
}
server/src/main/java/com/alibaba/otter/canal/common/CanalMessageSerializer.java → connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/CanalMessageSerializerUtil.java
+58 -7
View File
@@ -1,20 +1,25 @@
package com.alibaba.otter.canal.common;
package com.alibaba.otter.canal.connector.core.util;
import java.util.List;
import org.apache.kafka.common.errors.SerializationException;
import com.alibaba.otter.canal.protocol.exception.CanalClientException;
import org.springframework.util.CollectionUtils;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.CanalPacket;
import com.alibaba.otter.canal.protocol.CanalPacket.PacketType;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.server.netty.NettyUtils;
import com.google.protobuf.ByteString;
import com.google.protobuf.CodedOutputStream;
import com.google.protobuf.WireFormat;
public class CanalMessageSerializer {
/**
* Canal message 序列化工具类
*
* @author rewerma 2020-01-27
* @version 1.0.0
*/
public class CanalMessageSerializerUtil {
@SuppressWarnings("deprecation")
public static byte[] serializer(Message data, boolean filterTransactionEntry) {
@@ -58,7 +63,8 @@ public class CanalMessageSerializer {
CanalPacket.Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder();
for (CanalEntry.Entry entry : data.getEntries()) {
if (filterTransactionEntry
&& (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND)) {
&& (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
|| entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND)) {
continue;
}
@@ -67,15 +73,60 @@ public class CanalMessageSerializer {
CanalPacket.Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder();
packetBuilder.setType(PacketType.MESSAGES);
packetBuilder.setVersion(NettyUtils.VERSION);
packetBuilder.setVersion(1);
packetBuilder.setBody(messageBuilder.build().toByteString());
return packetBuilder.build().toByteArray();
}
}
}
} catch (Exception e) {
throw new SerializationException("Error when serializing message to byte[] ");
throw new RuntimeException("Error when serializing message to byte[] ");
}
return null;
}
public static Message deserializer(byte[] data) {
return deserializer(data, false);
}
public static Message deserializer(byte[] data, boolean lazyParseEntry) {
try {
if (data == null) {
return null;
} else {
CanalPacket.Packet p = CanalPacket.Packet.parseFrom(data);
switch (p.getType()) {
case MESSAGES: {
if (!p.getCompression().equals(CanalPacket.Compression.NONE)
&& !p.getCompression().equals(CanalPacket.Compression.COMPRESSIONCOMPATIBLEPROTO2)) {
throw new CanalClientException("compression is not supported in this connector");
}
CanalPacket.Messages messages = CanalPacket.Messages.parseFrom(p.getBody());
Message result = new Message(messages.getBatchId());
if (lazyParseEntry) {
// byteString
result.setRawEntries(messages.getMessagesList());
result.setRaw(true);
} else {
for (ByteString byteString : messages.getMessagesList()) {
result.addEntry(CanalEntry.Entry.parseFrom(byteString));
}
result.setRaw(false);
}
return result;
}
case ACK: {
CanalPacket.Ack ack = CanalPacket.Ack.parseFrom(p.getBody());
throw new CanalClientException("something goes wrong with reason: " + ack.getErrorMessage());
}
default: {
throw new CanalClientException("unexpected packet type: " + p.getType());
}
}
}
} catch (Exception e) {
throw new CanalClientException("deserializer failed", e);
}
}
}
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/DateUtil.java
+46
View File
@@ -0,0 +1,46 @@
package com.alibaba.otter.canal.connector.core.util;
import org.apache.commons.lang.StringUtils;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import java.util.Date;
public class DateUtil {
private static DateTimeZone dateTimeZone;
static {
dateTimeZone = DateTimeZone.forID(TimeZone.LOCATION_TIME_ZONE);
}
/**
* 通用日期时间字符解析
*
* @param datetimeStr 日期时间字符串
* @return Date
*/
public static Date parseDate(String datetimeStr) {
if (StringUtils.isEmpty(datetimeStr)) {
return null;
}
datetimeStr = datetimeStr.trim().replace('/', '-');
if (datetimeStr.contains("-")) {
if (datetimeStr.contains(":")) {
datetimeStr = datetimeStr.replace(" ", "T");
}
} else if (datetimeStr.contains(":")) {
datetimeStr = "T" + datetimeStr;
} else {
if (datetimeStr.length() == 8) {
String year = datetimeStr.substring(0, 4);
String month = datetimeStr.substring(4, 6);
String day = datetimeStr.substring(6, 8);
datetimeStr = year + "-" + month + "-" + day;
}
}
DateTime dateTime = new DateTime(datetimeStr, dateTimeZone);
return dateTime.toDate();
}
}
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/JdbcTypeUtil.java
+165
View File
@@ -0,0 +1,165 @@
package com.alibaba.otter.canal.connector.core.util;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.sql.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* 类型转换工具类
*
* @author rewerma 2018-8-19 下午06:14:23
* @version 1.0.0
*/
public class JdbcTypeUtil {
private static Logger logger = LoggerFactory.getLogger(JdbcTypeUtil.class);
public static Object getRSData(ResultSet rs, String columnName, int jdbcType) throws SQLException {
if (jdbcType == Types.BIT || jdbcType == Types.BOOLEAN) {
return rs.getByte(columnName);
} else {
return rs.getObject(columnName);
}
}
public static Class<?> jdbcType2javaType(int jdbcType) {
switch (jdbcType) {
case Types.BIT:
case Types.BOOLEAN:
// return Boolean.class;
case Types.TINYINT:
return Byte.TYPE;
case Types.SMALLINT:
return Short.class;
case Types.INTEGER:
return Integer.class;
case Types.BIGINT:
return Long.class;
case Types.DECIMAL:
case Types.NUMERIC:
return BigDecimal.class;
case Types.REAL:
return Float.class;
case Types.FLOAT:
case Types.DOUBLE:
return Double.class;
case Types.CHAR:
case Types.VARCHAR:
case Types.LONGVARCHAR:
return String.class;
case Types.BINARY:
case Types.VARBINARY:
case Types.LONGVARBINARY:
case Types.BLOB:
return byte[].class;
case Types.DATE:
return Date.class;
case Types.TIME:
return Time.class;
case Types.TIMESTAMP:
return Timestamp.class;
default:
return String.class;
}
}
private static boolean isText(String columnType) {
return "LONGTEXT".equalsIgnoreCase(columnType) || "MEDIUMTEXT".equalsIgnoreCase(columnType)
|| "TEXT".equalsIgnoreCase(columnType) || "TINYTEXT".equalsIgnoreCase(columnType);
}
public static Object typeConvert(String tableName, String columnName, String value, int sqlType, String mysqlType) {
if (value == null || (value.equals("") && !(isText(mysqlType) || sqlType == Types.CHAR
|| sqlType == Types.VARCHAR || sqlType == Types.LONGVARCHAR))) {
return null;
}
try {
Object res;
switch (sqlType) {
case Types.INTEGER:
res = Integer.parseInt(value);
break;
case Types.SMALLINT:
res = Short.parseShort(value);
break;
case Types.BIT:
case Types.TINYINT:
res = Byte.parseByte(value);
break;
case Types.BIGINT:
if (mysqlType != null && mysqlType.startsWith("bigint") && mysqlType.endsWith("unsigned")) {
res = new BigInteger(value);
} else {
res = Long.parseLong(value);
}
break;
// case Types.BIT:
case Types.BOOLEAN:
res = !"0".equals(value);
break;
case Types.DOUBLE:
case Types.FLOAT:
res = Double.parseDouble(value);
break;
case Types.REAL:
res = Float.parseFloat(value);
break;
case Types.DECIMAL:
case Types.NUMERIC:
res = new BigDecimal(value);
break;
case Types.BINARY:
case Types.VARBINARY:
case Types.LONGVARBINARY:
case Types.BLOB:
res = value.getBytes("ISO-8859-1");
break;
case Types.DATE:
if (!value.startsWith("0000-00-00")) {
java.util.Date date = DateUtil.parseDate(value);
if (date != null) {
res = new Date(date.getTime());
} else {
res = null;
}
} else {
res = null;
}
break;
case Types.TIME: {
java.util.Date date = DateUtil.parseDate(value);
if (date != null) {
res = new Time(date.getTime());
} else {
res = null;
}
break;
}
case Types.TIMESTAMP:
if (!value.startsWith("0000-00-00")) {
java.util.Date date = DateUtil.parseDate(value);
if (date != null) {
res = new Timestamp(date.getTime());
} else {
res = null;
}
} else {
res = null;
}
break;
case Types.CLOB:
default:
res = value;
break;
}
return res;
} catch (Exception e) {
logger.error("table: {} column: {}, failed convert type {} to {}", tableName, columnName, value, sqlType);
return value;
}
}
}
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/MessageUtil.java
+131
View File
@@ -0,0 +1,131 @@
package com.alibaba.otter.canal.connector.core.util;
import java.util.*;
import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.Message;
/**
* Message对象解析工具类
*
* @author rewerma 2018-8-19 下午06:14:23
* @version 1.0.0
*/
public class MessageUtil {
public static List<CommonMessage> convert(Message message) {
if (message == null) {
return null;
}
List<CanalEntry.Entry> entries = message.getEntries();
List<CommonMessage> msgs = new ArrayList<>(entries.size());
for (CanalEntry.Entry entry : entries) {
if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
|| entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
continue;
}
CanalEntry.RowChange rowChange;
try {
rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
} catch (Exception e) {
throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(),
e);
}
CanalEntry.EventType eventType = rowChange.getEventType();
final CommonMessage msg = new CommonMessage();
msg.setIsDdl(rowChange.getIsDdl());
msg.setDatabase(entry.getHeader().getSchemaName());
msg.setTable(entry.getHeader().getTableName());
msg.setType(eventType.toString());
msg.setEs(entry.getHeader().getExecuteTime());
msg.setIsDdl(rowChange.getIsDdl());
msg.setTs(System.currentTimeMillis());
msg.setSql(rowChange.getSql());
msgs.add(msg);
List<Map<String, Object>> data = new ArrayList<>();
List<Map<String, Object>> old = new ArrayList<>();
if (!rowChange.getIsDdl()) {
Set<String> updateSet = new HashSet<>();
msg.setPkNames(new ArrayList<>());
int i = 0;
for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
if (eventType != CanalEntry.EventType.INSERT && eventType != CanalEntry.EventType.UPDATE
&& eventType != CanalEntry.EventType.DELETE) {
continue;
}
Map<String, Object> row = new LinkedHashMap<>();
List<CanalEntry.Column> columns;
if (eventType == CanalEntry.EventType.DELETE) {
columns = rowData.getBeforeColumnsList();
} else {
columns = rowData.getAfterColumnsList();
}
for (CanalEntry.Column column : columns) {
if (i == 0) {
if (column.getIsKey()) {
msg.getPkNames().add(column.getName());
}
}
if (column.getIsNull()) {
row.put(column.getName(), null);
} else {
row.put(column.getName(),
JdbcTypeUtil.typeConvert(msg.getTable(),
column.getName(),
column.getValue(),
column.getSqlType(),
column.getMysqlType()));
}
// 获取update为true的字段
if (column.getUpdated()) {
updateSet.add(column.getName());
}
}
if (!row.isEmpty()) {
data.add(row);
}
if (eventType == CanalEntry.EventType.UPDATE) {
Map<String, Object> rowOld = new LinkedHashMap<>();
for (CanalEntry.Column column : rowData.getBeforeColumnsList()) {
if (updateSet.contains(column.getName())) {
if (column.getIsNull()) {
rowOld.put(column.getName(), null);
} else {
rowOld.put(column.getName(),
JdbcTypeUtil.typeConvert(msg.getTable(),
column.getName(),
column.getValue(),
column.getSqlType(),
column.getMysqlType()));
}
}
}
// update操作将记录修改前的值
if (!rowOld.isEmpty()) {
old.add(rowOld);
}
}
i++;
}
if (!data.isEmpty()) {
msg.setData(data);
}
if (!old.isEmpty()) {
msg.setOld(old);
}
}
}
return msgs;
}
}
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/TimeZone.java
+22
View File
@@ -0,0 +1,22 @@
package com.alibaba.otter.canal.connector.core.util;
public class TimeZone {
public final static String LOCATION_TIME_ZONE; // 当前时区
static {
java.util.TimeZone localTimeZone = java.util.TimeZone.getDefault();
int rawOffset = localTimeZone.getRawOffset();
String symbol = "+";
if (rawOffset < 0) {
symbol = "-";
}
rawOffset = Math.abs(rawOffset);
int offsetHour = rawOffset / 3600000;
int offsetMinute = rawOffset % 3600000 / 60000;
String hour = String.format("%1$02d", offsetHour);
String minute = String.format("%1$02d", offsetMinute);
LOCATION_TIME_ZONE = symbol + hour + ":" + minute;
java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone("GMT" + LOCATION_TIME_ZONE));
}
}
connector/kafka-connector/pom.xml
+94
View File
@@ -0,0 +1,94 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>canal.connector</artifactId>
<groupId>com.alibaba.otter</groupId>
<version>1.1.5-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.kafka</artifactId>
<packaging>jar</packaging>
<name>canal connector kafka module for otter ${project.version}</name>
<dependencies>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.protocol</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.core</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.11</artifactId>
<version>1.1.1</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4</version>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.8</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<tasks>
<copy todir="${project.basedir}/../../deployer/target/canal/plugin" overwrite="true">
<fileset dir="${project.basedir}/target" erroronmissingdir="true">
<include name="connector.kafka-${project.version}-jar-with-dependencies.jar" />
</fileset>
</copy>
<copy todir="${project.basedir}/../../client-adapter/launcher/target/canal-adapter/plugin" overwrite="true">
<fileset dir="${project.basedir}/target" erroronmissingdir="true">
<include name="connector.kafka-${project.version}-jar-with-dependencies.jar" />
</fileset>
</copy>
</tasks>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/config/KafkaConstants.java
+16
View File
@@ -0,0 +1,16 @@
package com.alibaba.otter.canal.connector.kafka.config;
/**
* Kafka 配置常量类
*
* @author rewerma 2020-01-27
* @version 1.0.0
*/
public class KafkaConstants {
public static final String ROOT = "canal";
public static final String CANAL_MQ_KAFKA_KERBEROS_ENABLE = ROOT + "." + "mq.kafka.kerberos.enable";
public static final String CANAL_MQ_KAFKA_KERBEROS_KRB5_FILE = ROOT + "." + "mq.kafka.kerberos.krb5.file";
public static final String CANAL_MQ_KAFKA_KERBEROS_JAAS_FILE = ROOT + "." + "mq.kafka.kerberos.jaas.file";
}
connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/config/KafkaProducerConfig.java
+53
View File
@@ -0,0 +1,53 @@
package com.alibaba.otter.canal.connector.kafka.config;
import java.util.LinkedHashMap;
import java.util.Map;
import com.alibaba.otter.canal.connector.core.config.MQProperties;
/**
* Kafka producer 配置类
*
* @author rewerma 2020-01-27
* @version 1.0.0
*/
public class KafkaProducerConfig extends MQProperties {
private Map<String, Object> kafkaProperties = new LinkedHashMap<>();
private boolean kerberosEnabled = false;
private String krb5File;
private String jaasFile;
public Map<String, Object> getKafkaProperties() {
return kafkaProperties;
}
public void setKafkaProperties(Map<String, Object> kafkaProperties) {
this.kafkaProperties = kafkaProperties;
}
public boolean isKerberosEnabled() {
return kerberosEnabled;
}
public void setKerberosEnabled(boolean kerberosEnabled) {
this.kerberosEnabled = kerberosEnabled;
}
public String getKrb5File() {
return krb5File;
}
public void setKrb5File(String krb5File) {
this.krb5File = krb5File;
}
public String getJaasFile() {
return jaasFile;
}
public void setJaasFile(String jaasFile) {
this.jaasFile = jaasFile;
}
}
connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/consumer/CanalKafkaConsumer.java
+137
View File
@@ -0,0 +1,137 @@
package com.alibaba.otter.canal.connector.kafka.consumer;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import com.alibaba.fastjson.JSON;
import com.alibaba.otter.canal.connector.core.config.CanalConstants;
import com.alibaba.otter.canal.connector.core.spi.SPI;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.serialization.StringDeserializer;
import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
import com.alibaba.otter.canal.connector.core.util.MessageUtil;
import com.alibaba.otter.canal.protocol.Message;
/**
* Kafka consumer SPI 实现
*
* @author rewerma @ 2020-02-01
* @version 1.0.0
*/
@SPI("kafka")
public class CanalKafkaConsumer implements CanalMsgConsumer {
private static final String PREFIX_KAFKA_CONFIG = "kafka.";
private KafkaConsumer<String, ?> kafkaConsumer;
private boolean flatMessage = true;
private String topic;
private Map<Integer, Long> currentOffsets = new ConcurrentHashMap<>();
private Properties kafkaProperties = new Properties();
@Override
public void init(Properties properties, String topic, String groupId) {
this.topic = topic;
Boolean flatMessage = (Boolean) properties.get(CanalConstants.CANAL_MQ_FLAT_MESSAGE);
if (flatMessage != null) {
this.flatMessage = flatMessage;
}
for (Map.Entry<Object, Object> entry : properties.entrySet()) {
String k = (String) entry.getKey();
Object v = entry.getValue();
if (k.startsWith(PREFIX_KAFKA_CONFIG) && v != null) {
kafkaProperties.put(k.substring(PREFIX_KAFKA_CONFIG.length()), v);
}
}
kafkaProperties.put("group.id", groupId);
kafkaProperties.put("key.deserializer", StringDeserializer.class);
kafkaProperties.put("client.id", UUID.randomUUID().toString().substring(0, 6));
}
@Override
public void connect() {
if (this.flatMessage) {
kafkaProperties.put("value.deserializer", StringDeserializer.class);
this.kafkaConsumer = new KafkaConsumer<String, String>(kafkaProperties);
} else {
kafkaProperties.put("value.deserializer", KafkaMessageDeserializer.class);
this.kafkaConsumer = new KafkaConsumer<String, Message>(kafkaProperties);
}
kafkaConsumer.subscribe(Collections.singletonList(topic));
}
@SuppressWarnings("unchecked")
@Override
public List<CommonMessage> getMessage(Long timeout, TimeUnit unit) {
if (!flatMessage) {
ConsumerRecords<String, Message> records = (ConsumerRecords<String, Message>) kafkaConsumer
.poll(unit.toMillis(timeout));
if (!records.isEmpty()) {
currentOffsets.clear();
List<CommonMessage> messages = new ArrayList<>();
for (ConsumerRecord<String, Message> record : records) {
if (currentOffsets.get(record.partition()) == null) {
currentOffsets.put(record.partition(), record.offset());
}
messages.addAll(MessageUtil.convert(record.value()));
}
return messages;
}
} else {
ConsumerRecords<String, String> records = (ConsumerRecords<String, String>) kafkaConsumer
.poll(unit.toMillis(timeout));
if (!records.isEmpty()) {
List<CommonMessage> messages = new ArrayList<>();
currentOffsets.clear();
for (ConsumerRecord<String, String> record : records) {
if (currentOffsets.get(record.partition()) == null) {
currentOffsets.put(record.partition(), record.offset());
}
String flatMessageJson = record.value();
CommonMessage flatMessages = JSON.parseObject(flatMessageJson, CommonMessage.class);
messages.add(flatMessages);
}
return messages;
}
}
return null;
}
@Override
public void rollback() {
// 回滚所有分区
if (kafkaConsumer != null) {
for (Map.Entry<Integer, Long> entry : currentOffsets.entrySet()) {
kafkaConsumer.seek(new TopicPartition(topic, entry.getKey()), currentOffsets.get(entry.getKey()));
kafkaConsumer.commitSync();
}
}
}
@Override
public void ack() {
if (kafkaConsumer != null) {
kafkaConsumer.commitSync();
}
}
@Override
public void disconnect() {
if (kafkaConsumer != null) {
kafkaConsumer.unsubscribe();
}
if (kafkaConsumer != null) {
kafkaConsumer.close();
kafkaConsumer = null;
}
}
}
connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/consumer/KafkaMessageDeserializer.java
+31
View File
@@ -0,0 +1,31 @@
package com.alibaba.otter.canal.connector.kafka.consumer;
import java.util.Map;
import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
import org.apache.kafka.common.serialization.Deserializer;
import com.alibaba.otter.canal.protocol.Message;
/**
* Kafka Message类的反序列化
*
* @author rewerma @ 2018-6-12
* @version 1.0.0
*/
public class KafkaMessageDeserializer implements Deserializer<Message> {
@Override
public void configure(Map<String, ?> configs, boolean isKey) {
}
@Override
public Message deserialize(String topic1, byte[] data) {
return CanalMessageSerializerUtil.deserializer(data);
}
@Override
public void close() {
// nothing to do
}
}
server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java → connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/producer/CanalKafkaProducer.java
+96 -82
View File
@@ -1,14 +1,12 @@
package com.alibaba.otter.canal.kafka;
package com.alibaba.otter.canal.connector.kafka.producer;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.*;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import com.alibaba.otter.canal.connector.kafka.config.KafkaConstants;
import org.apache.commons.lang.StringUtils;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
@@ -19,68 +17,90 @@ import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.alibaba.otter.canal.common.AbstractMQProducer;
import com.alibaba.otter.canal.common.CanalMessageSerializer;
import com.alibaba.otter.canal.common.MQMessageUtils;
import com.alibaba.otter.canal.common.MQMessageUtils.EntryRowData;
import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
import com.alibaba.otter.canal.connector.core.producer.AbstractMQProducer;
import com.alibaba.otter.canal.connector.core.util.Callback;
import com.alibaba.otter.canal.connector.core.producer.MQDestination;
import com.alibaba.otter.canal.connector.core.producer.MQMessageUtils;
import com.alibaba.otter.canal.connector.core.producer.MQMessageUtils.EntryRowData;
import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
import com.alibaba.otter.canal.connector.core.spi.SPI;
import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
import com.alibaba.otter.canal.connector.kafka.config.KafkaProducerConfig;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.spi.CanalMQProducer;
/**
* kafka producer 主操作类
* kafka producer SPI 实现
*
* @author machengyuan 2018-6-11 下午05:30:49
* @author rewerma 2018-6-11 下午05:30:49
* @version 1.0.0
*/
@SuppressWarnings({ "rawtypes", "unchecked" })
@SPI("kafka")
public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQProducer {
private static final Logger logger = LoggerFactory.getLogger(CanalKafkaProducer.class);
private static final Logger logger = LoggerFactory.getLogger(CanalKafkaProducer.class);
private static final String PREFIX_KAFKA_CONFIG = "kafka.";
private Producer<String, byte[]> producer;
private MQProperties kafkaProperties;
@Override
public void init(MQProperties kafkaProperties) {
super.init(kafkaProperties);
public void init(Properties properties) {
KafkaProducerConfig kafkaProducerConfig = new KafkaProducerConfig();
this.mqProperties = kafkaProducerConfig;
super.init(properties);
// load properties
this.loadKafkaProperties(properties);
this.kafkaProperties = kafkaProperties;
Properties properties = new Properties();
properties.put("bootstrap.servers", kafkaProperties.getServers());
properties.put("acks", kafkaProperties.getAcks());
properties.put("compression.type", kafkaProperties.getCompressionType());
properties.put("batch.size", kafkaProperties.getBatchSize());
properties.put("linger.ms", kafkaProperties.getLingerMs());
properties.put("max.request.size", kafkaProperties.getMaxRequestSize());
properties.put("buffer.memory", kafkaProperties.getBufferMemory());
properties.put("key.serializer", StringSerializer.class.getName());
properties.put("max.in.flight.requests.per.connection", 1);
if (!kafkaProperties.getProperties().isEmpty()) {
properties.putAll(kafkaProperties.getProperties());
}
properties.put("retries", kafkaProperties.getRetries());
if (kafkaProperties.isKerberosEnable()) {
File krb5File = new File(kafkaProperties.getKerberosKrb5FilePath());
File jaasFile = new File(kafkaProperties.getKerberosJaasFilePath());
Properties kafkaProperties = new Properties();
kafkaProperties.putAll(kafkaProducerConfig.getKafkaProperties());
kafkaProperties.put("key.serializer", StringSerializer.class);
if (kafkaProducerConfig.isKerberosEnabled()) {
File krb5File = new File(kafkaProducerConfig.getKrb5File());
File jaasFile = new File(kafkaProducerConfig.getJaasFile());
if (krb5File.exists() && jaasFile.exists()) {
// 配置kerberos认证，需要使用绝对路径
System.setProperty("java.security.krb5.conf", krb5File.getAbsolutePath());
System.setProperty("java.security.auth.login.config", jaasFile.getAbsolutePath());
System.setProperty("javax.security.auth.useSubjectCredsOnly", "false");
properties.put("security.protocol", "SASL_PLAINTEXT");
properties.put("sasl.kerberos.service.name", "kafka");
kafkaProperties.put("security.protocol", "SASL_PLAINTEXT");
kafkaProperties.put("sasl.kerberos.service.name", "kafka");
} else {
String errorMsg = "ERROR # The kafka kerberos configuration file does not exist! please check it";
logger.error(errorMsg);
throw new RuntimeException(errorMsg);
}
}
kafkaProperties.put("value.serializer", KafkaMessageSerializer.class);
producer = new KafkaProducer<>(kafkaProperties);
}
properties.put("value.serializer", KafkaMessageSerializer.class.getName());
producer = new KafkaProducer<String, byte[]>(properties);
private void loadKafkaProperties(Properties properties) {
KafkaProducerConfig kafkaProducerConfig = (KafkaProducerConfig) this.mqProperties;
Map<String, Object> kafkaProperties = kafkaProducerConfig.getKafkaProperties();
for (Map.Entry<Object, Object> entry : properties.entrySet()) {
String key = (String) entry.getKey();
Object value = entry.getValue();
if (key.startsWith(PREFIX_KAFKA_CONFIG) && value != null) {
key = key.substring(PREFIX_KAFKA_CONFIG.length());
kafkaProperties.put(key, value);
}
}
String kerberosEnabled = properties.getProperty(KafkaConstants.CANAL_MQ_KAFKA_KERBEROS_ENABLE);
if (!StringUtils.isEmpty(kerberosEnabled)) {
kafkaProducerConfig.setKerberosEnabled(Boolean.parseBoolean(kerberosEnabled));
}
String krb5File = properties.getProperty(KafkaConstants.CANAL_MQ_KAFKA_KERBEROS_KRB5_FILE);
if (!StringUtils.isEmpty(krb5File)) {
kafkaProducerConfig.setKrb5File(krb5File);
}
String jaasFile = properties.getProperty(KafkaConstants.CANAL_MQ_KAFKA_KERBEROS_JAAS_FILE);
if (!StringUtils.isEmpty(jaasFile)) {
kafkaProducerConfig.setJaasFile(jaasFile);
}
}
@Override
@@ -90,41 +110,34 @@ public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQPro
if (producer != null) {
producer.close();
}
super.stop();
} catch (Throwable e) {
logger.warn("##something goes wrong when stopping kafka producer:", e);
} finally {
logger.info("## kafka producer is down.");
}
super.stop();
}
@Override
public void send(MQProperties.CanalDestination canalDestination, Message message, Callback callback) {
public void send(MQDestination mqDestination, Message message, Callback callback) {
ExecutorTemplate template = new ExecutorTemplate(executor);
boolean flat = kafkaProperties.getFlatMessage();
try {
List result = null;
if (!StringUtils.isEmpty(canalDestination.getDynamicTopic())) {
List result;
if (!StringUtils.isEmpty(mqDestination.getDynamicTopic())) {
// 动态topic路由计算,只是基于schema/table,不涉及proto数据反序列化
Map<String, Message> messageMap = MQMessageUtils.messageTopics(message,
canalDestination.getTopic(),
canalDestination.getDynamicTopic());
Map<String, Message> messageMap = MQMessageUtils
.messageTopics(message, mqDestination.getTopic(), mqDestination.getDynamicTopic());
// 针对不同的topic,引入多线程提升效率
for (Map.Entry<String, Message> entry : messageMap.entrySet()) {
final String topicName = entry.getKey().replace('.', '_');
final Message messageSub = entry.getValue();
template.submit(new Callable() {
@Override
public List<Future> call() throws Exception {
try {
return send(canalDestination, topicName, messageSub, flat);
} catch (Exception e) {
throw new RuntimeException(e);
}
template.submit((Callable) () -> {
try {
return send(mqDestination, topicName, messageSub, mqProperties.isFlatMessage());
} catch (Exception e) {
throw new RuntimeException(e);
}
});
}
@@ -132,7 +145,10 @@ public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQPro
result = template.waitForResult();
} else {
result = new ArrayList();
List<Future> futures = send(canalDestination, canalDestination.getTopic(), message, flat);
List<Future> futures = send(mqDestination,
mqDestination.getTopic(),
message,
mqProperties.isFlatMessage());
result.add(futures);
}
@@ -161,36 +177,34 @@ public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQPro
}
}
private List<Future> send(MQProperties.CanalDestination canalDestination, String topicName, Message message,
boolean flat) throws Exception {
List<ProducerRecord<String, byte[]>> records = new ArrayList<ProducerRecord<String, byte[]>>();
private List<Future> send(MQDestination mqDestination, String topicName, Message message, boolean flat) {
List<ProducerRecord<String, byte[]>> records = new ArrayList<>();
if (!flat) {
if (canalDestination.getPartitionHash() != null && !canalDestination.getPartitionHash().isEmpty()) {
if (mqDestination.getPartitionHash() != null && !mqDestination.getPartitionHash().isEmpty()) {
// 并发构造
EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor);
// 串行分区
Message[] messages = MQMessageUtils.messagePartition(datas,
message.getId(),
canalDestination.getPartitionsNum(),
canalDestination.getPartitionHash(),
kafkaProperties.getDatabaseHash());
mqDestination.getPartitionsNum(),
mqDestination.getPartitionHash(),
this.mqProperties.isDatabaseHash());
int length = messages.length;
for (int i = 0; i < length; i++) {
Message messagePartition = messages[i];
if (messagePartition != null) {
records.add(new ProducerRecord<String, byte[]>(topicName,
records.add(new ProducerRecord<>(topicName,
i,
null,
CanalMessageSerializer.serializer(messagePartition,
kafkaProperties.isFilterTransactionEntry())));
CanalMessageSerializerUtil.serializer(messagePartition, true)));
}
}
} else {
final int partition = canalDestination.getPartition() != null ? canalDestination.getPartition() : 0;
records.add(new ProducerRecord<String, byte[]>(topicName,
final int partition = mqDestination.getPartition() != null ? mqDestination.getPartition() : 0;
records.add(new ProducerRecord<>(topicName,
partition,
null,
CanalMessageSerializer.serializer(message, kafkaProperties.isFilterTransactionEntry())));
CanalMessageSerializerUtil.serializer(message, true)));
}
} else {
// 发送扁平数据json
@@ -199,24 +213,24 @@ public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQPro
// 串行分区
List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, message.getId());
for (FlatMessage flatMessage : flatMessages) {
if (canalDestination.getPartitionHash() != null && !canalDestination.getPartitionHash().isEmpty()) {
if (mqDestination.getPartitionHash() != null && !mqDestination.getPartitionHash().isEmpty()) {
FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage,
canalDestination.getPartitionsNum(),
canalDestination.getPartitionHash(),
kafkaProperties.getDatabaseHash());
mqDestination.getPartitionsNum(),
mqDestination.getPartitionHash(),
this.mqProperties.isDatabaseHash());
int length = partitionFlatMessage.length;
for (int i = 0; i < length; i++) {
FlatMessage flatMessagePart = partitionFlatMessage[i];
if (flatMessagePart != null) {
records.add(new ProducerRecord<String, byte[]>(topicName,
records.add(new ProducerRecord<>(topicName,
i,
null,
JSON.toJSONBytes(flatMessagePart, SerializerFeature.WriteMapNullValue)));
}
}
} else {
final int partition = canalDestination.getPartition() != null ? canalDestination.getPartition() : 0;
records.add(new ProducerRecord<String, byte[]>(topicName,
final int partition = mqDestination.getPartition() != null ? mqDestination.getPartition() : 0;
records.add(new ProducerRecord<>(topicName,
partition,
null,
JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue)));
@@ -224,11 +238,11 @@ public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQPro
}
}
return produce(topicName, records, flat);
return produce(records);
}
private List<Future> produce(String topicName, List<ProducerRecord<String, byte[]>> records, boolean flatMessage) {
List<Future> futures = new ArrayList<Future>();
private List<Future> produce(List<ProducerRecord<String, byte[]>> records) {
List<Future> futures = new ArrayList<>();
// 异步发送，因为在partition hash的时候已经按照每个分区合并了消息，走到这一步不需要考虑单个分区内的顺序问题
for (ProducerRecord record : records) {
futures.add(producer.send(record));
server/src/main/java/com/alibaba/otter/canal/kafka/KafkaMessageSerializer.java → connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/producer/KafkaMessageSerializer.java
+4 -4
View File
@@ -1,13 +1,13 @@
package com.alibaba.otter.canal.kafka;
import java.util.Map;
package com.alibaba.otter.canal.connector.kafka.producer;
import org.apache.kafka.common.serialization.Serializer;
import java.util.Map;
/**
* Kafka Message类的序列化
*
* @author machengyuan 2018-6-11 下午05:30:49
* @author rewerma 2018-6-11 下午05:30:49
* @version 1.0.0
*/
public class KafkaMessageSerializer implements Serializer<byte[]> {
connector/kafka-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMQProducer
+1
View File
@@ -0,0 +1 @@
kafka=com.alibaba.otter.canal.connector.kafka.producer.CanalKafkaProducer
connector/kafka-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer
+1
View File
@@ -0,0 +1 @@
kafka=com.alibaba.otter.canal.connector.kafka.consumer.CanalKafkaConsumer
connector/kafka-connector/src/test/java/com/alibaba/otter/canal/connector/kafka/test/CanalKafkaProducerTest.java
+29
View File
@@ -0,0 +1,29 @@
package com.alibaba.otter.canal.connector.kafka.test;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.Properties;
import org.junit.Ignore;
import org.junit.Test;
import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
import com.alibaba.otter.canal.connector.core.spi.ExtensionLoader;
@Ignore
public class CanalKafkaProducerTest {
@Test
public void testLoadKafkaProducer() throws IOException {
Properties pro = new Properties();
FileInputStream in = new FileInputStream("../../deployer/src/main/resources/canal.properties");
pro.load(in);
ExtensionLoader<CanalMQProducer> loader = ExtensionLoader.getExtensionLoader(CanalMQProducer.class);
CanalMQProducer canalMQProducer = loader
.getExtension("kafka", "/../../deployer/target/canal/plugin", "/../../deployer/target/canal/plugin");
canalMQProducer.init(pro);
in.close();
}
}
connector/pom.xml
+131
View File
@@ -0,0 +1,131 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.1.5-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.connector</artifactId>
<version>1.1.5-SNAPSHOT</version>
<packaging>pom</packaging>
<name>canal connector module for otter ${project.version}</name>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.test.skip>true</maven.test.skip>
<downloadSources>true</downloadSources>
<java_source_version>1.8</java_source_version>
<java_target_version>1.8</java_target_version>
<file_encoding>UTF-8</file_encoding>
</properties>
<licenses>
<license>
<name>Apache License, Version 2.0</name>
<url>http://www.apache.org/licenses/LICENSE-2.0</url>
</license>
</licenses>
<scm>
<url>git@github.com:alibaba/canal.git</url>
<connection>scm:git:git@github.com:alibaba/canal.git</connection>
<developerConnection>scm:git:git@github.com:alibaba/canal.git</developerConnection>
</scm>
<repositories>
<repository>
<id>central</id>
<url>http://repo1.maven.org/maven2</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>java.net</id>
<url>http://download.java.net/maven/2/</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>aliyun</id>
<url>http://maven.aliyun.com/nexus/content/groups/public/</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>sonatype</id>
<name>sonatype</name>
<url>https://oss.sonatype.org/content/repositories/snapshots</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
<repository>
<id>sonatype-release</id>
<name>sonatype-release</name>
<url>https://oss.sonatype.org/service/local/repositories/releases/content</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
</repositories>
<modules>
<module>kafka-connector</module>
<module>core</module>
<module>rocketmq-connector</module>
<module>rabbitmq-connector</module>
<module>tcp-connector</module>
</modules>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.7.0</version>
<configuration>
<source>${java_source_version}</source>
<target>${java_target_version}</target>
<encoding>${file_encoding}</encoding>
</configuration>
</plugin>
</plugins>
</build>
<distributionManagement>
<snapshotRepository>
<id>sonatype-nexus-snapshots</id>
<name>Sonatype Nexus Snapshots</name>
<url>https://oss.sonatype.org/content/repositories/snapshots/</url>
</snapshotRepository>
<repository>
<id>sonatype-nexus-staging</id>
<name>Nexus Release Repository</name>
<url>https://oss.sonatype.org/service/local/staging/deploy/maven2/</url>
</repository>
</distributionManagement>
</project>
connector/rabbitmq-connector/pom.xml
+99
View File
@@ -0,0 +1,99 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>canal.connector</artifactId>
<groupId>com.alibaba.otter</groupId>
<version>1.1.5-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.rabbitmq</artifactId>
<packaging>jar</packaging>
<name>canal connector rabbitmq module for otter ${project.version}</name>
<dependencies>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.protocol</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.core</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.rabbitmq</groupId>
<artifactId>amqp-client</artifactId>
<version>5.5.0</version>
</dependency>
<dependency>
<groupId>com.alibaba.mq-amqp</groupId>
<artifactId>mq-amqp-client</artifactId>
<version>1.0.3</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4</version>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.8</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<tasks>
<copy todir="${project.basedir}/../../deployer/target/canal/plugin" overwrite="true">
<fileset dir="${project.basedir}/target" erroronmissingdir="true">
<include name="connector.rabbitmq-${project.version}-jar-with-dependencies.jar" />
</fileset>
</copy>
<copy todir="${project.basedir}/../../client-adapter/launcher/target/canal-adapter/plugin" overwrite="true">
<fileset dir="${project.basedir}/target" erroronmissingdir="true">
<include name="connector.rabbitmq-${project.version}-jar-with-dependencies.jar" />
</fileset>
</copy>
</tasks>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/config/RabbitMQConstants.java
+20
View File
@@ -0,0 +1,20 @@
package com.alibaba.otter.canal.connector.rabbitmq.config;
/**
* RabbitMQ 配置常量类
*
* @author rewerma 2020-01-27
* @version 1.0.0
*/
public class RabbitMQConstants {
public static final String ROOT = "rabbitmq";
public static final String RABBITMQ_HOST = ROOT + "." + "host";
public static final String RABBITMQ_EXCHANGE = ROOT + "." + "exchange";
public static final String RABBITMQ_VIRTUAL_HOST = ROOT + "." + "virtual.host";
public static final String RABBITMQ_USERNAME = ROOT + "." + "username";
public static final String RABBITMQ_PASSWORD = ROOT + "." + "password";
public static final String RABBITMQ_RESOURCE_OWNERID = ROOT + "." + "rabbitmq.resource.ownerId";
}
connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/config/RabbitMQProducerConfig.java
+58
View File
@@ -0,0 +1,58 @@
package com.alibaba.otter.canal.connector.rabbitmq.config;
import com.alibaba.otter.canal.connector.core.config.MQProperties;
/**
* RabbitMQ 配置类
*
* @author rewerma 2020-01-27
* @version 1.0.0
*/
public class RabbitMQProducerConfig extends MQProperties {
private String host;
private String virtualHost;
private String exchange;
private String username;
private String password;
public String getHost() {
return host;
}
public void setHost(String host) {
this.host = host;
}
public String getVirtualHost() {
return virtualHost;
}
public void setVirtualHost(String virtualHost) {
this.virtualHost = virtualHost;
}
public String getExchange() {
return exchange;
}
public void setExchange(String exchange) {
this.exchange = exchange;
}
public String getUsername() {
return username;
}
public void setUsername(String username) {
this.username = username;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
}
connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/consumer/CanalRabbitMQConsumer.java
+214
View File
@@ -0,0 +1,214 @@
package com.alibaba.otter.canal.connector.rabbitmq.consumer;
import java.io.IOException;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import com.alibaba.otter.canal.connector.core.spi.SPI;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSON;
import com.alibaba.otter.canal.connector.core.config.CanalConstants;
import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
import com.alibaba.otter.canal.connector.core.util.MessageUtil;
import com.alibaba.otter.canal.connector.rabbitmq.config.RabbitMQConstants;
import com.alibaba.otter.canal.connector.rabbitmq.producer.AliyunCredentialsProvider;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.protocol.exception.CanalClientException;
import com.google.common.collect.Lists;
import com.rabbitmq.client.*;
/**
* RabbitMQ consumer SPI 实现
*
* @author rewerma 2020-02-01
* @version 1.0.0
*/
@SPI("rabbitmq")
public class CanalRabbitMQConsumer implements CanalMsgConsumer {
private static final Logger logger = LoggerFactory
.getLogger(CanalRabbitMQConsumer.class);
// 链接地址
private String nameServer;
// 主机名
private String vhost;
private String queueName;
// 一些鉴权信息
private String accessKey;
private String secretKey;
private Long resourceOwnerId;
private String username;
private String password;
private boolean flatMessage;
private Connection connect;
private Channel channel;
private long batchProcessTimeout = 60 * 1000;
private BlockingQueue<ConsumerBatchMessage<CommonMessage>> messageBlockingQueue;
private volatile ConsumerBatchMessage<CommonMessage> lastGetBatchMessage = null;
@Override
public void init(Properties properties, String topic, String groupId) {
this.nameServer = properties.getProperty("rabbitmq.host");
this.vhost = properties.getProperty("rabbitmq.virtual.host");
this.queueName = topic;
this.accessKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_ACCESS_KEY);
this.secretKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_SECRET_KEY);
this.username = properties.getProperty(RabbitMQConstants.RABBITMQ_USERNAME);
this.password = properties.getProperty(RabbitMQConstants.RABBITMQ_PASSWORD);
Long resourceOwnerIdPro = (Long) properties.get(RabbitMQConstants.RABBITMQ_RESOURCE_OWNERID);
if (resourceOwnerIdPro != null) {
this.resourceOwnerId = resourceOwnerIdPro;
}
this.flatMessage = (Boolean) properties.get(CanalConstants.CANAL_MQ_FLAT_MESSAGE);
this.messageBlockingQueue = new LinkedBlockingQueue<>(1024);
}
@Override
public void connect() {
ConnectionFactory factory = new ConnectionFactory();
if (accessKey.length() > 0 && secretKey.length() > 0) {
factory.setCredentialsProvider(new AliyunCredentialsProvider(accessKey, secretKey, resourceOwnerId));
} else {
factory.setUsername(username);
factory.setPassword(password);
}
factory.setHost(nameServer);
factory.setAutomaticRecoveryEnabled(true);
factory.setNetworkRecoveryInterval(5000);
factory.setVirtualHost(vhost);
try {
connect = factory.newConnection();
channel = connect.createChannel();
} catch (IOException | TimeoutException e) {
throw new CanalClientException("Start RabbitMQ producer error", e);
}
// 不存在连接 则重新连接
if (connect == null) {
this.connect();
}
Consumer consumer = new DefaultConsumer(channel) {
@Override
public void handleDelivery(String consumerTag, Envelope envelope, AMQP.BasicProperties properties,
byte[] body) throws IOException {
if (body != null) {
channel.basicAck(envelope.getDeliveryTag(), process(body));
}
}
};
try {
channel.basicConsume(queueName, false, consumer);
} catch (IOException e) {
throw new CanalClientException("error", e);
}
}
private boolean process(byte[] messageData) {
if (logger.isDebugEnabled()) {
logger.debug("Get Message: {}", new String(messageData));
}
List<CommonMessage> messageList = Lists.newArrayList();
if (!flatMessage) {
Message message = CanalMessageSerializerUtil.deserializer(messageData);
messageList.addAll(MessageUtil.convert(message));
} else {
CommonMessage commonMessage = JSON.parseObject(messageData, CommonMessage.class);
messageList.add(commonMessage);
}
ConsumerBatchMessage<CommonMessage> batchMessage = new ConsumerBatchMessage<>(messageList);
try {
messageBlockingQueue.put(batchMessage);
} catch (InterruptedException e) {
logger.error("Put message to queue error", e);
throw new RuntimeException(e);
}
boolean isCompleted;
try {
isCompleted = batchMessage.waitFinish(batchProcessTimeout);
} catch (InterruptedException e) {
logger.error("Interrupted when waiting messages to be finished.", e);
throw new RuntimeException(e);
}
boolean isSuccess = batchMessage.isSuccess();
return isCompleted && isSuccess;
}
@Override
public List<CommonMessage> getMessage(Long timeout, TimeUnit unit) {
try {
if (this.lastGetBatchMessage != null) {
throw new CanalClientException("mq get/ack not support concurrent & async ack");
}
ConsumerBatchMessage<CommonMessage> batchMessage = messageBlockingQueue.poll(timeout, unit);
if (batchMessage != null) {
this.lastGetBatchMessage = batchMessage;
return batchMessage.getData();
}
} catch (InterruptedException ex) {
logger.warn("Get message timeout", ex);
throw new CanalClientException("Failed to fetch the data after: " + timeout);
}
return null;
}
@Override
public void rollback() {
try {
if (this.lastGetBatchMessage != null) {
this.lastGetBatchMessage.fail();
}
} finally {
this.lastGetBatchMessage = null;
}
}
@Override
public void ack() {
try {
if (this.lastGetBatchMessage != null) {
this.lastGetBatchMessage.ack();
}
} catch (Throwable e) {
if (this.lastGetBatchMessage != null) {
this.lastGetBatchMessage.fail();
}
} finally {
this.lastGetBatchMessage = null;
}
}
@Override
public void disconnect() {
if (connect != null) {
try {
connect.close();
} catch (IOException e) {
throw new CanalClientException("stop connect error", e);
}
}
if (channel != null) {
try {
channel.close();
} catch (IOException | TimeoutException e) {
throw new CanalClientException("stop channel error", e);
}
}
}
}
connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/consumer/ConsumerBatchMessage.java
+48
View File
@@ -0,0 +1,48 @@
package com.alibaba.otter.canal.connector.rabbitmq.consumer;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
public class ConsumerBatchMessage<T> {
private final List<T> data;
private CountDownLatch latch;
private boolean hasFailure = false;
public ConsumerBatchMessage(List<T> data){
this.data = data;
latch = new CountDownLatch(1);
}
public boolean waitFinish(long timeout) throws InterruptedException {
return latch.await(timeout, TimeUnit.MILLISECONDS);
}
public boolean isSuccess() {
return !hasFailure;
}
public List<T> getData() {
return data;
}
/**
* Countdown if the sub message is successful.
*/
public void ack() {
latch.countDown();
}
/**
* Countdown and fail-fast if the sub message is failed.
*/
public void fail() {
hasFailure = true;
// fail fast
long count = latch.getCount();
for (int i = 0; i < count; i++) {
latch.countDown();
}
}
}
server/src/main/java/com/alibaba/otter/canal/rabbitmq/AliyunCredentialsProvider.java → connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/producer/AliyunCredentialsProvider.java
+7 -5
View File
@@ -1,8 +1,4 @@
/**
* aliyun amqp协议 账号类
* 暂不支持STS授权情况
*/
package com.alibaba.otter.canal.rabbitmq;
package com.alibaba.otter.canal.connector.rabbitmq.producer;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
@@ -10,6 +6,12 @@ import java.security.NoSuchAlgorithmException;
import com.alibaba.mq.amqp.utils.UserUtils;
import com.rabbitmq.client.impl.CredentialsProvider;
/**
* aliyun amqp协议 账号类
* 暂不支持STS授权情况
*
* @version 1.0.0
*/
public class AliyunCredentialsProvider implements CredentialsProvider {
/**
connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/producer/CanalRabbitMQProducer.java
+180
View File
@@ -0,0 +1,180 @@
package com.alibaba.otter.canal.connector.rabbitmq.producer;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.TimeoutException;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.alibaba.otter.canal.common.CanalException;
import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
import com.alibaba.otter.canal.connector.core.producer.AbstractMQProducer;
import com.alibaba.otter.canal.connector.core.util.Callback;
import com.alibaba.otter.canal.connector.core.producer.MQDestination;
import com.alibaba.otter.canal.connector.core.producer.MQMessageUtils;
import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
import com.alibaba.otter.canal.connector.core.spi.SPI;
import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
import com.alibaba.otter.canal.connector.rabbitmq.config.RabbitMQConstants;
import com.alibaba.otter.canal.connector.rabbitmq.config.RabbitMQProducerConfig;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message;
import com.rabbitmq.client.Channel;
import com.rabbitmq.client.Connection;
import com.rabbitmq.client.ConnectionFactory;
/**
* RabbitMQ Producer SPI 实现
*
* @author rewerma 2020-01-27
* @version 1.0.0
*/
@SPI("rabbitmq")
public class CanalRabbitMQProducer extends AbstractMQProducer implements CanalMQProducer {
private static final Logger logger = LoggerFactory.getLogger(CanalRabbitMQProducer.class);
private Connection connect;
private Channel channel;
@Override
public void init(Properties properties) {
RabbitMQProducerConfig rabbitMQProperties = new RabbitMQProducerConfig();
this.mqProperties = rabbitMQProperties;
super.init(properties);
loadRabbitMQProperties(properties);
ConnectionFactory factory = new ConnectionFactory();
factory.setHost(rabbitMQProperties.getHost());
if (mqProperties.getAliyunAccessKey().length() > 0 && mqProperties.getAliyunSecretKey().length() > 0
&& mqProperties.getAliyunUid() > 0) {
factory.setCredentialsProvider(new AliyunCredentialsProvider(mqProperties.getAliyunAccessKey(),
mqProperties.getAliyunSecretKey(),
mqProperties.getAliyunUid()));
} else {
factory.setUsername(rabbitMQProperties.getUsername());
factory.setPassword(rabbitMQProperties.getPassword());
}
factory.setVirtualHost(rabbitMQProperties.getVirtualHost());
try {
connect = factory.newConnection();
channel = connect.createChannel();
// channel.exchangeDeclare(mqProperties.getExchange(), "topic");
} catch (IOException | TimeoutException ex) {
throw new CanalException("Start RabbitMQ producer error", ex);
}
}
private void loadRabbitMQProperties(Properties properties) {
RabbitMQProducerConfig rabbitMQProperties = (RabbitMQProducerConfig) this.mqProperties;
String host = properties.getProperty(RabbitMQConstants.RABBITMQ_HOST);
if (!StringUtils.isEmpty(host)) {
rabbitMQProperties.setHost(host);
}
String vhost = properties.getProperty(RabbitMQConstants.RABBITMQ_VIRTUAL_HOST);
if (!StringUtils.isEmpty(vhost)) {
rabbitMQProperties.setVirtualHost(vhost);
}
String exchange = properties.getProperty(RabbitMQConstants.RABBITMQ_EXCHANGE);
if (!StringUtils.isEmpty(exchange)) {
rabbitMQProperties.setExchange(exchange);
}
String username = properties.getProperty(RabbitMQConstants.RABBITMQ_USERNAME);
if (!StringUtils.isEmpty(username)) {
rabbitMQProperties.setUsername(username);
}
String password = properties.getProperty(RabbitMQConstants.RABBITMQ_PASSWORD);
if (!StringUtils.isEmpty(password)) {
rabbitMQProperties.setPassword(password);
}
}
@Override
public void send(final MQDestination destination, Message message, Callback callback) {
ExecutorTemplate template = new ExecutorTemplate(executor);
try {
if (!StringUtils.isEmpty(destination.getDynamicTopic())) {
// 动态topic
Map<String, Message> messageMap = MQMessageUtils
.messageTopics(message, destination.getTopic(), destination.getDynamicTopic());
for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) {
final String topicName = entry.getKey().replace('.', '_');
final com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue();
template.submit(new Runnable() {
@Override
public void run() {
send(destination, topicName, messageSub);
}
});
}
template.waitForResult();
} else {
send(destination, destination.getTopic(), message);
}
callback.commit();
} catch (Throwable e) {
logger.error(e.getMessage(), e);
callback.rollback();
} finally {
template.clear();
}
}
private void send(MQDestination canalDestination, String topicName, Message messageSub) {
if (!mqProperties.isFlatMessage()) {
byte[] message = CanalMessageSerializerUtil.serializer(messageSub, mqProperties.isFilterTransactionEntry());
if (logger.isDebugEnabled()) {
logger.debug("send message:{} to destination:{}", message, canalDestination.getCanalDestination());
}
sendMessage(topicName, message);
} else {
// 并发构造
MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(messageSub, executor);
// 串行分区
List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, messageSub.getId());
for (FlatMessage flatMessage : flatMessages) {
byte[] message = JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue);
if (logger.isDebugEnabled()) {
logger.debug("send message:{} to destination:{}", message, canalDestination.getCanalDestination());
}
sendMessage(topicName, message);
}
}
}
private void sendMessage(String queueName, byte[] message) {
// tips: 目前逻辑中暂不处理对exchange处理，请在Console后台绑定 才可使用routekey
try {
RabbitMQProducerConfig rabbitMQProperties = (RabbitMQProducerConfig) this.mqProperties;
channel.basicPublish(rabbitMQProperties.getExchange(), queueName, null, message);
} catch (Throwable e) {
throw new RuntimeException(e);
}
}
@Override
public void stop() {
logger.info("## Stop RabbitMQ producer##");
try {
this.connect.close();
this.channel.close();
super.stop();
} catch (IOException | TimeoutException ex) {
throw new CanalException("Stop RabbitMQ producer error", ex);
}
super.stop();
}
}
connector/rabbitmq-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMQProducer
+1
View File
@@ -0,0 +1 @@
rabbitmq=com.alibaba.otter.canal.connector.rabbitmq.producer.CanalRabbitMQProducer
connector/rabbitmq-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer
+1
View File
@@ -0,0 +1 @@
rabbitmq=com.alibaba.otter.canal.connector.rabbitmq.consumer.CanalRabbitMQConsumer
connector/rocketmq-connector/pom.xml
+104
View File
@@ -0,0 +1,104 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>canal.connector</artifactId>
<groupId>com.alibaba.otter</groupId>
<version>1.1.5-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.rocketmq</artifactId>
<packaging>jar</packaging>
<name>canal connector rocketMQ module for otter ${project.version}</name>
<properties>
<rocketmq_version>4.5.2</rocketmq_version>
</properties>
<dependencies>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.protocol</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.core</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-client</artifactId>
<version>${rocketmq_version}</version>
</dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-acl</artifactId>
<version>${rocketmq_version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4</version>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.8</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<tasks>
<copy todir="${project.basedir}/../../deployer/target/canal/plugin" overwrite="true">
<fileset dir="${project.basedir}/target" erroronmissingdir="true">
<include name="connector.rocketmq-${project.version}-jar-with-dependencies.jar"/>
</fileset>
</copy>
<copy todir="${project.basedir}/../../client-adapter/launcher/target/canal-adapter/plugin" overwrite="true">
<fileset dir="${project.basedir}/target" erroronmissingdir="true">
<include name="connector.rocketmq-${project.version}-jar-with-dependencies.jar" />
</fileset>
</copy>
</tasks>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/config/RocketMQConstants.java
+25
View File
@@ -0,0 +1,25 @@
package com.alibaba.otter.canal.connector.rocketmq.config;
/**
* RocketMQ 配置常量类
*
* @author rewerma 2020-01-27
* @version 1.0.0
*/
public class RocketMQConstants {
public static final String ROOT = "rocketmq";
public static final String ROCKETMQ_PRODUCER_GROUP = ROOT + "." + "producer.group";
public static final String ROCKETMQ_ENABLE_MESSAGE_TRACE = ROOT + "." + "enable.message.trace";
public static final String ROCKETMQ_CUSTOMIZED_TRACE_TOPIC = ROOT + "." + "customized.trace.topic";
public static final String ROCKETMQ_NAMESPACE = ROOT + "." + "namespace";
public static final String ROCKETMQ_NAMESRV_ADDR = ROOT + "." + "namesrv.addr";
public static final String ROCKETMQ_RETRY_TIMES_WHEN_SEND_FAILED = ROOT + "." + "retry.times.when.send.failed";
public static final String ROCKETMQ_VIP_CHANNEL_ENABLED = ROOT + "." + "vip.channel.enabled";
public static final String ROCKETMQ_ACCESS_CHANNEL = ROOT + "." + "access.channel";
public static final String ROCKETMQ_BATCH_SIZE = ROOT + "." + "batch.size";
public static final String ROCKETMQ_SUBSCRIBE_FILTER = ROOT + "." + "subscribe.filter";
}
connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/config/RocketMQProducerConfig.java
+76
View File
@@ -0,0 +1,76 @@
package com.alibaba.otter.canal.connector.rocketmq.config;
import com.alibaba.otter.canal.connector.core.config.MQProperties;
/**
* RocketMQ 配置类
*
* @author rewerma 2020-01-27
* @version 1.0.0
*/
public class RocketMQProducerConfig extends MQProperties {
private String producerGroup;
private boolean enableMessageTrace = false;
private String customizedTraceTopic;
private String namespace;
private String namesrvAddr;
private int retryTimesWhenSendFailed = 0;
private boolean vipChannelEnabled = false;
public String getProducerGroup() {
return producerGroup;
}
public void setProducerGroup(String producerGroup) {
this.producerGroup = producerGroup;
}
public boolean isEnableMessageTrace() {
return enableMessageTrace;
}
public void setEnableMessageTrace(boolean enableMessageTrace) {
this.enableMessageTrace = enableMessageTrace;
}
public String getCustomizedTraceTopic() {
return customizedTraceTopic;
}
public void setCustomizedTraceTopic(String customizedTraceTopic) {
this.customizedTraceTopic = customizedTraceTopic;
}
public String getNamespace() {
return namespace;
}
public void setNamespace(String namespace) {
this.namespace = namespace;
}
public String getNamesrvAddr() {
return namesrvAddr;
}
public void setNamesrvAddr(String namesrvAddr) {
this.namesrvAddr = namesrvAddr;
}
public int getRetryTimesWhenSendFailed() {
return retryTimesWhenSendFailed;
}
public void setRetryTimesWhenSendFailed(int retryTimesWhenSendFailed) {
this.retryTimesWhenSendFailed = retryTimesWhenSendFailed;
}
public boolean isVipChannelEnabled() {
return vipChannelEnabled;
}
public void setVipChannelEnabled(boolean vipChannelEnabled) {
this.vipChannelEnabled = vipChannelEnabled;
}
}
connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/consumer/CanalRocketMQConsumer.java
+237
View File
@@ -0,0 +1,237 @@
package com.alibaba.otter.canal.connector.rocketmq.consumer;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.StringUtils;
import org.apache.rocketmq.acl.common.AclClientRPCHook;
import org.apache.rocketmq.acl.common.SessionCredentials;
import org.apache.rocketmq.client.AccessChannel;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus;
import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly;
import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.remoting.RPCHook;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSON;
import com.alibaba.otter.canal.connector.core.config.CanalConstants;
import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
import com.alibaba.otter.canal.connector.core.spi.SPI;
import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
import com.alibaba.otter.canal.connector.core.util.MessageUtil;
import com.alibaba.otter.canal.connector.rocketmq.config.RocketMQConstants;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.protocol.exception.CanalClientException;
import com.google.common.collect.Lists;
/**
* RocketMQ consumer SPI 实现
*
* @author rewerma 2020-02-01
* @version 1.0.0
*/
@SPI("rocketmq")
public class CanalRocketMQConsumer implements CanalMsgConsumer {
private static final Logger logger = LoggerFactory
.getLogger(CanalRocketMQConsumer.class);
private static final String CLOUD_ACCESS_CHANNEL = "cloud";
private String nameServer;
private String topic;
private String groupName;
private volatile boolean connected = false;
private DefaultMQPushConsumer rocketMQConsumer;
private BlockingQueue<ConsumerBatchMessage<CommonMessage>> messageBlockingQueue;
private int batchSize = -1;
private long batchProcessTimeout = 60 * 1000;
private boolean flatMessage;
private volatile ConsumerBatchMessage<CommonMessage> lastGetBatchMessage = null;
private String accessKey;
private String secretKey;
private String customizedTraceTopic;
private boolean enableMessageTrace = false;
private String accessChannel;
private String namespace;
private String filter = "*";
@Override
public void init(Properties properties, String topic, String groupName) {
this.topic = topic;
this.groupName = groupName;
this.flatMessage = (Boolean) properties.get(CanalConstants.CANAL_MQ_FLAT_MESSAGE);
this.messageBlockingQueue = new LinkedBlockingQueue<>(1024);
this.accessKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_ACCESS_KEY);
this.secretKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_SECRET_KEY);
String enableMessageTrace = properties.getProperty(RocketMQConstants.ROCKETMQ_ENABLE_MESSAGE_TRACE);
if (StringUtils.isNotEmpty(enableMessageTrace)) {
this.enableMessageTrace = Boolean.parseBoolean(enableMessageTrace);
}
this.customizedTraceTopic = properties.getProperty(RocketMQConstants.ROCKETMQ_CUSTOMIZED_TRACE_TOPIC);
this.accessChannel = properties.getProperty(RocketMQConstants.ROCKETMQ_ACCESS_CHANNEL);
this.namespace = properties.getProperty(RocketMQConstants.ROCKETMQ_NAMESPACE);
this.nameServer = properties.getProperty(RocketMQConstants.ROCKETMQ_NAMESRV_ADDR);
String batchSize = properties.getProperty(RocketMQConstants.ROCKETMQ_BATCH_SIZE);
if (StringUtils.isNotEmpty(batchSize)) {
this.batchSize = Integer.parseInt(batchSize);
}
String subscribeFilter = properties.getProperty(RocketMQConstants.ROCKETMQ_SUBSCRIBE_FILTER);
if (StringUtils.isNotEmpty(subscribeFilter)) {
this.filter = subscribeFilter;
}
}
@Override
public void connect() {
RPCHook rpcHook = null;
if (null != accessKey && accessKey.length() > 0 && null != secretKey && secretKey.length() > 0) {
SessionCredentials sessionCredentials = new SessionCredentials();
sessionCredentials.setAccessKey(accessKey);
sessionCredentials.setSecretKey(secretKey);
rpcHook = new AclClientRPCHook(sessionCredentials);
}
rocketMQConsumer = new DefaultMQPushConsumer(groupName,
rpcHook,
new AllocateMessageQueueAveragely(),
enableMessageTrace,
customizedTraceTopic);
rocketMQConsumer.setVipChannelEnabled(false);
if (CLOUD_ACCESS_CHANNEL.equals(this.accessChannel)) {
rocketMQConsumer.setAccessChannel(AccessChannel.CLOUD);
}
if (!StringUtils.isEmpty(this.namespace)) {
rocketMQConsumer.setNamespace(this.namespace);
}
if (!StringUtils.isBlank(nameServer)) {
rocketMQConsumer.setNamesrvAddr(nameServer);
}
if (batchSize != -1) {
rocketMQConsumer.setConsumeMessageBatchMaxSize(batchSize);
}
try {
if (rocketMQConsumer == null) {
this.connect();
}
rocketMQConsumer.subscribe(this.topic, this.filter);
rocketMQConsumer.registerMessageListener((MessageListenerOrderly) (messageExts, context) -> {
context.setAutoCommit(true);
boolean isSuccess = process(messageExts);
if (isSuccess) {
return ConsumeOrderlyStatus.SUCCESS;
} else {
return ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT;
}
});
rocketMQConsumer.start();
} catch (MQClientException ex) {
connected = false;
logger.error("Start RocketMQ consumer error", ex);
}
}
private boolean process(List<MessageExt> messageExts) {
if (logger.isDebugEnabled()) {
logger.debug("Get Message: {}", messageExts);
}
List<CommonMessage> messageList = Lists.newArrayList();
for (MessageExt messageExt : messageExts) {
byte[] data = messageExt.getBody();
if (data != null) {
try {
if (!flatMessage) {
Message message = CanalMessageSerializerUtil.deserializer(data);
messageList.addAll(MessageUtil.convert(message));
} else {
CommonMessage commonMessage = JSON.parseObject(data, CommonMessage.class);
messageList.add(commonMessage);
}
} catch (Exception ex) {
logger.error("Add message error", ex);
throw new CanalClientException(ex);
}
} else {
logger.warn("Received message data is null");
}
}
ConsumerBatchMessage<CommonMessage> batchMessage = new ConsumerBatchMessage<>(messageList);
try {
messageBlockingQueue.put(batchMessage);
} catch (InterruptedException e) {
logger.error("Put message to queue error", e);
throw new RuntimeException(e);
}
boolean isCompleted;
try {
isCompleted = batchMessage.waitFinish(batchProcessTimeout);
} catch (InterruptedException e) {
logger.error("Interrupted when waiting messages to be finished.", e);
throw new RuntimeException(e);
}
boolean isSuccess = batchMessage.isSuccess();
return isCompleted && isSuccess;
}
@Override
public List<CommonMessage> getMessage(Long timeout, TimeUnit unit) {
try {
if (this.lastGetBatchMessage != null) {
throw new CanalClientException("mq get/ack not support concurrent & async ack");
}
ConsumerBatchMessage<CommonMessage> batchMessage = messageBlockingQueue.poll(timeout, unit);
if (batchMessage != null) {
this.lastGetBatchMessage = batchMessage;
return batchMessage.getData();
}
} catch (InterruptedException ex) {
logger.warn("Get message timeout", ex);
throw new CanalClientException("Failed to fetch the data after: " + timeout);
}
return null;
}
@Override
public void rollback() {
try {
if (this.lastGetBatchMessage != null) {
this.lastGetBatchMessage.fail();
}
} finally {
this.lastGetBatchMessage = null;
}
}
@Override
public void ack() {
try {
if (this.lastGetBatchMessage != null) {
this.lastGetBatchMessage.ack();
}
} catch (Throwable e) {
if (this.lastGetBatchMessage != null) {
this.lastGetBatchMessage.fail();
}
} finally {
this.lastGetBatchMessage = null;
}
}
@Override
public void disconnect() {
rocketMQConsumer.unsubscribe(topic);
rocketMQConsumer.shutdown();
connected = false;
}
}
connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/consumer/ConsumerBatchMessage.java
+48
View File
@@ -0,0 +1,48 @@
package com.alibaba.otter.canal.connector.rocketmq.consumer;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
public class ConsumerBatchMessage<T> {
private final List<T> data;
private CountDownLatch latch;
private boolean hasFailure = false;
public ConsumerBatchMessage(List<T> data){
this.data = data;
latch = new CountDownLatch(1);
}
public boolean waitFinish(long timeout) throws InterruptedException {
return latch.await(timeout, TimeUnit.MILLISECONDS);
}
public boolean isSuccess() {
return !hasFailure;
}
public List<T> getData() {
return data;
}
/**
* Countdown if the sub message is successful.
*/
public void ack() {
latch.countDown();
}
/**
* Countdown and fail-fast if the sub message is failed.
*/
public void fail() {
hasFailure = true;
// fail fast
long count = latch.getCount();
for (int i = 0; i < count; i++) {
latch.countDown();
}
}
}
connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/producer/CanalRocketMQProducer.java
+312
View File
@@ -0,0 +1,312 @@
package com.alibaba.otter.canal.connector.rocketmq.producer;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.stream.Collectors;
import org.apache.commons.lang.StringUtils;
import org.apache.rocketmq.acl.common.AclClientRPCHook;
import org.apache.rocketmq.acl.common.SessionCredentials;
import org.apache.rocketmq.client.AccessChannel;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;
import org.apache.rocketmq.client.impl.producer.TopicPublishInfo;
import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.client.producer.SendResult;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.remoting.RPCHook;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.alibaba.otter.canal.common.CanalException;
import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
import com.alibaba.otter.canal.connector.core.producer.AbstractMQProducer;
import com.alibaba.otter.canal.connector.core.util.Callback;
import com.alibaba.otter.canal.connector.core.producer.MQDestination;
import com.alibaba.otter.canal.connector.core.producer.MQMessageUtils;
import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
import com.alibaba.otter.canal.connector.core.spi.SPI;
import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
import com.alibaba.otter.canal.connector.rocketmq.config.RocketMQConstants;
import com.alibaba.otter.canal.connector.rocketmq.config.RocketMQProducerConfig;
import com.alibaba.otter.canal.protocol.FlatMessage;
/**
* RocketMQ Producer SPI 实现
*
* @author rewerma 2020-01-27
* @version 1.0.0
*/
@SPI("rocketmq")
public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQProducer {
private static final Logger logger = LoggerFactory.getLogger(CanalRocketMQProducer.class);
private DefaultMQProducer defaultMQProducer;
private static final String CLOUD_ACCESS_CHANNEL = "cloud";
@Override
public void init(Properties properties) {
RocketMQProducerConfig rocketMQProperties = new RocketMQProducerConfig();
this.mqProperties = rocketMQProperties;
super.init(properties);
loadRocketMQProperties(properties);
RPCHook rpcHook = null;
if (mqProperties.getAliyunAccessKey().length() > 0
&& mqProperties.getAliyunSecretKey().length() > 0) {
SessionCredentials sessionCredentials = new SessionCredentials();
sessionCredentials.setAccessKey(mqProperties.getAliyunAccessKey());
sessionCredentials.setSecretKey(mqProperties.getAliyunSecretKey());
rpcHook = new AclClientRPCHook(sessionCredentials);
}
defaultMQProducer = new DefaultMQProducer(rocketMQProperties.getProducerGroup(),
rpcHook,
rocketMQProperties.isEnableMessageTrace(),
rocketMQProperties.getCustomizedTraceTopic());
if (CLOUD_ACCESS_CHANNEL.equals(rocketMQProperties.getAccessChannel())) {
defaultMQProducer.setAccessChannel(AccessChannel.CLOUD);
}
if (!StringUtils.isEmpty(rocketMQProperties.getNamespace())) {
defaultMQProducer.setNamespace(rocketMQProperties.getNamespace());
}
defaultMQProducer.setNamesrvAddr(rocketMQProperties.getNamesrvAddr());
defaultMQProducer.setRetryTimesWhenSendFailed(rocketMQProperties.getRetryTimesWhenSendFailed());
defaultMQProducer.setVipChannelEnabled(rocketMQProperties.isVipChannelEnabled());
logger.info("##Start RocketMQ producer##");
try {
defaultMQProducer.start();
} catch (MQClientException ex) {
throw new CanalException("Start RocketMQ producer error", ex);
}
}
private void loadRocketMQProperties(Properties properties) {
RocketMQProducerConfig rocketMQProperties = (RocketMQProducerConfig) this.mqProperties;
String producerGroup = properties.getProperty(RocketMQConstants.ROCKETMQ_PRODUCER_GROUP);
if (!StringUtils.isEmpty(producerGroup)) {
rocketMQProperties.setProducerGroup(producerGroup);
}
String enableMessageTrace = properties.getProperty(RocketMQConstants.ROCKETMQ_ENABLE_MESSAGE_TRACE);
if (!StringUtils.isEmpty(enableMessageTrace)) {
rocketMQProperties.setEnableMessageTrace(Boolean.parseBoolean(enableMessageTrace));
}
String customizedTraceTopic = properties.getProperty(RocketMQConstants.ROCKETMQ_CUSTOMIZED_TRACE_TOPIC);
if (!StringUtils.isEmpty(customizedTraceTopic)) {
rocketMQProperties.setCustomizedTraceTopic(customizedTraceTopic);
}
String namespace = properties.getProperty(RocketMQConstants.ROCKETMQ_NAMESPACE);
if (!StringUtils.isEmpty(namespace)) {
rocketMQProperties.setNamespace(namespace);
}
String namesrvAddr = properties.getProperty(RocketMQConstants.ROCKETMQ_NAMESRV_ADDR);
if (!StringUtils.isEmpty(namesrvAddr)) {
rocketMQProperties.setNamesrvAddr(namesrvAddr);
}
String retry = properties.getProperty(RocketMQConstants.ROCKETMQ_RETRY_TIMES_WHEN_SEND_FAILED);
if (!StringUtils.isEmpty(retry)) {
rocketMQProperties.setRetryTimesWhenSendFailed(Integer.parseInt(retry));
}
String vipChannelEnabled = properties.getProperty(RocketMQConstants.ROCKETMQ_VIP_CHANNEL_ENABLED);
if (!StringUtils.isEmpty(vipChannelEnabled)) {
rocketMQProperties.setVipChannelEnabled(Boolean.parseBoolean(vipChannelEnabled));
}
}
@Override
public void send(MQDestination destination, com.alibaba.otter.canal.protocol.Message message, Callback callback) {
ExecutorTemplate template = new ExecutorTemplate(executor);
try {
if (!StringUtils.isEmpty(destination.getDynamicTopic())) {
// 动态topic
Map<String, com.alibaba.otter.canal.protocol.Message> messageMap = MQMessageUtils
.messageTopics(message, destination.getTopic(), destination.getDynamicTopic());
for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) {
String topicName = entry.getKey().replace('.', '_');
com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue();
template.submit(() -> {
try {
send(destination, topicName, messageSub);
} catch (Exception e) {
throw new RuntimeException(e);
}
});
}
template.waitForResult();
} else {
send(destination, destination.getTopic(), message);
}
callback.commit();
} catch (Throwable e) {
logger.error(e.getMessage(), e);
callback.rollback();
} finally {
template.clear();
}
}
public void send(final MQDestination destination, String topicName,
com.alibaba.otter.canal.protocol.Message message) {
if (!mqProperties.isFlatMessage()) {
if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
// 并发构造
MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor);
// 串行分区
com.alibaba.otter.canal.protocol.Message[] messages = MQMessageUtils.messagePartition(datas,
message.getId(),
destination.getPartitionsNum(),
destination.getPartitionHash(),
mqProperties.isDatabaseHash());
int length = messages.length;
ExecutorTemplate template = new ExecutorTemplate(executor);
for (int i = 0; i < length; i++) {
com.alibaba.otter.canal.protocol.Message dataPartition = messages[i];
if (dataPartition != null) {
final int index = i;
template.submit(() -> {
Message data = new Message(topicName,
CanalMessageSerializerUtil.serializer(dataPartition,
mqProperties.isFilterTransactionEntry()));
sendMessage(data, index);
});
}
}
// 等所有分片发送完毕
template.waitForResult();
} else {
final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
Message data = new Message(topicName,
CanalMessageSerializerUtil.serializer(message, mqProperties.isFilterTransactionEntry()));
sendMessage(data, partition);
}
} else {
// 并发构造
MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor);
// 串行分区
List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, message.getId());
// 初始化分区合并队列
if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
List<List<FlatMessage>> partitionFlatMessages = new ArrayList<>();
for (int i = 0; i < destination.getPartitionsNum(); i++) {
partitionFlatMessages.add(new ArrayList<>());
}
for (FlatMessage flatMessage : flatMessages) {
FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage,
destination.getPartitionsNum(),
destination.getPartitionHash(),
mqProperties.isDatabaseHash());
int length = partitionFlatMessage.length;
for (int i = 0; i < length; i++) {
partitionFlatMessages.get(i).add(partitionFlatMessage[i]);
}
}
ExecutorTemplate template = new ExecutorTemplate(executor);
for (int i = 0; i < partitionFlatMessages.size(); i++) {
final List<FlatMessage> flatMessagePart = partitionFlatMessages.get(i);
if (flatMessagePart != null) {
final int index = i;
template.submit(() -> {
List<Message> messages = flatMessagePart.stream()
.map(flatMessage -> new Message(topicName,
JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue)))
.collect(Collectors.toList());
// 批量发送
sendMessage(messages, index);
});
}
}
// 批量等所有分区的结果
template.waitForResult();
} else {
final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
List<Message> messages = flatMessages.stream()
.map(flatMessage -> new Message(topicName,
JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue)))
.collect(Collectors.toList());
// 批量发送
sendMessage(messages, partition);
}
}
}
private void sendMessage(Message message, int partition) {
try {
SendResult sendResult = this.defaultMQProducer.send(message, (mqs, msg, arg) -> {
if (partition > mqs.size()) {
return mqs.get(partition % mqs.size());
} else {
return mqs.get(partition);
}
}, null);
if (logger.isDebugEnabled()) {
logger.debug("Send Message Result: {}", sendResult);
}
} catch (Throwable e) {
throw new RuntimeException(e);
}
}
@SuppressWarnings("deprecation")
private void sendMessage(List<Message> messages, int partition) {
if (messages.isEmpty()) {
return;
}
// 获取一下messageQueue
DefaultMQProducerImpl innerProducer = this.defaultMQProducer.getDefaultMQProducerImpl();
TopicPublishInfo topicInfo = innerProducer.getTopicPublishInfoTable().get(messages.get(0).getTopic());
if (topicInfo == null) {
for (Message message : messages) {
sendMessage(message, partition);
}
} else {
// 批量发送
List<MessageQueue> queues = topicInfo.getMessageQueueList();
int size = queues.size();
if (size <= 0) {
// 可能是第一次创建
for (Message message : messages) {
sendMessage(message, partition);
}
} else {
MessageQueue queue;
if (partition > size) {
queue = queues.get(partition % size);
} else {
queue = queues.get(partition);
}
try {
SendResult sendResult = this.defaultMQProducer.send(messages, queue);
if (logger.isDebugEnabled()) {
logger.debug("Send Message Result: {}", sendResult);
}
} catch (Throwable e) {
throw new RuntimeException(e);
}
}
}
}
@Override
public void stop() {
logger.info("## Stop RocketMQ producer##");
this.defaultMQProducer.shutdown();
super.stop();
}
}
connector/rocketmq-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMQProducer
+1
View File
@@ -0,0 +1 @@
rocketmq=com.alibaba.otter.canal.connector.rocketmq.producer.CanalRocketMQProducer
connector/rocketmq-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer
+1
View File
@@ -0,0 +1 @@
rocketmq=com.alibaba.otter.canal.connector.rocketmq.consumer.CanalRocketMQConsumer
connector/tcp-connector/pom.xml
+124
View File
@@ -0,0 +1,124 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>canal.connector</artifactId>
<groupId>com.alibaba.otter</groupId>
<version>1.1.5-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.tcp</artifactId>
<packaging>jar</packaging>
<name>canal connector tcp module for otter ${project.version}</name>
<dependencies>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.core</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.protocol</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.client</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</exclusion>
<exclusion>
<groupId>org.springframework</groupId>
<artifactId>spring-orm</artifactId>
</exclusion>
<exclusion>
<groupId>org.springframework</groupId>
<artifactId>spring-jdbc</artifactId>
</exclusion>
<exclusion>
<groupId>org.springframework</groupId>
<artifactId>spring-aop</artifactId>
</exclusion>
<exclusion>
<groupId>org.springframework</groupId>
<artifactId>spring-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-client</artifactId>
</exclusion>
<exclusion>
<groupId>com.aliyun.openservices</groupId>
<artifactId>aliware-apache-rocketmq-cloud</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4</version>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.8</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<tasks>
<copy todir="${project.basedir}/../../client-adapter/launcher/target/canal-adapter/plugin"
overwrite="true">
<fileset dir="${project.basedir}/target" erroronmissingdir="true">
<include name="connector.tcp-${project.version}-jar-with-dependencies.jar"/>
</fileset>
</copy>
</tasks>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
connector/tcp-connector/src/main/java/com/alibaba/otter/canal/connector/tcp/config/TCPConstants.java
+12
View File
@@ -0,0 +1,12 @@
package com.alibaba.otter.canal.connector.tcp.config;
public class TCPConstants {
public static final String ROOT = "canal";
public static final String CANAL_TCP_HOST = ROOT + "." + "tcp.server.host";
public static final String CANAL_TCP_ZK_HOSTS = ROOT + "." + "tcp.zookeeper.hosts";
public static final String CANAL_TCP_USERNAME = ROOT + "." + "tcp.username";
public static final String CANAL_TCP_PASSWORD = ROOT + "." + "tcp.password";
public static final String CANAL_TCP_BATCH_SIZE = ROOT + "." + "tcp.batch.size";
}
connector/tcp-connector/src/main/java/com/alibaba/otter/canal/connector/tcp/consumer/CanalTCPConsumer.java
+106
View File
@@ -0,0 +1,106 @@
package com.alibaba.otter.canal.connector.tcp.consumer;
import com.alibaba.otter.canal.client.CanalConnector;
import com.alibaba.otter.canal.client.impl.ClusterCanalConnector;
import com.alibaba.otter.canal.client.impl.ClusterNodeAccessStrategy;
import com.alibaba.otter.canal.client.impl.SimpleCanalConnector;
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
import com.alibaba.otter.canal.connector.core.spi.SPI;
import com.alibaba.otter.canal.connector.core.util.MessageUtil;
import com.alibaba.otter.canal.connector.tcp.config.TCPConstants;
import com.alibaba.otter.canal.protocol.Message;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
/**
* TCP 消费者连接器, 一个destination对应一个SPI实例
*
* @author rewerma 2020-01-30
* @version 1.0.0
*/
@SPI("tcp")
public class CanalTCPConsumer implements CanalMsgConsumer {
private static final String PREFIX_TCP_CONFIG = "canal.tcp.";
private Long currentBatchId = null;
private CanalConnector canalConnector;
private int batchSize = 500;
@Override
public void init(Properties properties, String destination, String groupId) {
// load config
String host = properties.getProperty(TCPConstants.CANAL_TCP_HOST);
String username = properties.getProperty(TCPConstants.CANAL_TCP_USERNAME);
String password = properties.getProperty(TCPConstants.CANAL_TCP_PASSWORD);
String zkHosts = properties.getProperty(TCPConstants.CANAL_TCP_ZK_HOSTS);
String batchSizePro = properties.getProperty(TCPConstants.CANAL_TCP_BATCH_SIZE);
if (batchSizePro != null) {
batchSize = Integer.parseInt(batchSizePro);
}
if (host != null) {
String[] ipPort = host.split(":");
SocketAddress sa = new InetSocketAddress(ipPort[0], Integer.parseInt(ipPort[1]));
this.canalConnector = new SimpleCanalConnector(sa, username, password, destination);
} else {
this.canalConnector = new ClusterCanalConnector(username,
password,
destination,
new ClusterNodeAccessStrategy(destination, ZkClientx.getZkClient(zkHosts)));
}
}
@Override
public void connect() {
canalConnector.connect();
canalConnector.subscribe();
}
@Override
public List<CommonMessage> getMessage(Long timeout, TimeUnit unit) {
try {
Message message = canalConnector.getWithoutAck(batchSize, timeout, unit);
long batchId = message.getId();
currentBatchId = batchId;
int size = message.getEntries().size();
if (batchId == -1 || size == 0) {
return null;
} else {
return MessageUtil.convert(message);
}
} catch (Throwable e) {
throw new RuntimeException(e);
}
}
@Override
public void rollback() {
if (currentBatchId != null && currentBatchId != -1) {
canalConnector.rollback(currentBatchId);
currentBatchId = null;
}
}
@Override
public void ack() {
if (currentBatchId != null) {
canalConnector.ack(currentBatchId);
currentBatchId = null;
}
}
@Override
public void disconnect() {
canalConnector.unsubscribe();
canalConnector.disconnect();
}
}
connector/tcp-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer
+1
View File
@@ -0,0 +1 @@
tcp=com.alibaba.otter.canal.connector.tcp.consumer.CanalTCPConsumer
deployer/pom.xml
+177 -119
View File
@@ -1,127 +1,185 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.1.5-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.deployer</artifactId>
<packaging>jar</packaging>
<name>canal deployer module for otter ${project.version}</name>
<dependencies>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.server</artifactId>
<version>${project.version}</version>
</dependency>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.1.5-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.deployer</artifactId>
<packaging>jar</packaging>
<name>canal deployer module for otter ${project.version}</name>
<dependencies>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.server</artifactId>
<version>${project.version}</version>
</dependency>
<!-- 这里指定runtime的metrics provider-->
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.prometheus</artifactId>
<version>${project.version}</version>
<scope>runtime</scope>
</dependency>
</dependencies>
<build>
<plugins>
<!-- deploy模块的packaging通常是jar，如果项目中没有java 源代码或资源文件，加上这一段配置使项目能通过构建 -->
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<archive>
<addMavenDescriptor>true</addMavenDescriptor>
</archive>
<excludes>
<exclude>**/logback.xml</exclude>
<exclude>**/canal.properties</exclude>
<exclude>**/spring/**</exclude>
<exclude>**/example/**</exclude>
<exclude>**/mq.yml</exclude>
</excludes>
</configuration>
</plugin>
<!-- 这里指定runtime的metrics provider-->
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.prometheus</artifactId>
<version>${project.version}</version>
<scope>runtime</scope>
</dependency>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<!-- 这是最新版本，推荐使用这个版本 -->
<version>2.2.1</version>
<executions>
<execution>
<id>assemble</id>
<goals>
<goal>single</goal>
</goals>
<phase>package</phase>
</execution>
</executions>
<configuration>
<appendAssemblyId>false</appendAssemblyId>
<attach>false</attach>
</configuration>
</plugin>
</plugins>
</build>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.kafka</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
<artifactId>*</artifactId>
<groupId>*</groupId>
</exclusion>
</exclusions>
<classifier>jar-with-dependencies</classifier>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.rocketmq</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
<artifactId>*</artifactId>
<groupId>*</groupId>
</exclusion>
</exclusions>
<classifier>jar-with-dependencies</classifier>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.rabbitmq</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
<artifactId>*</artifactId>
<groupId>*</groupId>
</exclusion>
</exclusions>
<classifier>jar-with-dependencies</classifier>
<scope>provided</scope>
</dependency>
</dependencies>
<profiles>
<profile>
<id>dev</id>
<activation>
<activeByDefault>true</activeByDefault>
<property>
<name>env</name>
<value>!release</value>
</property>
</activation>
<build>
<plugins>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<archive>
<addMavenDescriptor>true</addMavenDescriptor>
</archive>
<excludes>
<exclude>**/logback.xml</exclude>
<exclude>**/canal.properties</exclude>
<exclude>**/spring/**</exclude>
<exclude>**/example/**</exclude>
<exclude>**/mq.yml</exclude>
</excludes>
</configuration>
</plugin>
<build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<!-- maven assembly插件需要一个描述文件 来告诉插件包的结构以及打包所需的文件来自哪里 -->
<descriptors>
<descriptor>${basedir}/src/main/assembly/dev.xml</descriptor>
</descriptors>
<finalName>canal</finalName>
<outputDirectory>${project.build.directory}</outputDirectory>
</configuration>
</plugin>
</plugins>
</build>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<!-- 这是最新版本，推荐使用这个版本 -->
<version>2.2.1</version>
<executions>
<execution>
<id>assemble</id>
<goals>
<goal>single</goal>
</goals>
<phase>package</phase>
</execution>
</executions>
<configuration>
<appendAssemblyId>false</appendAssemblyId>
<attach>false</attach>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.10</version>
<executions>
<execution>
<id>copy-dependencies-to-canal-client-service</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<includeClassifiers>jar-with-dependencies</includeClassifiers>
<outputDirectory>${project.basedir}/target/canal/plugin</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profiles>
<profile>
<id>dev</id>
<activation>
<activeByDefault>true</activeByDefault>
<property>
<name>env</name>
<value>!release</value>
</property>
</activation>
<profile>
<id>release</id>
<activation>
<property>
<name>env</name>
<value>release</value>
</property>
</activation>
<build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<!-- maven assembly插件需要一个描述文件 来告诉插件包的结构以及打包所需的文件来自哪里 -->
<descriptors>
<descriptor>${basedir}/src/main/assembly/dev.xml</descriptor>
</descriptors>
<finalName>canal</finalName>
<outputDirectory>${project.build.directory}</outputDirectory>
</configuration>
</plugin>
</plugins>
</build>
<build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<!-- 发布模式使用的maven assembly插件描述文件 -->
<descriptors>
<descriptor>${basedir}/src/main/assembly/release.xml</descriptor>
</descriptors>
<!-- 如果一个应用的包含多个deploy模块，如果使用同样的包名， 如果把它们复制的一个目录中可能会失败，所以包名加了 artifactId以示区分 -->
<finalName>${project.artifactId}-${project.version}</finalName>
<!-- scm 要求 release 模式打出的包放到顶级目录下的target子目录中 -->
<outputDirectory>${project.parent.build.directory}</outputDirectory>
</configuration>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</profile>
<profile>
<id>release</id>
<activation>
<property>
<name>env</name>
<value>release</value>
</property>
</activation>
<build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<!-- 发布模式使用的maven assembly插件描述文件 -->
<descriptors>
<descriptor>${basedir}/src/main/assembly/release.xml</descriptor>
</descriptors>
<!-- 如果一个应用的包含多个deploy模块，如果使用同样的包名， 如果把它们复制的一个目录中可能会失败，所以包名加了 artifactId以示区分 -->
<finalName>${project.artifactId}-${project.version}</finalName>
<!-- scm 要求 release 模式打出的包放到顶级目录下的target子目录中 -->
<outputDirectory>${project.parent.build.directory}</outputDirectory>
</configuration>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>
deployer/src/main/assembly/release.xml
+4
View File
@@ -42,6 +42,10 @@
<exclude>**/*</exclude>
</excludes>
</fileSet>
<fileSet>
<directory>${project.basedir}/target/canal/plugin</directory>
<outputDirectory>/plugin/</outputDirectory>
</fileSet>
</fileSets>
<dependencySets>
<dependencySet>
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalConstants.java
+29 -28
View File
@@ -46,36 +46,37 @@ public class CanalConstants {
public static final String CANAL_SOCKETCHANNEL = ROOT + "." + "socketChannel";
public static final String CANAL_MQ_SERVERS = ROOT + "." + "mq.servers";
public static final String CANAL_MQ_RETRIES = ROOT + "." + "mq.retries";
public static final String CANAL_MQ_BATCHSIZE = ROOT + "." + "mq.batchSize";
public static final String CANAL_MQ_LINGERMS = ROOT + "." + "mq.lingerMs";
public static final String CANAL_MQ_MAXREQUESTSIZE = ROOT + "." + "mq.maxRequestSize";
public static final String CANAL_MQ_BUFFERMEMORY = ROOT + "." + "mq.bufferMemory";
public static final String CANAL_MQ_CANALBATCHSIZE = ROOT + "." + "mq.canalBatchSize";
public static final String CANAL_MQ_CANALGETTIMEOUT = ROOT + "." + "mq.canalGetTimeout";
public static final String CANAL_MQ_FLATMESSAGE = ROOT + "." + "mq.flatMessage";
public static final String CANAL_MQ_PARALLELTHREADSIZE = ROOT + "." + "mq.parallelThreadSize";
public static final String CANAL_MQ_COMPRESSION_TYPE = ROOT + "." + "mq.compressionType";
public static final String CANAL_MQ_ACKS = ROOT + "." + "mq.acks";
public static final String CANAL_MQ_TRANSACTION = ROOT + "." + "mq.transaction";
public static final String CANAL_MQ_PRODUCERGROUP = ROOT + "." + "mq.producerGroup";
public static final String CANAL_ALIYUN_ACCESSKEY = ROOT + "." + "aliyun.accessKey";
public static final String CANAL_ALIYUN_SECRETKEY = ROOT + "." + "aliyun.secretKey";
public static final String CANAL_MQ_PROPERTIES = ROOT + "." + "mq.properties";
public static final String CANAL_MQ_ENABLE_MESSAGE_TRACE = ROOT + "." + "mq.enableMessageTrace";
public static final String CANAL_MQ_ACCESS_CHANNEL = ROOT + "." + "mq.accessChannel";
public static final String CANAL_MQ_CUSTOMIZED_TRACE_TOPIC = ROOT + "." + "mq.customizedTraceTopic";
public static final String CANAL_MQ_NAMESPACE = ROOT + "." + "mq.namespace";
public static final String CANAL_MQ_KAFKA_KERBEROS_ENABLE = ROOT + "." + "mq.kafka.kerberos.enable";
public static final String CANAL_MQ_KAFKA_KERBEROS_KRB5FILEPATH = ROOT + "." + "mq.kafka.kerberos.krb5FilePath";
public static final String CANAL_MQ_KAFKA_KERBEROS_JAASFILEPATH = ROOT + "." + "mq.kafka.kerberos.jaasFilePath";
public static final String CANAL_MQ_USERNAME = ROOT + "." + "mq.username";
public static final String CANAL_MQ_PASSWORD = ROOT + "." + "mq.password";
public static final String CANAL_MQ_VHOST = ROOT + "." + "mq.vhost";
public static final String CANAL_MQ_ALIYUN_UID = ROOT + "." + "mq.aliyunuid";
public static final String CANAL_MQ_EXCHANGE = ROOT + "." + "mq.exchange";
public static final String CANAL_MQ_DATABASE_HASH = ROOT + "." + "mq.database.hash";
// public static final String CANAL_MQ_SERVERS = ROOT + "." + "mq.servers";
// public static final String CANAL_MQ_RETRIES = ROOT + "." + "mq.retries";
// public static final String CANAL_MQ_BATCHSIZE = ROOT + "." + "mq.batchSize";
// public static final String CANAL_MQ_LINGERMS = ROOT + "." + "mq.lingerMs";
// public static final String CANAL_MQ_MAXREQUESTSIZE = ROOT + "." + "mq.maxRequestSize";
// public static final String CANAL_MQ_BUFFERMEMORY = ROOT + "." + "mq.bufferMemory";
// public static final String CANAL_MQ_CANALBATCHSIZE = ROOT + "." + "mq.canalBatchSize";
// public static final String CANAL_MQ_CANALGETTIMEOUT = ROOT + "." + "mq.canalGetTimeout";
// public static final String CANAL_MQ_FLATMESSAGE = ROOT + "." + "mq.flatMessage";
// public static final String CANAL_MQ_PARALLELTHREADSIZE = ROOT + "." + "mq.parallelThreadSize";
// public static final String CANAL_MQ_COMPRESSION_TYPE = ROOT + "." + "mq.compressionType";
// public static final String CANAL_MQ_ACKS = ROOT + "." + "mq.acks";
// public static final String CANAL_MQ_TRANSACTION = ROOT + "." + "mq.transaction";
// public static final String CANAL_MQ_PRODUCERGROUP = ROOT + "." + "mq.producerGroup";
// public static final String CANAL_MQ_PROPERTIES = ROOT + "." + "mq.properties";
// public static final String CANAL_MQ_ENABLE_MESSAGE_TRACE = ROOT + "." + "mq.enableMessageTrace";
// public static final String CANAL_MQ_ACCESS_CHANNEL = ROOT + "." + "mq.accessChannel";
// public static final String CANAL_MQ_CUSTOMIZED_TRACE_TOPIC = ROOT + "." + "mq.customizedTraceTopic";
// public static final String CANAL_MQ_NAMESPACE = ROOT + "." + "mq.namespace";
// public static final String CANAL_MQ_KAFKA_KERBEROS_ENABLE = ROOT + "." + "mq.kafka.kerberos.enable";
// public static final String CANAL_MQ_KAFKA_KERBEROS_KRB5FILEPATH = ROOT + "." + "mq.kafka.kerberos.krb5FilePath";
// public static final String CANAL_MQ_KAFKA_KERBEROS_JAASFILEPATH = ROOT + "." + "mq.kafka.kerberos.jaasFilePath";
// public static final String CANAL_MQ_USERNAME = ROOT + "." + "mq.username";
// public static final String CANAL_MQ_PASSWORD = ROOT + "." + "mq.password";
// public static final String CANAL_MQ_VHOST = ROOT + "." + "mq.vhost";
// public static final String CANAL_MQ_ALIYUN_UID = ROOT + "." + "mq.aliyunuid";
// public static final String CANAL_MQ_EXCHANGE = ROOT + "." + "mq.exchange";
// public static final String CANAL_MQ_DATABASE_HASH = ROOT + "." + "mq.database.hash";
public static String getInstanceModeKey(String destination) {
return MessageFormat.format(INSTANCE_MODE_TEMPLATE, destination);
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalStarter.java
+33 -177
View File
@@ -2,35 +2,36 @@ package com.alibaba.otter.canal.deployer;
import java.util.Properties;
import com.alibaba.otter.canal.connector.core.config.MQProperties;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.admin.netty.CanalAdminWithNetty;
import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
import com.alibaba.otter.canal.connector.core.spi.ExtensionLoader;
import com.alibaba.otter.canal.deployer.admin.CanalAdminController;
import com.alibaba.otter.canal.kafka.CanalKafkaProducer;
import com.alibaba.otter.canal.rabbitmq.CanalRabbitMQProducer;
import com.alibaba.otter.canal.rocketmq.CanalRocketMQProducer;
import com.alibaba.otter.canal.server.CanalMQStarter;
import com.alibaba.otter.canal.spi.CanalMQProducer;
/**
* Canal server 启动类
*
* @author rewerma 2018-12-30 下午05:12:16
* @version 1.0.1
* @author rewerma 2020-01-27
* @version 1.0.2
*/
public class CanalStarter {
private static final Logger logger = LoggerFactory.getLogger(CanalStarter.class);
private static final Logger logger = LoggerFactory.getLogger(CanalStarter.class);
private CanalController controller = null;
private CanalMQProducer canalMQProducer = null;
private Thread shutdownThread = null;
private CanalMQStarter canalMQStarter = null;
private static final String CONNECTOR_SPI_DIR = "/plugin";
private static final String CONNECTOR_STANDBY_SPI_DIR = "/canal/plugin";
private CanalController controller = null;
private CanalMQProducer canalMQProducer = null;
private Thread shutdownThread = null;
private CanalMQStarter canalMQStarter = null;
private volatile Properties properties;
private volatile boolean running = false;
private volatile boolean running = false;
private CanalAdminWithNetty canalAdmin;
@@ -61,20 +62,23 @@ public class CanalStarter {
*/
public synchronized void start() throws Throwable {
String serverMode = CanalController.getProperty(properties, CanalConstants.CANAL_SERVER_MODE);
if (serverMode.equalsIgnoreCase("kafka")) {
canalMQProducer = new CanalKafkaProducer();
} else if (serverMode.equalsIgnoreCase("rocketmq")) {
canalMQProducer = new CanalRocketMQProducer();
} else if (serverMode.equalsIgnoreCase("rabbitmq")) {
canalMQProducer = new CanalRabbitMQProducer();
if (!"tcp".equalsIgnoreCase(serverMode)) {
ExtensionLoader<CanalMQProducer> loader = ExtensionLoader.getExtensionLoader(CanalMQProducer.class);
canalMQProducer = loader
.getExtension(serverMode.toLowerCase(), CONNECTOR_SPI_DIR, CONNECTOR_STANDBY_SPI_DIR);
if (canalMQProducer != null) {
ClassLoader cl = Thread.currentThread().getContextClassLoader();
Thread.currentThread().setContextClassLoader(canalMQProducer.getClass().getClassLoader());
canalMQProducer.init(properties);
Thread.currentThread().setContextClassLoader(cl);
}
}
MQProperties mqProperties = null;
if (canalMQProducer != null) {
mqProperties = buildMQProperties(properties);
MQProperties mqProperties = canalMQProducer.getMqProperties();
// disable netty
System.setProperty(CanalConstants.CANAL_WITHOUT_NETTY, "true");
if (mqProperties.getFlatMessage()) {
if (mqProperties.isFlatMessage()) {
// 设置为raw避免ByteString->Entry的二次解析
System.setProperty("canal.instance.memory.rawEntry", "false");
}
@@ -104,7 +108,7 @@ public class CanalStarter {
if (canalMQProducer != null) {
canalMQStarter = new CanalMQStarter(canalMQProducer);
String destinations = CanalController.getProperty(properties, CanalConstants.CANAL_DESTINATIONS);
canalMQStarter.start(mqProperties, destinations);
canalMQStarter.start(destinations);
controller.setCanalMQStarter(canalMQStarter);
}
@@ -119,11 +123,15 @@ public class CanalStarter {
String ip = CanalController.getProperty(properties, CanalConstants.CANAL_IP);
logger.debug("canal admin port:{}, canal admin user:{}, canal admin password: {}, canal ip:{}", port, user, passwd, ip);
logger.debug("canal admin port:{}, canal admin user:{}, canal admin password: {}, canal ip:{}",
port,
user,
passwd,
ip);
CanalAdminWithNetty canalAdminWithNetty = CanalAdminWithNetty.instance();
canalAdminWithNetty.setCanalAdmin(canalAdmin);
canalAdminWithNetty.setPort(Integer.valueOf(port));
canalAdminWithNetty.setPort(Integer.parseInt(port));
canalAdminWithNetty.setIp(ip);
canalAdminWithNetty.start();
this.canalAdmin = canalAdminWithNetty;
@@ -162,156 +170,4 @@ public class CanalStarter {
}
running = false;
}
/**
* 构造MQ对应的配置
*
* @param properties canal.properties 配置
* @return
*/
private static MQProperties buildMQProperties(Properties properties) {
MQProperties mqProperties = new MQProperties();
String servers = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_SERVERS);
if (!StringUtils.isEmpty(servers)) {
mqProperties.setServers(servers);
}
String retires = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_RETRIES);
if (!StringUtils.isEmpty(retires)) {
mqProperties.setRetries(Integer.valueOf(retires));
}
String batchSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_BATCHSIZE);
if (!StringUtils.isEmpty(batchSize)) {
mqProperties.setBatchSize(Integer.valueOf(batchSize));
}
String lingerMs = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_LINGERMS);
if (!StringUtils.isEmpty(lingerMs)) {
mqProperties.setLingerMs(Integer.valueOf(lingerMs));
}
String maxRequestSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_MAXREQUESTSIZE);
if (!StringUtils.isEmpty(maxRequestSize)) {
mqProperties.setMaxRequestSize(Integer.valueOf(maxRequestSize));
}
String bufferMemory = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_BUFFERMEMORY);
if (!StringUtils.isEmpty(bufferMemory)) {
mqProperties.setBufferMemory(Long.valueOf(bufferMemory));
}
String canalBatchSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_CANALBATCHSIZE);
if (!StringUtils.isEmpty(canalBatchSize)) {
mqProperties.setCanalBatchSize(Integer.valueOf(canalBatchSize));
}
String canalGetTimeout = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_CANALGETTIMEOUT);
if (!StringUtils.isEmpty(canalGetTimeout)) {
mqProperties.setCanalGetTimeout(Long.valueOf(canalGetTimeout));
}
String flatMessage = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_FLATMESSAGE);
if (!StringUtils.isEmpty(flatMessage)) {
mqProperties.setFlatMessage(Boolean.valueOf(flatMessage));
}
String parallelThreadSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_PARALLELTHREADSIZE);
if (!StringUtils.isEmpty(parallelThreadSize)) {
mqProperties.setParallelThreadSize(Integer.valueOf(parallelThreadSize));
}
String compressionType = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_COMPRESSION_TYPE);
if (!StringUtils.isEmpty(compressionType)) {
mqProperties.setCompressionType(compressionType);
}
String acks = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_ACKS);
if (!StringUtils.isEmpty(acks)) {
mqProperties.setAcks(acks);
}
String aliyunAccessKey = CanalController.getProperty(properties, CanalConstants.CANAL_ALIYUN_ACCESSKEY);
if (!StringUtils.isEmpty(aliyunAccessKey)) {
mqProperties.setAliyunAccessKey(aliyunAccessKey);
}
String aliyunSecretKey = CanalController.getProperty(properties, CanalConstants.CANAL_ALIYUN_SECRETKEY);
if (!StringUtils.isEmpty(aliyunSecretKey)) {
mqProperties.setAliyunSecretKey(aliyunSecretKey);
}
String producerGroup = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_PRODUCERGROUP);
if (!StringUtils.isEmpty(producerGroup)) {
mqProperties.setProducerGroup(producerGroup);
}
String enableMessageTrace = CanalController.getProperty(properties,
CanalConstants.CANAL_MQ_ENABLE_MESSAGE_TRACE);
if (!StringUtils.isEmpty(enableMessageTrace)) {
mqProperties.setEnableMessageTrace(Boolean.valueOf(enableMessageTrace));
}
String accessChannel = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_ACCESS_CHANNEL);
if (!StringUtils.isEmpty(accessChannel)) {
mqProperties.setAccessChannel(accessChannel);
}
String customizedTraceTopic = CanalController.getProperty(properties,
CanalConstants.CANAL_MQ_CUSTOMIZED_TRACE_TOPIC);
if (!StringUtils.isEmpty(customizedTraceTopic)) {
mqProperties.setCustomizedTraceTopic(customizedTraceTopic);
}
String namespace = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_NAMESPACE);
if (!StringUtils.isEmpty(namespace)) {
mqProperties.setNamespace(namespace);
}
String kafkaKerberosEnable = CanalController.getProperty(properties,
CanalConstants.CANAL_MQ_KAFKA_KERBEROS_ENABLE);
if (!StringUtils.isEmpty(kafkaKerberosEnable)) {
mqProperties.setKerberosEnable(Boolean.valueOf(kafkaKerberosEnable));
}
String kafkaKerberosKrb5Filepath = CanalController.getProperty(properties,
CanalConstants.CANAL_MQ_KAFKA_KERBEROS_KRB5FILEPATH);
if (!StringUtils.isEmpty(kafkaKerberosKrb5Filepath)) {
mqProperties.setKerberosKrb5FilePath(kafkaKerberosKrb5Filepath);
}
String kafkaKerberosJaasFilepath = CanalController.getProperty(properties,
CanalConstants.CANAL_MQ_KAFKA_KERBEROS_JAASFILEPATH);
if (!StringUtils.isEmpty(kafkaKerberosJaasFilepath)) {
mqProperties.setKerberosJaasFilePath(kafkaKerberosJaasFilepath);
}
String vhost = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_VHOST);
if (!StringUtils.isEmpty(vhost)) {
mqProperties.setVhost(vhost);
}
String username = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_USERNAME);
if (!StringUtils.isEmpty(username)) {
mqProperties.setUsername(username);
}
String password = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_PASSWORD);
if (!StringUtils.isEmpty(password)) {
mqProperties.setPassword(password);
}
String aliyunUID = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_ALIYUN_UID);
if (!StringUtils.isEmpty(aliyunUID)) {
mqProperties.setAliyunUID(Long.valueOf(aliyunUID));
}
String exchange = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_EXCHANGE);
if (!StringUtils.isEmpty(exchange)) {
mqProperties.setExchange(exchange);
}
String databaseHash = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_DATABASE_HASH);
if (!StringUtils.isEmpty(databaseHash)){
mqProperties.setDatabaseHash(Boolean.valueOf(databaseHash));
}
for (Object key : properties.keySet()) {
key = StringUtils.trim(key.toString());
if (((String) key).startsWith(CanalConstants.CANAL_MQ_PROPERTIES)) {
String value = CanalController.getProperty(properties, (String) key);
String subKey = ((String) key).substring(CanalConstants.CANAL_MQ_PROPERTIES.length() + 1);
mqProperties.getProperties().put(subKey, value);
}
}
return mqProperties;
}
}
deployer/src/main/resources/canal.properties
+47 -32
View File
@@ -21,7 +21,7 @@ canal.zkServers =
# flush data to zk
canal.zookeeper.flush.period = 1000
canal.withoutNetty = false
# tcp, kafka, RocketMQ
# tcp, kafka, rocketMQ, rabbitMQ
canal.serverMode = tcp
# flush meta cursor/parse position to file
canal.file.data.dir = ${canal.conf.dir}
@@ -86,10 +86,6 @@ canal.instance.tsdb.snapshot.interval = 24
# purge snapshot expire , default 360 hour(15 days)
canal.instance.tsdb.snapshot.expire = 360
# aliyun ak/sk , support rds/mq
canal.aliyun.accessKey =
canal.aliyun.secretKey =
#################################################
######### destinations #############
#################################################
@@ -111,35 +107,54 @@ canal.instance.global.spring.xml = classpath:spring/file-instance.xml
#canal.instance.global.spring.xml = classpath:spring/default-instance.xml
##################################################
######### MQ #############
######### MQ Properties #############
##################################################
canal.mq.servers = 127.0.0.1:6667
canal.mq.retries = 0
canal.mq.batchSize = 16384
canal.mq.maxRequestSize = 1048576
canal.mq.lingerMs = 100
canal.mq.bufferMemory = 33554432
canal.mq.canalBatchSize = 50
canal.mq.canalGetTimeout = 100
canal.mq.parallelThreadSize = 8
canal.mq.flatMessage = true
canal.mq.compressionType = none
canal.mq.acks = all
#canal.mq.properties. =
canal.mq.producerGroup = test
# Set this value to "cloud", if you want open message trace feature in aliyun.
canal.mq.accessChannel = local
# aliyun mq namespace
#canal.mq.namespace =
canal.mq.vhost=
canal.mq.exchange=
canal.mq.username=
canal.mq.password=
canal.mq.aliyunuid=
canal.mq.flat.message = true
canal.mq.database.hash = true
canal.mq.parallel.thread.size = 8
canal.mq.canal.batch.size = 50
canal.mq.canal.fetch.timeout = 100
# Set this value to "cloud", if you want open message trace feature in aliyun.
canal.mq.access.channel = local
# aliyun ak/sk , support rds/mq
canal.aliyun.accessKey =
canal.aliyun.secretKey =
canal.aliyun.uid=
##################################################
######### Kafka Kerberos Info #############
######### Kafka #############
##################################################
kafka.bootstrap.servers = 127.0.0.1:9092
kafka.acks = all
kafka.compression.type = none
kafka.batch.size = 16384
kafka.linger.ms = 1
kafka.max.request.size = 1048576
kafka.buffer.memory = 33554432
kafka.max.in.flight.requests.per.connection = 1
kafka.retries = 0
canal.mq.kafka.kerberos.enable = false
canal.mq.kafka.kerberos.krb5FilePath = "../conf/kerberos/krb5.conf"
canal.mq.kafka.kerberos.jaasFilePath = "../conf/kerberos/jaas.conf"
canal.mq.kafka.kerberos.krb5.file = "../conf/kerberos/krb5.conf"
canal.mq.kafka.kerberos.jaas.file = "../conf/kerberos/jaas.conf"
##################################################
######### RocketMQ #############
##################################################
rocketmq.producer.group = test
rocketmq.enable.message.trace = false
rocketmq.customized.trace.topic =
rocketmq.namespace =
rocketmq.namesrv.addr = 127.0.0.1:9876
rocketmq.retry.times.when.send.failed = 0
rocketmq.vip.channel.enabled = false
##################################################
######### RabbitMQ #############
##################################################
rabbitmq.host =
rabbitmq.virtual.host =
rabbitmq.exchange =
rabbitmq.username =
rabbitmq.password =
deployer/src/main/resources/logback.xml
+6 -2
View File
@@ -73,11 +73,15 @@
<level value="INFO" />
<appender-ref ref="CANAL-META" />
</logger>
<logger name="com.alibaba.otter.canal.kafka" additivity="false">
<logger name="com.alibaba.otter.canal.connector.kafka" additivity="false">
<level value="INFO" />
<appender-ref ref="CANAL-ROOT" />
</logger>
<logger name="com.alibaba.otter.canal.rabbitmq" additivity="false">
<logger name="com.alibaba.otter.canal.connector.rocketmq" additivity="false">
<level value="INFO" />
<appender-ref ref="CANAL-ROOT" />
</logger>
<logger name="com.alibaba.otter.canal.connector.rabbitmq" additivity="false">
<level value="INFO" />
<appender-ref ref="CANAL-ROOT" />
</logger>
pom.xml
+6 -22
View File
@@ -100,8 +100,6 @@
<file_encoding>UTF-8</file_encoding>
<javadoc_skip>true</javadoc_skip>
<spring_version>3.2.18.RELEASE</spring_version>
<rocketmq_version>4.5.2</rocketmq_version>
<rabbitmq_version>5.5.0</rabbitmq_version>
<maven-jacoco-plugin.version>0.8.3</maven-jacoco-plugin.version>
<maven-surefire.version>2.22.1</maven-surefire.version>
<argline>-server -Xms512m -Xmx1024m -Dfile.encoding=UTF-8
@@ -129,6 +127,7 @@
<module>prometheus</module>
<module>admin</module>
<module>client-adapter</module>
<module>connector</module>
</modules>
<dependencyManagement>
@@ -233,6 +232,11 @@
<artifactId>aviator</artifactId>
<version>2.2.1</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.58.sec06</version>
</dependency>
<dependency>
<groupId>oro</groupId>
<artifactId>oro</artifactId>
@@ -336,26 +340,6 @@
<artifactId>jsr305</artifactId>
<version>3.0.2</version>
</dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-client</artifactId>
<version>${rocketmq_version}</version>
</dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-acl</artifactId>
<version>${rocketmq_version}</version>
</dependency>
<dependency>
<groupId>com.rabbitmq</groupId>
<artifactId>amqp-client</artifactId>
<version>${rabbitmq_version}</version>
</dependency>
<dependency>
<groupId>com.alibaba.mq-amqp</groupId>
<artifactId>mq-amqp-client</artifactId>
<version>1.0.3</version>
</dependency>
<dependency>
<groupId>javax.annotation</groupId>
<artifactId>javax.annotation-api</artifactId>
server/pom.xml
+6 -30
View File
@@ -25,39 +25,15 @@
<artifactId>canal.instance.manager</artifactId>
<version>${project.version}</version>
</dependency>
<!-- kafka -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.11</artifactId>
<version>1.1.1</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-client</artifactId>
</dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-acl</artifactId>
</dependency>
<dependency>
<groupId>com.rabbitmq</groupId>
<artifactId>amqp-client</artifactId>
</dependency>
<dependency>
<groupId>com.alibaba.mq-amqp</groupId>
<artifactId>mq-amqp-client</artifactId>
</dependency>
<!--kafka_2.11_1.1.1 exclusion掉了netty 的依赖，但CanalServerWithNetty 依赖 netty3,升级kafka至 1.1.1 需要显示加入,否则会启动失败 -->
<dependency>
<groupId>org.jboss.netty</groupId>
<artifactId>netty</artifactId>
<version>3.2.2.Final</version>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.7</version>
</dependency>
<!-- test dependency -->
server/src/main/java/com/alibaba/otter/canal/common/AbstractMQProducer.java
-37
View File
@@ -1,37 +0,0 @@
package com.alibaba.otter.canal.common;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
import com.alibaba.otter.canal.spi.CanalMQProducer;
/**
* @author agapple 2019年9月29日 上午11:17:11
* @since 1.1.5
*/
public abstract class AbstractMQProducer implements CanalMQProducer {
protected ThreadPoolExecutor executor;
@Override
public void init(MQProperties mqProperties) {
int parallelThreadSize = mqProperties.getParallelThreadSize();
executor = new ThreadPoolExecutor(parallelThreadSize,
parallelThreadSize,
0,
TimeUnit.SECONDS,
new ArrayBlockingQueue<Runnable>(parallelThreadSize * 2),
new NamedThreadFactory("MQParallel"),
new ThreadPoolExecutor.CallerRunsPolicy());
}
@Override
public void stop() {
executor.shutdownNow();
executor = null;
}
}
server/src/main/java/com/alibaba/otter/canal/common/MQProperties.java
-368
View File
@@ -1,368 +0,0 @@
package com.alibaba.otter.canal.common;
import java.util.Properties;
/**
* MQ 配置项
*
* @author machengyuan 2018-6-11 下午05:30:49
* @version 1.0.0
*/
public class MQProperties {
private String servers = "127.0.0.1:6667";
private int retries = 0;
private int batchSize = 16384;
private int lingerMs = 1;
private int maxRequestSize = 1048576;
private long bufferMemory = 33554432L;
private boolean filterTransactionEntry = true;
private String producerGroup = "Canal-Producer";
private int canalBatchSize = 50;
private Long canalGetTimeout = 100L;
private boolean flatMessage = true;
private String compressionType = "none";
private String acks = "all";
private String aliyunAccessKey = "";
private String aliyunSecretKey = "";
private Properties properties = new Properties();
private boolean enableMessageTrace = false;
private String accessChannel = null;
private String customizedTraceTopic = null;
private String namespace = "";
// kafka集群是否启动Kerberos认证
private boolean kerberosEnable = false;
// 启动Kerberos认证时配置为krb5.conf文件的路径
private String kerberosKrb5FilePath = "";
// 启动Kerberos认证时配置为jaas.conf文件的路径
private String kerberosJaasFilePath = "";
// rabbitmq 账号
private String username = "";
// rabbitmq 密码
private String password = "";
// rabbitmq 密码
private String vhost = "";
// aliyun 用户ID rabbitmq 阿里云需要使用
private long aliyunUID = 0;
// rabbitmq 交换机
private String exchange = "";
// 消息发送的并行度
private int parallelThreadSize = 8;
// 是否取消根据database进行hash
private boolean databaseHash = true;
public static class CanalDestination {
private String canalDestination;
private String topic;
private Integer partition;
private Integer partitionsNum;
private String partitionHash;
private String dynamicTopic;
public String getCanalDestination() {
return canalDestination;
}
public void setCanalDestination(String canalDestination) {
this.canalDestination = canalDestination;
}
public String getTopic() {
return topic;
}
public void setTopic(String topic) {
this.topic = topic;
}
public Integer getPartition() {
return partition;
}
public void setPartition(Integer partition) {
this.partition = partition;
}
public Integer getPartitionsNum() {
return partitionsNum;
}
public void setPartitionsNum(Integer partitionsNum) {
this.partitionsNum = partitionsNum;
}
public String getPartitionHash() {
return partitionHash;
}
public void setPartitionHash(String partitionHash) {
this.partitionHash = partitionHash;
}
public String getDynamicTopic() {
return dynamicTopic;
}
public void setDynamicTopic(String dynamicTopic) {
this.dynamicTopic = dynamicTopic;
}
}
public String getServers() {
return servers;
}
public void setServers(String servers) {
this.servers = servers;
}
public int getRetries() {
return retries;
}
public void setRetries(int retries) {
this.retries = retries;
}
public int getBatchSize() {
return batchSize;
}
public void setBatchSize(int batchSize) {
this.batchSize = batchSize;
}
public int getLingerMs() {
return lingerMs;
}
public void setLingerMs(int lingerMs) {
this.lingerMs = lingerMs;
}
public long getBufferMemory() {
return bufferMemory;
}
public void setBufferMemory(long bufferMemory) {
this.bufferMemory = bufferMemory;
}
public int getCanalBatchSize() {
return canalBatchSize;
}
public void setCanalBatchSize(int canalBatchSize) {
this.canalBatchSize = canalBatchSize;
}
public Long getCanalGetTimeout() {
return canalGetTimeout;
}
public void setCanalGetTimeout(Long canalGetTimeout) {
this.canalGetTimeout = canalGetTimeout;
}
public boolean getFlatMessage() {
return flatMessage;
}
public void setFlatMessage(boolean flatMessage) {
this.flatMessage = flatMessage;
}
public boolean isFilterTransactionEntry() {
return filterTransactionEntry;
}
public void setFilterTransactionEntry(boolean filterTransactionEntry) {
this.filterTransactionEntry = filterTransactionEntry;
}
public String getProducerGroup() {
return producerGroup;
}
public void setProducerGroup(String producerGroup) {
this.producerGroup = producerGroup;
}
public String getCompressionType() {
return compressionType;
}
public void setCompressionType(String compressionType) {
this.compressionType = compressionType;
}
public String getAcks() {
return acks;
}
public void setAcks(String acks) {
this.acks = acks;
}
public String getAliyunAccessKey() {
return aliyunAccessKey;
}
public void setAliyunAccessKey(String aliyunAccessKey) {
this.aliyunAccessKey = aliyunAccessKey;
}
public String getAliyunSecretKey() {
return aliyunSecretKey;
}
public void setAliyunSecretKey(String aliyunSecretKey) {
this.aliyunSecretKey = aliyunSecretKey;
}
public int getMaxRequestSize() {
return maxRequestSize;
}
public void setMaxRequestSize(int maxRequestSize) {
this.maxRequestSize = maxRequestSize;
}
public Properties getProperties() {
return properties;
}
public void setProperties(Properties properties) {
this.properties = properties;
}
public boolean isEnableMessageTrace() {
return enableMessageTrace;
}
public void setEnableMessageTrace(boolean enableMessageTrace) {
this.enableMessageTrace = enableMessageTrace;
}
public String getAccessChannel() {
return accessChannel;
}
public void setAccessChannel(String accessChannel) {
this.accessChannel = accessChannel;
}
public String getCustomizedTraceTopic() {
return customizedTraceTopic;
}
public void setCustomizedTraceTopic(String customizedTraceTopic) {
this.customizedTraceTopic = customizedTraceTopic;
}
public String getNamespace() {
return namespace;
}
public void setNamespace(String namespace) {
this.namespace = namespace;
}
public boolean isKerberosEnable() {
return kerberosEnable;
}
public void setKerberosEnable(boolean kerberosEnable) {
this.kerberosEnable = kerberosEnable;
}
public String getKerberosKrb5FilePath() {
return kerberosKrb5FilePath;
}
public void setKerberosKrb5FilePath(String kerberosKrb5FilePath) {
this.kerberosKrb5FilePath = kerberosKrb5FilePath;
}
public String getKerberosJaasFilePath() {
return kerberosJaasFilePath;
}
public void setKerberosJaasFilePath(String kerberosJaasFilePath) {
this.kerberosJaasFilePath = kerberosJaasFilePath;
}
public void setPassword(String password) {
this.password = password;
}
public String getPassword() {
return password;
}
public void setUsername(String username) {
this.username = username;
}
public String getUsername() {
return username;
}
public void setVhost(String vhost) {
this.vhost = vhost;
}
public String getVhost() {
return vhost;
}
public long getAliyunUID() {
return aliyunUID;
}
public void setAliyunUID(long aliyunUID) {
this.aliyunUID = aliyunUID;
}
public String getExchange() {
return exchange;
}
public void setExchange(String exchange) {
this.exchange = exchange;
}
public int getParallelThreadSize() {
return parallelThreadSize;
}
public void setParallelThreadSize(int parallelThreadSize) {
this.parallelThreadSize = parallelThreadSize;
}
public boolean getDatabaseHash() {
return databaseHash;
}
public void setDatabaseHash(boolean databaseHash) {
this.databaseHash = databaseHash;
}
@Override
public String toString() {
return "MQProperties [servers=" + servers + ", retries=" + retries + ", batchSize=" + batchSize + ", lingerMs="
+ lingerMs + ", maxRequestSize=" + maxRequestSize + ", bufferMemory=" + bufferMemory
+ ", filterTransactionEntry=" + filterTransactionEntry + ", producerGroup=" + producerGroup
+ ", canalBatchSize=" + canalBatchSize + ", canalGetTimeout=" + canalGetTimeout + ", flatMessage="
+ flatMessage + ", compressionType=" + compressionType + ", acks=" + acks + ", aliyunAccessKey="
+ aliyunAccessKey + ", aliyunSecretKey=" + aliyunSecretKey + ", properties=" + properties
+ ", enableMessageTrace=" + enableMessageTrace + ", accessChannel=" + accessChannel
+ ", customizedTraceTopic=" + customizedTraceTopic + ", namespace=" + namespace + ", kerberosEnable="
+ kerberosEnable + ", kerberosKrb5FilePath=" + kerberosKrb5FilePath + ", kerberosJaasFilePath="
+ kerberosJaasFilePath + ", username=" + username + ", password=" + password + ", vhost=" + vhost
+ ", aliyunUID=" + aliyunUID + ", exchange=" + exchange + ", parallelThreadSize=" + parallelThreadSize
+ ",databaseHash=" + databaseHash + "]";
}
}
server/src/main/java/com/alibaba/otter/canal/rabbitmq/CanalRabbitMQProducer.java
-145
View File
@@ -1,145 +0,0 @@
package com.alibaba.otter.canal.rabbitmq;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeoutException;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.alibaba.otter.canal.common.AbstractMQProducer;
import com.alibaba.otter.canal.common.CanalMessageSerializer;
import com.alibaba.otter.canal.common.MQMessageUtils;
import com.alibaba.otter.canal.common.MQMessageUtils.EntryRowData;
import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.server.exception.CanalServerException;
import com.alibaba.otter.canal.spi.CanalMQProducer;
import com.rabbitmq.client.Channel;
import com.rabbitmq.client.Connection;
import com.rabbitmq.client.ConnectionFactory;
public class CanalRabbitMQProducer extends AbstractMQProducer implements CanalMQProducer {
private static final Logger logger = LoggerFactory.getLogger(CanalRabbitMQProducer.class);
private MQProperties mqProperties;
private Connection connect;
private Channel channel;
@Override
public void init(MQProperties mqProperties) {
super.init(mqProperties);
this.mqProperties = mqProperties;
ConnectionFactory factory = new ConnectionFactory();
factory.setHost(mqProperties.getServers());
if (mqProperties.getAliyunAccessKey().length() > 0 && mqProperties.getAliyunSecretKey().length() > 0
&& mqProperties.getAliyunUID() > 0) {
factory.setCredentialsProvider(new AliyunCredentialsProvider(mqProperties.getAliyunAccessKey(),
mqProperties.getAliyunSecretKey(),
mqProperties.getAliyunUID()));
} else {
factory.setUsername(mqProperties.getUsername());
factory.setPassword(mqProperties.getPassword());
}
factory.setVirtualHost(mqProperties.getVhost());
try {
connect = factory.newConnection();
channel = connect.createChannel();
// channel.exchangeDeclare(mqProperties.getExchange(), "topic");
} catch (IOException | TimeoutException ex) {
throw new CanalServerException("Start RabbitMQ producer error", ex);
}
}
@Override
public void send(final MQProperties.CanalDestination canalDestination, Message message, Callback callback)
throws IOException {
ExecutorTemplate template = new ExecutorTemplate(executor);
try {
if (!StringUtils.isEmpty(canalDestination.getDynamicTopic())) {
// 动态topic
Map<String, com.alibaba.otter.canal.protocol.Message> messageMap = MQMessageUtils.messageTopics(message,
canalDestination.getTopic(),
canalDestination.getDynamicTopic());
for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) {
final String topicName = entry.getKey().replace('.', '_');
final com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue();
template.submit(new Runnable() {
@Override
public void run() {
send(canalDestination, topicName, messageSub);
}
});
}
template.waitForResult();
} else {
send(canalDestination, canalDestination.getTopic(), message);
}
callback.commit();
} catch (Throwable e) {
logger.error(e.getMessage(), e);
callback.rollback();
} finally {
template.clear();
}
}
private void send(MQProperties.CanalDestination canalDestination, String topicName, Message messageSub) {
if (!mqProperties.getFlatMessage()) {
byte[] message = CanalMessageSerializer.serializer(messageSub, mqProperties.isFilterTransactionEntry());
if (logger.isDebugEnabled()) {
logger.debug("send message:{} to destination:{}", message, canalDestination.getCanalDestination());
}
sendMessage(topicName, message);
} else {
// 并发构造
EntryRowData[] datas = MQMessageUtils.buildMessageData(messageSub, executor);
// 串行分区
List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, messageSub.getId());
if (flatMessages != null) {
for (FlatMessage flatMessage : flatMessages) {
byte[] message = JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue);
if (logger.isDebugEnabled()) {
logger.debug("send message:{} to destination:{}",
message,
canalDestination.getCanalDestination());
}
sendMessage(topicName, message);
}
}
}
}
private void sendMessage(String queueName, byte[] message) {
// tips: 目前逻辑中暂不处理对exchange处理，请在Console后台绑定 才可使用routekey
try {
channel.basicPublish(mqProperties.getExchange(), queueName, null, message);
} catch (Throwable e) {
throw new RuntimeException(e);
}
}
@Override
public void stop() {
logger.info("## Stop RabbitMQ producer##");
try {
this.connect.close();
this.channel.close();
} catch (IOException | TimeoutException ex) {
throw new CanalServerException("Stop RabbitMQ producer error", ex);
}
super.stop();
}
}
server/src/main/java/com/alibaba/otter/canal/rocketmq/CanalRocketMQProducer.java
-285
View File
@@ -1,285 +0,0 @@
package com.alibaba.otter.canal.rocketmq;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.commons.lang.StringUtils;
import org.apache.rocketmq.acl.common.AclClientRPCHook;
import org.apache.rocketmq.acl.common.SessionCredentials;
import org.apache.rocketmq.client.AccessChannel;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;
import org.apache.rocketmq.client.impl.producer.TopicPublishInfo;
import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.client.producer.MessageQueueSelector;
import org.apache.rocketmq.client.producer.SendResult;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.remoting.RPCHook;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.alibaba.otter.canal.common.AbstractMQProducer;
import com.alibaba.otter.canal.common.CanalMessageSerializer;
import com.alibaba.otter.canal.common.MQMessageUtils;
import com.alibaba.otter.canal.common.MQMessageUtils.EntryRowData;
import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.server.exception.CanalServerException;
import com.alibaba.otter.canal.spi.CanalMQProducer;
public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQProducer {
private static final Logger logger = LoggerFactory.getLogger(CanalRocketMQProducer.class);
private DefaultMQProducer defaultMQProducer;
private MQProperties mqProperties;
private static final String CLOUD_ACCESS_CHANNEL = "cloud";
@Override
public void init(MQProperties rocketMQProperties) {
super.init(rocketMQProperties);
this.mqProperties = rocketMQProperties;
RPCHook rpcHook = null;
if (rocketMQProperties.getAliyunAccessKey().length() > 0
&& rocketMQProperties.getAliyunSecretKey().length() > 0) {
SessionCredentials sessionCredentials = new SessionCredentials();
sessionCredentials.setAccessKey(rocketMQProperties.getAliyunAccessKey());
sessionCredentials.setSecretKey(rocketMQProperties.getAliyunSecretKey());
rpcHook = new AclClientRPCHook(sessionCredentials);
}
defaultMQProducer = new DefaultMQProducer(rocketMQProperties.getProducerGroup(),
rpcHook,
mqProperties.isEnableMessageTrace(),
mqProperties.getCustomizedTraceTopic());
if (CLOUD_ACCESS_CHANNEL.equals(rocketMQProperties.getAccessChannel())) {
defaultMQProducer.setAccessChannel(AccessChannel.CLOUD);
}
if (!StringUtils.isEmpty(mqProperties.getNamespace())) {
defaultMQProducer.setNamespace(mqProperties.getNamespace());
}
defaultMQProducer.setNamesrvAddr(rocketMQProperties.getServers());
defaultMQProducer.setRetryTimesWhenSendFailed(rocketMQProperties.getRetries());
defaultMQProducer.setVipChannelEnabled(false);
logger.info("##Start RocketMQ producer##");
try {
defaultMQProducer.start();
} catch (MQClientException ex) {
throw new CanalServerException("Start RocketMQ producer error", ex);
}
}
@Override
public void send(final MQProperties.CanalDestination destination, com.alibaba.otter.canal.protocol.Message data,
Callback callback) {
ExecutorTemplate template = new ExecutorTemplate(executor);
try {
if (!StringUtils.isEmpty(destination.getDynamicTopic())) {
// 动态topic
Map<String, com.alibaba.otter.canal.protocol.Message> messageMap = MQMessageUtils.messageTopics(data,
destination.getTopic(),
destination.getDynamicTopic());
for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) {
String topicName = entry.getKey().replace('.', '_');
com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue();
template.submit(new Runnable() {
@Override
public void run() {
try {
send(destination, topicName, messageSub);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
});
}
template.waitForResult();
} else {
send(destination, destination.getTopic(), data);
}
callback.commit();
} catch (Throwable e) {
logger.error(e.getMessage(), e);
callback.rollback();
} finally {
template.clear();
}
}
public void send(final MQProperties.CanalDestination destination, String topicName,
com.alibaba.otter.canal.protocol.Message message) throws Exception {
if (!mqProperties.getFlatMessage()) {
if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
// 并发构造
EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor);
// 串行分区
com.alibaba.otter.canal.protocol.Message[] messages = MQMessageUtils.messagePartition(datas,
message.getId(),
destination.getPartitionsNum(),
destination.getPartitionHash(),
mqProperties.getDatabaseHash());
int length = messages.length;
ExecutorTemplate template = new ExecutorTemplate(executor);
for (int i = 0; i < length; i++) {
com.alibaba.otter.canal.protocol.Message dataPartition = messages[i];
if (dataPartition != null) {
final int index = i;
template.submit(new Runnable() {
@Override
public void run() {
Message data = new Message(topicName, CanalMessageSerializer.serializer(dataPartition,
mqProperties.isFilterTransactionEntry()));
sendMessage(data, index);
}
});
}
}
// 等所有分片发送完毕
template.waitForResult();
} else {
final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
Message data = new Message(topicName, CanalMessageSerializer.serializer(message,
mqProperties.isFilterTransactionEntry()));
sendMessage(data, partition);
}
} else {
// 并发构造
EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor);
// 串行分区
List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, message.getId());
if (flatMessages != null) {
// 初始化分区合并队列
if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
List<List<FlatMessage>> partitionFlatMessages = new ArrayList<List<FlatMessage>>();
for (int i = 0; i < destination.getPartitionsNum(); i++) {
partitionFlatMessages.add(new ArrayList<FlatMessage>());
}
for (FlatMessage flatMessage : flatMessages) {
FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage,
destination.getPartitionsNum(),
destination.getPartitionHash(),
mqProperties.getDatabaseHash());
int length = partitionFlatMessage.length;
for (int i = 0; i < length; i++) {
partitionFlatMessages.get(i).add(partitionFlatMessage[i]);
}
}
ExecutorTemplate template = new ExecutorTemplate(executor);
for (int i = 0; i < partitionFlatMessages.size(); i++) {
final List<FlatMessage> flatMessagePart = partitionFlatMessages.get(i);
if (flatMessagePart != null) {
final int index = i;
template.submit(new Runnable() {
@Override
public void run() {
List<Message> messages = flatMessagePart.stream()
.map(flatMessage -> new Message(topicName, JSON.toJSONBytes(flatMessage,
SerializerFeature.WriteMapNullValue)))
.collect(Collectors.toList());
// 批量发送
sendMessage(messages, index);
}
});
}
}
// 批量等所有分区的结果
template.waitForResult();
} else {
final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
List<Message> messages = flatMessages.stream()
.map(flatMessage -> new Message(topicName, JSON.toJSONBytes(flatMessage,
SerializerFeature.WriteMapNullValue)))
.collect(Collectors.toList());
// 批量发送
sendMessage(messages, partition);
}
}
}
}
private void sendMessage(Message message, int partition) {
try {
SendResult sendResult = this.defaultMQProducer.send(message, new MessageQueueSelector() {
@Override
public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {
if (partition > mqs.size()) {
return mqs.get(partition % mqs.size());
} else {
return mqs.get(partition);
}
}
}, null);
if (logger.isDebugEnabled()) {
logger.debug("Send Message Result: {}", sendResult);
}
} catch (Throwable e) {
throw new RuntimeException(e);
}
}
@SuppressWarnings("deprecation")
private void sendMessage(List<Message> messages, int partition) {
if (messages.isEmpty()) {
return;
}
// 获取一下messageQueue
DefaultMQProducerImpl innerProducer = this.defaultMQProducer.getDefaultMQProducerImpl();
TopicPublishInfo topicInfo = innerProducer.getTopicPublishInfoTable().get(messages.get(0).getTopic());
if (topicInfo == null) {
for (Message message : messages) {
sendMessage(message, partition);
}
} else {
// 批量发送
List<MessageQueue> queues = topicInfo.getMessageQueueList();
int size = queues.size();
if (size <= 0) {
// 可能是第一次创建
for (Message message : messages) {
sendMessage(message, partition);
}
} else {
MessageQueue queue = null;
if (partition > size) {
queue = queues.get(partition % size);
} else {
queue = queues.get(partition);
}
try {
SendResult sendResult = this.defaultMQProducer.send(messages, queue);
if (logger.isDebugEnabled()) {
logger.debug("Send Message Result: {}", sendResult);
}
} catch (Throwable e) {
throw new RuntimeException(e);
}
}
}
}
@Override
public void stop() {
logger.info("## Stop RocketMQ producer##");
this.defaultMQProducer.shutdown();
super.stop();
}
}
server/src/main/java/com/alibaba/otter/canal/server/CanalMQStarter.java
+15 -15
View File
@@ -7,18 +7,21 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import com.alibaba.otter.canal.connector.core.util.Callback;
import com.alibaba.otter.canal.connector.core.producer.MQDestination;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
import com.alibaba.otter.canal.instance.core.CanalInstance;
import com.alibaba.otter.canal.instance.core.CanalMQConfig;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;
import com.alibaba.otter.canal.spi.CanalMQProducer;
import com.alibaba.otter.canal.connector.core.config.MQProperties;
public class CanalMQStarter {
@@ -30,7 +33,7 @@ public class CanalMQStarter {
private CanalMQProducer canalMQProducer;
private MQProperties properties;
private MQProperties mqProperties;
private CanalServerWithEmbedded canalServer;
@@ -42,15 +45,14 @@ public class CanalMQStarter {
this.canalMQProducer = canalMQProducer;
}
public synchronized void start(MQProperties properties, String destinations) {
public synchronized void start(String destinations) {
try {
if (running) {
return;
}
this.properties = properties;
canalMQProducer.init(properties);
mqProperties = canalMQProducer.getMqProperties();
// set filterTransactionEntry
if (properties.isFilterTransactionEntry()) {
if (mqProperties.isFilterTransactionEntry()) {
System.setProperty("canal.instance.filter.transaction.entry", "true");
}
@@ -151,7 +153,7 @@ public class CanalMQStarter {
}
continue;
}
MQProperties.CanalDestination canalDestination = new MQProperties.CanalDestination();
MQDestination canalDestination = new MQDestination();
canalDestination.setCanalDestination(destination);
CanalMQConfig mqConfig = canalInstance.getMqConfig();
canalDestination.setTopic(mqConfig.getTopic());
@@ -163,15 +165,13 @@ public class CanalMQStarter {
canalServer.subscribe(clientIdentity);
logger.info("## the MQ producer: {} is running now ......", destination);
Long getTimeout = properties.getCanalGetTimeout();
int getBatchSize = properties.getCanalBatchSize();
Integer getTimeout = mqProperties.getFetchTimeout();
Integer getBatchSize = mqProperties.getBatchSize();
while (running && destinationRunning.get()) {
Message message;
if (getTimeout != null && getTimeout > 0) {
message = canalServer.getWithoutAck(clientIdentity,
getBatchSize,
getTimeout,
TimeUnit.MILLISECONDS);
message = canalServer
.getWithoutAck(clientIdentity, getBatchSize, getTimeout.longValue(), TimeUnit.MILLISECONDS);
} else {
message = canalServer.getWithoutAck(clientIdentity, getBatchSize);
}
@@ -180,7 +180,7 @@ public class CanalMQStarter {
try {
int size = message.isRaw() ? message.getRawEntries().size() : message.getEntries().size();
if (batchId != -1 && size != 0) {
canalMQProducer.send(canalDestination, message, new CanalMQProducer.Callback() {
canalMQProducer.send(canalDestination, message, new Callback() {
@Override
public void commit() {
server/src/main/java/com/alibaba/otter/canal/spi/CanalMQProducer.java
-37
View File
@@ -1,37 +0,0 @@
package com.alibaba.otter.canal.spi;
import java.io.IOException;
import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.protocol.Message;
public interface CanalMQProducer {
/**
* Init producer.
*
* @param mqProperties MQ config
*/
void init(MQProperties mqProperties);
/**
* Send canal message to related topic
*
* @param canalDestination canal mq destination
* @param message canal message
* @throws IOException
*/
void send(MQProperties.CanalDestination canalDestination, Message message, Callback callback) throws IOException;
/**
* Stop MQ producer service
*/
void stop();
interface Callback {
void commit();
void rollback();
}
}