diff --git a/client-adapter/common/pom.xml b/client-adapter/common/pom.xml index 8775623e..2c9a7b5a 100644 --- a/client-adapter/common/pom.xml +++ b/client-adapter/common/pom.xml @@ -1,5 +1,6 @@ - + canal.client-adapter com.alibaba.otter @@ -17,6 +18,11 @@ canal.protocol ${project.version} + + com.alibaba.otter + connector.core + ${project.version} + org.springframework spring-core @@ -49,7 +55,7 @@ com.alibaba druid - + org.yaml snakeyaml diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java index 466ed20f..277c1348 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java @@ -3,6 +3,7 @@ package com.alibaba.otter.canal.client.adapter.support; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Properties; /** * 配置信息类 @@ -27,7 +28,7 @@ public class CanalClientConfig { // 重试次数 private Integer retries; // 消费超时时间 - private Long timeout; + private Long timeout = 500L; // 模式 tcp kafka rocketMQ private String mode = "tcp"; // aliyun ak/sk @@ -39,6 +40,8 @@ public class CanalClientConfig { // rabbitmq vhost private String vhost = "/"; + private Properties consumerProperties; + private Long resourceOwnerId; // 是否启用消息轨迹 private boolean enableMessageTrace; @@ -163,6 +166,14 @@ public class CanalClientConfig { this.vhost = vhost; } + public Properties getConsumerProperties() { + return consumerProperties; + } + + public void setConsumerProperties(Properties consumerProperties) { + this.consumerProperties = consumerProperties; + } + public Long getResourceOwnerId() { return resourceOwnerId; } diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java index 618f81d6..d0b42951 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java @@ -2,6 +2,7 @@ package com.alibaba.otter.canal.client.adapter.support; import java.util.*; +import com.alibaba.otter.canal.connector.core.consumer.CommonMessage; import com.alibaba.otter.canal.protocol.CanalEntry; import com.alibaba.otter.canal.protocol.FlatMessage; import com.alibaba.otter.canal.protocol.Message; @@ -131,10 +132,13 @@ public class MessageUtil { return dmls; } - public static List flatMessage2Dml(String destination, String groupId, List flatMessages) { - List dmls = new ArrayList(flatMessages.size()); - for (FlatMessage flatMessage : flatMessages) { - Dml dml = flatMessage2Dml(destination, groupId, flatMessage); + public static List flatMessage2Dml(String destination, String groupId, List commonMessages) { + if (commonMessages == null) { + return new ArrayList<>(); + } + List dmls = new ArrayList(commonMessages.size()); + for (CommonMessage commonMessage : commonMessages) { + Dml dml = flatMessage2Dml(destination, groupId, commonMessage); if (dml != null) { dmls.add(dml); } @@ -143,31 +147,31 @@ public class MessageUtil { return dmls; } - public static Dml flatMessage2Dml(String destination, String groupId, FlatMessage flatMessage) { - if (flatMessage == null) { + public static Dml flatMessage2Dml(String destination, String groupId, CommonMessage commonMessage) { + if (commonMessage == null) { return null; } Dml dml = new Dml(); dml.setDestination(destination); dml.setGroupId(groupId); - dml.setDatabase(flatMessage.getDatabase()); - dml.setTable(flatMessage.getTable()); - dml.setPkNames(flatMessage.getPkNames()); - dml.setIsDdl(flatMessage.getIsDdl()); - dml.setType(flatMessage.getType()); - dml.setTs(flatMessage.getTs()); - dml.setEs(flatMessage.getEs()); - dml.setSql(flatMessage.getSql()); + dml.setDatabase(commonMessage.getDatabase()); + dml.setTable(commonMessage.getTable()); + dml.setPkNames(commonMessage.getPkNames()); + dml.setIsDdl(commonMessage.getIsDdl()); + dml.setType(commonMessage.getType()); + dml.setTs(commonMessage.getTs()); + dml.setEs(commonMessage.getEs()); + dml.setSql(commonMessage.getSql()); // if (flatMessage.getSqlType() == null || flatMessage.getMysqlType() == null) { // throw new RuntimeException("SqlType or mysqlType is null"); // } - List> data = flatMessage.getData(); + List> data = commonMessage.getData(); if (data != null) { - dml.setData(changeRows(dml.getTable(), data, flatMessage.getSqlType(), flatMessage.getMysqlType())); + dml.setData(data); } - List> old = flatMessage.getOld(); + List> old = commonMessage.getOld(); if (old != null) { - dml.setOld(changeRows(dml.getTable(), old, flatMessage.getSqlType(), flatMessage.getMysqlType())); + dml.setOld(old); } return dml; } diff --git a/client-adapter/launcher/pom.xml b/client-adapter/launcher/pom.xml index 64e110b1..311254a3 100644 --- a/client-adapter/launcher/pom.xml +++ b/client-adapter/launcher/pom.xml @@ -30,11 +30,6 @@ client-adapter.common ${project.version} - - com.alibaba.otter - canal.client - ${project.version} - org.yaml snakeyaml @@ -56,25 +51,6 @@ org.apache.curator curator-recipes - - - org.apache.rocketmq - rocketmq-client - - - - org.apache.kafka - kafka-clients - - - - com.rabbitmq - amqp-client - - - com.alibaba.mq-amqp - mq-amqp-client - mysql @@ -159,6 +135,59 @@ jar-with-dependencies provided + + + com.alibaba.otter + connector.tcp + ${project.version} + + + * + * + + + jar-with-dependencies + provided + + + com.alibaba.otter + connector.kafka + ${project.version} + + + * + * + + + jar-with-dependencies + provided + + + com.alibaba.otter + connector.rocketmq + ${project.version} + + + * + * + + + jar-with-dependencies + provided + + + com.alibaba.otter + connector.rabbitmq + ${project.version} + + + * + * + + + jar-with-dependencies + provided + diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java deleted file mode 100644 index 9a4eede9..00000000 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java +++ /dev/null @@ -1,263 +0,0 @@ -package com.alibaba.otter.canal.adapter.launcher.loader; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.*; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch; -import com.alibaba.otter.canal.adapter.launcher.config.SpringContext; -import com.alibaba.otter.canal.client.CanalMQConnector; -import com.alibaba.otter.canal.client.adapter.OuterAdapter; -import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; -import com.alibaba.otter.canal.client.adapter.support.Dml; -import com.alibaba.otter.canal.client.adapter.support.MessageUtil; -import com.alibaba.otter.canal.client.adapter.support.Util; -import com.alibaba.otter.canal.protocol.FlatMessage; -import com.alibaba.otter.canal.protocol.Message; - -/** - * 适配器工作线程抽象类 - * - * @author rewerma 2018-8-19 下午11:30:49 - * @version 1.0.0 - */ -public abstract class AbstractCanalAdapterWorker { - - protected final Logger logger = LoggerFactory.getLogger(this.getClass()); - - protected String canalDestination; // canal实例 - protected String groupId = null; // groupId - protected List> canalOuterAdapters; // 外部适配器 - protected CanalClientConfig canalClientConfig; // 配置 - protected ExecutorService groupInnerExecutorService; // 组内工作线程池 - protected volatile boolean running = false; // 是否运行中 - protected Thread thread = null; - protected Thread.UncaughtExceptionHandler handler = (t, e) -> logger.error("parse events has an error", e); - - protected SyncSwitch syncSwitch; - - public AbstractCanalAdapterWorker(List> canalOuterAdapters){ - this.canalOuterAdapters = canalOuterAdapters; - this.groupInnerExecutorService = Util.newFixedThreadPool(canalOuterAdapters.size(), 5000L); - syncSwitch = (SyncSwitch) SpringContext.getBean(SyncSwitch.class); - } - - protected void writeOut(final Message message) { - List> futures = new ArrayList<>(); - // 组间适配器并行运行 - canalOuterAdapters.forEach(outerAdapters -> { - final List adapters = outerAdapters; - futures.add(groupInnerExecutorService.submit(() -> { - try { - // 组内适配器穿行运行，尽量不要配置组内适配器 - adapters.forEach(adapter -> { - long begin = System.currentTimeMillis(); - List dmls = MessageUtil.parse4Dml(canalDestination, groupId, message); - if (dmls != null) { - batchSync(dmls, adapter); - - if (logger.isDebugEnabled()) { - logger.debug("{} elapsed time: {}", - adapter.getClass().getName(), - (System.currentTimeMillis() - begin)); - } - } - }); - return true; - } catch (Exception e) { - logger.error(e.getMessage(), e); - return false; - } - })); - - // 等待所有适配器写入完成 - // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成 - RuntimeException exception = null; - for (Future future : futures) { - try { - if (!future.get()) { - exception = new RuntimeException("Outer adapter sync failed! "); - } - } catch (Exception e) { - exception = new RuntimeException(e); - } - } - if (exception != null) { - throw exception; - } - }); - } - - protected void writeOut(final List flatMessages) { - List> futures = new ArrayList<>(); - // 组间适配器并行运行 - canalOuterAdapters.forEach(outerAdapters -> { - futures.add(groupInnerExecutorService.submit(() -> { - try { - // 组内适配器穿行运行，尽量不要配置组内适配器 - outerAdapters.forEach(adapter -> { - long begin = System.currentTimeMillis(); - List dmls = MessageUtil.flatMessage2Dml(canalDestination, groupId, flatMessages); - batchSync(dmls, adapter); - - if (logger.isDebugEnabled()) { - logger.debug("{} elapsed time: {}", - adapter.getClass().getName(), - (System.currentTimeMillis() - begin)); - } - }); - return true; - } catch (Exception e) { - logger.error(e.getMessage(), e); - return false; - } - })); - - // 等待所有适配器写入完成 - // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成 - RuntimeException exception = null; - for (Future future : futures) { - try { - if (!future.get()) { - exception = new RuntimeException("Outer adapter sync failed! "); - } - } catch (Exception e) { - exception = new RuntimeException(e); - } - } - if (exception != null) { - throw exception; - } - }); - } - - @SuppressWarnings("unchecked") - protected boolean mqWriteOutData(int retry, long timeout, int i, final boolean flatMessage, - CanalMQConnector connector) { - ExecutorService workerExecutor = Util.newSingleDaemonThreadExecutor(5000); - try { - List messages; - if (!flatMessage) { - messages = connector.getListWithoutAck(100L, TimeUnit.MILLISECONDS); - } else { - messages = connector.getFlatListWithoutAck(100L, TimeUnit.MILLISECONDS); - } - if (messages != null && !messages.isEmpty()) { - Future future = workerExecutor.submit(() -> { - if (flatMessage) { - // batch write - writeOut((List) messages); - // FIXME xxx - // messages.forEach((message -> - // System.out.println(JSON.toJSONString(message)))); - } else { - for (final Object message : messages) { - writeOut((Message) message); - } - } - return true; - }); - - try { - future.get(timeout, TimeUnit.MILLISECONDS); - } catch (Exception e) { - future.cancel(true); - throw e; - } - connector.ack(); - } - return true; - } catch (Throwable e) { - if (i == retry - 1) { - connector.ack(); - logger.error(e.getMessage() + " Error sync but ACK!"); - return true; - } else { - connector.rollback(); - logger.error(e.getMessage() + " Error sync and rollback, execute times: " + (i + 1)); - } - try { - Thread.sleep(500); - } catch (InterruptedException e1) { - // ignore - } - } finally { - workerExecutor.shutdown(); - } - return false; - } - - /** - * 分批同步 - * - * @param dmls - * @param adapter - */ - private void batchSync(List dmls, OuterAdapter adapter) { - // 分批同步 - if (dmls.size() <= canalClientConfig.getSyncBatchSize()) { - adapter.sync(dmls); - } else { - int len = 0; - List dmlsBatch = new ArrayList<>(); - for (Dml dml : dmls) { - dmlsBatch.add(dml); - if (dml.getData() == null || dml.getData().isEmpty()) { - len += 1; - } else { - len += dml.getData().size(); - } - if (len >= canalClientConfig.getSyncBatchSize()) { - adapter.sync(dmlsBatch); - dmlsBatch.clear(); - len = 0; - } - } - if (!dmlsBatch.isEmpty()) { - adapter.sync(dmlsBatch); - } - } - } - - public void start() { - if (!running) { - thread = new Thread(this::process); - thread.setUncaughtExceptionHandler(handler); - thread.start(); - running = true; - } - } - - protected abstract void process(); - - public void stop() { - try { - if (!running) { - return; - } - - running = false; - - syncSwitch.release(canalDestination); - - logger.info("destination {} is waiting for adapters' worker thread die!", canalDestination); - if (thread != null) { - try { - thread.join(); - } catch (InterruptedException e) { - // ignore - } - } - groupInnerExecutorService.shutdown(); - logger.info("destination {} adapters worker thread dead!", canalDestination); - canalOuterAdapters.forEach(outerAdapters -> outerAdapters.forEach(OuterAdapter::destroy)); - logger.info("destination {} all adapters destroyed!", canalDestination); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - -} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AdapterProcessor.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AdapterProcessor.java new file mode 100644 index 00000000..287472b5 --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AdapterProcessor.java @@ -0,0 +1,268 @@ +package com.alibaba.otter.canal.adapter.launcher.loader; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import com.alibaba.otter.canal.adapter.launcher.config.SpringContext; +import com.alibaba.otter.canal.client.adapter.support.Util; +import com.alibaba.otter.canal.connector.core.config.CanalConstants; +import com.alibaba.otter.canal.connector.core.spi.ExtensionLoader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch; +import com.alibaba.otter.canal.client.adapter.OuterAdapter; +import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; +import com.alibaba.otter.canal.client.adapter.support.Dml; +import com.alibaba.otter.canal.client.adapter.support.MessageUtil; +import com.alibaba.otter.canal.connector.core.consumer.CommonMessage; +import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer; + +/** + * 适配处理器 + * + * @author rewerma 2020-02-01 + * @version 1.0.0 + */ +public class AdapterProcessor { + + private static final Logger logger = LoggerFactory.getLogger(AdapterProcessor.class); + + private static final String CONNECTOR_SPI_DIR = "/plugin"; + private static final String CONNECTOR_STANDBY_SPI_DIR = "/canal-adapter/plugin"; + + private CanalMsgConsumer canalMsgConsumer; + + private String canalDestination; // canal实例 + private String groupId = null; // groupId + private List> canalOuterAdapters; // 外部适配器 + private CanalClientConfig canalClientConfig; // 配置 + private ExecutorService groupInnerExecutorService; // 组内工作线程池 + private volatile boolean running = false; // 是否运行中 + private Thread thread = null; + private Thread.UncaughtExceptionHandler handler = (t, e) -> logger + .error("parse events has an error", e); + + private SyncSwitch syncSwitch; + + public AdapterProcessor(CanalClientConfig canalClientConfig, String destination, String groupId, + List> canalOuterAdapters){ + this.canalClientConfig = canalClientConfig; + this.canalDestination = destination; + this.groupId = groupId; + this.canalOuterAdapters = canalOuterAdapters; + + this.groupInnerExecutorService = Util.newFixedThreadPool(canalOuterAdapters.size(), 5000L); + syncSwitch = (SyncSwitch) SpringContext.getBean(SyncSwitch.class); + + // load connector consumer + ExtensionLoader loader = new ExtensionLoader<>(CanalMsgConsumer.class); + canalMsgConsumer = loader + .getExtension(canalClientConfig.getMode().toLowerCase(), CONNECTOR_SPI_DIR, CONNECTOR_STANDBY_SPI_DIR); + + Properties properties = canalClientConfig.getConsumerProperties(); + properties.put(CanalConstants.CANAL_MQ_FLAT_MESSAGE, canalClientConfig.getFlatMessage()); + properties.put(CanalConstants.CANAL_ALIYUN_ACCESS_KEY, canalClientConfig.getAccessKey()); + properties.put(CanalConstants.CANAL_ALIYUN_SECRET_KEY, canalClientConfig.getSecretKey()); + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + Thread.currentThread().setContextClassLoader(canalMsgConsumer.getClass().getClassLoader()); + canalMsgConsumer.init(properties, canalDestination, groupId); + Thread.currentThread().setContextClassLoader(cl); + } + + public void start() { + if (!running) { + thread = new Thread(this::process); + thread.setUncaughtExceptionHandler(handler); + thread.start(); + running = true; + } + } + + public void writeOut(final List commonMessages) { + List> futures = new ArrayList<>(); + // 组间适配器并行运行 + canalOuterAdapters.forEach(outerAdapters -> { + futures.add(groupInnerExecutorService.submit(() -> { + try { + // 组内适配器穿行运行，尽量不要配置组内适配器 + outerAdapters.forEach(adapter -> { + long begin = System.currentTimeMillis(); + List dmls = MessageUtil.flatMessage2Dml(canalDestination, groupId, commonMessages); + batchSync(dmls, adapter); + + if (logger.isDebugEnabled()) { + logger.debug("{} elapsed time: {}", + adapter.getClass().getName(), + (System.currentTimeMillis() - begin)); + } + }); + return true; + } catch (Exception e) { + logger.error(e.getMessage(), e); + return false; + } + })); + + // 等待所有适配器写入完成 + // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成 + RuntimeException exception = null; + for (Future future : futures) { + try { + if (!future.get()) { + exception = new RuntimeException("Outer adapter sync failed! "); + } + } catch (Exception e) { + exception = new RuntimeException(e); + } + } + if (exception != null) { + throw exception; + } + }); + } + + /** + * 分批同步 + * + * @param dmls + * @param adapter + */ + private void batchSync(List dmls, OuterAdapter adapter) { + // 分批同步 + if (dmls.size() <= canalClientConfig.getSyncBatchSize()) { + adapter.sync(dmls); + } else { + int len = 0; + List dmlsBatch = new ArrayList<>(); + for (Dml dml : dmls) { + dmlsBatch.add(dml); + if (dml.getData() == null || dml.getData().isEmpty()) { + len += 1; + } else { + len += dml.getData().size(); + } + if (len >= canalClientConfig.getSyncBatchSize()) { + adapter.sync(dmlsBatch); + dmlsBatch.clear(); + len = 0; + } + } + if (!dmlsBatch.isEmpty()) { + adapter.sync(dmlsBatch); + } + } + } + + private void process() { + while (!running) { // waiting until running == true + while (!running) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + } + } + } + + int retry = canalClientConfig.getRetries() == null + || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries(); + if (retry == -1) { + // 重试次数-1代表异常时一直阻塞重试 + retry = Integer.MAX_VALUE; + } + + while (running) { + try { + syncSwitch.get(canalDestination); + + logger.info("=============> Start to connect destination: {} <=============", this.canalDestination); + canalMsgConsumer.connect(); + logger.info("=============> Subscribe destination: {} succeed <=============", this.canalDestination); + while (running) { + try { + syncSwitch.get(canalDestination, 1L, TimeUnit.MINUTES); + } catch (TimeoutException e) { + break; + } + if (!running) { + break; + } + + for (int i = 0; i < retry; i++) { + if (!running) { + break; + } + try { + if (logger.isDebugEnabled()) { + logger.debug("destination: {} ", canalDestination); + } + long begin = System.currentTimeMillis(); + List commonMessages = canalMsgConsumer + .getMessage(this.canalClientConfig.getTimeout(), TimeUnit.MILLISECONDS); + writeOut(commonMessages); + canalMsgConsumer.ack(); + if (logger.isDebugEnabled()) { + logger.debug("destination: {} elapsed time: {} ms", + canalDestination, + System.currentTimeMillis() - begin); + } + } catch (Exception e) { + if (i != retry - 1) { + canalMsgConsumer.rollback(); // 处理失败, 回滚数据 + logger.error(e.getMessage() + " Error sync and rollback, execute times: " + (i + 1)); + } else { + canalMsgConsumer.ack(); + logger.error(e.getMessage() + " Error sync but ACK!"); + } + Thread.sleep(500); + } + } + } + + canalMsgConsumer.disconnect(); + } catch (Throwable e) { + logger.error("process error!", e); + } + + if (running) { // is reconnect + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + // ignore + } + } + } + } + + public void stop() { + try { + if (!running) { + return; + } + + running = false; + + syncSwitch.release(canalDestination); + + logger.info("destination {} is waiting for adapters' worker thread die!", canalDestination); + if (thread != null) { + try { + thread.join(); + } catch (InterruptedException e) { + // ignore + } + } + groupInnerExecutorService.shutdown(); + logger.info("destination {} adapters worker thread dead!", canalDestination); + canalOuterAdapters.forEach(outerAdapters -> outerAdapters.forEach(OuterAdapter::destroy)); + logger.info("destination {} all adapters destroyed!", canalDestination); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java deleted file mode 100644 index 35fa7072..00000000 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java +++ /dev/null @@ -1,96 +0,0 @@ -package com.alibaba.otter.canal.adapter.launcher.loader; - -import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.TimeUnit; - -import org.apache.kafka.common.errors.WakeupException; - -import com.alibaba.otter.canal.client.adapter.OuterAdapter; -import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; -import com.alibaba.otter.canal.client.adapter.support.Util; -import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector; - -/** - * kafka对应的client适配器工作线程 - * - * @author rewerma 2018-8-19 下午11:30:49 - * @version 1.0.0 - */ -public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker { - - private KafkaCanalConnector connector; - private String topic; - private boolean flatMessage; - - public CanalAdapterKafkaWorker(CanalClientConfig canalClientConfig, String bootstrapServers, String topic, - String groupId, List> canalOuterAdapters, boolean flatMessage){ - super(canalOuterAdapters); - this.canalClientConfig = canalClientConfig; - this.topic = topic; - super.canalDestination = topic; - super.groupId = groupId; - this.flatMessage = flatMessage; - this.connector = new KafkaCanalConnector(bootstrapServers, - topic, - null, - groupId, - canalClientConfig.getBatchSize(), - flatMessage); - connector.setSessionTimeout(30L, TimeUnit.SECONDS); - } - - @Override - protected void process() { - while (!running) { - try { - Thread.sleep(500); - } catch (InterruptedException e) { - // ignore - } - } - int retry = canalClientConfig.getRetries() == null - || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries(); - long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒 - - while (running) { - try { - syncSwitch.get(canalDestination); - logger.info("=============> Start to connect topic: {} <=============", this.topic); - connector.connect(); - logger.info("=============> Start to subscribe topic: {} <=============", this.topic); - connector.subscribe(); - logger.info("=============> Subscribe topic: {} succeed <=============", this.topic); - while (running) { - boolean status = syncSwitch.status(canalDestination); - if (!status) { - connector.disconnect(); - break; - } - if (retry == -1) { - retry = Integer.MAX_VALUE; - } - for (int i = 0; i < retry; i++) { - if (!running) { - break; - } - if (mqWriteOutData(retry, timeout, i, flatMessage, connector)) { - break; - } - } - } - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - - try { - connector.unsubscribe(); - } catch (WakeupException e) { - // No-op. Continue process - } - connector.disconnect(); - logger.info("=============> Disconnect topic: {} <=============", this.topic); - } -} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java index 3217af91..62ec8fa5 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java @@ -37,9 +37,7 @@ public class CanalAdapterLoader { private CanalClientConfig canalClientConfig; - private Map canalWorkers = new HashMap<>(); - - private Map canalMQWorker = new HashMap<>(); + private Map canalAdapterProcessors = new HashMap<>(); private ExtensionLoader loader; @@ -53,120 +51,132 @@ public class CanalAdapterLoader { public void init() { loader = ExtensionLoader.getExtensionLoader(OuterAdapter.class); - String canalServerHost = this.canalClientConfig.getCanalServerHost(); - SocketAddress sa = null; - if (canalServerHost != null) { - String[] ipPort = canalServerHost.split(":"); - sa = new InetSocketAddress(ipPort[0], Integer.parseInt(ipPort[1])); - } - String zkHosts = this.canalClientConfig.getZookeeperHosts(); - - if ("tcp".equalsIgnoreCase(canalClientConfig.getMode())) { - // 初始化canal-client的适配器 - for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) { + for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) { + for (CanalClientConfig.Group group : canalAdapter.getGroups()) { List> canalOuterAdapterGroups = new CopyOnWriteArrayList<>(); - - for (CanalClientConfig.Group connectorGroup : canalAdapter.getGroups()) { - List canalOutConnectors = new CopyOnWriteArrayList<>(); - for (OuterAdapterConfig c : connectorGroup.getOuterAdapters()) { - loadAdapter(c, canalOutConnectors); - } - canalOuterAdapterGroups.add(canalOutConnectors); + List canalOuterAdapters = new CopyOnWriteArrayList<>(); + for (OuterAdapterConfig config : group.getOuterAdapters()) { + loadAdapter(config, canalOuterAdapters); } - CanalAdapterWorker worker; - if (sa != null) { - worker = new CanalAdapterWorker(canalClientConfig, - canalAdapter.getInstance(), - sa, - canalOuterAdapterGroups); - } else if (zkHosts != null) { - worker = new CanalAdapterWorker(canalClientConfig, - canalAdapter.getInstance(), - zkHosts, - canalOuterAdapterGroups); - } else { - throw new RuntimeException("No canal server connector found"); - } - canalWorkers.put(canalAdapter.getInstance(), worker); - worker.start(); - logger.info("Start adapter for canal instance: {} succeed", canalAdapter.getInstance()); - } - } else if ("kafka".equalsIgnoreCase(canalClientConfig.getMode())) { - // 初始化canal-client-kafka的适配器 - for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) { - for (CanalClientConfig.Group group : canalAdapter.getGroups()) { - List> canalOuterAdapterGroups = new CopyOnWriteArrayList<>(); - List canalOuterAdapters = new CopyOnWriteArrayList<>(); - for (OuterAdapterConfig config : group.getOuterAdapters()) { - loadAdapter(config, canalOuterAdapters); - } - canalOuterAdapterGroups.add(canalOuterAdapters); + canalOuterAdapterGroups.add(canalOuterAdapters); - CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(canalClientConfig, - canalClientConfig.getMqServers(), - canalAdapter.getInstance(), - group.getGroupId(), - canalOuterAdapterGroups, - canalClientConfig.getFlatMessage()); - canalMQWorker.put(canalAdapter.getInstance() + "-kafka-" + group.getGroupId(), canalKafkaWorker); - canalKafkaWorker.start(); - logger.info("Start adapter for canal-client mq topic: {} succeed", + AdapterProcessor adapterProcessor = canalAdapterProcessors.computeIfAbsent(canalAdapter.getInstance()+"|"+StringUtils.trimToEmpty(group.getGroupId()), + f-> new AdapterProcessor(canalClientConfig,canalAdapter.getInstance(), + group.getGroupId(), canalOuterAdapterGroups)); + adapterProcessor.start(); + + logger.info("Start adapter for canal-client mq topic: {} succeed", canalAdapter.getInstance() + "-" + group.getGroupId()); - } } - } else if ("rocketMQ".equalsIgnoreCase(canalClientConfig.getMode())) { - // 初始化canal-client-rocketMQ的适配器 - for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) { - for (CanalClientConfig.Group group : canalAdapter.getGroups()) { - List> canalOuterAdapterGroups = new CopyOnWriteArrayList<>(); - List canalOuterAdapters = new CopyOnWriteArrayList<>(); - for (OuterAdapterConfig config : group.getOuterAdapters()) { - loadAdapter(config, canalOuterAdapters); - } - canalOuterAdapterGroups.add(canalOuterAdapters); - CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig, - canalClientConfig.getMqServers(), - canalAdapter.getInstance(), - group.getGroupId(), - canalOuterAdapterGroups, - canalClientConfig.getAccessKey(), - canalClientConfig.getSecretKey(), - canalClientConfig.getFlatMessage(), - canalClientConfig.isEnableMessageTrace(), - canalClientConfig.getCustomizedTraceTopic(), - canalClientConfig.getAccessChannel(), - canalClientConfig.getNamespace()); - canalMQWorker.put(canalAdapter.getInstance() + "-rocketmq-" + group.getGroupId(), rocketMQWorker); - rocketMQWorker.start(); - - logger.info("Start adapter for canal-client mq topic: {} succeed", - canalAdapter.getInstance() + "-" + group.getGroupId()); - } - } - } else if ("rabbitMQ".equalsIgnoreCase(canalClientConfig.getMode())) { - // 初始化canal-client-rabbitMQ的适配器 - for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) { - for (CanalClientConfig.Group group : canalAdapter.getGroups()) { - List> canalOuterAdapterGroups = new CopyOnWriteArrayList<>(); - List canalOuterAdapters = new CopyOnWriteArrayList<>(); - for (OuterAdapterConfig config : group.getOuterAdapters()) { - loadAdapter(config, canalOuterAdapters); - } - canalOuterAdapterGroups.add(canalOuterAdapters); - CanalAdapterRabbitMQWorker rabbitMQWork = new CanalAdapterRabbitMQWorker(canalClientConfig, - canalOuterAdapterGroups, - canalAdapter.getInstance(), - group.getGroupId(), - canalClientConfig.getFlatMessage()); - canalMQWorker.put(canalAdapter.getInstance() + "-rabbitmq-" + group.getGroupId(), rabbitMQWork); - rabbitMQWork.start(); - - logger.info("Start adapter for canal-client mq topic: {} succeed", - canalAdapter.getInstance() + "-" + group.getGroupId()); - } - } - // CanalAdapterRabbitMQWork } + +// if ("tcp".equalsIgnoreCase(canalClientConfig.getMode())) { +// // 初始化canal-client的适配器 +// for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) { +// List> canalOuterAdapterGroups = new CopyOnWriteArrayList<>(); +// +// for (CanalClientConfig.Group connectorGroup : canalAdapter.getGroups()) { +// List canalOutConnectors = new CopyOnWriteArrayList<>(); +// for (OuterAdapterConfig c : connectorGroup.getOuterAdapters()) { +// loadAdapter(c, canalOutConnectors); +// } +// canalOuterAdapterGroups.add(canalOutConnectors); +// } +// CanalAdapterWorker worker; +// if (StringUtils.isNotEmpty(canalServerHost)) { +// worker = new CanalAdapterWorker(canalClientConfig, +// canalAdapter.getInstance(), +// canalServerHost, +// zkHosts, +// canalOuterAdapterGroups); +// } else if (zkHosts != null) { +// worker = new CanalAdapterWorker(canalClientConfig, +// canalAdapter.getInstance(), +// zkHosts, +// canalOuterAdapterGroups); +// } else { +// throw new RuntimeException("No canal server connector found"); +// } +// canalWorkers.put(canalAdapter.getInstance(), worker); +// worker.start(); +// logger.info("Start adapter for canal instance: {} succeed", canalAdapter.getInstance()); +// } +// } else if ("kafka".equalsIgnoreCase(canalClientConfig.getMode())) { +// // 初始化canal-client-kafka的适配器 +// for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) { +// for (CanalClientConfig.Group group : canalAdapter.getGroups()) { +// List> canalOuterAdapterGroups = new CopyOnWriteArrayList<>(); +// List canalOuterAdapters = new CopyOnWriteArrayList<>(); +// for (OuterAdapterConfig config : group.getOuterAdapters()) { +// loadAdapter(config, canalOuterAdapters); +// } +// canalOuterAdapterGroups.add(canalOuterAdapters); +// +// CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(canalClientConfig, +// canalClientConfig.getMqServers(), +// canalAdapter.getInstance(), +// group.getGroupId(), +// canalOuterAdapterGroups, +// canalClientConfig.getFlatMessage()); +// canalMQWorker.put(canalAdapter.getInstance() + "-kafka-" + group.getGroupId(), canalKafkaWorker); +// canalKafkaWorker.start(); +// logger.info("Start adapter for canal-client mq topic: {} succeed", +// canalAdapter.getInstance() + "-" + group.getGroupId()); +// } +// } +// } else if ("rocketMQ".equalsIgnoreCase(canalClientConfig.getMode())) { +// // 初始化canal-client-rocketMQ的适配器 +// for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) { +// for (CanalClientConfig.Group group : canalAdapter.getGroups()) { +// List> canalOuterAdapterGroups = new CopyOnWriteArrayList<>(); +// List canalOuterAdapters = new CopyOnWriteArrayList<>(); +// for (OuterAdapterConfig config : group.getOuterAdapters()) { +// loadAdapter(config, canalOuterAdapters); +// } +// canalOuterAdapterGroups.add(canalOuterAdapters); +// CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig, +// canalClientConfig.getMqServers(), +// canalAdapter.getInstance(), +// group.getGroupId(), +// canalOuterAdapterGroups, +// canalClientConfig.getAccessKey(), +// canalClientConfig.getSecretKey(), +// canalClientConfig.getFlatMessage(), +// canalClientConfig.isEnableMessageTrace(), +// canalClientConfig.getCustomizedTraceTopic(), +// canalClientConfig.getAccessChannel(), +// canalClientConfig.getNamespace()); +// canalMQWorker.put(canalAdapter.getInstance() + "-rocketmq-" + group.getGroupId(), rocketMQWorker); +// rocketMQWorker.start(); +// +// logger.info("Start adapter for canal-client mq topic: {} succeed", +// canalAdapter.getInstance() + "-" + group.getGroupId()); +// } +// } +// } else if ("rabbitMQ".equalsIgnoreCase(canalClientConfig.getMode())) { +// // 初始化canal-client-rabbitMQ的适配器 +// for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) { +// for (CanalClientConfig.Group group : canalAdapter.getGroups()) { +// List> canalOuterAdapterGroups = new CopyOnWriteArrayList<>(); +// List canalOuterAdapters = new CopyOnWriteArrayList<>(); +// for (OuterAdapterConfig config : group.getOuterAdapters()) { +// loadAdapter(config, canalOuterAdapters); +// } +// canalOuterAdapterGroups.add(canalOuterAdapters); +// CanalAdapterRabbitMQWorker rabbitMQWork = new CanalAdapterRabbitMQWorker(canalClientConfig, +// canalOuterAdapterGroups, +// canalAdapter.getInstance(), +// group.getGroupId(), +// canalClientConfig.getFlatMessage()); +// canalMQWorker.put(canalAdapter.getInstance() + "-rabbitmq-" + group.getGroupId(), rabbitMQWork); +// rabbitMQWork.start(); +// +// logger.info("Start adapter for canal-client mq topic: {} succeed", +// canalAdapter.getInstance() + "-" + group.getGroupId()); +// } +// } +// // CanalAdapterRabbitMQWork +// } } private void loadAdapter(OuterAdapterConfig config, List canalOutConnectors) { @@ -206,10 +216,10 @@ public class CanalAdapterLoader { * 销毁所有适配器 为防止canal实例太多造成销毁阻塞, 并行销毁 */ public void destroy() { - if (!canalWorkers.isEmpty()) { - ExecutorService stopExecutorService = Executors.newFixedThreadPool(canalWorkers.size()); - for (CanalAdapterWorker canalAdapterWorker : canalWorkers.values()) { - stopExecutorService.execute(canalAdapterWorker::stop); + if (!canalAdapterProcessors.isEmpty()) { + ExecutorService stopExecutorService = Executors.newFixedThreadPool(canalAdapterProcessors.size()); + for (AdapterProcessor adapterProcessor : canalAdapterProcessors.values()) { + stopExecutorService.execute(adapterProcessor::stop); } stopExecutorService.shutdown(); try { @@ -220,21 +230,6 @@ public class CanalAdapterLoader { // ignore } } - - if (!canalMQWorker.isEmpty()) { - ExecutorService stopMQWorkerService = Executors.newFixedThreadPool(canalMQWorker.size()); - for (AbstractCanalAdapterWorker canalAdapterMQWorker : canalMQWorker.values()) { - stopMQWorkerService.execute(canalAdapterMQWorker::stop); - } - stopMQWorkerService.shutdown(); - try { - while (!stopMQWorkerService.awaitTermination(1, TimeUnit.SECONDS)) { - // ignore - } - } catch (InterruptedException e) { - // ignore - } - } logger.info("All canal adapters destroyed"); } } diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRabbitMQWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRabbitMQWorker.java deleted file mode 100644 index 41c08099..00000000 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRabbitMQWorker.java +++ /dev/null @@ -1,90 +0,0 @@ -package com.alibaba.otter.canal.adapter.launcher.loader; - -import com.alibaba.otter.canal.client.adapter.OuterAdapter; -import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; -import com.alibaba.otter.canal.client.adapter.support.Util; -import com.alibaba.otter.canal.client.rabbitmq.RabbitMQCanalConnector; -import org.apache.kafka.common.errors.WakeupException; - -import java.util.List; -import java.util.concurrent.ExecutorService; - -public class CanalAdapterRabbitMQWorker extends AbstractCanalAdapterWorker { - - private RabbitMQCanalConnector connector; - private String topic; - private boolean flatMessage; - - public CanalAdapterRabbitMQWorker(CanalClientConfig canalClientConfig, List> canalOuterAdapters, - String topic, String groupId, boolean flatMessage){ - super(canalOuterAdapters); - this.canalClientConfig = canalClientConfig; - this.topic = topic; - this.flatMessage = flatMessage; - this.canalDestination = topic; - this.groupId = groupId; - this.connector = new RabbitMQCanalConnector(canalClientConfig.getMqServers(), - canalClientConfig.getVhost(), - topic, - canalClientConfig.getAccessKey(), - canalClientConfig.getSecretKey(), - canalClientConfig.getUsername(), - canalClientConfig.getPassword(), - canalClientConfig.getResourceOwnerId(), - flatMessage); - } - - @Override - protected void process() { - while (!running) { - try { - Thread.sleep(1000); - } catch (InterruptedException ignored) { - } - } - - int retry = canalClientConfig.getRetries() == null - || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries(); - long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒 - while (running) { - try { - syncSwitch.get(canalDestination); - logger.info("=============> Start to connect topic: {} <=============", this.topic); - connector.connect(); - logger.info("=============> Start to subscribe topic: {}<=============", this.topic); - connector.subscribe(); - logger.info("=============> Subscribe topic: {} succeed<=============", this.topic); - while (running) { - boolean status = syncSwitch.status(canalDestination); - if (!status) { - connector.disconnect(); - break; - } - if (retry == -1) { - retry = Integer.MAX_VALUE; - } - for (int i = 0; i < retry; i++) { - if (!running) { - break; - } - if (mqWriteOutData(retry, timeout, i, flatMessage, connector)) { - break; - } - } - } - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - - try { - connector.unsubscribe(); - } catch (WakeupException e) { - // No-op. Continue process - } - connector.disconnect(); - logger.info("=============> Disconnect topic: {} <=============", this.topic); - - } -} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java deleted file mode 100644 index cf511c47..00000000 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java +++ /dev/null @@ -1,119 +0,0 @@ -package com.alibaba.otter.canal.adapter.launcher.loader; - -import java.util.List; -import java.util.concurrent.ExecutorService; - -import org.apache.kafka.common.errors.WakeupException; - -import com.alibaba.otter.canal.client.adapter.OuterAdapter; -import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; -import com.alibaba.otter.canal.client.adapter.support.Util; -import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnector; - -/** - * rocketmq对应的client适配器工作线程 - * - * @version 1.0.0 - */ -public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker { - - private RocketMQCanalConnector connector; - private String topic; - private boolean flatMessage; - - public CanalAdapterRocketMQWorker(CanalClientConfig canalClientConfig, String nameServers, String topic, - String groupId, List> canalOuterAdapters, String accessKey, - String secretKey, boolean flatMessage){ - super(canalOuterAdapters); - this.canalClientConfig = canalClientConfig; - this.topic = topic; - this.flatMessage = flatMessage; - super.canalDestination = topic; - super.groupId = groupId; - this.connector = new RocketMQCanalConnector(nameServers, - topic, - groupId, - accessKey, - secretKey, - canalClientConfig.getBatchSize(), - flatMessage); - logger.info("RocketMQ consumer config topic:{}, nameServer:{}, groupId:{}", topic, nameServers, groupId); - } - - public CanalAdapterRocketMQWorker(CanalClientConfig canalClientConfig, String nameServers, String topic, - String groupId, List> canalOuterAdapters, String accessKey, - String secretKey, boolean flatMessage, boolean enableMessageTrace, - String customizedTraceTopic, String accessChannel, String namespace) { - super(canalOuterAdapters); - this.canalClientConfig = canalClientConfig; - this.topic = topic; - this.flatMessage = flatMessage; - super.canalDestination = topic; - super.groupId = groupId; - this.connector = new RocketMQCanalConnector(nameServers, - topic, - groupId, - accessKey, - secretKey, - canalClientConfig.getBatchSize(), - flatMessage, - enableMessageTrace, - customizedTraceTopic, - accessChannel, - namespace); - logger.info("RocketMQ consumer config topic:{}, nameServer:{}, groupId:{}", topic, nameServers, groupId); - } - - @Override - protected void process() { - while (!running) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - } - } - - int retry = canalClientConfig.getRetries() == null - || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries(); - long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒 - - while (running) { - try { - syncSwitch.get(canalDestination); - logger.info("=============> Start to connect topic: {} <=============", this.topic); - connector.connect(); - logger.info("=============> Start to subscribe topic: {}<=============", this.topic); - connector.subscribe(); - logger.info("=============> Subscribe topic: {} succeed<=============", this.topic); - while (running) { - boolean status = syncSwitch.status(canalDestination); - if (!status) { - connector.disconnect(); - break; - } - if (retry == -1) { - retry = Integer.MAX_VALUE; - } - for (int i = 0; i < retry; i++) { - if (!running) { - break; - } - if (mqWriteOutData(retry, timeout, i, flatMessage, connector)) { - break; - } - } - } - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - try { - connector.unsubscribe(); - } catch (WakeupException e) { - // No-op. Continue process - } - connector.disconnect(); - logger.info("=============> Disconnect topic: {} <=============", this.topic); - } -} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java deleted file mode 100644 index 8f8cd06b..00000000 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java +++ /dev/null @@ -1,193 +0,0 @@ -package com.alibaba.otter.canal.adapter.launcher.loader; - -import java.net.SocketAddress; -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import com.alibaba.otter.canal.client.CanalConnector; -import com.alibaba.otter.canal.client.CanalConnectors; -import com.alibaba.otter.canal.client.adapter.OuterAdapter; -import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; -import com.alibaba.otter.canal.client.impl.ClusterCanalConnector; -import com.alibaba.otter.canal.client.impl.SimpleCanalConnector; -import com.alibaba.otter.canal.protocol.Message; - -/** - * 原生canal-server对应的client适配器工作线程 - * - * @author rewrema 2018-8-19 下午11:30:49 - * @version 1.0.0 - */ -public class CanalAdapterWorker extends AbstractCanalAdapterWorker { - - private static final int BATCH_SIZE = 50; - private static final int SO_TIMEOUT = 0; - - private CanalConnector connector; - - /** - * 单台client适配器worker的构造方法 - * - * @param canalDestination canal实例名 - * @param address canal-server地址 - * @param canalOuterAdapters 外部适配器组 - */ - public CanalAdapterWorker(CanalClientConfig canalClientConfig, String canalDestination, SocketAddress address, - List> canalOuterAdapters){ - super(canalOuterAdapters); - this.canalClientConfig = canalClientConfig; - this.canalDestination = canalDestination; - connector = CanalConnectors.newSingleConnector(address, canalDestination, "", ""); - } - - /** - * HA模式下client适配器worker的构造方法 - * - * @param canalDestination canal实例名 - * @param zookeeperHosts zookeeper地址 - * @param canalOuterAdapters 外部适配器组 - */ - public CanalAdapterWorker(CanalClientConfig canalClientConfig, String canalDestination, String zookeeperHosts, - List> canalOuterAdapters){ - super(canalOuterAdapters); - this.canalDestination = canalDestination; - this.canalClientConfig = canalClientConfig; - connector = CanalConnectors.newClusterConnector(zookeeperHosts, canalDestination, "", ""); - ((ClusterCanalConnector) connector).setSoTimeout(SO_TIMEOUT); - } - - @Override - protected void process() { - while (!running) { // waiting until running == true - while (!running) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - } - } - } - - int retry = canalClientConfig.getRetries() == null || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries(); - if (retry == -1) { - // 重试次数-1代表异常时一直阻塞重试 - retry = Integer.MAX_VALUE; - } - // long timeout = canalClientConfig.getTimeout() == null ? 300000 : - // canalClientConfig.getTimeout(); // 默认超时5分钟 - Integer batchSize = canalClientConfig.getBatchSize(); - if (batchSize == null) { - batchSize = BATCH_SIZE; - } - - while (running) { - try { - syncSwitch.get(canalDestination); - - logger.info("=============> Start to connect destination: {} <=============", this.canalDestination); - connector.connect(); - logger.info("=============> Start to subscribe destination: {} <=============", this.canalDestination); - connector.subscribe(); - logger.info("=============> Subscribe destination: {} succeed <=============", this.canalDestination); - while (running) { - try { - syncSwitch.get(canalDestination, 1L, TimeUnit.MINUTES); - } catch (TimeoutException e) { - break; - } - if (!running) { - break; - } - - for (int i = 0; i < retry; i++) { - if (!running) { - break; - } - Message message = connector.getWithoutAck(batchSize); // 获取指定数量的数据 - long batchId = message.getId(); - try { - int size = message.getEntries().size(); - if (batchId == -1 || size == 0) { - Thread.sleep(500); - } else { - if (logger.isDebugEnabled()) { - logger.debug("destination: {} batchId: {} batchSize: {} ", - canalDestination, - batchId, - size); - } - long begin = System.currentTimeMillis(); - writeOut(message); - if (logger.isDebugEnabled()) { - logger.debug("destination: {} batchId: {} elapsed time: {} ms", - canalDestination, - batchId, - System.currentTimeMillis() - begin); - } - } - connector.ack(batchId); // 提交确认 - break; - } catch (Exception e) { - if (i != retry - 1) { - connector.rollback(batchId); // 处理失败, 回滚数据 - logger.error(e.getMessage() + " Error sync and rollback, execute times: " + (i + 1)); - } else { - connector.ack(batchId); - logger.error(e.getMessage() + " Error sync but ACK!"); - } - Thread.sleep(500); - } - } - } - - } catch (Throwable e) { - logger.error("process error!", e); - } finally { - connector.disconnect(); - logger.info("=============> Disconnect destination: {} <=============", this.canalDestination); - } - - if (running) { // is reconnect - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - // ignore - } - } - } - } - - @Override - public void stop() { - try { - if (!running) { - return; - } - - if (connector instanceof ClusterCanalConnector) { - ((ClusterCanalConnector) connector).stopRunning(); - } else if (connector instanceof SimpleCanalConnector) { - ((SimpleCanalConnector) connector).stopRunning(); - } - - running = false; - - syncSwitch.release(canalDestination); - - logger.info("destination {} is waiting for adapters' worker thread die!", canalDestination); - if (thread != null) { - try { - thread.join(); - } catch (InterruptedException e) { - // ignore - } - } - groupInnerExecutorService.shutdown(); - logger.info("destination {} adapters worker thread dead!", canalDestination); - canalOuterAdapters.forEach(outerAdapters -> outerAdapters.forEach(OuterAdapter::destroy)); - logger.info("destination {} all adapters destroyed!", canalDestination); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } -} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/DbRemoteConfigLoader.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/DbRemoteConfigLoader.java index 0ba09bbf..085ff4d7 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/DbRemoteConfigLoader.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/DbRemoteConfigLoader.java @@ -15,14 +15,14 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import com.alibaba.otter.canal.common.utils.CommonUtils; +import com.alibaba.otter.canal.common.utils.NamedThreadFactory; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.druid.pool.DruidDataSource; import com.alibaba.otter.canal.adapter.launcher.config.AdapterConfigHolder; -import com.alibaba.otter.canal.common.utils.CommonUtils; -import com.alibaba.otter.canal.common.utils.NamedThreadFactory; import com.google.common.base.Joiner; /** diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/RemoteAdapterMonitorImpl.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/RemoteAdapterMonitorImpl.java index 2f0db822..cb3d2ec9 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/RemoteAdapterMonitorImpl.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/RemoteAdapterMonitorImpl.java @@ -5,11 +5,10 @@ import java.io.FileOutputStream; import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; +import com.alibaba.otter.canal.common.utils.CommonUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.alibaba.otter.canal.common.utils.CommonUtils; - /** * 远程配置监听器实现 * diff --git a/client-adapter/launcher/src/main/resources/application.yml b/client-adapter/launcher/src/main/resources/application.yml index f0f89a5e..60064186 100644 --- a/client-adapter/launcher/src/main/resources/application.yml +++ b/client-adapter/launcher/src/main/resources/application.yml @@ -7,20 +7,45 @@ spring: default-property-inclusion: non_null canal.conf: - mode: tcp # kafka rocketMQ - canalServerHost: 127.0.0.1:11111 -# zookeeperHosts: slave1:2181 -# mqServers: 127.0.0.1:9092 #or rocketmq -# flatMessage: true - batchSize: 500 + mode: tcp #tcp kafka rocketMQ rabbitMQ + flatMessage: true + zookeeperHosts: syncBatchSize: 1000 retries: 0 timeout: accessKey: secretKey: - username: - password: - vhost: + consumerProperties: + # canal tcp consumer + canal.tcp.server.host: 127.0.0.1:11111 + canal.tcp.zookeeper.hosts: + canal.tcp.batch.size: 500 + canal.tcp.username: + canal.tcp.password: + # kafka consumer + kafka.bootstrap.servers: 127.0.0.1:9092 + kafka.enable.auto.commit: false + kafka.auto.commit.interval.ms: 1000 + kafka.auto.offset.reset: latest + kafka.request.timeout.ms: 40000 + kafka.session.timeout.ms: 30000 + kafka.isolation.level: read_committed + kafka.max.poll.records: 1000 + # rocketMQ consumer + rocketmq.namespace: + rocketmq.namesrv.addr: 127.0.0.1:9876 + rocketmq.batch.size: 1000 + rocketmq.enable.message.trace: false + rocketmq.customized.trace.topic: + rocketmq.access.channel: + rocketmq.subscribe.filter: + # rabbitMQ consumer + rabbitmq.host: + rabbitmq.virtual.host: + rabbitmq.username: + rabbitmq.password: + rabbitmq.resource.ownerId: + # srcDataSources: # defaultDS: # url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true diff --git a/client/pom.xml b/client/pom.xml index 0ccf06eb..9bfbd404 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -101,23 +101,31 @@ ${spring_version} test + org.apache.rocketmq rocketmq-client + 4.5.2 + provided org.apache.rocketmq rocketmq-acl + 4.5.2 + provided com.rabbitmq amqp-client + 5.5.0 + provided com.alibaba.mq-amqp mq-amqp-client + 1.0.3 + provided - org.apache.kafka kafka-clients @@ -143,7 +151,7 @@ false - com.alibaba.otter:* + com.alibaba.otter:canal.client com.google.guava:* org.joda:* com.google.common:* diff --git a/connector/core/pom.xml b/connector/core/pom.xml new file mode 100644 index 00000000..46783ab7 --- /dev/null +++ b/connector/core/pom.xml @@ -0,0 +1,40 @@ + + + + canal.connector + com.alibaba.otter + 1.1.5-SNAPSHOT + + 4.0.0 + com.alibaba.otter + connector.core + jar + canal connector core module for otter ${project.version} + + + + com.alibaba.otter + canal.protocol + ${project.version} + provided + + + com.googlecode.aviator + aviator + 2.2.1 + + + oro + oro + 2.0.8 + + + joda-time + joda-time + 2.9.4 + + + + \ No newline at end of file diff --git a/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/config/CanalConstants.java b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/config/CanalConstants.java new file mode 100644 index 00000000..0a9f8256 --- /dev/null +++ b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/config/CanalConstants.java @@ -0,0 +1,26 @@ +package com.alibaba.otter.canal.connector.core.config; + +/** + * MQ配置常量 + * + * @author rewerma 2020-01-27 + * @version 1.0.0 + */ +public class CanalConstants { + + public static final String ROOT = "canal"; + + public static final String CANAL_FILTER_TRANSACTION_ENTRY = ROOT + "." + "instance.filter.transaction.entry"; + + public static final String CANAL_MQ_FLAT_MESSAGE = ROOT + "." + "mq.flat.message"; + public static final String CANAL_MQ_DATABASE_HASH = ROOT + "." + "mq.database.hash"; + public static final String CANAL_MQ_PARALLEL_THREAD_SIZE = ROOT + "." + "mq.parallel.thread.size"; + public static final String CANAL_MQ_CANAL_BATCH_SIZE = ROOT + "." + "mq.canal.batch.size"; + public static final String CANAL_MQ_CANAL_FETCH_TIMEOUT = ROOT + "." + "mq.canal.fetch.timeout"; + public static final String CANAL_MQ_ACCESS_CHANNEL = ROOT + "." + "mq.access.channel"; + + public static final String CANAL_ALIYUN_ACCESS_KEY = ROOT + "." + "aliyun.accessKey"; + public static final String CANAL_ALIYUN_SECRET_KEY = ROOT + "." + "aliyun.secretKey"; + public static final String CANAL_ALIYUN_UID = ROOT + "." + "aliyun.uid"; + +} diff --git a/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/config/MQProperties.java b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/config/MQProperties.java new file mode 100644 index 00000000..fcd7c3db --- /dev/null +++ b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/config/MQProperties.java @@ -0,0 +1,102 @@ +package com.alibaba.otter.canal.connector.core.config; + +/** + * MQ配置类 + * + * @author rewerma 2020-01-27 + * @version 1.0.0 + */ +public class MQProperties { + + private boolean flatMessage = true; + private boolean databaseHash = true; + private boolean filterTransactionEntry = true; + private Integer parallelThreadSize = 8; + private Integer fetchTimeout = 100; + private Integer batchSize = 50; + private String accessChannel = "local"; + + private String aliyunAccessKey = ""; + private String aliyunSecretKey = ""; + private int aliyunUid = 0; + + public boolean isFlatMessage() { + return flatMessage; + } + + public void setFlatMessage(boolean flatMessage) { + this.flatMessage = flatMessage; + } + + public boolean isDatabaseHash() { + return databaseHash; + } + + public void setDatabaseHash(boolean databaseHash) { + this.databaseHash = databaseHash; + } + + public boolean isFilterTransactionEntry() { + return filterTransactionEntry; + } + + public void setFilterTransactionEntry(boolean filterTransactionEntry) { + this.filterTransactionEntry = filterTransactionEntry; + } + + public Integer getParallelThreadSize() { + return parallelThreadSize; + } + + public void setParallelThreadSize(Integer parallelThreadSize) { + this.parallelThreadSize = parallelThreadSize; + } + + public Integer getFetchTimeout() { + return fetchTimeout; + } + + public void setFetchTimeout(Integer fetchTimeout) { + this.fetchTimeout = fetchTimeout; + } + + public Integer getBatchSize() { + return batchSize; + } + + public void setBatchSize(Integer batchSize) { + this.batchSize = batchSize; + } + + public String getAccessChannel() { + return accessChannel; + } + + public void setAccessChannel(String accessChannel) { + this.accessChannel = accessChannel; + } + + public String getAliyunAccessKey() { + return aliyunAccessKey; + } + + public void setAliyunAccessKey(String aliyunAccessKey) { + this.aliyunAccessKey = aliyunAccessKey; + } + + public String getAliyunSecretKey() { + return aliyunSecretKey; + } + + public void setAliyunSecretKey(String aliyunSecretKey) { + this.aliyunSecretKey = aliyunSecretKey; + } + + public int getAliyunUid() { + return aliyunUid; + } + + public void setAliyunUid(int aliyunUid) { + this.aliyunUid = aliyunUid; + } +} diff --git a/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/consumer/CommonMessage.java b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/consumer/CommonMessage.java new file mode 100644 index 00000000..6cd860c8 --- /dev/null +++ b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/consumer/CommonMessage.java @@ -0,0 +1,131 @@ +package com.alibaba.otter.canal.connector.core.consumer; + +import java.io.Serializable; +import java.util.List; +import java.util.Map; + +public class CommonMessage implements Serializable { + + private static final long serialVersionUID = 2611556444074013268L; + + private String database; // 数据库或schema + private String table; // 表名 + private List pkNames; + private Boolean isDdl; + private String type; // 类型: INSERT UPDATE DELETE + // binlog executeTime + private Long es; // 执行耗时 + // dml build timeStamp + private Long ts; // 同步时间 + private String sql; // 执行的sql, dml sql为空 + private List> data; // 数据列表 + private List> old; // 旧数据列表, 用于update, size和data的size一一对应 + + + public String getDatabase() { + return database; + } + + public void setDatabase(String database) { + this.database = database; + } + + public String getTable() { + return table; + } + + public void setTable(String table) { + this.table = table; + } + + public List getPkNames() { + return pkNames; + } + + public void setPkNames(List pkNames) { + this.pkNames = pkNames; + } + + public Boolean getIsDdl() { + return isDdl; + } + + public void setIsDdl(Boolean isDdl) { + this.isDdl = isDdl; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public Long getTs() { + return ts; + } + + public void setTs(Long ts) { + this.ts = ts; + } + + public String getSql() { + return sql; + } + + public void setSql(String sql) { + this.sql = sql; + } + + public List> getData() { + return data; + } + + public void setData(List> data) { + this.data = data; + } + + public List> getOld() { + return old; + } + + public void setOld(List> old) { + this.old = old; + } + + public Long getEs() { + return es; + } + + public void setEs(Long es) { + this.es = es; + } + + public void clear() { + database = null; + table = null; + type = null; + ts = null; + es = null; + data = null; + old = null; + sql = null; + } + + @Override + public String toString() { + return "CommonMessage{" + + "database='" + database + '\'' + + ", table='" + table + '\'' + + ", pkNames=" + pkNames + + ", isDdl=" + isDdl + + ", type='" + type + '\'' + + ", es=" + es + + ", ts=" + ts + + ", sql='" + sql + '\'' + + ", data=" + data + + ", old=" + old + + '}'; + } +} diff --git a/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/AviaterRegexFilter.java b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/AviaterRegexFilter.java new file mode 100644 index 00000000..f9ab3d6e --- /dev/null +++ b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/AviaterRegexFilter.java @@ -0,0 +1,125 @@ +package com.alibaba.otter.canal.connector.core.filter; + +import java.util.*; + +import org.apache.commons.lang.StringUtils; + +import com.googlecode.aviator.AviatorEvaluator; +import com.googlecode.aviator.Expression; + +/** + * 基于aviater进行tableName正则匹配的过滤算法 + * + * @author jianghang 2012-7-20 下午06:01:34 + */ +public class AviaterRegexFilter { + + private static final String SPLIT = ","; + private static final String PATTERN_SPLIT = "|"; + private static final String FILTER_EXPRESSION = "regex(pattern,target)"; + private static final RegexFunction regexFunction = new RegexFunction(); + private final Expression exp = AviatorEvaluator.compile(FILTER_EXPRESSION, true); + static { + AviatorEvaluator.addFunction(regexFunction); + } + + private static final Comparator COMPARATOR = new StringComparator(); + + final private String pattern; + final private boolean defaultEmptyValue; + + public AviaterRegexFilter(String pattern){ + this(pattern, true); + } + + public AviaterRegexFilter(String pattern, boolean defaultEmptyValue){ + this.defaultEmptyValue = defaultEmptyValue; + List list = null; + if (StringUtils.isEmpty(pattern)) { + list = new ArrayList(); + } else { + String[] ss = StringUtils.split(pattern, SPLIT); + list = Arrays.asList(ss); + } + + // 对pattern按照从长到短的排序 + // 因为 foo|foot 匹配 foot 会出错，原因是 foot 匹配了 foo 之后，会返回 foo，但是 foo 的长度和 foot + // 的长度不一样 + Collections.sort(list, COMPARATOR); + // 对pattern进行头尾完全匹配 + list = completionPattern(list); + this.pattern = StringUtils.join(list, PATTERN_SPLIT); + } + + public boolean filter(String filtered) { + if (StringUtils.isEmpty(pattern)) { + return defaultEmptyValue; + } + + if (StringUtils.isEmpty(filtered)) { + return defaultEmptyValue; + } + + Map env = new HashMap(); + env.put("pattern", pattern); + env.put("target", filtered.toLowerCase()); + return (Boolean) exp.execute(env); + } + + /** + * 修复正则表达式匹配的问题，因为使用了 oro 的 matches，会出现： + * + * 
+     * foo|foot 匹配 foot 出错，原因是 foot 匹配了 foo 之后，会返回 foo，但是 foo 的长度和 foot 的长度不一样
+     *
+ * + * 因此此类对正则表达式进行了从长到短的排序 + * + * @author zebin.xuzb 2012-10-22 下午2:02:26 + * @version 1.0.0 + */ + private static class StringComparator implements Comparator { + + @Override + public int compare(String str1, String str2) { + if (str1.length() > str2.length()) { + return -1; + } else if (str1.length() < str2.length()) { + return 1; + } else { + return 0; + } + } + } + + /** + * 修复正则表达式匹配的问题，即使按照长度递减排序，还是会出现以下问题： + * + * 
+     * foooo|f.*t 匹配 fooooot 出错，原因是 fooooot 匹配了 foooo 之后，会将 fooo 和数据进行匹配，但是 foooo 的长度和 fooooot 的长度不一样
+     *
+ * + * 因此此类对正则表达式进行头尾完全匹配 + * + * @author simon + * @version 1.0.0 + */ + + private List completionPattern(List patterns) { + List result = new ArrayList(); + for (String pattern : patterns) { + StringBuffer stringBuffer = new StringBuffer(); + stringBuffer.append("^"); + stringBuffer.append(pattern); + stringBuffer.append("$"); + result.add(stringBuffer.toString()); + } + return result; + } + + @Override + public String toString() { + return pattern; + } + +} diff --git a/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/PatternUtils.java b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/PatternUtils.java new file mode 100644 index 00000000..f008a142 --- /dev/null +++ b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/PatternUtils.java @@ -0,0 +1,33 @@ +package com.alibaba.otter.canal.connector.core.filter; + +import java.util.Map; + +import org.apache.oro.text.regex.MalformedPatternException; +import org.apache.oro.text.regex.Pattern; +import org.apache.oro.text.regex.PatternCompiler; +import org.apache.oro.text.regex.Perl5Compiler; + +import com.google.common.collect.MapMaker; +import com.google.common.collect.MigrateMap; + +public class PatternUtils { + + @SuppressWarnings("deprecation") + private static Map patterns = MigrateMap.makeComputingMap(new MapMaker().softValues(), pattern -> { + try { + PatternCompiler pc = new Perl5Compiler(); + return pc.compile(pattern, + Perl5Compiler.CASE_INSENSITIVE_MASK | Perl5Compiler.READ_ONLY_MASK | Perl5Compiler.SINGLELINE_MASK); + } catch (MalformedPatternException e) { + throw new RuntimeException(e); + } + }); + + public static Pattern getPattern(String pattern) { + return patterns.get(pattern); + } + + public static void clear() { + patterns.clear(); + } +} diff --git a/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/RegexFunction.java b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/RegexFunction.java new file mode 100644 index 00000000..5675ed4a --- /dev/null +++ b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/RegexFunction.java @@ -0,0 +1,31 @@ +package com.alibaba.otter.canal.connector.core.filter; + +import java.util.Map; + +import org.apache.oro.text.regex.Perl5Matcher; + +import com.googlecode.aviator.runtime.function.AbstractFunction; +import com.googlecode.aviator.runtime.function.FunctionUtils; +import com.googlecode.aviator.runtime.type.AviatorBoolean; +import com.googlecode.aviator.runtime.type.AviatorObject; + +/** + * 提供aviator regex的代码扩展 + * + * @author jianghang 2012-7-23 上午10:29:23 + */ +public class RegexFunction extends AbstractFunction { + + public AviatorObject call(Map env, AviatorObject arg1, AviatorObject arg2) { + String pattern = FunctionUtils.getStringValue(arg1, env); + String text = FunctionUtils.getStringValue(arg2, env); + Perl5Matcher matcher = new Perl5Matcher(); + boolean isMatch = matcher.matches(text, PatternUtils.getPattern(pattern)); + return AviatorBoolean.valueOf(isMatch); + } + + public String getName() { + return "regex"; + } + +} diff --git a/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/AbstractMQProducer.java b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/AbstractMQProducer.java new file mode 100644 index 00000000..cfe3d42b --- /dev/null +++ b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/AbstractMQProducer.java @@ -0,0 +1,110 @@ +package com.alibaba.otter.canal.connector.core.producer; + +import java.util.Properties; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import com.alibaba.otter.canal.connector.core.config.CanalConstants; +import org.apache.commons.lang.StringUtils; + +import com.alibaba.otter.canal.common.utils.NamedThreadFactory; +import com.alibaba.otter.canal.connector.core.config.MQProperties; +import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer; + +/** + * MQ producer 抽象类 + * + * @author rewerma 2020-01-27 + * @version 1.0.0 + */ +public abstract class AbstractMQProducer implements CanalMQProducer { + + protected MQProperties mqProperties; + + protected ThreadPoolExecutor executor; + + @Override + public void init(Properties properties) { + // parse canal mq properties + loadCanalMqProperties(properties); + + int parallelThreadSize = mqProperties.getParallelThreadSize(); + executor = new ThreadPoolExecutor(parallelThreadSize, + parallelThreadSize, + 0, + TimeUnit.SECONDS, + new ArrayBlockingQueue(parallelThreadSize * 2), + new NamedThreadFactory("MQParallel"), + new ThreadPoolExecutor.CallerRunsPolicy()); + } + + @Override + public MQProperties getMqProperties() { + return this.mqProperties; + } + + @Override + public void stop() { + if (executor != null) { + executor.shutdownNow(); + } + } + + /** + * 初始化配置 + *

+ * canal.mq.flat.message = true
+ * canal.mq.database.hash = true
+ * canal.mq.filter.transaction.entry = true
+ * canal.mq.parallel.thread.size = 8
+ * canal.mq.batch.size = 50
+ * canal.mq.timeout = 100
+ * canal.mq.access.channel = local
+ *

+ * + * @param properties 总配置对象 + */ + private void loadCanalMqProperties(Properties properties) { + String flatMessage = properties.getProperty(CanalConstants.CANAL_MQ_FLAT_MESSAGE); + if (!StringUtils.isEmpty(flatMessage)) { + mqProperties.setFlatMessage(Boolean.parseBoolean(flatMessage)); + } + String databaseHash = properties.getProperty(CanalConstants.CANAL_MQ_DATABASE_HASH); + if (!StringUtils.isEmpty(databaseHash)) { + mqProperties.setDatabaseHash(Boolean.parseBoolean(databaseHash)); + } + String filterTranEntry = properties.getProperty(CanalConstants.CANAL_FILTER_TRANSACTION_ENTRY); + if (!StringUtils.isEmpty(filterTranEntry)) { + mqProperties.setFilterTransactionEntry(Boolean.parseBoolean(filterTranEntry)); + } + String parallelThreadSize = properties.getProperty(CanalConstants.CANAL_MQ_PARALLEL_THREAD_SIZE); + if (!StringUtils.isEmpty(parallelThreadSize)) { + mqProperties.setParallelThreadSize(Integer.parseInt(parallelThreadSize)); + } + String batchSize = properties.getProperty(CanalConstants.CANAL_MQ_CANAL_BATCH_SIZE); + if (!StringUtils.isEmpty(batchSize)) { + mqProperties.setBatchSize(Integer.parseInt(batchSize)); + } + String timeOut = properties.getProperty(CanalConstants.CANAL_MQ_CANAL_FETCH_TIMEOUT); + if (!StringUtils.isEmpty(timeOut)) { + mqProperties.setFetchTimeout(Integer.parseInt(timeOut)); + } + String accessChannel = properties.getProperty(CanalConstants.CANAL_MQ_ACCESS_CHANNEL); + if (!StringUtils.isEmpty(accessChannel)) { + mqProperties.setAccessChannel(accessChannel); + } + String aliyunAccessKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_ACCESS_KEY); + if (!StringUtils.isEmpty(aliyunAccessKey)) { + mqProperties.setAliyunAccessKey(aliyunAccessKey); + } + String aliyunSecretKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_SECRET_KEY); + if (!StringUtils.isEmpty(aliyunSecretKey)) { + mqProperties.setAliyunAccessKey(aliyunSecretKey); + } + String aliyunUid = properties.getProperty(CanalConstants.CANAL_ALIYUN_UID); + if (!StringUtils.isEmpty(aliyunUid)) { + mqProperties.setAliyunUid(Integer.parseInt(aliyunUid)); + } + } +} diff --git a/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/MQDestination.java b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/MQDestination.java new file mode 100644 index 00000000..e0d9124f --- /dev/null +++ b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/MQDestination.java @@ -0,0 +1,64 @@ +package com.alibaba.otter.canal.connector.core.producer; + +/** + * MQ producer destination + * + * @author rewerma 2020-01-27 + * @version 1.0.0 + */ +public class MQDestination { + private String canalDestination; + private String topic; + private Integer partition; + private Integer partitionsNum; + private String partitionHash; + private String dynamicTopic; + + public String getCanalDestination() { + return canalDestination; + } + + public void setCanalDestination(String canalDestination) { + this.canalDestination = canalDestination; + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } + + public Integer getPartition() { + return partition; + } + + public void setPartition(Integer partition) { + this.partition = partition; + } + + public Integer getPartitionsNum() { + return partitionsNum; + } + + public void setPartitionsNum(Integer partitionsNum) { + this.partitionsNum = partitionsNum; + } + + public String getPartitionHash() { + return partitionHash; + } + + public void setPartitionHash(String partitionHash) { + this.partitionHash = partitionHash; + } + + public String getDynamicTopic() { + return dynamicTopic; + } + + public void setDynamicTopic(String dynamicTopic) { + this.dynamicTopic = dynamicTopic; + } +} diff --git a/server/src/main/java/com/alibaba/otter/canal/common/MQMessageUtils.java b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/MQMessageUtils.java similarity index 84% rename from server/src/main/java/com/alibaba/otter/canal/common/MQMessageUtils.java rename to connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/MQMessageUtils.java index 6bc275ac..5d1d6119 100644 --- a/server/src/main/java/com/alibaba/otter/canal/common/MQMessageUtils.java +++ b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/MQMessageUtils.java @@ -1,25 +1,18 @@ -package com.alibaba.otter.canal.common; +package com.alibaba.otter.canal.connector.core.producer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.concurrent.ThreadPoolExecutor; +import com.google.common.base.Function; import org.apache.commons.lang.StringUtils; import com.alibaba.otter.canal.common.utils.ExecutorTemplate; -import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter; +import com.alibaba.otter.canal.connector.core.filter.AviaterRegexFilter; import com.alibaba.otter.canal.protocol.CanalEntry; import com.alibaba.otter.canal.protocol.CanalEntry.Entry; import com.alibaba.otter.canal.protocol.CanalEntry.RowChange; import com.alibaba.otter.canal.protocol.FlatMessage; import com.alibaba.otter.canal.protocol.Message; -import com.google.common.base.Function; import com.google.common.collect.Lists; import com.google.common.collect.MapMaker; import com.google.common.collect.MigrateMap; @@ -29,77 +22,88 @@ import com.google.protobuf.InvalidProtocolBufferException; /** * process MQ Message utils * - * @author agapple 2018年12月11日 下午1:28:32 - */ -/** * @author agapple 2019年9月29日 下午12:36:26 * @since 5.0.0 */ public class MQMessageUtils { @SuppressWarnings("deprecation") - private static Map> partitionDatas = MigrateMap.makeComputingMap(new MapMaker().softValues(), - new Function>() { + private static Map> partitionDatas = MigrateMap + .makeComputingMap(new MapMaker().softValues(), pkHashConfigs -> { + List datas = Lists + .newArrayList(); - public List apply(String pkHashConfigs) { - List datas = Lists.newArrayList(); + String[] pkHashConfigArray = StringUtils + .split(StringUtils.replace( + pkHashConfigs, + ",", + ";"), ";"); + // schema.table:id^name + for (String pkHashConfig : pkHashConfigArray) { + PartitionData data = new PartitionData(); + int i = pkHashConfig + .lastIndexOf(":"); + if (i > 0) { + String pkStr = pkHashConfig + .substring(i + 1); + if (pkStr.equalsIgnoreCase( + "$pk$")) { + data.hashMode.autoPkHash = true; + } else { + data.hashMode.pkNames = Lists + .newArrayList( + StringUtils + .split( + pkStr, + '^')); + } - String[] pkHashConfigArray = StringUtils.split(StringUtils.replace(pkHashConfigs, - ",", - ";"), - ";"); - // schema.table:id^name - for (String pkHashConfig : pkHashConfigArray) { - PartitionData data = new PartitionData(); - int i = pkHashConfig.lastIndexOf(":"); - if (i > 0) { - String pkStr = pkHashConfig.substring(i + 1); - if (pkStr.equalsIgnoreCase("$pk$")) { - data.hashMode.autoPkHash = true; - } else { - data.hashMode.pkNames = Lists.newArrayList(StringUtils.split(pkStr, - '^')); - } + pkHashConfig = pkHashConfig + .substring(0, i); + } else { + data.hashMode.tableHash = true; + } - pkHashConfig = pkHashConfig.substring(0, - i); - } else { - data.hashMode.tableHash = true; - } + if (!isWildCard( + pkHashConfig)) { + data.simpleName = pkHashConfig; + } else { + data.regexFilter = new AviaterRegexFilter( + pkHashConfig); + } + datas.add(data); + } - if (!isWildCard(pkHashConfig)) { - data.simpleName = pkHashConfig; - } else { - data.regexFilter = new AviaterRegexFilter(pkHashConfig); - } - datas.add(data); - } - - return datas; - } - }); + return datas; + }); @SuppressWarnings("deprecation") - private static Map> dynamicTopicDatas = MigrateMap.makeComputingMap(new MapMaker().softValues(), - new Function>() { + private static Map> dynamicTopicDatas = MigrateMap + .makeComputingMap(new MapMaker().softValues(), new Function>() { public List apply(String pkHashConfigs) { - List datas = Lists.newArrayList(); - String[] dynamicTopicArray = StringUtils.split(StringUtils.replace(pkHashConfigs, - ",", - ";"), - ";"); + List datas = Lists + .newArrayList(); + String[] dynamicTopicArray = StringUtils + .split(StringUtils.replace( + pkHashConfigs, + ",", + ";"), ";"); // schema.table for (String dynamicTopic : dynamicTopicArray) { DynamicTopicData data = new DynamicTopicData(); - if (!isWildCard(dynamicTopic)) { + if (!isWildCard( + dynamicTopic)) { data.simpleName = dynamicTopic; } else { - if (dynamicTopic.contains("\\.")) { - data.tableRegexFilter = new AviaterRegexFilter(dynamicTopic); + if (dynamicTopic + .contains("\\.")) { + data.tableRegexFilter = new AviaterRegexFilter( + dynamicTopic); } else { - data.schemaRegexFilter = new AviaterRegexFilter(dynamicTopic); + data.schemaRegexFilter = new AviaterRegexFilter( + dynamicTopic); } } datas.add(data); @@ -238,8 +242,8 @@ public class MQMessageUtils { * @return 分区message数组 */ @SuppressWarnings("unchecked") - public static Message[] messagePartition(EntryRowData[] datas, long id, Integer partitionsNum, - String pkHashConfigs, boolean databaseHash) { + public static Message[] messagePartition(EntryRowData[] datas, long id, Integer partitionsNum, String pkHashConfigs, + boolean databaseHash) { if (partitionsNum == null) { partitionsNum = 1; } @@ -338,8 +342,8 @@ public class MQMessageUtils { /** * 将Message转换为FlatMessage * - * @param message 原生message * @return FlatMessage列表 + * @author agapple 2018年12月11日 下午1:28:32 */ public static List messageConverter(EntryRowData[] datas, long id) { List flatMessages = new ArrayList<>(); @@ -627,8 +631,8 @@ public class MQMessageUtils { private static boolean isWildCard(String value) { // not contaiins '.' ? - return StringUtils.containsAny(value, new char[] { '*', '?', '+', '|', '(', ')', '{', '}', '[', ']', '\\', '$', - '^' }); + return StringUtils.containsAny(value, + new char[] { '*', '?', '+', '|', '(', ')', '{', '}', '[', ']', '\\', '$', '^' }); } private static void put2MapMessage(Map messageMap, Long messageId, String topicName, diff --git a/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/CanalMQProducer.java b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/CanalMQProducer.java new file mode 100644 index 00000000..d0de4981 --- /dev/null +++ b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/CanalMQProducer.java @@ -0,0 +1,43 @@ +package com.alibaba.otter.canal.connector.core.spi; + +import java.util.Properties; + +import com.alibaba.otter.canal.connector.core.config.MQProperties; +import com.alibaba.otter.canal.connector.core.util.Callback; +import com.alibaba.otter.canal.connector.core.producer.MQDestination; +import com.alibaba.otter.canal.protocol.Message; + +/** + * MQ producer SPI 接口 + * + * @author rewerma 2020-01-27 + * @version 1.0.0 + */ +@SPI("kafka") +public interface CanalMQProducer { + + /** + * Init producer. + */ + void init(Properties properties); + + /** + * Get base mq properties + * + * @return MQProperties + */ + MQProperties getMqProperties(); + + /** + * Send canal message to related topic + * + * @param canalDestination canal mq destination + * @param message canal message + */ + void send(MQDestination canalDestination, Message message, Callback callback); + + /** + * Stop MQ producer service + */ + void stop(); +} diff --git a/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/CanalMsgConsumer.java b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/CanalMsgConsumer.java new file mode 100644 index 00000000..a80c69f1 --- /dev/null +++ b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/CanalMsgConsumer.java @@ -0,0 +1,55 @@ +package com.alibaba.otter.canal.connector.core.spi; + +import java.util.List; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +import com.alibaba.otter.canal.connector.core.consumer.CommonMessage; + +/** + * Canal/MQ consumer SPI 接口 + * + * @author rewerma @ 2020-02-01 + * @version 1.0.0 + */ +@SPI("kafka") +public interface CanalMsgConsumer { + + /** + * 初始化 + * + * @param properties consumer properties + * @param topic topic/destination + * @param groupId mq group id + */ + void init(Properties properties, String topic, String groupId); + + /** + * 连接Canal/MQ + */ + void connect(); + + /** + * 批量拉取数据 + * + * @param timeout 超时时间 + * @param unit 时间单位 + * @return Message列表 + */ + List getMessage(Long timeout, TimeUnit unit); + + /** + * 提交 + */ + void ack(); + + /** + * 回滚 + */ + void rollback(); + + /** + * 断开连接 + */ + void disconnect(); +} diff --git a/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/ExtensionLoader.java b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/ExtensionLoader.java new file mode 100644 index 00000000..e5c6708c --- /dev/null +++ b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/ExtensionLoader.java @@ -0,0 +1,435 @@ +package com.alibaba.otter.canal.connector.core.spi; + +import java.io.BufferedReader; +import java.io.File; +import java.io.InputStreamReader; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.regex.Pattern; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * SPI 类加载器 + * + * @author rewerma 2018-8-19 下午11:30:49 + * @version 1.0.0 + */ +public class ExtensionLoader { + + private static final Logger logger = LoggerFactory + .getLogger(ExtensionLoader.class); + + private static final String SERVICES_DIRECTORY = "META-INF/services/"; + + private static final String CANAL_DIRECTORY = "META-INF/canal/"; + + private static final String DEFAULT_CLASSLOADER_POLICY = "internal"; + + private static final Pattern NAME_SEPARATOR = Pattern + .compile("\\s*[,]+\\s*"); + + private static final ConcurrentMap, ExtensionLoader> EXTENSION_LOADERS = new ConcurrentHashMap<>(); + + private static final ConcurrentMap, Object> EXTENSION_INSTANCES = new ConcurrentHashMap<>(); + + private static final ConcurrentMap EXTENSION_KEY_INSTANCE = new ConcurrentHashMap<>(); + + private final Class type; + + private final String classLoaderPolicy; + + private final ConcurrentMap, String> cachedNames = new ConcurrentHashMap<>(); + + private final Holder>> cachedClasses = new Holder<>(); + + private final ConcurrentMap> cachedInstances = new ConcurrentHashMap<>(); + + private String cachedDefaultName; + + private ConcurrentHashMap exceptions = new ConcurrentHashMap<>(); + + private static boolean withExtensionAnnotation(Class type) { + return type.isAnnotationPresent(SPI.class); + } + + public static ExtensionLoader getExtensionLoader(Class type) { + return getExtensionLoader(type, DEFAULT_CLASSLOADER_POLICY); + } + + @SuppressWarnings("unchecked") + public static ExtensionLoader getExtensionLoader(Class type, String classLoaderPolicy) { + if (type == null) throw new IllegalArgumentException("Extension type == null"); + if (!type.isInterface()) { + throw new IllegalArgumentException("Extension type(" + type + ") is not interface!"); + } + if (!withExtensionAnnotation(type)) { + throw new IllegalArgumentException("Extension type(" + type + ") is not extension, because WITHOUT @" + + SPI.class.getSimpleName() + " Annotation!"); + } + + ExtensionLoader loader = (ExtensionLoader) EXTENSION_LOADERS.get(type); + if (loader == null) { + EXTENSION_LOADERS.putIfAbsent(type, new ExtensionLoader(type, classLoaderPolicy)); + loader = (ExtensionLoader) EXTENSION_LOADERS.get(type); + } + return loader; + } + + public ExtensionLoader(Class type){ + this.type = type; + this.classLoaderPolicy = DEFAULT_CLASSLOADER_POLICY; + } + + public ExtensionLoader(Class type, String classLoaderPolicy){ + this.type = type; + this.classLoaderPolicy = classLoaderPolicy; + } + + /** + * 返回指定名字的扩展 + * + * @param name + * @return + */ + @SuppressWarnings("unchecked") + public T getExtension(String name, String spiDir, String standbyDir) { + if (name == null || name.length() == 0) throw new IllegalArgumentException("Extension name == null"); + if ("true".equals(name)) { + return getDefaultExtension(spiDir, standbyDir); + } + Holder holder = cachedInstances.get(name); + if (holder == null) { + cachedInstances.putIfAbsent(name, new Holder<>()); + holder = cachedInstances.get(name); + } + Object instance = holder.get(); + if (instance == null) { + synchronized (holder) { + instance = holder.get(); + if (instance == null) { + instance = createExtension(name, spiDir, standbyDir); + holder.set(instance); + } + } + } + return (T) instance; + } + + @SuppressWarnings("unchecked") + public T getExtension(String name, String key, String spiDir, String standbyDir) { + if (name == null || name.length() == 0) throw new IllegalArgumentException("Extension name == null"); + if ("true".equals(name)) { + return getDefaultExtension(spiDir, standbyDir); + } + String extKey = name + "-" + StringUtils.trimToEmpty(key); + Holder holder = cachedInstances.get(extKey); + if (holder == null) { + cachedInstances.putIfAbsent(extKey, new Holder<>()); + holder = cachedInstances.get(extKey); + } + Object instance = holder.get(); + if (instance == null) { + synchronized (holder) { + instance = holder.get(); + if (instance == null) { + instance = createExtension(name, key, spiDir, standbyDir); + holder.set(instance); + } + } + } + return (T) instance; + } + + /** + * 返回缺省的扩展，如果没有设置则返回null + */ + public T getDefaultExtension(String spiDir, String standbyDir) { + getExtensionClasses(spiDir, standbyDir); + if (null == cachedDefaultName || cachedDefaultName.length() == 0 || "true".equals(cachedDefaultName)) { + return null; + } + return getExtension(cachedDefaultName, spiDir, standbyDir); + } + + @SuppressWarnings("unchecked") + public T createExtension(String name, String spiDir, String standbyDir) { + Class clazz = getExtensionClasses(spiDir, standbyDir).get(name); + if (clazz == null) { + throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type + + ") could not be instantiated: class could not be found"); + } + try { + T instance = (T) EXTENSION_INSTANCES.get(clazz); + if (instance == null) { + EXTENSION_INSTANCES.putIfAbsent(clazz, (T) clazz.newInstance()); + instance = (T) EXTENSION_INSTANCES.get(clazz); + } + return instance; + } catch (Throwable t) { + throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type + + ") could not be instantiated: " + t.getMessage(), + t); + } + } + + @SuppressWarnings("unchecked") + private T createExtension(String name, String key, String spiDir, String standbyDir) { + Class clazz = getExtensionClasses(spiDir, standbyDir).get(name); + if (clazz == null) { + throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type + + ") could not be instantiated: class could not be found"); + } + try { + T instance = (T) EXTENSION_KEY_INSTANCE.get(name + "-" + key); + if (instance == null) { + EXTENSION_KEY_INSTANCE.putIfAbsent(name + "-" + key, clazz.newInstance()); + instance = (T) EXTENSION_KEY_INSTANCE.get(name + "-" + key); + } + return instance; + } catch (Throwable t) { + throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type + + ") could not be instantiated: " + t.getMessage(), + t); + } + } + + private Map> getExtensionClasses(String spiDir, String standbyDir) { + Map> classes = cachedClasses.get(); + if (classes == null) { + synchronized (cachedClasses) { + classes = cachedClasses.get(); + if (classes == null) { + classes = loadExtensionClasses(spiDir, standbyDir); + cachedClasses.set(classes); + } + } + } + + return classes; + } + + private String getJarDirectoryPath() { + URL url = Thread.currentThread().getContextClassLoader().getResource(""); + String dirtyPath; + if (url != null) { + dirtyPath = url.toString(); + } else { + File file = new File(""); + dirtyPath = file.getAbsolutePath(); + } + String jarPath = dirtyPath.replaceAll("^.*file:/", ""); // removes + // file:/ and + // everything + // before it + jarPath = jarPath.replaceAll("jar!.*", "jar"); // removes everything + // after .jar, if .jar + // exists in dirtyPath + jarPath = jarPath.replaceAll("%20", " "); // necessary if path has + // spaces within + if (!jarPath.endsWith(".jar")) { // this is needed if you plan to run + // the app using Spring Tools Suit play + // button. + jarPath = jarPath.replaceAll("/classes/.*", "/classes/"); + } + Path path = Paths.get(jarPath).getParent(); // Paths - from java 8 + if (path != null) { + return path.toString(); + } + return null; + } + + private Map> loadExtensionClasses(String spiDir, String standbyDir) { + final SPI defaultAnnotation = type.getAnnotation(SPI.class); + if (defaultAnnotation != null) { + String value = defaultAnnotation.value(); + if ((value = value.trim()).length() > 0) { + String[] names = NAME_SEPARATOR.split(value); + if (names.length > 1) { + throw new IllegalStateException("more than 1 default extension name on extension " + type.getName() + + ": " + Arrays.toString(names)); + } + if (names.length == 1) cachedDefaultName = names[0]; + } + } + + Map> extensionClasses = new HashMap<>(); + + if (spiDir != null && standbyDir != null) { + // 1. plugin folder，customized extension classLoader （jar_dir/plugin） + String dir = File.separator + this.getJarDirectoryPath() + spiDir; // + "plugin"; + + File externalLibDir = new File(dir); + if (!externalLibDir.exists()) { + externalLibDir = new File(File.separator + this.getJarDirectoryPath() + standbyDir); + } + logger.info("extension classpath dir: " + externalLibDir.getAbsolutePath()); + if (externalLibDir.exists()) { + File[] files = externalLibDir.listFiles((dir1, name) -> name.endsWith(".jar")); + if (files != null) { + for (File f : files) { + URL url; + try { + url = f.toURI().toURL(); + } catch (MalformedURLException e) { + throw new RuntimeException("load extension jar failed!", e); + } + + ClassLoader parent = Thread.currentThread().getContextClassLoader(); + URLClassLoader localClassLoader; + if (classLoaderPolicy == null || "".equals(classLoaderPolicy) + || DEFAULT_CLASSLOADER_POLICY.equalsIgnoreCase(classLoaderPolicy)) { + localClassLoader = new URLClassExtensionLoader(new URL[] { url }); + } else { + localClassLoader = new URLClassLoader(new URL[] { url }, parent); + } + + loadFile(extensionClasses, CANAL_DIRECTORY, localClassLoader); + loadFile(extensionClasses, SERVICES_DIRECTORY, localClassLoader); + } + } + } + } + + // 2. load inner extension class with default classLoader + ClassLoader classLoader = findClassLoader(); + loadFile(extensionClasses, CANAL_DIRECTORY, classLoader); + loadFile(extensionClasses, SERVICES_DIRECTORY, classLoader); + + return extensionClasses; + } + + private void loadFile(Map> extensionClasses, String dir, ClassLoader classLoader) { + String fileName = dir + type.getName(); + try { + Enumeration urls; + if (classLoader != null) { + urls = classLoader.getResources(fileName); + } else { + urls = ClassLoader.getSystemResources(fileName); + } + if (urls != null) { + while (urls.hasMoreElements()) { + URL url = urls.nextElement(); + try { + BufferedReader reader = null; + try { + reader = new BufferedReader( + new InputStreamReader(url.openStream(), StandardCharsets.UTF_8)); + String line = null; + while ((line = reader.readLine()) != null) { + final int ci = line.indexOf('#'); + if (ci >= 0) line = line.substring(0, ci); + line = line.trim(); + if (line.length() > 0) { + try { + String name = null; + int i = line.indexOf('='); + if (i > 0) { + name = line.substring(0, i).trim(); + line = line.substring(i + 1).trim(); + } + if (line.length() > 0) { + Class clazz = classLoader.loadClass(line); + // Class clazz = + // Class.forName(line, true, + // classLoader); + if (!type.isAssignableFrom(clazz)) { + throw new IllegalStateException( + "Error when load extension class(interface: " + type + + ", class line: " + clazz.getName() + + "), class " + clazz.getName() + + "is not subtype of interface."); + } else { + try { + clazz.getConstructor(type); + } catch (NoSuchMethodException e) { + clazz.getConstructor(); + String[] names = NAME_SEPARATOR.split(name); + if (names != null && names.length > 0) { + for (String n : names) { + if (!cachedNames.containsKey(clazz)) { + cachedNames.put(clazz, n); + } + Class c = extensionClasses.get(n); + if (c == null) { + extensionClasses.put(n, clazz); + } else if (c != clazz) { + cachedNames.remove(clazz); + throw new IllegalStateException( + "Duplicate extension " + type.getName() + " name " + + n + " on " + + c.getName() + " and " + + clazz.getName()); + } + } + } + } + } + } + } catch (Throwable t) { + IllegalStateException e = new IllegalStateException( + "Failed to load extension class(interface: " + type + ", class line: " + + line + ") in " + url + + ", cause: " + + t.getMessage(), + t); + exceptions.put(line, e); + } + } + } // end of while read lines + } finally { + if (reader != null) { + reader.close(); + } + } + } catch (Throwable t) { + logger.error("Exception when load extension class(interface: " + type + ", class file: " + url + + ") in " + url, + t); + } + } // end of while urls + } + } catch (Throwable t) { + logger.error( + "Exception when load extension class(interface: " + type + ", description file: " + fileName + ").", + t); + } + } + + @SuppressWarnings("unused") + private static ClassLoader findClassLoader() { + return ExtensionLoader.class.getClassLoader(); + } + + @Override + public String toString() { + return this.getClass().getName() + "[" + type.getName() + "]"; + } + + private static class Holder { + + private volatile T value; + + private void set(T value) { + this.value = value; + } + + private T get() { + return value; + } + + } +} diff --git a/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/SPI.java b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/SPI.java new file mode 100644 index 00000000..e82620f3 --- /dev/null +++ b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/SPI.java @@ -0,0 +1,18 @@ +package com.alibaba.otter.canal.connector.core.spi; + +import java.lang.annotation.*; + +/** + * SPI装载器注解 + * + * @author rewerma @ 2018-10-20 + * @version 1.0.0 + */ +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target({ ElementType.TYPE }) +public @interface SPI { + + // Default SPI name + String value() default ""; +} diff --git a/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/URLClassExtensionLoader.java b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/URLClassExtensionLoader.java new file mode 100644 index 00000000..c502c6d7 --- /dev/null +++ b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/URLClassExtensionLoader.java @@ -0,0 +1,88 @@ +package com.alibaba.otter.canal.connector.core.spi; + +import java.io.IOException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.Enumeration; +import java.util.NoSuchElementException; + +public class URLClassExtensionLoader extends URLClassLoader { + public URLClassExtensionLoader(URL[] urls) { + super(urls); + } + + @Override + public Class loadClass(String name) throws ClassNotFoundException { + Class c = findLoadedClass(name); + if (c != null) { + return c; + } + + if (name.startsWith("java.") || name.startsWith("org.slf4j.") + || name.startsWith("org.apache.logging") + || name.startsWith("org.apache.commons.logging.")) { + // || name.startsWith("org.apache.hadoop.")) + // { + c = super.loadClass(name); + } + if (c != null) return c; + + try { + // 先加载jar内的class，可避免jar冲突 + c = findClass(name); + } catch (ClassNotFoundException e) { + c = null; + } + if (c != null) { + return c; + } + + return super.loadClass(name); + } + + @Override + public Enumeration getResources(String name) throws IOException { + @SuppressWarnings("unchecked") + Enumeration[] tmp = (Enumeration[]) new Enumeration[2]; + + tmp[0] = findResources(name); // local class + // path first + // tmp[1] = super.getResources(name); + + return new CompoundEnumeration<>(tmp); + } + + private static class CompoundEnumeration implements Enumeration { + + private Enumeration[] enums; + private int index = 0; + + public CompoundEnumeration(Enumeration[] enums){ + this.enums = enums; + } + + private boolean next() { + while (this.index < this.enums.length) { + if (this.enums[this.index] != null && this.enums[this.index].hasMoreElements()) { + return true; + } + + ++this.index; + } + + return false; + } + + public boolean hasMoreElements() { + return this.next(); + } + + public E nextElement() { + if (!this.next()) { + throw new NoSuchElementException(); + } else { + return this.enums[this.index].nextElement(); + } + } + } +} diff --git a/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/Callback.java b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/Callback.java new file mode 100644 index 00000000..fd019268 --- /dev/null +++ b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/Callback.java @@ -0,0 +1,13 @@ +package com.alibaba.otter.canal.connector.core.util; + +/** + * MQ 回调类 + * + * @author rewerma 2020-01-27 + * @version 1.0.0 + */ +public interface Callback { + void commit(); + + void rollback(); +} diff --git a/server/src/main/java/com/alibaba/otter/canal/common/CanalMessageSerializer.java b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/CanalMessageSerializerUtil.java similarity index 57% rename from server/src/main/java/com/alibaba/otter/canal/common/CanalMessageSerializer.java rename to connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/CanalMessageSerializerUtil.java index 5b80a922..c0becb2c 100644 --- a/server/src/main/java/com/alibaba/otter/canal/common/CanalMessageSerializer.java +++ b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/CanalMessageSerializerUtil.java @@ -1,20 +1,25 @@ -package com.alibaba.otter.canal.common; +package com.alibaba.otter.canal.connector.core.util; import java.util.List; -import org.apache.kafka.common.errors.SerializationException; +import com.alibaba.otter.canal.protocol.exception.CanalClientException; import org.springframework.util.CollectionUtils; import com.alibaba.otter.canal.protocol.CanalEntry; import com.alibaba.otter.canal.protocol.CanalPacket; import com.alibaba.otter.canal.protocol.CanalPacket.PacketType; import com.alibaba.otter.canal.protocol.Message; -import com.alibaba.otter.canal.server.netty.NettyUtils; import com.google.protobuf.ByteString; import com.google.protobuf.CodedOutputStream; import com.google.protobuf.WireFormat; -public class CanalMessageSerializer { +/** + * Canal message 序列化工具类 + * + * @author rewerma 2020-01-27 + * @version 1.0.0 + */ +public class CanalMessageSerializerUtil { @SuppressWarnings("deprecation") public static byte[] serializer(Message data, boolean filterTransactionEntry) { @@ -58,7 +63,8 @@ public class CanalMessageSerializer { CanalPacket.Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder(); for (CanalEntry.Entry entry : data.getEntries()) { if (filterTransactionEntry - && (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND)) { + && (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN + || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND)) { continue; } @@ -67,15 +73,60 @@ public class CanalMessageSerializer { CanalPacket.Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder(); packetBuilder.setType(PacketType.MESSAGES); - packetBuilder.setVersion(NettyUtils.VERSION); + packetBuilder.setVersion(1); packetBuilder.setBody(messageBuilder.build().toByteString()); return packetBuilder.build().toByteArray(); } } } } catch (Exception e) { - throw new SerializationException("Error when serializing message to byte[] "); + throw new RuntimeException("Error when serializing message to byte[] "); } return null; } + + public static Message deserializer(byte[] data) { + return deserializer(data, false); + } + + public static Message deserializer(byte[] data, boolean lazyParseEntry) { + try { + if (data == null) { + return null; + } else { + CanalPacket.Packet p = CanalPacket.Packet.parseFrom(data); + switch (p.getType()) { + case MESSAGES: { + if (!p.getCompression().equals(CanalPacket.Compression.NONE) + && !p.getCompression().equals(CanalPacket.Compression.COMPRESSIONCOMPATIBLEPROTO2)) { + throw new CanalClientException("compression is not supported in this connector"); + } + + CanalPacket.Messages messages = CanalPacket.Messages.parseFrom(p.getBody()); + Message result = new Message(messages.getBatchId()); + if (lazyParseEntry) { + // byteString + result.setRawEntries(messages.getMessagesList()); + result.setRaw(true); + } else { + for (ByteString byteString : messages.getMessagesList()) { + result.addEntry(CanalEntry.Entry.parseFrom(byteString)); + } + result.setRaw(false); + } + return result; + } + case ACK: { + CanalPacket.Ack ack = CanalPacket.Ack.parseFrom(p.getBody()); + throw new CanalClientException("something goes wrong with reason: " + ack.getErrorMessage()); + } + default: { + throw new CanalClientException("unexpected packet type: " + p.getType()); + } + } + } + } catch (Exception e) { + throw new CanalClientException("deserializer failed", e); + } + } } diff --git a/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/DateUtil.java b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/DateUtil.java new file mode 100644 index 00000000..21dfa894 --- /dev/null +++ b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/DateUtil.java @@ -0,0 +1,46 @@ +package com.alibaba.otter.canal.connector.core.util; + +import org.apache.commons.lang.StringUtils; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; + +import java.util.Date; + +public class DateUtil { + private static DateTimeZone dateTimeZone; + + static { + dateTimeZone = DateTimeZone.forID(TimeZone.LOCATION_TIME_ZONE); + } + + /** + * 通用日期时间字符解析 + * + * @param datetimeStr 日期时间字符串 + * @return Date + */ + public static Date parseDate(String datetimeStr) { + if (StringUtils.isEmpty(datetimeStr)) { + return null; + } + datetimeStr = datetimeStr.trim().replace('/', '-'); + if (datetimeStr.contains("-")) { + if (datetimeStr.contains(":")) { + datetimeStr = datetimeStr.replace(" ", "T"); + } + } else if (datetimeStr.contains(":")) { + datetimeStr = "T" + datetimeStr; + } else { + if (datetimeStr.length() == 8) { + String year = datetimeStr.substring(0, 4); + String month = datetimeStr.substring(4, 6); + String day = datetimeStr.substring(6, 8); + datetimeStr = year + "-" + month + "-" + day; + } + } + + DateTime dateTime = new DateTime(datetimeStr, dateTimeZone); + + return dateTime.toDate(); + } +} diff --git a/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/JdbcTypeUtil.java b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/JdbcTypeUtil.java new file mode 100644 index 00000000..f4ac08a6 --- /dev/null +++ b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/JdbcTypeUtil.java @@ -0,0 +1,165 @@ +package com.alibaba.otter.canal.connector.core.util; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.sql.*; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * 类型转换工具类 + * + * @author rewerma 2018-8-19 下午06:14:23 + * @version 1.0.0 + */ +public class JdbcTypeUtil { + + private static Logger logger = LoggerFactory.getLogger(JdbcTypeUtil.class); + + public static Object getRSData(ResultSet rs, String columnName, int jdbcType) throws SQLException { + if (jdbcType == Types.BIT || jdbcType == Types.BOOLEAN) { + return rs.getByte(columnName); + } else { + return rs.getObject(columnName); + } + } + + public static Class jdbcType2javaType(int jdbcType) { + switch (jdbcType) { + case Types.BIT: + case Types.BOOLEAN: + // return Boolean.class; + case Types.TINYINT: + return Byte.TYPE; + case Types.SMALLINT: + return Short.class; + case Types.INTEGER: + return Integer.class; + case Types.BIGINT: + return Long.class; + case Types.DECIMAL: + case Types.NUMERIC: + return BigDecimal.class; + case Types.REAL: + return Float.class; + case Types.FLOAT: + case Types.DOUBLE: + return Double.class; + case Types.CHAR: + case Types.VARCHAR: + case Types.LONGVARCHAR: + return String.class; + case Types.BINARY: + case Types.VARBINARY: + case Types.LONGVARBINARY: + case Types.BLOB: + return byte[].class; + case Types.DATE: + return Date.class; + case Types.TIME: + return Time.class; + case Types.TIMESTAMP: + return Timestamp.class; + default: + return String.class; + } + } + + private static boolean isText(String columnType) { + return "LONGTEXT".equalsIgnoreCase(columnType) || "MEDIUMTEXT".equalsIgnoreCase(columnType) + || "TEXT".equalsIgnoreCase(columnType) || "TINYTEXT".equalsIgnoreCase(columnType); + } + + public static Object typeConvert(String tableName, String columnName, String value, int sqlType, String mysqlType) { + if (value == null || (value.equals("") && !(isText(mysqlType) || sqlType == Types.CHAR + || sqlType == Types.VARCHAR || sqlType == Types.LONGVARCHAR))) { + return null; + } + + try { + Object res; + switch (sqlType) { + case Types.INTEGER: + res = Integer.parseInt(value); + break; + case Types.SMALLINT: + res = Short.parseShort(value); + break; + case Types.BIT: + case Types.TINYINT: + res = Byte.parseByte(value); + break; + case Types.BIGINT: + if (mysqlType != null && mysqlType.startsWith("bigint") && mysqlType.endsWith("unsigned")) { + res = new BigInteger(value); + } else { + res = Long.parseLong(value); + } + break; + // case Types.BIT: + case Types.BOOLEAN: + res = !"0".equals(value); + break; + case Types.DOUBLE: + case Types.FLOAT: + res = Double.parseDouble(value); + break; + case Types.REAL: + res = Float.parseFloat(value); + break; + case Types.DECIMAL: + case Types.NUMERIC: + res = new BigDecimal(value); + break; + case Types.BINARY: + case Types.VARBINARY: + case Types.LONGVARBINARY: + case Types.BLOB: + res = value.getBytes("ISO-8859-1"); + break; + case Types.DATE: + if (!value.startsWith("0000-00-00")) { + java.util.Date date = DateUtil.parseDate(value); + if (date != null) { + res = new Date(date.getTime()); + } else { + res = null; + } + } else { + res = null; + } + break; + case Types.TIME: { + java.util.Date date = DateUtil.parseDate(value); + if (date != null) { + res = new Time(date.getTime()); + } else { + res = null; + } + break; + } + case Types.TIMESTAMP: + if (!value.startsWith("0000-00-00")) { + java.util.Date date = DateUtil.parseDate(value); + if (date != null) { + res = new Timestamp(date.getTime()); + } else { + res = null; + } + } else { + res = null; + } + break; + case Types.CLOB: + default: + res = value; + break; + } + return res; + } catch (Exception e) { + logger.error("table: {} column: {}, failed convert type {} to {}", tableName, columnName, value, sqlType); + return value; + } + } +} diff --git a/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/MessageUtil.java b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/MessageUtil.java new file mode 100644 index 00000000..a6e519a2 --- /dev/null +++ b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/MessageUtil.java @@ -0,0 +1,131 @@ +package com.alibaba.otter.canal.connector.core.util; + +import java.util.*; + +import com.alibaba.otter.canal.connector.core.consumer.CommonMessage; +import com.alibaba.otter.canal.protocol.CanalEntry; +import com.alibaba.otter.canal.protocol.Message; + +/** + * Message对象解析工具类 + * + * @author rewerma 2018-8-19 下午06:14:23 + * @version 1.0.0 + */ +public class MessageUtil { + + public static List convert(Message message) { + if (message == null) { + return null; + } + List entries = message.getEntries(); + List msgs = new ArrayList<>(entries.size()); + for (CanalEntry.Entry entry : entries) { + if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN + || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) { + continue; + } + + CanalEntry.RowChange rowChange; + try { + rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue()); + } catch (Exception e) { + throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(), + e); + } + + CanalEntry.EventType eventType = rowChange.getEventType(); + + final CommonMessage msg = new CommonMessage(); + msg.setIsDdl(rowChange.getIsDdl()); + msg.setDatabase(entry.getHeader().getSchemaName()); + msg.setTable(entry.getHeader().getTableName()); + msg.setType(eventType.toString()); + msg.setEs(entry.getHeader().getExecuteTime()); + msg.setIsDdl(rowChange.getIsDdl()); + msg.setTs(System.currentTimeMillis()); + msg.setSql(rowChange.getSql()); + msgs.add(msg); + List> data = new ArrayList<>(); + List> old = new ArrayList<>(); + + if (!rowChange.getIsDdl()) { + Set updateSet = new HashSet<>(); + msg.setPkNames(new ArrayList<>()); + int i = 0; + for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) { + if (eventType != CanalEntry.EventType.INSERT && eventType != CanalEntry.EventType.UPDATE + && eventType != CanalEntry.EventType.DELETE) { + continue; + } + + Map row = new LinkedHashMap<>(); + List columns; + + if (eventType == CanalEntry.EventType.DELETE) { + columns = rowData.getBeforeColumnsList(); + } else { + columns = rowData.getAfterColumnsList(); + } + + for (CanalEntry.Column column : columns) { + if (i == 0) { + if (column.getIsKey()) { + msg.getPkNames().add(column.getName()); + } + } + if (column.getIsNull()) { + row.put(column.getName(), null); + } else { + row.put(column.getName(), + JdbcTypeUtil.typeConvert(msg.getTable(), + column.getName(), + column.getValue(), + column.getSqlType(), + column.getMysqlType())); + } + // 获取update为true的字段 + if (column.getUpdated()) { + updateSet.add(column.getName()); + } + } + if (!row.isEmpty()) { + data.add(row); + } + + if (eventType == CanalEntry.EventType.UPDATE) { + Map rowOld = new LinkedHashMap<>(); + for (CanalEntry.Column column : rowData.getBeforeColumnsList()) { + if (updateSet.contains(column.getName())) { + if (column.getIsNull()) { + rowOld.put(column.getName(), null); + } else { + rowOld.put(column.getName(), + JdbcTypeUtil.typeConvert(msg.getTable(), + column.getName(), + column.getValue(), + column.getSqlType(), + column.getMysqlType())); + } + } + } + // update操作将记录修改前的值 + if (!rowOld.isEmpty()) { + old.add(rowOld); + } + } + + i++; + } + if (!data.isEmpty()) { + msg.setData(data); + } + if (!old.isEmpty()) { + msg.setOld(old); + } + } + } + + return msgs; + } +} diff --git a/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/TimeZone.java b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/TimeZone.java new file mode 100644 index 00000000..d2af92d3 --- /dev/null +++ b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/TimeZone.java @@ -0,0 +1,22 @@ +package com.alibaba.otter.canal.connector.core.util; + + +public class TimeZone { + public final static String LOCATION_TIME_ZONE; // 当前时区 + + static { + java.util.TimeZone localTimeZone = java.util.TimeZone.getDefault(); + int rawOffset = localTimeZone.getRawOffset(); + String symbol = "+"; + if (rawOffset < 0) { + symbol = "-"; + } + rawOffset = Math.abs(rawOffset); + int offsetHour = rawOffset / 3600000; + int offsetMinute = rawOffset % 3600000 / 60000; + String hour = String.format("%1$02d", offsetHour); + String minute = String.format("%1$02d", offsetMinute); + LOCATION_TIME_ZONE = symbol + hour + ":" + minute; + java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone("GMT" + LOCATION_TIME_ZONE)); + } +} diff --git a/connector/kafka-connector/pom.xml b/connector/kafka-connector/pom.xml new file mode 100644 index 00000000..7a732626 --- /dev/null +++ b/connector/kafka-connector/pom.xml @@ -0,0 +1,94 @@ + + + + canal.connector + com.alibaba.otter + 1.1.5-SNAPSHOT + ../pom.xml + + + 4.0.0 + com.alibaba.otter + connector.kafka + jar + canal connector kafka module for otter ${project.version} + + + + com.alibaba.otter + canal.protocol + ${project.version} + provided + + + com.alibaba.otter + connector.core + ${project.version} + provided + + + org.apache.kafka + kafka_2.11 + 1.1.1 + + + junit + junit + test + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.4 + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + org.apache.maven.plugins + maven-antrun-plugin + 1.8 + + + package + + run + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/config/KafkaConstants.java b/connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/config/KafkaConstants.java new file mode 100644 index 00000000..8a6a1237 --- /dev/null +++ b/connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/config/KafkaConstants.java @@ -0,0 +1,16 @@ +package com.alibaba.otter.canal.connector.kafka.config; + +/** + * Kafka 配置常量类 + * + * @author rewerma 2020-01-27 + * @version 1.0.0 + */ +public class KafkaConstants { + + public static final String ROOT = "canal"; + + public static final String CANAL_MQ_KAFKA_KERBEROS_ENABLE = ROOT + "." + "mq.kafka.kerberos.enable"; + public static final String CANAL_MQ_KAFKA_KERBEROS_KRB5_FILE = ROOT + "." + "mq.kafka.kerberos.krb5.file"; + public static final String CANAL_MQ_KAFKA_KERBEROS_JAAS_FILE = ROOT + "." + "mq.kafka.kerberos.jaas.file"; +} diff --git a/connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/config/KafkaProducerConfig.java b/connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/config/KafkaProducerConfig.java new file mode 100644 index 00000000..4815edb3 --- /dev/null +++ b/connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/config/KafkaProducerConfig.java @@ -0,0 +1,53 @@ +package com.alibaba.otter.canal.connector.kafka.config; + +import java.util.LinkedHashMap; +import java.util.Map; + +import com.alibaba.otter.canal.connector.core.config.MQProperties; + +/** + * Kafka producer 配置类 + * + * @author rewerma 2020-01-27 + * @version 1.0.0 + */ +public class KafkaProducerConfig extends MQProperties { + + private Map kafkaProperties = new LinkedHashMap<>(); + + private boolean kerberosEnabled = false; + private String krb5File; + private String jaasFile; + + public Map getKafkaProperties() { + return kafkaProperties; + } + + public void setKafkaProperties(Map kafkaProperties) { + this.kafkaProperties = kafkaProperties; + } + + public boolean isKerberosEnabled() { + return kerberosEnabled; + } + + public void setKerberosEnabled(boolean kerberosEnabled) { + this.kerberosEnabled = kerberosEnabled; + } + + public String getKrb5File() { + return krb5File; + } + + public void setKrb5File(String krb5File) { + this.krb5File = krb5File; + } + + public String getJaasFile() { + return jaasFile; + } + + public void setJaasFile(String jaasFile) { + this.jaasFile = jaasFile; + } +} diff --git a/connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/consumer/CanalKafkaConsumer.java b/connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/consumer/CanalKafkaConsumer.java new file mode 100644 index 00000000..acb41b06 --- /dev/null +++ b/connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/consumer/CanalKafkaConsumer.java @@ -0,0 +1,137 @@ +package com.alibaba.otter.canal.connector.kafka.consumer; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +import com.alibaba.fastjson.JSON; +import com.alibaba.otter.canal.connector.core.config.CanalConstants; +import com.alibaba.otter.canal.connector.core.spi.SPI; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.serialization.StringDeserializer; + +import com.alibaba.otter.canal.connector.core.consumer.CommonMessage; +import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer; +import com.alibaba.otter.canal.connector.core.util.MessageUtil; +import com.alibaba.otter.canal.protocol.Message; + +/** + * Kafka consumer SPI 实现 + * + * @author rewerma @ 2020-02-01 + * @version 1.0.0 + */ +@SPI("kafka") +public class CanalKafkaConsumer implements CanalMsgConsumer { + + private static final String PREFIX_KAFKA_CONFIG = "kafka."; + + private KafkaConsumer kafkaConsumer; + private boolean flatMessage = true; + private String topic; + + private Map currentOffsets = new ConcurrentHashMap<>(); + private Properties kafkaProperties = new Properties(); + + @Override + public void init(Properties properties, String topic, String groupId) { + this.topic = topic; + + Boolean flatMessage = (Boolean) properties.get(CanalConstants.CANAL_MQ_FLAT_MESSAGE); + if (flatMessage != null) { + this.flatMessage = flatMessage; + } + for (Map.Entry entry : properties.entrySet()) { + String k = (String) entry.getKey(); + Object v = entry.getValue(); + if (k.startsWith(PREFIX_KAFKA_CONFIG) && v != null) { + kafkaProperties.put(k.substring(PREFIX_KAFKA_CONFIG.length()), v); + } + } + kafkaProperties.put("group.id", groupId); + kafkaProperties.put("key.deserializer", StringDeserializer.class); + kafkaProperties.put("client.id", UUID.randomUUID().toString().substring(0, 6)); + } + + @Override + public void connect() { + if (this.flatMessage) { + kafkaProperties.put("value.deserializer", StringDeserializer.class); + this.kafkaConsumer = new KafkaConsumer(kafkaProperties); + } else { + kafkaProperties.put("value.deserializer", KafkaMessageDeserializer.class); + this.kafkaConsumer = new KafkaConsumer(kafkaProperties); + } + kafkaConsumer.subscribe(Collections.singletonList(topic)); + } + + @SuppressWarnings("unchecked") + @Override + public List getMessage(Long timeout, TimeUnit unit) { + if (!flatMessage) { + ConsumerRecords records = (ConsumerRecords) kafkaConsumer + .poll(unit.toMillis(timeout)); + if (!records.isEmpty()) { + currentOffsets.clear(); + List messages = new ArrayList<>(); + for (ConsumerRecord record : records) { + if (currentOffsets.get(record.partition()) == null) { + currentOffsets.put(record.partition(), record.offset()); + } + messages.addAll(MessageUtil.convert(record.value())); + } + return messages; + } + } else { + ConsumerRecords records = (ConsumerRecords) kafkaConsumer + .poll(unit.toMillis(timeout)); + + if (!records.isEmpty()) { + List messages = new ArrayList<>(); + currentOffsets.clear(); + for (ConsumerRecord record : records) { + if (currentOffsets.get(record.partition()) == null) { + currentOffsets.put(record.partition(), record.offset()); + } + String flatMessageJson = record.value(); + CommonMessage flatMessages = JSON.parseObject(flatMessageJson, CommonMessage.class); + messages.add(flatMessages); + } + return messages; + } + } + return null; + } + + @Override + public void rollback() { + // 回滚所有分区 + if (kafkaConsumer != null) { + for (Map.Entry entry : currentOffsets.entrySet()) { + kafkaConsumer.seek(new TopicPartition(topic, entry.getKey()), currentOffsets.get(entry.getKey())); + kafkaConsumer.commitSync(); + } + } + } + + @Override + public void ack() { + if (kafkaConsumer != null) { + kafkaConsumer.commitSync(); + } + } + + @Override + public void disconnect() { + if (kafkaConsumer != null) { + kafkaConsumer.unsubscribe(); + } + if (kafkaConsumer != null) { + kafkaConsumer.close(); + kafkaConsumer = null; + } + } +} diff --git a/connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/consumer/KafkaMessageDeserializer.java b/connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/consumer/KafkaMessageDeserializer.java new file mode 100644 index 00000000..d8a75ec2 --- /dev/null +++ b/connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/consumer/KafkaMessageDeserializer.java @@ -0,0 +1,31 @@ +package com.alibaba.otter.canal.connector.kafka.consumer; + +import java.util.Map; + +import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil; +import org.apache.kafka.common.serialization.Deserializer; + +import com.alibaba.otter.canal.protocol.Message; + +/** + * Kafka Message类的反序列化 + * + * @author rewerma @ 2018-6-12 + * @version 1.0.0 + */ +public class KafkaMessageDeserializer implements Deserializer { + + @Override + public void configure(Map configs, boolean isKey) { + } + + @Override + public Message deserialize(String topic1, byte[] data) { + return CanalMessageSerializerUtil.deserializer(data); + } + + @Override + public void close() { + // nothing to do + } +} diff --git a/server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java b/connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/producer/CanalKafkaProducer.java similarity index 52% rename from server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java rename to connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/producer/CanalKafkaProducer.java index 0f0381df..9672a59a 100644 --- a/server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java +++ b/connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/producer/CanalKafkaProducer.java @@ -1,14 +1,12 @@ -package com.alibaba.otter.canal.kafka; +package com.alibaba.otter.canal.connector.kafka.producer; import java.io.File; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Properties; +import java.util.*; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; +import com.alibaba.otter.canal.connector.kafka.config.KafkaConstants; import org.apache.commons.lang.StringUtils; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.Producer; @@ -19,68 +17,90 @@ import org.slf4j.LoggerFactory; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.serializer.SerializerFeature; -import com.alibaba.otter.canal.common.AbstractMQProducer; -import com.alibaba.otter.canal.common.CanalMessageSerializer; -import com.alibaba.otter.canal.common.MQMessageUtils; -import com.alibaba.otter.canal.common.MQMessageUtils.EntryRowData; -import com.alibaba.otter.canal.common.MQProperties; import com.alibaba.otter.canal.common.utils.ExecutorTemplate; +import com.alibaba.otter.canal.connector.core.producer.AbstractMQProducer; +import com.alibaba.otter.canal.connector.core.util.Callback; +import com.alibaba.otter.canal.connector.core.producer.MQDestination; +import com.alibaba.otter.canal.connector.core.producer.MQMessageUtils; +import com.alibaba.otter.canal.connector.core.producer.MQMessageUtils.EntryRowData; +import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer; +import com.alibaba.otter.canal.connector.core.spi.SPI; +import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil; +import com.alibaba.otter.canal.connector.kafka.config.KafkaProducerConfig; import com.alibaba.otter.canal.protocol.FlatMessage; import com.alibaba.otter.canal.protocol.Message; -import com.alibaba.otter.canal.spi.CanalMQProducer; /** - * kafka producer 主操作类 + * kafka producer SPI 实现 * - * @author machengyuan 2018-6-11 下午05:30:49 + * @author rewerma 2018-6-11 下午05:30:49 * @version 1.0.0 */ +@SuppressWarnings({ "rawtypes", "unchecked" }) +@SPI("kafka") public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQProducer { - private static final Logger logger = LoggerFactory.getLogger(CanalKafkaProducer.class); + private static final Logger logger = LoggerFactory.getLogger(CanalKafkaProducer.class); + + private static final String PREFIX_KAFKA_CONFIG = "kafka."; private Producer producer; - private MQProperties kafkaProperties; @Override - public void init(MQProperties kafkaProperties) { - super.init(kafkaProperties); + public void init(Properties properties) { + KafkaProducerConfig kafkaProducerConfig = new KafkaProducerConfig(); + this.mqProperties = kafkaProducerConfig; + super.init(properties); + // load properties + this.loadKafkaProperties(properties); - this.kafkaProperties = kafkaProperties; - Properties properties = new Properties(); - properties.put("bootstrap.servers", kafkaProperties.getServers()); - properties.put("acks", kafkaProperties.getAcks()); - properties.put("compression.type", kafkaProperties.getCompressionType()); - properties.put("batch.size", kafkaProperties.getBatchSize()); - properties.put("linger.ms", kafkaProperties.getLingerMs()); - properties.put("max.request.size", kafkaProperties.getMaxRequestSize()); - properties.put("buffer.memory", kafkaProperties.getBufferMemory()); - properties.put("key.serializer", StringSerializer.class.getName()); - properties.put("max.in.flight.requests.per.connection", 1); - - if (!kafkaProperties.getProperties().isEmpty()) { - properties.putAll(kafkaProperties.getProperties()); - } - properties.put("retries", kafkaProperties.getRetries()); - if (kafkaProperties.isKerberosEnable()) { - File krb5File = new File(kafkaProperties.getKerberosKrb5FilePath()); - File jaasFile = new File(kafkaProperties.getKerberosJaasFilePath()); + Properties kafkaProperties = new Properties(); + kafkaProperties.putAll(kafkaProducerConfig.getKafkaProperties()); + kafkaProperties.put("key.serializer", StringSerializer.class); + if (kafkaProducerConfig.isKerberosEnabled()) { + File krb5File = new File(kafkaProducerConfig.getKrb5File()); + File jaasFile = new File(kafkaProducerConfig.getJaasFile()); if (krb5File.exists() && jaasFile.exists()) { // 配置kerberos认证，需要使用绝对路径 System.setProperty("java.security.krb5.conf", krb5File.getAbsolutePath()); System.setProperty("java.security.auth.login.config", jaasFile.getAbsolutePath()); System.setProperty("javax.security.auth.useSubjectCredsOnly", "false"); - properties.put("security.protocol", "SASL_PLAINTEXT"); - properties.put("sasl.kerberos.service.name", "kafka"); + kafkaProperties.put("security.protocol", "SASL_PLAINTEXT"); + kafkaProperties.put("sasl.kerberos.service.name", "kafka"); } else { String errorMsg = "ERROR # The kafka kerberos configuration file does not exist! please check it"; logger.error(errorMsg); throw new RuntimeException(errorMsg); } } + kafkaProperties.put("value.serializer", KafkaMessageSerializer.class); + producer = new KafkaProducer<>(kafkaProperties); + } - properties.put("value.serializer", KafkaMessageSerializer.class.getName()); - producer = new KafkaProducer(properties); + private void loadKafkaProperties(Properties properties) { + KafkaProducerConfig kafkaProducerConfig = (KafkaProducerConfig) this.mqProperties; + Map kafkaProperties = kafkaProducerConfig.getKafkaProperties(); + for (Map.Entry entry : properties.entrySet()) { + String key = (String) entry.getKey(); + Object value = entry.getValue(); + if (key.startsWith(PREFIX_KAFKA_CONFIG) && value != null) { + key = key.substring(PREFIX_KAFKA_CONFIG.length()); + kafkaProperties.put(key, value); + } + } + + String kerberosEnabled = properties.getProperty(KafkaConstants.CANAL_MQ_KAFKA_KERBEROS_ENABLE); + if (!StringUtils.isEmpty(kerberosEnabled)) { + kafkaProducerConfig.setKerberosEnabled(Boolean.parseBoolean(kerberosEnabled)); + } + String krb5File = properties.getProperty(KafkaConstants.CANAL_MQ_KAFKA_KERBEROS_KRB5_FILE); + if (!StringUtils.isEmpty(krb5File)) { + kafkaProducerConfig.setKrb5File(krb5File); + } + String jaasFile = properties.getProperty(KafkaConstants.CANAL_MQ_KAFKA_KERBEROS_JAAS_FILE); + if (!StringUtils.isEmpty(jaasFile)) { + kafkaProducerConfig.setJaasFile(jaasFile); + } } @Override @@ -90,41 +110,34 @@ public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQPro if (producer != null) { producer.close(); } + super.stop(); } catch (Throwable e) { logger.warn("##something goes wrong when stopping kafka producer:", e); } finally { logger.info("## kafka producer is down."); } - - super.stop(); } @Override - public void send(MQProperties.CanalDestination canalDestination, Message message, Callback callback) { + public void send(MQDestination mqDestination, Message message, Callback callback) { ExecutorTemplate template = new ExecutorTemplate(executor); - boolean flat = kafkaProperties.getFlatMessage(); try { - List result = null; - if (!StringUtils.isEmpty(canalDestination.getDynamicTopic())) { + List result; + if (!StringUtils.isEmpty(mqDestination.getDynamicTopic())) { // 动态topic路由计算,只是基于schema/table,不涉及proto数据反序列化 - Map messageMap = MQMessageUtils.messageTopics(message, - canalDestination.getTopic(), - canalDestination.getDynamicTopic()); + Map messageMap = MQMessageUtils + .messageTopics(message, mqDestination.getTopic(), mqDestination.getDynamicTopic()); // 针对不同的topic,引入多线程提升效率 for (Map.Entry entry : messageMap.entrySet()) { final String topicName = entry.getKey().replace('.', '_'); final Message messageSub = entry.getValue(); - template.submit(new Callable() { - - @Override - public List call() throws Exception { - try { - return send(canalDestination, topicName, messageSub, flat); - } catch (Exception e) { - throw new RuntimeException(e); - } + template.submit((Callable) () -> { + try { + return send(mqDestination, topicName, messageSub, mqProperties.isFlatMessage()); + } catch (Exception e) { + throw new RuntimeException(e); } }); } @@ -132,7 +145,10 @@ public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQPro result = template.waitForResult(); } else { result = new ArrayList(); - List futures = send(canalDestination, canalDestination.getTopic(), message, flat); + List futures = send(mqDestination, + mqDestination.getTopic(), + message, + mqProperties.isFlatMessage()); result.add(futures); } @@ -161,36 +177,34 @@ public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQPro } } - private List send(MQProperties.CanalDestination canalDestination, String topicName, Message message, - boolean flat) throws Exception { - List> records = new ArrayList>(); + private List send(MQDestination mqDestination, String topicName, Message message, boolean flat) { + List> records = new ArrayList<>(); if (!flat) { - if (canalDestination.getPartitionHash() != null && !canalDestination.getPartitionHash().isEmpty()) { + if (mqDestination.getPartitionHash() != null && !mqDestination.getPartitionHash().isEmpty()) { // 并发构造 EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor); // 串行分区 Message[] messages = MQMessageUtils.messagePartition(datas, message.getId(), - canalDestination.getPartitionsNum(), - canalDestination.getPartitionHash(), - kafkaProperties.getDatabaseHash()); + mqDestination.getPartitionsNum(), + mqDestination.getPartitionHash(), + this.mqProperties.isDatabaseHash()); int length = messages.length; for (int i = 0; i < length; i++) { Message messagePartition = messages[i]; if (messagePartition != null) { - records.add(new ProducerRecord(topicName, + records.add(new ProducerRecord<>(topicName, i, null, - CanalMessageSerializer.serializer(messagePartition, - kafkaProperties.isFilterTransactionEntry()))); + CanalMessageSerializerUtil.serializer(messagePartition, true))); } } } else { - final int partition = canalDestination.getPartition() != null ? canalDestination.getPartition() : 0; - records.add(new ProducerRecord(topicName, + final int partition = mqDestination.getPartition() != null ? mqDestination.getPartition() : 0; + records.add(new ProducerRecord<>(topicName, partition, null, - CanalMessageSerializer.serializer(message, kafkaProperties.isFilterTransactionEntry()))); + CanalMessageSerializerUtil.serializer(message, true))); } } else { // 发送扁平数据json @@ -199,24 +213,24 @@ public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQPro // 串行分区 List flatMessages = MQMessageUtils.messageConverter(datas, message.getId()); for (FlatMessage flatMessage : flatMessages) { - if (canalDestination.getPartitionHash() != null && !canalDestination.getPartitionHash().isEmpty()) { + if (mqDestination.getPartitionHash() != null && !mqDestination.getPartitionHash().isEmpty()) { FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage, - canalDestination.getPartitionsNum(), - canalDestination.getPartitionHash(), - kafkaProperties.getDatabaseHash()); + mqDestination.getPartitionsNum(), + mqDestination.getPartitionHash(), + this.mqProperties.isDatabaseHash()); int length = partitionFlatMessage.length; for (int i = 0; i < length; i++) { FlatMessage flatMessagePart = partitionFlatMessage[i]; if (flatMessagePart != null) { - records.add(new ProducerRecord(topicName, + records.add(new ProducerRecord<>(topicName, i, null, JSON.toJSONBytes(flatMessagePart, SerializerFeature.WriteMapNullValue))); } } } else { - final int partition = canalDestination.getPartition() != null ? canalDestination.getPartition() : 0; - records.add(new ProducerRecord(topicName, + final int partition = mqDestination.getPartition() != null ? mqDestination.getPartition() : 0; + records.add(new ProducerRecord<>(topicName, partition, null, JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue))); @@ -224,11 +238,11 @@ public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQPro } } - return produce(topicName, records, flat); + return produce(records); } - private List produce(String topicName, List> records, boolean flatMessage) { - List futures = new ArrayList(); + private List produce(List> records) { + List futures = new ArrayList<>(); // 异步发送，因为在partition hash的时候已经按照每个分区合并了消息，走到这一步不需要考虑单个分区内的顺序问题 for (ProducerRecord record : records) { futures.add(producer.send(record)); diff --git a/server/src/main/java/com/alibaba/otter/canal/kafka/KafkaMessageSerializer.java b/connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/producer/KafkaMessageSerializer.java similarity index 82% rename from server/src/main/java/com/alibaba/otter/canal/kafka/KafkaMessageSerializer.java rename to connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/producer/KafkaMessageSerializer.java index 0ea291f1..bf60c1a4 100644 --- a/server/src/main/java/com/alibaba/otter/canal/kafka/KafkaMessageSerializer.java +++ b/connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/producer/KafkaMessageSerializer.java @@ -1,13 +1,13 @@ -package com.alibaba.otter.canal.kafka; - -import java.util.Map; +package com.alibaba.otter.canal.connector.kafka.producer; import org.apache.kafka.common.serialization.Serializer; +import java.util.Map; + /** * Kafka Message类的序列化 * - * @author machengyuan 2018-6-11 下午05:30:49 + * @author rewerma 2018-6-11 下午05:30:49 * @version 1.0.0 */ public class KafkaMessageSerializer implements Serializer { diff --git a/connector/kafka-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMQProducer b/connector/kafka-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMQProducer new file mode 100644 index 00000000..c32cce0b --- /dev/null +++ b/connector/kafka-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMQProducer @@ -0,0 +1 @@ +kafka=com.alibaba.otter.canal.connector.kafka.producer.CanalKafkaProducer \ No newline at end of file diff --git a/connector/kafka-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer b/connector/kafka-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer new file mode 100644 index 00000000..24e60a9b --- /dev/null +++ b/connector/kafka-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer @@ -0,0 +1 @@ +kafka=com.alibaba.otter.canal.connector.kafka.consumer.CanalKafkaConsumer \ No newline at end of file diff --git a/connector/kafka-connector/src/test/java/com/alibaba/otter/canal/connector/kafka/test/CanalKafkaProducerTest.java b/connector/kafka-connector/src/test/java/com/alibaba/otter/canal/connector/kafka/test/CanalKafkaProducerTest.java new file mode 100644 index 00000000..1c23d43e --- /dev/null +++ b/connector/kafka-connector/src/test/java/com/alibaba/otter/canal/connector/kafka/test/CanalKafkaProducerTest.java @@ -0,0 +1,29 @@ +package com.alibaba.otter.canal.connector.kafka.test; + +import java.io.FileInputStream; +import java.io.IOException; +import java.util.Properties; + +import org.junit.Ignore; +import org.junit.Test; + +import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer; +import com.alibaba.otter.canal.connector.core.spi.ExtensionLoader; + +@Ignore +public class CanalKafkaProducerTest { + + @Test + public void testLoadKafkaProducer() throws IOException { + Properties pro = new Properties(); + FileInputStream in = new FileInputStream("../../deployer/src/main/resources/canal.properties"); + pro.load(in); + + ExtensionLoader loader = ExtensionLoader.getExtensionLoader(CanalMQProducer.class); + CanalMQProducer canalMQProducer = loader + .getExtension("kafka", "/../../deployer/target/canal/plugin", "/../../deployer/target/canal/plugin"); + canalMQProducer.init(pro); + + in.close(); + } +} diff --git a/connector/pom.xml b/connector/pom.xml new file mode 100644 index 00000000..b56937d5 --- /dev/null +++ b/connector/pom.xml @@ -0,0 +1,131 @@ + + + 4.0.0 + + com.alibaba.otter + canal + 1.1.5-SNAPSHOT + ../pom.xml + + + com.alibaba.otter + canal.connector + 1.1.5-SNAPSHOT + pom + canal connector module for otter ${project.version} + + + UTF-8 + true + true + 1.8 + 1.8 + UTF-8 + + + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0 + + + + + git@github.com:alibaba/canal.git + scm:git:git@github.com:alibaba/canal.git + scm:git:git@github.com:alibaba/canal.git + + + + + central + http://repo1.maven.org/maven2 + + true + + + false + + + + java.net + http://download.java.net/maven/2/ + + true + + + false + + + + aliyun + http://maven.aliyun.com/nexus/content/groups/public/ + + true + + + false + + + + sonatype + sonatype + https://oss.sonatype.org/content/repositories/snapshots + + false + + + true + + + + sonatype-release + sonatype-release + https://oss.sonatype.org/service/local/repositories/releases/content + + false + + + true + + + + + + kafka-connector + core + rocketmq-connector + rabbitmq-connector + tcp-connector + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.7.0 + + ${java_source_version} + ${java_target_version} + ${file_encoding} + + + + + + + + sonatype-nexus-snapshots + Sonatype Nexus Snapshots + https://oss.sonatype.org/content/repositories/snapshots/ + + + sonatype-nexus-staging + Nexus Release Repository + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + \ No newline at end of file diff --git a/connector/rabbitmq-connector/pom.xml b/connector/rabbitmq-connector/pom.xml new file mode 100644 index 00000000..679b9311 --- /dev/null +++ b/connector/rabbitmq-connector/pom.xml @@ -0,0 +1,99 @@ + + + + canal.connector + com.alibaba.otter + 1.1.5-SNAPSHOT + ../pom.xml + + + 4.0.0 + com.alibaba.otter + connector.rabbitmq + jar + canal connector rabbitmq module for otter ${project.version} + + + + com.alibaba.otter + canal.protocol + ${project.version} + provided + + + com.alibaba.otter + connector.core + ${project.version} + provided + + + com.rabbitmq + amqp-client + 5.5.0 + + + com.alibaba.mq-amqp + mq-amqp-client + 1.0.3 + + + junit + junit + test + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.4 + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + org.apache.maven.plugins + maven-antrun-plugin + 1.8 + + + package + + run + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/config/RabbitMQConstants.java b/connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/config/RabbitMQConstants.java new file mode 100644 index 00000000..f6b6f8e6 --- /dev/null +++ b/connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/config/RabbitMQConstants.java @@ -0,0 +1,20 @@ +package com.alibaba.otter.canal.connector.rabbitmq.config; + +/** + * RabbitMQ 配置常量类 + * + * @author rewerma 2020-01-27 + * @version 1.0.0 + */ +public class RabbitMQConstants { + + public static final String ROOT = "rabbitmq"; + + public static final String RABBITMQ_HOST = ROOT + "." + "host"; + public static final String RABBITMQ_EXCHANGE = ROOT + "." + "exchange"; + public static final String RABBITMQ_VIRTUAL_HOST = ROOT + "." + "virtual.host"; + public static final String RABBITMQ_USERNAME = ROOT + "." + "username"; + public static final String RABBITMQ_PASSWORD = ROOT + "." + "password"; + + public static final String RABBITMQ_RESOURCE_OWNERID = ROOT + "." + "rabbitmq.resource.ownerId"; +} diff --git a/connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/config/RabbitMQProducerConfig.java b/connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/config/RabbitMQProducerConfig.java new file mode 100644 index 00000000..57a9d12d --- /dev/null +++ b/connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/config/RabbitMQProducerConfig.java @@ -0,0 +1,58 @@ +package com.alibaba.otter.canal.connector.rabbitmq.config; + +import com.alibaba.otter.canal.connector.core.config.MQProperties; + +/** + * RabbitMQ 配置类 + * + * @author rewerma 2020-01-27 + * @version 1.0.0 + */ +public class RabbitMQProducerConfig extends MQProperties { + + private String host; + private String virtualHost; + private String exchange; + private String username; + private String password; + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public String getVirtualHost() { + return virtualHost; + } + + public void setVirtualHost(String virtualHost) { + this.virtualHost = virtualHost; + } + + public String getExchange() { + return exchange; + } + + public void setExchange(String exchange) { + this.exchange = exchange; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } +} diff --git a/connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/consumer/CanalRabbitMQConsumer.java b/connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/consumer/CanalRabbitMQConsumer.java new file mode 100644 index 00000000..d7b219a7 --- /dev/null +++ b/connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/consumer/CanalRabbitMQConsumer.java @@ -0,0 +1,214 @@ +package com.alibaba.otter.canal.connector.rabbitmq.consumer; + +import java.io.IOException; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import com.alibaba.otter.canal.connector.core.spi.SPI; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.fastjson.JSON; +import com.alibaba.otter.canal.connector.core.config.CanalConstants; +import com.alibaba.otter.canal.connector.core.consumer.CommonMessage; +import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer; +import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil; +import com.alibaba.otter.canal.connector.core.util.MessageUtil; +import com.alibaba.otter.canal.connector.rabbitmq.config.RabbitMQConstants; +import com.alibaba.otter.canal.connector.rabbitmq.producer.AliyunCredentialsProvider; +import com.alibaba.otter.canal.protocol.Message; +import com.alibaba.otter.canal.protocol.exception.CanalClientException; +import com.google.common.collect.Lists; +import com.rabbitmq.client.*; + +/** + * RabbitMQ consumer SPI 实现 + * + * @author rewerma 2020-02-01 + * @version 1.0.0 + */ +@SPI("rabbitmq") +public class CanalRabbitMQConsumer implements CanalMsgConsumer { + + private static final Logger logger = LoggerFactory + .getLogger(CanalRabbitMQConsumer.class); + + // 链接地址 + private String nameServer; + // 主机名 + private String vhost; + private String queueName; + + // 一些鉴权信息 + private String accessKey; + private String secretKey; + private Long resourceOwnerId; + private String username; + private String password; + + private boolean flatMessage; + + private Connection connect; + private Channel channel; + + private long batchProcessTimeout = 60 * 1000; + private BlockingQueue> messageBlockingQueue; + private volatile ConsumerBatchMessage lastGetBatchMessage = null; + + @Override + public void init(Properties properties, String topic, String groupId) { + this.nameServer = properties.getProperty("rabbitmq.host"); + this.vhost = properties.getProperty("rabbitmq.virtual.host"); + this.queueName = topic; + this.accessKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_ACCESS_KEY); + this.secretKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_SECRET_KEY); + this.username = properties.getProperty(RabbitMQConstants.RABBITMQ_USERNAME); + this.password = properties.getProperty(RabbitMQConstants.RABBITMQ_PASSWORD); + Long resourceOwnerIdPro = (Long) properties.get(RabbitMQConstants.RABBITMQ_RESOURCE_OWNERID); + if (resourceOwnerIdPro != null) { + this.resourceOwnerId = resourceOwnerIdPro; + } + this.flatMessage = (Boolean) properties.get(CanalConstants.CANAL_MQ_FLAT_MESSAGE); + this.messageBlockingQueue = new LinkedBlockingQueue<>(1024); + } + + @Override + public void connect() { + ConnectionFactory factory = new ConnectionFactory(); + if (accessKey.length() > 0 && secretKey.length() > 0) { + factory.setCredentialsProvider(new AliyunCredentialsProvider(accessKey, secretKey, resourceOwnerId)); + } else { + factory.setUsername(username); + factory.setPassword(password); + } + factory.setHost(nameServer); + factory.setAutomaticRecoveryEnabled(true); + factory.setNetworkRecoveryInterval(5000); + factory.setVirtualHost(vhost); + try { + connect = factory.newConnection(); + channel = connect.createChannel(); + } catch (IOException | TimeoutException e) { + throw new CanalClientException("Start RabbitMQ producer error", e); + } + + // 不存在连接 则重新连接 + if (connect == null) { + this.connect(); + } + + Consumer consumer = new DefaultConsumer(channel) { + + @Override + public void handleDelivery(String consumerTag, Envelope envelope, AMQP.BasicProperties properties, + byte[] body) throws IOException { + + if (body != null) { + channel.basicAck(envelope.getDeliveryTag(), process(body)); + } + } + }; + try { + channel.basicConsume(queueName, false, consumer); + } catch (IOException e) { + throw new CanalClientException("error", e); + } + } + + private boolean process(byte[] messageData) { + if (logger.isDebugEnabled()) { + logger.debug("Get Message: {}", new String(messageData)); + } + List messageList = Lists.newArrayList(); + if (!flatMessage) { + Message message = CanalMessageSerializerUtil.deserializer(messageData); + messageList.addAll(MessageUtil.convert(message)); + } else { + CommonMessage commonMessage = JSON.parseObject(messageData, CommonMessage.class); + messageList.add(commonMessage); + } + ConsumerBatchMessage batchMessage = new ConsumerBatchMessage<>(messageList); + try { + messageBlockingQueue.put(batchMessage); + } catch (InterruptedException e) { + logger.error("Put message to queue error", e); + throw new RuntimeException(e); + } + boolean isCompleted; + try { + isCompleted = batchMessage.waitFinish(batchProcessTimeout); + } catch (InterruptedException e) { + logger.error("Interrupted when waiting messages to be finished.", e); + throw new RuntimeException(e); + } + boolean isSuccess = batchMessage.isSuccess(); + return isCompleted && isSuccess; + } + + @Override + public List getMessage(Long timeout, TimeUnit unit) { + try { + if (this.lastGetBatchMessage != null) { + throw new CanalClientException("mq get/ack not support concurrent & async ack"); + } + + ConsumerBatchMessage batchMessage = messageBlockingQueue.poll(timeout, unit); + if (batchMessage != null) { + this.lastGetBatchMessage = batchMessage; + return batchMessage.getData(); + } + } catch (InterruptedException ex) { + logger.warn("Get message timeout", ex); + throw new CanalClientException("Failed to fetch the data after: " + timeout); + } + return null; + } + + @Override + public void rollback() { + try { + if (this.lastGetBatchMessage != null) { + this.lastGetBatchMessage.fail(); + } + } finally { + this.lastGetBatchMessage = null; + } + } + + @Override + public void ack() { + try { + if (this.lastGetBatchMessage != null) { + this.lastGetBatchMessage.ack(); + } + } catch (Throwable e) { + if (this.lastGetBatchMessage != null) { + this.lastGetBatchMessage.fail(); + } + } finally { + this.lastGetBatchMessage = null; + } + } + + @Override + public void disconnect() { + if (connect != null) { + try { + connect.close(); + } catch (IOException e) { + throw new CanalClientException("stop connect error", e); + } + } + if (channel != null) { + try { + channel.close(); + } catch (IOException | TimeoutException e) { + throw new CanalClientException("stop channel error", e); + } + } + } +} diff --git a/connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/consumer/ConsumerBatchMessage.java b/connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/consumer/ConsumerBatchMessage.java new file mode 100644 index 00000000..dc0a8ee8 --- /dev/null +++ b/connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/consumer/ConsumerBatchMessage.java @@ -0,0 +1,48 @@ +package com.alibaba.otter.canal.connector.rabbitmq.consumer; + +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +public class ConsumerBatchMessage { + + private final List data; + private CountDownLatch latch; + private boolean hasFailure = false; + + public ConsumerBatchMessage(List data){ + this.data = data; + latch = new CountDownLatch(1); + } + + public boolean waitFinish(long timeout) throws InterruptedException { + return latch.await(timeout, TimeUnit.MILLISECONDS); + } + + public boolean isSuccess() { + return !hasFailure; + } + + public List getData() { + return data; + } + + /** + * Countdown if the sub message is successful. + */ + public void ack() { + latch.countDown(); + } + + /** + * Countdown and fail-fast if the sub message is failed. + */ + public void fail() { + hasFailure = true; + // fail fast + long count = latch.getCount(); + for (int i = 0; i < count; i++) { + latch.countDown(); + } + } +} diff --git a/server/src/main/java/com/alibaba/otter/canal/rabbitmq/AliyunCredentialsProvider.java b/connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/producer/AliyunCredentialsProvider.java similarity index 93% rename from server/src/main/java/com/alibaba/otter/canal/rabbitmq/AliyunCredentialsProvider.java rename to connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/producer/AliyunCredentialsProvider.java index 9fb97a16..1843c137 100644 --- a/server/src/main/java/com/alibaba/otter/canal/rabbitmq/AliyunCredentialsProvider.java +++ b/connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/producer/AliyunCredentialsProvider.java @@ -1,8 +1,4 @@ -/** - * aliyun amqp协议 账号类 - * 暂不支持STS授权情况 - */ -package com.alibaba.otter.canal.rabbitmq; +package com.alibaba.otter.canal.connector.rabbitmq.producer; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; @@ -10,6 +6,12 @@ import java.security.NoSuchAlgorithmException; import com.alibaba.mq.amqp.utils.UserUtils; import com.rabbitmq.client.impl.CredentialsProvider; +/** + * aliyun amqp协议 账号类 + * 暂不支持STS授权情况 + * + * @version 1.0.0 + */ public class AliyunCredentialsProvider implements CredentialsProvider { /** diff --git a/connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/producer/CanalRabbitMQProducer.java b/connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/producer/CanalRabbitMQProducer.java new file mode 100644 index 00000000..fd72936c --- /dev/null +++ b/connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/producer/CanalRabbitMQProducer.java @@ -0,0 +1,180 @@ +package com.alibaba.otter.canal.connector.rabbitmq.producer; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.TimeoutException; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.serializer.SerializerFeature; +import com.alibaba.otter.canal.common.CanalException; +import com.alibaba.otter.canal.common.utils.ExecutorTemplate; +import com.alibaba.otter.canal.connector.core.producer.AbstractMQProducer; +import com.alibaba.otter.canal.connector.core.util.Callback; +import com.alibaba.otter.canal.connector.core.producer.MQDestination; +import com.alibaba.otter.canal.connector.core.producer.MQMessageUtils; +import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer; +import com.alibaba.otter.canal.connector.core.spi.SPI; +import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil; +import com.alibaba.otter.canal.connector.rabbitmq.config.RabbitMQConstants; +import com.alibaba.otter.canal.connector.rabbitmq.config.RabbitMQProducerConfig; +import com.alibaba.otter.canal.protocol.FlatMessage; +import com.alibaba.otter.canal.protocol.Message; +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.Connection; +import com.rabbitmq.client.ConnectionFactory; + +/** + * RabbitMQ Producer SPI 实现 + * + * @author rewerma 2020-01-27 + * @version 1.0.0 + */ +@SPI("rabbitmq") +public class CanalRabbitMQProducer extends AbstractMQProducer implements CanalMQProducer { + + private static final Logger logger = LoggerFactory.getLogger(CanalRabbitMQProducer.class); + + private Connection connect; + private Channel channel; + + @Override + public void init(Properties properties) { + RabbitMQProducerConfig rabbitMQProperties = new RabbitMQProducerConfig(); + this.mqProperties = rabbitMQProperties; + super.init(properties); + loadRabbitMQProperties(properties); + + ConnectionFactory factory = new ConnectionFactory(); + factory.setHost(rabbitMQProperties.getHost()); + if (mqProperties.getAliyunAccessKey().length() > 0 && mqProperties.getAliyunSecretKey().length() > 0 + && mqProperties.getAliyunUid() > 0) { + factory.setCredentialsProvider(new AliyunCredentialsProvider(mqProperties.getAliyunAccessKey(), + mqProperties.getAliyunSecretKey(), + mqProperties.getAliyunUid())); + } else { + factory.setUsername(rabbitMQProperties.getUsername()); + factory.setPassword(rabbitMQProperties.getPassword()); + } + factory.setVirtualHost(rabbitMQProperties.getVirtualHost()); + try { + connect = factory.newConnection(); + channel = connect.createChannel(); + // channel.exchangeDeclare(mqProperties.getExchange(), "topic"); + } catch (IOException | TimeoutException ex) { + throw new CanalException("Start RabbitMQ producer error", ex); + } + } + + private void loadRabbitMQProperties(Properties properties) { + RabbitMQProducerConfig rabbitMQProperties = (RabbitMQProducerConfig) this.mqProperties; + + String host = properties.getProperty(RabbitMQConstants.RABBITMQ_HOST); + if (!StringUtils.isEmpty(host)) { + rabbitMQProperties.setHost(host); + } + String vhost = properties.getProperty(RabbitMQConstants.RABBITMQ_VIRTUAL_HOST); + if (!StringUtils.isEmpty(vhost)) { + rabbitMQProperties.setVirtualHost(vhost); + } + String exchange = properties.getProperty(RabbitMQConstants.RABBITMQ_EXCHANGE); + if (!StringUtils.isEmpty(exchange)) { + rabbitMQProperties.setExchange(exchange); + } + String username = properties.getProperty(RabbitMQConstants.RABBITMQ_USERNAME); + if (!StringUtils.isEmpty(username)) { + rabbitMQProperties.setUsername(username); + } + String password = properties.getProperty(RabbitMQConstants.RABBITMQ_PASSWORD); + if (!StringUtils.isEmpty(password)) { + rabbitMQProperties.setPassword(password); + } + } + + @Override + public void send(final MQDestination destination, Message message, Callback callback) { + ExecutorTemplate template = new ExecutorTemplate(executor); + try { + if (!StringUtils.isEmpty(destination.getDynamicTopic())) { + // 动态topic + Map messageMap = MQMessageUtils + .messageTopics(message, destination.getTopic(), destination.getDynamicTopic()); + + for (Map.Entry entry : messageMap.entrySet()) { + final String topicName = entry.getKey().replace('.', '_'); + final com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue(); + + template.submit(new Runnable() { + + @Override + public void run() { + send(destination, topicName, messageSub); + } + }); + } + + template.waitForResult(); + } else { + send(destination, destination.getTopic(), message); + } + callback.commit(); + } catch (Throwable e) { + logger.error(e.getMessage(), e); + callback.rollback(); + } finally { + template.clear(); + } + } + + private void send(MQDestination canalDestination, String topicName, Message messageSub) { + if (!mqProperties.isFlatMessage()) { + byte[] message = CanalMessageSerializerUtil.serializer(messageSub, mqProperties.isFilterTransactionEntry()); + if (logger.isDebugEnabled()) { + logger.debug("send message:{} to destination:{}", message, canalDestination.getCanalDestination()); + } + sendMessage(topicName, message); + } else { + // 并发构造 + MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(messageSub, executor); + // 串行分区 + List flatMessages = MQMessageUtils.messageConverter(datas, messageSub.getId()); + for (FlatMessage flatMessage : flatMessages) { + byte[] message = JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue); + if (logger.isDebugEnabled()) { + logger.debug("send message:{} to destination:{}", message, canalDestination.getCanalDestination()); + } + sendMessage(topicName, message); + } + } + + } + + private void sendMessage(String queueName, byte[] message) { + // tips: 目前逻辑中暂不处理对exchange处理，请在Console后台绑定 才可使用routekey + try { + RabbitMQProducerConfig rabbitMQProperties = (RabbitMQProducerConfig) this.mqProperties; + channel.basicPublish(rabbitMQProperties.getExchange(), queueName, null, message); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public void stop() { + logger.info("## Stop RabbitMQ producer##"); + try { + this.connect.close(); + this.channel.close(); + super.stop(); + } catch (IOException | TimeoutException ex) { + throw new CanalException("Stop RabbitMQ producer error", ex); + } + + super.stop(); + } +} diff --git a/connector/rabbitmq-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMQProducer b/connector/rabbitmq-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMQProducer new file mode 100644 index 00000000..0271776d --- /dev/null +++ b/connector/rabbitmq-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMQProducer @@ -0,0 +1 @@ +rabbitmq=com.alibaba.otter.canal.connector.rabbitmq.producer.CanalRabbitMQProducer \ No newline at end of file diff --git a/connector/rabbitmq-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer b/connector/rabbitmq-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer new file mode 100644 index 00000000..c26a4ab9 --- /dev/null +++ b/connector/rabbitmq-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer @@ -0,0 +1 @@ +rabbitmq=com.alibaba.otter.canal.connector.rabbitmq.consumer.CanalRabbitMQConsumer \ No newline at end of file diff --git a/connector/rocketmq-connector/pom.xml b/connector/rocketmq-connector/pom.xml new file mode 100644 index 00000000..2a201832 --- /dev/null +++ b/connector/rocketmq-connector/pom.xml @@ -0,0 +1,104 @@ + + + + canal.connector + com.alibaba.otter + 1.1.5-SNAPSHOT + ../pom.xml + + + 4.0.0 + com.alibaba.otter + connector.rocketmq + jar + canal connector rocketMQ module for otter ${project.version} + + + 4.5.2 + + + + + com.alibaba.otter + canal.protocol + ${project.version} + provided + + + com.alibaba.otter + connector.core + ${project.version} + provided + + + org.apache.rocketmq + rocketmq-client + ${rocketmq_version} + + + org.apache.rocketmq + rocketmq-acl + ${rocketmq_version} + + + + junit + junit + test + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.4 + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + org.apache.maven.plugins + maven-antrun-plugin + 1.8 + + + package + + run + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/config/RocketMQConstants.java b/connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/config/RocketMQConstants.java new file mode 100644 index 00000000..5f728627 --- /dev/null +++ b/connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/config/RocketMQConstants.java @@ -0,0 +1,25 @@ +package com.alibaba.otter.canal.connector.rocketmq.config; + +/** + * RocketMQ 配置常量类 + * + * @author rewerma 2020-01-27 + * @version 1.0.0 + */ +public class RocketMQConstants { + + public static final String ROOT = "rocketmq"; + + public static final String ROCKETMQ_PRODUCER_GROUP = ROOT + "." + "producer.group"; + public static final String ROCKETMQ_ENABLE_MESSAGE_TRACE = ROOT + "." + "enable.message.trace"; + public static final String ROCKETMQ_CUSTOMIZED_TRACE_TOPIC = ROOT + "." + "customized.trace.topic"; + public static final String ROCKETMQ_NAMESPACE = ROOT + "." + "namespace"; + public static final String ROCKETMQ_NAMESRV_ADDR = ROOT + "." + "namesrv.addr"; + public static final String ROCKETMQ_RETRY_TIMES_WHEN_SEND_FAILED = ROOT + "." + "retry.times.when.send.failed"; + public static final String ROCKETMQ_VIP_CHANNEL_ENABLED = ROOT + "." + "vip.channel.enabled"; + + public static final String ROCKETMQ_ACCESS_CHANNEL = ROOT + "." + "access.channel"; + public static final String ROCKETMQ_BATCH_SIZE = ROOT + "." + "batch.size"; + public static final String ROCKETMQ_SUBSCRIBE_FILTER = ROOT + "." + "subscribe.filter"; + +} diff --git a/connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/config/RocketMQProducerConfig.java b/connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/config/RocketMQProducerConfig.java new file mode 100644 index 00000000..5aebb9fb --- /dev/null +++ b/connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/config/RocketMQProducerConfig.java @@ -0,0 +1,76 @@ +package com.alibaba.otter.canal.connector.rocketmq.config; + +import com.alibaba.otter.canal.connector.core.config.MQProperties; + +/** + * RocketMQ 配置类 + * + * @author rewerma 2020-01-27 + * @version 1.0.0 + */ +public class RocketMQProducerConfig extends MQProperties { + + private String producerGroup; + private boolean enableMessageTrace = false; + private String customizedTraceTopic; + private String namespace; + private String namesrvAddr; + private int retryTimesWhenSendFailed = 0; + private boolean vipChannelEnabled = false; + + public String getProducerGroup() { + return producerGroup; + } + + public void setProducerGroup(String producerGroup) { + this.producerGroup = producerGroup; + } + + public boolean isEnableMessageTrace() { + return enableMessageTrace; + } + + public void setEnableMessageTrace(boolean enableMessageTrace) { + this.enableMessageTrace = enableMessageTrace; + } + + public String getCustomizedTraceTopic() { + return customizedTraceTopic; + } + + public void setCustomizedTraceTopic(String customizedTraceTopic) { + this.customizedTraceTopic = customizedTraceTopic; + } + + public String getNamespace() { + return namespace; + } + + public void setNamespace(String namespace) { + this.namespace = namespace; + } + + public String getNamesrvAddr() { + return namesrvAddr; + } + + public void setNamesrvAddr(String namesrvAddr) { + this.namesrvAddr = namesrvAddr; + } + + public int getRetryTimesWhenSendFailed() { + return retryTimesWhenSendFailed; + } + + public void setRetryTimesWhenSendFailed(int retryTimesWhenSendFailed) { + this.retryTimesWhenSendFailed = retryTimesWhenSendFailed; + } + + public boolean isVipChannelEnabled() { + return vipChannelEnabled; + } + + public void setVipChannelEnabled(boolean vipChannelEnabled) { + this.vipChannelEnabled = vipChannelEnabled; + } +} diff --git a/connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/consumer/CanalRocketMQConsumer.java b/connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/consumer/CanalRocketMQConsumer.java new file mode 100644 index 00000000..79ee58a5 --- /dev/null +++ b/connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/consumer/CanalRocketMQConsumer.java @@ -0,0 +1,237 @@ +package com.alibaba.otter.canal.connector.rocketmq.consumer; + +import java.util.List; +import java.util.Properties; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.lang.StringUtils; +import org.apache.rocketmq.acl.common.AclClientRPCHook; +import org.apache.rocketmq.acl.common.SessionCredentials; +import org.apache.rocketmq.client.AccessChannel; +import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; +import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus; +import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly; +import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.remoting.RPCHook; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.fastjson.JSON; +import com.alibaba.otter.canal.connector.core.config.CanalConstants; +import com.alibaba.otter.canal.connector.core.consumer.CommonMessage; +import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer; +import com.alibaba.otter.canal.connector.core.spi.SPI; +import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil; +import com.alibaba.otter.canal.connector.core.util.MessageUtil; +import com.alibaba.otter.canal.connector.rocketmq.config.RocketMQConstants; +import com.alibaba.otter.canal.protocol.Message; +import com.alibaba.otter.canal.protocol.exception.CanalClientException; +import com.google.common.collect.Lists; + +/** + * RocketMQ consumer SPI 实现 + * + * @author rewerma 2020-02-01 + * @version 1.0.0 + */ +@SPI("rocketmq") +public class CanalRocketMQConsumer implements CanalMsgConsumer { + + private static final Logger logger = LoggerFactory + .getLogger(CanalRocketMQConsumer.class); + private static final String CLOUD_ACCESS_CHANNEL = "cloud"; + + private String nameServer; + private String topic; + private String groupName; + private volatile boolean connected = false; + private DefaultMQPushConsumer rocketMQConsumer; + private BlockingQueue> messageBlockingQueue; + private int batchSize = -1; + private long batchProcessTimeout = 60 * 1000; + private boolean flatMessage; + private volatile ConsumerBatchMessage lastGetBatchMessage = null; + private String accessKey; + private String secretKey; + private String customizedTraceTopic; + private boolean enableMessageTrace = false; + private String accessChannel; + private String namespace; + private String filter = "*"; + + @Override + public void init(Properties properties, String topic, String groupName) { + this.topic = topic; + this.groupName = groupName; + this.flatMessage = (Boolean) properties.get(CanalConstants.CANAL_MQ_FLAT_MESSAGE); + this.messageBlockingQueue = new LinkedBlockingQueue<>(1024); + this.accessKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_ACCESS_KEY); + this.secretKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_SECRET_KEY); + String enableMessageTrace = properties.getProperty(RocketMQConstants.ROCKETMQ_ENABLE_MESSAGE_TRACE); + if (StringUtils.isNotEmpty(enableMessageTrace)) { + this.enableMessageTrace = Boolean.parseBoolean(enableMessageTrace); + } + this.customizedTraceTopic = properties.getProperty(RocketMQConstants.ROCKETMQ_CUSTOMIZED_TRACE_TOPIC); + this.accessChannel = properties.getProperty(RocketMQConstants.ROCKETMQ_ACCESS_CHANNEL); + this.namespace = properties.getProperty(RocketMQConstants.ROCKETMQ_NAMESPACE); + this.nameServer = properties.getProperty(RocketMQConstants.ROCKETMQ_NAMESRV_ADDR); + String batchSize = properties.getProperty(RocketMQConstants.ROCKETMQ_BATCH_SIZE); + if (StringUtils.isNotEmpty(batchSize)) { + this.batchSize = Integer.parseInt(batchSize); + } + String subscribeFilter = properties.getProperty(RocketMQConstants.ROCKETMQ_SUBSCRIBE_FILTER); + if (StringUtils.isNotEmpty(subscribeFilter)) { + this.filter = subscribeFilter; + } + } + + @Override + public void connect() { + RPCHook rpcHook = null; + if (null != accessKey && accessKey.length() > 0 && null != secretKey && secretKey.length() > 0) { + SessionCredentials sessionCredentials = new SessionCredentials(); + sessionCredentials.setAccessKey(accessKey); + sessionCredentials.setSecretKey(secretKey); + rpcHook = new AclClientRPCHook(sessionCredentials); + } + + rocketMQConsumer = new DefaultMQPushConsumer(groupName, + rpcHook, + new AllocateMessageQueueAveragely(), + enableMessageTrace, + customizedTraceTopic); + rocketMQConsumer.setVipChannelEnabled(false); + if (CLOUD_ACCESS_CHANNEL.equals(this.accessChannel)) { + rocketMQConsumer.setAccessChannel(AccessChannel.CLOUD); + } + + if (!StringUtils.isEmpty(this.namespace)) { + rocketMQConsumer.setNamespace(this.namespace); + } + + if (!StringUtils.isBlank(nameServer)) { + rocketMQConsumer.setNamesrvAddr(nameServer); + } + if (batchSize != -1) { + rocketMQConsumer.setConsumeMessageBatchMaxSize(batchSize); + } + + try { + if (rocketMQConsumer == null) { + this.connect(); + } + rocketMQConsumer.subscribe(this.topic, this.filter); + rocketMQConsumer.registerMessageListener((MessageListenerOrderly) (messageExts, context) -> { + context.setAutoCommit(true); + boolean isSuccess = process(messageExts); + if (isSuccess) { + return ConsumeOrderlyStatus.SUCCESS; + } else { + return ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT; + } + }); + rocketMQConsumer.start(); + } catch (MQClientException ex) { + connected = false; + logger.error("Start RocketMQ consumer error", ex); + } + } + + private boolean process(List messageExts) { + if (logger.isDebugEnabled()) { + logger.debug("Get Message: {}", messageExts); + } + List messageList = Lists.newArrayList(); + for (MessageExt messageExt : messageExts) { + byte[] data = messageExt.getBody(); + if (data != null) { + try { + if (!flatMessage) { + Message message = CanalMessageSerializerUtil.deserializer(data); + messageList.addAll(MessageUtil.convert(message)); + } else { + CommonMessage commonMessage = JSON.parseObject(data, CommonMessage.class); + messageList.add(commonMessage); + } + } catch (Exception ex) { + logger.error("Add message error", ex); + throw new CanalClientException(ex); + } + } else { + logger.warn("Received message data is null"); + } + } + ConsumerBatchMessage batchMessage = new ConsumerBatchMessage<>(messageList); + try { + messageBlockingQueue.put(batchMessage); + } catch (InterruptedException e) { + logger.error("Put message to queue error", e); + throw new RuntimeException(e); + } + boolean isCompleted; + try { + isCompleted = batchMessage.waitFinish(batchProcessTimeout); + } catch (InterruptedException e) { + logger.error("Interrupted when waiting messages to be finished.", e); + throw new RuntimeException(e); + } + boolean isSuccess = batchMessage.isSuccess(); + return isCompleted && isSuccess; + } + + @Override + public List getMessage(Long timeout, TimeUnit unit) { + try { + if (this.lastGetBatchMessage != null) { + throw new CanalClientException("mq get/ack not support concurrent & async ack"); + } + + ConsumerBatchMessage batchMessage = messageBlockingQueue.poll(timeout, unit); + if (batchMessage != null) { + this.lastGetBatchMessage = batchMessage; + return batchMessage.getData(); + } + } catch (InterruptedException ex) { + logger.warn("Get message timeout", ex); + throw new CanalClientException("Failed to fetch the data after: " + timeout); + } + return null; + } + + @Override + public void rollback() { + try { + if (this.lastGetBatchMessage != null) { + this.lastGetBatchMessage.fail(); + } + } finally { + this.lastGetBatchMessage = null; + } + } + + @Override + public void ack() { + try { + if (this.lastGetBatchMessage != null) { + this.lastGetBatchMessage.ack(); + } + } catch (Throwable e) { + if (this.lastGetBatchMessage != null) { + this.lastGetBatchMessage.fail(); + } + } finally { + this.lastGetBatchMessage = null; + } + } + + @Override + public void disconnect() { + rocketMQConsumer.unsubscribe(topic); + rocketMQConsumer.shutdown(); + connected = false; + } +} diff --git a/connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/consumer/ConsumerBatchMessage.java b/connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/consumer/ConsumerBatchMessage.java new file mode 100644 index 00000000..02c0e01d --- /dev/null +++ b/connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/consumer/ConsumerBatchMessage.java @@ -0,0 +1,48 @@ +package com.alibaba.otter.canal.connector.rocketmq.consumer; + +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +public class ConsumerBatchMessage { + + private final List data; + private CountDownLatch latch; + private boolean hasFailure = false; + + public ConsumerBatchMessage(List data){ + this.data = data; + latch = new CountDownLatch(1); + } + + public boolean waitFinish(long timeout) throws InterruptedException { + return latch.await(timeout, TimeUnit.MILLISECONDS); + } + + public boolean isSuccess() { + return !hasFailure; + } + + public List getData() { + return data; + } + + /** + * Countdown if the sub message is successful. + */ + public void ack() { + latch.countDown(); + } + + /** + * Countdown and fail-fast if the sub message is failed. + */ + public void fail() { + hasFailure = true; + // fail fast + long count = latch.getCount(); + for (int i = 0; i < count; i++) { + latch.countDown(); + } + } +} diff --git a/connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/producer/CanalRocketMQProducer.java b/connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/producer/CanalRocketMQProducer.java new file mode 100644 index 00000000..b6fe4d2a --- /dev/null +++ b/connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/producer/CanalRocketMQProducer.java @@ -0,0 +1,312 @@ +package com.alibaba.otter.canal.connector.rocketmq.producer; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.stream.Collectors; + +import org.apache.commons.lang.StringUtils; +import org.apache.rocketmq.acl.common.AclClientRPCHook; +import org.apache.rocketmq.acl.common.SessionCredentials; +import org.apache.rocketmq.client.AccessChannel; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; +import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.remoting.RPCHook; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.serializer.SerializerFeature; +import com.alibaba.otter.canal.common.CanalException; +import com.alibaba.otter.canal.common.utils.ExecutorTemplate; +import com.alibaba.otter.canal.connector.core.producer.AbstractMQProducer; +import com.alibaba.otter.canal.connector.core.util.Callback; +import com.alibaba.otter.canal.connector.core.producer.MQDestination; +import com.alibaba.otter.canal.connector.core.producer.MQMessageUtils; +import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer; +import com.alibaba.otter.canal.connector.core.spi.SPI; +import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil; +import com.alibaba.otter.canal.connector.rocketmq.config.RocketMQConstants; +import com.alibaba.otter.canal.connector.rocketmq.config.RocketMQProducerConfig; +import com.alibaba.otter.canal.protocol.FlatMessage; + +/** + * RocketMQ Producer SPI 实现 + * + * @author rewerma 2020-01-27 + * @version 1.0.0 + */ +@SPI("rocketmq") +public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQProducer { + + private static final Logger logger = LoggerFactory.getLogger(CanalRocketMQProducer.class); + + private DefaultMQProducer defaultMQProducer; + private static final String CLOUD_ACCESS_CHANNEL = "cloud"; + + @Override + public void init(Properties properties) { + RocketMQProducerConfig rocketMQProperties = new RocketMQProducerConfig(); + this.mqProperties = rocketMQProperties; + super.init(properties); + loadRocketMQProperties(properties); + + RPCHook rpcHook = null; + if (mqProperties.getAliyunAccessKey().length() > 0 + && mqProperties.getAliyunSecretKey().length() > 0) { + SessionCredentials sessionCredentials = new SessionCredentials(); + sessionCredentials.setAccessKey(mqProperties.getAliyunAccessKey()); + sessionCredentials.setSecretKey(mqProperties.getAliyunSecretKey()); + rpcHook = new AclClientRPCHook(sessionCredentials); + } + + defaultMQProducer = new DefaultMQProducer(rocketMQProperties.getProducerGroup(), + rpcHook, + rocketMQProperties.isEnableMessageTrace(), + rocketMQProperties.getCustomizedTraceTopic()); + if (CLOUD_ACCESS_CHANNEL.equals(rocketMQProperties.getAccessChannel())) { + defaultMQProducer.setAccessChannel(AccessChannel.CLOUD); + } + if (!StringUtils.isEmpty(rocketMQProperties.getNamespace())) { + defaultMQProducer.setNamespace(rocketMQProperties.getNamespace()); + } + defaultMQProducer.setNamesrvAddr(rocketMQProperties.getNamesrvAddr()); + defaultMQProducer.setRetryTimesWhenSendFailed(rocketMQProperties.getRetryTimesWhenSendFailed()); + defaultMQProducer.setVipChannelEnabled(rocketMQProperties.isVipChannelEnabled()); + logger.info("##Start RocketMQ producer##"); + try { + defaultMQProducer.start(); + } catch (MQClientException ex) { + throw new CanalException("Start RocketMQ producer error", ex); + } + } + + private void loadRocketMQProperties(Properties properties) { + RocketMQProducerConfig rocketMQProperties = (RocketMQProducerConfig) this.mqProperties; + + String producerGroup = properties.getProperty(RocketMQConstants.ROCKETMQ_PRODUCER_GROUP); + if (!StringUtils.isEmpty(producerGroup)) { + rocketMQProperties.setProducerGroup(producerGroup); + } + String enableMessageTrace = properties.getProperty(RocketMQConstants.ROCKETMQ_ENABLE_MESSAGE_TRACE); + if (!StringUtils.isEmpty(enableMessageTrace)) { + rocketMQProperties.setEnableMessageTrace(Boolean.parseBoolean(enableMessageTrace)); + } + String customizedTraceTopic = properties.getProperty(RocketMQConstants.ROCKETMQ_CUSTOMIZED_TRACE_TOPIC); + if (!StringUtils.isEmpty(customizedTraceTopic)) { + rocketMQProperties.setCustomizedTraceTopic(customizedTraceTopic); + } + String namespace = properties.getProperty(RocketMQConstants.ROCKETMQ_NAMESPACE); + if (!StringUtils.isEmpty(namespace)) { + rocketMQProperties.setNamespace(namespace); + } + String namesrvAddr = properties.getProperty(RocketMQConstants.ROCKETMQ_NAMESRV_ADDR); + if (!StringUtils.isEmpty(namesrvAddr)) { + rocketMQProperties.setNamesrvAddr(namesrvAddr); + } + String retry = properties.getProperty(RocketMQConstants.ROCKETMQ_RETRY_TIMES_WHEN_SEND_FAILED); + if (!StringUtils.isEmpty(retry)) { + rocketMQProperties.setRetryTimesWhenSendFailed(Integer.parseInt(retry)); + } + String vipChannelEnabled = properties.getProperty(RocketMQConstants.ROCKETMQ_VIP_CHANNEL_ENABLED); + if (!StringUtils.isEmpty(vipChannelEnabled)) { + rocketMQProperties.setVipChannelEnabled(Boolean.parseBoolean(vipChannelEnabled)); + } + } + + @Override + public void send(MQDestination destination, com.alibaba.otter.canal.protocol.Message message, Callback callback) { + ExecutorTemplate template = new ExecutorTemplate(executor); + try { + if (!StringUtils.isEmpty(destination.getDynamicTopic())) { + // 动态topic + Map messageMap = MQMessageUtils + .messageTopics(message, destination.getTopic(), destination.getDynamicTopic()); + + for (Map.Entry entry : messageMap.entrySet()) { + String topicName = entry.getKey().replace('.', '_'); + com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue(); + template.submit(() -> { + try { + send(destination, topicName, messageSub); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + + template.waitForResult(); + } else { + send(destination, destination.getTopic(), message); + } + + callback.commit(); + } catch (Throwable e) { + logger.error(e.getMessage(), e); + callback.rollback(); + } finally { + template.clear(); + } + } + + public void send(final MQDestination destination, String topicName, + com.alibaba.otter.canal.protocol.Message message) { + if (!mqProperties.isFlatMessage()) { + if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) { + // 并发构造 + MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor); + // 串行分区 + com.alibaba.otter.canal.protocol.Message[] messages = MQMessageUtils.messagePartition(datas, + message.getId(), + destination.getPartitionsNum(), + destination.getPartitionHash(), + mqProperties.isDatabaseHash()); + int length = messages.length; + + ExecutorTemplate template = new ExecutorTemplate(executor); + for (int i = 0; i < length; i++) { + com.alibaba.otter.canal.protocol.Message dataPartition = messages[i]; + if (dataPartition != null) { + final int index = i; + template.submit(() -> { + Message data = new Message(topicName, + CanalMessageSerializerUtil.serializer(dataPartition, + mqProperties.isFilterTransactionEntry())); + sendMessage(data, index); + }); + } + } + // 等所有分片发送完毕 + template.waitForResult(); + } else { + final int partition = destination.getPartition() != null ? destination.getPartition() : 0; + Message data = new Message(topicName, + CanalMessageSerializerUtil.serializer(message, mqProperties.isFilterTransactionEntry())); + sendMessage(data, partition); + } + } else { + // 并发构造 + MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor); + // 串行分区 + List flatMessages = MQMessageUtils.messageConverter(datas, message.getId()); + // 初始化分区合并队列 + if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) { + List> partitionFlatMessages = new ArrayList<>(); + for (int i = 0; i < destination.getPartitionsNum(); i++) { + partitionFlatMessages.add(new ArrayList<>()); + } + + for (FlatMessage flatMessage : flatMessages) { + FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage, + destination.getPartitionsNum(), + destination.getPartitionHash(), + mqProperties.isDatabaseHash()); + int length = partitionFlatMessage.length; + for (int i = 0; i < length; i++) { + partitionFlatMessages.get(i).add(partitionFlatMessage[i]); + } + } + + ExecutorTemplate template = new ExecutorTemplate(executor); + for (int i = 0; i < partitionFlatMessages.size(); i++) { + final List flatMessagePart = partitionFlatMessages.get(i); + if (flatMessagePart != null) { + final int index = i; + template.submit(() -> { + List messages = flatMessagePart.stream() + .map(flatMessage -> new Message(topicName, + JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue))) + .collect(Collectors.toList()); + // 批量发送 + sendMessage(messages, index); + }); + } + } + + // 批量等所有分区的结果 + template.waitForResult(); + } else { + final int partition = destination.getPartition() != null ? destination.getPartition() : 0; + List messages = flatMessages.stream() + .map(flatMessage -> new Message(topicName, + JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue))) + .collect(Collectors.toList()); + // 批量发送 + sendMessage(messages, partition); + } + } + } + + private void sendMessage(Message message, int partition) { + try { + SendResult sendResult = this.defaultMQProducer.send(message, (mqs, msg, arg) -> { + if (partition > mqs.size()) { + return mqs.get(partition % mqs.size()); + } else { + return mqs.get(partition); + } + }, null); + + if (logger.isDebugEnabled()) { + logger.debug("Send Message Result: {}", sendResult); + } + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @SuppressWarnings("deprecation") + private void sendMessage(List messages, int partition) { + if (messages.isEmpty()) { + return; + } + + // 获取一下messageQueue + DefaultMQProducerImpl innerProducer = this.defaultMQProducer.getDefaultMQProducerImpl(); + TopicPublishInfo topicInfo = innerProducer.getTopicPublishInfoTable().get(messages.get(0).getTopic()); + if (topicInfo == null) { + for (Message message : messages) { + sendMessage(message, partition); + } + } else { + // 批量发送 + List queues = topicInfo.getMessageQueueList(); + int size = queues.size(); + if (size <= 0) { + // 可能是第一次创建 + for (Message message : messages) { + sendMessage(message, partition); + } + } else { + MessageQueue queue; + if (partition > size) { + queue = queues.get(partition % size); + } else { + queue = queues.get(partition); + } + + try { + SendResult sendResult = this.defaultMQProducer.send(messages, queue); + if (logger.isDebugEnabled()) { + logger.debug("Send Message Result: {}", sendResult); + } + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + } + } + + @Override + public void stop() { + logger.info("## Stop RocketMQ producer##"); + this.defaultMQProducer.shutdown(); + super.stop(); + } +} diff --git a/connector/rocketmq-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMQProducer b/connector/rocketmq-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMQProducer new file mode 100644 index 00000000..9d6c0ddc --- /dev/null +++ b/connector/rocketmq-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMQProducer @@ -0,0 +1 @@ +rocketmq=com.alibaba.otter.canal.connector.rocketmq.producer.CanalRocketMQProducer \ No newline at end of file diff --git a/connector/rocketmq-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer b/connector/rocketmq-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer new file mode 100644 index 00000000..287544e7 --- /dev/null +++ b/connector/rocketmq-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer @@ -0,0 +1 @@ +rocketmq=com.alibaba.otter.canal.connector.rocketmq.consumer.CanalRocketMQConsumer \ No newline at end of file diff --git a/connector/tcp-connector/pom.xml b/connector/tcp-connector/pom.xml new file mode 100644 index 00000000..f75ca080 --- /dev/null +++ b/connector/tcp-connector/pom.xml @@ -0,0 +1,124 @@ + + + + canal.connector + com.alibaba.otter + 1.1.5-SNAPSHOT + ../pom.xml + + + 4.0.0 + com.alibaba.otter + connector.tcp + jar + canal connector tcp module for otter ${project.version} + + + + com.alibaba.otter + connector.core + ${project.version} + provided + + + com.alibaba.otter + canal.protocol + ${project.version} + provided + + + com.alibaba.otter + canal.client + ${project.version} + + + com.google.protobuf + protobuf-java + + + org.springframework + spring-orm + + + org.springframework + spring-jdbc + + + org.springframework + spring-aop + + + org.springframework + spring-core + + + org.springframework + spring-context + + + org.apache.rocketmq + rocketmq-client + + + com.aliyun.openservices + aliware-apache-rocketmq-cloud + + + + + junit + junit + test + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.4 + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + org.apache.maven.plugins + maven-antrun-plugin + 1.8 + + + package + + run + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/connector/tcp-connector/src/main/java/com/alibaba/otter/canal/connector/tcp/config/TCPConstants.java b/connector/tcp-connector/src/main/java/com/alibaba/otter/canal/connector/tcp/config/TCPConstants.java new file mode 100644 index 00000000..68db8d4f --- /dev/null +++ b/connector/tcp-connector/src/main/java/com/alibaba/otter/canal/connector/tcp/config/TCPConstants.java @@ -0,0 +1,12 @@ +package com.alibaba.otter.canal.connector.tcp.config; + +public class TCPConstants { + + public static final String ROOT = "canal"; + + public static final String CANAL_TCP_HOST = ROOT + "." + "tcp.server.host"; + public static final String CANAL_TCP_ZK_HOSTS = ROOT + "." + "tcp.zookeeper.hosts"; + public static final String CANAL_TCP_USERNAME = ROOT + "." + "tcp.username"; + public static final String CANAL_TCP_PASSWORD = ROOT + "." + "tcp.password"; + public static final String CANAL_TCP_BATCH_SIZE = ROOT + "." + "tcp.batch.size"; +} diff --git a/connector/tcp-connector/src/main/java/com/alibaba/otter/canal/connector/tcp/consumer/CanalTCPConsumer.java b/connector/tcp-connector/src/main/java/com/alibaba/otter/canal/connector/tcp/consumer/CanalTCPConsumer.java new file mode 100644 index 00000000..fea52ec5 --- /dev/null +++ b/connector/tcp-connector/src/main/java/com/alibaba/otter/canal/connector/tcp/consumer/CanalTCPConsumer.java @@ -0,0 +1,106 @@ +package com.alibaba.otter.canal.connector.tcp.consumer; + +import com.alibaba.otter.canal.client.CanalConnector; +import com.alibaba.otter.canal.client.impl.ClusterCanalConnector; +import com.alibaba.otter.canal.client.impl.ClusterNodeAccessStrategy; +import com.alibaba.otter.canal.client.impl.SimpleCanalConnector; +import com.alibaba.otter.canal.common.zookeeper.ZkClientx; +import com.alibaba.otter.canal.connector.core.consumer.CommonMessage; +import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer; +import com.alibaba.otter.canal.connector.core.spi.SPI; +import com.alibaba.otter.canal.connector.core.util.MessageUtil; +import com.alibaba.otter.canal.connector.tcp.config.TCPConstants; +import com.alibaba.otter.canal.protocol.Message; + +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +/** + * TCP 消费者连接器, 一个destination对应一个SPI实例 + * + * @author rewerma 2020-01-30 + * @version 1.0.0 + */ +@SPI("tcp") +public class CanalTCPConsumer implements CanalMsgConsumer { + + private static final String PREFIX_TCP_CONFIG = "canal.tcp."; + + private Long currentBatchId = null; + + private CanalConnector canalConnector; + + private int batchSize = 500; + + @Override + public void init(Properties properties, String destination, String groupId) { + // load config + String host = properties.getProperty(TCPConstants.CANAL_TCP_HOST); + String username = properties.getProperty(TCPConstants.CANAL_TCP_USERNAME); + String password = properties.getProperty(TCPConstants.CANAL_TCP_PASSWORD); + String zkHosts = properties.getProperty(TCPConstants.CANAL_TCP_ZK_HOSTS); + String batchSizePro = properties.getProperty(TCPConstants.CANAL_TCP_BATCH_SIZE); + if (batchSizePro != null) { + batchSize = Integer.parseInt(batchSizePro); + } + if (host != null) { + String[] ipPort = host.split(":"); + SocketAddress sa = new InetSocketAddress(ipPort[0], Integer.parseInt(ipPort[1])); + this.canalConnector = new SimpleCanalConnector(sa, username, password, destination); + } else { + this.canalConnector = new ClusterCanalConnector(username, + password, + destination, + new ClusterNodeAccessStrategy(destination, ZkClientx.getZkClient(zkHosts))); + } + } + + @Override + public void connect() { + canalConnector.connect(); + canalConnector.subscribe(); + } + + @Override + public List getMessage(Long timeout, TimeUnit unit) { + try { + Message message = canalConnector.getWithoutAck(batchSize, timeout, unit); + long batchId = message.getId(); + currentBatchId = batchId; + int size = message.getEntries().size(); + if (batchId == -1 || size == 0) { + return null; + } else { + return MessageUtil.convert(message); + } + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public void rollback() { + if (currentBatchId != null && currentBatchId != -1) { + canalConnector.rollback(currentBatchId); + currentBatchId = null; + } + } + + @Override + public void ack() { + if (currentBatchId != null) { + canalConnector.ack(currentBatchId); + currentBatchId = null; + } + } + + @Override + public void disconnect() { + canalConnector.unsubscribe(); + canalConnector.disconnect(); + } +} diff --git a/connector/tcp-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer b/connector/tcp-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer new file mode 100644 index 00000000..4d56c7f3 --- /dev/null +++ b/connector/tcp-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer @@ -0,0 +1 @@ +tcp=com.alibaba.otter.canal.connector.tcp.consumer.CanalTCPConsumer \ No newline at end of file diff --git a/deployer/pom.xml b/deployer/pom.xml index 6b3026ba..47976554 100644 --- a/deployer/pom.xml +++ b/deployer/pom.xml @@ -1,127 +1,185 @@ - - 4.0.0 - - com.alibaba.otter - canal - 1.1.5-SNAPSHOT - ../pom.xml - - com.alibaba.otter - canal.deployer - jar - canal deployer module for otter ${project.version} - - - com.alibaba.otter - canal.server - ${project.version} - + + 4.0.0 + + com.alibaba.otter + canal + 1.1.5-SNAPSHOT + ../pom.xml + + com.alibaba.otter + canal.deployer + jar + canal deployer module for otter ${project.version} + + + com.alibaba.otter + canal.server + ${project.version} + - - - com.alibaba.otter - canal.prometheus - ${project.version} - runtime - - - - - - - - maven-jar-plugin - - - true - - - **/logback.xml - **/canal.properties - **/spring/** - **/example/** - **/mq.yml - - - + + + com.alibaba.otter + canal.prometheus + ${project.version} + runtime + - - org.apache.maven.plugins - maven-assembly-plugin - - 2.2.1 - - - assemble - - single - - package - - - - false - false - - - - + + com.alibaba.otter + connector.kafka + ${project.version} + + + * + * + + + jar-with-dependencies + provided + + + com.alibaba.otter + connector.rocketmq + ${project.version} + + + * + * + + + jar-with-dependencies + provided + + + com.alibaba.otter + connector.rabbitmq + ${project.version} + + + * + * + + + jar-with-dependencies + provided + + - - - dev - - true - - env - !release - - + + + + maven-jar-plugin + + + true + + + **/logback.xml + **/canal.properties + **/spring/** + **/example/** + **/mq.yml + + + - - - - maven-assembly-plugin - - - - ${basedir}/src/main/assembly/dev.xml - - canal - ${project.build.directory} - - - - + + org.apache.maven.plugins + maven-assembly-plugin + + 2.2.1 + + + assemble + + single + + package + + + + false + false + + + + org.apache.maven.plugins + maven-dependency-plugin + 2.10 + + + copy-dependencies-to-canal-client-service + package + + copy-dependencies + + + jar-with-dependencies + ${project.basedir}/target/canal/plugin + + + + + + - + + + dev + + true + + env + !release + + - - release - - - env - release - - + + + + maven-assembly-plugin + + + + ${basedir}/src/main/assembly/dev.xml + + canal + ${project.build.directory} + + + + - - - - maven-assembly-plugin - - - - ${basedir}/src/main/assembly/release.xml - - - ${project.artifactId}-${project.version} - - ${project.parent.build.directory} - - - - - - + + + + release + + + env + release + + + + + + + maven-assembly-plugin + + + + ${basedir}/src/main/assembly/release.xml + + + ${project.artifactId}-${project.version} + + ${project.parent.build.directory} + + + + + + diff --git a/deployer/src/main/assembly/release.xml b/deployer/src/main/assembly/release.xml index aada5f5d..66236e55 100644 --- a/deployer/src/main/assembly/release.xml +++ b/deployer/src/main/assembly/release.xml @@ -42,6 +42,10 @@ **/* + + ${project.basedir}/target/canal/plugin + /plugin/ + diff --git a/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalConstants.java b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalConstants.java index 4769c20f..5e691d40 100644 --- a/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalConstants.java +++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalConstants.java @@ -46,36 +46,37 @@ public class CanalConstants { public static final String CANAL_SOCKETCHANNEL = ROOT + "." + "socketChannel"; - public static final String CANAL_MQ_SERVERS = ROOT + "." + "mq.servers"; - public static final String CANAL_MQ_RETRIES = ROOT + "." + "mq.retries"; - public static final String CANAL_MQ_BATCHSIZE = ROOT + "." + "mq.batchSize"; - public static final String CANAL_MQ_LINGERMS = ROOT + "." + "mq.lingerMs"; - public static final String CANAL_MQ_MAXREQUESTSIZE = ROOT + "." + "mq.maxRequestSize"; - public static final String CANAL_MQ_BUFFERMEMORY = ROOT + "." + "mq.bufferMemory"; - public static final String CANAL_MQ_CANALBATCHSIZE = ROOT + "." + "mq.canalBatchSize"; - public static final String CANAL_MQ_CANALGETTIMEOUT = ROOT + "." + "mq.canalGetTimeout"; - public static final String CANAL_MQ_FLATMESSAGE = ROOT + "." + "mq.flatMessage"; - public static final String CANAL_MQ_PARALLELTHREADSIZE = ROOT + "." + "mq.parallelThreadSize"; - public static final String CANAL_MQ_COMPRESSION_TYPE = ROOT + "." + "mq.compressionType"; - public static final String CANAL_MQ_ACKS = ROOT + "." + "mq.acks"; - public static final String CANAL_MQ_TRANSACTION = ROOT + "." + "mq.transaction"; - public static final String CANAL_MQ_PRODUCERGROUP = ROOT + "." + "mq.producerGroup"; public static final String CANAL_ALIYUN_ACCESSKEY = ROOT + "." + "aliyun.accessKey"; public static final String CANAL_ALIYUN_SECRETKEY = ROOT + "." + "aliyun.secretKey"; - public static final String CANAL_MQ_PROPERTIES = ROOT + "." + "mq.properties"; - public static final String CANAL_MQ_ENABLE_MESSAGE_TRACE = ROOT + "." + "mq.enableMessageTrace"; - public static final String CANAL_MQ_ACCESS_CHANNEL = ROOT + "." + "mq.accessChannel"; - public static final String CANAL_MQ_CUSTOMIZED_TRACE_TOPIC = ROOT + "." + "mq.customizedTraceTopic"; - public static final String CANAL_MQ_NAMESPACE = ROOT + "." + "mq.namespace"; - public static final String CANAL_MQ_KAFKA_KERBEROS_ENABLE = ROOT + "." + "mq.kafka.kerberos.enable"; - public static final String CANAL_MQ_KAFKA_KERBEROS_KRB5FILEPATH = ROOT + "." + "mq.kafka.kerberos.krb5FilePath"; - public static final String CANAL_MQ_KAFKA_KERBEROS_JAASFILEPATH = ROOT + "." + "mq.kafka.kerberos.jaasFilePath"; - public static final String CANAL_MQ_USERNAME = ROOT + "." + "mq.username"; - public static final String CANAL_MQ_PASSWORD = ROOT + "." + "mq.password"; - public static final String CANAL_MQ_VHOST = ROOT + "." + "mq.vhost"; - public static final String CANAL_MQ_ALIYUN_UID = ROOT + "." + "mq.aliyunuid"; - public static final String CANAL_MQ_EXCHANGE = ROOT + "." + "mq.exchange"; - public static final String CANAL_MQ_DATABASE_HASH = ROOT + "." + "mq.database.hash"; + +// public static final String CANAL_MQ_SERVERS = ROOT + "." + "mq.servers"; +// public static final String CANAL_MQ_RETRIES = ROOT + "." + "mq.retries"; +// public static final String CANAL_MQ_BATCHSIZE = ROOT + "." + "mq.batchSize"; +// public static final String CANAL_MQ_LINGERMS = ROOT + "." + "mq.lingerMs"; +// public static final String CANAL_MQ_MAXREQUESTSIZE = ROOT + "." + "mq.maxRequestSize"; +// public static final String CANAL_MQ_BUFFERMEMORY = ROOT + "." + "mq.bufferMemory"; +// public static final String CANAL_MQ_CANALBATCHSIZE = ROOT + "." + "mq.canalBatchSize"; +// public static final String CANAL_MQ_CANALGETTIMEOUT = ROOT + "." + "mq.canalGetTimeout"; +// public static final String CANAL_MQ_FLATMESSAGE = ROOT + "." + "mq.flatMessage"; +// public static final String CANAL_MQ_PARALLELTHREADSIZE = ROOT + "." + "mq.parallelThreadSize"; +// public static final String CANAL_MQ_COMPRESSION_TYPE = ROOT + "." + "mq.compressionType"; +// public static final String CANAL_MQ_ACKS = ROOT + "." + "mq.acks"; +// public static final String CANAL_MQ_TRANSACTION = ROOT + "." + "mq.transaction"; +// public static final String CANAL_MQ_PRODUCERGROUP = ROOT + "." + "mq.producerGroup"; +// public static final String CANAL_MQ_PROPERTIES = ROOT + "." + "mq.properties"; +// public static final String CANAL_MQ_ENABLE_MESSAGE_TRACE = ROOT + "." + "mq.enableMessageTrace"; +// public static final String CANAL_MQ_ACCESS_CHANNEL = ROOT + "." + "mq.accessChannel"; +// public static final String CANAL_MQ_CUSTOMIZED_TRACE_TOPIC = ROOT + "." + "mq.customizedTraceTopic"; +// public static final String CANAL_MQ_NAMESPACE = ROOT + "." + "mq.namespace"; +// public static final String CANAL_MQ_KAFKA_KERBEROS_ENABLE = ROOT + "." + "mq.kafka.kerberos.enable"; +// public static final String CANAL_MQ_KAFKA_KERBEROS_KRB5FILEPATH = ROOT + "." + "mq.kafka.kerberos.krb5FilePath"; +// public static final String CANAL_MQ_KAFKA_KERBEROS_JAASFILEPATH = ROOT + "." + "mq.kafka.kerberos.jaasFilePath"; +// public static final String CANAL_MQ_USERNAME = ROOT + "." + "mq.username"; +// public static final String CANAL_MQ_PASSWORD = ROOT + "." + "mq.password"; +// public static final String CANAL_MQ_VHOST = ROOT + "." + "mq.vhost"; +// public static final String CANAL_MQ_ALIYUN_UID = ROOT + "." + "mq.aliyunuid"; +// public static final String CANAL_MQ_EXCHANGE = ROOT + "." + "mq.exchange"; +// public static final String CANAL_MQ_DATABASE_HASH = ROOT + "." + "mq.database.hash"; public static String getInstanceModeKey(String destination) { return MessageFormat.format(INSTANCE_MODE_TEMPLATE, destination); diff --git a/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalStarter.java b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalStarter.java index 64fd191a..e95dd554 100644 --- a/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalStarter.java +++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalStarter.java @@ -2,35 +2,36 @@ package com.alibaba.otter.canal.deployer; import java.util.Properties; +import com.alibaba.otter.canal.connector.core.config.MQProperties; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.otter.canal.admin.netty.CanalAdminWithNetty; -import com.alibaba.otter.canal.common.MQProperties; +import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer; +import com.alibaba.otter.canal.connector.core.spi.ExtensionLoader; import com.alibaba.otter.canal.deployer.admin.CanalAdminController; -import com.alibaba.otter.canal.kafka.CanalKafkaProducer; -import com.alibaba.otter.canal.rabbitmq.CanalRabbitMQProducer; -import com.alibaba.otter.canal.rocketmq.CanalRocketMQProducer; import com.alibaba.otter.canal.server.CanalMQStarter; -import com.alibaba.otter.canal.spi.CanalMQProducer; /** * Canal server 启动类 * - * @author rewerma 2018-12-30 下午05:12:16 - * @version 1.0.1 + * @author rewerma 2020-01-27 + * @version 1.0.2 */ public class CanalStarter { - private static final Logger logger = LoggerFactory.getLogger(CanalStarter.class); + private static final Logger logger = LoggerFactory.getLogger(CanalStarter.class); - private CanalController controller = null; - private CanalMQProducer canalMQProducer = null; - private Thread shutdownThread = null; - private CanalMQStarter canalMQStarter = null; + private static final String CONNECTOR_SPI_DIR = "/plugin"; + private static final String CONNECTOR_STANDBY_SPI_DIR = "/canal/plugin"; + + private CanalController controller = null; + private CanalMQProducer canalMQProducer = null; + private Thread shutdownThread = null; + private CanalMQStarter canalMQStarter = null; private volatile Properties properties; - private volatile boolean running = false; + private volatile boolean running = false; private CanalAdminWithNetty canalAdmin; @@ -61,20 +62,23 @@ public class CanalStarter { */ public synchronized void start() throws Throwable { String serverMode = CanalController.getProperty(properties, CanalConstants.CANAL_SERVER_MODE); - if (serverMode.equalsIgnoreCase("kafka")) { - canalMQProducer = new CanalKafkaProducer(); - } else if (serverMode.equalsIgnoreCase("rocketmq")) { - canalMQProducer = new CanalRocketMQProducer(); - } else if (serverMode.equalsIgnoreCase("rabbitmq")) { - canalMQProducer = new CanalRabbitMQProducer(); + if (!"tcp".equalsIgnoreCase(serverMode)) { + ExtensionLoader loader = ExtensionLoader.getExtensionLoader(CanalMQProducer.class); + canalMQProducer = loader + .getExtension(serverMode.toLowerCase(), CONNECTOR_SPI_DIR, CONNECTOR_STANDBY_SPI_DIR); + if (canalMQProducer != null) { + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + Thread.currentThread().setContextClassLoader(canalMQProducer.getClass().getClassLoader()); + canalMQProducer.init(properties); + Thread.currentThread().setContextClassLoader(cl); + } } - MQProperties mqProperties = null; if (canalMQProducer != null) { - mqProperties = buildMQProperties(properties); + MQProperties mqProperties = canalMQProducer.getMqProperties(); // disable netty System.setProperty(CanalConstants.CANAL_WITHOUT_NETTY, "true"); - if (mqProperties.getFlatMessage()) { + if (mqProperties.isFlatMessage()) { // 设置为raw避免ByteString->Entry的二次解析 System.setProperty("canal.instance.memory.rawEntry", "false"); } @@ -104,7 +108,7 @@ public class CanalStarter { if (canalMQProducer != null) { canalMQStarter = new CanalMQStarter(canalMQProducer); String destinations = CanalController.getProperty(properties, CanalConstants.CANAL_DESTINATIONS); - canalMQStarter.start(mqProperties, destinations); + canalMQStarter.start(destinations); controller.setCanalMQStarter(canalMQStarter); } @@ -119,11 +123,15 @@ public class CanalStarter { String ip = CanalController.getProperty(properties, CanalConstants.CANAL_IP); - logger.debug("canal admin port:{}, canal admin user:{}, canal admin password: {}, canal ip:{}", port, user, passwd, ip); + logger.debug("canal admin port:{}, canal admin user:{}, canal admin password: {}, canal ip:{}", + port, + user, + passwd, + ip); CanalAdminWithNetty canalAdminWithNetty = CanalAdminWithNetty.instance(); canalAdminWithNetty.setCanalAdmin(canalAdmin); - canalAdminWithNetty.setPort(Integer.valueOf(port)); + canalAdminWithNetty.setPort(Integer.parseInt(port)); canalAdminWithNetty.setIp(ip); canalAdminWithNetty.start(); this.canalAdmin = canalAdminWithNetty; @@ -162,156 +170,4 @@ public class CanalStarter { } running = false; } - - /** - * 构造MQ对应的配置 - * - * @param properties canal.properties 配置 - * @return - */ - private static MQProperties buildMQProperties(Properties properties) { - MQProperties mqProperties = new MQProperties(); - String servers = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_SERVERS); - if (!StringUtils.isEmpty(servers)) { - mqProperties.setServers(servers); - } - String retires = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_RETRIES); - if (!StringUtils.isEmpty(retires)) { - mqProperties.setRetries(Integer.valueOf(retires)); - } - String batchSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_BATCHSIZE); - if (!StringUtils.isEmpty(batchSize)) { - mqProperties.setBatchSize(Integer.valueOf(batchSize)); - } - String lingerMs = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_LINGERMS); - if (!StringUtils.isEmpty(lingerMs)) { - mqProperties.setLingerMs(Integer.valueOf(lingerMs)); - } - String maxRequestSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_MAXREQUESTSIZE); - if (!StringUtils.isEmpty(maxRequestSize)) { - mqProperties.setMaxRequestSize(Integer.valueOf(maxRequestSize)); - } - String bufferMemory = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_BUFFERMEMORY); - if (!StringUtils.isEmpty(bufferMemory)) { - mqProperties.setBufferMemory(Long.valueOf(bufferMemory)); - } - String canalBatchSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_CANALBATCHSIZE); - if (!StringUtils.isEmpty(canalBatchSize)) { - mqProperties.setCanalBatchSize(Integer.valueOf(canalBatchSize)); - } - String canalGetTimeout = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_CANALGETTIMEOUT); - if (!StringUtils.isEmpty(canalGetTimeout)) { - mqProperties.setCanalGetTimeout(Long.valueOf(canalGetTimeout)); - } - String flatMessage = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_FLATMESSAGE); - if (!StringUtils.isEmpty(flatMessage)) { - mqProperties.setFlatMessage(Boolean.valueOf(flatMessage)); - } - String parallelThreadSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_PARALLELTHREADSIZE); - if (!StringUtils.isEmpty(parallelThreadSize)) { - mqProperties.setParallelThreadSize(Integer.valueOf(parallelThreadSize)); - } - String compressionType = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_COMPRESSION_TYPE); - if (!StringUtils.isEmpty(compressionType)) { - mqProperties.setCompressionType(compressionType); - } - String acks = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_ACKS); - if (!StringUtils.isEmpty(acks)) { - mqProperties.setAcks(acks); - } - String aliyunAccessKey = CanalController.getProperty(properties, CanalConstants.CANAL_ALIYUN_ACCESSKEY); - if (!StringUtils.isEmpty(aliyunAccessKey)) { - mqProperties.setAliyunAccessKey(aliyunAccessKey); - } - String aliyunSecretKey = CanalController.getProperty(properties, CanalConstants.CANAL_ALIYUN_SECRETKEY); - if (!StringUtils.isEmpty(aliyunSecretKey)) { - mqProperties.setAliyunSecretKey(aliyunSecretKey); - } - - String producerGroup = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_PRODUCERGROUP); - if (!StringUtils.isEmpty(producerGroup)) { - mqProperties.setProducerGroup(producerGroup); - } - - String enableMessageTrace = CanalController.getProperty(properties, - CanalConstants.CANAL_MQ_ENABLE_MESSAGE_TRACE); - if (!StringUtils.isEmpty(enableMessageTrace)) { - mqProperties.setEnableMessageTrace(Boolean.valueOf(enableMessageTrace)); - } - - String accessChannel = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_ACCESS_CHANNEL); - if (!StringUtils.isEmpty(accessChannel)) { - mqProperties.setAccessChannel(accessChannel); - } - - String customizedTraceTopic = CanalController.getProperty(properties, - CanalConstants.CANAL_MQ_CUSTOMIZED_TRACE_TOPIC); - if (!StringUtils.isEmpty(customizedTraceTopic)) { - mqProperties.setCustomizedTraceTopic(customizedTraceTopic); - } - - String namespace = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_NAMESPACE); - if (!StringUtils.isEmpty(namespace)) { - mqProperties.setNamespace(namespace); - } - - String kafkaKerberosEnable = CanalController.getProperty(properties, - CanalConstants.CANAL_MQ_KAFKA_KERBEROS_ENABLE); - if (!StringUtils.isEmpty(kafkaKerberosEnable)) { - mqProperties.setKerberosEnable(Boolean.valueOf(kafkaKerberosEnable)); - } - - String kafkaKerberosKrb5Filepath = CanalController.getProperty(properties, - CanalConstants.CANAL_MQ_KAFKA_KERBEROS_KRB5FILEPATH); - if (!StringUtils.isEmpty(kafkaKerberosKrb5Filepath)) { - mqProperties.setKerberosKrb5FilePath(kafkaKerberosKrb5Filepath); - } - - String kafkaKerberosJaasFilepath = CanalController.getProperty(properties, - CanalConstants.CANAL_MQ_KAFKA_KERBEROS_JAASFILEPATH); - if (!StringUtils.isEmpty(kafkaKerberosJaasFilepath)) { - mqProperties.setKerberosJaasFilePath(kafkaKerberosJaasFilepath); - } - - String vhost = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_VHOST); - if (!StringUtils.isEmpty(vhost)) { - mqProperties.setVhost(vhost); - } - - String username = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_USERNAME); - if (!StringUtils.isEmpty(username)) { - mqProperties.setUsername(username); - } - - String password = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_PASSWORD); - if (!StringUtils.isEmpty(password)) { - mqProperties.setPassword(password); - } - - String aliyunUID = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_ALIYUN_UID); - if (!StringUtils.isEmpty(aliyunUID)) { - mqProperties.setAliyunUID(Long.valueOf(aliyunUID)); - } - - String exchange = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_EXCHANGE); - if (!StringUtils.isEmpty(exchange)) { - mqProperties.setExchange(exchange); - } - - String databaseHash = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_DATABASE_HASH); - if (!StringUtils.isEmpty(databaseHash)){ - mqProperties.setDatabaseHash(Boolean.valueOf(databaseHash)); - } - - for (Object key : properties.keySet()) { - key = StringUtils.trim(key.toString()); - if (((String) key).startsWith(CanalConstants.CANAL_MQ_PROPERTIES)) { - String value = CanalController.getProperty(properties, (String) key); - String subKey = ((String) key).substring(CanalConstants.CANAL_MQ_PROPERTIES.length() + 1); - mqProperties.getProperties().put(subKey, value); - } - } - - return mqProperties; - } } diff --git a/deployer/src/main/resources/canal.properties b/deployer/src/main/resources/canal.properties index 424af91d..234ffb00 100644 --- a/deployer/src/main/resources/canal.properties +++ b/deployer/src/main/resources/canal.properties @@ -21,7 +21,7 @@ canal.zkServers = # flush data to zk canal.zookeeper.flush.period = 1000 canal.withoutNetty = false -# tcp, kafka, RocketMQ +# tcp, kafka, rocketMQ, rabbitMQ canal.serverMode = tcp # flush meta cursor/parse position to file canal.file.data.dir = ${canal.conf.dir} @@ -86,10 +86,6 @@ canal.instance.tsdb.snapshot.interval = 24 # purge snapshot expire , default 360 hour(15 days) canal.instance.tsdb.snapshot.expire = 360 -# aliyun ak/sk , support rds/mq -canal.aliyun.accessKey = -canal.aliyun.secretKey = - ################################################# ######### destinations ############# ################################################# @@ -111,35 +107,54 @@ canal.instance.global.spring.xml = classpath:spring/file-instance.xml #canal.instance.global.spring.xml = classpath:spring/default-instance.xml ################################################## -######### MQ ############# +######### MQ Properties ############# ################################################## -canal.mq.servers = 127.0.0.1:6667 -canal.mq.retries = 0 -canal.mq.batchSize = 16384 -canal.mq.maxRequestSize = 1048576 -canal.mq.lingerMs = 100 -canal.mq.bufferMemory = 33554432 -canal.mq.canalBatchSize = 50 -canal.mq.canalGetTimeout = 100 -canal.mq.parallelThreadSize = 8 -canal.mq.flatMessage = true -canal.mq.compressionType = none -canal.mq.acks = all -#canal.mq.properties. = -canal.mq.producerGroup = test -# Set this value to "cloud", if you want open message trace feature in aliyun. -canal.mq.accessChannel = local -# aliyun mq namespace -#canal.mq.namespace = -canal.mq.vhost= -canal.mq.exchange= -canal.mq.username= -canal.mq.password= -canal.mq.aliyunuid= +canal.mq.flat.message = true canal.mq.database.hash = true +canal.mq.parallel.thread.size = 8 +canal.mq.canal.batch.size = 50 +canal.mq.canal.fetch.timeout = 100 +# Set this value to "cloud", if you want open message trace feature in aliyun. +canal.mq.access.channel = local + +# aliyun ak/sk , support rds/mq +canal.aliyun.accessKey = +canal.aliyun.secretKey = +canal.aliyun.uid= + ################################################## -######### Kafka Kerberos Info ############# +######### Kafka ############# ################################################## +kafka.bootstrap.servers = 127.0.0.1:9092 +kafka.acks = all +kafka.compression.type = none +kafka.batch.size = 16384 +kafka.linger.ms = 1 +kafka.max.request.size = 1048576 +kafka.buffer.memory = 33554432 +kafka.max.in.flight.requests.per.connection = 1 +kafka.retries = 0 + canal.mq.kafka.kerberos.enable = false -canal.mq.kafka.kerberos.krb5FilePath = "../conf/kerberos/krb5.conf" -canal.mq.kafka.kerberos.jaasFilePath = "../conf/kerberos/jaas.conf" \ No newline at end of file +canal.mq.kafka.kerberos.krb5.file = "../conf/kerberos/krb5.conf" +canal.mq.kafka.kerberos.jaas.file = "../conf/kerberos/jaas.conf" + +################################################## +######### RocketMQ ############# +################################################## +rocketmq.producer.group = test +rocketmq.enable.message.trace = false +rocketmq.customized.trace.topic = +rocketmq.namespace = +rocketmq.namesrv.addr = 127.0.0.1:9876 +rocketmq.retry.times.when.send.failed = 0 +rocketmq.vip.channel.enabled = false + +################################################## +######### RabbitMQ ############# +################################################## +rabbitmq.host = +rabbitmq.virtual.host = +rabbitmq.exchange = +rabbitmq.username = +rabbitmq.password = \ No newline at end of file diff --git a/deployer/src/main/resources/logback.xml b/deployer/src/main/resources/logback.xml index a3f66c60..4e80278c 100644 --- a/deployer/src/main/resources/logback.xml +++ b/deployer/src/main/resources/logback.xml @@ -73,11 +73,15 @@ - + - + + + + + diff --git a/pom.xml b/pom.xml index 6ddd9fae..c8983260 100644 --- a/pom.xml +++ b/pom.xml @@ -100,8 +100,6 @@ UTF-8 true 3.2.18.RELEASE - 4.5.2 - 5.5.0 0.8.3 2.22.1 -server -Xms512m -Xmx1024m -Dfile.encoding=UTF-8 @@ -129,6 +127,7 @@ prometheus admin client-adapter + connector @@ -233,6 +232,11 @@ aviator 2.2.1 + + com.alibaba + fastjson + 1.2.58.sec06 + oro oro @@ -336,26 +340,6 @@ jsr305 3.0.2 - - org.apache.rocketmq - rocketmq-client - ${rocketmq_version} - - - org.apache.rocketmq - rocketmq-acl - ${rocketmq_version} - - - com.rabbitmq - amqp-client - ${rabbitmq_version} - - - com.alibaba.mq-amqp - mq-amqp-client - 1.0.3 - javax.annotation javax.annotation-api diff --git a/server/pom.xml b/server/pom.xml index d9926645..06876c4c 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -25,39 +25,15 @@ canal.instance.manager ${project.version} - - org.apache.kafka - kafka_2.11 - 1.1.1 - - - org.slf4j - slf4j-log4j12 - - + com.alibaba.otter + connector.core + ${project.version} - org.apache.rocketmq - rocketmq-client - - - org.apache.rocketmq - rocketmq-acl - - - com.rabbitmq - amqp-client - - - com.alibaba.mq-amqp - mq-amqp-client - - - - org.jboss.netty - netty - 3.2.2.Final + org.apache.commons + commons-lang3 + 3.7 diff --git a/server/src/main/java/com/alibaba/otter/canal/common/AbstractMQProducer.java b/server/src/main/java/com/alibaba/otter/canal/common/AbstractMQProducer.java deleted file mode 100644 index 4d4aeffc..00000000 --- a/server/src/main/java/com/alibaba/otter/canal/common/AbstractMQProducer.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.alibaba.otter.canal.common; - -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -import com.alibaba.otter.canal.common.utils.NamedThreadFactory; -import com.alibaba.otter.canal.spi.CanalMQProducer; - -/** - * @author agapple 2019年9月29日 上午11:17:11 - * @since 1.1.5 - */ -public abstract class AbstractMQProducer implements CanalMQProducer { - - protected ThreadPoolExecutor executor; - - @Override - public void init(MQProperties mqProperties) { - int parallelThreadSize = mqProperties.getParallelThreadSize(); - executor = new ThreadPoolExecutor(parallelThreadSize, - parallelThreadSize, - 0, - TimeUnit.SECONDS, - new ArrayBlockingQueue(parallelThreadSize * 2), - new NamedThreadFactory("MQParallel"), - new ThreadPoolExecutor.CallerRunsPolicy()); - - } - - @Override - public void stop() { - executor.shutdownNow(); - executor = null; - } - -} diff --git a/server/src/main/java/com/alibaba/otter/canal/common/MQProperties.java b/server/src/main/java/com/alibaba/otter/canal/common/MQProperties.java deleted file mode 100644 index d4a809e1..00000000 --- a/server/src/main/java/com/alibaba/otter/canal/common/MQProperties.java +++ /dev/null @@ -1,368 +0,0 @@ -package com.alibaba.otter.canal.common; - -import java.util.Properties; - -/** - * MQ 配置项 - * - * @author machengyuan 2018-6-11 下午05:30:49 - * @version 1.0.0 - */ -public class MQProperties { - - private String servers = "127.0.0.1:6667"; - private int retries = 0; - private int batchSize = 16384; - private int lingerMs = 1; - private int maxRequestSize = 1048576; - private long bufferMemory = 33554432L; - private boolean filterTransactionEntry = true; - private String producerGroup = "Canal-Producer"; - private int canalBatchSize = 50; - private Long canalGetTimeout = 100L; - private boolean flatMessage = true; - private String compressionType = "none"; - private String acks = "all"; - private String aliyunAccessKey = ""; - private String aliyunSecretKey = ""; - private Properties properties = new Properties(); - private boolean enableMessageTrace = false; - private String accessChannel = null; - private String customizedTraceTopic = null; - private String namespace = ""; - // kafka集群是否启动Kerberos认证 - private boolean kerberosEnable = false; - // 启动Kerberos认证时配置为krb5.conf文件的路径 - private String kerberosKrb5FilePath = ""; - // 启动Kerberos认证时配置为jaas.conf文件的路径 - private String kerberosJaasFilePath = ""; - // rabbitmq 账号 - private String username = ""; - // rabbitmq 密码 - private String password = ""; - // rabbitmq 密码 - private String vhost = ""; - // aliyun 用户ID rabbitmq 阿里云需要使用 - private long aliyunUID = 0; - // rabbitmq 交换机 - private String exchange = ""; - // 消息发送的并行度 - private int parallelThreadSize = 8; - // 是否取消根据database进行hash - private boolean databaseHash = true; - - public static class CanalDestination { - - private String canalDestination; - private String topic; - private Integer partition; - private Integer partitionsNum; - private String partitionHash; - private String dynamicTopic; - - public String getCanalDestination() { - return canalDestination; - } - - public void setCanalDestination(String canalDestination) { - this.canalDestination = canalDestination; - } - - public String getTopic() { - return topic; - } - - public void setTopic(String topic) { - this.topic = topic; - } - - public Integer getPartition() { - return partition; - } - - public void setPartition(Integer partition) { - this.partition = partition; - } - - public Integer getPartitionsNum() { - return partitionsNum; - } - - public void setPartitionsNum(Integer partitionsNum) { - this.partitionsNum = partitionsNum; - } - - public String getPartitionHash() { - return partitionHash; - } - - public void setPartitionHash(String partitionHash) { - this.partitionHash = partitionHash; - } - - public String getDynamicTopic() { - return dynamicTopic; - } - - public void setDynamicTopic(String dynamicTopic) { - this.dynamicTopic = dynamicTopic; - } - } - - public String getServers() { - return servers; - } - - public void setServers(String servers) { - this.servers = servers; - } - - public int getRetries() { - return retries; - } - - public void setRetries(int retries) { - this.retries = retries; - } - - public int getBatchSize() { - return batchSize; - } - - public void setBatchSize(int batchSize) { - this.batchSize = batchSize; - } - - public int getLingerMs() { - return lingerMs; - } - - public void setLingerMs(int lingerMs) { - this.lingerMs = lingerMs; - } - - public long getBufferMemory() { - return bufferMemory; - } - - public void setBufferMemory(long bufferMemory) { - this.bufferMemory = bufferMemory; - } - - public int getCanalBatchSize() { - return canalBatchSize; - } - - public void setCanalBatchSize(int canalBatchSize) { - this.canalBatchSize = canalBatchSize; - } - - public Long getCanalGetTimeout() { - return canalGetTimeout; - } - - public void setCanalGetTimeout(Long canalGetTimeout) { - this.canalGetTimeout = canalGetTimeout; - } - - public boolean getFlatMessage() { - return flatMessage; - } - - public void setFlatMessage(boolean flatMessage) { - this.flatMessage = flatMessage; - } - - public boolean isFilterTransactionEntry() { - return filterTransactionEntry; - } - - public void setFilterTransactionEntry(boolean filterTransactionEntry) { - this.filterTransactionEntry = filterTransactionEntry; - } - - public String getProducerGroup() { - return producerGroup; - } - - public void setProducerGroup(String producerGroup) { - this.producerGroup = producerGroup; - } - - public String getCompressionType() { - return compressionType; - } - - public void setCompressionType(String compressionType) { - this.compressionType = compressionType; - } - - public String getAcks() { - return acks; - } - - public void setAcks(String acks) { - this.acks = acks; - } - - public String getAliyunAccessKey() { - return aliyunAccessKey; - } - - public void setAliyunAccessKey(String aliyunAccessKey) { - this.aliyunAccessKey = aliyunAccessKey; - } - - public String getAliyunSecretKey() { - return aliyunSecretKey; - } - - public void setAliyunSecretKey(String aliyunSecretKey) { - this.aliyunSecretKey = aliyunSecretKey; - } - - public int getMaxRequestSize() { - return maxRequestSize; - } - - public void setMaxRequestSize(int maxRequestSize) { - this.maxRequestSize = maxRequestSize; - } - - public Properties getProperties() { - return properties; - } - - public void setProperties(Properties properties) { - this.properties = properties; - } - - public boolean isEnableMessageTrace() { - return enableMessageTrace; - } - - public void setEnableMessageTrace(boolean enableMessageTrace) { - this.enableMessageTrace = enableMessageTrace; - } - - public String getAccessChannel() { - return accessChannel; - } - - public void setAccessChannel(String accessChannel) { - this.accessChannel = accessChannel; - } - - public String getCustomizedTraceTopic() { - return customizedTraceTopic; - } - - public void setCustomizedTraceTopic(String customizedTraceTopic) { - this.customizedTraceTopic = customizedTraceTopic; - } - - public String getNamespace() { - return namespace; - } - - public void setNamespace(String namespace) { - this.namespace = namespace; - } - - public boolean isKerberosEnable() { - return kerberosEnable; - } - - public void setKerberosEnable(boolean kerberosEnable) { - this.kerberosEnable = kerberosEnable; - } - - public String getKerberosKrb5FilePath() { - return kerberosKrb5FilePath; - } - - public void setKerberosKrb5FilePath(String kerberosKrb5FilePath) { - this.kerberosKrb5FilePath = kerberosKrb5FilePath; - } - - public String getKerberosJaasFilePath() { - return kerberosJaasFilePath; - } - - public void setKerberosJaasFilePath(String kerberosJaasFilePath) { - this.kerberosJaasFilePath = kerberosJaasFilePath; - } - - public void setPassword(String password) { - this.password = password; - } - - public String getPassword() { - return password; - } - - public void setUsername(String username) { - this.username = username; - } - - public String getUsername() { - return username; - } - - public void setVhost(String vhost) { - this.vhost = vhost; - } - - public String getVhost() { - return vhost; - } - - public long getAliyunUID() { - return aliyunUID; - } - - public void setAliyunUID(long aliyunUID) { - this.aliyunUID = aliyunUID; - } - - public String getExchange() { - return exchange; - } - - public void setExchange(String exchange) { - this.exchange = exchange; - } - - public int getParallelThreadSize() { - return parallelThreadSize; - } - - public void setParallelThreadSize(int parallelThreadSize) { - this.parallelThreadSize = parallelThreadSize; - } - - public boolean getDatabaseHash() { - return databaseHash; - } - - public void setDatabaseHash(boolean databaseHash) { - this.databaseHash = databaseHash; - } - - @Override - public String toString() { - return "MQProperties [servers=" + servers + ", retries=" + retries + ", batchSize=" + batchSize + ", lingerMs=" - + lingerMs + ", maxRequestSize=" + maxRequestSize + ", bufferMemory=" + bufferMemory - + ", filterTransactionEntry=" + filterTransactionEntry + ", producerGroup=" + producerGroup - + ", canalBatchSize=" + canalBatchSize + ", canalGetTimeout=" + canalGetTimeout + ", flatMessage=" - + flatMessage + ", compressionType=" + compressionType + ", acks=" + acks + ", aliyunAccessKey=" - + aliyunAccessKey + ", aliyunSecretKey=" + aliyunSecretKey + ", properties=" + properties - + ", enableMessageTrace=" + enableMessageTrace + ", accessChannel=" + accessChannel - + ", customizedTraceTopic=" + customizedTraceTopic + ", namespace=" + namespace + ", kerberosEnable=" - + kerberosEnable + ", kerberosKrb5FilePath=" + kerberosKrb5FilePath + ", kerberosJaasFilePath=" - + kerberosJaasFilePath + ", username=" + username + ", password=" + password + ", vhost=" + vhost - + ", aliyunUID=" + aliyunUID + ", exchange=" + exchange + ", parallelThreadSize=" + parallelThreadSize - + ",databaseHash=" + databaseHash + "]"; - } - -} diff --git a/server/src/main/java/com/alibaba/otter/canal/rabbitmq/CanalRabbitMQProducer.java b/server/src/main/java/com/alibaba/otter/canal/rabbitmq/CanalRabbitMQProducer.java deleted file mode 100644 index 8cf82caa..00000000 --- a/server/src/main/java/com/alibaba/otter/canal/rabbitmq/CanalRabbitMQProducer.java +++ /dev/null @@ -1,145 +0,0 @@ -package com.alibaba.otter.canal.rabbitmq; - -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeoutException; - -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.serializer.SerializerFeature; -import com.alibaba.otter.canal.common.AbstractMQProducer; -import com.alibaba.otter.canal.common.CanalMessageSerializer; -import com.alibaba.otter.canal.common.MQMessageUtils; -import com.alibaba.otter.canal.common.MQMessageUtils.EntryRowData; -import com.alibaba.otter.canal.common.MQProperties; -import com.alibaba.otter.canal.common.utils.ExecutorTemplate; -import com.alibaba.otter.canal.protocol.FlatMessage; -import com.alibaba.otter.canal.protocol.Message; -import com.alibaba.otter.canal.server.exception.CanalServerException; -import com.alibaba.otter.canal.spi.CanalMQProducer; -import com.rabbitmq.client.Channel; -import com.rabbitmq.client.Connection; -import com.rabbitmq.client.ConnectionFactory; - -public class CanalRabbitMQProducer extends AbstractMQProducer implements CanalMQProducer { - - private static final Logger logger = LoggerFactory.getLogger(CanalRabbitMQProducer.class); - private MQProperties mqProperties; - private Connection connect; - private Channel channel; - - @Override - public void init(MQProperties mqProperties) { - super.init(mqProperties); - this.mqProperties = mqProperties; - ConnectionFactory factory = new ConnectionFactory(); - factory.setHost(mqProperties.getServers()); - if (mqProperties.getAliyunAccessKey().length() > 0 && mqProperties.getAliyunSecretKey().length() > 0 - && mqProperties.getAliyunUID() > 0) { - factory.setCredentialsProvider(new AliyunCredentialsProvider(mqProperties.getAliyunAccessKey(), - mqProperties.getAliyunSecretKey(), - mqProperties.getAliyunUID())); - } else { - factory.setUsername(mqProperties.getUsername()); - factory.setPassword(mqProperties.getPassword()); - } - factory.setVirtualHost(mqProperties.getVhost()); - try { - connect = factory.newConnection(); - channel = connect.createChannel(); - // channel.exchangeDeclare(mqProperties.getExchange(), "topic"); - } catch (IOException | TimeoutException ex) { - throw new CanalServerException("Start RabbitMQ producer error", ex); - } - } - - @Override - public void send(final MQProperties.CanalDestination canalDestination, Message message, Callback callback) - throws IOException { - ExecutorTemplate template = new ExecutorTemplate(executor); - try { - if (!StringUtils.isEmpty(canalDestination.getDynamicTopic())) { - // 动态topic - Map messageMap = MQMessageUtils.messageTopics(message, - canalDestination.getTopic(), - canalDestination.getDynamicTopic()); - - for (Map.Entry entry : messageMap.entrySet()) { - final String topicName = entry.getKey().replace('.', '_'); - final com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue(); - - template.submit(new Runnable() { - - @Override - public void run() { - send(canalDestination, topicName, messageSub); - } - }); - } - - template.waitForResult(); - } else { - send(canalDestination, canalDestination.getTopic(), message); - } - callback.commit(); - } catch (Throwable e) { - logger.error(e.getMessage(), e); - callback.rollback(); - } finally { - template.clear(); - } - } - - private void send(MQProperties.CanalDestination canalDestination, String topicName, Message messageSub) { - if (!mqProperties.getFlatMessage()) { - byte[] message = CanalMessageSerializer.serializer(messageSub, mqProperties.isFilterTransactionEntry()); - if (logger.isDebugEnabled()) { - logger.debug("send message:{} to destination:{}", message, canalDestination.getCanalDestination()); - } - sendMessage(topicName, message); - } else { - // 并发构造 - EntryRowData[] datas = MQMessageUtils.buildMessageData(messageSub, executor); - // 串行分区 - List flatMessages = MQMessageUtils.messageConverter(datas, messageSub.getId()); - if (flatMessages != null) { - for (FlatMessage flatMessage : flatMessages) { - byte[] message = JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue); - if (logger.isDebugEnabled()) { - logger.debug("send message:{} to destination:{}", - message, - canalDestination.getCanalDestination()); - } - sendMessage(topicName, message); - } - } - } - - } - - private void sendMessage(String queueName, byte[] message) { - // tips: 目前逻辑中暂不处理对exchange处理，请在Console后台绑定 才可使用routekey - try { - channel.basicPublish(mqProperties.getExchange(), queueName, null, message); - } catch (Throwable e) { - throw new RuntimeException(e); - } - } - - @Override - public void stop() { - logger.info("## Stop RabbitMQ producer##"); - try { - this.connect.close(); - this.channel.close(); - } catch (IOException | TimeoutException ex) { - throw new CanalServerException("Stop RabbitMQ producer error", ex); - } - - super.stop(); - } -} diff --git a/server/src/main/java/com/alibaba/otter/canal/rocketmq/CanalRocketMQProducer.java b/server/src/main/java/com/alibaba/otter/canal/rocketmq/CanalRocketMQProducer.java deleted file mode 100644 index 88a22eaa..00000000 --- a/server/src/main/java/com/alibaba/otter/canal/rocketmq/CanalRocketMQProducer.java +++ /dev/null @@ -1,285 +0,0 @@ -package com.alibaba.otter.canal.rocketmq; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import org.apache.commons.lang.StringUtils; -import org.apache.rocketmq.acl.common.AclClientRPCHook; -import org.apache.rocketmq.acl.common.SessionCredentials; -import org.apache.rocketmq.client.AccessChannel; -import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; -import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; -import org.apache.rocketmq.client.producer.DefaultMQProducer; -import org.apache.rocketmq.client.producer.MessageQueueSelector; -import org.apache.rocketmq.client.producer.SendResult; -import org.apache.rocketmq.common.message.Message; -import org.apache.rocketmq.common.message.MessageQueue; -import org.apache.rocketmq.remoting.RPCHook; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.serializer.SerializerFeature; -import com.alibaba.otter.canal.common.AbstractMQProducer; -import com.alibaba.otter.canal.common.CanalMessageSerializer; -import com.alibaba.otter.canal.common.MQMessageUtils; -import com.alibaba.otter.canal.common.MQMessageUtils.EntryRowData; -import com.alibaba.otter.canal.common.MQProperties; -import com.alibaba.otter.canal.common.utils.ExecutorTemplate; -import com.alibaba.otter.canal.protocol.FlatMessage; -import com.alibaba.otter.canal.server.exception.CanalServerException; -import com.alibaba.otter.canal.spi.CanalMQProducer; - -public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQProducer { - - private static final Logger logger = LoggerFactory.getLogger(CanalRocketMQProducer.class); - private DefaultMQProducer defaultMQProducer; - private MQProperties mqProperties; - private static final String CLOUD_ACCESS_CHANNEL = "cloud"; - - @Override - public void init(MQProperties rocketMQProperties) { - super.init(rocketMQProperties); - this.mqProperties = rocketMQProperties; - RPCHook rpcHook = null; - if (rocketMQProperties.getAliyunAccessKey().length() > 0 - && rocketMQProperties.getAliyunSecretKey().length() > 0) { - SessionCredentials sessionCredentials = new SessionCredentials(); - sessionCredentials.setAccessKey(rocketMQProperties.getAliyunAccessKey()); - sessionCredentials.setSecretKey(rocketMQProperties.getAliyunSecretKey()); - rpcHook = new AclClientRPCHook(sessionCredentials); - } - - defaultMQProducer = new DefaultMQProducer(rocketMQProperties.getProducerGroup(), - rpcHook, - mqProperties.isEnableMessageTrace(), - mqProperties.getCustomizedTraceTopic()); - if (CLOUD_ACCESS_CHANNEL.equals(rocketMQProperties.getAccessChannel())) { - defaultMQProducer.setAccessChannel(AccessChannel.CLOUD); - } - if (!StringUtils.isEmpty(mqProperties.getNamespace())) { - defaultMQProducer.setNamespace(mqProperties.getNamespace()); - } - defaultMQProducer.setNamesrvAddr(rocketMQProperties.getServers()); - defaultMQProducer.setRetryTimesWhenSendFailed(rocketMQProperties.getRetries()); - defaultMQProducer.setVipChannelEnabled(false); - logger.info("##Start RocketMQ producer##"); - try { - defaultMQProducer.start(); - } catch (MQClientException ex) { - throw new CanalServerException("Start RocketMQ producer error", ex); - } - } - - @Override - public void send(final MQProperties.CanalDestination destination, com.alibaba.otter.canal.protocol.Message data, - Callback callback) { - ExecutorTemplate template = new ExecutorTemplate(executor); - try { - if (!StringUtils.isEmpty(destination.getDynamicTopic())) { - // 动态topic - Map messageMap = MQMessageUtils.messageTopics(data, - destination.getTopic(), - destination.getDynamicTopic()); - - for (Map.Entry entry : messageMap.entrySet()) { - String topicName = entry.getKey().replace('.', '_'); - com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue(); - template.submit(new Runnable() { - - @Override - public void run() { - try { - send(destination, topicName, messageSub); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - }); - } - - template.waitForResult(); - } else { - send(destination, destination.getTopic(), data); - } - - callback.commit(); - } catch (Throwable e) { - logger.error(e.getMessage(), e); - callback.rollback(); - } finally { - template.clear(); - } - } - - public void send(final MQProperties.CanalDestination destination, String topicName, - com.alibaba.otter.canal.protocol.Message message) throws Exception { - if (!mqProperties.getFlatMessage()) { - if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) { - // 并发构造 - EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor); - // 串行分区 - com.alibaba.otter.canal.protocol.Message[] messages = MQMessageUtils.messagePartition(datas, - message.getId(), - destination.getPartitionsNum(), - destination.getPartitionHash(), - mqProperties.getDatabaseHash()); - int length = messages.length; - - ExecutorTemplate template = new ExecutorTemplate(executor); - for (int i = 0; i < length; i++) { - com.alibaba.otter.canal.protocol.Message dataPartition = messages[i]; - if (dataPartition != null) { - final int index = i; - template.submit(new Runnable() { - - @Override - public void run() { - Message data = new Message(topicName, CanalMessageSerializer.serializer(dataPartition, - mqProperties.isFilterTransactionEntry())); - sendMessage(data, index); - } - }); - } - } - // 等所有分片发送完毕 - template.waitForResult(); - } else { - final int partition = destination.getPartition() != null ? destination.getPartition() : 0; - Message data = new Message(topicName, CanalMessageSerializer.serializer(message, - mqProperties.isFilterTransactionEntry())); - sendMessage(data, partition); - } - } else { - // 并发构造 - EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor); - // 串行分区 - List flatMessages = MQMessageUtils.messageConverter(datas, message.getId()); - if (flatMessages != null) { - // 初始化分区合并队列 - if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) { - List> partitionFlatMessages = new ArrayList>(); - for (int i = 0; i < destination.getPartitionsNum(); i++) { - partitionFlatMessages.add(new ArrayList()); - } - - for (FlatMessage flatMessage : flatMessages) { - FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage, - destination.getPartitionsNum(), - destination.getPartitionHash(), - mqProperties.getDatabaseHash()); - int length = partitionFlatMessage.length; - for (int i = 0; i < length; i++) { - partitionFlatMessages.get(i).add(partitionFlatMessage[i]); - } - } - - ExecutorTemplate template = new ExecutorTemplate(executor); - for (int i = 0; i < partitionFlatMessages.size(); i++) { - final List flatMessagePart = partitionFlatMessages.get(i); - if (flatMessagePart != null) { - final int index = i; - template.submit(new Runnable() { - - @Override - public void run() { - List messages = flatMessagePart.stream() - .map(flatMessage -> new Message(topicName, JSON.toJSONBytes(flatMessage, - SerializerFeature.WriteMapNullValue))) - .collect(Collectors.toList()); - // 批量发送 - sendMessage(messages, index); - } - }); - } - } - - // 批量等所有分区的结果 - template.waitForResult(); - } else { - final int partition = destination.getPartition() != null ? destination.getPartition() : 0; - List messages = flatMessages.stream() - .map(flatMessage -> new Message(topicName, JSON.toJSONBytes(flatMessage, - SerializerFeature.WriteMapNullValue))) - .collect(Collectors.toList()); - // 批量发送 - sendMessage(messages, partition); - } - } - } - } - - private void sendMessage(Message message, int partition) { - try { - SendResult sendResult = this.defaultMQProducer.send(message, new MessageQueueSelector() { - - @Override - public MessageQueue select(List mqs, Message msg, Object arg) { - if (partition > mqs.size()) { - return mqs.get(partition % mqs.size()); - } else { - return mqs.get(partition); - } - } - }, null); - - if (logger.isDebugEnabled()) { - logger.debug("Send Message Result: {}", sendResult); - } - } catch (Throwable e) { - throw new RuntimeException(e); - } - } - - @SuppressWarnings("deprecation") - private void sendMessage(List messages, int partition) { - if (messages.isEmpty()) { - return; - } - - // 获取一下messageQueue - DefaultMQProducerImpl innerProducer = this.defaultMQProducer.getDefaultMQProducerImpl(); - TopicPublishInfo topicInfo = innerProducer.getTopicPublishInfoTable().get(messages.get(0).getTopic()); - if (topicInfo == null) { - for (Message message : messages) { - sendMessage(message, partition); - } - } else { - // 批量发送 - List queues = topicInfo.getMessageQueueList(); - int size = queues.size(); - if (size <= 0) { - // 可能是第一次创建 - for (Message message : messages) { - sendMessage(message, partition); - } - } else { - MessageQueue queue = null; - if (partition > size) { - queue = queues.get(partition % size); - } else { - queue = queues.get(partition); - } - - try { - SendResult sendResult = this.defaultMQProducer.send(messages, queue); - if (logger.isDebugEnabled()) { - logger.debug("Send Message Result: {}", sendResult); - } - } catch (Throwable e) { - throw new RuntimeException(e); - } - } - } - } - - @Override - public void stop() { - logger.info("## Stop RocketMQ producer##"); - this.defaultMQProducer.shutdown(); - super.stop(); - } -} diff --git a/server/src/main/java/com/alibaba/otter/canal/server/CanalMQStarter.java b/server/src/main/java/com/alibaba/otter/canal/server/CanalMQStarter.java index fb0653b1..dae91df2 100644 --- a/server/src/main/java/com/alibaba/otter/canal/server/CanalMQStarter.java +++ b/server/src/main/java/com/alibaba/otter/canal/server/CanalMQStarter.java @@ -7,18 +7,21 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import com.alibaba.otter.canal.connector.core.util.Callback; +import com.alibaba.otter.canal.connector.core.producer.MQDestination; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; -import com.alibaba.otter.canal.common.MQProperties; +import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer; import com.alibaba.otter.canal.instance.core.CanalInstance; import com.alibaba.otter.canal.instance.core.CanalMQConfig; import com.alibaba.otter.canal.protocol.ClientIdentity; import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded; -import com.alibaba.otter.canal.spi.CanalMQProducer; + +import com.alibaba.otter.canal.connector.core.config.MQProperties; public class CanalMQStarter { @@ -30,7 +33,7 @@ public class CanalMQStarter { private CanalMQProducer canalMQProducer; - private MQProperties properties; + private MQProperties mqProperties; private CanalServerWithEmbedded canalServer; @@ -42,15 +45,14 @@ public class CanalMQStarter { this.canalMQProducer = canalMQProducer; } - public synchronized void start(MQProperties properties, String destinations) { + public synchronized void start(String destinations) { try { if (running) { return; } - this.properties = properties; - canalMQProducer.init(properties); + mqProperties = canalMQProducer.getMqProperties(); // set filterTransactionEntry - if (properties.isFilterTransactionEntry()) { + if (mqProperties.isFilterTransactionEntry()) { System.setProperty("canal.instance.filter.transaction.entry", "true"); } @@ -151,7 +153,7 @@ public class CanalMQStarter { } continue; } - MQProperties.CanalDestination canalDestination = new MQProperties.CanalDestination(); + MQDestination canalDestination = new MQDestination(); canalDestination.setCanalDestination(destination); CanalMQConfig mqConfig = canalInstance.getMqConfig(); canalDestination.setTopic(mqConfig.getTopic()); @@ -163,15 +165,13 @@ public class CanalMQStarter { canalServer.subscribe(clientIdentity); logger.info("## the MQ producer: {} is running now ......", destination); - Long getTimeout = properties.getCanalGetTimeout(); - int getBatchSize = properties.getCanalBatchSize(); + Integer getTimeout = mqProperties.getFetchTimeout(); + Integer getBatchSize = mqProperties.getBatchSize(); while (running && destinationRunning.get()) { Message message; if (getTimeout != null && getTimeout > 0) { - message = canalServer.getWithoutAck(clientIdentity, - getBatchSize, - getTimeout, - TimeUnit.MILLISECONDS); + message = canalServer + .getWithoutAck(clientIdentity, getBatchSize, getTimeout.longValue(), TimeUnit.MILLISECONDS); } else { message = canalServer.getWithoutAck(clientIdentity, getBatchSize); } @@ -180,7 +180,7 @@ public class CanalMQStarter { try { int size = message.isRaw() ? message.getRawEntries().size() : message.getEntries().size(); if (batchId != -1 && size != 0) { - canalMQProducer.send(canalDestination, message, new CanalMQProducer.Callback() { + canalMQProducer.send(canalDestination, message, new Callback() { @Override public void commit() { diff --git a/server/src/main/java/com/alibaba/otter/canal/spi/CanalMQProducer.java b/server/src/main/java/com/alibaba/otter/canal/spi/CanalMQProducer.java deleted file mode 100644 index 9ddbc80d..00000000 --- a/server/src/main/java/com/alibaba/otter/canal/spi/CanalMQProducer.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.alibaba.otter.canal.spi; - -import java.io.IOException; - -import com.alibaba.otter.canal.common.MQProperties; -import com.alibaba.otter.canal.protocol.Message; - -public interface CanalMQProducer { - - /** - * Init producer. - * - * @param mqProperties MQ config - */ - void init(MQProperties mqProperties); - - /** - * Send canal message to related topic - * - * @param canalDestination canal mq destination - * @param message canal message - * @throws IOException - */ - void send(MQProperties.CanalDestination canalDestination, Message message, Callback callback) throws IOException; - - /** - * Stop MQ producer service - */ - void stop(); - - interface Callback { - - void commit(); - - void rollback(); - } -}