From 5bc5eec85e99e3be0f068a556e7cb4d4be2647a0 Mon Sep 17 00:00:00 2001 From: "titeng.jiang" Date: Wed, 24 Oct 2018 10:02:50 +0800 Subject: [PATCH 01/35] Fix Kafka FlatMessage Model Null Value set to "" --- .../alibaba/otter/canal/protocol/FlatMessage.java | 12 ++++++++++-- .../otter/canal/kafka/CanalKafkaProducer.java | 3 ++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/protocol/src/main/java/com/alibaba/otter/canal/protocol/FlatMessage.java b/protocol/src/main/java/com/alibaba/otter/canal/protocol/FlatMessage.java index f66cf9b5..dfa4280b 100644 --- a/protocol/src/main/java/com/alibaba/otter/canal/protocol/FlatMessage.java +++ b/protocol/src/main/java/com/alibaba/otter/canal/protocol/FlatMessage.java @@ -204,7 +204,11 @@ public class FlatMessage implements Serializable { for (CanalEntry.Column column : columns) { sqlType.put(column.getName(), column.getSqlType()); mysqlType.put(column.getName(), column.getMysqlType()); - row.put(column.getName(), column.getValue()); + if (column.getIsNull()) { + row.put(column.getName(), null); + } else { + row.put(column.getName(), column.getValue()); + } // 获取update为true的字段 if (column.getUpdated()) { updateSet.add(column.getName()); @@ -218,7 +222,11 @@ public class FlatMessage implements Serializable { Map rowOld = new LinkedHashMap<>(); for (CanalEntry.Column column : rowData.getBeforeColumnsList()) { if (updateSet.contains(column.getName())) { - rowOld.put(column.getName(), column.getValue()); + if (column.getIsNull()) { + rowOld.put(column.getName(), null); + } else { + rowOld.put(column.getName(), column.getValue()); + } } } // update操作将记录修改前的值 diff --git a/server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java b/server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java index 15217f13..93f5f4ab 100644 --- a/server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java +++ b/server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java @@ -5,6 +5,7 @@ import java.util.List; import java.util.Properties; import java.util.concurrent.Future; +import com.alibaba.fastjson.serializer.SerializerFeature; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerRecord; @@ -141,7 +142,7 @@ public class CanalKafkaProducer implements CanalMQProducer { canalDestination.getTopic(), 0, null, - JSON.toJSONString(flatMessage)); + JSON.toJSONString(flatMessage, SerializerFeature.WriteMapNullValue)); producer2.send(record).get(); } catch (Exception e) { logger.error(e.getMessage(), e); From a91f204682ced2c193d187e76c65a1a2c33bcf75 Mon Sep 17 00:00:00 2001 From: mcy Date: Wed, 24 Oct 2018 11:29:28 +0800 Subject: [PATCH 02/35] =?UTF-8?q?=E6=96=B0=E5=A2=9Eadapter-launcher=20spri?= =?UTF-8?q?ngboot=E5=B7=A5=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client-adapter/adapter-connector/pom.xml | 64 ++++++ client-adapter/adapter-launcher/pom.xml | 91 +++++++++ .../launcher/CanalAdapterApplication.java | 56 ++++++ .../launcher/config/AdapterCanalConfig.java | 10 + .../launcher/config/AdapterConfigPath.java | 35 ++++ .../loader/AbstractCanalAdapterWorker.java | 160 +++++++++++++++ .../launcher/loader/CanalAdapterLoader.java | 112 +++++++++++ .../launcher/loader/CanalAdapterWorker.java | 186 ++++++++++++++++++ .../src/main/resources/application.yml | 21 ++ client-adapter/common/pom.xml | 2 +- .../adapter/support/AdapterConfigs.java | 8 + client-adapter/hbase/pom.xml | 30 +-- .../hbase/config/MappingConfigLoader.java | 14 +- .../main/resources/hbase-mapping/configs.conf | 5 - .../mytest_person2.yml | 0 client-adapter/pom.xml | 33 +++- client-launcher/pom.xml | 7 - deployer/src/main/resources/canal.properties | 2 +- .../resources/example/instance.properties | 8 +- deployer/src/main/resources/logback.xml | 6 +- pom.xml | 2 +- 21 files changed, 806 insertions(+), 46 deletions(-) create mode 100644 client-adapter/adapter-connector/pom.xml create mode 100644 client-adapter/adapter-launcher/pom.xml create mode 100644 client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/CanalAdapterApplication.java create mode 100644 client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java create mode 100644 client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfigPath.java create mode 100644 client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java create mode 100644 client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java create mode 100644 client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java create mode 100644 client-adapter/adapter-launcher/src/main/resources/application.yml create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/AdapterConfigs.java delete mode 100644 client-adapter/hbase/src/main/resources/hbase-mapping/configs.conf rename client-adapter/hbase/src/main/resources/{hbase-mapping => hbase}/mytest_person2.yml (100%) diff --git a/client-adapter/adapter-connector/pom.xml b/client-adapter/adapter-connector/pom.xml new file mode 100644 index 00000000..16891c35 --- /dev/null +++ b/client-adapter/adapter-connector/pom.xml @@ -0,0 +1,64 @@ + + + + canal.client-adapter + com.alibaba.otter + 1.1.1-SNAPSHOT + + 4.0.0 + com.alibaba.otter + adapter-connector + pom + canal client adapter connector module for otter ${project.version} + + + + com.alibaba.otter + client-adapter.logger + ${project.version} + + + * + * + + + jar-with-dependencies + + + com.alibaba.otter + client-adapter.hbase + ${project.version} + + + * + * + + + jar-with-dependencies + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + 2.10 + + + copy-dependencies-to-canal-client-service + package + + copy-dependencies + + + ${project.basedir}/../adapter-launcher/target/lib + + + + + + + \ No newline at end of file diff --git a/client-adapter/adapter-launcher/pom.xml b/client-adapter/adapter-launcher/pom.xml new file mode 100644 index 00000000..50197d56 --- /dev/null +++ b/client-adapter/adapter-launcher/pom.xml @@ -0,0 +1,91 @@ + + + + canal.client-adapter + com.alibaba.otter + 1.1.1-SNAPSHOT + + 4.0.0 + com.alibaba.otter + adapter-launcher + jar + canal client adapter launcher module for otter ${project.version} + + + + + org.springframework.boot + spring-boot-dependencies + 2.0.1.RELEASE + pom + import + + + + + + + com.alibaba.otter + client-adapter.common + ${project.version} + + + com.alibaba.otter + canal.client + ${canal_version} + + + org.yaml + snakeyaml + 1.19 + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-configuration-processor + true + + + + + + + org.springframework.boot + spring-boot-maven-plugin + 2.0.1.RELEASE + + + + repackage + + + + + + maven-antrun-plugin + + + package + + run + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/CanalAdapterApplication.java b/client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/CanalAdapterApplication.java new file mode 100644 index 00000000..4b0035d8 --- /dev/null +++ b/client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/CanalAdapterApplication.java @@ -0,0 +1,56 @@ +package com.alibaba.otter.canal.adapter.launcher; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import javax.annotation.Resource; + +import com.alibaba.otter.canal.adapter.launcher.loader.CanalAdapterLoader; +import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.builder.SpringApplicationBuilder; + + +@SpringBootApplication +public class CanalAdapterApplication { + private static final Logger logger = LoggerFactory.getLogger(CanalAdapterApplication.class); + + private static CanalAdapterLoader adapterLoader; + + public static void main(String[] args) { + new SpringApplicationBuilder(CanalAdapterApplication.class).run(args); + } + + @Resource + private CanalClientConfig canalClientConf; + + @PostConstruct + public void init() { + if (adapterLoader == null) { + try { + logger.info("## start the canal client adapters."); + adapterLoader = new CanalAdapterLoader(canalClientConf); + adapterLoader.init(); + logger.info("## the canal client adapters are running now ......"); + } catch (Throwable e) { + logger.error("## something goes wrong when starting up the canal client adapters:", e); + System.exit(0); + } + } + } + + @PreDestroy + public void destroy() { + try { + logger.info("## stop the canal client adapters"); + if (adapterLoader != null) { + adapterLoader.destroy(); + } + } catch (Throwable e) { + logger.warn("## something goes wrong when stopping canal client adapters:", e); + } finally { + logger.info("## canal client adapters are down."); + } + } +} diff --git a/client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java b/client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java new file mode 100644 index 00000000..5b232c36 --- /dev/null +++ b/client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java @@ -0,0 +1,10 @@ +package com.alibaba.otter.canal.adapter.launcher.config; + +import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@ConfigurationProperties(prefix = "canal.conf") +@Component +public class AdapterCanalConfig extends CanalClientConfig { +} diff --git a/client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfigPath.java b/client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfigPath.java new file mode 100644 index 00000000..474d43af --- /dev/null +++ b/client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfigPath.java @@ -0,0 +1,35 @@ +package com.alibaba.otter.canal.adapter.launcher.config; + +import java.util.List; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs; + +@Component +@ConfigurationProperties(prefix = "adapter.conf.path") +public class AdapterConfigPath { + + private List adapterConfigs; + + public List getAdapterConfigs() { + return adapterConfigs; + } + + public void setAdapterConfigs(List adapterConfigs) { + this.adapterConfigs = adapterConfigs; + + if (adapterConfigs != null) { + AdapterConfigs.configs.clear(); + for (String adapterConfig : adapterConfigs) { + int idx = adapterConfig.indexOf("/"); + if (idx > -1) { + String type = adapterConfig.substring(0, idx); + String ymlFile = adapterConfig.substring(idx + 1); + AdapterConfigs.configs.put(type, ymlFile); + } + } + } + } +} diff --git a/client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java b/client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java new file mode 100644 index 00000000..e0912bab --- /dev/null +++ b/client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java @@ -0,0 +1,160 @@ +package com.alibaba.otter.canal.adapter.launcher.loader; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter; +import com.alibaba.otter.canal.client.adapter.support.MessageUtil; +import com.alibaba.otter.canal.protocol.Message; + +/** + * 适配器工作线程抽象类 + * + * @author machengyuan 2018-8-19 下午11:30:49 + * @version 1.0.0 + */ +public abstract class AbstractCanalAdapterWorker { + + protected final Logger logger = LoggerFactory.getLogger(this.getClass()); + + protected String canalDestination; // canal实例 + protected List> canalOuterAdapters; // 外部适配器 + protected ExecutorService groupInnerExecutorService; // 组内工作线程池 + protected volatile boolean running = false; // 是否运行中 + protected Thread thread = null; + protected Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() { + + @Override + public void uncaughtException(Thread t, Throwable e) { + logger.error("parse events has an error", e); + } + }; + + protected void writeOut(final Message message) { + List> futures = new ArrayList<>(); + // 组间适配器并行运行 + for (List outerAdapters : canalOuterAdapters) { + final List adapters = outerAdapters; + futures.add(groupInnerExecutorService.submit(new Callable() { + + @Override + public Boolean call() { + try { + // 组内适配器穿行运行，尽量不要配置组内适配器 + for (final CanalOuterAdapter c : adapters) { + long begin = System.currentTimeMillis(); + MessageUtil.parse4Dml(message, c::writeOut); + + if (logger.isDebugEnabled()) { + logger.debug("{} elapsed time: {}", + c.getClass().getName(), + (System.currentTimeMillis() - begin)); + } + } + return true; + } catch (Exception e) { + return false; + } + } + })); + + // 等待所有适配器写入完成 + // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成 + for (Future f : futures) { + try { + if (!f.get()) { + logger.error("Outer adapter write failed"); + } + } catch (InterruptedException | ExecutionException e) { + // ignore + } + } + } + } + + // protected void writeOut(final FlatMessage flatMessage) { + // List> futures = new ArrayList<>(); + // // 组间适配器并行运行 + // for (List outerAdapters : canalOuterAdapters) { + // final List adapters = outerAdapters; + // futures.add(groupInnerExecutorService.submit(new Callable() { + // + // @Override + // public Boolean call() { + // try { + // // 组内适配器穿行运行，尽量不要配置组内适配器 + // for (CanalOuterAdapter c : adapters) { + // long begin = System.currentTimeMillis(); + // Dml dml = MessageUtil.flatMessage2Dml(flatMessage); + // c.writeOut(dml); + // if (logger.isDebugEnabled()) { + // logger.debug("{} elapsed time: {}", + // c.getClass().getName(), + // (System.currentTimeMillis() - begin)); + // } + // } + // return true; + // } catch (Exception e) { + // return false; + // } + // } + // })); + // + // // 等待所有适配器写入完成 + // // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成 + // for (Future f : futures) { + // try { + // if (!f.get()) { + // logger.error("Outer adapter write failed"); + // } + // } catch (InterruptedException | ExecutionException e) { + // // ignore + // } + // } + // } + // } + + protected void writeOut(Message message, String topic) { + if (logger.isDebugEnabled()) { + logger.debug("topic: {} batchId: {} batchSize: {} ", topic, message.getId(), message.getEntries().size()); + } + long begin = System.currentTimeMillis(); + writeOut(message); + long now = System.currentTimeMillis(); + if ((System.currentTimeMillis() - begin) > 5 * 60 * 1000) { + logger.error("topic: {} batchId {} elapsed time: {} ms", topic, message.getId(), now - begin); + } + if (logger.isDebugEnabled()) { + logger.debug("topic: {} batchId {} elapsed time: {} ms", topic, message.getId(), now - begin); + } + } + + protected void stopOutAdapters() { + if (thread != null) { + try { + thread.join(); + } catch (InterruptedException e) { + // ignore + } + } + groupInnerExecutorService.shutdown(); + logger.info("topic connectors' worker thread dead!"); + for (List outerAdapters : canalOuterAdapters) { + for (CanalOuterAdapter adapter : outerAdapters) { + adapter.destroy(); + } + } + logger.info("topic all connectors destroyed!"); + } + + public abstract void start(); + + public abstract void stop(); +} diff --git a/client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java b/client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java new file mode 100644 index 00000000..71c9d130 --- /dev/null +++ b/client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java @@ -0,0 +1,112 @@ +package com.alibaba.otter.canal.adapter.launcher.loader; + +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter; +import com.alibaba.otter.canal.client.adapter.support.CanalOuterAdapterConfiguration; +import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; + +/** + * 外部适配器的加载器 + * + * @version 1.0.0 + */ +public class CanalAdapterLoader { + + private static final Logger logger = LoggerFactory.getLogger(CanalAdapterLoader.class); + + private CanalClientConfig canalClientConfig; + + private Map canalWorkers = new HashMap<>(); + + private ExtensionLoader loader; + + public CanalAdapterLoader(CanalClientConfig canalClientConfig) { + this.canalClientConfig = canalClientConfig; + } + + /** + * 初始化canal-client + */ + public void init() { + // canal instances 和 mq topics 配置不能同时为空 + // if (canalClientConfig.getCanalInstances() == null && canalClientConfig.getMqTopics() == null) { + // throw new RuntimeException("Blank config property: canalInstances or canalMQTopics"); + // } + + loader = ExtensionLoader.getExtensionLoader(CanalOuterAdapter.class); + + String canalServerHost = this.canalClientConfig.getCanalServerHost(); + SocketAddress sa = null; + if (canalServerHost != null) { + String[] ipPort = canalServerHost.split(":"); + sa = new InetSocketAddress(ipPort[0], Integer.parseInt(ipPort[1])); + } + String zkHosts = this.canalClientConfig.getZookeeperHosts(); + + // 初始化canal-client的适配器 + if (canalClientConfig.getCanalInstances() != null) { + for (CanalClientConfig.CanalInstance instance : canalClientConfig.getCanalInstances()) { + List> canalOuterAdapterGroups = new ArrayList<>(); + + for (CanalClientConfig.AdapterGroup connectorGroup : instance.getAdapterGroups()) { + List canalOutConnectors = new ArrayList<>(); + for (CanalOuterAdapterConfiguration c : connectorGroup.getOutAdapters()) { + loadConnector(c, canalOutConnectors); + } + canalOuterAdapterGroups.add(canalOutConnectors); + } + CanalAdapterWorker worker; + if (zkHosts != null) { + worker = new CanalAdapterWorker(instance.getInstance(), zkHosts, canalOuterAdapterGroups); + } else { + worker = new CanalAdapterWorker(instance.getInstance(), sa, canalOuterAdapterGroups); + } + canalWorkers.put(instance.getInstance(), worker); + worker.start(); + logger.info("Start adapter for canal instance: {} succeed", instance.getInstance()); + } + } + } + + private void loadConnector(CanalOuterAdapterConfiguration config, List canalOutConnectors) { + try { + CanalOuterAdapter adapter = loader.getExtension(config.getName()); + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + // 替换ClassLoader + Thread.currentThread().setContextClassLoader(adapter.getClass().getClassLoader()); + adapter.init(config); + Thread.currentThread().setContextClassLoader(cl); + canalOutConnectors.add(adapter); + logger.info("Load canal adapter: {} succeed", config.getName()); + } catch (Exception e) { + logger.error("Load canal adapter: {} failed", config.getName(), e); + } + } + + /** + * 销毁所有适配器 为防止canal实例太多造成销毁阻塞, 并行销毁 + */ + public void destroy() { + if (canalWorkers.size() > 0) { + ExecutorService stopExecutorService = Executors.newFixedThreadPool(canalWorkers.size()); + for (CanalAdapterWorker v : canalWorkers.values()) { + final CanalAdapterWorker caw = v; + stopExecutorService.submit(() -> caw.stop()); + } + stopExecutorService.shutdown(); + } + logger.info("All canal adapters destroyed"); + } +} diff --git a/client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java b/client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java new file mode 100644 index 00000000..7e090c2d --- /dev/null +++ b/client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java @@ -0,0 +1,186 @@ +package com.alibaba.otter.canal.adapter.launcher.loader; + +import java.net.SocketAddress; +import java.util.List; +import java.util.concurrent.Executors; + +import com.alibaba.otter.canal.client.CanalConnector; +import com.alibaba.otter.canal.client.CanalConnectors; +import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter; +import com.alibaba.otter.canal.client.impl.ClusterCanalConnector; +import com.alibaba.otter.canal.protocol.Message; + +/** + * 原生canal-server对应的client适配器工作线程 + * + * @author machengyuan 2018-8-19 下午11:30:49 + * @version 1.0.0 + */ +public class CanalAdapterWorker extends AbstractCanalAdapterWorker { + + private static final int BATCH_SIZE = 50; + private static final int SO_TIMEOUT = 0; + + private CanalConnector connector; + + /** + * 单台client适配器worker的构造方法 + * + * @param canalDestination canal实例名 + * @param address canal-server地址 + * @param canalOuterAdapters 外部适配器组 + */ + public CanalAdapterWorker(String canalDestination, SocketAddress address, + List> canalOuterAdapters){ + this.canalOuterAdapters = canalOuterAdapters; + this.canalDestination = canalDestination; + groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size()); + connector = CanalConnectors.newSingleConnector(address, canalDestination, "", ""); + } + + /** + * HA模式下client适配器worker的构造方法 + * + * @param canalDestination canal实例名 + * @param zookeeperHosts zookeeper地址 + * @param canalOuterAdapters 外部适配器组 + */ + public CanalAdapterWorker(String canalDestination, String zookeeperHosts, + List> canalOuterAdapters){ + this.canalOuterAdapters = canalOuterAdapters; + this.canalDestination = canalDestination; + groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size()); + connector = CanalConnectors.newClusterConnector(zookeeperHosts, canalDestination, "", ""); + ((ClusterCanalConnector) connector).setSoTimeout(SO_TIMEOUT); + + // super.initSwitcher(canalDestination); + } + + @Override + public void start() { + if (!running) { + thread = new Thread(new Runnable() { + + @Override + public void run() { + process(); + } + }); + thread.setUncaughtExceptionHandler(handler); + thread.start(); + running = true; + } + } + + @Override + public void stop() { + try { + if (!running) { + return; + } + + // if (switcher != null && !switcher.state()) { + // switcher.set(true); + // } + + connector.stopRunning(); + running = false; + + logger.info("destination {} is waiting for adapters' worker thread die!", canalDestination); + if (thread != null) { + try { + thread.join(); + } catch (InterruptedException e) { + // ignore + } + } + groupInnerExecutorService.shutdown(); + logger.info("destination {} adapters' worker thread dead!", canalDestination); + for (List outerAdapters : canalOuterAdapters) { + for (CanalOuterAdapter adapter : outerAdapters) { + adapter.destroy(); + } + } + logger.info("destination {} all adapters destroyed!", canalDestination); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + private void process() { + while (!running) + ; // waiting until running == true + while (running) { + try { + // if (switcher != null) { + // switcher.get(); + // } + logger.info("=============> Start to connect destination: {} <=============", this.canalDestination); + connector.connect(); + logger.info("=============> Start to subscribe destination: {} <=============", this.canalDestination); + connector.subscribe(); + logger.info("=============> Subscribe destination: {} succeed <=============", this.canalDestination); + while (running) { + // try { + // if (switcher != null) { + // switcher.get(); + // } + // } catch (TimeoutException e) { + // break; + // } + + // server配置canal.instance.network.soTimeout(默认: 30s) + // 范围内未与server交互，server将关闭本次socket连接 + Message message = connector.getWithoutAck(BATCH_SIZE); // 获取指定数量的数据 + long batchId = message.getId(); + try { + int size = message.getEntries().size(); + + if (batchId == -1 || size == 0) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + // ignore + } + } else { + if (logger.isDebugEnabled()) { + logger.debug("destination: {} batchId: {} batchSize: {} ", + this.canalDestination, + batchId, + size); + } + long begin = System.currentTimeMillis(); + writeOut(message); + long now = System.currentTimeMillis(); + if (logger.isDebugEnabled()) { + logger.debug("destination: {} batchId: {} elapsed time: {} ms", + this.canalDestination, + batchId, + now - begin); + } + } + connector.ack(batchId); // 提交确认 + } catch (Exception e) { + connector.rollback(batchId); // 处理失败, 回滚数据 + throw e; + } + } + + } catch (Exception e) { + logger.error("process error!", e); + } finally { + connector.disconnect(); + logger.info("=============> Disconnect destination: {} <=============", this.canalDestination); + } + + if (running) { // is reconnect + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + // ignore + } + } + + } + } +} diff --git a/client-adapter/adapter-launcher/src/main/resources/application.yml b/client-adapter/adapter-launcher/src/main/resources/application.yml new file mode 100644 index 00000000..bb06e75b --- /dev/null +++ b/client-adapter/adapter-launcher/src/main/resources/application.yml @@ -0,0 +1,21 @@ +server: + port: 8081 + +canal.conf: + canalServerHost: 127.0.0.1:11111 + #zookeeperHosts: slave1:2181 + #bootstrapServers: localhost:9092 #or rocketmq nameservers:host1:9876;host2:9876 + flatMessage: false + + canalInstances: + - instance: example + adapterGroups: + - outAdapters: + - name: logger +# - name: hbase +# hosts: slave1:2181 +# properties: {znodeParent: "/hbase-unsecure"} + +adapter.conf.path: + adapterConfigs: + - hbase/mytest_person2.yml \ No newline at end of file diff --git a/client-adapter/common/pom.xml b/client-adapter/common/pom.xml index 93793d09..a581ec4c 100644 --- a/client-adapter/common/pom.xml +++ b/client-adapter/common/pom.xml @@ -16,7 +16,7 @@ com.alibaba.otter canal.protocol - ${project.version} + ${canal_version} org.slf4j diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/AdapterConfigs.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/AdapterConfigs.java new file mode 100644 index 00000000..13e51af1 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/AdapterConfigs.java @@ -0,0 +1,8 @@ +package com.alibaba.otter.canal.client.adapter.support; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; + +public interface AdapterConfigs { + Multimap configs = ArrayListMultimap.create(); +} diff --git a/client-adapter/hbase/pom.xml b/client-adapter/hbase/pom.xml index 1437ed5d..da2746c6 100644 --- a/client-adapter/hbase/pom.xml +++ b/client-adapter/hbase/pom.xml @@ -22,7 +22,8 @@ org.yaml snakeyaml - 1.17 + 1.19 + provided org.apache.hbase @@ -45,15 +46,23 @@ - maven-jar-plugin + org.apache.maven.plugins + maven-assembly-plugin + 2.4 - - true - - - **/hbase-mapping/** - + + jar-with-dependencies + + + + make-assembly + package + + single + + + maven-antrun-plugin @@ -65,9 +74,8 @@ - - - + + diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java index 929e9d6a..5922d63e 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java @@ -4,6 +4,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.util.Collection; import java.util.LinkedHashMap; import java.util.Map; @@ -12,12 +13,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.yaml.snakeyaml.Yaml; +import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs; + /** * HBase表映射配置加载器 - *

- * 配置统一从hbase-mapping/configs.conf文件作为入口, 该文件包含所有表映射配置的名称或者文件名列表。 - * 每个对应的表配置可以yml配置文件或者以database.table为配置名的简化形式 - *

* * @author machengyuan 2018-8-21 下午06:45:49 * @version 1.0.0 @@ -26,7 +25,7 @@ public class MappingConfigLoader { private static Logger logger = LoggerFactory.getLogger(MappingConfigLoader.class); - private static final String BASE_PATH = "hbase-mapping/"; + private static final String BASE_PATH = "hbase"; /** * 加载HBase表映射配置 @@ -35,12 +34,11 @@ public class MappingConfigLoader { */ public static Map load() { logger.info("## Start loading mapping config ... "); - String mappingConfigContent = readConfigContent(BASE_PATH + "configs.conf"); Map result = new LinkedHashMap<>(); - String[] configLines = mappingConfigContent.split("\n"); - for (String c : configLines) { + Collection configs = AdapterConfigs.configs.get("hbase"); + for (String c : configs) { if (c == null) { continue; } diff --git a/client-adapter/hbase/src/main/resources/hbase-mapping/configs.conf b/client-adapter/hbase/src/main/resources/hbase-mapping/configs.conf deleted file mode 100644 index ff5b292e..00000000 --- a/client-adapter/hbase/src/main/resources/hbase-mapping/configs.conf +++ /dev/null @@ -1,5 +0,0 @@ -# 详细映射配置 -mytest_person2.yml - -# 简易配置, 只用指定数据库名.表名, 详细配置全部使用默认 -mytest.person \ No newline at end of file diff --git a/client-adapter/hbase/src/main/resources/hbase-mapping/mytest_person2.yml b/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml similarity index 100% rename from client-adapter/hbase/src/main/resources/hbase-mapping/mytest_person2.yml rename to client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml diff --git a/client-adapter/pom.xml b/client-adapter/pom.xml index 792ce283..7007de33 100644 --- a/client-adapter/pom.xml +++ b/client-adapter/pom.xml @@ -2,20 +2,43 @@ - - canal - com.alibaba.otter - 1.1.1-SNAPSHOT - 4.0.0 com.alibaba.otter canal.client-adapter + 1.1.1-SNAPSHOT pom canal client adapter module for otter ${project.version} + + + UTF-8 + true + true + 1.8 + 1.8 + UTF-8 + 1.1.1-SNAPSHOT + + common logger hbase + adapter-launcher + adapter-connector + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.7.0 + + ${java_source_version} + ${java_target_version} + ${file_encoding} + + + + diff --git a/client-launcher/pom.xml b/client-launcher/pom.xml index 8c29047a..78c1f164 100644 --- a/client-launcher/pom.xml +++ b/client-launcher/pom.xml @@ -46,13 +46,6 @@ com.alibaba.otter client-adapter.logger ${project.version} - jar-with-dependencies - - - com.alibaba.otter - client-adapter.hbase - ${project.version} - jar-with-dependencies diff --git a/deployer/src/main/resources/canal.properties b/deployer/src/main/resources/canal.properties index ea096de8..c14fbfec 100644 --- a/deployer/src/main/resources/canal.properties +++ b/deployer/src/main/resources/canal.properties @@ -63,7 +63,7 @@ canal.instance.parser.parallel = true canal.instance.parser.parallelBufferSize = 256 # table meta tsdb info -canal.instance.tsdb.enable=true +canal.instance.tsdb.enable=false canal.instance.tsdb.dir=${canal.file.data.dir:../conf}/${canal.instance.destination:} canal.instance.tsdb.url=jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL; canal.instance.tsdb.dbUsername=canal diff --git a/deployer/src/main/resources/example/instance.properties b/deployer/src/main/resources/example/instance.properties index 81f78b65..27e120f5 100644 --- a/deployer/src/main/resources/example/instance.properties +++ b/deployer/src/main/resources/example/instance.properties @@ -18,7 +18,7 @@ canal.instance.rds.secretkey= canal.instance.rds.instanceId= # table meta tsdb info -canal.instance.tsdb.enable=true +canal.instance.tsdb.enable=false #canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb #canal.instance.tsdb.dbUsername=canal #canal.instance.tsdb.dbPassword=canal @@ -30,13 +30,13 @@ canal.instance.tsdb.enable=true #canal.instance.standby.gtid= # username/password -canal.instance.dbUsername=canal -canal.instance.dbPassword=cZozNf1mzW6EQLGO2q9u99619xbZLO0fbua3EX08r4BWNXb8lAt1aHrTEOBttd6UY8Vnuc0easlVXZDdLtt8BQ== +canal.instance.dbUsername=root +canal.instance.dbPassword=121212 canal.instance.pwdPublicKey=MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBALK4BUxdDltRRE5/zXpVEVPUgunvscYFtEip3pmLlhrWpacX7y7GCMo2/JM6LeHmiiNdH1FWgGCpUfircSwlWKUCAwEAAQ== canal.instance.connectionCharset = UTF-8 canal.instance.defaultDatabaseName =test # enable druid Decrypt database password -canal.instance.enableDruid=true +canal.instance.enableDruid=false # table regex canal.instance.filter.regex=.*\\..* diff --git a/deployer/src/main/resources/logback.xml b/deployer/src/main/resources/logback.xml index d27a1d78..034beb33 100644 --- a/deployer/src/main/resources/logback.xml +++ b/deployer/src/main/resources/logback.xml @@ -78,8 +78,8 @@ - - - + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index dc200785..49ddc369 100644 --- a/pom.xml +++ b/pom.xml @@ -247,7 +247,7 @@ com.alibaba.fastsql fastsql - 2.0.0_preview_630 + 2.0.0_preview_540 com.alibaba From 243e129710b957eb66851f1ec60793a9a2c4acb4 Mon Sep 17 00:00:00 2001 From: mcy Date: Wed, 24 Oct 2018 11:43:36 +0800 Subject: [PATCH 03/35] =?UTF-8?q?jar=E5=8C=85=E4=BE=9D=E8=B5=96=E8=B0=83?= =?UTF-8?q?=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client-adapter/adapter-connector/pom.xml | 64 ------------------------ client-adapter/adapter-launcher/pom.xml | 46 +++++++++++++++++ client-adapter/pom.xml | 1 - 3 files changed, 46 insertions(+), 65 deletions(-) delete mode 100644 client-adapter/adapter-connector/pom.xml diff --git a/client-adapter/adapter-connector/pom.xml b/client-adapter/adapter-connector/pom.xml deleted file mode 100644 index 16891c35..00000000 --- a/client-adapter/adapter-connector/pom.xml +++ /dev/null @@ -1,64 +0,0 @@ - - - - canal.client-adapter - com.alibaba.otter - 1.1.1-SNAPSHOT - - 4.0.0 - com.alibaba.otter - adapter-connector - pom - canal client adapter connector module for otter ${project.version} - - - - com.alibaba.otter - client-adapter.logger - ${project.version} - - - * - * - - - jar-with-dependencies - - - com.alibaba.otter - client-adapter.hbase - ${project.version} - - - * - * - - - jar-with-dependencies - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - 2.10 - - - copy-dependencies-to-canal-client-service - package - - copy-dependencies - - - ${project.basedir}/../adapter-launcher/target/lib - - - - - - - \ No newline at end of file diff --git a/client-adapter/adapter-launcher/pom.xml b/client-adapter/adapter-launcher/pom.xml index 50197d56..dbe22333 100644 --- a/client-adapter/adapter-launcher/pom.xml +++ b/client-adapter/adapter-launcher/pom.xml @@ -50,6 +50,34 @@ spring-boot-configuration-processor true + + + + com.alibaba.otter + client-adapter.logger + ${project.version} + + + * + * + + + jar-with-dependencies + true + + + com.alibaba.otter + client-adapter.hbase + ${project.version} + + + * + * + + + jar-with-dependencies + true + @@ -86,6 +114,24 @@ + + org.apache.maven.plugins + maven-dependency-plugin + 2.10 + + + copy-dependencies-to-canal-client-service + package + + copy-dependencies + + + jar-with-dependencies + ${project.basedir}/target/lib + + + + \ No newline at end of file diff --git a/client-adapter/pom.xml b/client-adapter/pom.xml index 7007de33..cb831332 100644 --- a/client-adapter/pom.xml +++ b/client-adapter/pom.xml @@ -24,7 +24,6 @@ logger hbase adapter-launcher - adapter-connector From 0ee46ad70eb3a15de56c0fb82ba93cd63c9456af Mon Sep 17 00:00:00 2001 From: mcy Date: Wed, 24 Oct 2018 11:49:58 +0800 Subject: [PATCH 04/35] =?UTF-8?q?jar=E5=8C=85=E4=BE=9D=E8=B5=96=E8=B0=83?= =?UTF-8?q?=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deployer/src/main/resources/canal.properties | 2 +- .../resources/example/instance.properties | 10 ++--- deployer/src/main/resources/logback.xml | 42 +++++++++---------- pom.xml | 2 +- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/deployer/src/main/resources/canal.properties b/deployer/src/main/resources/canal.properties index c14fbfec..ea096de8 100644 --- a/deployer/src/main/resources/canal.properties +++ b/deployer/src/main/resources/canal.properties @@ -63,7 +63,7 @@ canal.instance.parser.parallel = true canal.instance.parser.parallelBufferSize = 256 # table meta tsdb info -canal.instance.tsdb.enable=false +canal.instance.tsdb.enable=true canal.instance.tsdb.dir=${canal.file.data.dir:../conf}/${canal.instance.destination:} canal.instance.tsdb.url=jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL; canal.instance.tsdb.dbUsername=canal diff --git a/deployer/src/main/resources/example/instance.properties b/deployer/src/main/resources/example/instance.properties index 27e120f5..25bab241 100644 --- a/deployer/src/main/resources/example/instance.properties +++ b/deployer/src/main/resources/example/instance.properties @@ -18,25 +18,25 @@ canal.instance.rds.secretkey= canal.instance.rds.instanceId= # table meta tsdb info -canal.instance.tsdb.enable=false +canal.instance.tsdb.enable=true #canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb #canal.instance.tsdb.dbUsername=canal #canal.instance.tsdb.dbPassword=canal #canal.instance.standby.address = #canal.instance.standby.journal.name = -#canal.instance.standby.position = +#canal.instance.standby.position = #canal.instance.standby.timestamp = #canal.instance.standby.gtid= # username/password -canal.instance.dbUsername=root -canal.instance.dbPassword=121212 +canal.instance.dbUsername=canal +canal.instance.dbPassword=cZozNf1mzW6EQLGO2q9u99619xbZLO0fbua3EX08r4BWNXb8lAt1aHrTEOBttd6UY8Vnuc0easlVXZDdLtt8BQ== canal.instance.pwdPublicKey=MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBALK4BUxdDltRRE5/zXpVEVPUgunvscYFtEip3pmLlhrWpacX7y7GCMo2/JM6LeHmiiNdH1FWgGCpUfircSwlWKUCAwEAAQ== canal.instance.connectionCharset = UTF-8 canal.instance.defaultDatabaseName =test # enable druid Decrypt database password -canal.instance.enableDruid=false +canal.instance.enableDruid=true # table regex canal.instance.filter.regex=.*\\..* diff --git a/deployer/src/main/resources/logback.xml b/deployer/src/main/resources/logback.xml index 034beb33..aaaefac7 100644 --- a/deployer/src/main/resources/logback.xml +++ b/deployer/src/main/resources/logback.xml @@ -6,7 +6,7 @@ - + destination @@ -16,7 +16,7 @@ ../logs/${destination}/${destination}.log + class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> ../logs/${destination}/%d{yyyy-MM-dd}/${destination}-%d{yyyy-MM-dd}-%i.log.gz @@ -33,7 +33,7 @@ - + destination @@ -43,7 +43,7 @@ ../logs/${destination}/meta.log + class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> ../logs/${destination}/%d{yyyy-MM-dd}/meta-%d{yyyy-MM-dd}-%i.log.gz @@ -60,26 +60,26 @@ - - - - - - - - - - - - - + + + + + + + + + + + + + - - - - + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index 49ddc369..dc200785 100644 --- a/pom.xml +++ b/pom.xml @@ -247,7 +247,7 @@ com.alibaba.fastsql fastsql - 2.0.0_preview_540 + 2.0.0_preview_630 com.alibaba From bbb4c4347eaa9d0a56c55bad2fd50e9ef64dc1fa Mon Sep 17 00:00:00 2001 From: mcy Date: Wed, 24 Oct 2018 12:44:19 +0800 Subject: [PATCH 05/35] =?UTF-8?q?=E4=BF=AE=E6=94=B9adapter-launcher?= =?UTF-8?q?=E6=A8=A1=E5=9D=97=E5=90=8D=E4=B8=BAlauncher?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client-adapter/hbase/pom.xml | 2 +- client-adapter/{adapter-launcher => launcher}/pom.xml | 2 +- .../otter/canal/adapter/launcher/CanalAdapterApplication.java | 0 .../otter/canal/adapter/launcher/config/AdapterCanalConfig.java | 0 .../otter/canal/adapter/launcher/config/AdapterConfigPath.java | 0 .../adapter/launcher/loader/AbstractCanalAdapterWorker.java | 0 .../otter/canal/adapter/launcher/loader/CanalAdapterLoader.java | 0 .../otter/canal/adapter/launcher/loader/CanalAdapterWorker.java | 0 .../src/main/resources/application.yml | 0 client-adapter/pom.xml | 2 +- 10 files changed, 3 insertions(+), 3 deletions(-) rename client-adapter/{adapter-launcher => launcher}/pom.xml (99%) rename client-adapter/{adapter-launcher => launcher}/src/main/java/com/alibaba/otter/canal/adapter/launcher/CanalAdapterApplication.java (100%) rename client-adapter/{adapter-launcher => launcher}/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java (100%) rename client-adapter/{adapter-launcher => launcher}/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfigPath.java (100%) rename client-adapter/{adapter-launcher => launcher}/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java (100%) rename client-adapter/{adapter-launcher => launcher}/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java (100%) rename client-adapter/{adapter-launcher => launcher}/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java (100%) rename client-adapter/{adapter-launcher => launcher}/src/main/resources/application.yml (100%) diff --git a/client-adapter/hbase/pom.xml b/client-adapter/hbase/pom.xml index da2746c6..866bd31f 100644 --- a/client-adapter/hbase/pom.xml +++ b/client-adapter/hbase/pom.xml @@ -74,7 +74,7 @@ - + diff --git a/client-adapter/adapter-launcher/pom.xml b/client-adapter/launcher/pom.xml similarity index 99% rename from client-adapter/adapter-launcher/pom.xml rename to client-adapter/launcher/pom.xml index dbe22333..7b94bfc9 100644 --- a/client-adapter/adapter-launcher/pom.xml +++ b/client-adapter/launcher/pom.xml @@ -9,7 +9,7 @@ 4.0.0 com.alibaba.otter - adapter-launcher + launcher jar canal client adapter launcher module for otter ${project.version} diff --git a/client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/CanalAdapterApplication.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/CanalAdapterApplication.java similarity index 100% rename from client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/CanalAdapterApplication.java rename to client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/CanalAdapterApplication.java diff --git a/client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java similarity index 100% rename from client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java rename to client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java diff --git a/client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfigPath.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfigPath.java similarity index 100% rename from client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfigPath.java rename to client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfigPath.java diff --git a/client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java similarity index 100% rename from client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java rename to client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java diff --git a/client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java similarity index 100% rename from client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java rename to client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java diff --git a/client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java similarity index 100% rename from client-adapter/adapter-launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java rename to client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java diff --git a/client-adapter/adapter-launcher/src/main/resources/application.yml b/client-adapter/launcher/src/main/resources/application.yml similarity index 100% rename from client-adapter/adapter-launcher/src/main/resources/application.yml rename to client-adapter/launcher/src/main/resources/application.yml diff --git a/client-adapter/pom.xml b/client-adapter/pom.xml index cb831332..aafe6395 100644 --- a/client-adapter/pom.xml +++ b/client-adapter/pom.xml @@ -23,7 +23,7 @@ common logger hbase - adapter-launcher + launcher From 1b88430f71ca814badc85ae9582b5c8b3b4a8f5f Mon Sep 17 00:00:00 2001 From: mcy Date: Wed, 24 Oct 2018 12:55:42 +0800 Subject: [PATCH 06/35] =?UTF-8?q?=E5=8E=BB=E6=8E=89client-launcher,=20?= =?UTF-8?q?=E7=A7=BB=E5=85=A5client-adapter=E4=B8=AD=E9=87=8D=E5=91=BD?= =?UTF-8?q?=E5=90=8D=E4=B8=BAexample,=20=E8=B0=83=E6=95=B4=E5=85=B6?= =?UTF-8?q?=E4=BB=96=E7=9B=B8=E5=85=B3=E6=A8=A1=E5=9D=97=E7=BB=93=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../example}/pom.xml | 12 +++++----- .../example}/src/main/assembly/dev.xml | 0 .../example}/src/main/assembly/release.xml | 0 .../example}/src/main/bin/startup.bat | 0 .../example}/src/main/bin/startup.sh | 0 .../example}/src/main/bin/stop.sh | 0 .../otter/canal/client/ClientLauncher.java | 0 .../loader/AbstractCanalAdapterWorker.java | 0 .../loader/CanalAdapterKafkaWorker.java | 0 .../adapter/loader/CanalAdapterLoader.java | 0 .../loader/CanalAdapterRocketMQWorker.java | 0 .../adapter/loader/CanalAdapterWorker.java | 0 .../src/main/resources/canal-client.yml | 18 ++++++++++++++ .../example}/src/main/resources/logback.xml | 2 +- client-adapter/pom.xml | 1 + .../src/main/resources/canal-client.yml | 24 ------------------- pom.xml | 1 - 17 files changed, 26 insertions(+), 32 deletions(-) rename {client-launcher => client-adapter/example}/pom.xml (96%) rename {client-launcher => client-adapter/example}/src/main/assembly/dev.xml (100%) rename {client-launcher => client-adapter/example}/src/main/assembly/release.xml (100%) rename {client-launcher => client-adapter/example}/src/main/bin/startup.bat (100%) rename {client-launcher => client-adapter/example}/src/main/bin/startup.sh (100%) rename {client-launcher => client-adapter/example}/src/main/bin/stop.sh (100%) rename {client-launcher => client-adapter/example}/src/main/java/com/alibaba/otter/canal/client/ClientLauncher.java (100%) rename {client-launcher => client-adapter/example}/src/main/java/com/alibaba/otter/canal/client/adapter/loader/AbstractCanalAdapterWorker.java (100%) rename {client-launcher => client-adapter/example}/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterKafkaWorker.java (100%) rename {client-launcher => client-adapter/example}/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterLoader.java (100%) rename {client-launcher => client-adapter/example}/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterRocketMQWorker.java (100%) rename {client-launcher => client-adapter/example}/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterWorker.java (100%) create mode 100644 client-adapter/example/src/main/resources/canal-client.yml rename {client-launcher => client-adapter/example}/src/main/resources/logback.xml (97%) delete mode 100644 client-launcher/src/main/resources/canal-client.yml diff --git a/client-launcher/pom.xml b/client-adapter/example/pom.xml similarity index 96% rename from client-launcher/pom.xml rename to client-adapter/example/pom.xml index 78c1f164..3c68087a 100644 --- a/client-launcher/pom.xml +++ b/client-adapter/example/pom.xml @@ -3,15 +3,16 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - canal + canal.client-adapter com.alibaba.otter 1.1.1-SNAPSHOT 4.0.0 com.alibaba.otter - canal.client-launcher + example jar - canal client launcher module for otter ${project.version} + canal client adapter example module for otter ${project.version} + com.alibaba.otter @@ -21,7 +22,7 @@ com.alibaba.otter canal.client - ${project.version} + ${canal_version} @@ -144,5 +145,4 @@ - - + \ No newline at end of file diff --git a/client-launcher/src/main/assembly/dev.xml b/client-adapter/example/src/main/assembly/dev.xml similarity index 100% rename from client-launcher/src/main/assembly/dev.xml rename to client-adapter/example/src/main/assembly/dev.xml diff --git a/client-launcher/src/main/assembly/release.xml b/client-adapter/example/src/main/assembly/release.xml similarity index 100% rename from client-launcher/src/main/assembly/release.xml rename to client-adapter/example/src/main/assembly/release.xml diff --git a/client-launcher/src/main/bin/startup.bat b/client-adapter/example/src/main/bin/startup.bat similarity index 100% rename from client-launcher/src/main/bin/startup.bat rename to client-adapter/example/src/main/bin/startup.bat diff --git a/client-launcher/src/main/bin/startup.sh b/client-adapter/example/src/main/bin/startup.sh similarity index 100% rename from client-launcher/src/main/bin/startup.sh rename to client-adapter/example/src/main/bin/startup.sh diff --git a/client-launcher/src/main/bin/stop.sh b/client-adapter/example/src/main/bin/stop.sh similarity index 100% rename from client-launcher/src/main/bin/stop.sh rename to client-adapter/example/src/main/bin/stop.sh diff --git a/client-launcher/src/main/java/com/alibaba/otter/canal/client/ClientLauncher.java b/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/ClientLauncher.java similarity index 100% rename from client-launcher/src/main/java/com/alibaba/otter/canal/client/ClientLauncher.java rename to client-adapter/example/src/main/java/com/alibaba/otter/canal/client/ClientLauncher.java diff --git a/client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/AbstractCanalAdapterWorker.java b/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/AbstractCanalAdapterWorker.java similarity index 100% rename from client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/AbstractCanalAdapterWorker.java rename to client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/AbstractCanalAdapterWorker.java diff --git a/client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterKafkaWorker.java b/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterKafkaWorker.java similarity index 100% rename from client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterKafkaWorker.java rename to client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterKafkaWorker.java diff --git a/client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterLoader.java b/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterLoader.java similarity index 100% rename from client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterLoader.java rename to client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterLoader.java diff --git a/client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterRocketMQWorker.java b/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterRocketMQWorker.java similarity index 100% rename from client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterRocketMQWorker.java rename to client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterRocketMQWorker.java diff --git a/client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterWorker.java b/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterWorker.java similarity index 100% rename from client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterWorker.java rename to client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterWorker.java diff --git a/client-adapter/example/src/main/resources/canal-client.yml b/client-adapter/example/src/main/resources/canal-client.yml new file mode 100644 index 00000000..de24e756 --- /dev/null +++ b/client-adapter/example/src/main/resources/canal-client.yml @@ -0,0 +1,18 @@ +canalServerHost: 127.0.0.1:11111 +#zookeeperHosts: slave1:2181 +#bootstrapServers: slave1:6667,slave2:6667 #or rocketmq nameservers:host1:9876;host2:9876 +flatMessage: false + +canalInstances: +- instance: example + adapterGroups: + - outAdapters: + - name: logger + +#mqTopics: +#- mqMode: rocketmq +# topic: example +# groups: +# - groupId: example +# outAdapters: +# - name: logger diff --git a/client-launcher/src/main/resources/logback.xml b/client-adapter/example/src/main/resources/logback.xml similarity index 97% rename from client-launcher/src/main/resources/logback.xml rename to client-adapter/example/src/main/resources/logback.xml index 665954f7..e74c84f0 100644 --- a/client-launcher/src/main/resources/logback.xml +++ b/client-adapter/example/src/main/resources/logback.xml @@ -36,7 +36,7 @@ - + diff --git a/client-adapter/pom.xml b/client-adapter/pom.xml index aafe6395..7ca65de1 100644 --- a/client-adapter/pom.xml +++ b/client-adapter/pom.xml @@ -24,6 +24,7 @@ logger hbase launcher + example diff --git a/client-launcher/src/main/resources/canal-client.yml b/client-launcher/src/main/resources/canal-client.yml deleted file mode 100644 index d2cb2175..00000000 --- a/client-launcher/src/main/resources/canal-client.yml +++ /dev/null @@ -1,24 +0,0 @@ -#canalServerHost: 127.0.0.1:11111 -#zookeeperHosts: slave1:2181 -bootstrapServers: slave1:6667,slave2:6667 #or rocketmq nameservers:host1:9876;host2:9876 -flatMessage: true - -#canalInstances: -#- instance: example -# adapterGroups: -# - outAdapters: -# - name: logger -# - name: hbase -# hosts: slave1:2181 -# properties: {znodeParent: "/hbase-unsecure"} - -mqTopics: -- mqMode: rocketmq - topic: example - groups: - - groupId: example - outAdapters: - - name: logger -# - name: hbase -# hosts: slave1:2181 -# properties: {znodeParent: "/hbase-unsecure"} diff --git a/pom.xml b/pom.xml index dc200785..27038086 100644 --- a/pom.xml +++ b/pom.xml @@ -119,7 +119,6 @@ example prometheus client-adapter - client-launcher From c75c24808664ede8faadabd082b7a191f3c232fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Wed, 24 Oct 2018 13:45:58 +0800 Subject: [PATCH 07/35] fixed issue #1019 , support canal embedded with Entry --- .../otter/canal/protocol/FlatMessage.java | 16 +++- .../alibaba/otter/canal/protocol/Message.java | 1 - .../embedded/CanalServerWithEmbedded.java | 83 ++++++++++++++----- .../canal/sink/entry/EntryEventSink.java | 10 ++- .../memory/MemoryEventStoreWithBuffer.java | 11 +++ .../otter/canal/store/model/Event.java | 28 ++++++- 6 files changed, 116 insertions(+), 33 deletions(-) diff --git a/protocol/src/main/java/com/alibaba/otter/canal/protocol/FlatMessage.java b/protocol/src/main/java/com/alibaba/otter/canal/protocol/FlatMessage.java index dfa4280b..d8821f46 100644 --- a/protocol/src/main/java/com/alibaba/otter/canal/protocol/FlatMessage.java +++ b/protocol/src/main/java/com/alibaba/otter/canal/protocol/FlatMessage.java @@ -149,11 +149,19 @@ public class FlatMessage implements Serializable { } List flatMessages = new ArrayList<>(); + List entrys = null; + if (message.isRaw()) { + List rawEntries = message.getRawEntries(); + entrys = new ArrayList(rawEntries.size()); + for (ByteString byteString : rawEntries) { + CanalEntry.Entry entry = CanalEntry.Entry.parseFrom(byteString); + entrys.add(entry); + } + } else { + entrys = message.getEntries(); + } - List rawEntries = message.getRawEntries(); - - for (ByteString byteString : rawEntries) { - CanalEntry.Entry entry = CanalEntry.Entry.parseFrom(byteString); + for (CanalEntry.Entry entry : entrys) { if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) { continue; diff --git a/protocol/src/main/java/com/alibaba/otter/canal/protocol/Message.java b/protocol/src/main/java/com/alibaba/otter/canal/protocol/Message.java index 35adf15d..751e09d0 100644 --- a/protocol/src/main/java/com/alibaba/otter/canal/protocol/Message.java +++ b/protocol/src/main/java/com/alibaba/otter/canal/protocol/Message.java @@ -19,7 +19,6 @@ public class Message implements Serializable { private static final long serialVersionUID = 1234034768477580009L; private long id; - @Deprecated private List entries = new ArrayList(); // row data for performance, see: // https://github.com/alibaba/canal/issues/726 diff --git a/server/src/main/java/com/alibaba/otter/canal/server/embedded/CanalServerWithEmbedded.java b/server/src/main/java/com/alibaba/otter/canal/server/embedded/CanalServerWithEmbedded.java index a4f74c90..b07062ee 100644 --- a/server/src/main/java/com/alibaba/otter/canal/server/embedded/CanalServerWithEmbedded.java +++ b/server/src/main/java/com/alibaba/otter/canal/server/embedded/CanalServerWithEmbedded.java @@ -1,11 +1,12 @@ package com.alibaba.otter.canal.server.embedded; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.ServiceLoader; import java.util.concurrent.TimeUnit; -import com.alibaba.otter.canal.spi.CanalMetricsProvider; -import com.alibaba.otter.canal.spi.CanalMetricsService; -import com.alibaba.otter.canal.spi.NopCanalMetricsService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; @@ -14,6 +15,7 @@ import org.springframework.util.CollectionUtils; import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; import com.alibaba.otter.canal.instance.core.CanalInstance; import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator; +import com.alibaba.otter.canal.protocol.CanalEntry; import com.alibaba.otter.canal.protocol.ClientIdentity; import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.protocol.position.LogPosition; @@ -22,7 +24,11 @@ import com.alibaba.otter.canal.protocol.position.PositionRange; import com.alibaba.otter.canal.server.CanalServer; import com.alibaba.otter.canal.server.CanalService; import com.alibaba.otter.canal.server.exception.CanalServerException; +import com.alibaba.otter.canal.spi.CanalMetricsProvider; +import com.alibaba.otter.canal.spi.CanalMetricsService; +import com.alibaba.otter.canal.spi.NopCanalMetricsService; import com.alibaba.otter.canal.store.CanalEventStore; +import com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer; import com.alibaba.otter.canal.store.model.Event; import com.alibaba.otter.canal.store.model.Events; import com.google.common.base.Function; @@ -40,12 +46,12 @@ import com.google.protobuf.ByteString; */ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements CanalServer, CanalService { - private static final Logger logger = LoggerFactory.getLogger(CanalServerWithEmbedded.class); + private static final Logger logger = LoggerFactory.getLogger(CanalServerWithEmbedded.class); private Map canalInstances; // private Map lastRollbackPostions; private CanalInstanceGenerator canalInstanceGenerator; private int metricsPort; - private CanalMetricsService metrics = NopCanalMetricsService.NOP; + private CanalMetricsService metrics = NopCanalMetricsService.NOP; private static class SingletonHolder { @@ -207,7 +213,7 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C * b. 如果timeout不为null * 1. timeout为0，则采用get阻塞方式，获取数据，不设置超时，直到有足够的batchSize数据才返回 * 2. timeout不为0，则采用get+timeout方式，获取数据，超时还没有batchSize足够的数据，有多少返回多少 - * + * * 注意： meta获取和数据的获取需要保证顺序性，优先拿到meta的，一定也会是优先拿到数据，所以需要加同步. (不能出现先拿到meta，拿到第二批数据，这样就会导致数据顺序性出现问题) * */ @@ -239,12 +245,23 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C } else { // 记录到流式信息 Long batchId = canalInstance.getMetaManager().addBatch(clientIdentity, events.getPositionRange()); - List entrys = Lists.transform(events.getEvents(), new Function() { + boolean raw = isRaw(canalInstance.getEventStore()); + List entrys = null; + if (raw) { + entrys = Lists.transform(events.getEvents(), new Function() { - public ByteString apply(Event input) { - return input.getRawEntry(); - } - }); + public ByteString apply(Event input) { + return input.getRawEntry(); + } + }); + } else { + entrys = Lists.transform(events.getEvents(), new Function() { + + public CanalEntry.Entry apply(Event input) { + return input.getEntry(); + } + }); + } if (logger.isInfoEnabled()) { logger.info("get successfully, clientId:{} batchSize:{} real size is {} and result is [batchId:{} , position:{}]", clientIdentity.getClientId(), @@ -255,7 +272,7 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C } // 直接提交ack ack(clientIdentity, batchId); - return new Message(batchId, true, entrys); + return new Message(batchId, raw, entrys); } } } @@ -283,7 +300,7 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C * b. 如果timeout不为null * 1. timeout为0，则采用get阻塞方式，获取数据，不设置超时，直到有足够的batchSize数据才返回 * 2. timeout不为0，则采用get+timeout方式，获取数据，超时还没有batchSize足够的数据，有多少返回多少 - * + * * 注意： meta获取和数据的获取需要保证顺序性，优先拿到meta的，一定也会是优先拿到数据，所以需要加同步. (不能出现先拿到meta，拿到第二批数据，这样就会导致数据顺序性出现问题) * */ @@ -311,7 +328,8 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C } if (CollectionUtils.isEmpty(events.getEvents())) { - // logger.debug("getWithoutAck successfully, clientId:{} batchSize:{} but result + // logger.debug("getWithoutAck successfully, clientId:{} + // batchSize:{} but result // is null", // clientIdentity.getClientId(), // batchSize); @@ -319,12 +337,23 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C } else { // 记录到流式信息 Long batchId = canalInstance.getMetaManager().addBatch(clientIdentity, events.getPositionRange()); - List entrys = Lists.transform(events.getEvents(), new Function() { + boolean raw = isRaw(canalInstance.getEventStore()); + List entrys = null; + if (raw) { + entrys = Lists.transform(events.getEvents(), new Function() { - public ByteString apply(Event input) { - return input.getRawEntry(); - } - }); + public ByteString apply(Event input) { + return input.getRawEntry(); + } + }); + } else { + entrys = Lists.transform(events.getEvents(), new Function() { + + public CanalEntry.Entry apply(Event input) { + return input.getEntry(); + } + }); + } if (logger.isInfoEnabled()) { logger.info("getWithoutAck successfully, clientId:{} batchSize:{} real size is {} and result is [batchId:{} , position:{}]", clientIdentity.getClientId(), @@ -333,7 +362,7 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C batchId, events.getPositionRange()); } - return new Message(batchId, true, entrys); + return new Message(batchId, raw, entrys); } } @@ -515,17 +544,25 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C // 发现provider, 进行初始化 if (list.size() > 1) { logger.warn("Found more than one CanalMetricsProvider, use the first one."); - //报告冲突 + // 报告冲突 for (CanalMetricsProvider p : list) { logger.warn("Found CanalMetricsProvider: {}.", p.getClass().getName()); } } - //默认使用第一个 + // 默认使用第一个 CanalMetricsProvider provider = list.get(0); this.metrics = provider.getService(); } } + private boolean isRaw(CanalEventStore eventStore) { + if (eventStore instanceof MemoryEventStoreWithBuffer) { + return ((MemoryEventStoreWithBuffer) eventStore).isRaw(); + } + + return true; + } + // ========= setter ========== public void setCanalInstanceGenerator(CanalInstanceGenerator canalInstanceGenerator) { diff --git a/sink/src/main/java/com/alibaba/otter/canal/sink/entry/EntryEventSink.java b/sink/src/main/java/com/alibaba/otter/canal/sink/entry/EntryEventSink.java index 76124ec4..d34450ed 100644 --- a/sink/src/main/java/com/alibaba/otter/canal/sink/entry/EntryEventSink.java +++ b/sink/src/main/java/com/alibaba/otter/canal/sink/entry/EntryEventSink.java @@ -20,6 +20,7 @@ import com.alibaba.otter.canal.sink.CanalEventDownStreamHandler; import com.alibaba.otter.canal.sink.CanalEventSink; import com.alibaba.otter.canal.sink.exception.CanalSinkException; import com.alibaba.otter.canal.store.CanalEventStore; +import com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer; import com.alibaba.otter.canal.store.model.Event; /** @@ -42,7 +43,8 @@ public class EntryEventSink extends AbstractCanalEventSink props = entry.getHeader().getPropsList(); if (props != null) { @@ -60,6 +64,16 @@ public class Event implements Serializable { } } } + + if (raw) { + // build raw + this.rawEntry = entry.toByteString(); + this.rawLength = rawEntry.size(); + } else { + this.entry = entry; + // 按照3倍的event length预估 + this.rawLength = entry.getHeader().getEventLength() * 3; + } } public LogIdentity getLogIdentity() { @@ -150,6 +164,14 @@ public class Event implements Serializable { this.rowsCount = rowsCount; } + public CanalEntry.Entry getEntry() { + return entry; + } + + public void setEntry(CanalEntry.Entry entry) { + this.entry = entry; + } + public String toString() { return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE); } From 15e79c140075a145ba8a422cce2ba626fcb222c5 Mon Sep 17 00:00:00 2001 From: mcy Date: Wed, 24 Oct 2018 15:40:48 +0800 Subject: [PATCH 08/35] =?UTF-8?q?=E8=B0=83=E6=95=B4Hbase=E9=80=82=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client-adapter/common/pom.xml | 10 +++ ...nalOuterAdapter.java => OuterAdapter.java} | 18 ++++- .../adapter/support/AdapterConfigs.java | 28 ++++++- .../adapter/support/DatasourceConfig.java | 75 ++++++++++++++++++ .../loader/AbstractCanalAdapterWorker.java | 30 ++++---- .../loader/CanalAdapterKafkaWorker.java | 9 +-- .../adapter/loader/CanalAdapterLoader.java | 18 ++--- .../loader/CanalAdapterRocketMQWorker.java | 4 +- .../adapter/loader/CanalAdapterWorker.java | 10 +-- .../client/adapter/hbase/HbaseAdapter.java | 6 +- .../adapter/hbase/config/MappingConfig.java | 9 +++ .../hbase/config/MappingConfigLoader.java | 2 +- .../hbase/service/HbaseEtlService.java | 4 + ...a.otter.canal.client.adapter.OuterAdapter} | 0 .../main/resources/hbase/mytest_person2.yml | 3 +- client-adapter/launcher/pom.xml | 1 + .../launcher/CanalAdapterApplication.java | 44 ----------- .../launcher/config/AdapterCanalConfig.java | 2 +- .../launcher/config/AdapterConfig.java | 76 +++++++++++++++++++ .../launcher/config/AdapterConfigPath.java | 35 --------- .../loader/AbstractCanalAdapterWorker.java | 53 +++++-------- .../launcher/loader/CanalAdapterLoader.java | 27 +++---- .../launcher/loader/CanalAdapterService.java | 65 ++++++++++++++++ .../launcher/loader/CanalAdapterWorker.java | 10 +-- .../src/main/resources/application.yml | 7 +- .../adapter/logger/LoggerAdapterExample.java | 6 +- ...a.otter.canal.client.adapter.OuterAdapter} | 0 27 files changed, 366 insertions(+), 186 deletions(-) rename client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/{CanalOuterAdapter.java => OuterAdapter.java} (70%) create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/DatasourceConfig.java create mode 100644 client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java rename client-adapter/hbase/src/main/resources/META-INF/canal/{com.alibaba.otter.canal.client.adapter.CanalOuterAdapter => com.alibaba.otter.canal.client.adapter.OuterAdapter} (100%) create mode 100644 client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfig.java delete mode 100644 client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfigPath.java create mode 100644 client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterService.java rename client-adapter/logger/src/main/resources/META-INF/canal/{com.alibaba.otter.canal.client.adapter.CanalOuterAdapter => com.alibaba.otter.canal.client.adapter.OuterAdapter} (100%) diff --git a/client-adapter/common/pom.xml b/client-adapter/common/pom.xml index a581ec4c..d61d8d11 100644 --- a/client-adapter/common/pom.xml +++ b/client-adapter/common/pom.xml @@ -28,6 +28,16 @@ joda-time 2.9.4 + + com.alibaba + druid + 1.1.9 + + + mysql + mysql-connector-java + 5.1.40 + diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/CanalOuterAdapter.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java similarity index 70% rename from client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/CanalOuterAdapter.java rename to client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java index 5fe0a377..c4ee60fc 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/CanalOuterAdapter.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java @@ -1,5 +1,7 @@ package com.alibaba.otter.canal.client.adapter; +import java.util.Map; + import com.alibaba.otter.canal.client.adapter.support.CanalOuterAdapterConfiguration; import com.alibaba.otter.canal.client.adapter.support.Dml; import com.alibaba.otter.canal.client.adapter.support.SPI; @@ -11,7 +13,7 @@ import com.alibaba.otter.canal.client.adapter.support.SPI; * @version 1.0.0 */ @SPI("logger") -public interface CanalOuterAdapter { +public interface OuterAdapter { /** * 外部适配器初始化接口 @@ -21,11 +23,11 @@ public interface CanalOuterAdapter { void init(CanalOuterAdapterConfiguration configuration); /** - * 往适配器中写入数据 + * 往适配器中同步数据 * * @param dml 数据包 */ - void writeOut(Dml dml); + void sync(Dml dml); // void writeOut(FlatMessage flatMessage); @@ -33,4 +35,14 @@ public interface CanalOuterAdapter { * 外部适配器销毁接口 */ void destroy(); + + /** + * Etl操作 + * + * @param criteriaSql 条件拼接sql + */ + default void etl(String criteriaSql) { + throw new UnsupportedOperationException(); + } + } diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/AdapterConfigs.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/AdapterConfigs.java index 13e51af1..44d29ee9 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/AdapterConfigs.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/AdapterConfigs.java @@ -1,8 +1,28 @@ package com.alibaba.otter.canal.client.adapter.support; -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.Multimap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; -public interface AdapterConfigs { - Multimap configs = ArrayListMultimap.create(); +public class AdapterConfigs { + + private static Map> configs = new ConcurrentHashMap<>(); + + public static void put(String key, String value) { + Set values = configs.get(key); + if (values == null) { + values = new LinkedHashSet<>(); + } + values.add(value); + configs.put(key, values); + } + + public static Set get(String key) { + return configs.get(key); + } + + public static void clear() { + configs.clear(); + } } diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/DatasourceConfig.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/DatasourceConfig.java new file mode 100644 index 00000000..cf2ec471 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/DatasourceConfig.java @@ -0,0 +1,75 @@ +package com.alibaba.otter.canal.client.adapter.support; + +import com.alibaba.druid.pool.DruidDataSource; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class DatasourceConfig { + + public final static Map DATA_SOURCES = new ConcurrentHashMap<>(); + + private String driver = "com.mysql.jdbc.Driver"; + private String url; + private String database; + private String type = "mysql"; + private String username; + private String password; + private Integer maxActive = 3; + + public String getDriver() { + return driver; + } + + public void setDriver(String driver) { + this.driver = driver; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getDatabase() { + return database; + } + + public void setDatabase(String database) { + this.database = database; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public Integer getMaxActive() { + return maxActive; + } + + public void setMaxActive(Integer maxActive) { + this.maxActive = maxActive; + } +} diff --git a/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/AbstractCanalAdapterWorker.java b/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/AbstractCanalAdapterWorker.java index c1f55cd9..cd3f718d 100644 --- a/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/AbstractCanalAdapterWorker.java +++ b/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/AbstractCanalAdapterWorker.java @@ -10,7 +10,7 @@ import java.util.concurrent.Future; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter; +import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.adapter.support.Dml; import com.alibaba.otter.canal.client.adapter.support.MessageUtil; import com.alibaba.otter.canal.protocol.FlatMessage; @@ -26,10 +26,10 @@ public abstract class AbstractCanalAdapterWorker { protected final Logger logger = LoggerFactory.getLogger(this.getClass()); - protected String canalDestination; // canal实例 - protected List> canalOuterAdapters; // 外部适配器 - protected ExecutorService groupInnerExecutorService; // 组内工作线程池 - protected volatile boolean running = false; // 是否运行中 + protected String canalDestination; // canal实例 + protected List> canalOuterAdapters; // 外部适配器 + protected ExecutorService groupInnerExecutorService; // 组内工作线程池 + protected volatile boolean running = false; // 是否运行中 protected Thread thread = null; protected Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() { @@ -42,21 +42,21 @@ public abstract class AbstractCanalAdapterWorker { protected void writeOut(final Message message) { List> futures = new ArrayList<>(); // 组间适配器并行运行 - for (List outerAdapters : canalOuterAdapters) { - final List adapters = outerAdapters; + for (List outerAdapters : canalOuterAdapters) { + final List adapters = outerAdapters; futures.add(groupInnerExecutorService.submit(new Callable() { @Override public Boolean call() { try { // 组内适配器穿行运行，尽量不要配置组内适配器 - for (final CanalOuterAdapter c : adapters) { + for (final OuterAdapter c : adapters) { long begin = System.currentTimeMillis(); MessageUtil.parse4Dml(message, new MessageUtil.Consumer() { @Override public void accept(Dml dml) { - c.writeOut(dml); + c.sync(dml); } }); @@ -90,18 +90,18 @@ public abstract class AbstractCanalAdapterWorker { protected void writeOut(final FlatMessage flatMessage) { List> futures = new ArrayList<>(); // 组间适配器并行运行 - for (List outerAdapters : canalOuterAdapters) { - final List adapters = outerAdapters; + for (List outerAdapters : canalOuterAdapters) { + final List adapters = outerAdapters; futures.add(groupInnerExecutorService.submit(new Callable() { @Override public Boolean call() { try { // 组内适配器穿行运行，尽量不要配置组内适配器 - for (CanalOuterAdapter c : adapters) { + for (OuterAdapter c : adapters) { long begin = System.currentTimeMillis(); Dml dml = MessageUtil.flatMessage2Dml(flatMessage); - c.writeOut(dml); + c.sync(dml); if (logger.isDebugEnabled()) { logger.debug("{} elapsed time: {}", c.getClass().getName(), @@ -154,8 +154,8 @@ public abstract class AbstractCanalAdapterWorker { } groupInnerExecutorService.shutdown(); logger.info("topic connectors' worker thread dead!"); - for (List outerAdapters : canalOuterAdapters) { - for (CanalOuterAdapter adapter : outerAdapters) { + for (List outerAdapters : canalOuterAdapters) { + for (OuterAdapter adapter : outerAdapters) { adapter.destroy(); } } diff --git a/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterKafkaWorker.java b/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterKafkaWorker.java index 4429ca80..b2d3e583 100644 --- a/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterKafkaWorker.java +++ b/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterKafkaWorker.java @@ -4,12 +4,11 @@ import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import org.apache.kafka.clients.consumer.CommitFailedException; import org.apache.kafka.common.errors.WakeupException; -import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter; +import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector; import com.alibaba.otter.canal.client.kafka.KafkaCanalConnectors; import com.alibaba.otter.canal.protocol.FlatMessage; @@ -30,7 +29,7 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker { private boolean flatMessage; public CanalAdapterKafkaWorker(String bootstrapServers, String topic, String groupId, - List> canalOuterAdapters, boolean flatMessage){ + List> canalOuterAdapters, boolean flatMessage){ this.canalOuterAdapters = canalOuterAdapters; this.groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size()); this.topic = topic; @@ -81,8 +80,8 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker { } groupInnerExecutorService.shutdown(); logger.info("topic {} connectors' worker thread dead!", this.topic); - for (List outerAdapters : canalOuterAdapters) { - for (CanalOuterAdapter adapter : outerAdapters) { + for (List outerAdapters : canalOuterAdapters) { + for (OuterAdapter adapter : outerAdapters) { adapter.destroy(); } } diff --git a/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterLoader.java b/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterLoader.java index 471b6a3d..05ae6d2c 100644 --- a/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterLoader.java +++ b/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterLoader.java @@ -13,7 +13,7 @@ import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter; +import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; import com.alibaba.otter.canal.client.adapter.support.CanalOuterAdapterConfiguration; import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader; @@ -33,7 +33,7 @@ public class CanalAdapterLoader { private Map canalMQWorker = new HashMap<>(); - private ExtensionLoader loader; + private ExtensionLoader loader; public CanalAdapterLoader(CanalClientConfig canalClientConfig){ this.canalClientConfig = canalClientConfig; @@ -48,7 +48,7 @@ public class CanalAdapterLoader { throw new RuntimeException("Blank config property: canalInstances or canalMQTopics"); } - loader = ExtensionLoader.getExtensionLoader(CanalOuterAdapter.class, "" /* + loader = ExtensionLoader.getExtensionLoader(OuterAdapter.class, "" /* * TODO * canalClientConfig * . @@ -68,10 +68,10 @@ public class CanalAdapterLoader { // 初始化canal-client的适配器 if (canalClientConfig.getCanalInstances() != null) { for (CanalClientConfig.CanalInstance instance : canalClientConfig.getCanalInstances()) { - List> canalOuterAdapterGroups = new ArrayList<>(); + List> canalOuterAdapterGroups = new ArrayList<>(); for (CanalClientConfig.AdapterGroup connectorGroup : instance.getAdapterGroups()) { - List canalOutConnectors = new ArrayList<>(); + List canalOutConnectors = new ArrayList<>(); for (CanalOuterAdapterConfiguration c : connectorGroup.getOutAdapters()) { loadConnector(c, canalOutConnectors); } @@ -93,9 +93,9 @@ public class CanalAdapterLoader { if (canalClientConfig.getMqTopics() != null) { for (CanalClientConfig.MQTopic topic : canalClientConfig.getMqTopics()) { for (CanalClientConfig.Group group : topic.getGroups()) { - List> canalOuterAdapterGroups = new ArrayList<>(); + List> canalOuterAdapterGroups = new ArrayList<>(); - List canalOuterAdapters = new ArrayList<>(); + List canalOuterAdapters = new ArrayList<>(); for (CanalOuterAdapterConfiguration config : group.getOutAdapters()) { loadConnector(config, canalOuterAdapters); @@ -125,9 +125,9 @@ public class CanalAdapterLoader { } } - private void loadConnector(CanalOuterAdapterConfiguration config, List canalOutConnectors) { + private void loadConnector(CanalOuterAdapterConfiguration config, List canalOutConnectors) { try { - CanalOuterAdapter adapter = loader.getExtension(config.getName()); + OuterAdapter adapter = loader.getExtension(config.getName()); ClassLoader cl = Thread.currentThread().getContextClassLoader(); // 替换ClassLoader Thread.currentThread().setContextClassLoader(adapter.getClass().getClassLoader()); diff --git a/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterRocketMQWorker.java b/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterRocketMQWorker.java index f0f674db..9c126297 100644 --- a/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterRocketMQWorker.java +++ b/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterRocketMQWorker.java @@ -8,7 +8,7 @@ import java.util.concurrent.TimeUnit; import org.apache.kafka.clients.consumer.CommitFailedException; import org.apache.kafka.common.errors.WakeupException; -import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter; +import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnector; import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnectorProvider; import com.alibaba.otter.canal.protocol.Message; @@ -26,7 +26,7 @@ public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker { private String topic; public CanalAdapterRocketMQWorker(String nameServers, String topic, String groupId, - List> canalOuterAdapters){ + List> canalOuterAdapters){ logger.info("RocketMQ consumer config topic:{}, nameServer:{}, groupId:{}", topic, nameServers, groupId); this.canalOuterAdapters = canalOuterAdapters; this.groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size()); diff --git a/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterWorker.java b/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterWorker.java index e341c6a3..e72ed2d5 100644 --- a/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterWorker.java +++ b/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterWorker.java @@ -6,7 +6,7 @@ import java.util.concurrent.Executors; import com.alibaba.otter.canal.client.CanalConnector; import com.alibaba.otter.canal.client.CanalConnectors; -import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter; +import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.impl.ClusterCanalConnector; import com.alibaba.otter.canal.protocol.Message; @@ -31,7 +31,7 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker { * @param canalOuterAdapters 外部适配器组 */ public CanalAdapterWorker(String canalDestination, SocketAddress address, - List> canalOuterAdapters){ + List> canalOuterAdapters){ this.canalOuterAdapters = canalOuterAdapters; this.canalDestination = canalDestination; groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size()); @@ -46,7 +46,7 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker { * @param canalOuterAdapters 外部适配器组 */ public CanalAdapterWorker(String canalDestination, String zookeeperHosts, - List> canalOuterAdapters){ + List> canalOuterAdapters){ this.canalOuterAdapters = canalOuterAdapters; this.canalDestination = canalDestination; groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size()); @@ -96,8 +96,8 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker { } groupInnerExecutorService.shutdown(); logger.info("destination {} adapters' worker thread dead!", canalDestination); - for (List outerAdapters : canalOuterAdapters) { - for (CanalOuterAdapter adapter : outerAdapters) { + for (List outerAdapters : canalOuterAdapters) { + for (OuterAdapter adapter : outerAdapters) { adapter.destroy(); } } diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java index d75cea67..c9b67998 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java @@ -10,7 +10,7 @@ import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; -import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter; +import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig; import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfigLoader; import com.alibaba.otter.canal.client.adapter.hbase.service.HbaseSyncService; @@ -25,7 +25,7 @@ import com.alibaba.otter.canal.client.adapter.support.SPI; * @version 1.0.0 */ @SPI("hbase") -public class HbaseAdapter implements CanalOuterAdapter { +public class HbaseAdapter implements OuterAdapter { private static volatile Map mappingConfigCache = null; @@ -83,7 +83,7 @@ public class HbaseAdapter implements CanalOuterAdapter { } @Override - public void writeOut(Dml dml) { + public void sync(Dml dml) { if (dml == null) { return; } diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java index 6d70bdde..c3c30c1b 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java @@ -13,9 +13,18 @@ import java.util.Set; * @version 1.0.0 */ public class MappingConfig { + private String dataSourceKey; private HbaseOrm hbaseOrm; + public String getDataSourceKey() { + return dataSourceKey; + } + + public void setDataSourceKey(String dataSourceKey) { + this.dataSourceKey = dataSourceKey; + } + public HbaseOrm getHbaseOrm() { return hbaseOrm; } diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java index 5922d63e..173fc023 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java @@ -37,7 +37,7 @@ public class MappingConfigLoader { Map result = new LinkedHashMap<>(); - Collection configs = AdapterConfigs.configs.get("hbase"); + Collection configs = AdapterConfigs.get("hbase"); for (String c : configs) { if (c == null) { continue; diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java new file mode 100644 index 00000000..cccd2db6 --- /dev/null +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java @@ -0,0 +1,4 @@ +package com.alibaba.otter.canal.client.adapter.hbase.service; + +public class HbaseEtlService { +} diff --git a/client-adapter/hbase/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.CanalOuterAdapter b/client-adapter/hbase/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter similarity index 100% rename from client-adapter/hbase/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.CanalOuterAdapter rename to client-adapter/hbase/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter diff --git a/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml b/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml index 575a1bf5..f2b6a594 100644 --- a/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml +++ b/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml @@ -1,8 +1,9 @@ +dataSourceKey: defaultDS hbaseOrm: mode: PHOENIX #NATIVE #STRING database: mytest # 数据库名 table: person2 # 数据库表名 - hbaseTable: MYTEST.PERSON2 # HBase表名 + hbaseTable: MYTEST_PERSON2 # HBase表名 family: CF # 默认统一Family名称 uppercaseQualifier: true # 字段名转大写, 默认为true commitBatch: 3000 # 批量提交的大小 diff --git a/client-adapter/launcher/pom.xml b/client-adapter/launcher/pom.xml index 7b94bfc9..8b91f27f 100644 --- a/client-adapter/launcher/pom.xml +++ b/client-adapter/launcher/pom.xml @@ -81,6 +81,7 @@ + canal-client-launcher org.springframework.boot diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/CanalAdapterApplication.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/CanalAdapterApplication.java index 4b0035d8..56bbde8e 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/CanalAdapterApplication.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/CanalAdapterApplication.java @@ -1,56 +1,12 @@ package com.alibaba.otter.canal.adapter.launcher; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import javax.annotation.Resource; - -import com.alibaba.otter.canal.adapter.launcher.loader.CanalAdapterLoader; -import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.builder.SpringApplicationBuilder; - @SpringBootApplication public class CanalAdapterApplication { - private static final Logger logger = LoggerFactory.getLogger(CanalAdapterApplication.class); - - private static CanalAdapterLoader adapterLoader; public static void main(String[] args) { new SpringApplicationBuilder(CanalAdapterApplication.class).run(args); } - - @Resource - private CanalClientConfig canalClientConf; - - @PostConstruct - public void init() { - if (adapterLoader == null) { - try { - logger.info("## start the canal client adapters."); - adapterLoader = new CanalAdapterLoader(canalClientConf); - adapterLoader.init(); - logger.info("## the canal client adapters are running now ......"); - } catch (Throwable e) { - logger.error("## something goes wrong when starting up the canal client adapters:", e); - System.exit(0); - } - } - } - - @PreDestroy - public void destroy() { - try { - logger.info("## stop the canal client adapters"); - if (adapterLoader != null) { - adapterLoader.destroy(); - } - } catch (Throwable e) { - logger.warn("## something goes wrong when stopping canal client adapters:", e); - } finally { - logger.info("## canal client adapters are down."); - } - } } diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java index 5b232c36..447f896d 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java @@ -4,7 +4,7 @@ import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; -@ConfigurationProperties(prefix = "canal.conf") @Component +@ConfigurationProperties(prefix = "canal.conf") public class AdapterCanalConfig extends CanalClientConfig { } diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfig.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfig.java new file mode 100644 index 00000000..06cb1730 --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfig.java @@ -0,0 +1,76 @@ +package com.alibaba.otter.canal.adapter.launcher.config; + +import java.sql.SQLException; +import java.util.List; +import java.util.Map; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; + +@Component +@ConfigurationProperties(prefix = "adapter.conf") +public class AdapterConfig { + + private Map datasourceConfigs; + + private List adapterConfigs; + + public List getAdapterConfigs() { + return adapterConfigs; + } + + public Map getDatasourceConfigs() { + return datasourceConfigs; + } + + public void setDatasourceConfigs(Map datasourceConfigs) { + this.datasourceConfigs = datasourceConfigs; + + if (datasourceConfigs != null) { + for (Map.Entry entry : datasourceConfigs.entrySet()) { + DatasourceConfig datasourceConfig = entry.getValue(); + // 加载数据源连接池 + DruidDataSource ds = new DruidDataSource(); + ds.setDriverClassName(datasourceConfig.getDriver()); + ds.setUrl(datasourceConfig.getUrl()); + ds.setUsername(datasourceConfig.getUsername()); + ds.setPassword(datasourceConfig.getPassword()); + ds.setInitialSize(1); + ds.setMinIdle(1); + ds.setMaxActive(datasourceConfig.getMaxActive()); + ds.setMaxWait(60000); + ds.setTimeBetweenEvictionRunsMillis(60000); + ds.setMinEvictableIdleTimeMillis(300000); + ds.setPoolPreparedStatements(false); + ds.setMaxPoolPreparedStatementPerConnectionSize(20); + ds.setValidationQuery("select 1"); + try { + ds.init(); + } catch (SQLException e) { + throw new RuntimeException(e); + } + DatasourceConfig.DATA_SOURCES.put(entry.getKey(), ds); + } + } + } + + public void setAdapterConfigs(List adapterConfigs) { + this.adapterConfigs = adapterConfigs; + + if (adapterConfigs != null) { + AdapterConfigs.clear(); + for (String adapterConfig : adapterConfigs) { + int idx = adapterConfig.indexOf("/"); + if (idx > -1) { + String type = adapterConfig.substring(0, idx); + String ymlFile = adapterConfig.substring(idx + 1); + AdapterConfigs.put(type, ymlFile); + } + } + } + } +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfigPath.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfigPath.java deleted file mode 100644 index 474d43af..00000000 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfigPath.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.alibaba.otter.canal.adapter.launcher.config; - -import java.util.List; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs; - -@Component -@ConfigurationProperties(prefix = "adapter.conf.path") -public class AdapterConfigPath { - - private List adapterConfigs; - - public List getAdapterConfigs() { - return adapterConfigs; - } - - public void setAdapterConfigs(List adapterConfigs) { - this.adapterConfigs = adapterConfigs; - - if (adapterConfigs != null) { - AdapterConfigs.configs.clear(); - for (String adapterConfig : adapterConfigs) { - int idx = adapterConfig.indexOf("/"); - if (idx > -1) { - String type = adapterConfig.substring(0, idx); - String ymlFile = adapterConfig.substring(idx + 1); - AdapterConfigs.configs.put(type, ymlFile); - } - } - } - } -} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java index e0912bab..694251ac 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java @@ -10,7 +10,7 @@ import java.util.concurrent.Future; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter; +import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.adapter.support.MessageUtil; import com.alibaba.otter.canal.protocol.Message; @@ -25,43 +25,33 @@ public abstract class AbstractCanalAdapterWorker { protected final Logger logger = LoggerFactory.getLogger(this.getClass()); protected String canalDestination; // canal实例 - protected List> canalOuterAdapters; // 外部适配器 + protected List> canalOuterAdapters; // 外部适配器 protected ExecutorService groupInnerExecutorService; // 组内工作线程池 protected volatile boolean running = false; // 是否运行中 protected Thread thread = null; - protected Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() { - - @Override - public void uncaughtException(Thread t, Throwable e) { - logger.error("parse events has an error", e); - } - }; + protected Thread.UncaughtExceptionHandler handler = (t, e) -> logger.error("parse events has an error", e); protected void writeOut(final Message message) { List> futures = new ArrayList<>(); // 组间适配器并行运行 - for (List outerAdapters : canalOuterAdapters) { - final List adapters = outerAdapters; - futures.add(groupInnerExecutorService.submit(new Callable() { + for (List outerAdapters : canalOuterAdapters) { + final List adapters = outerAdapters; + futures.add(groupInnerExecutorService.submit(() -> { + try { + // 组内适配器穿行运行，尽量不要配置组内适配器 + for (final OuterAdapter c : adapters) { + long begin = System.currentTimeMillis(); + MessageUtil.parse4Dml(message, c::sync); - @Override - public Boolean call() { - try { - // 组内适配器穿行运行，尽量不要配置组内适配器 - for (final CanalOuterAdapter c : adapters) { - long begin = System.currentTimeMillis(); - MessageUtil.parse4Dml(message, c::writeOut); - - if (logger.isDebugEnabled()) { - logger.debug("{} elapsed time: {}", - c.getClass().getName(), - (System.currentTimeMillis() - begin)); - } + if (logger.isDebugEnabled()) { + logger.debug("{} elapsed time: {}", + c.getClass().getName(), + (System.currentTimeMillis() - begin)); } - return true; - } catch (Exception e) { - return false; } + return true; + } catch (Exception e) { + return false; } })); @@ -131,9 +121,6 @@ public abstract class AbstractCanalAdapterWorker { if ((System.currentTimeMillis() - begin) > 5 * 60 * 1000) { logger.error("topic: {} batchId {} elapsed time: {} ms", topic, message.getId(), now - begin); } - if (logger.isDebugEnabled()) { - logger.debug("topic: {} batchId {} elapsed time: {} ms", topic, message.getId(), now - begin); - } } protected void stopOutAdapters() { @@ -146,8 +133,8 @@ public abstract class AbstractCanalAdapterWorker { } groupInnerExecutorService.shutdown(); logger.info("topic connectors' worker thread dead!"); - for (List outerAdapters : canalOuterAdapters) { - for (CanalOuterAdapter adapter : outerAdapters) { + for (List outerAdapters : canalOuterAdapters) { + for (OuterAdapter adapter : outerAdapters) { adapter.destroy(); } } diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java index 71c9d130..6e7d2bb8 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java @@ -9,7 +9,7 @@ import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter; +import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.adapter.support.CanalOuterAdapterConfiguration; import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader; import org.slf4j.Logger; @@ -24,15 +24,15 @@ import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; */ public class CanalAdapterLoader { - private static final Logger logger = LoggerFactory.getLogger(CanalAdapterLoader.class); + private static final Logger logger = LoggerFactory.getLogger(CanalAdapterLoader.class); - private CanalClientConfig canalClientConfig; + private CanalClientConfig canalClientConfig; - private Map canalWorkers = new HashMap<>(); + private Map canalWorkers = new HashMap<>(); - private ExtensionLoader loader; + private ExtensionLoader loader; - public CanalAdapterLoader(CanalClientConfig canalClientConfig) { + public CanalAdapterLoader(CanalClientConfig canalClientConfig){ this.canalClientConfig = canalClientConfig; } @@ -40,12 +40,7 @@ public class CanalAdapterLoader { * 初始化canal-client */ public void init() { - // canal instances 和 mq topics 配置不能同时为空 - // if (canalClientConfig.getCanalInstances() == null && canalClientConfig.getMqTopics() == null) { - // throw new RuntimeException("Blank config property: canalInstances or canalMQTopics"); - // } - - loader = ExtensionLoader.getExtensionLoader(CanalOuterAdapter.class); + loader = ExtensionLoader.getExtensionLoader(OuterAdapter.class); String canalServerHost = this.canalClientConfig.getCanalServerHost(); SocketAddress sa = null; @@ -58,10 +53,10 @@ public class CanalAdapterLoader { // 初始化canal-client的适配器 if (canalClientConfig.getCanalInstances() != null) { for (CanalClientConfig.CanalInstance instance : canalClientConfig.getCanalInstances()) { - List> canalOuterAdapterGroups = new ArrayList<>(); + List> canalOuterAdapterGroups = new ArrayList<>(); for (CanalClientConfig.AdapterGroup connectorGroup : instance.getAdapterGroups()) { - List canalOutConnectors = new ArrayList<>(); + List canalOutConnectors = new ArrayList<>(); for (CanalOuterAdapterConfiguration c : connectorGroup.getOutAdapters()) { loadConnector(c, canalOutConnectors); } @@ -80,9 +75,9 @@ public class CanalAdapterLoader { } } - private void loadConnector(CanalOuterAdapterConfiguration config, List canalOutConnectors) { + private void loadConnector(CanalOuterAdapterConfiguration config, List canalOutConnectors) { try { - CanalOuterAdapter adapter = loader.getExtension(config.getName()); + OuterAdapter adapter = loader.getExtension(config.getName()); ClassLoader cl = Thread.currentThread().getContextClassLoader(); // 替换ClassLoader Thread.currentThread().setContextClassLoader(adapter.getClass().getClassLoader()); diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterService.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterService.java new file mode 100644 index 00000000..f7b9e445 --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterService.java @@ -0,0 +1,65 @@ +package com.alibaba.otter.canal.adapter.launcher.loader; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import javax.annotation.Resource; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.otter.canal.adapter.launcher.config.AdapterCanalConfig; +import com.alibaba.otter.canal.adapter.launcher.config.AdapterConfig; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; + +@Component +public class CanalAdapterService { + + private static final Logger logger = LoggerFactory.getLogger(CanalAdapterService.class); + + private static CanalAdapterLoader adapterLoader; + + @Resource + private AdapterCanalConfig adapterCanalConfig; + + // 注入配置保证配置优先先注册 + @Resource + private AdapterConfig adapterConfig; + + @PostConstruct + public void init() { + if (adapterLoader == null) { + try { + logger.info("## start the canal client adapters."); + adapterLoader = new CanalAdapterLoader(adapterCanalConfig); + adapterLoader.init(); + logger.info("## the canal client adapters are running now ......"); + } catch (Throwable e) { + logger.error("## something goes wrong when starting up the canal client adapters:", e); + System.exit(0); + } + } + } + + @PreDestroy + public void destroy() { + try { + logger.info("## stop the canal client adapters"); + if (adapterLoader != null) { + adapterLoader.destroy(); + } + for (DruidDataSource druidDataSource : DatasourceConfig.DATA_SOURCES.values()) { + try { + druidDataSource.close(); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + } catch (Throwable e) { + logger.warn("## something goes wrong when stopping canal client adapters:", e); + } finally { + logger.info("## canal client adapters are down."); + } + } +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java index 7e090c2d..1c38b353 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java @@ -6,7 +6,7 @@ import java.util.concurrent.Executors; import com.alibaba.otter.canal.client.CanalConnector; import com.alibaba.otter.canal.client.CanalConnectors; -import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter; +import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.impl.ClusterCanalConnector; import com.alibaba.otter.canal.protocol.Message; @@ -31,7 +31,7 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker { * @param canalOuterAdapters 外部适配器组 */ public CanalAdapterWorker(String canalDestination, SocketAddress address, - List> canalOuterAdapters){ + List> canalOuterAdapters){ this.canalOuterAdapters = canalOuterAdapters; this.canalDestination = canalDestination; groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size()); @@ -46,7 +46,7 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker { * @param canalOuterAdapters 外部适配器组 */ public CanalAdapterWorker(String canalDestination, String zookeeperHosts, - List> canalOuterAdapters){ + List> canalOuterAdapters){ this.canalOuterAdapters = canalOuterAdapters; this.canalDestination = canalDestination; groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size()); @@ -96,8 +96,8 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker { } groupInnerExecutorService.shutdown(); logger.info("destination {} adapters' worker thread dead!", canalDestination); - for (List outerAdapters : canalOuterAdapters) { - for (CanalOuterAdapter adapter : outerAdapters) { + for (List outerAdapters : canalOuterAdapters) { + for (OuterAdapter adapter : outerAdapters) { adapter.destroy(); } } diff --git a/client-adapter/launcher/src/main/resources/application.yml b/client-adapter/launcher/src/main/resources/application.yml index bb06e75b..7adef833 100644 --- a/client-adapter/launcher/src/main/resources/application.yml +++ b/client-adapter/launcher/src/main/resources/application.yml @@ -16,6 +16,11 @@ canal.conf: # hosts: slave1:2181 # properties: {znodeParent: "/hbase-unsecure"} -adapter.conf.path: +adapter.conf: + datasourceConfigs: + defaultDS: + url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true + username: root + password: 121212 adapterConfigs: - hbase/mytest_person2.yml \ No newline at end of file diff --git a/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java b/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java index 1885d6c2..27d99aae 100644 --- a/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java +++ b/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java @@ -3,7 +3,7 @@ package com.alibaba.otter.canal.client.adapter.logger; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter; +import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.adapter.support.CanalOuterAdapterConfiguration; import com.alibaba.otter.canal.client.adapter.support.Dml; import com.alibaba.otter.canal.client.adapter.support.SPI; @@ -16,7 +16,7 @@ import com.alibaba.otter.canal.client.adapter.support.SPI; */ @SPI("logger") // logger参数对应CanalOuterAdapterConfiguration配置中的name -public class LoggerAdapterExample implements CanalOuterAdapter { +public class LoggerAdapterExample implements OuterAdapter { private Logger logger = LoggerFactory.getLogger(this.getClass()); @@ -26,7 +26,7 @@ public class LoggerAdapterExample implements CanalOuterAdapter { } @Override - public void writeOut(Dml dml) { + public void sync(Dml dml) { logger.info(dml.toString()); } diff --git a/client-adapter/logger/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.CanalOuterAdapter b/client-adapter/logger/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter similarity index 100% rename from client-adapter/logger/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.CanalOuterAdapter rename to client-adapter/logger/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter From 575dd59c7893e6a41f8b0da6b9f5a107580d40d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Wed, 24 Oct 2018 16:25:28 +0800 Subject: [PATCH 09/35] fixed mysql set(8 bit) type parse --- .../java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java index c1b96c3c..9fe22820 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java @@ -859,7 +859,7 @@ public final class RowsLogBuffer { // 转化为unsign long switch (len) { case 1: - value = buffer.getInt8(); + value = buffer.getUint8(); break; case 2: value = buffer.getUint16(); From 92708346310f679db608a2f64bdde8b22b7d2468 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Wed, 24 Oct 2018 16:46:26 +0800 Subject: [PATCH 10/35] fixed rds ak/sk config for manager --- .../manager/CanalInstanceWithManager.java | 56 +++++++++++++++++-- .../manager/model/CanalParameter.java | 36 ++++++++++++ .../alibaba/otter/canal/protocol/Message.java | 1 + 3 files changed, 89 insertions(+), 4 deletions(-) diff --git a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java index 778a40aa..36774d94 100644 --- a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java +++ b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java @@ -6,7 +6,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import com.alibaba.otter.canal.meta.FileMixedMetaManager; +import org.apache.commons.lang.BooleanUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,7 +39,16 @@ import com.alibaba.otter.canal.parse.inbound.AbstractEventParser; import com.alibaba.otter.canal.parse.inbound.group.GroupEventParser; import com.alibaba.otter.canal.parse.inbound.mysql.LocalBinlogEventParser; import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser; -import com.alibaba.otter.canal.parse.index.*; +import com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsBinlogEventParserProxy; +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DefaultTableMetaTSDBFactory; +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaTSDB; +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaTSDBBuilder; +import com.alibaba.otter.canal.parse.index.CanalLogPositionManager; +import com.alibaba.otter.canal.parse.index.FailbackLogPositionManager; +import com.alibaba.otter.canal.parse.index.MemoryLogPositionManager; +import com.alibaba.otter.canal.parse.index.MetaLogPositionManager; +import com.alibaba.otter.canal.parse.index.PeriodMixedLogPositionManager; +import com.alibaba.otter.canal.parse.index.ZooKeeperLogPositionManager; import com.alibaba.otter.canal.parse.support.AuthenticationInfo; import com.alibaba.otter.canal.protocol.position.EntryPosition; import com.alibaba.otter.canal.sink.entry.EntryEventSink; @@ -229,7 +238,18 @@ public class CanalInstanceWithManager extends AbstractCanalInstance { private CanalEventParser doInitEventParser(SourcingType type, List dbAddresses) { CanalEventParser eventParser; if (type.isMysql()) { - MysqlEventParser mysqlEventParser = new MysqlEventParser(); + MysqlEventParser mysqlEventParser = null; + if (StringUtils.isNotEmpty(parameters.getRdsAccesskey()) + && StringUtils.isNotEmpty(parameters.getRdsSecretkey()) + && StringUtils.isNotEmpty(parameters.getRdsInstanceId())) { + + mysqlEventParser = new RdsBinlogEventParserProxy(); + ((RdsBinlogEventParserProxy) mysqlEventParser).setAccesskey(parameters.getRdsAccesskey()); + ((RdsBinlogEventParserProxy) mysqlEventParser).setSecretkey(parameters.getRdsSecretkey()); + ((RdsBinlogEventParserProxy) mysqlEventParser).setInstanceId(parameters.getRdsInstanceId()); + } else { + mysqlEventParser = new MysqlEventParser(); + } mysqlEventParser.setDestination(destination); // 编码参数 mysqlEventParser.setConnectionCharset(Charset.forName(parameters.getConnectionCharset())); @@ -273,6 +293,34 @@ public class CanalInstanceWithManager extends AbstractCanalInstance { mysqlEventParser.setFallbackIntervalInSeconds(parameters.getFallbackIntervalInSeconds()); mysqlEventParser.setProfilingEnabled(false); mysqlEventParser.setFilterTableError(parameters.getFilterTableError()); + mysqlEventParser.setIsGTIDMode(BooleanUtils.toBoolean(parameters.getGtidEnable())); + // tsdb + boolean tsdbEnable = BooleanUtils.toBoolean(parameters.getTsdbEnable()); + if (tsdbEnable) { + mysqlEventParser.setEnableTsdb(tsdbEnable); + mysqlEventParser.setTableMetaTSDBFactory(new DefaultTableMetaTSDBFactory() { + + @Override + public void destory(String destination) { + TableMetaTSDBBuilder.destory(destination); + } + + @Override + public TableMetaTSDB build(String destination, String springXml) { + try { + System.setProperty("canal.instance.tsdb.url", parameters.getTsdbJdbcUrl()); + System.setProperty("canal.instance.tsdb.dbUsername", parameters.getTsdbJdbcUserName()); + System.setProperty("canal.instance.tsdb.dbPassword", parameters.getTsdbJdbcPassword()); + + return TableMetaTSDBBuilder.build(destination, "classpath:spring/tsdb/mysql-tsdb.xml"); + } finally { + System.setProperty("canal.instance.tsdb.url", ""); + System.setProperty("canal.instance.tsdb.dbUsername", ""); + System.setProperty("canal.instance.tsdb.dbPassword", ""); + } + } + }); + } eventParser = mysqlEventParser; } else if (type.isLocalBinlog()) { LocalBinlogEventParser localBinlogEventParser = new LocalBinlogEventParser(); @@ -291,8 +339,8 @@ public class CanalInstanceWithManager extends AbstractCanalInstance { parameters.getDbUsername(), parameters.getDbPassword(), parameters.getDefaultDatabaseName())); - } + eventParser = localBinlogEventParser; } else if (type.isOracle()) { throw new CanalException("unsupport SourcingType for " + type); diff --git a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java index 40df1485..ece1196e 100644 --- a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java +++ b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java @@ -97,6 +97,10 @@ public class CanalParameter implements Serializable { private String tsdbJdbcUrl; private String tsdbJdbcUserName; private String tsdbJdbcPassword; + private String rdsAccesskey; + private String rdsSecretkey; + private String rdsInstanceId; + private Boolean gtidEnable = Boolean.FALSE; // 是否开启gtid // ================================== 兼容字段处理 private InetSocketAddress masterAddress; // 主库信息 private String masterUsername; // 帐号 @@ -919,6 +923,38 @@ public class CanalParameter implements Serializable { this.tsdbJdbcPassword = tsdbJdbcPassword; } + public String getRdsAccesskey() { + return rdsAccesskey; + } + + public void setRdsAccesskey(String rdsAccesskey) { + this.rdsAccesskey = rdsAccesskey; + } + + public String getRdsSecretkey() { + return rdsSecretkey; + } + + public void setRdsSecretkey(String rdsSecretkey) { + this.rdsSecretkey = rdsSecretkey; + } + + public String getRdsInstanceId() { + return rdsInstanceId; + } + + public void setRdsInstanceId(String rdsInstanceId) { + this.rdsInstanceId = rdsInstanceId; + } + + public Boolean getGtidEnable() { + return gtidEnable; + } + + public void setGtidEnable(Boolean gtidEnable) { + this.gtidEnable = gtidEnable; + } + public String toString() { return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE); } diff --git a/protocol/src/main/java/com/alibaba/otter/canal/protocol/Message.java b/protocol/src/main/java/com/alibaba/otter/canal/protocol/Message.java index 751e09d0..bf79ee45 100644 --- a/protocol/src/main/java/com/alibaba/otter/canal/protocol/Message.java +++ b/protocol/src/main/java/com/alibaba/otter/canal/protocol/Message.java @@ -38,6 +38,7 @@ public class Message implements Serializable { } else { this.entries = entries == null ? new ArrayList() : entries; } + this.raw = raw; } public Message(long id){ From b5dac60cbb192ff2f9722b4ff3a2c1e4ac103326 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Wed, 24 Oct 2018 17:00:56 +0800 Subject: [PATCH 11/35] fixed cliet-adapter artifactId --- client-adapter/example/pom.xml | 4 ++-- client-adapter/launcher/pom.xml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/client-adapter/example/pom.xml b/client-adapter/example/pom.xml index 3c68087a..606534e0 100644 --- a/client-adapter/example/pom.xml +++ b/client-adapter/example/pom.xml @@ -9,7 +9,7 @@ 4.0.0 com.alibaba.otter - example + client-adapter.example jar canal client adapter example module for otter ${project.version} @@ -145,4 +145,4 @@ - \ No newline at end of file + diff --git a/client-adapter/launcher/pom.xml b/client-adapter/launcher/pom.xml index 7b94bfc9..76480a70 100644 --- a/client-adapter/launcher/pom.xml +++ b/client-adapter/launcher/pom.xml @@ -9,7 +9,7 @@ 4.0.0 com.alibaba.otter - launcher + client-adapter.launcher jar canal client adapter launcher module for otter ${project.version} @@ -134,4 +134,4 @@ - \ No newline at end of file + From 0a7251add08f927f76bc5170258c0662a9c38c82 Mon Sep 17 00:00:00 2001 From: mcy Date: Wed, 24 Oct 2018 18:30:42 +0800 Subject: [PATCH 12/35] =?UTF-8?q?HBase=E7=9A=84etl=E5=88=9D=E6=AD=A5?= =?UTF-8?q?=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../canal/client/adapter/OuterAdapter.java | 13 +- .../adapter/support/CanalClientConfig.java | 12 +- .../client/adapter/support/JdbcTypeUtil.java | 54 ++- ...iguration.java => OuterAdapterConfig.java} | 16 +- .../adapter/loader/CanalAdapterLoader.java | 8 +- .../client/adapter/hbase/HbaseAdapter.java | 59 ++- .../hbase/service/HbaseEtlService.java | 347 ++++++++++++++++++ .../hbase/service/HbaseSyncService.java | 21 +- client-adapter/launcher/pom.xml | 12 + .../launcher/config/AdapterConfig.java | 6 +- .../loader/AbstractCanalAdapterWorker.java | 92 ++--- .../loader/CanalAdapterKafkaWorker.java | 148 ++++++++ .../launcher/loader/CanalAdapterLoader.java | 66 +++- .../loader/CanalAdapterRocketMQWorker.java | 124 +++++++ .../launcher/loader/CanalAdapterWorker.java | 8 +- .../src/main/resources/application.yml | 28 +- .../adapter/logger/LoggerAdapterExample.java | 4 +- 17 files changed, 869 insertions(+), 149 deletions(-) rename client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/{CanalOuterAdapterConfiguration.java => OuterAdapterConfig.java} (51%) create mode 100644 client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java create mode 100644 client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java index c4ee60fc..a508d1ef 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java @@ -1,8 +1,8 @@ package com.alibaba.otter.canal.client.adapter; -import java.util.Map; +import java.util.List; -import com.alibaba.otter.canal.client.adapter.support.CanalOuterAdapterConfiguration; +import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; import com.alibaba.otter.canal.client.adapter.support.Dml; import com.alibaba.otter.canal.client.adapter.support.SPI; @@ -20,7 +20,7 @@ public interface OuterAdapter { * * @param configuration 外部适配器配置信息 */ - void init(CanalOuterAdapterConfiguration configuration); + void init(OuterAdapterConfig configuration); /** * 往适配器中同步数据 @@ -29,8 +29,6 @@ public interface OuterAdapter { */ void sync(Dml dml); - // void writeOut(FlatMessage flatMessage); - /** * 外部适配器销毁接口 */ @@ -39,9 +37,10 @@ public interface OuterAdapter { /** * Etl操作 * - * @param criteriaSql 条件拼接sql + * @param task 任务名, 对应配置名 + * @param params etl筛选条件 */ - default void etl(String criteriaSql) { + default void etl(String task, List params) { throw new UnsupportedOperationException(); } diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java index d885e1cb..d7f7582a 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java @@ -109,13 +109,13 @@ public class CanalClientConfig { public static class AdapterGroup { - private List outAdapters; + private List outAdapters; - public List getOutAdapters() { + public List getOutAdapters() { return outAdapters; } - public void setOutAdapters(List outAdapters) { + public void setOutAdapters(List outAdapters) { this.outAdapters = outAdapters; } } @@ -159,7 +159,7 @@ public class CanalClientConfig { // private List adapters = new ArrayList<>(); - private List outAdapters; + private List outAdapters; public String getGroupId() { return groupId; @@ -169,11 +169,11 @@ public class CanalClientConfig { this.groupId = groupId; } - public List getOutAdapters() { + public List getOutAdapters() { return outAdapters; } - public void setOutAdapters(List outAdapters) { + public void setOutAdapters(List outAdapters) { this.outAdapters = outAdapters; } diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java index 777770ef..d4cb8a0e 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java @@ -2,10 +2,7 @@ package com.alibaba.otter.canal.client.adapter.support; import java.math.BigDecimal; import java.math.BigInteger; -import java.sql.Date; -import java.sql.Time; -import java.sql.Timestamp; -import java.sql.Types; +import java.sql.*; import org.joda.time.DateTime; import org.slf4j.Logger; @@ -21,6 +18,55 @@ public class JdbcTypeUtil { private static Logger logger = LoggerFactory.getLogger(JdbcTypeUtil.class); + public static Object getRSData(ResultSet rs, String columnName, int jdbcType) throws SQLException { + if (jdbcType == Types.BIT || jdbcType == Types.BOOLEAN) { + return rs.getByte(columnName); + } else { + return rs.getObject(columnName); + } + } + + public static Class jdbcType2javaType(int jdbcType) { + switch (jdbcType) { + case Types.BIT: + case Types.BOOLEAN: +// return Boolean.class; + case Types.TINYINT: + return Byte.TYPE; + case Types.SMALLINT: + return Short.class; + case Types.INTEGER: + return Integer.class; + case Types.BIGINT: + return Long.class; + case Types.DECIMAL: + case Types.NUMERIC: + return BigDecimal.class; + case Types.REAL: + return Float.class; + case Types.FLOAT: + case Types.DOUBLE: + return Double.class; + case Types.CHAR: + case Types.VARCHAR: + case Types.LONGVARCHAR: + return String.class; + case Types.BINARY: + case Types.VARBINARY: + case Types.LONGVARBINARY: + case Types.BLOB: + return byte[].class; + case Types.DATE: + return java.sql.Date.class; + case Types.TIME: + return Time.class; + case Types.TIMESTAMP: + return Timestamp.class; + default: + return String.class; + } + } + public static Object typeConvert(String columnName, String value, int sqlType, String mysqlType) { if (value == null || value.equals("")) { return null; diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalOuterAdapterConfiguration.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/OuterAdapterConfig.java similarity index 51% rename from client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalOuterAdapterConfiguration.java rename to client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/OuterAdapterConfig.java index 790044c8..3aed2f16 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalOuterAdapterConfiguration.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/OuterAdapterConfig.java @@ -1,6 +1,6 @@ package com.alibaba.otter.canal.client.adapter.support; -import java.util.Properties; +import java.util.Map; /** * 外部适配器配置信息类 @@ -8,15 +8,15 @@ import java.util.Properties; * @author machengyuan 2018-8-18 下午10:15:12 * @version 1.0.0 */ -public class CanalOuterAdapterConfiguration { +public class OuterAdapterConfig { - private String name; // 适配器名称, 如: logger, hbase, es + private String name; // 适配器名称, 如: logger, hbase, es - private String hosts; // 适配器内部的地址, 比如对应es该参数可以填写es的server地址 + private String hosts; // 适配器内部的地址, 比如对应es该参数可以填写es的server地址 - private String zkHosts; // 适配器内部的ZK地址, 比如对应HBase该参数可以填写HBase对应的ZK地址 + private String zkHosts; // 适配器内部的ZK地址, 比如对应HBase该参数可以填写HBase对应的ZK地址 - private Properties properties; // 其余参数, 可填写适配器中的所需的配置信息 + private Map properties; // 其余参数, 可填写适配器中的所需的配置信息 public String getName() { return name; @@ -34,11 +34,11 @@ public class CanalOuterAdapterConfiguration { this.hosts = hosts; } - public Properties getProperties() { + public Map getProperties() { return properties; } - public void setProperties(Properties properties) { + public void setProperties(Map properties) { this.properties = properties; } diff --git a/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterLoader.java b/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterLoader.java index 05ae6d2c..51b12a0a 100644 --- a/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterLoader.java +++ b/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterLoader.java @@ -15,7 +15,7 @@ import org.slf4j.LoggerFactory; import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; -import com.alibaba.otter.canal.client.adapter.support.CanalOuterAdapterConfiguration; +import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader; /** @@ -72,7 +72,7 @@ public class CanalAdapterLoader { for (CanalClientConfig.AdapterGroup connectorGroup : instance.getAdapterGroups()) { List canalOutConnectors = new ArrayList<>(); - for (CanalOuterAdapterConfiguration c : connectorGroup.getOutAdapters()) { + for (OuterAdapterConfig c : connectorGroup.getOutAdapters()) { loadConnector(c, canalOutConnectors); } canalOuterAdapterGroups.add(canalOutConnectors); @@ -97,7 +97,7 @@ public class CanalAdapterLoader { List canalOuterAdapters = new ArrayList<>(); - for (CanalOuterAdapterConfiguration config : group.getOutAdapters()) { + for (OuterAdapterConfig config : group.getOutAdapters()) { loadConnector(config, canalOuterAdapters); } canalOuterAdapterGroups.add(canalOuterAdapters); @@ -125,7 +125,7 @@ public class CanalAdapterLoader { } } - private void loadConnector(CanalOuterAdapterConfiguration config, List canalOutConnectors) { + private void loadConnector(OuterAdapterConfig config, List canalOutConnectors) { try { OuterAdapter adapter = loader.getExtension(config.getName()); ClassLoader cl = Thread.currentThread().getContextClassLoader(); diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java index c9b67998..da51ac88 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java @@ -2,9 +2,11 @@ package com.alibaba.otter.canal.client.adapter.hbase; import java.io.IOException; import java.util.HashMap; +import java.util.List; import java.util.Map; -import org.apache.commons.lang.StringUtils; +import javax.sql.DataSource; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.Connection; @@ -13,9 +15,12 @@ import org.apache.hadoop.hbase.client.ConnectionFactory; import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig; import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfigLoader; +import com.alibaba.otter.canal.client.adapter.hbase.service.HbaseEtlService; import com.alibaba.otter.canal.client.adapter.hbase.service.HbaseSyncService; -import com.alibaba.otter.canal.client.adapter.support.CanalOuterAdapterConfiguration; +import com.alibaba.otter.canal.client.adapter.hbase.support.HbaseTemplate; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; import com.alibaba.otter.canal.client.adapter.support.Dml; +import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; import com.alibaba.otter.canal.client.adapter.support.SPI; /** @@ -27,56 +32,37 @@ import com.alibaba.otter.canal.client.adapter.support.SPI; @SPI("hbase") public class HbaseAdapter implements OuterAdapter { - private static volatile Map mappingConfigCache = null; + private static volatile Map hbaseMapping = null; // 文件名对应配置 + private static volatile Map mappingConfigCache = null; // 库名-表名对应配置 private Connection conn; private HbaseSyncService hbaseSyncService; + private HbaseTemplate hbaseTemplate; @Override - public void init(CanalOuterAdapterConfiguration configuration) { + public void init(OuterAdapterConfig configuration) { try { if (mappingConfigCache == null) { synchronized (MappingConfig.class) { if (mappingConfigCache == null) { - Map hbaseMapping = MappingConfigLoader.load(); + hbaseMapping = MappingConfigLoader.load(); mappingConfigCache = new HashMap<>(); for (MappingConfig mappingConfig : hbaseMapping.values()) { mappingConfigCache.put(mappingConfig.getHbaseOrm().getDatabase() + "-" - + mappingConfig.getHbaseOrm().getTable(), mappingConfig); + + mappingConfig.getHbaseOrm().getTable(), + mappingConfig); } } } } - String hosts = configuration.getZkHosts(); - if (StringUtils.isEmpty(hosts)) { - hosts = configuration.getHosts(); - } - if (StringUtils.isEmpty(hosts)) { - throw new RuntimeException("Empty zookeeper hosts"); - } - String[] zkHosts = StringUtils.split(hosts, ","); - int zkPort = 0; - StringBuilder hostsWithoutPort = new StringBuilder(); - for (String host : zkHosts) { - int i = host.indexOf(":"); - hostsWithoutPort.append(host, 0, i); - hostsWithoutPort.append(","); - if (zkPort == 0) zkPort = Integer.parseInt(host.substring(i + 1)); - } - hostsWithoutPort.deleteCharAt(hostsWithoutPort.length() - 1); - - String znode = configuration.getProperties().getProperty("znodeParent"); - if (StringUtils.isEmpty(znode)) { - znode = "/hbase"; - } + Map propertites = configuration.getProperties(); Configuration hbaseConfig = HBaseConfiguration.create(); - hbaseConfig.set("hbase.zookeeper.quorum", hostsWithoutPort.toString()); - hbaseConfig.set("hbase.zookeeper.property.clientPort", Integer.toString(zkPort)); - hbaseConfig.set("zookeeper.znode.parent", znode); + propertites.forEach(hbaseConfig::set); conn = ConnectionFactory.createConnection(hbaseConfig); - hbaseSyncService = new HbaseSyncService(conn); + hbaseTemplate = new HbaseTemplate(conn); + hbaseSyncService = new HbaseSyncService(hbaseTemplate); } catch (Exception e) { throw new RuntimeException(e); } @@ -93,6 +79,15 @@ public class HbaseAdapter implements OuterAdapter { hbaseSyncService.sync(config, dml); } + @Override + public void etl(String task, List params) { + MappingConfig config = hbaseMapping.get(task); + DataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + if (dataSource != null) { + HbaseEtlService.importData(dataSource, hbaseTemplate, config, params); + } + } + @Override public void destroy() { if (conn != null) { diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java index cccd2db6..2c919955 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java @@ -1,4 +1,351 @@ package com.alibaba.otter.canal.client.adapter.hbase.service; +import java.sql.*; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; + +import javax.sql.DataSource; + +import org.apache.hadoop.hbase.util.Bytes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig; +import com.alibaba.otter.canal.client.adapter.hbase.support.*; +import com.alibaba.otter.canal.client.adapter.support.JdbcTypeUtil; +import com.google.common.base.Joiner; + public class HbaseEtlService { + + private static Logger logger = LoggerFactory.getLogger(HbaseEtlService.class); + + public static Object sqlRS(DataSource ds, String sql, Function fun) throws SQLException { + Connection conn = null; + Statement smt = null; + ResultSet rs = null; + try { + conn = ds.getConnection(); + smt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + smt.setFetchSize(Integer.MIN_VALUE); + rs = smt.executeQuery(sql); + return fun.apply(rs); + } finally { + if (rs != null) { + try { + rs.close(); + } catch (SQLException e) { + logger.error(e.getMessage(), e); + } + } + if (smt != null) { + try { + smt.close(); + } catch (SQLException e) { + logger.error(e.getMessage(), e); + } + } + if (conn != null) { + try { + conn.close(); + } catch (SQLException e) { + logger.error(e.getMessage(), e); + } + } + } + } + + public static void createTable(HbaseTemplate hbaseTemplate, MappingConfig config) { + try { + // 判断hbase表是否存在，不存在则建表 + MappingConfig.HbaseOrm hbaseOrm = config.getHbaseOrm(); + if (!hbaseTemplate.tableExists(hbaseOrm.getHbaseTable())) { + hbaseTemplate.createTable(hbaseOrm.getHbaseTable(), hbaseOrm.getFamilies().toArray(new String[0])); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + public static void importData(DataSource ds, HbaseTemplate hbaseTemplate, MappingConfig config, + List params) { + // EtlResult etlResult = new EtlResult(); + AtomicLong successCount = new AtomicLong(); + List errMsg = new ArrayList<>(); + String hbaseTable = ""; + try { + // if (config == null) { + // logger.error("Config is null!"); + // etlResult.setSucceeded(false); + // etlResult.setErrorMessage("Config is null!"); + // return etlResult; + // } + MappingConfig.HbaseOrm hbaseOrm = config.getHbaseOrm(); + hbaseTable = hbaseOrm.getHbaseTable(); + + long start = System.currentTimeMillis(); + + if (params != null && params.size() == 1 && "rebuild".equalsIgnoreCase(params.get(0))) { + logger.info(hbaseOrm.getHbaseTable() + " rebuild is starting!"); + // 如果表存在则删除 + if (hbaseTemplate.tableExists(hbaseOrm.getHbaseTable())) { + hbaseTemplate.disableTable(hbaseOrm.getHbaseTable()); + hbaseTemplate.deleteTable(hbaseOrm.getHbaseTable()); + } + params = null; + } else { + logger.info(hbaseOrm.getHbaseTable() + " etl is starting!"); + } + createTable(hbaseTemplate, config); + + // 拼接sql + String sql = "SELECT * FROM " + config.getHbaseOrm().getDatabase() + "." + hbaseOrm.getTable(); + + // 拼接条件 + if (params != null && params.size() == 1 && hbaseOrm.getEtlCondition() == null) { + AtomicBoolean stExists = new AtomicBoolean(false); + // 验证是否有SYS_TIME字段 + sqlRS(ds, sql, rs -> { + try { + ResultSetMetaData rsmd = rs.getMetaData(); + int cnt = rsmd.getColumnCount(); + for (int i = 1; i <= cnt; i++) { + String columnName = rsmd.getColumnName(i); + if ("SYS_TIME".equalsIgnoreCase(columnName)) { + stExists.set(true); + break; + } + } + } catch (Exception e) { + // ignore + } + return null; + }); + if (stExists.get()) { + sql += " WHERE SYS_TIME >= '" + params.get(0) + "' "; + } + } else if (hbaseOrm.getEtlCondition() != null && params != null) { + String etlCondition = hbaseOrm.getEtlCondition(); + int size = params.size(); + for (int i = 0; i < size; i++) { + etlCondition = etlCondition.replace("{" + i + "}", params.get(i)); + } + + sql += " " + etlCondition; + } + + // 获取总数 + String countSql = "SELECT COUNT(1) FROM ( " + sql + ") _CNT "; + long cnt = (Long) sqlRS(ds, countSql, rs -> { + Long count = null; + try { + if (rs.next()) { + count = ((Number) rs.getObject(1)).longValue(); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + return count == null ? 0 : count; + }); + + // 当大于1万条记录时开启多线程 + if (cnt >= 10000) { + int threadCount = 3; // TODO 从配置读取默认为3 + long perThreadCnt = cnt / threadCount; + ExecutorService executor = Executors.newFixedThreadPool(threadCount); + List> futures = new ArrayList<>(threadCount); + for (int i = 0; i < threadCount; i++) { + long offset = i * perThreadCnt; + Long size = null; + if (i != threadCount - 1) { + size = perThreadCnt; + } + String sqlFinal; + if (size != null) { + sqlFinal = sql + " LIMIT " + offset + "," + size; + } else { + sqlFinal = sql + " LIMIT " + offset + "," + cnt; + } + Future future = executor + .submit(() -> executeSqlImport(ds, sqlFinal, hbaseOrm, hbaseTemplate, successCount, errMsg)); + futures.add(future); + } + + for (Future future : futures) { + future.get(); + } + + executor.shutdown(); + } else { + executeSqlImport(ds, sql, hbaseOrm, hbaseTemplate, successCount, errMsg); + } + + logger.info( + hbaseOrm.getHbaseTable() + " etl completed in: " + (System.currentTimeMillis() - start) / 1000 + "s!"); + + // etlResult.setResultMessage("导入HBase表 " + hbaseOrm.getHbaseTable() + " 数据：" + + // successCount.get() + " 条"); + } catch (Exception e) { + logger.error(e.getMessage(), e); + errMsg.add(hbaseTable + " etl failed! ==>" + e.getMessage()); + } + + // if (errMsg.isEmpty()) { + // etlResult.setSucceeded(true); + // } else { + // etlResult.setErrorMessage(Joiner.on("\n").join(errMsg)); + // } + // return etlResult; + } + + private static boolean executeSqlImport(DataSource ds, String sql, MappingConfig.HbaseOrm hbaseOrm, + HbaseTemplate hbaseTemplate, AtomicLong successCount, List errMsg) { + try { + sqlRS(ds, sql, rs -> { + int i = 1; + + try { + boolean complete = false; + List rows = new ArrayList<>(); + String[] rowKeyColumns = null; + if (hbaseOrm.getRowKey() != null) { + rowKeyColumns = hbaseOrm.getRowKey().trim().split(","); + } + while (rs.next()) { + int cc = rs.getMetaData().getColumnCount(); + int[] jdbcTypes = new int[cc]; + Class[] classes = new Class[cc]; + for (int j = 1; j <= cc; j++) { + int jdbcType = rs.getMetaData().getColumnType(j); + jdbcTypes[j - 1] = jdbcType; + classes[j - 1] = JdbcTypeUtil.jdbcType2javaType(jdbcType); + } + HRow row = new HRow(); + + if (rowKeyColumns != null) { + // 取rowKey字段拼接 + StringBuilder rowKeyVale = new StringBuilder(); + for (String rowKeyColumnName : rowKeyColumns) { + Object obj = rs.getObject(rowKeyColumnName); + if (obj != null) { + rowKeyVale.append(obj.toString()); + } + rowKeyVale.append("|"); + } + int len = rowKeyVale.length(); + if (len > 0) { + rowKeyVale.delete(len - 1, len); + } + row.setRowKey(Bytes.toBytes(rowKeyVale.toString())); + } + + for (int j = 1; j <= cc; j++) { + String columnName = rs.getMetaData().getColumnName(j); + + Object val = JdbcTypeUtil.getRSData(rs, columnName, jdbcTypes[j - 1]); + if (val == null) { + continue; + } + + MappingConfig.ColumnItem columnItem = hbaseOrm.getColumnItems().get(columnName); + // 没有配置映射 + if (columnItem == null) { + String family = hbaseOrm.getFamily(); + String qualifile = columnName; + if (hbaseOrm.isUppercaseQualifier()) { + qualifile = qualifile.toUpperCase(); + } + if (MappingConfig.Mode.STRING == hbaseOrm.getMode()) { + if (hbaseOrm.getRowKey() == null && j == 1) { + row.setRowKey(Bytes.toBytes(val.toString())); + } else { + row.addCell(family, qualifile, Bytes.toBytes(val.toString())); + } + } else if (MappingConfig.Mode.NATIVE == hbaseOrm.getMode()) { + Type type = Type.getType(classes[j - 1]); + if (hbaseOrm.getRowKey() == null && j == 1) { + row.setRowKey(TypeUtil.toBytes(val, type)); + } else { + row.addCell(family, qualifile, TypeUtil.toBytes(val, type)); + } + } else if (MappingConfig.Mode.PHOENIX == hbaseOrm.getMode()) { + PhType phType = PhType.getType(classes[j - 1]); + if (hbaseOrm.getRowKey() == null && j == 1) { + row.setRowKey(PhTypeUtil.toBytes(val, phType)); + } else { + row.addCell(family, qualifile, PhTypeUtil.toBytes(val, phType)); + } + } + } else { + // 如果不需要类型转换 + if (columnItem.getType() == null || "".equals(columnItem.getType())) { + if (val instanceof java.sql.Date) { + SimpleDateFormat dateFmt = new SimpleDateFormat("yyyy-MM-dd"); + val = dateFmt.format((Date) val); + } else if (val instanceof Timestamp) { + SimpleDateFormat datetimeFmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + val = datetimeFmt.format((Date) val); + } + + byte[] valBytes = Bytes.toBytes(val.toString()); + if (columnItem.isRowKey()) { + row.setRowKey(valBytes); + } else { + row.addCell(columnItem.getFamily(), columnItem.getQualifier(), valBytes); + } + } else { + PhType phType = PhType.getType(columnItem.getType()); + if (columnItem.isRowKey()) { + row.setRowKey(PhTypeUtil.toBytes(val, phType)); + } else { + row.addCell(columnItem.getFamily(), + columnItem.getQualifier(), + PhTypeUtil.toBytes(val, phType)); + } + } + } + } + + if (row.getRowKey() == null) throw new RuntimeException("RowKey 值为空"); + + rows.add(row); + complete = false; + if (i % hbaseOrm.getCommitBatch() == 0 && !rows.isEmpty()) { + hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows); + rows.clear(); + complete = true; + } + if (logger.isDebugEnabled()) { + logger.debug("import count:" + i); + } + // System.out.println(i); + i++; + successCount.incrementAndGet(); + } + + if (!complete && !rows.isEmpty()) { + hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows); + } + + } catch (Exception e) { + logger.error(hbaseOrm.getHbaseTable() + " etl failed! ==>" + e.getMessage(), e); + errMsg.add(hbaseOrm.getHbaseTable() + " etl failed! ==>" + e.getMessage()); + throw new RuntimeException(e); + } + return i; + }); + return true; + } catch (Exception e) { + logger.error(e.getMessage(), e); + return false; + } + } } diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java index 7e69f9d0..1ff406fd 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java @@ -1,23 +1,13 @@ package com.alibaba.otter.canal.client.adapter.hbase.service; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; -import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.util.Bytes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig; -import com.alibaba.otter.canal.client.adapter.hbase.support.HRow; -import com.alibaba.otter.canal.client.adapter.hbase.support.HbaseTemplate; -import com.alibaba.otter.canal.client.adapter.hbase.support.PhType; -import com.alibaba.otter.canal.client.adapter.hbase.support.PhTypeUtil; -import com.alibaba.otter.canal.client.adapter.hbase.support.Type; -import com.alibaba.otter.canal.client.adapter.hbase.support.TypeUtil; +import com.alibaba.otter.canal.client.adapter.hbase.support.*; import com.alibaba.otter.canal.client.adapter.support.Dml; /** @@ -32,8 +22,8 @@ public class HbaseSyncService { private HbaseTemplate hbaseTemplate; // HBase操作模板 - public HbaseSyncService(Connection conn){ - hbaseTemplate = new HbaseTemplate(conn); + public HbaseSyncService(HbaseTemplate hbaseTemplate){ + this.hbaseTemplate = hbaseTemplate; } public void sync(MappingConfig config, Dml dml) { @@ -375,7 +365,8 @@ public class HbaseSyncService { * @param value 值 * @return 复合字段rowKey */ - private static byte[] typeConvert(MappingConfig.ColumnItem columnItem, MappingConfig.HbaseOrm hbaseOrm, Object value) { + private static byte[] typeConvert(MappingConfig.ColumnItem columnItem, MappingConfig.HbaseOrm hbaseOrm, + Object value) { if (value == null) { return null; } diff --git a/client-adapter/launcher/pom.xml b/client-adapter/launcher/pom.xml index 8b91f27f..2f268088 100644 --- a/client-adapter/launcher/pom.xml +++ b/client-adapter/launcher/pom.xml @@ -50,6 +50,18 @@ spring-boot-configuration-processor true + + + org.apache.rocketmq + rocketmq-client + 4.3.0 + + + + org.apache.kafka + kafka-clients + 1.1.1 + diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfig.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfig.java index 06cb1730..da37804f 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfig.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfig.java @@ -4,6 +4,8 @@ import java.sql.SQLException; import java.util.List; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; @@ -15,6 +17,8 @@ import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; @ConfigurationProperties(prefix = "adapter.conf") public class AdapterConfig { + private static Logger logger = LoggerFactory.getLogger(AdapterConfig.class); + private Map datasourceConfigs; private List adapterConfigs; @@ -51,7 +55,7 @@ public class AdapterConfig { try { ds.init(); } catch (SQLException e) { - throw new RuntimeException(e); + logger.error("#Failed to initial datasource: " + datasourceConfig.getUrl(), e); } DatasourceConfig.DATA_SOURCES.put(entry.getKey(), ds); } diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java index 694251ac..e007faa0 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java @@ -7,6 +7,8 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import com.alibaba.otter.canal.client.adapter.support.Dml; +import com.alibaba.otter.canal.protocol.FlatMessage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,10 +26,10 @@ public abstract class AbstractCanalAdapterWorker { protected final Logger logger = LoggerFactory.getLogger(this.getClass()); - protected String canalDestination; // canal实例 - protected List> canalOuterAdapters; // 外部适配器 - protected ExecutorService groupInnerExecutorService; // 组内工作线程池 - protected volatile boolean running = false; // 是否运行中 + protected String canalDestination; // canal实例 + protected List> canalOuterAdapters; // 外部适配器 + protected ExecutorService groupInnerExecutorService; // 组内工作线程池 + protected volatile boolean running = false; // 是否运行中 protected Thread thread = null; protected Thread.UncaughtExceptionHandler handler = (t, e) -> logger.error("parse events has an error", e); @@ -69,47 +71,47 @@ public abstract class AbstractCanalAdapterWorker { } } - // protected void writeOut(final FlatMessage flatMessage) { - // List> futures = new ArrayList<>(); - // // 组间适配器并行运行 - // for (List outerAdapters : canalOuterAdapters) { - // final List adapters = outerAdapters; - // futures.add(groupInnerExecutorService.submit(new Callable() { - // - // @Override - // public Boolean call() { - // try { - // // 组内适配器穿行运行，尽量不要配置组内适配器 - // for (CanalOuterAdapter c : adapters) { - // long begin = System.currentTimeMillis(); - // Dml dml = MessageUtil.flatMessage2Dml(flatMessage); - // c.writeOut(dml); - // if (logger.isDebugEnabled()) { - // logger.debug("{} elapsed time: {}", - // c.getClass().getName(), - // (System.currentTimeMillis() - begin)); - // } - // } - // return true; - // } catch (Exception e) { - // return false; - // } - // } - // })); - // - // // 等待所有适配器写入完成 - // // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成 - // for (Future f : futures) { - // try { - // if (!f.get()) { - // logger.error("Outer adapter write failed"); - // } - // } catch (InterruptedException | ExecutionException e) { - // // ignore - // } - // } - // } - // } + protected void writeOut(final FlatMessage flatMessage) { + List> futures = new ArrayList<>(); + // 组间适配器并行运行 + for (List outerAdapters : canalOuterAdapters) { + final List adapters = outerAdapters; + futures.add(groupInnerExecutorService.submit(new Callable() { + + @Override + public Boolean call() { + try { + // 组内适配器穿行运行，尽量不要配置组内适配器 + for (OuterAdapter c : adapters) { + long begin = System.currentTimeMillis(); + Dml dml = MessageUtil.flatMessage2Dml(flatMessage); + c.sync(dml); + if (logger.isDebugEnabled()) { + logger.debug("{} elapsed time: {}", + c.getClass().getName(), + (System.currentTimeMillis() - begin)); + } + } + return true; + } catch (Exception e) { + return false; + } + } + })); + + // 等待所有适配器写入完成 + // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成 + for (Future f : futures) { + try { + if (!f.get()) { + logger.error("Outer adapter write failed"); + } + } catch (InterruptedException | ExecutionException e) { + // ignore + } + } + } + } protected void writeOut(Message message, String topic) { if (logger.isDebugEnabled()) { diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java new file mode 100644 index 00000000..01f52e8e --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java @@ -0,0 +1,148 @@ +package com.alibaba.otter.canal.adapter.launcher.loader; + +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import org.apache.kafka.clients.consumer.CommitFailedException; +import org.apache.kafka.common.errors.WakeupException; + +import com.alibaba.otter.canal.client.adapter.OuterAdapter; +import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector; +import com.alibaba.otter.canal.client.kafka.KafkaCanalConnectors; +import com.alibaba.otter.canal.protocol.FlatMessage; +import com.alibaba.otter.canal.protocol.Message; + +/** + * kafka对应的client适配器工作线程 + * + * @author machengyuan 2018-8-19 下午11:30:49 + * @version 1.0.0 + */ +public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker { + + private KafkaCanalConnector connector; + + private String topic; + + private boolean flatMessage; + + public CanalAdapterKafkaWorker(String bootstrapServers, String topic, String groupId, + List> canalOuterAdapters, boolean flatMessage){ + this.canalOuterAdapters = canalOuterAdapters; + this.groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size()); + this.topic = topic; + this.canalDestination = topic; + this.flatMessage = flatMessage; + connector = KafkaCanalConnectors.newKafkaConnector(bootstrapServers, topic, null, groupId, flatMessage); + // connector.setSessionTimeout(1L, TimeUnit.MINUTES); + + // super.initSwitcher(topic); + } + + @Override + public void start() { + if (!running) { + thread = new Thread(new Runnable() { + + @Override + public void run() { + process(); + } + }); + thread.setUncaughtExceptionHandler(handler); + running = true; + thread.start(); + } + } + + @Override + public void stop() { + try { + if (!running) { + return; + } + + connector.stopRunning(); + running = false; + + // if (switcher != null && !switcher.state()) { + // switcher.set(true); + // } + + if (thread != null) { + try { + thread.join(); + } catch (InterruptedException e) { + // ignore + } + } + groupInnerExecutorService.shutdown(); + logger.info("topic {} connectors' worker thread dead!", this.topic); + for (List outerAdapters : canalOuterAdapters) { + for (OuterAdapter adapter : outerAdapters) { + adapter.destroy(); + } + } + logger.info("topic {} all connectors destroyed!", this.topic); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + private void process() { + while (!running) + ; + ExecutorService executor = Executors.newSingleThreadExecutor(); + // final AtomicBoolean executing = new AtomicBoolean(true); + while (running) { + try { + logger.info("=============> Start to connect topic: {} <=============", this.topic); + connector.connect(); + logger.info("=============> Start to subscribe topic: {} <=============", this.topic); + connector.subscribe(); + logger.info("=============> Subscribe topic: {} succeed <=============", this.topic); + while (running) { + try { + // switcher.get(); //等待开关开启 + + List messages; + if (!flatMessage) { + messages = connector.getWithoutAck(); + } else { + messages = connector.getFlatMessageWithoutAck(100L, TimeUnit.MILLISECONDS); + } + if (messages != null) { + for (final Object message : messages) { + if (message instanceof FlatMessage) { + writeOut((FlatMessage) message); + } else { + writeOut((Message) message); + } + } + } + connector.ack(); + } catch (CommitFailedException e) { + logger.warn(e.getMessage()); + } catch (Exception e) { + logger.error(e.getMessage(), e); + TimeUnit.SECONDS.sleep(1L); + } + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + executor.shutdown(); + + try { + connector.unsubscribe(); + } catch (WakeupException e) { + // No-op. Continue process + } + connector.disconnect(); + logger.info("=============> Disconnect topic: {} <=============", this.topic); + } +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java index 6e7d2bb8..d9cdaa7c 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java @@ -9,13 +9,14 @@ import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import com.alibaba.otter.canal.client.adapter.OuterAdapter; -import com.alibaba.otter.canal.client.adapter.support.CanalOuterAdapterConfiguration; -import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader; +import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; +import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; +import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader; /** * 外部适配器的加载器 @@ -24,13 +25,15 @@ import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; */ public class CanalAdapterLoader { - private static final Logger logger = LoggerFactory.getLogger(CanalAdapterLoader.class); + private static final Logger logger = LoggerFactory.getLogger(CanalAdapterLoader.class); - private CanalClientConfig canalClientConfig; + private CanalClientConfig canalClientConfig; - private Map canalWorkers = new HashMap<>(); + private Map canalWorkers = new HashMap<>(); - private ExtensionLoader loader; + private Map canalMQWorker = new HashMap<>(); + + private ExtensionLoader loader; public CanalAdapterLoader(CanalClientConfig canalClientConfig){ this.canalClientConfig = canalClientConfig; @@ -57,7 +60,7 @@ public class CanalAdapterLoader { for (CanalClientConfig.AdapterGroup connectorGroup : instance.getAdapterGroups()) { List canalOutConnectors = new ArrayList<>(); - for (CanalOuterAdapterConfiguration c : connectorGroup.getOutAdapters()) { + for (OuterAdapterConfig c : connectorGroup.getOutAdapters()) { loadConnector(c, canalOutConnectors); } canalOuterAdapterGroups.add(canalOutConnectors); @@ -73,9 +76,43 @@ public class CanalAdapterLoader { logger.info("Start adapter for canal instance: {} succeed", instance.getInstance()); } } + + // 初始化canal-client-mq的适配器 + if (canalClientConfig.getMqTopics() != null) { + for (CanalClientConfig.MQTopic topic : canalClientConfig.getMqTopics()) { + for (CanalClientConfig.Group group : topic.getGroups()) { + List> canalOuterAdapterGroups = new ArrayList<>(); + + List canalOuterAdapters = new ArrayList<>(); + + for (OuterAdapterConfig config : group.getOutAdapters()) { + loadConnector(config, canalOuterAdapters); + } + canalOuterAdapterGroups.add(canalOuterAdapters); + if (StringUtils.isBlank(topic.getMqMode()) || "rocketmq".equalsIgnoreCase(topic.getMqMode())) { + CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig + .getBootstrapServers(), topic.getTopic(), group.getGroupId(), canalOuterAdapterGroups); + canalMQWorker.put(topic.getTopic() + "-rocketmq-" + group.getGroupId(), rocketMQWorker); + rocketMQWorker.start(); + } else if ("kafka".equalsIgnoreCase(topic.getMqMode())) { + CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker( + canalClientConfig.getBootstrapServers(), + topic.getTopic(), + group.getGroupId(), + canalOuterAdapterGroups, + canalClientConfig.getFlatMessage()); + canalMQWorker.put(topic.getTopic() + "-kafka-" + group.getGroupId(), canalKafkaWorker); + canalKafkaWorker.start(); + } + logger.info("Start adapter for canal-client rocketmq topic: {} succeed", + topic.getTopic() + "-" + group.getGroupId()); + + } + } + } } - private void loadConnector(CanalOuterAdapterConfiguration config, List canalOutConnectors) { + private void loadConnector(OuterAdapterConfig config, List canalOutConnectors) { try { OuterAdapter adapter = loader.getExtension(config.getName()); ClassLoader cl = Thread.currentThread().getContextClassLoader(); @@ -98,10 +135,19 @@ public class CanalAdapterLoader { ExecutorService stopExecutorService = Executors.newFixedThreadPool(canalWorkers.size()); for (CanalAdapterWorker v : canalWorkers.values()) { final CanalAdapterWorker caw = v; - stopExecutorService.submit(() -> caw.stop()); + stopExecutorService.submit(caw::stop); } stopExecutorService.shutdown(); } + + if (canalMQWorker.size() > 0) { + ExecutorService stopMQWokerService = Executors.newFixedThreadPool(canalMQWorker.size()); + for (AbstractCanalAdapterWorker tmp : canalMQWorker.values()) { + final AbstractCanalAdapterWorker worker = tmp; + stopMQWokerService.submit(worker::stop); + } + stopMQWokerService.shutdown(); + } logger.info("All canal adapters destroyed"); } } diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java new file mode 100644 index 00000000..aa963bbe --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java @@ -0,0 +1,124 @@ +package com.alibaba.otter.canal.adapter.launcher.loader; + +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import org.apache.kafka.clients.consumer.CommitFailedException; +import org.apache.kafka.common.errors.WakeupException; + +import com.alibaba.otter.canal.client.adapter.OuterAdapter; +import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnector; +import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnectorProvider; +import com.alibaba.otter.canal.protocol.Message; + +/** + * kafka对应的client适配器工作线程 + * + * @author machengyuan 2018-8-19 下午11:30:49 + * @version 1.0.0 + */ +public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker { + + private RocketMQCanalConnector connector; + + private String topic; + + public CanalAdapterRocketMQWorker(String nameServers, String topic, String groupId, + List> canalOuterAdapters){ + logger.info("RocketMQ consumer config topic:{}, nameServer:{}, groupId:{}", topic, nameServers, groupId); + this.canalOuterAdapters = canalOuterAdapters; + this.groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size()); + this.topic = topic; + this.canalDestination = topic; + connector = RocketMQCanalConnectorProvider.newRocketMQConnector(nameServers, topic, groupId); + } + + @Override + public void start() { + if (!running) { + thread = new Thread(new Runnable() { + + @Override + public void run() { + process(); + } + }); + thread.setUncaughtExceptionHandler(handler); + running = true; + thread.start(); + } + } + + @Override + public void stop() { + try { + if (!running) { + return; + } + connector.stopRunning(); + running = false; + logger.info("Stop topic {} out adapters begin", this.topic); + stopOutAdapters(); + logger.info("Stop topic {} out adapters end", this.topic); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + private void process() { + while (!running) + ; + ExecutorService executor = Executors.newSingleThreadExecutor(); + while (running) { + try { + logger.info("=============> Start to connect topic: {} <=============", this.topic); + connector.connect(); + logger.info("=============> Start to subscribe topic: {}<=============", this.topic); + connector.subscribe(); + logger.info("=============> Subscribe topic: {} succeed<=============", this.topic); + while (running) { + try { + // switcher.get(); //等待开关开启 + + final Message message = connector.getWithoutAck(1); + if (message != null) { + executor.submit(new Runnable() { + + @Override + public void run() { + try { + writeOut(message, topic); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + connector.ack(message.getId()); + } + }); + } else { + logger.debug("Message is null"); + } + } catch (CommitFailedException e) { + logger.warn(e.getMessage()); + } catch (Exception e) { + logger.error(e.getMessage(), e); + TimeUnit.SECONDS.sleep(1L); + } + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + executor.shutdown(); + + try { + connector.unsubscribe(); + } catch (WakeupException e) { + // No-op. Continue process + } + connector.stopRunning(); + logger.info("=============> Disconnect topic: {} <=============", this.topic); + } +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java index 1c38b353..224e70d4 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java @@ -59,13 +59,7 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker { @Override public void start() { if (!running) { - thread = new Thread(new Runnable() { - - @Override - public void run() { - process(); - } - }); + thread = new Thread(() -> process()); thread.setUncaughtExceptionHandler(handler); thread.start(); running = true; diff --git a/client-adapter/launcher/src/main/resources/application.yml b/client-adapter/launcher/src/main/resources/application.yml index 7adef833..af787a9f 100644 --- a/client-adapter/launcher/src/main/resources/application.yml +++ b/client-adapter/launcher/src/main/resources/application.yml @@ -1,20 +1,32 @@ server: port: 8081 +logging: + level: + com.alibaba.otter.canal.client.adapter.hbase: DEBUG + canal.conf: canalServerHost: 127.0.0.1:11111 - #zookeeperHosts: slave1:2181 - #bootstrapServers: localhost:9092 #or rocketmq nameservers:host1:9876;host2:9876 - flatMessage: false - +# zookeeperHosts: slave1:2181 +# bootstrapServers: slave1:6667 #or rocketmq nameservers:host1:9876;host2:9876 + flatMessage: true canalInstances: - instance: example adapterGroups: - outAdapters: - - name: logger -# - name: hbase -# hosts: slave1:2181 -# properties: {znodeParent: "/hbase-unsecure"} +# - name: logger + - name: hbase + properties: + hbase.zookeeper.quorum: slave1 + hbase.zookeeper.property.clientPort: 2181 + zookeeper.znode.parent: /hbase-unsecure +# mqTopics: +# - mqMode: kafka +# topic: example +# groups: +# - groupId: g2 +# outAdapters: +# - name: logger adapter.conf: datasourceConfigs: diff --git a/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java b/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java index 27d99aae..985c7804 100644 --- a/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java +++ b/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java @@ -4,7 +4,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.otter.canal.client.adapter.OuterAdapter; -import com.alibaba.otter.canal.client.adapter.support.CanalOuterAdapterConfiguration; +import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; import com.alibaba.otter.canal.client.adapter.support.Dml; import com.alibaba.otter.canal.client.adapter.support.SPI; @@ -21,7 +21,7 @@ public class LoggerAdapterExample implements OuterAdapter { private Logger logger = LoggerFactory.getLogger(this.getClass()); @Override - public void init(CanalOuterAdapterConfiguration configuration) { + public void init(OuterAdapterConfig configuration) { } From 49caff57599fe550cc55c30a65ed8dfe6f7b8dfb Mon Sep 17 00:00:00 2001 From: mcy Date: Wed, 24 Oct 2018 19:12:16 +0800 Subject: [PATCH 13/35] =?UTF-8?q?HBase=E7=9A=84etl=E5=88=9D=E6=AD=A5?= =?UTF-8?q?=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../canal/client/adapter/OuterAdapter.java | 5 ++- .../client/adapter/support/EtlResult.java | 37 ++++++++++++++++ .../client/adapter/hbase/HbaseAdapter.java | 14 ++++--- .../hbase/service/HbaseEtlService.java | 36 ++++++++-------- .../canal/adapter/launcher/rest/EtlRest.java | 42 +++++++++++++++++++ .../src/main/resources/application.yml | 6 ++- 6 files changed, 113 insertions(+), 27 deletions(-) create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/EtlResult.java create mode 100644 client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/EtlRest.java diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java index a508d1ef..3fef6c0e 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java @@ -2,8 +2,9 @@ package com.alibaba.otter.canal.client.adapter; import java.util.List; -import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; import com.alibaba.otter.canal.client.adapter.support.Dml; +import com.alibaba.otter.canal.client.adapter.support.EtlResult; +import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; import com.alibaba.otter.canal.client.adapter.support.SPI; /** @@ -40,7 +41,7 @@ public interface OuterAdapter { * @param task 任务名, 对应配置名 * @param params etl筛选条件 */ - default void etl(String task, List params) { + default EtlResult etl(String task, List params) { throw new UnsupportedOperationException(); } diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/EtlResult.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/EtlResult.java new file mode 100644 index 00000000..0004e51a --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/EtlResult.java @@ -0,0 +1,37 @@ +package com.alibaba.otter.canal.client.adapter.support; + +import java.io.Serializable; + +public class EtlResult implements Serializable { + private static final long serialVersionUID = 4250522736289866505L; + + private boolean succeeded = false; + + private String resultMessage; + + private String errorMessage; + + public boolean getSucceeded() { + return succeeded; + } + + public void setSucceeded(boolean succeeded) { + this.succeeded = succeeded; + } + + public String getResultMessage() { + return resultMessage; + } + + public void setResultMessage(String resultMessage) { + this.resultMessage = resultMessage; + } + + public String getErrorMessage() { + return errorMessage; + } + + public void setErrorMessage(String errorMessage) { + this.errorMessage = errorMessage; + } +} \ No newline at end of file diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java index da51ac88..88c7fe22 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java @@ -18,10 +18,7 @@ import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfigLoader; import com.alibaba.otter.canal.client.adapter.hbase.service.HbaseEtlService; import com.alibaba.otter.canal.client.adapter.hbase.service.HbaseSyncService; import com.alibaba.otter.canal.client.adapter.hbase.support.HbaseTemplate; -import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; -import com.alibaba.otter.canal.client.adapter.support.Dml; -import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; -import com.alibaba.otter.canal.client.adapter.support.SPI; +import com.alibaba.otter.canal.client.adapter.support.*; /** * HBase外部适配器 @@ -80,11 +77,16 @@ public class HbaseAdapter implements OuterAdapter { } @Override - public void etl(String task, List params) { + public EtlResult etl(String task, List params) { MappingConfig config = hbaseMapping.get(task); DataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); if (dataSource != null) { - HbaseEtlService.importData(dataSource, hbaseTemplate, config, params); + return HbaseEtlService.importData(dataSource, hbaseTemplate, config, params); + } else { + EtlResult etlResult = new EtlResult(); + etlResult.setSucceeded(false); + etlResult.setErrorMessage("DataSource not found"); + return etlResult; } } diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java index 2c919955..e2b86df1 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java @@ -20,6 +20,7 @@ import org.slf4j.LoggerFactory; import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig; import com.alibaba.otter.canal.client.adapter.hbase.support.*; +import com.alibaba.otter.canal.client.adapter.support.EtlResult; import com.alibaba.otter.canal.client.adapter.support.JdbcTypeUtil; import com.google.common.base.Joiner; @@ -75,19 +76,19 @@ public class HbaseEtlService { } } - public static void importData(DataSource ds, HbaseTemplate hbaseTemplate, MappingConfig config, - List params) { - // EtlResult etlResult = new EtlResult(); + public static EtlResult importData(DataSource ds, HbaseTemplate hbaseTemplate, MappingConfig config, + List params) { + EtlResult etlResult = new EtlResult(); AtomicLong successCount = new AtomicLong(); List errMsg = new ArrayList<>(); String hbaseTable = ""; try { - // if (config == null) { - // logger.error("Config is null!"); - // etlResult.setSucceeded(false); - // etlResult.setErrorMessage("Config is null!"); - // return etlResult; - // } + if (config == null) { + logger.error("Config is null!"); + etlResult.setSucceeded(false); + etlResult.setErrorMessage("Config is null!"); + return etlResult; + } MappingConfig.HbaseOrm hbaseOrm = config.getHbaseOrm(); hbaseTable = hbaseOrm.getHbaseTable(); @@ -158,7 +159,7 @@ public class HbaseEtlService { // 当大于1万条记录时开启多线程 if (cnt >= 10000) { - int threadCount = 3; // TODO 从配置读取默认为3 + int threadCount = 3; long perThreadCnt = cnt / threadCount; ExecutorService executor = Executors.newFixedThreadPool(threadCount); List> futures = new ArrayList<>(threadCount); @@ -191,19 +192,18 @@ public class HbaseEtlService { logger.info( hbaseOrm.getHbaseTable() + " etl completed in: " + (System.currentTimeMillis() - start) / 1000 + "s!"); - // etlResult.setResultMessage("导入HBase表 " + hbaseOrm.getHbaseTable() + " 数据：" + - // successCount.get() + " 条"); + etlResult.setResultMessage("导入HBase表 " + hbaseOrm.getHbaseTable() + " 数据：" + successCount.get() + " 条"); } catch (Exception e) { logger.error(e.getMessage(), e); errMsg.add(hbaseTable + " etl failed! ==>" + e.getMessage()); } - // if (errMsg.isEmpty()) { - // etlResult.setSucceeded(true); - // } else { - // etlResult.setErrorMessage(Joiner.on("\n").join(errMsg)); - // } - // return etlResult; + if (errMsg.isEmpty()) { + etlResult.setSucceeded(true); + } else { + etlResult.setErrorMessage(Joiner.on("\n").join(errMsg)); + } + return etlResult; } private static boolean executeSqlImport(DataSource ds, String sql, MappingConfig.HbaseOrm hbaseOrm, diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/EtlRest.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/EtlRest.java new file mode 100644 index 00000000..33b1d8f9 --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/EtlRest.java @@ -0,0 +1,42 @@ +package com.alibaba.otter.canal.adapter.launcher.rest; + +import java.util.Arrays; +import java.util.List; + +import javax.annotation.PostConstruct; + +import com.alibaba.otter.canal.client.adapter.support.EtlResult; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import com.alibaba.otter.canal.client.adapter.OuterAdapter; +import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader; + +@RestController +public class EtlRest { + + private ExtensionLoader loader; + + @PostConstruct + public void init() { + loader = ExtensionLoader.getExtensionLoader(OuterAdapter.class); + } + + /** + * Demo: http://127.0.0.1:8081/etl/hbase/mytest_person2.yml + */ + @GetMapping("/etl/{type}/{task}") + public EtlResult etl(@PathVariable String type, @PathVariable String task, + @RequestParam(required = false) String params) { + OuterAdapter adapter = loader.getExtension(type); + List paramArr = null; + if (params != null) { + String[] parmaArray = params.trim().split(";"); + paramArr = Arrays.asList(parmaArray); + } + + return adapter.etl(task, paramArr); + } +} diff --git a/client-adapter/launcher/src/main/resources/application.yml b/client-adapter/launcher/src/main/resources/application.yml index af787a9f..7a319012 100644 --- a/client-adapter/launcher/src/main/resources/application.yml +++ b/client-adapter/launcher/src/main/resources/application.yml @@ -1,9 +1,13 @@ server: port: 8081 - logging: level: com.alibaba.otter.canal.client.adapter.hbase: DEBUG +spring: + jackson: + date-format: yyyy-MM-dd HH:mm:ss + time-zone: GMT+8 + default-property-inclusion: non_null canal.conf: canalServerHost: 127.0.0.1:11111 From 30a11c75cbaecfb290b5ae43df2191b82818d7aa Mon Sep 17 00:00:00 2001 From: mcy Date: Wed, 24 Oct 2018 20:47:25 +0800 Subject: [PATCH 14/35] =?UTF-8?q?HBase=E7=9A=84etl=E5=88=9D=E6=AD=A5?= =?UTF-8?q?=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../canal/client/adapter/OuterAdapter.java | 12 ++++- .../canal/client/adapter/support/Result.java | 50 +++++++++++++++++++ client-adapter/example/pom.xml | 2 +- .../client/adapter/hbase/HbaseAdapter.java | 39 +++++++++++++-- .../hbase/config/MappingConfigLoader.java | 4 +- .../hbase/service/HbaseEtlService.java | 2 +- .../main/resources/hbase/mytest_person2.yml | 2 +- client-adapter/launcher/pom.xml | 2 +- .../rest/{EtlRest.java => CommonRest.java} | 25 ++++++---- .../src/main/resources/application.yml | 5 +- 10 files changed, 121 insertions(+), 22 deletions(-) create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Result.java rename client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/{EtlRest.java => CommonRest.java} (62%) diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java index 3fef6c0e..bf6d5fce 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java @@ -1,6 +1,7 @@ package com.alibaba.otter.canal.client.adapter; import java.util.List; +import java.util.Map; import com.alibaba.otter.canal.client.adapter.support.Dml; import com.alibaba.otter.canal.client.adapter.support.EtlResult; @@ -42,7 +43,16 @@ public interface OuterAdapter { * @param params etl筛选条件 */ default EtlResult etl(String task, List params) { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException("unsupported operation"); } + /** + * 计算总数 + * + * @param task 任务名, 对应配置名 + * @return 总数 + */ + default Map count(String task) { + throw new UnsupportedOperationException("unsupported operation"); + } } diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Result.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Result.java new file mode 100644 index 00000000..0ea913e1 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Result.java @@ -0,0 +1,50 @@ +package com.alibaba.otter.canal.client.adapter.support; + +import java.io.Serializable; +import java.util.Date; + +public class Result implements Serializable { + + public Integer code = 20000; + public Object data; + public String message; + public Date sysTime = new Date(); + + public static Result createSuccess(Object data) { + Result result = new Result(); + result.setData(data); + return result; + } + + public Integer getCode() { + return code; + } + + public void setCode(Integer code) { + this.code = code; + } + + public Object getData() { + return data; + } + + public void setData(Object data) { + this.data = data; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public Date getSysTime() { + return sysTime; + } + + public void setSysTime(Date sysTime) { + this.sysTime = sysTime; + } +} diff --git a/client-adapter/example/pom.xml b/client-adapter/example/pom.xml index 3c68087a..7433e948 100644 --- a/client-adapter/example/pom.xml +++ b/client-adapter/example/pom.xml @@ -9,7 +9,7 @@ 4.0.0 com.alibaba.otter - example + client-adapter.example jar canal client adapter example module for otter ${project.version} diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java index 88c7fe22..97a7bd70 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java @@ -2,6 +2,7 @@ package com.alibaba.otter.canal.client.adapter.hbase; import java.io.IOException; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -9,8 +10,11 @@ import javax.sql.DataSource; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig; @@ -29,12 +33,15 @@ import com.alibaba.otter.canal.client.adapter.support.*; @SPI("hbase") public class HbaseAdapter implements OuterAdapter { - private static volatile Map hbaseMapping = null; // 文件名对应配置 - private static volatile Map mappingConfigCache = null; // 库名-表名对应配置 + private static Logger logger = LoggerFactory.getLogger(HbaseAdapter.class); + + private static volatile Map hbaseMapping = null; // 文件名对应配置 + private static volatile Map mappingConfigCache = null; // 库名-表名对应配置 private Connection conn; private HbaseSyncService hbaseSyncService; private HbaseTemplate hbaseTemplate; + private Configuration hbaseConfig; @Override public void init(OuterAdapterConfig configuration) { @@ -55,7 +62,7 @@ public class HbaseAdapter implements OuterAdapter { Map propertites = configuration.getProperties(); - Configuration hbaseConfig = HBaseConfiguration.create(); + hbaseConfig = HBaseConfiguration.create(); propertites.forEach(hbaseConfig::set); conn = ConnectionFactory.createConnection(hbaseConfig); hbaseTemplate = new HbaseTemplate(conn); @@ -90,6 +97,28 @@ public class HbaseAdapter implements OuterAdapter { } } + @Override + public Map count(String task) { + MappingConfig config = hbaseMapping.get(task); + String hbaseTable = config.getHbaseOrm().getHbaseTable(); + long rowCount = 0L; + try { + HTable table = new HTable(hbaseConfig, hbaseTable); + Scan scan = new Scan(); + scan.setFilter(new FirstKeyOnlyFilter()); + ResultScanner resultScanner = table.getScanner(scan); + for (Result result : resultScanner) { + rowCount += result.size(); + } + } catch (IOException e) { + logger.error(e.getMessage(), e); + } + Map res = new LinkedHashMap<>(); + res.put("hbaseTable", hbaseTable); + res.put("count", rowCount); + return res; + } + @Override public void destroy() { if (conn != null) { diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java index 173fc023..0fb727d8 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java @@ -72,6 +72,7 @@ public class MappingConfigLoader { String[] dbTable; if (dsKey == null) { dbTable = srcMeta.split("\\."); + } else { dbTable = srcMeta.split("@")[0].split("\\."); } @@ -84,7 +85,7 @@ public class MappingConfigLoader { hbaseOrm.setAutoCreateTable(true); hbaseOrm.setDatabase(dbTable[0]); hbaseOrm.setTable(dbTable[1]); - hbaseOrm.setMode(MappingConfig.Mode.STRING); + hbaseOrm.setMode(MappingConfig.Mode.PHOENIX); hbaseOrm.setRowKey(rowKey); // 有定义rowKey if (rowKey != null) { @@ -94,6 +95,7 @@ public class MappingConfigLoader { hbaseOrm.setRowKeyColumn(columnItem); } config.setHbaseOrm(hbaseOrm); + config.setDataSourceKey(dsKey); } else { throw new RuntimeException(String.format("配置项[%s]内容为空, 或格式不符合database.table", c)); diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java index e2b86df1..6b955096 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java @@ -68,7 +68,7 @@ public class HbaseEtlService { // 判断hbase表是否存在，不存在则建表 MappingConfig.HbaseOrm hbaseOrm = config.getHbaseOrm(); if (!hbaseTemplate.tableExists(hbaseOrm.getHbaseTable())) { - hbaseTemplate.createTable(hbaseOrm.getHbaseTable(), hbaseOrm.getFamilies().toArray(new String[0])); + hbaseTemplate.createTable(hbaseOrm.getHbaseTable(), hbaseOrm.getFamily()); } } catch (Exception e) { logger.error(e.getMessage(), e); diff --git a/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml b/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml index f2b6a594..cbf14d48 100644 --- a/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml +++ b/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml @@ -3,7 +3,7 @@ hbaseOrm: mode: PHOENIX #NATIVE #STRING database: mytest # 数据库名 table: person2 # 数据库表名 - hbaseTable: MYTEST_PERSON2 # HBase表名 + hbaseTable: MYTEST.PERSON2 # HBase表名 family: CF # 默认统一Family名称 uppercaseQualifier: true # 字段名转大写, 默认为true commitBatch: 3000 # 批量提交的大小 diff --git a/client-adapter/launcher/pom.xml b/client-adapter/launcher/pom.xml index 2f268088..d5e43708 100644 --- a/client-adapter/launcher/pom.xml +++ b/client-adapter/launcher/pom.xml @@ -9,7 +9,7 @@ 4.0.0 com.alibaba.otter - launcher + client-adapter.launcher jar canal client adapter launcher module for otter ${project.version} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/EtlRest.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java similarity index 62% rename from client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/EtlRest.java rename to client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java index 33b1d8f9..388f42cd 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/EtlRest.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java @@ -2,20 +2,18 @@ package com.alibaba.otter.canal.adapter.launcher.rest; import java.util.Arrays; import java.util.List; +import java.util.Map; import javax.annotation.PostConstruct; -import com.alibaba.otter.canal.client.adapter.support.EtlResult; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import com.alibaba.otter.canal.client.adapter.OuterAdapter; +import com.alibaba.otter.canal.client.adapter.support.EtlResult; import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader; @RestController -public class EtlRest { +public class CommonRest { private ExtensionLoader loader; @@ -25,9 +23,9 @@ public class EtlRest { } /** - * Demo: http://127.0.0.1:8081/etl/hbase/mytest_person2.yml + * Demo: POST: http://127.0.0.1:8081/etl/hbase/mytest_person2.yml */ - @GetMapping("/etl/{type}/{task}") + @PostMapping("/etl/{type}/{task}") public EtlResult etl(@PathVariable String type, @PathVariable String task, @RequestParam(required = false) String params) { OuterAdapter adapter = loader.getExtension(type); @@ -37,6 +35,15 @@ public class EtlRest { paramArr = Arrays.asList(parmaArray); } - return adapter.etl(task, paramArr); + return adapter.etl(task, paramArr); + } + + /** + * Demo: GET: http://127.0.0.1:8081/count/hbase/mytest_person2.yml + */ + @GetMapping("/count/{type}/{task}") + public Map count(@PathVariable String type, @PathVariable String task) { + OuterAdapter adapter = loader.getExtension(type); + return adapter.count(task); } } diff --git a/client-adapter/launcher/src/main/resources/application.yml b/client-adapter/launcher/src/main/resources/application.yml index 7a319012..b61a4749 100644 --- a/client-adapter/launcher/src/main/resources/application.yml +++ b/client-adapter/launcher/src/main/resources/application.yml @@ -18,7 +18,7 @@ canal.conf: - instance: example adapterGroups: - outAdapters: -# - name: logger + - name: logger - name: hbase properties: hbase.zookeeper.quorum: slave1 @@ -39,4 +39,5 @@ adapter.conf: username: root password: 121212 adapterConfigs: - - hbase/mytest_person2.yml \ No newline at end of file + - hbase/mytest_person2.yml +# - hbase/mytest.person2@defaultDS From 8dbf63e4364bcb64b63627ef0632549d10965b6e Mon Sep 17 00:00:00 2001 From: mcy Date: Thu, 25 Oct 2018 10:05:05 +0800 Subject: [PATCH 15/35] =?UTF-8?q?=E5=A2=9E=E5=8A=A0adapter=20assembly?= =?UTF-8?q?=E6=89=93=E5=8C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client-adapter/example/pom.xml | 1 + client-adapter/hbase/pom.xml | 19 ----------- .../client/adapter/hbase/HbaseAdapter.java | 6 ++-- .../adapter/hbase/support/TypeUtil.java | 4 +-- client-adapter/launcher/assembly.xml | 34 +++++++++++++++++++ client-adapter/launcher/pom.xml | 20 ++++++++++- 6 files changed, 59 insertions(+), 25 deletions(-) create mode 100644 client-adapter/launcher/assembly.xml diff --git a/client-adapter/example/pom.xml b/client-adapter/example/pom.xml index 7433e948..f21649db 100644 --- a/client-adapter/example/pom.xml +++ b/client-adapter/example/pom.xml @@ -55,6 +55,7 @@ maven-jar-plugin + 3.0.2 true diff --git a/client-adapter/hbase/pom.xml b/client-adapter/hbase/pom.xml index 866bd31f..a48d7401 100644 --- a/client-adapter/hbase/pom.xml +++ b/client-adapter/hbase/pom.xml @@ -84,25 +84,6 @@ - - org.apache.maven.plugins - maven-assembly-plugin - 2.4 - - - jar-with-dependencies - - - - - make-assembly - package - - single - - - - diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java index 97a7bd70..88fa9a4a 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java @@ -10,6 +10,7 @@ import javax.sql.DataSource; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; @@ -41,7 +42,6 @@ public class HbaseAdapter implements OuterAdapter { private Connection conn; private HbaseSyncService hbaseSyncService; private HbaseTemplate hbaseTemplate; - private Configuration hbaseConfig; @Override public void init(OuterAdapterConfig configuration) { @@ -62,7 +62,7 @@ public class HbaseAdapter implements OuterAdapter { Map propertites = configuration.getProperties(); - hbaseConfig = HBaseConfiguration.create(); + Configuration hbaseConfig = HBaseConfiguration.create(); propertites.forEach(hbaseConfig::set); conn = ConnectionFactory.createConnection(hbaseConfig); hbaseTemplate = new HbaseTemplate(conn); @@ -103,7 +103,7 @@ public class HbaseAdapter implements OuterAdapter { String hbaseTable = config.getHbaseOrm().getHbaseTable(); long rowCount = 0L; try { - HTable table = new HTable(hbaseConfig, hbaseTable); + HTable table = (HTable) conn.getTable(TableName.valueOf(hbaseTable)); Scan scan = new Scan(); scan.setFilter(new FirstKeyOnlyFilter()); ResultScanner resultScanner = table.getScanner(scan); diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/TypeUtil.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/TypeUtil.java index 07aad00f..f90e5bd8 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/TypeUtil.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/TypeUtil.java @@ -92,6 +92,7 @@ public class TypeUtil { return b; } + @SuppressWarnings("unchecked") public static T toObject(byte[] bytes, Class clazz) { if (bytes == null) { return null; @@ -132,10 +133,10 @@ public class TypeUtil { } else { throw new IllegalArgumentException("mismatch class type"); } - // noinspection unchecked return (T) res; } + @SuppressWarnings("unchecked") public static T toObject(byte[] bytes, Type type) { if (bytes == null) { return null; @@ -182,7 +183,6 @@ public class TypeUtil { } else { throw new IllegalArgumentException("mismatch class type"); } - // noinspection unchecked return (T) res; } } diff --git a/client-adapter/launcher/assembly.xml b/client-adapter/launcher/assembly.xml new file mode 100644 index 00000000..7bb12314 --- /dev/null +++ b/client-adapter/launcher/assembly.xml @@ -0,0 +1,34 @@ + + canal-adapter + + tar.gz + + + + ${project.build.directory} + / + + *.jar + + + + + ${project.build.directory}/lib + /lib/ + + *.jar + + + + + ${project.build.directory}/config + /config/ + + ** + + + + / + \ No newline at end of file diff --git a/client-adapter/launcher/pom.xml b/client-adapter/launcher/pom.xml index d5e43708..7d54cd58 100644 --- a/client-adapter/launcher/pom.xml +++ b/client-adapter/launcher/pom.xml @@ -93,7 +93,7 @@ - canal-client-launcher + canal-adapter-launcher org.springframework.boot @@ -145,6 +145,24 @@ + + maven-assembly-plugin + + false + + assembly.xml + + + + + make-assembly + package + + single + + + + \ No newline at end of file From 27dba9ec1bf921eddc6f2dff9f9a8b5cd797032b Mon Sep 17 00:00:00 2001 From: mcy Date: Thu, 25 Oct 2018 10:52:55 +0800 Subject: [PATCH 16/35] =?UTF-8?q?HBase=E6=A0=B9=E6=8D=AEdatasourceKey?= =?UTF-8?q?=E5=AF=BC=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../client/adapter/hbase/HbaseAdapter.java | 47 ++++++++++++++++--- .../hbase/service/HbaseEtlService.java | 9 ++-- .../adapter/launcher/rest/CommonRest.java | 6 +-- 3 files changed, 47 insertions(+), 15 deletions(-) diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java index 88fa9a4a..22731960 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java @@ -85,16 +85,49 @@ public class HbaseAdapter implements OuterAdapter { @Override public EtlResult etl(String task, List params) { + EtlResult etlResult = new EtlResult(); MappingConfig config = hbaseMapping.get(task); - DataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); - if (dataSource != null) { - return HbaseEtlService.importData(dataSource, hbaseTemplate, config, params); + if (config != null) { + DataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + if (dataSource != null) { + return HbaseEtlService.importData(dataSource, hbaseTemplate, config, params); + } else { + etlResult.setSucceeded(false); + etlResult.setErrorMessage("DataSource not found"); + return etlResult; + } } else { - EtlResult etlResult = new EtlResult(); - etlResult.setSucceeded(false); - etlResult.setErrorMessage("DataSource not found"); - return etlResult; + DataSource dataSource = DatasourceConfig.DATA_SOURCES.get(task); + if (dataSource != null) { + StringBuilder resultMsg = new StringBuilder(); + boolean resSucc = true; + // ds不为空说明传入的是datasourceKey + for (MappingConfig configTmp : hbaseMapping.values()) { + // 取所有的datasourceKey为task的配置 + if (configTmp.getDataSourceKey().equals(task)) { + EtlResult etlRes = HbaseEtlService.importData(dataSource, hbaseTemplate, configTmp, params); + if (!etlRes.getSucceeded()) { + resSucc = false; + resultMsg.append(etlRes.getErrorMessage()).append("\n"); + } else { + resultMsg.append(etlRes.getResultMessage()).append("\n"); + } + } + } + if (resultMsg.length() > 0) { + etlResult.setSucceeded(resSucc); + if (resSucc) { + etlResult.setResultMessage(resultMsg.toString()); + } else { + etlResult.setErrorMessage(resultMsg.toString()); + } + return etlResult; + } + } } + etlResult.setSucceeded(false); + etlResult.setErrorMessage("Task not found"); + return etlResult; } @Override diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java index 6b955096..d40278f0 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java @@ -323,12 +323,11 @@ public class HbaseEtlService { rows.clear(); complete = true; } - if (logger.isDebugEnabled()) { - logger.debug("import count:" + i); - } - // System.out.println(i); i++; successCount.incrementAndGet(); + if (logger.isDebugEnabled()) { + logger.debug("successful import count:" + successCount.get()); + } } if (!complete && !rows.isEmpty()) { @@ -338,7 +337,7 @@ public class HbaseEtlService { } catch (Exception e) { logger.error(hbaseOrm.getHbaseTable() + " etl failed! ==>" + e.getMessage(), e); errMsg.add(hbaseOrm.getHbaseTable() + " etl failed! ==>" + e.getMessage()); - throw new RuntimeException(e); + // throw new RuntimeException(e); } return i; }); diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java index 388f42cd..c326c574 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java @@ -23,11 +23,11 @@ public class CommonRest { } /** - * Demo: POST: http://127.0.0.1:8081/etl/hbase/mytest_person2.yml + * Demo: curl http://127.0.0.1:8081/etl/hbase/mytest_person2.yml -X POST -d "params=0,1,2" */ @PostMapping("/etl/{type}/{task}") public EtlResult etl(@PathVariable String type, @PathVariable String task, - @RequestParam(required = false) String params) { + @RequestParam(name = "params", required = false) String params) { OuterAdapter adapter = loader.getExtension(type); List paramArr = null; if (params != null) { @@ -39,7 +39,7 @@ public class CommonRest { } /** - * Demo: GET: http://127.0.0.1:8081/count/hbase/mytest_person2.yml + * Demo: curl http://127.0.0.1:8081/count/hbase/mytest_person2.yml */ @GetMapping("/count/{type}/{task}") public Map count(@PathVariable String type, @PathVariable String task) { From 346d091d003a8b604ae2a462bc88bcabc4a57317 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Thu, 25 Oct 2018 13:11:34 +0800 Subject: [PATCH 17/35] fixed issue #1032 , support raw config --- deployer/src/main/resources/canal.properties | 1 + .../resources/spring/default-instance.xml | 1 + .../main/resources/spring/file-instance.xml | 1 + .../main/resources/spring/group-instance.xml | 1 + .../main/resources/spring/memory-instance.xml | 1 + .../manager/CanalInstanceWithManager.java | 1 + .../manager/model/CanalParameter.java | 9 +++++++++ .../otter/canal/kafka/CanalKafkaProducer.java | 19 ++++++++----------- .../otter/canal/server/CanalMQStarter.java | 5 +++++ 9 files changed, 28 insertions(+), 11 deletions(-) diff --git a/deployer/src/main/resources/canal.properties b/deployer/src/main/resources/canal.properties index ea096de8..2a24b66f 100644 --- a/deployer/src/main/resources/canal.properties +++ b/deployer/src/main/resources/canal.properties @@ -20,6 +20,7 @@ canal.instance.memory.buffer.size = 16384 canal.instance.memory.buffer.memunit = 1024 ## meory store gets mode used MEMSIZE or ITEMSIZE canal.instance.memory.batch.mode = MEMSIZE +canal.instance.memory.rawEntry = true ## detecing config canal.instance.detecting.enable = false diff --git a/deployer/src/main/resources/spring/default-instance.xml b/deployer/src/main/resources/spring/default-instance.xml index 95ec6a29..fe9cc7a5 100644 --- a/deployer/src/main/resources/spring/default-instance.xml +++ b/deployer/src/main/resources/spring/default-instance.xml @@ -58,6 +58,7 @@ + diff --git a/deployer/src/main/resources/spring/file-instance.xml b/deployer/src/main/resources/spring/file-instance.xml index df30174a..23d8e4bd 100644 --- a/deployer/src/main/resources/spring/file-instance.xml +++ b/deployer/src/main/resources/spring/file-instance.xml @@ -44,6 +44,7 @@ + diff --git a/deployer/src/main/resources/spring/group-instance.xml b/deployer/src/main/resources/spring/group-instance.xml index 71788f66..e614b86c 100644 --- a/deployer/src/main/resources/spring/group-instance.xml +++ b/deployer/src/main/resources/spring/group-instance.xml @@ -41,6 +41,7 @@ + diff --git a/deployer/src/main/resources/spring/memory-instance.xml b/deployer/src/main/resources/spring/memory-instance.xml index 751e9f4b..f785a46e 100644 --- a/deployer/src/main/resources/spring/memory-instance.xml +++ b/deployer/src/main/resources/spring/memory-instance.xml @@ -41,6 +41,7 @@ + diff --git a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java index 36774d94..dff18278 100644 --- a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java +++ b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java @@ -147,6 +147,7 @@ public class CanalInstanceWithManager extends AbstractCanalInstance { memoryEventStore.setBufferMemUnit(parameters.getMemoryStorageBufferMemUnit()); memoryEventStore.setBatchMode(BatchMode.valueOf(parameters.getStorageBatchMode().name())); memoryEventStore.setDdlIsolation(parameters.getDdlIsolation()); + memoryEventStore.setRaw(parameters.getMemoryStorageRawEntry()); eventStore = memoryEventStore; } else if (mode.isFile()) { // 后续版本支持 diff --git a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java index ece1196e..ca5e1af7 100644 --- a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java +++ b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java @@ -39,6 +39,7 @@ public class CanalParameter implements Serializable { private BatchMode storageBatchMode = BatchMode.MEMSIZE; // 基于大小返回结果 private Integer memoryStorageBufferSize = 16 * 1024; // 内存存储的buffer大小 private Integer memoryStorageBufferMemUnit = 1024; // 内存存储的buffer内存占用单位，默认为1kb + private Boolean memoryStorageRawEntry = Boolean.TRUE; // 内存存储的对象是否启用raw的ByteString模式 private String fileStorageDirectory; // 文件存储的目录位置 private Integer fileStorageStoreCount; // 每个文件store存储的记录数 private Integer fileStorageRollverCount; // store文件的个数 @@ -955,6 +956,14 @@ public class CanalParameter implements Serializable { this.gtidEnable = gtidEnable; } + public Boolean getMemoryStorageRawEntry() { + return memoryStorageRawEntry; + } + + public void setMemoryStorageRawEntry(Boolean memoryStorageRawEntry) { + this.memoryStorageRawEntry = memoryStorageRawEntry; + } + public String toString() { return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE); } diff --git a/server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java b/server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java index 93f5f4ab..654a4b6b 100644 --- a/server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java +++ b/server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java @@ -1,24 +1,21 @@ package com.alibaba.otter.canal.kafka; -import java.util.ArrayList; import java.util.List; import java.util.Properties; -import java.util.concurrent.Future; -import com.alibaba.fastjson.serializer.SerializerFeature; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.common.serialization.StringSerializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.serializer.SerializerFeature; import com.alibaba.otter.canal.common.MQProperties; import com.alibaba.otter.canal.protocol.FlatMessage; import com.alibaba.otter.canal.protocol.Message; -import com.alibaba.otter.canal.spi.CanalMQProducer;; +import com.alibaba.otter.canal.spi.CanalMQProducer; /** * kafka producer 主操作类 @@ -104,8 +101,10 @@ public class CanalKafkaProducer implements CanalMQProducer { for (FlatMessage flatMessage : flatMessages) { if (canalDestination.getPartition() != null) { try { - ProducerRecord record = new ProducerRecord(canalDestination - .getTopic(), canalDestination.getPartition(), null, JSON.toJSONString(flatMessage)); + ProducerRecord record = new ProducerRecord(canalDestination.getTopic(), + canalDestination.getPartition(), + null, + JSON.toJSONString(flatMessage)); producer2.send(record); } catch (Exception e) { logger.error(e.getMessage(), e); @@ -123,8 +122,7 @@ public class CanalKafkaProducer implements CanalMQProducer { FlatMessage flatMessagePart = partitionFlatMessage[i]; if (flatMessagePart != null) { try { - ProducerRecord record = new ProducerRecord( - canalDestination.getTopic(), + ProducerRecord record = new ProducerRecord(canalDestination.getTopic(), i, null, JSON.toJSONString(flatMessagePart)); @@ -138,8 +136,7 @@ public class CanalKafkaProducer implements CanalMQProducer { } } else { try { - ProducerRecord record = new ProducerRecord( - canalDestination.getTopic(), + ProducerRecord record = new ProducerRecord(canalDestination.getTopic(), 0, null, JSON.toJSONString(flatMessage, SerializerFeature.WriteMapNullValue)); diff --git a/server/src/main/java/com/alibaba/otter/canal/server/CanalMQStarter.java b/server/src/main/java/com/alibaba/otter/canal/server/CanalMQStarter.java index b3ed906f..413c42b2 100644 --- a/server/src/main/java/com/alibaba/otter/canal/server/CanalMQStarter.java +++ b/server/src/main/java/com/alibaba/otter/canal/server/CanalMQStarter.java @@ -57,6 +57,11 @@ public class CanalMQStarter { System.setProperty("canal.instance.filter.transaction.entry", "true"); } + if (properties.getFlatMessage()) { + // 针对flat message模式,设置为raw避免ByteString->Entry的二次解析 + System.setProperty("canal.instance.memory.rawEntry", "false"); + } + // 对应每个instance启动一个worker线程 List destinations = properties.getCanalDestinations(); From 0f1334c92c8ac1f3753a8a6f9934d4570000a3a3 Mon Sep 17 00:00:00 2001 From: mcy Date: Thu, 25 Oct 2018 13:52:52 +0800 Subject: [PATCH 18/35] =?UTF-8?q?=E5=8A=A0=E5=85=A5=E5=90=8C=E6=AD=A5?= =?UTF-8?q?=E5=BC=80=E5=85=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../adapter/support/CanalClientConfig.java | 10 +- .../canal/client/adapter/support/Dml.java | 14 ++- .../client/adapter/support/MessageUtil.java | 22 ++-- .../canal/client/adapter/support/Result.java | 6 +- .../loader/AbstractCanalAdapterWorker.java | 4 +- .../hbase/service/HbaseEtlService.java | 15 +-- .../adapter/launcher/common/SyncSwitch.java | 102 ++++++++++++++++++ .../launcher/config/AdapterCanalConfig.java | 37 ++++++- .../launcher/config/SpringContext.java | 28 +++++ .../loader/AbstractCanalAdapterWorker.java | 24 ++--- .../loader/CanalAdapterKafkaWorker.java | 8 +- .../loader/CanalAdapterRocketMQWorker.java | 10 +- .../launcher/loader/CanalAdapterService.java | 8 +- .../launcher/loader/CanalAdapterWorker.java | 30 +++--- .../adapter/launcher/rest/CommonRest.java | 70 +++++++++++- .../src/main/resources/application.yml | 11 +- 16 files changed, 323 insertions(+), 76 deletions(-) create mode 100644 client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/SyncSwitch.java create mode 100644 client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/SpringContext.java diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java index d7f7582a..08246a42 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java @@ -62,6 +62,10 @@ public class CanalClientConfig { return mqTopics; } + public void setMqTopics(List mqTopics) { + this.mqTopics = mqTopics; + } + public Boolean getFlatMessage() { return flatMessage; } @@ -70,10 +74,6 @@ public class CanalClientConfig { this.flatMessage = flatMessage; } - public void setMqTopics(List mqTopics) { - this.mqTopics = mqTopics; - } - public List getCanalInstances() { return canalInstances; } @@ -155,7 +155,7 @@ public class CanalClientConfig { public static class Group { - private String groupId; + private String groupId; // private List adapters = new ArrayList<>(); diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Dml.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Dml.java index d4cbbf7d..8e5af059 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Dml.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Dml.java @@ -14,6 +14,7 @@ public class Dml implements Serializable { private static final long serialVersionUID = 2611556444074013268L; + private String destination; private String database; private String table; private String type; @@ -25,6 +26,14 @@ public class Dml implements Serializable { private List> data; private List> old; + public String getDestination() { + return destination; + } + + public void setDestination(String destination) { + this.destination = destination; + } + public String getDatabase() { return database; } @@ -102,7 +111,8 @@ public class Dml implements Serializable { @Override public String toString() { - return "Dml [database=" + database + ", table=" + table + ", type=" + type + ", es=" + es + ", ts=" + ts - + ", sql=" + sql + ", data=" + data + ", old=" + old + "]"; + return "Dml{" + "destination='" + destination + '\'' + ", database='" + database + '\'' + ", table='" + table + + '\'' + ", type='" + type + '\'' + ", es=" + es + ", ts=" + ts + ", sql='" + sql + '\'' + ", data=" + + data + ", old=" + old + '}'; } } diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java index 02723b41..826ed917 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java @@ -1,11 +1,6 @@ package com.alibaba.otter.canal.client.adapter.support; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import com.alibaba.otter.canal.protocol.CanalEntry; import com.alibaba.otter.canal.protocol.FlatMessage; @@ -19,7 +14,7 @@ import com.alibaba.otter.canal.protocol.Message; */ public class MessageUtil { - public static void parse4Dml(Message message, Consumer consumer) { + public static void parse4Dml(String destination, Message message, Consumer consumer) { if (message == null) { return; } @@ -42,6 +37,7 @@ public class MessageUtil { CanalEntry.EventType eventType = rowChange.getEventType(); final Dml dml = new Dml(); + dml.setDestination(destination); dml.setDatabase(entry.getHeader().getSchemaName()); dml.setTable(entry.getHeader().getTableName()); dml.setType(eventType.toString()); @@ -87,10 +83,11 @@ public class MessageUtil { Map rowOld = new LinkedHashMap<>(); for (CanalEntry.Column column : rowData.getBeforeColumnsList()) { if (updateSet.contains(column.getName())) { - rowOld.put(column.getName(), JdbcTypeUtil.typeConvert(column.getName(), - column.getValue(), - column.getSqlType(), - column.getMysqlType())); + rowOld.put(column.getName(), + JdbcTypeUtil.typeConvert(column.getName(), + column.getValue(), + column.getSqlType(), + column.getMysqlType())); } } // update操作将记录修改前的值 @@ -110,11 +107,12 @@ public class MessageUtil { } } - public static Dml flatMessage2Dml(FlatMessage flatMessage) { + public static Dml flatMessage2Dml(String destination, FlatMessage flatMessage) { if (flatMessage == null) { return null; } Dml dml = new Dml(); + dml.setDestination(destination); dml.setDatabase(flatMessage.getDatabase()); dml.setTable(flatMessage.getTable()); dml.setType(flatMessage.getType()); diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Result.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Result.java index 0ea913e1..03831f1f 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Result.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Result.java @@ -8,11 +8,11 @@ public class Result implements Serializable { public Integer code = 20000; public Object data; public String message; - public Date sysTime = new Date(); + public Date sysTime; - public static Result createSuccess(Object data) { + public static Result createSuccess(String message) { Result result = new Result(); - result.setData(data); + result.setMessage(message); return result; } diff --git a/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/AbstractCanalAdapterWorker.java b/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/AbstractCanalAdapterWorker.java index cd3f718d..bbfd462d 100644 --- a/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/AbstractCanalAdapterWorker.java +++ b/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/AbstractCanalAdapterWorker.java @@ -52,7 +52,7 @@ public abstract class AbstractCanalAdapterWorker { // 组内适配器穿行运行，尽量不要配置组内适配器 for (final OuterAdapter c : adapters) { long begin = System.currentTimeMillis(); - MessageUtil.parse4Dml(message, new MessageUtil.Consumer() { + MessageUtil.parse4Dml(canalDestination, message, new MessageUtil.Consumer() { @Override public void accept(Dml dml) { @@ -100,7 +100,7 @@ public abstract class AbstractCanalAdapterWorker { // 组内适配器穿行运行，尽量不要配置组内适配器 for (OuterAdapter c : adapters) { long begin = System.currentTimeMillis(); - Dml dml = MessageUtil.flatMessage2Dml(flatMessage); + Dml dml = MessageUtil.flatMessage2Dml(canalDestination, flatMessage); c.sync(dml); if (logger.isDebugEnabled()) { logger.debug("{} elapsed time: {}", diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java index d40278f0..ee110755 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java @@ -30,13 +30,13 @@ public class HbaseEtlService { public static Object sqlRS(DataSource ds, String sql, Function fun) throws SQLException { Connection conn = null; - Statement smt = null; + Statement stmt = null; ResultSet rs = null; try { conn = ds.getConnection(); - smt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); - smt.setFetchSize(Integer.MIN_VALUE); - rs = smt.executeQuery(sql); + stmt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + stmt.setFetchSize(Integer.MIN_VALUE); + rs = stmt.executeQuery(sql); return fun.apply(rs); } finally { if (rs != null) { @@ -46,9 +46,9 @@ public class HbaseEtlService { logger.error(e.getMessage(), e); } } - if (smt != null) { + if (stmt != null) { try { - smt.close(); + stmt.close(); } catch (SQLException e) { logger.error(e.getMessage(), e); } @@ -60,6 +60,9 @@ public class HbaseEtlService { logger.error(e.getMessage(), e); } } + rs = null; + stmt = null; + conn = null; } } diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/SyncSwitch.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/SyncSwitch.java new file mode 100644 index 00000000..2a2f1220 --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/SyncSwitch.java @@ -0,0 +1,102 @@ +package com.alibaba.otter.canal.adapter.launcher.common; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import javax.annotation.PostConstruct; +import javax.annotation.Resource; + +import org.apache.commons.lang.StringUtils; +import org.springframework.stereotype.Component; + +import com.alibaba.otter.canal.adapter.launcher.config.AdapterCanalConfig; +import com.alibaba.otter.canal.common.utils.BooleanMutex; + +@Component +public class SyncSwitch { + + private static final Map LOCAL_LOCK = new ConcurrentHashMap<>(); + + private Mode mode = Mode.LOCAL; + + @Resource + private AdapterCanalConfig adapterCanalConfig; + + @PostConstruct + public void init() { + if (StringUtils.isEmpty(adapterCanalConfig.getZookeeperHosts())) { + mode = Mode.LOCAL; + LOCAL_LOCK.clear(); + for (String destination : AdapterCanalConfig.DESTINATIONS) { + // 对应每个destination注册锁 + LOCAL_LOCK.put(destination, new BooleanMutex(true)); + } + } else { + mode = Mode.DISTRIBUTED; + } + } + + public synchronized void off(String destination) { + if (mode == Mode.LOCAL) { + BooleanMutex mutex = LOCAL_LOCK.get(destination); + if (mutex != null && mutex.state()) { + mutex.set(false); + } + } + } + + public synchronized void on(String destination) { + if (mode == Mode.LOCAL) { + BooleanMutex mutex = LOCAL_LOCK.get(destination); + if (mutex != null && !mutex.state()) { + mutex.set(true); + } + } + } + + public synchronized void release(String destination) { + if (mode == Mode.LOCAL) { + BooleanMutex mutex = LOCAL_LOCK.get(destination); + if (mutex != null && mutex.state()) { + mutex.set(false); + } + } + } + + public Boolean status(String destination) { + if (mode == Mode.LOCAL) { + BooleanMutex mutex = LOCAL_LOCK.get(destination); + if (mutex != null) { + return mutex.state(); + } else { + return null; + } + } + return null; + } + + public void get(String destination) throws InterruptedException { + if (mode == Mode.LOCAL) { + BooleanMutex mutex = LOCAL_LOCK.get(destination); + if (mutex != null) { + mutex.get(); + } + } + } + + public void get(String destination, long timeout, TimeUnit unit) throws InterruptedException, TimeoutException { + if (mode == Mode.LOCAL) { + BooleanMutex mutex = LOCAL_LOCK.get(destination); + if (mutex != null) { + mutex.get(timeout, unit); + } + } + } + + enum Mode { + LOCAL, // 本地模式 + DISTRIBUTED // 分布式 + } +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java index 447f896d..3dcfe4e7 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java @@ -1,10 +1,45 @@ package com.alibaba.otter.canal.adapter.launcher.config; -import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; +import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; + @Component @ConfigurationProperties(prefix = "canal.conf") public class AdapterCanalConfig extends CanalClientConfig { + + public static final Set DESTINATIONS = new LinkedHashSet<>(); + + @Override + public void setCanalInstances(List canalInstances) { + super.setCanalInstances(canalInstances); + + if (canalInstances != null) { + synchronized (DESTINATIONS) { + DESTINATIONS.clear(); + for (CanalInstance canalInstance : canalInstances) { + DESTINATIONS.add(canalInstance.getInstance()); + } + } + } + } + + @Override + public void setMqTopics(List mqTopics) { + super.setMqTopics(mqTopics); + + if (mqTopics != null) { + synchronized (DESTINATIONS) { + DESTINATIONS.clear(); + for (MQTopic mqTopic : mqTopics) { + DESTINATIONS.add(mqTopic.getTopic()); + } + } + } + } } diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/SpringContext.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/SpringContext.java new file mode 100644 index 00000000..ac860bea --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/SpringContext.java @@ -0,0 +1,28 @@ +package com.alibaba.otter.canal.adapter.launcher.config; + +import org.springframework.beans.BeansException; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.stereotype.Component; + +@Component +public class SpringContext implements ApplicationContextAware { + + private static ApplicationContext context; + + /* + * 注入ApplicationContext + */ + public void setApplicationContext(final ApplicationContext context) throws BeansException { + // 在加载Spring时自动获得context + SpringContext.context = context; + } + + public static Object getBean(final String beanName) { + return SpringContext.context.getBean(beanName); + } + + public static Object getBean(final Class clz) { + return context.getBean(clz); + } +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java index e007faa0..bd3a47a4 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java @@ -7,6 +7,8 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch; +import com.alibaba.otter.canal.adapter.launcher.config.SpringContext; import com.alibaba.otter.canal.client.adapter.support.Dml; import com.alibaba.otter.canal.protocol.FlatMessage; import org.slf4j.Logger; @@ -33,6 +35,12 @@ public abstract class AbstractCanalAdapterWorker { protected Thread thread = null; protected Thread.UncaughtExceptionHandler handler = (t, e) -> logger.error("parse events has an error", e); + protected SyncSwitch syncSwitch; + + public AbstractCanalAdapterWorker(){ + syncSwitch = (SyncSwitch) SpringContext.getBean(SyncSwitch.class); + } + protected void writeOut(final Message message) { List> futures = new ArrayList<>(); // 组间适配器并行运行 @@ -43,7 +51,7 @@ public abstract class AbstractCanalAdapterWorker { // 组内适配器穿行运行，尽量不要配置组内适配器 for (final OuterAdapter c : adapters) { long begin = System.currentTimeMillis(); - MessageUtil.parse4Dml(message, c::sync); + MessageUtil.parse4Dml(canalDestination, message, c::sync); if (logger.isDebugEnabled()) { logger.debug("{} elapsed time: {}", @@ -84,7 +92,7 @@ public abstract class AbstractCanalAdapterWorker { // 组内适配器穿行运行，尽量不要配置组内适配器 for (OuterAdapter c : adapters) { long begin = System.currentTimeMillis(); - Dml dml = MessageUtil.flatMessage2Dml(flatMessage); + Dml dml = MessageUtil.flatMessage2Dml(canalDestination, flatMessage); c.sync(dml); if (logger.isDebugEnabled()) { logger.debug("{} elapsed time: {}", @@ -113,18 +121,6 @@ public abstract class AbstractCanalAdapterWorker { } } - protected void writeOut(Message message, String topic) { - if (logger.isDebugEnabled()) { - logger.debug("topic: {} batchId: {} batchSize: {} ", topic, message.getId(), message.getEntries().size()); - } - long begin = System.currentTimeMillis(); - writeOut(message); - long now = System.currentTimeMillis(); - if ((System.currentTimeMillis() - begin) > 5 * 60 * 1000) { - logger.error("topic: {} batchId {} elapsed time: {} ms", topic, message.getId(), now - begin); - } - } - protected void stopOutAdapters() { if (thread != null) { try { diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java index 01f52e8e..22272e80 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java @@ -44,13 +44,7 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker { @Override public void start() { if (!running) { - thread = new Thread(new Runnable() { - - @Override - public void run() { - process(); - } - }); + thread = new Thread(() -> process()); thread.setUncaughtExceptionHandler(handler); running = true; thread.start(); diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java index aa963bbe..09863a33 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java @@ -89,7 +89,15 @@ public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker { @Override public void run() { try { - writeOut(message, topic); + if (logger.isDebugEnabled()) { + logger.debug("topic: {} batchId: {} batchSize: {} ", topic, message.getId(), message.getEntries().size()); + } + long begin = System.currentTimeMillis(); + writeOut(message); + long now = System.currentTimeMillis(); + if ((System.currentTimeMillis() - begin) > 5 * 60 * 1000) { + logger.error("topic: {} batchId {} elapsed time: {} ms", topic, message.getId(), now - begin); + } } catch (Exception e) { logger.error(e.getMessage(), e); } diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterService.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterService.java index f7b9e445..03f515bf 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterService.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterService.java @@ -4,6 +4,8 @@ import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import javax.annotation.Resource; +import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch; +import com.alibaba.otter.canal.adapter.launcher.config.SpringContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -23,9 +25,13 @@ public class CanalAdapterService { @Resource private AdapterCanalConfig adapterCanalConfig; - // 注入配置保证配置优先先注册 + // 注入bean保证优先注册 @Resource private AdapterConfig adapterConfig; + @Resource + private SpringContext springContext; + @Resource + private SyncSwitch syncSwitch; @PostConstruct public void init() { diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java index 224e70d4..b976bb82 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java @@ -3,6 +3,8 @@ package com.alibaba.otter.canal.adapter.launcher.loader; import java.net.SocketAddress; import java.util.List; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import com.alibaba.otter.canal.client.CanalConnector; import com.alibaba.otter.canal.client.CanalConnectors; @@ -59,7 +61,7 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker { @Override public void start() { if (!running) { - thread = new Thread(() -> process()); + thread = new Thread(this::process); thread.setUncaughtExceptionHandler(handler); thread.start(); running = true; @@ -73,13 +75,11 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker { return; } - // if (switcher != null && !switcher.state()) { - // switcher.set(true); - // } - connector.stopRunning(); running = false; + syncSwitch.release(canalDestination); + logger.info("destination {} is waiting for adapters' worker thread die!", canalDestination); if (thread != null) { try { @@ -106,22 +106,22 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker { ; // waiting until running == true while (running) { try { - // if (switcher != null) { - // switcher.get(); - // } + syncSwitch.get(canalDestination); + logger.info("=============> Start to connect destination: {} <=============", this.canalDestination); connector.connect(); logger.info("=============> Start to subscribe destination: {} <=============", this.canalDestination); connector.subscribe(); logger.info("=============> Subscribe destination: {} succeed <=============", this.canalDestination); while (running) { - // try { - // if (switcher != null) { - // switcher.get(); - // } - // } catch (TimeoutException e) { - // break; - // } + try { + syncSwitch.get(canalDestination, 1L, TimeUnit.MINUTES); + } catch (TimeoutException e) { + break; + } + if (!running) { + break; + } // server配置canal.instance.network.soTimeout(默认: 30s) // 范围内未与server交互，server将关闭本次socket连接 diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java index c326c574..4e4aa7cb 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java @@ -1,29 +1,39 @@ package com.alibaba.otter.canal.adapter.launcher.rest; -import java.util.Arrays; -import java.util.List; -import java.util.Map; +import java.util.*; import javax.annotation.PostConstruct; +import javax.annotation.Resource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.web.bind.annotation.*; +import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch; +import com.alibaba.otter.canal.adapter.launcher.config.AdapterCanalConfig; import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.adapter.support.EtlResult; import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader; +import com.alibaba.otter.canal.client.adapter.support.Result; @RestController public class CommonRest { + private static Logger logger = LoggerFactory.getLogger(CommonRest.class); + private ExtensionLoader loader; + @Resource + private SyncSwitch syncSwitch; + @PostConstruct public void init() { loader = ExtensionLoader.getExtensionLoader(OuterAdapter.class); } /** - * Demo: curl http://127.0.0.1:8081/etl/hbase/mytest_person2.yml -X POST -d "params=0,1,2" + * Demo: curl http://127.0.0.1:8081/etl/hbase/mytest_person2.yml -X POST -d + * "params=0,1,2" */ @PostMapping("/etl/{type}/{task}") public EtlResult etl(@PathVariable String type, @PathVariable String task, @@ -46,4 +56,56 @@ public class CommonRest { OuterAdapter adapter = loader.getExtension(type); return adapter.count(task); } + + @GetMapping("/destinations") + public List> destinations() { + List> result = new ArrayList<>(); + Set destinations = AdapterCanalConfig.DESTINATIONS; + for (String destination : destinations) { + Map resMap = new LinkedHashMap<>(); + Boolean status = syncSwitch.status(destination); + String resStatus = "none"; + if (status != null && status) { + resStatus = "on"; + } else if (status != null && !status) { + resStatus = "off"; + } + resMap.put("destination", destination); + resMap.put("status", resStatus); + result.add(resMap); + } + return result; + } + + @PutMapping("/syncSwitch/{destination}/{status}") + public Result etl(@PathVariable String destination, @PathVariable String status) { + if (status.equals("on")) { + syncSwitch.on(destination); + logger.info("#Destination: {} sync on", destination); + return Result.createSuccess("实例: " + destination + " 开启同步成功"); + } else if (status.equals("off")) { + syncSwitch.off(destination); + logger.info("#Destination: {} sync off", destination); + return Result.createSuccess("实例: " + destination + " 关闭同步成功"); + } else { + Result result = new Result(); + result.setCode(50000); + result.setMessage("实例: " + destination + " 操作失败"); + return result; + } + } + + @GetMapping("/syncSwitch/{destination}") + public Map etl(@PathVariable String destination) { + Boolean status = syncSwitch.status(destination); + String resStatus = "none"; + if (status != null && status) { + resStatus = "on"; + } else if (status != null && !status) { + resStatus = "off"; + } + Map res = new LinkedHashMap<>(); + res.put("stauts", resStatus); + return res; + } } diff --git a/client-adapter/launcher/src/main/resources/application.yml b/client-adapter/launcher/src/main/resources/application.yml index b61a4749..83ccb66d 100644 --- a/client-adapter/launcher/src/main/resources/application.yml +++ b/client-adapter/launcher/src/main/resources/application.yml @@ -9,6 +9,11 @@ spring: time-zone: GMT+8 default-property-inclusion: non_null + +hbase.zookeeper.quorum: slave1 +hbase.zookeeper.property.clientPort: 2181 +hbase.zookeeper.znode.parent: /hbase-unsecure + canal.conf: canalServerHost: 127.0.0.1:11111 # zookeeperHosts: slave1:2181 @@ -21,9 +26,9 @@ canal.conf: - name: logger - name: hbase properties: - hbase.zookeeper.quorum: slave1 - hbase.zookeeper.property.clientPort: 2181 - zookeeper.znode.parent: /hbase-unsecure + hbase.zookeeper.quorum: ${hbase.zookeeper.quorum} + hbase.zookeeper.property.clientPort: ${hbase.zookeeper.property.clientPort} + zookeeper.znode.parent: ${hbase.zookeeper.znode.parent} # mqTopics: # - mqMode: kafka # topic: example From 42a4abe4c7ff6abb0d57adfbac4afee955a88bf7 Mon Sep 17 00:00:00 2001 From: mcy Date: Thu, 25 Oct 2018 14:24:33 +0800 Subject: [PATCH 19/35] =?UTF-8?q?HBase=20etl=E9=94=81=E5=AE=9A=E5=90=8C?= =?UTF-8?q?=E6=AD=A5=E7=AD=89=E5=BE=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../canal/client/adapter/OuterAdapter.java | 10 ++++ .../client/adapter/hbase/HbaseAdapter.java | 9 ++++ .../adapter/hbase/config/MappingConfig.java | 9 ++++ .../main/resources/hbase/mytest_person2.yml | 1 + .../adapter/launcher/rest/CommonRest.java | 53 +++++++++++++++---- 5 files changed, 73 insertions(+), 9 deletions(-) diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java index bf6d5fce..286ba398 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java @@ -55,4 +55,14 @@ public interface OuterAdapter { default Map count(String task) { throw new UnsupportedOperationException("unsupported operation"); } + + /** + * 通过task获取对应的destination + * + * @param task 任务名, 对应配置名 + * @return destination + */ + default String getDestination(String task) { + return null; + } } diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java index 22731960..50eb271c 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java @@ -162,4 +162,13 @@ public class HbaseAdapter implements OuterAdapter { } } } + + @Override + public String getDestination(String task) { + MappingConfig config = hbaseMapping.get(task); + if (config != null && config.getHbaseOrm() != null) { + return config.getHbaseOrm().getDestination(); + } + return null; + } } diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java index c3c30c1b..a5ea5707 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java @@ -146,6 +146,7 @@ public class MappingConfig { public static class HbaseOrm { private Mode mode = Mode.STRING; + private String destination; private String database; private String table; private String hbaseTable; @@ -170,6 +171,14 @@ public class MappingConfig { this.mode = mode; } + public String getDestination() { + return destination; + } + + public void setDestination(String destination) { + this.destination = destination; + } + public String getDatabase() { return database; } diff --git a/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml b/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml index cbf14d48..88cd99e2 100644 --- a/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml +++ b/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml @@ -1,6 +1,7 @@ dataSourceKey: defaultDS hbaseOrm: mode: PHOENIX #NATIVE #STRING + destination: example database: mytest # 数据库名 table: person2 # 数据库表名 hbaseTable: MYTEST.PERSON2 # HBase表名 diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java index 4e4aa7cb..61c2b47b 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java @@ -32,24 +32,43 @@ public class CommonRest { } /** - * Demo: curl http://127.0.0.1:8081/etl/hbase/mytest_person2.yml -X POST -d - * "params=0,1,2" + * ETL curl http://127.0.0.1:8081/etl/hbase/mytest_person2.yml -X POST + * + * @param type 类型 hbase, es + * @param task 任务名对应配置文件名 mytest_person2.yml + * @param params etl where条件参数, 为空全部导入 + * @return */ @PostMapping("/etl/{type}/{task}") public EtlResult etl(@PathVariable String type, @PathVariable String task, @RequestParam(name = "params", required = false) String params) { OuterAdapter adapter = loader.getExtension(type); - List paramArr = null; - if (params != null) { - String[] parmaArray = params.trim().split(";"); - paramArr = Arrays.asList(parmaArray); + String destination = adapter.getDestination(task); + Boolean oriSwithcStatus = null; + if (destination != null) { + oriSwithcStatus = syncSwitch.status(destination); + syncSwitch.off(destination); + } + try { + List paramArr = null; + if (params != null) { + String[] parmaArray = params.trim().split(";"); + paramArr = Arrays.asList(parmaArray); + } + return adapter.etl(task, paramArr); + } finally { + if (destination != null && oriSwithcStatus != null && oriSwithcStatus) { + syncSwitch.on(destination); + } } - - return adapter.etl(task, paramArr); } /** - * Demo: curl http://127.0.0.1:8081/count/hbase/mytest_person2.yml + * 统计总数 curl http://127.0.0.1:8081/count/hbase/mytest_person2.yml + * + * @param type 类型 hbase, es + * @param task 任务名对应配置文件名 mytest_person2.yml + * @return */ @GetMapping("/count/{type}/{task}") public Map count(@PathVariable String type, @PathVariable String task) { @@ -57,6 +76,9 @@ public class CommonRest { return adapter.count(task); } + /** + * 返回所有实例 curl http://127.0.0.1:8081/destinations + */ @GetMapping("/destinations") public List> destinations() { List> result = new ArrayList<>(); @@ -77,6 +99,13 @@ public class CommonRest { return result; } + /** + * 实例同步开关 curl http://127.0.0.1:8081/syncSwitch/example/off -X PUT + * + * @param destination 实例名称 + * @param status 开关状态: off on + * @return + */ @PutMapping("/syncSwitch/{destination}/{status}") public Result etl(@PathVariable String destination, @PathVariable String status) { if (status.equals("on")) { @@ -95,6 +124,12 @@ public class CommonRest { } } + /** + * 获取实例开关状态 curl http://127.0.0.1:8081/syncSwitch/example + * + * @param destination 实例名称 + * @return + */ @GetMapping("/syncSwitch/{destination}") public Map etl(@PathVariable String destination) { Boolean status = syncSwitch.status(destination); From 6ee6b3c3980f422921dce524fdf806286572589f Mon Sep 17 00:00:00 2001 From: mcy Date: Thu, 25 Oct 2018 16:43:33 +0800 Subject: [PATCH 20/35] =?UTF-8?q?=E5=90=8C=E6=AD=A5=E5=88=86=E5=B8=83?= =?UTF-8?q?=E5=BC=8F=E5=BC=80=E5=85=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client-adapter/common/pom.xml | 5 - .../client/adapter/hbase/HbaseAdapter.java | 7 +- client-adapter/launcher/pom.xml | 5 + .../adapter/launcher/common/SyncSwitch.java | 130 ++++++++++++++++-- .../launcher/config/AdapterCanalConfig.java | 2 +- .../launcher/config/CuratorClient.java | 36 +++++ .../launcher/loader/CanalAdapterLoader.java | 8 +- .../launcher/loader/CanalAdapterWorker.java | 2 - .../adapter/launcher/rest/CommonRest.java | 5 +- .../src/main/resources/application.yml | 5 +- 10 files changed, 178 insertions(+), 27 deletions(-) create mode 100644 client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/CuratorClient.java diff --git a/client-adapter/common/pom.xml b/client-adapter/common/pom.xml index d61d8d11..eb3a21a3 100644 --- a/client-adapter/common/pom.xml +++ b/client-adapter/common/pom.xml @@ -18,11 +18,6 @@ canal.protocol ${canal_version} - - org.slf4j - slf4j-api - 1.7.12 - joda-time joda-time diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java index 50eb271c..ababeb6e 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java @@ -8,6 +8,7 @@ import java.util.Map; import javax.sql.DataSource; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.TableName; @@ -52,7 +53,8 @@ public class HbaseAdapter implements OuterAdapter { hbaseMapping = MappingConfigLoader.load(); mappingConfigCache = new HashMap<>(); for (MappingConfig mappingConfig : hbaseMapping.values()) { - mappingConfigCache.put(mappingConfig.getHbaseOrm().getDatabase() + "-" + mappingConfigCache.put(StringUtils.trimToEmpty(mappingConfig.getHbaseOrm().getDestination()) + + "." + mappingConfig.getHbaseOrm().getDatabase() + "." + mappingConfig.getHbaseOrm().getTable(), mappingConfig); } @@ -77,9 +79,10 @@ public class HbaseAdapter implements OuterAdapter { if (dml == null) { return; } + String destination = StringUtils.trimToEmpty(dml.getDestination()); String database = dml.getDatabase(); String table = dml.getTable(); - MappingConfig config = mappingConfigCache.get(database + "-" + table); + MappingConfig config = mappingConfigCache.get(destination + "." + database + "." + table); hbaseSyncService.sync(config, dml); } diff --git a/client-adapter/launcher/pom.xml b/client-adapter/launcher/pom.xml index 7d54cd58..0f39411e 100644 --- a/client-adapter/launcher/pom.xml +++ b/client-adapter/launcher/pom.xml @@ -50,6 +50,11 @@ spring-boot-configuration-processor true + + org.apache.curator + curator-recipes + 2.10.0 + org.apache.rocketmq diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/SyncSwitch.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/SyncSwitch.java index 2a2f1220..e9e647fd 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/SyncSwitch.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/SyncSwitch.java @@ -1,5 +1,6 @@ package com.alibaba.otter.canal.adapter.launcher.common; +import java.nio.charset.StandardCharsets; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; @@ -8,42 +9,114 @@ import java.util.concurrent.TimeoutException; import javax.annotation.PostConstruct; import javax.annotation.Resource; -import org.apache.commons.lang.StringUtils; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.recipes.cache.NodeCache; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.data.Stat; import org.springframework.stereotype.Component; import com.alibaba.otter.canal.adapter.launcher.config.AdapterCanalConfig; +import com.alibaba.otter.canal.adapter.launcher.config.CuratorClient; import com.alibaba.otter.canal.common.utils.BooleanMutex; @Component public class SyncSwitch { - private static final Map LOCAL_LOCK = new ConcurrentHashMap<>(); + private static final String SYN_SWITCH_ZKNODE = "/sync-switch/"; - private Mode mode = Mode.LOCAL; + private static final Map LOCAL_LOCK = new ConcurrentHashMap<>(); + + private static final Map DISTRIBUTED_LOCK = new ConcurrentHashMap<>(); + + private static Mode mode = Mode.LOCAL; @Resource private AdapterCanalConfig adapterCanalConfig; + @Resource + private CuratorClient curatorClient; @PostConstruct public void init() { - if (StringUtils.isEmpty(adapterCanalConfig.getZookeeperHosts())) { + CuratorFramework curator = curatorClient.getCurator(); + if (curator != null) { + mode = Mode.DISTRIBUTED; + DISTRIBUTED_LOCK.clear(); + for (String destination : adapterCanalConfig.DESTINATIONS) { + // 对应每个destination注册锁 + BooleanMutex mutex = new BooleanMutex(true); + initMutex(curator, destination, mutex); + DISTRIBUTED_LOCK.put(destination, mutex); + startListen(destination, mutex); + } + } else { mode = Mode.LOCAL; LOCAL_LOCK.clear(); - for (String destination : AdapterCanalConfig.DESTINATIONS) { + for (String destination : adapterCanalConfig.DESTINATIONS) { // 对应每个destination注册锁 LOCAL_LOCK.put(destination, new BooleanMutex(true)); } - } else { - mode = Mode.DISTRIBUTED; + } + } + + private synchronized void startListen(String destination, BooleanMutex mutex) { + try { + String path = SYN_SWITCH_ZKNODE + destination; + CuratorFramework curator = curatorClient.getCurator(); + final NodeCache nodeCache = new NodeCache(curator, path); + nodeCache.start(); + nodeCache.getListenable().addListener(() -> initMutex(curator, destination, mutex)); + } catch (Exception e) { + throw new RuntimeException(e.getMessage()); + } + } + + private synchronized void initMutex(CuratorFramework curator, String destination, BooleanMutex mutex) { + try { + String path = SYN_SWITCH_ZKNODE + destination; + Stat stat = curator.checkExists().forPath(path); + if (stat == null) { + if (!mutex.state()) { + mutex.set(true); + } + } else { + String data = new String(curator.getData().forPath(path), StandardCharsets.UTF_8); + if ("on".equals(data)) { + if (!mutex.state()) { + mutex.set(true); + } + } else { + if (mutex.state()) { + mutex.set(false); + } + } + } + } catch (Exception e) { + throw new RuntimeException(e.getMessage()); } } public synchronized void off(String destination) { + if (mode == Mode.LOCAL) { BooleanMutex mutex = LOCAL_LOCK.get(destination); if (mutex != null && mutex.state()) { mutex.set(false); } + } else { + try { + String path = SYN_SWITCH_ZKNODE + destination; + try { + curatorClient.getCurator() + .create() + .creatingParentContainersIfNeeded() + .withMode(CreateMode.PERSISTENT) + .forPath(path, "off".getBytes(StandardCharsets.UTF_8)); + } catch (Exception e) { + curatorClient.getCurator().setData().forPath(path, "off".getBytes(StandardCharsets.UTF_8)); + } + } catch (Exception e) { + throw new RuntimeException(e); + } } } @@ -53,14 +126,35 @@ public class SyncSwitch { if (mutex != null && !mutex.state()) { mutex.set(true); } + } else { + try { + String path = SYN_SWITCH_ZKNODE + destination; + try { + curatorClient.getCurator() + .create() + .creatingParentContainersIfNeeded() + .withMode(CreateMode.PERSISTENT) + .forPath(path, "on".getBytes(StandardCharsets.UTF_8)); + } catch (Exception e) { + curatorClient.getCurator().setData().forPath(path, "on".getBytes(StandardCharsets.UTF_8)); + } + } catch (Exception e) { + throw new RuntimeException(e); + } } } public synchronized void release(String destination) { if (mode == Mode.LOCAL) { BooleanMutex mutex = LOCAL_LOCK.get(destination); - if (mutex != null && mutex.state()) { - mutex.set(false); + if (mutex != null && !mutex.state()) { + mutex.set(true); + } + } + if (mode == Mode.DISTRIBUTED) { + BooleanMutex mutex = DISTRIBUTED_LOCK.get(destination); + if (mutex != null && !mutex.state()) { + mutex.set(true); } } } @@ -73,8 +167,14 @@ public class SyncSwitch { } else { return null; } + } else { + BooleanMutex mutex = DISTRIBUTED_LOCK.get(destination); + if (mutex != null) { + return mutex.state(); + } else { + return null; + } } - return null; } public void get(String destination) throws InterruptedException { @@ -83,6 +183,11 @@ public class SyncSwitch { if (mutex != null) { mutex.get(); } + } else { + BooleanMutex mutex = DISTRIBUTED_LOCK.get(destination); + if (mutex != null) { + mutex.get(); + } } } @@ -92,6 +197,11 @@ public class SyncSwitch { if (mutex != null) { mutex.get(timeout, unit); } + } else { + BooleanMutex mutex = DISTRIBUTED_LOCK.get(destination); + if (mutex != null) { + mutex.get(timeout, unit); + } } } diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java index 3dcfe4e7..18a12d7c 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java @@ -13,7 +13,7 @@ import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; @ConfigurationProperties(prefix = "canal.conf") public class AdapterCanalConfig extends CanalClientConfig { - public static final Set DESTINATIONS = new LinkedHashSet<>(); + public final Set DESTINATIONS = new LinkedHashSet<>(); @Override public void setCanalInstances(List canalInstances) { diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/CuratorClient.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/CuratorClient.java new file mode 100644 index 00000000..b7944852 --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/CuratorClient.java @@ -0,0 +1,36 @@ +package com.alibaba.otter.canal.adapter.launcher.config; + +import javax.annotation.PostConstruct; +import javax.annotation.Resource; + +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.springframework.stereotype.Component; + +@Component +public class CuratorClient { + + @Resource + private AdapterCanalConfig adapterCanalConfig; + + private CuratorFramework curator = null; + + @PostConstruct + public void init() { + if (adapterCanalConfig.getZookeeperHosts() != null) { + curator = CuratorFrameworkFactory.builder() + .connectString(adapterCanalConfig.getZookeeperHosts()) + .retryPolicy(new ExponentialBackoffRetry(1000, 3)) + .sessionTimeoutMs(6000) + .connectionTimeoutMs(3000) + .namespace("canal-adapter") + .build(); + curator.start(); + } + } + + public CuratorFramework getCurator() { + return curator; + } +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java index d9cdaa7c..3bb5b4a4 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java @@ -15,8 +15,8 @@ import org.slf4j.LoggerFactory; import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; -import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader; +import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; /** * 外部适配器的加载器 @@ -66,10 +66,12 @@ public class CanalAdapterLoader { canalOuterAdapterGroups.add(canalOutConnectors); } CanalAdapterWorker worker; - if (zkHosts != null) { + if (sa != null) { + worker = new CanalAdapterWorker(instance.getInstance(), sa, canalOuterAdapterGroups); + } else if (zkHosts != null) { worker = new CanalAdapterWorker(instance.getInstance(), zkHosts, canalOuterAdapterGroups); } else { - worker = new CanalAdapterWorker(instance.getInstance(), sa, canalOuterAdapterGroups); + throw new RuntimeException("No canal server connector found"); } canalWorkers.put(instance.getInstance(), worker); worker.start(); diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java index b976bb82..1c324996 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java @@ -54,8 +54,6 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker { groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size()); connector = CanalConnectors.newClusterConnector(zookeeperHosts, canalDestination, "", ""); ((ClusterCanalConnector) connector).setSoTimeout(SO_TIMEOUT); - - // super.initSwitcher(canalDestination); } @Override diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java index 61c2b47b..d08af780 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java @@ -26,6 +26,9 @@ public class CommonRest { @Resource private SyncSwitch syncSwitch; + @Resource + private AdapterCanalConfig adapterCanalConfig; + @PostConstruct public void init() { loader = ExtensionLoader.getExtensionLoader(OuterAdapter.class); @@ -82,7 +85,7 @@ public class CommonRest { @GetMapping("/destinations") public List> destinations() { List> result = new ArrayList<>(); - Set destinations = AdapterCanalConfig.DESTINATIONS; + Set destinations = adapterCanalConfig.DESTINATIONS; for (String destination : destinations) { Map resMap = new LinkedHashMap<>(); Boolean status = syncSwitch.status(destination); diff --git a/client-adapter/launcher/src/main/resources/application.yml b/client-adapter/launcher/src/main/resources/application.yml index 83ccb66d..9030cb71 100644 --- a/client-adapter/launcher/src/main/resources/application.yml +++ b/client-adapter/launcher/src/main/resources/application.yml @@ -16,14 +16,14 @@ hbase.zookeeper.znode.parent: /hbase-unsecure canal.conf: canalServerHost: 127.0.0.1:11111 -# zookeeperHosts: slave1:2181 + zookeeperHosts: slave1:2181 # bootstrapServers: slave1:6667 #or rocketmq nameservers:host1:9876;host2:9876 flatMessage: true canalInstances: - instance: example adapterGroups: - outAdapters: - - name: logger +# - name: logger - name: hbase properties: hbase.zookeeper.quorum: ${hbase.zookeeper.quorum} @@ -45,4 +45,3 @@ adapter.conf: password: 121212 adapterConfigs: - hbase/mytest_person2.yml -# - hbase/mytest.person2@defaultDS From c3402a97f377b3cb8dc1d69a5fab027f293088ee Mon Sep 17 00:00:00 2001 From: mcy Date: Thu, 25 Oct 2018 17:34:15 +0800 Subject: [PATCH 21/35] =?UTF-8?q?etl=20=E5=A2=9E=E5=8A=A0=E5=90=8C?= =?UTF-8?q?=E6=AD=A5=E9=94=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../adapter/launcher/common/EtlLock.java | 110 ++++++++++++++++++ .../canal/adapter/launcher/common/Mode.java | 6 + .../adapter/launcher/common/SyncSwitch.java | 16 +-- .../adapter/launcher/rest/CommonRest.java | 49 +++++--- .../src/main/resources/application.yml | 30 ++--- 5 files changed, 172 insertions(+), 39 deletions(-) create mode 100644 client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/EtlLock.java create mode 100644 client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/Mode.java diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/EtlLock.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/EtlLock.java new file mode 100644 index 00000000..35ba5ddf --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/EtlLock.java @@ -0,0 +1,110 @@ +package com.alibaba.otter.canal.adapter.launcher.common; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; + +import javax.annotation.PostConstruct; +import javax.annotation.Resource; + +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.recipes.locks.InterProcessMutex; +import org.springframework.stereotype.Component; + +import com.alibaba.otter.canal.adapter.launcher.config.CuratorClient; + +@Component +public class EtlLock { + + private static final Map LOCAL_LOCK = new ConcurrentHashMap<>(); + + private static final Map DISTRIBUTED_LOCK = new ConcurrentHashMap<>(); + + private static Mode mode = Mode.LOCAL; + + @Resource + private CuratorClient curatorClient; + + @PostConstruct + public void init() { + CuratorFramework curator = curatorClient.getCurator(); + if (curator != null) { + mode = Mode.DISTRIBUTED; + } else { + mode = Mode.LOCAL; + } + } + + private ReentrantLock getLock(String key) { + ReentrantLock lock = LOCAL_LOCK.get(key); + if (lock == null) { + synchronized (EtlLock.class) { + lock = LOCAL_LOCK.get(key); + if (lock == null) { + lock = new ReentrantLock(); + LOCAL_LOCK.put(key, lock); + } + } + } + return lock; + } + + private InterProcessMutex getRemoteLock(String key) { + InterProcessMutex lock = DISTRIBUTED_LOCK.get(key); + if (lock == null) { + synchronized (EtlLock.class) { + lock = DISTRIBUTED_LOCK.get(key); + if (lock == null) { + lock = new InterProcessMutex(curatorClient.getCurator(), key); + DISTRIBUTED_LOCK.put(key, lock); + } + } + } + return lock; + } + + public void lock(String key) throws Exception { + if (mode == Mode.LOCAL) { + getLock(key).lock(); + } else { + InterProcessMutex lock = getRemoteLock(key); + lock.acquire(); + } + } + + public boolean tryLock(String key, long timeout, TimeUnit unit) { + try { + if (mode == Mode.LOCAL) { + return getLock(key).tryLock(timeout, unit); + } else { + InterProcessMutex lock = getRemoteLock(key); + return lock.acquire(timeout, unit); + } + } catch (Exception e) { + return false; + } + } + + public boolean tryLock(String key) { + try { + if (mode == Mode.LOCAL) { + return getLock(key).tryLock(); + } else { + InterProcessMutex lock = getRemoteLock(key); + return lock.acquire(500, TimeUnit.MILLISECONDS); + } + } catch (Exception e) { + return false; + } + } + + public void unlock(String key) throws Exception { + if (mode == Mode.LOCAL) { + getLock(key).unlock(); + } else { + InterProcessMutex lock = getRemoteLock(key); + lock.release(); + } + } +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/Mode.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/Mode.java new file mode 100644 index 00000000..ac73b85e --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/Mode.java @@ -0,0 +1,6 @@ +package com.alibaba.otter.canal.adapter.launcher.common; + +public enum Mode { + LOCAL, // 本地模式 + DISTRIBUTED // 分布式 +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/SyncSwitch.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/SyncSwitch.java index e9e647fd..d96c753e 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/SyncSwitch.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/SyncSwitch.java @@ -22,7 +22,7 @@ import com.alibaba.otter.canal.common.utils.BooleanMutex; @Component public class SyncSwitch { - private static final String SYN_SWITCH_ZKNODE = "/sync-switch/"; + private static final String SYN_SWITCH_ZK_NODE = "/sync-switch/"; private static final Map LOCAL_LOCK = new ConcurrentHashMap<>(); @@ -60,7 +60,7 @@ public class SyncSwitch { private synchronized void startListen(String destination, BooleanMutex mutex) { try { - String path = SYN_SWITCH_ZKNODE + destination; + String path = SYN_SWITCH_ZK_NODE + destination; CuratorFramework curator = curatorClient.getCurator(); final NodeCache nodeCache = new NodeCache(curator, path); nodeCache.start(); @@ -72,7 +72,7 @@ public class SyncSwitch { private synchronized void initMutex(CuratorFramework curator, String destination, BooleanMutex mutex) { try { - String path = SYN_SWITCH_ZKNODE + destination; + String path = SYN_SWITCH_ZK_NODE + destination; Stat stat = curator.checkExists().forPath(path); if (stat == null) { if (!mutex.state()) { @@ -96,7 +96,6 @@ public class SyncSwitch { } public synchronized void off(String destination) { - if (mode == Mode.LOCAL) { BooleanMutex mutex = LOCAL_LOCK.get(destination); if (mutex != null && mutex.state()) { @@ -104,7 +103,7 @@ public class SyncSwitch { } } else { try { - String path = SYN_SWITCH_ZKNODE + destination; + String path = SYN_SWITCH_ZK_NODE + destination; try { curatorClient.getCurator() .create() @@ -128,7 +127,7 @@ public class SyncSwitch { } } else { try { - String path = SYN_SWITCH_ZKNODE + destination; + String path = SYN_SWITCH_ZK_NODE + destination; try { curatorClient.getCurator() .create() @@ -205,8 +204,5 @@ public class SyncSwitch { } } - enum Mode { - LOCAL, // 本地模式 - DISTRIBUTED // 分布式 - } + } diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java index d08af780..0c4392a3 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java @@ -9,6 +9,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.web.bind.annotation.*; +import com.alibaba.otter.canal.adapter.launcher.common.EtlLock; import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch; import com.alibaba.otter.canal.adapter.launcher.config.AdapterCanalConfig; import com.alibaba.otter.canal.client.adapter.OuterAdapter; @@ -19,12 +20,16 @@ import com.alibaba.otter.canal.client.adapter.support.Result; @RestController public class CommonRest { - private static Logger logger = LoggerFactory.getLogger(CommonRest.class); + private static Logger logger = LoggerFactory.getLogger(CommonRest.class); + + private static final String ETL_LOCK_ZK_NODE = "/sync-etl/"; private ExtensionLoader loader; @Resource private SyncSwitch syncSwitch; + @Resource + private EtlLock etlLock; @Resource private AdapterCanalConfig adapterCanalConfig; @@ -45,23 +50,39 @@ public class CommonRest { @PostMapping("/etl/{type}/{task}") public EtlResult etl(@PathVariable String type, @PathVariable String task, @RequestParam(name = "params", required = false) String params) { - OuterAdapter adapter = loader.getExtension(type); - String destination = adapter.getDestination(task); - Boolean oriSwithcStatus = null; - if (destination != null) { - oriSwithcStatus = syncSwitch.status(destination); - syncSwitch.off(destination); + + boolean locked = etlLock.tryLock(ETL_LOCK_ZK_NODE + type + "-" + task); + if (!locked) { + EtlResult result = new EtlResult(); + result.setSucceeded(false); + result.setErrorMessage(task + " 有其他进程正在导入中, 请稍后再试"); + return result; } try { - List paramArr = null; - if (params != null) { - String[] parmaArray = params.trim().split(";"); - paramArr = Arrays.asList(parmaArray); + OuterAdapter adapter = loader.getExtension(type); + String destination = adapter.getDestination(task); + Boolean oriSwithcStatus = null; + if (destination != null) { + oriSwithcStatus = syncSwitch.status(destination); + syncSwitch.off(destination); + } + try { + List paramArr = null; + if (params != null) { + String[] parmaArray = params.trim().split(";"); + paramArr = Arrays.asList(parmaArray); + } + return adapter.etl(task, paramArr); + } finally { + if (destination != null && oriSwithcStatus != null && oriSwithcStatus) { + syncSwitch.on(destination); + } } - return adapter.etl(task, paramArr); } finally { - if (destination != null && oriSwithcStatus != null && oriSwithcStatus) { - syncSwitch.on(destination); + try { + etlLock.unlock(ETL_LOCK_ZK_NODE + type + "-" + task); + } catch (Exception e) { + logger.error(e.getMessage(), e); } } } diff --git a/client-adapter/launcher/src/main/resources/application.yml b/client-adapter/launcher/src/main/resources/application.yml index 9030cb71..43cc4e38 100644 --- a/client-adapter/launcher/src/main/resources/application.yml +++ b/client-adapter/launcher/src/main/resources/application.yml @@ -16,19 +16,19 @@ hbase.zookeeper.znode.parent: /hbase-unsecure canal.conf: canalServerHost: 127.0.0.1:11111 - zookeeperHosts: slave1:2181 +# zookeeperHosts: slave1:2181 # bootstrapServers: slave1:6667 #or rocketmq nameservers:host1:9876;host2:9876 flatMessage: true canalInstances: - instance: example adapterGroups: - outAdapters: -# - name: logger - - name: hbase - properties: - hbase.zookeeper.quorum: ${hbase.zookeeper.quorum} - hbase.zookeeper.property.clientPort: ${hbase.zookeeper.property.clientPort} - zookeeper.znode.parent: ${hbase.zookeeper.znode.parent} + - name: logger +# - name: hbase +# properties: +# hbase.zookeeper.quorum: ${hbase.zookeeper.quorum} +# hbase.zookeeper.property.clientPort: ${hbase.zookeeper.property.clientPort} +# zookeeper.znode.parent: ${hbase.zookeeper.znode.parent} # mqTopics: # - mqMode: kafka # topic: example @@ -37,11 +37,11 @@ canal.conf: # outAdapters: # - name: logger -adapter.conf: - datasourceConfigs: - defaultDS: - url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true - username: root - password: 121212 - adapterConfigs: - - hbase/mytest_person2.yml +#adapter.conf: +# datasourceConfigs: +# defaultDS: +# url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true +# username: root +# password: 121212 +# adapterConfigs: +# - hbase/mytest_person2.yml From ddfc2464f379a66663720294a341ba533f1c7d83 Mon Sep 17 00:00:00 2001 From: mcy Date: Thu, 25 Oct 2018 17:43:11 +0800 Subject: [PATCH 22/35] =?UTF-8?q?etl=20=E5=A2=9E=E5=8A=A0=E5=90=8C?= =?UTF-8?q?=E6=AD=A5=E9=94=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../otter/canal/adapter/launcher/common/EtlLock.java | 8 ++++++-- .../otter/canal/adapter/launcher/rest/CommonRest.java | 6 +----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/EtlLock.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/EtlLock.java index 35ba5ddf..67d0ac41 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/EtlLock.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/EtlLock.java @@ -99,12 +99,16 @@ public class EtlLock { } } - public void unlock(String key) throws Exception { + public void unlock(String key) { if (mode == Mode.LOCAL) { getLock(key).unlock(); } else { InterProcessMutex lock = getRemoteLock(key); - lock.release(); + try { + lock.release(); + } catch (Exception e) { + // ignore + } } } } diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java index 0c4392a3..bee78e04 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java @@ -79,11 +79,7 @@ public class CommonRest { } } } finally { - try { - etlLock.unlock(ETL_LOCK_ZK_NODE + type + "-" + task); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } + etlLock.unlock(ETL_LOCK_ZK_NODE + type + "-" + task); } } From d70dc8b88c5cd1a5506b191226fc89d3b8e72244 Mon Sep 17 00:00:00 2001 From: mcy Date: Thu, 25 Oct 2018 21:21:06 +0800 Subject: [PATCH 23/35] =?UTF-8?q?=E8=A7=84=E8=8C=83=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../canal/client/adapter/OuterAdapter.java | 2 +- .../adapter/support/AdapterConfigs.java | 17 ++++- .../adapter/support/CanalClientConfig.java | 41 +++++------- .../adapter/support/DatasourceConfig.java | 26 ++++--- .../canal/client/adapter/support/Dml.java | 20 +++--- .../client/adapter/support/EtlResult.java | 15 +++-- .../adapter/support/ExtensionLoader.java | 67 ++++++++----------- .../client/adapter/support/JdbcTypeUtil.java | 4 +- .../client/adapter/support/MessageUtil.java | 2 +- .../adapter/support/OuterAdapterConfig.java | 2 +- .../canal/client/adapter/support/Result.java | 8 ++- .../canal/client/adapter/support/SPI.java | 6 ++ .../adapter/hbase/config/MappingConfig.java | 45 ++++++------- .../hbase/config/MappingConfigLoader.java | 2 +- .../hbase/service/HbaseEtlService.java | 32 +++++++++ .../hbase/service/HbaseSyncService.java | 2 +- .../launcher/CanalAdapterApplication.java | 6 ++ .../adapter/launcher/common/EtlLock.java | 6 ++ .../adapter/launcher/common/SyncSwitch.java | 13 ++-- .../launcher/config/AdapterCanalConfig.java | 6 ++ .../launcher/config/AdapterConfig.java | 8 ++- .../launcher/config/CuratorClient.java | 6 ++ .../launcher/config/SpringContext.java | 6 ++ .../loader/AbstractCanalAdapterWorker.java | 2 +- .../loader/CanalAdapterKafkaWorker.java | 2 +- .../loader/CanalAdapterRocketMQWorker.java | 2 +- .../launcher/loader/CanalAdapterService.java | 6 ++ .../launcher/loader/CanalAdapterWorker.java | 2 +- .../adapter/launcher/rest/CommonRest.java | 6 ++ 29 files changed, 230 insertions(+), 132 deletions(-) diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java index 286ba398..edb34049 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java @@ -11,7 +11,7 @@ import com.alibaba.otter.canal.client.adapter.support.SPI; /** * 外部适配器接口 * - * @author machengyuan 2018-8-18 下午10:14:02 + * @author reweerma 2018-8-18 下午10:14:02 * @version 1.0.0 */ @SPI("logger") diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/AdapterConfigs.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/AdapterConfigs.java index 44d29ee9..0ccfb509 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/AdapterConfigs.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/AdapterConfigs.java @@ -1,13 +1,26 @@ package com.alibaba.otter.canal.client.adapter.support; +import java.util.HashMap; import java.util.LinkedHashSet; import java.util.Map; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; +/** + * 适配器配置集合, 用于配置加载, 线程不安全 + * + * @author rewerma @ 2018-10-20 + * @version 1.0.0 + */ public class AdapterConfigs { - private static Map> configs = new ConcurrentHashMap<>(); + /** + * 类型下对应所有配置名, 如: + * hbase + * ┗━ mytest_person.yml + * ┗━ mytest_role.yml + * ┗━ mytest_department.yml + */ + private static Map> configs = new HashMap<>(); public static void put(String key, String value) { Set values = configs.get(key); diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java index 08246a42..29da946e 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java @@ -2,29 +2,26 @@ package com.alibaba.otter.canal.client.adapter.support; import java.util.ArrayList; import java.util.List; -import java.util.Properties; /** * 配置信息类 * - * @author machengyuan 2018-8-18 下午10:40:12 + * @author rewerma 2018-8-18 下午10:40:12 * @version 1.0.0 */ public class CanalClientConfig { - private String canalServerHost; + private String canalServerHost; // 单机模式下canal server的 ip:port - private String zookeeperHosts; + private String zookeeperHosts; // 集群模式下的zk地址, 如果配置了单机地址则以单机为准!! - private Properties properties; + private String bootstrapServers; // kafka or rocket mq 地址 - private String bootstrapServers; + private Boolean flatMessage = true; // 是否已flatMessage模式传输, 只适用于mq模式 - private List mqTopics; + private List mqTopics; // mq topic 列表 - private Boolean flatMessage = true; - - private List canalInstances; + private List canalInstances; // tcp 模式下 canal 实例列表, 与mq模式不能共存!! public String getCanalServerHost() { return canalServerHost; @@ -42,14 +39,6 @@ public class CanalClientConfig { this.zookeeperHosts = zookeeperHosts; } - public Properties getProperties() { - return properties; - } - - public void setProperties(Properties properties) { - this.properties = properties; - } - public String getBootstrapServers() { return bootstrapServers; } @@ -84,9 +73,9 @@ public class CanalClientConfig { public static class CanalInstance { - private String instance; + private String instance; // 实例名 - private List adapterGroups; + private List adapterGroups; // 适配器分组列表 public String getInstance() { return instance; @@ -109,7 +98,7 @@ public class CanalClientConfig { public static class AdapterGroup { - private List outAdapters; + private List outAdapters; // 适配器列表 public List getOutAdapters() { return outAdapters; @@ -122,11 +111,11 @@ public class CanalClientConfig { public static class MQTopic { - private String mqMode; + private String mqMode; // mq模式 kafka or rocketMQ - private String topic; + private String topic; // topic名 - private List groups = new ArrayList<>(); + private List groups = new ArrayList<>(); // 分组列表 public String getMqMode() { return mqMode; @@ -155,11 +144,11 @@ public class CanalClientConfig { public static class Group { - private String groupId; + private String groupId; // group id // private List adapters = new ArrayList<>(); - private List outAdapters; + private List outAdapters; // 适配器配置列表 public String getGroupId() { return groupId; diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/DatasourceConfig.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/DatasourceConfig.java index cf2ec471..acc07dd1 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/DatasourceConfig.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/DatasourceConfig.java @@ -1,21 +1,27 @@ package com.alibaba.otter.canal.client.adapter.support; -import com.alibaba.druid.pool.DruidDataSource; - import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import com.alibaba.druid.pool.DruidDataSource; + +/** + * 数据源配置 + * + * @author rewerma @ 2018-10-20 + * @version 1.0.0 + */ public class DatasourceConfig { - public final static Map DATA_SOURCES = new ConcurrentHashMap<>(); + public final static Map DATA_SOURCES = new ConcurrentHashMap<>(); // key对应的数据源 - private String driver = "com.mysql.jdbc.Driver"; - private String url; - private String database; - private String type = "mysql"; - private String username; - private String password; - private Integer maxActive = 3; + private String driver = "com.mysql.jdbc.Driver"; // 默认为mysql jdbc驱动 + private String url; // jdbc url + private String database; // jdbc database + private String type = "mysql"; // 类型, 默认为mysql + private String username; // jdbc username + private String password; // jdbc password + private Integer maxActive = 3; // 连接池最大连接数,默认为3 public String getDriver() { return driver; diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Dml.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Dml.java index 8e5af059..1d5357b4 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Dml.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Dml.java @@ -7,24 +7,24 @@ import java.util.Map; /** * DML操作转换对象 * - * @author machengyuan 2018-8-19 下午11:30:49 + * @author rewerma 2018-8-19 下午11:30:49 * @version 1.0.0 */ public class Dml implements Serializable { private static final long serialVersionUID = 2611556444074013268L; - private String destination; - private String database; - private String table; - private String type; + private String destination; // 对应canal的实例或者MQ的topic + private String database; // 数据库或schema + private String table; // 表名 + private String type; // 类型: INSERT UPDATE DELETE // binlog executeTime - private Long es; + private Long es; // 执行耗时 // dml build timeStamp - private Long ts; - private String sql; - private List> data; - private List> old; + private Long ts; // 同步时间 + private String sql; // 执行的sql, dml sql为空 + private List> data; // 数据列表 + private List> old; // 旧数据列表, 用于update, size和data的size一一对应 public String getDestination() { return destination; diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/EtlResult.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/EtlResult.java index 0004e51a..498af7a3 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/EtlResult.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/EtlResult.java @@ -2,14 +2,21 @@ package com.alibaba.otter.canal.client.adapter.support; import java.io.Serializable; +/** + * ETL的结果对象 + * + * @author rewerma @ 2018-10-20 + * @version 1.0.0 + */ public class EtlResult implements Serializable { + private static final long serialVersionUID = 4250522736289866505L; - private boolean succeeded = false; + private boolean succeeded = false; - private String resultMessage; + private String resultMessage; - private String errorMessage; + private String errorMessage; public boolean getSucceeded() { return succeeded; @@ -34,4 +41,4 @@ public class EtlResult implements Serializable { public void setErrorMessage(String errorMessage) { this.errorMessage = errorMessage; } -} \ No newline at end of file +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/ExtensionLoader.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/ExtensionLoader.java index b5de0d82..5b5c1bbb 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/ExtensionLoader.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/ExtensionLoader.java @@ -1,22 +1,11 @@ package com.alibaba.otter.canal.client.adapter.support; -import java.io.BufferedReader; -import java.io.File; -import java.io.FilenameFilter; -import java.io.IOException; -import java.io.InputStreamReader; +import java.io.*; import java.net.MalformedURLException; import java.net.URL; import java.net.URLClassLoader; import java.nio.file.Paths; -import java.util.Arrays; -import java.util.Collections; -import java.util.Enumeration; -import java.util.HashMap; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Set; -import java.util.TreeSet; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.regex.Pattern; @@ -27,12 +16,13 @@ import org.slf4j.LoggerFactory; /** * SPI 类加载器 * - * @author machengyuan 2018-8-19 下午11:30:49 + * @author rewerma 2018-8-19 下午11:30:49 * @version 1.0.0 */ public class ExtensionLoader { - private static final Logger logger = LoggerFactory.getLogger(ExtensionLoader.class); + private static final Logger logger = LoggerFactory + .getLogger(ExtensionLoader.class); private static final String SERVICES_DIRECTORY = "META-INF/services/"; @@ -40,7 +30,8 @@ public class ExtensionLoader { private static final String DEFAULT_CLASSLOADER_POLICY = "internal"; - private static final Pattern NAME_SEPARATOR = Pattern.compile("\\s*[,]+\\s*"); + private static final Pattern NAME_SEPARATOR = Pattern + .compile("\\s*[,]+\\s*"); private static final ConcurrentMap, ExtensionLoader> EXTENSION_LOADERS = new ConcurrentHashMap<>(); @@ -271,7 +262,8 @@ public class ExtensionLoader { return instance; } catch (Throwable t) { throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type - + ") could not be instantiated: " + t.getMessage(), t); + + ") could not be instantiated: " + t.getMessage(), + t); } } @@ -279,8 +271,8 @@ public class ExtensionLoader { if (type == null) throw new IllegalArgumentException("Extension type == null"); if (name == null) throw new IllegalArgumentException("Extension name == null"); Class clazz = getExtensionClasses().get(name); - if (clazz == null) throw new IllegalStateException("No such extension \"" + name + "\" for " + type.getName() - + "!"); + if (clazz == null) + throw new IllegalStateException("No such extension \"" + name + "\" for " + type.getName() + "!"); return clazz; } @@ -342,8 +334,8 @@ public class ExtensionLoader { logger.info("extension classpath dir: " + dir); File externalLibDir = new File(dir); if (!externalLibDir.exists()) { - externalLibDir = new File(File.separator + this.getJarDirectoryPath() + File.separator + "canal_client" - + File.separator + "lib"); + externalLibDir = new File( + File.separator + this.getJarDirectoryPath() + File.separator + "canal_client" + File.separator + "lib"); } if (externalLibDir.exists()) { File[] files = externalLibDir.listFiles(new FilenameFilter() { @@ -495,12 +487,10 @@ public class ExtensionLoader { // Class.forName(line, true, // classLoader); if (!type.isAssignableFrom(clazz)) { - throw new IllegalStateException("Error when load extension class(interface: " - + type - + ", class line: " - + clazz.getName() - + "), class " - + clazz.getName() + throw new IllegalStateException( + "Error when load extension class(interface: " + type + + ", class line: " + clazz.getName() + + "), class " + clazz.getName() + "is not subtype of interface."); } else { try { @@ -518,9 +508,9 @@ public class ExtensionLoader { extensionClasses.put(n, clazz); } else if (c != clazz) { cachedNames.remove(clazz); - throw new IllegalStateException("Duplicate extension " - + type.getName() - + " name " + n + " on " + throw new IllegalStateException( + "Duplicate extension " + type.getName() + " name " + + n + " on " + c.getName() + " and " + clazz.getName()); } @@ -530,12 +520,9 @@ public class ExtensionLoader { } } } catch (Throwable t) { - IllegalStateException e = new IllegalStateException("Failed to load extension class(interface: " - + type - + ", class line: " - + line - + ") in " - + url + IllegalStateException e = new IllegalStateException( + "Failed to load extension class(interface: " + type + ", class line: " + + line + ") in " + url + ", cause: " + t.getMessage(), t); @@ -550,13 +537,15 @@ public class ExtensionLoader { } } catch (Throwable t) { logger.error("Exception when load extension class(interface: " + type + ", class file: " + url - + ") in " + url, t); + + ") in " + url, + t); } } // end of while urls } } catch (Throwable t) { - logger.error("Exception when load extension class(interface: " + type + ", description file: " + fileName - + ").", t); + logger.error( + "Exception when load extension class(interface: " + type + ", description file: " + fileName + ").", + t); } } diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java index d4cb8a0e..c24ff7de 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java @@ -11,7 +11,7 @@ import org.slf4j.LoggerFactory; /** * 类型转换工具类 * - * @author machengyuan 2018-8-19 下午06:14:23 + * @author rewerma 2018-8-19 下午06:14:23 * @version 1.0.0 */ public class JdbcTypeUtil { @@ -30,7 +30,7 @@ public class JdbcTypeUtil { switch (jdbcType) { case Types.BIT: case Types.BOOLEAN: -// return Boolean.class; + // return Boolean.class; case Types.TINYINT: return Byte.TYPE; case Types.SMALLINT: diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java index 826ed917..ed5a4358 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java @@ -9,7 +9,7 @@ import com.alibaba.otter.canal.protocol.Message; /** * Message对象解析工具类 * - * @author machengyuan 2018-8-19 下午06:14:23 + * @author rewerma 2018-8-19 下午06:14:23 * @version 1.0.0 */ public class MessageUtil { diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/OuterAdapterConfig.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/OuterAdapterConfig.java index 3aed2f16..4e73791e 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/OuterAdapterConfig.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/OuterAdapterConfig.java @@ -5,7 +5,7 @@ import java.util.Map; /** * 外部适配器配置信息类 * - * @author machengyuan 2018-8-18 下午10:15:12 + * @author rewerma 2018-8-18 下午10:15:12 * @version 1.0.0 */ public class OuterAdapterConfig { diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Result.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Result.java index 03831f1f..a9602d05 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Result.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Result.java @@ -3,9 +3,15 @@ package com.alibaba.otter.canal.client.adapter.support; import java.io.Serializable; import java.util.Date; +/** + * 用于rest的结果返回对象 + * + * @author rewerma @ 2018-10-20 + * @version 1.0.0 + */ public class Result implements Serializable { - public Integer code = 20000; + public Integer code = 20000; public Object data; public String message; public Date sysTime; diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/SPI.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/SPI.java index 7aa69938..b4f89b6a 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/SPI.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/SPI.java @@ -6,6 +6,12 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +/** + * SPI装载器注解 + * + * @author rewerma @ 2018-10-20 + * @version 1.0.0 + */ @Documented @Retention(RetentionPolicy.RUNTIME) @Target({ ElementType.TYPE }) diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java index a5ea5707..8027fb51 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java @@ -1,21 +1,18 @@ package com.alibaba.otter.canal.client.adapter.hbase.config; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.Map; -import java.util.Objects; -import java.util.Set; +import java.util.*; /** * HBase表映射配置 * - * @author machengyuan 2018-8-21 下午06:45:49 + * @author rewerma 2018-8-21 下午06:45:49 * @version 1.0.0 */ public class MappingConfig { - private String dataSourceKey; - private HbaseOrm hbaseOrm; + private String dataSourceKey; // 数据源key + + private HbaseOrm hbaseOrm; // hbase映射配置 public String getDataSourceKey() { return dataSourceKey; @@ -130,7 +127,7 @@ public class MappingConfig { } public enum Mode { - STRING("STRING"), NATIVE("NATIVE"), PHOENIX("PHOENIX"); + STRING("STRING"), NATIVE("NATIVE"), PHOENIX("PHOENIX"); private String type; @@ -145,23 +142,23 @@ public class MappingConfig { public static class HbaseOrm { - private Mode mode = Mode.STRING; - private String destination; - private String database; - private String table; - private String hbaseTable; - private String family = "CF"; - private boolean uppercaseQualifier = true; - private boolean autoCreateTable = false; // 同步时HBase中表不存在的情况下自动建表 - private String rowKey; // 指定复合主键为rowKey - private Map columns; - private ColumnItem rowKeyColumn; - private String etlCondition; + private Mode mode = Mode.STRING; // hbase默认转换格式 + private String destination; // canal实例或MQ的topic + private String database; // 数据库名或schema名 + private String table; // 表面名 + private String hbaseTable; // hbase表名 + private String family = "CF"; // 默认统一column family + private boolean uppercaseQualifier = true; // 是否转大写 + private boolean autoCreateTable = false; // 同步时HBase中表不存在的情况下自动建表 + private String rowKey; // 指定复合主键为rowKey + private Map columns; // 字段映射 + private ColumnItem rowKeyColumn; // rowKey字段 + private String etlCondition; // etl条件sql - private Map columnItems = new LinkedHashMap<>(); - private Set families = new LinkedHashSet<>(); + private Map columnItems = new LinkedHashMap<>(); // 转换后的字段映射列表 + private Set families = new LinkedHashSet<>(); // column family列表 private int readBatch = 5000; - private int commitBatch = 5000; + private int commitBatch = 5000; // etl等批量提交大小 public Mode getMode() { return mode; diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java index 0fb727d8..83ba56df 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java @@ -18,7 +18,7 @@ import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs; /** * HBase表映射配置加载器 * - * @author machengyuan 2018-8-21 下午06:45:49 + * @author rewerma 2018-8-21 下午06:45:49 * @version 1.0.0 */ public class MappingConfigLoader { diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java index ee110755..7e3c817a 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java @@ -24,6 +24,12 @@ import com.alibaba.otter.canal.client.adapter.support.EtlResult; import com.alibaba.otter.canal.client.adapter.support.JdbcTypeUtil; import com.google.common.base.Joiner; +/** + * HBase ETL 操作业务类 + * + * @author rewerma @ 2018-10-20 + * @version 1.0.0 + */ public class HbaseEtlService { private static Logger logger = LoggerFactory.getLogger(HbaseEtlService.class); @@ -66,6 +72,12 @@ public class HbaseEtlService { } } + /** + * 建表 + * + * @param hbaseTemplate + * @param config + */ public static void createTable(HbaseTemplate hbaseTemplate, MappingConfig config) { try { // 判断hbase表是否存在，不存在则建表 @@ -79,6 +91,15 @@ public class HbaseEtlService { } } + /** + * 导入数据 + * + * @param ds 数据源 + * @param hbaseTemplate hbaseTemplate + * @param config 配置 + * @param params 筛选条件 + * @return 导入结果 + */ public static EtlResult importData(DataSource ds, HbaseTemplate hbaseTemplate, MappingConfig config, List params) { EtlResult etlResult = new EtlResult(); @@ -209,6 +230,17 @@ public class HbaseEtlService { return etlResult; } + /** + * 执行导入 + * + * @param ds + * @param sql + * @param hbaseOrm + * @param hbaseTemplate + * @param successCount + * @param errMsg + * @return + */ private static boolean executeSqlImport(DataSource ds, String sql, MappingConfig.HbaseOrm hbaseOrm, HbaseTemplate hbaseTemplate, AtomicLong successCount, List errMsg) { try { diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java index 1ff406fd..2edc396b 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java @@ -13,7 +13,7 @@ import com.alibaba.otter.canal.client.adapter.support.Dml; /** * HBase同步操作业务 * - * @author machengyuan 2018-8-21 下午06:45:49 + * @author rewerma 2018-8-21 下午06:45:49 * @version 1.0.0 */ public class HbaseSyncService { diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/CanalAdapterApplication.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/CanalAdapterApplication.java index 56bbde8e..99140b0c 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/CanalAdapterApplication.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/CanalAdapterApplication.java @@ -3,6 +3,12 @@ package com.alibaba.otter.canal.adapter.launcher; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.builder.SpringApplicationBuilder; +/** + * 启动入口 + * + * @author rewerma @ 2018-10-20 + * @version 1.0.0 + */ @SpringBootApplication public class CanalAdapterApplication { diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/EtlLock.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/EtlLock.java index 67d0ac41..5fd5f3fe 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/EtlLock.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/EtlLock.java @@ -14,6 +14,12 @@ import org.springframework.stereotype.Component; import com.alibaba.otter.canal.adapter.launcher.config.CuratorClient; +/** + * Etl 同步锁 + * + * @author rewerma @ 2018-10-20 + * @version 1.0.0 + */ @Component public class EtlLock { diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/SyncSwitch.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/SyncSwitch.java index d96c753e..aa6e55bc 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/SyncSwitch.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/SyncSwitch.java @@ -19,16 +19,22 @@ import com.alibaba.otter.canal.adapter.launcher.config.AdapterCanalConfig; import com.alibaba.otter.canal.adapter.launcher.config.CuratorClient; import com.alibaba.otter.canal.common.utils.BooleanMutex; +/** + * 同步开关 + * + * @author rewerma @ 2018-10-20 + * @version 1.0.0 + */ @Component public class SyncSwitch { private static final String SYN_SWITCH_ZK_NODE = "/sync-switch/"; - private static final Map LOCAL_LOCK = new ConcurrentHashMap<>(); + private static final Map LOCAL_LOCK = new ConcurrentHashMap<>(); - private static final Map DISTRIBUTED_LOCK = new ConcurrentHashMap<>(); + private static final Map DISTRIBUTED_LOCK = new ConcurrentHashMap<>(); - private static Mode mode = Mode.LOCAL; + private static Mode mode = Mode.LOCAL; @Resource private AdapterCanalConfig adapterCanalConfig; @@ -204,5 +210,4 @@ public class SyncSwitch { } } - } diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java index 18a12d7c..2579e1f7 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java @@ -9,6 +9,12 @@ import org.springframework.stereotype.Component; import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; +/** + * canal 的相关配置类 + * + * @author rewerma @ 2018-10-20 + * @version 1.0.0 + */ @Component @ConfigurationProperties(prefix = "canal.conf") public class AdapterCanalConfig extends CanalClientConfig { diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfig.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfig.java index da37804f..f3e265f8 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfig.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfig.java @@ -13,6 +13,12 @@ import com.alibaba.druid.pool.DruidDataSource; import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs; import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +/** + * 适配器数据源及配置文件列表配置类 + * + * @author rewerma @ 2018-10-20 + * @version 1.0.0 + */ @Component @ConfigurationProperties(prefix = "adapter.conf") public class AdapterConfig { @@ -55,7 +61,7 @@ public class AdapterConfig { try { ds.init(); } catch (SQLException e) { - logger.error("#Failed to initial datasource: " + datasourceConfig.getUrl(), e); + logger.error("ERROR ## failed to initial datasource: " + datasourceConfig.getUrl(), e); } DatasourceConfig.DATA_SOURCES.put(entry.getKey(), ds); } diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/CuratorClient.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/CuratorClient.java index b7944852..0ec60392 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/CuratorClient.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/CuratorClient.java @@ -8,6 +8,12 @@ import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.ExponentialBackoffRetry; import org.springframework.stereotype.Component; +/** + * curator 配置类 + * + * @author rewerma @ 2018-10-20 + * @version 1.0.0 + */ @Component public class CuratorClient { diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/SpringContext.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/SpringContext.java index ac860bea..987371e2 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/SpringContext.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/SpringContext.java @@ -5,6 +5,12 @@ import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; import org.springframework.stereotype.Component; +/** + * spring util配置类 + * + * @author rewerma @ 2018-10-20 + * @version 1.0.0 + */ @Component public class SpringContext implements ApplicationContextAware { diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java index bd3a47a4..cd818a9d 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java @@ -21,7 +21,7 @@ import com.alibaba.otter.canal.protocol.Message; /** * 适配器工作线程抽象类 * - * @author machengyuan 2018-8-19 下午11:30:49 + * @author rewerma 2018-8-19 下午11:30:49 * @version 1.0.0 */ public abstract class AbstractCanalAdapterWorker { diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java index 22272e80..4c48125a 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java @@ -17,7 +17,7 @@ import com.alibaba.otter.canal.protocol.Message; /** * kafka对应的client适配器工作线程 * - * @author machengyuan 2018-8-19 下午11:30:49 + * @author rewerma 2018-8-19 下午11:30:49 * @version 1.0.0 */ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker { diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java index 09863a33..4632d7f0 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java @@ -16,7 +16,7 @@ import com.alibaba.otter.canal.protocol.Message; /** * kafka对应的client适配器工作线程 * - * @author machengyuan 2018-8-19 下午11:30:49 + * @author rewerma 2018-8-19 下午11:30:49 * @version 1.0.0 */ public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker { diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterService.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterService.java index 03f515bf..2e535535 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterService.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterService.java @@ -15,6 +15,12 @@ import com.alibaba.otter.canal.adapter.launcher.config.AdapterCanalConfig; import com.alibaba.otter.canal.adapter.launcher.config.AdapterConfig; import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +/** + * 适配器启动业务类 + * + * @author rewerma @ 2018-10-20 + * @version 1.0.0 + */ @Component public class CanalAdapterService { diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java index 1c324996..c728a15a 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java @@ -15,7 +15,7 @@ import com.alibaba.otter.canal.protocol.Message; /** * 原生canal-server对应的client适配器工作线程 * - * @author machengyuan 2018-8-19 下午11:30:49 + * @author rewrema 2018-8-19 下午11:30:49 * @version 1.0.0 */ public class CanalAdapterWorker extends AbstractCanalAdapterWorker { diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java index bee78e04..d2c163fb 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java @@ -17,6 +17,12 @@ import com.alibaba.otter.canal.client.adapter.support.EtlResult; import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader; import com.alibaba.otter.canal.client.adapter.support.Result; +/** + * 适配器操作Rest + * + * @author rewerma @ 2018-10-20 + * @version 1.0.0 + */ @RestController public class CommonRest { From 848cbe11dd024c8226544dbf59fed137a25e39e1 Mon Sep 17 00:00:00 2001 From: mcy Date: Fri, 26 Oct 2018 10:33:16 +0800 Subject: [PATCH 24/35] =?UTF-8?q?=E4=BB=A3=E7=A0=81=E6=95=B4=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../loader/AbstractCanalAdapterWorker.java | 120 ++++++++++-------- .../loader/CanalAdapterKafkaWorker.java | 49 +------ .../launcher/loader/CanalAdapterLoader.java | 14 +- .../loader/CanalAdapterRocketMQWorker.java | 68 ++++------ .../launcher/loader/CanalAdapterWorker.java | 55 +------- 5 files changed, 106 insertions(+), 200 deletions(-) diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java index cd818a9d..0486045c 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java @@ -2,20 +2,19 @@ package com.alibaba.otter.canal.adapter.launcher.loader; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; -import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch; -import com.alibaba.otter.canal.adapter.launcher.config.SpringContext; -import com.alibaba.otter.canal.client.adapter.support.Dml; -import com.alibaba.otter.canal.protocol.FlatMessage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch; +import com.alibaba.otter.canal.adapter.launcher.config.SpringContext; import com.alibaba.otter.canal.client.adapter.OuterAdapter; +import com.alibaba.otter.canal.client.adapter.support.Dml; import com.alibaba.otter.canal.client.adapter.support.MessageUtil; +import com.alibaba.otter.canal.protocol.FlatMessage; import com.alibaba.otter.canal.protocol.Message; /** @@ -44,21 +43,21 @@ public abstract class AbstractCanalAdapterWorker { protected void writeOut(final Message message) { List> futures = new ArrayList<>(); // 组间适配器并行运行 - for (List outerAdapters : canalOuterAdapters) { + canalOuterAdapters.forEach(outerAdapters -> { final List adapters = outerAdapters; futures.add(groupInnerExecutorService.submit(() -> { try { // 组内适配器穿行运行，尽量不要配置组内适配器 - for (final OuterAdapter c : adapters) { + adapters.forEach(adapter -> { long begin = System.currentTimeMillis(); - MessageUtil.parse4Dml(canalDestination, message, c::sync); + MessageUtil.parse4Dml(canalDestination, message, adapter::sync); if (logger.isDebugEnabled()) { logger.debug("{} elapsed time: {}", - c.getClass().getName(), + adapter.getClass().getName(), (System.currentTimeMillis() - begin)); } - } + }); return true; } catch (Exception e) { return false; @@ -67,79 +66,94 @@ public abstract class AbstractCanalAdapterWorker { // 等待所有适配器写入完成 // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成 - for (Future f : futures) { + futures.forEach(future -> { try { - if (!f.get()) { + if (!future.get()) { logger.error("Outer adapter write failed"); } } catch (InterruptedException | ExecutionException e) { // ignore } - } - } + }); + }); } protected void writeOut(final FlatMessage flatMessage) { List> futures = new ArrayList<>(); // 组间适配器并行运行 - for (List outerAdapters : canalOuterAdapters) { - final List adapters = outerAdapters; - futures.add(groupInnerExecutorService.submit(new Callable() { - - @Override - public Boolean call() { - try { - // 组内适配器穿行运行，尽量不要配置组内适配器 - for (OuterAdapter c : adapters) { - long begin = System.currentTimeMillis(); - Dml dml = MessageUtil.flatMessage2Dml(canalDestination, flatMessage); - c.sync(dml); - if (logger.isDebugEnabled()) { - logger.debug("{} elapsed time: {}", - c.getClass().getName(), - (System.currentTimeMillis() - begin)); - } + canalOuterAdapters.forEach(outerAdapters -> { + futures.add(groupInnerExecutorService.submit(() -> { + try { + // 组内适配器穿行运行，尽量不要配置组内适配器 + outerAdapters.forEach(adapter -> { + long begin = System.currentTimeMillis(); + Dml dml = MessageUtil.flatMessage2Dml(canalDestination, flatMessage); + adapter.sync(dml); + if (logger.isDebugEnabled()) { + logger.debug("{} elapsed time: {}", + adapter.getClass().getName(), + (System.currentTimeMillis() - begin)); } - return true; - } catch (Exception e) { - return false; - } + }); + return true; + } catch (Exception e) { + return false; } })); // 等待所有适配器写入完成 // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成 - for (Future f : futures) { + futures.forEach(future -> { try { - if (!f.get()) { + if (!future.get()) { logger.error("Outer adapter write failed"); } } catch (InterruptedException | ExecutionException e) { // ignore } - } + }); + }); + } + + public void start() { + if (!running) { + thread = new Thread(this::process); + thread.setUncaughtExceptionHandler(handler); + thread.start(); + running = true; } } - protected void stopOutAdapters() { - if (thread != null) { - try { - thread.join(); - } catch (InterruptedException e) { - // ignore + protected abstract void process(); + + public void stop() { + try { + if (!running) { + return; } - } - groupInnerExecutorService.shutdown(); - logger.info("topic connectors' worker thread dead!"); - for (List outerAdapters : canalOuterAdapters) { - for (OuterAdapter adapter : outerAdapters) { - adapter.destroy(); + + closeConnection(); + running = false; + + syncSwitch.release(canalDestination); + + logger.info("destination {} is waiting for adapters' worker thread die!", canalDestination); + if (thread != null) { + try { + thread.join(); + } catch (InterruptedException e) { + // ignore + } } + groupInnerExecutorService.shutdown(); + logger.info("destination {} adapters worker thread dead!", canalDestination); + canalOuterAdapters.forEach(outerAdapters -> outerAdapters.forEach(OuterAdapter::destroy)); + logger.info("destination {} all adapters destroyed!", canalDestination); + } catch (Exception e) { + logger.error(e.getMessage(), e); } - logger.info("topic all connectors destroyed!"); } - public abstract void start(); + protected abstract void closeConnection(); - public abstract void stop(); } diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java index 4c48125a..665ad38e 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java @@ -37,59 +37,18 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker { this.flatMessage = flatMessage; connector = KafkaCanalConnectors.newKafkaConnector(bootstrapServers, topic, null, groupId, flatMessage); // connector.setSessionTimeout(1L, TimeUnit.MINUTES); - - // super.initSwitcher(topic); } @Override - public void start() { - if (!running) { - thread = new Thread(() -> process()); - thread.setUncaughtExceptionHandler(handler); - running = true; - thread.start(); - } + protected void closeConnection(){ + connector.stopRunning(); } @Override - public void stop() { - try { - if (!running) { - return; - } - - connector.stopRunning(); - running = false; - - // if (switcher != null && !switcher.state()) { - // switcher.set(true); - // } - - if (thread != null) { - try { - thread.join(); - } catch (InterruptedException e) { - // ignore - } - } - groupInnerExecutorService.shutdown(); - logger.info("topic {} connectors' worker thread dead!", this.topic); - for (List outerAdapters : canalOuterAdapters) { - for (OuterAdapter adapter : outerAdapters) { - adapter.destroy(); - } - } - logger.info("topic {} all connectors destroyed!", this.topic); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - private void process() { + protected void process() { while (!running) ; ExecutorService executor = Executors.newSingleThreadExecutor(); - // final AtomicBoolean executing = new AtomicBoolean(true); while (running) { try { logger.info("=============> Start to connect topic: {} <=============", this.topic); @@ -99,7 +58,7 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker { logger.info("=============> Subscribe topic: {} succeed <=============", this.topic); while (running) { try { - // switcher.get(); //等待开关开启 + syncSwitch.get(canalDestination); List messages; if (!flatMessage) { diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java index 3bb5b4a4..0447036a 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java @@ -135,20 +135,18 @@ public class CanalAdapterLoader { public void destroy() { if (canalWorkers.size() > 0) { ExecutorService stopExecutorService = Executors.newFixedThreadPool(canalWorkers.size()); - for (CanalAdapterWorker v : canalWorkers.values()) { - final CanalAdapterWorker caw = v; - stopExecutorService.submit(caw::stop); + for (CanalAdapterWorker canalAdapterWorker : canalWorkers.values()) { + stopExecutorService.submit(canalAdapterWorker::stop); } stopExecutorService.shutdown(); } if (canalMQWorker.size() > 0) { - ExecutorService stopMQWokerService = Executors.newFixedThreadPool(canalMQWorker.size()); - for (AbstractCanalAdapterWorker tmp : canalMQWorker.values()) { - final AbstractCanalAdapterWorker worker = tmp; - stopMQWokerService.submit(worker::stop); + ExecutorService stopMQWorkerService = Executors.newFixedThreadPool(canalMQWorker.size()); + for (AbstractCanalAdapterWorker canalAdapterMQWorker : canalMQWorker.values()) { + stopMQWorkerService.submit(canalAdapterMQWorker::stop); } - stopMQWokerService.shutdown(); + stopMQWorkerService.shutdown(); } logger.info("All canal adapters destroyed"); } diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java index 4632d7f0..f4c2e704 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java @@ -36,38 +36,12 @@ public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker { } @Override - public void start() { - if (!running) { - thread = new Thread(new Runnable() { - - @Override - public void run() { - process(); - } - }); - thread.setUncaughtExceptionHandler(handler); - running = true; - thread.start(); - } + protected void closeConnection() { + connector.stopRunning(); } @Override - public void stop() { - try { - if (!running) { - return; - } - connector.stopRunning(); - running = false; - logger.info("Stop topic {} out adapters begin", this.topic); - stopOutAdapters(); - logger.info("Stop topic {} out adapters end", this.topic); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - private void process() { + protected void process() { while (!running) ; ExecutorService executor = Executors.newSingleThreadExecutor(); @@ -84,25 +58,27 @@ public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker { final Message message = connector.getWithoutAck(1); if (message != null) { - executor.submit(new Runnable() { - - @Override - public void run() { - try { - if (logger.isDebugEnabled()) { - logger.debug("topic: {} batchId: {} batchSize: {} ", topic, message.getId(), message.getEntries().size()); - } - long begin = System.currentTimeMillis(); - writeOut(message); - long now = System.currentTimeMillis(); - if ((System.currentTimeMillis() - begin) > 5 * 60 * 1000) { - logger.error("topic: {} batchId {} elapsed time: {} ms", topic, message.getId(), now - begin); - } - } catch (Exception e) { - logger.error(e.getMessage(), e); + executor.submit(() -> { + try { + if (logger.isDebugEnabled()) { + logger.debug("topic: {} batchId: {} batchSize: {} ", + topic, + message.getId(), + message.getEntries().size()); } - connector.ack(message.getId()); + long begin = System.currentTimeMillis(); + writeOut(message); + long now = System.currentTimeMillis(); + if ((System.currentTimeMillis() - begin) > 5 * 60 * 1000) { + logger.error("topic: {} batchId {} elapsed time: {} ms", + topic, + message.getId(), + now - begin); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); } + connector.ack(message.getId()); }); } else { logger.debug("Message is null"); diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java index c728a15a..33a70f89 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java @@ -57,49 +57,12 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker { } @Override - public void start() { - if (!running) { - thread = new Thread(this::process); - thread.setUncaughtExceptionHandler(handler); - thread.start(); - running = true; - } + protected void closeConnection() { + connector.stopRunning(); } @Override - public void stop() { - try { - if (!running) { - return; - } - - connector.stopRunning(); - running = false; - - syncSwitch.release(canalDestination); - - logger.info("destination {} is waiting for adapters' worker thread die!", canalDestination); - if (thread != null) { - try { - thread.join(); - } catch (InterruptedException e) { - // ignore - } - } - groupInnerExecutorService.shutdown(); - logger.info("destination {} adapters' worker thread dead!", canalDestination); - for (List outerAdapters : canalOuterAdapters) { - for (OuterAdapter adapter : outerAdapters) { - adapter.destroy(); - } - } - logger.info("destination {} all adapters destroyed!", canalDestination); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - private void process() { + protected void process() { while (!running) ; // waiting until running == true while (running) { @@ -129,11 +92,7 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker { int size = message.getEntries().size(); if (batchId == -1 || size == 0) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - // ignore - } + Thread.sleep(1000); } else { if (logger.isDebugEnabled()) { logger.debug("destination: {} batchId: {} batchSize: {} ", @@ -143,18 +102,18 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker { } long begin = System.currentTimeMillis(); writeOut(message); - long now = System.currentTimeMillis(); if (logger.isDebugEnabled()) { logger.debug("destination: {} batchId: {} elapsed time: {} ms", this.canalDestination, batchId, - now - begin); + System.currentTimeMillis() - begin); } } connector.ack(batchId); // 提交确认 } catch (Exception e) { connector.rollback(batchId); // 处理失败, 回滚数据 - throw e; + logger.error("sync error!", e); + Thread.sleep(500); } } From bbfd4b499662c91662acdb81cd2439c0844a6a0a Mon Sep 17 00:00:00 2001 From: mcy Date: Fri, 26 Oct 2018 13:23:22 +0800 Subject: [PATCH 25/35] canal-adapter readme.md --- client-adapter/README.md | 251 ++++++++++++++++++ .../main/resources/hbase/mytest_person2.yml | 2 +- 2 files changed, 252 insertions(+), 1 deletion(-) create mode 100644 client-adapter/README.md diff --git a/client-adapter/README.md b/client-adapter/README.md new file mode 100644 index 00000000..47a5dcc4 --- /dev/null +++ b/client-adapter/README.md @@ -0,0 +1,251 @@ +## 基本说明 +canal 1.1.1版本之后, 增加客户端数据落地的适配及启动功能, 目前支持功能: +* 客户端启动器 +* 同步管理REST接口 +* 日志适配器, 作为DEMO +* HBase的数据同步(表对表同步), ETL功能 +* (后续支持) ElasticSearch多表数据同步,ETL功能 + +## 环境版本 +* 操作系统：无要求 +* java版本: jdk1.8 以上 +* canal 版本: 请下载最新的安装包，本文以当前v1.1.1 的canal.deployer-1.1.1.tar.gz为例 +* MySQL版本 ：5.7.18 +* HBase版本: Apache HBase 1.1.2, 若和服务端版本不一致可自行替换客户端HBase依赖 + +## 一、适配器启动器 +client-adapter分为适配器和启动器两部分, 适配器为多个fat jar, 每个适配器会将自己所需的依赖打成一个包, 以SPI的方式让启动器动态加载 + + +启动器为 SpringBoot 项目, 支持canal-client启动的同时提供相关REST管理接口, 运行目录结构为: +``` +canal-adapter-launcher.jar +- lib + client-adapter.logger-1.1.1-jar-with-dependencies.jar + client-adapter.hbase-1.1.1-jar-with-dependencies.jar +- config + application.yml + - hbase + mytest_person2.yml +``` +以上目录结构最终会打包成 canal-adapter-launcher.tar.gz 压缩包 + +## 二、启动器 +### 2.1 启动器配置 application.yml +#### canal相关配置部分说明 +``` +canal.conf: + canalServerHost: 127.0.0.1:11111 # 对应单机模式下的canal server的ip:port + zookeeperHosts: slave1:2181 # 对应集群模式下的zk地址, 如果配置了canalServerHost, 则以canalServerHost为准 + bootstrapServers: slave1:6667 # kafka或rocketMQ地址, 与canalServerHost不能并存 + flatMessage: true # 扁平message开关, 是否以json字符串形式投递数据, 仅在kafka/rocketMQ模式下有效 + canalInstances: # canal实例组, 如果是tcp模式可配置此项 + - instance: example # 对应canal destination + adapterGroups: # 对应适配器分组, 分组间的适配器并行运行 + - outAdapters: # 适配器列表, 分组内的适配串行运行 + - name: logger # 适配器SPI名 + - name: hbase + properties: # HBase相关连接参数 + hbase.zookeeper.quorum: slave1 + hbase.zookeeper.property.clientPort: 2181 + zookeeper.znode.parent: /hbase + mqTopics: # MQ topic租, 如果是kafka或者rockeMQ模式可配置此项, 与canalInstances不能并存 + - mqMode: kafka # MQ的模式: kafak/rocketMQ + topic: example # MQ topic + groups: # group组 + - groupId: g2 # group id + outAdapters: # 适配器列表, 以下配置和canalInstances中一样 + - name: logger +``` +#### 适配器相关配置部分说明 +``` +adapter.conf: + datasourceConfigs: # 数据源配置列表, 数据源将在适配器中用于ETL、数据同步回查等使用 + defaultDS: # 数据源 dataSourceKey, 适配器中通过该值获取指定数据源 + url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true + username: root + password: 121212 + adapterConfigs: # 适配器内部配置列表 + - hbase/mytest_person2.yml # 类型/配置文件名, 这里示例就是对应HBase适配器hbase目录下的mytest_person2.yml文件 +``` +## 2.2 适配器管理REST接口 +#### 2.2.1 查询所有订阅同步的canal destination或MQ topic +``` +curl http://127.0.0.1:8081/destinations +``` +#### 2.2.2 数据同步开关 +``` +curl http://127.0.0.1:8081/syncSwitch/example/off -X PUT +``` +针对 example 这个canal destination/MQ topic 进行开关操作. off代表关闭, 该destination/topic下的同步将阻塞或者断开连接不再接收数据, on代表开启 + +注: 如果在配置文件中配置了 zookeeperHosts 项, 则会使用分布式锁来控制HA中的数据同步开关, 如果是单机模式则使用本地锁来控制开关 +#### 2.2.3 数据同步开关状态 +``` +curl http://127.0.0.1:8081/syncSwitch/example +``` +查看指定 canal destination/MQ topic 的数据同步开关状态 +#### 2.2.4 手动ETL +``` +curl http://127.0.0.1:8081/etl/hbase/mytest_person2.yml -X POST -d "params=2018-10-21 00:00:00" +``` +导入数据到指定类型的库 +#### 2.2.5 查看相关库总数据 +``` +curl http://127.0.0.1:8081/count/hbase/mytest_person2.yml +``` +### 2.3 启动canal-adapter示例 +#### 2.3.1 启动canal server (单机模式), 参考: [Canal QuickStart](https://github.com/alibaba/canal/wiki/QuickStart) +#### 2.3.2 修改config/application.yml为: +``` +server: + port: 8081 +logging: + level: + com.alibaba.otter.canal.client.adapter.hbase: DEBUG +spring: + jackson: + date-format: yyyy-MM-dd HH:mm:ss + time-zone: GMT+8 + default-property-inclusion: non_null + +canal.conf: + canalServerHost: 127.0.0.1:11111 + flatMessage: true + canalInstances: + - instance: example + adapterGroups: + - outAdapters: + - name: logger +``` +启动 canal-adapter-launcher.jar +``` +java -jar canal-adapter-launcher.jar +``` + +## 三、HBase适配器 +### 3.1 修改启动器配置: application.yml +``` +server: + port: 8081 +logging: + level: + com.alibaba.otter.canal.client.adapter.hbase: DEBUG +spring: + jackson: + date-format: yyyy-MM-dd HH:mm:ss + time-zone: GMT+8 + default-property-inclusion: non_null + +canal.conf: + canalServerHost: 127.0.0.1:11111 + flatMessage: true + canalInstances: + - instance: example + adapterGroups: + - outAdapters: + - name: hbase + properties: + hbase.zookeeper.quorum: slave1 + hbase.zookeeper.property.clientPort: 2181 + zookeeper.znode.parent: /hbase + +adapter.conf: + datasourceConfigs: + defaultDS: + url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true + username: root + password: 121212 + adapterConfigs: + - hbase/mytest_person.yml +``` +其中指定了一个HBase表映射文件: mytest_person.yml +### 3.2 适配器表映射文件 +修改 config/hbase/mytest_person.yml文件: +``` +dataSourceKey: defaultDS # 对应application.yml中的datasourceConfigs下的配置 +hbaseOrm: # mysql--HBase的单表映射配置 + mode: STRING # HBase中的存储类型, 默认统一存为String, 可选: #PHOENIX #NATIVE #STRING + # NATIVE: 以java类型为主, PHOENIX: 将类型转换为Phoenix对应的类型 + destination: example # 对应 canal destination/MQ topic 名称 + database: mytest # 数据库名/schema名 + table: person # 表名 + hbaseTable: MYTEST.PERSON # HBase表名 + family: CF # 默认统一Column Family名称 + uppercaseQualifier: true # 字段名转大写, 默认为true + commitBatch: 3000 # 批量提交的大小, ETL中用到 + #rowKey: id,type # 复合字段rowKey不能和columns中的rowKey并存 + # 复合rowKey会以 '|' 分隔 + columns: # 字段映射, 如果不配置将自动映射所有字段, + # 并取第一个字段为rowKey, HBase字段名以mysql字段名为主 + id: ROWKE + name: CF:NAME + email: EMAIL # 如果column family为默认CF, 则可以省略 + type: # 如果HBase字段和mysql字段名一致, 则可以省略 + c_time: + birthday: +``` +如果涉及到类型转换,可以如下形式: +``` +... + columns: + id: ROWKE$STRING + ... + type: TYPE$BYTE + ... +``` +类型转换涉及到Java类型和Phoenix类型两种, 分别定义如下: +``` +#Java 类型转换, 对应配置 mode: NATIVE +$DEFAULT +$STRING +$INTEGER +$LONG +$SHORT +$BOOLEAN +$FLOAT +$DOUBLE +$BIGDECIMAL +$DATE +$BYTE +$BYTES +``` +``` +#Phoenix 类型转换, 对应配置 mode: PHOENIX +$DEFAULT 对应PHOENIX里的VARCHAR +$UNSIGNED_INT 对应PHOENIX里的UNSIGNED_INT 4字节 +$UNSIGNED_LONG 对应PHOENIX里的UNSIGNED_LONG 8字节 +$UNSIGNED_TINYINT 对应PHOENIX里的UNSIGNED_TINYINT 1字节 +$UNSIGNED_SMALLINT 对应PHOENIX里的UNSIGNED_SMALLINT 2字节 +$UNSIGNED_FLOAT 对应PHOENIX里的UNSIGNED_FLOAT 4字节 +$UNSIGNED_DOUBLE 对应PHOENIX里的UNSIGNED_DOUBLE 8字节 +$INTEGER 对应PHOENIX里的INTEGER 4字节 +$BIGINT 对应PHOENIX里的BIGINT 8字节 +$TINYINT 对应PHOENIX里的TINYINT 1字节 +$SMALLINT 对应PHOENIX里的SMALLINT 2字节 +$FLOAT 对应PHOENIX里的FLOAT 4字节 +$DOUBLE 对应PHOENIX里的DOUBLE 8字节 +$BOOLEAN 对应PHOENIX里的BOOLEAN 1字节 +$TIME 对应PHOENIX里的TIME 8字节 +$DATE 对应PHOENIX里的DATE 8字节 +$TIMESTAMP 对应PHOENIX里的TIMESTAMP 12字节 +$UNSIGNED_TIME 对应PHOENIX里的UNSIGNED_TIME 8字节 +$UNSIGNED_DATE 对应PHOENIX里的UNSIGNED_DATE 8字节 +$UNSIGNED_TIMESTAMP 对应PHOENIX里的UNSIGNED_TIMESTAMP 12字节 +$VARCHAR 对应PHOENIX里的VARCHAR 动态长度 +$VARBINARY 对应PHOENIX里的VARBINARY 动态长度 +$DECIMAL 对应PHOENIX里的DECIMAL 动态长度 +``` +如果不配置将以java对象原生类型默认映射转换 +### 3.3 启动HBase数据同步 +#### 创建HBase表 +在HBase shell中运行: +``` +create 'MYTEST.PERSON', {NAME=>'CF'} +``` +#### 启动canal-adapter启动器 +``` +java -jar canal-adapter-launcher.jar +``` +#### 验证 +修改mysql mytest.person表的数据, 将会自动同步到HBase的MYTEST.PERSON表下面, 并会打出DML的log \ No newline at end of file diff --git a/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml b/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml index 88cd99e2..b22ffe5c 100644 --- a/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml +++ b/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml @@ -45,7 +45,7 @@ hbaseOrm: # $TINYINT 对应PHOENIX里的TINYINT 1字节 # $SMALLINT 对应PHOENIX里的SMALLINT 2字节 # $FLOAT 对应PHOENIX里的FLOAT 4字节 -# DOUBLE 对应PHOENIX里的DOUBLE 8字节 +# $DOUBLE 对应PHOENIX里的DOUBLE 8字节 # $BOOLEAN 对应PHOENIX里的BOOLEAN 1字节 # $TIME 对应PHOENIX里的TIME 8字节 # $DATE 对应PHOENIX里的DATE 8字节 From 3b1f6ce4fbb0ab00a82d097093d2b7c8958426f3 Mon Sep 17 00:00:00 2001 From: rewerma Date: Fri, 26 Oct 2018 13:25:40 +0800 Subject: [PATCH 26/35] Update README.md --- client-adapter/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client-adapter/README.md b/client-adapter/README.md index 47a5dcc4..642bf50b 100644 --- a/client-adapter/README.md +++ b/client-adapter/README.md @@ -224,7 +224,7 @@ $BIGINT 对应PHOENIX里的BIGINT 8字节 $TINYINT 对应PHOENIX里的TINYINT 1字节 $SMALLINT 对应PHOENIX里的SMALLINT 2字节 $FLOAT 对应PHOENIX里的FLOAT 4字节 -$DOUBLE 对应PHOENIX里的DOUBLE 8字节 +$DOUBLE 对应PHOENIX里的DOUBLE 8字节 $BOOLEAN 对应PHOENIX里的BOOLEAN 1字节 $TIME 对应PHOENIX里的TIME 8字节 $DATE 对应PHOENIX里的DATE 8字节 @@ -248,4 +248,4 @@ create 'MYTEST.PERSON', {NAME=>'CF'} java -jar canal-adapter-launcher.jar ``` #### 验证 -修改mysql mytest.person表的数据, 将会自动同步到HBase的MYTEST.PERSON表下面, 并会打出DML的log \ No newline at end of file +修改mysql mytest.person表的数据, 将会自动同步到HBase的MYTEST.PERSON表下面, 并会打出DML的log From 5cfbc6b91d0e55baba1fe525c8c3f3d4f63dd82a Mon Sep 17 00:00:00 2001 From: rewerma Date: Fri, 26 Oct 2018 13:36:59 +0800 Subject: [PATCH 27/35] Update README.md --- client-adapter/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client-adapter/README.md b/client-adapter/README.md index 642bf50b..31d6b5a1 100644 --- a/client-adapter/README.md +++ b/client-adapter/README.md @@ -68,7 +68,7 @@ adapter.conf: adapterConfigs: # 适配器内部配置列表 - hbase/mytest_person2.yml # 类型/配置文件名, 这里示例就是对应HBase适配器hbase目录下的mytest_person2.yml文件 ``` -## 2.2 适配器管理REST接口 +## 2.2 同步管理REST接口 #### 2.2.1 查询所有订阅同步的canal destination或MQ topic ``` curl http://127.0.0.1:8081/destinations From 0d0fcbbc0b885d577ebe08eed33ae1b9e413e0fc Mon Sep 17 00:00:00 2001 From: mcy Date: Fri, 26 Oct 2018 14:03:29 +0800 Subject: [PATCH 28/35] =?UTF-8?q?=E4=BF=AE=E6=94=B9HBase=E9=80=82=E9=85=8D?= =?UTF-8?q?=E5=99=A8=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../adapter/hbase/config/MappingConfig.java | 9 ++++++ .../hbase/service/HbaseSyncService.java | 6 ++++ .../main/resources/hbase/mytest_person2.yml | 2 ++ .../src/main/resources/application.yml | 28 +++++++++---------- 4 files changed, 31 insertions(+), 14 deletions(-) diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java index 8027fb51..93968610 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java @@ -152,6 +152,7 @@ public class MappingConfig { private boolean autoCreateTable = false; // 同步时HBase中表不存在的情况下自动建表 private String rowKey; // 指定复合主键为rowKey private Map columns; // 字段映射 + private List excludeColumns; // 不映射的字段 private ColumnItem rowKeyColumn; // rowKey字段 private String etlCondition; // etl条件sql @@ -292,6 +293,14 @@ public class MappingConfig { } } + public List getExcludeColumns() { + return excludeColumns; + } + + public void setExcludeColumns(List excludeColumns) { + this.excludeColumns = excludeColumns; + } + public String getFamily() { return family; } diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java index 2edc396b..4f31d0d3 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java @@ -110,6 +110,9 @@ public class HbaseSyncService { Map columnItems = hbaseOrm.getColumnItems(); int i = 0; for (Map.Entry entry : data.entrySet()) { + if (hbaseOrm.getExcludeColumns() != null && hbaseOrm.getExcludeColumns().contains(entry.getKey())) { + continue; + } if (entry.getValue() != null) { MappingConfig.ColumnItem columnItem = columnItems.get(entry.getKey()); @@ -196,6 +199,9 @@ public class HbaseSyncService { Map columnItems = hbaseOrm.getColumnItems(); HRow hRow = new HRow(rowKeyBytes); for (String updateColumn : old.get(index).keySet()) { + if (hbaseOrm.getExcludeColumns() != null && hbaseOrm.getExcludeColumns().contains(updateColumn)) { + continue; + } MappingConfig.ColumnItem columnItem = columnItems.get(updateColumn); if (columnItem == null) { String family = hbaseOrm.getFamily(); diff --git a/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml b/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml index b22ffe5c..d80b95eb 100644 --- a/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml +++ b/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml @@ -17,6 +17,8 @@ hbaseOrm: type: $DECIMAL c_time: C_TIME$UNSIGNED_TIMESTAMP birthday: BIRTHDAY$DATE + excludeColumns: + - lat # -- NATIVE类型 # $DEFAULT diff --git a/client-adapter/launcher/src/main/resources/application.yml b/client-adapter/launcher/src/main/resources/application.yml index 43cc4e38..1b1139c1 100644 --- a/client-adapter/launcher/src/main/resources/application.yml +++ b/client-adapter/launcher/src/main/resources/application.yml @@ -23,12 +23,12 @@ canal.conf: - instance: example adapterGroups: - outAdapters: - - name: logger -# - name: hbase -# properties: -# hbase.zookeeper.quorum: ${hbase.zookeeper.quorum} -# hbase.zookeeper.property.clientPort: ${hbase.zookeeper.property.clientPort} -# zookeeper.znode.parent: ${hbase.zookeeper.znode.parent} +# - name: logger + - name: hbase + properties: + hbase.zookeeper.quorum: ${hbase.zookeeper.quorum} + hbase.zookeeper.property.clientPort: ${hbase.zookeeper.property.clientPort} + zookeeper.znode.parent: ${hbase.zookeeper.znode.parent} # mqTopics: # - mqMode: kafka # topic: example @@ -37,11 +37,11 @@ canal.conf: # outAdapters: # - name: logger -#adapter.conf: -# datasourceConfigs: -# defaultDS: -# url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true -# username: root -# password: 121212 -# adapterConfigs: -# - hbase/mytest_person2.yml +adapter.conf: + datasourceConfigs: + defaultDS: + url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true + username: root + password: 121212 + adapterConfigs: + - hbase/mytest_person2.yml From fb936f896eb6d2b33c1e3ad6a95c6ddfe4b26ffa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Fri, 26 Oct 2018 14:57:52 +0800 Subject: [PATCH 29/35] fixed client-launcher deploy --- client-adapter/example/pom.xml | 149 -------------- .../example/src/main/assembly/dev.xml | 57 ------ .../example/src/main/assembly/release.xml | 57 ------ .../example/src/main/bin/startup.bat | 25 --- .../example/src/main/bin/startup.sh | 100 ---------- client-adapter/example/src/main/bin/stop.sh | 65 ------ .../otter/canal/client/ClientLauncher.java | 66 ------- .../loader/AbstractCanalAdapterWorker.java | 168 ---------------- .../loader/CanalAdapterKafkaWorker.java | 178 ----------------- .../adapter/loader/CanalAdapterLoader.java | 177 ----------------- .../loader/CanalAdapterRocketMQWorker.java | 124 ------------ .../adapter/loader/CanalAdapterWorker.java | 186 ------------------ .../src/main/resources/canal-client.yml | 18 -- .../example/src/main/resources/logback.xml | 47 ----- client-adapter/hbase/pom.xml | 98 +++++---- client-adapter/launcher/pom.xml | 67 +++++-- .../main/assembly/release.xml} | 0 client-adapter/logger/pom.xml | 58 +++--- client-adapter/pom.xml | 1 - 19 files changed, 139 insertions(+), 1502 deletions(-) delete mode 100644 client-adapter/example/pom.xml delete mode 100644 client-adapter/example/src/main/assembly/dev.xml delete mode 100644 client-adapter/example/src/main/assembly/release.xml delete mode 100644 client-adapter/example/src/main/bin/startup.bat delete mode 100644 client-adapter/example/src/main/bin/startup.sh delete mode 100644 client-adapter/example/src/main/bin/stop.sh delete mode 100644 client-adapter/example/src/main/java/com/alibaba/otter/canal/client/ClientLauncher.java delete mode 100644 client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/AbstractCanalAdapterWorker.java delete mode 100644 client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterKafkaWorker.java delete mode 100644 client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterLoader.java delete mode 100644 client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterRocketMQWorker.java delete mode 100644 client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterWorker.java delete mode 100644 client-adapter/example/src/main/resources/canal-client.yml delete mode 100644 client-adapter/example/src/main/resources/logback.xml rename client-adapter/launcher/{assembly.xml => src/main/assembly/release.xml} (100%) diff --git a/client-adapter/example/pom.xml b/client-adapter/example/pom.xml deleted file mode 100644 index bf68d731..00000000 --- a/client-adapter/example/pom.xml +++ /dev/null @@ -1,149 +0,0 @@ - - - - canal.client-adapter - com.alibaba.otter - 1.1.1-SNAPSHOT - - 4.0.0 - com.alibaba.otter - client-adapter.example - jar - canal client adapter example module for otter ${project.version} - - - - com.alibaba.otter - client-adapter.common - ${project.version} - - - com.alibaba.otter - canal.client - ${canal_version} - - - - org.apache.rocketmq - rocketmq-client - 4.3.0 - - - - org.apache.kafka - kafka-clients - 1.1.1 - - - org.yaml - snakeyaml - 1.17 - - - - - com.alibaba.otter - client-adapter.logger - ${project.version} - - - - - - - - maven-jar-plugin - 3.0.2 - - - true - - - **/logback.xml - **/canal-client.yml - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - 2.2.1 - - - assemble - - single - - package - - - - false - false - - - - - - - - dev - - true - - env - !release - - - - - - - maven-assembly-plugin - - - - ${basedir}/src/main/assembly/dev.xml - - canal_client - ${project.build.directory} - - - - - - - - - release - - - env - release - - - - - - - maven-assembly-plugin - - - - ${basedir}/src/main/assembly/release.xml - - - ${project.artifactId}-${project.version} - - ${project.parent.build.directory} - - - - - - - diff --git a/client-adapter/example/src/main/assembly/dev.xml b/client-adapter/example/src/main/assembly/dev.xml deleted file mode 100644 index a3cbc20c..00000000 --- a/client-adapter/example/src/main/assembly/dev.xml +++ /dev/null @@ -1,57 +0,0 @@ - - dist - - dir - - false - - - . - / - - README* - - - - ./src/main/bin - bin - - **/* - - 0755 - - - ./src/main/conf - /conf - - **/* - - - - ./src/main/resources - /conf - - **/* - - - META-INF/** - - - - target - logs - - **/* - - - - - - lib - - junit:junit - - - - diff --git a/client-adapter/example/src/main/assembly/release.xml b/client-adapter/example/src/main/assembly/release.xml deleted file mode 100644 index 9350da6e..00000000 --- a/client-adapter/example/src/main/assembly/release.xml +++ /dev/null @@ -1,57 +0,0 @@ - - dist - - tar.gz - - false - - - . - / - - README* - - - - ./src/main/bin - bin - - **/* - - 0755 - - - ./src/main/conf - /conf - - **/* - - - - ./src/main/resources - /conf - - **/* - - - META-INF/** - - - - target - logs - - **/* - - - - - - lib - - junit:junit - - - - diff --git a/client-adapter/example/src/main/bin/startup.bat b/client-adapter/example/src/main/bin/startup.bat deleted file mode 100644 index 33fd7945..00000000 --- a/client-adapter/example/src/main/bin/startup.bat +++ /dev/null @@ -1,25 +0,0 @@ -@echo off -@if not "%ECHO%" == "" echo %ECHO% -@if "%OS%" == "Windows_NT" setlocal - -set ENV_PATH=.\ -if "%OS%" == "Windows_NT" set ENV_PATH=%~dp0% - -set conf_dir=%ENV_PATH%\..\conf -set canal_conf=%conf_dir%\canal-client.yml -set logback_configurationFile=%conf_dir%\logback.xml - -set CLASSPATH=%conf_dir% -set CLASSPATH=%conf_dir%\..\lib\*;%CLASSPATH% - -set JAVA_MEM_OPTS= -Xms128m -Xmx512m -XX:PermSize=128m -set JAVA_OPTS_EXT= -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dapplication.codeset=UTF-8 -Dfile.encoding=UTF-8 -set JAVA_DEBUG_OPT= -server -Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=9099,server=y,suspend=n -set CANAL_OPTS= -DappName=otter-canal -Dlogback.configurationFile="%logback_configurationFile%" -Dcanal.conf="%canal_conf%" - -set JAVA_OPTS= %JAVA_MEM_OPTS% %JAVA_OPTS_EXT% %JAVA_DEBUG_OPT% %CANAL_OPTS% - -set CMD_STR= java %JAVA_OPTS% -classpath "%CLASSPATH%" java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.client.ClientLauncher -echo start cmd : %CMD_STR% - -java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.client.ClientLauncher \ No newline at end of file diff --git a/client-adapter/example/src/main/bin/startup.sh b/client-adapter/example/src/main/bin/startup.sh deleted file mode 100644 index 0e09ddff..00000000 --- a/client-adapter/example/src/main/bin/startup.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/bash - -current_path=`pwd` -case "`uname`" in - Linux) - bin_abs_path=$(readlink -f $(dirname $0)) - ;; - *) - bin_abs_path=`cd $(dirname $0); pwd` - ;; -esac -base=${bin_abs_path}/.. -canal_conf=$base/conf/canal-client.yml -logback_configurationFile=$base/conf/logback.xml -export LANG=en_US.UTF-8 -export BASE=$base - -if [ -f $base/bin/canal_client.pid ] ; then - echo "found canal_client.pid , Please run stop.sh first ,then startup.sh" 2>&2 - exit 1 -fi - -## set java path -if [ -z "$JAVA" ] ; then - JAVA=$(which java) -fi - -ALIBABA_JAVA="/usr/alibaba/java/bin/java" -TAOBAO_JAVA="/opt/taobao/java/bin/java" -if [ -z "$JAVA" ]; then - if [ -f $ALIBABA_JAVA ] ; then - JAVA=$ALIBABA_JAVA - elif [ -f $TAOBAO_JAVA ] ; then - JAVA=$TAOBAO_JAVA - else - echo "Cannot find a Java JDK. Please set either set JAVA or put java (>=1.5) in your PATH." 2>&2 - exit 1 - fi -fi - -case "$#" -in -0 ) - ;; -1 ) - var=$* - if [ -f $var ] ; then - canal_conf=$var - else - echo "THE PARAMETER IS NOT CORRECT.PLEASE CHECK AGAIN." - exit - fi;; -2 ) - var=$1 - if [ -f $var ] ; then - canal_conf=$var - else - if [ "$1" = "debug" ]; then - DEBUG_PORT=$2 - DEBUG_SUSPEND="n" - JAVA_DEBUG_OPT="-Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=$DEBUG_PORT,server=y,suspend=$DEBUG_SUSPEND" - fi - fi;; -* ) - echo "THE PARAMETERS MUST BE TWO OR LESS.PLEASE CHECK AGAIN." - exit;; -esac - -str=`file -L $JAVA | grep 64-bit` -if [ -n "$str" ]; then - JAVA_OPTS="-server -Xms2048m -Xmx3072m -Xmn1024m -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -Xss256k -XX:-UseAdaptiveSizePolicy -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+UseCMSCompactAtFullCollection -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:+HeapDumpOnOutOfMemoryError" -else - JAVA_OPTS="-server -Xms1024m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:MaxPermSize=128m " -fi - -JAVA_OPTS=" $JAVA_OPTS -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8" -CANAL_OPTS="-DappName=otter-canal-client -Dlogback.configurationFile=$logback_configurationFile -Dcanal.conf=$canal_conf" - -if [ -e $canal_conf -a -e $logback_configurationFile ] -then - - for i in $base/lib/*; - do CLASSPATH=$i:"$CLASSPATH"; - done - CLASSPATH="$base/conf:$CLASSPATH"; - - echo "cd to $bin_abs_path for workaround relative path" - cd $bin_abs_path - - echo LOG CONFIGURATION : $logback_configurationFile - echo canal conf : $canal_conf - echo CLASSPATH :$CLASSPATH - $JAVA $JAVA_OPTS $JAVA_DEBUG_OPT $CANAL_OPTS -classpath .:$CLASSPATH com.alibaba.otter.canal.client.ClientLauncher 1>>$base/logs/canal_client.log 2>&1 & - echo $! > $base/bin/canal_client.pid - - echo "cd to $current_path for continue" - cd $current_path -else - echo "canal client conf("$canal_conf") OR log configration file($logback_configurationFile) is not exist,please create then first!" -fi diff --git a/client-adapter/example/src/main/bin/stop.sh b/client-adapter/example/src/main/bin/stop.sh deleted file mode 100644 index c96f2807..00000000 --- a/client-adapter/example/src/main/bin/stop.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash - -cygwin=false; -linux=false; -case "`uname`" in - CYGWIN*) - cygwin=true - ;; - Linux*) - linux=true - ;; -esac - -get_pid() { - STR=$1 - PID=$2 - if $cygwin; then - JAVA_CMD="$JAVA_HOME\bin\java" - JAVA_CMD=`cygpath --path --unix $JAVA_CMD` - JAVA_PID=`ps |grep $JAVA_CMD |awk '{print $1}'` - else - if $linux; then - if [ ! -z "$PID" ]; then - JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'` - else - JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep -v grep|awk '{print $2}'` - fi - else - if [ ! -z "$PID" ]; then - JAVA_PID=`ps aux |grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'` - else - JAVA_PID=`ps aux |grep "$STR"|grep -v grep|awk '{print $2}'` - fi - fi - fi - echo $JAVA_PID; -} - -base=`dirname $0`/.. -pidfile=$base/bin/canal_client.pid -if [ ! -f "$pidfile" ];then - echo "canal client is not running. exists" - exit -fi - -pid=`cat $pidfile` -if [ "$pid" == "" ] ; then - pid=`get_pid "appName=otter-canal-client"` -fi - -echo -e "`hostname`: stopping canal $pid ... " -kill $pid - -LOOPS=0 -while (true); -do - gpid=`get_pid "appName=otter-canal-client" "$pid"` - if [ "$gpid" == "" ] ; then - echo "Oook! cost:$LOOPS" - `rm $pidfile` - break; - fi - let LOOPS=LOOPS+1 - sleep 1 -done \ No newline at end of file diff --git a/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/ClientLauncher.java b/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/ClientLauncher.java deleted file mode 100644 index a705a526..00000000 --- a/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/ClientLauncher.java +++ /dev/null @@ -1,66 +0,0 @@ -package com.alibaba.otter.canal.client; - -import java.io.FileInputStream; - -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.yaml.snakeyaml.Yaml; - -import com.alibaba.otter.canal.client.adapter.loader.CanalAdapterLoader; -import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; - -public class ClientLauncher { - - private static final String CLASSPATH_URL_PREFIX = "classpath:"; - private static final Logger logger = LoggerFactory.getLogger(ClientLauncher.class); - - public static void main(String[] args) { - try { - logger.info("## set default uncaught exception handler"); - setGlobalUncaughtExceptionHandler(); - - logger.info("## load canal client configurations"); - String conf = System.getProperty("client.conf", "classpath:canal-client.yml"); - CanalClientConfig canalClientConfig; - if (conf.startsWith(CLASSPATH_URL_PREFIX)) { - conf = StringUtils.substringAfter(conf, CLASSPATH_URL_PREFIX); - canalClientConfig = new Yaml().loadAs(ClientLauncher.class.getClassLoader().getResourceAsStream(conf), - CanalClientConfig.class); - } else { - canalClientConfig = new Yaml().loadAs(new FileInputStream(conf), CanalClientConfig.class); - } - logger.info("## start the canal client adapters."); - final CanalAdapterLoader adapterLoader = new CanalAdapterLoader(canalClientConfig); - adapterLoader.init(); - logger.info("## the canal client adapters are running now ......"); - Runtime.getRuntime().addShutdownHook(new Thread() { - - public void run() { - try { - logger.info("## stop the canal client adapters"); - adapterLoader.destroy(); - } catch (Throwable e) { - logger.warn("## something goes wrong when stopping canal client adapters:", e); - } finally { - logger.info("## canal client adapters are down."); - } - } - - }); - } catch (Throwable e) { - logger.error("## something goes wrong when starting up the canal client adapters:", e); - System.exit(0); - } - } - - private static void setGlobalUncaughtExceptionHandler() { - Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { - - @Override - public void uncaughtException(Thread t, Throwable e) { - logger.error("UnCaughtException", e); - } - }); - } -} diff --git a/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/AbstractCanalAdapterWorker.java b/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/AbstractCanalAdapterWorker.java deleted file mode 100644 index bbfd462d..00000000 --- a/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/AbstractCanalAdapterWorker.java +++ /dev/null @@ -1,168 +0,0 @@ -package com.alibaba.otter.canal.client.adapter.loader; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.alibaba.otter.canal.client.adapter.OuterAdapter; -import com.alibaba.otter.canal.client.adapter.support.Dml; -import com.alibaba.otter.canal.client.adapter.support.MessageUtil; -import com.alibaba.otter.canal.protocol.FlatMessage; -import com.alibaba.otter.canal.protocol.Message; - -/** - * 适配器工作线程抽象类 - * - * @author machengyuan 2018-8-19 下午11:30:49 - * @version 1.0.0 - */ -public abstract class AbstractCanalAdapterWorker { - - protected final Logger logger = LoggerFactory.getLogger(this.getClass()); - - protected String canalDestination; // canal实例 - protected List> canalOuterAdapters; // 外部适配器 - protected ExecutorService groupInnerExecutorService; // 组内工作线程池 - protected volatile boolean running = false; // 是否运行中 - protected Thread thread = null; - protected Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() { - - @Override - public void uncaughtException(Thread t, Throwable e) { - logger.error("parse events has an error", e); - } - }; - - protected void writeOut(final Message message) { - List> futures = new ArrayList<>(); - // 组间适配器并行运行 - for (List outerAdapters : canalOuterAdapters) { - final List adapters = outerAdapters; - futures.add(groupInnerExecutorService.submit(new Callable() { - - @Override - public Boolean call() { - try { - // 组内适配器穿行运行，尽量不要配置组内适配器 - for (final OuterAdapter c : adapters) { - long begin = System.currentTimeMillis(); - MessageUtil.parse4Dml(canalDestination, message, new MessageUtil.Consumer() { - - @Override - public void accept(Dml dml) { - c.sync(dml); - } - }); - - if (logger.isDebugEnabled()) { - logger.debug("{} elapsed time: {}", - c.getClass().getName(), - (System.currentTimeMillis() - begin)); - } - } - return true; - } catch (Exception e) { - return false; - } - } - })); - - // 等待所有适配器写入完成 - // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成 - for (Future f : futures) { - try { - if (!f.get()) { - logger.error("Outer adapter write failed"); - } - } catch (InterruptedException | ExecutionException e) { - // ignore - } - } - } - } - - protected void writeOut(final FlatMessage flatMessage) { - List> futures = new ArrayList<>(); - // 组间适配器并行运行 - for (List outerAdapters : canalOuterAdapters) { - final List adapters = outerAdapters; - futures.add(groupInnerExecutorService.submit(new Callable() { - - @Override - public Boolean call() { - try { - // 组内适配器穿行运行，尽量不要配置组内适配器 - for (OuterAdapter c : adapters) { - long begin = System.currentTimeMillis(); - Dml dml = MessageUtil.flatMessage2Dml(canalDestination, flatMessage); - c.sync(dml); - if (logger.isDebugEnabled()) { - logger.debug("{} elapsed time: {}", - c.getClass().getName(), - (System.currentTimeMillis() - begin)); - } - } - return true; - } catch (Exception e) { - return false; - } - } - })); - - // 等待所有适配器写入完成 - // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成 - for (Future f : futures) { - try { - if (!f.get()) { - logger.error("Outer adapter write failed"); - } - } catch (InterruptedException | ExecutionException e) { - // ignore - } - } - } - } - - protected void writeOut(Message message, String topic) { - if (logger.isDebugEnabled()) { - logger.debug("topic: {} batchId: {} batchSize: {} ", topic, message.getId(), message.getEntries().size()); - } - long begin = System.currentTimeMillis(); - writeOut(message); - long now = System.currentTimeMillis(); - if ((System.currentTimeMillis() - begin) > 5 * 60 * 1000) { - logger.error("topic: {} batchId {} elapsed time: {} ms", topic, message.getId(), now - begin); - } - if (logger.isDebugEnabled()) { - logger.debug("topic: {} batchId {} elapsed time: {} ms", topic, message.getId(), now - begin); - } - } - - protected void stopOutAdapters() { - if (thread != null) { - try { - thread.join(); - } catch (InterruptedException e) { - // ignore - } - } - groupInnerExecutorService.shutdown(); - logger.info("topic connectors' worker thread dead!"); - for (List outerAdapters : canalOuterAdapters) { - for (OuterAdapter adapter : outerAdapters) { - adapter.destroy(); - } - } - logger.info("topic all connectors destroyed!"); - } - - public abstract void start(); - - public abstract void stop(); -} diff --git a/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterKafkaWorker.java b/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterKafkaWorker.java deleted file mode 100644 index b2d3e583..00000000 --- a/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterKafkaWorker.java +++ /dev/null @@ -1,178 +0,0 @@ -package com.alibaba.otter.canal.client.adapter.loader; - -import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -import org.apache.kafka.clients.consumer.CommitFailedException; -import org.apache.kafka.common.errors.WakeupException; - -import com.alibaba.otter.canal.client.adapter.OuterAdapter; -import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector; -import com.alibaba.otter.canal.client.kafka.KafkaCanalConnectors; -import com.alibaba.otter.canal.protocol.FlatMessage; -import com.alibaba.otter.canal.protocol.Message; - -/** - * kafka对应的client适配器工作线程 - * - * @author machengyuan 2018-8-19 下午11:30:49 - * @version 1.0.0 - */ -public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker { - - private KafkaCanalConnector connector; - - private String topic; - - private boolean flatMessage; - - public CanalAdapterKafkaWorker(String bootstrapServers, String topic, String groupId, - List> canalOuterAdapters, boolean flatMessage){ - this.canalOuterAdapters = canalOuterAdapters; - this.groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size()); - this.topic = topic; - this.canalDestination = topic; - this.flatMessage = flatMessage; - connector = KafkaCanalConnectors.newKafkaConnector(bootstrapServers, topic, null, groupId, flatMessage); - // connector.setSessionTimeout(1L, TimeUnit.MINUTES); - - // super.initSwitcher(topic); - } - - @Override - public void start() { - if (!running) { - thread = new Thread(new Runnable() { - - @Override - public void run() { - process(); - } - }); - thread.setUncaughtExceptionHandler(handler); - running = true; - thread.start(); - } - } - - @Override - public void stop() { - try { - if (!running) { - return; - } - - connector.stopRunning(); - running = false; - - // if (switcher != null && !switcher.state()) { - // switcher.set(true); - // } - - if (thread != null) { - try { - thread.join(); - } catch (InterruptedException e) { - // ignore - } - } - groupInnerExecutorService.shutdown(); - logger.info("topic {} connectors' worker thread dead!", this.topic); - for (List outerAdapters : canalOuterAdapters) { - for (OuterAdapter adapter : outerAdapters) { - adapter.destroy(); - } - } - logger.info("topic {} all connectors destroyed!", this.topic); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - private void process() { - while (!running) - ; - ExecutorService executor = Executors.newSingleThreadExecutor(); - // final AtomicBoolean executing = new AtomicBoolean(true); - while (running) { - try { - logger.info("=============> Start to connect topic: {} <=============", this.topic); - connector.connect(); - logger.info("=============> Start to subscribe topic: {} <=============", this.topic); - connector.subscribe(); - logger.info("=============> Subscribe topic: {} succeed <=============", this.topic); - while (running) { - try { - // switcher.get(); //等待开关开启 - - List messages; - if (!flatMessage) { - messages = connector.getWithoutAck(); - } else { - messages = connector.getFlatMessageWithoutAck(100L, TimeUnit.MILLISECONDS); - } - if (messages != null) { - for (final Object message : messages) { - if (message instanceof FlatMessage) { - writeOut((FlatMessage) message); - } else { - writeOut((Message) message); - } - // executing.set(true); - // if (message != null) { - // executor.submit(new Runnable() { - // - // @Override - // public void run() { - // try { - // if (message instanceof FlatMessage) { - // writeOut((FlatMessage) message); - // } else { - // writeOut((Message) message); - // } - // } catch (Exception e) { - // logger.error(e.getMessage(), e); - // } finally { - // executing.compareAndSet(true, false); - // } - // } - // }); - // - // // 间隔一段时间ack一次, 防止因超时未响应切换到另外台客户端 - // long currentTS = System.currentTimeMillis(); - // while (executing.get()) { - // // 大于10秒未消费完ack一次keep alive - // if (System.currentTimeMillis() - currentTS > 10000) { - // connector.ack(); - // currentTS = System.currentTimeMillis(); - // } - // } - // } - } - } - connector.ack(); - } catch (CommitFailedException e) { - logger.warn(e.getMessage()); - } catch (Exception e) { - logger.error(e.getMessage(), e); - TimeUnit.SECONDS.sleep(1L); - } - } - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - executor.shutdown(); - - try { - connector.unsubscribe(); - } catch (WakeupException e) { - // No-op. Continue process - } - connector.disconnect(); - logger.info("=============> Disconnect topic: {} <=============", this.topic); - } -} diff --git a/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterLoader.java b/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterLoader.java deleted file mode 100644 index 51b12a0a..00000000 --- a/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterLoader.java +++ /dev/null @@ -1,177 +0,0 @@ -package com.alibaba.otter.canal.client.adapter.loader; - -import java.net.InetSocketAddress; -import java.net.SocketAddress; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.alibaba.otter.canal.client.adapter.OuterAdapter; -import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; -import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; -import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader; - -/** - * MQ外部适配器的加载器 - * - * @version 1.0.0 - */ -public class CanalAdapterLoader { - - private static final Logger logger = LoggerFactory.getLogger(CanalAdapterLoader.class); - - private CanalClientConfig canalClientConfig; - - private Map canalWorkers = new HashMap<>(); - - private Map canalMQWorker = new HashMap<>(); - - private ExtensionLoader loader; - - public CanalAdapterLoader(CanalClientConfig canalClientConfig){ - this.canalClientConfig = canalClientConfig; - } - - /** - * 初始化canal-client、 canal-client-rocketmq的适配器 - */ - public void init() { - // canal instances 和 mq topics 配置不能同时为空 - if (canalClientConfig.getCanalInstances() == null && canalClientConfig.getMqTopics() == null) { - throw new RuntimeException("Blank config property: canalInstances or canalMQTopics"); - } - - loader = ExtensionLoader.getExtensionLoader(OuterAdapter.class, "" /* - * TODO - * canalClientConfig - * . - * getClassloaderPolicy - * ( - * ) - */); - - String canalServerHost = this.canalClientConfig.getCanalServerHost(); - SocketAddress sa = null; - if (canalServerHost != null) { - String[] ipPort = canalServerHost.split(":"); - sa = new InetSocketAddress(ipPort[0], Integer.parseInt(ipPort[1])); - } - String zkHosts = this.canalClientConfig.getZookeeperHosts(); - - // 初始化canal-client的适配器 - if (canalClientConfig.getCanalInstances() != null) { - for (CanalClientConfig.CanalInstance instance : canalClientConfig.getCanalInstances()) { - List> canalOuterAdapterGroups = new ArrayList<>(); - - for (CanalClientConfig.AdapterGroup connectorGroup : instance.getAdapterGroups()) { - List canalOutConnectors = new ArrayList<>(); - for (OuterAdapterConfig c : connectorGroup.getOutAdapters()) { - loadConnector(c, canalOutConnectors); - } - canalOuterAdapterGroups.add(canalOutConnectors); - } - CanalAdapterWorker worker; - if (zkHosts != null) { - worker = new CanalAdapterWorker(instance.getInstance(), zkHosts, canalOuterAdapterGroups); - } else { - worker = new CanalAdapterWorker(instance.getInstance(), sa, canalOuterAdapterGroups); - } - canalWorkers.put(instance.getInstance(), worker); - worker.start(); - logger.info("Start adapter for canal instance: {} succeed", instance.getInstance()); - } - } - - // 初始化canal-client-mq的适配器 - if (canalClientConfig.getMqTopics() != null) { - for (CanalClientConfig.MQTopic topic : canalClientConfig.getMqTopics()) { - for (CanalClientConfig.Group group : topic.getGroups()) { - List> canalOuterAdapterGroups = new ArrayList<>(); - - List canalOuterAdapters = new ArrayList<>(); - - for (OuterAdapterConfig config : group.getOutAdapters()) { - loadConnector(config, canalOuterAdapters); - } - canalOuterAdapterGroups.add(canalOuterAdapters); - if (StringUtils.isBlank(topic.getMqMode()) || "rocketmq".equalsIgnoreCase(topic.getMqMode())) { - CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig.getBootstrapServers(), - topic.getTopic(), - group.getGroupId(), - canalOuterAdapterGroups); - canalMQWorker.put(topic.getTopic() + "-rocketmq-" + group.getGroupId(), rocketMQWorker); - rocketMQWorker.start(); - } else if ("kafka".equalsIgnoreCase(topic.getMqMode())) { - CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(canalClientConfig.getBootstrapServers(), - topic.getTopic(), - group.getGroupId(), - canalOuterAdapterGroups, - canalClientConfig.getFlatMessage()); - canalMQWorker.put(topic.getTopic() + "-kafka-" + group.getGroupId(), canalKafkaWorker); - canalKafkaWorker.start(); - } - logger.info("Start adapter for canal-client rocketmq topic: {} succeed", topic.getTopic() + "-" - + group.getGroupId()); - - } - } - } - } - - private void loadConnector(OuterAdapterConfig config, List canalOutConnectors) { - try { - OuterAdapter adapter = loader.getExtension(config.getName()); - ClassLoader cl = Thread.currentThread().getContextClassLoader(); - // 替换ClassLoader - Thread.currentThread().setContextClassLoader(adapter.getClass().getClassLoader()); - adapter.init(config); - Thread.currentThread().setContextClassLoader(cl); - canalOutConnectors.add(adapter); - logger.info("Load canal adapter: {} succeed", config.getName()); - } catch (Exception e) { - logger.error("Load canal adapter: {} failed", config.getName(), e); - } - } - - /** - * 销毁所有适配器 为防止canal实例太多造成销毁阻塞, 并行销毁 - */ - public void destroy() { - if (canalWorkers.size() > 0) { - ExecutorService stopExecutorService = Executors.newFixedThreadPool(canalWorkers.size()); - for (CanalAdapterWorker v : canalWorkers.values()) { - final CanalAdapterWorker caw = v; - stopExecutorService.submit(new Runnable() { - - @Override - public void run() { - caw.stop(); - } - }); - } - stopExecutorService.shutdown(); - } - if (canalMQWorker.size() > 0) { - ExecutorService stopMQWokerService = Executors.newFixedThreadPool(canalMQWorker.size()); - for (AbstractCanalAdapterWorker tmp : canalMQWorker.values()) { - final AbstractCanalAdapterWorker worker = tmp; - stopMQWokerService.submit(new Runnable() { - - @Override - public void run() { - worker.stop(); - } - }); - } - stopMQWokerService.shutdown(); - } - logger.info("All canal adapters destroyed"); - } -} diff --git a/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterRocketMQWorker.java b/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterRocketMQWorker.java deleted file mode 100644 index 9c126297..00000000 --- a/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterRocketMQWorker.java +++ /dev/null @@ -1,124 +0,0 @@ -package com.alibaba.otter.canal.client.adapter.loader; - -import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -import org.apache.kafka.clients.consumer.CommitFailedException; -import org.apache.kafka.common.errors.WakeupException; - -import com.alibaba.otter.canal.client.adapter.OuterAdapter; -import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnector; -import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnectorProvider; -import com.alibaba.otter.canal.protocol.Message; - -/** - * kafka对应的client适配器工作线程 - * - * @author machengyuan 2018-8-19 下午11:30:49 - * @version 1.0.0 - */ -public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker { - - private RocketMQCanalConnector connector; - - private String topic; - - public CanalAdapterRocketMQWorker(String nameServers, String topic, String groupId, - List> canalOuterAdapters){ - logger.info("RocketMQ consumer config topic:{}, nameServer:{}, groupId:{}", topic, nameServers, groupId); - this.canalOuterAdapters = canalOuterAdapters; - this.groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size()); - this.topic = topic; - this.canalDestination = topic; - connector = RocketMQCanalConnectorProvider.newRocketMQConnector(nameServers, topic, groupId); - } - - @Override - public void start() { - if (!running) { - thread = new Thread(new Runnable() { - - @Override - public void run() { - process(); - } - }); - thread.setUncaughtExceptionHandler(handler); - running = true; - thread.start(); - } - } - - @Override - public void stop() { - try { - if (!running) { - return; - } - connector.stopRunning(); - running = false; - logger.info("Stop topic {} out adapters begin", this.topic); - stopOutAdapters(); - logger.info("Stop topic {} out adapters end", this.topic); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - private void process() { - while (!running) - ; - ExecutorService executor = Executors.newSingleThreadExecutor(); - while (running) { - try { - logger.info("=============> Start to connect topic: {} <=============", this.topic); - connector.connect(); - logger.info("=============> Start to subscribe topic: {}<=============", this.topic); - connector.subscribe(); - logger.info("=============> Subscribe topic: {} succeed<=============", this.topic); - while (running) { - try { - // switcher.get(); //等待开关开启 - - final Message message = connector.getWithoutAck(1); - if (message != null) { - executor.submit(new Runnable() { - - @Override - public void run() { - try { - writeOut(message, topic); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - connector.ack(message.getId()); - } - }); - } else { - logger.debug("Message is null"); - } - } catch (CommitFailedException e) { - logger.warn(e.getMessage()); - } catch (Exception e) { - logger.error(e.getMessage(), e); - TimeUnit.SECONDS.sleep(1L); - } - } - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - executor.shutdown(); - - try { - connector.unsubscribe(); - } catch (WakeupException e) { - // No-op. Continue process - } - connector.stopRunning(); - logger.info("=============> Disconnect topic: {} <=============", this.topic); - } -} diff --git a/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterWorker.java b/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterWorker.java deleted file mode 100644 index e72ed2d5..00000000 --- a/client-adapter/example/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterWorker.java +++ /dev/null @@ -1,186 +0,0 @@ -package com.alibaba.otter.canal.client.adapter.loader; - -import java.net.SocketAddress; -import java.util.List; -import java.util.concurrent.Executors; - -import com.alibaba.otter.canal.client.CanalConnector; -import com.alibaba.otter.canal.client.CanalConnectors; -import com.alibaba.otter.canal.client.adapter.OuterAdapter; -import com.alibaba.otter.canal.client.impl.ClusterCanalConnector; -import com.alibaba.otter.canal.protocol.Message; - -/** - * 原生canal-server对应的client适配器工作线程 - * - * @author machengyuan 2018-8-19 下午11:30:49 - * @version 1.0.0 - */ -public class CanalAdapterWorker extends AbstractCanalAdapterWorker { - - private static final int BATCH_SIZE = 50; - private static final int SO_TIMEOUT = 0; - - private CanalConnector connector; - - /** - * 单台client适配器worker的构造方法 - * - * @param canalDestination canal实例名 - * @param address canal-server地址 - * @param canalOuterAdapters 外部适配器组 - */ - public CanalAdapterWorker(String canalDestination, SocketAddress address, - List> canalOuterAdapters){ - this.canalOuterAdapters = canalOuterAdapters; - this.canalDestination = canalDestination; - groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size()); - connector = CanalConnectors.newSingleConnector(address, canalDestination, "", ""); - } - - /** - * HA模式下client适配器worker的构造方法 - * - * @param canalDestination canal实例名 - * @param zookeeperHosts zookeeper地址 - * @param canalOuterAdapters 外部适配器组 - */ - public CanalAdapterWorker(String canalDestination, String zookeeperHosts, - List> canalOuterAdapters){ - this.canalOuterAdapters = canalOuterAdapters; - this.canalDestination = canalDestination; - groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size()); - connector = CanalConnectors.newClusterConnector(zookeeperHosts, canalDestination, "", ""); - ((ClusterCanalConnector) connector).setSoTimeout(SO_TIMEOUT); - - // super.initSwitcher(canalDestination); - } - - @Override - public void start() { - if (!running) { - thread = new Thread(new Runnable() { - - @Override - public void run() { - process(); - } - }); - thread.setUncaughtExceptionHandler(handler); - thread.start(); - running = true; - } - } - - @Override - public void stop() { - try { - if (!running) { - return; - } - - // if (switcher != null && !switcher.state()) { - // switcher.set(true); - // } - - connector.stopRunning(); - running = false; - - logger.info("destination {} is waiting for adapters' worker thread die!", canalDestination); - if (thread != null) { - try { - thread.join(); - } catch (InterruptedException e) { - // ignore - } - } - groupInnerExecutorService.shutdown(); - logger.info("destination {} adapters' worker thread dead!", canalDestination); - for (List outerAdapters : canalOuterAdapters) { - for (OuterAdapter adapter : outerAdapters) { - adapter.destroy(); - } - } - logger.info("destination {} all adapters destroyed!", canalDestination); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - private void process() { - while (!running) - ; // waiting until running == true - while (running) { - try { - // if (switcher != null) { - // switcher.get(); - // } - logger.info("=============> Start to connect destination: {} <=============", this.canalDestination); - connector.connect(); - logger.info("=============> Start to subscribe destination: {} <=============", this.canalDestination); - connector.subscribe(); - logger.info("=============> Subscribe destination: {} succeed <=============", this.canalDestination); - while (running) { - // try { - // if (switcher != null) { - // switcher.get(); - // } - // } catch (TimeoutException e) { - // break; - // } - - // server配置canal.instance.network.soTimeout(默认: 30s) - // 范围内未与server交互，server将关闭本次socket连接 - Message message = connector.getWithoutAck(BATCH_SIZE); // 获取指定数量的数据 - long batchId = message.getId(); - try { - int size = message.getEntries().size(); - - if (batchId == -1 || size == 0) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - // ignore - } - } else { - if (logger.isDebugEnabled()) { - logger.debug("destination: {} batchId: {} batchSize: {} ", - this.canalDestination, - batchId, - size); - } - long begin = System.currentTimeMillis(); - writeOut(message); - long now = System.currentTimeMillis(); - if (logger.isDebugEnabled()) { - logger.debug("destination: {} batchId: {} elapsed time: {} ms", - this.canalDestination, - batchId, - now - begin); - } - } - connector.ack(batchId); // 提交确认 - } catch (Exception e) { - connector.rollback(batchId); // 处理失败, 回滚数据 - throw e; - } - } - - } catch (Exception e) { - logger.error("process error!", e); - } finally { - connector.disconnect(); - logger.info("=============> Disconnect destination: {} <=============", this.canalDestination); - } - - if (running) { // is reconnect - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - // ignore - } - } - - } - } -} diff --git a/client-adapter/example/src/main/resources/canal-client.yml b/client-adapter/example/src/main/resources/canal-client.yml deleted file mode 100644 index de24e756..00000000 --- a/client-adapter/example/src/main/resources/canal-client.yml +++ /dev/null @@ -1,18 +0,0 @@ -canalServerHost: 127.0.0.1:11111 -#zookeeperHosts: slave1:2181 -#bootstrapServers: slave1:6667,slave2:6667 #or rocketmq nameservers:host1:9876;host2:9876 -flatMessage: false - -canalInstances: -- instance: example - adapterGroups: - - outAdapters: - - name: logger - -#mqTopics: -#- mqMode: rocketmq -# topic: example -# groups: -# - groupId: example -# outAdapters: -# - name: logger diff --git a/client-adapter/example/src/main/resources/logback.xml b/client-adapter/example/src/main/resources/logback.xml deleted file mode 100644 index e74c84f0..00000000 --- a/client-adapter/example/src/main/resources/logback.xml +++ /dev/null @@ -1,47 +0,0 @@ - - - - - %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n - - - - - - - destination - canal_client - - - - ../logs/${destination}.log - - - ../logs/%d{yyyy-MM-dd}/${destination}-%d{yyyy-MM-dd}-%i.log.gz - - - 512MB - - 60 - - - - %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/client-adapter/hbase/pom.xml b/client-adapter/hbase/pom.xml index a48d7401..f48cdd10 100644 --- a/client-adapter/hbase/pom.xml +++ b/client-adapter/hbase/pom.xml @@ -43,48 +43,60 @@ - - - - org.apache.maven.plugins - maven-assembly-plugin - 2.4 - - - jar-with-dependencies - - - - - make-assembly - package - - single - - - - - - maven-antrun-plugin - - - package - - run - - - - - - - - - - - - - - - + + + release + + + env + release + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.4 + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + maven-antrun-plugin + + + package + + run + + + + + + + + + + + + + + + + + diff --git a/client-adapter/launcher/pom.xml b/client-adapter/launcher/pom.xml index cb8aee72..f10dfaac 100644 --- a/client-adapter/launcher/pom.xml +++ b/client-adapter/launcher/pom.xml @@ -104,6 +104,18 @@ org.springframework.boot spring-boot-maven-plugin 2.0.1.RELEASE + + + + com.alibaba.otter + client-adapter.hbase + + + com.alibaba.otter + client-adapter.logger + + + @@ -150,24 +162,43 @@ - - maven-assembly-plugin - - false - - assembly.xml - - - - - make-assembly - package - - single - - - - + + + + release + + + env + release + + + + + + + maven-assembly-plugin + + false + + ${basedir}/src/main/assembly/release.xml + + ${project.artifactId}-${project.version} + ${project.basedir}/../../target + + + + make-assembly + package + + single + + + + + + + + diff --git a/client-adapter/launcher/assembly.xml b/client-adapter/launcher/src/main/assembly/release.xml similarity index 100% rename from client-adapter/launcher/assembly.xml rename to client-adapter/launcher/src/main/assembly/release.xml diff --git a/client-adapter/logger/pom.xml b/client-adapter/logger/pom.xml index 8f2b4edb..e9a146aa 100644 --- a/client-adapter/logger/pom.xml +++ b/client-adapter/logger/pom.xml @@ -21,27 +21,39 @@ - - - - org.apache.maven.plugins - maven-assembly-plugin - 2.4 - - - jar-with-dependencies - - - - - make-assembly - package - - single - - - - - - + + + release + + + env + release + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.4 + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + + + diff --git a/client-adapter/pom.xml b/client-adapter/pom.xml index 7ca65de1..aafe6395 100644 --- a/client-adapter/pom.xml +++ b/client-adapter/pom.xml @@ -24,7 +24,6 @@ logger hbase launcher - example From 0d22dbc9828e24cc5ffbf8bd77914db8b93b990a Mon Sep 17 00:00:00 2001 From: mcy Date: Fri, 26 Oct 2018 15:56:05 +0800 Subject: [PATCH 30/35] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=89=93=E5=8C=85?= =?UTF-8?q?=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client-adapter/hbase/pom.xml | 78 ++++++++++----------------------- client-adapter/launcher/pom.xml | 5 +++ client-adapter/logger/pom.xml | 57 ++++++++++-------------- 3 files changed, 51 insertions(+), 89 deletions(-) diff --git a/client-adapter/hbase/pom.xml b/client-adapter/hbase/pom.xml index f48cdd10..b3d274e4 100644 --- a/client-adapter/hbase/pom.xml +++ b/client-adapter/hbase/pom.xml @@ -43,60 +43,28 @@ + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.4 + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + - - - release - - - env - release - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - 2.4 - - - jar-with-dependencies - - - - - make-assembly - package - - single - - - - - - maven-antrun-plugin - - - package - - run - - - - - - - - - - - - - - - - - diff --git a/client-adapter/launcher/pom.xml b/client-adapter/launcher/pom.xml index f10dfaac..8159bb66 100644 --- a/client-adapter/launcher/pom.xml +++ b/client-adapter/launcher/pom.xml @@ -139,6 +139,11 @@ + + + + + diff --git a/client-adapter/logger/pom.xml b/client-adapter/logger/pom.xml index e9a146aa..af7ec379 100644 --- a/client-adapter/logger/pom.xml +++ b/client-adapter/logger/pom.xml @@ -21,39 +21,28 @@ - - - release - - - env - release - - - - - - org.apache.maven.plugins - maven-assembly-plugin - 2.4 - - - jar-with-dependencies - - - - - make-assembly - package - - single - - - - - - - - + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.4 + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + From 2d930894ffc944af6d62bffe60118a7a0c98d3e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Fri, 26 Oct 2018 21:50:12 +0800 Subject: [PATCH 31/35] fixed issue #1046, tinyint(1) unsigned with value 255 --- .../canal/parse/inbound/mysql/dbsync/LogEventConvert.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java index 7fcd1a1f..ec71465b 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java @@ -671,9 +671,6 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar } int javaType = buffer.getJavaType(); - if (isSingleBit && javaType == Types.TINYINT) { - javaType = Types.BIT; - } if (buffer.isNull()) { columnBuilder.setIsNull(true); } else { @@ -722,6 +719,10 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar // 对象为number类型，直接valueof即可 columnBuilder.setValue(String.valueOf(value)); } + + if (isSingleBit && javaType == Types.TINYINT) { + javaType = Types.BIT; + } break; case Types.REAL: // float case Types.DOUBLE: // double From 1e2cf9d47636b382fe1c76f8d00e9e641699603a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Fri, 26 Oct 2018 21:51:19 +0800 Subject: [PATCH 32/35] fixed bit(8) with value 255 --- .../java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java index 9fe22820..8cd57332 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java @@ -367,7 +367,7 @@ public final class RowsLogBuffer { // 转化为unsign long switch (len) { case 1: - value = buffer.getInt8(); + value = buffer.getUint8(); break; case 2: value = buffer.getBeUint16(); From f822761ab27bc5ee262ca8b80cd2e0dcb283624a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Fri, 26 Oct 2018 23:46:41 +0800 Subject: [PATCH 33/35] fixed issue #1047 , support snapshot expire --- deployer/src/main/resources/canal.properties | 4 ++ .../resources/spring/default-instance.xml | 2 + .../main/resources/spring/file-instance.xml | 2 + .../main/resources/spring/memory-instance.xml | 2 + .../spring/tsdb/sql-map/sqlmap_history.xml | 4 +- .../spring/tsdb/sql-map/sqlmap_snapshot.xml | 4 +- .../manager/CanalInstanceWithManager.java | 6 ++ .../manager/model/CanalParameter.java | 18 ++++++ .../mysql/AbstractMysqlEventParser.java | 18 ++++++ .../inbound/mysql/LocalBinlogEventParser.java | 2 + .../parse/inbound/mysql/MysqlEventParser.java | 2 + .../inbound/mysql/tsdb/DatabaseTableMeta.java | 61 ++++++++++++++----- .../mysql/tsdb/dao/MetaHistoryDAO.java | 11 ++-- .../mysql/tsdb/dao/MetaSnapshotDAO.java | 11 ++-- .../mysql/tsdb/MetaHistoryDAOTest.java | 19 +++++- .../mysql/tsdb/MetaSnapshotDAOTest.java | 41 +++++++++++++ .../resources/tsdb/sql-map/sqlmap_history.xml | 4 +- .../tsdb/sql-map/sqlmap_snapshot.xml | 4 +- 18 files changed, 176 insertions(+), 39 deletions(-) create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MetaSnapshotDAOTest.java diff --git a/deployer/src/main/resources/canal.properties b/deployer/src/main/resources/canal.properties index 2a24b66f..b5284dec 100644 --- a/deployer/src/main/resources/canal.properties +++ b/deployer/src/main/resources/canal.properties @@ -69,6 +69,10 @@ canal.instance.tsdb.dir=${canal.file.data.dir:../conf}/${canal.instance.destinat canal.instance.tsdb.url=jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL; canal.instance.tsdb.dbUsername=canal canal.instance.tsdb.dbPassword=canal +# dump snapshot interval, default 24 hour +canal.instance.tsdb.snapshot.interval=24 +# purge snapshot expire , default 360 hour(15 days) +canal.instance.tsdb.snapshot.expire=360 # rds oss binlog account canal.instance.rds.accesskey = diff --git a/deployer/src/main/resources/spring/default-instance.xml b/deployer/src/main/resources/spring/default-instance.xml index fe9cc7a5..f8c19bd0 100644 --- a/deployer/src/main/resources/spring/default-instance.xml +++ b/deployer/src/main/resources/spring/default-instance.xml @@ -176,6 +176,8 @@ + + diff --git a/deployer/src/main/resources/spring/file-instance.xml b/deployer/src/main/resources/spring/file-instance.xml index 23d8e4bd..5524f73f 100644 --- a/deployer/src/main/resources/spring/file-instance.xml +++ b/deployer/src/main/resources/spring/file-instance.xml @@ -161,6 +161,8 @@ + + diff --git a/deployer/src/main/resources/spring/memory-instance.xml b/deployer/src/main/resources/spring/memory-instance.xml index f785a46e..8fdfb178 100644 --- a/deployer/src/main/resources/spring/memory-instance.xml +++ b/deployer/src/main/resources/spring/memory-instance.xml @@ -149,6 +149,8 @@ + + diff --git a/deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_history.xml b/deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_history.xml index 7445bf69..58957c28 100644 --- a/deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_history.xml +++ b/deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_history.xml @@ -36,10 +36,10 @@ - + \ No newline at end of file diff --git a/deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_snapshot.xml b/deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_snapshot.xml index 79aa25fe..2ee0f8f4 100644 --- a/deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_snapshot.xml +++ b/deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_snapshot.xml @@ -42,10 +42,10 @@ where destination=#destination# - + 0 ]]> \ No newline at end of file diff --git a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java index dff18278..154cd53f 100644 --- a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java +++ b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java @@ -296,6 +296,12 @@ public class CanalInstanceWithManager extends AbstractCanalInstance { mysqlEventParser.setFilterTableError(parameters.getFilterTableError()); mysqlEventParser.setIsGTIDMode(BooleanUtils.toBoolean(parameters.getGtidEnable())); // tsdb + if (parameters.getTsdbSnapshotInterval() != null) { + mysqlEventParser.setTsdbSnapshotInterval(parameters.getTsdbSnapshotInterval()); + } + if (parameters.getTsdbSnapshotExpire() != null) { + mysqlEventParser.setTsdbSnapshotExpire(parameters.getTsdbSnapshotExpire()); + } boolean tsdbEnable = BooleanUtils.toBoolean(parameters.getTsdbEnable()); if (tsdbEnable) { mysqlEventParser.setEnableTsdb(tsdbEnable); diff --git a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java index ca5e1af7..af3841dd 100644 --- a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java +++ b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java @@ -98,6 +98,8 @@ public class CanalParameter implements Serializable { private String tsdbJdbcUrl; private String tsdbJdbcUserName; private String tsdbJdbcPassword; + private Integer tsdbSnapshotInterval = 24; + private Integer tsdbSnapshotExpire = 360; private String rdsAccesskey; private String rdsSecretkey; private String rdsInstanceId; @@ -964,6 +966,22 @@ public class CanalParameter implements Serializable { this.memoryStorageRawEntry = memoryStorageRawEntry; } + public Integer getTsdbSnapshotInterval() { + return tsdbSnapshotInterval; + } + + public void setTsdbSnapshotInterval(Integer tsdbSnapshotInterval) { + this.tsdbSnapshotInterval = tsdbSnapshotInterval; + } + + public Integer getTsdbSnapshotExpire() { + return tsdbSnapshotExpire; + } + + public void setTsdbSnapshotExpire(Integer tsdbSnapshotExpire) { + this.tsdbSnapshotExpire = tsdbSnapshotExpire; + } + public String toString() { return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE); } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java index 00e302e6..8050186d 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java @@ -26,6 +26,8 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser { protected TableMetaTSDBFactory tableMetaTSDBFactory = new DefaultTableMetaTSDBFactory(); protected boolean enableTsdb = false; + protected int tsdbSnapshotInterval = 24; + protected int tsdbSnapshotExpire = 360; protected String tsdbSpringXml; protected TableMetaTSDB tableMetaTSDB; @@ -210,4 +212,20 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser { return this.receivedBinlogBytes; } + public int getTsdbSnapshotInterval() { + return tsdbSnapshotInterval; + } + + public void setTsdbSnapshotInterval(int tsdbSnapshotInterval) { + this.tsdbSnapshotInterval = tsdbSnapshotInterval; + } + + public int getTsdbSnapshotExpire() { + return tsdbSnapshotExpire; + } + + public void setTsdbSnapshotExpire(int tsdbSnapshotExpire) { + this.tsdbSnapshotExpire = tsdbSnapshotExpire; + } + } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java index ea6b0365..4f6ace4f 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java @@ -55,6 +55,8 @@ public class LocalBinlogEventParser extends AbstractMysqlEventParser implements ((DatabaseTableMeta) tableMetaTSDB).setConnection(metaConnection); ((DatabaseTableMeta) tableMetaTSDB).setFilter(eventFilter); ((DatabaseTableMeta) tableMetaTSDB).setBlackFilter(eventBlackFilter); + ((DatabaseTableMeta) tableMetaTSDB).setSnapshotInterval(tsdbSnapshotInterval); + ((DatabaseTableMeta) tableMetaTSDB).setSnapshotExpire(tsdbSnapshotExpire); } tableMetaCache = new TableMetaCache(metaConnection, tableMetaTSDB); diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java index cf102d3d..7f83b6d2 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java @@ -121,6 +121,8 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE ((DatabaseTableMeta) tableMetaTSDB).setConnection(metaConnection); ((DatabaseTableMeta) tableMetaTSDB).setFilter(eventFilter); ((DatabaseTableMeta) tableMetaTSDB).setBlackFilter(eventBlackFilter); + ((DatabaseTableMeta) tableMetaTSDB).setSnapshotInterval(tsdbSnapshotInterval); + ((DatabaseTableMeta) tableMetaTSDB).setSnapshotExpire(tsdbSnapshotExpire); } tableMetaCache = new TableMetaCache(metaConnection, tableMetaTSDB); diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java index bb2af45f..d7b2f7d2 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java @@ -44,19 +44,21 @@ import com.alibaba.otter.canal.protocol.position.EntryPosition; */ public class DatabaseTableMeta implements TableMetaTSDB { - public static final EntryPosition INIT_POSITION = new EntryPosition("0", 0L, -2L, -1L); - private static Logger logger = LoggerFactory.getLogger(DatabaseTableMeta.class); - private static Pattern pattern = Pattern.compile("Duplicate entry '.*' for key '*'"); - private static Pattern h2Pattern = Pattern.compile("Unique index or primary key violation"); + public static final EntryPosition INIT_POSITION = new EntryPosition("0", 0L, -2L, -1L); + private static Logger logger = LoggerFactory.getLogger(DatabaseTableMeta.class); + private static Pattern pattern = Pattern.compile("Duplicate entry '.*' for key '*'"); + private static Pattern h2Pattern = Pattern.compile("Unique index or primary key violation"); private String destination; private MemoryTableMeta memoryTableMeta; - private MysqlConnection connection; // 查询meta信息的链接 + private MysqlConnection connection; // 查询meta信息的链接 private CanalEventFilter filter; private CanalEventFilter blackFilter; private EntryPosition lastPosition; private ScheduledExecutorService scheduler; private MetaHistoryDAO metaHistoryDAO; private MetaSnapshotDAO metaSnapshotDAO; + private int snapshotInterval = 24; + private int snapshotExpire = 360; public DatabaseTableMeta(){ @@ -77,18 +79,27 @@ public class DatabaseTableMeta implements TableMetaTSDB { }); // 24小时生成一份snapshot - scheduler.scheduleWithFixedDelay(new Runnable() { + if (snapshotInterval > 0) { + scheduler.scheduleWithFixedDelay(new Runnable() { - @Override - public void run() { - try { - MDC.put("destination", destination); - applySnapshotToDB(lastPosition, false); - } catch (Throwable e) { - logger.error("scheudle applySnapshotToDB faield", e); + @Override + public void run() { + try { + MDC.put("destination", destination); + applySnapshotToDB(lastPosition, false); + } catch (Throwable e) { + logger.error("scheudle applySnapshotToDB faield", e); + } + + try { + MDC.put("destination", destination); + snapshotExpire((int) TimeUnit.HOURS.toSeconds(snapshotExpire)); + } catch (Throwable e) { + logger.error("scheudle snapshotExpire faield", e); + } } - } - }, 24, 24, TimeUnit.HOURS); + }, snapshotInterval, snapshotInterval, TimeUnit.HOURS); + } return true; } @@ -461,6 +472,10 @@ public class DatabaseTableMeta implements TableMetaTSDB { return true; } + private int snapshotExpire(int expireTimestamp) { + return metaSnapshotDAO.deleteByTimestamp(destination, expireTimestamp); + } + public void setConnection(MysqlConnection connection) { this.connection = connection; } @@ -489,6 +504,22 @@ public class DatabaseTableMeta implements TableMetaTSDB { this.blackFilter = blackFilter; } + public int getSnapshotInterval() { + return snapshotInterval; + } + + public void setSnapshotInterval(int snapshotInterval) { + this.snapshotInterval = snapshotInterval; + } + + public int getSnapshotExpire() { + return snapshotExpire; + } + + public void setSnapshotExpire(int snapshotExpire) { + this.snapshotExpire = snapshotExpire; + } + public MysqlConnection getConnection() { return connection; } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaHistoryDAO.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaHistoryDAO.java index 6ec51614..c214d88f 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaHistoryDAO.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaHistoryDAO.java @@ -1,7 +1,5 @@ package com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao; -import java.text.SimpleDateFormat; -import java.util.Date; import java.util.HashMap; import java.util.List; @@ -37,13 +35,12 @@ public class MetaHistoryDAO extends MetaBaseDAO { /** * 删除interval秒之前的数据 */ - public Integer deleteByGmtModified(int interval) { + public Integer deleteByTimestamp(String destination, int interval) { HashMap params = Maps.newHashMapWithExpectedSize(2); long timestamp = System.currentTimeMillis() - interval * 1000; - Date date = new Date(timestamp); - SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - params.put("timestamp", format.format(date)); - return getSqlMapClientTemplate().delete("meta_history.deleteByGmtModified", params); + params.put("timestamp", timestamp); + params.put("destination", destination); + return getSqlMapClientTemplate().delete("meta_history.deleteByTimestamp", params); } protected void initDao() throws Exception { diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaSnapshotDAO.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaSnapshotDAO.java index f525031a..ff79b107 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaSnapshotDAO.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaSnapshotDAO.java @@ -1,7 +1,5 @@ package com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao; -import java.text.SimpleDateFormat; -import java.util.Date; import java.util.HashMap; import com.google.common.collect.Maps; @@ -40,13 +38,12 @@ public class MetaSnapshotDAO extends MetaBaseDAO { /** * 删除interval秒之前的数据 */ - public Integer deleteByGmtModified(int interval) { + public Integer deleteByTimestamp(String destination, int interval) { HashMap params = Maps.newHashMapWithExpectedSize(2); long timestamp = System.currentTimeMillis() - interval * 1000; - Date date = new Date(timestamp); - SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - params.put("timestamp", format.format(date)); - return getSqlMapClientTemplate().delete("meta_snapshot.deleteByGmtModified", params); + params.put("timestamp", timestamp); + params.put("destination", destination); + return getSqlMapClientTemplate().delete("meta_snapshot.deleteByTimestamp", params); } protected void initDao() throws Exception { diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MetaHistoryDAOTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MetaHistoryDAOTest.java index b199a326..cea6c087 100644 --- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MetaHistoryDAOTest.java +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MetaHistoryDAOTest.java @@ -16,7 +16,7 @@ import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDO; * Created by wanshao Date: 2017/9/20 Time: 下午5:00 **/ @RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(locations = { "/tsdb/mysql-tsdb.xml" }) +@ContextConfiguration(locations = { "/tsdb/h2-tsdb.xml" }) public class MetaHistoryDAOTest { @Resource @@ -24,7 +24,22 @@ public class MetaHistoryDAOTest { @Test public void testSimple() { - List metaHistoryDOList = metaHistoryDAO.findByTimestamp("test", 0L, 0L); + MetaHistoryDO historyDO = new MetaHistoryDO(); + historyDO.setDestination("test"); + historyDO.setBinlogFile("000001"); + historyDO.setBinlogOffest(4L); + historyDO.setBinlogMasterId("1"); + historyDO.setBinlogTimestamp(System.currentTimeMillis() - 7300 * 1000); + historyDO.setSqlSchema("test"); + historyDO.setUseSchema("test"); + historyDO.setSqlTable("testTable"); + historyDO.setSqlTable("drop table testTable"); + metaHistoryDAO.insert(historyDO); + + int count = metaHistoryDAO.deleteByTimestamp("test", 7200); + System.out.println(count); + + List metaHistoryDOList = metaHistoryDAO.findByTimestamp("test", 0L, System.currentTimeMillis()); for (MetaHistoryDO metaHistoryDO : metaHistoryDOList) { System.out.println(metaHistoryDO.getId()); } diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MetaSnapshotDAOTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MetaSnapshotDAOTest.java new file mode 100644 index 00000000..f503e4b2 --- /dev/null +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MetaSnapshotDAOTest.java @@ -0,0 +1,41 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.tsdb; + +import javax.annotation.Resource; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDAO; +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDO; + +/** + * Created by wanshao Date: 2017/9/20 Time: 下午5:00 + **/ +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(locations = { "/tsdb/h2-tsdb.xml" }) +public class MetaSnapshotDAOTest { + + @Resource + MetaSnapshotDAO metaSnapshotDAO; + + @Test + public void testSimple() { + MetaSnapshotDO metaSnapshotDO = new MetaSnapshotDO(); + metaSnapshotDO.setDestination("test"); + metaSnapshotDO.setBinlogFile("000001"); + metaSnapshotDO.setBinlogOffest(4L); + metaSnapshotDO.setBinlogMasterId("1"); + metaSnapshotDO.setBinlogTimestamp(System.currentTimeMillis() - 7300 * 1000); + metaSnapshotDO.setData("test"); + metaSnapshotDAO.insert(metaSnapshotDO); + + MetaSnapshotDO snapshotDO = metaSnapshotDAO.findByTimestamp("test", System.currentTimeMillis()); + System.out.println(snapshotDO); + + int count = metaSnapshotDAO.deleteByTimestamp("test", 7200); + System.out.println(count); + } + +} diff --git a/parse/src/test/resources/tsdb/sql-map/sqlmap_history.xml b/parse/src/test/resources/tsdb/sql-map/sqlmap_history.xml index 7445bf69..58957c28 100644 --- a/parse/src/test/resources/tsdb/sql-map/sqlmap_history.xml +++ b/parse/src/test/resources/tsdb/sql-map/sqlmap_history.xml @@ -36,10 +36,10 @@ - + \ No newline at end of file diff --git a/parse/src/test/resources/tsdb/sql-map/sqlmap_snapshot.xml b/parse/src/test/resources/tsdb/sql-map/sqlmap_snapshot.xml index 7ba0a435..60827c6c 100644 --- a/parse/src/test/resources/tsdb/sql-map/sqlmap_snapshot.xml +++ b/parse/src/test/resources/tsdb/sql-map/sqlmap_snapshot.xml @@ -54,10 +54,10 @@ where destination=#destination# - + 0 ]]> \ No newline at end of file From fd7b9d766a89f8a6621d37ac6a109e98f57cd97f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Sat, 27 Oct 2018 00:47:50 +0800 Subject: [PATCH 34/35] fixed snapshot expire --- .../canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java index d7b2f7d2..b0dd89ce 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java @@ -84,16 +84,19 @@ public class DatabaseTableMeta implements TableMetaTSDB { @Override public void run() { + boolean applyResult = false; try { MDC.put("destination", destination); - applySnapshotToDB(lastPosition, false); + applyResult = applySnapshotToDB(lastPosition, false); } catch (Throwable e) { logger.error("scheudle applySnapshotToDB faield", e); } try { MDC.put("destination", destination); - snapshotExpire((int) TimeUnit.HOURS.toSeconds(snapshotExpire)); + if (applyResult) { + snapshotExpire((int) TimeUnit.HOURS.toSeconds(snapshotExpire)); + } } catch (Throwable e) { logger.error("scheudle snapshotExpire faield", e); } From 87ac3daecc3f45e796ed56bf1a1922200533c26b Mon Sep 17 00:00:00 2001 From: mcy Date: Sat, 27 Oct 2018 20:03:26 +0800 Subject: [PATCH 35/35] =?UTF-8?q?=E4=BF=AE=E6=94=B9hbase=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E9=A1=B9=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../client/adapter/hbase/HbaseAdapter.java | 12 +- .../adapter/hbase/config/MappingConfig.java | 42 +++---- .../hbase/config/MappingConfigLoader.java | 18 +-- .../hbase/service/HbaseEtlService.java | 72 +++++------ .../hbase/service/HbaseSyncService.java | 116 +++++++++--------- .../main/resources/hbase/mytest_person2.yml | 4 +- .../src/main/resources/application.yml | 32 ++--- 7 files changed, 148 insertions(+), 148 deletions(-) diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java index ababeb6e..e97035ed 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java @@ -53,9 +53,9 @@ public class HbaseAdapter implements OuterAdapter { hbaseMapping = MappingConfigLoader.load(); mappingConfigCache = new HashMap<>(); for (MappingConfig mappingConfig : hbaseMapping.values()) { - mappingConfigCache.put(StringUtils.trimToEmpty(mappingConfig.getHbaseOrm().getDestination()) - + "." + mappingConfig.getHbaseOrm().getDatabase() + "." - + mappingConfig.getHbaseOrm().getTable(), + mappingConfigCache.put(StringUtils.trimToEmpty(mappingConfig.getHbaseMapping().getDestination()) + + "." + mappingConfig.getHbaseMapping().getDatabase() + "." + + mappingConfig.getHbaseMapping().getTable(), mappingConfig); } } @@ -136,7 +136,7 @@ public class HbaseAdapter implements OuterAdapter { @Override public Map count(String task) { MappingConfig config = hbaseMapping.get(task); - String hbaseTable = config.getHbaseOrm().getHbaseTable(); + String hbaseTable = config.getHbaseMapping().getHbaseTable(); long rowCount = 0L; try { HTable table = (HTable) conn.getTable(TableName.valueOf(hbaseTable)); @@ -169,8 +169,8 @@ public class HbaseAdapter implements OuterAdapter { @Override public String getDestination(String task) { MappingConfig config = hbaseMapping.get(task); - if (config != null && config.getHbaseOrm() != null) { - return config.getHbaseOrm().getDestination(); + if (config != null && config.getHbaseMapping() != null) { + return config.getHbaseMapping().getDestination(); } return null; } diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java index 93968610..3ff1bb85 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java @@ -10,9 +10,9 @@ import java.util.*; */ public class MappingConfig { - private String dataSourceKey; // 数据源key + private String dataSourceKey; // 数据源key - private HbaseOrm hbaseOrm; // hbase映射配置 + private HbaseMapping hbaseMapping; // hbase映射配置 public String getDataSourceKey() { return dataSourceKey; @@ -22,28 +22,28 @@ public class MappingConfig { this.dataSourceKey = dataSourceKey; } - public HbaseOrm getHbaseOrm() { - return hbaseOrm; + public HbaseMapping getHbaseMapping() { + return hbaseMapping; } - public void setHbaseOrm(HbaseOrm hbaseOrm) { - this.hbaseOrm = hbaseOrm; + public void setHbaseMapping(HbaseMapping hbaseMapping) { + this.hbaseMapping = hbaseMapping; } public void validate() { - if (hbaseOrm.database == null || hbaseOrm.database.isEmpty()) { - throw new NullPointerException("hbaseOrm.database"); + if (hbaseMapping.database == null || hbaseMapping.database.isEmpty()) { + throw new NullPointerException("hbaseMapping.database"); } - if (hbaseOrm.table == null || hbaseOrm.table.isEmpty()) { - throw new NullPointerException("hbaseOrm.table"); + if (hbaseMapping.table == null || hbaseMapping.table.isEmpty()) { + throw new NullPointerException("hbaseMapping.table"); } - if (hbaseOrm.hbaseTable == null || hbaseOrm.hbaseTable.isEmpty()) { - throw new NullPointerException("hbaseOrm.hbaseTable"); + if (hbaseMapping.hbaseTable == null || hbaseMapping.hbaseTable.isEmpty()) { + throw new NullPointerException("hbaseMapping.hbaseTable"); } - if (hbaseOrm.mode == null) { - throw new NullPointerException("hbaseOrm.mode"); + if (hbaseMapping.mode == null) { + throw new NullPointerException("hbaseMapping.mode"); } - if (hbaseOrm.rowKey != null && hbaseOrm.rowKeyColumn != null) { + if (hbaseMapping.rowKey != null && hbaseMapping.rowKeyColumn != null) { throw new RuntimeException("已配置了复合主键作为RowKey，无需再指定RowKey列"); } } @@ -55,12 +55,12 @@ public class MappingConfig { MappingConfig config = (MappingConfig) o; - return hbaseOrm != null ? hbaseOrm.equals(config.hbaseOrm) : config.hbaseOrm == null; + return hbaseMapping != null ? hbaseMapping.equals(config.hbaseMapping) : config.hbaseMapping == null; } @Override public int hashCode() { - return hbaseOrm != null ? hbaseOrm.hashCode() : 0; + return hbaseMapping != null ? hbaseMapping.hashCode() : 0; } public static class ColumnItem { @@ -140,7 +140,7 @@ public class MappingConfig { } } - public static class HbaseOrm { + public static class HbaseMapping { private Mode mode = Mode.STRING; // hbase默认转换格式 private String destination; // canal实例或MQ的topic @@ -349,10 +349,10 @@ public class MappingConfig { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - HbaseOrm hbaseOrm = (HbaseOrm) o; + HbaseMapping hbaseMapping = (HbaseMapping) o; - if (table != null ? !table.equals(hbaseOrm.table) : hbaseOrm.table != null) return false; - return hbaseTable != null ? hbaseTable.equals(hbaseOrm.hbaseTable) : hbaseOrm.hbaseTable == null; + if (table != null ? !table.equals(hbaseMapping.table) : hbaseMapping.table != null) return false; + return hbaseTable != null ? hbaseTable.equals(hbaseMapping.hbaseTable) : hbaseMapping.hbaseTable == null; } @Override diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java index 83ba56df..c0e9496a 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java @@ -80,21 +80,21 @@ public class MappingConfigLoader { if (dbTable.length == 2) { config = new MappingConfig(); - MappingConfig.HbaseOrm hbaseOrm = new MappingConfig.HbaseOrm(); - hbaseOrm.setHbaseTable(dbTable[0].toUpperCase() + "." + dbTable[1].toUpperCase()); - hbaseOrm.setAutoCreateTable(true); - hbaseOrm.setDatabase(dbTable[0]); - hbaseOrm.setTable(dbTable[1]); - hbaseOrm.setMode(MappingConfig.Mode.PHOENIX); - hbaseOrm.setRowKey(rowKey); + MappingConfig.HbaseMapping hbaseMapping = new MappingConfig.HbaseMapping(); + hbaseMapping.setHbaseTable(dbTable[0].toUpperCase() + "." + dbTable[1].toUpperCase()); + hbaseMapping.setAutoCreateTable(true); + hbaseMapping.setDatabase(dbTable[0]); + hbaseMapping.setTable(dbTable[1]); + hbaseMapping.setMode(MappingConfig.Mode.PHOENIX); + hbaseMapping.setRowKey(rowKey); // 有定义rowKey if (rowKey != null) { MappingConfig.ColumnItem columnItem = new MappingConfig.ColumnItem(); columnItem.setRowKey(true); columnItem.setColumn(rowKey); - hbaseOrm.setRowKeyColumn(columnItem); + hbaseMapping.setRowKeyColumn(columnItem); } - config.setHbaseOrm(hbaseOrm); + config.setHbaseMapping(hbaseMapping); config.setDataSourceKey(dsKey); } else { diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java index 7e3c817a..7c57a9ab 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java @@ -81,9 +81,9 @@ public class HbaseEtlService { public static void createTable(HbaseTemplate hbaseTemplate, MappingConfig config) { try { // 判断hbase表是否存在，不存在则建表 - MappingConfig.HbaseOrm hbaseOrm = config.getHbaseOrm(); - if (!hbaseTemplate.tableExists(hbaseOrm.getHbaseTable())) { - hbaseTemplate.createTable(hbaseOrm.getHbaseTable(), hbaseOrm.getFamily()); + MappingConfig.HbaseMapping hbaseMapping = config.getHbaseMapping(); + if (!hbaseTemplate.tableExists(hbaseMapping.getHbaseTable())) { + hbaseTemplate.createTable(hbaseMapping.getHbaseTable(), hbaseMapping.getFamily()); } } catch (Exception e) { logger.error(e.getMessage(), e); @@ -113,29 +113,29 @@ public class HbaseEtlService { etlResult.setErrorMessage("Config is null!"); return etlResult; } - MappingConfig.HbaseOrm hbaseOrm = config.getHbaseOrm(); - hbaseTable = hbaseOrm.getHbaseTable(); + MappingConfig.HbaseMapping hbaseMapping = config.getHbaseMapping(); + hbaseTable = hbaseMapping.getHbaseTable(); long start = System.currentTimeMillis(); if (params != null && params.size() == 1 && "rebuild".equalsIgnoreCase(params.get(0))) { - logger.info(hbaseOrm.getHbaseTable() + " rebuild is starting!"); + logger.info(hbaseMapping.getHbaseTable() + " rebuild is starting!"); // 如果表存在则删除 - if (hbaseTemplate.tableExists(hbaseOrm.getHbaseTable())) { - hbaseTemplate.disableTable(hbaseOrm.getHbaseTable()); - hbaseTemplate.deleteTable(hbaseOrm.getHbaseTable()); + if (hbaseTemplate.tableExists(hbaseMapping.getHbaseTable())) { + hbaseTemplate.disableTable(hbaseMapping.getHbaseTable()); + hbaseTemplate.deleteTable(hbaseMapping.getHbaseTable()); } params = null; } else { - logger.info(hbaseOrm.getHbaseTable() + " etl is starting!"); + logger.info(hbaseMapping.getHbaseTable() + " etl is starting!"); } createTable(hbaseTemplate, config); // 拼接sql - String sql = "SELECT * FROM " + config.getHbaseOrm().getDatabase() + "." + hbaseOrm.getTable(); + String sql = "SELECT * FROM " + config.getHbaseMapping().getDatabase() + "." + hbaseMapping.getTable(); // 拼接条件 - if (params != null && params.size() == 1 && hbaseOrm.getEtlCondition() == null) { + if (params != null && params.size() == 1 && hbaseMapping.getEtlCondition() == null) { AtomicBoolean stExists = new AtomicBoolean(false); // 验证是否有SYS_TIME字段 sqlRS(ds, sql, rs -> { @@ -157,8 +157,8 @@ public class HbaseEtlService { if (stExists.get()) { sql += " WHERE SYS_TIME >= '" + params.get(0) + "' "; } - } else if (hbaseOrm.getEtlCondition() != null && params != null) { - String etlCondition = hbaseOrm.getEtlCondition(); + } else if (hbaseMapping.getEtlCondition() != null && params != null) { + String etlCondition = hbaseMapping.getEtlCondition(); int size = params.size(); for (int i = 0; i < size; i++) { etlCondition = etlCondition.replace("{" + i + "}", params.get(i)); @@ -200,7 +200,7 @@ public class HbaseEtlService { sqlFinal = sql + " LIMIT " + offset + "," + cnt; } Future future = executor - .submit(() -> executeSqlImport(ds, sqlFinal, hbaseOrm, hbaseTemplate, successCount, errMsg)); + .submit(() -> executeSqlImport(ds, sqlFinal, hbaseMapping, hbaseTemplate, successCount, errMsg)); futures.add(future); } @@ -210,13 +210,13 @@ public class HbaseEtlService { executor.shutdown(); } else { - executeSqlImport(ds, sql, hbaseOrm, hbaseTemplate, successCount, errMsg); + executeSqlImport(ds, sql, hbaseMapping, hbaseTemplate, successCount, errMsg); } logger.info( - hbaseOrm.getHbaseTable() + " etl completed in: " + (System.currentTimeMillis() - start) / 1000 + "s!"); + hbaseMapping.getHbaseTable() + " etl completed in: " + (System.currentTimeMillis() - start) / 1000 + "s!"); - etlResult.setResultMessage("导入HBase表 " + hbaseOrm.getHbaseTable() + " 数据：" + successCount.get() + " 条"); + etlResult.setResultMessage("导入HBase表 " + hbaseMapping.getHbaseTable() + " 数据：" + successCount.get() + " 条"); } catch (Exception e) { logger.error(e.getMessage(), e); errMsg.add(hbaseTable + " etl failed! ==>" + e.getMessage()); @@ -235,13 +235,13 @@ public class HbaseEtlService { * * @param ds * @param sql - * @param hbaseOrm + * @param hbaseMapping * @param hbaseTemplate * @param successCount * @param errMsg * @return */ - private static boolean executeSqlImport(DataSource ds, String sql, MappingConfig.HbaseOrm hbaseOrm, + private static boolean executeSqlImport(DataSource ds, String sql, MappingConfig.HbaseMapping hbaseMapping, HbaseTemplate hbaseTemplate, AtomicLong successCount, List errMsg) { try { sqlRS(ds, sql, rs -> { @@ -251,8 +251,8 @@ public class HbaseEtlService { boolean complete = false; List rows = new ArrayList<>(); String[] rowKeyColumns = null; - if (hbaseOrm.getRowKey() != null) { - rowKeyColumns = hbaseOrm.getRowKey().trim().split(","); + if (hbaseMapping.getRowKey() != null) { + rowKeyColumns = hbaseMapping.getRowKey().trim().split(","); } while (rs.next()) { int cc = rs.getMetaData().getColumnCount(); @@ -290,30 +290,30 @@ public class HbaseEtlService { continue; } - MappingConfig.ColumnItem columnItem = hbaseOrm.getColumnItems().get(columnName); + MappingConfig.ColumnItem columnItem = hbaseMapping.getColumnItems().get(columnName); // 没有配置映射 if (columnItem == null) { - String family = hbaseOrm.getFamily(); + String family = hbaseMapping.getFamily(); String qualifile = columnName; - if (hbaseOrm.isUppercaseQualifier()) { + if (hbaseMapping.isUppercaseQualifier()) { qualifile = qualifile.toUpperCase(); } - if (MappingConfig.Mode.STRING == hbaseOrm.getMode()) { - if (hbaseOrm.getRowKey() == null && j == 1) { + if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) { + if (hbaseMapping.getRowKey() == null && j == 1) { row.setRowKey(Bytes.toBytes(val.toString())); } else { row.addCell(family, qualifile, Bytes.toBytes(val.toString())); } - } else if (MappingConfig.Mode.NATIVE == hbaseOrm.getMode()) { + } else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) { Type type = Type.getType(classes[j - 1]); - if (hbaseOrm.getRowKey() == null && j == 1) { + if (hbaseMapping.getRowKey() == null && j == 1) { row.setRowKey(TypeUtil.toBytes(val, type)); } else { row.addCell(family, qualifile, TypeUtil.toBytes(val, type)); } - } else if (MappingConfig.Mode.PHOENIX == hbaseOrm.getMode()) { + } else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) { PhType phType = PhType.getType(classes[j - 1]); - if (hbaseOrm.getRowKey() == null && j == 1) { + if (hbaseMapping.getRowKey() == null && j == 1) { row.setRowKey(PhTypeUtil.toBytes(val, phType)); } else { row.addCell(family, qualifile, PhTypeUtil.toBytes(val, phType)); @@ -353,8 +353,8 @@ public class HbaseEtlService { rows.add(row); complete = false; - if (i % hbaseOrm.getCommitBatch() == 0 && !rows.isEmpty()) { - hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows); + if (i % hbaseMapping.getCommitBatch() == 0 && !rows.isEmpty()) { + hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows); rows.clear(); complete = true; } @@ -366,12 +366,12 @@ public class HbaseEtlService { } if (!complete && !rows.isEmpty()) { - hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows); + hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows); } } catch (Exception e) { - logger.error(hbaseOrm.getHbaseTable() + " etl failed! ==>" + e.getMessage(), e); - errMsg.add(hbaseOrm.getHbaseTable() + " etl failed! ==>" + e.getMessage()); + logger.error(hbaseMapping.getHbaseTable() + " etl failed! ==>" + e.getMessage(), e); + errMsg.add(hbaseMapping.getHbaseTable() + " etl failed! ==>" + e.getMessage()); // throw new RuntimeException(e); } return i; diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java index 4f31d0d3..3dd03b91 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java @@ -59,11 +59,11 @@ public class HbaseSyncService { return; } - MappingConfig.HbaseOrm hbaseOrm = config.getHbaseOrm(); + MappingConfig.HbaseMapping hbaseMapping = config.getHbaseMapping(); // if (!validHTable(config)) { // logger.error("HBase table '{}' not exists", - // hbaseOrm.getHbaseTable()); + // hbaseMapping.getHbaseTable()); // return; // } int i = 1; @@ -73,28 +73,28 @@ public class HbaseSyncService { HRow hRow = new HRow(); // 拼接复合rowKey - if (hbaseOrm.getRowKey() != null) { - String[] rowKeyColumns = hbaseOrm.getRowKey().trim().split(","); + if (hbaseMapping.getRowKey() != null) { + String[] rowKeyColumns = hbaseMapping.getRowKey().trim().split(","); String rowKeyVale = getRowKeys(rowKeyColumns, r); // params.put("rowKey", Bytes.toBytes(rowKeyVale)); hRow.setRowKey(Bytes.toBytes(rowKeyVale)); } - convertData2Row(hbaseOrm, hRow, r); + convertData2Row(hbaseMapping, hRow, r); if (hRow.getRowKey() == null) { throw new RuntimeException("empty rowKey"); } rows.add(hRow); complete = false; - if (i % config.getHbaseOrm().getCommitBatch() == 0 && !rows.isEmpty()) { - hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows); + if (i % config.getHbaseMapping().getCommitBatch() == 0 && !rows.isEmpty()) { + hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows); rows.clear(); complete = true; } i++; } if (!complete && !rows.isEmpty()) { - hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows); + hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows); } } @@ -102,30 +102,30 @@ public class HbaseSyncService { /** * 将Map数据转换为HRow行数据 * - * @param hbaseOrm hbase映射配置 + * @param hbaseMapping hbase映射配置 * @param hRow 行对象 * @param data Map数据 */ - private static void convertData2Row(MappingConfig.HbaseOrm hbaseOrm, HRow hRow, Map data) { - Map columnItems = hbaseOrm.getColumnItems(); + private static void convertData2Row(MappingConfig.HbaseMapping hbaseMapping, HRow hRow, Map data) { + Map columnItems = hbaseMapping.getColumnItems(); int i = 0; for (Map.Entry entry : data.entrySet()) { - if (hbaseOrm.getExcludeColumns() != null && hbaseOrm.getExcludeColumns().contains(entry.getKey())) { + if (hbaseMapping.getExcludeColumns() != null && hbaseMapping.getExcludeColumns().contains(entry.getKey())) { continue; } if (entry.getValue() != null) { MappingConfig.ColumnItem columnItem = columnItems.get(entry.getKey()); - byte[] bytes = typeConvert(columnItem, hbaseOrm, entry.getValue()); + byte[] bytes = typeConvert(columnItem, hbaseMapping, entry.getValue()); if (columnItem == null) { - String familyName = hbaseOrm.getFamily(); + String familyName = hbaseMapping.getFamily(); String qualifier = entry.getKey(); - if (hbaseOrm.isUppercaseQualifier()) { + if (hbaseMapping.isUppercaseQualifier()) { qualifier = qualifier.toUpperCase(); } - if (hbaseOrm.getRowKey() == null && i == 0) { + if (hbaseMapping.getRowKey() == null && i == 0) { hRow.setRowKey(bytes); } else { hRow.addCell(familyName, qualifier, bytes); @@ -156,15 +156,15 @@ public class HbaseSyncService { return; } - MappingConfig.HbaseOrm hbaseOrm = config.getHbaseOrm(); + MappingConfig.HbaseMapping hbaseMapping = config.getHbaseMapping(); // if (!validHTable(config)) { // logger.error("HBase table '{}' not exists", - // hbaseOrm.getHbaseTable()); + // hbaseMapping.getHbaseTable()); // return; // } - MappingConfig.ColumnItem rowKeyColumn = hbaseOrm.getRowKeyColumn(); + MappingConfig.ColumnItem rowKeyColumn = hbaseMapping.getRowKeyColumn(); int index = 0; int i = 1; boolean complete = false; @@ -172,8 +172,8 @@ public class HbaseSyncService { out: for (Map r : data) { byte[] rowKeyBytes; - if (hbaseOrm.getRowKey() != null) { - String[] rowKeyColumns = hbaseOrm.getRowKey().trim().split(","); + if (hbaseMapping.getRowKey() != null) { + String[] rowKeyColumns = hbaseMapping.getRowKey().trim().split(","); // 判断是否有复合主键修改 for (String updateColumn : old.get(index).keySet()) { @@ -190,23 +190,23 @@ public class HbaseSyncService { rowKeyBytes = Bytes.toBytes(rowKeyVale); } else if (rowKeyColumn == null) { Map rowKey = data.get(0); - rowKeyBytes = typeConvert(null, hbaseOrm, rowKey.values().iterator().next()); + rowKeyBytes = typeConvert(null, hbaseMapping, rowKey.values().iterator().next()); } else { - rowKeyBytes = typeConvert(rowKeyColumn, hbaseOrm, r.get(rowKeyColumn.getColumn())); + rowKeyBytes = typeConvert(rowKeyColumn, hbaseMapping, r.get(rowKeyColumn.getColumn())); } if (rowKeyBytes == null) throw new RuntimeException("rowKey值为空"); - Map columnItems = hbaseOrm.getColumnItems(); + Map columnItems = hbaseMapping.getColumnItems(); HRow hRow = new HRow(rowKeyBytes); for (String updateColumn : old.get(index).keySet()) { - if (hbaseOrm.getExcludeColumns() != null && hbaseOrm.getExcludeColumns().contains(updateColumn)) { + if (hbaseMapping.getExcludeColumns() != null && hbaseMapping.getExcludeColumns().contains(updateColumn)) { continue; } MappingConfig.ColumnItem columnItem = columnItems.get(updateColumn); if (columnItem == null) { - String family = hbaseOrm.getFamily(); + String family = hbaseMapping.getFamily(); String qualifier = updateColumn; - if (hbaseOrm.isUppercaseQualifier()) { + if (hbaseMapping.isUppercaseQualifier()) { qualifier = qualifier.toUpperCase(); } @@ -215,7 +215,7 @@ public class HbaseSyncService { if (newVal == null) { hRow.addCell(family, qualifier, null); } else { - hRow.addCell(family, qualifier, typeConvert(null, hbaseOrm, newVal)); + hRow.addCell(family, qualifier, typeConvert(null, hbaseMapping, newVal)); } } else { // 排除修改id的情况 @@ -227,14 +227,14 @@ public class HbaseSyncService { } else { hRow.addCell(columnItem.getFamily(), columnItem.getQualifier(), - typeConvert(columnItem, hbaseOrm, newVal)); + typeConvert(columnItem, hbaseMapping, newVal)); } } } rows.add(hRow); complete = false; - if (i % config.getHbaseOrm().getCommitBatch() == 0 && !rows.isEmpty()) { - hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows); + if (i % config.getHbaseMapping().getCommitBatch() == 0 && !rows.isEmpty()) { + hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows); rows.clear(); complete = true; } @@ -242,7 +242,7 @@ public class HbaseSyncService { index++; } if (!complete && !rows.isEmpty()) { - hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows); + hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows); } } @@ -252,45 +252,45 @@ public class HbaseSyncService { return; } - MappingConfig.HbaseOrm hbaseOrm = config.getHbaseOrm(); + MappingConfig.HbaseMapping hbaseMapping = config.getHbaseMapping(); // if (!validHTable(config)) { // logger.error("HBase table '{}' not exists", - // hbaseOrm.getHbaseTable()); + // hbaseMapping.getHbaseTable()); // return; // } - MappingConfig.ColumnItem rowKeyColumn = hbaseOrm.getRowKeyColumn(); + MappingConfig.ColumnItem rowKeyColumn = hbaseMapping.getRowKeyColumn(); boolean complete = false; int i = 1; Set rowKeys = new HashSet<>(); for (Map r : data) { byte[] rowKeyBytes; - if (hbaseOrm.getRowKey() != null) { - String[] rowKeyColumns = hbaseOrm.getRowKey().trim().split(","); + if (hbaseMapping.getRowKey() != null) { + String[] rowKeyColumns = hbaseMapping.getRowKey().trim().split(","); String rowKeyVale = getRowKeys(rowKeyColumns, r); rowKeyBytes = Bytes.toBytes(rowKeyVale); } else if (rowKeyColumn == null) { // 如果不需要类型转换 Map rowKey = data.get(0); - rowKeyBytes = typeConvert(null, hbaseOrm, rowKey.values().iterator().next()); + rowKeyBytes = typeConvert(null, hbaseMapping, rowKey.values().iterator().next()); } else { Object val = r.get(rowKeyColumn.getColumn()); - rowKeyBytes = typeConvert(rowKeyColumn, hbaseOrm, val); + rowKeyBytes = typeConvert(rowKeyColumn, hbaseMapping, val); } if (rowKeyBytes == null) throw new RuntimeException("rowKey值为空"); rowKeys.add(rowKeyBytes); complete = false; - if (i % config.getHbaseOrm().getCommitBatch() == 0 && !rowKeys.isEmpty()) { - hbaseTemplate.deletes(hbaseOrm.getHbaseTable(), rowKeys); + if (i % config.getHbaseMapping().getCommitBatch() == 0 && !rowKeys.isEmpty()) { + hbaseTemplate.deletes(hbaseMapping.getHbaseTable(), rowKeys); rowKeys.clear(); complete = true; } i++; } if (!complete && !rowKeys.isEmpty()) { - hbaseTemplate.deletes(hbaseOrm.getHbaseTable(), rowKeys); + hbaseTemplate.deletes(hbaseMapping.getHbaseTable(), rowKeys); } } @@ -300,9 +300,9 @@ public class HbaseSyncService { if (old == null || old.isEmpty() || data == null || data.isEmpty()) { return; } - MappingConfig.HbaseOrm hbaseOrm = config.getHbaseOrm(); + MappingConfig.HbaseMapping hbaseMapping = config.getHbaseMapping(); - String[] rowKeyColumns = hbaseOrm.getRowKey().trim().split(","); + String[] rowKeyColumns = hbaseMapping.getRowKey().trim().split(","); int index = 0; int i = 1; @@ -343,13 +343,13 @@ public class HbaseSyncService { rowKeys.add(oldRowKeyBytes); HRow row = new HRow(newRowKeyBytes); - convertData2Row(hbaseOrm, row, r); + convertData2Row(hbaseMapping, row, r); rows.add(row); complete = false; - if (i % config.getHbaseOrm().getCommitBatch() == 0 && !rows.isEmpty()) { - hbaseTemplate.deletes(hbaseOrm.getHbaseTable(), rowKeys); + if (i % config.getHbaseMapping().getCommitBatch() == 0 && !rows.isEmpty()) { + hbaseTemplate.deletes(hbaseMapping.getHbaseTable(), rowKeys); - hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows); + hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows); rowKeys.clear(); rows.clear(); complete = true; @@ -358,8 +358,8 @@ public class HbaseSyncService { index++; } if (!complete && !rows.isEmpty()) { - hbaseTemplate.deletes(hbaseOrm.getHbaseTable(), rowKeys); - hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows); + hbaseTemplate.deletes(hbaseMapping.getHbaseTable(), rowKeys); + hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows); } } @@ -367,32 +367,32 @@ public class HbaseSyncService { * 根据对应的类型进行转换 * * @param columnItem 列项配置 - * @param hbaseOrm hbase映射配置 + * @param hbaseMapping hbase映射配置 * @param value 值 * @return 复合字段rowKey */ - private static byte[] typeConvert(MappingConfig.ColumnItem columnItem, MappingConfig.HbaseOrm hbaseOrm, + private static byte[] typeConvert(MappingConfig.ColumnItem columnItem, MappingConfig.HbaseMapping hbaseMapping, Object value) { if (value == null) { return null; } byte[] bytes = null; if (columnItem == null || columnItem.getType() == null || "".equals(columnItem.getType())) { - if (MappingConfig.Mode.STRING == hbaseOrm.getMode()) { + if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) { bytes = Bytes.toBytes(value.toString()); - } else if (MappingConfig.Mode.NATIVE == hbaseOrm.getMode()) { + } else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) { bytes = TypeUtil.toBytes(value); - } else if (MappingConfig.Mode.PHOENIX == hbaseOrm.getMode()) { + } else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) { PhType phType = PhType.getType(value.getClass()); bytes = PhTypeUtil.toBytes(value, phType); } } else { - if (hbaseOrm.getMode() == MappingConfig.Mode.STRING) { + if (hbaseMapping.getMode() == MappingConfig.Mode.STRING) { bytes = Bytes.toBytes(value.toString()); - } else if (hbaseOrm.getMode() == MappingConfig.Mode.NATIVE) { + } else if (hbaseMapping.getMode() == MappingConfig.Mode.NATIVE) { Type type = Type.getType(columnItem.getType()); bytes = TypeUtil.toBytes(value, type); - } else if (hbaseOrm.getMode() == MappingConfig.Mode.PHOENIX) { + } else if (hbaseMapping.getMode() == MappingConfig.Mode.PHOENIX) { PhType phType = PhType.getType(columnItem.getType()); bytes = PhTypeUtil.toBytes(value, phType); } diff --git a/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml b/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml index d80b95eb..f31341a3 100644 --- a/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml +++ b/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml @@ -1,5 +1,5 @@ dataSourceKey: defaultDS -hbaseOrm: +hbaseMapping: mode: PHOENIX #NATIVE #STRING destination: example database: mytest # 数据库名 @@ -18,7 +18,7 @@ hbaseOrm: c_time: C_TIME$UNSIGNED_TIMESTAMP birthday: BIRTHDAY$DATE excludeColumns: - - lat + - lat # 忽略字段 # -- NATIVE类型 # $DEFAULT diff --git a/client-adapter/launcher/src/main/resources/application.yml b/client-adapter/launcher/src/main/resources/application.yml index 1b1139c1..19fa6182 100644 --- a/client-adapter/launcher/src/main/resources/application.yml +++ b/client-adapter/launcher/src/main/resources/application.yml @@ -10,9 +10,9 @@ spring: default-property-inclusion: non_null -hbase.zookeeper.quorum: slave1 +hbasezookeeper.quorum: 127.0.0.1 hbase.zookeeper.property.clientPort: 2181 -hbase.zookeeper.znode.parent: /hbase-unsecure +hbase.zookeeper.znode.parent: /hbase canal.conf: canalServerHost: 127.0.0.1:11111 @@ -23,12 +23,12 @@ canal.conf: - instance: example adapterGroups: - outAdapters: -# - name: logger - - name: hbase - properties: - hbase.zookeeper.quorum: ${hbase.zookeeper.quorum} - hbase.zookeeper.property.clientPort: ${hbase.zookeeper.property.clientPort} - zookeeper.znode.parent: ${hbase.zookeeper.znode.parent} + - name: logger +# - name: hbase +# properties: +# hbase.zookeeper.quorum: ${hbase.zookeeper.quorum} +# hbase.zookeeper.property.clientPort: ${hbase.zookeeper.property.clientPort} +# zookeeper.znode.parent: ${hbase.zookeeper.znode.parent} # mqTopics: # - mqMode: kafka # topic: example @@ -37,11 +37,11 @@ canal.conf: # outAdapters: # - name: logger -adapter.conf: - datasourceConfigs: - defaultDS: - url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true - username: root - password: 121212 - adapterConfigs: - - hbase/mytest_person2.yml +#adapter.conf: +# datasourceConfigs: +# defaultDS: +# url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true +# username: root +# password: 121212 +# adapterConfigs: +# - hbase/mytest_person2.yml