支持不同topic指定专属partitionNum (#2479)

* MemoryTableMeta ddl 处理注释

* MemoryTableMeta ddl 处理注释2

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/config/EbeanConfig.java

@@ -1,5 +1,10 @@
 package com.alibaba.otter.canal.admin.config;
 
+import io.ebean.EbeanServer;
+import io.ebean.EbeanServerFactory;
+import io.ebean.config.ServerConfig;
+import io.ebean.config.UnderscoreNamingConvention;
+
 import java.util.ArrayList;
 import java.util.List;
 
@@ -8,11 +13,6 @@ import javax.sql.DataSource;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
-import io.ebean.EbeanServer;
-import io.ebean.EbeanServerFactory;
-import io.ebean.config.ServerConfig;
-import io.ebean.config.UnderscoreNamingConvention;
-
 /**
  * Ebean 配置
  *

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/connector/SimpleAdminConnectors.java

@@ -2,11 +2,12 @@ package com.alibaba.otter.canal.admin.connector;
 
 import java.util.function.Function;
 
-import com.alibaba.otter.canal.admin.config.SpringContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.core.env.Environment;
 
+import com.alibaba.otter.canal.admin.config.SpringContext;
+
 public class SimpleAdminConnectors {
 
     private static final Logger logger = LoggerFactory.getLogger(SimpleAdminConnectors.class);
@@ -25,7 +26,7 @@ public class SimpleAdminConnectors {
             connector.connect();
             return function.apply(connector);
         } catch (Exception e) {
-            logger.error("connect to ip:{},port:{},user:{},password:{}, failed", ip, port, user,passwd);
+            logger.error("connect to ip:{},port:{},user:{},password:{}, failed", ip, port, user, passwd);
             logger.error(e.getMessage());
         } finally {
             connector.disconnect();

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/controller/CanalClusterController.java

@@ -4,7 +4,14 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.web.bind.annotation.*;
+import org.springframework.web.bind.annotation.DeleteMapping;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.PutMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
 
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
@@ -22,7 +29,7 @@ public class CanalClusterController {
     CanalClusterService canalClusterServic;
 
     @Autowired
-    NodeServerService  nodeServerService;
+    NodeServerService   nodeServerService;
 
     @GetMapping(value = "/clusters")
     public BaseModel<List<CanalCluster>> clusters(CanalCluster canalCluster, @PathVariable String env) {

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/controller/CanalConfigController.java

@@ -1,6 +1,5 @@
 package com.alibaba.otter.canal.admin.controller;
 
-import com.alibaba.otter.canal.admin.common.TemplateConfigLoader;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PathVariable;
@@ -9,6 +8,7 @@ import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
 
+import com.alibaba.otter.canal.admin.common.TemplateConfigLoader;
 import com.alibaba.otter.canal.admin.model.BaseModel;
 import com.alibaba.otter.canal.admin.model.CanalConfig;
 import com.alibaba.otter.canal.admin.service.CanalConfigService;

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/controller/CanalInstanceController.java

@@ -3,7 +3,6 @@ package com.alibaba.otter.canal.admin.controller;
 import java.util.List;
 import java.util.Map;
 
-import com.alibaba.otter.canal.admin.common.TemplateConfigLoader;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.DeleteMapping;
 import org.springframework.web.bind.annotation.GetMapping;
@@ -15,6 +14,7 @@ import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
+import com.alibaba.otter.canal.admin.common.TemplateConfigLoader;
 import com.alibaba.otter.canal.admin.model.BaseModel;
 import com.alibaba.otter.canal.admin.model.CanalInstanceConfig;
 import com.alibaba.otter.canal.admin.model.Pager;
@@ -143,8 +143,7 @@ public class CanalInstanceController {
      * @return 是否成功
      */
     @PutMapping(value = "/instance/status/{id}")
-    public BaseModel<Boolean> instanceStart(@PathVariable Long id, @RequestParam String option,
-                                            @PathVariable String env) {
+    public BaseModel<Boolean> instanceStart(@PathVariable Long id, @RequestParam String option, @PathVariable String env) {
         return BaseModel.getInstance(canalInstanceConfigService.instanceOperation(id, option));
     }
 

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/controller/UserController.java

@@ -1,20 +1,28 @@
 package com.alibaba.otter.canal.admin.controller;
 
-import com.alibaba.otter.canal.admin.service.UserService;
-import com.github.benmanes.caffeine.cache.Caffeine;
-import com.github.benmanes.caffeine.cache.LoadingCache;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.web.bind.annotation.*;
-
-import com.alibaba.otter.canal.admin.model.BaseModel;
-import com.alibaba.otter.canal.admin.model.User;
-
-import javax.servlet.http.HttpServletRequest;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 
+import javax.servlet.http.HttpServletRequest;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.PutMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import com.alibaba.otter.canal.admin.model.BaseModel;
+import com.alibaba.otter.canal.admin.model.User;
+import com.alibaba.otter.canal.admin.service.UserService;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+
 /**
  * 用户管理控制层
  *
@@ -26,9 +34,9 @@ import java.util.concurrent.TimeUnit;
 public class UserController {
 
     public static final LoadingCache<String, User> loginUsers = Caffeine.newBuilder()
-        .maximumSize(10_000)
-        .expireAfterAccess(30, TimeUnit.MINUTES)
-        .build(key -> null);                                                         // 用户登录信息缓存
+                                                                  .maximumSize(10_000)
+                                                                  .expireAfterAccess(30, TimeUnit.MINUTES)
+                                                                  .build(key -> null); // 用户登录信息缓存
 
     @Autowired
     UserService                                    userService;

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/handler/CustomExceptionHandler.java

@@ -1,7 +1,5 @@
 package com.alibaba.otter.canal.admin.handler;
 
-import com.alibaba.otter.canal.admin.common.exception.ServiceException;
-import com.alibaba.otter.canal.admin.model.BaseModel;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.http.HttpStatus;
@@ -10,6 +8,9 @@ import org.springframework.web.bind.annotation.ExceptionHandler;
 import org.springframework.web.bind.annotation.ResponseBody;
 import org.springframework.web.bind.annotation.ResponseStatus;
 
+import com.alibaba.otter.canal.admin.common.exception.ServiceException;
+import com.alibaba.otter.canal.admin.model.BaseModel;
+
 /**
  * MVC异常拦截器
  *

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/model/CanalCluster.java

@@ -3,10 +3,11 @@ package com.alibaba.otter.canal.admin.model;
 import io.ebean.Finder;
 import io.ebean.annotation.WhenModified;
 
+import java.util.Date;
+
 import javax.persistence.Entity;
 import javax.persistence.Id;
 import javax.persistence.Table;
-import java.util.Date;
 
 /**
  * Canal集群信息实体类

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/model/CanalInstanceConfig.java

@@ -1,12 +1,18 @@
 package com.alibaba.otter.canal.admin.model;
 
-import java.util.Date;
-
-import javax.persistence.*;
-
 import io.ebean.Finder;
 import io.ebean.annotation.WhenModified;
 
+import java.util.Date;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.FetchType;
+import javax.persistence.Id;
+import javax.persistence.JoinColumn;
+import javax.persistence.ManyToOne;
+import javax.persistence.Transient;
+
 /**
  * Canal实例配置信息实体类
  *
@@ -44,14 +50,14 @@ public class CanalInstanceConfig extends Model {
     private String       name;
     private String       content;
     private String       contentMd5;
-    private String       status;         // 1: 正常 0: 停止
+    private String       status;             // 1: 正常 0: 停止
     @WhenModified
     private Date         modifiedTime;
 
     @Transient
     private String       clusterServerId;
     @Transient
-    private String       runningStatus = "0";  // 1: 运行中 0: 停止
+    private String       runningStatus = "0"; // 1: 运行中 0: 停止
 
     public void init() {
         status = "1";

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/model/Model.java

@@ -1,13 +1,13 @@
 package com.alibaba.otter.canal.admin.model;
 
+import io.ebean.Ebean;
+import io.ebean.EbeanServer;
+
 import javax.persistence.MappedSuperclass;
 import javax.persistence.OptimisticLockException;
 
 import org.apache.commons.beanutils.PropertyUtils;
 
-import io.ebean.Ebean;
-import io.ebean.EbeanServer;
-
 /**
  * EBean Model扩展类
  *

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/model/NodeServer.java

@@ -5,7 +5,13 @@ import io.ebean.annotation.WhenModified;
 
 import java.util.Date;
 
-import javax.persistence.*;
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.FetchType;
+import javax.persistence.Id;
+import javax.persistence.JoinColumn;
+import javax.persistence.ManyToOne;
+import javax.persistence.Table;
 
 /**
  * 节点信息实体类

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/model/User.java

@@ -3,11 +3,12 @@ package com.alibaba.otter.canal.admin.model;
 import io.ebean.Finder;
 import io.ebean.annotation.WhenCreated;
 
+import java.util.Date;
+
 import javax.persistence.Entity;
 import javax.persistence.Id;
 import javax.persistence.Table;
 import javax.persistence.Transient;
-import java.util.Date;
 
 /**
  * 用户信息实体类

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/service/NodeServerService.java

@@ -1,10 +1,10 @@
 package com.alibaba.otter.canal.admin.service;
 
+import java.util.List;
+
 import com.alibaba.otter.canal.admin.model.NodeServer;
 import com.alibaba.otter.canal.admin.model.Pager;
 
-import java.util.List;
-
 public interface NodeServerService {
 
     void save(NodeServer nodeServer);

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/service/impl/CanalInstanceServiceImpl.java

@@ -109,12 +109,12 @@ public class CanalInstanceServiceImpl implements CanalInstanceService {
             }));
         }
 
-        for (Future<Void> f :futures){
+        for (Future<Void> f : futures) {
             try {
                 f.get(3, TimeUnit.SECONDS);
             } catch (InterruptedException | ExecutionException e) {
                 // ignore
-            } catch (TimeoutException e){
+            } catch (TimeoutException e) {
                 break;
             }
         }

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/service/impl/NodeServerServiceImpl.java

@@ -154,12 +154,12 @@ public class NodeServerServiceImpl implements NodeServerService {
                 return !status;
             }));
         }
-        for (Future<Boolean> f :futures){
+        for (Future<Boolean> f : futures) {
             try {
                 f.get(3, TimeUnit.SECONDS);
             } catch (InterruptedException | ExecutionException e) {
                 // ignore
-            } catch (TimeoutException e){
+            } catch (TimeoutException e) {
                 break;
             }
         }

admin/admin-web/src/main/resources/canal-template.properties

@@ -8,11 +8,11 @@ canal.register.ip =
 canal.port = 11111
 canal.metrics.pull.port = 11112
 # canal instance user/passwd
-canal.user = canal
-canal.passwd = E3619321C1A937C46A0D8BD1DAC39F93B27D4458
+# canal.user = canal
+# canal.passwd = E3619321C1A937C46A0D8BD1DAC39F93B27D4458
 
 # canal admin config
-canal.admin.manager = 127.0.0.1:8089
+#canal.admin.manager = 127.0.0.1:8089
 canal.admin.port = 11110
 canal.admin.user = admin
 canal.admin.passwd = 4ACFE3202A5FF5CF467898FC58AAB1D615029441
@@ -21,7 +21,7 @@ canal.zkServers =
 # flush data to zk
 canal.zookeeper.flush.period = 1000
 canal.withoutNetty = false
-# tcp, kafka, RocketMQ
+# tcp, kafka, rocketMQ, rabbitMQ
 canal.serverMode = tcp
 # flush meta cursor/parse position to file
 canal.file.data.dir = ${canal.conf.dir}
@@ -86,14 +86,10 @@ canal.instance.tsdb.snapshot.interval = 24
 # purge snapshot expire , default 360 hour(15 days)
 canal.instance.tsdb.snapshot.expire = 360
 
-# aliyun ak/sk , support rds/mq
-canal.aliyun.accessKey =
-canal.aliyun.secretKey =
-
 #################################################
 ######### 		destinations		#############
 #################################################
-canal.destinations =
+canal.destinations = 
 # conf root dir
 canal.conf.dir = ../conf
 # auto scan instance dir add/remove and start/stop instance
@@ -111,29 +107,56 @@ canal.instance.global.spring.xml = classpath:spring/file-instance.xml
 #canal.instance.global.spring.xml = classpath:spring/default-instance.xml
 
 ##################################################
-######### 		     MQ 		     #############
+######### 	      MQ Properties      #############
 ##################################################
-canal.mq.servers = 127.0.0.1:6667
-canal.mq.retries = 0
-canal.mq.batchSize = 16384
-canal.mq.maxRequestSize = 1048576
-canal.mq.lingerMs = 100
-canal.mq.bufferMemory = 33554432
+# aliyun ak/sk , support rds/mq
+canal.aliyun.accessKey =
+canal.aliyun.secretKey =
+canal.aliyun.uid=
+
+canal.mq.flatMessage = true
 canal.mq.canalBatchSize = 50
 canal.mq.canalGetTimeout = 100
-canal.mq.flatMessage = true
-canal.mq.compressionType = none
-canal.mq.acks = all
-#canal.mq.properties. =
-canal.mq.producerGroup = test
 # Set this value to "cloud", if you want open message trace feature in aliyun.
 canal.mq.accessChannel = local
-# aliyun mq namespace
-#canal.mq.namespace =
+
+canal.mq.database.hash = true
+canal.mq.send.thread.size = 30
+canal.mq.build.thread.size = 8
 
 ##################################################
-#########     Kafka Kerberos Info    #############
+######### 		     Kafka 		     #############
 ##################################################
-canal.mq.kafka.kerberos.enable = false
-canal.mq.kafka.kerberos.krb5FilePath = "../conf/kerberos/krb5.conf"
-canal.mq.kafka.kerberos.jaasFilePath = "../conf/kerberos/jaas.conf"
+kafka.bootstrap.servers = 127.0.0.1:6667
+kafka.acks = all
+kafka.compression.type = none
+kafka.batch.size = 16384
+kafka.linger.ms = 1
+kafka.max.request.size = 1048576
+kafka.buffer.memory = 33554432
+kafka.max.in.flight.requests.per.connection = 1
+kafka.retries = 0
+
+kafka.kerberos.enable = false
+kafka.kerberos.krb5.file = "../conf/kerberos/krb5.conf"
+kafka.kerberos.jaas.file = "../conf/kerberos/jaas.conf"
+
+##################################################
+######### 		    RocketMQ	     #############
+##################################################
+rocketmq.producer.group = test
+rocketmq.enable.message.trace = false
+rocketmq.customized.trace.topic =
+rocketmq.namespace =
+rocketmq.namesrv.addr = 127.0.0.1:9876
+rocketmq.retry.times.when.send.failed = 0
+rocketmq.vip.channel.enabled = false
+
+##################################################
+######### 		    RabbitMQ	     #############
+##################################################
+rabbitmq.host =
+rabbitmq.virtual.host =
+rabbitmq.exchange =
+rabbitmq.username =
+rabbitmq.password =

admin/pom.xml

@@ -69,7 +69,7 @@
             <dependency>
                 <groupId>mysql</groupId>
                 <artifactId>mysql-connector-java</artifactId>
-                <version>8.0.17</version>
+                <version>5.1.48</version>
             </dependency>
             <dependency>
                 <groupId>com.github.ben-manes.caffeine</groupId>

client-adapter/common/pom.xml

@@ -1,5 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
@@ -17,6 +18,11 @@
             <artifactId>canal.protocol</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>connector.core</artifactId>
+            <version>${project.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.springframework</groupId>
             <artifactId>spring-core</artifactId>
@@ -49,7 +55,7 @@
         <dependency>
             <groupId>com.alibaba</groupId>
             <artifactId>druid</artifactId>
-        </dependency>        
+        </dependency>
         <dependency>
             <groupId>org.yaml</groupId>
             <artifactId>snakeyaml</artifactId>

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java

@@ -3,6 +3,7 @@ package com.alibaba.otter.canal.client.adapter.support;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Properties;
 
 /**
  * 配置信息类
@@ -27,7 +28,7 @@ public class CanalClientConfig {
     // 重试次数
     private Integer            retries;
     // 消费超时时间
-    private Long               timeout;
+    private Long               timeout       = 500L;
     // 模式 tcp kafka rocketMQ
     private String             mode          = "tcp";
     // aliyun ak/sk
@@ -39,6 +40,8 @@ public class CanalClientConfig {
     // rabbitmq vhost
     private String             vhost         = "/";
 
+    private Properties         consumerProperties;
+
     private Long               resourceOwnerId;
     // 是否启用消息轨迹
     private boolean            enableMessageTrace;
@@ -163,6 +166,14 @@ public class CanalClientConfig {
         this.vhost = vhost;
     }
 
+    public Properties getConsumerProperties() {
+        return consumerProperties;
+    }
+
+    public void setConsumerProperties(Properties consumerProperties) {
+        this.consumerProperties = consumerProperties;
+    }
+
     public Long getResourceOwnerId() {
         return resourceOwnerId;
     }

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java

@@ -2,6 +2,7 @@ package com.alibaba.otter.canal.client.adapter.support;
 
 import java.util.*;
 
+import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
 import com.alibaba.otter.canal.protocol.CanalEntry;
 import com.alibaba.otter.canal.protocol.FlatMessage;
 import com.alibaba.otter.canal.protocol.Message;
@@ -131,10 +132,13 @@ public class MessageUtil {
         return dmls;
     }
 
-    public static List<Dml> flatMessage2Dml(String destination, String groupId, List<FlatMessage> flatMessages) {
-        List<Dml> dmls = new ArrayList<Dml>(flatMessages.size());
-        for (FlatMessage flatMessage : flatMessages) {
-            Dml dml = flatMessage2Dml(destination, groupId, flatMessage);
+    public static List<Dml> flatMessage2Dml(String destination, String groupId, List<CommonMessage> commonMessages) {
+        if (commonMessages == null) {
+            return new ArrayList<>();
+        }
+        List<Dml> dmls = new ArrayList<Dml>(commonMessages.size());
+        for (CommonMessage commonMessage : commonMessages) {
+            Dml dml = flatMessage2Dml(destination, groupId, commonMessage);
             if (dml != null) {
                 dmls.add(dml);
             }
@@ -143,31 +147,31 @@ public class MessageUtil {
         return dmls;
     }
 
-    public static Dml flatMessage2Dml(String destination, String groupId, FlatMessage flatMessage) {
-        if (flatMessage == null) {
+    public static Dml flatMessage2Dml(String destination, String groupId, CommonMessage commonMessage) {
+        if (commonMessage == null) {
             return null;
         }
         Dml dml = new Dml();
         dml.setDestination(destination);
         dml.setGroupId(groupId);
-        dml.setDatabase(flatMessage.getDatabase());
-        dml.setTable(flatMessage.getTable());
-        dml.setPkNames(flatMessage.getPkNames());
-        dml.setIsDdl(flatMessage.getIsDdl());
-        dml.setType(flatMessage.getType());
-        dml.setTs(flatMessage.getTs());
-        dml.setEs(flatMessage.getEs());
-        dml.setSql(flatMessage.getSql());
+        dml.setDatabase(commonMessage.getDatabase());
+        dml.setTable(commonMessage.getTable());
+        dml.setPkNames(commonMessage.getPkNames());
+        dml.setIsDdl(commonMessage.getIsDdl());
+        dml.setType(commonMessage.getType());
+        dml.setTs(commonMessage.getTs());
+        dml.setEs(commonMessage.getEs());
+        dml.setSql(commonMessage.getSql());
         // if (flatMessage.getSqlType() == null || flatMessage.getMysqlType() == null) {
         // throw new RuntimeException("SqlType or mysqlType is null");
         // }
-        List<Map<String, String>> data = flatMessage.getData();
+        List<Map<String, Object>> data = commonMessage.getData();
         if (data != null) {
-            dml.setData(changeRows(dml.getTable(), data, flatMessage.getSqlType(), flatMessage.getMysqlType()));
+            dml.setData(data);
         }
-        List<Map<String, String>> old = flatMessage.getOld();
+        List<Map<String, Object>> old = commonMessage.getOld();
         if (old != null) {
-            dml.setOld(changeRows(dml.getTable(), old, flatMessage.getSqlType(), flatMessage.getMysqlType()));
+            dml.setOld(old);
         }
         return dml;
     }

client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/ES6xAdapter.java

@@ -5,6 +5,10 @@ import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 
+import javax.sql.DataSource;
+
+import org.elasticsearch.action.search.SearchResponse;
+
 import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter;
 import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
 import com.alibaba.otter.canal.client.adapter.es6x.etl.ESEtlService;
@@ -14,9 +18,6 @@ import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
 import com.alibaba.otter.canal.client.adapter.support.EtlResult;
 import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
 import com.alibaba.otter.canal.client.adapter.support.SPI;
-import org.elasticsearch.action.search.SearchResponse;
-
-import javax.sql.DataSource;
 
 /**
  * ES 6.x 外部适配器

client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/etl/ESEtlService.java

@@ -10,7 +10,6 @@ import java.util.concurrent.atomic.AtomicLong;
 import javax.sql.DataSource;
 
 import org.apache.commons.lang.StringUtils;
-import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
@@ -79,119 +78,118 @@ public class ESEtlService extends AbstractEtlService {
                             }
 
                             // 如果是主键字段则不插入
-                            if (fieldItem.getFieldName().equals(mapping.get_id())) {
-                                idVal = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName);
-                            } else {
-                                Object val = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName);
-                                esFieldData.put(Util.cleanColumn(fieldName), val);
-                            }
-
-                        }
-
-                        if (!mapping.getRelations().isEmpty()) {
-                            mapping.getRelations().forEach((relationField, relationMapping) -> {
-                                Map<String, Object> relations = new HashMap<>();
-                                relations.put("name", relationMapping.getName());
-                                if (StringUtils.isNotEmpty(relationMapping.getParent())) {
-                                    FieldItem parentFieldItem = mapping.getSchemaItem()
-                                        .getSelectFields()
-                                        .get(relationMapping.getParent());
-                                    Object parentVal;
-                                    try {
-                                        parentVal = esTemplate.getValFromRS(mapping,
-                                            rs,
-                                            parentFieldItem.getFieldName(),
-                                            parentFieldItem.getFieldName());
-                                    } catch (SQLException e) {
-                                        throw new RuntimeException(e);
-                                    }
-                                    if (parentVal != null) {
-                                        relations.put("parent", parentVal.toString());
-                                        esFieldData.put("$parent_routing", parentVal.toString());
-
-                                    }
-                                }
-                                esFieldData.put(Util.cleanColumn(relationField), relations);
-                            });
-                        }
-
-                        if (idVal != null) {
-                            String parentVal = (String) esFieldData.remove("$parent_routing");
-                            if (mapping.isUpsert()) {
-                                ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(
-                                    mapping.get_index(),
-                                    mapping.get_type(),
-                                    idVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
-
-                                if (StringUtils.isNotEmpty(parentVal)) {
-                                    esUpdateRequest.setRouting(parentVal);
-                                }
-
-                                esBulkRequest.add(esUpdateRequest);
-                            } else {
-                                ESIndexRequest esIndexRequest = this.esConnection.new ES6xIndexRequest(mapping
-                                    .get_index(), mapping.get_type(), idVal.toString()).setSource(esFieldData);
-                                if (StringUtils.isNotEmpty(parentVal)) {
-                                    esIndexRequest.setRouting(parentVal);
-                                }
-                                esBulkRequest.add(esIndexRequest);
-                            }
-                        } else {
-                            idVal = esFieldData.get(mapping.getPk());
-                            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(),
-                                mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), idVal))
-                                    .size(10000);
-                            SearchResponse response = esSearchRequest.getResponse();
-                            for (SearchHit hit : response.getHits()) {
-                                ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping
-                                    .get_index(), mapping.get_type(), hit.getId()).setDoc(esFieldData);
-                                esBulkRequest.add(esUpdateRequest);
-                            }
-                        }
-
-                        if (esBulkRequest.numberOfActions() % mapping.getCommitBatch() == 0
-                            && esBulkRequest.numberOfActions() > 0) {
-                            long esBatchBegin = System.currentTimeMillis();
-                            ESBulkResponse rp = esBulkRequest.bulk();
-                            if (rp.hasFailures()) {
-                                rp.processFailBulkResponse("全量数据 etl 异常 ");
-                            }
-
-                            if (logger.isTraceEnabled()) {
-                                logger.trace("全量数据批量导入批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}",
-                                    (System.currentTimeMillis() - batchBegin),
-                                    (System.currentTimeMillis() - esBatchBegin),
-                                    esBulkRequest.numberOfActions(),
-                                    mapping.get_index());
-                            }
-                            batchBegin = System.currentTimeMillis();
-                            esBulkRequest.resetBulk();
-                        }
-                        count++;
-                        impCount.incrementAndGet();
-                    }
-
-                    if (esBulkRequest.numberOfActions() > 0) {
-                        long esBatchBegin = System.currentTimeMillis();
-                        ESBulkResponse rp = esBulkRequest.bulk();
-                        if (rp.hasFailures()) {
-                            rp.processFailBulkResponse("全量数据 etl 异常 ");
-                        }
-                        if (logger.isTraceEnabled()) {
-                            logger.trace("全量数据批量导入最后批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}",
-                                (System.currentTimeMillis() - batchBegin),
-                                (System.currentTimeMillis() - esBatchBegin),
-                                esBulkRequest.numberOfActions(),
-                                mapping.get_index());
-                        }
-                    }
-                } catch (Exception e) {
-                    logger.error(e.getMessage(), e);
-                    errMsg.add(mapping.get_index() + " etl failed! ==>" + e.getMessage());
-                    throw new RuntimeException(e);
+                if (fieldItem.getFieldName().equals(mapping.get_id())) {
+                    idVal = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName);
+                } else {
+                    Object val = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName);
+                    esFieldData.put(Util.cleanColumn(fieldName), val);
                 }
-                return count;
-            });
+
+            }
+
+            if (!mapping.getRelations().isEmpty()) {
+                mapping.getRelations().forEach((relationField, relationMapping) -> {
+                    Map<String, Object> relations = new HashMap<>();
+                    relations.put("name", relationMapping.getName());
+                    if (StringUtils.isNotEmpty(relationMapping.getParent())) {
+                        FieldItem parentFieldItem = mapping.getSchemaItem()
+                            .getSelectFields()
+                            .get(relationMapping.getParent());
+                        Object parentVal;
+                        try {
+                            parentVal = esTemplate.getValFromRS(mapping,
+                                rs,
+                                parentFieldItem.getFieldName(),
+                                parentFieldItem.getFieldName());
+                        } catch (SQLException e) {
+                            throw new RuntimeException(e);
+                        }
+                        if (parentVal != null) {
+                            relations.put("parent", parentVal.toString());
+                            esFieldData.put("$parent_routing", parentVal.toString());
+
+                        }
+                    }
+                    esFieldData.put(Util.cleanColumn(relationField), relations);
+                });
+            }
+
+            if (idVal != null) {
+                String parentVal = (String) esFieldData.remove("$parent_routing");
+                if (mapping.isUpsert()) {
+                    ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.get_index(),
+                        mapping.get_type(),
+                        idVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
+
+                    if (StringUtils.isNotEmpty(parentVal)) {
+                        esUpdateRequest.setRouting(parentVal);
+                    }
+
+                    esBulkRequest.add(esUpdateRequest);
+                } else {
+                    ESIndexRequest esIndexRequest = this.esConnection.new ES6xIndexRequest(mapping.get_index(),
+                        mapping.get_type(),
+                        idVal.toString()).setSource(esFieldData);
+                    if (StringUtils.isNotEmpty(parentVal)) {
+                        esIndexRequest.setRouting(parentVal);
+                    }
+                    esBulkRequest.add(esIndexRequest);
+                }
+            } else {
+                idVal = esFieldData.get(mapping.getPk());
+                ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(),
+                    mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), idVal)).size(10000);
+                SearchResponse response = esSearchRequest.getResponse();
+                for (SearchHit hit : response.getHits()) {
+                    ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.get_index(),
+                        mapping.get_type(),
+                        hit.getId()).setDoc(esFieldData);
+                    esBulkRequest.add(esUpdateRequest);
+                }
+            }
+
+            if (esBulkRequest.numberOfActions() % mapping.getCommitBatch() == 0 && esBulkRequest.numberOfActions() > 0) {
+                long esBatchBegin = System.currentTimeMillis();
+                ESBulkResponse rp = esBulkRequest.bulk();
+                if (rp.hasFailures()) {
+                    rp.processFailBulkResponse("全量数据 etl 异常 ");
+                }
+
+                if (logger.isTraceEnabled()) {
+                    logger.trace("全量数据批量导入批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}",
+                        (System.currentTimeMillis() - batchBegin),
+                        (System.currentTimeMillis() - esBatchBegin),
+                        esBulkRequest.numberOfActions(),
+                        mapping.get_index());
+                }
+                batchBegin = System.currentTimeMillis();
+                esBulkRequest.resetBulk();
+            }
+            count++;
+            impCount.incrementAndGet();
+        }
+
+        if (esBulkRequest.numberOfActions() > 0) {
+            long esBatchBegin = System.currentTimeMillis();
+            ESBulkResponse rp = esBulkRequest.bulk();
+            if (rp.hasFailures()) {
+                rp.processFailBulkResponse("全量数据 etl 异常 ");
+            }
+            if (logger.isTraceEnabled()) {
+                logger.trace("全量数据批量导入最后批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}",
+                    (System.currentTimeMillis() - batchBegin),
+                    (System.currentTimeMillis() - esBatchBegin),
+                    esBulkRequest.numberOfActions(),
+                    mapping.get_index());
+            }
+        }
+    } catch (Exception e) {
+        logger.error(e.getMessage(), e);
+        errMsg.add(mapping.get_index() + " etl failed! ==>" + e.getMessage());
+        throw new RuntimeException(e);
+    }
+    return count;
+}           );
 
             return true;
         } catch (Exception e) {

client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/support/ES6xTemplate.java

@@ -2,7 +2,11 @@ package com.alibaba.otter.canal.client.adapter.es6x.support;
 
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 
@@ -20,8 +24,8 @@ import org.slf4j.LoggerFactory;
 import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
 import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping;
 import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem;
-import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem;
 import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.ColumnItem;
+import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem;
 import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest;
 import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESBulkResponse;
 import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESDeleteRequest;

client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/support/ESConnection.java

@@ -5,7 +5,6 @@ import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.util.Map;
 
-import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest;
 import org.apache.commons.lang.StringUtils;
 import org.apache.http.HttpHost;
 import org.apache.http.auth.AuthScope;
@@ -27,7 +26,11 @@ import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.action.update.UpdateRequestBuilder;
-import org.elasticsearch.client.*;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestClientBuilder;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.client.RestHighLevelClientExt;
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
@@ -40,6 +43,8 @@ import org.elasticsearch.transport.client.PreBuiltTransportClient;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest;
+
 /**
  * ES 连接器, Transport Rest 两种方式
  *
@@ -51,7 +56,7 @@ public class ESConnection {
     private static final Logger logger = LoggerFactory.getLogger(ESConnection.class);
 
     public enum ESClientMode {
-                              TRANSPORT, REST
+        TRANSPORT, REST
     }
 
     private ESClientMode        mode;
@@ -86,10 +91,9 @@ public class ESConnection {
             if (StringUtils.isNotEmpty(nameAndPwd) && nameAndPwd.contains(":")) {
                 String[] nameAndPwdArr = nameAndPwd.split(":");
                 final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
-                credentialsProvider.setCredentials(AuthScope.ANY,
-                    new UsernamePasswordCredentials(nameAndPwdArr[0], nameAndPwdArr[1]));
-                restClientBuilder.setHttpClientConfigCallback(
-                    httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
+                credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(nameAndPwdArr[0],
+                    nameAndPwdArr[1]));
+                restClientBuilder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
             }
             restHighLevelClient = new RestHighLevelClient(restClientBuilder);
         }
@@ -418,6 +422,7 @@ public class ESConnection {
             }
         }
 
+        @SuppressWarnings("deprecation")
         public ESBulkResponse bulk() {
             if (mode == ESClientMode.TRANSPORT) {
                 BulkResponse responses = bulkRequestBuilder.execute().actionGet();

client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/LabelSyncJoinSub2Test.java

@@ -1,6 +1,10 @@
 package com.alibaba.otter.canal.client.adapter.es6x.test.sync;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
 
 import javax.sql.DataSource;
 

client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/LabelSyncJoinSubTest.java

@@ -1,6 +1,10 @@
 package com.alibaba.otter.canal.client.adapter.es6x.test.sync;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
 
 import javax.sql.DataSource;
 

client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/RoleSyncJoinOne2Test.java

@@ -1,6 +1,10 @@
 package com.alibaba.otter.canal.client.adapter.es6x.test.sync;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
 
 import javax.sql.DataSource;
 

client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/RoleSyncJoinOneTest.java

@@ -1,6 +1,10 @@
 package com.alibaba.otter.canal.client.adapter.es6x.test.sync;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
 
 import javax.sql.DataSource;
 

client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/UserSyncJoinOneTest.java

@@ -1,6 +1,10 @@
 package com.alibaba.otter.canal.client.adapter.es6x.test.sync;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
 
 import javax.sql.DataSource;
 

client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/UserSyncSingleTest.java

@@ -1,6 +1,10 @@
 package com.alibaba.otter.canal.client.adapter.es6x.test.sync;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
 
 import org.elasticsearch.action.get.GetResponse;
 import org.junit.Assert;

client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/ES7xAdapter.java

@@ -5,20 +5,20 @@ import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 
-import com.alibaba.otter.canal.client.adapter.es7x.etl.ESEtlService;
-import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
+import javax.sql.DataSource;
+
 import org.elasticsearch.action.search.SearchResponse;
 
 import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter;
 import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
+import com.alibaba.otter.canal.client.adapter.es7x.etl.ESEtlService;
 import com.alibaba.otter.canal.client.adapter.es7x.support.ES7xTemplate;
 import com.alibaba.otter.canal.client.adapter.es7x.support.ESConnection;
+import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
 import com.alibaba.otter.canal.client.adapter.support.EtlResult;
 import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
 import com.alibaba.otter.canal.client.adapter.support.SPI;
 
-import javax.sql.DataSource;
-
 /**
  * ES 7.x 外部适配器
  *

client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/support/ES7xTemplate.java

@@ -2,7 +2,11 @@ package com.alibaba.otter.canal.client.adapter.es7x.support;
 
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 

client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/support/ESConnection.java

@@ -54,11 +54,12 @@ public class ESConnection {
     private static final Logger logger = LoggerFactory.getLogger(ESConnection.class);
 
     public enum ESClientMode {
-                              TRANSPORT, REST
+        TRANSPORT, REST
     }
 
     private ESClientMode        mode;
 
+    @SuppressWarnings("deprecation")
     private TransportClient     transportClient;
 
     private RestHighLevelClient restHighLevelClient;
@@ -89,10 +90,9 @@ public class ESConnection {
             if (StringUtils.isNotEmpty(nameAndPwd) && nameAndPwd.contains(":")) {
                 String[] nameAndPwdArr = nameAndPwd.split(":");
                 final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
-                credentialsProvider.setCredentials(AuthScope.ANY,
-                    new UsernamePasswordCredentials(nameAndPwdArr[0], nameAndPwdArr[1]));
-                restClientBuilder.setHttpClientConfigCallback(
-                    httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
+                credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(nameAndPwdArr[0],
+                    nameAndPwdArr[1]));
+                restClientBuilder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
             }
             restHighLevelClient = new RestHighLevelClient(restClientBuilder);
         }

client-adapter/es7x/src/test/java/com/alibaba/otter/canal/client/adapter/es7x/test/ES7xTest.java

@@ -24,6 +24,7 @@ import org.junit.Test;
 @Ignore
 public class ES7xTest {
 
+    @SuppressWarnings("deprecation")
     private TransportClient transportClient;
 
     @Before
@@ -65,8 +66,9 @@ public class ES7xTest {
         relations.put("parent", "2");
 
         BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
-        bulkRequestBuilder
-            .add(transportClient.prepareIndex("test", "osm", "2_4").setRouting("2").setSource(esFieldData));
+        bulkRequestBuilder.add(transportClient.prepareIndex("test", "osm", "2_4")
+            .setRouting("2")
+            .setSource(esFieldData));
         commit(bulkRequestBuilder);
     }
 

client-adapter/es7x/src/test/java/com/alibaba/otter/canal/client/adapter/es7x/test/ESConnectionTest.java

@@ -8,9 +8,9 @@ import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Test;
+import org.springframework.util.Assert;
 
 import com.alibaba.otter.canal.client.adapter.es7x.support.ESConnection;
-import org.springframework.util.Assert;
 
 @Ignore
 public class ESConnectionTest {

client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/ESAdapter.java

@@ -1,6 +1,5 @@
 package com.alibaba.otter.canal.client.adapter.es.core;
 
-import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
@@ -8,7 +7,6 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import com.alibaba.otter.canal.client.adapter.es.core.support.ESTemplate;
 import org.apache.commons.lang.StringUtils;
 
 import com.alibaba.druid.pool.DruidDataSource;
@@ -19,6 +17,7 @@ import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem;
 import com.alibaba.otter.canal.client.adapter.es.core.config.SqlParser;
 import com.alibaba.otter.canal.client.adapter.es.core.monitor.ESConfigMonitor;
 import com.alibaba.otter.canal.client.adapter.es.core.service.ESSyncService;
+import com.alibaba.otter.canal.client.adapter.es.core.support.ESTemplate;
 import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
 import com.alibaba.otter.canal.client.adapter.support.Dml;
 import com.alibaba.otter.canal.client.adapter.support.EtlResult;
@@ -63,8 +62,8 @@ public abstract class ESAdapter implements OuterAdapter {
             // 过滤不匹配的key的配置
             esSyncConfigTmp.forEach((key, config) -> {
                 if ((config.getOuterAdapterKey() == null && configuration.getKey() == null)
-                    || (config.getOuterAdapterKey() != null
-                        && config.getOuterAdapterKey().equalsIgnoreCase(configuration.getKey()))) {
+                    || (config.getOuterAdapterKey() != null && config.getOuterAdapterKey()
+                        .equalsIgnoreCase(configuration.getKey()))) {
                     esSyncConfig.put(key, config);
                 }
             });
@@ -104,12 +103,12 @@ public abstract class ESAdapter implements OuterAdapter {
         String table = dml.getTable();
         Map<String, ESSyncConfig> configMap;
         if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
-            configMap = dbTableEsSyncConfig
-                .get(StringUtils.trimToEmpty(dml.getDestination()) + "-" + StringUtils.trimToEmpty(dml.getGroupId())
-                     + "_" + database + "-" + table);
+            configMap = dbTableEsSyncConfig.get(StringUtils.trimToEmpty(dml.getDestination()) + "-"
+                                                + StringUtils.trimToEmpty(dml.getGroupId()) + "_" + database + "-"
+                                                + table);
         } else {
-            configMap = dbTableEsSyncConfig
-                .get(StringUtils.trimToEmpty(dml.getDestination()) + "_" + database + "-" + table);
+            configMap = dbTableEsSyncConfig.get(StringUtils.trimToEmpty(dml.getDestination()) + "_" + database + "-"
+                                                + table);
         }
 
         if (configMap != null && !configMap.values().isEmpty()) {
@@ -155,23 +154,29 @@ public abstract class ESAdapter implements OuterAdapter {
         }
         String schema = matcher.group(2);
 
-        schemaItem.getAliasTableItems().values().forEach(tableItem -> {
-            Map<String, ESSyncConfig> esSyncConfigMap;
-            if (envProperties != null
-                    && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
-                esSyncConfigMap = dbTableEsSyncConfig
-                        .computeIfAbsent(StringUtils.trimToEmpty(config.getDestination()) + "-"
-                                        + StringUtils.trimToEmpty(config.getGroupId()) + "_" + schema + "-"
-                                        + tableItem.getTableName(),
-                                k -> new ConcurrentHashMap<>());
-            } else {
-                esSyncConfigMap = dbTableEsSyncConfig
-                        .computeIfAbsent(StringUtils.trimToEmpty(config.getDestination()) + "_" + schema + "-"
-                                        + tableItem.getTableName(),
-                                k -> new ConcurrentHashMap<>());
-            }
+        schemaItem.getAliasTableItems()
+            .values()
+            .forEach(tableItem -> {
+                Map<String, ESSyncConfig> esSyncConfigMap;
+                if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
+                    esSyncConfigMap = dbTableEsSyncConfig.computeIfAbsent(StringUtils.trimToEmpty(config.getDestination())
+                                                                          + "-"
+                                                                          + StringUtils.trimToEmpty(config.getGroupId())
+                                                                          + "_"
+                                                                          + schema
+                                                                          + "-"
+                                                                          + tableItem.getTableName(),
+                        k -> new ConcurrentHashMap<>());
+                } else {
+                    esSyncConfigMap = dbTableEsSyncConfig.computeIfAbsent(StringUtils.trimToEmpty(config.getDestination())
+                                                                          + "_"
+                                                                          + schema
+                                                                          + "-"
+                                                                          + tableItem.getTableName(),
+                        k -> new ConcurrentHashMap<>());
+                }
 
-            esSyncConfigMap.put(configName, config);
-        });
+                esSyncConfigMap.put(configName, config);
+            });
     }
 }

client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/SqlParser.java

@@ -9,8 +9,19 @@ import java.util.stream.Collectors;
 
 import com.alibaba.fastsql.sql.SQLUtils;
 import com.alibaba.fastsql.sql.ast.SQLExpr;
-import com.alibaba.fastsql.sql.ast.expr.*;
-import com.alibaba.fastsql.sql.ast.statement.*;
+import com.alibaba.fastsql.sql.ast.expr.SQLBinaryOpExpr;
+import com.alibaba.fastsql.sql.ast.expr.SQLCaseExpr;
+import com.alibaba.fastsql.sql.ast.expr.SQLCharExpr;
+import com.alibaba.fastsql.sql.ast.expr.SQLIdentifierExpr;
+import com.alibaba.fastsql.sql.ast.expr.SQLMethodInvokeExpr;
+import com.alibaba.fastsql.sql.ast.expr.SQLPropertyExpr;
+import com.alibaba.fastsql.sql.ast.statement.SQLExprTableSource;
+import com.alibaba.fastsql.sql.ast.statement.SQLJoinTableSource;
+import com.alibaba.fastsql.sql.ast.statement.SQLSelectGroupByClause;
+import com.alibaba.fastsql.sql.ast.statement.SQLSelectItem;
+import com.alibaba.fastsql.sql.ast.statement.SQLSelectStatement;
+import com.alibaba.fastsql.sql.ast.statement.SQLSubqueryTableSource;
+import com.alibaba.fastsql.sql.ast.statement.SQLTableSource;
 import com.alibaba.fastsql.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
 import com.alibaba.fastsql.sql.dialect.mysql.parser.MySqlStatementParser;
 import com.alibaba.fastsql.sql.parser.ParserException;

client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/monitor/ESConfigMonitor.java

@@ -1,13 +1,9 @@
 package com.alibaba.otter.canal.client.adapter.es.core.monitor;
 
 import java.io.File;
-import java.util.HashMap;
 import java.util.Map;
 import java.util.Properties;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
-import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter;
 import org.apache.commons.io.filefilter.FileFilterUtils;
 import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
 import org.apache.commons.io.monitor.FileAlterationMonitor;
@@ -15,12 +11,9 @@ import org.apache.commons.io.monitor.FileAlterationObserver;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.alibaba.druid.pool.DruidDataSource;
 import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
+import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter;
 import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
-import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem;
-import com.alibaba.otter.canal.client.adapter.es.core.config.SqlParser;
-import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
 import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
 import com.alibaba.otter.canal.client.adapter.support.Util;
 
@@ -70,8 +63,11 @@ public class ESConfigMonitor {
             try {
                 // 加载新增的配置文件
                 String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName());
-                ESSyncConfig config = YmlConfigBinder
-                    .bindYmlToObj(null, configContent, ESSyncConfig.class, null, envProperties);
+                ESSyncConfig config = YmlConfigBinder.bindYmlToObj(null,
+                    configContent,
+                    ESSyncConfig.class,
+                    null,
+                    envProperties);
                 if (config != null) {
                     config.validate();
                     addConfigToCache(file, config);
@@ -89,14 +85,17 @@ public class ESConfigMonitor {
             try {
                 if (esAdapter.getEsSyncConfig().containsKey(file.getName())) {
                     // 加载配置文件
-                    String configContent = MappingConfigsLoader
-                        .loadConfig(adapterName + File.separator + file.getName());
+                    String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator
+                                                                           + file.getName());
                     if (configContent == null) {
                         onFileDelete(file);
                         return;
                     }
-                    ESSyncConfig config = YmlConfigBinder
-                        .bindYmlToObj(null, configContent, ESSyncConfig.class, null, envProperties);
+                    ESSyncConfig config = YmlConfigBinder.bindYmlToObj(null,
+                        configContent,
+                        ESSyncConfig.class,
+                        null,
+                        envProperties);
                     if (config == null) {
                         return;
                     }

client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/service/ESSyncService.java

@@ -1,22 +1,26 @@
 package com.alibaba.otter.canal.client.adapter.es.core.service;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
 
 import javax.sql.DataSource;
 
-import com.alibaba.fastsql.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
-import com.alibaba.otter.canal.client.adapter.es.core.config.SqlParser;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.serializer.SerializerFeature;
+import com.alibaba.fastsql.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
 import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
 import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping;
 import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem;
 import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.ColumnItem;
 import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem;
 import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.TableItem;
+import com.alibaba.otter.canal.client.adapter.es.core.config.SqlParser;
 import com.alibaba.otter.canal.client.adapter.es.core.support.ESSyncUtil;
 import com.alibaba.otter.canal.client.adapter.es.core.support.ESTemplate;
 import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;

client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/support/ESSyncUtil.java

@@ -5,7 +5,13 @@ import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 import java.sql.Blob;
 import java.sql.SQLException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
 
 import org.apache.commons.codec.binary.Base64;
 import org.joda.time.DateTime;

client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java

@@ -5,19 +5,24 @@ import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
 import java.util.concurrent.atomic.AtomicLong;
 
 import javax.sql.DataSource;
 
-import com.alibaba.otter.canal.client.adapter.support.*;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig;
-import com.alibaba.otter.canal.client.adapter.hbase.support.*;
+import com.alibaba.otter.canal.client.adapter.hbase.support.HRow;
+import com.alibaba.otter.canal.client.adapter.hbase.support.HbaseTemplate;
+import com.alibaba.otter.canal.client.adapter.hbase.support.PhType;
+import com.alibaba.otter.canal.client.adapter.hbase.support.PhTypeUtil;
+import com.alibaba.otter.canal.client.adapter.hbase.support.Type;
+import com.alibaba.otter.canal.client.adapter.hbase.support.TypeUtil;
+import com.alibaba.otter.canal.client.adapter.support.AbstractEtlService;
+import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
+import com.alibaba.otter.canal.client.adapter.support.EtlResult;
+import com.alibaba.otter.canal.client.adapter.support.JdbcTypeUtil;
+import com.alibaba.otter.canal.client.adapter.support.Util;
 import com.google.common.base.Joiner;
 
 /**
@@ -79,7 +84,8 @@ public class HbaseEtlService extends AbstractEtlService {
             createTable();
 
             // 拼接sql
-            String sql = "SELECT * FROM `" + config.getHbaseMapping().getDatabase() + "`.`" + hbaseMapping.getTable() + "`";
+            String sql = "SELECT * FROM `" + config.getHbaseMapping().getDatabase() + "`.`" + hbaseMapping.getTable()
+                         + "`";
 
             return super.importData(sql, params);
         } catch (Exception e) {
@@ -120,146 +126,146 @@ public class HbaseEtlService extends AbstractEtlService {
 
                         if (rowKeyColumns != null) {
                             // 取rowKey字段拼接
-                            StringBuilder rowKeyVale = new StringBuilder();
-                            for (String rowKeyColumnName : rowKeyColumns) {
-                                Object obj = rs.getObject(rowKeyColumnName);
-                                if (obj != null) {
-                                    rowKeyVale.append(obj.toString());
-                                }
-                                rowKeyVale.append("|");
-                            }
-                            int len = rowKeyVale.length();
-                            if (len > 0) {
-                                rowKeyVale.delete(len - 1, len);
-                            }
-                            row.setRowKey(Bytes.toBytes(rowKeyVale.toString()));
+                StringBuilder rowKeyVale = new StringBuilder();
+                for (String rowKeyColumnName : rowKeyColumns) {
+                    Object obj = rs.getObject(rowKeyColumnName);
+                    if (obj != null) {
+                        rowKeyVale.append(obj.toString());
+                    }
+                    rowKeyVale.append("|");
+                }
+                int len = rowKeyVale.length();
+                if (len > 0) {
+                    rowKeyVale.delete(len - 1, len);
+                }
+                row.setRowKey(Bytes.toBytes(rowKeyVale.toString()));
+            }
+
+            for (int j = 1; j <= cc; j++) {
+                String columnName = rs.getMetaData().getColumnName(j);
+
+                Object val = JdbcTypeUtil.getRSData(rs, columnName, jdbcTypes[j - 1]);
+                if (val == null) {
+                    continue;
+                }
+
+                MappingConfig.ColumnItem columnItem = hbaseMapping.getColumnItems().get(columnName);
+                // 没有配置映射
+                if (columnItem == null) {
+                    String family = hbaseMapping.getFamily();
+                    String qualifile = columnName;
+                    if (hbaseMapping.isUppercaseQualifier()) {
+                        qualifile = qualifile.toUpperCase();
+                    }
+                    if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) {
+                        if (hbaseMapping.getRowKey() == null && j == 1) {
+                            row.setRowKey(Bytes.toBytes(val.toString()));
+                        } else {
+                            row.addCell(family, qualifile, Bytes.toBytes(val.toString()));
+                        }
+                    } else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) {
+                        Type type = Type.getType(classes[j - 1]);
+                        if (hbaseMapping.getRowKey() == null && j == 1) {
+                            row.setRowKey(TypeUtil.toBytes(val, type));
+                        } else {
+                            row.addCell(family, qualifile, TypeUtil.toBytes(val, type));
+                        }
+                    } else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) {
+                        PhType phType = PhType.getType(classes[j - 1]);
+                        if (hbaseMapping.getRowKey() == null && j == 1) {
+                            row.setRowKey(PhTypeUtil.toBytes(val, phType));
+                        } else {
+                            row.addCell(family, qualifile, PhTypeUtil.toBytes(val, phType));
+                        }
+                    }
+                } else {
+                    // 如果不需要类型转换
+                    if (columnItem.getType() == null || "".equals(columnItem.getType())) {
+                        if (val instanceof java.sql.Date) {
+                            SimpleDateFormat dateFmt = new SimpleDateFormat("yyyy-MM-dd");
+                            val = dateFmt.format((Date) val);
+                        } else if (val instanceof Timestamp) {
+                            SimpleDateFormat datetimeFmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+                            val = datetimeFmt.format((Date) val);
                         }
 
-                        for (int j = 1; j <= cc; j++) {
-                            String columnName = rs.getMetaData().getColumnName(j);
-
-                            Object val = JdbcTypeUtil.getRSData(rs, columnName, jdbcTypes[j - 1]);
-                            if (val == null) {
-                                continue;
+                        byte[] valBytes = Bytes.toBytes(val.toString());
+                        if (columnItem.isRowKey()) {
+                            if (columnItem.getRowKeyLen() != null) {
+                                valBytes = Bytes.toBytes(limitLenNum(columnItem.getRowKeyLen(), val));
+                                row.setRowKey(valBytes);
+                            } else {
+                                row.setRowKey(valBytes);
                             }
-
-                            MappingConfig.ColumnItem columnItem = hbaseMapping.getColumnItems().get(columnName);
-                            // 没有配置映射
-                            if (columnItem == null) {
-                                String family = hbaseMapping.getFamily();
-                                String qualifile = columnName;
-                                if (hbaseMapping.isUppercaseQualifier()) {
-                                    qualifile = qualifile.toUpperCase();
+                        } else {
+                            row.addCell(columnItem.getFamily(), columnItem.getQualifier(), valBytes);
+                        }
+                    } else {
+                        if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) {
+                            byte[] valBytes = Bytes.toBytes(val.toString());
+                            if (columnItem.isRowKey()) {
+                                if (columnItem.getRowKeyLen() != null) {
+                                    valBytes = Bytes.toBytes(limitLenNum(columnItem.getRowKeyLen(), val));
                                 }
-                                if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) {
-                                    if (hbaseMapping.getRowKey() == null && j == 1) {
-                                        row.setRowKey(Bytes.toBytes(val.toString()));
-                                    } else {
-                                        row.addCell(family, qualifile, Bytes.toBytes(val.toString()));
-                                    }
-                                } else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) {
-                                    Type type = Type.getType(classes[j - 1]);
-                                    if (hbaseMapping.getRowKey() == null && j == 1) {
-                                        row.setRowKey(TypeUtil.toBytes(val, type));
-                                    } else {
-                                        row.addCell(family, qualifile, TypeUtil.toBytes(val, type));
-                                    }
-                                } else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) {
-                                    PhType phType = PhType.getType(classes[j - 1]);
-                                    if (hbaseMapping.getRowKey() == null && j == 1) {
-                                        row.setRowKey(PhTypeUtil.toBytes(val, phType));
-                                    } else {
-                                        row.addCell(family, qualifile, PhTypeUtil.toBytes(val, phType));
-                                    }
+                                row.setRowKey(valBytes);
+                            } else {
+                                row.addCell(columnItem.getFamily(), columnItem.getQualifier(), valBytes);
+                            }
+                        } else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) {
+                            Type type = Type.getType(columnItem.getType());
+                            if (columnItem.isRowKey()) {
+                                if (columnItem.getRowKeyLen() != null) {
+                                    String v = limitLenNum(columnItem.getRowKeyLen(), val);
+                                    row.setRowKey(Bytes.toBytes(v));
+                                } else {
+                                    row.setRowKey(TypeUtil.toBytes(val, type));
                                 }
                             } else {
-                                // 如果不需要类型转换
-                                if (columnItem.getType() == null || "".equals(columnItem.getType())) {
-                                    if (val instanceof java.sql.Date) {
-                                        SimpleDateFormat dateFmt = new SimpleDateFormat("yyyy-MM-dd");
-                                        val = dateFmt.format((Date) val);
-                                    } else if (val instanceof Timestamp) {
-                                        SimpleDateFormat datetimeFmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
-                                        val = datetimeFmt.format((Date) val);
-                                    }
-
-                                    byte[] valBytes = Bytes.toBytes(val.toString());
-                                    if (columnItem.isRowKey()) {
-                                        if (columnItem.getRowKeyLen() != null) {
-                                            valBytes = Bytes.toBytes(limitLenNum(columnItem.getRowKeyLen(), val));
-                                            row.setRowKey(valBytes);
-                                        } else {
-                                            row.setRowKey(valBytes);
-                                        }
-                                    } else {
-                                        row.addCell(columnItem.getFamily(), columnItem.getQualifier(), valBytes);
-                                    }
-                                } else {
-                                    if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) {
-                                        byte[] valBytes = Bytes.toBytes(val.toString());
-                                        if (columnItem.isRowKey()) {
-                                            if (columnItem.getRowKeyLen() != null) {
-                                                valBytes = Bytes.toBytes(limitLenNum(columnItem.getRowKeyLen(), val));
-                                            }
-                                            row.setRowKey(valBytes);
-                                        } else {
-                                            row.addCell(columnItem.getFamily(), columnItem.getQualifier(), valBytes);
-                                        }
-                                    } else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) {
-                                        Type type = Type.getType(columnItem.getType());
-                                        if (columnItem.isRowKey()) {
-                                            if (columnItem.getRowKeyLen() != null) {
-                                                String v = limitLenNum(columnItem.getRowKeyLen(), val);
-                                                row.setRowKey(Bytes.toBytes(v));
-                                            } else {
-                                                row.setRowKey(TypeUtil.toBytes(val, type));
-                                            }
-                                        } else {
-                                            row.addCell(columnItem.getFamily(),
-                                                columnItem.getQualifier(),
-                                                TypeUtil.toBytes(val, type));
-                                        }
-                                    } else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) {
-                                        PhType phType = PhType.getType(columnItem.getType());
-                                        if (columnItem.isRowKey()) {
-                                            row.setRowKey(PhTypeUtil.toBytes(val, phType));
-                                        } else {
-                                            row.addCell(columnItem.getFamily(),
-                                                columnItem.getQualifier(),
-                                                PhTypeUtil.toBytes(val, phType));
-                                        }
-                                    }
-                                }
+                                row.addCell(columnItem.getFamily(),
+                                    columnItem.getQualifier(),
+                                    TypeUtil.toBytes(val, type));
+                            }
+                        } else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) {
+                            PhType phType = PhType.getType(columnItem.getType());
+                            if (columnItem.isRowKey()) {
+                                row.setRowKey(PhTypeUtil.toBytes(val, phType));
+                            } else {
+                                row.addCell(columnItem.getFamily(),
+                                    columnItem.getQualifier(),
+                                    PhTypeUtil.toBytes(val, phType));
                             }
                         }
-
-                        if (row.getRowKey() == null) throw new RuntimeException("RowKey 值为空");
-
-                        rows.add(row);
-                        complete = false;
-                        if (i % hbaseMapping.getCommitBatch() == 0 && !rows.isEmpty()) {
-                            hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows);
-                            rows.clear();
-                            complete = true;
-                        }
-                        i++;
-                        impCount.incrementAndGet();
-                        if (logger.isDebugEnabled()) {
-                            logger.debug("successful import count:" + impCount.get());
-                        }
                     }
-
-                    if (!complete && !rows.isEmpty()) {
-                        hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows);
-                    }
-
-                } catch (Exception e) {
-                    logger.error(hbaseMapping.getHbaseTable() + " etl failed! ==>" + e.getMessage(), e);
-                    errMsg.add(hbaseMapping.getHbaseTable() + " etl failed! ==>" + e.getMessage());
-                    // throw new RuntimeException(e);
                 }
-                return i;
-            });
+            }
+
+            if (row.getRowKey() == null) throw new RuntimeException("RowKey 值为空");
+
+            rows.add(row);
+            complete = false;
+            if (i % hbaseMapping.getCommitBatch() == 0 && !rows.isEmpty()) {
+                hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows);
+                rows.clear();
+                complete = true;
+            }
+            i++;
+            impCount.incrementAndGet();
+            if (logger.isDebugEnabled()) {
+                logger.debug("successful import count:" + impCount.get());
+            }
+        }
+
+        if (!complete && !rows.isEmpty()) {
+            hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows);
+        }
+
+    } catch (Exception e) {
+        logger.error(hbaseMapping.getHbaseTable() + " etl failed! ==>" + e.getMessage(), e);
+        errMsg.add(hbaseMapping.getHbaseTable() + " etl failed! ==>" + e.getMessage());
+        // throw new RuntimeException(e);
+    }
+    return i;
+}           );
             return true;
         } catch (Exception e) {
             logger.error(e.getMessage(), e);

client-adapter/kudu/pom.xml

@@ -0,0 +1,87 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>canal.client-adapter</artifactId>
+        <groupId>com.alibaba.otter</groupId>
+        <version>1.1.5-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.alibaba.otter</groupId>
+    <artifactId>client-adapter.kudu</artifactId>
+    <packaging>jar</packaging>
+    <name>canal client adapter kudu module for otter ${project.version}</name>
+    <dependencies>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>client-adapter.common</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.kudu</groupId>
+            <artifactId>kudu-client</artifactId>
+            <version>1.6.0</version>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.12</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>mysql</groupId>
+            <artifactId>mysql-connector-java</artifactId>
+            <version>5.1.48</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>2.4</version>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <artifactId>maven-antrun-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+                            <tasks>
+                                <copy todir="${project.basedir}/../launcher/target/classes/kudu" overwrite="true">
+                                    <fileset dir="${project.basedir}/target/classes/kudu" erroronmissingdir="true">
+                                        <include name="*.yml"/>
+                                    </fileset>
+                                </copy>
+                            </tasks>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/KuduAdapter.java

@@ -0,0 +1,206 @@
+package com.alibaba.otter.canal.client.adapter.kudu;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.client.adapter.OuterAdapter;
+import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
+import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfigLoader;
+import com.alibaba.otter.canal.client.adapter.kudu.monitor.KuduConfigMonitor;
+import com.alibaba.otter.canal.client.adapter.kudu.service.KuduEtlService;
+import com.alibaba.otter.canal.client.adapter.kudu.service.KuduSyncService;
+import com.alibaba.otter.canal.client.adapter.kudu.support.KuduTemplate;
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+import com.alibaba.otter.canal.client.adapter.support.EtlResult;
+import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
+import com.alibaba.otter.canal.client.adapter.support.SPI;
+
+/**
+ * @author liuyadong
+ * @description kudu适配器主类
+ */
+@SPI("kudu")
+public class KuduAdapter implements OuterAdapter {
+
+    private static Logger                               logger             = LoggerFactory.getLogger(KuduAdapter.class);
+
+    private Map<String, KuduMappingConfig>              kuduMapping        = new ConcurrentHashMap<>();                 // 文件名对应配置
+    private Map<String, Map<String, KuduMappingConfig>> mappingConfigCache = new ConcurrentHashMap<>();                 // 库名-表名对应配置
+
+    private String                                      dataSourceKey;
+
+    private KuduTemplate                                kuduTemplate;
+
+    private KuduSyncService                             kuduSyncService;
+
+    private KuduConfigMonitor                           kuduConfigMonitor;
+
+    private Properties                                  envProperties;
+
+    public Map<String, KuduMappingConfig> getKuduMapping() {
+        return kuduMapping;
+    }
+
+    public Map<String, Map<String, KuduMappingConfig>> getMappingConfigCache() {
+        return mappingConfigCache;
+    }
+
+    @Override
+    public void init(OuterAdapterConfig configuration, Properties envProperties) {
+        this.envProperties = envProperties;
+        Map<String, KuduMappingConfig> kuduMappingTmp = KuduMappingConfigLoader.load(envProperties);
+        // 过滤不匹配的key的配置,获取连接key，key为配置文件名称
+        kuduMappingTmp.forEach((key, mappingConfig) -> {
+            if ((mappingConfig.getOuterAdapterKey() == null && configuration.getKey() == null)
+                || (mappingConfig.getOuterAdapterKey() != null && mappingConfig.getOuterAdapterKey()
+                    .equalsIgnoreCase(configuration.getKey()))) {
+                kuduMapping.put(key, mappingConfig);
+                dataSourceKey = mappingConfig.getDataSourceKey();
+            }
+        });
+        // 判断目标字段是否为空
+        if (kuduMapping.isEmpty()) {
+            throw new RuntimeException("No kudu adapter found for config key: " + configuration.getKey());
+        }
+        for (Map.Entry<String, KuduMappingConfig> entry : kuduMapping.entrySet()) {
+            String configName = entry.getKey();
+            KuduMappingConfig mappingConfig = entry.getValue();
+            String k;
+            if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
+                k = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "-"
+                    + StringUtils.trimToEmpty(mappingConfig.getGroupId()) + "_"
+                    + mappingConfig.getKuduMapping().getDatabase() + "-" + mappingConfig.getKuduMapping().getTable();
+            } else {
+                k = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "_"
+                    + mappingConfig.getKuduMapping().getDatabase() + "-" + mappingConfig.getKuduMapping().getTable();
+            }
+            Map<String, KuduMappingConfig> configMap = mappingConfigCache.computeIfAbsent(k,
+                k1 -> new ConcurrentHashMap<>());
+            configMap.put(configName, mappingConfig);
+        }
+        Map<String, String> properties = configuration.getProperties();
+
+        String kudu_master = properties.get("kudu.master.address");
+        kuduTemplate = new KuduTemplate(kudu_master);
+        kuduSyncService = new KuduSyncService(kuduTemplate);
+
+        kuduConfigMonitor = new KuduConfigMonitor();
+        kuduConfigMonitor.init(this, envProperties);
+    }
+
+    @Override
+    public void sync(List<Dml> dmls) {
+        if (dmls == null || dmls.isEmpty()) {
+            return;
+        }
+        for (Dml dml : dmls) {
+            if (dml == null) {
+                return;
+            }
+            String destination = StringUtils.trimToEmpty(dml.getDestination());
+            String groupId = StringUtils.trimToEmpty(dml.getGroupId());
+            String database = dml.getDatabase();
+            String table = dml.getTable();
+            Map<String, KuduMappingConfig> configMap;
+            if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
+                configMap = mappingConfigCache.get(destination + "-" + groupId + "_" + database + "-" + table);
+            } else {
+                configMap = mappingConfigCache.get(destination + "_" + database + "-" + table);
+            }
+            if (configMap != null) {
+                List<KuduMappingConfig> configs = new ArrayList<>();
+                configMap.values().forEach(config -> {
+                    if (StringUtils.isNotEmpty(config.getGroupId())) {
+                        if (config.getGroupId().equals(dml.getGroupId())) {
+                            configs.add(config);
+                        }
+                    } else {
+                        configs.add(config);
+                    }
+                });
+                if (!configs.isEmpty()) {
+                    configs.forEach(config -> kuduSyncService.sync(config, dml));
+                } else {
+                    logger.error("groupID didn't mach,please check your gruopId ");
+                }
+            } else {
+                logger.error("{} config didn't get,please check your map key ", destination + "_" + database + "-"
+                                                                                + table);
+            }
+        }
+    }
+
+    @Override
+    public void destroy() {
+        if (kuduConfigMonitor != null) {
+            kuduConfigMonitor.destroy();
+        }
+        // 加入kudu client 关闭钩子
+        kuduTemplate.closeKuduClient();
+    }
+
+    @Override
+    public EtlResult etl(String task, List<String> params) {
+        EtlResult etlResult = new EtlResult();
+        KuduMappingConfig config = kuduMapping.get(task);
+        KuduEtlService hbaseEtlService = new KuduEtlService(kuduTemplate, config);
+        if (config != null) {
+            return hbaseEtlService.importData(params);
+        } else {
+            StringBuilder resultMsg = new StringBuilder();
+            boolean resSucc = true;
+            for (KuduMappingConfig configTmp : kuduMapping.values()) {
+                // 取所有的destination为task的配置
+                if (configTmp.getDestination().equals(task)) {
+                    EtlResult etlRes = hbaseEtlService.importData(params);
+                    if (!etlRes.getSucceeded()) {
+                        resSucc = false;
+                        resultMsg.append(etlRes.getErrorMessage()).append("\n");
+                    } else {
+                        resultMsg.append(etlRes.getResultMessage()).append("\n");
+                    }
+                }
+            }
+            if (resultMsg.length() > 0) {
+                etlResult.setSucceeded(resSucc);
+                if (resSucc) {
+                    etlResult.setResultMessage(resultMsg.toString());
+                } else {
+                    etlResult.setErrorMessage(resultMsg.toString());
+                }
+                return etlResult;
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Map<String, Object> count(String task) {
+        Map<String, Object> res = new LinkedHashMap<>();
+        KuduMappingConfig config = kuduMapping.get(task);
+        if (config != null && config.getKuduMapping() != null) {
+            String tableName = config.getKuduMapping().getTargetTable();
+            long rowCount = kuduTemplate.countRow(tableName);
+            res.put("kuduTable", tableName);
+            res.put("count", rowCount);
+        }
+        return res;
+    }
+
+    @Override
+    public String getDestination(String task) {
+        KuduMappingConfig config = kuduMapping.get(task);
+        if (config != null && config.getKuduMapping() != null) {
+            return config.getDestination();
+        }
+        return null;
+    }
+}

client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/config/KuduMappingConfig.java

@@ -0,0 +1,200 @@
+package com.alibaba.otter.canal.client.adapter.kudu.config;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
+
+/**
+ * @author liuyadong
+ * @description kudu配置文件映射
+ */
+public class KuduMappingConfig implements AdapterConfig {
+
+    private String      dataSourceKey;     // 数据源key
+
+    private String      destination;       // canal实例或MQ的topic
+
+    private String      groupId;           // groupId
+
+    private String      outerAdapterKey;   // 对应适配器的key
+
+    private boolean     concurrent = false; // 是否并行同步
+
+    private KuduMapping kuduMapping;       // db映射配置
+
+    @Override
+    public String getDataSourceKey() {
+        return dataSourceKey;
+    }
+
+    @Override
+    public AdapterMapping getMapping() {
+        return kuduMapping;
+    }
+
+    public void setDataSourceKey(String dataSourceKey) {
+        this.dataSourceKey = dataSourceKey;
+    }
+
+    public String getGroupId() {
+        return groupId;
+    }
+
+    public void setGroupId(String groupId) {
+        this.groupId = groupId;
+    }
+
+    public String getOuterAdapterKey() {
+        return outerAdapterKey;
+    }
+
+    public void setOuterAdapterKey(String outerAdapterKey) {
+        this.outerAdapterKey = outerAdapterKey;
+    }
+
+    public boolean getConcurrent() {
+        return concurrent;
+    }
+
+    public void setConcurrent(boolean concurrent) {
+        this.concurrent = concurrent;
+    }
+
+    public KuduMapping getKuduMapping() {
+        return kuduMapping;
+    }
+
+    public void setKuduMapping(KuduMapping kuduMapping) {
+        this.kuduMapping = kuduMapping;
+    }
+
+    public String getDestination() {
+        return destination;
+    }
+
+    public void setDestination(String destination) {
+        this.destination = destination;
+    }
+
+    public void validate() {
+        if (kuduMapping.database == null || kuduMapping.database.isEmpty()) {
+            throw new NullPointerException("KuduMapping.database");
+        }
+    }
+
+    public static class KuduMapping implements AdapterMapping {
+
+        private String              database;                           // 数据库名或schema名
+        private String              table;                              // 表名
+        private Map<String, String> targetPk    = new LinkedHashMap<>(); // 目标表主键字段
+        private boolean             mapAll      = false;                // 映射所有字段
+        private String              targetDb;                           // 目标库名
+        private String              targetTable;                        // 目标表名
+        private Map<String, String> targetColumns;                      // 目标表字段映射
+
+        private String              etlCondition;                       // etl条件sql
+
+        private int                 readBatch   = 5000;
+        private int                 commitBatch = 5000;                 // etl等批量提交大小
+
+        private Map<String, String> allMapColumns;
+
+        public String getDatabase() {
+            return database;
+        }
+
+        public void setDatabase(String database) {
+            this.database = database;
+        }
+
+        public String getTable() {
+            return table;
+        }
+
+        public void setTable(String table) {
+            this.table = table;
+        }
+
+        public Map<String, String> getTargetPk() {
+            return targetPk;
+        }
+
+        public void setTargetPk(Map<String, String> targetPk) {
+            this.targetPk = targetPk;
+        }
+
+        public Boolean getMapAll() {
+            return mapAll;
+        }
+
+        public void setMapAll(Boolean mapAll) {
+            this.mapAll = mapAll;
+        }
+
+        public String getTargetDb() {
+            return targetDb;
+        }
+
+        public void setTargetDb(String targetDb) {
+            this.targetDb = targetDb;
+        }
+
+        public String getTargetTable() {
+            return targetTable;
+        }
+
+        public void setTargetTable(String targetTable) {
+            this.targetTable = targetTable;
+        }
+
+        public Map<String, String> getTargetColumns() {
+            if (targetColumns != null) {
+                targetColumns.forEach((key, value) -> {
+                    if (StringUtils.isEmpty(value)) {
+                        targetColumns.put(key, key);
+                    }
+                });
+            }
+            return targetColumns;
+        }
+
+        public void setTargetColumns(Map<String, String> targetColumns) {
+            this.targetColumns = targetColumns;
+        }
+
+        public String getEtlCondition() {
+            return etlCondition;
+        }
+
+        public void setEtlCondition(String etlCondition) {
+            this.etlCondition = etlCondition;
+        }
+
+        public int getReadBatch() {
+            return readBatch;
+        }
+
+        public void setReadBatch(int readBatch) {
+            this.readBatch = readBatch;
+        }
+
+        public int getCommitBatch() {
+            return commitBatch;
+        }
+
+        public void setCommitBatch(int commitBatch) {
+            this.commitBatch = commitBatch;
+        }
+
+        public Map<String, String> getAllMapColumns() {
+            return allMapColumns;
+        }
+
+        public void setAllMapColumns(Map<String, String> allMapColumns) {
+            this.allMapColumns = allMapColumns;
+        }
+    }
+}

client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/config/KuduMappingConfigLoader.java

@@ -0,0 +1,52 @@
+package com.alibaba.otter.canal.client.adapter.kudu.config;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
+import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
+
+/**
+ * @author liuyadong
+ * @description kudu表信息加载
+ */
+public class KuduMappingConfigLoader {
+
+    private static Logger logger = LoggerFactory.getLogger(KuduMappingConfigLoader.class);
+
+    /**
+     * 加载HBase表映射配置
+     *
+     * @return 配置名/配置文件名--对象
+     */
+    public static Map<String, KuduMappingConfig> load(Properties envProperties) {
+        logger.info("## Start loading kudu mapping config ... ");
+
+        Map<String, KuduMappingConfig> result = new LinkedHashMap<>();
+
+        Map<String, String> configContentMap = MappingConfigsLoader.loadConfigs("kudu");
+        configContentMap.forEach((fileName, content) -> {
+            KuduMappingConfig config = YmlConfigBinder.bindYmlToObj(null,
+                content,
+                KuduMappingConfig.class,
+                null,
+                envProperties);
+            if (config == null) {
+                return;
+            }
+            try {
+                config.validate();
+            } catch (Exception e) {
+                throw new RuntimeException("ERROR load Config: " + fileName + " " + e.getMessage(), e);
+            }
+            result.put(fileName, config);
+        });
+
+        logger.info("## kudu mapping config loaded");
+        return result;
+    }
+}

client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/monitor/KuduConfigMonitor.java

@@ -0,0 +1,170 @@
+package com.alibaba.otter.canal.client.adapter.kudu.monitor;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.commons.io.filefilter.FileFilterUtils;
+import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
+import org.apache.commons.io.monitor.FileAlterationMonitor;
+import org.apache.commons.io.monitor.FileAlterationObserver;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
+import com.alibaba.otter.canal.client.adapter.kudu.KuduAdapter;
+import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
+import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
+import com.alibaba.otter.canal.client.adapter.support.Util;
+
+/**
+ * @author liuyadong
+ * @description 配置文件监听
+ */
+public class KuduConfigMonitor {
+
+    private static final Logger   logger      = LoggerFactory.getLogger(KuduConfigMonitor.class);
+
+    private static final String   adapterName = "kudu";
+
+    private KuduAdapter           kuduAdapter;
+
+    private Properties            envProperties;
+
+    private FileAlterationMonitor fileMonitor;
+
+    public void init(KuduAdapter kuduAdapter, Properties envProperties) {
+        this.kuduAdapter = kuduAdapter;
+        this.envProperties = envProperties;
+        File confDir = Util.getConfDirPath(adapterName);
+        try {
+            FileAlterationObserver observer = new FileAlterationObserver(confDir,
+                FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml")));
+            FileListener listener = new FileListener();
+            observer.addListener(listener);
+            fileMonitor = new FileAlterationMonitor(3000, observer);
+            fileMonitor.start();
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+
+    /**
+     * 停止监听配置文件
+     */
+    public void destroy() {
+        try {
+            fileMonitor.stop();
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+
+    /**
+     * 配置文件监听
+     */
+    private class FileListener extends FileAlterationListenerAdaptor {
+
+        @Override
+        public void onFileCreate(File file) {
+            super.onFileCreate(file);
+
+            try {
+                // 加载新增的配置文件
+                String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName());
+                KuduMappingConfig config = YmlConfigBinder.bindYmlToObj(null,
+                    configContent,
+                    KuduMappingConfig.class,
+                    null,
+                    envProperties);
+                if (config == null) {
+                    return;
+                }
+                config.validate();
+                addConfigToCache(file, config);
+
+                logger.info("Add a new kudu mapping config: {} to canal adapter", file.getName());
+            } catch (Exception e) {
+                logger.error(e.getMessage(), e);
+            }
+        }
+
+        @Override
+        public void onFileChange(File file) {
+            super.onFileChange(file);
+
+            try {
+                if (kuduAdapter.getKuduMapping().containsKey(file.getName())) {
+                    // 加载配置文件
+                    String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator
+                                                                           + file.getName());
+                    if (configContent == null) {
+                        onFileDelete(file);
+                        return;
+                    }
+                    KuduMappingConfig config = YmlConfigBinder.bindYmlToObj(null,
+                        configContent,
+                        KuduMappingConfig.class,
+                        null,
+                        envProperties);
+                    if (config == null) {
+                        return;
+                    }
+                    config.validate();
+                    if (kuduAdapter.getKuduMapping().containsKey(file.getName())) {
+                        deleteConfigFromCache(file);
+                    }
+                    addConfigToCache(file, config);
+                }
+            } catch (Exception e) {
+                logger.error(e.getMessage(), e);
+            }
+        }
+
+        @Override
+        public void onFileDelete(File file) {
+            super.onFileDelete(file);
+
+            try {
+                if (kuduAdapter.getKuduMapping().containsKey(file.getName())) {
+                    deleteConfigFromCache(file);
+                    logger.info("Delete a hbase mapping config: {} of canal adapter", file.getName());
+                }
+            } catch (Exception e) {
+                logger.error(e.getMessage(), e);
+            }
+        }
+
+        /**
+         * 添加配置文件信息到缓存
+         *
+         * @param file
+         * @param config
+         */
+        private void addConfigToCache(File file, KuduMappingConfig config) {
+            kuduAdapter.getKuduMapping().put(file.getName(), config);
+            Map<String, KuduMappingConfig> configMap = kuduAdapter.getMappingConfigCache()
+                .computeIfAbsent(StringUtils.trimToEmpty(config.getDestination()) + "."
+                                 + config.getKuduMapping().getDatabase() + "." + config.getKuduMapping().getTable(),
+                    k1 -> new HashMap<>());
+            configMap.put(file.getName(), config);
+        }
+
+        /**
+         * 从缓存中删除配置
+         *
+         * @param file 文件
+         */
+        private void deleteConfigFromCache(File file) {
+            kuduAdapter.getKuduMapping().remove(file.getName());
+            for (Map<String, KuduMappingConfig> configMap : kuduAdapter.getMappingConfigCache().values()) {
+                if (configMap != null) {
+                    configMap.remove(file.getName());
+                }
+            }
+        }
+
+    }
+}

client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/service/KuduEtlService.java

@@ -0,0 +1,137 @@
+package com.alibaba.otter.canal.client.adapter.kudu.service;
+
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+import javax.sql.DataSource;
+
+import org.apache.kudu.client.KuduException;
+
+import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
+import com.alibaba.otter.canal.client.adapter.kudu.support.KuduTemplate;
+import com.alibaba.otter.canal.client.adapter.kudu.support.SyncUtil;
+import com.alibaba.otter.canal.client.adapter.support.AbstractEtlService;
+import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
+import com.alibaba.otter.canal.client.adapter.support.EtlResult;
+import com.alibaba.otter.canal.client.adapter.support.Util;
+import com.google.common.base.Joiner;
+
+/**
+ * @author liuyadong
+ * @description kudu 拉取历史数据
+ */
+public class KuduEtlService extends AbstractEtlService {
+
+    private KuduTemplate      kuduTemplate;
+    private KuduMappingConfig config;
+
+    public KuduEtlService(KuduTemplate kuduTemplate, KuduMappingConfig config){
+        super("kudu", config);
+        this.kuduTemplate = kuduTemplate;
+        this.config = config;
+    }
+
+    public EtlResult importData(List<String> params) {
+        EtlResult etlResult = new EtlResult();
+        List<String> errMsg = new ArrayList<>();
+
+        KuduMappingConfig.KuduMapping kuduMapping = config.getKuduMapping();
+        boolean flag = kuduTemplate.tableExists(kuduMapping.getTargetTable());
+        // 表不存在，停止导入
+        if (!flag) {
+            logger.info("{} is don't hava,please check your kudu table !", kuduMapping.getTargetTable());
+            errMsg.add(kuduMapping.getTargetTable() + " is don't hava,please check your kudu table !");
+            etlResult.setErrorMessage(Joiner.on("\n").join(errMsg));
+            return etlResult;
+        }
+        logger.info("{} etl is starting!", kuduMapping.getTargetTable());
+        String sql = "SELECT * FROM " + kuduMapping.getDatabase() + "." + kuduMapping.getTable();
+        return importData(sql, params);
+    }
+
+    @Override
+    protected boolean executeSqlImport(DataSource ds, String sql, List<Object> values,
+                                       AdapterConfig.AdapterMapping mapping, AtomicLong impCount, List<String> errMsg) {
+        KuduMappingConfig.KuduMapping kuduMapping = (KuduMappingConfig.KuduMapping) mapping;
+        // 获取字段元数据
+        Map<String, String> columnsMap = new LinkedHashMap<>();// 需要同步的字段
+
+        try {
+            Util.sqlRS(ds, "SELECT * FROM " + SyncUtil.getDbTableName(kuduMapping) + " LIMIT 1", rs -> {
+                try {
+                    ResultSetMetaData rsd = rs.getMetaData();
+                    int columnCount = rsd.getColumnCount();
+                    List<String> columns = new ArrayList<>();
+                    for (int i = 1; i <= columnCount; i++) {
+                        columns.add(rsd.getColumnName(i).toLowerCase());
+                    }
+                    columnsMap.putAll(SyncUtil.getColumnsMap(kuduMapping, columns));
+                    return true;
+                } catch (Exception e) {
+                    logger.error(e.getMessage(), e);
+                    return false;
+                }
+            });
+            // 写入数据
+            logger.info("etl select data sql is :{}", sql);
+            Util.sqlRS(ds, sql, values, rs -> {
+                int idx = 1;
+                try {
+                    List<Map<String, Object>> dataList = new ArrayList<>();
+                    while (rs.next()) {
+                        Map<String, Object> data = new HashMap<>();
+                        for (Map.Entry<String, String> entry : columnsMap.entrySet()) {
+                            String mysqlColumnName = entry.getKey();// mysql字段名
+                String kuduColumnName = entry.getValue();// kudu字段名
+                if (kuduColumnName == null) {
+                    kuduColumnName = mysqlColumnName;
+                }
+                Object value = rs.getObject(kuduColumnName);
+                if (value != null) {
+                    data.put(kuduColumnName, value);
+                } else {
+                    data.put(kuduColumnName, null);
+                }
+            }
+            dataList.add(data);
+            idx++;
+            impCount.incrementAndGet();
+            if (logger.isDebugEnabled()) {
+                logger.debug("successful import count:" + impCount.get());
+            }
+            if (idx % kuduMapping.getCommitBatch() == 0) {
+                kuduTemplate.upsert(kuduMapping.getTargetTable(), dataList);
+                dataList.clear();
+            }
+        }
+        if (!dataList.isEmpty()) {
+            kuduTemplate.upsert(kuduMapping.getTargetTable(), dataList);
+        }
+        return true;
+
+    } catch (SQLException e) {
+        e.printStackTrace();
+        logger.error(kuduMapping.getTargetTable() + " etl failed! ==>" + e.getMessage(), e);
+        errMsg.add(kuduMapping.getTargetTable() + " etl failed! ==>" + e.getMessage());
+        return false;
+    } catch (KuduException e) {
+        e.printStackTrace();
+        logger.error(kuduMapping.getTargetTable() + " etl failed! ==>" + e.getMessage(), e);
+        errMsg.add(kuduMapping.getTargetTable() + " etl failed! ==>" + e.getMessage());
+        return false;
+    }
+}           );
+            return true;
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+            return false;
+        }
+
+    }
+}

client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/service/KuduSyncService.java

@@ -0,0 +1,209 @@
+package com.alibaba.otter.canal.client.adapter.kudu.service;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.kudu.client.KuduException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.serializer.SerializerFeature;
+import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
+import com.alibaba.otter.canal.client.adapter.kudu.support.KuduTemplate;
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+
+/**
+ * @author liuyadong
+ * @description kudu实时同步
+ */
+public class KuduSyncService {
+
+    private static Logger logger = LoggerFactory.getLogger(KuduSyncService.class);
+
+    private KuduTemplate  kuduTemplate;
+
+    // 源库表字段类型缓存: instance.schema.table -> <columnName, jdbcType>
+    // private Map<String, Map<String, Integer>> columnsTypeCache = new
+    // ConcurrentHashMap<>();
+
+    public KuduSyncService(KuduTemplate kuduTemplate){
+        this.kuduTemplate = kuduTemplate;
+    }
+
+    // public Map<String, Map<String, Integer>> getColumnsTypeCache() {
+    // return columnsTypeCache;
+    // }
+
+    /**
+     * 同步事件处理
+     *
+     * @param config
+     * @param dml
+     */
+    public void sync(KuduMappingConfig config, Dml dml) {
+        if (config != null) {
+            String type = dml.getType();
+            if (type != null && type.equalsIgnoreCase("INSERT")) {
+                insert(config, dml);
+            } else if (type != null && type.equalsIgnoreCase("UPDATE")) {
+                upsert(config, dml);
+            } else if (type != null && type.equalsIgnoreCase("DELETE")) {
+                delete(config, dml);
+            }
+            if (logger.isDebugEnabled()) {
+                logger.debug("DML: {}", JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue));
+            }
+        }
+    }
+
+    /**
+     * 删除事件
+     *
+     * @param config
+     * @param dml
+     */
+    private void delete(KuduMappingConfig config, Dml dml) {
+        KuduMappingConfig.KuduMapping kuduMapping = config.getKuduMapping();
+        String configTable = kuduMapping.getTable();
+        String configDatabase = kuduMapping.getDatabase();
+        String table = dml.getTable();
+        String database = dml.getDatabase();
+        if (configTable.equals(table) && configDatabase.equals(database)) {
+            List<Map<String, Object>> data = dml.getData();
+            if (data == null || data.isEmpty()) {
+                return;
+            }
+            // 判定主键映射
+            String pkId = "";
+            Map<String, String> targetPk = kuduMapping.getTargetPk();
+            for (Map.Entry<String, String> entry : targetPk.entrySet()) {
+                String mysqlID = entry.getKey().toLowerCase();
+                String kuduID = entry.getValue();
+                if (kuduID == null) {
+                    pkId = mysqlID;
+                } else {
+                    pkId = kuduID;
+                }
+            }
+            // 切割联合主键
+            List<String> pkIds = Arrays.asList(pkId.split(","));
+            try {
+                int idx = 1;
+                boolean completed = false;
+                List<Map<String, Object>> dataList = new ArrayList<>();
+
+                for (Map<String, Object> item : data) {
+                    Map<String, Object> primaryKeyMap = new HashMap<>();
+                    for (Map.Entry<String, Object> entry : item.entrySet()) {
+                        String columnName = entry.getKey().toLowerCase();
+                        Object value = entry.getValue();
+                        if (pkIds.contains(columnName)) {
+                            primaryKeyMap.put(columnName, value);
+                        }
+                    }
+                    dataList.add(primaryKeyMap);
+                    idx++;
+                    if (idx % kuduMapping.getCommitBatch() == 0) {
+                        kuduTemplate.delete(kuduMapping.getTargetTable(), dataList);
+                        dataList.clear();
+                        completed = true;
+                    }
+                }
+                if (!completed) {
+                    kuduTemplate.delete(kuduMapping.getTargetTable(), dataList);
+                }
+            } catch (KuduException e) {
+                logger.error(e.getMessage());
+                logger.error("DML: {}", JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue));
+            }
+        }
+    }
+
+    /**
+     * 更新插入事件
+     *
+     * @param config
+     * @param dml
+     */
+    private void upsert(KuduMappingConfig config, Dml dml) {
+        KuduMappingConfig.KuduMapping kuduMapping = config.getKuduMapping();
+        String configTable = kuduMapping.getTable();
+        String configDatabase = kuduMapping.getDatabase();
+        String table = dml.getTable();
+        String database = dml.getDatabase();
+        if (configTable.equals(table) && configDatabase.equals(database)) {
+            List<Map<String, Object>> data = dml.getData();
+            if (data == null || data.isEmpty()) {
+                return;
+            }
+            try {
+                int idx = 1;
+                boolean completed = false;
+                List<Map<String, Object>> dataList = new ArrayList<>();
+
+                for (Map<String, Object> entry : data) {
+                    dataList.add(entry);
+                    idx++;
+                    if (idx % kuduMapping.getCommitBatch() == 0) {
+                        kuduTemplate.upsert(kuduMapping.getTargetTable(), dataList);
+                        dataList.clear();
+                        completed = true;
+                    }
+                }
+                if (!completed) {
+                    kuduTemplate.upsert(kuduMapping.getTargetTable(), dataList);
+                }
+            } catch (KuduException e) {
+                logger.error(e.getMessage());
+                logger.error("DML: {}", JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue));
+            }
+        }
+
+    }
+
+    /**
+     * 插入事件
+     *
+     * @param config
+     * @param dml
+     */
+    private void insert(KuduMappingConfig config, Dml dml) {
+        KuduMappingConfig.KuduMapping kuduMapping = config.getKuduMapping();
+        String configTable = kuduMapping.getTable();
+        String configDatabase = kuduMapping.getDatabase();
+        String table = dml.getTable();
+        String database = dml.getDatabase();
+        if (configTable.equals(table) && configDatabase.equals(database)) {
+            List<Map<String, Object>> data = dml.getData();
+            if (data == null || data.isEmpty()) {
+                return;
+            }
+            try {
+                int idx = 1;
+                boolean completed = false;
+                List<Map<String, Object>> dataList = new ArrayList<>();
+
+                for (Map<String, Object> entry : data) {
+                    dataList.add(entry);
+                    idx++;
+                    if (idx % kuduMapping.getCommitBatch() == 0) {
+                        kuduTemplate.insert(kuduMapping.getTargetTable(), dataList);
+                        dataList.clear();
+                        completed = true;
+                    }
+                }
+                if (!completed) {
+                    kuduTemplate.insert(kuduMapping.getTargetTable(), dataList);
+                }
+            } catch (KuduException e) {
+                logger.error(e.getMessage());
+                logger.error("DML: {}", JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue));
+            }
+        }
+    }
+
+}

client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/support/KuduTemplate.java

@@ -0,0 +1,424 @@
+package com.alibaba.otter.canal.client.adapter.kudu.support;
+
+import java.io.IOException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Schema;
+import org.apache.kudu.Type;
+import org.apache.kudu.client.Delete;
+import org.apache.kudu.client.Insert;
+import org.apache.kudu.client.KuduClient;
+import org.apache.kudu.client.KuduException;
+import org.apache.kudu.client.KuduScanner;
+import org.apache.kudu.client.KuduSession;
+import org.apache.kudu.client.KuduTable;
+import org.apache.kudu.client.OperationResponse;
+import org.apache.kudu.client.PartialRow;
+import org.apache.kudu.client.SessionConfiguration;
+import org.apache.kudu.client.Upsert;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author liuyadong
+ * @description kudu 操作工具类
+ */
+public class KuduTemplate {
+
+    private Logger           logger          = LoggerFactory.getLogger(this.getClass());
+
+    private KuduClient       kuduClient;
+    private String           masters;
+
+    private final static int OPERATION_BATCH = 500;
+
+    private SimpleDateFormat sdf             = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+
+    public KuduTemplate(String master_str){
+        this.masters = master_str;
+        checkClient();
+    }
+
+    /**
+     * 检车连接
+     */
+    private void checkClient() {
+        if (kuduClient == null) {
+            // kudu master 以逗号分隔
+            List<String> masterList = Arrays.asList(masters.split(","));
+            kuduClient = new KuduClient.KuduClientBuilder(masterList).defaultOperationTimeoutMs(60000)
+                .defaultSocketReadTimeoutMs(60000)
+                .defaultAdminOperationTimeoutMs(60000)
+                .build();
+        }
+    }
+
+    /**
+     * 查询表是否存在
+     *
+     * @param tableName
+     * @return
+     */
+    public boolean tableExists(String tableName) {
+        this.checkClient();
+        try {
+            return kuduClient.tableExists(tableName);
+        } catch (KuduException e) {
+            logger.error("kudu table exists check fail,message :{}", e.getMessage());
+            return true;
+        }
+    }
+
+    /**
+     * 删除行
+     *
+     * @param tableName
+     * @param dataList
+     * @throws KuduException
+     */
+    public void delete(String tableName, List<Map<String, Object>> dataList) throws KuduException {
+        this.checkClient();
+        KuduTable kuduTable = kuduClient.openTable(tableName);
+        KuduSession session = kuduClient.newSession();
+        try {
+            session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
+            session.setMutationBufferSpace(OPERATION_BATCH);
+            // 获取元数据结构
+            Map<String, Type> metaMap = new HashMap<>();
+            Schema schema = kuduTable.getSchema();
+            for (ColumnSchema columnSchema : schema.getColumns()) {
+                String colName = columnSchema.getName().toLowerCase();
+                Type type = columnSchema.getType();
+                metaMap.put(colName, type);
+            }
+            int uncommit = 0;
+            for (Map<String, Object> data : dataList) {
+                Delete delete = kuduTable.newDelete();
+                PartialRow row = delete.getRow();
+                for (Map.Entry<String, Object> entry : data.entrySet()) {
+                    String name = entry.getKey().toLowerCase();
+                    Type type = metaMap.get(name);
+                    Object value = entry.getValue();
+                    fillRow(row, name, value, type); // 填充行数据
+                }
+                session.apply(delete);
+                // 对于手工提交, 需要buffer在未满的时候flush,这里采用了buffer一半时即提交
+                uncommit = uncommit + 1;
+                if (uncommit > OPERATION_BATCH / 3 * 2) {
+                    List<OperationResponse> delete_option = session.flush();
+                    if (delete_option.size() > 0) {
+                        OperationResponse response = delete_option.get(0);
+                        if (response.hasRowError()) {
+                            logger.error("delete row fail table name is :{} ", tableName);
+                            logger.error("error list is :{}", response.getRowError().getMessage());
+                        }
+                    }
+                    uncommit = 0;
+                }
+            }
+            List<OperationResponse> delete_option = session.flush();
+            if (delete_option.size() > 0) {
+                OperationResponse response = delete_option.get(0);
+                if (response.hasRowError()) {
+                    logger.error("delete row fail table name is :{} ", tableName);
+                    logger.error("error list is :{}", response.getRowError().getMessage());
+                }
+            }
+
+        } catch (KuduException e) {
+            logger.error("error message is :{}", dataList.toString());
+            throw e;
+        } finally {
+            if (!session.isClosed()) {
+                session.close();
+            }
+        }
+    }
+
+    /**
+     * 更新/插入字段
+     *
+     * @param tableName
+     * @param dataList
+     * @throws KuduException
+     */
+    public void upsert(String tableName, List<Map<String, Object>> dataList) throws KuduException {
+        this.checkClient();
+        KuduTable kuduTable = kuduClient.openTable(tableName);
+        KuduSession session = kuduClient.newSession();
+        try {
+            session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
+            session.setMutationBufferSpace(OPERATION_BATCH);
+            // 获取元数据结构
+            Map<String, Type> metaMap = new HashMap<>();
+            Schema schema = kuduTable.getSchema();
+            for (ColumnSchema columnSchema : schema.getColumns()) {
+                String colName = columnSchema.getName().toLowerCase();
+                Type type = columnSchema.getType();
+                metaMap.put(colName, type);
+            }
+            int uncommit = 0;
+            for (Map<String, Object> data : dataList) {
+                Upsert upsert = kuduTable.newUpsert();
+                PartialRow row = upsert.getRow();
+                for (Map.Entry<String, Object> entry : data.entrySet()) {
+                    String name = entry.getKey().toLowerCase();
+                    Type type = metaMap.get(name);
+                    Object value = entry.getValue();
+                    fillRow(row, name, value, type); // 填充行数据
+                }
+                session.apply(upsert);
+                // 对于手工提交, 需要buffer在未满的时候flush,这里采用了buffer一半时即提交
+                uncommit = uncommit + 1;
+                if (uncommit > OPERATION_BATCH / 3 * 2) {
+                    List<OperationResponse> update_option = session.flush();
+                    if (update_option.size() > 0) {
+                        OperationResponse response = update_option.get(0);
+                        if (response.hasRowError()) {
+                            logger.error("update row fail table name is :{} ", tableName);
+                            logger.error("update list is :{}", response.getRowError().getMessage());
+                        }
+                    }
+                    uncommit = 0;
+                }
+            }
+            List<OperationResponse> update_option = session.flush();
+            if (update_option.size() > 0) {
+                OperationResponse response = update_option.get(0);
+                if (response.hasRowError()) {
+                    logger.error("update row fail table name is :{} ", tableName);
+                    logger.error("update list is :{}", response.getRowError().getMessage());
+                }
+            }
+        } catch (KuduException e) {
+            logger.error("error message is :{}", dataList.toString());
+            throw e;
+        } finally {
+            if (!session.isClosed()) {
+                session.close();
+            }
+        }
+
+    }
+
+    /**
+     * 插入数据
+     *
+     * @param tableName
+     * @param dataList
+     * @throws KuduException
+     */
+    public void insert(String tableName, List<Map<String, Object>> dataList) throws KuduException {
+        this.checkClient();
+        KuduTable kuduTable = kuduClient.openTable(tableName);// 打开表
+        KuduSession session = kuduClient.newSession(); // 创建写session,kudu必须通过session写入
+        try {
+            session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH); // 采取Flush方式
+                                                                               // 手动刷新
+            session.setMutationBufferSpace(OPERATION_BATCH);
+            // 获取元数据结构
+            Map<String, Type> metaMap = new HashMap<>();
+            Schema schema = kuduTable.getSchema();
+            for (ColumnSchema columnSchema : schema.getColumns()) {
+                String colName = columnSchema.getName().toLowerCase();
+                Type type = columnSchema.getType();
+                metaMap.put(colName, type);
+            }
+            int uncommit = 0;
+            for (Map<String, Object> data : dataList) {
+                Insert insert = kuduTable.newInsert();
+                PartialRow row = insert.getRow();
+                for (Map.Entry<String, Object> entry : data.entrySet()) {
+                    String name = entry.getKey().toLowerCase();
+                    Type type = metaMap.get(name);
+                    Object value = entry.getValue();
+                    fillRow(row, name, value, type); // 填充行数据
+                }
+                session.apply(insert);
+                // 对于手工提交, 需要buffer在未满的时候flush,这里采用了buffer一半时即提交
+                uncommit = uncommit + 1;
+                if (uncommit > OPERATION_BATCH / 3 * 2) {
+                    List<OperationResponse> insert_option = session.flush();
+                    if (insert_option.size() > 0) {
+                        OperationResponse response = insert_option.get(0);
+                        if (response.hasRowError()) {
+                            logger.error("insert row fail table name is :{} ", tableName);
+                            logger.error("insert list is :{}", response.getRowError().getMessage());
+                        }
+                    }
+                    uncommit = 0;
+                }
+            }
+            List<OperationResponse> insert_option = session.flush();
+            if (insert_option.size() > 0) {
+                OperationResponse response = insert_option.get(0);
+                if (response.hasRowError()) {
+                    logger.error("insert row fail table name is :{} ", tableName);
+                    logger.error("insert list is :{}", response.getRowError().getMessage());
+                }
+            }
+        } catch (KuduException e) {
+            logger.error("error message is :{}", dataList.toString());
+            throw e;
+        } finally {
+            if (!session.isClosed()) {
+                session.close();
+            }
+        }
+
+    }
+
+    /**
+     * 统计kudu表数据
+     *
+     * @param tableName
+     * @return
+     */
+    public long countRow(String tableName) {
+        this.checkClient();
+        long rowCount = 0L;
+        try {
+            KuduTable kuduTable = kuduClient.openTable(tableName);
+            // 创建scanner扫描
+            KuduScanner scanner = kuduClient.newScannerBuilder(kuduTable).build();
+            // 遍历数据
+            while (scanner.hasMoreRows()) {
+                while (scanner.nextRows().hasNext()) {
+                    rowCount++;
+                }
+            }
+        } catch (KuduException e) {
+            e.printStackTrace();
+        }
+        return rowCount;
+    }
+
+    /**
+     * 关闭钩子
+     *
+     * @throws IOException
+     */
+    public void closeKuduClient() {
+        if (kuduClient != null) {
+            try {
+                kuduClient.close();
+            } catch (Exception e) {
+                logger.error("ShutdownHook Close KuduClient Error! error message {}", e.getMessage());
+            }
+        }
+    }
+
+    /**
+     * 封装kudu行数据
+     *
+     * @param row
+     * @param rawVal
+     * @param type
+     */
+    private void fillRow(PartialRow row, String colName, Object rawVal, Type type) {
+        String rowValue = "0";
+        if (!(rawVal == null || "".equals(rawVal))) {
+            rowValue = rawVal + "";
+        } else {
+            return;
+        }
+        if (type == null) {
+            logger.warn("got unknown type {} for column '{}'-- ignoring this column", type, colName);
+            return;
+        }
+        try {
+            switch (type) {
+                case INT8:
+                    row.addByte(colName, Byte.parseByte(rowValue));
+                    break;
+                case INT16:
+                    row.addShort(colName, Short.parseShort(rowValue));
+                    break;
+                case INT32:
+                    row.addInt(colName, Integer.parseInt(rowValue));
+                    break;
+                case INT64:
+                    row.addLong(colName, Long.parseLong(rowValue));
+                    break;
+                case BINARY:
+                    row.addBinary(colName, rowValue.getBytes());
+                    break;
+                case STRING:
+                    row.addString(colName, rowValue);
+                    break;
+                case BOOL:
+                    if (!("true".equalsIgnoreCase(rowValue) || "false".equalsIgnoreCase(rowValue))) {
+                        return;
+                    }
+                    row.addBoolean(colName, Boolean.parseBoolean(rowValue));
+                    break;
+                case FLOAT:
+                    row.addFloat(colName, Float.parseFloat(rowValue));
+                    break;
+                case DOUBLE:
+                    row.addDouble(colName, Double.parseDouble(rowValue));
+                    break;
+                case UNIXTIME_MICROS:
+                    if ("0".equals(rowValue)) {
+                        try {
+                            Date parse = sdf.parse("2099-11-11 11:11:11");
+                            row.addLong(colName, parse.getTime());
+                        } catch (ParseException e) {
+                            logger.warn("date column is null");
+                        }
+                    } else {
+                        try {
+                            Date parse = rowValue.length() > 19 ? sdf.parse(rowValue.substring(0, 19)) : sdf.parse(rowValue);
+                            row.addLong(colName, parse.getTime());
+                        } catch (ParseException e) {
+                            logger.warn("date format error, error data is :{}", rowValue);
+                            try {
+                                Date parse = sdf.parse("2099-11-11 11:11:11");
+                                row.addLong(colName, parse.getTime());
+                            } catch (ParseException ie) {
+                                logger.warn("date column is null");
+                            }
+                        }
+                    }
+                    break;
+                default:
+                    logger.warn("got unknown type {} for column '{}'-- ignoring this column", type, colName);
+            }
+        } catch (NumberFormatException e) {
+            logger.error(e.getMessage());
+            logger.error("column parse fail==> column name=>{},column type=>{},column value=>{}", colName, type, rawVal);
+        }
+    }
+
+    /**
+     * kudu数据类型映射
+     *
+     * @param
+     */
+    private Type toKuduType(String mysqlType) throws IllegalArgumentException {
+
+        switch (mysqlType) {
+            case "varchar":
+                return Type.STRING;
+            case "int":
+                return Type.INT8;
+            case "decimal":
+                return Type.DOUBLE;
+            case "double":
+                return Type.DOUBLE;
+            case "datetime":
+                return Type.STRING;
+            case "timestamp":
+                return Type.STRING;
+            default:
+                throw new IllegalArgumentException("The provided data type doesn't map to know any known one.");
+        }
+    }
+}

client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/support/SyncUtil.java

@@ -0,0 +1,68 @@
+package com.alibaba.otter.canal.client.adapter.kudu.support;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
+
+/**
+ * @author liuyadong
+ * @description 工具
+ */
+public class SyncUtil {
+
+    /**
+     * 获取源数据库 元数据结构
+     *
+     * @param kuduMapping
+     * @return
+     */
+    public static String getDbTableName(KuduMappingConfig.KuduMapping kuduMapping) {
+        String result = "";
+        if (StringUtils.isNotEmpty(kuduMapping.getTable())) {
+            result += kuduMapping.getDatabase() + ".";
+        }
+        result += kuduMapping.getTable();
+        return result;
+    }
+
+    /**
+     * 过滤转换字段
+     *
+     * @param kuduMapping
+     * @param columns
+     * @return
+     */
+    public static Map<String, String> getColumnsMap(KuduMappingConfig.KuduMapping kuduMapping, List<String> columns) {
+        Map<String, String> columnsMap;
+        if (kuduMapping.getMapAll()) {
+            if (kuduMapping.getAllMapColumns() != null) {
+                return kuduMapping.getAllMapColumns();
+            }
+            columnsMap = new LinkedHashMap<>();
+            for (String srcColumn : columns) {
+                boolean flag = true;
+                if (kuduMapping.getTargetColumns() != null) {
+                    for (Map.Entry<String, String> entry : kuduMapping.getTargetColumns().entrySet()) {
+                        if (srcColumn.equals(entry.getValue())) {
+                            columnsMap.put(entry.getKey(), srcColumn);
+                            flag = false;
+                            break;
+                        }
+                    }
+                }
+                if (flag) {
+                    columnsMap.put(srcColumn, srcColumn);
+                }
+            }
+            kuduMapping.setAllMapColumns(columnsMap);
+        } else {
+            columnsMap = kuduMapping.getTargetColumns();
+        }
+        return columnsMap;
+    }
+
+}

client-adapter/kudu/src/main/resources/META-INF.canal/com.alibaba.otter.canal.client.adapter.OuterAdapter

@@ -0,0 +1 @@
+kudu=com.alibaba.otter.canal.client.adapter.kudu.KuduAdapter

client-adapter/kudu/src/main/resources/kudu/kudutest_user.yml

@@ -0,0 +1,19 @@
+dataSourceKey: defaultDS
+destination: test_instance
+outerAdapterKey: kudu
+groupId: g1
+kuduMapping:
+  database: test
+  table: test
+  targetTable: test
+  targetPk:
+    id: id
+  mapAll: true
+  targetColumns:
+    id:
+    name:
+    role_id:
+    c_time:
+    test1:
+  etlCondition:
+  commitBatch: 3000 # 批量提交的大小

client-adapter/kudu/src/test/java/com/alibaba/otter/canal/client/adapter/kudu/test/KuduConnectionTest.java

@@ -0,0 +1,29 @@
+package com.alibaba.otter.canal.client.adapter.kudu.test;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.kudu.client.KuduClient;
+import org.apache.kudu.client.KuduException;
+import org.apache.kudu.client.KuduTable;
+import org.junit.Test;
+
+public class KuduConnectionTest {
+
+    @Test
+    public void test01() {
+        List<String> masterList = Arrays.asList("10.6.36.102:7051,10.6.36.187:7051,10.6.36.229:7051".split(","));
+        KuduClient kuduClient = new KuduClient.KuduClientBuilder(masterList).defaultOperationTimeoutMs(60000)
+            .defaultSocketReadTimeoutMs(30000)
+            .defaultAdminOperationTimeoutMs(60000)
+            .build();
+        try {
+            List<String> tablesList = kuduClient.getTablesList().getTablesList();
+            System.out.println(tablesList.toString());
+            KuduTable web_white_list = kuduClient.openTable("web_white_list");
+
+        } catch (KuduException e) {
+            e.printStackTrace();
+        }
+    }
+}

client-adapter/kudu/src/test/java/com/alibaba/otter/canal/client/adapter/kudu/test/TestConfig.java

@@ -0,0 +1,29 @@
+package com.alibaba.otter.canal.client.adapter.kudu.test;
+
+import java.util.Map;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
+import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfigLoader;
+import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
+
+/**
+ * @description
+ */
+public class TestConfig {
+
+    @Before
+    public void before() {
+        // 加载数据源连接池
+        DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource);
+    }
+
+    @Test
+    public void testLoad() {
+        Map<String, KuduMappingConfig> configMap = KuduMappingConfigLoader.load(null);
+        Assert.assertFalse(configMap.isEmpty());
+    }
+}

client-adapter/kudu/src/test/java/com/alibaba/otter/canal/client/adapter/kudu/test/TestConstant.java

@@ -0,0 +1,37 @@
+package com.alibaba.otter.canal.client.adapter.kudu.test;
+
+import java.sql.SQLException;
+
+import com.alibaba.druid.pool.DruidDataSource;
+
+public class TestConstant {
+
+    public final static String          jdbcUrl      = "jdbc:mysql://10.0.9.5:3306/canal_manager?useUnicode=true";
+    public final static String          jdbcUser     = "axtest";
+    public final static String          jdbcPassword = "axtest123";
+
+    public final static DruidDataSource dataSource;
+
+    static {
+        dataSource = new DruidDataSource();
+        dataSource.setDriverClassName("com.mysql.jdbc.Driver");
+        dataSource.setUrl(jdbcUrl);
+        dataSource.setUsername(jdbcUser);
+        dataSource.setPassword(jdbcPassword);
+        dataSource.setInitialSize(1);
+        dataSource.setMinIdle(1);
+        dataSource.setMaxActive(1);
+        dataSource.setMaxWait(60000);
+        dataSource.setTimeBetweenEvictionRunsMillis(60000);
+        dataSource.setMinEvictableIdleTimeMillis(300000);
+        dataSource.setPoolPreparedStatements(false);
+        dataSource.setMaxPoolPreparedStatementPerConnectionSize(20);
+        dataSource.setValidationQuery("select 1");
+        try {
+            dataSource.init();
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+    }
+
+}

client-adapter/kudu/src/test/java/com/alibaba/otter/canal/client/adapter/kudu/test/sync/Common.java

@@ -0,0 +1,30 @@
+package com.alibaba.otter.canal.client.adapter.kudu.test.sync;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import com.alibaba.otter.canal.client.adapter.kudu.KuduAdapter;
+import com.alibaba.otter.canal.client.adapter.kudu.test.TestConstant;
+import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
+import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
+
+/**
+ * @description
+ */
+public class Common {
+
+    public static KuduAdapter init() {
+        DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource);
+
+        OuterAdapterConfig outerAdapterConfig = new OuterAdapterConfig();
+        outerAdapterConfig.setName("kudu");
+        outerAdapterConfig.setKey("kudu");
+        Map<String, String> properties = new HashMap<>();
+        properties.put("kudu.master.address", "10.6.36.102,10.6.36.187,10.6.36.229");
+        outerAdapterConfig.setProperties(properties);
+
+        KuduAdapter adapter = new KuduAdapter();
+        adapter.init(outerAdapterConfig, null);
+        return adapter;
+    }
+}

client-adapter/kudu/src/test/java/com/alibaba/otter/canal/client/adapter/kudu/test/sync/TestSyncKudu.java

@@ -0,0 +1,59 @@
+package com.alibaba.otter.canal.client.adapter.kudu.test.sync;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.client.adapter.kudu.KuduAdapter;
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+
+/**
+ * @description
+ */
+public class TestSyncKudu {
+
+    private KuduAdapter kuduAdapter;
+
+    @Before
+    public void init() {
+        kuduAdapter = Common.init();
+    }
+
+    @Test
+    public void testEtl() {
+        List<String> param = new ArrayList<>();
+        kuduAdapter.etl("kudutest_user.yml", param);
+    }
+
+    @Test
+    public void testCount() {
+        kuduAdapter.count("kudutest_user.yml");
+    }
+
+    @Test
+    public void testSync() {
+        Dml dml = new Dml();
+        dml.setDestination("example");
+        dml.setGroupId("g1");
+        dml.setTs(new Date().getTime());
+        dml.setType("DELETE");
+        dml.setDatabase("test1");
+        dml.setTable("user");
+        List<Map<String, Object>> dataList = new ArrayList<>();
+        Map<String, Object> data = new LinkedHashMap<>();
+        dataList.add(data);
+        data.put("id", 1L);
+        data.put("name", "liuyadong");
+        data.put("role_id", 1L);
+        data.put("c_time", new Date());
+        dml.setData(dataList);
+
+        kuduAdapter.sync(Collections.singletonList(dml));
+    }
+}

client-adapter/launcher/pom.xml

@@ -30,11 +30,6 @@
             <artifactId>client-adapter.common</artifactId>
             <version>${project.version}</version>
         </dependency>
-        <dependency>
-            <groupId>com.alibaba.otter</groupId>
-            <artifactId>canal.client</artifactId>
-            <version>${project.version}</version>
-        </dependency>
         <dependency>
             <groupId>org.yaml</groupId>
             <artifactId>snakeyaml</artifactId>
@@ -56,25 +51,6 @@
             <groupId>org.apache.curator</groupId>
             <artifactId>curator-recipes</artifactId>
         </dependency>
-        <!-- 单独引入rocketmq依赖 -->
-        <dependency>
-            <groupId>org.apache.rocketmq</groupId>
-            <artifactId>rocketmq-client</artifactId>
-        </dependency>
-        <!-- 单独引入kafka依赖 -->
-        <dependency>
-            <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka-clients</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>com.rabbitmq</groupId>
-            <artifactId>amqp-client</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>com.alibaba.mq-amqp</groupId>
-            <artifactId>mq-amqp-client</artifactId>
-        </dependency>
         <!-- jdbc -->
         <dependency>
             <groupId>mysql</groupId>
@@ -159,6 +135,59 @@
             <classifier>jar-with-dependencies</classifier>
             <scope>provided</scope>
         </dependency>
+        <!-- connector plugin -->
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>connector.tcp</artifactId>
+            <version>${project.version}</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>*</artifactId>
+                    <groupId>*</groupId>
+                </exclusion>
+            </exclusions>
+            <classifier>jar-with-dependencies</classifier>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>connector.kafka</artifactId>
+            <version>${project.version}</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>*</artifactId>
+                    <groupId>*</groupId>
+                </exclusion>
+            </exclusions>
+            <classifier>jar-with-dependencies</classifier>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>connector.rocketmq</artifactId>
+            <version>${project.version}</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>*</artifactId>
+                    <groupId>*</groupId>
+                </exclusion>
+            </exclusions>
+            <classifier>jar-with-dependencies</classifier>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>connector.rabbitmq</artifactId>
+            <version>${project.version}</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>*</artifactId>
+                    <groupId>*</groupId>
+                </exclusion>
+            </exclusions>
+            <classifier>jar-with-dependencies</classifier>
+            <scope>provided</scope>
+        </dependency>
     </dependencies>
 
     <build>

client-adapter/launcher/src/main/assembly/dev.xml

@@ -50,6 +50,13 @@
                 <include>**/*</include>
             </includes>
         </fileSet>
+        <fileSet>
+            <directory>../kudu/src/main/resources/kudu</directory>
+            <outputDirectory>/conf/kudu</outputDirectory>
+            <includes>
+                <include>**/*</include>
+            </includes>
+        </fileSet>
         <fileSet>
             <directory>../rdb/src/main/resources/</directory>
             <outputDirectory>/conf</outputDirectory>

client-adapter/launcher/src/main/assembly/release.xml

@@ -51,6 +51,13 @@
                 <include>**/*</include>
             </includes>
         </fileSet>
+        <fileSet>
+            <directory>../kudu/src/main/resources/kudu</directory>
+            <outputDirectory>/conf/kudu</outputDirectory>
+            <includes>
+                <include>**/*</include>
+            </includes>
+        </fileSet>
         <fileSet>
             <directory>../rdb/src/main/resources/</directory>
             <outputDirectory>/conf</outputDirectory>

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfigHolder.java

@@ -1,10 +1,10 @@
 package com.alibaba.otter.canal.adapter.launcher.config;
 
-import com.alibaba.otter.canal.adapter.launcher.monitor.remote.ConfigItem;
-
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
+import com.alibaba.otter.canal.adapter.launcher.monitor.remote.ConfigItem;
+
 /**
  * @author L.J.R @ 2019-09-03
  * @version 1.1.4

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/CuratorClient.java

@@ -3,6 +3,7 @@ package com.alibaba.otter.canal.adapter.launcher.config;
 import javax.annotation.PostConstruct;
 import javax.annotation.Resource;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.CuratorFrameworkFactory;
 import org.apache.curator.retry.ExponentialBackoffRetry;
@@ -24,7 +25,7 @@ public class CuratorClient {
 
     @PostConstruct
     public void init() {
-        if (adapterCanalConfig.getZookeeperHosts() != null) {
+        if (StringUtils.isNotEmpty(adapterCanalConfig.getZookeeperHosts())) {
             curator = CuratorFrameworkFactory.builder()
                 .connectString(adapterCanalConfig.getZookeeperHosts())
                 .retryPolicy(new ExponentialBackoffRetry(1000, 3))

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java

@@ -1,263 +0,0 @@
-package com.alibaba.otter.canal.adapter.launcher.loader;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.*;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch;
-import com.alibaba.otter.canal.adapter.launcher.config.SpringContext;
-import com.alibaba.otter.canal.client.CanalMQConnector;
-import com.alibaba.otter.canal.client.adapter.OuterAdapter;
-import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
-import com.alibaba.otter.canal.client.adapter.support.Dml;
-import com.alibaba.otter.canal.client.adapter.support.MessageUtil;
-import com.alibaba.otter.canal.client.adapter.support.Util;
-import com.alibaba.otter.canal.protocol.FlatMessage;
-import com.alibaba.otter.canal.protocol.Message;
-
-/**
- * 适配器工作线程抽象类
- *
- * @author rewerma 2018-8-19 下午11:30:49
- * @version 1.0.0
- */
-public abstract class AbstractCanalAdapterWorker {
-
-    protected final Logger                    logger  = LoggerFactory.getLogger(this.getClass());
-
-    protected String                          canalDestination;                                                // canal实例
-    protected String                          groupId = null;                                                  // groupId
-    protected List<List<OuterAdapter>>        canalOuterAdapters;                                              // 外部适配器
-    protected CanalClientConfig               canalClientConfig;                                               // 配置
-    protected ExecutorService                 groupInnerExecutorService;                                       // 组内工作线程池
-    protected volatile boolean                running = false;                                                 // 是否运行中
-    protected Thread                          thread  = null;
-    protected Thread.UncaughtExceptionHandler handler = (t, e) -> logger.error("parse events has an error", e);
-
-    protected SyncSwitch                      syncSwitch;
-
-    public AbstractCanalAdapterWorker(List<List<OuterAdapter>> canalOuterAdapters){
-        this.canalOuterAdapters = canalOuterAdapters;
-        this.groupInnerExecutorService = Util.newFixedThreadPool(canalOuterAdapters.size(), 5000L);
-        syncSwitch = (SyncSwitch) SpringContext.getBean(SyncSwitch.class);
-    }
-
-    protected void writeOut(final Message message) {
-        List<Future<Boolean>> futures = new ArrayList<>();
-        // 组间适配器并行运行
-        canalOuterAdapters.forEach(outerAdapters -> {
-            final List<OuterAdapter> adapters = outerAdapters;
-            futures.add(groupInnerExecutorService.submit(() -> {
-                try {
-                    // 组内适配器穿行运行，尽量不要配置组内适配器
-                    adapters.forEach(adapter -> {
-                        long begin = System.currentTimeMillis();
-                        List<Dml> dmls = MessageUtil.parse4Dml(canalDestination, groupId, message);
-                        if (dmls != null) {
-                            batchSync(dmls, adapter);
-
-                            if (logger.isDebugEnabled()) {
-                                logger.debug("{} elapsed time: {}",
-                                    adapter.getClass().getName(),
-                                    (System.currentTimeMillis() - begin));
-                            }
-                        }
-                    });
-                    return true;
-                } catch (Exception e) {
-                    logger.error(e.getMessage(), e);
-                    return false;
-                }
-            }));
-
-            // 等待所有适配器写入完成
-            // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成
-            RuntimeException exception = null;
-            for (Future<Boolean> future : futures) {
-                try {
-                    if (!future.get()) {
-                        exception = new RuntimeException("Outer adapter sync failed! ");
-                    }
-                } catch (Exception e) {
-                    exception = new RuntimeException(e);
-                }
-            }
-            if (exception != null) {
-                throw exception;
-            }
-        });
-    }
-
-    protected void writeOut(final List<FlatMessage> flatMessages) {
-        List<Future<Boolean>> futures = new ArrayList<>();
-        // 组间适配器并行运行
-        canalOuterAdapters.forEach(outerAdapters -> {
-            futures.add(groupInnerExecutorService.submit(() -> {
-                try {
-                    // 组内适配器穿行运行，尽量不要配置组内适配器
-                    outerAdapters.forEach(adapter -> {
-                        long begin = System.currentTimeMillis();
-                        List<Dml> dmls = MessageUtil.flatMessage2Dml(canalDestination, groupId, flatMessages);
-                        batchSync(dmls, adapter);
-
-                        if (logger.isDebugEnabled()) {
-                            logger.debug("{} elapsed time: {}",
-                                adapter.getClass().getName(),
-                                (System.currentTimeMillis() - begin));
-                        }
-                    });
-                    return true;
-                } catch (Exception e) {
-                    logger.error(e.getMessage(), e);
-                    return false;
-                }
-            }));
-
-            // 等待所有适配器写入完成
-            // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成
-            RuntimeException exception = null;
-            for (Future<Boolean> future : futures) {
-                try {
-                    if (!future.get()) {
-                        exception = new RuntimeException("Outer adapter sync failed! ");
-                    }
-                } catch (Exception e) {
-                    exception = new RuntimeException(e);
-                }
-            }
-            if (exception != null) {
-                throw exception;
-            }
-        });
-    }
-
-    @SuppressWarnings("unchecked")
-    protected boolean mqWriteOutData(int retry, long timeout, int i, final boolean flatMessage,
-                                     CanalMQConnector connector) {
-        ExecutorService workerExecutor =  Util.newSingleDaemonThreadExecutor(5000);
-        try {
-            List<?> messages;
-            if (!flatMessage) {
-                messages = connector.getListWithoutAck(100L, TimeUnit.MILLISECONDS);
-            } else {
-                messages = connector.getFlatListWithoutAck(100L, TimeUnit.MILLISECONDS);
-            }
-            if (messages != null && !messages.isEmpty()) {
-                Future<Boolean> future = workerExecutor.submit(() -> {
-                    if (flatMessage) {
-                        // batch write
-                        writeOut((List<FlatMessage>) messages);
-                        // FIXME xxx
-                        // messages.forEach((message ->
-                        // System.out.println(JSON.toJSONString(message))));
-                    } else {
-                        for (final Object message : messages) {
-                            writeOut((Message) message);
-                        }
-                    }
-                    return true;
-                });
-
-                try {
-                    future.get(timeout, TimeUnit.MILLISECONDS);
-                } catch (Exception e) {
-                    future.cancel(true);
-                    throw e;
-                }
-                connector.ack();
-            }
-            return true;
-        } catch (Throwable e) {
-            if (i == retry - 1) {
-                connector.ack();
-                logger.error(e.getMessage() + " Error sync but ACK!");
-                return true;
-            } else {
-                connector.rollback();
-                logger.error(e.getMessage() + " Error sync and rollback, execute times: " + (i + 1));
-            }
-            try {
-                Thread.sleep(500);
-            } catch (InterruptedException e1) {
-                // ignore
-            }
-        } finally {
-            workerExecutor.shutdown();
-        }
-        return false;
-    }
-
-    /**
-     * 分批同步
-     *
-     * @param dmls
-     * @param adapter
-     */
-    private void batchSync(List<Dml> dmls, OuterAdapter adapter) {
-        // 分批同步
-        if (dmls.size() <= canalClientConfig.getSyncBatchSize()) {
-            adapter.sync(dmls);
-        } else {
-            int len = 0;
-            List<Dml> dmlsBatch = new ArrayList<>();
-            for (Dml dml : dmls) {
-                dmlsBatch.add(dml);
-                if (dml.getData() == null || dml.getData().isEmpty()) {
-                    len += 1;
-                } else {
-                    len += dml.getData().size();
-                }
-                if (len >= canalClientConfig.getSyncBatchSize()) {
-                    adapter.sync(dmlsBatch);
-                    dmlsBatch.clear();
-                    len = 0;
-                }
-            }
-            if (!dmlsBatch.isEmpty()) {
-                adapter.sync(dmlsBatch);
-            }
-        }
-    }
-
-    public void start() {
-        if (!running) {
-            thread = new Thread(this::process);
-            thread.setUncaughtExceptionHandler(handler);
-            thread.start();
-            running = true;
-        }
-    }
-
-    protected abstract void process();
-
-    public void stop() {
-        try {
-            if (!running) {
-                return;
-            }
-
-            running = false;
-
-            syncSwitch.release(canalDestination);
-
-            logger.info("destination {} is waiting for adapters' worker thread die!", canalDestination);
-            if (thread != null) {
-                try {
-                    thread.join();
-                } catch (InterruptedException e) {
-                    // ignore
-                }
-            }
-            groupInnerExecutorService.shutdown();
-            logger.info("destination {} adapters worker thread dead!", canalDestination);
-            canalOuterAdapters.forEach(outerAdapters -> outerAdapters.forEach(OuterAdapter::destroy));
-            logger.info("destination {} all adapters destroyed!", canalDestination);
-        } catch (Exception e) {
-            logger.error(e.getMessage(), e);
-        }
-    }
-
-}

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AdapterProcessor.java

@@ -0,0 +1,268 @@
+package com.alibaba.otter.canal.adapter.launcher.loader;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch;
+import com.alibaba.otter.canal.adapter.launcher.config.SpringContext;
+import com.alibaba.otter.canal.client.adapter.OuterAdapter;
+import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+import com.alibaba.otter.canal.client.adapter.support.MessageUtil;
+import com.alibaba.otter.canal.client.adapter.support.Util;
+import com.alibaba.otter.canal.connector.core.config.CanalConstants;
+import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
+import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
+import com.alibaba.otter.canal.connector.core.spi.ExtensionLoader;
+
+/**
+ * 适配处理器
+ * 
+ * @author rewerma 2020-02-01
+ * @version 1.0.0
+ */
+public class AdapterProcessor {
+
+    private static final Logger             logger                    = LoggerFactory.getLogger(AdapterProcessor.class);
+
+    private static final String             CONNECTOR_SPI_DIR         = "/plugin";
+    private static final String             CONNECTOR_STANDBY_SPI_DIR = "/canal-adapter/plugin";
+
+    private CanalMsgConsumer                canalMsgConsumer;
+
+    private String                          canalDestination;                                                           // canal实例
+    private String                          groupId                   = null;                                           // groupId
+    private List<List<OuterAdapter>>        canalOuterAdapters;                                                         // 外部适配器
+    private CanalClientConfig               canalClientConfig;                                                          // 配置
+    private ExecutorService                 groupInnerExecutorService;                                                  // 组内工作线程池
+    private volatile boolean                running                   = false;                                          // 是否运行中
+    private Thread                          thread                    = null;
+    private Thread.UncaughtExceptionHandler handler                   = (t, e) -> logger
+        .error("parse events has an error", e);
+
+    private SyncSwitch                      syncSwitch;
+
+    public AdapterProcessor(CanalClientConfig canalClientConfig, String destination, String groupId,
+                            List<List<OuterAdapter>> canalOuterAdapters){
+        this.canalClientConfig = canalClientConfig;
+        this.canalDestination = destination;
+        this.groupId = groupId;
+        this.canalOuterAdapters = canalOuterAdapters;
+
+        this.groupInnerExecutorService = Util.newFixedThreadPool(canalOuterAdapters.size(), 5000L);
+        syncSwitch = (SyncSwitch) SpringContext.getBean(SyncSwitch.class);
+
+        // load connector consumer
+        ExtensionLoader<CanalMsgConsumer> loader = new ExtensionLoader<>(CanalMsgConsumer.class);
+        canalMsgConsumer = loader
+            .getExtension(canalClientConfig.getMode().toLowerCase(), CONNECTOR_SPI_DIR, CONNECTOR_STANDBY_SPI_DIR);
+
+        Properties properties = canalClientConfig.getConsumerProperties();
+        properties.put(CanalConstants.CANAL_MQ_FLAT_MESSAGE, canalClientConfig.getFlatMessage());
+        properties.put(CanalConstants.CANAL_ALIYUN_ACCESS_KEY, canalClientConfig.getAccessKey());
+        properties.put(CanalConstants.CANAL_ALIYUN_SECRET_KEY, canalClientConfig.getSecretKey());
+        ClassLoader cl = Thread.currentThread().getContextClassLoader();
+        Thread.currentThread().setContextClassLoader(canalMsgConsumer.getClass().getClassLoader());
+        canalMsgConsumer.init(properties, canalDestination, groupId);
+        Thread.currentThread().setContextClassLoader(cl);
+    }
+
+    public void start() {
+        if (!running) {
+            thread = new Thread(this::process);
+            thread.setUncaughtExceptionHandler(handler);
+            thread.start();
+            running = true;
+        }
+    }
+
+    public void writeOut(final List<CommonMessage> commonMessages) {
+        List<Future<Boolean>> futures = new ArrayList<>();
+        // 组间适配器并行运行
+        canalOuterAdapters.forEach(outerAdapters -> {
+            futures.add(groupInnerExecutorService.submit(() -> {
+                try {
+                    // 组内适配器穿行运行，尽量不要配置组内适配器
+                    outerAdapters.forEach(adapter -> {
+                        long begin = System.currentTimeMillis();
+                        List<Dml> dmls = MessageUtil.flatMessage2Dml(canalDestination, groupId, commonMessages);
+                        batchSync(dmls, adapter);
+
+                        if (logger.isDebugEnabled()) {
+                            logger.debug("{} elapsed time: {}",
+                                adapter.getClass().getName(),
+                                (System.currentTimeMillis() - begin));
+                        }
+                    });
+                    return true;
+                } catch (Exception e) {
+                    logger.error(e.getMessage(), e);
+                    return false;
+                }
+            }));
+
+            // 等待所有适配器写入完成
+            // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成
+            RuntimeException exception = null;
+            for (Future<Boolean> future : futures) {
+                try {
+                    if (!future.get()) {
+                        exception = new RuntimeException("Outer adapter sync failed! ");
+                    }
+                } catch (Exception e) {
+                    exception = new RuntimeException(e);
+                }
+            }
+            if (exception != null) {
+                throw exception;
+            }
+        });
+    }
+
+    /**
+     * 分批同步
+     *
+     * @param dmls
+     * @param adapter
+     */
+    private void batchSync(List<Dml> dmls, OuterAdapter adapter) {
+        // 分批同步
+        if (dmls.size() <= canalClientConfig.getSyncBatchSize()) {
+            adapter.sync(dmls);
+        } else {
+            int len = 0;
+            List<Dml> dmlsBatch = new ArrayList<>();
+            for (Dml dml : dmls) {
+                dmlsBatch.add(dml);
+                if (dml.getData() == null || dml.getData().isEmpty()) {
+                    len += 1;
+                } else {
+                    len += dml.getData().size();
+                }
+                if (len >= canalClientConfig.getSyncBatchSize()) {
+                    adapter.sync(dmlsBatch);
+                    dmlsBatch.clear();
+                    len = 0;
+                }
+            }
+            if (!dmlsBatch.isEmpty()) {
+                adapter.sync(dmlsBatch);
+            }
+        }
+    }
+
+    private void process() {
+        while (!running) { // waiting until running == true
+            while (!running) {
+                try {
+                    Thread.sleep(1000);
+                } catch (InterruptedException e) {
+                }
+            }
+        }
+
+        int retry = canalClientConfig.getRetries() == null
+                    || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
+        if (retry == -1) {
+            // 重试次数-1代表异常时一直阻塞重试
+            retry = Integer.MAX_VALUE;
+        }
+
+        while (running) {
+            try {
+                syncSwitch.get(canalDestination);
+
+                logger.info("=============> Start to connect destination: {} <=============", this.canalDestination);
+                canalMsgConsumer.connect();
+                logger.info("=============> Subscribe destination: {} succeed <=============", this.canalDestination);
+                while (running) {
+                    try {
+                        syncSwitch.get(canalDestination, 1L, TimeUnit.MINUTES);
+                    } catch (TimeoutException e) {
+                        break;
+                    }
+                    if (!running) {
+                        break;
+                    }
+
+                    for (int i = 0; i < retry; i++) {
+                        if (!running) {
+                            break;
+                        }
+                        try {
+                            if (logger.isDebugEnabled()) {
+                                logger.debug("destination: {} ", canalDestination);
+                            }
+                            long begin = System.currentTimeMillis();
+                            List<CommonMessage> commonMessages = canalMsgConsumer
+                                .getMessage(this.canalClientConfig.getTimeout(), TimeUnit.MILLISECONDS);
+                            writeOut(commonMessages);
+                            canalMsgConsumer.ack();
+                            if (logger.isDebugEnabled()) {
+                                logger.debug("destination: {} elapsed time: {} ms",
+                                    canalDestination,
+                                    System.currentTimeMillis() - begin);
+                            }
+                        } catch (Exception e) {
+                            if (i != retry - 1) {
+                                canalMsgConsumer.rollback(); // 处理失败, 回滚数据
+                                logger.error(e.getMessage() + " Error sync and rollback, execute times: " + (i + 1));
+                            } else {
+                                canalMsgConsumer.ack();
+                                logger.error(e.getMessage() + " Error sync but ACK!");
+                            }
+                            Thread.sleep(500);
+                        }
+                    }
+                }
+
+                canalMsgConsumer.disconnect();
+            } catch (Throwable e) {
+                logger.error("process error!", e);
+            }
+
+            if (running) { // is reconnect
+                try {
+                    Thread.sleep(1000);
+                } catch (InterruptedException e) {
+                    // ignore
+                }
+            }
+        }
+    }
+
+    public void stop() {
+        try {
+            if (!running) {
+                return;
+            }
+
+            running = false;
+
+            syncSwitch.release(canalDestination);
+
+            logger.info("destination {} is waiting for adapters' worker thread die!", canalDestination);
+            if (thread != null) {
+                try {
+                    thread.join();
+                } catch (InterruptedException e) {
+                    // ignore
+                }
+            }
+            groupInnerExecutorService.shutdown();
+            logger.info("destination {} adapters worker thread dead!", canalDestination);
+            canalOuterAdapters.forEach(outerAdapters -> outerAdapters.forEach(OuterAdapter::destroy));
+            logger.info("destination {} all adapters destroyed!", canalDestination);
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+}

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java

@@ -1,96 +0,0 @@
-package com.alibaba.otter.canal.adapter.launcher.loader;
-
-import java.util.List;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.kafka.common.errors.WakeupException;
-
-import com.alibaba.otter.canal.client.adapter.OuterAdapter;
-import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
-import com.alibaba.otter.canal.client.adapter.support.Util;
-import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector;
-
-/**
- * kafka对应的client适配器工作线程
- *
- * @author rewerma 2018-8-19 下午11:30:49
- * @version 1.0.0
- */
-public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
-
-    private KafkaCanalConnector connector;
-    private String              topic;
-    private boolean             flatMessage;
-
-    public CanalAdapterKafkaWorker(CanalClientConfig canalClientConfig, String bootstrapServers, String topic,
-                                   String groupId, List<List<OuterAdapter>> canalOuterAdapters, boolean flatMessage){
-        super(canalOuterAdapters);
-        this.canalClientConfig = canalClientConfig;
-        this.topic = topic;
-        super.canalDestination = topic;
-        super.groupId = groupId;
-        this.flatMessage = flatMessage;
-        this.connector = new KafkaCanalConnector(bootstrapServers,
-            topic,
-            null,
-            groupId,
-            canalClientConfig.getBatchSize(),
-            flatMessage);
-        connector.setSessionTimeout(30L, TimeUnit.SECONDS);
-    }
-
-    @Override
-    protected void process() {
-        while (!running) {
-            try {
-                Thread.sleep(500);
-            } catch (InterruptedException e) {
-                // ignore
-            }
-        }
-        int retry = canalClientConfig.getRetries() == null
-                    || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
-        long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒
-
-        while (running) {
-            try {
-                syncSwitch.get(canalDestination);
-                logger.info("=============> Start to connect topic: {} <=============", this.topic);
-                connector.connect();
-                logger.info("=============> Start to subscribe topic: {} <=============", this.topic);
-                connector.subscribe();
-                logger.info("=============> Subscribe topic: {} succeed <=============", this.topic);
-                while (running) {
-                    boolean status = syncSwitch.status(canalDestination);
-                    if (!status) {
-                        connector.disconnect();
-                        break;
-                    }
-                    if (retry == -1) {
-                        retry = Integer.MAX_VALUE;
-                    }
-                    for (int i = 0; i < retry; i++) {
-                        if (!running) {
-                            break;
-                        }
-                        if (mqWriteOutData(retry, timeout, i, flatMessage, connector)) {
-                            break;
-                        }
-                    }
-                }
-            } catch (Exception e) {
-                logger.error(e.getMessage(), e);
-            }
-        }
-
-
-        try {
-            connector.unsubscribe();
-        } catch (WakeupException e) {
-            // No-op. Continue process
-        }
-        connector.disconnect();
-        logger.info("=============> Disconnect topic: {} <=============", this.topic);
-    }
-}

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java

@@ -1,8 +1,5 @@
 package com.alibaba.otter.canal.adapter.launcher.loader;
 
-import java.net.InetSocketAddress;
-import java.net.SocketAddress;
-import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -33,15 +30,13 @@ import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
  */
 public class CanalAdapterLoader {
 
-    private static final Logger                     logger        = LoggerFactory.getLogger(CanalAdapterLoader.class);
+    private static final Logger           logger                 = LoggerFactory.getLogger(CanalAdapterLoader.class);
 
-    private CanalClientConfig                       canalClientConfig;
+    private CanalClientConfig             canalClientConfig;
 
-    private Map<String, CanalAdapterWorker>         canalWorkers  = new HashMap<>();
+    private Map<String, AdapterProcessor> canalAdapterProcessors = new HashMap<>();
 
-    private Map<String, AbstractCanalAdapterWorker> canalMQWorker = new HashMap<>();
-
-    private ExtensionLoader<OuterAdapter>           loader;
+    private ExtensionLoader<OuterAdapter> loader;
 
     public CanalAdapterLoader(CanalClientConfig canalClientConfig){
         this.canalClientConfig = canalClientConfig;
@@ -53,120 +48,154 @@ public class CanalAdapterLoader {
     public void init() {
         loader = ExtensionLoader.getExtensionLoader(OuterAdapter.class);
 
-        String canalServerHost = this.canalClientConfig.getCanalServerHost();
-        SocketAddress sa = null;
-        if (canalServerHost != null) {
-            String[] ipPort = canalServerHost.split(":");
-            sa = new InetSocketAddress(ipPort[0], Integer.parseInt(ipPort[1]));
-        }
-        String zkHosts = this.canalClientConfig.getZookeeperHosts();
-
-        if ("tcp".equalsIgnoreCase(canalClientConfig.getMode())) {
-            // 初始化canal-client的适配器
-            for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
+        for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
+            for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
                 List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
-
-                for (CanalClientConfig.Group connectorGroup : canalAdapter.getGroups()) {
-                    List<OuterAdapter> canalOutConnectors = new CopyOnWriteArrayList<>();
-                    for (OuterAdapterConfig c : connectorGroup.getOuterAdapters()) {
-                        loadAdapter(c, canalOutConnectors);
-                    }
-                    canalOuterAdapterGroups.add(canalOutConnectors);
+                List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
+                for (OuterAdapterConfig config : group.getOuterAdapters()) {
+                    loadAdapter(config, canalOuterAdapters);
                 }
-                CanalAdapterWorker worker;
-                if (sa != null) {
-                    worker = new CanalAdapterWorker(canalClientConfig,
-                        canalAdapter.getInstance(),
-                        sa,
-                        canalOuterAdapterGroups);
-                } else if (zkHosts != null) {
-                    worker = new CanalAdapterWorker(canalClientConfig,
-                        canalAdapter.getInstance(),
-                        zkHosts,
-                        canalOuterAdapterGroups);
-                } else {
-                    throw new RuntimeException("No canal server connector found");
-                }
-                canalWorkers.put(canalAdapter.getInstance(), worker);
-                worker.start();
-                logger.info("Start adapter for canal instance: {} succeed", canalAdapter.getInstance());
-            }
-        } else if ("kafka".equalsIgnoreCase(canalClientConfig.getMode())) {
-            // 初始化canal-client-kafka的适配器
-            for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
-                for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
-                    List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
-                    List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
-                    for (OuterAdapterConfig config : group.getOuterAdapters()) {
-                        loadAdapter(config, canalOuterAdapters);
-                    }
-                    canalOuterAdapterGroups.add(canalOuterAdapters);
+                canalOuterAdapterGroups.add(canalOuterAdapters);
 
-                    CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(canalClientConfig,
-                        canalClientConfig.getMqServers(),
+                AdapterProcessor adapterProcessor = canalAdapterProcessors.computeIfAbsent(canalAdapter.getInstance()
+                                                                                           + "|"
+                                                                                           + StringUtils.trimToEmpty(group.getGroupId()),
+                    f -> new AdapterProcessor(canalClientConfig,
                         canalAdapter.getInstance(),
                         group.getGroupId(),
-                        canalOuterAdapterGroups,
-                        canalClientConfig.getFlatMessage());
-                    canalMQWorker.put(canalAdapter.getInstance() + "-kafka-" + group.getGroupId(), canalKafkaWorker);
-                    canalKafkaWorker.start();
-                    logger.info("Start adapter for canal-client mq topic: {} succeed",
-                        canalAdapter.getInstance() + "-" + group.getGroupId());
-                }
-            }
-        } else if ("rocketMQ".equalsIgnoreCase(canalClientConfig.getMode())) {
-            // 初始化canal-client-rocketMQ的适配器
-            for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
-                for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
-                    List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
-                    List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
-                    for (OuterAdapterConfig config : group.getOuterAdapters()) {
-                        loadAdapter(config, canalOuterAdapters);
-                    }
-                    canalOuterAdapterGroups.add(canalOuterAdapters);
-                    CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig,
-                        canalClientConfig.getMqServers(),
-                        canalAdapter.getInstance(),
-                        group.getGroupId(),
-                        canalOuterAdapterGroups,
-                        canalClientConfig.getAccessKey(),
-                        canalClientConfig.getSecretKey(),
-                        canalClientConfig.getFlatMessage(),
-                        canalClientConfig.isEnableMessageTrace(),
-                        canalClientConfig.getCustomizedTraceTopic(),
-                        canalClientConfig.getAccessChannel(),
-                        canalClientConfig.getNamespace());
-                    canalMQWorker.put(canalAdapter.getInstance() + "-rocketmq-" + group.getGroupId(), rocketMQWorker);
-                    rocketMQWorker.start();
+                        canalOuterAdapterGroups));
+                adapterProcessor.start();
 
-                    logger.info("Start adapter for canal-client mq topic: {} succeed",
-                        canalAdapter.getInstance() + "-" + group.getGroupId());
-                }
+                logger.info("Start adapter for canal-client mq topic: {} succeed", canalAdapter.getInstance() + "-"
+                                                                                   + group.getGroupId());
             }
-        } else if ("rabbitMQ".equalsIgnoreCase(canalClientConfig.getMode())) {
-            // 初始化canal-client-rabbitMQ的适配器
-            for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
-                for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
-                    List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
-                    List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
-                    for (OuterAdapterConfig config : group.getOuterAdapters()) {
-                        loadAdapter(config, canalOuterAdapters);
-                    }
-                    canalOuterAdapterGroups.add(canalOuterAdapters);
-                    CanalAdapterRabbitMQWorker rabbitMQWork = new CanalAdapterRabbitMQWorker(canalClientConfig,
-                        canalOuterAdapterGroups,
-                        canalAdapter.getInstance(),
-                        group.getGroupId(),
-                        canalClientConfig.getFlatMessage());
-                    canalMQWorker.put(canalAdapter.getInstance() + "-rabbitmq-" + group.getGroupId(), rabbitMQWork);
-                    rabbitMQWork.start();
-
-                    logger.info("Start adapter for canal-client mq topic: {} succeed",
-                        canalAdapter.getInstance() + "-" + group.getGroupId());
-                }
-            }
-            // CanalAdapterRabbitMQWork
         }
+
+        // if ("tcp".equalsIgnoreCase(canalClientConfig.getMode())) {
+        // // 初始化canal-client的适配器
+        // for (CanalClientConfig.CanalAdapter canalAdapter :
+        // canalClientConfig.getCanalAdapters()) {
+        // List<List<OuterAdapter>> canalOuterAdapterGroups = new
+        // CopyOnWriteArrayList<>();
+        //
+        // for (CanalClientConfig.Group connectorGroup :
+        // canalAdapter.getGroups()) {
+        // List<OuterAdapter> canalOutConnectors = new CopyOnWriteArrayList<>();
+        // for (OuterAdapterConfig c : connectorGroup.getOuterAdapters()) {
+        // loadAdapter(c, canalOutConnectors);
+        // }
+        // canalOuterAdapterGroups.add(canalOutConnectors);
+        // }
+        // CanalAdapterWorker worker;
+        // if (StringUtils.isNotEmpty(canalServerHost)) {
+        // worker = new CanalAdapterWorker(canalClientConfig,
+        // canalAdapter.getInstance(),
+        // canalServerHost,
+        // zkHosts,
+        // canalOuterAdapterGroups);
+        // } else if (zkHosts != null) {
+        // worker = new CanalAdapterWorker(canalClientConfig,
+        // canalAdapter.getInstance(),
+        // zkHosts,
+        // canalOuterAdapterGroups);
+        // } else {
+        // throw new RuntimeException("No canal server connector found");
+        // }
+        // canalWorkers.put(canalAdapter.getInstance(), worker);
+        // worker.start();
+        // logger.info("Start adapter for canal instance: {} succeed",
+        // canalAdapter.getInstance());
+        // }
+        // } else if ("kafka".equalsIgnoreCase(canalClientConfig.getMode())) {
+        // // 初始化canal-client-kafka的适配器
+        // for (CanalClientConfig.CanalAdapter canalAdapter :
+        // canalClientConfig.getCanalAdapters()) {
+        // for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
+        // List<List<OuterAdapter>> canalOuterAdapterGroups = new
+        // CopyOnWriteArrayList<>();
+        // List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
+        // for (OuterAdapterConfig config : group.getOuterAdapters()) {
+        // loadAdapter(config, canalOuterAdapters);
+        // }
+        // canalOuterAdapterGroups.add(canalOuterAdapters);
+        //
+        // CanalAdapterKafkaWorker canalKafkaWorker = new
+        // CanalAdapterKafkaWorker(canalClientConfig,
+        // canalClientConfig.getMqServers(),
+        // canalAdapter.getInstance(),
+        // group.getGroupId(),
+        // canalOuterAdapterGroups,
+        // canalClientConfig.getFlatMessage());
+        // canalMQWorker.put(canalAdapter.getInstance() + "-kafka-" +
+        // group.getGroupId(), canalKafkaWorker);
+        // canalKafkaWorker.start();
+        // logger.info("Start adapter for canal-client mq topic: {} succeed",
+        // canalAdapter.getInstance() + "-" + group.getGroupId());
+        // }
+        // }
+        // } else if ("rocketMQ".equalsIgnoreCase(canalClientConfig.getMode()))
+        // {
+        // // 初始化canal-client-rocketMQ的适配器
+        // for (CanalClientConfig.CanalAdapter canalAdapter :
+        // canalClientConfig.getCanalAdapters()) {
+        // for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
+        // List<List<OuterAdapter>> canalOuterAdapterGroups = new
+        // CopyOnWriteArrayList<>();
+        // List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
+        // for (OuterAdapterConfig config : group.getOuterAdapters()) {
+        // loadAdapter(config, canalOuterAdapters);
+        // }
+        // canalOuterAdapterGroups.add(canalOuterAdapters);
+        // CanalAdapterRocketMQWorker rocketMQWorker = new
+        // CanalAdapterRocketMQWorker(canalClientConfig,
+        // canalClientConfig.getMqServers(),
+        // canalAdapter.getInstance(),
+        // group.getGroupId(),
+        // canalOuterAdapterGroups,
+        // canalClientConfig.getAccessKey(),
+        // canalClientConfig.getSecretKey(),
+        // canalClientConfig.getFlatMessage(),
+        // canalClientConfig.isEnableMessageTrace(),
+        // canalClientConfig.getCustomizedTraceTopic(),
+        // canalClientConfig.getAccessChannel(),
+        // canalClientConfig.getNamespace());
+        // canalMQWorker.put(canalAdapter.getInstance() + "-rocketmq-" +
+        // group.getGroupId(), rocketMQWorker);
+        // rocketMQWorker.start();
+        //
+        // logger.info("Start adapter for canal-client mq topic: {} succeed",
+        // canalAdapter.getInstance() + "-" + group.getGroupId());
+        // }
+        // }
+        // } else if ("rabbitMQ".equalsIgnoreCase(canalClientConfig.getMode()))
+        // {
+        // // 初始化canal-client-rabbitMQ的适配器
+        // for (CanalClientConfig.CanalAdapter canalAdapter :
+        // canalClientConfig.getCanalAdapters()) {
+        // for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
+        // List<List<OuterAdapter>> canalOuterAdapterGroups = new
+        // CopyOnWriteArrayList<>();
+        // List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
+        // for (OuterAdapterConfig config : group.getOuterAdapters()) {
+        // loadAdapter(config, canalOuterAdapters);
+        // }
+        // canalOuterAdapterGroups.add(canalOuterAdapters);
+        // CanalAdapterRabbitMQWorker rabbitMQWork = new
+        // CanalAdapterRabbitMQWorker(canalClientConfig,
+        // canalOuterAdapterGroups,
+        // canalAdapter.getInstance(),
+        // group.getGroupId(),
+        // canalClientConfig.getFlatMessage());
+        // canalMQWorker.put(canalAdapter.getInstance() + "-rabbitmq-" +
+        // group.getGroupId(), rabbitMQWork);
+        // rabbitMQWork.start();
+        //
+        // logger.info("Start adapter for canal-client mq topic: {} succeed",
+        // canalAdapter.getInstance() + "-" + group.getGroupId());
+        // }
+        // }
+        // // CanalAdapterRabbitMQWork
+        // }
     }
 
     private void loadAdapter(OuterAdapterConfig config, List<OuterAdapter> canalOutConnectors) {
@@ -206,10 +235,10 @@ public class CanalAdapterLoader {
      * 销毁所有适配器 为防止canal实例太多造成销毁阻塞, 并行销毁
      */
     public void destroy() {
-        if (!canalWorkers.isEmpty()) {
-            ExecutorService stopExecutorService = Executors.newFixedThreadPool(canalWorkers.size());
-            for (CanalAdapterWorker canalAdapterWorker : canalWorkers.values()) {
-                stopExecutorService.execute(canalAdapterWorker::stop);
+        if (!canalAdapterProcessors.isEmpty()) {
+            ExecutorService stopExecutorService = Executors.newFixedThreadPool(canalAdapterProcessors.size());
+            for (AdapterProcessor adapterProcessor : canalAdapterProcessors.values()) {
+                stopExecutorService.execute(adapterProcessor::stop);
             }
             stopExecutorService.shutdown();
             try {
@@ -220,21 +249,6 @@ public class CanalAdapterLoader {
                 // ignore
             }
         }
-
-        if (!canalMQWorker.isEmpty()) {
-            ExecutorService stopMQWorkerService = Executors.newFixedThreadPool(canalMQWorker.size());
-            for (AbstractCanalAdapterWorker canalAdapterMQWorker : canalMQWorker.values()) {
-                stopMQWorkerService.execute(canalAdapterMQWorker::stop);
-            }
-            stopMQWorkerService.shutdown();
-            try {
-                while (!stopMQWorkerService.awaitTermination(1, TimeUnit.SECONDS)) {
-                    // ignore
-                }
-            } catch (InterruptedException e) {
-                // ignore
-            }
-        }
         logger.info("All canal adapters destroyed");
     }
 }

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRabbitMQWorker.java

@@ -1,90 +0,0 @@
-package com.alibaba.otter.canal.adapter.launcher.loader;
-
-import com.alibaba.otter.canal.client.adapter.OuterAdapter;
-import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
-import com.alibaba.otter.canal.client.adapter.support.Util;
-import com.alibaba.otter.canal.client.rabbitmq.RabbitMQCanalConnector;
-import org.apache.kafka.common.errors.WakeupException;
-
-import java.util.List;
-import java.util.concurrent.ExecutorService;
-
-public class CanalAdapterRabbitMQWorker extends AbstractCanalAdapterWorker {
-
-    private RabbitMQCanalConnector connector;
-    private String                 topic;
-    private boolean                flatMessage;
-
-    public CanalAdapterRabbitMQWorker(CanalClientConfig canalClientConfig, List<List<OuterAdapter>> canalOuterAdapters,
-                                      String topic, String groupId, boolean flatMessage){
-        super(canalOuterAdapters);
-        this.canalClientConfig = canalClientConfig;
-        this.topic = topic;
-        this.flatMessage = flatMessage;
-        this.canalDestination = topic;
-        this.groupId = groupId;
-        this.connector = new RabbitMQCanalConnector(canalClientConfig.getMqServers(),
-            canalClientConfig.getVhost(),
-            topic,
-            canalClientConfig.getAccessKey(),
-            canalClientConfig.getSecretKey(),
-            canalClientConfig.getUsername(),
-            canalClientConfig.getPassword(),
-            canalClientConfig.getResourceOwnerId(),
-            flatMessage);
-    }
-
-    @Override
-    protected void process() {
-        while (!running) {
-            try {
-                Thread.sleep(1000);
-            } catch (InterruptedException ignored) {
-            }
-        }
-
-        int retry = canalClientConfig.getRetries() == null
-                    || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
-        long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒
-        while (running) {
-            try {
-                syncSwitch.get(canalDestination);
-                logger.info("=============> Start to connect topic: {} <=============", this.topic);
-                connector.connect();
-                logger.info("=============> Start to subscribe topic: {}<=============", this.topic);
-                connector.subscribe();
-                logger.info("=============> Subscribe topic: {} succeed<=============", this.topic);
-                while (running) {
-                    boolean status = syncSwitch.status(canalDestination);
-                    if (!status) {
-                        connector.disconnect();
-                        break;
-                    }
-                    if (retry == -1) {
-                        retry = Integer.MAX_VALUE;
-                    }
-                    for (int i = 0; i < retry; i++) {
-                        if (!running) {
-                            break;
-                        }
-                        if (mqWriteOutData(retry, timeout, i, flatMessage, connector)) {
-                            break;
-                        }
-                    }
-                }
-            } catch (Exception e) {
-                logger.error(e.getMessage(), e);
-            }
-        }
-
-
-        try {
-            connector.unsubscribe();
-        } catch (WakeupException e) {
-            // No-op. Continue process
-        }
-        connector.disconnect();
-        logger.info("=============> Disconnect topic: {} <=============", this.topic);
-
-    }
-}

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java

@@ -1,119 +0,0 @@
-package com.alibaba.otter.canal.adapter.launcher.loader;
-
-import java.util.List;
-import java.util.concurrent.ExecutorService;
-
-import org.apache.kafka.common.errors.WakeupException;
-
-import com.alibaba.otter.canal.client.adapter.OuterAdapter;
-import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
-import com.alibaba.otter.canal.client.adapter.support.Util;
-import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnector;
-
-/**
- * rocketmq对应的client适配器工作线程
- *
- * @version 1.0.0
- */
-public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker {
-
-    private RocketMQCanalConnector connector;
-    private String                 topic;
-    private boolean                flatMessage;
-
-    public CanalAdapterRocketMQWorker(CanalClientConfig canalClientConfig, String nameServers, String topic,
-                                      String groupId, List<List<OuterAdapter>> canalOuterAdapters, String accessKey,
-                                      String secretKey, boolean flatMessage){
-        super(canalOuterAdapters);
-        this.canalClientConfig = canalClientConfig;
-        this.topic = topic;
-        this.flatMessage = flatMessage;
-        super.canalDestination = topic;
-        super.groupId = groupId;
-        this.connector = new RocketMQCanalConnector(nameServers,
-            topic,
-            groupId,
-            accessKey,
-            secretKey,
-            canalClientConfig.getBatchSize(),
-            flatMessage);
-        logger.info("RocketMQ consumer config topic:{}, nameServer:{}, groupId:{}", topic, nameServers, groupId);
-    }
-
-    public CanalAdapterRocketMQWorker(CanalClientConfig canalClientConfig, String nameServers, String topic,
-        String groupId, List<List<OuterAdapter>> canalOuterAdapters, String accessKey,
-        String secretKey, boolean flatMessage, boolean enableMessageTrace,
-        String customizedTraceTopic, String accessChannel, String namespace) {
-        super(canalOuterAdapters);
-        this.canalClientConfig = canalClientConfig;
-        this.topic = topic;
-        this.flatMessage = flatMessage;
-        super.canalDestination = topic;
-        super.groupId = groupId;
-        this.connector = new RocketMQCanalConnector(nameServers,
-            topic,
-            groupId,
-            accessKey,
-            secretKey,
-            canalClientConfig.getBatchSize(),
-            flatMessage,
-            enableMessageTrace,
-            customizedTraceTopic,
-            accessChannel,
-            namespace);
-        logger.info("RocketMQ consumer config topic:{}, nameServer:{}, groupId:{}", topic, nameServers, groupId);
-    }
-
-    @Override
-    protected void process() {
-        while (!running) {
-            try {
-                Thread.sleep(1000);
-            } catch (InterruptedException e) {
-            }
-        }
-
-        int retry = canalClientConfig.getRetries() == null
-                    || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
-        long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒
-
-        while (running) {
-            try {
-                syncSwitch.get(canalDestination);
-                logger.info("=============> Start to connect topic: {} <=============", this.topic);
-                connector.connect();
-                logger.info("=============> Start to subscribe topic: {}<=============", this.topic);
-                connector.subscribe();
-                logger.info("=============> Subscribe topic: {} succeed<=============", this.topic);
-                while (running) {
-                    boolean status = syncSwitch.status(canalDestination);
-                    if (!status) {
-                        connector.disconnect();
-                        break;
-                    }
-                    if (retry == -1) {
-                        retry = Integer.MAX_VALUE;
-                    }
-                    for (int i = 0; i < retry; i++) {
-                        if (!running) {
-                            break;
-                        }
-                        if (mqWriteOutData(retry, timeout, i, flatMessage, connector)) {
-                            break;
-                        }
-                    }
-                }
-            } catch (Exception e) {
-                logger.error(e.getMessage(), e);
-            }
-        }
-
-        try {
-            connector.unsubscribe();
-        } catch (WakeupException e) {
-            // No-op. Continue process
-        }
-        connector.disconnect();
-        logger.info("=============> Disconnect topic: {} <=============", this.topic);
-    }
-}

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java

@@ -1,193 +0,0 @@
-package com.alibaba.otter.canal.adapter.launcher.loader;
-
-import java.net.SocketAddress;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-import com.alibaba.otter.canal.client.CanalConnector;
-import com.alibaba.otter.canal.client.CanalConnectors;
-import com.alibaba.otter.canal.client.adapter.OuterAdapter;
-import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
-import com.alibaba.otter.canal.client.impl.ClusterCanalConnector;
-import com.alibaba.otter.canal.client.impl.SimpleCanalConnector;
-import com.alibaba.otter.canal.protocol.Message;
-
-/**
- * 原生canal-server对应的client适配器工作线程
- *
- * @author rewrema 2018-8-19 下午11:30:49
- * @version 1.0.0
- */
-public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
-
-    private static final int BATCH_SIZE = 50;
-    private static final int SO_TIMEOUT = 0;
-
-    private CanalConnector   connector;
-
-    /**
-     * 单台client适配器worker的构造方法
-     *
-     * @param canalDestination canal实例名
-     * @param address canal-server地址
-     * @param canalOuterAdapters 外部适配器组
-     */
-    public CanalAdapterWorker(CanalClientConfig canalClientConfig, String canalDestination, SocketAddress address,
-                              List<List<OuterAdapter>> canalOuterAdapters){
-        super(canalOuterAdapters);
-        this.canalClientConfig = canalClientConfig;
-        this.canalDestination = canalDestination;
-        connector = CanalConnectors.newSingleConnector(address, canalDestination, "", "");
-    }
-
-    /**
-     * HA模式下client适配器worker的构造方法
-     *
-     * @param canalDestination canal实例名
-     * @param zookeeperHosts zookeeper地址
-     * @param canalOuterAdapters 外部适配器组
-     */
-    public CanalAdapterWorker(CanalClientConfig canalClientConfig, String canalDestination, String zookeeperHosts,
-                              List<List<OuterAdapter>> canalOuterAdapters){
-        super(canalOuterAdapters);
-        this.canalDestination = canalDestination;
-        this.canalClientConfig = canalClientConfig;
-        connector = CanalConnectors.newClusterConnector(zookeeperHosts, canalDestination, "", "");
-        ((ClusterCanalConnector) connector).setSoTimeout(SO_TIMEOUT);
-    }
-
-    @Override
-    protected void process() {
-        while (!running) { // waiting until running == true
-            while (!running) {
-                try {
-                    Thread.sleep(1000);
-                } catch (InterruptedException e) {
-                }
-            }
-        }
-
-        int retry = canalClientConfig.getRetries() == null || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
-        if (retry == -1) {
-            // 重试次数-1代表异常时一直阻塞重试
-            retry = Integer.MAX_VALUE;
-        }
-        // long timeout = canalClientConfig.getTimeout() == null ? 300000 :
-        // canalClientConfig.getTimeout(); // 默认超时5分钟
-        Integer batchSize = canalClientConfig.getBatchSize();
-        if (batchSize == null) {
-            batchSize = BATCH_SIZE;
-        }
-
-        while (running) {
-            try {
-                syncSwitch.get(canalDestination);
-
-                logger.info("=============> Start to connect destination: {} <=============", this.canalDestination);
-                connector.connect();
-                logger.info("=============> Start to subscribe destination: {} <=============", this.canalDestination);
-                connector.subscribe();
-                logger.info("=============> Subscribe destination: {} succeed <=============", this.canalDestination);
-                while (running) {
-                    try {
-                        syncSwitch.get(canalDestination, 1L, TimeUnit.MINUTES);
-                    } catch (TimeoutException e) {
-                        break;
-                    }
-                    if (!running) {
-                        break;
-                    }
-
-                    for (int i = 0; i < retry; i++) {
-                        if (!running) {
-                            break;
-                        }
-                        Message message = connector.getWithoutAck(batchSize); // 获取指定数量的数据
-                        long batchId = message.getId();
-                        try {
-                            int size = message.getEntries().size();
-                            if (batchId == -1 || size == 0) {
-                                Thread.sleep(500);
-                            } else {
-                                if (logger.isDebugEnabled()) {
-                                    logger.debug("destination: {} batchId: {} batchSize: {} ",
-                                        canalDestination,
-                                        batchId,
-                                        size);
-                                }
-                                long begin = System.currentTimeMillis();
-                                writeOut(message);
-                                if (logger.isDebugEnabled()) {
-                                    logger.debug("destination: {} batchId: {} elapsed time: {} ms",
-                                        canalDestination,
-                                        batchId,
-                                        System.currentTimeMillis() - begin);
-                                }
-                            }
-                            connector.ack(batchId); // 提交确认
-                            break;
-                        } catch (Exception e) {
-                            if (i != retry - 1) {
-                                connector.rollback(batchId); // 处理失败, 回滚数据
-                                logger.error(e.getMessage() + " Error sync and rollback, execute times: " + (i + 1));
-                            } else {
-                                connector.ack(batchId);
-                                logger.error(e.getMessage() + " Error sync but ACK!");
-                            }
-                            Thread.sleep(500);
-                        }
-                    }
-                }
-
-            } catch (Throwable e) {
-                logger.error("process error!", e);
-            } finally {
-                connector.disconnect();
-                logger.info("=============> Disconnect destination: {} <=============", this.canalDestination);
-            }
-
-            if (running) { // is reconnect
-                try {
-                    Thread.sleep(1000);
-                } catch (InterruptedException e) {
-                    // ignore
-                }
-            }
-        }
-    }
-
-    @Override
-    public void stop() {
-        try {
-            if (!running) {
-                return;
-            }
-
-            if (connector instanceof ClusterCanalConnector) {
-                ((ClusterCanalConnector) connector).stopRunning();
-            } else if (connector instanceof SimpleCanalConnector) {
-                ((SimpleCanalConnector) connector).stopRunning();
-            }
-
-            running = false;
-
-            syncSwitch.release(canalDestination);
-
-            logger.info("destination {} is waiting for adapters' worker thread die!", canalDestination);
-            if (thread != null) {
-                try {
-                    thread.join();
-                } catch (InterruptedException e) {
-                    // ignore
-                }
-            }
-            groupInnerExecutorService.shutdown();
-            logger.info("destination {} adapters worker thread dead!", canalDestination);
-            canalOuterAdapters.forEach(outerAdapters -> outerAdapters.forEach(OuterAdapter::destroy));
-            logger.info("destination {} all adapters destroyed!", canalDestination);
-        } catch (Exception e) {
-            logger.error(e.getMessage(), e);
-        }
-    }
-}

client-adapter/launcher/src/main/resources/application.yml

@@ -7,20 +7,45 @@ spring:
     default-property-inclusion: non_null
 
 canal.conf:
-  mode: tcp # kafka rocketMQ
-  canalServerHost: 127.0.0.1:11111
-#  zookeeperHosts: slave1:2181
-#  mqServers: 127.0.0.1:9092 #or rocketmq
-#  flatMessage: true
-  batchSize: 500
+  mode: tcp #tcp kafka rocketMQ rabbitMQ
+  flatMessage: true
+  zookeeperHosts:
   syncBatchSize: 1000
   retries: 0
   timeout:
   accessKey:
   secretKey:
-  username:
-  password:
-  vhost:
+  consumerProperties:
+    # canal tcp consumer
+    canal.tcp.server.host: 127.0.0.1:11111
+    canal.tcp.zookeeper.hosts:
+    canal.tcp.batch.size: 500
+    canal.tcp.username:
+    canal.tcp.password:
+    # kafka consumer
+    kafka.bootstrap.servers: 127.0.0.1:9092
+    kafka.enable.auto.commit: false
+    kafka.auto.commit.interval.ms: 1000
+    kafka.auto.offset.reset: latest
+    kafka.request.timeout.ms: 40000
+    kafka.session.timeout.ms: 30000
+    kafka.isolation.level: read_committed
+    kafka.max.poll.records: 1000
+    # rocketMQ consumer
+    rocketmq.namespace:
+    rocketmq.namesrv.addr: 127.0.0.1:9876
+    rocketmq.batch.size: 1000
+    rocketmq.enable.message.trace: false
+    rocketmq.customized.trace.topic:
+    rocketmq.access.channel:
+    rocketmq.subscribe.filter:
+    # rabbitMQ consumer
+    rabbitmq.host:
+    rabbitmq.virtual.host:
+    rabbitmq.username:
+    rabbitmq.password:
+    rabbitmq.resource.ownerId:
+
 #  srcDataSources:
 #    defaultDS:
 #      url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true
@@ -66,4 +91,7 @@ canal.conf:
 #          mode: transport # or rest
 #          # security.auth: test:123456 #  only used for rest mode
 #          cluster.name: elasticsearch
-
+#        - name: kudu
+#          key: kudu
+#          properties:
+#            kudu.master.address: 127.0.0.1 # ',' split multi address

client-adapter/pom.xml

@@ -31,6 +31,7 @@
         <module>es6x</module>
         <module>es7x</module>
         <module>escore</module>
+        <module>kudu</module>
     </modules>
 
     <licenses>
@@ -171,7 +172,7 @@
             <dependency>
                 <groupId>mysql</groupId>
                 <artifactId>mysql-connector-java</artifactId>
-                <version>5.1.47</version>
+                <version>5.1.48</version>
             </dependency>
             <dependency>
                 <groupId>org.postgresql</groupId>

client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/MappingConfig.java

@@ -3,9 +3,10 @@ package com.alibaba.otter.canal.client.adapter.rdb.config;
 import java.util.LinkedHashMap;
 import java.util.Map;
 
-import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
 import org.apache.commons.lang.StringUtils;
 
+import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
+
 /**
  * RDB表映射配置
  *
@@ -92,19 +93,21 @@ public class MappingConfig implements AdapterConfig {
 
     public static class DbMapping implements AdapterMapping {
 
-        private boolean             mirrorDb    = false;                 // 是否镜像库
-        private String              database;                            // 数据库名或schema名
-        private String              table;                               // 表名
-        private Map<String, String> targetPk    = new LinkedHashMap<>(); // 目标表主键字段
-        private boolean             mapAll      = false;                 // 映射所有字段
-        private String              targetDb;                            // 目标库名
-        private String              targetTable;                         // 目标表名
-        private Map<String, String> targetColumns;                       // 目标表字段映射
+        private boolean             mirrorDb        = false;                 // 是否镜像库
+        private String              database;                                // 数据库名或schema名
+        private String              table;                                   // 表名
+        private Map<String, String> targetPk        = new LinkedHashMap<>(); // 目标表主键字段
+        private boolean             mapAll          = false;                 // 映射所有字段
+        private String              targetDb;                                // 目标库名
+        private String              targetTable;                             // 目标表名
+        private Map<String, String> targetColumns;                           // 目标表字段映射
 
-        private String              etlCondition;                        // etl条件sql
+        private boolean             caseInsensitive = false;                 // 目标表不区分大小写，默认是否
 
-        private int                 readBatch   = 5000;
-        private int                 commitBatch = 5000;                  // etl等批量提交大小
+        private String              etlCondition;                            // etl条件sql
+
+        private int                 readBatch       = 5000;
+        private int                 commitBatch     = 5000;                  // etl等批量提交大小
 
         private Map<String, String> allMapColumns;
 
@@ -179,6 +182,14 @@ public class MappingConfig implements AdapterConfig {
             this.targetColumns = targetColumns;
         }
 
+        public boolean isCaseInsensitive() {
+            return caseInsensitive;
+        }
+
+        public void setCaseInsensitive(boolean caseInsensitive) {
+            this.caseInsensitive = caseInsensitive;
+        }
+
         public String getEtlCondition() {
             return etlCondition;
         }

client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbEtlService.java

@@ -1,6 +1,10 @@
 package com.alibaba.otter.canal.client.adapter.rdb.service;
 
-import java.sql.*;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;

client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbMirrorDbSyncService.java

@@ -91,9 +91,10 @@ public class RdbMirrorDbSyncService {
                 if (config == null) {
                     return false;
                 }
-
+                // 是否区分大小写
+                boolean caseInsensitive = config.getDbMapping().isCaseInsensitive();
                 if (config.getConcurrent()) {
-                    List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml);
+                    List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml, caseInsensitive);
                     singleDmls.forEach(singleDml -> {
                         int hash = rdbSyncService.pkHash(config.getDbMapping(), singleDml.getData());
                         RdbSyncService.SyncItem syncItem = new RdbSyncService.SyncItem(config, singleDml);
@@ -101,7 +102,7 @@ public class RdbMirrorDbSyncService {
                     });
                 } else {
                     int hash = 0;
-                    List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml);
+                    List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml, caseInsensitive);
                     singleDmls.forEach(singleDml -> {
                         RdbSyncService.SyncItem syncItem = new RdbSyncService.SyncItem(config, singleDml);
                         rdbSyncService.getDmlsPartition()[hash].add(syncItem);

client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java

@@ -119,6 +119,7 @@ public class RdbSyncService {
                             batchExecutors[j].commit();
                             return true;
                         } catch (Throwable e) {
+                            dmlsPartition[j].clear();
                             batchExecutors[j].rollback();
                             throw new RuntimeException(e);
                         }
@@ -176,8 +177,9 @@ public class RdbSyncService {
             }
 
             for (MappingConfig config : configMap.values()) {
+                boolean caseInsensitive = config.getDbMapping().isCaseInsensitive();
                 if (config.getConcurrent()) {
-                    List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml);
+                    List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml, caseInsensitive);
                     singleDmls.forEach(singleDml -> {
                         int hash = pkHash(config.getDbMapping(), singleDml.getData());
                         SyncItem syncItem = new SyncItem(config, singleDml);
@@ -185,7 +187,7 @@ public class RdbSyncService {
                     });
                 } else {
                     int hash = 0;
-                    List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml);
+                    List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml, caseInsensitive);
                     singleDmls.forEach(singleDml -> {
                         SyncItem syncItem = new SyncItem(config, singleDml);
                         dmlsPartition[hash].add(syncItem);
@@ -245,7 +247,10 @@ public class RdbSyncService {
         StringBuilder insertSql = new StringBuilder();
         insertSql.append("INSERT INTO ").append(SyncUtil.getDbTableName(dbMapping)).append(" (");
 
-        columnsMap.forEach((targetColumnName, srcColumnName) -> insertSql.append(targetColumnName).append(","));
+        columnsMap.forEach((targetColumnName, srcColumnName) -> insertSql.append("`")
+            .append(targetColumnName)
+            .append("`")
+            .append(","));
         int len = insertSql.length();
         insertSql.delete(len - 1, len).append(") VALUES (");
         int mapLen = columnsMap.size();
@@ -327,7 +332,7 @@ public class RdbSyncService {
             if (!targetColumnNames.isEmpty()) {
                 hasMatched = true;
                 for (String targetColumnName : targetColumnNames) {
-                    updateSql.append(targetColumnName).append("=?, ");
+                    updateSql.append("`").append(targetColumnName).append("`").append("=?, ");
                     Integer type = ctype.get(Util.cleanColumn(targetColumnName).toLowerCase());
                     if (type == null) {
                         throw new RuntimeException("Target column: " + targetColumnName + " not matched");
@@ -447,7 +452,7 @@ public class RdbSyncService {
             if (srcColumnName == null) {
                 srcColumnName = Util.cleanColumn(targetColumnName);
             }
-            sql.append(targetColumnName).append("=? AND ");
+            sql.append("`").append(targetColumnName).append("`").append("=? AND ");
             Integer type = ctype.get(Util.cleanColumn(targetColumnName).toLowerCase());
             if (type == null) {
                 throw new RuntimeException("Target column: " + targetColumnName + " not matched");

client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/support/SingleDml.java

@@ -4,6 +4,8 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
+import org.springframework.util.LinkedCaseInsensitiveMap;
+
 import com.alibaba.otter.canal.client.adapter.support.Dml;
 
 public class SingleDml {
@@ -63,7 +65,7 @@ public class SingleDml {
         this.old = old;
     }
 
-    public static List<SingleDml> dml2SingleDmls(Dml dml) {
+    public static List<SingleDml> dml2SingleDmls(Dml dml, boolean caseInsensitive) {
         List<SingleDml> singleDmls = new ArrayList<>();
         if (dml.getData() != null) {
             int size = dml.getData().size();
@@ -73,9 +75,17 @@ public class SingleDml {
                 singleDml.setDatabase(dml.getDatabase());
                 singleDml.setTable(dml.getTable());
                 singleDml.setType(dml.getType());
-                singleDml.setData(dml.getData().get(i));
+                Map<String, Object> data = dml.getData().get(i);
+                if (caseInsensitive) {
+                    data = toCaseInsensitiveMap(data);
+                }
+                singleDml.setData(data);
                 if (dml.getOld() != null) {
-                    singleDml.setOld(dml.getOld().get(i));
+                    Map<String, Object> oldData = dml.getOld().get(i);
+                    if (caseInsensitive) {
+                        oldData = toCaseInsensitiveMap(oldData);
+                    }
+                    singleDml.setOld(oldData);
                 }
                 singleDmls.add(singleDml);
             }
@@ -89,4 +99,14 @@ public class SingleDml {
         }
         return singleDmls;
     }
+
+    public static List<SingleDml> dml2SingleDmls(Dml dml) {
+        return dml2SingleDmls(dml, false);
+    }
+
+    private static <V> LinkedCaseInsensitiveMap<V> toCaseInsensitiveMap(Map<String, V> data) {
+        LinkedCaseInsensitiveMap map = new LinkedCaseInsensitiveMap();
+        map.putAll(data);
+        return map;
+    }
 }

client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/DBTest.java

@@ -11,6 +11,7 @@ import org.junit.Ignore;
 import org.junit.Test;
 
 import com.alibaba.druid.pool.DruidDataSource;
+
 @Ignore
 public class DBTest {
 
@@ -39,8 +40,7 @@ public class DBTest {
         Connection conn = dataSource.getConnection();
 
         conn.setAutoCommit(false);
-        PreparedStatement pstmt = conn
-            .prepareStatement("insert into user (id,name,role_id,c_time,test1,test2) values (?,?,?,?,?,?)");
+        PreparedStatement pstmt = conn.prepareStatement("insert into user (id,name,role_id,c_time,test1,test2) values (?,?,?,?,?,?)");
 
         java.util.Date now = new java.util.Date();
         for (int i = 1; i <= 10000; i++) {
@@ -67,14 +67,16 @@ public class DBTest {
         // ResultSetMetaData rsm = rs.getMetaData();
         // int cnt = rsm.getColumnCount();
         // for (int i = 1; i <= cnt; i++) {
-        // System.out.println(rsm.getColumnName(i) + " " + rsm.getColumnType(i));
+        // System.out.println(rsm.getColumnName(i) + " " +
+        // rsm.getColumnType(i));
         // }
 
         // rs.close();
         // stmt.close();
 
         // PreparedStatement pstmt = conn
-        // .prepareStatement("insert into tb_user (id,name,role_id,c_time,test1,test2)
+        // .prepareStatement("insert into tb_user
+        // (id,name,role_id,c_time,test1,test2)
         // values (?,?,?,?,?,?)");
         // pstmt.setBigDecimal(1, new BigDecimal("5"));
         // pstmt.setString(2, "test");
@@ -91,6 +93,7 @@ public class DBTest {
         dataSource.close();
     }
 
+    @SuppressWarnings("unused")
     private String clob2Str(Clob clob) {
         String content = "";
         try (Reader is = clob.getCharacterStream(); BufferedReader buff = new BufferedReader(is)) {

client/pom.xml

@@ -101,23 +101,31 @@
 			<version>${spring_version}</version>
 			<scope>test</scope>
 		</dependency>
+		<!-- 客户端要使用请单独引入mq-clients依赖 -->
 		<dependency>
 			<groupId>org.apache.rocketmq</groupId>
 			<artifactId>rocketmq-client</artifactId>
+			<version>4.5.2</version>
+			<scope>provided</scope>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.rocketmq</groupId>
 			<artifactId>rocketmq-acl</artifactId>
+			<version>4.5.2</version>
+			<scope>provided</scope>
 		</dependency>
 		<dependency>
 			<groupId>com.rabbitmq</groupId>
 			<artifactId>amqp-client</artifactId>
+			<version>5.5.0</version>
+			<scope>provided</scope>
 		</dependency>
 		<dependency>
 			<groupId>com.alibaba.mq-amqp</groupId>
 			<artifactId>mq-amqp-client</artifactId>
+			<version>1.0.3</version>
+			<scope>provided</scope>
 		</dependency>
-		<!-- 客户端要使用请单独引入kafka-clients依赖 -->
 		<dependency>
 			<groupId>org.apache.kafka</groupId>
 			<artifactId>kafka-clients</artifactId>
@@ -143,7 +151,7 @@
 					<createDependencyReducedPom>false</createDependencyReducedPom>
 					<artifactSet>
 						<includes>
-							<include>com.alibaba.otter:*</include>
+							<include>com.alibaba.otter:canal.client</include>
 							<include>com.google.guava:*</include>
 							<include>org.joda:*</include>
 							<include>com.google.common:*</include>

client/src/main/java/com/alibaba/otter/canal/client/CanalConnector.java

@@ -1,6 +1,5 @@
 package com.alibaba.otter.canal.client;
 
-import java.io.IOException;
 import java.util.concurrent.TimeUnit;
 
 import com.alibaba.otter.canal.protocol.Message;

common/src/main/java/com/google/common/collect/MigrateMap.java

@@ -1,19 +1,134 @@
 package com.google.common.collect;
 
+import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutionException;
 
 import com.google.common.base.Function;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
 
 public class MigrateMap {
 
-    @SuppressWarnings("deprecation")
-    public static <K, V> ConcurrentMap<K, V> makeComputingMap(MapMaker maker,
+    public static <K, V> ConcurrentMap<K, V> makeComputingMap(CacheBuilder<Object, Object> builder,
                                                               Function<? super K, ? extends V> computingFunction) {
-        return maker.makeComputingMap(computingFunction);
+        final Function<? super K, ? extends V> function = computingFunction;
+        LoadingCache<K, V> computingCache = builder.build(new CacheLoader<K, V>() {
+
+            @Override
+            public V load(K key) throws Exception {
+                return function.apply(key);
+            }
+        });
+
+        return new MigrateConcurrentMap<K, V>(computingCache);
     }
 
-    @SuppressWarnings("deprecation")
     public static <K, V> ConcurrentMap<K, V> makeComputingMap(Function<? super K, ? extends V> computingFunction) {
-        return new MapMaker().makeComputingMap(computingFunction);
+        return makeComputingMap(CacheBuilder.newBuilder(), computingFunction);
+    }
+
+    final static class MigrateConcurrentMap<K, V> implements ConcurrentMap<K, V> {
+
+        private final LoadingCache<K, V>  computingCache;
+
+        private final ConcurrentMap<K, V> cacheView;
+
+        MigrateConcurrentMap(LoadingCache<K, V> computingCache){
+            this.computingCache = computingCache;
+            this.cacheView = computingCache.asMap();
+        }
+
+        @Override
+        public int size() {
+            return cacheView.size();
+        }
+
+        @Override
+        public boolean isEmpty() {
+            return cacheView.isEmpty();
+        }
+
+        @Override
+        public boolean containsKey(Object key) {
+            return cacheView.containsKey(key);
+        }
+
+        @Override
+        public boolean containsValue(Object value) {
+            return cacheView.containsValue(value);
+        }
+
+        @Override
+        public V get(Object key) {
+            try {
+                return computingCache.get((K) key);
+            } catch (ExecutionException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        @Override
+        public V put(K key, V value) {
+            return cacheView.put(key, value);
+        }
+
+        @Override
+        public V remove(Object key) {
+            return cacheView.remove(key);
+        }
+
+        @Override
+        public void putAll(Map<? extends K, ? extends V> m) {
+            cacheView.putAll(m);
+        }
+
+        @Override
+        public void clear() {
+            cacheView.clear();
+        }
+
+        @Override
+        public Set<K> keySet() {
+            return cacheView.keySet();
+        }
+
+        @Override
+        public Collection<V> values() {
+            return cacheView.values();
+        }
+
+        @Override
+        public Set<Entry<K, V>> entrySet() {
+            return cacheView.entrySet();
+        }
+
+        @Override
+        public V putIfAbsent(K key, V value) {
+            return cacheView.putIfAbsent(key, value);
+        }
+
+        @Override
+        public boolean remove(Object key, Object value) {
+            return cacheView.remove(key, value);
+        }
+
+        @Override
+        public boolean replace(K key, V oldValue, V newValue) {
+            return cacheView.replace(key, oldValue, newValue);
+        }
+
+        @Override
+        public V replace(K key, V value) {
+            return cacheView.replace(key, value);
+        }
+
+        @Override
+        public String toString() {
+            return cacheView.toString();
+        }
     }
 }

connector/core/pom.xml

@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>canal.connector</artifactId>
+        <groupId>com.alibaba.otter</groupId>
+        <version>1.1.5-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.alibaba.otter</groupId>
+    <artifactId>connector.core</artifactId>
+    <packaging>jar</packaging>
+    <name>canal connector core module for otter ${project.version}</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>canal.protocol</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.googlecode.aviator</groupId>
+            <artifactId>aviator</artifactId>
+            <version>2.2.1</version>
+        </dependency>
+        <dependency>
+            <groupId>oro</groupId>
+            <artifactId>oro</artifactId>
+            <version>2.0.8</version>
+        </dependency>
+        <dependency>
+            <groupId>joda-time</groupId>
+            <artifactId>joda-time</artifactId>
+            <version>2.9.4</version>
+        </dependency>
+    </dependencies>
+
+</project>

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/config/CanalConstants.java

@@ -0,0 +1,28 @@
+package com.alibaba.otter.canal.connector.core.config;
+
+/**
+ * MQ配置常量
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public class CanalConstants {
+
+    public static final String ROOT                           = "canal";
+
+    public static final String CANAL_FILTER_TRANSACTION_ENTRY = ROOT + "." + "instance.filter.transaction.entry";
+
+    public static final String CANAL_MQ_ACCESS_CHANNEL        = ROOT + "." + "mq.accessChannel";
+    public static final String CANAL_MQ_CANAL_BATCH_SIZE      = ROOT + "." + "mq.canalBatchSize";
+    public static final String CANAL_MQ_CANAL_GET_TIMEOUT     = ROOT + "." + "mq.canalGetTimeout";
+    public static final String CANAL_MQ_FLAT_MESSAGE          = ROOT + "." + "mq.flatMessage";
+
+    public static final String CANAL_MQ_DATABASE_HASH         = ROOT + "." + "mq.database.hash";
+    public static final String CANAL_MQ_BUILD_THREAD_SIZE     = ROOT + "." + "mq.build.thread.size";
+    public static final String CANAL_MQ_SEND_THREAD_SIZE      = ROOT + "." + "mq.send.thread.size";
+
+    public static final String CANAL_ALIYUN_ACCESS_KEY        = ROOT + "." + "aliyun.accessKey";
+    public static final String CANAL_ALIYUN_SECRET_KEY        = ROOT + "." + "aliyun.secretKey";
+    public static final String CANAL_ALIYUN_UID               = ROOT + "." + "aliyun.uid";
+
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/config/MQProperties.java

@@ -0,0 +1,111 @@
+package com.alibaba.otter.canal.connector.core.config;
+
+/**
+ * MQ配置类
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public class MQProperties {
+
+    private boolean flatMessage             = true;
+    private boolean databaseHash            = true;
+    private boolean filterTransactionEntry  = true;
+    private Integer parallelBuildThreadSize = 8;
+    private Integer parallelSendThreadSize  = 30;
+    private Integer fetchTimeout            = 100;
+    private Integer batchSize               = 50;
+    private String  accessChannel           = "local";
+
+    private String  aliyunAccessKey         = "";
+    private String  aliyunSecretKey         = "";
+    private int     aliyunUid               = 0;
+
+    public boolean isFlatMessage() {
+        return flatMessage;
+    }
+
+    public void setFlatMessage(boolean flatMessage) {
+        this.flatMessage = flatMessage;
+    }
+
+    public boolean isDatabaseHash() {
+        return databaseHash;
+    }
+
+    public void setDatabaseHash(boolean databaseHash) {
+        this.databaseHash = databaseHash;
+    }
+
+    public boolean isFilterTransactionEntry() {
+        return filterTransactionEntry;
+    }
+
+    public void setFilterTransactionEntry(boolean filterTransactionEntry) {
+        this.filterTransactionEntry = filterTransactionEntry;
+    }
+
+    public Integer getParallelBuildThreadSize() {
+        return parallelBuildThreadSize;
+    }
+
+    public void setParallelBuildThreadSize(Integer parallelBuildThreadSize) {
+        this.parallelBuildThreadSize = parallelBuildThreadSize;
+    }
+
+    public Integer getParallelSendThreadSize() {
+        return parallelSendThreadSize;
+    }
+
+    public void setParallelSendThreadSize(Integer parallelSendThreadSize) {
+        this.parallelSendThreadSize = parallelSendThreadSize;
+    }
+
+    public Integer getFetchTimeout() {
+        return fetchTimeout;
+    }
+
+    public void setFetchTimeout(Integer fetchTimeout) {
+        this.fetchTimeout = fetchTimeout;
+    }
+
+    public Integer getBatchSize() {
+        return batchSize;
+    }
+
+    public void setBatchSize(Integer batchSize) {
+        this.batchSize = batchSize;
+    }
+
+    public String getAccessChannel() {
+        return accessChannel;
+    }
+
+    public void setAccessChannel(String accessChannel) {
+        this.accessChannel = accessChannel;
+    }
+
+    public String getAliyunAccessKey() {
+        return aliyunAccessKey;
+    }
+
+    public void setAliyunAccessKey(String aliyunAccessKey) {
+        this.aliyunAccessKey = aliyunAccessKey;
+    }
+
+    public String getAliyunSecretKey() {
+        return aliyunSecretKey;
+    }
+
+    public void setAliyunSecretKey(String aliyunSecretKey) {
+        this.aliyunSecretKey = aliyunSecretKey;
+    }
+
+    public int getAliyunUid() {
+        return aliyunUid;
+    }
+
+    public void setAliyunUid(int aliyunUid) {
+        this.aliyunUid = aliyunUid;
+    }
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/consumer/CommonMessage.java

@@ -0,0 +1,125 @@
+package com.alibaba.otter.canal.connector.core.consumer;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+
+public class CommonMessage implements Serializable {
+
+    private static final long         serialVersionUID = 2611556444074013268L;
+
+    private String                    database;                               // 数据库或schema
+    private String                    table;                                  // 表名
+    private List<String>              pkNames;
+    private Boolean                   isDdl;
+    // 类型:INSERT/UPDATE/DELETE
+    private String                    type;
+    // binlog executeTime, 执行耗时
+    private Long                      es;
+    // dml build timeStamp, 同步时间
+    private Long                      ts;
+    // 执行的sql,dml sql为空
+    private String                    sql;
+    // 数据列表
+    private List<Map<String, Object>> data;
+    // 旧数据列表,用于update,size和data的size一一对应
+    private List<Map<String, Object>> old;
+
+    public String getDatabase() {
+        return database;
+    }
+
+    public void setDatabase(String database) {
+        this.database = database;
+    }
+
+    public String getTable() {
+        return table;
+    }
+
+    public void setTable(String table) {
+        this.table = table;
+    }
+
+    public List<String> getPkNames() {
+        return pkNames;
+    }
+
+    public void setPkNames(List<String> pkNames) {
+        this.pkNames = pkNames;
+    }
+
+    public Boolean getIsDdl() {
+        return isDdl;
+    }
+
+    public void setIsDdl(Boolean isDdl) {
+        this.isDdl = isDdl;
+    }
+
+    public String getType() {
+        return type;
+    }
+
+    public void setType(String type) {
+        this.type = type;
+    }
+
+    public Long getTs() {
+        return ts;
+    }
+
+    public void setTs(Long ts) {
+        this.ts = ts;
+    }
+
+    public String getSql() {
+        return sql;
+    }
+
+    public void setSql(String sql) {
+        this.sql = sql;
+    }
+
+    public List<Map<String, Object>> getData() {
+        return data;
+    }
+
+    public void setData(List<Map<String, Object>> data) {
+        this.data = data;
+    }
+
+    public List<Map<String, Object>> getOld() {
+        return old;
+    }
+
+    public void setOld(List<Map<String, Object>> old) {
+        this.old = old;
+    }
+
+    public Long getEs() {
+        return es;
+    }
+
+    public void setEs(Long es) {
+        this.es = es;
+    }
+
+    public void clear() {
+        database = null;
+        table = null;
+        type = null;
+        ts = null;
+        es = null;
+        data = null;
+        old = null;
+        sql = null;
+    }
+
+    @Override
+    public String toString() {
+        return "CommonMessage{" + "database='" + database + '\'' + ", table='" + table + '\'' + ", pkNames=" + pkNames
+               + ", isDdl=" + isDdl + ", type='" + type + '\'' + ", es=" + es + ", ts=" + ts + ", sql='" + sql + '\''
+               + ", data=" + data + ", old=" + old + '}';
+    }
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/AviaterRegexFilter.java

@@ -0,0 +1,131 @@
+package com.alibaba.otter.canal.connector.core.filter;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.googlecode.aviator.AviatorEvaluator;
+import com.googlecode.aviator.Expression;
+
+/**
+ * 基于aviater进行tableName正则匹配的过滤算法
+ *
+ * @author jianghang 2012-7-20 下午06:01:34
+ */
+public class AviaterRegexFilter {
+
+    private static final String             SPLIT             = ",";
+    private static final String             PATTERN_SPLIT     = "|";
+    private static final String             FILTER_EXPRESSION = "regex(pattern,target)";
+    private static final RegexFunction      regexFunction     = new RegexFunction();
+    private final Expression                exp               = AviatorEvaluator.compile(FILTER_EXPRESSION, true);
+    static {
+        AviatorEvaluator.addFunction(regexFunction);
+    }
+
+    private static final Comparator<String> COMPARATOR        = new StringComparator();
+
+    final private String                    pattern;
+    final private boolean                   defaultEmptyValue;
+
+    public AviaterRegexFilter(String pattern){
+        this(pattern, true);
+    }
+
+    public AviaterRegexFilter(String pattern, boolean defaultEmptyValue){
+        this.defaultEmptyValue = defaultEmptyValue;
+        List<String> list = null;
+        if (StringUtils.isEmpty(pattern)) {
+            list = new ArrayList<String>();
+        } else {
+            String[] ss = StringUtils.split(pattern, SPLIT);
+            list = Arrays.asList(ss);
+        }
+
+        // 对pattern按照从长到短的排序
+        // 因为 foo|foot 匹配 foot 会出错，原因是 foot 匹配了 foo 之后，会返回 foo，但是 foo 的长度和 foot
+        // 的长度不一样
+        Collections.sort(list, COMPARATOR);
+        // 对pattern进行头尾完全匹配
+        list = completionPattern(list);
+        this.pattern = StringUtils.join(list, PATTERN_SPLIT);
+    }
+
+    public boolean filter(String filtered) {
+        if (StringUtils.isEmpty(pattern)) {
+            return defaultEmptyValue;
+        }
+
+        if (StringUtils.isEmpty(filtered)) {
+            return defaultEmptyValue;
+        }
+
+        Map<String, Object> env = new HashMap<String, Object>();
+        env.put("pattern", pattern);
+        env.put("target", filtered.toLowerCase());
+        return (Boolean) exp.execute(env);
+    }
+
+    /**
+     * 修复正则表达式匹配的问题，因为使用了 oro 的 matches，会出现：
+     *
+     * <pre>
+     * foo|foot 匹配 foot 出错，原因是 foot 匹配了 foo 之后，会返回 foo，但是 foo 的长度和 foot 的长度不一样
+     * </pre>
+     *
+     * 因此此类对正则表达式进行了从长到短的排序
+     *
+     * @author zebin.xuzb 2012-10-22 下午2:02:26
+     * @version 1.0.0
+     */
+    private static class StringComparator implements Comparator<String> {
+
+        @Override
+        public int compare(String str1, String str2) {
+            if (str1.length() > str2.length()) {
+                return -1;
+            } else if (str1.length() < str2.length()) {
+                return 1;
+            } else {
+                return 0;
+            }
+        }
+    }
+
+    /**
+     * 修复正则表达式匹配的问题，即使按照长度递减排序，还是会出现以下问题：
+     *
+     * <pre>
+     * foooo|f.*t 匹配 fooooot 出错，原因是 fooooot 匹配了 foooo 之后，会将 fooo 和数据进行匹配，但是 foooo 的长度和 fooooot 的长度不一样
+     * </pre>
+     *
+     * 因此此类对正则表达式进行头尾完全匹配
+     *
+     * @author simon
+     * @version 1.0.0
+     */
+
+    private List<String> completionPattern(List<String> patterns) {
+        List<String> result = new ArrayList<String>();
+        for (String pattern : patterns) {
+            StringBuffer stringBuffer = new StringBuffer();
+            stringBuffer.append("^");
+            stringBuffer.append(pattern);
+            stringBuffer.append("$");
+            result.add(stringBuffer.toString());
+        }
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return pattern;
+    }
+
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/PatternUtils.java

@@ -0,0 +1,35 @@
+package com.alibaba.otter.canal.connector.core.filter;
+
+import java.util.Map;
+
+import org.apache.oro.text.regex.MalformedPatternException;
+import org.apache.oro.text.regex.Pattern;
+import org.apache.oro.text.regex.PatternCompiler;
+import org.apache.oro.text.regex.Perl5Compiler;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.collect.MigrateMap;
+
+public class PatternUtils {
+
+    private static Map<String, Pattern> patterns = MigrateMap.makeComputingMap(CacheBuilder.newBuilder().softValues(),
+                                                     pattern -> {
+                                                         try {
+                                                             PatternCompiler pc = new Perl5Compiler();
+                                                             return pc.compile(pattern,
+                                                                 Perl5Compiler.CASE_INSENSITIVE_MASK
+                                                                         | Perl5Compiler.READ_ONLY_MASK
+                                                                         | Perl5Compiler.SINGLELINE_MASK);
+                                                         } catch (MalformedPatternException e) {
+                                                             throw new RuntimeException(e);
+                                                         }
+                                                     });
+
+    public static Pattern getPattern(String pattern) {
+        return patterns.get(pattern);
+    }
+
+    public static void clear() {
+        patterns.clear();
+    }
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/RegexFunction.java

@@ -0,0 +1,31 @@
+package com.alibaba.otter.canal.connector.core.filter;
+
+import java.util.Map;
+
+import org.apache.oro.text.regex.Perl5Matcher;
+
+import com.googlecode.aviator.runtime.function.AbstractFunction;
+import com.googlecode.aviator.runtime.function.FunctionUtils;
+import com.googlecode.aviator.runtime.type.AviatorBoolean;
+import com.googlecode.aviator.runtime.type.AviatorObject;
+
+/**
+ * 提供aviator regex的代码扩展
+ *
+ * @author jianghang 2012-7-23 上午10:29:23
+ */
+public class RegexFunction extends AbstractFunction {
+
+    public AviatorObject call(Map<String, Object> env, AviatorObject arg1, AviatorObject arg2) {
+        String pattern = FunctionUtils.getStringValue(arg1, env);
+        String text = FunctionUtils.getStringValue(arg2, env);
+        Perl5Matcher matcher = new Perl5Matcher();
+        boolean isMatch = matcher.matches(text, PatternUtils.getPattern(pattern));
+        return AviatorBoolean.valueOf(isMatch);
+    }
+
+    public String getName() {
+        return "regex";
+    }
+
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/AbstractMQProducer.java

@@ -0,0 +1,140 @@
+package com.alibaba.otter.canal.connector.core.producer;
+
+import java.util.Properties;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
+import com.alibaba.otter.canal.connector.core.config.CanalConstants;
+import com.alibaba.otter.canal.connector.core.config.MQProperties;
+import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
+
+/**
+ * MQ producer 抽象类
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public abstract class AbstractMQProducer implements CanalMQProducer {
+
+    protected MQProperties       mqProperties;
+
+    protected ThreadPoolExecutor sendExecutor;
+    protected ThreadPoolExecutor buildExecutor;
+
+    @Override
+    public void init(Properties properties) {
+        // parse canal mq properties
+        loadCanalMqProperties(properties);
+
+        int parallelBuildThreadSize = mqProperties.getParallelBuildThreadSize();
+        buildExecutor = new ThreadPoolExecutor(parallelBuildThreadSize,
+            parallelBuildThreadSize,
+            0,
+            TimeUnit.SECONDS,
+            new ArrayBlockingQueue<Runnable>(parallelBuildThreadSize * 2),
+            new NamedThreadFactory("MQ-Parallel-Builder"),
+            new ThreadPoolExecutor.CallerRunsPolicy());
+
+        int parallelSendThreadSize = mqProperties.getParallelSendThreadSize();
+        sendExecutor = new ThreadPoolExecutor(parallelSendThreadSize,
+            parallelSendThreadSize,
+            0,
+            TimeUnit.SECONDS,
+            new ArrayBlockingQueue<Runnable>(parallelSendThreadSize * 2),
+            new NamedThreadFactory("MQ-Parallel-Sender"),
+            new ThreadPoolExecutor.CallerRunsPolicy());
+    }
+
+    @Override
+    public MQProperties getMqProperties() {
+        return this.mqProperties;
+    }
+
+    @Override
+    public void stop() {
+        if (buildExecutor != null) {
+            buildExecutor.shutdownNow();
+        }
+
+        if (sendExecutor != null) {
+            sendExecutor.shutdownNow();
+        }
+    }
+
+    /**
+     * 初始化配置
+     * <p>
+     * canal.mq.flat.message = true <br/>
+     * canal.mq.database.hash = true <br/>
+     * canal.mq.filter.transaction.entry = true <br/>
+     * canal.mq.parallel.build.thread.size = 8 <br/>
+     * canal.mq.parallel.send.thread.size = 8 <br/>
+     * canal.mq.batch.size = 50 <br/>
+     * canal.mq.timeout = 100 <br/>
+     * canal.mq.access.channel = local <br/>
+     * </p>
+     * 
+     * @param properties 总配置对象
+     */
+    private void loadCanalMqProperties(Properties properties) {
+        String flatMessage = properties.getProperty(CanalConstants.CANAL_MQ_FLAT_MESSAGE);
+        if (!StringUtils.isEmpty(flatMessage)) {
+            mqProperties.setFlatMessage(Boolean.parseBoolean(flatMessage));
+        }
+
+        String databaseHash = properties.getProperty(CanalConstants.CANAL_MQ_DATABASE_HASH);
+        if (!StringUtils.isEmpty(databaseHash)) {
+            mqProperties.setDatabaseHash(Boolean.parseBoolean(databaseHash));
+        }
+        String filterTranEntry = properties.getProperty(CanalConstants.CANAL_FILTER_TRANSACTION_ENTRY);
+        if (!StringUtils.isEmpty(filterTranEntry)) {
+            mqProperties.setFilterTransactionEntry(Boolean.parseBoolean(filterTranEntry));
+        }
+        String parallelBuildThreadSize = properties.getProperty(CanalConstants.CANAL_MQ_BUILD_THREAD_SIZE);
+        if (!StringUtils.isEmpty(parallelBuildThreadSize)) {
+            mqProperties.setParallelBuildThreadSize(Integer.parseInt(parallelBuildThreadSize));
+        }
+        String parallelSendThreadSize = properties.getProperty(CanalConstants.CANAL_MQ_SEND_THREAD_SIZE);
+        if (!StringUtils.isEmpty(parallelSendThreadSize)) {
+            mqProperties.setParallelSendThreadSize(Integer.parseInt(parallelSendThreadSize));
+        }
+        String batchSize = properties.getProperty(CanalConstants.CANAL_MQ_CANAL_BATCH_SIZE);
+        if (!StringUtils.isEmpty(batchSize)) {
+            mqProperties.setBatchSize(Integer.parseInt(batchSize));
+        }
+        String timeOut = properties.getProperty(CanalConstants.CANAL_MQ_CANAL_GET_TIMEOUT);
+        if (!StringUtils.isEmpty(timeOut)) {
+            mqProperties.setFetchTimeout(Integer.parseInt(timeOut));
+        }
+        String accessChannel = properties.getProperty(CanalConstants.CANAL_MQ_ACCESS_CHANNEL);
+        if (!StringUtils.isEmpty(accessChannel)) {
+            mqProperties.setAccessChannel(accessChannel);
+        }
+        String aliyunAccessKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_ACCESS_KEY);
+        if (!StringUtils.isEmpty(aliyunAccessKey)) {
+            mqProperties.setAliyunAccessKey(aliyunAccessKey);
+        }
+        String aliyunSecretKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_SECRET_KEY);
+        if (!StringUtils.isEmpty(aliyunSecretKey)) {
+            mqProperties.setAliyunAccessKey(aliyunSecretKey);
+        }
+        String aliyunUid = properties.getProperty(CanalConstants.CANAL_ALIYUN_UID);
+        if (!StringUtils.isEmpty(aliyunUid)) {
+            mqProperties.setAliyunUid(Integer.parseInt(aliyunUid));
+        }
+    }
+
+    /**
+     * 兼容下<=1.1.4的mq配置项
+     */
+    protected void doMoreCompatibleConvert(String oldKey, String newKey, Properties properties) {
+        String value = properties.getProperty(oldKey);
+        if (StringUtils.isNotEmpty(value)) {
+            properties.setProperty(newKey, value);
+        }
+    }
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/MQDestination.java

@@ -0,0 +1,74 @@
+package com.alibaba.otter.canal.connector.core.producer;
+
+/**
+ * MQ producer destination
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public class MQDestination {
+
+    private String  canalDestination;
+    private String  topic;
+    private Integer partition;
+    private Integer partitionsNum;
+    private String  partitionHash;
+    private String  dynamicTopic;
+    private String  dynamicTopicPartitionNum;
+
+    public String getCanalDestination() {
+        return canalDestination;
+    }
+
+    public void setCanalDestination(String canalDestination) {
+        this.canalDestination = canalDestination;
+    }
+
+    public String getTopic() {
+        return topic;
+    }
+
+    public void setTopic(String topic) {
+        this.topic = topic;
+    }
+
+    public Integer getPartition() {
+        return partition;
+    }
+
+    public void setPartition(Integer partition) {
+        this.partition = partition;
+    }
+
+    public Integer getPartitionsNum() {
+        return partitionsNum;
+    }
+
+    public void setPartitionsNum(Integer partitionsNum) {
+        this.partitionsNum = partitionsNum;
+    }
+
+    public String getPartitionHash() {
+        return partitionHash;
+    }
+
+    public void setPartitionHash(String partitionHash) {
+        this.partitionHash = partitionHash;
+    }
+
+    public String getDynamicTopic() {
+        return dynamicTopic;
+    }
+
+    public void setDynamicTopic(String dynamicTopic) {
+        this.dynamicTopic = dynamicTopic;
+    }
+
+    public String getDynamicTopicPartitionNum() {
+        return dynamicTopicPartitionNum;
+    }
+
+    public void setDynamicTopicPartitionNum(String dynamicTopicPartitionNum) {
+        this.dynamicTopicPartitionNum = dynamicTopicPartitionNum;
+    }
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/MQMessageUtils.java

@@ -1,4 +1,4 @@
-package com.alibaba.otter.canal.common;
+package com.alibaba.otter.canal.connector.core.producer;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -13,15 +13,15 @@ import java.util.concurrent.ThreadPoolExecutor;
 import org.apache.commons.lang.StringUtils;
 
 import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
-import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
+import com.alibaba.otter.canal.connector.core.filter.AviaterRegexFilter;
 import com.alibaba.otter.canal.protocol.CanalEntry;
 import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
 import com.alibaba.otter.canal.protocol.CanalEntry.RowChange;
 import com.alibaba.otter.canal.protocol.FlatMessage;
 import com.alibaba.otter.canal.protocol.Message;
 import com.google.common.base.Function;
+import com.google.common.cache.CacheBuilder;
 import com.google.common.collect.Lists;
-import com.google.common.collect.MapMaker;
 import com.google.common.collect.MigrateMap;
 import com.google.protobuf.ByteString;
 import com.google.protobuf.InvalidProtocolBufferException;
@@ -29,58 +29,52 @@ import com.google.protobuf.InvalidProtocolBufferException;
 /**
  * process MQ Message utils
  *
- * @author agapple 2018年12月11日 下午1:28:32
- */
-/**
  * @author agapple 2019年9月29日 下午12:36:26
  * @since 5.0.0
  */
 public class MQMessageUtils {
 
-    @SuppressWarnings("deprecation")
-    private static Map<String, List<PartitionData>>    partitionDatas    = MigrateMap.makeComputingMap(new MapMaker().softValues(),
-                                                                             new Function<String, List<PartitionData>>() {
+    private static Map<String, List<PartitionData>>    partitionDatas    = MigrateMap.makeComputingMap(CacheBuilder.newBuilder()
+                                                                             .softValues(),
+                                                                             pkHashConfigs -> {
+                                                                                 List<PartitionData> datas = Lists.newArrayList();
 
-                                                                                 public List<PartitionData> apply(String pkHashConfigs) {
-                                                                                     List<PartitionData> datas = Lists.newArrayList();
-
-                                                                                     String[] pkHashConfigArray = StringUtils.split(StringUtils.replace(pkHashConfigs,
-                                                                                         ",",
-                                                                                         ";"),
-                                                                                         ";");
-                                                                                     // schema.table:id^name
-                                                                                     for (String pkHashConfig : pkHashConfigArray) {
-                                                                                         PartitionData data = new PartitionData();
-                                                                                         int i = pkHashConfig.lastIndexOf(":");
-                                                                                         if (i > 0) {
-                                                                                             String pkStr = pkHashConfig.substring(i + 1);
-                                                                                             if (pkStr.equalsIgnoreCase("$pk$")) {
-                                                                                                 data.hashMode.autoPkHash = true;
-                                                                                             } else {
-                                                                                                 data.hashMode.pkNames = Lists.newArrayList(StringUtils.split(pkStr,
-                                                                                                     '^'));
-                                                                                             }
-
-                                                                                             pkHashConfig = pkHashConfig.substring(0,
-                                                                                                 i);
+                                                                                 String[] pkHashConfigArray = StringUtils.split(StringUtils.replace(pkHashConfigs,
+                                                                                     ",",
+                                                                                     ";"),
+                                                                                     ";");
+                                                                                 // schema.table:id^name
+                                                                                 for (String pkHashConfig : pkHashConfigArray) {
+                                                                                     PartitionData data = new PartitionData();
+                                                                                     int i = pkHashConfig.lastIndexOf(":");
+                                                                                     if (i > 0) {
+                                                                                         String pkStr = pkHashConfig.substring(i + 1);
+                                                                                         if (pkStr.equalsIgnoreCase("$pk$")) {
+                                                                                             data.hashMode.autoPkHash = true;
                                                                                          } else {
-                                                                                             data.hashMode.tableHash = true;
+                                                                                             data.hashMode.pkNames = Lists.newArrayList(StringUtils.split(pkStr,
+                                                                                                 '^'));
                                                                                          }
 
-                                                                                         if (!isWildCard(pkHashConfig)) {
-                                                                                             data.simpleName = pkHashConfig;
-                                                                                         } else {
-                                                                                             data.regexFilter = new AviaterRegexFilter(pkHashConfig);
-                                                                                         }
-                                                                                         datas.add(data);
+                                                                                         pkHashConfig = pkHashConfig.substring(0,
+                                                                                             i);
+                                                                                     } else {
+                                                                                         data.hashMode.tableHash = true;
                                                                                      }
 
-                                                                                     return datas;
+                                                                                     if (!isWildCard(pkHashConfig)) {
+                                                                                         data.simpleName = pkHashConfig;
+                                                                                     } else {
+                                                                                         data.regexFilter = new AviaterRegexFilter(pkHashConfig);
+                                                                                     }
+                                                                                     datas.add(data);
                                                                                  }
+
+                                                                                 return datas;
                                                                              });
 
-    @SuppressWarnings("deprecation")
-    private static Map<String, List<DynamicTopicData>> dynamicTopicDatas = MigrateMap.makeComputingMap(new MapMaker().softValues(),
+    private static Map<String, List<DynamicTopicData>> dynamicTopicDatas = MigrateMap.makeComputingMap(CacheBuilder.newBuilder()
+                                                                             .softValues(),
                                                                              new Function<String, List<DynamicTopicData>>() {
 
                                                                                  public List<DynamicTopicData> apply(String pkHashConfigs) {
@@ -109,6 +103,38 @@ public class MQMessageUtils {
                                                                                  }
                                                                              });
 
+    private static Map<String, List<TopicPartitionData>> topicPartitionDatas = MigrateMap.makeComputingMap(CacheBuilder.newBuilder()
+                                                                                .softValues(),
+                                                                                new Function<String, List<TopicPartitionData>>() {
+
+                                                                                    public List<TopicPartitionData> apply(String tPConfigs) {
+                                                                                        List<TopicPartitionData> datas = Lists.newArrayList();
+                                                                                        String[] tPArray = StringUtils.split(StringUtils.replace(tPConfigs,
+                                                                                                ",",
+                                                                                                ";"),
+                                                                                                ";");
+                                                                                        for (String tPConfig : tPArray) {
+                                                                                            TopicPartitionData data = new TopicPartitionData();
+                                                                                            int i = tPConfig.lastIndexOf(":");
+                                                                                            if (i > 0) {
+                                                                                                String tStr = tPConfig.substring(0, i);
+                                                                                                String pStr = tPConfig.substring(i + 1);
+                                                                                                if (!isWildCard(tStr)) {
+                                                                                                    data.simpleName = tStr;
+                                                                                                } else {
+                                                                                                    data.regexFilter = new AviaterRegexFilter(tStr);
+                                                                                                }
+                                                                                                if (!StringUtils.isEmpty(pStr) && StringUtils.isNumeric(pStr)) {
+                                                                                                    data.partitionNum = Integer.valueOf(pStr);
+                                                                                                }
+                                                                                                datas.add(data);
+                                                                                            }
+                                                                                        }
+
+                                                                                        return datas;
+                                                                                    }
+                                                                                });
+
     /**
      * 按 schema 或者 schema+table 将 message 分配到对应topic
      *
@@ -338,8 +364,8 @@ public class MQMessageUtils {
     /**
      * 将Message转换为FlatMessage
      *
-     * @param message 原生message
      * @return FlatMessage列表
+     * @author agapple 2018年12月11日 下午1:28:32
      */
     public static List<FlatMessage> messageConverter(EntryRowData[] datas, long id) {
         List<FlatMessage> flatMessages = new ArrayList<>();
@@ -625,6 +651,24 @@ public class MQMessageUtils {
         return false;
     }
 
+    public static Integer parseDynamicTopicPartition(String name, String tPConfigs) {
+        if (!StringUtils.isEmpty(tPConfigs)) {
+            List<TopicPartitionData> datas = topicPartitionDatas.get(tPConfigs);
+            for (TopicPartitionData data : datas) {
+                if (data.simpleName != null) {
+                    if (data.simpleName.equalsIgnoreCase(name)) {
+                        return data.partitionNum;
+                    }
+                } else {
+                    if (data.regexFilter.filter(name)) {
+                        return data.partitionNum;
+                    }
+                }
+            }
+        }
+        return null;
+    }
+
     private static boolean isWildCard(String value) {
         // not contaiins '.' ?
         return StringUtils.containsAny(value, new char[] { '*', '?', '+', '|', '(', ')', '{', '}', '[', ']', '\\', '$',
@@ -662,6 +706,13 @@ public class MQMessageUtils {
         public AviaterRegexFilter tableRegexFilter;
     }
 
+    public static class TopicPartitionData {
+
+        public String             simpleName;
+        public AviaterRegexFilter regexFilter;
+        public Integer            partitionNum;
+    }
+
     public static class EntryRowData {
 
         public Entry     entry;

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/CanalMQProducer.java

@@ -0,0 +1,43 @@
+package com.alibaba.otter.canal.connector.core.spi;
+
+import java.util.Properties;
+
+import com.alibaba.otter.canal.connector.core.config.MQProperties;
+import com.alibaba.otter.canal.connector.core.producer.MQDestination;
+import com.alibaba.otter.canal.connector.core.util.Callback;
+import com.alibaba.otter.canal.protocol.Message;
+
+/**
+ * MQ producer SPI 接口
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+@SPI("kafka")
+public interface CanalMQProducer {
+
+    /**
+     * Init producer.
+     */
+    void init(Properties properties);
+
+    /**
+     * Get base mq properties
+     * 
+     * @return MQProperties
+     */
+    MQProperties getMqProperties();
+
+    /**
+     * Send canal message to related topic
+     *
+     * @param canalDestination canal mq destination
+     * @param message canal message
+     */
+    void send(MQDestination canalDestination, Message message, Callback callback);
+
+    /**
+     * Stop MQ producer service
+     */
+    void stop();
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/CanalMsgConsumer.java

@@ -0,0 +1,55 @@
+package com.alibaba.otter.canal.connector.core.spi;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
+
+/**
+ * Canal/MQ consumer SPI 接口
+ *
+ * @author rewerma @ 2020-02-01
+ * @version 1.0.0
+ */
+@SPI("kafka")
+public interface CanalMsgConsumer {
+
+    /**
+     * 初始化
+     * 
+     * @param properties consumer properties
+     * @param topic topic/destination
+     * @param groupId mq group id
+     */
+    void init(Properties properties, String topic, String groupId);
+
+    /**
+     * 连接Canal/MQ
+     */
+    void connect();
+
+    /**
+     * 批量拉取数据
+     * 
+     * @param timeout 超时时间
+     * @param unit 时间单位
+     * @return Message列表
+     */
+    List<CommonMessage> getMessage(Long timeout, TimeUnit unit);
+
+    /**
+     * 提交
+     */
+    void ack();
+
+    /**
+     * 回滚
+     */
+    void rollback();
+
+    /**
+     * 断开连接
+     */
+    void disconnect();
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/ExtensionLoader.java

@@ -0,0 +1,434 @@
+package com.alibaba.otter.canal.connector.core.spi;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.InputStreamReader;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.regex.Pattern;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * SPI 类加载器
+ *
+ * @author rewerma 2018-8-19 下午11:30:49
+ * @version 1.0.0
+ */
+public class ExtensionLoader<T> {
+
+    private static final Logger                                      logger                     = LoggerFactory.getLogger(ExtensionLoader.class);
+
+    private static final String                                      SERVICES_DIRECTORY         = "META-INF/services/";
+
+    private static final String                                      CANAL_DIRECTORY            = "META-INF/canal/";
+
+    private static final String                                      DEFAULT_CLASSLOADER_POLICY = "internal";
+
+    private static final Pattern                                     NAME_SEPARATOR             = Pattern.compile("\\s*[,]+\\s*");
+
+    private static final ConcurrentMap<Class<?>, ExtensionLoader<?>> EXTENSION_LOADERS          = new ConcurrentHashMap<>();
+
+    private static final ConcurrentMap<Class<?>, Object>             EXTENSION_INSTANCES        = new ConcurrentHashMap<>();
+
+    private static final ConcurrentMap<String, Object>               EXTENSION_KEY_INSTANCE     = new ConcurrentHashMap<>();
+
+    private final Class<?>                                           type;
+
+    private final String                                             classLoaderPolicy;
+
+    private final ConcurrentMap<Class<?>, String>                    cachedNames                = new ConcurrentHashMap<>();
+
+    private final Holder<Map<String, Class<?>>>                      cachedClasses              = new Holder<>();
+
+    private final ConcurrentMap<String, Holder<Object>>              cachedInstances            = new ConcurrentHashMap<>();
+
+    private String                                                   cachedDefaultName;
+
+    private ConcurrentHashMap<String, IllegalStateException>         exceptions                 = new ConcurrentHashMap<>();
+
+    private static <T> boolean withExtensionAnnotation(Class<T> type) {
+        return type.isAnnotationPresent(SPI.class);
+    }
+
+    public static <T> ExtensionLoader<T> getExtensionLoader(Class<T> type) {
+        return getExtensionLoader(type, DEFAULT_CLASSLOADER_POLICY);
+    }
+
+    @SuppressWarnings("unchecked")
+    public static <T> ExtensionLoader<T> getExtensionLoader(Class<T> type, String classLoaderPolicy) {
+        if (type == null) throw new IllegalArgumentException("Extension type == null");
+        if (!type.isInterface()) {
+            throw new IllegalArgumentException("Extension type(" + type + ") is not interface!");
+        }
+        if (!withExtensionAnnotation(type)) {
+            throw new IllegalArgumentException("Extension type(" + type + ") is not extension, because WITHOUT @"
+                                               + SPI.class.getSimpleName() + " Annotation!");
+        }
+
+        ExtensionLoader<T> loader = (ExtensionLoader<T>) EXTENSION_LOADERS.get(type);
+        if (loader == null) {
+            EXTENSION_LOADERS.putIfAbsent(type, new ExtensionLoader<T>(type, classLoaderPolicy));
+            loader = (ExtensionLoader<T>) EXTENSION_LOADERS.get(type);
+        }
+        return loader;
+    }
+
+    public ExtensionLoader(Class<?> type){
+        this.type = type;
+        this.classLoaderPolicy = DEFAULT_CLASSLOADER_POLICY;
+    }
+
+    public ExtensionLoader(Class<?> type, String classLoaderPolicy){
+        this.type = type;
+        this.classLoaderPolicy = classLoaderPolicy;
+    }
+
+    /**
+     * 返回指定名字的扩展
+     *
+     * @param name
+     * @return
+     */
+    @SuppressWarnings("unchecked")
+    public T getExtension(String name, String spiDir, String standbyDir) {
+        if (name == null || name.length() == 0) throw new IllegalArgumentException("Extension name == null");
+        if ("true".equals(name)) {
+            return getDefaultExtension(spiDir, standbyDir);
+        }
+        Holder<Object> holder = cachedInstances.get(name);
+        if (holder == null) {
+            cachedInstances.putIfAbsent(name, new Holder<>());
+            holder = cachedInstances.get(name);
+        }
+        Object instance = holder.get();
+        if (instance == null) {
+            synchronized (holder) {
+                instance = holder.get();
+                if (instance == null) {
+                    instance = createExtension(name, spiDir, standbyDir);
+                    holder.set(instance);
+                }
+            }
+        }
+        return (T) instance;
+    }
+
+    @SuppressWarnings("unchecked")
+    public T getExtension(String name, String key, String spiDir, String standbyDir) {
+        if (name == null || name.length() == 0) throw new IllegalArgumentException("Extension name == null");
+        if ("true".equals(name)) {
+            return getDefaultExtension(spiDir, standbyDir);
+        }
+        String extKey = name + "-" + StringUtils.trimToEmpty(key);
+        Holder<Object> holder = cachedInstances.get(extKey);
+        if (holder == null) {
+            cachedInstances.putIfAbsent(extKey, new Holder<>());
+            holder = cachedInstances.get(extKey);
+        }
+        Object instance = holder.get();
+        if (instance == null) {
+            synchronized (holder) {
+                instance = holder.get();
+                if (instance == null) {
+                    instance = createExtension(name, key, spiDir, standbyDir);
+                    holder.set(instance);
+                }
+            }
+        }
+        return (T) instance;
+    }
+
+    /**
+     * 返回缺省的扩展，如果没有设置则返回<code>null</code>
+     */
+    public T getDefaultExtension(String spiDir, String standbyDir) {
+        getExtensionClasses(spiDir, standbyDir);
+        if (null == cachedDefaultName || cachedDefaultName.length() == 0 || "true".equals(cachedDefaultName)) {
+            return null;
+        }
+        return getExtension(cachedDefaultName, spiDir, standbyDir);
+    }
+
+    @SuppressWarnings("unchecked")
+    public T createExtension(String name, String spiDir, String standbyDir) {
+        Class<?> clazz = getExtensionClasses(spiDir, standbyDir).get(name);
+        if (clazz == null) {
+            throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type
+                                            + ")  could not be instantiated: class could not be found");
+        }
+        try {
+            T instance = (T) EXTENSION_INSTANCES.get(clazz);
+            if (instance == null) {
+                EXTENSION_INSTANCES.putIfAbsent(clazz, (T) clazz.newInstance());
+                instance = (T) EXTENSION_INSTANCES.get(clazz);
+            }
+            return instance;
+        } catch (Throwable t) {
+            throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type
+                                            + ")  could not be instantiated: " + t.getMessage(), t);
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private T createExtension(String name, String key, String spiDir, String standbyDir) {
+        Class<?> clazz = getExtensionClasses(spiDir, standbyDir).get(name);
+        if (clazz == null) {
+            throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type
+                                            + ")  could not be instantiated: class could not be found");
+        }
+        try {
+            T instance = (T) EXTENSION_KEY_INSTANCE.get(name + "-" + key);
+            if (instance == null) {
+                EXTENSION_KEY_INSTANCE.putIfAbsent(name + "-" + key, clazz.newInstance());
+                instance = (T) EXTENSION_KEY_INSTANCE.get(name + "-" + key);
+            }
+            return instance;
+        } catch (Throwable t) {
+            throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type
+                                            + ")  could not be instantiated: " + t.getMessage(), t);
+        }
+    }
+
+    private Map<String, Class<?>> getExtensionClasses(String spiDir, String standbyDir) {
+        Map<String, Class<?>> classes = cachedClasses.get();
+        if (classes == null) {
+            synchronized (cachedClasses) {
+                classes = cachedClasses.get();
+                if (classes == null) {
+                    classes = loadExtensionClasses(spiDir, standbyDir);
+                    cachedClasses.set(classes);
+                }
+            }
+        }
+
+        return classes;
+    }
+
+    private String getJarDirectoryPath() {
+        URL url = Thread.currentThread().getContextClassLoader().getResource("");
+        String dirtyPath;
+        if (url != null) {
+            dirtyPath = url.toString();
+        } else {
+            File file = new File("");
+            dirtyPath = file.getAbsolutePath();
+        }
+        String jarPath = dirtyPath.replaceAll("^.*file:/", ""); // removes
+                                                                // file:/ and
+                                                                // everything
+                                                                // before it
+        jarPath = jarPath.replaceAll("jar!.*", "jar"); // removes everything
+                                                       // after .jar, if .jar
+                                                       // exists in dirtyPath
+        jarPath = jarPath.replaceAll("%20", " "); // necessary if path has
+                                                  // spaces within
+        if (!jarPath.endsWith(".jar")) { // this is needed if you plan to run
+                                         // the app using Spring Tools Suit play
+                                         // button.
+            jarPath = jarPath.replaceAll("/classes/.*", "/classes/");
+        }
+        Path path = Paths.get(jarPath).getParent(); // Paths - from java 8
+        if (path != null) {
+            return path.toString();
+        }
+        return null;
+    }
+
+    private Map<String, Class<?>> loadExtensionClasses(String spiDir, String standbyDir) {
+        final SPI defaultAnnotation = type.getAnnotation(SPI.class);
+        if (defaultAnnotation != null) {
+            String value = defaultAnnotation.value();
+            if ((value = value.trim()).length() > 0) {
+                String[] names = NAME_SEPARATOR.split(value);
+                if (names.length > 1) {
+                    throw new IllegalStateException("more than 1 default extension name on extension " + type.getName()
+                                                    + ": " + Arrays.toString(names));
+                }
+                if (names.length == 1) cachedDefaultName = names[0];
+            }
+        }
+
+        Map<String, Class<?>> extensionClasses = new HashMap<>();
+
+        if (spiDir != null && standbyDir != null) {
+            // 1. plugin folder，customized extension classLoader
+            // （jar_dir/plugin）
+            String dir = File.separator + this.getJarDirectoryPath() + spiDir; // +
+                                                                               // "plugin";
+
+            File externalLibDir = new File(dir);
+            if (!externalLibDir.exists()) {
+                externalLibDir = new File(File.separator + this.getJarDirectoryPath() + standbyDir);
+            }
+            logger.info("extension classpath dir: " + externalLibDir.getAbsolutePath());
+            if (externalLibDir.exists()) {
+                File[] files = externalLibDir.listFiles((dir1, name) -> name.endsWith(".jar"));
+                if (files != null) {
+                    for (File f : files) {
+                        URL url;
+                        try {
+                            url = f.toURI().toURL();
+                        } catch (MalformedURLException e) {
+                            throw new RuntimeException("load extension jar failed!", e);
+                        }
+
+                        ClassLoader parent = Thread.currentThread().getContextClassLoader();
+                        URLClassLoader localClassLoader;
+                        if (classLoaderPolicy == null || "".equals(classLoaderPolicy)
+                            || DEFAULT_CLASSLOADER_POLICY.equalsIgnoreCase(classLoaderPolicy)) {
+                            localClassLoader = new URLClassExtensionLoader(new URL[] { url });
+                        } else {
+                            localClassLoader = new URLClassLoader(new URL[] { url }, parent);
+                        }
+
+                        loadFile(extensionClasses, CANAL_DIRECTORY, localClassLoader);
+                        loadFile(extensionClasses, SERVICES_DIRECTORY, localClassLoader);
+                    }
+                }
+            }
+        }
+
+        // 2. load inner extension class with default classLoader
+        ClassLoader classLoader = findClassLoader();
+        loadFile(extensionClasses, CANAL_DIRECTORY, classLoader);
+        loadFile(extensionClasses, SERVICES_DIRECTORY, classLoader);
+
+        return extensionClasses;
+    }
+
+    private void loadFile(Map<String, Class<?>> extensionClasses, String dir, ClassLoader classLoader) {
+        String fileName = dir + type.getName();
+        try {
+            Enumeration<URL> urls;
+            if (classLoader != null) {
+                urls = classLoader.getResources(fileName);
+            } else {
+                urls = ClassLoader.getSystemResources(fileName);
+            }
+            if (urls != null) {
+                while (urls.hasMoreElements()) {
+                    URL url = urls.nextElement();
+                    try {
+                        BufferedReader reader = null;
+                        try {
+                            reader = new BufferedReader(new InputStreamReader(url.openStream(), StandardCharsets.UTF_8));
+                            String line = null;
+                            while ((line = reader.readLine()) != null) {
+                                final int ci = line.indexOf('#');
+                                if (ci >= 0) line = line.substring(0, ci);
+                                line = line.trim();
+                                if (line.length() > 0) {
+                                    try {
+                                        String name = null;
+                                        int i = line.indexOf('=');
+                                        if (i > 0) {
+                                            name = line.substring(0, i).trim();
+                                            line = line.substring(i + 1).trim();
+                                        }
+                                        if (line.length() > 0) {
+                                            Class<?> clazz = classLoader.loadClass(line);
+                                            // Class<?> clazz =
+                                            // Class.forName(line, true,
+                                            // classLoader);
+                                            if (!type.isAssignableFrom(clazz)) {
+                                                throw new IllegalStateException("Error when load extension class(interface: "
+                                                                                + type
+                                                                                + ", class line: "
+                                                                                + clazz.getName()
+                                                                                + "), class "
+                                                                                + clazz.getName()
+                                                                                + "is not subtype of interface.");
+                                            } else {
+                                                try {
+                                                    clazz.getConstructor(type);
+                                                } catch (NoSuchMethodException e) {
+                                                    clazz.getConstructor();
+                                                    String[] names = NAME_SEPARATOR.split(name);
+                                                    if (names != null && names.length > 0) {
+                                                        for (String n : names) {
+                                                            if (!cachedNames.containsKey(clazz)) {
+                                                                cachedNames.put(clazz, n);
+                                                            }
+                                                            Class<?> c = extensionClasses.get(n);
+                                                            if (c == null) {
+                                                                extensionClasses.put(n, clazz);
+                                                            } else if (c != clazz) {
+                                                                cachedNames.remove(clazz);
+                                                                throw new IllegalStateException("Duplicate extension "
+                                                                                                + type.getName()
+                                                                                                + " name " + n + " on "
+                                                                                                + c.getName() + " and "
+                                                                                                + clazz.getName());
+                                                            }
+                                                        }
+                                                    }
+                                                }
+                                            }
+                                        }
+                                    } catch (Throwable t) {
+                                        IllegalStateException e = new IllegalStateException("Failed to load extension class(interface: "
+                                                                                            + type
+                                                                                            + ", class line: "
+                                                                                            + line
+                                                                                            + ") in "
+                                                                                            + url
+                                                                                            + ", cause: "
+                                                                                            + t.getMessage(),
+                                            t);
+                                        exceptions.put(line, e);
+                                    }
+                                }
+                            } // end of while read lines
+                        } finally {
+                            if (reader != null) {
+                                reader.close();
+                            }
+                        }
+                    } catch (Throwable t) {
+                        logger.error("Exception when load extension class(interface: " + type + ", class file: " + url
+                                     + ") in " + url, t);
+                    }
+                } // end of while urls
+            }
+        } catch (Throwable t) {
+            logger.error("Exception when load extension class(interface: " + type + ", description file: " + fileName
+                         + ").", t);
+        }
+    }
+
+    private static ClassLoader findClassLoader() {
+        return ExtensionLoader.class.getClassLoader();
+    }
+
+    @Override
+    public String toString() {
+        return this.getClass().getName() + "[" + type.getName() + "]";
+    }
+
+    private static class Holder<T> {
+
+        private volatile T value;
+
+        private void set(T value) {
+            this.value = value;
+        }
+
+        private T get() {
+            return value;
+        }
+
+    }
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/SPI.java

@@ -0,0 +1,22 @@
+package com.alibaba.otter.canal.connector.core.spi;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * SPI装载器注解
+ *
+ * @author rewerma @ 2018-10-20
+ * @version 1.0.0
+ */
+@Documented
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ ElementType.TYPE })
+public @interface SPI {
+
+    // Default SPI name
+    String value() default "";
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/URLClassExtensionLoader.java

@@ -0,0 +1,88 @@
+package com.alibaba.otter.canal.connector.core.spi;
+
+import java.io.IOException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.Enumeration;
+import java.util.NoSuchElementException;
+
+public class URLClassExtensionLoader extends URLClassLoader {
+
+    public URLClassExtensionLoader(URL[] urls){
+        super(urls);
+    }
+
+    @Override
+    public Class<?> loadClass(String name) throws ClassNotFoundException {
+        Class<?> c = findLoadedClass(name);
+        if (c != null) {
+            return c;
+        }
+
+        if (name.startsWith("java.") || name.startsWith("org.slf4j.") || name.startsWith("org.apache.logging")
+            || name.startsWith("org.apache.commons.logging.")) {
+            // || name.startsWith("org.apache.hadoop."))
+            // {
+            c = super.loadClass(name);
+        }
+        if (c != null) return c;
+
+        try {
+            // 先加载jar内的class，可避免jar冲突
+            c = findClass(name);
+        } catch (ClassNotFoundException e) {
+            c = null;
+        }
+        if (c != null) {
+            return c;
+        }
+
+        return super.loadClass(name);
+    }
+
+    @Override
+    public Enumeration<URL> getResources(String name) throws IOException {
+        @SuppressWarnings("unchecked")
+        Enumeration<URL>[] tmp = (Enumeration<URL>[]) new Enumeration<?>[2];
+
+        tmp[0] = findResources(name); // local class
+        // path first
+        // tmp[1] = super.getResources(name);
+
+        return new CompoundEnumeration<>(tmp);
+    }
+
+    private static class CompoundEnumeration<E> implements Enumeration<E> {
+
+        private Enumeration<E>[] enums;
+        private int              index = 0;
+
+        public CompoundEnumeration(Enumeration<E>[] enums){
+            this.enums = enums;
+        }
+
+        private boolean next() {
+            while (this.index < this.enums.length) {
+                if (this.enums[this.index] != null && this.enums[this.index].hasMoreElements()) {
+                    return true;
+                }
+
+                ++this.index;
+            }
+
+            return false;
+        }
+
+        public boolean hasMoreElements() {
+            return this.next();
+        }
+
+        public E nextElement() {
+            if (!this.next()) {
+                throw new NoSuchElementException();
+            } else {
+                return this.enums[this.index].nextElement();
+            }
+        }
+    }
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/Callback.java

@@ -0,0 +1,14 @@
+package com.alibaba.otter.canal.connector.core.util;
+
+/**
+ * MQ 回调类
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public interface Callback {
+
+    void commit();
+
+    void rollback();
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/CanalMessageSerializerUtil.java

@@ -1,20 +1,25 @@
-package com.alibaba.otter.canal.common;
+package com.alibaba.otter.canal.connector.core.util;
 
 import java.util.List;
 
-import org.apache.kafka.common.errors.SerializationException;
 import org.springframework.util.CollectionUtils;
 
 import com.alibaba.otter.canal.protocol.CanalEntry;
 import com.alibaba.otter.canal.protocol.CanalPacket;
 import com.alibaba.otter.canal.protocol.CanalPacket.PacketType;
 import com.alibaba.otter.canal.protocol.Message;
-import com.alibaba.otter.canal.server.netty.NettyUtils;
+import com.alibaba.otter.canal.protocol.exception.CanalClientException;
 import com.google.protobuf.ByteString;
 import com.google.protobuf.CodedOutputStream;
 import com.google.protobuf.WireFormat;
 
-public class CanalMessageSerializer {
+/**
+ * Canal message 序列化工具类
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public class CanalMessageSerializerUtil {
 
     @SuppressWarnings("deprecation")
     public static byte[] serializer(Message data, boolean filterTransactionEntry) {
@@ -54,7 +59,7 @@ public class CanalMessageSerializer {
                         output.checkNoSpaceLeft();
                         return body;
                     } else if (!CollectionUtils.isEmpty(data.getEntries())) {
-                        // mq模式只会走到非rowEntry模式
+                        // mq模式只会走到非rawEntry模式
                         CanalPacket.Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder();
                         for (CanalEntry.Entry entry : data.getEntries()) {
                             if (filterTransactionEntry
@@ -67,15 +72,60 @@ public class CanalMessageSerializer {
 
                         CanalPacket.Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder();
                         packetBuilder.setType(PacketType.MESSAGES);
-                        packetBuilder.setVersion(NettyUtils.VERSION);
+                        packetBuilder.setVersion(1);
                         packetBuilder.setBody(messageBuilder.build().toByteString());
                         return packetBuilder.build().toByteArray();
                     }
                 }
             }
         } catch (Exception e) {
-            throw new SerializationException("Error when serializing message to byte[] ");
+            throw new RuntimeException("Error when serializing message to byte[] ");
         }
         return null;
     }
+
+    public static Message deserializer(byte[] data) {
+        return deserializer(data, false);
+    }
+
+    public static Message deserializer(byte[] data, boolean lazyParseEntry) {
+        try {
+            if (data == null) {
+                return null;
+            } else {
+                CanalPacket.Packet p = CanalPacket.Packet.parseFrom(data);
+                switch (p.getType()) {
+                    case MESSAGES: {
+                        if (!p.getCompression().equals(CanalPacket.Compression.NONE)
+                            && !p.getCompression().equals(CanalPacket.Compression.COMPRESSIONCOMPATIBLEPROTO2)) {
+                            throw new CanalClientException("compression is not supported in this connector");
+                        }
+
+                        CanalPacket.Messages messages = CanalPacket.Messages.parseFrom(p.getBody());
+                        Message result = new Message(messages.getBatchId());
+                        if (lazyParseEntry) {
+                            // byteString
+                            result.setRawEntries(messages.getMessagesList());
+                            result.setRaw(true);
+                        } else {
+                            for (ByteString byteString : messages.getMessagesList()) {
+                                result.addEntry(CanalEntry.Entry.parseFrom(byteString));
+                            }
+                            result.setRaw(false);
+                        }
+                        return result;
+                    }
+                    case ACK: {
+                        CanalPacket.Ack ack = CanalPacket.Ack.parseFrom(p.getBody());
+                        throw new CanalClientException("something goes wrong with reason: " + ack.getErrorMessage());
+                    }
+                    default: {
+                        throw new CanalClientException("unexpected packet type: " + p.getType());
+                    }
+                }
+            }
+        } catch (Exception e) {
+            throw new CanalClientException("deserializer failed", e);
+        }
+    }
 }

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/DateUtil.java

@@ -0,0 +1,47 @@
+package com.alibaba.otter.canal.connector.core.util;
+
+import java.util.Date;
+
+import org.apache.commons.lang.StringUtils;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+
+public class DateUtil {
+
+    private static DateTimeZone dateTimeZone;
+
+    static {
+        dateTimeZone = DateTimeZone.forID(TimeZone.LOCATION_TIME_ZONE);
+    }
+
+    /**
+     * 通用日期时间字符解析
+     *
+     * @param datetimeStr 日期时间字符串
+     * @return Date
+     */
+    public static Date parseDate(String datetimeStr) {
+        if (StringUtils.isEmpty(datetimeStr)) {
+            return null;
+        }
+        datetimeStr = datetimeStr.trim().replace('/', '-');
+        if (datetimeStr.contains("-")) {
+            if (datetimeStr.contains(":")) {
+                datetimeStr = datetimeStr.replace(" ", "T");
+            }
+        } else if (datetimeStr.contains(":")) {
+            datetimeStr = "T" + datetimeStr;
+        } else {
+            if (datetimeStr.length() == 8) {
+                String year = datetimeStr.substring(0, 4);
+                String month = datetimeStr.substring(4, 6);
+                String day = datetimeStr.substring(6, 8);
+                datetimeStr = year + "-" + month + "-" + day;
+            }
+        }
+
+        DateTime dateTime = new DateTime(datetimeStr, dateTimeZone);
+
+        return dateTime.toDate();
+    }
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/JdbcTypeUtil.java

@@ -0,0 +1,170 @@
+package com.alibaba.otter.canal.connector.core.util;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.sql.Date;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.sql.Types;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * 类型转换工具类
+ *
+ * @author rewerma 2018-8-19 下午06:14:23
+ * @version 1.0.0
+ */
+public class JdbcTypeUtil {
+
+    private static Logger logger = LoggerFactory.getLogger(JdbcTypeUtil.class);
+
+    public static Object getRSData(ResultSet rs, String columnName, int jdbcType) throws SQLException {
+        if (jdbcType == Types.BIT || jdbcType == Types.BOOLEAN) {
+            return rs.getByte(columnName);
+        } else {
+            return rs.getObject(columnName);
+        }
+    }
+
+    public static Class<?> jdbcType2javaType(int jdbcType) {
+        switch (jdbcType) {
+            case Types.BIT:
+            case Types.BOOLEAN:
+                // return Boolean.class;
+            case Types.TINYINT:
+                return Byte.TYPE;
+            case Types.SMALLINT:
+                return Short.class;
+            case Types.INTEGER:
+                return Integer.class;
+            case Types.BIGINT:
+                return Long.class;
+            case Types.DECIMAL:
+            case Types.NUMERIC:
+                return BigDecimal.class;
+            case Types.REAL:
+                return Float.class;
+            case Types.FLOAT:
+            case Types.DOUBLE:
+                return Double.class;
+            case Types.CHAR:
+            case Types.VARCHAR:
+            case Types.LONGVARCHAR:
+                return String.class;
+            case Types.BINARY:
+            case Types.VARBINARY:
+            case Types.LONGVARBINARY:
+            case Types.BLOB:
+                return byte[].class;
+            case Types.DATE:
+                return Date.class;
+            case Types.TIME:
+                return Time.class;
+            case Types.TIMESTAMP:
+                return Timestamp.class;
+            default:
+                return String.class;
+        }
+    }
+
+    private static boolean isText(String columnType) {
+        return "LONGTEXT".equalsIgnoreCase(columnType) || "MEDIUMTEXT".equalsIgnoreCase(columnType)
+               || "TEXT".equalsIgnoreCase(columnType) || "TINYTEXT".equalsIgnoreCase(columnType);
+    }
+
+    public static Object typeConvert(String tableName, String columnName, String value, int sqlType, String mysqlType) {
+        if (value == null
+            || (value.equals("") && !(isText(mysqlType) || sqlType == Types.CHAR || sqlType == Types.VARCHAR || sqlType == Types.LONGVARCHAR))) {
+            return null;
+        }
+
+        try {
+            Object res;
+            switch (sqlType) {
+                case Types.INTEGER:
+                    res = Integer.parseInt(value);
+                    break;
+                case Types.SMALLINT:
+                    res = Short.parseShort(value);
+                    break;
+                case Types.BIT:
+                case Types.TINYINT:
+                    res = Byte.parseByte(value);
+                    break;
+                case Types.BIGINT:
+                    if (mysqlType != null && mysqlType.startsWith("bigint") && mysqlType.endsWith("unsigned")) {
+                        res = new BigInteger(value);
+                    } else {
+                        res = Long.parseLong(value);
+                    }
+                    break;
+                // case Types.BIT:
+                case Types.BOOLEAN:
+                    res = !"0".equals(value);
+                    break;
+                case Types.DOUBLE:
+                case Types.FLOAT:
+                    res = Double.parseDouble(value);
+                    break;
+                case Types.REAL:
+                    res = Float.parseFloat(value);
+                    break;
+                case Types.DECIMAL:
+                case Types.NUMERIC:
+                    res = new BigDecimal(value);
+                    break;
+                case Types.BINARY:
+                case Types.VARBINARY:
+                case Types.LONGVARBINARY:
+                case Types.BLOB:
+                    res = value.getBytes("ISO-8859-1");
+                    break;
+                case Types.DATE:
+                    if (!value.startsWith("0000-00-00")) {
+                        java.util.Date date = DateUtil.parseDate(value);
+                        if (date != null) {
+                            res = new Date(date.getTime());
+                        } else {
+                            res = null;
+                        }
+                    } else {
+                        res = null;
+                    }
+                    break;
+                case Types.TIME: {
+                    java.util.Date date = DateUtil.parseDate(value);
+                    if (date != null) {
+                        res = new Time(date.getTime());
+                    } else {
+                        res = null;
+                    }
+                    break;
+                }
+                case Types.TIMESTAMP:
+                    if (!value.startsWith("0000-00-00")) {
+                        java.util.Date date = DateUtil.parseDate(value);
+                        if (date != null) {
+                            res = new Timestamp(date.getTime());
+                        } else {
+                            res = null;
+                        }
+                    } else {
+                        res = null;
+                    }
+                    break;
+                case Types.CLOB:
+                default:
+                    res = value;
+                    break;
+            }
+            return res;
+        } catch (Exception e) {
+            logger.error("table: {} column: {}, failed convert type {} to {}", tableName, columnName, value, sqlType);
+            return value;
+        }
+    }
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/MessageUtil.java

@@ -0,0 +1,135 @@
+package com.alibaba.otter.canal.connector.core.util;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
+import com.alibaba.otter.canal.protocol.CanalEntry;
+import com.alibaba.otter.canal.protocol.Message;
+
+/**
+ * Message对象解析工具类
+ *
+ * @author rewerma 2018-8-19 下午06:14:23
+ * @version 1.0.0
+ */
+public class MessageUtil {
+
+    public static List<CommonMessage> convert(Message message) {
+        if (message == null) {
+            return null;
+        }
+        List<CanalEntry.Entry> entries = message.getEntries();
+        List<CommonMessage> msgs = new ArrayList<>(entries.size());
+        for (CanalEntry.Entry entry : entries) {
+            if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
+                || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
+                continue;
+            }
+
+            CanalEntry.RowChange rowChange;
+            try {
+                rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
+            } catch (Exception e) {
+                throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(),
+                    e);
+            }
+
+            CanalEntry.EventType eventType = rowChange.getEventType();
+
+            final CommonMessage msg = new CommonMessage();
+            msg.setIsDdl(rowChange.getIsDdl());
+            msg.setDatabase(entry.getHeader().getSchemaName());
+            msg.setTable(entry.getHeader().getTableName());
+            msg.setType(eventType.toString());
+            msg.setEs(entry.getHeader().getExecuteTime());
+            msg.setIsDdl(rowChange.getIsDdl());
+            msg.setTs(System.currentTimeMillis());
+            msg.setSql(rowChange.getSql());
+            msgs.add(msg);
+            List<Map<String, Object>> data = new ArrayList<>();
+            List<Map<String, Object>> old = new ArrayList<>();
+
+            if (!rowChange.getIsDdl()) {
+                Set<String> updateSet = new HashSet<>();
+                msg.setPkNames(new ArrayList<>());
+                int i = 0;
+                for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
+                    if (eventType != CanalEntry.EventType.INSERT && eventType != CanalEntry.EventType.UPDATE
+                        && eventType != CanalEntry.EventType.DELETE) {
+                        continue;
+                    }
+
+                    Map<String, Object> row = new LinkedHashMap<>();
+                    List<CanalEntry.Column> columns;
+
+                    if (eventType == CanalEntry.EventType.DELETE) {
+                        columns = rowData.getBeforeColumnsList();
+                    } else {
+                        columns = rowData.getAfterColumnsList();
+                    }
+
+                    for (CanalEntry.Column column : columns) {
+                        if (i == 0) {
+                            if (column.getIsKey()) {
+                                msg.getPkNames().add(column.getName());
+                            }
+                        }
+                        if (column.getIsNull()) {
+                            row.put(column.getName(), null);
+                        } else {
+                            row.put(column.getName(),
+                                JdbcTypeUtil.typeConvert(msg.getTable(),
+                                    column.getName(),
+                                    column.getValue(),
+                                    column.getSqlType(),
+                                    column.getMysqlType()));
+                        }
+                        // 获取update为true的字段
+                        if (column.getUpdated()) {
+                            updateSet.add(column.getName());
+                        }
+                    }
+                    if (!row.isEmpty()) {
+                        data.add(row);
+                    }
+
+                    if (eventType == CanalEntry.EventType.UPDATE) {
+                        Map<String, Object> rowOld = new LinkedHashMap<>();
+                        for (CanalEntry.Column column : rowData.getBeforeColumnsList()) {
+                            if (updateSet.contains(column.getName())) {
+                                if (column.getIsNull()) {
+                                    rowOld.put(column.getName(), null);
+                                } else {
+                                    rowOld.put(column.getName(), JdbcTypeUtil.typeConvert(msg.getTable(),
+                                        column.getName(),
+                                        column.getValue(),
+                                        column.getSqlType(),
+                                        column.getMysqlType()));
+                                }
+                            }
+                        }
+                        // update操作将记录修改前的值
+                        if (!rowOld.isEmpty()) {
+                            old.add(rowOld);
+                        }
+                    }
+
+                    i++;
+                }
+                if (!data.isEmpty()) {
+                    msg.setData(data);
+                }
+                if (!old.isEmpty()) {
+                    msg.setOld(old);
+                }
+            }
+        }
+
+        return msgs;
+    }
+}