add polardb-x parser test

* 修复升级2.0.4导致兼容的问题

* update druid & fastjson version

.travis.yml

@@ -14,7 +14,7 @@ cache:
 install: true
 
 script:
-  - travis_wait 30 ./mvnw clean install -DskipTests=false -Dcheckstyle.skip=true -Dmaven.javadoc.skip=true
+  - travis_wait 30 ./mvnw -T 1C clean install -DskipTests=false -Dcheckstyle.skip=true -Dmaven.javadoc.skip=true
 
 after_success:
   - bash <(curl -s https://codecov.io/bash)

README.md

@@ -90,10 +90,13 @@ canal 作为 MySQL binlog 增量获取和解析工具，可将变更记录投递
 
 - 参考文档: [Canal Kafka/RocketMQ QuickStart](https://github.com/alibaba/canal/wiki/Canal-Kafka-RocketMQ-QuickStart)
 
-## 相关开源
+## 相关开源&产品
 
 - [canal 消费端开源项目: Otter](http://github.com/alibaba/otter)
 - [阿里巴巴去 Oracle 数据迁移同步工具: yugong](http://github.com/alibaba/yugong)
+- [阿里巴巴离线同步开源项目 DataX](https://github.com/alibaba/datax)
+- [阿里巴巴数据库连接池开源项目 Druid](https://github.com/alibaba/druid)
+- [阿里巴巴实时数据同步工具 DTS](https://www.aliyun.com/product/dts)
 
 ## 问题反馈
 - 报告 issue: [github issues](https://github.com/alibaba/canal/issues)

admin/admin-ui/pom.xml

@@ -3,7 +3,7 @@
   <parent>
     <artifactId>canal.admin</artifactId>
     <groupId>com.alibaba.otter</groupId>
-    <version>1.1.5-SNAPSHOT</version>
+    <version>1.1.7-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

admin/admin-web/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.admin</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5-SNAPSHOT</version>
+        <version>1.1.7-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>canal.admin-web</artifactId>

admin/admin-web/src/main/bin/restart.sh

@@ -1,5 +1,13 @@
 #!/bin/bash
 
-sh stop.sh
+case $(uname) in
+Linux)
+  bin_abs_path=$(readlink -f $(dirname $0))
+  ;;
+*)
+  bin_abs_path=$(cd $(dirname $0) ||exit ; pwd)
+  ;;
+esac
 
-sh startup.sh
+sh "$bin_abs_path"/stop.sh
+sh "$bin_abs_path"/startup.sh

admin/admin-web/src/main/bin/startup.sh

@@ -56,14 +56,26 @@ in
   exit;;
 esac
 
+JavaVersion=`$JAVA -version 2>&1 |awk 'NR==1{ gsub(/"/,""); print $3 }' | awk  -F '.' '{print $1}'`
 str=`file -L $JAVA | grep 64-bit`
-if [ -n "$str" ]; then
-	JAVA_OPTS="-server -Xms2048m -Xmx3072m"
+
+JAVA_OPTS="$JAVA_OPTS -Xss256k -XX:+AggressiveOpts -XX:-UseBiasedLocking -XX:-OmitStackTraceInFastThrow -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$base/logs"
+if [ $JavaVersion -ge 11 ] ; then
+  #JAVA_OPTS="$JAVA_OPTS -Xlog:gc*:$base_log/gc.log:time "
+  JAVA_OPTS="$JAVA_OPTS"
 else
-	JAVA_OPTS="-server -Xms1024m -Xmx1024m"
+  #JAVA_OPTS="$JAVA_OPTS -Xloggc:$base/logs/canal/gc.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime"
+  JAVA_OPTS="$JAVA_OPTS -XX:+UseFastAccessorMethods -XX:+PrintAdaptiveSizePolicy -XX:+PrintTenuringDistribution"
+fi
+
+if [ -n "$str" ]; then
+	# JAVA_OPTS="-server -Xms2048m -Xmx3072m -Xmn1024m -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC $JAVA_OPTS"
+  # For G1
+  JAVA_OPTS="-server -Xms2g -Xmx3g -XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent $JAVA_OPTS"
+else
+	JAVA_OPTS="-server -Xms1024m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:MaxPermSize=128m $JAVA_OPTS"
 fi
 
-JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent -XX:+PrintAdaptiveSizePolicy -XX:+PrintTenuringDistribution"
 JAVA_OPTS=" $JAVA_OPTS -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8"
 CANAL_OPTS="-DappName=canal-admin"
 
@@ -81,4 +93,4 @@ $JAVA $JAVA_OPTS $JAVA_DEBUG_OPT $CANAL_OPTS -classpath .:$CLASSPATH com.alibaba
 echo $! > $base/bin/admin.pid
 
 echo "cd to $current_path for continue"
-cd $current_path
+cd $current_path

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/common/Threads.java

@@ -12,6 +12,6 @@ public class Threads {
                                                             pool,
                                                             0L,
                                                             TimeUnit.MILLISECONDS,
-                                                            new ArrayBlockingQueue<Runnable>(pool * 20),
+                                                            new ArrayBlockingQueue<>(pool * 20),
                                                             DaemonThreadFactory.daemonThreadFactory);
 }

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/config/WebConfig.java

@@ -84,7 +84,9 @@ public class WebConfig implements WebMvcConfigurer {
             }
         })
             .addPathPatterns("/api/**")
-            .excludePathPatterns("/api/**/config/**")
+            .excludePathPatterns("/api/**/config/server_polling")
+            .excludePathPatterns("/api/**/config/instances_polling")
+            .excludePathPatterns("/api/**/config/instance_polling/**")
             .excludePathPatterns("/api/**/user/login")
             .excludePathPatterns("/api/**/user/logout")
             .excludePathPatterns("/api/**/user/info");

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/connector/SimpleAdminConnector.java

@@ -108,9 +108,7 @@ public class SimpleAdminConnector implements AdminConnector {
             }
 
             connected = true;
-        } catch (IOException e) {
-            throw new ServiceException(e);
-        } catch (NoSuchAlgorithmException e) {
+        } catch (IOException | NoSuchAlgorithmException e) {
             throw new ServiceException(e);
         }
     }

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/connector/SimpleAdminConnectors.java

@@ -2,6 +2,7 @@ package com.alibaba.otter.canal.admin.connector;
 
 import java.util.function.Function;
 
+import org.assertj.core.util.Strings;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.core.env.Environment;
@@ -26,7 +27,8 @@ public class SimpleAdminConnectors {
             connector.connect();
             return function.apply(connector);
         } catch (Exception e) {
-            logger.error("connect to ip:{},port:{},user:{},password:{}, failed", ip, port, user, passwd);
+            logger.error("connect to ip:{},port:{},user:{},password:{}, failed",
+                    ip, port, user, getDesensitizationPassword(passwd));
             logger.error(e.getMessage());
         } finally {
             connector.disconnect();
@@ -34,4 +36,14 @@ public class SimpleAdminConnectors {
 
         return null;
     }
+
+    private static String getDesensitizationPassword(String password) {
+        String defaultPassword = "******";
+
+        if (Strings.isNullOrEmpty(password) || password.length() < 6) {
+            return defaultPassword;
+        } else {
+            return String.format("%s******", password.substring(0, 3));
+        }
+    }
 }

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/controller/CanalClusterController.java

@@ -13,8 +13,8 @@ import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
 
-import com.alibaba.fastjson.JSONArray;
-import com.alibaba.fastjson.JSONObject;
+import com.alibaba.fastjson2.JSONArray;
+import com.alibaba.fastjson2.JSONObject;
 import com.alibaba.otter.canal.admin.model.BaseModel;
 import com.alibaba.otter.canal.admin.model.CanalCluster;
 import com.alibaba.otter.canal.admin.model.NodeServer;

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/controller/PollingConfigController.java

@@ -46,14 +46,15 @@ public class PollingConfigController {
     public BaseModel<CanalConfig> canalConfigPoll(@RequestHeader String user, @RequestHeader String passwd,
                                                   @RequestParam String ip, @RequestParam Integer port,
                                                   @RequestParam String md5, @RequestParam boolean register,
-                                                  @RequestParam String cluster, @PathVariable String env) {
+                                                  @RequestParam String cluster, @RequestParam String name,
+                                                  @PathVariable String env) {
         if (!auth(user, passwd)) {
             throw new RuntimeException("auth :" + user + " is failed");
         }
 
         if (StringUtils.isEmpty(md5) && register) {
             // do something
-            pollingConfigService.autoRegister(ip, port, cluster);
+            pollingConfigService.autoRegister(ip, port, cluster, StringUtils.trimToNull(name));
         }
 
         CanalConfig canalConfig = pollingConfigService.getChangedConfig(ip, port, md5);

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/model/Pager.java

@@ -9,7 +9,7 @@ public class Pager<T> implements Serializable {
     private static final long serialVersionUID = -986577815091763517L;
 
     private Long              count            = 0L;
-    private List<T>           items            = new ArrayList<T>();
+    private List<T>           items            = new ArrayList<>();
     private Integer           page             = 1;
     private Integer           size             = 20;
     private Long              offset           = 0L;

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/service/PollingConfigService.java

@@ -5,7 +5,7 @@ import com.alibaba.otter.canal.admin.model.CanalInstanceConfig;
 
 public interface PollingConfigService {
 
-    public boolean autoRegister(String ip, Integer adminPort, String cluster);
+    public boolean autoRegister(String ip, Integer adminPort, String cluster, String name);
 
     CanalConfig getChangedConfig(String ip, Integer port, String md5);
 

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/service/impl/PollingConfigServiceImpl.java

@@ -2,6 +2,7 @@ package com.alibaba.otter.canal.admin.service.impl;
 
 import java.security.NoSuchAlgorithmException;
 import java.util.List;
+import java.util.Optional;
 import java.util.stream.Collectors;
 
 import org.apache.commons.lang.StringUtils;
@@ -28,11 +29,11 @@ public class PollingConfigServiceImpl implements PollingConfigService {
     @Autowired
     CanalClusterService canalClusterService;
 
-    public boolean autoRegister(String ip, Integer adminPort, String cluster) {
+    public boolean autoRegister(String ip, Integer adminPort, String cluster, String name) {
         NodeServer server = NodeServer.find.query().where().eq("ip", ip).eq("adminPort", adminPort).findOne();
         if (server == null) {
             server = new NodeServer();
-            server.setName(ip);
+            server.setName(Optional.ofNullable(name).orElse(ip));
             server.setIp(ip);
             server.setAdminPort(adminPort);
             server.setTcpPort(adminPort + 1);

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/service/impl/UserServiceImpl.java

@@ -21,6 +21,8 @@ public class UserServiceImpl implements UserService {
 
     private static byte[] seeds = "canal is best!".getBytes();
 
+    private static final Integer PASSWORD_LENGTH = 6;
+
     public User find4Login(String username, String password) {
         if (StringUtils.isEmpty(username) || StringUtils.isEmpty(password)) {
             return null;
@@ -43,6 +45,9 @@ public class UserServiceImpl implements UserService {
     }
 
     public void update(User user) {
+        if (user.getPassword().length() < PASSWORD_LENGTH) {
+            throw new ServiceException("The new password is too short,must more than 6 digits");
+        }
         User userTmp = User.find.query().where().eq("username", user.getUsername()).findOne();
         if (userTmp == null) {
             throw new ServiceException();

admin/admin-web/src/main/resources/canal-template.properties

@@ -16,6 +16,10 @@ canal.metrics.pull.port = 11112
 canal.admin.port = 11110
 canal.admin.user = admin
 canal.admin.passwd = 4ACFE3202A5FF5CF467898FC58AAB1D615029441
+# admin auto register
+#canal.admin.register.auto = true
+#canal.admin.register.cluster =
+#canal.admin.register.name =
 
 canal.zkServers =
 # flush data to zk
@@ -60,6 +64,9 @@ canal.instance.filter.query.ddl = false
 canal.instance.filter.table.error = false
 canal.instance.filter.rows = false
 canal.instance.filter.transaction.entry = false
+canal.instance.filter.dml.insert = false
+canal.instance.filter.dml.update = false
+canal.instance.filter.dml.delete = false
 
 # binlog format/image check
 canal.instance.binlog.format = ROW,STATEMENT,MIXED 
@@ -95,6 +102,9 @@ canal.conf.dir = ../conf
 # auto scan instance dir add/remove and start/stop instance
 canal.auto.scan = true
 canal.auto.scan.interval = 5
+# set this value to 'true' means that when binlog pos not found, skip to latest.
+# WARN: pls keep 'false' in production env, or if you know what you want.
+canal.auto.reset.latest.pos.mode = false
 
 canal.instance.tsdb.spring.xml = classpath:spring/tsdb/h2-tsdb.xml
 #canal.instance.tsdb.spring.xml = classpath:spring/tsdb/mysql-tsdb.xml
@@ -151,6 +161,7 @@ rocketmq.namespace =
 rocketmq.namesrv.addr = 127.0.0.1:9876
 rocketmq.retry.times.when.send.failed = 0
 rocketmq.vip.channel.enabled = false
+rocketmq.tag = 
 
 ##################################################
 ######### 		    RabbitMQ	     #############
@@ -159,4 +170,5 @@ rabbitmq.host =
 rabbitmq.virtual.host =
 rabbitmq.exchange =
 rabbitmq.username =
-rabbitmq.password =
+rabbitmq.password =
+rabbitmq.deliveryMode =

admin/pom.xml

@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.alibaba.otter</groupId>
     <artifactId>canal.admin</artifactId>
-    <version>1.1.5-SNAPSHOT</version>
+    <version>1.1.7-SNAPSHOT</version>
     <packaging>pom</packaging>
     <name>canal admin module for otter ${project.version}</name>
     <modules>
@@ -18,6 +18,7 @@
         <java_source_version>1.8</java_source_version>
         <java_target_version>1.8</java_target_version>
         <file_encoding>UTF-8</file_encoding>
+        <log4j_version>2.17.0</log4j_version>
     </properties>
 
     <dependencyManagement>
@@ -91,6 +92,21 @@
                 <artifactId>ebean</artifactId>
                 <version>11.41.1</version>
             </dependency>
+            <dependency>
+                <groupId>org.apache.logging.log4j</groupId>
+                <artifactId>log4j-core</artifactId>
+                <version>${log4j_version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.logging.log4j</groupId>
+                <artifactId>log4j-api</artifactId>
+                <version>${log4j_version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.logging.log4j</groupId>
+                <artifactId>log4j-to-slf4j</artifactId>
+                <version>${log4j_version}</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 

client-adapter/common/pom.xml

@@ -1,10 +1,9 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5-SNAPSHOT</version>
+        <version>1.1.7-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/YmlConfigBinder.java

@@ -140,7 +140,7 @@ public class YmlConfigBinder {
             MutablePropertySources propertySources = new MutablePropertySources();
             propertySources.addFirst(propertySource);
 
-            PropertiesConfigurationFactory<Object> factory = new PropertiesConfigurationFactory<Object>(target);
+            PropertiesConfigurationFactory<Object> factory = new PropertiesConfigurationFactory<>(target);
             factory.setPropertySources(propertySources);
             factory.setIgnoreInvalidFields(true);
             factory.setIgnoreUnknownFields(true);

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/DefaultPropertyNamePatternsMatcher.java

@@ -25,7 +25,7 @@ class DefaultPropertyNamePatternsMatcher implements PropertyNamePatternsMatcher
     }
 
     protected DefaultPropertyNamePatternsMatcher(char[] delimiters, boolean ignoreCase, String... names){
-        this(delimiters, ignoreCase, new HashSet<String>(Arrays.asList(names)));
+        this(delimiters, ignoreCase, new HashSet<>(Arrays.asList(names)));
     }
 
     DefaultPropertyNamePatternsMatcher(char[] delimiters, boolean ignoreCase, Set<String> names){
@@ -70,8 +70,8 @@ class DefaultPropertyNamePatternsMatcher implements PropertyNamePatternsMatcher
                 return false;
             }
         }
-        for (int i = 0; i < match.length; i++) {
-            if (match[i]) {
+        for (boolean b : match) {
+            if (b) {
                 return true;
             }
         }

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertiesConfigurationFactory.java

@@ -277,7 +277,7 @@ public class PropertiesConfigurationFactory<T> implements FactoryBean<T>, Applic
     }
 
     private Set<String> getNames(Iterable<String> prefixes) {
-        Set<String> names = new LinkedHashSet<String>();
+        Set<String> names = new LinkedHashSet<>();
         if (this.target != null) {
             PropertyDescriptor[] descriptors = BeanUtils.getPropertyDescriptors(this.target.getClass());
             for (PropertyDescriptor descriptor : descriptors) {
@@ -318,7 +318,7 @@ public class PropertiesConfigurationFactory<T> implements FactoryBean<T>, Applic
             // We can filter properties to those starting with the target name, but
             // we can't do a complete filter since we need to trigger the
             // unknown fields check
-            Set<String> relaxedNames = new HashSet<String>();
+            Set<String> relaxedNames = new HashSet<>();
             for (String relaxedTargetName : relaxedTargetNames) {
                 relaxedNames.add(relaxedTargetName);
             }

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyNamePatternsMatcher.java

@@ -9,23 +9,9 @@ package com.alibaba.otter.canal.client.adapter.config.bind;
  */
 interface PropertyNamePatternsMatcher {
 
-    PropertyNamePatternsMatcher ALL  = new PropertyNamePatternsMatcher() {
+    PropertyNamePatternsMatcher ALL  = propertyName -> true;
 
-                                         @Override
-                                         public boolean matches(String propertyName) {
-                                             return true;
-                                         }
-
-                                     };
-
-    PropertyNamePatternsMatcher NONE = new PropertyNamePatternsMatcher() {
-
-                                         @Override
-                                         public boolean matches(String propertyName) {
-                                             return false;
-                                         }
-
-                                     };
+    PropertyNamePatternsMatcher NONE = propertyName -> false;
 
     /**
      * Return {@code true} of the property name matches.

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyValues.java

@@ -37,9 +37,9 @@ public class PropertySourcesPropertyValues implements PropertyValues {
 
     private final PropertyNamePatternsMatcher                  includes;
 
-    private final Map<String, PropertyValue>                   propertyValues      = new LinkedHashMap<String, PropertyValue>();
+    private final Map<String, PropertyValue>                   propertyValues      = new LinkedHashMap<>();
 
-    private final ConcurrentHashMap<String, PropertySource<?>> collectionOwners    = new ConcurrentHashMap<String, PropertySource<?>>();
+    private final ConcurrentHashMap<String, PropertySource<?>> collectionOwners    = new ConcurrentHashMap<>();
 
     private final boolean                                      resolvePlaceholders;
 

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedDataBinder.java

@@ -32,7 +32,7 @@ public class RelaxedDataBinder extends DataBinder {
     private static final Set<Class<?>> EXCLUDED_EDITORS;
 
     static {
-        Set<Class<?>> excluded = new HashSet<Class<?>>();
+        Set<Class<?>> excluded = new HashSet<>();
         excluded.add(FileEditor.class);
         EXCLUDED_EDITORS = Collections.unmodifiableSet(excluded);
     }
@@ -43,7 +43,7 @@ public class RelaxedDataBinder extends DataBinder {
 
     private boolean                       ignoreNestedProperties;
 
-    private MultiValueMap<String, String> nameAliases = new LinkedMultiValueMap<String, String>();
+    private MultiValueMap<String, String> nameAliases = new LinkedMultiValueMap<>();
 
     /**
      * Create a new {@link RelaxedDataBinder} instance.
@@ -89,7 +89,7 @@ public class RelaxedDataBinder extends DataBinder {
      * @param aliases a map of property name to aliases
      */
     public void setNameAliases(Map<String, List<String>> aliases) {
-        this.nameAliases = new LinkedMultiValueMap<String, String>(aliases);
+        this.nameAliases = new LinkedMultiValueMap<>(aliases);
     }
 
     /**
@@ -129,8 +129,8 @@ public class RelaxedDataBinder extends DataBinder {
         BeanWrapper wrapper = new BeanWrapperImpl(target);
         wrapper.setConversionService(new RelaxedConversionService(getConversionService()));
         wrapper.setAutoGrowNestedPaths(true);
-        List<PropertyValue> sortedValues = new ArrayList<PropertyValue>();
-        Set<String> modifiedNames = new HashSet<String>();
+        List<PropertyValue> sortedValues = new ArrayList<>();
+        Set<String> modifiedNames = new HashSet<>();
         List<String> sortedNames = getSortedPropertyNames(propertyValues);
         for (String name : sortedNames) {
             PropertyValue propertyValue = propertyValues.getPropertyValue(name);
@@ -143,7 +143,7 @@ public class RelaxedDataBinder extends DataBinder {
     }
 
     private List<String> getSortedPropertyNames(MutablePropertyValues propertyValues) {
-        List<String> names = new LinkedList<String>();
+        List<String> names = new LinkedList<>();
         for (PropertyValue propertyValue : propertyValues.getPropertyValueList()) {
             names.add(propertyValue.getName());
         }
@@ -161,7 +161,7 @@ public class RelaxedDataBinder extends DataBinder {
      * @param names the names to sort
      */
     private void sortPropertyNames(List<String> names) {
-        for (String name : new ArrayList<String>(names)) {
+        for (String name : new ArrayList<>(names)) {
             int propertyIndex = names.indexOf(name);
             RelaxedDataBinder.BeanPath path = new RelaxedDataBinder.BeanPath(name);
             for (String prefix : path.prefixes()) {
@@ -319,7 +319,7 @@ public class RelaxedDataBinder extends DataBinder {
         }
         Object extend = new LinkedHashMap<String, Object>();
         if (!elementDescriptor.isMap() && path.isArrayIndex(index)) {
-            extend = new ArrayList<Object>();
+            extend = new ArrayList<>();
         }
         wrapper.setPropertyValue(path.prefix(index + 1), extend);
     }
@@ -347,7 +347,7 @@ public class RelaxedDataBinder extends DataBinder {
         }
         Object extend = new LinkedHashMap<String, Object>();
         if (descriptor.isCollection()) {
-            extend = new ArrayList<Object>();
+            extend = new ArrayList<>();
         }
         if (descriptor.getType().equals(Object.class) && path.isLastNode(index)) {
             extend = BLANK;
@@ -411,7 +411,7 @@ public class RelaxedDataBinder extends DataBinder {
         if (aliases == null) {
             return Collections.singleton(name);
         }
-        List<String> nameAndAliases = new ArrayList<String>(aliases.size() + 1);
+        List<String> nameAndAliases = new ArrayList<>(aliases.size() + 1);
         nameAndAliases.add(name);
         nameAndAliases.addAll(aliases);
         return nameAndAliases;
@@ -473,7 +473,7 @@ public class RelaxedDataBinder extends DataBinder {
         }
 
         public List<String> prefixes() {
-            List<String> prefixes = new ArrayList<String>();
+            List<String> prefixes = new ArrayList<>();
             for (int index = 1; index < this.nodes.size(); index++) {
                 prefixes.add(prefix(index));
             }
@@ -485,7 +485,7 @@ public class RelaxedDataBinder extends DataBinder {
         }
 
         private List<PathNode> splitPath(String path) {
-            List<PathNode> nodes = new ArrayList<PathNode>();
+            List<PathNode> nodes = new ArrayList<>();
             String current = extractIndexedPaths(path, nodes);
             for (String name : StringUtils.delimitedListToStringArray(current, ".")) {
                 if (StringUtils.hasText(name)) {
@@ -518,7 +518,7 @@ public class RelaxedDataBinder extends DataBinder {
         }
 
         public void collapseKeys(int index) {
-            List<PathNode> revised = new ArrayList<PathNode>();
+            List<PathNode> revised = new ArrayList<>();
             for (int i = 0; i < index; i++) {
                 revised.add(this.nodes.get(i));
             }
@@ -668,7 +668,7 @@ public class RelaxedDataBinder extends DataBinder {
         private static final Set<String> BENIGN_PROPERTY_SOURCE_NAMES;
 
         static {
-            Set<String> names = new HashSet<String>();
+            Set<String> names = new HashSet<>();
             names.add(StandardEnvironment.SYSTEM_ENVIRONMENT_PROPERTY_SOURCE_NAME);
             names.add(StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME);
             BENIGN_PROPERTY_SOURCE_NAMES = Collections.unmodifiableSet(names);

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedNames.java

@@ -24,7 +24,7 @@ public final class RelaxedNames implements Iterable<String> {
 
     private final String         name;
 
-    private final Set<String>    values                          = new LinkedHashSet<String>();
+    private final Set<String>    values                          = new LinkedHashSet<>();
 
     /**
      * Create a new {@link RelaxedNames} instance.

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/CompositePropertySource.java

@@ -22,7 +22,7 @@ import org.springframework.util.StringUtils;
  */
 public class CompositePropertySource extends EnumerablePropertySource<Object> {
 
-    private final Set<PropertySource<?>> propertySources = new LinkedHashSet<PropertySource<?>>();
+    private final Set<PropertySource<?>> propertySources = new LinkedHashSet<>();
 
     /**
      * Create a new {@code CompositePropertySource}.
@@ -56,7 +56,7 @@ public class CompositePropertySource extends EnumerablePropertySource<Object> {
 
     @Override
     public String[] getPropertyNames() {
-        Set<String> names = new LinkedHashSet<String>();
+        Set<String> names = new LinkedHashSet<>();
         for (PropertySource<?> propertySource : this.propertySources) {
             if (!(propertySource instanceof EnumerablePropertySource)) {
                 throw new IllegalStateException(
@@ -83,7 +83,7 @@ public class CompositePropertySource extends EnumerablePropertySource<Object> {
      * @since 4.1
      */
     public void addFirstPropertySource(PropertySource<?> propertySource) {
-        List<PropertySource<?>> existing = new ArrayList<PropertySource<?>>(this.propertySources);
+        List<PropertySource<?>> existing = new ArrayList<>(this.propertySources);
         this.propertySources.clear();
         this.propertySources.add(propertySource);
         this.propertySources.addAll(existing);

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MutablePropertySources.java

@@ -28,7 +28,7 @@ public class MutablePropertySources implements PropertySources {
 
     private final Log                     logger;
 
-    private final List<PropertySource<?>> propertySourceList = new CopyOnWriteArrayList<PropertySource<?>>();
+    private final List<PropertySource<?>> propertySourceList = new CopyOnWriteArrayList<>();
 
     /**
      * Create a new {@link MutablePropertySources}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/SpringProfileDocumentMatcher.java

@@ -31,7 +31,7 @@ public class SpringProfileDocumentMatcher implements YamlProcessor.DocumentMatch
     }
 
     public void addActiveProfiles(String... profiles) {
-        LinkedHashSet<String> set = new LinkedHashSet<String>(Arrays.asList(this.activeProfiles));
+        LinkedHashSet<String> set = new LinkedHashSet<>(Arrays.asList(this.activeProfiles));
         Collections.addAll(set, profiles);
         this.activeProfiles = set.toArray(new String[set.size()]);
     }
@@ -67,14 +67,14 @@ public class SpringProfileDocumentMatcher implements YamlProcessor.DocumentMatch
 
     private ProfilesMatcher getProfilesMatcher() {
         return (this.activeProfiles.length != 0 ? new ActiveProfilesMatcher(
-            new HashSet<String>(Arrays.asList(this.activeProfiles))) : new EmptyProfilesMatcher());
+                new HashSet<>(Arrays.asList(this.activeProfiles))) : new EmptyProfilesMatcher());
     }
 
     private Set<String> extractProfiles(List<String> profiles, ProfileType type) {
         if (CollectionUtils.isEmpty(profiles)) {
             return null;
         }
-        Set<String> extractedProfiles = new HashSet<String>();
+        Set<String> extractedProfiles = new HashSet<>();
         for (String candidate : profiles) {
             ProfileType candidateType = ProfileType.POSITIVE;
             if (candidate.startsWith("!")) {
@@ -167,7 +167,7 @@ public class SpringProfileDocumentMatcher implements YamlProcessor.DocumentMatch
      */
     static class SpringProperties {
 
-        private List<String> profiles = new ArrayList<String>();
+        private List<String> profiles = new ArrayList<>();
 
         public List<String> getProfiles() {
             return this.profiles;

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlProcessor.java

@@ -130,8 +130,7 @@ public abstract class YamlProcessor {
             if (logger.isDebugEnabled()) {
                 logger.debug("Loading from YAML: " + resource);
             }
-            Reader reader = new UnicodeReader(resource.getInputStream());
-            try {
+            try (Reader reader = new UnicodeReader(resource.getInputStream())) {
                 for (Object object : yaml.loadAll(reader)) {
                     if (object != null && process(asMap(object), callback)) {
                         count++;
@@ -142,10 +141,8 @@ public abstract class YamlProcessor {
                 }
                 if (logger.isDebugEnabled()) {
                     logger.debug(
-                        "Loaded " + count + " document" + (count > 1 ? "s" : "") + " from YAML resource: " + resource);
+                            "Loaded " + count + " document" + (count > 1 ? "s" : "") + " from YAML resource: " + resource);
                 }
-            } finally {
-                reader.close();
             }
         } catch (IOException ex) {
             handleProcessError(resource, ex);
@@ -166,7 +163,7 @@ public abstract class YamlProcessor {
     @SuppressWarnings("unchecked")
     private Map<String, Object> asMap(Object object) {
         // YAML can have numbers as keys
-        Map<String, Object> result = new LinkedHashMap<String, Object>();
+        Map<String, Object> result = new LinkedHashMap<>();
         if (!(object instanceof Map)) {
             // A document can be a text literal
             result.put("document", object);
@@ -249,7 +246,7 @@ public abstract class YamlProcessor {
      * @since 4.1.3
      */
     protected final Map<String, Object> getFlattenedMap(Map<String, Object> source) {
-        Map<String, Object> result = new LinkedHashMap<String, Object>();
+        Map<String, Object> result = new LinkedHashMap<>();
         buildFlattenedMap(result, source, null);
         return result;
     }

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlPropertySourceLoader.java

@@ -3,7 +3,6 @@ package com.alibaba.otter.canal.client.adapter.config.common;
 import java.io.IOException;
 import java.util.LinkedHashMap;
 import java.util.Map;
-import java.util.Properties;
 import java.util.regex.Pattern;
 
 import org.springframework.util.ClassUtils;
@@ -72,14 +71,8 @@ public class YamlPropertySourceLoader implements PropertySourceLoader {
         }
 
         public Map<String, Object> process() {
-            final Map<String, Object> result = new LinkedHashMap<String, Object>();
-            process(new MatchCallback() {
-
-                @Override
-                public void process(Properties properties, Map<String, Object> map) {
-                    result.putAll(getFlattenedMap(map));
-                }
-            });
+            final Map<String, Object> result = new LinkedHashMap<>();
+            process((properties, map) -> result.putAll(getFlattenedMap(map)));
             return result;
         }
 

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java

@@ -54,6 +54,8 @@ public class CanalClientConfig {
     // canal adapters 配置
     private List<CanalAdapter> canalAdapters;
 
+    private Boolean terminateOnException = false;
+
     public String getCanalServerHost() {
         return canalServerHost;
     }
@@ -222,6 +224,14 @@ public class CanalClientConfig {
         this.namespace = namespace;
     }
 
+    public Boolean getTerminateOnException() {
+        return terminateOnException;
+    }
+
+    public void setTerminateOnException(Boolean terminateOnException) {
+        this.terminateOnException = terminateOnException;
+    }
+
     public static class CanalAdapter {
 
         private String      instance; // 实例名

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/ExtensionLoader.java

@@ -6,6 +6,7 @@ import java.io.InputStreamReader;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLClassLoader;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Arrays;
@@ -259,7 +260,7 @@ public class ExtensionLoader<T> {
             }
         }
 
-        Map<String, Class<?>> extensionClasses = new HashMap<String, Class<?>>();
+        Map<String, Class<?>> extensionClasses = new HashMap<>();
 
         // 1. plugin folder，customized extension classLoader （jar_dir/plugin）
         String dir = File.separator + this.getJarDirectoryPath() + File.separator + "plugin";
@@ -317,10 +318,8 @@ public class ExtensionLoader<T> {
                 while (urls.hasMoreElements()) {
                     URL url = urls.nextElement();
                     try {
-                        BufferedReader reader = null;
-                        try {
-                            reader = new BufferedReader(new InputStreamReader(url.openStream(), "utf-8"));
-                            String line = null;
+                        try (BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream(), StandardCharsets.UTF_8))) {
+                            String line;
                             while ((line = reader.readLine()) != null) {
                                 final int ci = line.indexOf('#');
                                 if (ci >= 0) line = line.substring(0, ci);
@@ -340,10 +339,10 @@ public class ExtensionLoader<T> {
                                             // classLoader);
                                             if (!type.isAssignableFrom(clazz)) {
                                                 throw new IllegalStateException(
-                                                    "Error when load extension class(interface: " + type
-                                                                                + ", class line: " + clazz.getName()
-                                                                                + "), class " + clazz.getName()
-                                                                                + "is not subtype of interface.");
+                                                        "Error when load extension class(interface: " + type
+                                                                + ", class line: " + clazz.getName()
+                                                                + "), class " + clazz.getName()
+                                                                + "is not subtype of interface.");
                                             } else {
                                                 try {
                                                     clazz.getConstructor(type);
@@ -361,10 +360,10 @@ public class ExtensionLoader<T> {
                                                             } else if (c != clazz) {
                                                                 cachedNames.remove(clazz);
                                                                 throw new IllegalStateException(
-                                                                    "Duplicate extension " + type.getName() + " name "
-                                                                                                + n + " on "
-                                                                                                + c.getName() + " and "
-                                                                                                + clazz.getName());
+                                                                        "Duplicate extension " + type.getName() + " name "
+                                                                                + n + " on "
+                                                                                + c.getName() + " and "
+                                                                                + clazz.getName());
                                                             }
                                                         }
                                                     }
@@ -373,19 +372,15 @@ public class ExtensionLoader<T> {
                                         }
                                     } catch (Throwable t) {
                                         IllegalStateException e = new IllegalStateException(
-                                            "Failed to load extension class(interface: " + type + ", class line: "
-                                                                                            + line + ") in " + url
-                                                                                            + ", cause: "
-                                                                                            + t.getMessage(),
-                                            t);
+                                                "Failed to load extension class(interface: " + type + ", class line: "
+                                                        + line + ") in " + url
+                                                        + ", cause: "
+                                                        + t.getMessage(),
+                                                t);
                                         exceptions.put(line, e);
                                     }
                                 }
                             } // end of while read lines
-                        } finally {
-                            if (reader != null) {
-                                reader.close();
-                            }
                         }
                     } catch (Throwable t) {
                         logger.error("Exception when load extension class(interface: " + type + ", class file: " + url

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/FileName2KeyMapping.java

@@ -0,0 +1,28 @@
+package com.alibaba.otter.canal.client.adapter.support;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Created by @author zhuchao on @date 2021/11/11.
+ */
+public class FileName2KeyMapping {
+
+    private static Map<String, String> MAP = new ConcurrentHashMap<>();
+
+    public static void register(String type, String fileName, String key) {
+        MAP.putIfAbsent(join(type, fileName), key);
+    }
+
+    public static void unregister(String type, String fileName) {
+        MAP.remove(join(type, fileName));
+    }
+
+    public static String getKey(String type, String fileName) {
+        return MAP.get(join(type, fileName));
+    }
+
+    private static String join(String type, String fileName) {
+        return type + "|" + fileName;
+    }
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MappingConfigsLoader.java

@@ -55,7 +55,7 @@ public class MappingConfigsLoader {
         }
         if (filePath.exists()) {
             String fileName = filePath.getName();
-            if (!fileName.endsWith(".yml")) {
+            if (!(fileName.endsWith(".yml") || fileName.endsWith(".yaml"))) {
                 return null;
             }
             try (InputStream in = new FileInputStream(filePath)) {

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java

@@ -4,7 +4,6 @@ import java.util.*;
 
 import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
 import com.alibaba.otter.canal.protocol.CanalEntry;
-import com.alibaba.otter.canal.protocol.FlatMessage;
 import com.alibaba.otter.canal.protocol.Message;
 
 /**
@@ -20,7 +19,7 @@ public class MessageUtil {
             return null;
         }
         List<CanalEntry.Entry> entries = message.getEntries();
-        List<Dml> dmls = new ArrayList<Dml>(entries.size());
+        List<Dml> dmls = new ArrayList<>(entries.size());
         for (CanalEntry.Entry entry : entries) {
             if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
                 || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
@@ -136,7 +135,7 @@ public class MessageUtil {
         if (commonMessages == null) {
             return new ArrayList<>();
         }
-        List<Dml> dmls = new ArrayList<Dml>(commonMessages.size());
+        List<Dml> dmls = new ArrayList<>(commonMessages.size());
         for (CommonMessage commonMessage : commonMessages) {
             Dml dml = flatMessage2Dml(destination, groupId, commonMessage);
             if (dml != null) {

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/URLClassExtensionLoader.java

@@ -7,7 +7,8 @@ import java.util.Enumeration;
 import java.util.NoSuchElementException;
 
 public class URLClassExtensionLoader extends URLClassLoader {
-    public URLClassExtensionLoader(URL[] urls) {
+
+    public URLClassExtensionLoader(URL[] urls){
         super(urls);
     }
 
@@ -18,9 +19,11 @@ public class URLClassExtensionLoader extends URLClassLoader {
             return c;
         }
 
-        if (name.startsWith("java.") || name.startsWith("org.slf4j.")
-                || name.startsWith("org.apache.logging")
-                || name.startsWith("org.apache.commons.logging.")) {
+        if (name.startsWith("java.") || name.startsWith("org.slf4j.") || name.startsWith("org.apache.logging")
+            || name.startsWith("org.apache.zookeeper.") || name.startsWith("org.I0Itec.zkclient.")
+            || name.startsWith("org.apache.commons.logging.")
+            || name.startsWith("com.alibaba.druid")
+            ) {
             // || name.startsWith("org.apache.hadoop."))
             // {
             c = super.loadClass(name);

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java

@@ -30,6 +30,8 @@ public class Util {
 
     private static final Logger logger = LoggerFactory.getLogger(Util.class);
 
+    public static final String AUTO_GENERATED_PREFIX = "AUTO_GENERATED_";
+
     /**
      * 通过DS执行sql
      */

client-adapter/common/src/test/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtilTest.java

@@ -1,24 +1,13 @@
 package com.alibaba.otter.canal.client.adapter.support;
 
-import static org.mockito.AdditionalMatchers.or;
-import static org.mockito.Matchers.isA;
-import static org.mockito.Matchers.isNull;
-import static org.powermock.api.mockito.PowerMockito.mockStatic;
-
-import com.alibaba.otter.canal.client.adapter.support.JdbcTypeUtil;
-import com.alibaba.otter.canal.client.adapter.support.Util;
 import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
-import org.powermock.api.mockito.PowerMockito;
-import org.powermock.core.classloader.annotations.PrepareForTest;
 import org.powermock.modules.junit4.PowerMockRunner;
 
-import java.lang.reflect.Array;
-
 @RunWith(PowerMockRunner.class)
 public class JdbcTypeUtilTest {
 

client-adapter/common/src/test/java/com/alibaba/otter/canal/client/adapter/support/UtilTest.java

@@ -3,9 +3,7 @@ package com.alibaba.otter.canal.client.adapter.support;
 import static org.mockito.AdditionalMatchers.or;
 import static org.mockito.Matchers.isA;
 import static org.mockito.Matchers.isNull;
-import static org.powermock.api.mockito.PowerMockito.mockStatic;
 
-import com.alibaba.otter.canal.client.adapter.support.Util;
 import com.diffblue.deeptestutils.mock.DTUMemberMatcher;
 import org.apache.commons.lang.StringUtils;
 import org.junit.Assert;

client-adapter/es6x/pom.xml

@@ -1,11 +1,9 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5-SNAPSHOT</version>
+        <version>1.1.7-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
@@ -29,22 +27,22 @@
         <dependency>
             <groupId>org.elasticsearch</groupId>
             <artifactId>elasticsearch</artifactId>
-            <version>6.4.3</version>
+            <version>6.8.22</version>
         </dependency>
         <dependency>
             <groupId>org.elasticsearch.client</groupId>
             <artifactId>transport</artifactId>
-            <version>6.4.3</version>
+            <version>6.8.22</version>
         </dependency>
         <dependency>
             <groupId>org.elasticsearch.client</groupId>
             <artifactId>elasticsearch-rest-client</artifactId>
-            <version>6.4.3</version>
+            <version>6.8.22</version>
         </dependency>
         <dependency>
             <groupId>org.elasticsearch.client</groupId>
             <artifactId>elasticsearch-rest-high-level-client</artifactId>
-            <version>6.4.3</version>
+            <version>6.8.22</version>
         </dependency>
 
         <dependency>

client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/support/ES6xTemplate.java

@@ -316,13 +316,17 @@ public class ES6xTemplate implements ESTemplate {
     }
 
     @Override
-    public Object getESDataFromDmlData(ESSyncConfig.ESMapping mapping, Map<String, Object> dmlData,
+    public Object getESDataFromDmlData(ESSyncConfig.ESMapping mapping,String owner, Map<String, Object> dmlData,
                                        Map<String, Object> dmlOld, Map<String, Object> esFieldData) {
         SchemaItem schemaItem = mapping.getSchemaItem();
         String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
         Object resultIdVal = null;
         for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
-            String columnName = fieldItem.getColumnItems().iterator().next().getColumnName();
+            ColumnItem columnItem = fieldItem.getColumnItems().iterator().next();
+            if (!columnItem.getOwner().equals(owner)) {
+                continue;
+            }
+            String columnName = columnItem.getColumnName();
 
             if (fieldItem.getFieldName().equals(idFieldName)) {
                 resultIdVal = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName);

client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/support/ESConnection.java

@@ -2,7 +2,10 @@ package com.alibaba.otter.canal.client.adapter.es6x.support;
 
 import java.io.IOException;
 import java.net.InetAddress;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.UnknownHostException;
+import java.util.Arrays;
 import java.util.Map;
 
 import org.apache.commons.lang.StringUtils;
@@ -78,14 +81,7 @@ public class ESConnection {
                     Integer.parseInt(host.substring(i + 1))));
             }
         } else {
-            HttpHost[] httpHosts = new HttpHost[hosts.length];
-            for (int i = 0; i < hosts.length; i++) {
-                String host = hosts[i];
-                int j = host.indexOf(":");
-                HttpHost httpHost = new HttpHost(InetAddress.getByName(host.substring(0, j)),
-                    Integer.parseInt(host.substring(j + 1)));
-                httpHosts[i] = httpHost;
-            }
+            HttpHost[] httpHosts = Arrays.stream(hosts).map(this::createHttpHost).toArray(HttpHost[]::new);
             RestClientBuilder restClientBuilder = RestClient.builder(httpHosts);
             String nameAndPwd = properties.get("security.auth");
             if (StringUtils.isNotEmpty(nameAndPwd) && nameAndPwd.contains(":")) {
@@ -154,7 +150,13 @@ public class ESConnection {
                 logger.error(e.getMessage(), e);
                 return null;
             }
-            mappingMetaData = mappings.get(index).get(type);
+
+            //通过别名查询mapping返回的是真实索引名称，mappings.get(index)返回null，为兼容别名情况修改如下：
+            ImmutableOpenMap<String, MappingMetaData> esIndex = mappings.get(index);
+            if(esIndex == null){
+                esIndex = mappings.valuesIt().next();
+            }
+            mappingMetaData = esIndex.get(type);
         }
         return mappingMetaData;
     }
@@ -507,4 +509,17 @@ public class ESConnection {
     public void setRestHighLevelClient(RestHighLevelClient restHighLevelClient) {
         this.restHighLevelClient = restHighLevelClient;
     }
+
+    private HttpHost createHttpHost(String uriStr) {
+        URI uri = URI.create(uriStr);
+        if (!org.springframework.util.StringUtils.hasLength(uri.getUserInfo())) {
+            return HttpHost.create(uri.toString());
+        }
+        try {
+            return HttpHost.create(new URI(uri.getScheme(), null, uri.getHost(), uri.getPort(), uri.getPath(),
+                                           uri.getQuery(), uri.getFragment()).toString());
+        } catch (URISyntaxException ex) {
+            throw new IllegalStateException(ex);
+        }
+    }
 }

client-adapter/es7x/pom.xml

@@ -1,11 +1,9 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5-SNAPSHOT</version>
+        <version>1.1.7-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/support/ES7xTemplate.java

@@ -287,8 +287,7 @@ public class ES7xTemplate implements ESTemplate {
     }
 
     @Override
-    public Object getESDataFromDmlData(ESMapping mapping, Map<String, Object> dmlData,
-                                       Map<String, Object> esFieldData) {
+    public Object getESDataFromDmlData(ESMapping mapping, Map<String, Object> dmlData, Map<String, Object> esFieldData) {
         SchemaItem schemaItem = mapping.getSchemaItem();
         String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
         Object resultIdVal = null;
@@ -312,13 +311,17 @@ public class ES7xTemplate implements ESTemplate {
     }
 
     @Override
-    public Object getESDataFromDmlData(ESMapping mapping, Map<String, Object> dmlData, Map<String, Object> dmlOld,
+    public Object getESDataFromDmlData(ESMapping mapping,String owner, Map<String, Object> dmlData, Map<String, Object> dmlOld,
                                        Map<String, Object> esFieldData) {
         SchemaItem schemaItem = mapping.getSchemaItem();
         String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
         Object resultIdVal = null;
         for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
-            String columnName = fieldItem.getColumnItems().iterator().next().getColumnName();
+            ColumnItem columnItem = fieldItem.getColumnItems().iterator().next();
+            if (!columnItem.getOwner().equals(owner)) {
+                continue;
+            }
+            String columnName = columnItem.getColumnName();
 
             if (fieldItem.getFieldName().equals(idFieldName)) {
                 resultIdVal = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName);

client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/support/ESConnection.java

@@ -2,7 +2,10 @@ package com.alibaba.otter.canal.client.adapter.es7x.support;
 
 import java.io.IOException;
 import java.net.InetAddress;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.UnknownHostException;
+import java.util.Arrays;
 import java.util.Map;
 
 import org.apache.commons.lang.StringUtils;
@@ -77,14 +80,7 @@ public class ESConnection {
                     Integer.parseInt(host.substring(i + 1))));
             }
         } else {
-            HttpHost[] httpHosts = new HttpHost[hosts.length];
-            for (int i = 0; i < hosts.length; i++) {
-                String host = hosts[i];
-                int j = host.indexOf(":");
-                HttpHost httpHost = new HttpHost(InetAddress.getByName(host.substring(0, j)),
-                    Integer.parseInt(host.substring(j + 1)));
-                httpHosts[i] = httpHost;
-            }
+            HttpHost[] httpHosts = Arrays.stream(hosts).map(this::createHttpHost).toArray(HttpHost[]::new);
             RestClientBuilder restClientBuilder = RestClient.builder(httpHosts);
             String nameAndPwd = properties.get("security.auth");
             if (StringUtils.isNotEmpty(nameAndPwd) && nameAndPwd.contains(":")) {
@@ -501,4 +497,17 @@ public class ESConnection {
     public void setRestHighLevelClient(RestHighLevelClient restHighLevelClient) {
         this.restHighLevelClient = restHighLevelClient;
     }
+
+    private HttpHost createHttpHost(String uriStr) {
+        URI uri = URI.create(uriStr);
+        if (!org.springframework.util.StringUtils.hasLength(uri.getUserInfo())) {
+            return HttpHost.create(uri.toString());
+        }
+        try {
+            return HttpHost.create(new URI(uri.getScheme(), null, uri.getHost(), uri.getPort(), uri.getPath(),
+                                           uri.getQuery(), uri.getFragment()).toString());
+        } catch (URISyntaxException ex) {
+            throw new IllegalStateException(ex);
+        }
+    }
 }

client-adapter/escore/pom.xml

@@ -1,11 +1,9 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5-SNAPSHOT</version>
+        <version>1.1.7-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
@@ -22,8 +20,9 @@
             <scope>provided</scope>
         </dependency>
         <dependency>
-            <groupId>com.alibaba.fastsql</groupId>
-            <artifactId>fastsql</artifactId>
+            <groupId>com.alibaba</groupId>
+            <artifactId>druid</artifactId>
+            <scope>provided</scope>
         </dependency>
 
         <dependency>

client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/ESAdapter.java

@@ -18,10 +18,7 @@ import com.alibaba.otter.canal.client.adapter.es.core.config.SqlParser;
 import com.alibaba.otter.canal.client.adapter.es.core.monitor.ESConfigMonitor;
 import com.alibaba.otter.canal.client.adapter.es.core.service.ESSyncService;
 import com.alibaba.otter.canal.client.adapter.es.core.support.ESTemplate;
-import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
-import com.alibaba.otter.canal.client.adapter.support.Dml;
-import com.alibaba.otter.canal.client.adapter.support.EtlResult;
-import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
+import com.alibaba.otter.canal.client.adapter.support.*;
 
 /**
  * ES外部适配器
@@ -42,6 +39,8 @@ public abstract class ESAdapter implements OuterAdapter {
 
     protected Properties                             envProperties;
 
+    protected OuterAdapterConfig                     configuration;
+
     public ESSyncService getEsSyncService() {
         return esSyncService;
     }
@@ -58,23 +57,13 @@ public abstract class ESAdapter implements OuterAdapter {
     public void init(OuterAdapterConfig configuration, Properties envProperties) {
         try {
             this.envProperties = envProperties;
+            this.configuration = configuration;
             Map<String, ESSyncConfig> esSyncConfigTmp = ESSyncConfigLoader.load(envProperties);
             // 过滤不匹配的key的配置
             esSyncConfigTmp.forEach((key, config) -> {
-                if ((config.getOuterAdapterKey() == null && configuration.getKey() == null)
-                    || (config.getOuterAdapterKey() != null && config.getOuterAdapterKey()
-                        .equalsIgnoreCase(configuration.getKey()))) {
-                    esSyncConfig.put(key, config);
-                }
+                addConfig(key, config);
             });
 
-            for (Map.Entry<String, ESSyncConfig> entry : esSyncConfig.entrySet()) {
-                String configName = entry.getKey();
-                ESSyncConfig config = entry.getValue();
-
-                addSyncConfigToCache(configName, config);
-            }
-
             esSyncService = new ESSyncService(esTemplate);
 
             esConfigMonitor = new ESConfigMonitor();
@@ -103,12 +92,12 @@ public abstract class ESAdapter implements OuterAdapter {
         String table = dml.getTable();
         Map<String, ESSyncConfig> configMap;
         if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
-            configMap = dbTableEsSyncConfig.get(StringUtils.trimToEmpty(dml.getDestination()) + "-"
-                                                + StringUtils.trimToEmpty(dml.getGroupId()) + "_" + database + "-"
-                                                + table);
+            configMap = dbTableEsSyncConfig
+                .get(StringUtils.trimToEmpty(dml.getDestination()) + "-" + StringUtils.trimToEmpty(dml.getGroupId())
+                     + "_" + database + "-" + table);
         } else {
-            configMap = dbTableEsSyncConfig.get(StringUtils.trimToEmpty(dml.getDestination()) + "_" + database + "-"
-                                                + table);
+            configMap = dbTableEsSyncConfig
+                .get(StringUtils.trimToEmpty(dml.getDestination()) + "_" + database + "-" + table);
         }
 
         if (configMap != null && !configMap.values().isEmpty()) {
@@ -138,7 +127,7 @@ public abstract class ESAdapter implements OuterAdapter {
         return null;
     }
 
-    public void addSyncConfigToCache(String configName, ESSyncConfig config) {
+    private void addSyncConfigToCache(String configName, ESSyncConfig config) {
         Properties envProperties = this.envProperties;
         SchemaItem schemaItem = SqlParser.parse(config.getEsMapping().getSql());
         config.getEsMapping().setSchemaItem(schemaItem);
@@ -153,30 +142,60 @@ public abstract class ESAdapter implements OuterAdapter {
             throw new RuntimeException("Not found the schema of jdbc-url: " + config.getDataSourceKey());
         }
         String schema = matcher.group(2);
-
-        schemaItem.getAliasTableItems()
-            .values()
-            .forEach(tableItem -> {
-                Map<String, ESSyncConfig> esSyncConfigMap;
-                if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
-                    esSyncConfigMap = dbTableEsSyncConfig.computeIfAbsent(StringUtils.trimToEmpty(config.getDestination())
-                                                                          + "-"
-                                                                          + StringUtils.trimToEmpty(config.getGroupId())
-                                                                          + "_"
-                                                                          + schema
-                                                                          + "-"
-                                                                          + tableItem.getTableName(),
+        schemaItem.getAliasTableItems().values().forEach(tableItem -> {
+            Map<String, ESSyncConfig> esSyncConfigMap;
+            String schemaKey = tableItem.getSchema() == null ? schema : tableItem.getSchema();
+            if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
+                esSyncConfigMap = dbTableEsSyncConfig
+                    .computeIfAbsent(StringUtils.trimToEmpty(config.getDestination()) + "-"
+                                     + StringUtils.trimToEmpty(config.getGroupId()) + "_" + schemaKey + "-"
+                                     + tableItem.getTableName(),
                         k -> new ConcurrentHashMap<>());
-                } else {
-                    esSyncConfigMap = dbTableEsSyncConfig.computeIfAbsent(StringUtils.trimToEmpty(config.getDestination())
-                                                                          + "_"
-                                                                          + schema
-                                                                          + "-"
-                                                                          + tableItem.getTableName(),
-                        k -> new ConcurrentHashMap<>());
-                }
+            } else {
+                esSyncConfigMap = dbTableEsSyncConfig.computeIfAbsent(
+                    StringUtils.trimToEmpty(config.getDestination()) + "_" + schemaKey + "-" + tableItem.getTableName(),
+                    k -> new ConcurrentHashMap<>());
+            }
 
-                esSyncConfigMap.put(configName, config);
-            });
+            esSyncConfigMap.put(configName, config);
+        });
+    }
+
+    public boolean addConfig(String fileName, ESSyncConfig config) {
+        if (match(config)) {
+            esSyncConfig.put(fileName, config);
+            addSyncConfigToCache(fileName, config);
+            FileName2KeyMapping.register(getClass().getAnnotation(SPI.class).value(), fileName, configuration.getKey());
+            return true;
+        }
+        return false;
+    }
+
+    public void updateConfig(String fileName, ESSyncConfig config) {
+        if (config.getOuterAdapterKey() != null && !config.getOuterAdapterKey().equals(configuration.getKey())) {
+            // 理论上不允许改这个 因为本身就是通过这个关联起Adapter和Config的
+            throw new RuntimeException("not allow to change outAdapterKey");
+        }
+        esSyncConfig.put(fileName, config);
+        addSyncConfigToCache(fileName, config);
+    }
+
+    public void deleteConfig(String fileName) {
+        esSyncConfig.remove(fileName);
+        for (Map<String, ESSyncConfig> configMap : dbTableEsSyncConfig.values()) {
+            if (configMap != null) {
+                configMap.remove(fileName);
+            }
+        }
+        FileName2KeyMapping.unregister(getClass().getAnnotation(SPI.class).value(), fileName);
+    }
+
+    private boolean match(ESSyncConfig config) {
+        boolean sameMatch = config.getOuterAdapterKey() != null
+                            && config.getOuterAdapterKey().equalsIgnoreCase(configuration.getKey());
+        boolean prefixMatch = config.getOuterAdapterKey() == null && configuration.getKey()
+            .startsWith(StringUtils
+                .join(new String[] { Util.AUTO_GENERATED_PREFIX, config.getDestination(), config.getGroupId() }, '-'));
+        return sameMatch || prefixMatch;
     }
 }

client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/SqlParser.java

@@ -1,31 +1,31 @@
 package com.alibaba.otter.canal.client.adapter.es.core.config;
 
-import static com.alibaba.fastsql.sql.ast.expr.SQLBinaryOperator.BooleanAnd;
-import static com.alibaba.fastsql.sql.ast.expr.SQLBinaryOperator.Equality;
+import static com.alibaba.druid.sql.ast.expr.SQLBinaryOperator.BooleanAnd;
+import static com.alibaba.druid.sql.ast.expr.SQLBinaryOperator.Equality;
 
 import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.Collectors;
 
-import com.alibaba.fastsql.sql.SQLUtils;
-import com.alibaba.fastsql.sql.ast.SQLExpr;
-import com.alibaba.fastsql.sql.ast.expr.SQLBinaryOpExpr;
-import com.alibaba.fastsql.sql.ast.expr.SQLCaseExpr;
-import com.alibaba.fastsql.sql.ast.expr.SQLCharExpr;
-import com.alibaba.fastsql.sql.ast.expr.SQLIdentifierExpr;
-import com.alibaba.fastsql.sql.ast.expr.SQLMethodInvokeExpr;
-import com.alibaba.fastsql.sql.ast.expr.SQLPropertyExpr;
-import com.alibaba.fastsql.sql.ast.statement.SQLExprTableSource;
-import com.alibaba.fastsql.sql.ast.statement.SQLJoinTableSource;
-import com.alibaba.fastsql.sql.ast.statement.SQLSelectGroupByClause;
-import com.alibaba.fastsql.sql.ast.statement.SQLSelectItem;
-import com.alibaba.fastsql.sql.ast.statement.SQLSelectStatement;
-import com.alibaba.fastsql.sql.ast.statement.SQLSubqueryTableSource;
-import com.alibaba.fastsql.sql.ast.statement.SQLTableSource;
-import com.alibaba.fastsql.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
-import com.alibaba.fastsql.sql.dialect.mysql.parser.MySqlStatementParser;
-import com.alibaba.fastsql.sql.parser.ParserException;
-import com.alibaba.fastsql.sql.parser.SQLStatementParser;
+import com.alibaba.druid.sql.SQLUtils;
+import com.alibaba.druid.sql.ast.SQLExpr;
+import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr;
+import com.alibaba.druid.sql.ast.expr.SQLCaseExpr;
+import com.alibaba.druid.sql.ast.expr.SQLCharExpr;
+import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr;
+import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr;
+import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr;
+import com.alibaba.druid.sql.ast.statement.SQLExprTableSource;
+import com.alibaba.druid.sql.ast.statement.SQLJoinTableSource;
+import com.alibaba.druid.sql.ast.statement.SQLSelectGroupByClause;
+import com.alibaba.druid.sql.ast.statement.SQLSelectItem;
+import com.alibaba.druid.sql.ast.statement.SQLSelectStatement;
+import com.alibaba.druid.sql.ast.statement.SQLSubqueryTableSource;
+import com.alibaba.druid.sql.ast.statement.SQLTableSource;
+import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
+import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser;
+import com.alibaba.druid.sql.parser.ParserException;
+import com.alibaba.druid.sql.parser.SQLStatementParser;
 import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.ColumnItem;
 import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem;
 import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.RelationFieldsPair;
@@ -230,8 +230,7 @@ public class SqlParser {
             }
             FieldItem rightFieldItem = new FieldItem();
             visitColumn(sqlBinaryOpExpr.getRight(), rightFieldItem);
-            if (rightFieldItem.getColumnItems().size() != 1 || rightFieldItem.isMethod()
-                || rightFieldItem.isBinaryOp()) {
+            if (rightFieldItem.getColumnItems().size() != 1 || rightFieldItem.isMethod() || rightFieldItem.isBinaryOp()) {
                 throw new UnsupportedOperationException("Unsupported for complex of on-condition");
             }
             tableItem.getRelationFields().add(new RelationFieldsPair(leftFieldItem, rightFieldItem));

client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/monitor/ESConfigMonitor.java

@@ -1,21 +1,18 @@
 package com.alibaba.otter.canal.client.adapter.es.core.monitor;
 
-import java.io.File;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.commons.io.filefilter.FileFilterUtils;
-import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
-import org.apache.commons.io.monitor.FileAlterationMonitor;
-import org.apache.commons.io.monitor.FileAlterationObserver;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
 import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter;
 import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
 import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
 import com.alibaba.otter.canal.client.adapter.support.Util;
+import java.io.File;
+import java.util.Properties;
+import org.apache.commons.io.filefilter.FileFilterUtils;
+import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
+import org.apache.commons.io.monitor.FileAlterationMonitor;
+import org.apache.commons.io.monitor.FileAlterationObserver;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class ESConfigMonitor {
 
@@ -36,7 +33,7 @@ public class ESConfigMonitor {
         File confDir = Util.getConfDirPath(adapterName);
         try {
             FileAlterationObserver observer = new FileAlterationObserver(confDir,
-                FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml")));
+                    FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml")));
             FileListener listener = new FileListener();
             observer.addListener(listener);
             fileMonitor = new FileAlterationMonitor(3000, observer);
@@ -69,9 +66,14 @@ public class ESConfigMonitor {
                     null,
                     envProperties);
                 if (config != null) {
+                    // 这里要记得设置esVersion bugfix
+                    config.setEsVersion(adapterName);
                     config.validate();
-                    addConfigToCache(file, config);
-                    logger.info("Add a new es mapping config: {} to canal adapter", file.getName());
+                    boolean result = esAdapter.addConfig(file.getName(), config);
+                    if (result) {
+                        logger.info("Add a new es mapping config: {} to canal adapter",
+                                file.getName());
+                    }
                 }
             } catch (Exception e) {
                 logger.error(e.getMessage(), e);
@@ -99,12 +101,10 @@ public class ESConfigMonitor {
                     if (config == null) {
                         return;
                     }
+                    // 这里要记得设置esVersion bugfix
+                    config.setEsVersion(adapterName);
                     config.validate();
-                    if (esAdapter.getEsSyncConfig().containsKey(file.getName())) {
-                        deleteConfigFromCache(file);
-                    }
-                    addConfigToCache(file, config);
-
+                    esAdapter.updateConfig(file.getName(), config);
                     logger.info("Change a es mapping config: {} of canal adapter", file.getName());
                 }
             } catch (Exception e) {
@@ -118,29 +118,12 @@ public class ESConfigMonitor {
 
             try {
                 if (esAdapter.getEsSyncConfig().containsKey(file.getName())) {
-                    deleteConfigFromCache(file);
-
+                    esAdapter.deleteConfig(file.getName());
                     logger.info("Delete a es mapping config: {} of canal adapter", file.getName());
                 }
             } catch (Exception e) {
                 logger.error(e.getMessage(), e);
             }
         }
-
-        private void addConfigToCache(File file, ESSyncConfig config) {
-            esAdapter.getEsSyncConfig().put(file.getName(), config);
-
-            esAdapter.addSyncConfigToCache(file.getName(), config);
-        }
-
-        private void deleteConfigFromCache(File file) {
-            esAdapter.getEsSyncConfig().remove(file.getName());
-            for (Map<String, ESSyncConfig> configMap : esAdapter.getDbTableEsSyncConfig().values()) {
-                if (configMap != null) {
-                    configMap.remove(file.getName());
-                }
-            }
-
-        }
     }
 }

client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/service/ESSyncService.java

@@ -1,19 +1,15 @@
 package com.alibaba.otter.canal.client.adapter.es.core.service;
 
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
 import javax.sql.DataSource;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.serializer.SerializerFeature;
-import com.alibaba.fastsql.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
+import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
+import com.alibaba.fastjson2.JSON;
+import com.alibaba.fastjson2.JSONWriter;
 import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
 import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping;
 import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem;
@@ -79,7 +75,7 @@ public class ESSyncService {
                 esSyncConfigs
                     .forEach(esSyncConfig -> configIndexes.append(esSyncConfig.getEsMapping().get_index()).append(" "));
                 logger.debug("DML: {} \nAffected indexes: {}",
-                    JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue),
+                    JSON.toJSONString(dml, JSONWriter.Feature.WriteNulls),
                     configIndexes.toString());
             }
         }
@@ -209,7 +205,7 @@ public class ESSyncService {
 
             if (schemaItem.getAliasTableItems().size() == 1 && schemaItem.isAllFieldsSimple()) {
                 // ------单表 & 所有字段都为简单字段------
-                singleTableSimpleFiledUpdate(config, dml, data, old);
+                singleTableSimpleFiledUpdate(config, schemaItem.getMainTable().getAlias(), dml, data, old);
             } else {
                 // ------主表 查询sql来更新------
                 if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) {
@@ -263,7 +259,7 @@ public class ESSyncService {
 
                     // 判断主键和所更新的字段是否全为简单字段
                     if (idFieldSimple && allUpdateFieldSimple && !fkChanged) {
-                        singleTableSimpleFiledUpdate(config, dml, data, old);
+                        singleTableSimpleFiledUpdate(config, schemaItem.getMainTable().getAlias(), dml, data, old);
                     } else {
                         mainTableUpdate(config, dml, data, old);
                     }
@@ -810,12 +806,12 @@ public class ESSyncService {
      * @param data 单行data数据
      * @param old 单行old数据
      */
-    private void singleTableSimpleFiledUpdate(ESSyncConfig config, Dml dml, Map<String, Object> data,
+    private void singleTableSimpleFiledUpdate(ESSyncConfig config, String owner, Dml dml, Map<String, Object> data,
                                               Map<String, Object> old) {
         ESMapping mapping = config.getEsMapping();
         Map<String, Object> esFieldData = new LinkedHashMap<>();
 
-        Object idVal = esTemplate.getESDataFromDmlData(mapping, data, old, esFieldData);
+        Object idVal = esTemplate.getESDataFromDmlData(mapping, owner, data, old, esFieldData);
 
         if (logger.isTraceEnabled()) {
             logger.trace("Main table update to es index, destination:{}, table: {}, index: {}, id: {}",

client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/support/ESSyncUtil.java

@@ -18,7 +18,7 @@ import org.joda.time.DateTime;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson2.JSON;
 import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping;
 import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem;
 import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.ColumnItem;
@@ -63,172 +63,187 @@ public class ESSyncUtil {
             return val;
         }
         Object res = null;
-        if ("integer".equals(esType)) {
-            if (val instanceof Number) {
-                res = ((Number) val).intValue();
-            } else {
-                res = Integer.parseInt(val.toString());
-            }
-        } else if ("long".equals(esType)) {
-            if (val instanceof Number) {
-                res = ((Number) val).longValue();
-            } else {
-                res = Long.parseLong(val.toString());
-            }
-        } else if ("short".equals(esType)) {
-            if (val instanceof Number) {
-                res = ((Number) val).shortValue();
-            } else {
-                res = Short.parseShort(val.toString());
-            }
-        } else if ("byte".equals(esType)) {
-            if (val instanceof Number) {
-                res = ((Number) val).byteValue();
-            } else {
-                res = Byte.parseByte(val.toString());
-            }
-        } else if ("double".equals(esType)) {
-            if (val instanceof Number) {
-                res = ((Number) val).doubleValue();
-            } else {
-                res = Double.parseDouble(val.toString());
-            }
-        } else if ("float".equals(esType) || "half_float".equals(esType) || "scaled_float".equals(esType)) {
-            if (val instanceof Number) {
-                res = ((Number) val).floatValue();
-            } else {
-                res = Float.parseFloat(val.toString());
-            }
-        } else if ("boolean".equals(esType)) {
-            if (val instanceof Boolean) {
-                res = val;
-            } else if (val instanceof Number) {
-                int v = ((Number) val).intValue();
-                res = v != 0;
-            } else {
-                res = Boolean.parseBoolean(val.toString());
-            }
-        } else if ("date".equals(esType)) {
-            if (val instanceof java.sql.Time) {
-                DateTime dateTime = new DateTime(((java.sql.Time) val).getTime());
-                if (dateTime.getMillisOfSecond() != 0) {
-                    res = dateTime.toString("HH:mm:ss.SSS");
+        switch (esType) {
+            case "integer":
+                if (val instanceof Number) {
+                    res = ((Number) val).intValue();
                 } else {
-                    res = dateTime.toString("HH:mm:ss");
+                    res = Integer.parseInt(val.toString());
                 }
-            } else if (val instanceof java.sql.Timestamp) {
-                DateTime dateTime = new DateTime(((java.sql.Timestamp) val).getTime());
-                if (dateTime.getMillisOfSecond() != 0) {
-                    res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS" + Util.timeZone);
+                break;
+            case "long":
+                if (val instanceof Number) {
+                    res = ((Number) val).longValue();
                 } else {
-                    res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss" + Util.timeZone);
+                    res = Long.parseLong(val.toString());
                 }
-            } else if (val instanceof java.sql.Date || val instanceof Date) {
-                DateTime dateTime;
-                if (val instanceof java.sql.Date) {
-                    dateTime = new DateTime(((java.sql.Date) val).getTime());
+                break;
+            case "short":
+                if (val instanceof Number) {
+                    res = ((Number) val).shortValue();
                 } else {
-                    dateTime = new DateTime(((Date) val).getTime());
+                    res = Short.parseShort(val.toString());
                 }
-                if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0
-                    && dateTime.getMillisOfSecond() == 0) {
-                    res = dateTime.toString("yyyy-MM-dd");
+                break;
+            case "byte":
+                if (val instanceof Number) {
+                    res = ((Number) val).byteValue();
                 } else {
+                    res = Byte.parseByte(val.toString());
+                }
+                break;
+            case "double":
+                if (val instanceof Number) {
+                    res = ((Number) val).doubleValue();
+                } else {
+                    res = Double.parseDouble(val.toString());
+                }
+                break;
+            case "float":
+            case "half_float":
+            case "scaled_float":
+                if (val instanceof Number) {
+                    res = ((Number) val).floatValue();
+                } else {
+                    res = Float.parseFloat(val.toString());
+                }
+                break;
+            case "boolean":
+                if (val instanceof Boolean) {
+                    res = val;
+                } else if (val instanceof Number) {
+                    int v = ((Number) val).intValue();
+                    res = v != 0;
+                } else {
+                    res = Boolean.parseBoolean(val.toString());
+                }
+                break;
+            case "date":
+                if (val instanceof java.sql.Time) {
+                    DateTime dateTime = new DateTime(((java.sql.Time) val).getTime());
+                    if (dateTime.getMillisOfSecond() != 0) {
+                        res = dateTime.toString("HH:mm:ss.SSS");
+                    } else {
+                        res = dateTime.toString("HH:mm:ss");
+                    }
+                } else if (val instanceof java.sql.Timestamp) {
+                    DateTime dateTime = new DateTime(((java.sql.Timestamp) val).getTime());
                     if (dateTime.getMillisOfSecond() != 0) {
                         res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS" + Util.timeZone);
                     } else {
                         res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss" + Util.timeZone);
                     }
-                }
-            } else if (val instanceof Long) {
-                DateTime dateTime = new DateTime(((Long) val).longValue());
-                if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0
-                    && dateTime.getMillisOfSecond() == 0) {
-                    res = dateTime.toString("yyyy-MM-dd");
-                } else if (dateTime.getMillisOfSecond() != 0) {
-                    res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS" + Util.timeZone);
-                } else {
-                    res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss" + Util.timeZone);
-                }
-            } else if (val instanceof String) {
-                String v = ((String) val).trim();
-                if (v.length() > 18 && v.charAt(4) == '-' && v.charAt(7) == '-' && v.charAt(10) == ' '
-                    && v.charAt(13) == ':' && v.charAt(16) == ':') {
-                    String dt = v.substring(0, 10) + "T" + v.substring(11);
-                    Date date = Util.parseDate(dt);
-                    if (date != null) {
-                        DateTime dateTime = new DateTime(date);
+                } else if (val instanceof java.sql.Date || val instanceof Date) {
+                    DateTime dateTime;
+                    if (val instanceof java.sql.Date) {
+                        dateTime = new DateTime(((java.sql.Date) val).getTime());
+                    } else {
+                        dateTime = new DateTime(((Date) val).getTime());
+                    }
+                    if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0
+                            && dateTime.getMillisOfSecond() == 0) {
+                        res = dateTime.toString("yyyy-MM-dd");
+                    } else {
                         if (dateTime.getMillisOfSecond() != 0) {
                             res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS" + Util.timeZone);
                         } else {
                             res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss" + Util.timeZone);
                         }
                     }
-                } else if (v.length() == 10 && v.charAt(4) == '-' && v.charAt(7) == '-') {
-                    Date date = Util.parseDate(v);
-                    if (date != null) {
-                        DateTime dateTime = new DateTime(date);
+                } else if (val instanceof Long) {
+                    DateTime dateTime = new DateTime(((Long) val).longValue());
+                    if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0
+                            && dateTime.getMillisOfSecond() == 0) {
                         res = dateTime.toString("yyyy-MM-dd");
+                    } else if (dateTime.getMillisOfSecond() != 0) {
+                        res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS" + Util.timeZone);
+                    } else {
+                        res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss" + Util.timeZone);
+                    }
+                } else if (val instanceof String) {
+                    String v = ((String) val).trim();
+                    if (v.length() > 18 && v.charAt(4) == '-' && v.charAt(7) == '-' && v.charAt(10) == ' '
+                            && v.charAt(13) == ':' && v.charAt(16) == ':') {
+                        String dt = v.substring(0, 10) + "T" + v.substring(11);
+                        Date date = Util.parseDate(dt);
+                        if (date != null) {
+                            DateTime dateTime = new DateTime(date);
+                            if (dateTime.getMillisOfSecond() != 0) {
+                                res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS" + Util.timeZone);
+                            } else {
+                                res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss" + Util.timeZone);
+                            }
+                        }
+                    } else if (v.length() == 10 && v.charAt(4) == '-' && v.charAt(7) == '-') {
+                        Date date = Util.parseDate(v);
+                        if (date != null) {
+                            DateTime dateTime = new DateTime(date);
+                            res = dateTime.toString("yyyy-MM-dd");
+                        }
                     }
                 }
-            }
-        } else if ("binary".equals(esType)) {
-            if (val instanceof byte[]) {
-                Base64 base64 = new Base64();
-                res = base64.encodeAsString((byte[]) val);
-            } else if (val instanceof Blob) {
-                byte[] b = blobToBytes((Blob) val);
-                Base64 base64 = new Base64();
-                res = base64.encodeAsString(b);
-            } else if (val instanceof String) {
-                // 对应canal中的单字节编码
-                byte[] b = ((String) val).getBytes(StandardCharsets.ISO_8859_1);
-                Base64 base64 = new Base64();
-                res = base64.encodeAsString(b);
-            }
-        } else if ("geo_point".equals(esType)) {
-            if (!(val instanceof String)) {
-                logger.error("es type is geo_point, but source type is not String");
-                return val;
-            }
-
-            if (!((String) val).contains(",")) {
-                logger.error("es type is geo_point, source value not contains ',' separator");
-                return val;
-            }
-
-            String[] point = ((String) val).split(",");
-            Map<String, Double> location = new HashMap<>();
-            location.put("lat", Double.valueOf(point[0].trim()));
-            location.put("lon", Double.valueOf(point[1].trim()));
-            return location;
-        } else if ("array".equals(esType)) {
-            if ("".equals(val.toString().trim())) {
-                res = new ArrayList<>();
-            } else {
-                String value = val.toString();
-                String separator = ",";
-                if (!value.contains(",")) {
-                    if (value.contains(";")) {
-                        separator = ";";
-                    } else if (value.contains("|")) {
-                        separator = "\\|";
-                    } else if (value.contains("-")) {
-                        separator = "-";
-                    }
+                break;
+            case "binary":
+                if (val instanceof byte[]) {
+                    Base64 base64 = new Base64();
+                    res = base64.encodeAsString((byte[]) val);
+                } else if (val instanceof Blob) {
+                    byte[] b = blobToBytes((Blob) val);
+                    Base64 base64 = new Base64();
+                    res = base64.encodeAsString(b);
+                } else if (val instanceof String) {
+                    // 对应canal中的单字节编码
+                    byte[] b = ((String) val).getBytes(StandardCharsets.ISO_8859_1);
+                    Base64 base64 = new Base64();
+                    res = base64.encodeAsString(b);
                 }
-                String[] values = value.split(separator);
-                return Arrays.asList(values);
-            }
-        } else if ("object".equals(esType)) {
-            if ("".equals(val.toString().trim())) {
-                res = new HashMap<>();
-            } else {
-                res = JSON.parseObject(val.toString(), Map.class);
-            }
-        } else {
-            // 其他类全以字符串处理
-            res = val.toString();
+                break;
+            case "geo_point":
+                if (!(val instanceof String)) {
+                    logger.error("es type is geo_point, but source type is not String");
+                    return val;
+                }
+
+                if (!((String) val).contains(",")) {
+                    logger.error("es type is geo_point, source value not contains ',' separator");
+                    return val;
+                }
+
+                String[] point = ((String) val).split(",");
+                Map<String, Double> location = new HashMap<>();
+                location.put("lat", Double.valueOf(point[0].trim()));
+                location.put("lon", Double.valueOf(point[1].trim()));
+                return location;
+            case "array":
+                if ("".equals(val.toString().trim())) {
+                    res = new ArrayList<>();
+                } else {
+                    String value = val.toString();
+                    String separator = ",";
+                    if (!value.contains(",")) {
+                        if (value.contains(";")) {
+                            separator = ";";
+                        } else if (value.contains("|")) {
+                            separator = "\\|";
+                        } else if (value.contains("-")) {
+                            separator = "-";
+                        }
+                    }
+                    String[] values = value.split(separator);
+                    return Arrays.asList(values);
+                }
+                break;
+            case "object":
+                if ("".equals(val.toString().trim())) {
+                    res = new HashMap<>();
+                } else {
+                    res = JSON.parseObject(val.toString(), Map.class);
+                }
+                break;
+            default:
+                // 其他类全以字符串处理
+                res = val.toString();
+                break;
         }
 
         return res;

client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/support/ESTemplate.java

@@ -57,13 +57,11 @@ public interface ESTemplate {
 
     Object getIdValFromRS(ESMapping mapping, ResultSet resultSet) throws SQLException;
 
-    Object getESDataFromRS(ESMapping mapping, ResultSet resultSet, Map<String, Object> dmlOld,
-                           Map<String, Object> esFieldData) throws SQLException;
+    Object getESDataFromRS(ESMapping mapping, ResultSet resultSet, Map<String, Object> dmlOld, Map<String, Object> esFieldData) throws SQLException;
 
     Object getValFromData(ESMapping mapping, Map<String, Object> dmlData, String fieldName, String columnName);
 
     Object getESDataFromDmlData(ESMapping mapping, Map<String, Object> dmlData, Map<String, Object> esFieldData);
 
-    Object getESDataFromDmlData(ESMapping mapping, Map<String, Object> dmlData, Map<String, Object> dmlOld,
-                                Map<String, Object> esFieldData);
+    Object getESDataFromDmlData(ESMapping mapping,String owner, Map<String, Object> dmlData, Map<String, Object> dmlOld, Map<String, Object> esFieldData);
 }

client-adapter/hbase/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5-SNAPSHOT</version>
+        <version>1.1.7-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
@@ -20,7 +20,7 @@
         </dependency>
         <dependency>
             <groupId>org.apache.hbase</groupId>
-            <artifactId>hbase-client</artifactId>
+            <artifactId>hbase-shaded-client</artifactId>
             <version>1.1.2</version>
             <exclusions>
                 <exclusion>

client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java

@@ -1,9 +1,25 @@
 package com.alibaba.otter.canal.client.adapter.hbase;
 
+import com.alibaba.otter.canal.client.adapter.OuterAdapter;
+import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig;
+import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfigLoader;
+import com.alibaba.otter.canal.client.adapter.hbase.monitor.HbaseConfigMonitor;
+import com.alibaba.otter.canal.client.adapter.hbase.service.HbaseEtlService;
+import com.alibaba.otter.canal.client.adapter.hbase.service.HbaseSyncService;
+import com.alibaba.otter.canal.client.adapter.hbase.support.HbaseTemplate;
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+import com.alibaba.otter.canal.client.adapter.support.EtlResult;
+import com.alibaba.otter.canal.client.adapter.support.FileName2KeyMapping;
+import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
+import com.alibaba.otter.canal.client.adapter.support.SPI;
+import com.alibaba.otter.canal.client.adapter.support.Util;
 import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
 import java.util.concurrent.ConcurrentHashMap;
-
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -16,18 +32,6 @@ import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.alibaba.otter.canal.client.adapter.OuterAdapter;
-import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig;
-import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfigLoader;
-import com.alibaba.otter.canal.client.adapter.hbase.monitor.HbaseConfigMonitor;
-import com.alibaba.otter.canal.client.adapter.hbase.service.HbaseEtlService;
-import com.alibaba.otter.canal.client.adapter.hbase.service.HbaseSyncService;
-import com.alibaba.otter.canal.client.adapter.hbase.support.HbaseTemplate;
-import com.alibaba.otter.canal.client.adapter.support.Dml;
-import com.alibaba.otter.canal.client.adapter.support.EtlResult;
-import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
-import com.alibaba.otter.canal.client.adapter.support.SPI;
-
 /**
  * HBase外部适配器
  *
@@ -49,6 +53,8 @@ public class HbaseAdapter implements OuterAdapter {
 
     private Properties                              envProperties;
 
+    private OuterAdapterConfig                      configuration;
+
     public Map<String, MappingConfig> getHbaseMapping() {
         return hbaseMapping;
     }
@@ -61,33 +67,12 @@ public class HbaseAdapter implements OuterAdapter {
     public void init(OuterAdapterConfig configuration, Properties envProperties) {
         try {
             this.envProperties = envProperties;
+            this.configuration = configuration;
             Map<String, MappingConfig> hbaseMappingTmp = MappingConfigLoader.load(envProperties);
             // 过滤不匹配的key的配置
-            hbaseMappingTmp.forEach((key, mappingConfig) -> {
-                if ((mappingConfig.getOuterAdapterKey() == null && configuration.getKey() == null)
-                    || (mappingConfig.getOuterAdapterKey() != null
-                        && mappingConfig.getOuterAdapterKey().equalsIgnoreCase(configuration.getKey()))) {
-                    hbaseMapping.put(key, mappingConfig);
-                }
+            hbaseMappingTmp.forEach((key, config) -> {
+                addConfig(key, config);
             });
-            for (Map.Entry<String, MappingConfig> entry : hbaseMapping.entrySet()) {
-                String configName = entry.getKey();
-                MappingConfig mappingConfig = entry.getValue();
-                String k;
-                if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
-                    k = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "-"
-                        + StringUtils.trimToEmpty(mappingConfig.getGroupId()) + "_"
-                        + mappingConfig.getHbaseMapping().getDatabase() + "-"
-                        + mappingConfig.getHbaseMapping().getTable();
-                } else {
-                    k = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "_"
-                        + mappingConfig.getHbaseMapping().getDatabase() + "-"
-                        + mappingConfig.getHbaseMapping().getTable();
-                }
-                Map<String, MappingConfig> configMap = mappingConfigCache.computeIfAbsent(k,
-                    k1 -> new ConcurrentHashMap<>());
-                configMap.put(configName, mappingConfig);
-            }
 
             Map<String, String> properties = configuration.getProperties();
 
@@ -223,4 +208,62 @@ public class HbaseAdapter implements OuterAdapter {
         }
         return null;
     }
+
+    private void addSyncConfigToCache(String configName, MappingConfig mappingConfig) {
+        String k;
+        if (envProperties != null && !"tcp"
+                .equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
+            k = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "-" + StringUtils
+                    .trimToEmpty(mappingConfig.getGroupId()) + "_" + mappingConfig.getHbaseMapping()
+                    .getDatabase() + "-" + mappingConfig.getHbaseMapping().getTable();
+        } else {
+            k = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "_" + mappingConfig
+                    .getHbaseMapping().getDatabase() + "-" + mappingConfig.getHbaseMapping()
+                    .getTable();
+        }
+        Map<String, MappingConfig> configMap = mappingConfigCache
+                .computeIfAbsent(k, k1 -> new ConcurrentHashMap<>());
+        configMap.put(configName, mappingConfig);
+    }
+
+    public boolean addConfig(String fileName, MappingConfig config) {
+        if (match(config)) {
+            hbaseMapping.put(fileName, config);
+            addSyncConfigToCache(fileName, config);
+            FileName2KeyMapping.register(getClass().getAnnotation(SPI.class).value(), fileName,
+                    configuration.getKey());
+            return true;
+        }
+        return false;
+    }
+
+    public void updateConfig(String fileName, MappingConfig config) {
+        if (config.getOuterAdapterKey() != null && !config.getOuterAdapterKey()
+                .equals(configuration.getKey())) {
+            // 理论上不允许改这个 因为本身就是通过这个关联起Adapter和Config的
+            throw new RuntimeException("not allow to change outAdapterKey");
+        }
+        hbaseMapping.put(fileName, config);
+        addSyncConfigToCache(fileName, config);
+    }
+
+    public void deleteConfig(String fileName) {
+        hbaseMapping.remove(fileName);
+        for (Map<String, MappingConfig> configMap : mappingConfigCache.values()) {
+            if (configMap != null) {
+                configMap.remove(fileName);
+            }
+        }
+        FileName2KeyMapping.unregister(getClass().getAnnotation(SPI.class).value(), fileName);
+    }
+
+    private boolean match(MappingConfig config) {
+        boolean sameMatch = config.getOuterAdapterKey() != null && config.getOuterAdapterKey()
+                .equalsIgnoreCase(configuration.getKey());
+        boolean prefixMatch = config.getOuterAdapterKey() == null && configuration.getKey()
+                .startsWith(StringUtils
+                        .join(new String[]{Util.AUTO_GENERATED_PREFIX, config.getDestination(),
+                                config.getGroupId()}, '-'));
+        return sameMatch || prefixMatch;
+    }
 }

client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/monitor/HbaseConfigMonitor.java

@@ -1,23 +1,18 @@
 package com.alibaba.otter.canal.client.adapter.hbase.monitor;
 
-import java.io.File;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.commons.io.filefilter.FileFilterUtils;
-import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
-import org.apache.commons.io.monitor.FileAlterationMonitor;
-import org.apache.commons.io.monitor.FileAlterationObserver;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
 import com.alibaba.otter.canal.client.adapter.hbase.HbaseAdapter;
 import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig;
 import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
 import com.alibaba.otter.canal.client.adapter.support.Util;
+import java.io.File;
+import java.util.Properties;
+import org.apache.commons.io.filefilter.FileFilterUtils;
+import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
+import org.apache.commons.io.monitor.FileAlterationMonitor;
+import org.apache.commons.io.monitor.FileAlterationObserver;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class HbaseConfigMonitor {
 
@@ -37,7 +32,7 @@ public class HbaseConfigMonitor {
         File confDir = Util.getConfDirPath(adapterName);
         try {
             FileAlterationObserver observer = new FileAlterationObserver(confDir,
-                FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml")));
+                    FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml")));
             FileListener listener = new FileListener();
             observer.addListener(listener);
             fileMonitor = new FileAlterationMonitor(3000, observer);
@@ -70,9 +65,11 @@ public class HbaseConfigMonitor {
                     return;
                 }
                 config.validate();
-                addConfigToCache(file, config);
-
-                logger.info("Add a new hbase mapping config: {} to canal adapter", file.getName());
+                boolean result = hbaseAdapter.addConfig(file.getName(), config);
+                if (result) {
+                    logger.info("Add a new hbase mapping config: {} to canal adapter",
+                            file.getName());
+                }
             } catch (Exception e) {
                 logger.error(e.getMessage(), e);
             }
@@ -97,10 +94,7 @@ public class HbaseConfigMonitor {
                         return;
                     }
                     config.validate();
-                    if (hbaseAdapter.getHbaseMapping().containsKey(file.getName())) {
-                        deleteConfigFromCache(file);
-                    }
-                    addConfigToCache(file, config);
+                    hbaseAdapter.updateConfig(file.getName(), config);
                 }
             } catch (Exception e) {
                 logger.error(e.getMessage(), e);
@@ -113,33 +107,12 @@ public class HbaseConfigMonitor {
 
             try {
                 if (hbaseAdapter.getHbaseMapping().containsKey(file.getName())) {
-                    deleteConfigFromCache(file);
-
+                    hbaseAdapter.deleteConfig(file.getName());
                     logger.info("Delete a hbase mapping config: {} of canal adapter", file.getName());
                 }
             } catch (Exception e) {
                 logger.error(e.getMessage(), e);
             }
         }
-
-        private void addConfigToCache(File file, MappingConfig config) {
-            hbaseAdapter.getHbaseMapping().put(file.getName(), config);
-            Map<String, MappingConfig> configMap = hbaseAdapter.getMappingConfigCache()
-                .computeIfAbsent(StringUtils.trimToEmpty(config.getDestination()) + "."
-                                 + config.getHbaseMapping().getDatabase() + "." + config.getHbaseMapping().getTable(),
-                    k1 -> new HashMap<>());
-            configMap.put(file.getName(), config);
-        }
-
-        private void deleteConfigFromCache(File file) {
-
-            hbaseAdapter.getHbaseMapping().remove(file.getName());
-            for (Map<String, MappingConfig> configMap : hbaseAdapter.getMappingConfigCache().values()) {
-                if (configMap != null) {
-                    configMap.remove(file.getName());
-                }
-            }
-
-        }
     }
 }

client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java

@@ -6,8 +6,8 @@ import org.apache.hadoop.hbase.util.Bytes;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.serializer.SerializerFeature;
+import com.alibaba.fastjson2.JSON;
+import com.alibaba.fastjson2.JSONWriter.Feature;
 import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig;
 import com.alibaba.otter.canal.client.adapter.hbase.support.*;
 import com.alibaba.otter.canal.client.adapter.support.Dml;
@@ -39,7 +39,7 @@ public class HbaseSyncService {
                 delete(config, dml);
             }
             if (logger.isDebugEnabled()) {
-                logger.debug("DML: {}", JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue));
+                logger.debug("DML: {}", JSON.toJSONString(dml, Feature.WriteNulls));
             }
         }
     }

client-adapter/kudu/pom.xml

@@ -1,11 +1,9 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5-SNAPSHOT</version>
+        <version>1.1.7-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
@@ -73,7 +71,7 @@
                             <tasks>
                                 <copy todir="${project.basedir}/../launcher/target/classes/kudu" overwrite="true">
                                     <fileset dir="${project.basedir}/target/classes/kudu" erroronmissingdir="true">
-                                        <include name="*.yml"/>
+                                        <include name="*.yml" />
                                     </fileset>
                                 </copy>
                             </tasks>

client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/KuduAdapter.java

@@ -1,16 +1,5 @@
 package com.alibaba.otter.canal.client.adapter.kudu;
 
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import com.alibaba.otter.canal.client.adapter.OuterAdapter;
 import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
 import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfigLoader;
@@ -20,8 +9,19 @@ import com.alibaba.otter.canal.client.adapter.kudu.service.KuduSyncService;
 import com.alibaba.otter.canal.client.adapter.kudu.support.KuduTemplate;
 import com.alibaba.otter.canal.client.adapter.support.Dml;
 import com.alibaba.otter.canal.client.adapter.support.EtlResult;
+import com.alibaba.otter.canal.client.adapter.support.FileName2KeyMapping;
 import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
 import com.alibaba.otter.canal.client.adapter.support.SPI;
+import com.alibaba.otter.canal.client.adapter.support.Util;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * @author liuyadong
@@ -35,8 +35,6 @@ public class KuduAdapter implements OuterAdapter {
     private Map<String, KuduMappingConfig>              kuduMapping        = new ConcurrentHashMap<>();                 // 文件名对应配置
     private Map<String, Map<String, KuduMappingConfig>> mappingConfigCache = new ConcurrentHashMap<>();                 // 库名-表名对应配置
 
-    private String                                      dataSourceKey;
-
     private KuduTemplate                                kuduTemplate;
 
     private KuduSyncService                             kuduSyncService;
@@ -45,6 +43,8 @@ public class KuduAdapter implements OuterAdapter {
 
     private Properties                                  envProperties;
 
+    private OuterAdapterConfig                          configuration;
+
     public Map<String, KuduMappingConfig> getKuduMapping() {
         return kuduMapping;
     }
@@ -56,36 +56,17 @@ public class KuduAdapter implements OuterAdapter {
     @Override
     public void init(OuterAdapterConfig configuration, Properties envProperties) {
         this.envProperties = envProperties;
+        this.configuration = configuration;
         Map<String, KuduMappingConfig> kuduMappingTmp = KuduMappingConfigLoader.load(envProperties);
         // 过滤不匹配的key的配置,获取连接key，key为配置文件名称
-        kuduMappingTmp.forEach((key, mappingConfig) -> {
-            if ((mappingConfig.getOuterAdapterKey() == null && configuration.getKey() == null)
-                || (mappingConfig.getOuterAdapterKey() != null && mappingConfig.getOuterAdapterKey()
-                    .equalsIgnoreCase(configuration.getKey()))) {
-                kuduMapping.put(key, mappingConfig);
-                dataSourceKey = mappingConfig.getDataSourceKey();
-            }
+        kuduMappingTmp.forEach((key, config) -> {
+            addConfig(key, config);
         });
         // 判断目标字段是否为空
         if (kuduMapping.isEmpty()) {
             throw new RuntimeException("No kudu adapter found for config key: " + configuration.getKey());
         }
-        for (Map.Entry<String, KuduMappingConfig> entry : kuduMapping.entrySet()) {
-            String configName = entry.getKey();
-            KuduMappingConfig mappingConfig = entry.getValue();
-            String k;
-            if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
-                k = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "-"
-                    + StringUtils.trimToEmpty(mappingConfig.getGroupId()) + "_"
-                    + mappingConfig.getKuduMapping().getDatabase() + "-" + mappingConfig.getKuduMapping().getTable();
-            } else {
-                k = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "_"
-                    + mappingConfig.getKuduMapping().getDatabase() + "-" + mappingConfig.getKuduMapping().getTable();
-            }
-            Map<String, KuduMappingConfig> configMap = mappingConfigCache.computeIfAbsent(k,
-                k1 -> new ConcurrentHashMap<>());
-            configMap.put(configName, mappingConfig);
-        }
+
         Map<String, String> properties = configuration.getProperties();
 
         String kudu_master = properties.get("kudu.master.address");
@@ -203,4 +184,60 @@ public class KuduAdapter implements OuterAdapter {
         }
         return null;
     }
+
+    private void addSyncConfigToCache(String configName, KuduMappingConfig mappingConfig) {
+        String k;
+        if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
+            k = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "-"
+                    + StringUtils.trimToEmpty(mappingConfig.getGroupId()) + "_"
+                    + mappingConfig.getKuduMapping().getDatabase() + "-" + mappingConfig.getKuduMapping().getTable();
+        } else {
+            k = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "_"
+                    + mappingConfig.getKuduMapping().getDatabase() + "-" + mappingConfig.getKuduMapping().getTable();
+        }
+        Map<String, KuduMappingConfig> configMap = mappingConfigCache.computeIfAbsent(k,
+                k1 -> new ConcurrentHashMap<>());
+        configMap.put(configName, mappingConfig);
+    }
+
+    public boolean addConfig(String fileName, KuduMappingConfig config) {
+        if (match(config)) {
+            kuduMapping.put(fileName, config);
+            addSyncConfigToCache(fileName, config);
+            FileName2KeyMapping.register(getClass().getAnnotation(SPI.class).value(), fileName,
+                    configuration.getKey());
+            return true;
+        }
+        return false;
+    }
+
+    public void updateConfig(String fileName, KuduMappingConfig config) {
+        if (config.getOuterAdapterKey() != null && !config.getOuterAdapterKey()
+                .equals(configuration.getKey())) {
+            // 理论上不允许改这个 因为本身就是通过这个关联起Adapter和Config的
+            throw new RuntimeException("not allow to change outAdapterKey");
+        }
+        kuduMapping.put(fileName, config);
+        addSyncConfigToCache(fileName, config);
+    }
+
+    public void deleteConfig(String fileName) {
+        kuduMapping.remove(fileName);
+        for (Map<String, KuduMappingConfig> configMap : mappingConfigCache.values()) {
+            if (configMap != null) {
+                configMap.remove(fileName);
+            }
+        }
+        FileName2KeyMapping.unregister(getClass().getAnnotation(SPI.class).value(), fileName);
+    }
+
+    private boolean match(KuduMappingConfig config) {
+        boolean sameMatch = config.getOuterAdapterKey() != null && config.getOuterAdapterKey()
+                .equalsIgnoreCase(configuration.getKey());
+        boolean prefixMatch = config.getOuterAdapterKey() == null && configuration.getKey()
+                .startsWith(StringUtils
+                        .join(new String[]{Util.AUTO_GENERATED_PREFIX, config.getDestination(),
+                                config.getGroupId()}, '-'));
+        return sameMatch || prefixMatch;
+    }
 }

client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/monitor/KuduConfigMonitor.java

@@ -1,23 +1,18 @@
 package com.alibaba.otter.canal.client.adapter.kudu.monitor;
 
-import java.io.File;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.commons.io.filefilter.FileFilterUtils;
-import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
-import org.apache.commons.io.monitor.FileAlterationMonitor;
-import org.apache.commons.io.monitor.FileAlterationObserver;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
 import com.alibaba.otter.canal.client.adapter.kudu.KuduAdapter;
 import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
 import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
 import com.alibaba.otter.canal.client.adapter.support.Util;
+import java.io.File;
+import java.util.Properties;
+import org.apache.commons.io.filefilter.FileFilterUtils;
+import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
+import org.apache.commons.io.monitor.FileAlterationMonitor;
+import org.apache.commons.io.monitor.FileAlterationObserver;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * @author liuyadong
@@ -41,7 +36,7 @@ public class KuduConfigMonitor {
         File confDir = Util.getConfDirPath(adapterName);
         try {
             FileAlterationObserver observer = new FileAlterationObserver(confDir,
-                FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml")));
+                    FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml")));
             FileListener listener = new FileListener();
             observer.addListener(listener);
             fileMonitor = new FileAlterationMonitor(3000, observer);
@@ -83,9 +78,11 @@ public class KuduConfigMonitor {
                     return;
                 }
                 config.validate();
-                addConfigToCache(file, config);
-
-                logger.info("Add a new kudu mapping config: {} to canal adapter", file.getName());
+                boolean result = kuduAdapter.addConfig(file.getName(), config);
+                if (result) {
+                    logger.info("Add a new kudu mapping config: {} to canal adapter",
+                            file.getName());
+                }
             } catch (Exception e) {
                 logger.error(e.getMessage(), e);
             }
@@ -113,10 +110,7 @@ public class KuduConfigMonitor {
                         return;
                     }
                     config.validate();
-                    if (kuduAdapter.getKuduMapping().containsKey(file.getName())) {
-                        deleteConfigFromCache(file);
-                    }
-                    addConfigToCache(file, config);
+                    kuduAdapter.updateConfig(file.getName(), config);
                 }
             } catch (Exception e) {
                 logger.error(e.getMessage(), e);
@@ -129,42 +123,12 @@ public class KuduConfigMonitor {
 
             try {
                 if (kuduAdapter.getKuduMapping().containsKey(file.getName())) {
-                    deleteConfigFromCache(file);
+                    kuduAdapter.deleteConfig(file.getName());
                     logger.info("Delete a hbase mapping config: {} of canal adapter", file.getName());
                 }
             } catch (Exception e) {
                 logger.error(e.getMessage(), e);
             }
         }
-
-        /**
-         * 添加配置文件信息到缓存
-         *
-         * @param file
-         * @param config
-         */
-        private void addConfigToCache(File file, KuduMappingConfig config) {
-            kuduAdapter.getKuduMapping().put(file.getName(), config);
-            Map<String, KuduMappingConfig> configMap = kuduAdapter.getMappingConfigCache()
-                .computeIfAbsent(StringUtils.trimToEmpty(config.getDestination()) + "."
-                                 + config.getKuduMapping().getDatabase() + "." + config.getKuduMapping().getTable(),
-                    k1 -> new HashMap<>());
-            configMap.put(file.getName(), config);
-        }
-
-        /**
-         * 从缓存中删除配置
-         *
-         * @param file 文件
-         */
-        private void deleteConfigFromCache(File file) {
-            kuduAdapter.getKuduMapping().remove(file.getName());
-            for (Map<String, KuduMappingConfig> configMap : kuduAdapter.getMappingConfigCache().values()) {
-                if (configMap != null) {
-                    configMap.remove(file.getName());
-                }
-            }
-        }
-
     }
 }

client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/service/KuduEtlService.java

@@ -88,45 +88,40 @@ public class KuduEtlService extends AbstractEtlService {
                         Map<String, Object> data = new HashMap<>();
                         for (Map.Entry<String, String> entry : columnsMap.entrySet()) {
                             String mysqlColumnName = entry.getKey();// mysql字段名
-                String kuduColumnName = entry.getValue();// kudu字段名
-                if (kuduColumnName == null) {
-                    kuduColumnName = mysqlColumnName;
-                }
-                Object value = rs.getObject(kuduColumnName);
-                if (value != null) {
-                    data.put(kuduColumnName, value);
-                } else {
-                    data.put(kuduColumnName, null);
-                }
-            }
-            dataList.add(data);
-            idx++;
-            impCount.incrementAndGet();
-            if (logger.isDebugEnabled()) {
-                logger.debug("successful import count:" + impCount.get());
-            }
-            if (idx % kuduMapping.getCommitBatch() == 0) {
-                kuduTemplate.upsert(kuduMapping.getTargetTable(), dataList);
-                dataList.clear();
-            }
-        }
-        if (!dataList.isEmpty()) {
-            kuduTemplate.upsert(kuduMapping.getTargetTable(), dataList);
-        }
-        return true;
+                            String kuduColumnName = entry.getValue();// kudu字段名
+                            if (kuduColumnName == null) {
+                                kuduColumnName = mysqlColumnName;
+                            }
+                            Object value = rs.getObject(kuduColumnName);
+                            if (value != null) {
+                                data.put(kuduColumnName, value);
+                            } else {
+                                data.put(kuduColumnName, null);
+                            }
+                        }
+                        dataList.add(data);
+                        idx++;
+                        impCount.incrementAndGet();
+                        if (logger.isDebugEnabled()) {
+                            logger.debug("successful import count:" + impCount.get());
+                        }
+                        if (idx % kuduMapping.getCommitBatch() == 0) {
+                            kuduTemplate.upsert(kuduMapping.getTargetTable(), dataList);
+                            dataList.clear();
+                        }
+                    }
+                    if (!dataList.isEmpty()) {
+                        kuduTemplate.upsert(kuduMapping.getTargetTable(), dataList);
+                    }
+                    return true;
 
-    } catch (SQLException e) {
-        e.printStackTrace();
-        logger.error(kuduMapping.getTargetTable() + " etl failed! ==>" + e.getMessage(), e);
-        errMsg.add(kuduMapping.getTargetTable() + " etl failed! ==>" + e.getMessage());
-        return false;
-    } catch (KuduException e) {
-        e.printStackTrace();
-        logger.error(kuduMapping.getTargetTable() + " etl failed! ==>" + e.getMessage(), e);
-        errMsg.add(kuduMapping.getTargetTable() + " etl failed! ==>" + e.getMessage());
-        return false;
-    }
-}           );
+                } catch (SQLException | KuduException e) {
+                    e.printStackTrace();
+                    logger.error(kuduMapping.getTargetTable() + " etl failed! ==>" + e.getMessage(), e);
+                    errMsg.add(kuduMapping.getTargetTable() + " etl failed! ==>" + e.getMessage());
+                    return false;
+                }
+            });
             return true;
         } catch (Exception e) {
             logger.error(e.getMessage(), e);

client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/service/KuduSyncService.java

@@ -10,8 +10,8 @@ import org.apache.kudu.client.KuduException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.serializer.SerializerFeature;
+import com.alibaba.fastjson2.JSON;
+import com.alibaba.fastjson2.JSONWriter.Feature;
 import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
 import com.alibaba.otter.canal.client.adapter.kudu.support.KuduTemplate;
 import com.alibaba.otter.canal.client.adapter.support.Dml;
@@ -55,7 +55,7 @@ public class KuduSyncService {
                 delete(config, dml);
             }
             if (logger.isDebugEnabled()) {
-                logger.debug("DML: {}", JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue));
+                logger.debug("DML: {}", JSON.toJSONString(dml, Feature.WriteNulls));
             }
         }
     }
@@ -118,7 +118,7 @@ public class KuduSyncService {
                 }
             } catch (KuduException e) {
                 logger.error(e.getMessage());
-                logger.error("DML: {}", JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue));
+                logger.error("DML: {}", JSON.toJSONString(dml, Feature.WriteNulls));
             }
         }
     }
@@ -159,7 +159,7 @@ public class KuduSyncService {
                 }
             } catch (KuduException e) {
                 logger.error(e.getMessage());
-                logger.error("DML: {}", JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue));
+                logger.error("DML: {}", JSON.toJSONString(dml, Feature.WriteNulls));
             }
         }
 
@@ -201,7 +201,7 @@ public class KuduSyncService {
                 }
             } catch (KuduException e) {
                 logger.error(e.getMessage());
-                logger.error("DML: {}", JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue));
+                logger.error("DML: {}", JSON.toJSONString(dml, Feature.WriteNulls));
             }
         }
     }

client-adapter/launcher/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5-SNAPSHOT</version>
+        <version>1.1.7-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
@@ -61,7 +61,7 @@
             <artifactId>postgresql</artifactId>
         </dependency>
         <dependency>
-            <groupId>com.oracle</groupId>
+            <groupId>com.oracle.database.jdbc</groupId>
             <artifactId>ojdbc6</artifactId>
         </dependency>
         <dependency>
@@ -69,6 +69,12 @@
             <artifactId>mssql-jdbc</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.hbase</groupId>
+            <artifactId>hbase-shaded-client</artifactId>
+            <scope>runtime</scope>
+        </dependency>
+
         <!-- outer adapter jar with dependencies-->
         <dependency>
             <groupId>com.alibaba.otter</groupId>
@@ -135,6 +141,19 @@
             <classifier>jar-with-dependencies</classifier>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>client-adapter.tablestore</artifactId>
+            <version>${project.version}</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>*</artifactId>
+                    <groupId>*</groupId>
+                </exclusion>
+            </exclusions>
+            <classifier>jar-with-dependencies</classifier>
+            <scope>provided</scope>
+        </dependency>
         <!-- connector plugin -->
         <dependency>
             <groupId>com.alibaba.otter</groupId>
@@ -188,6 +207,19 @@
             <classifier>jar-with-dependencies</classifier>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>connector.pulsarmq</artifactId>
+            <version>${project.version}</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>*</artifactId>
+                    <groupId>*</groupId>
+                </exclusion>
+            </exclusions>
+            <classifier>jar-with-dependencies</classifier>
+            <scope>provided</scope>
+        </dependency>
     </dependencies>
 
     <build>

client-adapter/launcher/src/main/assembly/dev.xml

@@ -64,6 +64,13 @@
                 <exclude>META-INF/**</exclude>
             </excludes>
         </fileSet>
+        <fileSet>
+            <directory>../tablestore/src/main/resources/</directory>
+            <outputDirectory>/conf</outputDirectory>
+            <excludes>
+                <exclude>META-INF/**</exclude>
+            </excludes>
+        </fileSet>
         <fileSet>
             <directory>target</directory>
             <outputDirectory>logs</outputDirectory>

client-adapter/launcher/src/main/assembly/release.xml

@@ -65,6 +65,13 @@
                 <exclude>META-INF/**</exclude>
             </excludes>
         </fileSet>
+        <fileSet>
+            <directory>../tablestore/src/main/resources/</directory>
+            <outputDirectory>/conf</outputDirectory>
+            <excludes>
+                <exclude>META-INF/**</exclude>
+            </excludes>
+        </fileSet>
         <fileSet>
             <directory>target</directory>
             <outputDirectory>logs</outputDirectory>

client-adapter/launcher/src/main/bin/restart.sh

@@ -1,5 +1,13 @@
 #!/bin/bash
 
-sh stop.sh
+case $(uname) in
+Linux)
+  bin_abs_path=$(readlink -f $(dirname $0))
+  ;;
+*)
+  bin_abs_path=$(cd $(dirname $0) ||exit ; pwd)
+  ;;
+esac
 
-sh startup.sh
+sh "$bin_abs_path"/stop.sh
+sh "$bin_abs_path"/startup.sh

client-adapter/launcher/src/main/bin/startup.sh

@@ -56,11 +56,24 @@ in
   exit;;
 esac
 
+JavaVersion=`$JAVA -version 2>&1 |awk 'NR==1{ gsub(/"/,""); print $3 }' | awk  -F '.' '{print $1}'`
 str=`file -L $JAVA | grep 64-bit`
-if [ -n "$str" ]; then
-	JAVA_OPTS="-server -Xms2048m -Xmx3072m -Xmn1024m -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -Xss256k -XX:-UseAdaptiveSizePolicy -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+UseCMSCompactAtFullCollection -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:+HeapDumpOnOutOfMemoryError"
+
+JAVA_OPTS="$JAVA_OPTS -Xss256k -XX:+AggressiveOpts -XX:-UseBiasedLocking -XX:-OmitStackTraceInFastThrow -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$base/logs"
+if [ $JavaVersion -ge 11 ] ; then
+  #JAVA_OPTS="$JAVA_OPTS -Xlog:gc*:$base_log/gc.log:time "
+  JAVA_OPTS="$JAVA_OPTS"
 else
-	JAVA_OPTS="-server -Xms1024m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:MaxPermSize=128m "
+  #JAVA_OPTS="$JAVA_OPTS -Xloggc:$base/logs/canal/gc.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime"
+  JAVA_OPTS="$JAVA_OPTS -XX:+UseFastAccessorMethods -XX:+PrintAdaptiveSizePolicy -XX:+PrintTenuringDistribution"
+fi
+
+if [ -n "$str" ]; then
+	# JAVA_OPTS="-server -Xms2048m -Xmx3072m -Xmn1024m -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC $JAVA_OPTS"
+  # For G1
+  JAVA_OPTS="-server -Xms2g -Xmx3g -XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent $JAVA_OPTS"
+else
+	JAVA_OPTS="-server -Xms1024m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:MaxPermSize=128m $JAVA_OPTS"
 fi
 
 JAVA_OPTS=" $JAVA_OPTS -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8"

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/CanalAdapterApplication.java

@@ -12,8 +12,10 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
  */
 @SpringBootApplication
 public class CanalAdapterApplication {
-
     public static void main(String[] args) {
+        // 支持rocketmq client 配置日志路径
+        System.setProperty("rocketmq.client.logUseSlf4j","true");
+
         SpringApplication application = new SpringApplication(CanalAdapterApplication.class);
         application.setBannerMode(Banner.Mode.OFF);
         application.run(args);

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/DruidConfig.java

@@ -0,0 +1,23 @@
+package com.alibaba.otter.canal.adapter.launcher.config;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.springframework.boot.web.servlet.ServletRegistrationBean;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import com.alibaba.druid.support.http.StatViewServlet;
+
+@Configuration
+public class DruidConfig {
+
+    @Bean
+    public ServletRegistrationBean<StatViewServlet> statViewServlet(){
+        ServletRegistrationBean<StatViewServlet> bean = new ServletRegistrationBean<>( new StatViewServlet(),"/druid/*");
+        Map<String,String> initParams = new HashMap<>();
+        initParams.put("allow","");
+        bean.setInitParameters(initParams);
+        return  bean;
+    }
+}

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AdapterProcessor.java

@@ -7,6 +7,7 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -63,7 +64,7 @@ public class AdapterProcessor {
         // load connector consumer
         ExtensionLoader<CanalMsgConsumer> loader = new ExtensionLoader<>(CanalMsgConsumer.class);
         canalMsgConsumer = loader
-            .getExtension(canalClientConfig.getMode().toLowerCase(), CONNECTOR_SPI_DIR, CONNECTOR_STANDBY_SPI_DIR);
+            .getExtension(canalClientConfig.getMode().toLowerCase(),destination ,CONNECTOR_SPI_DIR, CONNECTOR_STANDBY_SPI_DIR);
 
         Properties properties = canalClientConfig.getConsumerProperties();
         properties.put(CanalConstants.CANAL_MQ_FLAT_MESSAGE, canalClientConfig.getFlatMessage());
@@ -211,13 +212,21 @@ public class AdapterProcessor {
                                     canalDestination,
                                     System.currentTimeMillis() - begin);
                             }
+                            break;
                         } catch (Exception e) {
                             if (i != retry - 1) {
                                 canalMsgConsumer.rollback(); // 处理失败, 回滚数据
                                 logger.error(e.getMessage() + " Error sync and rollback, execute times: " + (i + 1));
                             } else {
-                                canalMsgConsumer.ack();
-                                logger.error(e.getMessage() + " Error sync but ACK!");
+                                if (canalClientConfig.getTerminateOnException()) {
+                                    canalMsgConsumer.rollback();
+                                    logger.error("Retry fail, turn switch off and abort data transfer.");
+                                    syncSwitch.off(canalDestination);
+                                    logger.error("finish turn off switch of destination:" + canalDestination);
+                                } else {
+                                    canalMsgConsumer.ack();
+                                    logger.error(e.getMessage() + " Error sync but ACK!");
+                                }
                             }
                             Thread.sleep(500);
                         }

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java

@@ -22,6 +22,7 @@ import com.alibaba.otter.canal.client.adapter.OuterAdapter;
 import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
 import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader;
 import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
+import com.alibaba.otter.canal.client.adapter.support.Util;
 
 /**
  * 外部适配器的加载器
@@ -50,158 +51,43 @@ public class CanalAdapterLoader {
 
         for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
             for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
+                int autoGenId = 0;
                 List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
                 List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
+
                 for (OuterAdapterConfig config : group.getOuterAdapters()) {
+                    // 保证一定有key
+                    if (StringUtils.isEmpty(config.getKey())) {
+                        String key = StringUtils.join(
+                            new String[] { Util.AUTO_GENERATED_PREFIX, canalAdapter.getInstance(), group.getGroupId(),
+                                           String.valueOf(autoGenId) },
+                            '-');
+                        //gen keyId
+                        config.setKey(key);
+                    }
+                    autoGenId++;
                     loadAdapter(config, canalOuterAdapters);
                 }
                 canalOuterAdapterGroups.add(canalOuterAdapters);
 
-                AdapterProcessor adapterProcessor = canalAdapterProcessors.computeIfAbsent(canalAdapter.getInstance()
-                                                                                           + "|"
-                                                                                           + StringUtils.trimToEmpty(group.getGroupId()),
+                AdapterProcessor adapterProcessor = canalAdapterProcessors.computeIfAbsent(
+                    canalAdapter.getInstance() + "|" + StringUtils.trimToEmpty(group.getGroupId()),
                     f -> new AdapterProcessor(canalClientConfig,
                         canalAdapter.getInstance(),
                         group.getGroupId(),
                         canalOuterAdapterGroups));
                 adapterProcessor.start();
 
-                logger.info("Start adapter for canal-client mq topic: {} succeed", canalAdapter.getInstance() + "-"
-                                                                                   + group.getGroupId());
+                logger.info("Start adapter for canal-client mq topic: {} succeed",
+                    canalAdapter.getInstance() + "-" + group.getGroupId());
             }
         }
-
-        // if ("tcp".equalsIgnoreCase(canalClientConfig.getMode())) {
-        // // 初始化canal-client的适配器
-        // for (CanalClientConfig.CanalAdapter canalAdapter :
-        // canalClientConfig.getCanalAdapters()) {
-        // List<List<OuterAdapter>> canalOuterAdapterGroups = new
-        // CopyOnWriteArrayList<>();
-        //
-        // for (CanalClientConfig.Group connectorGroup :
-        // canalAdapter.getGroups()) {
-        // List<OuterAdapter> canalOutConnectors = new CopyOnWriteArrayList<>();
-        // for (OuterAdapterConfig c : connectorGroup.getOuterAdapters()) {
-        // loadAdapter(c, canalOutConnectors);
-        // }
-        // canalOuterAdapterGroups.add(canalOutConnectors);
-        // }
-        // CanalAdapterWorker worker;
-        // if (StringUtils.isNotEmpty(canalServerHost)) {
-        // worker = new CanalAdapterWorker(canalClientConfig,
-        // canalAdapter.getInstance(),
-        // canalServerHost,
-        // zkHosts,
-        // canalOuterAdapterGroups);
-        // } else if (zkHosts != null) {
-        // worker = new CanalAdapterWorker(canalClientConfig,
-        // canalAdapter.getInstance(),
-        // zkHosts,
-        // canalOuterAdapterGroups);
-        // } else {
-        // throw new RuntimeException("No canal server connector found");
-        // }
-        // canalWorkers.put(canalAdapter.getInstance(), worker);
-        // worker.start();
-        // logger.info("Start adapter for canal instance: {} succeed",
-        // canalAdapter.getInstance());
-        // }
-        // } else if ("kafka".equalsIgnoreCase(canalClientConfig.getMode())) {
-        // // 初始化canal-client-kafka的适配器
-        // for (CanalClientConfig.CanalAdapter canalAdapter :
-        // canalClientConfig.getCanalAdapters()) {
-        // for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
-        // List<List<OuterAdapter>> canalOuterAdapterGroups = new
-        // CopyOnWriteArrayList<>();
-        // List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
-        // for (OuterAdapterConfig config : group.getOuterAdapters()) {
-        // loadAdapter(config, canalOuterAdapters);
-        // }
-        // canalOuterAdapterGroups.add(canalOuterAdapters);
-        //
-        // CanalAdapterKafkaWorker canalKafkaWorker = new
-        // CanalAdapterKafkaWorker(canalClientConfig,
-        // canalClientConfig.getMqServers(),
-        // canalAdapter.getInstance(),
-        // group.getGroupId(),
-        // canalOuterAdapterGroups,
-        // canalClientConfig.getFlatMessage());
-        // canalMQWorker.put(canalAdapter.getInstance() + "-kafka-" +
-        // group.getGroupId(), canalKafkaWorker);
-        // canalKafkaWorker.start();
-        // logger.info("Start adapter for canal-client mq topic: {} succeed",
-        // canalAdapter.getInstance() + "-" + group.getGroupId());
-        // }
-        // }
-        // } else if ("rocketMQ".equalsIgnoreCase(canalClientConfig.getMode()))
-        // {
-        // // 初始化canal-client-rocketMQ的适配器
-        // for (CanalClientConfig.CanalAdapter canalAdapter :
-        // canalClientConfig.getCanalAdapters()) {
-        // for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
-        // List<List<OuterAdapter>> canalOuterAdapterGroups = new
-        // CopyOnWriteArrayList<>();
-        // List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
-        // for (OuterAdapterConfig config : group.getOuterAdapters()) {
-        // loadAdapter(config, canalOuterAdapters);
-        // }
-        // canalOuterAdapterGroups.add(canalOuterAdapters);
-        // CanalAdapterRocketMQWorker rocketMQWorker = new
-        // CanalAdapterRocketMQWorker(canalClientConfig,
-        // canalClientConfig.getMqServers(),
-        // canalAdapter.getInstance(),
-        // group.getGroupId(),
-        // canalOuterAdapterGroups,
-        // canalClientConfig.getAccessKey(),
-        // canalClientConfig.getSecretKey(),
-        // canalClientConfig.getFlatMessage(),
-        // canalClientConfig.isEnableMessageTrace(),
-        // canalClientConfig.getCustomizedTraceTopic(),
-        // canalClientConfig.getAccessChannel(),
-        // canalClientConfig.getNamespace());
-        // canalMQWorker.put(canalAdapter.getInstance() + "-rocketmq-" +
-        // group.getGroupId(), rocketMQWorker);
-        // rocketMQWorker.start();
-        //
-        // logger.info("Start adapter for canal-client mq topic: {} succeed",
-        // canalAdapter.getInstance() + "-" + group.getGroupId());
-        // }
-        // }
-        // } else if ("rabbitMQ".equalsIgnoreCase(canalClientConfig.getMode()))
-        // {
-        // // 初始化canal-client-rabbitMQ的适配器
-        // for (CanalClientConfig.CanalAdapter canalAdapter :
-        // canalClientConfig.getCanalAdapters()) {
-        // for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
-        // List<List<OuterAdapter>> canalOuterAdapterGroups = new
-        // CopyOnWriteArrayList<>();
-        // List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
-        // for (OuterAdapterConfig config : group.getOuterAdapters()) {
-        // loadAdapter(config, canalOuterAdapters);
-        // }
-        // canalOuterAdapterGroups.add(canalOuterAdapters);
-        // CanalAdapterRabbitMQWorker rabbitMQWork = new
-        // CanalAdapterRabbitMQWorker(canalClientConfig,
-        // canalOuterAdapterGroups,
-        // canalAdapter.getInstance(),
-        // group.getGroupId(),
-        // canalClientConfig.getFlatMessage());
-        // canalMQWorker.put(canalAdapter.getInstance() + "-rabbitmq-" +
-        // group.getGroupId(), rabbitMQWork);
-        // rabbitMQWork.start();
-        //
-        // logger.info("Start adapter for canal-client mq topic: {} succeed",
-        // canalAdapter.getInstance() + "-" + group.getGroupId());
-        // }
-        // }
-        // // CanalAdapterRabbitMQWork
-        // }
     }
 
     private void loadAdapter(OuterAdapterConfig config, List<OuterAdapter> canalOutConnectors) {
         try {
             OuterAdapter adapter;
-            adapter = loader.getExtension(config.getName(), StringUtils.trimToEmpty(config.getKey()));
+            adapter = loader.getExtension(config.getName(), config.getKey());
 
             ClassLoader cl = Thread.currentThread().getContextClassLoader();
             // 替换ClassLoader

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java

@@ -1,15 +1,21 @@
 package com.alibaba.otter.canal.adapter.launcher.rest;
 
+import com.alibaba.otter.canal.adapter.launcher.common.EtlLock;
+import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch;
+import com.alibaba.otter.canal.adapter.launcher.config.AdapterCanalConfig;
+import com.alibaba.otter.canal.client.adapter.OuterAdapter;
+import com.alibaba.otter.canal.client.adapter.support.EtlResult;
+import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader;
+import com.alibaba.otter.canal.client.adapter.support.FileName2KeyMapping;
+import com.alibaba.otter.canal.client.adapter.support.Result;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-
 import javax.annotation.PostConstruct;
 import javax.annotation.Resource;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.web.bind.annotation.GetMapping;
@@ -19,14 +25,6 @@ import org.springframework.web.bind.annotation.PutMapping;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
-import com.alibaba.otter.canal.adapter.launcher.common.EtlLock;
-import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch;
-import com.alibaba.otter.canal.adapter.launcher.config.AdapterCanalConfig;
-import com.alibaba.otter.canal.client.adapter.OuterAdapter;
-import com.alibaba.otter.canal.client.adapter.support.EtlResult;
-import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader;
-import com.alibaba.otter.canal.client.adapter.support.Result;
-
 /**
  * 适配器操作Rest
  *
@@ -66,6 +64,9 @@ public class CommonRest {
     @PostMapping("/etl/{type}/{key}/{task}")
     public EtlResult etl(@PathVariable String type, @PathVariable String key, @PathVariable String task,
                          @RequestParam(name = "params", required = false) String params) {
+        if (key == null) {
+            key = FileName2KeyMapping.getKey(type, task);
+        }
         OuterAdapter adapter = loader.getExtension(type, key);
         String destination = adapter.getDestination(task);
         String lockKey = destination == null ? task : destination;
@@ -133,6 +134,9 @@ public class CommonRest {
      */
     @GetMapping("/count/{type}/{key}/{task}")
     public Map<String, Object> count(@PathVariable String type, @PathVariable String key, @PathVariable String task) {
+        if (key == null) {
+            key = FileName2KeyMapping.getKey(type, task);
+        }
         OuterAdapter adapter = loader.getExtension(type, key);
         return adapter.count(task);
     }

client-adapter/launcher/src/main/resources/application.yml

@@ -11,7 +11,7 @@ canal.conf:
   flatMessage: true
   zookeeperHosts:
   syncBatchSize: 1000
-  retries: 0
+  retries: -1
   timeout:
   accessKey:
   secretKey:
@@ -64,6 +64,8 @@ canal.conf:
 #          jdbc.url: jdbc:mysql://127.0.0.1:3306/mytest2?useUnicode=true
 #          jdbc.username: root
 #          jdbc.password: 121212
+#          druid.stat.enable: false
+#          druid.stat.slowSqlMillis: 1000
 #      - name: rdb
 #        key: oracle1
 #        properties:
@@ -91,7 +93,14 @@ canal.conf:
 #          mode: transport # or rest
 #          # security.auth: test:123456 #  only used for rest mode
 #          cluster.name: elasticsearch
-#        - name: kudu
-#          key: kudu
-#          properties:
-#            kudu.master.address: 127.0.0.1 # ',' split multi address
+#      - name: kudu
+#        key: kudu
+#        properties:
+#          kudu.master.address: 127.0.0.1 # ',' split multi address
+#      - name: phoenix
+#        key: phoenix
+#        properties:
+#          jdbc.driverClassName: org.apache.phoenix.jdbc.PhoenixDriver
+#          jdbc.url: jdbc:phoenix:127.0.0.1:2181:/hbase/db
+#          jdbc.username:
+#          jdbc.password:

client-adapter/launcher/src/main/resources/bootstrap.yml

@@ -1,6 +1,6 @@
-#canal:
-#  manager:
-#    jdbc:
-#      url: jdbc:mysql://127.0.0.1:3306/canal_manager?useUnicode=true&characterEncoding=UTF-8
-#      username: root
-#      password: 121212
+canal:
+  manager:
+    jdbc:
+      url: jdbc:mysql://127.0.0.1:3306/canal_manager?useUnicode=true&characterEncoding=UTF-8
+      username: canal
+      password: canal

client-adapter/launcher/src/main/resources/logback.xml

@@ -34,6 +34,23 @@
 		</sift>
 	</appender>
 
+	<appender name="RocketmqClientAppender" class="ch.qos.logback.core.rolling.RollingFileAppender">
+		<file>../logs/adapter/rocketmq_client.log</file>
+		<rollingPolicy
+				class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+			<!-- rollover daily -->
+			<fileNamePattern>../logs/adapter/%d{yyyy-MM-dd}/rocketmq_client-%d{yyyy-MM-dd}-%i.log.gz</fileNamePattern>
+			<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+				<!-- or whenever the file size reaches 100MB -->
+				<maxFileSize>512MB</maxFileSize>
+			</timeBasedFileNamingAndTriggeringPolicy>
+			<maxHistory>60</maxHistory>
+		</rollingPolicy>
+		<encoder charset="UTF-8">
+			<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n</pattern>
+		</encoder>
+	</appender>
+
 	<logger name="com.alibaba.otter.canal.client.adapter.hbase" additivity="false">
 		<level value="DEBUG" />
 		<appender-ref ref="STDOUT"/>
@@ -54,6 +71,10 @@
 		<appender-ref ref="STDOUT"/>
 		<appender-ref ref="CANAL-ROOT" />
 	</logger>
+	<logger name="RocketmqClient" additivity="false">
+		<level value="INFO" />
+		<appender-ref ref="RocketmqClientAppender" />
+	</logger>
 
 	<root level="INFO">
 		<appender-ref ref="STDOUT"/>

client-adapter/logger/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5-SNAPSHOT</version>
+        <version>1.1.7-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java

@@ -6,8 +6,8 @@ import java.util.Properties;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.serializer.SerializerFeature;
+import com.alibaba.fastjson2.JSON;
+import com.alibaba.fastjson2.JSONWriter.Feature;
 import com.alibaba.otter.canal.client.adapter.OuterAdapter;
 import com.alibaba.otter.canal.client.adapter.support.Dml;
 import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
@@ -36,7 +36,7 @@ public class LoggerAdapterExample implements OuterAdapter {
     }
 
     public void sync(Dml dml) {
-        logger.info("DML: {}", JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue));
+        logger.info("DML: {}", JSON.toJSONString(dml, Feature.WriteNulls));
     }
 
     @Override

client-adapter/phoenix/pom.xml

@@ -0,0 +1,144 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>canal.client-adapter</artifactId>
+        <groupId>com.alibaba.otter</groupId>
+        <version>1.1.7-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.alibaba.otter</groupId>
+    <artifactId>client-adapter.phoenix</artifactId>
+    <packaging>jar</packaging>
+    <name>canal client adapter phoenix module for otter ${project.version}</name>
+
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.phoenix</groupId>
+            <artifactId>phoenix-core</artifactId>
+            <version>4.14.1-HBase-1.4</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hbase</groupId>
+            <artifactId>hbase-client</artifactId>
+            <version>1.4.8</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.protobuf</groupId>
+            <artifactId>protobuf-java</artifactId>
+            <version>2.5.0</version>
+        </dependency>
+
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>client-adapter.common</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>commons-lang</groupId>
+            <artifactId>commons-lang</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba</groupId>
+            <artifactId>druid</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba.fastjson2</groupId>
+            <artifactId>fastjson2</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>mysql</groupId>
+            <artifactId>mysql-connector-java</artifactId>
+            <scope>provided</scope>
+        </dependency>
+    </dependencies>
+
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>2.4</version>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <artifactId>maven-antrun-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+<!--                            <tasks>-->
+<!--                                <copy todir="${project.basedir}/../launcher/target/classes/phoenix" overwrite="true">-->
+<!--                                    <fileset dir="${project.basedir}/target/classes/phoenix" erroronmissingdir="true">-->
+<!--                                        <include name="*.yml" />-->
+<!--                                    </fileset>-->
+<!--                                </copy>-->
+<!--                            </tasks>-->
+                            <tasks>
+                                <copy todir="${project.basedir}/../launcher/target/canal-adapter/conf/phoenix" overwrite="true">
+                                    <fileset dir="${project.basedir}/target/classes/phoenix" erroronmissingdir="true">
+                                        <include name="*.yml" />
+                                    </fileset>
+                                </copy>
+                                <copy todir="${project.basedir}/../launcher/target/canal-adapter/plugin" overwrite="true">
+                                    <fileset dir="${project.basedir}/target/" erroronmissingdir="true">
+                                        <include name="*with-dependencies.jar" />
+                                    </fileset>
+                                </copy>
+                                <copy todir="${project.basedir}/../launcher/target/classes/phoenix" overwrite="true">
+                                    <fileset dir="${project.basedir}/target/classes/phoenix" erroronmissingdir="true">
+                                        <include name="*.yml" />
+                                    </fileset>
+                                </copy>
+                            </tasks>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/PhoenixAdapter.java

@@ -0,0 +1,326 @@
+package com.alibaba.otter.canal.client.adapter.phoenix;
+import com.alibaba.otter.canal.client.adapter.phoenix.config.ConfigLoader;
+import com.alibaba.otter.canal.client.adapter.phoenix.config.MappingConfig;
+import com.alibaba.otter.canal.client.adapter.phoenix.monitor.PhoenixConfigMonitor;
+import com.alibaba.otter.canal.client.adapter.phoenix.service.PhoenixEtlService;
+import com.alibaba.otter.canal.client.adapter.phoenix.service.PhoenixSyncService;
+import com.alibaba.otter.canal.client.adapter.OuterAdapter;
+import com.alibaba.otter.canal.client.adapter.phoenix.support.SyncUtil;
+import com.alibaba.otter.canal.client.adapter.support.*;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import javax.sql.DataSource;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * @author: lihua
+ * @date: 2021/1/5 15:01
+ * @Description: Phoenix适配器实现类
+ */
+@SPI("phoenix")
+public class PhoenixAdapter implements OuterAdapter {
+
+    private static Logger logger = LoggerFactory.getLogger(PhoenixAdapter.class);
+
+    private Map<String, MappingConfig> phoenixMapping = new ConcurrentHashMap<>();                // 文件名对应配置
+    private Map<String, Map<String, MappingConfig>> mappingConfigCache = new ConcurrentHashMap<>();                // 库名-表名对应配置
+
+    private static  String DriverClass;
+    private static  String PhoenixUrl;
+    private static  Properties phoenixPro = new Properties();
+
+    private PhoenixSyncService phoenixSyncService;
+
+    private PhoenixConfigMonitor phoenixConfigMonitor;
+
+    private Properties envProperties;
+
+    private OuterAdapterConfig configuration;
+
+    public Map<String, MappingConfig> getPhoenixMapping() {
+        return phoenixMapping;
+    }
+
+    public Map<String, Map<String, MappingConfig>> getMappingConfigCache() {
+        return mappingConfigCache;
+    }
+
+    public PhoenixAdapter() {
+        logger.info("PhoenixAdapter create: {} {}", this, Thread.currentThread().getStackTrace());
+    }
+    /**
+     * 初始化方法
+     *
+     * @param configuration 外部适配器配置信息
+     */
+    @Override
+    public void init(OuterAdapterConfig configuration, Properties envProperties) {
+        this.envProperties = envProperties;
+        this.configuration = configuration;
+        Map<String, MappingConfig> phoenixMappingTmp = ConfigLoader.load(envProperties);
+        // 过滤不匹配的key的配置
+        phoenixMappingTmp.forEach((key, config) -> {
+            addConfig(key, config);
+        });
+
+        if (phoenixMapping.isEmpty()) {
+            throw new RuntimeException("No phoenix adapter found for config key: " + configuration.getKey());
+        } else {
+            logger.info("[{}]phoenix config mapping: {}", this, phoenixMapping.keySet());
+        }
+
+        Map<String, String> properties = configuration.getProperties();
+
+        DriverClass= properties.get("jdbc.driverClassName");
+        PhoenixUrl=properties.get("jdbc.url");
+
+        try {
+            //phoenix内部本身有连接池，不需要使用Druid初始化
+            phoenixPro.setProperty("hbase.rpc.timeout","600000");
+            phoenixPro.setProperty("hbase.client.scanner.timeout.period","600000");
+            phoenixPro.setProperty("dfs.client.socket-timeout","600000");
+            phoenixPro.setProperty("phoenix.query.keepAliveMs","600000");
+            phoenixPro.setProperty("phoenix.query.timeoutMs","3600000");
+            Class.forName(DriverClass);
+        } catch (ClassNotFoundException e) {
+            e.printStackTrace();
+        }
+        String threads = properties.get("threads");
+        phoenixSyncService = new PhoenixSyncService(
+                threads != null ? Integer.valueOf(threads) : null
+        );
+
+        phoenixConfigMonitor = new PhoenixConfigMonitor();
+        phoenixConfigMonitor.init(configuration.getKey(), this, envProperties);
+    }
+
+
+    /**
+     * 获取phoenix连接
+     * @return
+     */
+    public static  Connection  getPhoenixConnection() {
+        try {
+            return DriverManager.getConnection(PhoenixUrl,phoenixPro);
+        } catch (SQLException e) {
+            logger.error("getPhoenixConnection Exception"+e.getMessage());
+        }
+        return  null;
+    }
+
+
+
+    /**
+     * 同步方法
+     *
+     * @param dmls 数据包
+     */
+    @Override
+    public void sync(List<Dml> dmls) {
+        if (dmls == null || dmls.isEmpty()) {
+            return;
+        }
+        try {
+            phoenixSyncService.sync(mappingConfigCache, dmls, envProperties);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * ETL方法
+     *
+     * @param task   任务名, 对应配置名
+     * @param params etl筛选条件
+     * @return ETL结果
+     */
+    @Override
+    public EtlResult etl(String task, List<String> params) {
+        EtlResult etlResult = new EtlResult();
+        MappingConfig config = phoenixMapping.get(task);
+        if (config != null) {
+            DataSource srcDataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
+            if (srcDataSource != null) {
+                return PhoenixEtlService.importData(srcDataSource, getPhoenixConnection(), config, params);
+            } else {
+                etlResult.setSucceeded(false);
+                etlResult.setErrorMessage("DataSource not found");
+                return etlResult;
+            }
+        } else {
+            StringBuilder resultMsg = new StringBuilder();
+            boolean resSucc = true;
+            // ds不为空说明传入的是destination
+            for (MappingConfig configTmp : phoenixMapping.values()) {
+                // 取所有的destination为task的配置
+                if (configTmp.getDestination().equals(task)) {
+                    DataSource srcDataSource = DatasourceConfig.DATA_SOURCES.get(configTmp.getDataSourceKey());
+                    if (srcDataSource == null) {
+                        continue;
+                    }
+                    EtlResult etlRes = PhoenixEtlService.importData(srcDataSource,getPhoenixConnection(), configTmp, params);
+                    if (!etlRes.getSucceeded()) {
+                        resSucc = false;
+                        resultMsg.append(etlRes.getErrorMessage()).append("\n");
+                    } else {
+                        resultMsg.append(etlRes.getResultMessage()).append("\n");
+                    }
+                }
+            }
+            if (resultMsg.length() > 0) {
+                etlResult.setSucceeded(resSucc);
+                if (resSucc) {
+                    etlResult.setResultMessage(resultMsg.toString());
+                } else {
+                    etlResult.setErrorMessage(resultMsg.toString());
+                }
+                return etlResult;
+            }
+        }
+        etlResult.setSucceeded(false);
+        etlResult.setErrorMessage("Task not found");
+        return etlResult;
+    }
+
+    /**
+     * 获取总数方法
+     *
+     * @param task 任务名, 对应配置名
+     * @return 总数
+     */
+    @Override
+    public Map<String, Object> count(String task) {
+        Map<String, Object> res = new LinkedHashMap<>();
+        MappingConfig config = phoenixMapping.get(task);
+        if (config == null) {
+            logger.info("[{}]phoenix config mapping: {}", this, phoenixMapping.keySet());
+            res.put("succeeded", false);
+            res.put("errorMessage", "Task[" + task + "] not found");
+            res.put("tasks", phoenixMapping.keySet());
+            return res;
+        }
+        MappingConfig.DbMapping dbMapping = config.getDbMapping();
+        String sql = "SELECT COUNT(1) AS cnt FROM " + SyncUtil.getDbTableName(dbMapping);
+        Connection conn = null;
+        try {
+            //conn = dataSource.getConnection();
+            conn = getPhoenixConnection();
+            Util.sqlRS(conn, sql, rs -> {
+                try {
+                    if (rs.next()) {
+                        Long rowCount = rs.getLong("cnt");
+                        res.put("count", rowCount);
+                    }
+                } catch (SQLException e) {
+                    logger.error(e.getMessage(), e);
+                }
+            });
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        } finally {
+            if (conn != null) {
+                try {
+                    conn.close();
+                } catch (SQLException e) {
+                    logger.error(e.getMessage(), e);
+                }
+            }
+        }
+        res.put("targetTable", SyncUtil.getDbTableName(dbMapping));
+
+        return res;
+    }
+
+    /**
+     * 获取对应canal instance name 或 mq topic
+     *
+     * @param task 任务名, 对应配置名
+     * @return destination
+     */
+    @Override
+    public String getDestination(String task) {
+        MappingConfig config = phoenixMapping.get(task);
+        if (config != null) {
+            return config.getDestination();
+        }
+        return null;
+    }
+
+    /**
+     * 销毁方法
+     */
+    @Override
+    public void destroy() {
+        if (phoenixConfigMonitor != null) {
+            phoenixConfigMonitor.destroy();
+        }
+
+        if (phoenixSyncService != null) {
+            phoenixSyncService.close();
+        }
+    }
+
+    private void addSyncConfigToCache(String configName, MappingConfig mappingConfig) {
+        String key;
+        if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
+            key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "-"
+                    + StringUtils.trimToEmpty(mappingConfig.getGroupId()) + "_"
+                    + mappingConfig.getDbMapping().getDatabase() + "-" + mappingConfig.getDbMapping().getTable().toLowerCase();
+        } else {
+            key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "_"
+                    + mappingConfig.getDbMapping().getDatabase() + "-" + mappingConfig.getDbMapping().getTable().toLowerCase();
+        }
+        Map<String, MappingConfig> configMap = mappingConfigCache.computeIfAbsent(key,
+                k1 -> new ConcurrentHashMap<>());
+        configMap.put(configName, mappingConfig);
+    }
+
+    public boolean addConfig(String fileName, MappingConfig config) {
+        if (match(config)) {
+            phoenixMapping.put(fileName, config);
+            addSyncConfigToCache(fileName, config);
+            FileName2KeyMapping.register(getClass().getAnnotation(SPI.class).value(), fileName,
+                    configuration.getKey());
+            return true;
+        }
+        return false;
+    }
+
+    public void updateConfig(String fileName, MappingConfig config) {
+        if (config.getOuterAdapterKey() != null && !config.getOuterAdapterKey()
+                .equals(configuration.getKey())) {
+            // 理论上不允许改这个 因为本身就是通过这个关联起Adapter和Config的
+            throw new RuntimeException("not allow to change outAdapterKey");
+        }
+        phoenixMapping.put(fileName, config);
+        addSyncConfigToCache(fileName, config);
+    }
+
+    public void deleteConfig(String fileName) {
+        phoenixMapping.remove(fileName);
+        for (Map<String, MappingConfig> configMap : mappingConfigCache.values()) {
+            if (configMap != null) {
+                configMap.remove(fileName);
+            }
+        }
+        FileName2KeyMapping.unregister(getClass().getAnnotation(SPI.class).value(), fileName);
+    }
+
+    private boolean match(MappingConfig config) {
+        boolean sameMatch = config.getOuterAdapterKey() != null && config.getOuterAdapterKey()
+                .equalsIgnoreCase(configuration.getKey());
+        boolean prefixMatch = config.getOuterAdapterKey() == null && configuration.getKey()
+                .startsWith(StringUtils
+                        .join(new String[]{Util.AUTO_GENERATED_PREFIX, config.getDestination(),
+                                config.getGroupId()}, '-'));
+        return sameMatch || prefixMatch;
+    }
+}

client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/config/ConfigLoader.java

@@ -0,0 +1,48 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.config;
+
+import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
+import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * Phoenix表映射配置加载器
+ */
+public class ConfigLoader {
+
+    private static Logger logger = LoggerFactory.getLogger(ConfigLoader.class);
+
+    /**
+     * 加载Phoenix表映射配置
+     *
+     * @return 配置名/配置文件名--对象
+     */
+    public static Map<String, MappingConfig> load(Properties envProperties) {
+        logger.info("## Start loading phoenix mapping config ... ");
+
+        Map<String, MappingConfig> result = new LinkedHashMap<>();
+
+        Map<String, String> configContentMap = MappingConfigsLoader.loadConfigs("phoenix");
+        configContentMap.forEach((fileName, content) -> {
+            MappingConfig config = YmlConfigBinder
+                    .bindYmlToObj(null, content, MappingConfig.class, null, envProperties);
+            if (config == null) {
+                return;
+            }
+            try {
+                config.validate();
+            } catch (Exception e) {
+                throw new RuntimeException("ERROR Config: " + fileName + " " + e.getMessage(), e);
+            }
+            result.put(fileName, config);
+        });
+
+        logger.info("## Phoenix mapping config loaded");
+        logger.info("## Phoenix sync threads: " + ConfigurationManager.getInteger("threads"));
+        return result;
+    }
+}

client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/config/ConfigurationManager.java

@@ -0,0 +1,49 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.config;
+
+import java.io.InputStream;
+import java.util.Properties;
+
+/**
+ * 配置管理组件
+ * @author Administrator
+ *
+ */
+public class ConfigurationManager {
+
+	private static Properties prop = new Properties();
+
+	static {
+		try {
+			InputStream in = ConfigurationManager.class
+					.getClassLoader().getResourceAsStream("phoenix/phoenix_common.properties");
+			prop.load(in);  
+		} catch (Exception e) {
+			e.printStackTrace();  
+		}
+	}
+	
+	/**
+	 * 获取指定key对应的value
+	 *
+	 * @param key 
+	 * @return 返回value是字符串
+	 */
+	public static String getProperty(String key) {
+		return prop.getProperty(key);
+	}
+	
+	/**
+	 * 获取整数类型的配置项
+	 * @param key StringKye
+	 * @return value
+	 */
+	public static Integer getInteger(String key) {
+		String value = getProperty(key);
+		try {
+			return Integer.valueOf(value);
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+		return 0;
+	}
+}

client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/config/MappingConfig.java

@@ -0,0 +1,288 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.config;
+
+import org.apache.commons.lang.StringUtils;
+
+import java.util.*;
+
+/**
+ * Phoenix表映射配置
+ */
+@SuppressWarnings("unused")
+public class MappingConfig {
+
+    private String dataSourceKey;           // 数据源key
+    private String destination;             // canal实例或MQ的topic
+    private String groupId;                 // groupId
+    private String outerAdapterKey;         // 对应适配器的key
+    private boolean concurrent = false;     // 是否并行同步
+    private DbMapping dbMapping;            // db映射配置
+    private boolean debug = false;          // 调试
+
+    public String getDataSourceKey() {
+        return dataSourceKey;
+    }
+
+    public void setDataSourceKey(String dataSourceKey) {
+        this.dataSourceKey = dataSourceKey;
+    }
+
+    public String getGroupId() {
+        return groupId;
+    }
+
+    public void setGroupId(String groupId) {
+        this.groupId = groupId;
+    }
+
+    public String getOuterAdapterKey() {
+        return outerAdapterKey;
+    }
+
+    public void setOuterAdapterKey(String outerAdapterKey) {
+        this.outerAdapterKey = outerAdapterKey;
+    }
+
+    public boolean getConcurrent() {
+        return concurrent;
+    }
+
+    public void setConcurrent(boolean concurrent) {
+        this.concurrent = concurrent;
+    }
+
+    public DbMapping getDbMapping() {
+        return dbMapping;
+    }
+
+    public void setDbMapping(DbMapping dbMapping) {
+        this.dbMapping = dbMapping;
+    }
+
+    public String getDestination() {
+        return destination;
+    }
+
+    public void setDestination(String destination) {
+        this.destination = destination;
+    }
+
+    public boolean isDebug() {
+        return debug;
+    }
+
+    public void setDebug(boolean debug) {
+        this.debug = debug;
+    }
+
+    public void validate() {
+        if (dbMapping.database == null || dbMapping.database.isEmpty()) {
+            throw new NullPointerException("dbMapping.database");
+        }
+        if ((dbMapping.table == null || dbMapping.table.isEmpty())) {
+            throw new NullPointerException("dbMapping.table");
+        }
+        if ((dbMapping.targetTable == null || dbMapping.targetTable.isEmpty())) {
+            throw new NullPointerException("dbMapping.targetTable");
+        }
+    }
+
+    public static class DbMapping {
+
+        private String database;                            // 数据库名或schema名
+        private String table;                               // 表名
+        private Map<String, String> targetPk = new LinkedHashMap<>(); // 目标表主键字段
+        private boolean mapAll = true;                      // 映射所有字段
+
+        private boolean alter = true;                       // 是否允许修改表
+        private boolean drop = false;                       // 是否允许删除字段
+        private boolean limit = false;                      // 是否限制字段长度
+        private boolean skipMissing = false;                // 是否跳过丢失的字段
+        private boolean escapeUpper = true;                 // 字段默认大写加双引号
+
+        private String targetDb;                            // 目标库名
+        private String targetTable;                         // 目标表名
+        private Map<String, String> targetColumns;          // 目标表字段映射
+
+        private List<String> excludeColumns;                // 不映射的字段
+
+        private String etlCondition;                        // etl条件sql
+        private int readBatch = 5000;
+        private int commitBatch = 5000;                     // etl等批量提交大小
+        private Map<String, String> allMapColumns;
+
+        public String escape(String name) {
+            if (escapeUpper) {
+                return "\"" + name.toUpperCase() + "\"";
+            } else {
+                return name;
+            }
+        }
+
+        public String getDatabase() {
+            return database;
+        }
+
+        public void setDatabase(String database) {
+            this.database = database;
+        }
+
+        public String getTable() {
+            return table;
+        }
+
+        public void setTable(String table) {
+            this.table = table;
+        }
+
+        public Map<String, String> getTargetPk() {
+            return targetPk;
+        }
+
+        public void setTargetPk(Map<String, String> targetPk) {
+            this.targetPk = targetPk;
+        }
+
+        public Boolean getMapAll() {
+            return mapAll;
+        }
+
+        public void setMapAll(Boolean mapAll) {
+            this.mapAll = mapAll;
+        }
+
+        public boolean isAlter() {
+            return alter;
+        }
+
+        public void setAlter(boolean alter) {
+            this.alter = alter;
+        }
+
+        public boolean isLimit() {
+            return limit;
+        }
+
+        public void setLimit(boolean limit) {
+            this.limit = limit;
+        }
+
+        public boolean isDrop() {
+            return drop;
+        }
+
+        public void setDrop(boolean drop) {
+            this.drop = drop;
+        }
+
+        public boolean isSkipMissing() {
+            return skipMissing;
+        }
+
+        public void setSkipMissing(boolean skipMissing) {
+            this.skipMissing = skipMissing;
+        }
+
+        public boolean isEscapeUpper() {
+            return escapeUpper;
+        }
+
+        public void setEscapeUpper(boolean escapeUpper) {
+            this.escapeUpper = escapeUpper;
+        }
+
+        public String getTargetDb() {
+            return targetDb;
+        }
+
+        public void setTargetDb(String targetDb) {
+            this.targetDb = targetDb;
+        }
+
+        public String getTargetTable() {
+            return targetTable;
+        }
+
+        public void setTargetTable(String targetTable) {
+            this.targetTable = targetTable;
+        }
+
+        public Map<String, String> getTargetColumns() {
+            if (targetColumns != null) {
+                targetColumns.forEach((key, value) -> {
+                    if (StringUtils.isEmpty(value)) {
+                        targetColumns.put(key, key);
+                    }
+                });
+            } else {
+                targetColumns = new HashMap<>();
+            }
+            return targetColumns;
+        }
+
+        public void setTargetColumns(Map<String, String> targetColumns) {
+            this.targetColumns = targetColumns;
+        }
+
+        public void addTargetColumn(String key, String value) {
+            if (targetColumns == null) {
+                targetColumns = new HashMap<>();
+            }
+            targetColumns.put(key, value);
+            if (allMapColumns != null) {
+                allMapColumns.put(key, value);
+            }
+        }
+
+        public void removeTargetColumn(String key) {
+            if (targetColumns != null) {
+                targetColumns.remove(key);
+            }
+            if (allMapColumns != null) {
+                allMapColumns.remove(key);
+            }
+        }
+
+        public List<String> getExcludeColumns() {
+            if (excludeColumns == null) {
+                excludeColumns = new ArrayList<>();
+            }
+            return excludeColumns;
+        }
+
+        public void setExcludeColumns(List<String> excludeColumns) {
+            this.excludeColumns = excludeColumns;
+        }
+
+        public String getEtlCondition() {
+            return etlCondition;
+        }
+
+        public void setEtlCondition(String etlCondition) {
+            this.etlCondition = etlCondition;
+        }
+
+        public int getReadBatch() {
+            return readBatch;
+        }
+
+        public void setReadBatch(int readBatch) {
+            this.readBatch = readBatch;
+        }
+
+        public int getCommitBatch() {
+            return commitBatch;
+        }
+
+        public void setCommitBatch(int commitBatch) {
+            this.commitBatch = commitBatch;
+        }
+
+        public Map<String, String> getAllMapColumns() {
+            return allMapColumns;
+        }
+
+        public void setAllMapColumns(Map<String, String> allMapColumns) {
+            this.allMapColumns = allMapColumns;
+        }
+    }
+}

client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/monitor/PhoenixConfigMonitor.java

@@ -0,0 +1,125 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.monitor;
+
+import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
+import com.alibaba.otter.canal.client.adapter.phoenix.PhoenixAdapter;
+import com.alibaba.otter.canal.client.adapter.phoenix.config.MappingConfig;
+import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
+import com.alibaba.otter.canal.client.adapter.support.Util;
+import java.io.File;
+import java.util.Properties;
+import org.apache.commons.io.filefilter.FileFilterUtils;
+import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
+import org.apache.commons.io.monitor.FileAlterationMonitor;
+import org.apache.commons.io.monitor.FileAlterationObserver;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * phoenix config monitor
+ */
+public class PhoenixConfigMonitor {
+
+    private static final Logger logger = LoggerFactory.getLogger(PhoenixConfigMonitor.class);
+
+    private static final String adapterName = "phoenix";  //相应组件名字
+
+    private String key;
+
+    private PhoenixAdapter phoenixAdapter;   //相应适配器名实现类
+
+    private Properties envProperties;
+
+    private FileAlterationMonitor fileMonitor;
+
+    public void init(String key, PhoenixAdapter phoenixAdapter, Properties envProperties) {
+        this.key = key;
+        this.phoenixAdapter = phoenixAdapter;
+        this.envProperties = envProperties;
+        File confDir = Util.getConfDirPath(adapterName);
+        try {
+            FileAlterationObserver observer = new FileAlterationObserver(confDir,
+                    FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml")));
+            FileListener listener = new FileListener();
+            observer.addListener(listener);
+            fileMonitor = new FileAlterationMonitor(3000, observer);
+            fileMonitor.start();
+
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+
+    public void destroy() {
+        try {
+            fileMonitor.stop();
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+
+    private class FileListener extends FileAlterationListenerAdaptor {
+
+        @Override
+        public void onFileCreate(File file) {
+            super.onFileCreate(file);
+            try {
+                // 加载新增的配置文件
+                String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName());
+                MappingConfig config = YmlConfigBinder
+                        .bindYmlToObj(null, configContent, MappingConfig.class, null, envProperties);
+                if (config == null) {
+                    return;
+                }
+                config.validate();
+                boolean result = phoenixAdapter.addConfig(file.getName(), config);
+                if (result) {
+                    logger.info("Add a new phoenix mapping config: {} to canal adapter",
+                            file.getName());
+                }
+            } catch (Exception e) {
+                logger.error(e.getMessage(), e);
+            }
+        }
+
+        @Override
+        public void onFileChange(File file) {
+            super.onFileChange(file);
+
+            try {
+                if (phoenixAdapter.getPhoenixMapping().containsKey(file.getName())) {
+                    // 加载配置文件
+                    String configContent = MappingConfigsLoader
+                            .loadConfig(adapterName + File.separator + file.getName());
+                    if (configContent == null) {
+                        onFileDelete(file);
+                        return;
+                    }
+                    MappingConfig config = YmlConfigBinder
+                            .bindYmlToObj(null, configContent, MappingConfig.class, null, envProperties);
+                    if (config == null) {
+                        return;
+                    }
+                    config.validate();
+                    phoenixAdapter.updateConfig(file.getName(), config);
+                    logger.info("Change a phoenix mapping config: {} of canal adapter", file.getName());
+                }
+            } catch (Exception e) {
+                logger.error(e.getMessage(), e);
+            }
+        }
+
+        @Override
+        public void onFileDelete(File file) {
+            super.onFileDelete(file);
+
+            try {
+                if (phoenixAdapter.getPhoenixMapping().containsKey(file.getName())) {
+                    phoenixAdapter.deleteConfig(file.getName());
+                    logger.info("Delete a phoenix mapping config: {} of canal adapter", file.getName());
+                }
+            } catch (Exception e) {
+                logger.error(e.getMessage(), e);
+            }
+        }
+    }
+}

client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/service/PhoenixEtlService.java

@@ -0,0 +1,429 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.service;
+
+import com.alibaba.otter.canal.client.adapter.phoenix.PhoenixAdapter;
+import com.alibaba.otter.canal.client.adapter.phoenix.config.MappingConfig;
+import com.alibaba.otter.canal.client.adapter.phoenix.config.MappingConfig.DbMapping;
+import com.alibaba.otter.canal.client.adapter.phoenix.support.PhoenixSupportUtil;
+import com.alibaba.otter.canal.client.adapter.phoenix.support.SyncUtil;
+import com.alibaba.otter.canal.client.adapter.phoenix.support.TypeUtil;
+import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
+import com.alibaba.otter.canal.client.adapter.support.EtlResult;
+import com.alibaba.otter.canal.client.adapter.support.Util;
+import com.google.common.base.Joiner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.sql.DataSource;
+import java.sql.*;
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Phoenix ETL 操作业务类
+ */
+public class PhoenixEtlService {
+
+    private static final Logger logger = LoggerFactory.getLogger(PhoenixEtlService.class);
+
+    private static String[] splitNotEmpty(String s) {
+        if (s != null && s.trim().length() > 0) {
+            return s.trim().split(",");
+        }
+        return new String[]{};
+    }
+
+
+    static boolean syncSchema(Connection targetDSConnection, MappingConfig config) {
+        DataSource srcDataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
+        if (srcDataSource == null) {
+            return false;
+        }
+        try {
+            return syncSchema(srcDataSource.getConnection(), targetDSConnection, config);
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private static boolean syncSchema(DataSource srcDS,Connection targetDSConnection, MappingConfig config) {
+        try {
+            return syncSchema(srcDS.getConnection(),targetDSConnection, config);
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private static boolean syncSchema(Connection srcDS, Connection targetDS, MappingConfig config) {
+        DbMapping dbMapping = config.getDbMapping();
+        if (dbMapping.getMapAll() && dbMapping.isAlter()) { // 检查字段是否缺失
+            Map<String, Integer> targetColumnType = new LinkedHashMap<>();
+            String targetTable = SyncUtil.getDbTableName(dbMapping);
+            try {
+                Util.sqlRS(targetDS, "SELECT * FROM " + targetTable + " LIMIT 1", rs -> {
+                    try {
+                        ResultSetMetaData rsd = rs.getMetaData();
+                        int columnCount = rsd.getColumnCount();
+                        for (int i = 1; i <= columnCount; i++) {
+                            targetColumnType.put(rsd.getColumnName(i).toLowerCase(), rsd.getColumnType(i));
+                        }
+                    } catch (Exception e) {
+                        logger.error(dbMapping.getTable() + " etl failed! ==>" + e.getMessage(), e);
+                    }
+                });
+            } catch (RuntimeException e) {
+                if (!e.getCause().getClass().getName().endsWith("TableNotFoundException")) {
+                    throw e;
+                }
+            }
+            StringBuilder missing = new StringBuilder();
+            StringBuilder constraint = new StringBuilder();
+            Util.sqlRS(srcDS, "SELECT * FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '" + dbMapping.getDatabase() + "'  AND TABLE_NAME = '" + dbMapping.getTable() + "'", rs -> {
+                try {
+                    List<String> excludeColumns = config.getDbMapping().getExcludeColumns();
+                    while (rs.next()) {
+                        String name = rs.getString("COLUMN_NAME");
+                        String lower = name.toLowerCase();
+                        String colType = rs.getString("COLUMN_TYPE");
+                        if (targetColumnType.get(lower) == null && !excludeColumns.contains(lower)) {
+                            boolean isPri = rs.getString("COLUMN_KEY").equals("PRI");
+                            String[] args = splitNotEmpty(colType.replaceAll("^\\w+(?:\\(([^)]*)\\))?[\\s\\S]*$", "$1"));
+                            missing.append(dbMapping.escape(name)).append(" ").append(TypeUtil.getPhoenixType(
+                                    rs.getString("DATA_TYPE").toUpperCase(),
+                                    args,
+                                    colType.contains("unsigned"),
+                                    dbMapping.isLimit()
+                            ));
+                            if (isPri) {
+                                if (args.length > 0 && dbMapping.isLimit() || rs.getString("IS_NULLABLE").equals("NO")) {
+                                    missing.append(" NOT NULL");
+                                }
+                                constraint.append(dbMapping.escape(name)).append(',');
+                            }
+                            missing.append(',');
+                        }
+                    }
+                } catch (Exception e) {
+                    logger.error(dbMapping.getDatabase() + "." + dbMapping.getTable() + " schema failed! ==>" + e.getMessage(), e);
+                    throw new RuntimeException(e);
+                }
+            });
+            if (missing.length() > 0) {
+                String sql;
+                if (targetColumnType.isEmpty()) {
+                    if (constraint.length() > 0) {
+                        constraint.deleteCharAt(constraint.length() - 1);
+                        missing.append("CONSTRAINT pk PRIMARY KEY(").append(constraint.toString()).append(")");
+                    } else {
+                        missing.deleteCharAt(missing.length() - 1);
+                    }
+                    sql = "CREATE TABLE " + targetTable + " (" + missing.toString() + ")";
+                } else {
+                    missing.deleteCharAt(missing.length() - 1);
+                    sql = "ALTER TABLE " + targetTable + " ADD " + missing.toString();
+                }
+                logger.info("schema missing: {} {}", targetColumnType, sql);
+                try (PreparedStatement pstmt = targetDS.prepareStatement(sql)) {
+                    pstmt.executeUpdate();
+                } catch (SQLException e) {
+                    logger.error("sync schema error: " + e.getMessage(), e);
+                }
+            } else {
+                logger.debug("schema ok: {}", targetColumnType);
+            }
+            return true;
+        }
+        return false;
+    }
+
+    /**
+     * 导入数据
+     */
+    public static EtlResult importData(DataSource srcDS, Connection targetDSConnection, MappingConfig config,
+                                       List<String> params) {
+        EtlResult etlResult = new EtlResult();
+        AtomicLong successCount = new AtomicLong();
+        List<String> errMsg = new ArrayList<>();
+        String hbaseTable = "";
+        try {
+            if (config == null) {
+                logger.error("Config is null!");
+                etlResult.setSucceeded(false);
+                etlResult.setErrorMessage("Config is null!");
+                return etlResult;
+            }
+            boolean debug = params != null && params.get(0).equals("_debug");
+            if (debug) {
+                params = params.subList(1, params.size());
+            }
+            syncSchema(srcDS, targetDSConnection, config);
+            DbMapping dbMapping = config.getDbMapping();
+
+            long start = System.currentTimeMillis();
+
+            // 拼接sql
+            StringBuilder sql = new StringBuilder(
+                    "SELECT * FROM " + dbMapping.getDatabase() + "." + dbMapping.getTable());
+
+            // 拼接条件
+            appendCondition(params, dbMapping, srcDS, sql);
+
+            // 获取总数
+            String countSql = "SELECT COUNT(1) FROM ( " + sql + ") _CNT ";
+            long cnt = (Long) Util.sqlRS(srcDS, countSql, rs -> {
+                Long count = null;
+                try {
+                    if (rs.next()) {
+                        count = ((Number) rs.getObject(1)).longValue();
+                    }
+                } catch (Exception e) {
+                    logger.error(e.getMessage(), e);
+                }
+                return count == null ? 0 : count;
+            });
+
+            // 当大于1万条记录时开启多线程
+            if (cnt >= 10000) {
+                int threadCount = 3;
+                long perThreadCnt = cnt / threadCount;
+                ExecutorService executor = Util.newFixedThreadPool(threadCount, 5000L);
+                for (int i = 0; i < threadCount; i++) {
+                    long offset = i * perThreadCnt;
+                    Long size = null;
+                    if (i != threadCount - 1) {
+                        size = perThreadCnt;
+                    }
+                    String sqlFinal;
+                    if (size != null) {
+                        sqlFinal = sql + " LIMIT " + offset + "," + size;
+                    } else {
+                        sqlFinal = sql + " LIMIT " + offset + "," + cnt;
+                    }
+                    executor
+                            .execute(() -> executeSqlImport(srcDS, targetDSConnection, sqlFinal, dbMapping, successCount, errMsg, debug));
+                }
+
+                executor.shutdown();
+                //noinspection StatementWithEmptyBody
+                while (!executor.awaitTermination(3, TimeUnit.SECONDS)) ;
+            } else {
+                executeSqlImport(srcDS, targetDSConnection, sql.toString(), dbMapping, successCount, errMsg, debug);
+            }
+
+            logger.info(
+                    dbMapping.getTable() + " etl completed in: " + (System.currentTimeMillis() - start) / 1000 + "s!");
+
+            etlResult
+                    .setResultMessage("导入目标表 " + SyncUtil.getDbTableName(dbMapping) + " 数据：" + successCount.get() + " 条");
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+            errMsg.add(hbaseTable + " etl failed! ==>" + e.getMessage());
+        }
+
+        if (errMsg.isEmpty()) {
+            etlResult.setSucceeded(true);
+        } else {
+            etlResult.setErrorMessage(Joiner.on("\n").join(errMsg));
+        }
+        return etlResult;
+    }
+
+    private static void appendCondition(List<String> params, DbMapping dbMapping, DataSource ds,
+                                        StringBuilder sql) {
+        if (params != null && params.size() == 1 && dbMapping.getEtlCondition() == null) {
+            AtomicBoolean stExists = new AtomicBoolean(false);
+            // 验证是否有SYS_TIME字段
+            Util.sqlRS(ds, sql.toString(), rs -> {
+                try {
+                    ResultSetMetaData rsmd = rs.getMetaData();
+                    int cnt = rsmd.getColumnCount();
+                    for (int i = 1; i <= cnt; i++) {
+                        String columnName = rsmd.getColumnName(i);
+                        if ("SYS_TIME".equalsIgnoreCase(columnName)) {
+                            stExists.set(true);
+                            break;
+                        }
+                    }
+                } catch (Exception e) {
+                    // ignore
+                }
+                return null;
+            });
+            if (stExists.get()) {
+                sql.append(" WHERE SYS_TIME >= '").append(params.get(0)).append("' ");
+            }
+        } else if (dbMapping.getEtlCondition() != null && params != null && params.size() > 0) {
+            String etlCondition = dbMapping.getEtlCondition();
+            int size = params.size();
+            for (int i = 0; i < size; i++) {
+                etlCondition = etlCondition.replace("{" + i + "}", params.get(i));
+            }
+
+            sql.append(" ").append(etlCondition);
+        }
+    }
+
+    /**
+     * 执行导入
+     */
+    private static boolean executeSqlImport(DataSource srcDS, Connection targetDSConnection, String sql, DbMapping dbMapping,
+                                            AtomicLong successCount, List<String> errMsg, boolean debug) {
+        try {
+            Map<String, String> columnsMap = new LinkedHashMap<>();
+            Map<String, Integer> columnType = new LinkedHashMap<>();
+
+
+            PhoenixSupportUtil.sqlRS(targetDSConnection, "SELECT * FROM " + SyncUtil.getDbTableName(dbMapping) + " LIMIT 1 ", rs -> {
+                try {
+
+                    ResultSetMetaData rsd = rs.getMetaData();
+                    int columnCount = rsd.getColumnCount();
+                    List<String> columns = new ArrayList<>();
+                    List<String> excludeColumns = dbMapping.getExcludeColumns();
+                    for (int i = 1; i <= columnCount; i++) {
+                        String lower = rsd.getColumnName(i).toLowerCase();
+                        if (!excludeColumns.contains(lower)) {
+                            columnType.put(lower, rsd.getColumnType(i));
+                            columns.add(lower);
+                        }
+                    }
+                    columnsMap.putAll(SyncUtil.getColumnsMap(dbMapping, columns));
+                    return true;
+                } catch (Exception e) {
+                    logger.error(e.getMessage(), e);
+                    return false;
+                }
+            });
+            Util.sqlRS(srcDS, sql, rs -> {
+                int idx = 1;
+
+                try {
+                    boolean completed = false;
+
+                    // if (dbMapping.isMapAll()) {
+                    // columnsMap = dbMapping.getAllColumns();
+                    // } else {
+                    // columnsMap = dbMapping.getTargetColumns();
+                    // }
+
+                    StringBuilder insertSql = new StringBuilder();
+                    insertSql.append("UPSERT INTO ").append(SyncUtil.getDbTableName(dbMapping)).append(" (");
+                    columnsMap
+                            .forEach((targetColumnName, srcColumnName) -> insertSql.append(dbMapping.escape(targetColumnName)).append(","));
+
+                    int len = insertSql.length();
+                    insertSql.delete(len - 1, len).append(") VALUES (");
+                    int mapLen = columnsMap.size();
+                    for (int i = 0; i < mapLen; i++) {
+                        insertSql.append("?,");
+                    }
+                    len = insertSql.length();
+                    insertSql.delete(len - 1, len).append(")");
+                    try (
+                         //Connection connTarget = targetDS.getConnection();
+                         Connection connTarget =PhoenixAdapter.getPhoenixConnection();
+                         PreparedStatement pstmt = connTarget.prepareStatement(insertSql.toString())) {
+                        connTarget.setAutoCommit(false);
+
+                        while (rs.next()) {
+                            completed = false;
+
+                            pstmt.clearParameters();
+
+                            // 删除数据
+                            Map<String, Object> values = new LinkedHashMap<>();
+                            StringBuilder deleteSql = new StringBuilder(
+                                    "DELETE FROM " + SyncUtil.getDbTableName(dbMapping) + " WHERE ");
+                            appendCondition(dbMapping, deleteSql, values, rs);
+                            try (PreparedStatement pstmt2 = connTarget.prepareStatement(deleteSql.toString())) {
+                                int k = 1;
+                                for (Object val : values.values()) {
+                                    pstmt2.setObject(k++, val);
+                                }
+                                pstmt2.execute();
+                            }
+
+                            Map<String, Object> insertValues = new HashMap<>();
+                            int i = 1;
+                            for (Map.Entry<String, String> entry : columnsMap.entrySet()) {
+                                String targetClolumnName = entry.getKey();
+                                String srcColumnName = entry.getValue();
+                                if (srcColumnName == null) {
+                                    srcColumnName = targetClolumnName;
+                                }
+
+                                Integer type = columnType.get(targetClolumnName.toLowerCase());
+
+                                try {
+                                    Object value = rs.getObject(srcColumnName);
+                                    insertValues.put(srcColumnName, value);
+                                    if (value != null) {
+                                        SyncUtil.setPStmt(type, pstmt, value, i);
+                                    } else {
+                                        pstmt.setNull(i, type);
+                                    }
+                                } catch (SQLException e) {
+                                    insertValues.put(srcColumnName, null);
+                                    pstmt.setNull(i, type);
+                                }
+                                i++;
+                            }
+                            if (debug) {
+                                logger.info("insert sql: {} {} {}", insertSql, insertValues, pstmt);
+                            }
+
+                            pstmt.execute();
+                            if (logger.isTraceEnabled()) {
+                                logger.trace("Insert into target table, sql: {}", insertSql);
+                            }
+
+                            if (idx % dbMapping.getCommitBatch() == 0) {
+                                connTarget.commit();
+                                completed = true;
+                            }
+                            idx++;
+                            successCount.incrementAndGet();
+                            if (logger.isDebugEnabled()) {
+                                logger.debug("successful import count:" + successCount.get());
+                            }
+                        }
+                        if (!completed) {
+                            connTarget.commit();
+                        }
+                    }
+
+                } catch (Exception e) {
+                    logger.error(dbMapping.getTable() + " etl failed! ==>" + e.getMessage(), e);
+                    errMsg.add(dbMapping.getTable() + " etl failed! ==>" + e.getMessage());
+                }
+                return idx;
+            });
+            return true;
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+            return false;
+        }
+    }
+
+    /**
+     * 拼接目标表主键where条件
+     */
+    private static void appendCondition(DbMapping dbMapping, StringBuilder sql, Map<String, Object> values,
+                                        ResultSet rs) throws SQLException {
+        // 拼接主键
+        for (Map.Entry<String, String> entry : dbMapping.getTargetPk().entrySet()) {
+            String targetColumnName = entry.getKey();
+            String srcColumnName = entry.getValue();
+            if (srcColumnName == null) {
+                srcColumnName = targetColumnName;
+            }
+            sql.append(dbMapping.escape(targetColumnName)).append("=? AND ");
+            values.put(targetColumnName, rs.getObject(srcColumnName));
+        }
+        int len = sql.length();
+        sql.delete(len - 4, len);
+    }
+}

client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/service/PhoenixSyncService.java

@@ -0,0 +1,644 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.service;
+
+import com.alibaba.druid.sql.SQLUtils;
+import com.alibaba.druid.sql.ast.SQLName;
+import com.alibaba.druid.sql.ast.SQLStatement;
+import com.alibaba.druid.sql.ast.statement.*;
+import com.alibaba.druid.sql.parser.ParserException;
+import com.alibaba.druid.util.JdbcConstants;
+import com.alibaba.fastjson2.JSON;
+import com.alibaba.fastjson2.JSONWriter.Feature;
+import com.alibaba.otter.canal.client.adapter.phoenix.config.ConfigurationManager;
+import com.alibaba.otter.canal.client.adapter.phoenix.config.MappingConfig;
+import com.alibaba.otter.canal.client.adapter.phoenix.config.MappingConfig.DbMapping;
+import com.alibaba.otter.canal.client.adapter.phoenix.support.BatchExecutor;
+import com.alibaba.otter.canal.client.adapter.phoenix.support.SingleDml;
+import com.alibaba.otter.canal.client.adapter.phoenix.support.SyncUtil;
+import com.alibaba.otter.canal.client.adapter.phoenix.support.TypeUtil;
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+import com.alibaba.otter.canal.client.adapter.support.Util;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Connection;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.function.Function;
+
+/**
+ * Phoenix同步操作业务
+ */
+public class PhoenixSyncService {
+
+    private static final Logger logger = LoggerFactory.getLogger(PhoenixSyncService.class);
+
+    // 源库表字段类型缓存: instance.schema.table -> <columnName, jdbcType>
+    private Map<String, Map<String, Integer>> columnsTypeCache;
+
+    //同步线程数
+    //默认开启3个线程同步,此处配置了自定义同步线程数
+    private int threads = ConfigurationManager.getInteger("threads");
+
+    private List<SyncItem>[] dmlsPartition;
+    private BatchExecutor[] batchExecutors;
+    private ExecutorService[] executorThreads;
+
+    public PhoenixSyncService(Integer threads) {
+        this(threads, new ConcurrentHashMap<>());
+    }
+
+
+    @SuppressWarnings("unchecked")
+    private PhoenixSyncService(Integer threads, Map<String, Map<String, Integer>> columnsTypeCache) {
+        this.columnsTypeCache = columnsTypeCache;
+        try {
+            if (threads != null) {
+                this.threads = threads;
+            }
+            this.dmlsPartition = new List[this.threads];
+            this.batchExecutors = new BatchExecutor[this.threads];
+            this.executorThreads = new ExecutorService[this.threads];
+            for (int i = 0; i < this.threads; i++) {
+                dmlsPartition[i] = new ArrayList<>();
+                batchExecutors[i] = new BatchExecutor();
+                //创建单个线程，用来操作一个无界的队列任务，不会使用额外的线程。如果线程崩溃会重新创建一个，直到任务完成。
+                executorThreads[i] = Executors.newSingleThreadExecutor();
+            }
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * 批量同步回调
+     *
+     * @param dmls     批量 DML
+     * @param function 回调方法
+     */
+    private void sync(List<Dml> dmls, Function<Dml, Boolean> function) {
+        try {
+            boolean toExecute = false;
+            for (Dml dml : dmls) {
+                if (!toExecute) {
+                    toExecute = function.apply(dml);
+                } else {
+                    function.apply(dml);
+                }
+            }
+            if (toExecute) {
+                List<Future<Boolean>> futures = new ArrayList<>();
+                for (int i = 0; i < threads; i++) {
+                    int j = i;
+                    if (dmlsPartition[j].isEmpty()) {
+                        // bypass
+                        continue;
+                    }
+
+                    futures.add(executorThreads[i].submit(() -> {
+                        try {
+                            dmlsPartition[j].forEach(syncItem -> sync(batchExecutors[j],
+                                    syncItem.config,
+                                    syncItem.singleDml));
+                            //相对于RDB同步 少了  dmlsPartition[j].clear();
+                            //在 try catch中获取异常后再次执行一次batchExecutors[j].commit()
+                            batchExecutors[j].commit();
+                            return true;
+                        } catch (Throwable e) {
+                            batchExecutors[j].rollback();
+                            if (!e.getClass().getName().endsWith("ColumnNotFoundException")
+                                    && !e.getClass().getName().endsWith("TableNotFoundException")) {
+                                throw new RuntimeException(e);
+                            }
+                            logger.info("table or column not found: " + e.getMessage());
+                            boolean synced = false;
+                            for (SyncItem syncItem : dmlsPartition[j]) {
+                                if (PhoenixEtlService.syncSchema(batchExecutors[j].getConn(), syncItem.config)) {
+                                    synced = true;
+                                }
+                            }
+                            if (!synced) {
+                                throw new RuntimeException(e);
+                            }
+                            dmlsPartition[j].forEach(syncItem -> sync(batchExecutors[j],
+                                    syncItem.config,
+                                    syncItem.singleDml));
+                            try {
+                                batchExecutors[j].commit();
+                                return true;
+                            } catch (Throwable e1) {
+                                batchExecutors[j].rollback();
+                                throw new RuntimeException(e1);
+                            }
+                        } finally {
+                            dmlsPartition[j].clear();
+                        }
+                    }));
+                }
+                futures.forEach(future -> {
+                    try {
+                        future.get();
+                    } catch (ExecutionException | InterruptedException e) {
+                        throw new RuntimeException(e);
+                    }
+                });
+            }
+        } finally {
+            for (BatchExecutor batchExecutor : batchExecutors) {
+                if (batchExecutor != null) {
+                    batchExecutor.close();
+                }
+            }
+        }
+    }
+
+    /**
+     * 批量同步 :将批量DML进行解析并放入List<SingleDml> --> dmlsPartition[hash].add(syncItem);
+     * @param mappingConfig 配置集合
+     * @param dmls   批量 DML
+     */
+    public void sync(Map<String, Map<String, MappingConfig>> mappingConfig, List<Dml> dmls, Properties envProperties) {
+        sync(dmls, dml -> {
+            String destination = StringUtils.trimToEmpty(dml.getDestination());
+            String groupId = StringUtils.trimToEmpty(dml.getGroupId());
+            String database = dml.getDatabase();
+            String table = dml.getTable().toLowerCase();
+            Map<String, MappingConfig> configMap;
+            if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
+                //tcp 模式
+                configMap = mappingConfig.get(destination + "-" + groupId + "_" + database + "-" + table);
+            } else {
+                //kafka 模式 或者 RocketMQ模式
+                configMap = mappingConfig.get(destination + "_" + database + "-" + table);
+            }
+
+            if (configMap == null) {
+                if (logger.isTraceEnabled()) {
+                    logger.trace("no config map: destination={},groupId={}, database={}, table={}, keys={}", destination, groupId, database, table, mappingConfig.keySet());
+                }
+                return false;
+            }
+            if (configMap.values().isEmpty()) {
+                logger.info("config map empty: destination={},groupId={}, database={}, table={}, keys={}", destination, groupId, database, table, mappingConfig.keySet());
+                return false;
+            }
+            if (dml.getIsDdl() != null && dml.getIsDdl() && StringUtils.isNotEmpty(dml.getSql())) {
+                // DDL
+                columnsTypeCache.remove(dml.getDestination() + "." + dml.getDatabase() + "." + dml.getTable());
+                List<SQLStatement> stmtList;
+                try {
+                    stmtList = SQLUtils.parseStatements(dml.getSql(), JdbcConstants.MYSQL, false);
+                } catch (ParserException e) {
+                    // 可能存在一些SQL是不支持的，比如存储过程
+                    logger.info("parse sql error: " + dml.getSql(), e);
+                    return false;
+                }
+                for (Map.Entry<String, MappingConfig> entry : configMap.entrySet()) {
+                    try {
+                        alter(batchExecutors[0], entry.getValue(), dml, stmtList, entry.getKey());
+                    } catch (SQLException e) {
+                        throw new RuntimeException(e);
+                    }
+                }
+                return false;
+            } else {
+                // DML
+                for (Map.Entry<String, MappingConfig> entry : configMap.entrySet()) {
+                    MappingConfig config = entry.getValue();
+                    if (config.isDebug()) {
+                        logger.info("DML: {} {}", entry.getKey(), JSON.toJSONString(dml, Feature.WriteNulls));
+                    }
+                    if (config.getConcurrent()) {
+                        //并行同步
+                        //将一批DML转成SingleDml
+                        List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml);
+                        singleDmls.forEach(singleDml -> {
+                            //取主键hash
+                            int hash = pkHash(config.getDbMapping(), singleDml.getData());
+                            SyncItem syncItem = new SyncItem(config, singleDml);
+                            //相同的主键数据的顺序是可以保证的
+                            dmlsPartition[hash].add(syncItem);
+                        });
+                    } else {
+                        //不并行同步
+                        int hash = 0;
+                        List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml);
+                        singleDmls.forEach(singleDml -> {
+                            SyncItem syncItem = new SyncItem(config, singleDml);
+                            //这里线程默认是3个,如果不并行，则会出现2个线程空跑
+                            dmlsPartition[hash].add(syncItem);
+                        });
+                    }
+                }
+                return true;
+            }
+        });
+    }
+
+    /**
+     * 单条 dml 同步
+     *
+     * @param batchExecutor 批量事务执行器
+     * @param config        对应配置对象
+     * @param dml           DML
+     */
+    private void sync(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) {
+        if (config != null) {
+            try {
+                String type = dml.getType();
+                if (type != null && type.equalsIgnoreCase("INSERT")) {
+                    insert(batchExecutor, config, dml);
+                } else if (type != null && type.equalsIgnoreCase("UPDATE")) {
+                    insert(batchExecutor, config, dml);
+                } else if (type != null && type.equalsIgnoreCase("DELETE")) {
+                    delete(batchExecutor, config, dml);
+                } else if (type != null && type.equalsIgnoreCase("TRUNCATE")) {
+                    truncate(batchExecutor, config);
+                } else if (logger.isInfoEnabled()){
+                    logger.info("SingleDml: {}", JSON.toJSONString(dml, Feature.WriteNulls));
+                }
+            } catch (SQLException e) {
+                logger.error("sync error: " + e.getMessage(), e);
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
+    private void alter(BatchExecutor batchExecutor, MappingConfig config, Dml dml, List<SQLStatement> stmtList, String configFile) throws SQLException {
+        if (config.isDebug()) {
+            logger.info("DML: {} {}", configFile, JSON.toJSONString(dml, Feature.WriteNulls));
+        }
+        DbMapping dbMapping = config.getDbMapping();
+        if (!dbMapping.isAlter()) {
+            logger.info("not alterable table: {} {}", dml.getTable(), configFile);
+            return;
+        }
+
+        Map<String, String> columnsMap = dbMapping.getTargetColumns();
+
+        Map<String, String> columnsMap1 = new HashMap<>();
+        for (Map.Entry<String, String> entry : columnsMap.entrySet()) {
+            columnsMap1.put(entry.getValue(), entry.getKey());
+        }
+
+        String targetTable = SyncUtil.getDbTableName(dbMapping);
+        Map<String, String> defValues = new HashMap<>();
+        for (SQLStatement statement : stmtList) {
+            if (statement instanceof SQLAlterTableStatement) {
+                SQLAlterTableStatement alterTable = (SQLAlterTableStatement) statement;
+                for (SQLAlterTableItem item : alterTable.getItems()) {
+                    if (item instanceof SQLAlterTableDropColumnItem) {
+                        SQLAlterTableDropColumnItem dropColumnItem = (SQLAlterTableDropColumnItem) item;
+                        if (!dbMapping.isDrop()) {
+                            logger.info("drop table column disabled: {} {}", targetTable, dropColumnItem.getColumns());
+                            continue;
+                        }
+                        for (SQLName sqlName : dropColumnItem.getColumns()) {
+                            String name = Util.cleanColumn(sqlName.getSimpleName());
+                            String sql = "ALTER TABLE " + targetTable + " DROP COLUMN IF EXISTS " +
+                                    dbMapping.escape(columnsMap1.getOrDefault(name, name));
+                            try {
+                                logger.info("drop table column: {} {}", sql, batchExecutor.executeUpdate(sql));
+                                dbMapping.removeTargetColumn(name);
+                            } catch (Exception e) {
+                                logger.warn("drop table column error: " + sql, e);
+                            }
+                        }
+                    } else if (item instanceof SQLAlterTableAddColumn) {
+                        SQLAlterTableAddColumn addColumn = (SQLAlterTableAddColumn) item;
+                        if (!dbMapping.getMapAll()) {
+                            logger.info("add table column disabled: {} {}", targetTable, addColumn.getColumns());
+                            continue;
+                        }
+                        for (SQLColumnDefinition definition : addColumn.getColumns()) {
+                            String name = Util.cleanColumn(definition.getNameAsString());
+                            if (dbMapping.getExcludeColumns().contains(name)) {
+                                continue;
+                            }
+                            String sql = "ALTER TABLE " + targetTable +
+                                    " ADD IF NOT EXISTS " +
+                                    dbMapping.escape(name) + " " + TypeUtil.getPhoenixType(definition, dbMapping.isLimit());
+                            try {
+                                logger.info("add table column: {} {}", sql, batchExecutor.executeUpdate(sql));
+                                dbMapping.addTargetColumn(name, name);
+                                if (definition.getDefaultExpr() != null) {
+                                    String defVal = definition.getDefaultExpr().toString();
+                                    if (!defVal.equalsIgnoreCase("NULL") && !defVal.equalsIgnoreCase("NOT NULL") && name.length() > 0) {
+                                        defValues.put(name, defVal);
+                                    }
+                                }
+                            } catch (Exception e) {
+                                logger.error("add table column error: " + sql, e);
+                                throw e;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        if (!defValues.isEmpty()) {
+            StringBuilder defSql = new StringBuilder();
+            defSql.append("UPSERT INTO ").append(targetTable).append("(");
+            Set<Map.Entry<String, String>> pkSet = dbMapping.getTargetPk().entrySet();
+            Set<Map.Entry<String, String>> defSet = defValues.entrySet();
+            for (Map.Entry<String, String> entry : pkSet) {
+                defSql.append(dbMapping.escape(entry.getKey())).append(",");
+            }
+            for (Map.Entry<String, String> entry : defSet) {
+                defSql.append(dbMapping.escape(entry.getKey())).append(",");
+            }
+            defSql.deleteCharAt(defSql.length() - 1).append(") SELECT ");
+            for (Map.Entry<String, String> entry : pkSet) {
+                defSql.append(dbMapping.escape(entry.getKey())).append(",");
+            }
+            for (Map.Entry<String, String> entry : defSet) {
+                defSql.append(entry.getValue()).append(",");
+            }
+            defSql.deleteCharAt(defSql.length() - 1).append(" FROM ").append(targetTable);
+            try {
+                logger.info("set column default value: {} {}", defSql, batchExecutor.executeUpdate(defSql.toString()));
+                batchExecutor.commit();
+            } catch (SQLException e) {
+                logger.error("set column default value error: {}", defSql, e);
+                batchExecutor.rollback();
+                throw e;
+            }
+        }
+    }
+
+    /**
+     * 插入操作
+     *
+     * @param config 配置项
+     * @param dml    DML数据
+     */
+    private void insert(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) throws SQLException {
+        Map<String, Object> data = dml.getData();
+        if (data == null || data.isEmpty()) {
+            return;
+        }
+
+        DbMapping dbMapping = config.getDbMapping();
+        Map<String, String> columnsMap = SyncUtil.getColumnsMap(dbMapping, data);
+
+        StringBuilder insertSql = new StringBuilder();
+        insertSql.append("UPSERT INTO ").append(SyncUtil.getDbTableName(dbMapping)).append(" (");
+
+        Map<String, Integer> ctype = getTargetColumnType(batchExecutor.getConn(), config);
+
+        int mapLen = columnsMap.size();
+        List<Map<String, ?>> values = new ArrayList<>();
+        for (Map.Entry<String, String> entry : columnsMap.entrySet()) {
+            String targetColumnName = entry.getKey();
+            String srcColumnName = entry.getValue();
+            if (srcColumnName == null) {
+                srcColumnName = Util.cleanColumn(targetColumnName);
+            }
+
+            Integer type = ctype.get(Util.cleanColumn(targetColumnName).toLowerCase());
+            if (type == null) {
+                if (dbMapping.isSkipMissing()) {
+                    logger.warn("Target missing field: {}", targetColumnName);
+                    mapLen -= 1;
+                    continue;
+                } else if (dbMapping.getMapAll() && dbMapping.isAlter() && PhoenixEtlService.syncSchema(batchExecutor.getConn(), config)) {
+                    columnsTypeCache.remove(config.getDestination() + "." + dbMapping.getDatabase() + "." + dbMapping.getTable());
+                    ctype = getTargetColumnType(batchExecutor.getConn(), config);
+                    type = ctype.get(Util.cleanColumn(targetColumnName).toLowerCase());
+                }
+                if (type == null) {
+                    throw new RuntimeException("Target column: " + targetColumnName + " not matched");
+                }
+            }
+            insertSql.append(dbMapping.escape(targetColumnName)).append(",");
+            Object value = data.get(srcColumnName);
+            BatchExecutor.setValue(values, type, value);
+        }
+
+        int len = insertSql.length();
+        insertSql.delete(len - 1, len).append(") VALUES (");
+        for (int i = 0; i < mapLen; i++) {
+            insertSql.append("?,");
+        }
+        len = insertSql.length();
+        insertSql.delete(len - 1, len).append(")");
+
+        Map<String, Object> old = dml.getOld();
+        try {
+            if (old != null && !old.isEmpty()) {
+                boolean keyChanged = false;
+                List<Map<String, ?>> delValues = new ArrayList<>();
+                StringBuilder deleteSql = new StringBuilder();
+                deleteSql.append("DELETE FROM ").append(SyncUtil.getDbTableName(dbMapping)).append(" WHERE ");
+                for (Map.Entry<String, String> entry : dbMapping.getTargetPk().entrySet()) {
+                    String targetColumnName = entry.getKey();
+                    String srcColumnName = entry.getValue();
+                    if (srcColumnName == null) {
+                        srcColumnName = Util.cleanColumn(targetColumnName);
+                    }
+                    Integer type = ctype.get(Util.cleanColumn(targetColumnName).toLowerCase());
+                    if (type != null) {
+                        deleteSql.append(dbMapping.escape(targetColumnName)).append("=? AND ");
+                        // 如果有修改主键的情况
+                        if (old.containsKey(srcColumnName)) {
+                            keyChanged = true;
+                            BatchExecutor.setValue(delValues, type, old.get(srcColumnName));
+                        } else {
+                            BatchExecutor.setValue(delValues, type, data.get(srcColumnName));
+                        }
+                    }
+                }
+                if (keyChanged) {
+                    if (config.isDebug()) {
+                        logger.info("insert into table: {} {}", deleteSql, delValues);
+                    }
+                    batchExecutor.execute(deleteSql.toString(), delValues);
+                }
+            }
+            if (config.isDebug()) {
+                logger.info("insert into table: {} {}", insertSql, values);
+            }
+            batchExecutor.execute(insertSql.toString(), values);
+        } catch (SQLException | RuntimeException e) {
+            logger.warn("Insert into target table, sql: {} {}", insertSql, values ,e);
+            throw e;
+        }
+        if (logger.isTraceEnabled()) {
+            logger.trace("Insert into target table, sql: {}", insertSql);
+        }
+    }
+
+    /**
+     * 删除操作 没有改动
+     *
+     * @param config MappingConfig
+     * @param dml    Single DML
+     */
+    private void delete(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) throws SQLException {
+        Map<String, Object> data = dml.getData();
+        if (data == null || data.isEmpty()) {
+            return;
+        }
+
+        DbMapping dbMapping = config.getDbMapping();
+
+        Map<String, Integer> ctype = getTargetColumnType(batchExecutor.getConn(), config);
+
+        StringBuilder sql = new StringBuilder();
+        sql.append("DELETE FROM ").append(SyncUtil.getDbTableName(dbMapping)).append(" WHERE ");
+
+        List<Map<String, ?>> values = new ArrayList<>();
+        // 拼接主键
+        appendCondition(dbMapping, sql, ctype, values, data);
+        try {
+            batchExecutor.execute(sql.toString(), values);
+            if (logger.isTraceEnabled()) {
+                logger.trace("Delete from target table, sql: {}", sql);
+            }
+        } catch (SQLException e) {
+            logger.warn("Delete from target error, sql: {} {}", sql, values);
+            throw e;
+        }
+    }
+
+    /**
+     * truncate操作   没有改动
+     *
+     * @param config MappingConfig
+     */
+    private void truncate(BatchExecutor batchExecutor, MappingConfig config) throws SQLException {
+        DbMapping dbMapping = config.getDbMapping();
+        StringBuilder sql = new StringBuilder();
+        sql.append("TRUNCATE TABLE ").append(SyncUtil.getDbTableName(dbMapping));
+        batchExecutor.execute(sql.toString(), new ArrayList<>());
+        if (logger.isTraceEnabled()) {
+            logger.trace("Truncate target table, sql: {}", sql);
+        }
+    }
+
+    /**
+     * 获取目标字段类型
+     *
+     * @param conn   sql connection
+     * @param config 映射配置
+     * @return 字段sqlType
+     */
+    private Map<String, Integer> getTargetColumnType(Connection conn, MappingConfig config) {
+        DbMapping dbMapping = config.getDbMapping();
+        String cacheKey = config.getDestination() + "." + dbMapping.getDatabase() + "." + dbMapping.getTable();
+        Map<String, Integer> columnType = columnsTypeCache.get(cacheKey);
+        if (columnType == null) {
+            synchronized (PhoenixSyncService.class) {
+                columnType = columnsTypeCache.get(cacheKey);
+                if (columnType == null) {
+                    columnType = new LinkedHashMap<>();
+                    final Map<String, Integer> columnTypeTmp = columnType;
+                    String sql = "SELECT * FROM " + SyncUtil.getDbTableName(dbMapping) + " WHERE 1=2";
+                    try {
+                        Util.sqlRS(conn, sql, rs -> {
+                            try {
+                                ResultSetMetaData rsd = rs.getMetaData();
+                                int columnCount = rsd.getColumnCount();
+                                for (int i = 1; i <= columnCount; i++) {
+                                    columnTypeTmp.put(rsd.getColumnName(i).toLowerCase(), rsd.getColumnType(i));
+                                }
+                                columnsTypeCache.put(cacheKey, columnTypeTmp);
+                            } catch (SQLException e) {
+                                logger.error(e.getMessage(), e);
+                            }
+                        });
+                    } catch (RuntimeException e) {
+                        //新增catch 里面做了操作
+                        if (!e.getCause().getClass().getName().endsWith("TableNotFoundException")) {
+                            throw e;
+                        }
+                        if (!PhoenixEtlService.syncSchema(conn, config)) {
+                            throw e;
+                        }
+                        Util.sqlRS(conn, sql, rs -> {
+                            try {
+                                ResultSetMetaData rsd = rs.getMetaData();
+                                int columnCount = rsd.getColumnCount();
+                                for (int i = 1; i <= columnCount; i++) {
+                                    columnTypeTmp.put(rsd.getColumnName(i).toLowerCase(), rsd.getColumnType(i));
+                                }
+                                columnsTypeCache.put(cacheKey, columnTypeTmp);
+                            } catch (SQLException e1) {
+                                logger.error(e1.getMessage(), e1);
+                            }
+                        });
+                    }
+                }
+            }
+        }
+        return columnType;
+    }
+
+    /**
+     * 拼接主键 where条件
+     */
+    private void appendCondition(DbMapping dbMapping, StringBuilder sql, Map<String, Integer> ctype,
+                                 List<Map<String, ?>> values, Map<String, Object> d) {
+        // 拼接主键
+        for (Map.Entry<String, String> entry : dbMapping.getTargetPk().entrySet()) {
+            String targetColumnName = entry.getKey();
+            String srcColumnName = entry.getValue();
+            if (srcColumnName == null) {
+                srcColumnName = Util.cleanColumn(targetColumnName);
+            }
+            sql.append(dbMapping.escape(targetColumnName)).append("=? AND ");
+            Integer type = ctype.get(Util.cleanColumn(targetColumnName).toLowerCase());
+            if (type == null) {
+                throw new RuntimeException("Target column: " + targetColumnName + " not matched");
+            }
+            BatchExecutor.setValue(values, type, d.get(srcColumnName));
+        }
+        int len = sql.length();
+        sql.delete(len - 4, len);
+    }
+
+    public static class SyncItem {
+
+        private MappingConfig config;
+        private SingleDml singleDml;
+
+        SyncItem(MappingConfig config, SingleDml singleDml) {
+            this.config = config;
+            this.singleDml = singleDml;
+        }
+    }
+
+    /**
+     * 取主键hash
+     */
+    private int pkHash(DbMapping dbMapping, Map<String, Object> d) {
+        int hash = 0;
+        // 取主键
+        for (Map.Entry<String, String> entry : dbMapping.getTargetPk().entrySet()) {
+            String targetColumnName = entry.getKey();
+            String srcColumnName = entry.getValue();
+            if (srcColumnName == null) {
+                srcColumnName = Util.cleanColumn(targetColumnName);
+            }
+            Object value = null;
+            if (d != null) {
+                value = d.get(srcColumnName);
+            }
+            if (value != null) {
+                hash += value.hashCode();
+
+            }
+        }
+        hash = Math.abs(hash) % threads;
+        return Math.abs(hash);
+    }
+
+
+    public void close() {
+        for (int i = 0; i < threads; i++) {
+            executorThreads[i].shutdown();
+        }
+    }
+}

client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/support/BatchExecutor.java

@@ -0,0 +1,108 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.support;
+
+import com.alibaba.otter.canal.client.adapter.phoenix.PhoenixAdapter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.sql.DataSource;
+import java.io.Closeable;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * sql批量执行器   基本没有变动
+ */
+public class BatchExecutor implements Closeable {
+
+    private static final Logger logger = LoggerFactory.getLogger(BatchExecutor.class);
+
+    private DataSource          dataSource;
+    private Connection          conn;
+    private AtomicInteger       idx    = new AtomicInteger(0);
+
+    public BatchExecutor(DataSource dataSource){
+        this.dataSource = dataSource;
+    }
+    public BatchExecutor(){
+    }
+
+    public Connection getConn() {
+        if (conn == null) {
+            try {
+                conn=PhoenixAdapter.getPhoenixConnection();
+                this.conn.setAutoCommit(false);
+            } catch (SQLException e) {
+                logger.error(e.getMessage(), e);
+            }
+        }
+        return conn;
+    }
+
+    public static void setValue(List<Map<String, ?>> values, int type, Object value) {
+        Map<String, Object> valueItem = new HashMap<>();
+        valueItem.put("type", type);
+        valueItem.put("value", value);
+        values.add(valueItem);
+    }
+
+    public int executeUpdate(String sql) throws SQLException {
+        logger.debug("execute: {}", sql);
+        Statement statement = getConn().createStatement();
+        int ret = statement.executeUpdate(sql);
+        statement.close();
+        return ret;
+    }
+
+    public void execute(String sql, List<Map<String, ?>> values) throws SQLException {
+        if (logger.isDebugEnabled()) {
+            logger.debug("execute: {} {}", sql, Arrays.toString(values.toArray()));
+        }
+        PreparedStatement pstmt = getConn().prepareStatement(sql);
+        int len = values.size();
+        for (int i = 0; i < len; i++) {
+            int type = (Integer) values.get(i).get("type");
+            Object value = values.get(i).get("value");
+            SyncUtil.setPStmt(type, pstmt, value, i + 1);
+        }
+
+        pstmt.execute();
+        idx.incrementAndGet();
+        pstmt.close();
+    }
+
+    public void commit() throws SQLException {
+        getConn().commit();
+        if (logger.isTraceEnabled()) {
+            logger.trace("Batch executor commit " + idx.get() + " rows");
+        }
+        idx.set(0);
+    }
+
+    public void rollback() throws SQLException {
+        getConn().rollback();
+        if (logger.isTraceEnabled()) {
+            logger.trace("Batch executor rollback " + idx.get() + " rows");
+        }
+        idx.set(0);
+    }
+
+    @Override
+    public void close() {
+        if (conn != null) {
+            try {
+                conn.close();
+            } catch (SQLException e) {
+                logger.error(e.getMessage(), e);
+            } finally {
+                conn = null;
+            }
+        }
+    }
+}

client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/support/PhoenixSupportUtil.java

@@ -0,0 +1,97 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.support;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.function.Function;
+
+/**
+ * @author: lihua
+ * @date: 2020/12/30 18:43
+ * @Description:
+ */
+public class PhoenixSupportUtil {
+
+
+    public static final  Logger logger = LoggerFactory.getLogger(PhoenixSupportUtil.class);
+    public static Object sqlRS(Connection dsConnection, String sql, Function<ResultSet, Object> fun) {
+
+        try {
+            Connection conn = dsConnection;
+            Throwable var4 = null;
+
+            Object var9;
+            try {
+                Statement stmt = conn.createStatement(1003, 1007);
+                Throwable var6 = null;
+
+                try {
+                    stmt.setFetchSize(-2147483648);
+                    ResultSet rs = stmt.executeQuery(sql);
+                    Throwable var8 = null;
+
+                    try {
+                        var9 = fun.apply(rs);
+                    } catch (Throwable var56) {
+                        var9 = var56;
+                        var8 = var56;
+                        throw var56;
+                    } finally {
+                        if (rs != null) {
+                            if (var8 != null) {
+                                try {
+                                    rs.close();
+                                } catch (Throwable var55) {
+                                    var8.addSuppressed(var55);
+                                }
+                            } else {
+                                rs.close();
+                            }
+                        }
+
+                    }
+                } catch (Throwable var58) {
+                    var6 = var58;
+                    throw var58;
+                } finally {
+                    if (stmt != null) {
+                        if (var6 != null) {
+                            try {
+                                stmt.close();
+                            } catch (Throwable var54) {
+                                var6.addSuppressed(var54);
+                            }
+                        } else {
+                            stmt.close();
+                        }
+                    }
+
+                }
+            } catch (Throwable var60) {
+                var4 = var60;
+                throw var60;
+            } finally {
+                if (conn != null) {
+                    if (var4 != null) {
+                        try {
+                            conn.close();
+                        } catch (Throwable var53) {
+                            var4.addSuppressed(var53);
+                        }
+                    } else {
+                        conn.close();
+                    }
+                }
+
+            }
+
+            return var9;
+        } catch (Exception var62) {
+            logger.error("sqlRs has error, sql: {} ", sql);
+            throw new RuntimeException(var62);
+        }
+    }
+}

client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/support/SingleDml.java

@@ -0,0 +1,97 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.support;
+
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * 没有改动 单个DML类
+ */
+@SuppressWarnings({"unused", "WeakerAccess"})
+public class SingleDml {
+
+    private String              destination;
+    private String              database;
+    private String              table;
+    private String              type;
+    private Map<String, Object> data;
+    private Map<String, Object> old;
+
+    public String getDestination() {
+        return destination;
+    }
+
+    public void setDestination(String destination) {
+        this.destination = destination;
+    }
+
+    public String getDatabase() {
+        return database;
+    }
+
+    public void setDatabase(String database) {
+        this.database = database;
+    }
+
+    public String getTable() {
+        return table;
+    }
+
+    public void setTable(String table) {
+        this.table = table;
+    }
+
+    public String getType() {
+        return type;
+    }
+
+    public void setType(String type) {
+        this.type = type;
+    }
+
+    public Map<String, Object> getData() {
+        return data;
+    }
+
+    public void setData(Map<String, Object> data) {
+        this.data = data;
+    }
+
+    public Map<String, Object> getOld() {
+        return old;
+    }
+
+    public void setOld(Map<String, Object> old) {
+        this.old = old;
+    }
+
+    public static List<SingleDml> dml2SingleDmls(Dml dml) {
+        List<SingleDml> singleDmls = new ArrayList<>();
+        if (dml.getData() != null) {
+            int size = dml.getData().size();
+            for (int i = 0; i < size; i++) {
+                SingleDml singleDml = new SingleDml();
+                singleDml.setDestination(dml.getDestination());
+                singleDml.setDatabase(dml.getDatabase());
+                singleDml.setTable(dml.getTable());
+                singleDml.setType(dml.getType());
+                singleDml.setData(dml.getData().get(i));
+                if (dml.getOld() != null) {
+                    singleDml.setOld(dml.getOld().get(i));
+                }
+                singleDmls.add(singleDml);
+            }
+            //MaxWell 中没有对TRUNCATE的DML操作进行解析
+        } else if ("TRUNCATE".equalsIgnoreCase(dml.getType())) {
+            SingleDml singleDml = new SingleDml();
+            singleDml.setDestination(dml.getDestination());
+            singleDml.setDatabase(dml.getDatabase());
+            singleDml.setTable(dml.getTable());
+            singleDml.setType(dml.getType());
+            singleDmls.add(singleDml);
+        }
+        return singleDmls;
+    }
+}

client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/support/SyncUtil.java

@@ -0,0 +1,274 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.support;
+
+import com.alibaba.otter.canal.client.adapter.phoenix.config.MappingConfig;
+import com.alibaba.otter.canal.client.adapter.support.Util;
+import org.apache.commons.lang.StringUtils;
+
+import java.io.Reader;
+import java.io.StringReader;
+import java.math.BigDecimal;
+import java.nio.charset.StandardCharsets;
+import java.sql.*;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class SyncUtil {
+
+    public static Map<String, String> getColumnsMap(MappingConfig.DbMapping dbMapping, Map<String, Object> data) {
+        return getColumnsMap(dbMapping, data.keySet());
+    }
+
+    public static Map<String, String> getColumnsMap(MappingConfig.DbMapping dbMapping, Collection<String> columns) {
+        Map<String, String> columnsMap;
+        if (dbMapping.getMapAll()) {
+            if (dbMapping.getAllMapColumns() != null) {
+                return dbMapping.getAllMapColumns();
+            }
+            columnsMap = new LinkedHashMap<>();
+            for (String srcColumn : columns) {
+                boolean flag = true;
+                if (dbMapping.getTargetColumns() != null) {
+                    for (Map.Entry<String, String> entry : dbMapping.getTargetColumns().entrySet()) {
+                        if (srcColumn.equals(entry.getValue())) {
+                            columnsMap.put(entry.getKey(), srcColumn);
+                            flag = false;
+                            break;
+                        }
+                    }
+                }
+                //新增 排除掉getExcludeColumns() 去除的列
+                if (flag && !dbMapping.getExcludeColumns().contains(srcColumn)) {
+                    columnsMap.put(srcColumn, srcColumn);
+                }
+            }
+            dbMapping.setAllMapColumns(columnsMap);
+        } else {
+            columnsMap = dbMapping.getTargetColumns();
+        }
+        return columnsMap;
+    }
+
+    /**
+     * 设置 preparedStatement
+     *
+     * @param type sqlType
+     * @param pstmt 需要设置的preparedStatement
+     * @param value 值
+     * @param i 索引号
+     */
+    public static void setPStmt(int type, PreparedStatement pstmt, Object value, int i) throws SQLException {
+        switch (type) {
+            case Types.BIT:
+            case Types.BOOLEAN:
+                if (value instanceof Boolean) {
+                    pstmt.setBoolean(i, (Boolean) value);
+                } else if (value instanceof String) {
+                    boolean v = !value.equals("0");
+                    pstmt.setBoolean(i, v);
+                } else if (value instanceof Number) {
+                    boolean v = ((Number) value).intValue() != 0;
+                    pstmt.setBoolean(i, v);
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.CHAR:
+            case Types.NCHAR:
+            case Types.VARCHAR:
+            case Types.LONGVARCHAR:
+                if (value instanceof String) {
+                    pstmt.setString(i, (String) value);
+                } else if (value == null) {
+                    pstmt.setNull(i, type);
+                } else {
+                    pstmt.setString(i, value.toString());
+                }
+                break;
+            case Types.TINYINT:
+                if (value instanceof Number) {
+                    pstmt.setByte(i, ((Number) value).byteValue());
+                } else if (value instanceof String) {
+                    pstmt.setByte(i, Byte.parseByte((String) value));
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.SMALLINT:
+                if (value instanceof Number) {
+                    pstmt.setShort(i, ((Number) value).shortValue());
+                } else if (value instanceof String) {
+                    pstmt.setShort(i, Short.parseShort((String) value));
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.INTEGER:
+                if (value instanceof Number) {
+                    pstmt.setInt(i, ((Number) value).intValue());
+                } else if (value instanceof String) {
+                    pstmt.setInt(i, Integer.parseInt((String) value));
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.BIGINT:
+                if (value instanceof Number) {
+                    pstmt.setLong(i, ((Number) value).longValue());
+                } else if (value instanceof String) {
+                    pstmt.setLong(i, Long.parseLong((String) value));
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.DECIMAL:
+            case Types.NUMERIC:
+                if (value instanceof BigDecimal) {
+                    pstmt.setBigDecimal(i, (BigDecimal) value);
+                } else if (value instanceof Byte) {
+                    pstmt.setInt(i, ((Byte) value).intValue());
+                } else if (value instanceof Short) {
+                    pstmt.setInt(i, ((Short) value).intValue());
+                } else if (value instanceof Integer) {
+                    pstmt.setInt(i, (Integer) value);
+                } else if (value instanceof Long) {
+                    pstmt.setLong(i, (Long) value);
+                } else if (value instanceof Float) {
+                    pstmt.setBigDecimal(i, new BigDecimal((float) value));
+                } else if (value instanceof Double) {
+                    pstmt.setBigDecimal(i, new BigDecimal((double) value));
+                } else if (value != null) {
+                    pstmt.setBigDecimal(i, new BigDecimal(value.toString()));
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.REAL:
+                if (value instanceof Number) {
+                    pstmt.setFloat(i, ((Number) value).floatValue());
+                } else if (value instanceof String) {
+                    pstmt.setFloat(i, Float.parseFloat((String) value));
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.FLOAT:
+            case Types.DOUBLE:
+                if (value instanceof Number) {
+                    pstmt.setDouble(i, ((Number) value).doubleValue());
+                } else if (value instanceof String) {
+                    pstmt.setDouble(i, Double.parseDouble((String) value));
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.BINARY:
+            case Types.VARBINARY:
+            case Types.LONGVARBINARY:
+            case Types.BLOB:
+                if (value instanceof Blob) {
+                    pstmt.setBlob(i, (Blob) value);
+                } else if (value instanceof byte[]) {
+                    pstmt.setBytes(i, (byte[]) value);
+                } else if (value instanceof String) {
+                    pstmt.setBytes(i, ((String) value).getBytes(StandardCharsets.ISO_8859_1));
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.CLOB:
+                if (value instanceof Clob) {
+                    pstmt.setClob(i, (Clob) value);
+                } else if (value instanceof byte[]) {
+                    pstmt.setBytes(i, (byte[]) value);
+                } else if (value instanceof String) {
+                    Reader clobReader = new StringReader((String) value);
+                    pstmt.setCharacterStream(i, clobReader);
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.DATE:
+                if (value instanceof Date) {
+                    pstmt.setDate(i, (Date) value);
+                } else if (value instanceof java.util.Date) {
+                    pstmt.setDate(i, new Date(((java.util.Date) value).getTime()));
+                } else if (value instanceof String) {
+                    String v = (String) value;
+                    if (!v.startsWith("0000-00-00")) {
+                        java.util.Date date = Util.parseDate(v);
+                        if (date != null) {
+                            pstmt.setDate(i, new Date(date.getTime()));
+                        } else {
+                            pstmt.setNull(i, type);
+                        }
+                    } else {
+                        pstmt.setObject(i, value);
+                    }
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.TIME:
+                if (value instanceof Time) {
+                    pstmt.setTime(i, (Time) value);
+                } else if (value instanceof java.util.Date) {
+                    pstmt.setTime(i, new Time(((java.util.Date) value).getTime()));
+                } else if (value instanceof String) {
+                    String v = (String) value;
+                    java.util.Date date = Util.parseDate(v);
+                    if (date != null) {
+                        pstmt.setTime(i, new Time(date.getTime()));
+                    } else {
+                        pstmt.setNull(i, type);
+                    }
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.TIMESTAMP:
+                if (value instanceof Timestamp) {
+                    pstmt.setTimestamp(i, (Timestamp) value);
+                } else if (value instanceof java.util.Date) {
+                    pstmt.setTimestamp(i, new Timestamp(((java.util.Date) value).getTime()));
+                } else if (value instanceof String) {
+                    String v = (String) value;
+                    if (!v.startsWith("0000-00-00")) {
+                        java.util.Date date = Util.parseDate(v);
+                        if (date != null) {
+                            pstmt.setTimestamp(i, new Timestamp(date.getTime()));
+                        } else {
+                            pstmt.setNull(i, type);
+                        }
+                    } else {
+                        pstmt.setObject(i, value);
+                    }
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            default:
+                pstmt.setObject(i, value, type);
+        }
+    }
+
+    public static String getDbTableName(MappingConfig.DbMapping dbMapping) {
+        String result = "";
+        if (StringUtils.isNotEmpty(dbMapping.getTargetDb())) {
+            if (dbMapping.isEscapeUpper()) {
+                //新增 字段默认大写加双引号
+                result += "\"" + dbMapping.getTargetDb() + "\".";
+            } else {
+                result += dbMapping.getTargetDb() + ".";
+            }
+        }
+
+        if (dbMapping.isEscapeUpper()) {
+            //新增  字段默认大写加双引号
+            result += "\"" + dbMapping.getTargetTable().replaceAll("\\.", "\".\"") + "\"";
+        } else {
+            result += dbMapping.getTargetTable();
+        }
+        return result;
+    }
+}

client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/support/TypeUtil.java

@@ -0,0 +1,142 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.support;
+
+import com.alibaba.druid.sql.ast.SQLDataType;
+import com.alibaba.druid.sql.ast.SQLDataTypeImpl;
+import com.alibaba.druid.sql.ast.statement.SQLColumnDefinition;
+import org.apache.commons.lang.StringUtils;
+
+/**
+ * Phoenix类型   此类型完全新增
+ */
+public class TypeUtil {
+
+    private static String joinArgs(String type, Object[] args) {
+        if (args.length > 0) {
+            return type + "(" + StringUtils.join(args, ",") + ")";
+        }
+        return type;
+    }
+
+    /**
+     * 根据SQL的定义返回Phoenix的类型定义
+     * @see "https://dev.mysql.com/doc/connector-j/5.1/en/connector-j-reference-type-conversions.html"
+     * @param definition SQL的字段定义
+     * @param limit 是否启用字段长度限制
+     * @return Phoenix字段类型定义
+     */
+    public static String getPhoenixType(SQLColumnDefinition definition, boolean limit) {
+        if (definition == null) return "VARCHAR";
+        SQLDataType sqlDataType = definition.getDataType();
+        SQLDataTypeImpl sqlDataType1 = sqlDataType instanceof SQLDataTypeImpl ? (SQLDataTypeImpl)sqlDataType : null;
+        boolean isUnsigned = sqlDataType1 != null && sqlDataType1.isUnsigned();
+
+        return getPhoenixType(sqlDataType.getName().toUpperCase(), sqlDataType.getArguments().toArray(), isUnsigned, limit);
+    }
+
+    //MySQL类型和Phoenix类型一一对应
+    public static String getPhoenixType(String name, Object[] args, boolean isUnsigned, boolean limit) {
+        switch (name) {
+            case "BIT":
+                if (limit) {
+                    return joinArgs("BINARY", args);
+                }
+                return "BINARY";
+            case "TINYINT":
+                if (isUnsigned) {
+                    return "UNSIGNED_TINYINT";
+                }
+                return "TINYINT";
+            case "BOOLEAN":
+            case "BOOL":
+                return "BOOLEAN";
+            case "SMALLINT":
+                if (isUnsigned) {
+                    return "UNSIGNED_SMALLINT";
+                }
+                return "SMALLINT";
+            case "MEDIUMINT":
+                return "INTEGER";
+            case "INT":
+            case "INTEGER":
+                if (isUnsigned) {
+                    return "UNSIGNED_INT";
+                }
+                return "INTEGER";
+            case "BIGINT":
+                if (isUnsigned) {
+                    return "UNSIGNED_LONG";
+                }
+                return "BIGINT";
+            case "FLOAT":
+                if (isUnsigned) {
+                    return "UNSIGNED_FLOAT";
+                }
+                return "FLOAT";
+            case "DOUBLE":
+                if (isUnsigned) {
+                    return "UNSIGNED_DOUBLE";
+                }
+                return "DOUBLE";
+            case "DECIMAL":
+                if (limit) {
+                    return joinArgs("DECIMAL", args);
+                }
+                return "DECIMAL";
+            case "DATE":
+                if (isUnsigned) {
+                    return "UNSIGNED_DATE";
+                }
+                return "DATE";
+            case "DATETIME":
+            case "TIMESTAMP":
+                if (isUnsigned) {
+                    return "UNSIGNED_TIMESTAMP";
+                }
+                return "TIMESTAMP";
+            case "TIME":
+                if (isUnsigned) {
+                    return "UNSIGNED_TIME";
+                }
+                return "TIME";
+            case "YEAR":
+                return "INTEGER";
+            case "CHAR":
+                if (limit) {
+                    return joinArgs(name, args);
+                }
+                return "VARCHAR";
+            case "VARCHAR":
+                if (limit) {
+                    return joinArgs(name, args);
+                }
+                return "VARCHAR";
+            case "BINARY":
+                if (limit) {
+                    return joinArgs(name, args);
+                }
+                return "VARBINARY";
+            case "VARBINARY":
+                return "VARBINARY";
+            case "TINYBLOB":
+                return "VARBINARY";
+            case "TINYTEXT":
+                return "VARCHAR";
+            case "BLOB":
+                return "VARBINARY";
+            case "TEXT":
+                return "VARCHAR";
+            case "MEDIUMBLOB":
+                return "VARBINARY";
+            case "MEDIUMTEXT":
+                return "VARCHAR";
+            case "LONGBLOB":
+                return "VARBINARY";
+            case "LONGTEXT":
+                return "VARCHAR";
+            case "ENUM":
+            case "SET":
+                return "VARCHAR";
+        }
+        return "VARCHAR";
+    }
+}

client-adapter/phoenix/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter

@@ -0,0 +1 @@
+phoenix=com.alibaba.otter.canal.client.adapter.phoenix.PhoenixAdapter

client-adapter/phoenix/src/main/resources/hbase-site.xml

@@ -0,0 +1,99 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+
+  <!--支持NAMESPACE映射 -->
+   <property>
+          <name>phoenix.schema.isNamespaceMappingEnabled</name>
+          <value>true</value>
+   </property>
+   <property>
+          <name>phoenix.schema.mapSystemTablesToNamespace</name>
+          <value>true</value>
+   </property>
+  <!--支持phoenix的二级索引 -->
+   <property>
+          <name>hbase.regionserver.wal.codec</name>
+          <value>org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec</value>
+   </property>
+   <property>  
+        <name>hbase.table.sanity.checks</name>  
+        <value>false</value> 
+   </property> 
+  <!--启动压缩文件检查 -->
+   <property>
+     <name>hbase.regionserver.codecs</name>
+     <value>snappy,lzo</value>
+   </property>
+  <!-- rpc 超时时长 -->
+	<property>
+	  <name>hbase.rpc.timeout</name>
+	  <value>1200000</value>
+	</property>
+	<property>
+	  <name>hbase.client.operation.timeout</name>
+	  <value>600000</value>
+	</property>
+	<property>
+	  <name>hbase.client.scanner.timeout.period</name>
+	  <value>1200000</value>
+	</property>
+	<property>
+	  <name>hbase.client.scanner.timeout.period</name>
+	  <value>1200000</value>
+	</property>
+	<property>
+	 <name>phoenix.query.timeoutMs</name>
+	 <value>1800000</value>
+	</property>
+	<property>
+	 <name>phoenix.query.keepAliveMs</name>
+	 <value>600000</value>
+	</property>
+	<property>
+	  <name>hbase.client.ipc.pool.type</name>
+	  <value>RoundRobinPool</value>
+	</property>
+	<property>
+	  <name>hbase.client.ipc.pool.size</name>
+	  <value>10</value>
+	</property>
+	<property>
+	  <name>index.builder.threads.keepalivetime</name>
+	  <value>1200000</value>
+	</property>
+	<property>
+	  <name>index.write.threads.keepalivetime</name>
+	  <value>1200000</value>
+	</property>
+	<property>
+	  <name>hbase.htable.threads.keepalivetime</name>
+	  <value>1200000</value>
+	</property>
+<!--允许写入自定义WAL编辑，确保索引更新的正确性 -->
+	<property>
+	  <name>hbase.regionserver.wal.codec</name>
+	  <value>org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec</value>
+	</property>
+
+</configuration>

client-adapter/phoenix/src/main/resources/phoenix/phoenix_common.properties

@@ -0,0 +1,2 @@
+# 同步线程数
+threads =3

client-adapter/phoenix/src/main/resources/phoenix/phoenixtest_user.yml

@@ -0,0 +1,22 @@
+dataSourceKey: defaultDS
+destination: example
+groupId: g1
+outerAdapterKey: phoenix
+concurrent: true
+dbMapping:
+  database: mytest
+  table: user
+  targetTable: mytest.user
+  escapeUpper: true  # 字段默认大写，并用双引号引起来
+  targetPk:
+    id: ID
+  mapAll: true  # 映射所有字段(默认true，不包含排除的字段)
+  alter: true   # 允许修改表结构(默认true，mapAll=true时可以新增，drop=true时可以删除字段)
+  drop: false   # 允许删除字段(默认false)
+  skipMissing: false # 是否跳过缺失的字段(默认false，允许新增字段时会自动同步缺失的字段;true时跳过缺失的字段)
+  limit: false  # 是否限与数据长度限制一致(默认false，不限制长度避免修改长度而无法修改)
+  targetColumns:
+    id: ID
+    name: NAME
+  excludeColumns: # 排除字段
+    - password

client-adapter/phoenix/src/test/java/com/alibaba/otter/canal/client/adapter/phoenix/test/PhoenixConnectionTest.java

@@ -0,0 +1,32 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.test;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.util.Properties;
+
+/**
+ * @author: lihua
+ * @date: 2021/1/5 16:58
+ * @Description:
+ */
+public class PhoenixConnectionTest {
+
+    public static void main(String[] args) {
+        Properties phoenixPro = new Properties();
+        //phoenix内部本身有连接池，不需要使用Druid初始化
+        phoenixPro.setProperty("hbase.rpc.timeout","600000");
+        phoenixPro.setProperty("hbase.client.scanner.timeout.period","600000");
+        phoenixPro.setProperty("dfs.client.socket-timeout","600000");
+        phoenixPro.setProperty("phoenix.query.keepAliveMs","600000");
+        phoenixPro.setProperty("phoenix.query.timeoutMs","3600000");
+        try {
+            Class.forName("org.apache.phoenix.jdbc.PhoenixDriver");
+            Connection connection = DriverManager.getConnection("jdbc:phoenix:zookeeper01,zookeeper02,zookeeper03:2181:/hbase/db", phoenixPro);
+            System.out.println(connection);
+            connection.close();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+
+    }
+}

client-adapter/phoenix/src/test/java/com/alibaba/otter/canal/client/adapter/phoenix/test/TestConfigLoad.java

@@ -0,0 +1,29 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.test;
+
+import com.alibaba.otter.canal.client.adapter.phoenix.config.ConfigLoader;
+import com.alibaba.otter.canal.client.adapter.phoenix.config.MappingConfig;
+import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Map;
+
+/**
+ * @author: lihua
+ * @date: 2021/1/5 17:07
+ * @Description:
+ */
+public class TestConfigLoad {
+    @Before
+    public void before() {
+        // 加载数据源连接池
+        DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource);
+    }
+
+    @Test
+    public void testLoad() {
+        Map<String, MappingConfig> configMap = ConfigLoader.load(null);
+        Assert.assertFalse(configMap.isEmpty());
+    }
+}

client-adapter/phoenix/src/test/java/com/alibaba/otter/canal/client/adapter/phoenix/test/TestConstant.java

@@ -0,0 +1,40 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.test;
+
+import com.alibaba.druid.pool.DruidDataSource;
+
+import java.sql.SQLException;
+
+/**
+ * @author: lihua
+ * @date: 2021/1/5 17:09
+ * @Description:
+ */
+public class TestConstant {
+    public final static String          jdbcUrl      = "jdbc:mysql://127.0.0.1:3306/canal_adapter?useUnicode=true";
+    public final static String          jdbcUser     = "root";
+    public final static String          jdbcPassword = "!123456";
+
+    public final static DruidDataSource dataSource;
+
+    static {
+        dataSource = new DruidDataSource();
+        dataSource.setDriverClassName("com.mysql.jdbc.Driver");
+        dataSource.setUrl(jdbcUrl);
+        dataSource.setUsername(jdbcUser);
+        dataSource.setPassword(jdbcPassword);
+        dataSource.setInitialSize(1);
+        dataSource.setMinIdle(1);
+        dataSource.setMaxActive(1);
+        dataSource.setMaxWait(60000);
+        dataSource.setTimeBetweenEvictionRunsMillis(60000);
+        dataSource.setMinEvictableIdleTimeMillis(300000);
+        dataSource.setPoolPreparedStatements(false);
+        dataSource.setMaxPoolPreparedStatementPerConnectionSize(20);
+        dataSource.setValidationQuery("select 1");
+        try {
+            dataSource.init();
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+    }
+}

client-adapter/phoenix/src/test/java/com/alibaba/otter/canal/client/adapter/phoenix/test/sync/Common.java

@@ -0,0 +1,36 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.test.sync;
+
+import com.alibaba.otter.canal.client.adapter.phoenix.PhoenixAdapter;
+import com.alibaba.otter.canal.client.adapter.phoenix.test.TestConstant;
+import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
+import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author: lihua
+ * @date: 2021/1/5 23:15
+ * @Description:
+ */
+public class Common {
+    public static PhoenixAdapter init() {
+        DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource);
+
+        OuterAdapterConfig outerAdapterConfig = new OuterAdapterConfig();
+        outerAdapterConfig.setName("phoenix");
+        outerAdapterConfig.setKey("phoenix");
+        Map<String, String> properties = new HashMap<>();
+        properties.put("jdbc.driverClassName", "org.apache.phoenix.jdbc.PhoenixDriver");
+        properties.put("jdbc.url", "jdbc:phoenix:zookeeper01,zookeeper02,zookeeper03:2181:/hbase/db");
+        outerAdapterConfig.setProperties(properties);
+
+        PhoenixAdapter adapter = new PhoenixAdapter();
+        adapter.init(outerAdapterConfig, null);
+        return adapter;
+    }
+
+    public static void main(String[] args) {
+        init();
+    }
+}

client-adapter/phoenix/src/test/java/com/alibaba/otter/canal/client/adapter/phoenix/test/sync/PhoenixSyncTest.java

@@ -0,0 +1,73 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.test.sync;
+
+import com.alibaba.otter.canal.client.adapter.phoenix.PhoenixAdapter;
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.*;
+
+/**
+ * @author: lihua
+ * @date: 2021/1/5 23:16
+ * @Description:
+ */
+public class PhoenixSyncTest {
+
+    private PhoenixAdapter phoenixAdapter;
+
+    @Before
+    public void init() {
+        phoenixAdapter = Common.init();
+    }
+
+    @Test
+    public void testEtl() {
+        List<String> param = new ArrayList<>();
+        phoenixAdapter.etl("phoenixtest_user.yml", param);
+    }
+    @Test
+    public void testCount() {
+        phoenixAdapter.count("phoenixtest_user.yml");
+    }
+    @Test
+    public void test01() {
+        Dml dml = new Dml();
+        dml.setDestination("example");
+        dml.setTs(new Date().getTime());
+        dml.setType("INSERT");
+        dml.setDatabase("mytest");
+        dml.setTable("user");
+        List<Map<String, Object>> dataList = new ArrayList<>();
+        Map<String, Object> data = new LinkedHashMap<>();
+        dataList.add(data);
+        data.put("id", 1);
+        data.put("name", "sixPulseExcalibur");
+        data.put("password", "123456");
+        dml.setData(dataList);
+
+        phoenixAdapter.sync(Collections.singletonList(dml));
+    }
+
+    @Test
+    public void test02() {
+        Dml dml = new Dml();
+        dml.setDestination("example");
+        dml.setTs(new Date().getTime());
+        dml.setType("UPDATE");
+        dml.setDatabase("mytest");
+        dml.setTable("user");
+        List<Map<String, Object>> dataList = new ArrayList<>();
+        Map<String, Object> data = new LinkedHashMap<>();
+        dataList.add(data);
+        data.put("id", 1);
+        data.put("name", "sixPulseExcalibur2");
+        dml.setData(dataList);
+        List<Map<String, Object>> oldList = new ArrayList<>();
+        Map<String, Object> old = new LinkedHashMap<>();
+        oldList.add(old);
+        old.put("name", "sixPulseExcalibur");
+        dml.setOld(oldList);
+        phoenixAdapter.sync(Collections.singletonList(dml));
+    }
+}

client-adapter/pom.xml

@@ -4,12 +4,12 @@
     <parent>
         <groupId>com.alibaba.otter</groupId>
         <artifactId>canal</artifactId>
-        <version>1.1.5-SNAPSHOT</version>
+        <version>1.1.7-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <groupId>com.alibaba.otter</groupId>
     <artifactId>canal.client-adapter</artifactId>
-    <version>1.1.5-SNAPSHOT</version>
+    <version>1.1.7-SNAPSHOT</version>
     <packaging>pom</packaging>
     <name>canal client adapter module for otter ${project.version}</name>
 
@@ -20,6 +20,7 @@
         <java_source_version>1.8</java_source_version>
         <java_target_version>1.8</java_target_version>
         <file_encoding>UTF-8</file_encoding>
+        <log4j_version>2.17.0</log4j_version>
     </properties>
 
     <modules>
@@ -32,6 +33,8 @@
         <module>es7x</module>
         <module>escore</module>
         <module>kudu</module>
+        <module>phoenix</module>
+        <module>tablestore</module>
     </modules>
 
     <licenses>
@@ -180,9 +183,9 @@
                 <version>42.1.4</version>
             </dependency>
             <dependency>
-                <groupId>com.oracle</groupId>
+                <groupId>com.oracle.database.jdbc</groupId>
                 <artifactId>ojdbc6</artifactId>
-                <version>11.2.0.3</version>
+                <version>11.2.0.4</version>
             </dependency>
             <dependency>
                 <groupId>com.microsoft.sqlserver</groupId>
@@ -192,7 +195,7 @@
 
             <dependency>
                 <groupId>org.apache.hbase</groupId>
-                <artifactId>hbase-client</artifactId>
+                <artifactId>hbase-shaded-client</artifactId>
                 <version>1.1.2</version>
                 <exclusions>
                     <exclusion>
@@ -210,6 +213,12 @@
                 <artifactId>curator-recipes</artifactId>
                 <version>2.10.0</version>
             </dependency>
+            <!-- 单独指定guava版本，兼容curator-client -->
+            <dependency>
+                <groupId>com.google.guava</groupId>
+                <artifactId>guava</artifactId>
+                <version>18.0</version>
+            </dependency>
             <dependency>
                 <groupId>joda-time</groupId>
                 <artifactId>joda-time</artifactId>
@@ -250,6 +259,39 @@
                 <version>1.9.0</version>
                 <scope>test</scope>
             </dependency>
+
+            <dependency>
+                <groupId>com.aliyun.openservices</groupId>
+                <artifactId>tablestore</artifactId>
+                <version>5.10.3</version>
+                <classifier>jar-with-dependencies</classifier>
+                <exclusions>
+                    <exclusion>
+                        <groupId>com.google.protobuf</groupId>
+                        <artifactId>protobuf-java</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>org.apache.httpcomponents</groupId>
+                        <artifactId>httpasyncclient</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
+
+            <dependency>
+                <groupId>org.apache.logging.log4j</groupId>
+                <artifactId>log4j-core</artifactId>
+                <version>${log4j_version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.logging.log4j</groupId>
+                <artifactId>log4j-api</artifactId>
+                <version>${log4j_version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.logging.log4j</groupId>
+                <artifactId>log4j-to-slf4j</artifactId>
+                <version>${log4j_version}</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>